parser, baby steps. Soft keywords highlighting

.idea/projectCodeStyle.xml

@@ -3,7 +3,9 @@
   <component name="CodeStyleSettingsManager">
     <option name="PER_PROJECT_SETTINGS">
       <value>
+        <option name="ELSE_ON_NEW_LINE" value="true" />
         <option name="ALIGN_GROUP_FIELD_DECLARATIONS" value="true" />
+        <option name="IF_BRACE_FORCE" value="1" />
         <ADDITIONAL_INDENT_OPTIONS fileType="groovy">
           <option name="INDENT_SIZE" value="2" />
           <option name="CONTINUATION_INDENT_SIZE" value="8" />

idea/META-INF/plugin.xml

@@ -20,5 +20,6 @@
       <fileTypeFactory implementation="org.jetbrains.jet.lang.JetFileFactory"/>
       <lang.braceMatcher language="jet" implementationClass="org.jetbrains.jet.lang.JetPairMatcher"/>
       <lang.parserDefinition language="jet" implementationClass="org.jetbrains.jet.lang.parsing.JetParserDefinition"/>
+      <annotator language="jet" implementationClass="org.jetbrains.jet.lang.annotations.SoftKeywordsAnnotator"/>
   </extensions>
 </idea-plugin>

idea/src/org/jetbrains/jet/JetNodeType.java

@@ -0,0 +1,17 @@
+/*
+ * @author max
+ */
+package org.jetbrains.jet;
+
+import com.intellij.lang.Language;
+import com.intellij.psi.tree.IElementType;
+import org.jetbrains.annotations.NonNls;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+import org.jetbrains.jet.lang.JetLanguage;
+
+public class JetNodeType extends IElementType {
+    public JetNodeType(@NotNull @NonNls String debugName) {
+        super(debugName, JetLanguage.INSTANCE);
+    }
+}

idea/src/org/jetbrains/jet/JetNodeTypes.java

@@ -3,10 +3,30 @@
  */
 package org.jetbrains.jet;
 
+import com.intellij.psi.tree.IElementType;
 import com.intellij.psi.tree.IFileElementType;
 import org.jetbrains.jet.lang.JetLanguage;
 
 public interface JetNodeTypes {
-    IFileElementType JET_FILE_NODE = new IFileElementType(JetLanguage.INSTANCE);
+    IFileElementType JET_FILE = new IFileElementType(JetLanguage.INSTANCE);
+
+    JetNodeType NAMESPACE = new JetNodeType("NAMESPACE");
+    JetNodeType CLASS = new JetNodeType("CLASS");
+    JetNodeType PROPERTY = new JetNodeType("PROPERTY");
+    JetNodeType FUN = new JetNodeType("FUN");
+    JetNodeType EXTENSION = new JetNodeType("EXTENSION");
+    JetNodeType TYPEDEF = new JetNodeType("TYPEDEF");
+    JetNodeType TYPE_PARAMETER_LIST = new JetNodeType("TYPE_PARAMETER_LIST");
+    JetNodeType TYPE_PARAMETER = new JetNodeType("TYPE_PARAMETER");
+    JetNodeType PRIMARY_CONSTRUCTOR_PARAMETERS_LIST = new JetNodeType("PRIMARY_CONSTRUCTOR_PARAMETERS_LIST");
+    JetNodeType PRIMARY_CONSTRUCTOR_PARAMETER = new JetNodeType("PRIMARY_CONSTRUCTOR_PARAMETER");
+    JetNodeType DELEGATION_SPECIFIER_LIST = new JetNodeType("DELEGATION_SPECIFIER_LIST");
+    JetNodeType DELEGATION_SPECIFIER = new JetNodeType("DELEGATION_SPECIFIER");
+    JetNodeType DELEGATOR_BY = new JetNodeType("DELEGATOR_BY");
+    JetNodeType DELEGATOR_SUPER_CALL = new JetNodeType("DELEGATOR_SUPER_CALL");
+    JetNodeType DELEGATOR_SUPER_CLASS = new JetNodeType("DELEGATOR_SUPER_CLASS");
+    JetNodeType VALUE_PARAMETER_LIST = new JetNodeType("VALUE_PARAMETER_LIST");
+    JetNodeType NAMED_ARGUMENT = new JetNodeType("NAMED_ARGUMENT");
+    JetNodeType CLASS_BODY = new JetNodeType("CLASS_BODY");
 
 }

idea/src/org/jetbrains/jet/lang/JetHighlighter.java

@@ -25,7 +25,7 @@ public class JetHighlighter extends SyntaxHighlighterBase {
                                                   SyntaxHighlighterColors.KEYWORD.getDefaultAttributes()
                                                  );
 
-    static final TextAttributesKey JET_SOFT_KEYWORD = TextAttributesKey.createTextAttributesKey(
+    public static final TextAttributesKey JET_SOFT_KEYWORD = TextAttributesKey.createTextAttributesKey(
                                                   "JET.SOFT.KEYWORD",
                                                   SyntaxHighlighterColors.KEYWORD.getDefaultAttributes()
                                                  );

idea/src/org/jetbrains/jet/lang/annotations/SoftKeywordsAnnotator.java

@@ -0,0 +1,22 @@
+/*
+ * @author max
+ */
+package org.jetbrains.jet.lang.annotations;
+
+import com.intellij.lang.annotation.AnnotationHolder;
+import com.intellij.lang.annotation.Annotator;
+import com.intellij.psi.PsiElement;
+import com.intellij.psi.impl.source.tree.LeafPsiElement;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.jet.lang.JetHighlighter;
+import org.jetbrains.jet.lexer.JetTokens;
+
+public class SoftKeywordsAnnotator implements Annotator {
+    public void annotate(@NotNull PsiElement element, @NotNull AnnotationHolder holder) {
+        if (element instanceof LeafPsiElement) {
+            if (JetTokens.SOFT_KEYWORDS.contains(((LeafPsiElement) element).getElementType())) {
+                holder.createInfoAnnotation(element, null).setTextAttributes(JetHighlighter.JET_SOFT_KEYWORD);
+            }
+        }
+    }
+}

idea/src/org/jetbrains/jet/lang/parsing/JetParser.java

@@ -13,12 +13,7 @@ import org.jetbrains.jet.JetNodeTypes;
 public class JetParser implements PsiParser {
     @NotNull
     public ASTNode parse(IElementType iElementType, PsiBuilder psiBuilder) {
-        PsiBuilder.Marker mark = psiBuilder.mark();
+        new JetParsing(psiBuilder).parseFile();
-        while (!psiBuilder.eof()) {
-            psiBuilder.advanceLexer();
-        }
-        mark.done(JetNodeTypes.JET_FILE_NODE);
-
         return psiBuilder.getTreeBuilt();
     }
 }

idea/src/org/jetbrains/jet/lang/parsing/JetParserDefinition.java

@@ -12,14 +12,12 @@ import com.intellij.openapi.project.Project;
 import com.intellij.psi.FileViewProvider;
 import com.intellij.psi.PsiElement;
 import com.intellij.psi.PsiFile;
-import com.intellij.psi.TokenType;
 import com.intellij.psi.tree.IFileElementType;
 import com.intellij.psi.tree.TokenSet;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.jet.JetNodeTypes;
 import org.jetbrains.jet.lang.psi.JetFile;
 import org.jetbrains.jet.lexer.JetLexer;
-import org.jetbrains.jet.lexer.JetToken;
 import org.jetbrains.jet.lexer.JetTokens;
 
 public class JetParserDefinition implements ParserDefinition {
@@ -33,7 +31,7 @@ public class JetParserDefinition implements ParserDefinition {
     }
 
     public IFileElementType getFileNodeType() {
-        return JetNodeTypes.JET_FILE_NODE;
+        return JetNodeTypes.JET_FILE;
     }
 
     @NotNull
@@ -54,7 +52,7 @@ public class JetParserDefinition implements ParserDefinition {
     @NotNull
     public PsiElement createElement(ASTNode astNode) {
         // TODO
-        return new ASTWrapperPsiElement(astNode);
+        return new CompositeNode(astNode);
     }
 
     public PsiFile createFile(FileViewProvider fileViewProvider) {
@@ -64,4 +62,15 @@ public class JetParserDefinition implements ParserDefinition {
     public SpaceRequirements spaceExistanceTypeBetweenTokens(ASTNode astNode, ASTNode astNode1) {
         return SpaceRequirements.MAY;
     }
+
+    private static class CompositeNode extends ASTWrapperPsiElement {
+        private CompositeNode(@NotNull ASTNode node) {
+            super(node);
+        }
+
+        @Override
+        public String toString() {
+            return getNode().getElementType().toString();
+        }
+    }
 }

idea/src/org/jetbrains/jet/lang/parsing/JetParsing.java

@@ -0,0 +1,417 @@
+/*
+ * @author max
+ */
+package org.jetbrains.jet.lang.parsing;
+
+import com.intellij.lang.ITokenTypeRemapper;
+import com.intellij.lang.PsiBuilder;
+import com.intellij.psi.tree.IElementType;
+import com.intellij.psi.tree.TokenSet;
+import org.jetbrains.annotations.Nullable;
+import org.jetbrains.jet.JetNodeType;
+import org.jetbrains.jet.lexer.JetSoftKeywordToken;
+import org.jetbrains.jet.lexer.JetToken;
+import org.jetbrains.jet.lexer.JetTokens;
+
+import static org.jetbrains.jet.JetNodeTypes.*;
+import static org.jetbrains.jet.lexer.JetTokens.*;
+
+public class JetParsing {
+    public static final TokenSet CLASS_NAME_FOLLOW = TokenSet.create(LT, WRAPS_KEYWORD, LPAR, COLON, LBRACE);
+    public static final TokenSet TYPE_PARAMETER_GT_FOLLOW = TokenSet.create(WHERE_KEYWORD, WRAPS_KEYWORD, LPAR, COLON, LBRACE, GT);
+    public static final TokenSet PARAMETER_NAME_FOLLOW = TokenSet.create(COLON, EQ, COMMA, RPAR);
+    private final PsiBuilder myBuilder;
+
+    public JetParsing(PsiBuilder myBuilder) {
+        this.myBuilder = myBuilder;
+    }
+
+
+    public void parseFile() {
+        PsiBuilder.Marker fileMarker = mark();
+        PsiBuilder.Marker namespaceMarker = parsePreamble();
+
+        while (!eof()) {
+            parseTopLevelObject();
+        }
+
+        if (namespaceMarker != null) {
+            namespaceMarker.done(NAMESPACE);
+        }
+
+        fileMarker.done(JET_FILE);
+    }
+
+    @Nullable
+    private PsiBuilder.Marker parsePreamble() {
+        if (at(NAMESPACE_KEYWORD)) {
+            advance();
+
+            PsiBuilder.Marker namespaceMarker = mark();
+            parseQualifiedName();
+            return namespaceMarker;
+        }
+
+        while (at(IMPORT_KEYWORD)) {
+            parseImportDirective();
+        }
+
+        return null;
+    }
+
+    private void parseImportDirective() {
+        // TODO
+    }
+
+    private PsiBuilder.Marker mark() {
+        return myBuilder.mark();
+    }
+
+    private void parseQualifiedName() {
+        while (true) {
+            if (tt() != IDENTIFIER) {
+                if (tt() != DOT) {
+                    error("Expecting qualified name");
+                    break;
+                }
+                error("Qualified name must start with an identifier");
+                advance();
+            }
+            else {
+                advance();
+                if (tt() != DOT) break;
+                advance();
+            }
+        }
+    }
+
+    private void error(String message) {
+        myBuilder.error(message);
+    }
+
+    private void parseTopLevelObject() {
+        PsiBuilder.Marker decl = mark();
+        if (!parseModifierList()) {
+            decl.drop();
+            return;
+        }
+
+        JetToken keywordToken = tt();
+        JetNodeType declType = null;
+        if (keywordToken == CLASS_KEYWORD) {
+            declType = parseClass();
+        }
+        else if (keywordToken == EXTENSION_KEYWORD) {
+            declType = parseExtension();
+        }
+        else if (keywordToken == FUN_KEYWORD) {
+            declType = parseFunction();
+        }
+        else if (keywordToken == VAL_KEYWORD || keywordToken == VAR_KEYWORD) {
+            declType = parseProperty();
+        }
+        else if (keywordToken == NAMESPACE_KEYWORD) {
+            declType = parseBlockNamespace();
+        }
+        else if (keywordToken == TYPE_KEYWORD) {
+            declType = parseTypedef();
+        }
+
+        if (declType == null) {
+            errorAndAdvance("Expecting namespace or top level declaration");
+            decl.drop();
+        }
+        else {
+            decl.done(declType);
+        }
+    }
+
+    private JetNodeType parseTypedef() {
+        return TYPEDEF;
+    }
+
+    private JetNodeType parseBlockNamespace() {
+        return NAMESPACE;
+    }
+
+    private JetNodeType parseProperty() {
+        return PROPERTY;
+    }
+
+    private JetNodeType parseFunction() {
+        return FUN;
+    }
+
+    private JetNodeType parseExtension() {
+        return EXTENSION;
+    }
+
+    private JetNodeType parseClass() {
+        assert at(CLASS_KEYWORD);
+        advance();
+
+        expect(IDENTIFIER, "Class name expected", CLASS_NAME_FOLLOW);
+        parseTypeParameterList();
+
+        if (at(WRAPS_KEYWORD)) advance();
+        if (at(LPAR)) {
+            parsePrimaryConstructorParameterList();
+        }
+
+        if (at(COLON)) {
+            advance();
+            parseDelegationSpecifierList();
+        }
+
+        if (at(LBRACE)) {
+            parseClassBody();
+        }
+
+        return CLASS;
+    }
+
+    private void parseClassBody() {
+        assert at(LBRACE);
+        PsiBuilder.Marker body = mark();
+        advance();
+
+        while (!eof()) {
+            if (at(RBRACE)) {
+                break;
+            }
+            advance();
+        }
+        expect(RBRACE, "Missing '}");
+        body.done(CLASS_BODY);
+    }
+
+    private void parseDelegationSpecifierList() {
+        PsiBuilder.Marker list = mark();
+        while (true) {
+            parseDelegationSpecifier();
+            if (!at(COMMA)) break;
+            advance();
+        }
+
+        list.done(DELEGATION_SPECIFIER_LIST);
+    }
+
+    private void parseDelegationSpecifier() {
+        PsiBuilder.Marker specifier = mark();
+        parseAttributeList();
+
+        PsiBuilder.Marker delegator = mark();
+        parseTypeRef(); // TODO: Error recovery!!!
+        if (at(BY_KEYWORD)) {
+            advance();
+            parseExpression();
+            delegator.done(DELEGATOR_BY);
+        }
+        else if (at(LPAR)) {
+            parseValueParameterList();
+            delegator.done(DELEGATOR_SUPER_CALL);
+        }
+        else {
+            delegator.done(DELEGATOR_SUPER_CLASS);
+        }
+
+        specifier.done(DELEGATION_SPECIFIER);
+    }
+
+    private void parseValueParameterList() {
+        assert at(LPAR);
+        PsiBuilder.Marker list = mark();
+        advance();
+        while (true) {
+            if (at(IDENTIFIER) && lookahead(1) == EQ) {
+                PsiBuilder.Marker named = mark();
+                advance();
+                advance();
+                parseExpression();
+                named.done(NAMED_ARGUMENT);
+            }
+            else {
+                parseExpression();
+            }
+
+            if (!at(COMMA)) {
+                break;
+            }
+        }
+
+        expect(RPAR, "Missing ')'");
+
+        list.done(VALUE_PARAMETER_LIST);
+
+    }
+
+    private void parseAttributeList() {
+        // TODO
+    }
+
+    private void parsePrimaryConstructorParameterList() {
+        assert at(LPAR);
+        PsiBuilder.Marker cons = mark();
+        advance();
+
+        while (true) {
+            parsePrimaryConstructorParameter();
+            if (!at(COMMA)) break;
+            advance();
+        }
+
+        expect(RPAR, "')' expected");
+
+        cons.done(PRIMARY_CONSTRUCTOR_PARAMETERS_LIST);
+    }
+
+    private void parsePrimaryConstructorParameter() {
+        PsiBuilder.Marker param = mark();
+        parseModifierList();
+
+        if (at(VAR_KEYWORD) || at(VAL_KEYWORD)) {
+            advance();
+        }
+
+        parseFunctionParameterRest();
+
+        param.done(PRIMARY_CONSTRUCTOR_PARAMETER);
+    }
+
+    private void parseFunctionParameterRest() {
+        expect(IDENTIFIER, "Parameter name expected", PARAMETER_NAME_FOLLOW);
+
+        if (at(COLON)) {
+            advance();
+            parseTypeRef();
+        }
+        else {
+            error("Parameters must have type annotation");
+        }
+
+        if (at(EQ)) {
+            advance();
+            parseExpression();
+        }
+    }
+
+    private void parseExpression() {
+        advance();
+        // TODO
+    }
+
+    private void parseTypeParameterList() {
+        PsiBuilder.Marker list = mark();
+        if (tt() == LT) {
+            advance();
+
+            while (true) {
+                parseTypeParameter();
+
+                if (!at(COMMA)) break;
+                advance();
+            }
+
+            expect(GT, "Missing '>'", TYPE_PARAMETER_GT_FOLLOW);
+        }
+        list.done(TYPE_PARAMETER_LIST);
+    }
+
+    private void parseTypeParameter() {
+        if (TYPE_PARAMETER_GT_FOLLOW.contains(tt())) {
+            error("Type parameter declaration expected");
+            return;
+        }
+
+        PsiBuilder.Marker mark = mark();
+        parseModifierList();
+        expect(IDENTIFIER, "Type parameter name expected", TokenSet.EMPTY);
+
+        if (at(COLON)) {
+            advance();
+            parseTypeRef();
+        }
+
+        mark.done(TYPE_PARAMETER);
+
+    }
+
+    private void parseTypeRef() {
+        advance();
+        // tODO:
+    }
+
+    private boolean expect(JetToken expectation, String message) {
+        return expect(expectation, message, null);
+    }
+
+    private boolean expect(JetToken expectation, String message, TokenSet recoverySet) {
+        JetToken tt = tt();
+        if (tt == expectation) {
+            advance();
+            return true;
+        }
+
+        if (recoverySet == null || recoverySet.contains(tt)) {
+            error(message);
+        }
+        else {
+            errorAndAdvance(message);
+        }
+
+        return false;
+    }
+
+    private void errorAndAdvance(String message) {
+        PsiBuilder.Marker err = mark();
+        advance();
+        err.error(message);
+    }
+
+    private boolean parseModifierList() {
+        // TODO
+        return true;
+    }
+
+    private boolean eof() {
+        return myBuilder.eof();
+    }
+
+    private void advance() {
+        myBuilder.advanceLexer();
+    }
+
+    private JetToken tt() {
+        return (JetToken) myBuilder.getTokenType();
+    }
+
+
+
+    private boolean at(final IElementType expectation) {
+        JetToken token = tt();
+        if (token == expectation) return true;
+        if (token == IDENTIFIER && expectation instanceof JetSoftKeywordToken) {
+            if (((JetSoftKeywordToken) expectation).getValue().equals(myBuilder.getTokenText())) {
+                myBuilder.setTokenTypeRemapper(new ITokenTypeRemapper() {
+                    public IElementType filter(IElementType source, int start, int end, CharSequence text) {
+                        return expectation;
+                    }
+                });
+
+                tt();
+                myBuilder.setTokenTypeRemapper(null);
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private IElementType lookahead(int k) {
+        PsiBuilder.Marker tmp = mark();
+        for (int i = 0; i < k; i++) advance();
+
+        JetToken tt = tt();
+        tmp.rollbackTo();
+        return tt;
+    }
+}

idea/src/org/jetbrains/jet/lexer/JetSoftKeywordToken.java

@@ -0,0 +1,20 @@
+/*
+ * @author max
+ */
+package org.jetbrains.jet.lexer;
+
+import org.jetbrains.annotations.NonNls;
+import org.jetbrains.annotations.NotNull;
+
+public class JetSoftKeywordToken extends JetToken {
+    private final String myValue;
+
+    public JetSoftKeywordToken(@NotNull @NonNls String value) {
+        super(value);
+        myValue = value;
+    }
+
+    public String getValue() {
+        return myValue;
+    }
+}

idea/src/org/jetbrains/jet/lexer/JetTokens.java

@@ -98,24 +98,24 @@ public interface JetTokens {
     JetToken MINUSEQ     = new JetToken("MINUSEQ");
     JetToken COMMA       = new JetToken("COMMA");
 
-    JetToken WRAPS_KEYWORD     = new JetToken("wraps");
+    JetSoftKeywordToken WRAPS_KEYWORD     = new JetSoftKeywordToken("wraps");
-    JetToken WHERE_KEYWORD     = new JetToken("where");
+    JetSoftKeywordToken WHERE_KEYWORD     = new JetSoftKeywordToken("where");
-    JetToken BY_KEYWORD        = new JetToken("by");
+    JetSoftKeywordToken BY_KEYWORD        = new JetSoftKeywordToken("by");
-    JetToken LAZY_KEYWORD      = new JetToken("lazy");
+    JetSoftKeywordToken LAZY_KEYWORD      = new JetSoftKeywordToken("lazy");
-    JetToken GET_KEYWORD       = new JetToken("get");
+    JetSoftKeywordToken GET_KEYWORD       = new JetSoftKeywordToken("get");
-    JetToken SET_KEYWORD       = new JetToken("set");
+    JetSoftKeywordToken SET_KEYWORD       = new JetSoftKeywordToken("set");
-    JetToken ABSTRACT_KEYWORD  = new JetToken("abstract");
+    JetSoftKeywordToken ABSTRACT_KEYWORD  = new JetSoftKeywordToken("abstract");
-    JetToken VIRTUAL_KEYWORD   = new JetToken("virtual");
+    JetSoftKeywordToken VIRTUAL_KEYWORD   = new JetSoftKeywordToken("virtual");
-    JetToken ENUM_KEYWORD      = new JetToken("enum");
+    JetSoftKeywordToken ENUM_KEYWORD      = new JetSoftKeywordToken("enum");
-    JetToken OPEN_KEYWORD      = new JetToken("open");
+    JetSoftKeywordToken OPEN_KEYWORD      = new JetSoftKeywordToken("open");
-    JetToken ATTRIBUTE_KEYWORD = new JetToken("attribute");
+    JetSoftKeywordToken ATTRIBUTE_KEYWORD = new JetSoftKeywordToken("attribute");
-    JetToken OVERRIDE_KEYWORD  = new JetToken("override");
+    JetSoftKeywordToken OVERRIDE_KEYWORD  = new JetSoftKeywordToken("override");
-    JetToken PRIVATE_KEYWORD   = new JetToken("private");
+    JetSoftKeywordToken PRIVATE_KEYWORD   = new JetSoftKeywordToken("private");
-    JetToken PUBLIC_KEYWORD    = new JetToken("public");
+    JetSoftKeywordToken PUBLIC_KEYWORD    = new JetSoftKeywordToken("public");
-    JetToken INTERNAL_KEYWORD  = new JetToken("internal");
+    JetSoftKeywordToken INTERNAL_KEYWORD  = new JetSoftKeywordToken("internal");
-    JetToken PROTECTED_KEYWORD = new JetToken("protected");
+    JetSoftKeywordToken PROTECTED_KEYWORD = new JetSoftKeywordToken("protected");
-    JetToken OUT_KEYWORD       = new JetToken("out");
+    JetSoftKeywordToken OUT_KEYWORD       = new JetSoftKeywordToken("out");
-    JetToken REF_KEYWORD       = new JetToken("ref");
+    JetSoftKeywordToken REF_KEYWORD       = new JetSoftKeywordToken("ref");
 
     TokenSet KEYWORDS = TokenSet.create(NAMESPACE_KEYWORD, IMPORT_KEYWORD, AS_KEYWORD, TYPE_KEYWORD, CLASS_KEYWORD,
             THIS_KEYWORD, VAL_KEYWORD, VAR_KEYWORD, FUN_KEYWORD, DECOMPOSER_KEYWORD, EXTENSION_KEYWORD, FOR_KEYWORD,