KT-482 Support nested comments: comments nest properly, but strings are not accounted for inside comments

#KT-482 In progress
2012-03-21 15:51:48 +01:00
parent 01679d315d
commit 3a73a96e06
21 changed files with 749 additions and 491 deletions
@@ -1,5 +1,13 @@
 <project name="JetLexer" default="lexer">
    <property name="home" value="${basedir}"/>
+<!--
+    This script relies on a custom JFlex that is available in the sources of IntelliJ IDEA Community Edition
+    put the idea.properties file to the root directory of the frontend module, and put the following property there:
+
+       idea.home=../../../idea/src
+
+    where "../../../idea/src" is the relative location of the sources of IntelliJ IDEA Community
+-->
    <property file="${home}/idea.properties"/>

    <property name="flex.base" value="${idea.home}/tools/lexer/jflex-1.4"/>
@@ -22,11 +22,19 @@ import org.jetbrains.jet.lexer.JetTokens;
            this.state = state;
            this.lBraceCount = lBraceCount;
        }
+
+        @Override
+        public String toString() {
+            return "yystate = " + state + (lBraceCount == 0 ? "" : "lBraceCount = " + lBraceCount);
+        }
    }

    private final Stack<State> states = new Stack<State>();
    private int lBraceCount;

+    private int commentStart;
+    private int commentDepth;
+
    private void pushState(int state) {
        states.push(new State(yystate(), lBraceCount));
        lBraceCount = 0;
@@ -38,14 +46,26 @@ import org.jetbrains.jet.lexer.JetTokens;
        lBraceCount = state.lBraceCount;
        yybegin(state.state);
    }
+
+    private IElementType commentStateToTokenType(int state) {
+        switch (state) {
+            case BLOCK_COMMENT:
+                return JetTokens.BLOCK_COMMENT;
+            case DOC_COMMENT:
+                return JetTokens.DOC_COMMENT;
+            default:
+                throw new IllegalArgumentException("Unexpected state: " + state);
+        }
+    }
 %}

 %function advance
 %type IElementType
-%eof{  return;
+%eof{
+  return;
 %eof}

-%xstate STRING RAW_STRING SHORT_TEMPLATE_ENTRY
+%xstate STRING RAW_STRING SHORT_TEMPLATE_ENTRY BLOCK_COMMENT DOC_COMMENT
 %state LONG_TEMPLATE_ENTRY

 DIGIT=[0-9]
@@ -63,10 +83,6 @@ IDENTIFIER = {PLAIN_IDENTIFIER}|{ESCAPED_IDENTIFIER}
 FIELD_IDENTIFIER = \${IDENTIFIER}
 LABEL_IDENTIFIER = \@{IDENTIFIER}

-BLOCK_COMMENT=("/*"[^"*"]{COMMENT_TAIL})|"/*"
-// TODO: Wiki markup for doc comments?
-DOC_COMMENT="/*""*"+("/"|([^"/""*"]{COMMENT_TAIL}))?
-COMMENT_TAIL=([^"*"]*("*"+[^"*""/"])?)*("*"+"/")?
 EOL_COMMENT="/""/"[^\n]*

 INTEGER_LITERAL={DECIMAL_INTEGER_LITERAL}|{HEX_INTEGER_LITERAL}|{BIN_INTEGER_LITERAL}
@@ -146,10 +162,48 @@ LONG_TEMPLATE_ENTRY_END=\}
                                           return JetTokens.RBRACE;
                                       }

-// Mere mortals
+// (Nested) comments

-{BLOCK_COMMENT} { return JetTokens.BLOCK_COMMENT; }
-{DOC_COMMENT} { return JetTokens.DOC_COMMENT; }
+"/**" {
+    pushState(DOC_COMMENT);
+    commentDepth = 0;
+    commentStart = getTokenStart();
+}
+
+"/*" {
+    pushState(BLOCK_COMMENT);
+    commentDepth = 0;
+    commentStart = getTokenStart();
+}
+
+<BLOCK_COMMENT, DOC_COMMENT> {
+    "/*" {
+         commentDepth++;
+    }
+
+    <<EOF>> {
+        int state = yystate();
+        popState();
+        zzStartRead = commentStart;
+        return commentStateToTokenType(state);
+    }
+
+    "*/" {
+        if (commentDepth > 0) {
+            commentDepth--;
+        }
+        else {
+             int state = yystate();
+             popState();
+             zzStartRead = commentStart;
+             return commentStateToTokenType(state);
+        }
+    }
+
+    .|{WHITE_SPACE_CHAR} {}
+}
+
+// Mere mortals

 ({WHITE_SPACE_CHAR})+ { return JetTokens.WHITE_SPACE; }

@@ -162,7 +216,6 @@ LONG_TEMPLATE_ENTRY_END=\}
 {HEX_DOUBLE_LITERAL} { return JetTokens.FLOAT_LITERAL; }

 {CHARACTER_LITERAL} { return JetTokens.CHARACTER_LITERAL; }
-//{STRING_LITERAL} { return JetTokens.STRING_LITERAL; }

 "continue"   { return JetTokens.CONTINUE_KEYWORD ;}
 "package"    { return JetTokens.PACKAGE_KEYWORD ;}
@@ -0,0 +1 @@
+/*
@@ -0,0 +1,4 @@
+JetFile: BlockCommentAtBeginningOfFile1.jet
+  PsiComment(BLOCK_COMMENT)('/*')
+  NAMESPACE_HEADER
+    <empty list>
@@ -0,0 +1,2 @@
+/*
+/*
@@ -0,0 +1,4 @@
+JetFile: BlockCommentAtBeginningOfFile2.jet
+  PsiComment(BLOCK_COMMENT)('/*\n/*')
+  NAMESPACE_HEADER
+    <empty list>
@@ -0,0 +1,3 @@
+/*
+
+fooo
@@ -0,0 +1,4 @@
+JetFile: BlockCommentAtBeginningOfFile3.jet
+  PsiComment(BLOCK_COMMENT)('/*\n\nfooo')
+  NAMESPACE_HEADER
+    <empty list>
@@ -0,0 +1,5 @@
+/*
+
+/*foo*/
+
+asdfas
@@ -0,0 +1,4 @@
+JetFile: BlockCommentAtBeginningOfFile4.jet
+  PsiComment(BLOCK_COMMENT)('/*\n\n/*foo*/\n\nasdfas')
+  NAMESPACE_HEADER
+    <empty list>
@@ -0,0 +1 @@
+/**
@@ -0,0 +1,4 @@
+JetFile: DocCommentAtBeginningOfFile1.jet
+  PsiComment(DOC_COMMENT)('/**')
+  NAMESPACE_HEADER
+    <empty list>
@@ -0,0 +1,2 @@
+/**
+/**
@@ -0,0 +1,4 @@
+JetFile: DocCommentAtBeginningOfFile2.jet
+  PsiComment(DOC_COMMENT)('/**\n/**')
+  NAMESPACE_HEADER
+    <empty list>
@@ -0,0 +1,3 @@
+/**
+
+fooo
@@ -0,0 +1,4 @@
+JetFile: DocCommentAtBeginningOfFile3.jet
+  PsiComment(DOC_COMMENT)('/**\n\nfooo')
+  NAMESPACE_HEADER
+    <empty list>
@@ -0,0 +1,5 @@
+/**
+
+/**foo*/
+
+asdfas
@@ -0,0 +1,4 @@
+JetFile: DocCommentAtBeginningOfFile4.jet
+  PsiComment(DOC_COMMENT)('/**\n\n/**foo*/\n\nasdfas')
+  NAMESPACE_HEADER
+    <empty list>
@@ -0,0 +1,19 @@
+fun doo() {
+/*/b
+// */
+b
+/**/
+b
+/* */
+b
+/*/**/*/
+b
+/***/
+b
+/**/***/*/
+b
+/** /**
+
+*/***/
+b
+}
@@ -0,0 +1,34 @@
+JetFile: NestedComments.jet
+  NAMESPACE_HEADER
+    <empty list>
+  FUN
+    PsiElement(fun)('fun')
+    PsiWhiteSpace(' ')
+    PsiElement(IDENTIFIER)('doo')
+    VALUE_PARAMETER_LIST
+      PsiElement(LPAR)('(')
+      PsiElement(RPAR)(')')
+    PsiWhiteSpace(' ')
+    BLOCK
+      PsiElement(LBRACE)('{')
+      PsiWhiteSpace('\n')
+      PsiComment(BLOCK_COMMENT)('/*/b\n// */')
+      PsiWhiteSpace('\n')
+      REFERENCE_EXPRESSION
+        PsiElement(IDENTIFIER)('b')
+      PsiWhiteSpace('\n')
+      PsiComment(DOC_COMMENT)('/**/\nb\n/* */\nb\n/*/**/*/\nb\n/***/\nb\n/**/***/')
+      PsiErrorElement:Expecting an element
+        PsiElement(MUL)('*')
+      PsiErrorElement:Unexpected tokens (use ';' to separate expressions on the same line)
+        PsiElement(DIV)('/')
+      PsiWhiteSpace('\n')
+      REFERENCE_EXPRESSION
+        PsiElement(IDENTIFIER)('b')
+      PsiWhiteSpace('\n')
+      PsiComment(DOC_COMMENT)('/** /**\n\n*/***/')
+      PsiWhiteSpace('\n')
+      REFERENCE_EXPRESSION
+        PsiElement(IDENTIFIER)('b')
+      PsiWhiteSpace('\n')
+      PsiElement(RBRACE)('}')