KT-482 Support nested comments: comments nest properly, but strings are not accounted for inside comments

#KT-482 In progress

compiler/frontend/buildLexer.xml

@@ -1,5 +1,13 @@
 <project name="JetLexer" default="lexer">
     <property name="home" value="${basedir}"/>
+<!--
+    This script relies on a custom JFlex that is available in the sources of IntelliJ IDEA Community Edition
+    put the idea.properties file to the root directory of the frontend module, and put the following property there:
+
+       idea.home=../../../idea/src
+
+    where "../../../idea/src" is the relative location of the sources of IntelliJ IDEA Community
+-->
     <property file="${home}/idea.properties"/>
 
     <property name="flex.base" value="${idea.home}/tools/lexer/jflex-1.4"/>

compiler/frontend/src/org/jetbrains/jet/lexer/Jet.flex

@@ -22,11 +22,19 @@ import org.jetbrains.jet.lexer.JetTokens;
             this.state = state;
             this.lBraceCount = lBraceCount;
         }
+
+        @Override
+        public String toString() {
+            return "yystate = " + state + (lBraceCount == 0 ? "" : "lBraceCount = " + lBraceCount);
+        }
     }
 
     private final Stack<State> states = new Stack<State>();
     private int lBraceCount;
 
+    private int commentStart;
+    private int commentDepth;
+
     private void pushState(int state) {
         states.push(new State(yystate(), lBraceCount));
         lBraceCount = 0;
@@ -38,14 +46,26 @@ import org.jetbrains.jet.lexer.JetTokens;
         lBraceCount = state.lBraceCount;
         yybegin(state.state);
     }
+
+    private IElementType commentStateToTokenType(int state) {
+        switch (state) {
+            case BLOCK_COMMENT:
+                return JetTokens.BLOCK_COMMENT;
+            case DOC_COMMENT:
+                return JetTokens.DOC_COMMENT;
+            default:
+                throw new IllegalArgumentException("Unexpected state: " + state);
+        }
+    }
 %}
 
 %function advance
 %type IElementType
-%eof{  return;
+%eof{
+  return;
 %eof}
 
-%xstate STRING RAW_STRING SHORT_TEMPLATE_ENTRY
+%xstate STRING RAW_STRING SHORT_TEMPLATE_ENTRY BLOCK_COMMENT DOC_COMMENT
 %state LONG_TEMPLATE_ENTRY
 
 DIGIT=[0-9]
@@ -63,10 +83,6 @@ IDENTIFIER = {PLAIN_IDENTIFIER}|{ESCAPED_IDENTIFIER}
 FIELD_IDENTIFIER = \${IDENTIFIER}
 LABEL_IDENTIFIER = \@{IDENTIFIER}
 
-BLOCK_COMMENT=("/*"[^"*"]{COMMENT_TAIL})|"/*"
-// TODO: Wiki markup for doc comments?
-DOC_COMMENT="/*""*"+("/"|([^"/""*"]{COMMENT_TAIL}))?
-COMMENT_TAIL=([^"*"]*("*"+[^"*""/"])?)*("*"+"/")?
 EOL_COMMENT="/""/"[^\n]*
 
 INTEGER_LITERAL={DECIMAL_INTEGER_LITERAL}|{HEX_INTEGER_LITERAL}|{BIN_INTEGER_LITERAL}
@@ -146,10 +162,48 @@ LONG_TEMPLATE_ENTRY_END=\}
                                            return JetTokens.RBRACE;
                                        }
 
-// Mere mortals
+// (Nested) comments
 
-{BLOCK_COMMENT} { return JetTokens.BLOCK_COMMENT; }
+"/**" {
-{DOC_COMMENT} { return JetTokens.DOC_COMMENT; }
+    pushState(DOC_COMMENT);
+    commentDepth = 0;
+    commentStart = getTokenStart();
+}
+
+"/*" {
+    pushState(BLOCK_COMMENT);
+    commentDepth = 0;
+    commentStart = getTokenStart();
+}
+
+<BLOCK_COMMENT, DOC_COMMENT> {
+    "/*" {
+         commentDepth++;
+    }
+
+    <<EOF>> {
+        int state = yystate();
+        popState();
+        zzStartRead = commentStart;
+        return commentStateToTokenType(state);
+    }
+
+    "*/" {
+        if (commentDepth > 0) {
+            commentDepth--;
+        }
+        else {
+             int state = yystate();
+             popState();
+             zzStartRead = commentStart;
+             return commentStateToTokenType(state);
+        }
+    }
+
+    .|{WHITE_SPACE_CHAR} {}
+}
+
+// Mere mortals
 
 ({WHITE_SPACE_CHAR})+ { return JetTokens.WHITE_SPACE; }
 
@@ -162,7 +216,6 @@ LONG_TEMPLATE_ENTRY_END=\}
 {HEX_DOUBLE_LITERAL} { return JetTokens.FLOAT_LITERAL; }
 
 {CHARACTER_LITERAL} { return JetTokens.CHARACTER_LITERAL; }
-//{STRING_LITERAL} { return JetTokens.STRING_LITERAL; }
 
 "continue"   { return JetTokens.CONTINUE_KEYWORD ;}
 "package"    { return JetTokens.PACKAGE_KEYWORD ;}

compiler/testData/psi/BlockCommentAtBeginningOfFile1.jet

@@ -0,0 +1 @@
+/*

compiler/testData/psi/BlockCommentAtBeginningOfFile1.txt

@@ -0,0 +1,4 @@
+JetFile: BlockCommentAtBeginningOfFile1.jet
+  PsiComment(BLOCK_COMMENT)('/*')
+  NAMESPACE_HEADER
+    <empty list>

compiler/testData/psi/BlockCommentAtBeginningOfFile2.jet

@@ -0,0 +1,2 @@
+/*
+/*

compiler/testData/psi/BlockCommentAtBeginningOfFile2.txt

@@ -0,0 +1,4 @@
+JetFile: BlockCommentAtBeginningOfFile2.jet
+  PsiComment(BLOCK_COMMENT)('/*\n/*')
+  NAMESPACE_HEADER
+    <empty list>

compiler/testData/psi/BlockCommentAtBeginningOfFile3.jet

@@ -0,0 +1,3 @@
+/*
+
+fooo

compiler/testData/psi/BlockCommentAtBeginningOfFile3.txt

@@ -0,0 +1,4 @@
+JetFile: BlockCommentAtBeginningOfFile3.jet
+  PsiComment(BLOCK_COMMENT)('/*\n\nfooo')
+  NAMESPACE_HEADER
+    <empty list>

compiler/testData/psi/BlockCommentAtBeginningOfFile4.jet

@@ -0,0 +1,5 @@
+/*
+
+/*foo*/
+
+asdfas

compiler/testData/psi/BlockCommentAtBeginningOfFile4.txt

@@ -0,0 +1,4 @@
+JetFile: BlockCommentAtBeginningOfFile4.jet
+  PsiComment(BLOCK_COMMENT)('/*\n\n/*foo*/\n\nasdfas')
+  NAMESPACE_HEADER
+    <empty list>

compiler/testData/psi/DocCommentAtBeginningOfFile1.jet

@@ -0,0 +1 @@
+/**

compiler/testData/psi/DocCommentAtBeginningOfFile1.txt

@@ -0,0 +1,4 @@
+JetFile: DocCommentAtBeginningOfFile1.jet
+  PsiComment(DOC_COMMENT)('/**')
+  NAMESPACE_HEADER
+    <empty list>

compiler/testData/psi/DocCommentAtBeginningOfFile2.jet

@@ -0,0 +1,2 @@
+/**
+/**

compiler/testData/psi/DocCommentAtBeginningOfFile2.txt

@@ -0,0 +1,4 @@
+JetFile: DocCommentAtBeginningOfFile2.jet
+  PsiComment(DOC_COMMENT)('/**\n/**')
+  NAMESPACE_HEADER
+    <empty list>

compiler/testData/psi/DocCommentAtBeginningOfFile3.jet

@@ -0,0 +1,3 @@
+/**
+
+fooo

compiler/testData/psi/DocCommentAtBeginningOfFile3.txt

@@ -0,0 +1,4 @@
+JetFile: DocCommentAtBeginningOfFile3.jet
+  PsiComment(DOC_COMMENT)('/**\n\nfooo')
+  NAMESPACE_HEADER
+    <empty list>

compiler/testData/psi/DocCommentAtBeginningOfFile4.jet

@@ -0,0 +1,5 @@
+/**
+
+/**foo*/
+
+asdfas

compiler/testData/psi/DocCommentAtBeginningOfFile4.txt

@@ -0,0 +1,4 @@
+JetFile: DocCommentAtBeginningOfFile4.jet
+  PsiComment(DOC_COMMENT)('/**\n\n/**foo*/\n\nasdfas')
+  NAMESPACE_HEADER
+    <empty list>

compiler/testData/psi/NestedComments.jet

@@ -0,0 +1,19 @@
+fun doo() {
+/*/b
+// */
+b
+/**/
+b
+/* */
+b
+/*/**/*/
+b
+/***/
+b
+/**/***/*/
+b
+/** /**
+
+*/***/
+b
+}

compiler/testData/psi/NestedComments.txt

@@ -0,0 +1,34 @@
+JetFile: NestedComments.jet
+  NAMESPACE_HEADER
+    <empty list>
+  FUN
+    PsiElement(fun)('fun')
+    PsiWhiteSpace(' ')
+    PsiElement(IDENTIFIER)('doo')
+    VALUE_PARAMETER_LIST
+      PsiElement(LPAR)('(')
+      PsiElement(RPAR)(')')
+    PsiWhiteSpace(' ')
+    BLOCK
+      PsiElement(LBRACE)('{')
+      PsiWhiteSpace('\n')
+      PsiComment(BLOCK_COMMENT)('/*/b\n// */')
+      PsiWhiteSpace('\n')
+      REFERENCE_EXPRESSION
+        PsiElement(IDENTIFIER)('b')
+      PsiWhiteSpace('\n')
+      PsiComment(DOC_COMMENT)('/**/\nb\n/* */\nb\n/*/**/*/\nb\n/***/\nb\n/**/***/')
+      PsiErrorElement:Expecting an element
+        PsiElement(MUL)('*')
+      PsiErrorElement:Unexpected tokens (use ';' to separate expressions on the same line)
+        PsiElement(DIV)('/')
+      PsiWhiteSpace('\n')
+      REFERENCE_EXPRESSION
+        PsiElement(IDENTIFIER)('b')
+      PsiWhiteSpace('\n')
+      PsiComment(DOC_COMMENT)('/** /**\n\n*/***/')
+      PsiWhiteSpace('\n')
+      REFERENCE_EXPRESSION
+        PsiElement(IDENTIFIER)('b')
+      PsiWhiteSpace('\n')
+      PsiElement(RBRACE)('}')