KDoc lexer refactored (got rid of hacks and separate tokens for words). Fixed bug with isInComment(element) detection.

compiler/frontend/src/org/jetbrains/jet/kdoc/lexer/KDoc.flex

@@ -23,18 +23,6 @@ import com.intellij.util.text.CharArrayUtil;
   private final Boolean yytextContainLineBreaks() {
     return CharArrayUtil.containLineBreaks(zzBuffer, zzStartRead, zzMarkedPos);
   }
-
-  private final void pushbackEnd() {
-    if (isLastToken() && CharArrayUtil.regionMatches(zzBuffer, zzMarkedPos - 2, zzMarkedPos, "*/")) {
-      yypushback(2);
-    }
-  }
-
-  private final void pushbackText() {
-    int i = zzStartRead;
-    while (zzBuffer.charAt(i) == '*') i++;
-    yypushback(zzMarkedPos - i);
-  }
 %}
 
 %function advance
@@ -54,22 +42,25 @@ NOT_WHITE_SPACE_CHAR=[^\ \t\f\n\r]
 
 <YYINITIAL> "/**"                         { yybegin(CONTENTS);
                                             return KDocTokens.START;            }
-"*"+ "/"                                  { if (isLastToken())
+"*"+ "/"                                  { if (isLastToken()) return KDocTokens.END;
-                                              return KDocTokens.END;            }
+                                            else return KDocTokens.TEXT; }
 
-// hack: make longest match
+<LINE_BEGINNING> "*"+                     { yybegin(CONTENTS);
-<LINE_BEGINNING> "*"+ {NOT_WHITE_SPACE_CHAR}* { pushbackText();
+                                            return KDocTokens.LEADING_ASTERISK; }
-                                                yybegin(CONTENTS);
-                                                return KDocTokens.LEADING_ASTERISK; }
 
 <CONTENTS, LINE_BEGINNING> {
-            // todo: put markdown and @tags token patterns here
+    {WHITE_SPACE_CHAR}+ {
+        if (yytextContainLineBreaks()) {
+            yybegin(LINE_BEGINNING);
+            return TokenType.WHITE_SPACE;
+        } else {
+            yybegin(CONTENTS);
+            return KDocTokens.TEXT;  // internal white space
+        }
+    }
 
-            {NOT_WHITE_SPACE_CHAR}+        { pushbackEnd();
+  . { yybegin(CONTENTS);
-                                             return KDocTokens.TEXT;            }
+      return KDocTokens.TEXT; }
-            {WHITE_SPACE_CHAR}+            { if (yytextContainLineBreaks())
-                                               yybegin(LINE_BEGINNING);
-                                             return TokenType.WHITE_SPACE;      }
 }
 
-.                                         { return TokenType.BAD_CHARACTER;     }
+. { return TokenType.BAD_CHARACTER; }

compiler/frontend/src/org/jetbrains/jet/kdoc/lexer/KDocLexer.java

@@ -17,11 +17,18 @@
 package org.jetbrains.jet.kdoc.lexer;
 
 import com.intellij.lexer.FlexAdapter;
+import com.intellij.lexer.MergingLexerAdapter;
+import com.intellij.psi.tree.TokenSet;
 
 import java.io.Reader;
 
-public class KDocLexer extends FlexAdapter {
+public class KDocLexer extends MergingLexerAdapter {
     public KDocLexer() {
-        super(new _KDocLexer((Reader) null));
+        super(
+                new FlexAdapter(
+                        new _KDocLexer((Reader) null)
+                ),
+                TokenSet.create(KDocTokens.TEXT)
+        );
     }
 }

compiler/frontend/src/org/jetbrains/jet/kdoc/lexer/KDocTokens.java

@@ -23,7 +23,6 @@ import com.intellij.lang.PsiParser;
 import com.intellij.openapi.project.Project;
 import com.intellij.psi.PsiElement;
 import com.intellij.psi.tree.ILazyParseableElementType;
-import com.intellij.psi.tree.TokenSet;
 import org.jetbrains.annotations.Nullable;
 import org.jetbrains.jet.kdoc.parser.KDocParser;
 import org.jetbrains.jet.kdoc.psi.impl.KDocImpl;
@@ -53,7 +52,5 @@ public interface KDocTokens {
     KDocToken END                   = new KDocToken("KDOC_END");
     KDocToken LEADING_ASTERISK      = new KDocToken("KDOC_LEADING_ASTERISK");
 
-    KDocToken TEXT      = new KDocToken("KDOC_TEXT");
+    KDocToken TEXT                  = new KDocToken("KDOC_TEXT");
-
-    TokenSet ALL_KDOC_TOKENS = TokenSet.create(START, END, LEADING_ASTERISK, TEXT);
 }

compiler/frontend/src/org/jetbrains/jet/kdoc/lexer/_KDocLexer.java

@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.4.3 on 23.05.13 19:26 */
+/* The following code was generated by JFlex 1.4.3 on 25.05.13 15:08 */
 
 package org.jetbrains.jet.kdoc.lexer;
 
@@ -11,7 +11,7 @@ import com.intellij.util.text.CharArrayUtil;
 /**
  * This class is a scanner generated by 
  * <a href="http://www.jflex.de/">JFlex</a> 1.4.3
- * on 23.05.13 19:26 from the specification file
+ * on 25.05.13 15:08 from the specification file
  * <tt>/Users/factitious/Documents/kotlin/compiler/frontend/src/org/jetbrains/jet/kdoc/lexer/KDoc.flex</tt>
  */
 class _KDocLexer implements FlexLexer {
@@ -19,9 +19,11 @@ class _KDocLexer implements FlexLexer {
   private static final int ZZ_BUFFERSIZE = 16384;
 
   /** lexical states */
-  public static final int LINE_BEGINNING = 4;
+  public static final int LINE_BEGINNING = 8;
+  public static final int STRONG_CONTENTS = 6;
   public static final int CONTENTS = 2;
   public static final int YYINITIAL = 0;
+  public static final int EMPHASIS_CONTENTS = 4;
 
   /**
    * ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l
@@ -30,7 +32,7 @@ class _KDocLexer implements FlexLexer {
    * l is of the form l = 2*k, k a non negative integer
    */
   private static final int ZZ_LEXSTATE[] = { 
-     0,  0,  1,  1,  2, 2
+     0,  0,  1,  1,  1,  1,  1,  1,  2, 2
   };
 
   /** 
@@ -52,10 +54,10 @@ class _KDocLexer implements FlexLexer {
 
   private static final String ZZ_ACTION_PACKED_0 =
     "\3\0\3\1\1\2\1\3\1\2\1\4\1\0\1\5"+
-    "\1\0\1\5\1\4\1\5\1\6";
+    "\1\0\1\6";
 
   private static int [] zzUnpackAction() {
-    int [] result = new int[17];
+    int [] result = new int[14];
     int offset = 0;
     offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result);
     return result;
@@ -80,12 +82,11 @@ class _KDocLexer implements FlexLexer {
   private static final int [] ZZ_ROWMAP = zzUnpackRowMap();
 
   private static final String ZZ_ROWMAP_PACKED_0 =
-    "\0\0\0\5\0\12\0\17\0\24\0\31\0\36\0\43"+
+    "\0\0\0\5\0\12\0\17\0\24\0\31\0\17\0\36"+
-    "\0\50\0\55\0\62\0\17\0\31\0\36\0\67\0\67"+
+    "\0\31\0\43\0\50\0\17\0\31\0\17";
-    "\0\17";
 
   private static int [] zzUnpackRowMap() {
-    int [] result = new int[17];
+    int [] result = new int[14];
     int offset = 0;
     offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result);
     return result;
@@ -110,13 +111,11 @@ class _KDocLexer implements FlexLexer {
   private static final String ZZ_TRANS_PACKED_0 =
     "\2\4\1\5\1\6\1\0\1\7\1\10\1\7\1\11"+
     "\1\10\1\7\1\10\1\7\1\12\1\10\10\0\1\13"+
-    "\3\0\1\14\1\15\1\0\1\7\1\0\2\7\2\0"+
+    "\3\0\1\14\1\15\2\0\1\10\2\0\1\10\2\0"+
-    "\1\10\2\0\1\10\1\7\1\0\1\16\1\11\1\0"+
+    "\1\14\1\12\4\0\1\16\1\0";
-    "\1\17\1\0\1\20\1\12\4\0\1\21\1\0\1\17"+
-    "\1\0\2\17\1\0";
 
   private static int [] zzUnpackTrans() {
-    int [] result = new int[60];
+    int [] result = new int[45];
     int offset = 0;
     offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result);
     return result;
@@ -157,10 +156,11 @@ class _KDocLexer implements FlexLexer {
   private static final int [] ZZ_ATTRIBUTE = zzUnpackAttribute();
 
   private static final String ZZ_ATTRIBUTE_PACKED_0 =
-    "\3\0\1\11\6\1\1\0\1\11\1\0\3\1\1\11";
+    "\3\0\1\11\2\1\1\11\3\1\1\0\1\11\1\0"+
+    "\1\11";
 
   private static int [] zzUnpackAttribute() {
-    int [] result = new int[17];
+    int [] result = new int[14];
     int offset = 0;
     offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result);
     return result;
@@ -231,18 +231,6 @@ class _KDocLexer implements FlexLexer {
     return CharArrayUtil.containLineBreaks(zzBuffer, zzStartRead, zzMarkedPos);
   }
 
-  private final void pushbackEnd() {
-    if (isLastToken() && CharArrayUtil.regionMatches(zzBuffer, zzMarkedPos - 2, zzMarkedPos, "*/")) {
-      yypushback(2);
-    }
-  }
-
-  private final void pushbackText() {
-    int i = zzStartRead;
-    while (zzBuffer.charAt(i) == '*') i++;
-    yypushback(zzMarkedPos - i);
-  }
-
 
   _KDocLexer(java.io.Reader in) {
     this.zzReader = in;
@@ -497,8 +485,8 @@ class _KDocLexer implements FlexLexer {
           }
         case 7: break;
         case 5: 
-          { if (isLastToken())
+          { if (isLastToken()) return KDocTokens.END;
-                                              return KDocTokens.END;
+                                            else return KDocTokens.TEXT;
           }
         case 8: break;
         case 1: 
@@ -506,20 +494,23 @@ class _KDocLexer implements FlexLexer {
           }
         case 9: break;
         case 2: 
-          { pushbackEnd();
+          { yybegin(CONTENTS);
-                                             return KDocTokens.TEXT;
+      return KDocTokens.TEXT;
           }
         case 10: break;
         case 4: 
-          { pushbackText();
+          { yybegin(CONTENTS);
-                                                yybegin(CONTENTS);
+                                            return KDocTokens.LEADING_ASTERISK;
-                                                return KDocTokens.LEADING_ASTERISK;
           }
         case 11: break;
         case 3: 
-          { if (yytextContainLineBreaks())
+          { if (yytextContainLineBreaks()) {
-                                               yybegin(LINE_BEGINNING);
+            yybegin(LINE_BEGINNING);
-                                             return TokenType.WHITE_SPACE;
+            return TokenType.WHITE_SPACE;
+        } else {
+            yybegin(CONTENTS);
+            return KDocTokens.TEXT;  // internal white space
+        }
           }
         case 12: break;
         default:

compiler/frontend/src/org/jetbrains/jet/kdoc/psi/api/KDocElement.java

@@ -16,7 +16,7 @@
 
 package org.jetbrains.jet.kdoc.psi.api;
 
-import org.jetbrains.jet.lang.psi.JetElement;
+import com.intellij.psi.PsiElement;
 
-public interface KDocElement extends JetElement {
+public interface KDocElement extends PsiElement {
 }

compiler/frontend/src/org/jetbrains/jet/lang/psi/JetPsiUtil.java

@@ -28,10 +28,12 @@ import com.intellij.psi.impl.CheckUtil;
 import com.intellij.psi.impl.source.codeStyle.CodeEditUtil;
 import com.intellij.psi.tree.IElementType;
 import com.intellij.psi.util.PsiTreeUtil;
+import com.intellij.util.codeInsight.CommentUtilCore;
 import com.intellij.util.containers.ContainerUtil;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.jetbrains.jet.JetNodeTypes;
+import org.jetbrains.jet.kdoc.psi.api.KDocElement;
 import org.jetbrains.jet.lang.parsing.JetExpressionParsing;
 import org.jetbrains.jet.lang.resolve.ImportPath;
 import org.jetbrains.jet.lang.resolve.name.FqName;
@@ -740,4 +742,17 @@ public class JetPsiUtil {
     public static String getText(@Nullable PsiElement element) {
         return element != null ? element.getText() : "";
     }
+
+    /**
+     * CommentUtilCore.isComment fails if element <strong>inside</strong> comment.
+     *
+     * Also, we can not add KDocTokens to COMMENTS TokenSet, because it is used in JetParserDefinition.getCommentTokens(),
+     * and therefor all COMMENTS tokens will be ignored by PsiBuilder.
+     *
+     * @param element
+     * @return
+     */
+    public static boolean isInComment(PsiElement element) {
+        return CommentUtilCore.isComment(element) || element instanceof KDocElement;
+    }
 }

compiler/frontend/src/org/jetbrains/jet/lexer/JetTokens.java

@@ -184,8 +184,14 @@ public interface JetTokens {
             PROTECTED_KEYWORD, OUT_KEYWORD, IN_KEYWORD, FINAL_KEYWORD, VARARG_KEYWORD, INLINE_KEYWORD, REIFIED_KEYWORD
     );
     TokenSet WHITESPACES = TokenSet.create(TokenType.WHITE_SPACE);
-    TokenSet COMMENTS = TokenSet.orSet(TokenSet.create(EOL_COMMENT, BLOCK_COMMENT, DOC_COMMENT, SHEBANG_COMMENT),
+
-                                       KDocTokens.ALL_KDOC_TOKENS);
+    /**
+     * Don't add KDocTokens to COMMENTS TokenSet, because it is used in JetParserDefinition.getCommentTokens(),
+     * and therefor all COMMENTS tokens will be ignored by PsiBuilder.
+     *
+     * @see org.jetbrains.jet.lang.psi.JetPsiUtil.isInComment()
+     */
+    TokenSet COMMENTS = TokenSet.create(EOL_COMMENT, BLOCK_COMMENT, DOC_COMMENT, SHEBANG_COMMENT);
     TokenSet WHITE_SPACE_OR_COMMENT_BIT_SET = TokenSet.orSet(COMMENTS, TokenSet.create(WHITE_SPACE));
 
     TokenSet STRINGS = TokenSet.create(CHARACTER_LITERAL, REGULAR_STRING_PART);

compiler/frontend/src/org/jetbrains/jet/lexer/_JetLexer.java

@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.4.3 on 23.05.13 19:26 */
+/* The following code was generated by JFlex 1.4.3 on 25.05.13 15:08 */
 
 package org.jetbrains.jet.lexer;
 
@@ -11,7 +11,7 @@ import com.intellij.util.containers.Stack;
 /**
  * This class is a scanner generated by 
  * <a href="http://www.jflex.de/">JFlex</a> 1.4.3
- * on 23.05.13 19:26 from the specification file
+ * on 25.05.13 15:08 from the specification file
  * <tt>/Users/factitious/Documents/kotlin/compiler/frontend/src/org/jetbrains/jet/lexer/Jet.flex</tt>
  */
 class _JetLexer implements FlexLexer {

compiler/testData/psi/EOLsInComments.txt

@@ -27,7 +27,7 @@ JetFile: EOLsInComments.jet
       PsiWhiteSpace('\n  ')
       KDoc
         PsiElement(KDOC_START)('/**')
-        PsiWhiteSpace(' ')
+        PsiElement(KDOC_TEXT)(' ')
         PsiElement(KDOC_END)('*/')
       PREFIX_EXPRESSION
         OPERATION_REFERENCE

compiler/testData/psi/NestedComments.txt

@@ -41,8 +41,7 @@ JetFile: NestedComments.jet
       PsiWhiteSpace('\n')
       KDoc
         PsiElement(KDOC_START)('/**')
-        PsiWhiteSpace(' ')
+        PsiElement(KDOC_TEXT)(' /***/')
-        PsiElement(KDOC_TEXT)('/***/')
         PsiElement(KDOC_END)('*/')
       PsiWhiteSpace('\n')
       REFERENCE_EXPRESSION
@@ -50,12 +49,10 @@ JetFile: NestedComments.jet
       PsiWhiteSpace('\n')
       KDoc
         PsiElement(KDOC_START)('/**')
-        PsiWhiteSpace(' ')
+        PsiElement(KDOC_TEXT)(' /**')
-        PsiElement(KDOC_TEXT)('/**')
         PsiWhiteSpace('\n\n')
-        PsiElement(KDOC_LEADING_ASTERISK)('*')
+        PsiElement(KDOC_TEXT)('*/')
-        PsiElement(KDOC_TEXT)('/**')
+        PsiElement(KDOC_END)('***/')
-        PsiElement(KDOC_END)('*/')
       PsiWhiteSpace('\n')
       REFERENCE_EXPRESSION
         PsiElement(IDENTIFIER)('b')

compiler/testData/psi/examples/array/MutableArray.txt

@@ -2,41 +2,7 @@ JetFile: MutableArray.jet
   KDoc
     PsiElement(KDOC_START)('/**')
     PsiWhiteSpace('\n  ')
-    PsiElement(KDOC_TEXT)('These')
+    PsiElement(KDOC_TEXT)('These declarations are "shallow" in the sense that they are not really compiled, only the type-checker uses them')
-    PsiWhiteSpace(' ')
-    PsiElement(KDOC_TEXT)('declarations')
-    PsiWhiteSpace(' ')
-    PsiElement(KDOC_TEXT)('are')
-    PsiWhiteSpace(' ')
-    PsiElement(KDOC_TEXT)('"shallow"')
-    PsiWhiteSpace(' ')
-    PsiElement(KDOC_TEXT)('in')
-    PsiWhiteSpace(' ')
-    PsiElement(KDOC_TEXT)('the')
-    PsiWhiteSpace(' ')
-    PsiElement(KDOC_TEXT)('sense')
-    PsiWhiteSpace(' ')
-    PsiElement(KDOC_TEXT)('that')
-    PsiWhiteSpace(' ')
-    PsiElement(KDOC_TEXT)('they')
-    PsiWhiteSpace(' ')
-    PsiElement(KDOC_TEXT)('are')
-    PsiWhiteSpace(' ')
-    PsiElement(KDOC_TEXT)('not')
-    PsiWhiteSpace(' ')
-    PsiElement(KDOC_TEXT)('really')
-    PsiWhiteSpace(' ')
-    PsiElement(KDOC_TEXT)('compiled,')
-    PsiWhiteSpace(' ')
-    PsiElement(KDOC_TEXT)('only')
-    PsiWhiteSpace(' ')
-    PsiElement(KDOC_TEXT)('the')
-    PsiWhiteSpace(' ')
-    PsiElement(KDOC_TEXT)('type-checker')
-    PsiWhiteSpace(' ')
-    PsiElement(KDOC_TEXT)('uses')
-    PsiWhiteSpace(' ')
-    PsiElement(KDOC_TEXT)('them')
     PsiWhiteSpace('\n')
     PsiElement(KDOC_END)('*/')
   PsiWhiteSpace('\n\n')

compiler/testData/psi/kdoc/Simple.txt

@@ -1,52 +1,23 @@
 JetFile: Simple.kt
   KDoc
     PsiElement(KDOC_START)('/**')
-    PsiWhiteSpace(' ')
+    PsiElement(KDOC_TEXT)(' line 0')
-    PsiElement(KDOC_TEXT)('line')
-    PsiWhiteSpace(' ')
-    PsiElement(KDOC_TEXT)('0')
     PsiWhiteSpace('\n  ')
-    PsiElement(KDOC_TEXT)('line')
+    PsiElement(KDOC_TEXT)('line 1      //')
-    PsiWhiteSpace(' ')
-    PsiElement(KDOC_TEXT)('1')
-    PsiWhiteSpace('      ')
-    PsiElement(KDOC_TEXT)('//')
     PsiWhiteSpace('\n   ')
     PsiElement(KDOC_LEADING_ASTERISK)('**')
-    PsiWhiteSpace(' ')
+    PsiElement(KDOC_TEXT)(' line 2  /*')
-    PsiElement(KDOC_TEXT)('line')
-    PsiWhiteSpace(' ')
-    PsiElement(KDOC_TEXT)('2')
-    PsiWhiteSpace('  ')
-    PsiElement(KDOC_TEXT)('/*')
     PsiWhiteSpace('\n    ')
-    PsiElement(KDOC_TEXT)('line')
+    PsiElement(KDOC_TEXT)('line 3    /**')
-    PsiWhiteSpace(' ')
-    PsiElement(KDOC_TEXT)('3')
-    PsiWhiteSpace('    ')
-    PsiElement(KDOC_TEXT)('/**')
     PsiWhiteSpace('\n')
     PsiElement(KDOC_LEADING_ASTERISK)('*')
-    PsiWhiteSpace(' ')
+    PsiElement(KDOC_TEXT)(' line * 4')
-    PsiElement(KDOC_TEXT)('line')
-    PsiWhiteSpace(' ')
-    PsiElement(KDOC_TEXT)('*')
-    PsiWhiteSpace(' ')
-    PsiElement(KDOC_TEXT)('4')
     PsiWhiteSpace('\n    ')
     PsiElement(KDOC_LEADING_ASTERISK)('***')
-    PsiWhiteSpace(' ')
+    PsiElement(KDOC_TEXT)(' line */ 5')
-    PsiElement(KDOC_TEXT)('line')
-    PsiWhiteSpace(' */')
-    PsiWhiteSpace(' ')
-    PsiElement(KDOC_TEXT)('5')
     PsiWhiteSpace('\n    ')
     PsiElement(KDOC_LEADING_ASTERISK)('**')
-    PsiWhiteSpace(' ')
+    PsiElement(KDOC_TEXT)(' line 6 ')
-    PsiElement(KDOC_TEXT)('line')
-    PsiWhiteSpace(' ')
-    PsiElement(KDOC_TEXT)('6')
-    PsiWhiteSpace(' ')
     PsiElement(KDOC_END)('*/')
   NAMESPACE_HEADER
     <empty list>

idea/src/org/jetbrains/jet/plugin/completion/JetKeywordCompletionContributor.java

@@ -36,7 +36,6 @@ import com.intellij.psi.impl.source.tree.LeafPsiElement;
 import com.intellij.psi.util.PsiTreeUtil;
 import com.intellij.util.ArrayUtil;
 import com.intellij.util.ProcessingContext;
-import com.intellij.util.codeInsight.CommentUtilCore;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.jet.lang.psi.*;
 import org.jetbrains.jet.lexer.JetToken;
@@ -103,7 +102,7 @@ public class JetKeywordCompletionContributor extends CompletionContributor {
                 return false;
             }
 
-            return CommentUtilCore.isComment((PsiElement) element);
+            return JetPsiUtil.isInComment((PsiElement) element);
         }
 
         @Override

idea/src/org/jetbrains/jet/plugin/formatter/JetFormattingModelBuilder.java

@@ -25,7 +25,6 @@ import com.intellij.psi.codeStyle.CodeStyleSettings;
 import com.intellij.psi.codeStyle.CommonCodeStyleSettings;
 import com.intellij.psi.tree.TokenSet;
 import org.jetbrains.annotations.NotNull;
-import org.jetbrains.jet.kdoc.lexer.KDocTokens;
 import org.jetbrains.jet.plugin.JetLanguage;
 
 import static org.jetbrains.jet.JetNodeTypes.*;
@@ -108,9 +107,6 @@ public class JetFormattingModelBuilder implements FormattingModelBuilder {
 
                 .between(VALUE_ARGUMENT_LIST, FUNCTION_LITERAL_EXPRESSION).spaces(1)
                 .aroundInside(ARROW, WHEN_ENTRY).spaces(1)
-
-                // KDoc
-                .between(KDocTokens.LEADING_ASTERISK, KDocTokens.TEXT).spacing(1, 100, 0, true, 100)
                 ;
     }