add a token for inline Markdown links; add test for identifier links with labels

2015-04-09 14:49:16 +02:00
parent 2aba59cbec
commit 5bbbadeb37
7 changed files with 92 additions and 53 deletions
@@ -139,6 +139,11 @@ CODE_LINK=\[{QUALIFIED_NAME}\]
    "\\"[\[\]]   { yybegin(CONTENTS);
                   return KDocTokens.MARKDOWN_ESCAPED_CHAR; }

+    "[" [^\[]* "](" [^)]* ")" {
+        yybegin(CONTENTS);
+        return KDocTokens.MARKDOWN_INLINE_LINK;
+    }
+
    /* We're only interested in parsing links that can become code references,
       meaning they contain only identifier characters and characters that can be
       used in type declarations. No brackets, backticks, asterisks or anything like that.
@@ -65,7 +65,8 @@ public interface KDocTokens {
    };

    KDocToken MARKDOWN_ESCAPED_CHAR = new KDocToken("KDOC_MARKDOWN_ESCAPED_CHAR");
+    KDocToken MARKDOWN_INLINE_LINK = new KDocToken("KDOC_MARKDOWN_INLINE_LINK");

-    TokenSet KDOC_HIGHLIGHT_TOKENS = TokenSet.create(START, END, LEADING_ASTERISK, TEXT, MARKDOWN_LINK, MARKDOWN_ESCAPED_CHAR);
-    TokenSet CONTENT_TOKENS = TokenSet.create(TEXT, TAG_NAME, MARKDOWN_LINK, MARKDOWN_ESCAPED_CHAR);
+    TokenSet KDOC_HIGHLIGHT_TOKENS = TokenSet.create(START, END, LEADING_ASTERISK, TEXT, MARKDOWN_LINK, MARKDOWN_ESCAPED_CHAR, MARKDOWN_INLINE_LINK);
+    TokenSet CONTENT_TOKENS = TokenSet.create(TEXT, TAG_NAME, MARKDOWN_LINK, MARKDOWN_ESCAPED_CHAR, MARKDOWN_INLINE_LINK);
 }
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.4.3 on 3/5/15 7:51 PM */
+/* The following code was generated by JFlex 1.4.3 on 4/9/15 2:44 PM */

 package org.jetbrains.kotlin.kdoc.lexer;

@@ -14,7 +14,7 @@ import org.jetbrains.kotlin.kdoc.parser.KDocKnownTag;
 /**
 * This class is a scanner generated by 
 * <a href="http://www.jflex.de/">JFlex</a> 1.4.3
- * on 3/5/15 7:51 PM from the specification file
+ * on 4/9/15 2:44 PM from the specification file
 * <tt>/Users/yole/jetbrains/kotlin/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/KDoc.flex</tt>
 */
 class _KDocLexer implements FlexLexer {
@@ -44,7 +44,7 @@ class _KDocLexer implements FlexLexer {
   */
  private static final String ZZ_CMAP_PACKED = 
    "\11\0\1\1\1\12\1\0\2\1\22\0\1\1\3\0\1\3\3\0"+
-    "\1\14\1\0\1\10\3\0\1\4\1\7\12\2\6\0\1\11\32\3"+
+    "\1\14\1\15\1\10\3\0\1\4\1\7\12\2\6\0\1\11\32\3"+
    "\1\5\1\13\1\6\1\0\1\3\1\0\32\3\47\0\4\3\4\0"+
    "\1\3\12\0\1\3\4\0\1\3\5\0\27\3\1\0\37\3\1\0"+
    "\u013f\3\31\0\162\3\4\0\14\3\16\0\5\3\11\0\1\3\213\0"+
@@ -117,11 +117,11 @@ class _KDocLexer implements FlexLexer {

  private static final String ZZ_ACTION_PACKED_0 =
    "\6\0\3\1\1\2\1\3\1\2\1\4\3\2\1\5"+
-    "\1\6\1\7\3\5\1\0\1\10\2\0\1\11\1\12"+
-    "\2\0\1\13\1\0\1\14\1\15\1\16";
+    "\1\6\1\7\3\5\1\0\1\10\4\0\1\11\1\12"+
+    "\2\0\1\13\2\0\1\14\1\15\2\16\1\0\2\17";

  private static int [] zzUnpackAction() {
-    int [] result = new int[35];
+    int [] result = new int[42];
    int offset = 0;
    offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result);
    return result;
@@ -146,14 +146,15 @@ class _KDocLexer implements FlexLexer {
  private static final int [] ZZ_ROWMAP = zzUnpackRowMap();

  private static final String ZZ_ROWMAP_PACKED_0 =
-    "\0\0\0\15\0\32\0\47\0\64\0\101\0\116\0\133"+
-    "\0\150\0\116\0\165\0\202\0\217\0\234\0\150\0\251"+
-    "\0\116\0\266\0\303\0\320\0\150\0\335\0\352\0\116"+
-    "\0\150\0\367\0\116\0\u0104\0\u0111\0\u011e\0\116\0\u012b"+
-    "\0\116\0\116\0\116";
+    "\0\0\0\16\0\34\0\52\0\70\0\106\0\124\0\142"+
+    "\0\160\0\124\0\176\0\214\0\232\0\250\0\160\0\266"+
+    "\0\124\0\304\0\322\0\340\0\160\0\356\0\374\0\124"+
+    "\0\160\0\u010a\0\u0118\0\u0126\0\124\0\u0134\0\u0142\0\u0150"+
+    "\0\124\0\u015e\0\u016c\0\124\0\124\0\u010a\0\u0126\0\u017a"+
+    "\0\u010a\0\124";

  private static int [] zzUnpackRowMap() {
-    int [] result = new int[35];
+    int [] result = new int[42];
    int offset = 0;
    offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result);
    return result;
@@ -176,22 +177,25 @@ class _KDocLexer implements FlexLexer {
  private static final int [] ZZ_TRANS = zzUnpackTrans();

  private static final String ZZ_TRANS_PACKED_0 =
-    "\7\7\1\10\1\11\1\7\1\0\2\7\1\12\1\13"+
-    "\3\12\1\14\2\12\1\15\1\12\1\13\1\16\2\12"+
+    "\7\7\1\10\1\11\1\7\1\0\3\7\1\12\1\13"+
+    "\3\12\1\14\2\12\1\15\1\12\1\13\1\16\3\12"+
    "\1\13\3\12\1\14\2\12\1\17\1\20\1\13\1\16"+
-    "\1\12\1\21\1\22\1\21\1\23\1\21\1\24\2\21"+
-    "\1\25\1\21\1\22\3\21\1\22\3\21\1\26\2\21"+
-    "\1\25\1\21\1\22\2\21\1\12\1\13\3\12\1\14"+
-    "\2\12\1\17\1\12\1\13\1\16\1\12\25\0\1\27"+
-    "\13\0\1\30\1\31\5\0\1\13\10\0\1\13\5\0"+
-    "\1\32\20\0\1\30\1\15\11\0\2\33\11\0\1\34"+
-    "\12\0\1\22\10\0\1\22\4\0\3\23\13\0\1\35"+
-    "\14\0\1\36\21\0\1\37\6\0\3\32\1\0\1\40"+
-    "\10\0\2\34\13\0\3\35\1\0\1\41\10\0\3\36"+
-    "\1\0\1\42\6\0\5\43\1\0\6\43\1\0";
+    "\2\12\1\21\1\22\1\21\1\23\1\21\1\24\2\21"+
+    "\1\25\1\21\1\22\4\21\1\22\3\21\1\26\2\21"+
+    "\1\25\1\21\1\22\3\21\1\12\1\13\3\12\1\14"+
+    "\2\12\1\17\1\12\1\13\1\16\2\12\26\0\1\27"+
+    "\14\0\1\30\1\31\6\0\1\13\10\0\1\13\3\0"+
+    "\3\32\1\33\1\32\1\0\1\34\7\32\7\0\1\30"+
+    "\1\15\12\0\2\35\12\0\1\36\13\0\1\22\10\0"+
+    "\1\22\5\0\3\23\14\0\1\37\15\0\1\40\22\0"+
+    "\1\41\5\0\5\32\1\0\1\34\11\32\3\33\1\0"+
+    "\1\42\14\32\1\0\1\34\5\32\1\43\1\32\2\0"+
+    "\2\36\14\0\3\37\1\0\1\44\11\0\3\40\1\0"+
+    "\1\45\7\0\5\46\1\0\1\47\5\46\1\43\1\46"+
+    "\5\43\1\50\7\43\1\51\15\50\1\52";

  private static int [] zzUnpackTrans() {
-    int [] result = new int[312];
+    int [] result = new int[392];
    int offset = 0;
    offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result);
    return result;
@@ -233,10 +237,11 @@ class _KDocLexer implements FlexLexer {

  private static final String ZZ_ATTRIBUTE_PACKED_0 =
    "\6\0\1\11\2\1\1\11\6\1\1\11\5\1\1\0"+
-    "\1\11\2\0\1\11\1\1\2\0\1\11\1\0\3\11";
+    "\1\11\4\0\1\11\1\1\2\0\1\11\2\0\2\11"+
+    "\2\1\1\0\1\1\1\11";

  private static int [] zzUnpackAttribute() {
-    int [] result = new int[35];
+    int [] result = new int[42];
    int offset = 0;
    offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result);
    return result;
@@ -316,19 +321,15 @@ class _KDocLexer implements FlexLexer {
  }


+  /**
+   * Creates a new scanner
+   *
+   * @param   in  the java.io.Reader to read input from.
+   */
  _KDocLexer(java.io.Reader in) {
    this.zzReader = in;
  }

-  /**
-   * Creates a new scanner.
-   * There is also java.io.Reader version of this constructor.
-   *
-   * @param   in  the java.io.Inputstream to read input from.
-   */
-  _KDocLexer(java.io.InputStream in) {
-    this(new java.io.InputStreamReader(in));
-  }

  /** 
   * Unpacks the compressed character translation table.
@@ -568,7 +569,7 @@ class _KDocLexer implements FlexLexer {
    yybegin(tag != null && tag.isReferenceRequired() ? TAG_BEGINNING : TAG_TEXT_BEGINNING);
    return KDocTokens.TAG_NAME;
          }
-        case 15: break;
+        case 16: break;
        case 3: 
          { if (yytextContainLineBreaks()) {
            yybegin(LINE_BEGINNING);
@@ -578,70 +579,75 @@ class _KDocLexer implements FlexLexer {
            return KDocTokens.TEXT;  // internal white space
        }
          }
-        case 16: break;
+        case 17: break;
        case 11: 
          { yybegin(CONTENTS_BEGINNING);
                                            return KDocTokens.START;
          }
-        case 17: break;
+        case 18: break;
+        case 15: 
+          { yybegin(CONTENTS);
+        return KDocTokens.MARKDOWN_INLINE_LINK;
+          }
+        case 19: break;
        case 8: 
          { if (isLastToken()) return KDocTokens.END;
                                            else return KDocTokens.TEXT;
          }
-        case 18: break;
+        case 20: break;
        case 5: 
          { yybegin(CONTENTS);
        return KDocTokens.TEXT;
          }
-        case 19: break;
+        case 21: break;
        case 12: 
          { yybegin(TAG_TEXT_BEGINNING);
                  return KDocTokens.MARKDOWN_LINK;
          }
-        case 20: break;
+        case 22: break;
        case 7: 
          { yybegin(TAG_TEXT_BEGINNING);
        return KDocTokens.MARKDOWN_LINK;
          }
-        case 21: break;
+        case 23: break;
        case 6: 
          { if (yytextContainLineBreaks()) {
            yybegin(LINE_BEGINNING);
        }
        return TokenType.WHITE_SPACE;
          }
-        case 22: break;
+        case 24: break;
        case 14: 
          // lookahead expression with fixed lookahead length
          yypushback(1);
          { yybegin(CONTENTS);
                  return KDocTokens.MARKDOWN_LINK;
          }
-        case 23: break;
+        case 25: break;
        case 13: 
          { yybegin(CONTENTS);
                  return KDocTokens.MARKDOWN_LINK;
          }
-        case 24: break;
+        case 26: break;
        case 1: 
          { return TokenType.BAD_CHARACTER;
          }
-        case 25: break;
+        case 27: break;
        case 9: 
          { yybegin(CONTENTS);
                   return KDocTokens.MARKDOWN_ESCAPED_CHAR;
          }
-        case 26: break;
+        case 28: break;
        case 2: 
          { yybegin(CONTENTS);
            return KDocTokens.TEXT;
          }
-        case 27: break;
+        case 29: break;
        case 4: 
          { yybegin(CONTENTS_BEGINNING);
                                            return KDocTokens.LEADING_ASTERISK;
          }
-        case 28: break;
+        case 30: break;
        default:
          if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
            zzAtEOF = true;
@@ -6,7 +6,9 @@ JetFile: HttpLink.kt
    PsiWhiteSpace('\n ')
    KDOC_SECTION
      PsiElement(KDOC_LEADING_ASTERISK)('*')
-      PsiElement(KDOC_TEXT)(' [Kotlin](http://kotlinlang.org) is made by [JetBrains][1]')
+      PsiElement(KDOC_TEXT)(' ')
+      PsiElement(KDOC_MARKDOWN_INLINE_LINK)('[Kotlin](http://kotlinlang.org)')
+      PsiElement(KDOC_TEXT)(' is made by [JetBrains][1]')
      PsiWhiteSpace('\n ')
      PsiElement(KDOC_LEADING_ASTERISK)('*')
      PsiElement(KDOC_TEXT)(' [1]: http://www.jetbrains.com/')
@@ -0,0 +1,3 @@
+/**
+ * [foo][bar]
+ */
@@ -0,0 +1,16 @@
+JetFile: MarkdownLinkWithLabel.kt
+  PACKAGE_DIRECTIVE
+    <empty list>
+  KDoc
+    PsiElement(KDOC_START)('/**')
+    PsiWhiteSpace('\n ')
+    KDOC_SECTION
+      PsiElement(KDOC_LEADING_ASTERISK)('*')
+      PsiElement(KDOC_TEXT)(' [foo]')
+      KDOC_MARKDOWN_LINK
+        PsiElement(LBRACKET)('[')
+        KDOC_NAME
+          PsiElement(IDENTIFIER)('bar')
+        PsiElement(RBRACKET)(']')
+    PsiWhiteSpace('\n ')
+    PsiElement(KDOC_END)('*/')
@@ -1134,6 +1134,12 @@ public class JetParsingTestGenerated extends AbstractJetParsingTest {
                doParsingTest(fileName);
            }

+            @TestMetadata("MarkdownLinkWithLabel.kt")
+            public void testMarkdownLinkWithLabel() throws Exception {
+                String fileName = JetTestUtils.navigationMetadata("compiler/testData/psi/kdoc/MarkdownLinkWithLabel.kt");
+                doParsingTest(fileName);
+            }
+
            @TestMetadata("MarkdownQualifiedName.kt")
            public void testMarkdownQualifiedName() throws Exception {
                String fileName = JetTestUtils.navigationMetadata("compiler/testData/psi/kdoc/MarkdownQualifiedName.kt");