From 5bbbadeb37028dd424b421290c4be9a08c9f6fb1 Mon Sep 17 00:00:00 2001 From: Dmitry Jemerov Date: Thu, 9 Apr 2015 14:49:16 +0200 Subject: [PATCH] add a token for inline Markdown links; add test for identifier links with labels --- .../org/jetbrains/kotlin/kdoc/lexer/KDoc.flex | 5 + .../kotlin/kdoc/lexer/KDocTokens.java | 5 +- .../kotlin/kdoc/lexer/_KDocLexer.java | 106 +++++++++--------- compiler/testData/psi/kdoc/HttpLink.txt | 4 +- .../psi/kdoc/MarkdownLinkWithLabel.kt | 3 + .../psi/kdoc/MarkdownLinkWithLabel.txt | 16 +++ .../parsing/JetParsingTestGenerated.java | 6 + 7 files changed, 92 insertions(+), 53 deletions(-) create mode 100644 compiler/testData/psi/kdoc/MarkdownLinkWithLabel.kt create mode 100644 compiler/testData/psi/kdoc/MarkdownLinkWithLabel.txt diff --git a/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/KDoc.flex b/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/KDoc.flex index f798bd2dd7c..aef2fb3a15e 100644 --- a/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/KDoc.flex +++ b/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/KDoc.flex @@ -139,6 +139,11 @@ CODE_LINK=\[{QUALIFIED_NAME}\] "\\"[\[\]] { yybegin(CONTENTS); return KDocTokens.MARKDOWN_ESCAPED_CHAR; } + "[" [^\[]* "](" [^)]* ")" { + yybegin(CONTENTS); + return KDocTokens.MARKDOWN_INLINE_LINK; + } + /* We're only interested in parsing links that can become code references, meaning they contain only identifier characters and characters that can be used in type declarations. No brackets, backticks, asterisks or anything like that. diff --git a/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/KDocTokens.java b/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/KDocTokens.java index b24eeb09cf1..9198f6130b9 100644 --- a/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/KDocTokens.java +++ b/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/KDocTokens.java @@ -65,7 +65,8 @@ public interface KDocTokens { }; KDocToken MARKDOWN_ESCAPED_CHAR = new KDocToken("KDOC_MARKDOWN_ESCAPED_CHAR"); + KDocToken MARKDOWN_INLINE_LINK = new KDocToken("KDOC_MARKDOWN_INLINE_LINK"); - TokenSet KDOC_HIGHLIGHT_TOKENS = TokenSet.create(START, END, LEADING_ASTERISK, TEXT, MARKDOWN_LINK, MARKDOWN_ESCAPED_CHAR); - TokenSet CONTENT_TOKENS = TokenSet.create(TEXT, TAG_NAME, MARKDOWN_LINK, MARKDOWN_ESCAPED_CHAR); + TokenSet KDOC_HIGHLIGHT_TOKENS = TokenSet.create(START, END, LEADING_ASTERISK, TEXT, MARKDOWN_LINK, MARKDOWN_ESCAPED_CHAR, MARKDOWN_INLINE_LINK); + TokenSet CONTENT_TOKENS = TokenSet.create(TEXT, TAG_NAME, MARKDOWN_LINK, MARKDOWN_ESCAPED_CHAR, MARKDOWN_INLINE_LINK); } diff --git a/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/_KDocLexer.java b/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/_KDocLexer.java index 1365b992e75..c0de155ca2b 100644 --- a/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/_KDocLexer.java +++ b/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/_KDocLexer.java @@ -1,4 +1,4 @@ -/* The following code was generated by JFlex 1.4.3 on 3/5/15 7:51 PM */ +/* The following code was generated by JFlex 1.4.3 on 4/9/15 2:44 PM */ package org.jetbrains.kotlin.kdoc.lexer; @@ -14,7 +14,7 @@ import org.jetbrains.kotlin.kdoc.parser.KDocKnownTag; /** * This class is a scanner generated by * JFlex 1.4.3 - * on 3/5/15 7:51 PM from the specification file + * on 4/9/15 2:44 PM from the specification file * /Users/yole/jetbrains/kotlin/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/KDoc.flex */ class _KDocLexer implements FlexLexer { @@ -44,7 +44,7 @@ class _KDocLexer implements FlexLexer { */ private static final String ZZ_CMAP_PACKED = "\11\0\1\1\1\12\1\0\2\1\22\0\1\1\3\0\1\3\3\0"+ - "\1\14\1\0\1\10\3\0\1\4\1\7\12\2\6\0\1\11\32\3"+ + "\1\14\1\15\1\10\3\0\1\4\1\7\12\2\6\0\1\11\32\3"+ "\1\5\1\13\1\6\1\0\1\3\1\0\32\3\47\0\4\3\4\0"+ "\1\3\12\0\1\3\4\0\1\3\5\0\27\3\1\0\37\3\1\0"+ "\u013f\3\31\0\162\3\4\0\14\3\16\0\5\3\11\0\1\3\213\0"+ @@ -117,11 +117,11 @@ class _KDocLexer implements FlexLexer { private static final String ZZ_ACTION_PACKED_0 = "\6\0\3\1\1\2\1\3\1\2\1\4\3\2\1\5"+ - "\1\6\1\7\3\5\1\0\1\10\2\0\1\11\1\12"+ - "\2\0\1\13\1\0\1\14\1\15\1\16"; + "\1\6\1\7\3\5\1\0\1\10\4\0\1\11\1\12"+ + "\2\0\1\13\2\0\1\14\1\15\2\16\1\0\2\17"; private static int [] zzUnpackAction() { - int [] result = new int[35]; + int [] result = new int[42]; int offset = 0; offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result); return result; @@ -146,14 +146,15 @@ class _KDocLexer implements FlexLexer { private static final int [] ZZ_ROWMAP = zzUnpackRowMap(); private static final String ZZ_ROWMAP_PACKED_0 = - "\0\0\0\15\0\32\0\47\0\64\0\101\0\116\0\133"+ - "\0\150\0\116\0\165\0\202\0\217\0\234\0\150\0\251"+ - "\0\116\0\266\0\303\0\320\0\150\0\335\0\352\0\116"+ - "\0\150\0\367\0\116\0\u0104\0\u0111\0\u011e\0\116\0\u012b"+ - "\0\116\0\116\0\116"; + "\0\0\0\16\0\34\0\52\0\70\0\106\0\124\0\142"+ + "\0\160\0\124\0\176\0\214\0\232\0\250\0\160\0\266"+ + "\0\124\0\304\0\322\0\340\0\160\0\356\0\374\0\124"+ + "\0\160\0\u010a\0\u0118\0\u0126\0\124\0\u0134\0\u0142\0\u0150"+ + "\0\124\0\u015e\0\u016c\0\124\0\124\0\u010a\0\u0126\0\u017a"+ + "\0\u010a\0\124"; private static int [] zzUnpackRowMap() { - int [] result = new int[35]; + int [] result = new int[42]; int offset = 0; offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result); return result; @@ -176,22 +177,25 @@ class _KDocLexer implements FlexLexer { private static final int [] ZZ_TRANS = zzUnpackTrans(); private static final String ZZ_TRANS_PACKED_0 = - "\7\7\1\10\1\11\1\7\1\0\2\7\1\12\1\13"+ - "\3\12\1\14\2\12\1\15\1\12\1\13\1\16\2\12"+ + "\7\7\1\10\1\11\1\7\1\0\3\7\1\12\1\13"+ + "\3\12\1\14\2\12\1\15\1\12\1\13\1\16\3\12"+ "\1\13\3\12\1\14\2\12\1\17\1\20\1\13\1\16"+ - "\1\12\1\21\1\22\1\21\1\23\1\21\1\24\2\21"+ - "\1\25\1\21\1\22\3\21\1\22\3\21\1\26\2\21"+ - "\1\25\1\21\1\22\2\21\1\12\1\13\3\12\1\14"+ - "\2\12\1\17\1\12\1\13\1\16\1\12\25\0\1\27"+ - "\13\0\1\30\1\31\5\0\1\13\10\0\1\13\5\0"+ - "\1\32\20\0\1\30\1\15\11\0\2\33\11\0\1\34"+ - "\12\0\1\22\10\0\1\22\4\0\3\23\13\0\1\35"+ - "\14\0\1\36\21\0\1\37\6\0\3\32\1\0\1\40"+ - "\10\0\2\34\13\0\3\35\1\0\1\41\10\0\3\36"+ - "\1\0\1\42\6\0\5\43\1\0\6\43\1\0"; + "\2\12\1\21\1\22\1\21\1\23\1\21\1\24\2\21"+ + "\1\25\1\21\1\22\4\21\1\22\3\21\1\26\2\21"+ + "\1\25\1\21\1\22\3\21\1\12\1\13\3\12\1\14"+ + "\2\12\1\17\1\12\1\13\1\16\2\12\26\0\1\27"+ + "\14\0\1\30\1\31\6\0\1\13\10\0\1\13\3\0"+ + "\3\32\1\33\1\32\1\0\1\34\7\32\7\0\1\30"+ + "\1\15\12\0\2\35\12\0\1\36\13\0\1\22\10\0"+ + "\1\22\5\0\3\23\14\0\1\37\15\0\1\40\22\0"+ + "\1\41\5\0\5\32\1\0\1\34\11\32\3\33\1\0"+ + "\1\42\14\32\1\0\1\34\5\32\1\43\1\32\2\0"+ + "\2\36\14\0\3\37\1\0\1\44\11\0\3\40\1\0"+ + "\1\45\7\0\5\46\1\0\1\47\5\46\1\43\1\46"+ + "\5\43\1\50\7\43\1\51\15\50\1\52"; private static int [] zzUnpackTrans() { - int [] result = new int[312]; + int [] result = new int[392]; int offset = 0; offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result); return result; @@ -233,10 +237,11 @@ class _KDocLexer implements FlexLexer { private static final String ZZ_ATTRIBUTE_PACKED_0 = "\6\0\1\11\2\1\1\11\6\1\1\11\5\1\1\0"+ - "\1\11\2\0\1\11\1\1\2\0\1\11\1\0\3\11"; + "\1\11\4\0\1\11\1\1\2\0\1\11\2\0\2\11"+ + "\2\1\1\0\1\1\1\11"; private static int [] zzUnpackAttribute() { - int [] result = new int[35]; + int [] result = new int[42]; int offset = 0; offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result); return result; @@ -316,19 +321,15 @@ class _KDocLexer implements FlexLexer { } + /** + * Creates a new scanner + * + * @param in the java.io.Reader to read input from. + */ _KDocLexer(java.io.Reader in) { this.zzReader = in; } - /** - * Creates a new scanner. - * There is also java.io.Reader version of this constructor. - * - * @param in the java.io.Inputstream to read input from. - */ - _KDocLexer(java.io.InputStream in) { - this(new java.io.InputStreamReader(in)); - } /** * Unpacks the compressed character translation table. @@ -568,7 +569,7 @@ class _KDocLexer implements FlexLexer { yybegin(tag != null && tag.isReferenceRequired() ? TAG_BEGINNING : TAG_TEXT_BEGINNING); return KDocTokens.TAG_NAME; } - case 15: break; + case 16: break; case 3: { if (yytextContainLineBreaks()) { yybegin(LINE_BEGINNING); @@ -578,70 +579,75 @@ class _KDocLexer implements FlexLexer { return KDocTokens.TEXT; // internal white space } } - case 16: break; + case 17: break; case 11: { yybegin(CONTENTS_BEGINNING); return KDocTokens.START; } - case 17: break; + case 18: break; + case 15: + { yybegin(CONTENTS); + return KDocTokens.MARKDOWN_INLINE_LINK; + } + case 19: break; case 8: { if (isLastToken()) return KDocTokens.END; else return KDocTokens.TEXT; } - case 18: break; + case 20: break; case 5: { yybegin(CONTENTS); return KDocTokens.TEXT; } - case 19: break; + case 21: break; case 12: { yybegin(TAG_TEXT_BEGINNING); return KDocTokens.MARKDOWN_LINK; } - case 20: break; + case 22: break; case 7: { yybegin(TAG_TEXT_BEGINNING); return KDocTokens.MARKDOWN_LINK; } - case 21: break; + case 23: break; case 6: { if (yytextContainLineBreaks()) { yybegin(LINE_BEGINNING); } return TokenType.WHITE_SPACE; } - case 22: break; + case 24: break; case 14: // lookahead expression with fixed lookahead length yypushback(1); { yybegin(CONTENTS); return KDocTokens.MARKDOWN_LINK; } - case 23: break; + case 25: break; case 13: { yybegin(CONTENTS); return KDocTokens.MARKDOWN_LINK; } - case 24: break; + case 26: break; case 1: { return TokenType.BAD_CHARACTER; } - case 25: break; + case 27: break; case 9: { yybegin(CONTENTS); return KDocTokens.MARKDOWN_ESCAPED_CHAR; } - case 26: break; + case 28: break; case 2: { yybegin(CONTENTS); return KDocTokens.TEXT; } - case 27: break; + case 29: break; case 4: { yybegin(CONTENTS_BEGINNING); return KDocTokens.LEADING_ASTERISK; } - case 28: break; + case 30: break; default: if (zzInput == YYEOF && zzStartRead == zzCurrentPos) { zzAtEOF = true; diff --git a/compiler/testData/psi/kdoc/HttpLink.txt b/compiler/testData/psi/kdoc/HttpLink.txt index d8e17c5cc05..5f3b749ba9f 100644 --- a/compiler/testData/psi/kdoc/HttpLink.txt +++ b/compiler/testData/psi/kdoc/HttpLink.txt @@ -6,7 +6,9 @@ JetFile: HttpLink.kt PsiWhiteSpace('\n ') KDOC_SECTION PsiElement(KDOC_LEADING_ASTERISK)('*') - PsiElement(KDOC_TEXT)(' [Kotlin](http://kotlinlang.org) is made by [JetBrains][1]') + PsiElement(KDOC_TEXT)(' ') + PsiElement(KDOC_MARKDOWN_INLINE_LINK)('[Kotlin](http://kotlinlang.org)') + PsiElement(KDOC_TEXT)(' is made by [JetBrains][1]') PsiWhiteSpace('\n ') PsiElement(KDOC_LEADING_ASTERISK)('*') PsiElement(KDOC_TEXT)(' [1]: http://www.jetbrains.com/') diff --git a/compiler/testData/psi/kdoc/MarkdownLinkWithLabel.kt b/compiler/testData/psi/kdoc/MarkdownLinkWithLabel.kt new file mode 100644 index 00000000000..2bd2de37847 --- /dev/null +++ b/compiler/testData/psi/kdoc/MarkdownLinkWithLabel.kt @@ -0,0 +1,3 @@ +/** + * [foo][bar] + */ diff --git a/compiler/testData/psi/kdoc/MarkdownLinkWithLabel.txt b/compiler/testData/psi/kdoc/MarkdownLinkWithLabel.txt new file mode 100644 index 00000000000..ff29e2bd922 --- /dev/null +++ b/compiler/testData/psi/kdoc/MarkdownLinkWithLabel.txt @@ -0,0 +1,16 @@ +JetFile: MarkdownLinkWithLabel.kt + PACKAGE_DIRECTIVE + + KDoc + PsiElement(KDOC_START)('/**') + PsiWhiteSpace('\n ') + KDOC_SECTION + PsiElement(KDOC_LEADING_ASTERISK)('*') + PsiElement(KDOC_TEXT)(' [foo]') + KDOC_MARKDOWN_LINK + PsiElement(LBRACKET)('[') + KDOC_NAME + PsiElement(IDENTIFIER)('bar') + PsiElement(RBRACKET)(']') + PsiWhiteSpace('\n ') + PsiElement(KDOC_END)('*/') \ No newline at end of file diff --git a/compiler/tests/org/jetbrains/kotlin/parsing/JetParsingTestGenerated.java b/compiler/tests/org/jetbrains/kotlin/parsing/JetParsingTestGenerated.java index 5d361d731cf..05ffb9008e7 100644 --- a/compiler/tests/org/jetbrains/kotlin/parsing/JetParsingTestGenerated.java +++ b/compiler/tests/org/jetbrains/kotlin/parsing/JetParsingTestGenerated.java @@ -1134,6 +1134,12 @@ public class JetParsingTestGenerated extends AbstractJetParsingTest { doParsingTest(fileName); } + @TestMetadata("MarkdownLinkWithLabel.kt") + public void testMarkdownLinkWithLabel() throws Exception { + String fileName = JetTestUtils.navigationMetadata("compiler/testData/psi/kdoc/MarkdownLinkWithLabel.kt"); + doParsingTest(fileName); + } + @TestMetadata("MarkdownQualifiedName.kt") public void testMarkdownQualifiedName() throws Exception { String fileName = JetTestUtils.navigationMetadata("compiler/testData/psi/kdoc/MarkdownQualifiedName.kt");