azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

don't parse lines starting with @ as KDoc tags if they're inside a Markdown code fence

Browse Source 
#KT-11789 Fixed
This commit is contained in:
Dmitry Jemerov
2016-03-31 20:48:35 +02:00
parent ad002c4f09
commit 01b0cfb552
10 changed files with 449 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/KDoc.flex
+55 -2
View File
@@ -46,9 +46,12 @@ import org.jetbrains.kotlin.kdoc.parser.KDocKnownTag;
%state TAG_BEGINNING
%state TAG_TEXT_BEGINNING
%state CONTENTS
%state CODE_BLOCK
%state CODE_BLOCK_LINE_BEGINNING
%state CODE_BLOCK_CONTENTS_BEGINNING
WHITE_SPACE_CHAR =[\ \t\f\n\r]
NOT_WHITE_SPACE_CHAR=[^\ \t\f\n\r]
WHITE_SPACE_CHAR =[\ \t\f\n]
NOT_WHITE_SPACE_CHAR=[^\ \t\f\n]
DIGIT=[0-9]
ALPHA=[:jletter:]
@@ -58,6 +61,8 @@ QUALIFIED_NAME_START={ALPHA}
QUALIFIED_NAME_CHAR={ALPHA}|{DIGIT}|[\.]
QUALIFIED_NAME={QUALIFIED_NAME_START}{QUALIFIED_NAME_CHAR}*
CODE_LINK=\[{QUALIFIED_NAME}\]
CODE_FENCE_START=("```" | "~~~").*
CODE_FENCE_END=("```" | "~~~")
%%
@@ -143,6 +148,11 @@ CODE_LINK=\[{QUALIFIED_NAME}\]
return KDocTokens.MARKDOWN_INLINE_LINK;
}
{CODE_FENCE_START} {
yybegin(CODE_BLOCK_LINE_BEGINNING);
return KDocTokens.TEXT;
}
/* We're only interested in parsing links that can become code references,
meaning they contain only identifier characters and characters that can be
used in type declarations. No brackets, backticks, asterisks or anything like that.
@@ -155,4 +165,47 @@ CODE_LINK=\[{QUALIFIED_NAME}\]
return KDocTokens.TEXT; }
}
<CODE_BLOCK_LINE_BEGINNING> {
"*"+ {
yybegin(CODE_BLOCK_CONTENTS_BEGINNING);
return KDocTokens.LEADING_ASTERISK;
}
}
<CODE_BLOCK_LINE_BEGINNING, CODE_BLOCK_CONTENTS_BEGINNING> {
{CODE_FENCE_END} / [ \t\f]* [\n] [ \t\f]* {
// Code fence end
yybegin(CONTENTS);
return KDocTokens.TEXT;
}
{WHITE_SPACE_CHAR}+ {
if (yytextContainLineBreaks()) {
yybegin(CODE_BLOCK_LINE_BEGINNING);
return TokenType.WHITE_SPACE;
}
return KDocTokens.TEXT;
}
. {
yybegin(CODE_BLOCK);
return KDocTokens.TEXT;
}
}
<CODE_BLOCK_LINE_BEGINNING, CODE_BLOCK_CONTENTS_BEGINNING, CODE_BLOCK> {
{WHITE_SPACE_CHAR}+ {
if (yytextContainLineBreaks()) {
yybegin(CODE_BLOCK_LINE_BEGINNING);
return TokenType.WHITE_SPACE;
}
return KDocTokens.TEXT;
}
. {
yybegin(CODE_BLOCK);
return KDocTokens.TEXT;
}
}
. { return TokenType.BAD_CHARACTER; }
compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/_KDocLexer.java
+203 -174
View File
@@ -1,4 +1,4 @@
/* The following code was generated by JFlex 1.4.3 on 12/9/15 7:45 PM */
/* The following code was generated by JFlex 1.4.3 on 4/5/16 7:39 PM */
package org.jetbrains.kotlin.kdoc.lexer;
@@ -6,25 +6,29 @@ import com.intellij.lexer.FlexLexer;
import com.intellij.psi.TokenType;
import com.intellij.psi.tree.IElementType;
import com.intellij.util.text.CharArrayUtil;
import java.lang.Character;
import org.jetbrains.kotlin.kdoc.parser.KDocKnownTag;
/**
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.4.3
* on 12/9/15 7:45 PM from the specification file
* <tt>/home/user/projects/kotlin/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/KDoc.flex</tt>
* on 4/5/16 7:39 PM from the specification file
* <tt>/Users/yole/jetbrains/kotlin/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/KDoc.flex</tt>
*/
class _KDocLexer implements FlexLexer {
/** initial size of the lookahead buffer */
private static final int ZZ_BUFFERSIZE = 16384;
/** lexical states */
public static final int CODE_BLOCK_CONTENTS_BEGINNING = 16;
public static final int TAG_TEXT_BEGINNING = 8;
public static final int CODE_BLOCK = 12;
public static final int CODE_BLOCK_LINE_BEGINNING = 14;
public static final int CONTENTS_BEGINNING = 4;
public static final int LINE_BEGINNING = 2;
public static final int CONTENTS = 10;
public static final int YYINITIAL = 0;
public static final int TAG_TEXT_BEGINNING = 8;
public static final int LINE_BEGINNING = 2;
public static final int TAG_BEGINNING = 6;
/**
@@ -34,92 +38,75 @@ class _KDocLexer implements FlexLexer {
* l is of the form l = 2*k, k a non negative integer
*/
private static final int ZZ_LEXSTATE[] = {
0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5
0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7,
8, 8
};
/**
* Translates characters to character classes
*/
private static final String ZZ_CMAP_PACKED =
"\11\0\1\1\1\12\1\0\2\1\22\0\1\1\3\0\1\3\3\0"+
"\1\14\1\15\1\10\3\0\1\4\1\7\12\2\6\0\1\11\32\3"+
"\1\5\1\13\1\6\1\0\1\3\1\0\32\3\47\0\4\3\4\0"+
"\1\3\12\0\1\3\4\0\1\3\5\0\27\3\1\0\37\3\1\0"+
"\u01ca\3\4\0\14\3\16\0\5\3\7\0\1\3\1\0\1\3\201\0"+
"\5\3\1\0\2\3\2\0\4\3\10\0\1\3\1\0\3\3\1\0"+
"\1\3\1\0\24\3\1\0\123\3\1\0\213\3\10\0\236\3\11\0"+
"\46\3\2\0\1\3\7\0\47\3\110\0\33\3\5\0\3\3\30\0"+
"\1\3\24\0\53\3\43\0\2\3\1\0\143\3\1\0\1\3\17\0"+
"\2\3\7\0\2\3\12\0\3\3\2\0\1\3\20\0\1\3\1\0"+
"\36\3\35\0\131\3\13\0\1\3\30\0\41\3\11\0\2\3\4\0"+
"\1\3\5\0\26\3\4\0\1\3\11\0\1\3\3\0\1\3\27\0"+
"\31\3\253\0\66\3\3\0\1\3\22\0\1\3\7\0\12\3\17\0"+
"\7\3\1\0\7\3\5\0\10\3\2\0\2\3\2\0\26\3\1\0"+
"\7\3\1\0\1\3\3\0\4\3\3\0\1\3\20\0\1\3\15\0"+
"\2\3\1\0\3\3\16\0\4\3\7\0\1\3\11\0\6\3\4\0"+
"\2\3\2\0\26\3\1\0\7\3\1\0\2\3\1\0\2\3\1\0"+
"\2\3\37\0\4\3\1\0\1\3\23\0\3\3\20\0\11\3\1\0"+
"\3\3\1\0\26\3\1\0\7\3\1\0\2\3\1\0\5\3\3\0"+
"\1\3\22\0\1\3\17\0\2\3\17\0\1\3\23\0\10\3\2\0"+
"\2\3\2\0\26\3\1\0\7\3\1\0\2\3\1\0\5\3\3\0"+
"\1\3\36\0\2\3\1\0\3\3\17\0\1\3\21\0\1\3\1\0"+
"\6\3\3\0\3\3\1\0\4\3\3\0\2\3\1\0\1\3\1\0"+
"\2\3\3\0\2\3\3\0\3\3\3\0\14\3\26\0\1\3\50\0"+
"\1\3\13\0\10\3\1\0\3\3\1\0\27\3\1\0\12\3\1\0"+
"\5\3\3\0\1\3\32\0\2\3\6\0\2\3\43\0\10\3\1\0"+
"\3\3\1\0\27\3\1\0\12\3\1\0\5\3\3\0\1\3\40\0"+
"\1\3\1\0\2\3\17\0\2\3\22\0\10\3\1\0\3\3\1\0"+
"\51\3\2\0\1\3\20\0\1\3\21\0\2\3\30\0\6\3\5\0"+
"\22\3\3\0\30\3\1\0\11\3\1\0\1\3\2\0\7\3\72\0"+
"\60\3\1\0\2\3\13\0\10\3\72\0\2\3\1\0\1\3\2\0"+
"\2\3\1\0\1\3\2\0\1\3\6\0\4\3\1\0\7\3\1\0"+
"\3\3\1\0\1\3\1\0\1\3\2\0\2\3\1\0\4\3\1\0"+
"\2\3\11\0\1\3\2\0\5\3\1\0\1\3\25\0\2\3\42\0"+
"\1\3\77\0\10\3\1\0\44\3\33\0\5\3\163\0\53\3\24\0"+
"\1\3\20\0\6\3\4\0\4\3\3\0\1\3\3\0\2\3\7\0"+
"\3\3\4\0\15\3\14\0\1\3\21\0\46\3\12\0\53\3\1\0"+
"\1\3\3\0\u0149\3\1\0\4\3\2\0\7\3\1\0\1\3\1\0"+
"\4\3\2\0\51\3\1\0\4\3\2\0\41\3\1\0\4\3\2\0"+
"\7\3\1\0\1\3\1\0\4\3\2\0\17\3\1\0\71\3\1\0"+
"\4\3\2\0\103\3\45\0\20\3\20\0\125\3\14\0\u026c\3\2\0"+
"\21\3\1\0\32\3\5\0\113\3\3\0\3\3\17\0\15\3\1\0"+
"\4\3\16\0\22\3\16\0\22\3\16\0\15\3\1\0\3\3\17\0"+
"\64\3\43\0\1\3\3\0\2\3\103\0\130\3\10\0\51\3\1\0"+
"\1\3\5\0\106\3\12\0\35\3\63\0\36\3\2\0\5\3\13\0"+
"\54\3\25\0\7\3\70\0\27\3\11\0\65\3\122\0\1\3\135\0"+
"\57\3\21\0\7\3\67\0\36\3\15\0\2\3\20\0\46\3\32\0"+
"\44\3\51\0\3\3\12\0\44\3\153\0\4\3\1\0\4\3\16\0"+
"\300\3\100\0\u0116\3\2\0\6\3\2\0\46\3\2\0\6\3\2\0"+
"\10\3\1\0\1\3\1\0\1\3\1\0\1\3\1\0\37\3\2\0"+
"\65\3\1\0\7\3\1\0\1\3\3\0\3\3\1\0\7\3\3\0"+
"\4\3\2\0\6\3\4\0\15\3\5\0\3\3\1\0\7\3\102\0"+
"\2\3\23\0\1\3\34\0\1\3\15\0\1\3\20\0\15\3\3\0"+
"\32\3\110\0\1\3\4\0\1\3\2\0\12\3\1\0\1\3\3\0"+
"\5\3\6\0\1\3\1\0\1\3\1\0\1\3\1\0\4\3\1\0"+
"\13\3\2\0\4\3\5\0\5\3\4\0\1\3\21\0\51\3\u0a77\0"+
"\57\3\1\0\57\3\1\0\205\3\6\0\4\3\21\0\46\3\12\0"+
"\66\3\11\0\1\3\20\0\27\3\11\0\7\3\1\0\7\3\1\0"+
"\7\3\1\0\7\3\1\0\7\3\1\0\7\3\1\0\7\3\1\0"+
"\7\3\120\0\1\3\u01d5\0\3\3\31\0\11\3\7\0\5\3\2\0"+
"\5\3\4\0\126\3\6\0\3\3\1\0\132\3\1\0\4\3\5\0"+
"\51\3\3\0\136\3\21\0\33\3\65\0\20\3\u0200\0\u19b6\3\112\0"+
"\u51cc\3\64\0\u048d\3\103\0\56\3\2\0\u010d\3\3\0\20\3\12\0"+
"\2\3\24\0\57\3\20\0\31\3\10\0\120\3\47\0\11\3\2\0"+
"\147\3\2\0\4\3\1\0\2\3\16\0\12\3\120\0\10\3\1\0"+
"\3\3\1\0\4\3\1\0\27\3\25\0\1\3\7\0\64\3\16\0"+
"\62\3\76\0\6\3\3\0\1\3\16\0\34\3\12\0\27\3\31\0"+
"\35\3\7\0\57\3\34\0\1\3\60\0\51\3\27\0\3\3\1\0"+
"\10\3\24\0\27\3\3\0\1\3\5\0\60\3\1\0\1\3\3\0"+
"\2\3\2\0\5\3\2\0\1\3\1\0\1\3\30\0\3\3\43\0"+
"\6\3\2\0\6\3\2\0\6\3\11\0\7\3\1\0\7\3\221\0"+
"\43\3\35\0\u2ba4\3\14\0\27\3\4\0\61\3\u2104\0\u012e\3\2\0"+
"\76\3\2\0\152\3\46\0\7\3\14\0\5\3\5\0\1\3\1\0"+
"\12\3\1\0\15\3\1\0\5\3\1\0\1\3\1\0\2\3\1\0"+
"\2\3\1\0\154\3\41\0\u016b\3\22\0\100\3\2\0\66\3\50\0"+
"\15\3\66\0\2\3\30\0\3\3\31\0\1\3\6\0\5\3\1\0"+
"\207\3\7\0\1\3\34\0\32\3\4\0\1\3\1\0\32\3\13\0"+
"\131\3\3\0\6\3\2\0\6\3\2\0\6\3\2\0\3\3\3\0"+
"\2\3\3\0\2\3\31\0";
"\11\0\1\1\1\11\1\0\1\1\23\0\1\1\3\0\1\3\3\0"+
"\1\16\1\17\1\13\3\0\1\4\1\12\12\2\6\0\1\14\32\3"+
"\1\5\1\15\1\6\1\0\1\3\1\7\32\3\3\0\1\10\43\0"+
"\4\3\4\0\1\3\12\0\1\3\4\0\1\3\5\0\27\3\1\0"+
"\37\3\1\0\u013f\3\31\0\162\3\4\0\14\3\16\0\5\3\11\0"+
"\1\3\213\0\1\3\13\0\1\3\1\0\3\3\1\0\1\3\1\0"+
"\24\3\1\0\54\3\1\0\46\3\1\0\5\3\4\0\202\3\10\0"+
"\105\3\1\0\46\3\2\0\2\3\6\0\20\3\41\0\46\3\2\0"+
"\1\3\7\0\47\3\110\0\33\3\5\0\3\3\56\0\32\3\5\0"+
"\13\3\43\0\2\3\1\0\143\3\1\0\1\3\17\0\2\3\7\0"+
"\2\3\12\0\3\3\2\0\1\3\20\0\1\3\1\0\36\3\35\0"+
"\3\3\60\0\46\3\13\0\1\3\u0152\0\66\3\3\0\1\3\22\0"+
"\1\3\7\0\12\3\43\0\10\3\2\0\2\3\2\0\26\3\1\0"+
"\7\3\1\0\1\3\3\0\4\3\3\0\1\3\36\0\2\3\1\0"+
"\3\3\16\0\4\3\21\0\6\3\4\0\2\3\2\0\26\3\1\0"+
"\7\3\1\0\2\3\1\0\2\3\1\0\2\3\37\0\4\3\1\0"+
"\1\3\23\0\3\3\20\0\11\3\1\0\3\3\1\0\26\3\1\0"+
"\7\3\1\0\2\3\1\0\5\3\3\0\1\3\22\0\1\3\17\0"+
"\2\3\17\0\1\3\23\0\10\3\2\0\2\3\2\0\26\3\1\0"+
"\7\3\1\0\2\3\1\0\5\3\3\0\1\3\36\0\2\3\1\0"+
"\3\3\17\0\1\3\21\0\1\3\1\0\6\3\3\0\3\3\1\0"+
"\4\3\3\0\2\3\1\0\1\3\1\0\2\3\3\0\2\3\3\0"+
"\3\3\3\0\10\3\1\0\3\3\77\0\1\3\13\0\10\3\1\0"+
"\3\3\1\0\27\3\1\0\12\3\1\0\5\3\46\0\2\3\43\0"+
"\10\3\1\0\3\3\1\0\27\3\1\0\12\3\1\0\5\3\3\0"+
"\1\3\40\0\1\3\1\0\2\3\43\0\10\3\1\0\3\3\1\0"+
"\27\3\1\0\20\3\46\0\2\3\43\0\22\3\3\0\30\3\1\0"+
"\11\3\1\0\1\3\2\0\7\3\72\0\60\3\1\0\2\3\13\0"+
"\10\3\72\0\2\3\1\0\1\3\2\0\2\3\1\0\1\3\2\0"+
"\1\3\6\0\4\3\1\0\7\3\1\0\3\3\1\0\1\3\1\0"+
"\1\3\2\0\2\3\1\0\4\3\1\0\2\3\11\0\1\3\2\0"+
"\5\3\1\0\1\3\25\0\2\3\42\0\1\3\77\0\10\3\1\0"+
"\42\3\35\0\4\3\164\0\42\3\1\0\5\3\1\0\2\3\45\0"+
"\6\3\112\0\46\3\12\0\51\3\7\0\132\3\5\0\104\3\5\0"+
"\122\3\6\0\7\3\1\0\77\3\1\0\1\3\1\0\4\3\2\0"+
"\7\3\1\0\1\3\1\0\4\3\2\0\47\3\1\0\1\3\1\0"+
"\4\3\2\0\37\3\1\0\1\3\1\0\4\3\2\0\7\3\1\0"+
"\1\3\1\0\4\3\2\0\7\3\1\0\7\3\1\0\27\3\1\0"+
"\37\3\1\0\1\3\1\0\4\3\2\0\7\3\1\0\47\3\1\0"+
"\23\3\105\0\125\3\14\0\u026c\3\2\0\10\3\12\0\32\3\5\0"+
"\113\3\3\0\3\3\17\0\15\3\1\0\4\3\16\0\22\3\16\0"+
"\22\3\16\0\15\3\1\0\3\3\17\0\64\3\43\0\1\3\3\0"+
"\2\3\103\0\130\3\10\0\51\3\127\0\35\3\63\0\36\3\2\0"+
"\5\3\u038b\0\154\3\224\0\234\3\4\0\132\3\6\0\26\3\2\0"+
"\6\3\2\0\46\3\2\0\6\3\2\0\10\3\1\0\1\3\1\0"+
"\1\3\1\0\1\3\1\0\37\3\2\0\65\3\1\0\7\3\1\0"+
"\1\3\3\0\3\3\1\0\7\3\3\0\4\3\2\0\6\3\4\0"+
"\15\3\5\0\3\3\1\0\7\3\102\0\2\3\23\0\1\3\34\0"+
"\1\3\15\0\1\3\40\0\22\3\120\0\1\3\4\0\1\3\2\0"+
"\12\3\1\0\1\3\3\0\5\3\6\0\1\3\1\0\1\3\1\0"+
"\1\3\1\0\4\3\1\0\3\3\1\0\7\3\3\0\3\3\5\0"+
"\5\3\26\0\44\3\u0e81\0\3\3\31\0\11\3\7\0\5\3\2\0"+
"\5\3\4\0\126\3\6\0\3\3\1\0\137\3\5\0\50\3\4\0"+
"\136\3\21\0\30\3\70\0\20\3\u0200\0\u19b6\3\112\0\u51a6\3\132\0"+
"\u048d\3\u0773\0\u2ba4\3\u215c\0\u012e\3\2\0\73\3\225\0\7\3\14\0"+
"\5\3\5\0\1\3\1\0\12\3\1\0\15\3\1\0\5\3\1\0"+
"\1\3\1\0\2\3\1\0\2\3\1\0\154\3\41\0\u016b\3\22\0"+
"\100\3\2\0\66\3\50\0\15\3\66\0\2\3\30\0\3\3\31\0"+
"\1\3\6\0\5\3\1\0\207\3\7\0\1\3\34\0\32\3\4\0"+
"\1\3\1\0\32\3\12\0\132\3\3\0\6\3\2\0\6\3\2\0"+
"\6\3\2\0\3\3\3\0\2\3\3\0\2\3\31\0";
/**
* Translates characters to character classes
@@ -132,12 +119,13 @@ class _KDocLexer implements FlexLexer {
private static final int [] ZZ_ACTION = zzUnpackAction();
private static final String ZZ_ACTION_PACKED_0 =
"\6\0\3\1\1\2\1\3\1\2\1\4\3\2\1\5"+
"\1\6\1\7\3\5\1\0\1\10\4\0\1\11\1\12"+
"\2\0\1\13\2\0\1\14\1\15\2\16\1\0\2\17";
"\11\0\3\1\1\2\1\3\3\2\1\4\3\2\1\5"+
"\1\6\1\7\3\5\1\10\1\11\3\10\1\12\1\0"+
"\1\13\6\0\1\14\1\15\4\0\1\16\2\0\1\17"+
"\1\20\1\21\1\0\2\22\1\0\1\23\1\24\1\23";
private static int [] zzUnpackAction() {
int [] result = new int[42];
int [] result = new int[60];
int offset = 0;
offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result);
return result;
@@ -162,15 +150,17 @@ class _KDocLexer implements FlexLexer {
private static final int [] ZZ_ROWMAP = zzUnpackRowMap();
private static final String ZZ_ROWMAP_PACKED_0 =
"\0\0\0\16\0\34\0\52\0\70\0\106\0\124\0\142"+
"\0\160\0\124\0\176\0\214\0\232\0\250\0\160\0\266"+
"\0\124\0\304\0\322\0\340\0\160\0\356\0\374\0\124"+
"\0\160\0\u010a\0\u0118\0\u0126\0\124\0\u0134\0\u0142\0\u0150"+
"\0\124\0\u015e\0\u016c\0\124\0\124\0\u010a\0\u0126\0\u017a"+
"\0\u010a\0\124";
"\0\0\0\20\0\40\0\60\0\100\0\120\0\140\0\160"+
"\0\200\0\220\0\240\0\260\0\220\0\300\0\320\0\340"+
"\0\360\0\u0100\0\u0110\0\260\0\u0120\0\220\0\u0130\0\u0140"+
"\0\u0150\0\260\0\u0160\0\220\0\u0170\0\260\0\u0180\0\u0190"+
"\0\u01a0\0\u01b0\0\220\0\260\0\u01c0\0\u01d0\0\u01e0\0\u01f0"+
"\0\u0200\0\220\0\u0210\0\u0220\0\u0230\0\u0240\0\u0250\0\220"+
"\0\u0260\0\u0270\0\u0280\0\220\0\220\0\u0290\0\u01c0\0\u01e0"+
"\0\u02a0\0\u01c0\0\u02b0\0\220";
private static int [] zzUnpackRowMap() {
int [] result = new int[42];
int [] result = new int[60];
int offset = 0;
offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result);
return result;
@@ -193,25 +183,33 @@ class _KDocLexer implements FlexLexer {
private static final int [] ZZ_TRANS = zzUnpackTrans();
private static final String ZZ_TRANS_PACKED_0 =
"\7\7\1\10\1\11\1\7\1\0\3\7\1\12\1\13"+
"\3\12\1\14\2\12\1\15\1\12\1\13\1\16\3\12"+
"\1\13\3\12\1\14\2\12\1\17\1\20\1\13\1\16"+
"\2\12\1\21\1\22\1\21\1\23\1\21\1\24\2\21"+
"\1\25\1\21\1\22\4\21\1\22\3\21\1\26\2\21"+
"\1\25\1\21\1\22\3\21\1\12\1\13\3\12\1\14"+
"\2\12\1\17\1\12\1\13\1\16\2\12\26\0\1\27"+
"\14\0\1\30\1\31\6\0\1\13\10\0\1\13\3\0"+
"\3\32\1\33\1\32\1\0\1\34\7\32\7\0\1\30"+
"\1\15\12\0\2\35\12\0\1\36\13\0\1\22\10\0"+
"\1\22\5\0\3\23\14\0\1\37\15\0\1\40\22\0"+
"\1\41\5\0\5\32\1\0\1\34\11\32\3\33\1\0"+
"\1\42\14\32\1\0\1\34\5\32\1\43\1\32\2\0"+
"\2\36\14\0\3\37\1\0\1\44\11\0\3\40\1\0"+
"\1\45\7\0\5\46\1\0\1\47\5\46\1\43\1\46"+
"\5\43\1\50\7\43\1\51\15\50\1\52";
"\11\12\1\0\1\13\1\14\4\12\1\15\1\16\3\15"+
"\1\17\1\15\1\20\1\21\1\16\1\15\1\22\1\15"+
"\1\23\3\15\1\16\3\15\1\17\1\15\1\20\1\21"+
"\1\16\1\15\1\24\1\25\1\23\2\15\1\26\1\27"+
"\1\26\1\30\1\26\1\31\3\26\1\27\1\26\1\32"+
"\5\26\1\27\3\26\1\33\3\26\1\27\1\26\1\32"+
"\4\26\1\15\1\16\3\15\1\17\1\15\1\20\1\21"+
"\1\16\1\15\1\24\1\15\1\23\2\15\1\34\1\35"+
"\7\34\1\35\1\34\1\36\5\34\1\35\5\34\1\37"+
"\1\40\1\35\1\34\1\41\5\34\1\35\5\34\1\37"+
"\1\40\1\35\1\34\1\36\4\34\33\0\1\42\16\0"+
"\1\43\1\44\5\0\1\16\7\0\1\16\6\0\3\45"+
"\1\46\1\45\1\0\1\47\11\45\7\0\1\50\20\0"+
"\1\51\21\0\1\43\1\22\11\0\2\52\14\0\1\53"+
"\15\0\1\27\7\0\1\27\10\0\3\30\16\0\1\54"+
"\17\0\1\55\15\0\1\35\7\0\1\35\15\0\1\56"+
"\20\0\1\57\21\0\1\43\1\41\17\0\1\60\4\0"+
"\5\45\1\0\1\47\13\45\3\46\1\0\1\61\16\45"+
"\1\0\1\47\7\45\1\62\1\45\7\0\1\63\20\0"+
"\1\63\11\0\2\53\16\0\3\54\1\0\1\64\13\0"+
"\3\55\1\0\1\65\20\0\1\66\20\0\1\66\7\0"+
"\5\67\1\0\1\70\7\67\1\62\1\67\5\62\1\71"+
"\11\62\1\72\11\63\1\0\6\63\1\0\1\66\7\0"+
"\1\73\6\0\17\71\1\74\1\0\1\73\16\0";
private static int [] zzUnpackTrans() {
int [] result = new int[392];
int [] result = new int[704];
int offset = 0;
offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result);
return result;
@@ -252,12 +250,12 @@ class _KDocLexer implements FlexLexer {
private static final int [] ZZ_ATTRIBUTE = zzUnpackAttribute();
private static final String ZZ_ATTRIBUTE_PACKED_0 =
"\6\0\1\11\2\1\1\11\6\1\1\11\5\1\1\0"+
"\1\11\4\0\1\11\1\1\2\0\1\11\2\0\2\11"+
"\2\1\1\0\1\1\1\11";
"\11\0\1\11\2\1\1\11\10\1\1\11\5\1\1\11"+
"\5\1\1\0\1\11\6\0\1\11\1\1\4\0\1\11"+
"\2\0\1\1\2\11\1\0\2\1\1\0\2\1\1\11";
private static int [] zzUnpackAttribute() {
int [] result = new int[42];
int [] result = new int[60];
int offset = 0;
offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result);
return result;
@@ -357,7 +355,7 @@ class _KDocLexer implements FlexLexer {
char [] map = new char[0x10000];
int i = 0; /* index in packed string */
int j = 0; /* index in unpacked array */
while (i < 1568) {
while (i < 1216) {
int count = packed.charAt(i++);
char value = packed.charAt(i++);
do map[j++] = value; while (--count > 0);
@@ -580,12 +578,82 @@ class _KDocLexer implements FlexLexer {
zzMarkedPos = zzMarkedPosL;
switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
case 10:
case 15:
{ yybegin(CODE_BLOCK_LINE_BEGINNING);
return KDocTokens.TEXT;
}
case 21: break;
case 13:
{ KDocKnownTag tag = KDocKnownTag.findByTagName(zzBuffer.subSequence(zzStartRead, zzMarkedPos));
yybegin(tag != null && tag.isReferenceRequired() ? TAG_BEGINNING : TAG_TEXT_BEGINNING);
return KDocTokens.TAG_NAME;
}
case 16: break;
case 22: break;
case 10:
{ yybegin(CODE_BLOCK_CONTENTS_BEGINNING);
return KDocTokens.LEADING_ASTERISK;
}
case 23: break;
case 1:
{ return TokenType.BAD_CHARACTER;
}
case 24: break;
case 14:
{ yybegin(CONTENTS_BEGINNING);
return KDocTokens.START;
}
case 25: break;
case 2:
{ yybegin(CONTENTS);
return KDocTokens.TEXT;
}
case 26: break;
case 9:
{ if (yytextContainLineBreaks()) {
yybegin(CODE_BLOCK_LINE_BEGINNING);
return TokenType.WHITE_SPACE;
}
return KDocTokens.TEXT;
}
case 27: break;
case 20:
// lookahead expression with fixed base length
zzMarkedPos = zzStartRead + 3;
{ // Code fence end
yybegin(CONTENTS);
return KDocTokens.TEXT;
}
case 28: break;
case 19:
{ yybegin(CONTENTS);
return KDocTokens.MARKDOWN_INLINE_LINK;
}
case 29: break;
case 5:
{ yybegin(CONTENTS);
return KDocTokens.TEXT;
}
case 30: break;
case 12:
{ yybegin(CONTENTS);
return KDocTokens.MARKDOWN_ESCAPED_CHAR;
}
case 31: break;
case 16:
{ yybegin(TAG_TEXT_BEGINNING);
return KDocTokens.MARKDOWN_LINK;
}
case 32: break;
case 4:
{ yybegin(CONTENTS_BEGINNING);
return KDocTokens.LEADING_ASTERISK;
}
case 33: break;
case 8:
{ yybegin(CODE_BLOCK);
return KDocTokens.TEXT;
}
case 34: break;
case 3:
{ if (yytextContainLineBreaks()) {
yybegin(LINE_BEGINNING);
@@ -595,75 +663,36 @@ class _KDocLexer implements FlexLexer {
return KDocTokens.TEXT; // internal white space
}
}
case 17: break;
case 11:
{ yybegin(CONTENTS_BEGINNING);
return KDocTokens.START;
}
case 18: break;
case 15:
case 35: break;
case 18:
// lookahead expression with fixed lookahead length
yypushback(1);
{ yybegin(CONTENTS);
return KDocTokens.MARKDOWN_INLINE_LINK;
return KDocTokens.MARKDOWN_LINK;
}
case 19: break;
case 8:
case 36: break;
case 17:
{ yybegin(CONTENTS);
return KDocTokens.MARKDOWN_LINK;
}
case 37: break;
case 11:
{ if (isLastToken()) return KDocTokens.END;
else return KDocTokens.TEXT;
}
case 20: break;
case 5:
{ yybegin(CONTENTS);
return KDocTokens.TEXT;
}
case 21: break;
case 12:
{ yybegin(TAG_TEXT_BEGINNING);
return KDocTokens.MARKDOWN_LINK;
}
case 22: break;
case 38: break;
case 7:
{ yybegin(TAG_TEXT_BEGINNING);
return KDocTokens.MARKDOWN_LINK;
}
case 23: break;
case 39: break;
case 6:
{ if (yytextContainLineBreaks()) {
yybegin(LINE_BEGINNING);
}
return TokenType.WHITE_SPACE;
}
case 24: break;
case 14:
// lookahead expression with fixed lookahead length
yypushback(1);
{ yybegin(CONTENTS);
return KDocTokens.MARKDOWN_LINK;
}
case 25: break;
case 13:
{ yybegin(CONTENTS);
return KDocTokens.MARKDOWN_LINK;
}
case 26: break;
case 1:
{ return TokenType.BAD_CHARACTER;
}
case 27: break;
case 9:
{ yybegin(CONTENTS);
return KDocTokens.MARKDOWN_ESCAPED_CHAR;
}
case 28: break;
case 2:
{ yybegin(CONTENTS);
return KDocTokens.TEXT;
}
case 29: break;
case 4:
{ yybegin(CONTENTS_BEGINNING);
return KDocTokens.LEADING_ASTERISK;
}
case 30: break;
case 40: break;
default:
if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
zzAtEOF = true;
compiler/frontend/src/org/jetbrains/kotlin/lexer/_JetLexer.java
+10 -10
View File
@@ -9,7 +9,7 @@ import com.intellij.util.containers.Stack;
/**
* This class is a scanner generated by
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.4.3
* on 12/9/15 7:45 PM from the specification file
* <tt>/home/user/projects/kotlin/compiler/frontend/src/org/jetbrains/kotlin/lexer/Kotlin.flex</tt>
@@ -33,14 +33,14 @@ class _JetLexer implements FlexLexer {
* at the beginning of a line
* l is of the form l = 2*k, k a non negative integer
*/
private static final int ZZ_LEXSTATE[] = {
private static final int ZZ_LEXSTATE[] = {
0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 4, 5, 5
};
/**
/**
* Translates characters to character classes
*/
private static final String ZZ_CMAP_PACKED =
private static final String ZZ_CMAP_PACKED =
"\11\0\1\3\1\7\1\0\1\3\23\0\1\3\1\13\1\27\1\12"+
"\1\10\1\67\1\65\1\24\1\73\1\74\1\36\1\62\1\76\1\23"+
"\1\20\1\11\1\14\11\1\1\70\1\75\1\63\1\60\1\64\1\61"+
@@ -128,12 +128,12 @@ class _JetLexer implements FlexLexer {
"\32\4\13\0\131\4\3\0\6\4\2\0\6\4\2\0\6\4\2\0"+
"\3\4\43\0";
/**
/**
* Translates characters to character classes
*/
private static final char [] ZZ_CMAP = zzUnpackCMap(ZZ_CMAP_PACKED);
/**
/**
* Translates DFA states to action switch labels.
*/
private static final int [] ZZ_ACTION = zzUnpackAction();
@@ -181,7 +181,7 @@ class _JetLexer implements FlexLexer {
}
/**
/**
* Translates a state to a row index in the transition table
*/
private static final int [] ZZ_ROWMAP = zzUnpackRowMap();
@@ -236,7 +236,7 @@ class _JetLexer implements FlexLexer {
return j;
}
/**
/**
* The transition table of the DFA
*/
private static final int [] ZZ_TRANS = zzUnpackTrans();
@@ -618,7 +618,7 @@ class _JetLexer implements FlexLexer {
private final Stack<State> states = new Stack<State>();
private int lBraceCount;
private int commentStart;
private int commentDepth;
@@ -656,7 +656,7 @@ class _JetLexer implements FlexLexer {
}
/**
/**
* Unpacks the compressed character translation table.
*
* @param packed the packed character translation table
compiler/testData/kdoc/lexer/codeBlocks.kt
Vendored
+14
View File
@@ -0,0 +1,14 @@
/**
* @foo This is text.
* ``` kotlin
* @foo This is code block.
* ```
* @foo This is text.
* ~~~
* @foo This is code block
* With multiple lines.
* ~~~
* @foo This is yet more text.
* ```
* @foo This is an unclosed code block.
*/
compiler/testData/kdoc/lexer/codeBlocks.txt
Vendored
+48
View File
@@ -0,0 +1,48 @@
KDOC_START ('/**')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_TEXT (' ')
KDOC_TAG_NAME ('@foo')
WHITE_SPACE (' ')
KDOC_TEXT ('This is text.')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_TEXT (' ``` kotlin')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_TEXT (' @foo This is code block.')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_TEXT (' ```')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_TEXT (' ')
KDOC_TAG_NAME ('@foo')
WHITE_SPACE (' ')
KDOC_TEXT ('This is text.')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_TEXT (' ~~~')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_TEXT (' @foo This is code block')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_TEXT (' With multiple lines.')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_TEXT (' ~~~')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_TEXT (' ')
KDOC_TAG_NAME ('@foo')
WHITE_SPACE (' ')
KDOC_TEXT ('This is yet more text.')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_TEXT (' ```')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_TEXT (' @foo This is an unclosed code block.')
WHITE_SPACE ('\n ')
KDOC_END ('*/')
compiler/testData/psi/kdoc/AtTags.kt
Vendored
+3
View File
@@ -1,5 +1,8 @@
/**
* @tag
* text @notATag
* ```
* @notATag and some description
* ```
* @
*/
compiler/testData/psi/kdoc/AtTags.txt
Vendored
+10 -1
View File
@@ -16,6 +16,15 @@ JetFile: AtTags.kt
PsiElement(KDOC_TEXT)(' text @notATag')
PsiWhiteSpace('\n ')
PsiElement(KDOC_LEADING_ASTERISK)('*')
PsiElement(KDOC_TEXT)(' ```')
PsiWhiteSpace('\n ')
PsiElement(KDOC_LEADING_ASTERISK)('*')
PsiElement(KDOC_TEXT)(' @notATag and some description')
PsiWhiteSpace('\n ')
PsiElement(KDOC_LEADING_ASTERISK)('*')
PsiElement(KDOC_TEXT)(' ```')
PsiWhiteSpace('\n ')
PsiElement(KDOC_LEADING_ASTERISK)('*')
PsiElement(KDOC_TEXT)(' @')
PsiWhiteSpace('\n ')
PsiElement(KDOC_END)('*/')
PsiElement(KDOC_END)('*/')
compiler/tests/org/jetbrains/kotlin/kdoc/AbstractKDocLexerTest.kt
+58
View File
@@ -0,0 +1,58 @@
/*
* Copyright 2010-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.kdoc
import com.intellij.lang.TokenWrapper
import com.intellij.lexer.Lexer
import com.intellij.openapi.util.text.StringUtil
import junit.framework.TestCase
import org.jetbrains.kotlin.kdoc.lexer.KDocLexer
import org.jetbrains.kotlin.test.testFramework.KtUsefulTestCase
import java.io.File
abstract class AbstractKDocLexerTest : TestCase() {
protected fun doTest(fileName: String) {
val text = File(fileName).readText()
val lexerResult = printTokens(text, 0, KDocLexer())
KtUsefulTestCase.assertSameLinesWithFile(fileName.replaceAfterLast(".", "txt"), lexerResult)
}
fun printTokens(text: CharSequence, start: Int, lexer: Lexer): String {
lexer.start(text, start, text.length)
var result = ""
while (true) {
val tokenType = lexer.tokenType ?: break
val tokenText = getTokenText(lexer)
val tokenTypeName = tokenType.toString()
val line = "$tokenTypeName ('$tokenText')\n"
result += line
lexer.advance()
}
return result
}
private fun getTokenText(lexer: Lexer): String {
val tokenType = lexer.tokenType
if (tokenType is TokenWrapper) {
return tokenType.value
}
var text = lexer.bufferSequence.subSequence(lexer.tokenStart, lexer.tokenEnd).toString()
text = StringUtil.replace(text, "\n", "\\n")
return text
}
}
compiler/tests/org/jetbrains/kotlin/kdoc/KDocLexerTestGenerated.java
+43
View File
@@ -0,0 +1,43 @@
/*
* Copyright 2010-2016 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.kdoc;
import com.intellij.testFramework.TestDataPath;
import org.jetbrains.kotlin.test.JUnit3RunnerWithInners;
import org.jetbrains.kotlin.test.KotlinTestUtils;
import org.jetbrains.kotlin.test.TestMetadata;
import org.junit.runner.RunWith;
import java.io.File;
import java.util.regex.Pattern;
/** This class is generated by {@link org.jetbrains.kotlin.generators.tests.TestsPackage}. DO NOT MODIFY MANUALLY */
@SuppressWarnings("all")
@TestMetadata("compiler/testData/kdoc/lexer")
@TestDataPath("$PROJECT_ROOT")
@RunWith(JUnit3RunnerWithInners.class)
public class KDocLexerTestGenerated extends AbstractKDocLexerTest {
public void testAllFilesPresentInLexer() throws Exception {
KotlinTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("compiler/testData/kdoc/lexer"), Pattern.compile("^(.+)\\.kt$"), true);
}
@TestMetadata("codeBlocks.kt")
public void testCodeBlocks() throws Exception {
String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/kdoc/lexer/codeBlocks.kt");
doTest(fileName);
}
}
generators/src/org/jetbrains/kotlin/generators/tests/GenerateTests.kt
+5
View File
@@ -117,6 +117,7 @@ import org.jetbrains.kotlin.js.test.semantics.*
import org.jetbrains.kotlin.jvm.compiler.*
import org.jetbrains.kotlin.jvm.runtime.AbstractJvm8RuntimeDescriptorLoaderTest
import org.jetbrains.kotlin.jvm.runtime.AbstractJvmRuntimeDescriptorLoaderTest
import org.jetbrains.kotlin.kdoc.AbstractKDocLexerTest
import org.jetbrains.kotlin.lang.resolve.android.test.AbstractAndroidBoxTest
import org.jetbrains.kotlin.lang.resolve.android.test.AbstractAndroidBytecodeShapeTest
import org.jetbrains.kotlin.lang.resolve.android.test.AbstractAndroidSyntheticPropertyDescriptorTest
@@ -336,6 +337,10 @@ fun main(args: Array<String>) {
testClass<AbstractLocalClassProtoTest>() {
model("serialization/local")
}
testClass<AbstractKDocLexerTest> {
model("kdoc/lexer")
}
}
testGroup("compiler/tests-java8/tests", "compiler/testData") {