diff --git a/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/KDoc.flex b/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/KDoc.flex index 30e1dd6e747..8a8f6b1001e 100644 --- a/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/KDoc.flex +++ b/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/KDoc.flex @@ -46,9 +46,12 @@ import org.jetbrains.kotlin.kdoc.parser.KDocKnownTag; %state TAG_BEGINNING %state TAG_TEXT_BEGINNING %state CONTENTS +%state CODE_BLOCK +%state CODE_BLOCK_LINE_BEGINNING +%state CODE_BLOCK_CONTENTS_BEGINNING -WHITE_SPACE_CHAR =[\ \t\f\n\r] -NOT_WHITE_SPACE_CHAR=[^\ \t\f\n\r] +WHITE_SPACE_CHAR =[\ \t\f\n] +NOT_WHITE_SPACE_CHAR=[^\ \t\f\n] DIGIT=[0-9] ALPHA=[:jletter:] @@ -58,6 +61,8 @@ QUALIFIED_NAME_START={ALPHA} QUALIFIED_NAME_CHAR={ALPHA}|{DIGIT}|[\.] QUALIFIED_NAME={QUALIFIED_NAME_START}{QUALIFIED_NAME_CHAR}* CODE_LINK=\[{QUALIFIED_NAME}\] +CODE_FENCE_START=("```" | "~~~").* +CODE_FENCE_END=("```" | "~~~") %% @@ -143,6 +148,11 @@ CODE_LINK=\[{QUALIFIED_NAME}\] return KDocTokens.MARKDOWN_INLINE_LINK; } + {CODE_FENCE_START} { + yybegin(CODE_BLOCK_LINE_BEGINNING); + return KDocTokens.TEXT; + } + /* We're only interested in parsing links that can become code references, meaning they contain only identifier characters and characters that can be used in type declarations. No brackets, backticks, asterisks or anything like that. @@ -155,4 +165,47 @@ CODE_LINK=\[{QUALIFIED_NAME}\] return KDocTokens.TEXT; } } + { + "*"+ { + yybegin(CODE_BLOCK_CONTENTS_BEGINNING); + return KDocTokens.LEADING_ASTERISK; + } +} + + { + {CODE_FENCE_END} / [ \t\f]* [\n] [ \t\f]* { + // Code fence end + yybegin(CONTENTS); + return KDocTokens.TEXT; + } + + {WHITE_SPACE_CHAR}+ { + if (yytextContainLineBreaks()) { + yybegin(CODE_BLOCK_LINE_BEGINNING); + return TokenType.WHITE_SPACE; + } + return KDocTokens.TEXT; + } + + . { + yybegin(CODE_BLOCK); + return KDocTokens.TEXT; + } +} + + { + + {WHITE_SPACE_CHAR}+ { + if (yytextContainLineBreaks()) { + yybegin(CODE_BLOCK_LINE_BEGINNING); + return TokenType.WHITE_SPACE; + } + return KDocTokens.TEXT; + } + . { + yybegin(CODE_BLOCK); + return KDocTokens.TEXT; + } +} + . { return TokenType.BAD_CHARACTER; } diff --git a/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/_KDocLexer.java b/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/_KDocLexer.java index d8f1bdec4c9..214c6ec7fb2 100644 --- a/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/_KDocLexer.java +++ b/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/_KDocLexer.java @@ -1,4 +1,4 @@ -/* The following code was generated by JFlex 1.4.3 on 12/9/15 7:45 PM */ +/* The following code was generated by JFlex 1.4.3 on 4/5/16 7:39 PM */ package org.jetbrains.kotlin.kdoc.lexer; @@ -6,25 +6,29 @@ import com.intellij.lexer.FlexLexer; import com.intellij.psi.TokenType; import com.intellij.psi.tree.IElementType; import com.intellij.util.text.CharArrayUtil; +import java.lang.Character; import org.jetbrains.kotlin.kdoc.parser.KDocKnownTag; /** * This class is a scanner generated by * JFlex 1.4.3 - * on 12/9/15 7:45 PM from the specification file - * /home/user/projects/kotlin/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/KDoc.flex + * on 4/5/16 7:39 PM from the specification file + * /Users/yole/jetbrains/kotlin/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/KDoc.flex */ class _KDocLexer implements FlexLexer { /** initial size of the lookahead buffer */ private static final int ZZ_BUFFERSIZE = 16384; /** lexical states */ + public static final int CODE_BLOCK_CONTENTS_BEGINNING = 16; + public static final int TAG_TEXT_BEGINNING = 8; + public static final int CODE_BLOCK = 12; + public static final int CODE_BLOCK_LINE_BEGINNING = 14; public static final int CONTENTS_BEGINNING = 4; - public static final int LINE_BEGINNING = 2; public static final int CONTENTS = 10; public static final int YYINITIAL = 0; - public static final int TAG_TEXT_BEGINNING = 8; + public static final int LINE_BEGINNING = 2; public static final int TAG_BEGINNING = 6; /** @@ -34,92 +38,75 @@ class _KDocLexer implements FlexLexer { * l is of the form l = 2*k, k a non negative integer */ private static final int ZZ_LEXSTATE[] = { - 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5 + 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, + 8, 8 }; /** * Translates characters to character classes */ private static final String ZZ_CMAP_PACKED = - "\11\0\1\1\1\12\1\0\2\1\22\0\1\1\3\0\1\3\3\0"+ - "\1\14\1\15\1\10\3\0\1\4\1\7\12\2\6\0\1\11\32\3"+ - "\1\5\1\13\1\6\1\0\1\3\1\0\32\3\47\0\4\3\4\0"+ - "\1\3\12\0\1\3\4\0\1\3\5\0\27\3\1\0\37\3\1\0"+ - "\u01ca\3\4\0\14\3\16\0\5\3\7\0\1\3\1\0\1\3\201\0"+ - "\5\3\1\0\2\3\2\0\4\3\10\0\1\3\1\0\3\3\1\0"+ - "\1\3\1\0\24\3\1\0\123\3\1\0\213\3\10\0\236\3\11\0"+ - "\46\3\2\0\1\3\7\0\47\3\110\0\33\3\5\0\3\3\30\0"+ - "\1\3\24\0\53\3\43\0\2\3\1\0\143\3\1\0\1\3\17\0"+ - "\2\3\7\0\2\3\12\0\3\3\2\0\1\3\20\0\1\3\1\0"+ - "\36\3\35\0\131\3\13\0\1\3\30\0\41\3\11\0\2\3\4\0"+ - "\1\3\5\0\26\3\4\0\1\3\11\0\1\3\3\0\1\3\27\0"+ - "\31\3\253\0\66\3\3\0\1\3\22\0\1\3\7\0\12\3\17\0"+ - "\7\3\1\0\7\3\5\0\10\3\2\0\2\3\2\0\26\3\1\0"+ - "\7\3\1\0\1\3\3\0\4\3\3\0\1\3\20\0\1\3\15\0"+ - "\2\3\1\0\3\3\16\0\4\3\7\0\1\3\11\0\6\3\4\0"+ - "\2\3\2\0\26\3\1\0\7\3\1\0\2\3\1\0\2\3\1\0"+ - "\2\3\37\0\4\3\1\0\1\3\23\0\3\3\20\0\11\3\1\0"+ - "\3\3\1\0\26\3\1\0\7\3\1\0\2\3\1\0\5\3\3\0"+ - "\1\3\22\0\1\3\17\0\2\3\17\0\1\3\23\0\10\3\2\0"+ - "\2\3\2\0\26\3\1\0\7\3\1\0\2\3\1\0\5\3\3\0"+ - "\1\3\36\0\2\3\1\0\3\3\17\0\1\3\21\0\1\3\1\0"+ - "\6\3\3\0\3\3\1\0\4\3\3\0\2\3\1\0\1\3\1\0"+ - "\2\3\3\0\2\3\3\0\3\3\3\0\14\3\26\0\1\3\50\0"+ - "\1\3\13\0\10\3\1\0\3\3\1\0\27\3\1\0\12\3\1\0"+ - "\5\3\3\0\1\3\32\0\2\3\6\0\2\3\43\0\10\3\1\0"+ - "\3\3\1\0\27\3\1\0\12\3\1\0\5\3\3\0\1\3\40\0"+ - "\1\3\1\0\2\3\17\0\2\3\22\0\10\3\1\0\3\3\1\0"+ - "\51\3\2\0\1\3\20\0\1\3\21\0\2\3\30\0\6\3\5\0"+ - "\22\3\3\0\30\3\1\0\11\3\1\0\1\3\2\0\7\3\72\0"+ - "\60\3\1\0\2\3\13\0\10\3\72\0\2\3\1\0\1\3\2\0"+ - "\2\3\1\0\1\3\2\0\1\3\6\0\4\3\1\0\7\3\1\0"+ - "\3\3\1\0\1\3\1\0\1\3\2\0\2\3\1\0\4\3\1\0"+ - "\2\3\11\0\1\3\2\0\5\3\1\0\1\3\25\0\2\3\42\0"+ - "\1\3\77\0\10\3\1\0\44\3\33\0\5\3\163\0\53\3\24\0"+ - "\1\3\20\0\6\3\4\0\4\3\3\0\1\3\3\0\2\3\7\0"+ - "\3\3\4\0\15\3\14\0\1\3\21\0\46\3\12\0\53\3\1\0"+ - "\1\3\3\0\u0149\3\1\0\4\3\2\0\7\3\1\0\1\3\1\0"+ - "\4\3\2\0\51\3\1\0\4\3\2\0\41\3\1\0\4\3\2\0"+ - "\7\3\1\0\1\3\1\0\4\3\2\0\17\3\1\0\71\3\1\0"+ - "\4\3\2\0\103\3\45\0\20\3\20\0\125\3\14\0\u026c\3\2\0"+ - "\21\3\1\0\32\3\5\0\113\3\3\0\3\3\17\0\15\3\1\0"+ - "\4\3\16\0\22\3\16\0\22\3\16\0\15\3\1\0\3\3\17\0"+ - "\64\3\43\0\1\3\3\0\2\3\103\0\130\3\10\0\51\3\1\0"+ - "\1\3\5\0\106\3\12\0\35\3\63\0\36\3\2\0\5\3\13\0"+ - "\54\3\25\0\7\3\70\0\27\3\11\0\65\3\122\0\1\3\135\0"+ - "\57\3\21\0\7\3\67\0\36\3\15\0\2\3\20\0\46\3\32\0"+ - "\44\3\51\0\3\3\12\0\44\3\153\0\4\3\1\0\4\3\16\0"+ - "\300\3\100\0\u0116\3\2\0\6\3\2\0\46\3\2\0\6\3\2\0"+ - "\10\3\1\0\1\3\1\0\1\3\1\0\1\3\1\0\37\3\2\0"+ - "\65\3\1\0\7\3\1\0\1\3\3\0\3\3\1\0\7\3\3\0"+ - "\4\3\2\0\6\3\4\0\15\3\5\0\3\3\1\0\7\3\102\0"+ - "\2\3\23\0\1\3\34\0\1\3\15\0\1\3\20\0\15\3\3\0"+ - "\32\3\110\0\1\3\4\0\1\3\2\0\12\3\1\0\1\3\3\0"+ - "\5\3\6\0\1\3\1\0\1\3\1\0\1\3\1\0\4\3\1\0"+ - "\13\3\2\0\4\3\5\0\5\3\4\0\1\3\21\0\51\3\u0a77\0"+ - "\57\3\1\0\57\3\1\0\205\3\6\0\4\3\21\0\46\3\12\0"+ - "\66\3\11\0\1\3\20\0\27\3\11\0\7\3\1\0\7\3\1\0"+ - "\7\3\1\0\7\3\1\0\7\3\1\0\7\3\1\0\7\3\1\0"+ - "\7\3\120\0\1\3\u01d5\0\3\3\31\0\11\3\7\0\5\3\2\0"+ - "\5\3\4\0\126\3\6\0\3\3\1\0\132\3\1\0\4\3\5\0"+ - "\51\3\3\0\136\3\21\0\33\3\65\0\20\3\u0200\0\u19b6\3\112\0"+ - "\u51cc\3\64\0\u048d\3\103\0\56\3\2\0\u010d\3\3\0\20\3\12\0"+ - "\2\3\24\0\57\3\20\0\31\3\10\0\120\3\47\0\11\3\2\0"+ - "\147\3\2\0\4\3\1\0\2\3\16\0\12\3\120\0\10\3\1\0"+ - "\3\3\1\0\4\3\1\0\27\3\25\0\1\3\7\0\64\3\16\0"+ - "\62\3\76\0\6\3\3\0\1\3\16\0\34\3\12\0\27\3\31\0"+ - "\35\3\7\0\57\3\34\0\1\3\60\0\51\3\27\0\3\3\1\0"+ - "\10\3\24\0\27\3\3\0\1\3\5\0\60\3\1\0\1\3\3\0"+ - "\2\3\2\0\5\3\2\0\1\3\1\0\1\3\30\0\3\3\43\0"+ - "\6\3\2\0\6\3\2\0\6\3\11\0\7\3\1\0\7\3\221\0"+ - "\43\3\35\0\u2ba4\3\14\0\27\3\4\0\61\3\u2104\0\u012e\3\2\0"+ - "\76\3\2\0\152\3\46\0\7\3\14\0\5\3\5\0\1\3\1\0"+ - "\12\3\1\0\15\3\1\0\5\3\1\0\1\3\1\0\2\3\1\0"+ - "\2\3\1\0\154\3\41\0\u016b\3\22\0\100\3\2\0\66\3\50\0"+ - "\15\3\66\0\2\3\30\0\3\3\31\0\1\3\6\0\5\3\1\0"+ - "\207\3\7\0\1\3\34\0\32\3\4\0\1\3\1\0\32\3\13\0"+ - "\131\3\3\0\6\3\2\0\6\3\2\0\6\3\2\0\3\3\3\0"+ - "\2\3\3\0\2\3\31\0"; + "\11\0\1\1\1\11\1\0\1\1\23\0\1\1\3\0\1\3\3\0"+ + "\1\16\1\17\1\13\3\0\1\4\1\12\12\2\6\0\1\14\32\3"+ + "\1\5\1\15\1\6\1\0\1\3\1\7\32\3\3\0\1\10\43\0"+ + "\4\3\4\0\1\3\12\0\1\3\4\0\1\3\5\0\27\3\1\0"+ + "\37\3\1\0\u013f\3\31\0\162\3\4\0\14\3\16\0\5\3\11\0"+ + "\1\3\213\0\1\3\13\0\1\3\1\0\3\3\1\0\1\3\1\0"+ + "\24\3\1\0\54\3\1\0\46\3\1\0\5\3\4\0\202\3\10\0"+ + "\105\3\1\0\46\3\2\0\2\3\6\0\20\3\41\0\46\3\2\0"+ + "\1\3\7\0\47\3\110\0\33\3\5\0\3\3\56\0\32\3\5\0"+ + "\13\3\43\0\2\3\1\0\143\3\1\0\1\3\17\0\2\3\7\0"+ + "\2\3\12\0\3\3\2\0\1\3\20\0\1\3\1\0\36\3\35\0"+ + "\3\3\60\0\46\3\13\0\1\3\u0152\0\66\3\3\0\1\3\22\0"+ + "\1\3\7\0\12\3\43\0\10\3\2\0\2\3\2\0\26\3\1\0"+ + "\7\3\1\0\1\3\3\0\4\3\3\0\1\3\36\0\2\3\1\0"+ + "\3\3\16\0\4\3\21\0\6\3\4\0\2\3\2\0\26\3\1\0"+ + "\7\3\1\0\2\3\1\0\2\3\1\0\2\3\37\0\4\3\1\0"+ + "\1\3\23\0\3\3\20\0\11\3\1\0\3\3\1\0\26\3\1\0"+ + "\7\3\1\0\2\3\1\0\5\3\3\0\1\3\22\0\1\3\17\0"+ + "\2\3\17\0\1\3\23\0\10\3\2\0\2\3\2\0\26\3\1\0"+ + "\7\3\1\0\2\3\1\0\5\3\3\0\1\3\36\0\2\3\1\0"+ + "\3\3\17\0\1\3\21\0\1\3\1\0\6\3\3\0\3\3\1\0"+ + "\4\3\3\0\2\3\1\0\1\3\1\0\2\3\3\0\2\3\3\0"+ + "\3\3\3\0\10\3\1\0\3\3\77\0\1\3\13\0\10\3\1\0"+ + "\3\3\1\0\27\3\1\0\12\3\1\0\5\3\46\0\2\3\43\0"+ + "\10\3\1\0\3\3\1\0\27\3\1\0\12\3\1\0\5\3\3\0"+ + "\1\3\40\0\1\3\1\0\2\3\43\0\10\3\1\0\3\3\1\0"+ + "\27\3\1\0\20\3\46\0\2\3\43\0\22\3\3\0\30\3\1\0"+ + "\11\3\1\0\1\3\2\0\7\3\72\0\60\3\1\0\2\3\13\0"+ + "\10\3\72\0\2\3\1\0\1\3\2\0\2\3\1\0\1\3\2\0"+ + "\1\3\6\0\4\3\1\0\7\3\1\0\3\3\1\0\1\3\1\0"+ + "\1\3\2\0\2\3\1\0\4\3\1\0\2\3\11\0\1\3\2\0"+ + "\5\3\1\0\1\3\25\0\2\3\42\0\1\3\77\0\10\3\1\0"+ + "\42\3\35\0\4\3\164\0\42\3\1\0\5\3\1\0\2\3\45\0"+ + "\6\3\112\0\46\3\12\0\51\3\7\0\132\3\5\0\104\3\5\0"+ + "\122\3\6\0\7\3\1\0\77\3\1\0\1\3\1\0\4\3\2\0"+ + "\7\3\1\0\1\3\1\0\4\3\2\0\47\3\1\0\1\3\1\0"+ + "\4\3\2\0\37\3\1\0\1\3\1\0\4\3\2\0\7\3\1\0"+ + "\1\3\1\0\4\3\2\0\7\3\1\0\7\3\1\0\27\3\1\0"+ + "\37\3\1\0\1\3\1\0\4\3\2\0\7\3\1\0\47\3\1\0"+ + "\23\3\105\0\125\3\14\0\u026c\3\2\0\10\3\12\0\32\3\5\0"+ + "\113\3\3\0\3\3\17\0\15\3\1\0\4\3\16\0\22\3\16\0"+ + "\22\3\16\0\15\3\1\0\3\3\17\0\64\3\43\0\1\3\3\0"+ + "\2\3\103\0\130\3\10\0\51\3\127\0\35\3\63\0\36\3\2\0"+ + "\5\3\u038b\0\154\3\224\0\234\3\4\0\132\3\6\0\26\3\2\0"+ + "\6\3\2\0\46\3\2\0\6\3\2\0\10\3\1\0\1\3\1\0"+ + "\1\3\1\0\1\3\1\0\37\3\2\0\65\3\1\0\7\3\1\0"+ + "\1\3\3\0\3\3\1\0\7\3\3\0\4\3\2\0\6\3\4\0"+ + "\15\3\5\0\3\3\1\0\7\3\102\0\2\3\23\0\1\3\34\0"+ + "\1\3\15\0\1\3\40\0\22\3\120\0\1\3\4\0\1\3\2\0"+ + "\12\3\1\0\1\3\3\0\5\3\6\0\1\3\1\0\1\3\1\0"+ + "\1\3\1\0\4\3\1\0\3\3\1\0\7\3\3\0\3\3\5\0"+ + "\5\3\26\0\44\3\u0e81\0\3\3\31\0\11\3\7\0\5\3\2\0"+ + "\5\3\4\0\126\3\6\0\3\3\1\0\137\3\5\0\50\3\4\0"+ + "\136\3\21\0\30\3\70\0\20\3\u0200\0\u19b6\3\112\0\u51a6\3\132\0"+ + "\u048d\3\u0773\0\u2ba4\3\u215c\0\u012e\3\2\0\73\3\225\0\7\3\14\0"+ + "\5\3\5\0\1\3\1\0\12\3\1\0\15\3\1\0\5\3\1\0"+ + "\1\3\1\0\2\3\1\0\2\3\1\0\154\3\41\0\u016b\3\22\0"+ + "\100\3\2\0\66\3\50\0\15\3\66\0\2\3\30\0\3\3\31\0"+ + "\1\3\6\0\5\3\1\0\207\3\7\0\1\3\34\0\32\3\4\0"+ + "\1\3\1\0\32\3\12\0\132\3\3\0\6\3\2\0\6\3\2\0"+ + "\6\3\2\0\3\3\3\0\2\3\3\0\2\3\31\0"; /** * Translates characters to character classes @@ -132,12 +119,13 @@ class _KDocLexer implements FlexLexer { private static final int [] ZZ_ACTION = zzUnpackAction(); private static final String ZZ_ACTION_PACKED_0 = - "\6\0\3\1\1\2\1\3\1\2\1\4\3\2\1\5"+ - "\1\6\1\7\3\5\1\0\1\10\4\0\1\11\1\12"+ - "\2\0\1\13\2\0\1\14\1\15\2\16\1\0\2\17"; + "\11\0\3\1\1\2\1\3\3\2\1\4\3\2\1\5"+ + "\1\6\1\7\3\5\1\10\1\11\3\10\1\12\1\0"+ + "\1\13\6\0\1\14\1\15\4\0\1\16\2\0\1\17"+ + "\1\20\1\21\1\0\2\22\1\0\1\23\1\24\1\23"; private static int [] zzUnpackAction() { - int [] result = new int[42]; + int [] result = new int[60]; int offset = 0; offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result); return result; @@ -162,15 +150,17 @@ class _KDocLexer implements FlexLexer { private static final int [] ZZ_ROWMAP = zzUnpackRowMap(); private static final String ZZ_ROWMAP_PACKED_0 = - "\0\0\0\16\0\34\0\52\0\70\0\106\0\124\0\142"+ - "\0\160\0\124\0\176\0\214\0\232\0\250\0\160\0\266"+ - "\0\124\0\304\0\322\0\340\0\160\0\356\0\374\0\124"+ - "\0\160\0\u010a\0\u0118\0\u0126\0\124\0\u0134\0\u0142\0\u0150"+ - "\0\124\0\u015e\0\u016c\0\124\0\124\0\u010a\0\u0126\0\u017a"+ - "\0\u010a\0\124"; + "\0\0\0\20\0\40\0\60\0\100\0\120\0\140\0\160"+ + "\0\200\0\220\0\240\0\260\0\220\0\300\0\320\0\340"+ + "\0\360\0\u0100\0\u0110\0\260\0\u0120\0\220\0\u0130\0\u0140"+ + "\0\u0150\0\260\0\u0160\0\220\0\u0170\0\260\0\u0180\0\u0190"+ + "\0\u01a0\0\u01b0\0\220\0\260\0\u01c0\0\u01d0\0\u01e0\0\u01f0"+ + "\0\u0200\0\220\0\u0210\0\u0220\0\u0230\0\u0240\0\u0250\0\220"+ + "\0\u0260\0\u0270\0\u0280\0\220\0\220\0\u0290\0\u01c0\0\u01e0"+ + "\0\u02a0\0\u01c0\0\u02b0\0\220"; private static int [] zzUnpackRowMap() { - int [] result = new int[42]; + int [] result = new int[60]; int offset = 0; offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result); return result; @@ -193,25 +183,33 @@ class _KDocLexer implements FlexLexer { private static final int [] ZZ_TRANS = zzUnpackTrans(); private static final String ZZ_TRANS_PACKED_0 = - "\7\7\1\10\1\11\1\7\1\0\3\7\1\12\1\13"+ - "\3\12\1\14\2\12\1\15\1\12\1\13\1\16\3\12"+ - "\1\13\3\12\1\14\2\12\1\17\1\20\1\13\1\16"+ - "\2\12\1\21\1\22\1\21\1\23\1\21\1\24\2\21"+ - "\1\25\1\21\1\22\4\21\1\22\3\21\1\26\2\21"+ - "\1\25\1\21\1\22\3\21\1\12\1\13\3\12\1\14"+ - "\2\12\1\17\1\12\1\13\1\16\2\12\26\0\1\27"+ - "\14\0\1\30\1\31\6\0\1\13\10\0\1\13\3\0"+ - "\3\32\1\33\1\32\1\0\1\34\7\32\7\0\1\30"+ - "\1\15\12\0\2\35\12\0\1\36\13\0\1\22\10\0"+ - "\1\22\5\0\3\23\14\0\1\37\15\0\1\40\22\0"+ - "\1\41\5\0\5\32\1\0\1\34\11\32\3\33\1\0"+ - "\1\42\14\32\1\0\1\34\5\32\1\43\1\32\2\0"+ - "\2\36\14\0\3\37\1\0\1\44\11\0\3\40\1\0"+ - "\1\45\7\0\5\46\1\0\1\47\5\46\1\43\1\46"+ - "\5\43\1\50\7\43\1\51\15\50\1\52"; + "\11\12\1\0\1\13\1\14\4\12\1\15\1\16\3\15"+ + "\1\17\1\15\1\20\1\21\1\16\1\15\1\22\1\15"+ + "\1\23\3\15\1\16\3\15\1\17\1\15\1\20\1\21"+ + "\1\16\1\15\1\24\1\25\1\23\2\15\1\26\1\27"+ + "\1\26\1\30\1\26\1\31\3\26\1\27\1\26\1\32"+ + "\5\26\1\27\3\26\1\33\3\26\1\27\1\26\1\32"+ + "\4\26\1\15\1\16\3\15\1\17\1\15\1\20\1\21"+ + "\1\16\1\15\1\24\1\15\1\23\2\15\1\34\1\35"+ + "\7\34\1\35\1\34\1\36\5\34\1\35\5\34\1\37"+ + "\1\40\1\35\1\34\1\41\5\34\1\35\5\34\1\37"+ + "\1\40\1\35\1\34\1\36\4\34\33\0\1\42\16\0"+ + "\1\43\1\44\5\0\1\16\7\0\1\16\6\0\3\45"+ + "\1\46\1\45\1\0\1\47\11\45\7\0\1\50\20\0"+ + "\1\51\21\0\1\43\1\22\11\0\2\52\14\0\1\53"+ + "\15\0\1\27\7\0\1\27\10\0\3\30\16\0\1\54"+ + "\17\0\1\55\15\0\1\35\7\0\1\35\15\0\1\56"+ + "\20\0\1\57\21\0\1\43\1\41\17\0\1\60\4\0"+ + "\5\45\1\0\1\47\13\45\3\46\1\0\1\61\16\45"+ + "\1\0\1\47\7\45\1\62\1\45\7\0\1\63\20\0"+ + "\1\63\11\0\2\53\16\0\3\54\1\0\1\64\13\0"+ + "\3\55\1\0\1\65\20\0\1\66\20\0\1\66\7\0"+ + "\5\67\1\0\1\70\7\67\1\62\1\67\5\62\1\71"+ + "\11\62\1\72\11\63\1\0\6\63\1\0\1\66\7\0"+ + "\1\73\6\0\17\71\1\74\1\0\1\73\16\0"; private static int [] zzUnpackTrans() { - int [] result = new int[392]; + int [] result = new int[704]; int offset = 0; offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result); return result; @@ -252,12 +250,12 @@ class _KDocLexer implements FlexLexer { private static final int [] ZZ_ATTRIBUTE = zzUnpackAttribute(); private static final String ZZ_ATTRIBUTE_PACKED_0 = - "\6\0\1\11\2\1\1\11\6\1\1\11\5\1\1\0"+ - "\1\11\4\0\1\11\1\1\2\0\1\11\2\0\2\11"+ - "\2\1\1\0\1\1\1\11"; + "\11\0\1\11\2\1\1\11\10\1\1\11\5\1\1\11"+ + "\5\1\1\0\1\11\6\0\1\11\1\1\4\0\1\11"+ + "\2\0\1\1\2\11\1\0\2\1\1\0\2\1\1\11"; private static int [] zzUnpackAttribute() { - int [] result = new int[42]; + int [] result = new int[60]; int offset = 0; offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result); return result; @@ -357,7 +355,7 @@ class _KDocLexer implements FlexLexer { char [] map = new char[0x10000]; int i = 0; /* index in packed string */ int j = 0; /* index in unpacked array */ - while (i < 1568) { + while (i < 1216) { int count = packed.charAt(i++); char value = packed.charAt(i++); do map[j++] = value; while (--count > 0); @@ -580,12 +578,82 @@ class _KDocLexer implements FlexLexer { zzMarkedPos = zzMarkedPosL; switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) { - case 10: + case 15: + { yybegin(CODE_BLOCK_LINE_BEGINNING); + return KDocTokens.TEXT; + } + case 21: break; + case 13: { KDocKnownTag tag = KDocKnownTag.findByTagName(zzBuffer.subSequence(zzStartRead, zzMarkedPos)); yybegin(tag != null && tag.isReferenceRequired() ? TAG_BEGINNING : TAG_TEXT_BEGINNING); return KDocTokens.TAG_NAME; } - case 16: break; + case 22: break; + case 10: + { yybegin(CODE_BLOCK_CONTENTS_BEGINNING); + return KDocTokens.LEADING_ASTERISK; + } + case 23: break; + case 1: + { return TokenType.BAD_CHARACTER; + } + case 24: break; + case 14: + { yybegin(CONTENTS_BEGINNING); + return KDocTokens.START; + } + case 25: break; + case 2: + { yybegin(CONTENTS); + return KDocTokens.TEXT; + } + case 26: break; + case 9: + { if (yytextContainLineBreaks()) { + yybegin(CODE_BLOCK_LINE_BEGINNING); + return TokenType.WHITE_SPACE; + } + return KDocTokens.TEXT; + } + case 27: break; + case 20: + // lookahead expression with fixed base length + zzMarkedPos = zzStartRead + 3; + { // Code fence end + yybegin(CONTENTS); + return KDocTokens.TEXT; + } + case 28: break; + case 19: + { yybegin(CONTENTS); + return KDocTokens.MARKDOWN_INLINE_LINK; + } + case 29: break; + case 5: + { yybegin(CONTENTS); + return KDocTokens.TEXT; + } + case 30: break; + case 12: + { yybegin(CONTENTS); + return KDocTokens.MARKDOWN_ESCAPED_CHAR; + } + case 31: break; + case 16: + { yybegin(TAG_TEXT_BEGINNING); + return KDocTokens.MARKDOWN_LINK; + } + case 32: break; + case 4: + { yybegin(CONTENTS_BEGINNING); + return KDocTokens.LEADING_ASTERISK; + } + case 33: break; + case 8: + { yybegin(CODE_BLOCK); + return KDocTokens.TEXT; + } + case 34: break; case 3: { if (yytextContainLineBreaks()) { yybegin(LINE_BEGINNING); @@ -595,75 +663,36 @@ class _KDocLexer implements FlexLexer { return KDocTokens.TEXT; // internal white space } } - case 17: break; - case 11: - { yybegin(CONTENTS_BEGINNING); - return KDocTokens.START; - } - case 18: break; - case 15: + case 35: break; + case 18: + // lookahead expression with fixed lookahead length + yypushback(1); { yybegin(CONTENTS); - return KDocTokens.MARKDOWN_INLINE_LINK; + return KDocTokens.MARKDOWN_LINK; } - case 19: break; - case 8: + case 36: break; + case 17: + { yybegin(CONTENTS); + return KDocTokens.MARKDOWN_LINK; + } + case 37: break; + case 11: { if (isLastToken()) return KDocTokens.END; else return KDocTokens.TEXT; } - case 20: break; - case 5: - { yybegin(CONTENTS); - return KDocTokens.TEXT; - } - case 21: break; - case 12: - { yybegin(TAG_TEXT_BEGINNING); - return KDocTokens.MARKDOWN_LINK; - } - case 22: break; + case 38: break; case 7: { yybegin(TAG_TEXT_BEGINNING); return KDocTokens.MARKDOWN_LINK; } - case 23: break; + case 39: break; case 6: { if (yytextContainLineBreaks()) { yybegin(LINE_BEGINNING); } return TokenType.WHITE_SPACE; } - case 24: break; - case 14: - // lookahead expression with fixed lookahead length - yypushback(1); - { yybegin(CONTENTS); - return KDocTokens.MARKDOWN_LINK; - } - case 25: break; - case 13: - { yybegin(CONTENTS); - return KDocTokens.MARKDOWN_LINK; - } - case 26: break; - case 1: - { return TokenType.BAD_CHARACTER; - } - case 27: break; - case 9: - { yybegin(CONTENTS); - return KDocTokens.MARKDOWN_ESCAPED_CHAR; - } - case 28: break; - case 2: - { yybegin(CONTENTS); - return KDocTokens.TEXT; - } - case 29: break; - case 4: - { yybegin(CONTENTS_BEGINNING); - return KDocTokens.LEADING_ASTERISK; - } - case 30: break; + case 40: break; default: if (zzInput == YYEOF && zzStartRead == zzCurrentPos) { zzAtEOF = true; diff --git a/compiler/frontend/src/org/jetbrains/kotlin/lexer/_JetLexer.java b/compiler/frontend/src/org/jetbrains/kotlin/lexer/_JetLexer.java index 456022afde9..173fb23adf0 100644 --- a/compiler/frontend/src/org/jetbrains/kotlin/lexer/_JetLexer.java +++ b/compiler/frontend/src/org/jetbrains/kotlin/lexer/_JetLexer.java @@ -9,7 +9,7 @@ import com.intellij.util.containers.Stack; /** - * This class is a scanner generated by + * This class is a scanner generated by * JFlex 1.4.3 * on 12/9/15 7:45 PM from the specification file * /home/user/projects/kotlin/compiler/frontend/src/org/jetbrains/kotlin/lexer/Kotlin.flex @@ -33,14 +33,14 @@ class _JetLexer implements FlexLexer { * at the beginning of a line * l is of the form l = 2*k, k a non negative integer */ - private static final int ZZ_LEXSTATE[] = { + private static final int ZZ_LEXSTATE[] = { 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 4, 5, 5 }; - /** + /** * Translates characters to character classes */ - private static final String ZZ_CMAP_PACKED = + private static final String ZZ_CMAP_PACKED = "\11\0\1\3\1\7\1\0\1\3\23\0\1\3\1\13\1\27\1\12"+ "\1\10\1\67\1\65\1\24\1\73\1\74\1\36\1\62\1\76\1\23"+ "\1\20\1\11\1\14\11\1\1\70\1\75\1\63\1\60\1\64\1\61"+ @@ -128,12 +128,12 @@ class _JetLexer implements FlexLexer { "\32\4\13\0\131\4\3\0\6\4\2\0\6\4\2\0\6\4\2\0"+ "\3\4\43\0"; - /** + /** * Translates characters to character classes */ private static final char [] ZZ_CMAP = zzUnpackCMap(ZZ_CMAP_PACKED); - /** + /** * Translates DFA states to action switch labels. */ private static final int [] ZZ_ACTION = zzUnpackAction(); @@ -181,7 +181,7 @@ class _JetLexer implements FlexLexer { } - /** + /** * Translates a state to a row index in the transition table */ private static final int [] ZZ_ROWMAP = zzUnpackRowMap(); @@ -236,7 +236,7 @@ class _JetLexer implements FlexLexer { return j; } - /** + /** * The transition table of the DFA */ private static final int [] ZZ_TRANS = zzUnpackTrans(); @@ -618,7 +618,7 @@ class _JetLexer implements FlexLexer { private final Stack states = new Stack(); private int lBraceCount; - + private int commentStart; private int commentDepth; @@ -656,7 +656,7 @@ class _JetLexer implements FlexLexer { } - /** + /** * Unpacks the compressed character translation table. * * @param packed the packed character translation table diff --git a/compiler/testData/kdoc/lexer/codeBlocks.kt b/compiler/testData/kdoc/lexer/codeBlocks.kt new file mode 100644 index 00000000000..2b7a2d7a0b3 --- /dev/null +++ b/compiler/testData/kdoc/lexer/codeBlocks.kt @@ -0,0 +1,14 @@ +/** + * @foo This is text. + * ``` kotlin + * @foo This is code block. + * ``` + * @foo This is text. + * ~~~ + * @foo This is code block + * With multiple lines. + * ~~~ + * @foo This is yet more text. + * ``` + * @foo This is an unclosed code block. + */ \ No newline at end of file diff --git a/compiler/testData/kdoc/lexer/codeBlocks.txt b/compiler/testData/kdoc/lexer/codeBlocks.txt new file mode 100644 index 00000000000..e8ea11b5b9d --- /dev/null +++ b/compiler/testData/kdoc/lexer/codeBlocks.txt @@ -0,0 +1,48 @@ +KDOC_START ('/**') +WHITE_SPACE ('\n ') +KDOC_LEADING_ASTERISK ('*') +KDOC_TEXT (' ') +KDOC_TAG_NAME ('@foo') +WHITE_SPACE (' ') +KDOC_TEXT ('This is text.') +WHITE_SPACE ('\n ') +KDOC_LEADING_ASTERISK ('*') +KDOC_TEXT (' ``` kotlin') +WHITE_SPACE ('\n ') +KDOC_LEADING_ASTERISK ('*') +KDOC_TEXT (' @foo This is code block.') +WHITE_SPACE ('\n ') +KDOC_LEADING_ASTERISK ('*') +KDOC_TEXT (' ```') +WHITE_SPACE ('\n ') +KDOC_LEADING_ASTERISK ('*') +KDOC_TEXT (' ') +KDOC_TAG_NAME ('@foo') +WHITE_SPACE (' ') +KDOC_TEXT ('This is text.') +WHITE_SPACE ('\n ') +KDOC_LEADING_ASTERISK ('*') +KDOC_TEXT (' ~~~') +WHITE_SPACE ('\n ') +KDOC_LEADING_ASTERISK ('*') +KDOC_TEXT (' @foo This is code block') +WHITE_SPACE ('\n ') +KDOC_LEADING_ASTERISK ('*') +KDOC_TEXT (' With multiple lines.') +WHITE_SPACE ('\n ') +KDOC_LEADING_ASTERISK ('*') +KDOC_TEXT (' ~~~') +WHITE_SPACE ('\n ') +KDOC_LEADING_ASTERISK ('*') +KDOC_TEXT (' ') +KDOC_TAG_NAME ('@foo') +WHITE_SPACE (' ') +KDOC_TEXT ('This is yet more text.') +WHITE_SPACE ('\n ') +KDOC_LEADING_ASTERISK ('*') +KDOC_TEXT (' ```') +WHITE_SPACE ('\n ') +KDOC_LEADING_ASTERISK ('*') +KDOC_TEXT (' @foo This is an unclosed code block.') +WHITE_SPACE ('\n ') +KDOC_END ('*/') \ No newline at end of file diff --git a/compiler/testData/psi/kdoc/AtTags.kt b/compiler/testData/psi/kdoc/AtTags.kt index 8798fcad57b..eda6e13be0e 100644 --- a/compiler/testData/psi/kdoc/AtTags.kt +++ b/compiler/testData/psi/kdoc/AtTags.kt @@ -1,5 +1,8 @@ /** * @tag * text @notATag + * ``` + * @notATag and some description + * ``` * @ */ \ No newline at end of file diff --git a/compiler/testData/psi/kdoc/AtTags.txt b/compiler/testData/psi/kdoc/AtTags.txt index e386619a476..dd452f5925c 100644 --- a/compiler/testData/psi/kdoc/AtTags.txt +++ b/compiler/testData/psi/kdoc/AtTags.txt @@ -16,6 +16,15 @@ JetFile: AtTags.kt PsiElement(KDOC_TEXT)(' text @notATag') PsiWhiteSpace('\n ') PsiElement(KDOC_LEADING_ASTERISK)('*') + PsiElement(KDOC_TEXT)(' ```') + PsiWhiteSpace('\n ') + PsiElement(KDOC_LEADING_ASTERISK)('*') + PsiElement(KDOC_TEXT)(' @notATag and some description') + PsiWhiteSpace('\n ') + PsiElement(KDOC_LEADING_ASTERISK)('*') + PsiElement(KDOC_TEXT)(' ```') + PsiWhiteSpace('\n ') + PsiElement(KDOC_LEADING_ASTERISK)('*') PsiElement(KDOC_TEXT)(' @') PsiWhiteSpace('\n ') - PsiElement(KDOC_END)('*/') + PsiElement(KDOC_END)('*/') \ No newline at end of file diff --git a/compiler/tests/org/jetbrains/kotlin/kdoc/AbstractKDocLexerTest.kt b/compiler/tests/org/jetbrains/kotlin/kdoc/AbstractKDocLexerTest.kt new file mode 100644 index 00000000000..6ade22734f0 --- /dev/null +++ b/compiler/tests/org/jetbrains/kotlin/kdoc/AbstractKDocLexerTest.kt @@ -0,0 +1,58 @@ +/* + * Copyright 2010-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.kdoc + +import com.intellij.lang.TokenWrapper +import com.intellij.lexer.Lexer +import com.intellij.openapi.util.text.StringUtil +import junit.framework.TestCase +import org.jetbrains.kotlin.kdoc.lexer.KDocLexer +import org.jetbrains.kotlin.test.testFramework.KtUsefulTestCase +import java.io.File + +abstract class AbstractKDocLexerTest : TestCase() { + protected fun doTest(fileName: String) { + val text = File(fileName).readText() + val lexerResult = printTokens(text, 0, KDocLexer()) + KtUsefulTestCase.assertSameLinesWithFile(fileName.replaceAfterLast(".", "txt"), lexerResult) + } + + fun printTokens(text: CharSequence, start: Int, lexer: Lexer): String { + lexer.start(text, start, text.length) + var result = "" + while (true) { + val tokenType = lexer.tokenType ?: break + val tokenText = getTokenText(lexer) + val tokenTypeName = tokenType.toString() + val line = "$tokenTypeName ('$tokenText')\n" + result += line + lexer.advance() + } + return result + } + + private fun getTokenText(lexer: Lexer): String { + val tokenType = lexer.tokenType + if (tokenType is TokenWrapper) { + return tokenType.value + } + + var text = lexer.bufferSequence.subSequence(lexer.tokenStart, lexer.tokenEnd).toString() + text = StringUtil.replace(text, "\n", "\\n") + return text + } +} \ No newline at end of file diff --git a/compiler/tests/org/jetbrains/kotlin/kdoc/KDocLexerTestGenerated.java b/compiler/tests/org/jetbrains/kotlin/kdoc/KDocLexerTestGenerated.java new file mode 100644 index 00000000000..5e493727682 --- /dev/null +++ b/compiler/tests/org/jetbrains/kotlin/kdoc/KDocLexerTestGenerated.java @@ -0,0 +1,43 @@ +/* + * Copyright 2010-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.kdoc; + +import com.intellij.testFramework.TestDataPath; +import org.jetbrains.kotlin.test.JUnit3RunnerWithInners; +import org.jetbrains.kotlin.test.KotlinTestUtils; +import org.jetbrains.kotlin.test.TestMetadata; +import org.junit.runner.RunWith; + +import java.io.File; +import java.util.regex.Pattern; + +/** This class is generated by {@link org.jetbrains.kotlin.generators.tests.TestsPackage}. DO NOT MODIFY MANUALLY */ +@SuppressWarnings("all") +@TestMetadata("compiler/testData/kdoc/lexer") +@TestDataPath("$PROJECT_ROOT") +@RunWith(JUnit3RunnerWithInners.class) +public class KDocLexerTestGenerated extends AbstractKDocLexerTest { + public void testAllFilesPresentInLexer() throws Exception { + KotlinTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("compiler/testData/kdoc/lexer"), Pattern.compile("^(.+)\\.kt$"), true); + } + + @TestMetadata("codeBlocks.kt") + public void testCodeBlocks() throws Exception { + String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/kdoc/lexer/codeBlocks.kt"); + doTest(fileName); + } +} diff --git a/generators/src/org/jetbrains/kotlin/generators/tests/GenerateTests.kt b/generators/src/org/jetbrains/kotlin/generators/tests/GenerateTests.kt index 59367f9c1ac..13ac71e1d55 100644 --- a/generators/src/org/jetbrains/kotlin/generators/tests/GenerateTests.kt +++ b/generators/src/org/jetbrains/kotlin/generators/tests/GenerateTests.kt @@ -117,6 +117,7 @@ import org.jetbrains.kotlin.js.test.semantics.* import org.jetbrains.kotlin.jvm.compiler.* import org.jetbrains.kotlin.jvm.runtime.AbstractJvm8RuntimeDescriptorLoaderTest import org.jetbrains.kotlin.jvm.runtime.AbstractJvmRuntimeDescriptorLoaderTest +import org.jetbrains.kotlin.kdoc.AbstractKDocLexerTest import org.jetbrains.kotlin.lang.resolve.android.test.AbstractAndroidBoxTest import org.jetbrains.kotlin.lang.resolve.android.test.AbstractAndroidBytecodeShapeTest import org.jetbrains.kotlin.lang.resolve.android.test.AbstractAndroidSyntheticPropertyDescriptorTest @@ -336,6 +337,10 @@ fun main(args: Array) { testClass() { model("serialization/local") } + + testClass { + model("kdoc/lexer") + } } testGroup("compiler/tests-java8/tests", "compiler/testData") {