Fix the lack of the handling of whitespace characters in the KDoc lexer
Affected characters are U+000B, U+000C, U+000D, U+0085, U+2028, U+2029 This problem reproduces with JFlex since 1.5.0: http://jflex.de/changelog.html#jflex-1.5.0-jan-21-2014
This commit is contained in:
victor.petukhov
@@ -104,7 +104,7 @@ CODE_FENCE_END=("```" | "~~~")
|
||||
return KDocTokens.MARKDOWN_LINK;
|
||||
}
|
||||
|
||||
. {
|
||||
[^\n] {
|
||||
yybegin(CONTENTS);
|
||||
return KDocTokens.TEXT;
|
||||
}
|
||||
@@ -124,7 +124,7 @@ CODE_FENCE_END=("```" | "~~~")
|
||||
{CODE_LINK} { yybegin(CONTENTS);
|
||||
return KDocTokens.MARKDOWN_LINK; }
|
||||
|
||||
. {
|
||||
[^\n] {
|
||||
yybegin(CONTENTS);
|
||||
return KDocTokens.TEXT;
|
||||
}
|
||||
@@ -174,7 +174,7 @@ CODE_FENCE_END=("```" | "~~~")
|
||||
return KDocTokens.MARKDOWN_LINK;
|
||||
}
|
||||
|
||||
. {
|
||||
[^\n] {
|
||||
yybegin(CONTENTS);
|
||||
return KDocTokens.TEXT;
|
||||
}
|
||||
@@ -204,10 +204,10 @@ CODE_FENCE_END=("```" | "~~~")
|
||||
return KDocTokens.CODE_BLOCK_TEXT;
|
||||
}
|
||||
|
||||
. {
|
||||
[^\n] {
|
||||
yybegin(yystate() == INDENTED_CODE_BLOCK ? INDENTED_CODE_BLOCK : CODE_BLOCK);
|
||||
return KDocTokens.CODE_BLOCK_TEXT;
|
||||
}
|
||||
}
|
||||
|
||||
. { return TokenType.BAD_CHARACTER; }
|
||||
[\s\S] { return TokenType.BAD_CHARACTER; }
|
||||
|
||||
@@ -210,40 +210,40 @@ class _KDocLexer implements FlexLexer {
|
||||
private static final int [] ZZ_TRANS = zzUnpackTrans();
|
||||
|
||||
private static final String ZZ_TRANS_PACKED_0 =
|
||||
"\11\12\3\0\1\13\1\14\5\12\1\15\1\16\3\15"+
|
||||
"\1\17\1\15\1\20\1\21\1\22\1\0\1\22\1\15"+
|
||||
"\1\23\1\15\1\24\1\25\3\15\1\16\3\15\1\17"+
|
||||
"\1\15\1\20\1\21\1\22\1\0\1\22\1\15\1\26"+
|
||||
"\1\27\1\24\1\25\3\15\1\30\1\15\1\31\1\15"+
|
||||
"\1\32\3\15\1\30\1\0\1\30\1\15\1\26\1\15"+
|
||||
"\1\30\4\15\1\30\3\15\1\33\3\15\1\30\1\0"+
|
||||
"\1\30\1\15\1\26\1\15\1\30\4\15\1\16\3\15"+
|
||||
"\1\17\1\15\1\20\1\21\1\22\1\0\1\22\1\15"+
|
||||
"\1\26\1\15\1\24\1\25\2\15\1\34\1\35\7\34"+
|
||||
"\1\35\1\0\1\35\1\34\1\36\1\34\1\35\4\34"+
|
||||
"\1\35\5\34\1\37\1\40\1\35\1\0\1\35\1\34"+
|
||||
"\1\41\1\34\1\35\4\34\1\35\5\34\1\37\1\40"+
|
||||
"\1\35\1\0\1\35\1\34\1\36\1\34\1\35\3\34"+
|
||||
"\40\0\1\42\21\0\1\43\1\44\6\0\1\16\7\0"+
|
||||
"\1\22\1\0\1\22\3\0\1\22\3\0\3\45\1\46"+
|
||||
"\1\45\1\0\1\47\14\45\7\0\1\50\23\0\1\51"+
|
||||
"\13\0\1\22\7\0\1\22\1\0\1\22\3\0\1\22"+
|
||||
"\17\0\1\43\1\23\6\0\1\22\7\0\1\22\1\0"+
|
||||
"\1\22\3\0\1\52\10\0\2\53\17\0\1\54\20\0"+
|
||||
"\1\30\7\0\1\30\1\0\1\30\3\0\1\30\5\0"+
|
||||
"\3\31\21\0\1\55\22\0\1\56\20\0\1\35\7\0"+
|
||||
"\1\35\1\0\1\35\3\0\1\35\12\0\1\57\23\0"+
|
||||
"\1\60\26\0\1\43\1\41\22\0\1\61\5\0\5\45"+
|
||||
"\1\0\1\47\16\45\3\46\1\0\1\62\21\45\1\0"+
|
||||
"\1\47\12\45\1\63\1\45\7\0\1\64\23\0\1\64"+
|
||||
"\13\0\1\22\7\0\1\22\1\0\1\22\3\0\1\65"+
|
||||
"\5\0\2\54\21\0\3\55\1\0\1\66\16\0\3\56"+
|
||||
"\1\0\1\67\23\0\1\70\23\0\1\70\12\0\5\71"+
|
||||
"\1\0\1\72\12\71\1\63\1\71\5\63\1\73\14\63"+
|
||||
"\1\74\11\64\3\0\7\64\1\0\1\22\7\0\1\22"+
|
||||
"\1\0\1\22\3\0\1\75\4\0\1\70\7\0\1\76"+
|
||||
"\1\0\1\70\3\0\1\70\3\0\22\73\1\77\1\0"+
|
||||
"\1\76\11\0\1\76\3\0\1\76\3\0";
|
||||
"\14\12\1\13\1\14\5\12\1\15\1\16\3\15\1\17"+
|
||||
"\1\15\1\20\1\21\1\22\1\15\1\22\1\15\1\23"+
|
||||
"\1\15\1\24\1\25\3\15\1\16\3\15\1\17\1\15"+
|
||||
"\1\20\1\21\1\22\1\15\1\22\1\15\1\26\1\27"+
|
||||
"\1\24\1\25\3\15\1\30\1\15\1\31\1\15\1\32"+
|
||||
"\3\15\1\30\1\15\1\30\1\15\1\26\1\15\1\30"+
|
||||
"\4\15\1\30\3\15\1\33\3\15\1\30\1\15\1\30"+
|
||||
"\1\15\1\26\1\15\1\30\4\15\1\16\3\15\1\17"+
|
||||
"\1\15\1\20\1\21\1\22\1\15\1\22\1\15\1\26"+
|
||||
"\1\15\1\24\1\25\2\15\1\34\1\35\7\34\1\35"+
|
||||
"\1\34\1\35\1\34\1\36\1\34\1\35\4\34\1\35"+
|
||||
"\5\34\1\37\1\40\1\35\1\34\1\35\1\34\1\41"+
|
||||
"\1\34\1\35\4\34\1\35\5\34\1\37\1\40\1\35"+
|
||||
"\1\34\1\35\1\34\1\36\1\34\1\35\3\34\40\0"+
|
||||
"\1\42\21\0\1\43\1\44\6\0\1\16\7\0\1\22"+
|
||||
"\1\0\1\22\3\0\1\22\3\0\3\45\1\46\1\45"+
|
||||
"\1\0\1\47\14\45\7\0\1\50\23\0\1\51\13\0"+
|
||||
"\1\22\7\0\1\22\1\0\1\22\3\0\1\22\17\0"+
|
||||
"\1\43\1\23\6\0\1\22\7\0\1\22\1\0\1\22"+
|
||||
"\3\0\1\52\10\0\2\53\17\0\1\54\20\0\1\30"+
|
||||
"\7\0\1\30\1\0\1\30\3\0\1\30\5\0\3\31"+
|
||||
"\21\0\1\55\22\0\1\56\20\0\1\35\7\0\1\35"+
|
||||
"\1\0\1\35\3\0\1\35\12\0\1\57\23\0\1\60"+
|
||||
"\26\0\1\43\1\41\22\0\1\61\5\0\5\45\1\0"+
|
||||
"\1\47\16\45\3\46\1\0\1\62\21\45\1\0\1\47"+
|
||||
"\12\45\1\63\1\45\7\0\1\64\23\0\1\64\13\0"+
|
||||
"\1\22\7\0\1\22\1\0\1\22\3\0\1\65\5\0"+
|
||||
"\2\54\21\0\3\55\1\0\1\66\16\0\3\56\1\0"+
|
||||
"\1\67\23\0\1\70\23\0\1\70\12\0\5\71\1\0"+
|
||||
"\1\72\12\71\1\63\1\71\5\63\1\73\14\63\1\74"+
|
||||
"\11\64\3\0\7\64\1\0\1\22\7\0\1\22\1\0"+
|
||||
"\1\22\3\0\1\75\4\0\1\70\7\0\1\76\1\0"+
|
||||
"\1\70\3\0\1\70\3\0\22\73\1\77\1\0\1\76"+
|
||||
"\11\0\1\76\3\0\1\76\3\0";
|
||||
|
||||
private static int [] zzUnpackTrans() {
|
||||
int [] result = new int[912];
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
/**
|
||||
* @foo This is text.
|
||||
* This is indented code block
|
||||
* Second line of code block
|
||||
* @foo This is text .
|
||||
* Tabulated code block
|
||||
* Tabulated second line
|
||||
* @foo This is text.
|
||||
* ``` kotlin
|
||||
* @foo This is code block.
|
||||
* ```
|
||||
* @foo This is text .
|
||||
* ~~~
|
||||
* @foo This is code block
|
||||
* With multiple lines.
|
||||
* ~~~
|
||||
* @foo This is yet more text.
|
||||
* ```
|
||||
* @foo This is an unclosed code block .
|
||||
*/
|
||||
@@ -0,0 +1,74 @@
|
||||
KDOC_START ('/**')
|
||||
WHITE_SPACE ('\n ')
|
||||
KDOC_LEADING_ASTERISK ('*')
|
||||
KDOC_TEXT (' ')
|
||||
KDOC_TAG_NAME ('@foo')
|
||||
WHITE_SPACE (' ')
|
||||
KDOC_TEXT ('This is text.')
|
||||
WHITE_SPACE ('\n ')
|
||||
KDOC_LEADING_ASTERISK ('*')
|
||||
KDOC_CODE_BLOCK_TEXT (' This is indented code block')
|
||||
WHITE_SPACE ('\n ')
|
||||
KDOC_LEADING_ASTERISK ('*')
|
||||
KDOC_CODE_BLOCK_TEXT (' Second line of code block')
|
||||
WHITE_SPACE ('\n ')
|
||||
KDOC_LEADING_ASTERISK ('*')
|
||||
KDOC_TEXT (' ')
|
||||
KDOC_TAG_NAME ('@foo')
|
||||
WHITE_SPACE (' ')
|
||||
KDOC_TEXT ('This is text .')
|
||||
WHITE_SPACE ('\n ')
|
||||
KDOC_LEADING_ASTERISK ('*')
|
||||
KDOC_CODE_BLOCK_TEXT (' Tabulated code block')
|
||||
WHITE_SPACE ('\n ')
|
||||
KDOC_LEADING_ASTERISK ('*')
|
||||
KDOC_CODE_BLOCK_TEXT (' Tabulated second line')
|
||||
WHITE_SPACE ('\n ')
|
||||
KDOC_LEADING_ASTERISK ('*')
|
||||
KDOC_TEXT (' ')
|
||||
KDOC_TAG_NAME ('@foo')
|
||||
WHITE_SPACE (' ')
|
||||
KDOC_TEXT ('This is text.')
|
||||
WHITE_SPACE ('\n ')
|
||||
KDOC_LEADING_ASTERISK ('*')
|
||||
KDOC_TEXT (' ``` kotlin')
|
||||
WHITE_SPACE ('\n ')
|
||||
KDOC_LEADING_ASTERISK ('*')
|
||||
KDOC_CODE_BLOCK_TEXT (' @foo This is code block.')
|
||||
WHITE_SPACE ('\n ')
|
||||
KDOC_LEADING_ASTERISK ('*')
|
||||
KDOC_CODE_BLOCK_TEXT (' ')
|
||||
KDOC_TEXT ('```')
|
||||
WHITE_SPACE ('\n ')
|
||||
KDOC_LEADING_ASTERISK ('*')
|
||||
KDOC_TEXT (' ')
|
||||
KDOC_TAG_NAME ('@foo')
|
||||
WHITE_SPACE (' ')
|
||||
KDOC_TEXT ('This is text .')
|
||||
WHITE_SPACE ('\n ')
|
||||
KDOC_LEADING_ASTERISK ('*')
|
||||
KDOC_TEXT (' ~~~')
|
||||
WHITE_SPACE ('\n ')
|
||||
KDOC_LEADING_ASTERISK ('*')
|
||||
KDOC_CODE_BLOCK_TEXT (' @foo This is code block')
|
||||
WHITE_SPACE ('\n ')
|
||||
KDOC_LEADING_ASTERISK ('*')
|
||||
KDOC_CODE_BLOCK_TEXT (' With multiple lines.')
|
||||
WHITE_SPACE ('\n ')
|
||||
KDOC_LEADING_ASTERISK ('*')
|
||||
KDOC_CODE_BLOCK_TEXT (' ')
|
||||
KDOC_TEXT ('~~~')
|
||||
WHITE_SPACE ('\n ')
|
||||
KDOC_LEADING_ASTERISK ('*')
|
||||
KDOC_TEXT (' ')
|
||||
KDOC_TAG_NAME ('@foo')
|
||||
WHITE_SPACE (' ')
|
||||
KDOC_TEXT ('This is yet more text.')
|
||||
WHITE_SPACE ('\n ')
|
||||
KDOC_LEADING_ASTERISK ('*')
|
||||
KDOC_TEXT (' ```')
|
||||
WHITE_SPACE ('\n ')
|
||||
KDOC_LEADING_ASTERISK ('*')
|
||||
KDOC_CODE_BLOCK_TEXT (' @foo This is an unclosed code block .')
|
||||
WHITE_SPACE ('\n ')
|
||||
KDOC_END ('*/')
|
||||
Reference in New Issue
Block a user