azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix the lack of the handling of whitespace characters in the KDoc lexer

Browse Source 
Affected characters are U+000B, U+000C, U+000D, U+0085, U+2028, U+2029
This problem reproduces with JFlex since 1.5.0: http://jflex.de/changelog.html#jflex-1.5.0-jan-21-2014
This commit is contained in:
victor.petukhov
2018-10-25 18:55:08 +03:00
parent 9e6ac6cb2a
commit fe072e3366
4 changed files with 133 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/psi/src/org/jetbrains/kotlin/kdoc/lexer/KDoc.flex
+5 -5
View File
@@ -104,7 +104,7 @@ CODE_FENCE_END=("```" | "~~~")
return KDocTokens.MARKDOWN_LINK; return KDocTokens.MARKDOWN_LINK;
} }
. { [^\n] {
yybegin(CONTENTS); yybegin(CONTENTS);
return KDocTokens.TEXT; return KDocTokens.TEXT;
} }
@@ -124,7 +124,7 @@ CODE_FENCE_END=("```" | "~~~")
{CODE_LINK} { yybegin(CONTENTS); {CODE_LINK} { yybegin(CONTENTS);
return KDocTokens.MARKDOWN_LINK; } return KDocTokens.MARKDOWN_LINK; }
. { [^\n] {
yybegin(CONTENTS); yybegin(CONTENTS);
return KDocTokens.TEXT; return KDocTokens.TEXT;
} }
@@ -174,7 +174,7 @@ CODE_FENCE_END=("```" | "~~~")
return KDocTokens.MARKDOWN_LINK; return KDocTokens.MARKDOWN_LINK;
} }
. { [^\n] {
yybegin(CONTENTS); yybegin(CONTENTS);
return KDocTokens.TEXT; return KDocTokens.TEXT;
} }
@@ -204,10 +204,10 @@ CODE_FENCE_END=("```" | "~~~")
return KDocTokens.CODE_BLOCK_TEXT; return KDocTokens.CODE_BLOCK_TEXT;
} }
. { [^\n] {
yybegin(yystate() == INDENTED_CODE_BLOCK ? INDENTED_CODE_BLOCK : CODE_BLOCK); yybegin(yystate() == INDENTED_CODE_BLOCK ? INDENTED_CODE_BLOCK : CODE_BLOCK);
return KDocTokens.CODE_BLOCK_TEXT; return KDocTokens.CODE_BLOCK_TEXT;
} }
} }
. { return TokenType.BAD_CHARACTER; } [\s\S] { return TokenType.BAD_CHARACTER; }
compiler/psi/src/org/jetbrains/kotlin/kdoc/lexer/_KDocLexer.java
+34 -34
View File
@@ -210,40 +210,40 @@ class _KDocLexer implements FlexLexer {
private static final int [] ZZ_TRANS = zzUnpackTrans(); private static final int [] ZZ_TRANS = zzUnpackTrans();
private static final String ZZ_TRANS_PACKED_0 = private static final String ZZ_TRANS_PACKED_0 =
"\11\12\3\0\1\13\1\14\5\12\1\15\1\16\3\15"+ "\14\12\1\13\1\14\5\12\1\15\1\16\3\15\1\17"+
"\1\17\1\15\1\20\1\21\1\22\1\0\1\22\1\15"+ "\1\15\1\20\1\21\1\22\1\15\1\22\1\15\1\23"+
"\1\23\1\15\1\24\1\25\3\15\1\16\3\15\1\17"+ "\1\15\1\24\1\25\3\15\1\16\3\15\1\17\1\15"+
"\1\15\1\20\1\21\1\22\1\0\1\22\1\15\1\26"+ "\1\20\1\21\1\22\1\15\1\22\1\15\1\26\1\27"+
"\1\27\1\24\1\25\3\15\1\30\1\15\1\31\1\15"+ "\1\24\1\25\3\15\1\30\1\15\1\31\1\15\1\32"+
"\1\32\3\15\1\30\1\0\1\30\1\15\1\26\1\15"+ "\3\15\1\30\1\15\1\30\1\15\1\26\1\15\1\30"+
"\1\30\4\15\1\30\3\15\1\33\3\15\1\30\1\0"+ "\4\15\1\30\3\15\1\33\3\15\1\30\1\15\1\30"+
"\1\30\1\15\1\26\1\15\1\30\4\15\1\16\3\15"+ "\1\15\1\26\1\15\1\30\4\15\1\16\3\15\1\17"+
"\1\17\1\15\1\20\1\21\1\22\1\0\1\22\1\15"+ "\1\15\1\20\1\21\1\22\1\15\1\22\1\15\1\26"+
"\1\26\1\15\1\24\1\25\2\15\1\34\1\35\7\34"+ "\1\15\1\24\1\25\2\15\1\34\1\35\7\34\1\35"+
"\1\35\1\0\1\35\1\34\1\36\1\34\1\35\4\34"+ "\1\34\1\35\1\34\1\36\1\34\1\35\4\34\1\35"+
"\1\35\5\34\1\37\1\40\1\35\1\0\1\35\1\34"+ "\5\34\1\37\1\40\1\35\1\34\1\35\1\34\1\41"+
"\1\41\1\34\1\35\4\34\1\35\5\34\1\37\1\40"+ "\1\34\1\35\4\34\1\35\5\34\1\37\1\40\1\35"+
"\1\35\1\0\1\35\1\34\1\36\1\34\1\35\3\34"+ "\1\34\1\35\1\34\1\36\1\34\1\35\3\34\40\0"+
"\40\0\1\42\21\0\1\43\1\44\6\0\1\16\7\0"+ "\1\42\21\0\1\43\1\44\6\0\1\16\7\0\1\22"+
"\1\22\1\0\1\22\3\0\1\22\3\0\3\45\1\46"+ "\1\0\1\22\3\0\1\22\3\0\3\45\1\46\1\45"+
"\1\45\1\0\1\47\14\45\7\0\1\50\23\0\1\51"+ "\1\0\1\47\14\45\7\0\1\50\23\0\1\51\13\0"+
"\13\0\1\22\7\0\1\22\1\0\1\22\3\0\1\22"+ "\1\22\7\0\1\22\1\0\1\22\3\0\1\22\17\0"+
"\17\0\1\43\1\23\6\0\1\22\7\0\1\22\1\0"+ "\1\43\1\23\6\0\1\22\7\0\1\22\1\0\1\22"+
"\1\22\3\0\1\52\10\0\2\53\17\0\1\54\20\0"+ "\3\0\1\52\10\0\2\53\17\0\1\54\20\0\1\30"+
"\1\30\7\0\1\30\1\0\1\30\3\0\1\30\5\0"+ "\7\0\1\30\1\0\1\30\3\0\1\30\5\0\3\31"+
"\3\31\21\0\1\55\22\0\1\56\20\0\1\35\7\0"+ "\21\0\1\55\22\0\1\56\20\0\1\35\7\0\1\35"+
"\1\35\1\0\1\35\3\0\1\35\12\0\1\57\23\0"+ "\1\0\1\35\3\0\1\35\12\0\1\57\23\0\1\60"+
"\1\60\26\0\1\43\1\41\22\0\1\61\5\0\5\45"+ "\26\0\1\43\1\41\22\0\1\61\5\0\5\45\1\0"+
"\1\0\1\47\16\45\3\46\1\0\1\62\21\45\1\0"+ "\1\47\16\45\3\46\1\0\1\62\21\45\1\0\1\47"+
"\1\47\12\45\1\63\1\45\7\0\1\64\23\0\1\64"+ "\12\45\1\63\1\45\7\0\1\64\23\0\1\64\13\0"+
"\13\0\1\22\7\0\1\22\1\0\1\22\3\0\1\65"+ "\1\22\7\0\1\22\1\0\1\22\3\0\1\65\5\0"+
"\5\0\2\54\21\0\3\55\1\0\1\66\16\0\3\56"+ "\2\54\21\0\3\55\1\0\1\66\16\0\3\56\1\0"+
"\1\0\1\67\23\0\1\70\23\0\1\70\12\0\5\71"+ "\1\67\23\0\1\70\23\0\1\70\12\0\5\71\1\0"+
"\1\0\1\72\12\71\1\63\1\71\5\63\1\73\14\63"+ "\1\72\12\71\1\63\1\71\5\63\1\73\14\63\1\74"+
"\1\74\11\64\3\0\7\64\1\0\1\22\7\0\1\22"+ "\11\64\3\0\7\64\1\0\1\22\7\0\1\22\1\0"+
"\1\0\1\22\3\0\1\75\4\0\1\70\7\0\1\76"+ "\1\22\3\0\1\75\4\0\1\70\7\0\1\76\1\0"+
"\1\0\1\70\3\0\1\70\3\0\22\73\1\77\1\0"+ "\1\70\3\0\1\70\3\0\22\73\1\77\1\0\1\76"+
"\1\76\11\0\1\76\3\0\1\76\3\0"; "\11\0\1\76\3\0\1\76\3\0";
private static int [] zzUnpackTrans() { private static int [] zzUnpackTrans() {
int [] result = new int[912]; int [] result = new int[912];
compiler/testData/kdoc/lexer/codeBlocksWithVerticalTabs.kt
Vendored
+20
View File
@@ -0,0 +1,20 @@
/**
* @foo This is text.
* This is indented code block
* Second line of code block
* @foo This is text .
* Tabulated code block
* Tabulated second line
* @foo This is text.
* ``` kotlin
* @foo This is code block.
* ```
* @foo This is text .
* ~~~
* @foo This is code block
* With multiple lines.
* ~~~
* @foo This is yet more text.
* ```
* @foo This is an unclosed code block .
*/
compiler/testData/kdoc/lexer/codeBlocksWithVerticalTabs.txt
Vendored
+74
View File
@@ -0,0 +1,74 @@
KDOC_START ('/**')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_TEXT (' ')
KDOC_TAG_NAME ('@foo')
WHITE_SPACE (' ')
KDOC_TEXT ('This is text.')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_CODE_BLOCK_TEXT (' This is indented code block')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_CODE_BLOCK_TEXT (' Second line of code block')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_TEXT (' ')
KDOC_TAG_NAME ('@foo')
WHITE_SPACE (' ')
KDOC_TEXT ('This is text .')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_CODE_BLOCK_TEXT (' Tabulated code block')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_CODE_BLOCK_TEXT (' Tabulated second line')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_TEXT (' ')
KDOC_TAG_NAME ('@foo')
WHITE_SPACE (' ')
KDOC_TEXT ('This is text.')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_TEXT (' ``` kotlin')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_CODE_BLOCK_TEXT (' @foo This is code block.')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_CODE_BLOCK_TEXT (' ')
KDOC_TEXT ('```')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_TEXT (' ')
KDOC_TAG_NAME ('@foo')
WHITE_SPACE (' ')
KDOC_TEXT ('This is text .')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_TEXT (' ~~~')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_CODE_BLOCK_TEXT (' @foo This is code block')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_CODE_BLOCK_TEXT (' With multiple lines.')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_CODE_BLOCK_TEXT (' ')
KDOC_TEXT ('~~~')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_TEXT (' ')
KDOC_TAG_NAME ('@foo')
WHITE_SPACE (' ')
KDOC_TEXT ('This is yet more text.')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_TEXT (' ```')
WHITE_SPACE ('\n ')
KDOC_LEADING_ASTERISK ('*')
KDOC_CODE_BLOCK_TEXT (' @foo This is an unclosed code block .')
WHITE_SPACE ('\n ')
KDOC_END ('*/')