azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Parse ;; as a single token

Browse Source
This commit is contained in:
Yan Zhulanow
2015-10-02 12:53:34 +03:00
parent 8d5cbeeab0
commit e85d76f9cc
7 changed files with 428 additions and 440 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/_KDocLexer.java
+60 -78
View File
@@ -1,4 +1,4 @@
/* The following code was generated by JFlex 1.4.3 on 4/24/15 7:12 PM */
/* The following code was generated by JFlex 1.4.3 on 10/1/15 9:28 PM */
package org.jetbrains.kotlin.kdoc.lexer;
@@ -14,8 +14,8 @@ import org.jetbrains.kotlin.kdoc.parser.KDocKnownTag;
/**
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.4.3
* on 4/24/15 7:12 PM from the specification file
* <tt>/home/sufix/work/kotlin/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/KDoc.flex</tt>
* on 10/1/15 9:28 PM from the specification file
* <tt>/Users/yan/jb/kotlin/compiler/frontend/src/org/jetbrains/kotlin/kdoc/lexer/KDoc.flex</tt>
*/
class _KDocLexer implements FlexLexer {
/** initial size of the lookahead buffer */
@@ -47,81 +47,63 @@ class _KDocLexer implements FlexLexer {
"\1\14\1\15\1\10\3\0\1\4\1\7\12\2\6\0\1\11\32\3"+
"\1\5\1\13\1\6\1\0\1\3\1\0\32\3\47\0\4\3\4\0"+
"\1\3\12\0\1\3\4\0\1\3\5\0\27\3\1\0\37\3\1\0"+
"\u01ca\3\4\0\14\3\16\0\5\3\7\0\1\3\1\0\1\3\201\0"+
"\5\3\1\0\2\3\2\0\4\3\10\0\1\3\1\0\3\3\1\0"+
"\1\3\1\0\24\3\1\0\123\3\1\0\213\3\10\0\236\3\11\0"+
"\46\3\2\0\1\3\7\0\47\3\110\0\33\3\5\0\3\3\30\0"+
"\1\3\24\0\53\3\43\0\2\3\1\0\143\3\1\0\1\3\17\0"+
"\2\3\7\0\2\3\12\0\3\3\2\0\1\3\20\0\1\3\1\0"+
"\36\3\35\0\131\3\13\0\1\3\30\0\41\3\11\0\2\3\4\0"+
"\1\3\5\0\26\3\4\0\1\3\11\0\1\3\3\0\1\3\27\0"+
"\31\3\253\0\66\3\3\0\1\3\22\0\1\3\7\0\12\3\17\0"+
"\7\3\1\0\7\3\5\0\10\3\2\0\2\3\2\0\26\3\1\0"+
"\7\3\1\0\1\3\3\0\4\3\3\0\1\3\20\0\1\3\15\0"+
"\2\3\1\0\3\3\16\0\4\3\7\0\1\3\11\0\6\3\4\0"+
"\2\3\2\0\26\3\1\0\7\3\1\0\2\3\1\0\2\3\1\0"+
"\2\3\37\0\4\3\1\0\1\3\23\0\3\3\20\0\11\3\1\0"+
"\3\3\1\0\26\3\1\0\7\3\1\0\2\3\1\0\5\3\3\0"+
"\1\3\22\0\1\3\17\0\2\3\17\0\1\3\23\0\10\3\2\0"+
"\2\3\2\0\26\3\1\0\7\3\1\0\2\3\1\0\5\3\3\0"+
"\1\3\36\0\2\3\1\0\3\3\17\0\1\3\21\0\1\3\1\0"+
"\6\3\3\0\3\3\1\0\4\3\3\0\2\3\1\0\1\3\1\0"+
"\2\3\3\0\2\3\3\0\3\3\3\0\14\3\26\0\1\3\50\0"+
"\1\3\13\0\10\3\1\0\3\3\1\0\27\3\1\0\12\3\1\0"+
"\5\3\3\0\1\3\32\0\2\3\6\0\2\3\43\0\10\3\1\0"+
"\u013f\3\31\0\162\3\4\0\14\3\16\0\5\3\11\0\1\3\213\0"+
"\1\3\13\0\1\3\1\0\3\3\1\0\1\3\1\0\24\3\1\0"+
"\54\3\1\0\46\3\1\0\5\3\4\0\202\3\10\0\105\3\1\0"+
"\46\3\2\0\2\3\6\0\20\3\41\0\46\3\2\0\1\3\7\0"+
"\47\3\110\0\33\3\5\0\3\3\56\0\32\3\5\0\13\3\43\0"+
"\2\3\1\0\143\3\1\0\1\3\17\0\2\3\7\0\2\3\12\0"+
"\3\3\2\0\1\3\20\0\1\3\1\0\36\3\35\0\3\3\60\0"+
"\46\3\13\0\1\3\u0152\0\66\3\3\0\1\3\22\0\1\3\7\0"+
"\12\3\43\0\10\3\2\0\2\3\2\0\26\3\1\0\7\3\1\0"+
"\1\3\3\0\4\3\3\0\1\3\36\0\2\3\1\0\3\3\16\0"+
"\4\3\21\0\6\3\4\0\2\3\2\0\26\3\1\0\7\3\1\0"+
"\2\3\1\0\2\3\1\0\2\3\37\0\4\3\1\0\1\3\23\0"+
"\3\3\20\0\11\3\1\0\3\3\1\0\26\3\1\0\7\3\1\0"+
"\2\3\1\0\5\3\3\0\1\3\22\0\1\3\17\0\2\3\17\0"+
"\1\3\23\0\10\3\2\0\2\3\2\0\26\3\1\0\7\3\1\0"+
"\2\3\1\0\5\3\3\0\1\3\36\0\2\3\1\0\3\3\17\0"+
"\1\3\21\0\1\3\1\0\6\3\3\0\3\3\1\0\4\3\3\0"+
"\2\3\1\0\1\3\1\0\2\3\3\0\2\3\3\0\3\3\3\0"+
"\10\3\1\0\3\3\77\0\1\3\13\0\10\3\1\0\3\3\1\0"+
"\27\3\1\0\12\3\1\0\5\3\46\0\2\3\43\0\10\3\1\0"+
"\3\3\1\0\27\3\1\0\12\3\1\0\5\3\3\0\1\3\40\0"+
"\1\3\1\0\2\3\17\0\2\3\22\0\10\3\1\0\3\3\1\0"+
"\51\3\2\0\1\3\20\0\1\3\21\0\2\3\30\0\6\3\5\0"+
"\22\3\3\0\30\3\1\0\11\3\1\0\1\3\2\0\7\3\72\0"+
"\60\3\1\0\2\3\13\0\10\3\72\0\2\3\1\0\1\3\2\0"+
"\2\3\1\0\1\3\2\0\1\3\6\0\4\3\1\0\7\3\1\0"+
"\3\3\1\0\1\3\1\0\1\3\2\0\2\3\1\0\4\3\1\0"+
"\2\3\11\0\1\3\2\0\5\3\1\0\1\3\25\0\2\3\42\0"+
"\1\3\77\0\10\3\1\0\44\3\33\0\5\3\163\0\53\3\24\0"+
"\1\3\20\0\6\3\4\0\4\3\3\0\1\3\3\0\2\3\7\0"+
"\3\3\4\0\15\3\14\0\1\3\21\0\46\3\12\0\53\3\1\0"+
"\1\3\3\0\u0149\3\1\0\4\3\2\0\7\3\1\0\1\3\1\0"+
"\4\3\2\0\51\3\1\0\4\3\2\0\41\3\1\0\4\3\2\0"+
"\7\3\1\0\1\3\1\0\4\3\2\0\17\3\1\0\71\3\1\0"+
"\4\3\2\0\103\3\45\0\20\3\20\0\125\3\14\0\u026c\3\2\0"+
"\21\3\1\0\32\3\5\0\113\3\3\0\3\3\17\0\15\3\1\0"+
"\4\3\16\0\22\3\16\0\22\3\16\0\15\3\1\0\3\3\17\0"+
"\64\3\43\0\1\3\3\0\2\3\103\0\130\3\10\0\51\3\1\0"+
"\1\3\5\0\106\3\12\0\35\3\63\0\36\3\2\0\5\3\13\0"+
"\54\3\25\0\7\3\70\0\27\3\11\0\65\3\122\0\1\3\135\0"+
"\57\3\21\0\7\3\67\0\36\3\15\0\2\3\20\0\46\3\32\0"+
"\44\3\51\0\3\3\12\0\44\3\153\0\4\3\1\0\4\3\16\0"+
"\300\3\100\0\u0116\3\2\0\6\3\2\0\46\3\2\0\6\3\2\0"+
"\10\3\1\0\1\3\1\0\1\3\1\0\1\3\1\0\37\3\2\0"+
"\65\3\1\0\7\3\1\0\1\3\3\0\3\3\1\0\7\3\3\0"+
"\4\3\2\0\6\3\4\0\15\3\5\0\3\3\1\0\7\3\102\0"+
"\2\3\23\0\1\3\34\0\1\3\15\0\1\3\20\0\15\3\3\0"+
"\32\3\110\0\1\3\4\0\1\3\2\0\12\3\1\0\1\3\3\0"+
"\5\3\6\0\1\3\1\0\1\3\1\0\1\3\1\0\4\3\1\0"+
"\13\3\2\0\4\3\5\0\5\3\4\0\1\3\21\0\51\3\u0a77\0"+
"\57\3\1\0\57\3\1\0\205\3\6\0\4\3\21\0\46\3\12\0"+
"\66\3\11\0\1\3\20\0\27\3\11\0\7\3\1\0\7\3\1\0"+
"\7\3\1\0\7\3\1\0\7\3\1\0\7\3\1\0\7\3\1\0"+
"\7\3\120\0\1\3\u01d5\0\3\3\31\0\11\3\7\0\5\3\2\0"+
"\5\3\4\0\126\3\6\0\3\3\1\0\132\3\1\0\4\3\5\0"+
"\51\3\3\0\136\3\21\0\33\3\65\0\20\3\u0200\0\u19b6\3\112\0"+
"\u51cc\3\64\0\u048d\3\103\0\56\3\2\0\u010d\3\3\0\20\3\12\0"+
"\2\3\24\0\57\3\20\0\31\3\10\0\120\3\47\0\11\3\2\0"+
"\147\3\2\0\4\3\1\0\2\3\16\0\12\3\120\0\10\3\1\0"+
"\3\3\1\0\4\3\1\0\27\3\25\0\1\3\7\0\64\3\16\0"+
"\62\3\76\0\6\3\3\0\1\3\16\0\34\3\12\0\27\3\31\0"+
"\35\3\7\0\57\3\34\0\1\3\60\0\51\3\27\0\3\3\1\0"+
"\10\3\24\0\27\3\3\0\1\3\5\0\60\3\1\0\1\3\3\0"+
"\2\3\2\0\5\3\2\0\1\3\1\0\1\3\30\0\3\3\43\0"+
"\6\3\2\0\6\3\2\0\6\3\11\0\7\3\1\0\7\3\221\0"+
"\43\3\35\0\u2ba4\3\14\0\27\3\4\0\61\3\u2104\0\u012e\3\2\0"+
"\76\3\2\0\152\3\46\0\7\3\14\0\5\3\5\0\1\3\1\0"+
"\12\3\1\0\15\3\1\0\5\3\1\0\1\3\1\0\2\3\1\0"+
"\2\3\1\0\154\3\41\0\u016b\3\22\0\100\3\2\0\66\3\50\0"+
"\15\3\66\0\2\3\30\0\3\3\31\0\1\3\6\0\5\3\1\0"+
"\207\3\7\0\1\3\34\0\32\3\4\0\1\3\1\0\32\3\13\0"+
"\131\3\3\0\6\3\2\0\6\3\2\0\6\3\2\0\3\3\3\0"+
"\2\3\3\0\2\3\31\0";
"\1\3\1\0\2\3\43\0\10\3\1\0\3\3\1\0\27\3\1\0"+
"\20\3\46\0\2\3\43\0\22\3\3\0\30\3\1\0\11\3\1\0"+
"\1\3\2\0\7\3\72\0\60\3\1\0\2\3\13\0\10\3\72\0"+
"\2\3\1\0\1\3\2\0\2\3\1\0\1\3\2\0\1\3\6\0"+
"\4\3\1\0\7\3\1\0\3\3\1\0\1\3\1\0\1\3\2\0"+
"\2\3\1\0\4\3\1\0\2\3\11\0\1\3\2\0\5\3\1\0"+
"\1\3\25\0\2\3\42\0\1\3\77\0\10\3\1\0\42\3\35\0"+
"\4\3\164\0\42\3\1\0\5\3\1\0\2\3\45\0\6\3\112\0"+
"\46\3\12\0\51\3\7\0\132\3\5\0\104\3\5\0\122\3\6\0"+
"\7\3\1\0\77\3\1\0\1\3\1\0\4\3\2\0\7\3\1\0"+
"\1\3\1\0\4\3\2\0\47\3\1\0\1\3\1\0\4\3\2\0"+
"\37\3\1\0\1\3\1\0\4\3\2\0\7\3\1\0\1\3\1\0"+
"\4\3\2\0\7\3\1\0\7\3\1\0\27\3\1\0\37\3\1\0"+
"\1\3\1\0\4\3\2\0\7\3\1\0\47\3\1\0\23\3\105\0"+
"\125\3\14\0\u026c\3\2\0\10\3\12\0\32\3\5\0\113\3\3\0"+
"\3\3\17\0\15\3\1\0\4\3\16\0\22\3\16\0\22\3\16\0"+
"\15\3\1\0\3\3\17\0\64\3\43\0\1\3\3\0\2\3\103\0"+
"\130\3\10\0\51\3\127\0\35\3\63\0\36\3\2\0\5\3\u038b\0"+
"\154\3\224\0\234\3\4\0\132\3\6\0\26\3\2\0\6\3\2\0"+
"\46\3\2\0\6\3\2\0\10\3\1\0\1\3\1\0\1\3\1\0"+
"\1\3\1\0\37\3\2\0\65\3\1\0\7\3\1\0\1\3\3\0"+
"\3\3\1\0\7\3\3\0\4\3\2\0\6\3\4\0\15\3\5\0"+
"\3\3\1\0\7\3\102\0\2\3\23\0\1\3\34\0\1\3\15\0"+
"\1\3\40\0\22\3\120\0\1\3\4\0\1\3\2\0\12\3\1\0"+
"\1\3\3\0\5\3\6\0\1\3\1\0\1\3\1\0\1\3\1\0"+
"\4\3\1\0\3\3\1\0\7\3\3\0\3\3\5\0\5\3\26\0"+
"\44\3\u0e81\0\3\3\31\0\11\3\7\0\5\3\2\0\5\3\4\0"+
"\126\3\6\0\3\3\1\0\137\3\5\0\50\3\4\0\136\3\21\0"+
"\30\3\70\0\20\3\u0200\0\u19b6\3\112\0\u51a6\3\132\0\u048d\3\u0773\0"+
"\u2ba4\3\u215c\0\u012e\3\2\0\73\3\225\0\7\3\14\0\5\3\5\0"+
"\1\3\1\0\12\3\1\0\15\3\1\0\5\3\1\0\1\3\1\0"+
"\2\3\1\0\2\3\1\0\154\3\41\0\u016b\3\22\0\100\3\2\0"+
"\66\3\50\0\15\3\66\0\2\3\30\0\3\3\31\0\1\3\6\0"+
"\5\3\1\0\207\3\7\0\1\3\34\0\32\3\4\0\1\3\1\0"+
"\32\3\12\0\132\3\3\0\6\3\2\0\6\3\2\0\6\3\2\0"+
"\3\3\3\0\2\3\3\0\2\3\31\0";
/**
* Translates characters to character classes
@@ -359,7 +341,7 @@ class _KDocLexer implements FlexLexer {
char [] map = new char[0x10000];
int i = 0; /* index in packed string */
int j = 0; /* index in unpacked array */
while (i < 1568) {
while (i < 1212) {
int count = packed.charAt(i++);
char value = packed.charAt(i++);
do map[j++] = value; while (--count > 0);
compiler/frontend/src/org/jetbrains/kotlin/lexer/JetTokens.java
+1
View File
@@ -116,6 +116,7 @@ public interface JetTokens {
JetSingleValueToken COLONCOLON = new JetSingleValueToken("COLONCOLON", "::");
JetSingleValueToken COLON = new JetSingleValueToken("COLON", ":");
JetSingleValueToken SEMICOLON = new JetSingleValueToken("SEMICOLON", ";");
JetSingleValueToken DOUBLE_SEMICOLON = new JetSingleValueToken("DOUBLE_SEMICOLON", ";;");
JetSingleValueToken RANGE = new JetSingleValueToken("RANGE", "..");
JetSingleValueToken EQ = new JetSingleValueToken("EQ", "=");
JetSingleValueToken MULTEQ = new JetSingleValueToken("MULTEQ", "*=");
compiler/frontend/src/org/jetbrains/kotlin/lexer/Kotlin.flex
+1
View File
@@ -319,6 +319,7 @@ LONG_TEMPLATE_ENTRY_START=\$\{
">" { return JetTokens.GT ; }
"?" { return JetTokens.QUEST ; }
":" { return JetTokens.COLON ; }
";;" { return JetTokens.DOUBLE_SEMICOLON;}
";" { return JetTokens.SEMICOLON ; }
"=" { return JetTokens.EQ ; }
"," { return JetTokens.COMMA ; }
compiler/frontend/src/org/jetbrains/kotlin/lexer/_JetLexer.java
+345 -340
View File
File diff suppressed because it is too large Load Diff
compiler/testData/psi/ControlStructures.txt
Vendored
+6 -7
View File
@@ -970,18 +970,17 @@ JetFile: ControlStructures.kt
BODY
BLOCK
PsiElement(LBRACE)('{')
PsiElement(SEMICOLON)(';')
PsiElement(SEMICOLON)(';')
PsiErrorElement:Expecting an element
PsiElement(DOUBLE_SEMICOLON)(';;')
PsiElement(SEMICOLON)(';')
REFERENCE_EXPRESSION
PsiElement(IDENTIFIER)('foo')
PsiElement(SEMICOLON)(';')
REFERENCE_EXPRESSION
PsiElement(IDENTIFIER)('bar')
PsiElement(SEMICOLON)(';')
PsiElement(SEMICOLON)(';')
PsiElement(SEMICOLON)(';')
PsiElement(SEMICOLON)(';')
PsiErrorElement:Unexpected tokens (use ';' to separate expressions on the same line)
PsiElement(DOUBLE_SEMICOLON)(';;')
PsiElement(DOUBLE_SEMICOLON)(';;')
PsiWhiteSpace(' ')
PsiElement(RBRACE)('}')
PsiWhiteSpace(' ')
@@ -993,4 +992,4 @@ JetFile: ControlStructures.kt
PsiElement(IDENTIFIER)('r')
PsiElement(RPAR)(')')
PsiWhiteSpace('\n')
PsiElement(RBRACE)('}')
PsiElement(RBRACE)('}')
compiler/testData/psi/Expressions_ERR.txt
Vendored
+11 -10
View File
@@ -34,19 +34,20 @@ JetFile: Expressions_ERR.kt
PsiWhiteSpace('\n ')
PsiErrorElement:Expecting an element
PsiElement(RPAR)(')')
PsiElement(SEMICOLON)(';')
PsiElement(SEMICOLON)(';')
PsiElement(SEMICOLON)(';')
PsiErrorElement:Unexpected tokens (use ';' to separate expressions on the same line)
PsiElement(DOUBLE_SEMICOLON)(';;')
PsiErrorElement:Expecting an element
PsiElement(SEMICOLON)(';')
PsiErrorElement:Unexpected tokens (use ';' to separate expressions on the same line)
PsiElement(RPAR)(')')
PsiElement(SEMICOLON)(';')
PsiElement(SEMICOLON)(';')
PsiElement(SEMICOLON)(';')
PsiElement(DOUBLE_SEMICOLON)(';;')
PsiErrorElement:Expecting an element
PsiElement(SEMICOLON)(';')
PsiErrorElement:Unexpected tokens (use ';' to separate expressions on the same line)
PsiElement(BAD_CHARACTER)('~')
PsiElement(SEMICOLON)(';')
PsiElement(SEMICOLON)(';')
PsiElement(SEMICOLON)(';')
PsiElement(DOUBLE_SEMICOLON)(';;')
PsiErrorElement:Expecting an element
PsiElement(SEMICOLON)(';')
PsiWhiteSpace('\n ')
REFERENCE_EXPRESSION
PsiElement(IDENTIFIER)('srgsdfg')
@@ -57,4 +58,4 @@ JetFile: Expressions_ERR.kt
PsiErrorElement:Expecting an element
PsiElement(RBRACKET)(']')
PsiWhiteSpace('\n')
PsiElement(RBRACE)('}')
PsiElement(RBRACE)('}')
compiler/testData/psi/semicolonBetweenDeclarations.txt
Vendored
+4 -5
View File
@@ -30,10 +30,9 @@ JetFile: semicolonBetweenDeclarations.kt
PsiWhiteSpace(' ')
INTEGER_CONSTANT
PsiElement(INTEGER_LITERAL)('1')
PsiElement(SEMICOLON)(';')
PsiElement(SEMICOLON)(';')
PsiElement(SEMICOLON)(';')
PsiElement(SEMICOLON)(';')
PsiErrorElement:Property getter or setter expected
PsiElement(DOUBLE_SEMICOLON)(';;')
PsiElement(DOUBLE_SEMICOLON)(';;')
PsiWhiteSpace('\n')
CLASS
PsiElement(class)('class')
@@ -178,4 +177,4 @@ JetFile: semicolonBetweenDeclarations.kt
<empty list>
PsiElement(SEMICOLON)(';')
PsiWhiteSpace('\n')
PsiElement(RBRACE)('}')
PsiElement(RBRACE)('}')