diff --git a/compiler/frontend/src/org/jetbrains/kotlin/lexer/Kotlin.flex b/compiler/frontend/src/org/jetbrains/kotlin/lexer/Kotlin.flex index 4540ad37a43..b394b459477 100644 --- a/compiler/frontend/src/org/jetbrains/kotlin/lexer/Kotlin.flex +++ b/compiler/frontend/src/org/jetbrains/kotlin/lexer/Kotlin.flex @@ -69,7 +69,7 @@ import org.jetbrains.kotlin.lexer.KtTokens; %eof} %xstate STRING RAW_STRING SHORT_TEMPLATE_ENTRY BLOCK_COMMENT DOC_COMMENT -%state LONG_TEMPLATE_ENTRY +%state LONG_TEMPLATE_ENTRY UNMATCHED_BACKTICK DIGIT=[0-9] DIGIT_OR_UNDERSCORE = [_0-9] @@ -119,6 +119,7 @@ REGULAR_STRING_PART=[^\\\"\n\$]+ SHORT_TEMPLATE_ENTRY=\${IDENTIFIER} LONELY_DOLLAR=\$ LONG_TEMPLATE_ENTRY_START=\$\{ +LONELY_BACKTICK=` %% @@ -318,6 +319,8 @@ LONG_TEMPLATE_ENTRY_START=\$\{ "#" { return KtTokens.HASH ; } "@" { return KtTokens.AT ; } +{LONELY_BACKTICK} { pushState(UNMATCHED_BACKTICK); return TokenType.BAD_CHARACTER; } + // error fallback . { return TokenType.BAD_CHARACTER; } // error fallback for exclusive states diff --git a/compiler/frontend/src/org/jetbrains/kotlin/lexer/_JetLexer.java b/compiler/frontend/src/org/jetbrains/kotlin/lexer/_JetLexer.java index 695a9609373..2b08500cbab 100644 --- a/compiler/frontend/src/org/jetbrains/kotlin/lexer/_JetLexer.java +++ b/compiler/frontend/src/org/jetbrains/kotlin/lexer/_JetLexer.java @@ -27,7 +27,7 @@ import com.intellij.util.containers.Stack; /** * This class is a scanner generated by * JFlex 1.7.0-SNAPSHOT - * from the specification file /Users/endermz/projects/kotlin/compiler/frontend/src/org/jetbrains/kotlin/lexer/Kotlin.flex + * from the specification file /Users/yole/jetbrains/kotlin/compiler/frontend/src/org/jetbrains/kotlin/lexer/Kotlin.flex */ class _JetLexer implements FlexLexer { @@ -45,6 +45,7 @@ class _JetLexer implements FlexLexer { public static final int BLOCK_COMMENT = 8; public static final int DOC_COMMENT = 10; public static final int LONG_TEMPLATE_ENTRY = 12; + public static final int UNMATCHED_BACKTICK = 14; /** * ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l @@ -53,7 +54,7 @@ class _JetLexer implements FlexLexer { * l is of the form l = 2*k, k a non negative integer */ private static final int ZZ_LEXSTATE[] = { - 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 4, 5, 5 + 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 4, 5, 5, 0, 0 }; /** @@ -183,26 +184,26 @@ class _JetLexer implements FlexLexer { private static final int [] ZZ_ACTION = zzUnpackAction(); private static final String ZZ_ACTION_PACKED_0 = - "\6\0\1\1\1\2\1\3\1\4\2\1\1\5\1\6"+ - "\1\7\1\2\1\10\1\11\1\12\1\13\1\14\3\3"+ - "\1\15\1\16\14\3\1\17\1\20\1\21\1\22\1\23"+ - "\2\1\1\24\1\25\1\26\1\27\1\30\1\31\1\32"+ - "\1\33\1\34\1\35\1\36\1\35\1\1\1\37\2\35"+ - "\1\40\1\1\1\40\3\41\1\42\1\43\1\2\1\0"+ - "\2\44\1\0\1\45\1\0\1\46\1\47\1\50\1\51"+ - "\1\0\1\52\1\0\2\2\1\44\1\53\1\54\1\55"+ - "\1\56\2\12\1\0\3\3\1\57\1\60\1\61\1\3"+ - "\1\62\2\3\1\63\13\3\1\64\1\65\1\66\1\67"+ + "\6\0\1\1\1\2\1\3\1\4\1\5\1\1\1\6"+ + "\1\7\1\10\1\2\1\11\1\12\1\13\1\14\1\15"+ + "\3\3\1\16\1\17\14\3\1\20\1\21\1\22\1\23"+ + "\1\24\2\1\1\25\1\26\1\27\1\30\1\31\1\32"+ + "\1\33\1\34\1\35\1\36\1\37\1\36\1\1\1\40"+ + "\2\36\1\41\1\1\1\41\3\42\1\43\1\44\1\2"+ + "\1\0\2\45\1\0\1\46\1\0\1\47\1\50\1\51"+ + "\1\52\1\0\1\53\1\0\2\2\1\45\1\54\1\55"+ + "\1\56\1\57\2\13\1\0\3\3\1\60\1\61\1\62"+ + "\1\3\1\63\2\3\1\64\13\3\1\65\1\66\1\67"+ "\1\70\1\71\1\72\1\73\1\74\1\75\1\76\1\77"+ - "\1\100\1\0\1\101\2\102\2\0\1\40\1\103\1\104"+ - "\1\0\1\105\1\44\1\3\1\0\1\106\1\107\1\110"+ - "\1\111\1\0\1\112\1\113\4\3\1\114\4\3\1\115"+ - "\2\3\1\116\1\3\1\117\6\3\1\120\1\121\1\122"+ - "\2\0\1\123\2\40\1\45\1\124\1\125\1\126\2\3"+ - "\1\127\3\3\1\130\1\131\7\3\1\132\1\100\1\0"+ - "\1\133\1\134\3\3\1\135\2\3\1\136\1\137\2\3"+ - "\1\140\1\141\1\0\1\3\1\142\2\3\1\143\1\3"+ - "\1\144\2\3\1\145\3\3\1\146\1\147\1\150"; + "\1\100\1\101\1\0\1\102\2\103\2\0\1\41\1\104"+ + "\1\105\1\0\1\106\1\45\1\3\1\0\1\107\1\110"+ + "\1\111\1\112\1\0\1\113\1\114\4\3\1\115\4\3"+ + "\1\116\2\3\1\117\1\3\1\120\6\3\1\121\1\122"+ + "\1\123\2\0\1\124\2\41\1\46\1\125\1\126\1\127"+ + "\2\3\1\130\3\3\1\131\1\132\7\3\1\133\1\101"+ + "\1\0\1\134\1\135\3\3\1\136\2\3\1\137\1\140"+ + "\2\3\1\141\1\142\1\0\1\3\1\143\2\3\1\144"+ + "\1\3\1\145\2\3\1\146\3\3\1\147\1\150\1\151"; private static int [] zzUnpackAction() { int [] result = new int[231]; @@ -646,7 +647,7 @@ class _JetLexer implements FlexLexer { } } - private final Stack states = new Stack<>(); + private final Stack states = new Stack(); private int lBraceCount; private int commentStart; @@ -954,140 +955,144 @@ class _JetLexer implements FlexLexer { case 1: { return TokenType.BAD_CHARACTER; } - case 105: break; + case 106: break; case 2: { return KtTokens.INTEGER_LITERAL; } - case 106: break; + case 107: break; case 3: { return KtTokens.IDENTIFIER; } - case 107: break; + case 108: break; case 4: { return KtTokens.WHITE_SPACE; } - case 108: break; - case 5: - { return KtTokens.DIV ; - } case 109: break; - case 6: - { return KtTokens.HASH ; + case 5: + { pushState(UNMATCHED_BACKTICK); return TokenType.BAD_CHARACTER; } case 110: break; - case 7: - { return KtTokens.EXCL ; + case 6: + { return KtTokens.DIV ; } case 111: break; - case 8: - { return KtTokens.DOT ; + case 7: + { return KtTokens.HASH ; } case 112: break; - case 9: - { return KtTokens.MINUS ; + case 8: + { return KtTokens.EXCL ; } case 113: break; - case 10: - { return KtTokens.CHARACTER_LITERAL; + case 9: + { return KtTokens.DOT ; } case 114: break; - case 11: - { pushState(STRING); return KtTokens.OPEN_QUOTE; + case 10: + { return KtTokens.MINUS ; } case 115: break; - case 12: - { return KtTokens.LBRACE ; + case 11: + { return KtTokens.CHARACTER_LITERAL; } case 116: break; - case 13: - { return KtTokens.RBRACE ; + case 12: + { pushState(STRING); return KtTokens.OPEN_QUOTE; } case 117: break; - case 14: - { return KtTokens.MUL ; + case 13: + { return KtTokens.LBRACE ; } case 118: break; - case 15: - { return KtTokens.EQ ; + case 14: + { return KtTokens.RBRACE ; } case 119: break; - case 16: - { return KtTokens.QUEST ; + case 15: + { return KtTokens.MUL ; } case 120: break; - case 17: - { return KtTokens.PLUS ; + case 16: + { return KtTokens.EQ ; } case 121: break; - case 18: - { return KtTokens.LT ; + case 17: + { return KtTokens.QUEST ; } case 122: break; - case 19: - { return KtTokens.GT ; + case 18: + { return KtTokens.PLUS ; } case 123: break; - case 20: - { return KtTokens.PERC ; + case 19: + { return KtTokens.LT ; } case 124: break; - case 21: - { return KtTokens.COLON ; + case 20: + { return KtTokens.GT ; } case 125: break; - case 22: - { return KtTokens.LBRACKET ; + case 21: + { return KtTokens.PERC ; } case 126: break; - case 23: - { return KtTokens.RBRACKET ; + case 22: + { return KtTokens.COLON ; } case 127: break; - case 24: - { return KtTokens.LPAR ; + case 23: + { return KtTokens.LBRACKET ; } case 128: break; - case 25: - { return KtTokens.RPAR ; + case 24: + { return KtTokens.RBRACKET ; } case 129: break; - case 26: - { return KtTokens.SEMICOLON ; + case 25: + { return KtTokens.LPAR ; } case 130: break; - case 27: - { return KtTokens.COMMA ; + case 26: + { return KtTokens.RPAR ; } case 131: break; - case 28: - { return KtTokens.AT ; + case 27: + { return KtTokens.SEMICOLON ; } case 132: break; - case 29: - { return KtTokens.REGULAR_STRING_PART; + case 28: + { return KtTokens.COMMA ; } case 133: break; - case 30: - { popState(); yypushback(1); return KtTokens.DANGLING_NEWLINE; + case 29: + { return KtTokens.AT ; } case 134: break; - case 31: - { popState(); return KtTokens.CLOSING_QUOTE; + case 30: + { return KtTokens.REGULAR_STRING_PART; } case 135: break; - case 32: - { popState(); return KtTokens.IDENTIFIER; + case 31: + { popState(); yypushback(1); return KtTokens.DANGLING_NEWLINE; } case 136: break; - case 33: - { + case 32: + { popState(); return KtTokens.CLOSING_QUOTE; } case 137: break; - case 34: - { lBraceCount++; return KtTokens.LBRACE; + case 33: + { popState(); return KtTokens.IDENTIFIER; } case 138: break; + case 34: + { + } + case 139: break; case 35: + { lBraceCount++; return KtTokens.LBRACE; + } + case 140: break; + case 36: { if (lBraceCount == 0) { popState(); return KtTokens.LONG_TEMPLATE_ENTRY_END; @@ -1095,30 +1100,30 @@ class _JetLexer implements FlexLexer { lBraceCount--; return KtTokens.RBRACE; } - case 139: break; - case 36: + case 141: break; + case 37: { return KtTokens.FLOAT_LITERAL; } - case 140: break; - case 37: + case 142: break; + case 38: { return KtTokens.FIELD_IDENTIFIER; } - case 141: break; - case 38: + case 143: break; + case 39: { return KtTokens.EOL_COMMENT; } - case 142: break; - case 39: + case 144: break; + case 40: { pushState(BLOCK_COMMENT); commentDepth = 0; commentStart = getTokenStart(); } - case 143: break; - case 40: + case 145: break; + case 41: { return KtTokens.DIVEQ ; } - case 144: break; - case 41: + case 146: break; + case 42: { if (zzCurrentPos == 0) { return KtTokens.SHEBANG_COMMENT; } @@ -1127,114 +1132,114 @@ class _JetLexer implements FlexLexer { return KtTokens.HASH; } } - case 145: break; - case 42: + case 147: break; + case 43: { return KtTokens.EXCLEQ ; } - case 146: break; - case 43: + case 148: break; + case 44: { return KtTokens.RANGE ; } - case 147: break; - case 44: + case 149: break; + case 45: { return KtTokens.MINUSMINUS; } - case 148: break; - case 45: + case 150: break; + case 46: { return KtTokens.MINUSEQ ; } - case 149: break; - case 46: + case 151: break; + case 47: { return KtTokens.ARROW ; } - case 150: break; - case 47: + case 152: break; + case 48: { return KtTokens.IS_KEYWORD ; } - case 151: break; - case 48: + case 153: break; + case 49: { return KtTokens.IN_KEYWORD ; } - case 152: break; - case 49: + case 154: break; + case 50: { return KtTokens.IF_KEYWORD ; } - case 153: break; - case 50: + case 155: break; + case 51: { return KtTokens.MULTEQ ; } - case 154: break; - case 51: + case 156: break; + case 52: { return KtTokens.AS_KEYWORD ; } - case 155: break; - case 52: + case 157: break; + case 53: { return KtTokens.DO_KEYWORD ; } - case 156: break; - case 53: + case 158: break; + case 54: { return KtTokens.EQEQ ; } - case 157: break; - case 54: + case 159: break; + case 55: { return KtTokens.DOUBLE_ARROW; } - case 158: break; - case 55: + case 160: break; + case 56: { return KtTokens.PLUSEQ ; } - case 159: break; - case 56: + case 161: break; + case 57: { return KtTokens.PLUSPLUS ; } - case 160: break; - case 57: + case 162: break; + case 58: { return KtTokens.LTEQ ; } - case 161: break; - case 58: + case 163: break; + case 59: { return KtTokens.GTEQ ; } - case 162: break; - case 59: + case 164: break; + case 60: { return KtTokens.ANDAND ; } - case 163: break; - case 60: + case 165: break; + case 61: { return KtTokens.OROR ; } - case 164: break; - case 61: + case 166: break; + case 62: { return KtTokens.PERCEQ ; } - case 165: break; - case 62: + case 167: break; + case 63: { return KtTokens.COLONCOLON; } - case 166: break; - case 63: + case 168: break; + case 64: { return KtTokens.DOUBLE_SEMICOLON; } - case 167: break; - case 64: + case 169: break; + case 65: { pushState(SHORT_TEMPLATE_ENTRY); yypushback(yylength() - 1); return KtTokens.SHORT_TEMPLATE_ENTRY_START; } - case 168: break; - case 65: + case 170: break; + case 66: { pushState(LONG_TEMPLATE_ENTRY); return KtTokens.LONG_TEMPLATE_ENTRY_START; } - case 169: break; - case 66: + case 171: break; + case 67: { return KtTokens.ESCAPE_SEQUENCE; } - case 170: break; - case 67: + case 172: break; + case 68: { commentDepth++; } - case 171: break; - case 68: + case 173: break; + case 69: { if (commentDepth > 0) { commentDepth--; } @@ -1245,66 +1250,66 @@ class _JetLexer implements FlexLexer { return commentStateToTokenType(state); } } - case 172: break; - case 69: + case 174: break; + case 70: { yypushback(2); return KtTokens.INTEGER_LITERAL; } - case 173: break; - case 70: + case 175: break; + case 71: { pushState(DOC_COMMENT); commentDepth = 0; commentStart = getTokenStart(); } - case 174: break; - case 71: + case 176: break; + case 72: { return KtTokens.NOT_IS; } - case 175: break; - case 72: + case 177: break; + case 73: { return KtTokens.NOT_IN; } - case 176: break; - case 73: + case 178: break; + case 74: { return KtTokens.EXCLEQEQEQ; } - case 177: break; - case 74: + case 179: break; + case 75: { return KtTokens.RESERVED ; } - case 178: break; - case 75: + case 180: break; + case 76: { pushState(RAW_STRING); return KtTokens.OPEN_QUOTE; } - case 179: break; - case 76: + case 181: break; + case 77: { return KtTokens.TRY_KEYWORD ; } - case 180: break; - case 77: + case 182: break; + case 78: { return KtTokens.AS_SAFE; } - case 181: break; - case 78: + case 183: break; + case 79: { return KtTokens.FUN_KEYWORD ; } - case 182: break; - case 79: + case 184: break; + case 80: { return KtTokens.FOR_KEYWORD ; } - case 183: break; - case 80: + case 185: break; + case 81: { return KtTokens.VAL_KEYWORD ; } - case 184: break; - case 81: + case 186: break; + case 82: { return KtTokens.VAR_KEYWORD ; } - case 185: break; - case 82: + case 187: break; + case 83: { return KtTokens.EQEQEQ ; } - case 186: break; - case 83: + case 188: break; + case 84: { int length = yytext().length(); if (length <= 3) { // closing """ popState(); @@ -1315,91 +1320,91 @@ class _JetLexer implements FlexLexer { return KtTokens.REGULAR_STRING_PART; } } - case 187: break; - case 84: + case 189: break; + case 85: { return KtTokens.BLOCK_COMMENT; } - case 188: break; - case 85: + case 190: break; + case 86: { yypushback(3); return KtTokens.EXCL; } - case 189: break; - case 86: + case 191: break; + case 87: { return KtTokens.THIS_KEYWORD ; } - case 190: break; - case 87: + case 192: break; + case 88: { return KtTokens.TRUE_KEYWORD ; } - case 191: break; - case 88: + case 193: break; + case 89: { return KtTokens.ELSE_KEYWORD ; } - case 192: break; - case 89: + case 194: break; + case 90: { return KtTokens.NULL_KEYWORD ; } - case 193: break; - case 90: + case 195: break; + case 91: { return KtTokens.WHEN_KEYWORD ; } - case 194: break; - case 91: + case 196: break; + case 92: { popState(); return KtTokens.THIS_KEYWORD; } - case 195: break; - case 92: + case 197: break; + case 93: { return KtTokens.THROW_KEYWORD ; } - case 196: break; - case 93: + case 198: break; + case 94: { return KtTokens.SUPER_KEYWORD ; } - case 197: break; - case 94: + case 199: break; + case 95: { return KtTokens.FALSE_KEYWORD ; } - case 198: break; - case 95: + case 200: break; + case 96: { return KtTokens.CLASS_KEYWORD ; } - case 199: break; - case 96: + case 201: break; + case 97: { return KtTokens.BREAK_KEYWORD ; } - case 200: break; - case 97: + case 202: break; + case 98: { return KtTokens.WHILE_KEYWORD ; } - case 201: break; - case 98: + case 203: break; + case 99: { return KtTokens.TYPEOF_KEYWORD ; } - case 202: break; - case 99: + case 204: break; + case 100: { return KtTokens.RETURN_KEYWORD ; } - case 203: break; - case 100: + case 205: break; + case 101: { return KtTokens.OBJECT_KEYWORD ; } - case 204: break; - case 101: + case 206: break; + case 102: { return KtTokens.PACKAGE_KEYWORD ; } - case 205: break; - case 102: + case 207: break; + case 103: { return KtTokens.CONTINUE_KEYWORD ; } - case 206: break; - case 103: + case 208: break; + case 104: { return KtTokens.TYPE_ALIAS_KEYWORD ; } - case 207: break; - case 104: + case 209: break; + case 105: { return KtTokens.INTERFACE_KEYWORD ; } - case 208: break; + case 210: break; default: zzScanError(ZZ_NO_MATCH); } diff --git a/idea/tests/org/jetbrains/kotlin/idea/highlighter/HighlightingUpdateTest.kt b/idea/tests/org/jetbrains/kotlin/idea/highlighter/HighlightingUpdateTest.kt new file mode 100644 index 00000000000..c9ef56312cd --- /dev/null +++ b/idea/tests/org/jetbrains/kotlin/idea/highlighter/HighlightingUpdateTest.kt @@ -0,0 +1,58 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.idea.highlighter + +import com.intellij.openapi.editor.ex.EditorEx +import com.intellij.openapi.editor.ex.util.LexerEditorHighlighter +import com.intellij.psi.TokenType +import com.intellij.psi.tree.IElementType +import org.jetbrains.kotlin.idea.test.KotlinLightCodeInsightFixtureTestCase +import org.jetbrains.kotlin.idea.test.KotlinLightProjectDescriptor +import org.jetbrains.kotlin.idea.util.application.executeWriteCommand +import org.jetbrains.kotlin.lexer.KtTokens +import kotlin.test.assertEquals + +class HighlightingUpdateTest : KotlinLightCodeInsightFixtureTestCase() { + override fun getProjectDescriptor(): KotlinLightProjectDescriptor = KotlinLightProjectDescriptor.INSTANCE + + fun testUpdateWithBackticks() { + val editorText = "fun `two words`() { }" + myFixture.configureByText("foo.kt", editorText) + val editorHighlighter = (myFixture.editor as EditorEx).highlighter as LexerEditorHighlighter + + val index = editorText.indexOf("`()") + myFixture.project.executeWriteCommand("") { + editorHighlighter.assertHighlighterIteratorTokens(KtTokens.FUN_KEYWORD, KtTokens.WHITE_SPACE, KtTokens.IDENTIFIER) + + myFixture.editor.document.deleteString(index, index + 1) + + editorHighlighter.assertHighlighterIteratorTokens(KtTokens.FUN_KEYWORD, KtTokens.WHITE_SPACE, TokenType.BAD_CHARACTER) + + myFixture.editor.document.insertString(index, "`") + + editorHighlighter.assertHighlighterIteratorTokens(KtTokens.FUN_KEYWORD, KtTokens.WHITE_SPACE, KtTokens.IDENTIFIER) + } + } +} + +private fun LexerEditorHighlighter.assertHighlighterIteratorTokens(vararg tokens: IElementType) { + val iterator = createIterator(0) + for (token in tokens) { + assertEquals(token, iterator.tokenType) + iterator.advance() + } +}