Return non-default lexer state for unmatched backtick

When a document is changed, the editor highlighter restarts the lexer at the last point where it was in the default state. Previously, if a matching backtick was deleted and then reinserted, the lexer was resumed just before the reinsertion point, and as the result the two backticks were parsed not as two delimiters of a single token but as two BAD_CHARACTER tokens. With this change, all tokens after the unmatched backtick will be in a non-default state, so the lexer will restart at the first backtick and will correctly see the two backticks as a single token. #KT-9091 Fixed
2017-05-09 17:32:48 +02:00
parent 66877c138d
commit ec53a6dbe7
3 changed files with 293 additions and 227 deletions
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.idea.highlighter
+
+import com.intellij.openapi.editor.ex.EditorEx
+import com.intellij.openapi.editor.ex.util.LexerEditorHighlighter
+import com.intellij.psi.TokenType
+import com.intellij.psi.tree.IElementType
+import org.jetbrains.kotlin.idea.test.KotlinLightCodeInsightFixtureTestCase
+import org.jetbrains.kotlin.idea.test.KotlinLightProjectDescriptor
+import org.jetbrains.kotlin.idea.util.application.executeWriteCommand
+import org.jetbrains.kotlin.lexer.KtTokens
+import kotlin.test.assertEquals
+
+class HighlightingUpdateTest : KotlinLightCodeInsightFixtureTestCase() {
+    override fun getProjectDescriptor(): KotlinLightProjectDescriptor = KotlinLightProjectDescriptor.INSTANCE
+
+    fun testUpdateWithBackticks() {
+        val editorText = "fun `two words`() { }"
+        myFixture.configureByText("foo.kt", editorText)
+        val editorHighlighter = (myFixture.editor as EditorEx).highlighter as LexerEditorHighlighter
+
+        val index = editorText.indexOf("`()")
+        myFixture.project.executeWriteCommand("") {
+            editorHighlighter.assertHighlighterIteratorTokens(KtTokens.FUN_KEYWORD, KtTokens.WHITE_SPACE, KtTokens.IDENTIFIER)
+
+            myFixture.editor.document.deleteString(index, index + 1)
+
+            editorHighlighter.assertHighlighterIteratorTokens(KtTokens.FUN_KEYWORD, KtTokens.WHITE_SPACE, TokenType.BAD_CHARACTER)
+
+            myFixture.editor.document.insertString(index, "`")
+
+            editorHighlighter.assertHighlighterIteratorTokens(KtTokens.FUN_KEYWORD, KtTokens.WHITE_SPACE, KtTokens.IDENTIFIER)
+        }
+    }
+}
+
+private fun LexerEditorHighlighter.assertHighlighterIteratorTokens(vararg tokens: IElementType) {
+    val iterator = createIterator(0)
+    for (token in tokens) {
+        assertEquals(token, iterator.tokenType)
+        iterator.advance()
+    }
+}