use new API for indexing identifiers and TODO items in Kotlin files that allows reusing the lexer for building both indices; index TODO items in KDoc comments

#KT-5677 Fixed

idea/idea-analysis/src/org/jetbrains/kotlin/idea/search/KotlinIndexPatternBuilder.kt

@@ -0,0 +1,46 @@
+/*
+ * Copyright 2010-2015 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.idea.search
+
+import com.intellij.lexer.Lexer
+import com.intellij.psi.PsiFile
+import com.intellij.psi.impl.search.IndexPatternBuilder
+import com.intellij.psi.tree.IElementType
+import com.intellij.psi.tree.TokenSet
+import org.jetbrains.kotlin.kdoc.lexer.KDocTokens
+import org.jetbrains.kotlin.lexer.JetLexer
+import org.jetbrains.kotlin.lexer.JetTokens
+import org.jetbrains.kotlin.psi.JetFile
+
+public class KotlinIndexPatternBuilder: IndexPatternBuilder {
+    private val TODO_COMMENT_TOKENS = TokenSet.orSet(JetTokens.COMMENTS, TokenSet.create(KDocTokens.KDOC))
+
+    override fun getCommentTokenSet(file: PsiFile): TokenSet? {
+        return if (file is JetFile) TODO_COMMENT_TOKENS else null
+    }
+
+    override fun getIndexingLexer(file: PsiFile): Lexer? {
+        return if (file is JetFile) JetLexer() else null
+    }
+
+    override fun getCommentStartDelta(tokenType: IElementType?): Int = 0
+
+    override fun getCommentEndDelta(tokenType: IElementType?): Int = when(tokenType) {
+        JetTokens.BLOCK_COMMENT -> "*/".length()
+        else -> 0
+    }
+}

idea/idea-analysis/src/org/jetbrains/kotlin/idea/search/KotlinIndexers.kt

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2010-2015 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.idea.search
+
+import com.intellij.lexer.Lexer
+import com.intellij.psi.TokenType
+import com.intellij.psi.impl.cache.impl.BaseFilterLexer
+import com.intellij.psi.impl.cache.impl.IdAndToDoScannerBasedOnFilterLexer
+import com.intellij.psi.impl.cache.impl.OccurrenceConsumer
+import com.intellij.psi.impl.cache.impl.id.LexerBasedIdIndexer
+import com.intellij.psi.impl.cache.impl.todo.LexerBasedTodoIndexer
+import com.intellij.psi.search.UsageSearchContext
+import com.intellij.psi.tree.TokenSet
+import org.jetbrains.kotlin.idea.search.usagesSearch.ALL_SEARCHABLE_OPERATIONS
+import org.jetbrains.kotlin.kdoc.lexer.KDocTokens
+import org.jetbrains.kotlin.lexer.JetLexer
+import org.jetbrains.kotlin.lexer.JetTokens
+
+class KotlinFilterLexer(table: OccurrenceConsumer): BaseFilterLexer(JetLexer(), table) {
+    private val codeTokens = TokenSet.orSet(
+            TokenSet.create(*ALL_SEARCHABLE_OPERATIONS.toTypedArray()),
+            TokenSet.create(JetTokens.IDENTIFIER, JetTokens.FIELD_IDENTIFIER, JetTokens.LABEL_IDENTIFIER)
+    )
+
+    private val commentTokens = TokenSet.orSet(JetTokens.COMMENTS, TokenSet.create(KDocTokens.KDOC))
+
+    private val skipTokens = TokenSet.create(
+            TokenType.WHITE_SPACE, JetTokens.RPAR, JetTokens.LBRACE, JetTokens.RBRACE,
+            JetTokens.RBRACKET, JetTokens.SEMICOLON, JetTokens.COMMA, JetTokens.DOT
+    )
+
+    override fun advance() {
+        val tokenType = myDelegate.getTokenType()
+
+        when(tokenType) {
+            in codeTokens -> addOccurrenceInToken(UsageSearchContext.IN_CODE.toInt())
+            in JetTokens.STRINGS -> scanWordsInToken(UsageSearchContext.IN_STRINGS + UsageSearchContext.IN_FOREIGN_LANGUAGES, false, true)
+            in commentTokens -> {
+                scanWordsInToken(UsageSearchContext.IN_COMMENTS.toInt(), false, false)
+                advanceTodoItemCountsInToken()
+            }
+            !in skipTokens -> scanWordsInToken(UsageSearchContext.IN_PLAIN_TEXT.toInt(), false, false)
+        }
+
+        myDelegate.advance()
+    }
+}
+
+class KotlinIdIndexer: LexerBasedIdIndexer() {
+    override fun createLexer(consumer: OccurrenceConsumer): Lexer = KotlinFilterLexer(consumer)
+}
+
+class KotlinTodoIndexer: LexerBasedTodoIndexer(), IdAndToDoScannerBasedOnFilterLexer {
+    override fun createLexer(consumer: OccurrenceConsumer): Lexer = KotlinFilterLexer(consumer)
+}

idea/src/META-INF/plugin.xml

@@ -453,6 +453,10 @@
     <fileBasedIndex implementation="org.jetbrains.kotlin.idea.versions.KotlinAbiVersionIndex"/>
     <fileBasedIndex implementation="org.jetbrains.kotlin.idea.vfilefinder.KotlinClassFileIndex"/>
 
+    <idIndexer filetype="Kotlin" implementationClass="org.jetbrains.kotlin.idea.search.KotlinIdIndexer"/>
+    <todoIndexer filetype="Kotlin" implementationClass="org.jetbrains.kotlin.idea.search.KotlinTodoIndexer"/>
+    <indexPatternBuilder implementation="org.jetbrains.kotlin.idea.search.KotlinIndexPatternBuilder"/>
+
     <editorNotificationProvider implementation="org.jetbrains.kotlin.idea.versions.UnsupportedAbiVersionNotificationPanelProvider"/>
     <editorNotificationProvider implementation="org.jetbrains.kotlin.idea.highlighter.ErrorDuringFileAnalyzeNotificationProvider"/>
 

idea/testData/highlighter/Todo.kt

@@ -0,0 +1,7 @@
+// <info descr="TODO This is a todo in line comment">TODO This is a todo in line comment</info>
+
+/* <info descr="TODO This is a todo in block comment ">TODO This is a todo in block comment </info>*/
+
+/**
+ * <info descr="TODO This is a todo in KDoc">TODO This is a todo in KDoc</info>
+ */

idea/tests/org/jetbrains/kotlin/idea/highlighter/HighlightingTestGenerated.java

@@ -71,6 +71,12 @@ public class HighlightingTestGenerated extends AbstractHighlightingTest {
         doTest(fileName);
     }
 
+    @TestMetadata("Todo.kt")
+    public void testTodo() throws Exception {
+        String fileName = JetTestUtils.navigationMetadata("idea/testData/highlighter/Todo.kt");
+        doTest(fileName);
+    }
+
     @TestMetadata("TypesAndAnnotations.kt")
     public void testTypesAndAnnotations() throws Exception {
         String fileName = JetTestUtils.navigationMetadata("idea/testData/highlighter/TypesAndAnnotations.kt");