Add benchmark action for highlighting

idea/src/META-INF/plugin.xml

@@ -146,6 +146,10 @@
                 text="Local scenario"/>
       </group>
 
+      <action id="HighlightingBenchmarkAction"
+              class="org.jetbrains.kotlin.idea.actions.internal.benchmark.HighlightingBenchmarkAction"
+              text="Benchmark highlighting"/>
+
       <action id="CompletionBindingContextCachingToggleAction"
               class="org.jetbrains.kotlin.idea.actions.internal.CompletionBindingContextCachingToggleAction"
               text="Enable completion binding context caching"/>

idea/src/org/jetbrains/kotlin/idea/actions/internal/benchmark/AbstractCompletionBenchmarkAction.kt

@@ -30,11 +30,16 @@ import com.intellij.openapi.ui.popup.JBPopupFactory
 import com.intellij.openapi.wm.WindowManager
 import com.intellij.psi.PsiDocumentManager
 import com.intellij.psi.search.DelegatingGlobalSearchScope
+import com.intellij.psi.search.FileTypeIndex
 import com.intellij.psi.search.GlobalSearchScope
+import com.intellij.ui.components.JBLabel
+import com.intellij.ui.components.JBTextField
+import com.intellij.uiDesigner.core.GridConstraints
 import kotlinx.coroutines.experimental.CancellationException
 import kotlinx.coroutines.experimental.delay
 import kotlinx.coroutines.experimental.launch
 import kotlinx.coroutines.experimental.withTimeout
+import org.jetbrains.kotlin.idea.KotlinFileType
 import org.jetbrains.kotlin.idea.actions.internal.KotlinInternalMode
 import org.jetbrains.kotlin.idea.caches.resolve.ModuleOrigin
 import org.jetbrains.kotlin.idea.caches.resolve.getNullableModuleInfo
@@ -42,11 +47,12 @@ import org.jetbrains.kotlin.idea.completion.CompletionBenchmarkSink
 import org.jetbrains.kotlin.idea.core.moveCaret
 import org.jetbrains.kotlin.idea.core.util.EDT
 import org.jetbrains.kotlin.idea.refactoring.getLineCount
-import org.jetbrains.kotlin.idea.stubindex.KotlinExactPackagesIndex
+import org.jetbrains.kotlin.idea.refactoring.toPsiFile
 import org.jetbrains.kotlin.idea.util.application.runWriteAction
 import org.jetbrains.kotlin.psi.KtFile
 import java.util.*
 import javax.swing.JFileChooser
+import javax.swing.JPanel
 
 abstract class AbstractCompletionBenchmarkAction : AnAction() {
     override fun actionPerformed(e: AnActionEvent?) {
@@ -74,31 +80,36 @@ abstract class AbstractCompletionBenchmarkAction : AnAction() {
 
         internal fun <T> List<T>.randomElement(random: Random): T? = if (this.isNotEmpty()) this[random.nextInt(this.size)] else null
         internal fun <T> Array<T>.randomElement(random: Random): T? = if (this.isNotEmpty()) this[random.nextInt(this.size)] else null
+        internal fun <T : Any> List<T>.shuffledSequence(random: Random): Sequence<T> = generateSequence { this.randomElement(random) }.distinct()
 
-    }
+        internal fun collectSuitableKotlinFiles(project: Project, filePredicate: (KtFile) -> Boolean): MutableList<KtFile> {
-
+            val scope = object : DelegatingGlobalSearchScope(GlobalSearchScope.allScope(project)) {
-    fun collectSuitableKotlinFiles(project: Project, filter: (KtFile) -> Boolean): MutableList<KtFile> {
+                override fun isSearchOutsideRootModel(): Boolean = false
-        val scope = object : DelegatingGlobalSearchScope(GlobalSearchScope.allScope(project)) {
-            override fun isSearchOutsideRootModel(): Boolean = false
-        }
-        val exactPackageIndex = KotlinExactPackagesIndex.getInstance()
-
-        fun KtFile.isUsableForBenchmark(): Boolean {
-            val moduleInfo = this.getNullableModuleInfo() ?: return false
-            if (this.isCompiled || !this.isWritable || this.isScript) return false
-            return moduleInfo.moduleOrigin == ModuleOrigin.MODULE
-        }
-
-        val ktFiles = mutableListOf<KtFile>()
-        exactPackageIndex.processAllKeys(project) {
-            exactPackageIndex.get(it, project, scope).forEach {
-                if (it.isUsableForBenchmark() && filter(it)) {
-                    ktFiles += it
-                }
             }
-            true
+
+            fun KtFile.isUsableForBenchmark(): Boolean {
+                val moduleInfo = this.getNullableModuleInfo() ?: return false
+                if (this.isCompiled || !this.isWritable || this.isScript) return false
+                return moduleInfo.moduleOrigin == ModuleOrigin.MODULE
+            }
+
+            val kotlinVFiles = FileTypeIndex.getFiles(KotlinFileType.INSTANCE, scope)
+
+            return kotlinVFiles
+                    .asSequence()
+                    .mapNotNull { vfile -> (vfile.toPsiFile(project) as? KtFile) }
+                    .filterTo(mutableListOf()) { it.isUsableForBenchmark() && filePredicate(it) }
+        }
+
+        internal fun JPanel.addBoxWithLabel(tooltip: String, label: String = tooltip + ":", default: String, i: Int): JBTextField {
+            this.add(JBLabel(label), GridConstraints().apply { row = i; column = 0 })
+            val textField = JBTextField().apply {
+                text = default
+                toolTipText = tooltip
+            }
+            this.add(textField, GridConstraints().apply { row = i; column = 1; fill = GridConstraints.FILL_HORIZONTAL })
+            return textField
         }
-        return ktFiles
     }
 
     override fun update(e: AnActionEvent) {

idea/src/org/jetbrains/kotlin/idea/actions/internal/benchmark/HighlightingBenchmarkAction.kt

@@ -0,0 +1,222 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.idea.actions.internal.benchmark
+
+import com.intellij.codeInsight.daemon.DaemonCodeAnalyzer
+import com.intellij.codeInsight.daemon.impl.DaemonCodeAnalyzerImpl
+import com.intellij.codeInsight.daemon.impl.HighlightInfo
+import com.intellij.codeInsight.daemon.impl.SeverityRegistrar
+import com.intellij.codeInsight.navigation.NavigationUtil
+import com.intellij.openapi.actionSystem.ActionManager
+import com.intellij.openapi.actionSystem.AnAction
+import com.intellij.openapi.actionSystem.AnActionEvent
+import com.intellij.openapi.editor.impl.DocumentMarkupModel
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.ui.DialogBuilder
+import com.intellij.psi.PsiDocumentManager
+import com.intellij.ui.components.JBPanel
+import com.intellij.ui.components.JBTextField
+import com.intellij.uiDesigner.core.GridLayoutManager
+import kotlinx.coroutines.experimental.channels.ConflatedChannel
+import kotlinx.coroutines.experimental.delay
+import kotlinx.coroutines.experimental.launch
+import org.jetbrains.kotlin.idea.actions.internal.KotlinInternalMode
+import org.jetbrains.kotlin.idea.actions.internal.benchmark.AbstractCompletionBenchmarkAction.Companion.addBoxWithLabel
+import org.jetbrains.kotlin.idea.actions.internal.benchmark.AbstractCompletionBenchmarkAction.Companion.collectSuitableKotlinFiles
+import org.jetbrains.kotlin.idea.actions.internal.benchmark.AbstractCompletionBenchmarkAction.Companion.shuffledSequence
+import org.jetbrains.kotlin.idea.core.util.EDT
+import org.jetbrains.kotlin.idea.refactoring.getLineCount
+import org.jetbrains.kotlin.psi.KtFile
+import java.util.*
+import javax.swing.JFileChooser
+import kotlin.properties.Delegates
+
+class HighlightingBenchmarkAction : AnAction() {
+    override fun actionPerformed(e: AnActionEvent?) {
+        val project = e?.project ?: return
+
+        val settings = showSettingsDialog() ?: return
+
+        val random = Random(settings.seed)
+
+        fun collectFiles(): List<KtFile>? {
+
+            val ktFiles = collectSuitableKotlinFiles(project, { it.getLineCount() >= settings.lines })
+
+            if (ktFiles.size < settings.files) {
+                AbstractCompletionBenchmarkAction.showPopup(project, "Number of attempts > then files in project, ${ktFiles.size}")
+                return null
+            }
+
+            return ktFiles
+        }
+
+
+        val ktFiles = collectFiles() ?: return
+
+        val results = mutableListOf<Result>()
+
+        val connection = project.messageBus.connect()
+
+        ActionManager.getInstance().getAction("CloseAllEditors").actionPerformed(e)
+
+        val finishListener = DaemonFinishListener()
+        connection.subscribe(DaemonCodeAnalyzer.DAEMON_EVENT_TOPIC, finishListener)
+        launch(EDT) {
+            try {
+                delay(100)
+                ktFiles
+                        .shuffledSequence(random)
+                        .take(settings.files)
+                        .forEach { file ->
+                            results += openFileAndMeasureTimeToHighlight(file, project, finishListener)
+                        }
+
+                saveResults(results, project)
+            }
+            finally {
+                connection.disconnect()
+                finishListener.channel.close()
+            }
+        }
+    }
+
+    private data class Settings(val seed: Long, val files: Int, val lines: Int)
+
+    private inner class DaemonFinishListener : DaemonCodeAnalyzer.DaemonListener {
+        val channel = ConflatedChannel<String>()
+
+        override fun daemonFinished() {
+            channel.offer(SUCCESS)
+        }
+
+        override fun daemonCancelEventOccurred(reason: String) {
+            channel.offer(reason)
+        }
+    }
+
+    companion object {
+        private const val SUCCESS = "Success"
+    }
+
+    private fun showSettingsDialog(): Settings? {
+        var cSeed: JBTextField by Delegates.notNull()
+        var cFiles: JBTextField by Delegates.notNull()
+        var cLines: JBTextField by Delegates.notNull()
+        val dialogBuilder = DialogBuilder()
+
+
+        val jPanel = JBPanel<JBPanel<*>>(GridLayoutManager(3, 2)).apply {
+            var i = 0
+            cSeed = addBoxWithLabel("Random seed", default = "0", i = i++)
+            cFiles = addBoxWithLabel("Files to visit", default = "20", i = i++)
+            cLines = addBoxWithLabel("Minimal line count", default = "100", i = i)
+        }
+        dialogBuilder.centerPanel(jPanel)
+        if (!dialogBuilder.showAndGet()) return null
+
+        return Settings(cSeed.text.toLong(), cFiles.text.toInt(), cLines.text.toInt())
+    }
+
+    private sealed class Result(val location: String, val lines: Int) {
+        abstract fun toCSV(builder: StringBuilder)
+
+        class Success(location: String, lines: Int, val time: Long, val status: String) : Result(location, lines) {
+            override fun toCSV(builder: StringBuilder): Unit = with(builder) {
+                append(location)
+                append(", ")
+                append(lines)
+                append(", ")
+                append(status)
+                append(", ")
+                append(time)
+            }
+        }
+
+        class Error(location: String, lines: Int = 0, val reason: String) : Result(location, lines) {
+            override fun toCSV(builder: StringBuilder): Unit = with(builder) {
+                append(location)
+                append(", ")
+                append(lines)
+                append(", fail: ")
+                append(reason)
+                append(", ")
+            }
+        }
+    }
+
+    private suspend fun openFileAndMeasureTimeToHighlight(file: KtFile, project: Project, finishListener: DaemonFinishListener): Result {
+
+        NavigationUtil.openFileWithPsiElement(file.navigationElement, true, true)
+        val location = file.virtualFile.path
+
+        val lines = file.getLineCount()
+
+        val document = PsiDocumentManager.getInstance(project).getDocument(file) ?: return Result.Error(location, lines, "No document")
+
+        val daemon = DaemonCodeAnalyzer.getInstance(project) as DaemonCodeAnalyzerImpl
+
+        if (!daemon.isHighlightingAvailable(file)) return Result.Error(location, lines, "Highlighting not available")
+
+        if (!daemon.isRunningOrPending) return Result.Error(location, lines, "Analysis not running or pending")
+
+        val start = System.currentTimeMillis()
+        val outcome = finishListener.channel.receive()
+        if (outcome != SUCCESS) {
+            return Result.Error(location, lines, outcome)
+        }
+
+        val analysisTime = System.currentTimeMillis() - start
+
+        val model = DocumentMarkupModel.forDocument(document, project, true)
+
+        val severityRegistrar = SeverityRegistrar.getSeverityRegistrar(project)
+
+        val maxSeverity = model.allHighlighters
+                .mapNotNull { highlighter ->
+                    val info = highlighter.errorStripeTooltip as? HighlightInfo ?: return@mapNotNull null
+                    info.severity
+                }.maxWith(Comparator { o1, o2 -> severityRegistrar.compare(o1, o2) })
+        return Result.Success(location, lines, analysisTime, maxSeverity?.myName ?: "clean")
+    }
+
+
+    private fun saveResults(allResults: List<Result>, project: Project) {
+        val jfc = JFileChooser()
+        val result = jfc.showSaveDialog(null)
+        if (result == JFileChooser.APPROVE_OPTION) {
+            val file = jfc.selectedFile
+            file.writeText(buildString {
+                appendln("n, file, lines, status, time")
+                var i = 0
+                allResults.forEach {
+                    append(i++)
+                    append(", ")
+                    it.toCSV(this)
+                    appendln()
+                }
+            })
+        }
+        AbstractCompletionBenchmarkAction.showPopup(project, "Done")
+    }
+
+    override fun update(e: AnActionEvent) {
+        e.presentation.isEnabledAndVisible = KotlinInternalMode.enabled
+    }
+}
+
+

idea/src/org/jetbrains/kotlin/idea/actions/internal/benchmark/LocalCompletionBenchmark.kt

@@ -18,10 +18,8 @@ package org.jetbrains.kotlin.idea.actions.internal.benchmark
 
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.ui.DialogBuilder
-import com.intellij.ui.components.JBLabel
 import com.intellij.ui.components.JBPanel
 import com.intellij.ui.components.JBTextField
-import com.intellij.uiDesigner.core.GridConstraints
 import com.intellij.uiDesigner.core.GridLayoutManager
 import org.jetbrains.kotlin.idea.actions.internal.benchmark.AbstractCompletionBenchmarkAction.Companion.randomElement
 import org.jetbrains.kotlin.idea.completion.CompletionBenchmarkSink
@@ -76,22 +74,12 @@ class LocalCompletionBenchmarkAction : AbstractCompletionBenchmarkAction() {
 
 
         val jPanel = JBPanel<JBPanel<*>>(GridLayoutManager(4, 2)).apply {
-
             var i = 0
-            fun addBoxWithLabel(tooltip: String, label: String = tooltip + ":", default: String): JBTextField {
-                this.add(JBLabel(label), GridConstraints().apply { row = i; column = 0 })
-                val textField = JBTextField().apply {
-                    text = default
-                    toolTipText = tooltip
-                }
-                this.add(textField, GridConstraints().apply { row = i++; column = 1; fill = GridConstraints.FILL_HORIZONTAL })
-                return textField
-            }
-            cSeed = addBoxWithLabel("Random seed", default = "0")
-            cFiles = addBoxWithLabel("Files to visit", default = "20")
-            cFunctions = addBoxWithLabel("Max functions to visit", default = "20")
-            cLines = addBoxWithLabel("File lines", default = "100")
 
+            cSeed = addBoxWithLabel("Random seed", default = "0", i = i++)
+            cFiles = addBoxWithLabel("Files to visit", default = "20", i = i++)
+            cFunctions = addBoxWithLabel("Max functions to visit", default = "20", i = i++)
+            cLines = addBoxWithLabel("File lines", default = "100", i = i)
         }
         dialogBuilder.centerPanel(jPanel)
         if (!dialogBuilder.showAndGet()) return null

idea/src/org/jetbrains/kotlin/idea/actions/internal/benchmark/TopLevelCompletionBenchmark.kt

@@ -77,20 +77,10 @@ class TopLevelCompletionBenchmarkAction : AbstractCompletionBenchmarkAction() {
 
 
         val jPanel = JBPanel<JBPanel<*>>(GridLayoutManager(3, 2)).apply {
-
             var i = 0
-            fun addBoxWithLabel(tooltip: String, label: String = tooltip + ":", default: String): JBTextField {
+            cSeed = addBoxWithLabel("Random seed", default = "0", i = i++)
-                this.add(JBLabel(label), GridConstraints().apply { row = i; column = 0 })
+            cFiles = addBoxWithLabel("Files to visit", default = "20", i = i++)
-                val textField = JBTextField().apply {
+            cLines = addBoxWithLabel("File lines", default = "100", i = i)
-                    text = default
-                    toolTipText = tooltip
-                }
-                this.add(textField, GridConstraints().apply { row = i++; column = 1; fill = GridConstraints.FILL_HORIZONTAL })
-                return textField
-            }
-            cSeed = addBoxWithLabel("Random seed", default = "0")
-            cFiles = addBoxWithLabel("Files to visit", default = "20")
-            cLines = addBoxWithLabel("File lines", default = "100")
         }
         dialogBuilder.centerPanel(jPanel)
         if (!dialogBuilder.showAndGet()) return null