Add benchmark for local completion and improve benchmarking

2017-07-19 04:43:49 +03:00
parent 4df6db141c
commit ccfcfd8721
8 changed files with 516 additions and 293 deletions
@@ -17,12 +17,13 @@
 package org.jetbrains.kotlin.idea.completion

 import kotlinx.coroutines.experimental.channels.ConflatedChannel
+import java.lang.System.currentTimeMillis


 interface CompletionBenchmarkSink {
    fun onCompletionStarted(completionSession: CompletionSession)
-    fun onCompletionEnded(completionSession: CompletionSession)
-    fun onFirstFlush(completionSession: CompletionSession)
+    fun onCompletionEnded(completionSession: CompletionSession, canceled: Boolean)
+    fun onFlush(completionSession: CompletionSession)

    companion object {

@@ -40,49 +41,60 @@ interface CompletionBenchmarkSink {
    private object Empty : CompletionBenchmarkSink {
        override fun onCompletionStarted(completionSession: CompletionSession) {}

-        override fun onCompletionEnded(completionSession: CompletionSession) {}
+        override fun onCompletionEnded(completionSession: CompletionSession, canceled: Boolean) {}

-        override fun onFirstFlush(completionSession: CompletionSession) {}
+        override fun onFlush(completionSession: CompletionSession) {}
    }

    class Impl : CompletionBenchmarkSink {
        private val pendingSessions = mutableListOf<CompletionSession>()
-        private lateinit var results: CompletionBenchmarkResults
        val channel = ConflatedChannel<CompletionBenchmarkResults>()

+        private val perSessionResults = LinkedHashMap<CompletionSession, PerSessionResults>()
+        private var start: Long = 0
+
        override fun onCompletionStarted(completionSession: CompletionSession) = synchronized(this) {
            if (pendingSessions.isEmpty())
-                results = CompletionBenchmarkResults()
+                start = currentTimeMillis()
            pendingSessions += completionSession
+            perSessionResults[completionSession] = PerSessionResults()
        }

-        override fun onCompletionEnded(completionSession: CompletionSession) = synchronized(this) {
+        override fun onCompletionEnded(completionSession: CompletionSession, canceled: Boolean) = synchronized(this) {
            pendingSessions -= completionSession
+            perSessionResults[completionSession]?.onEnd(canceled)
            if (pendingSessions.isEmpty()) {
-                results.onEnd()
-                channel.offer(results)
+                val firstFlush = perSessionResults.values.filterNot { results -> results.canceled }.map { it.firstFlush }.min() ?: 0
+                val full = perSessionResults.values.map { it.full }.max() ?: 0
+                channel.offer(CompletionBenchmarkResults(firstFlush, full))
+                reset()
            }
        }

-        override fun onFirstFlush(completionSession: CompletionSession) = synchronized(this) {
-            results.onFirstFlush()
+        override fun onFlush(completionSession: CompletionSession) = synchronized(this) {
+            perSessionResults[completionSession]?.onFirstFlush()
+            Unit
        }

        fun reset() = synchronized(this) {
            pendingSessions.clear()
+            perSessionResults.clear()
        }

-        class CompletionBenchmarkResults {
-            var start: Long = System.currentTimeMillis()
-            var firstFlush: Long = 0
-            var full: Long = 0
+        data class CompletionBenchmarkResults(var firstFlush: Long = 0, var full: Long = 0)
+
+        private inner class PerSessionResults {
+            var firstFlush = 0L
+            var full = 0L
+            var canceled = false
+
            fun onFirstFlush() {
-                if (firstFlush == 0L)
-                    firstFlush = System.currentTimeMillis() - start
+                firstFlush = currentTimeMillis() - start
            }

-            fun onEnd() {
-                full = System.currentTimeMillis() - start
+            fun onEnd(canceled: Boolean) {
+                full = currentTimeMillis() - start
+                this.canceled = canceled
            }
        }
    }
@@ -23,6 +23,7 @@ import com.intellij.codeInsight.completion.CompletionUtil
 import com.intellij.codeInsight.completion.impl.CamelHumpMatcher
 import com.intellij.codeInsight.completion.impl.RealPrefixMatchingWeigher
 import com.intellij.codeInsight.lookup.LookupElement
+import com.intellij.openapi.progress.ProcessCanceledException
 import com.intellij.patterns.PatternCondition
 import com.intellij.patterns.StandardPatterns
 import com.intellij.psi.search.GlobalSearchScope
@@ -144,7 +145,7 @@ abstract class CompletionSession(

    // LookupElementsCollector instantiation is deferred because virtual call to createSorter uses data from derived classes
    protected val collector: LookupElementsCollector by lazy(LazyThreadSafetyMode.NONE) {
-        LookupElementsCollector(this, prefixMatcher, parameters, resultSet, createSorter(), (file as? KtCodeFragment)?.extraCompletionFilter)
+        LookupElementsCollector({ CompletionBenchmarkSink.instance.onFlush(this) }, prefixMatcher, parameters, resultSet, createSorter(), (file as? KtCodeFragment)?.extraCompletionFilter)
    }

    protected val searchScope: GlobalSearchScope = getResolveScope(parameters.originalFile as KtFile)
@@ -204,9 +205,12 @@ abstract class CompletionSession(

    fun complete(): Boolean {
        return try {
-            _complete()
-        } finally {
-            CompletionBenchmarkSink.instance.onCompletionEnded(this)
+            _complete().also {
+                CompletionBenchmarkSink.instance.onCompletionEnded(this, false)
+            }
+        } catch (pce: ProcessCanceledException) {
+            CompletionBenchmarkSink.instance.onCompletionEnded(this, true)
+            throw pce
        }
    }

@@ -29,7 +29,7 @@ import org.jetbrains.kotlin.idea.core.completion.DeclarationLookupObject
 import java.util.*

 class LookupElementsCollector(
-        private val session: CompletionSession,
+        private val onFlush: () -> Unit,
        private val prefixMatcher: PrefixMatcher,
        private val completionParameters: CompletionParameters,
        resultSet: CompletionResultSet,
@@ -54,7 +54,7 @@ class LookupElementsCollector(

    fun flushToResultSet() {
        if (!elements.isEmpty()) {
-            CompletionBenchmarkSink.instance.onFirstFlush(session)
+            onFlush()

            resultSet.addAllElements(elements)
            elements.clear()
@@ -134,8 +134,17 @@
    <group id="KotlinInternalGroup" popup="true" text="Internal" icon="/general/balloonWarning.png"
           class="org.jetbrains.kotlin.idea.actions.internal.KotlinInternalActionGroup">

-      <action id="BenchmarkCompletionAction" class="org.jetbrains.kotlin.idea.actions.internal.BenchmarkCompletionAction"
-              text="Benchmark completion"/>
+      <group id="KotlinCompletionBenchmarkGroup" popup="true" text="Benchmark completion"
+             class="org.jetbrains.kotlin.idea.actions.internal.KotlinInternalActionGroup" >
+
+        <action id="TopLevelCompletionBenchmarkAction"
+                class="org.jetbrains.kotlin.idea.actions.internal.benchmark.TopLevelCompletionBenchmarkAction"
+                text="Top-level scenario"/>
+
+        <action id="LocalCompletionBenchmarkAction"
+                class="org.jetbrains.kotlin.idea.actions.internal.benchmark.LocalCompletionBenchmarkAction"
+                text="Local scenario"/>
+      </group>

      <action id="CheckComponentsUsageSearchAction" class="org.jetbrains.kotlin.idea.actions.internal.CheckComponentsUsageSearchAction"
              text="Check Component Functions Usage Search"/>
@@ -1,266 +0,0 @@
-/*
- * Copyright 2010-2017 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.jetbrains.kotlin.idea.actions.internal
-
-import com.intellij.codeInsight.navigation.NavigationUtil
-import com.intellij.openapi.actionSystem.AnAction
-import com.intellij.openapi.actionSystem.AnActionEvent
-import com.intellij.openapi.command.CommandProcessor
-import com.intellij.openapi.editor.EditorFactory
-import com.intellij.openapi.editor.ScrollType
-import com.intellij.openapi.project.Project
-import com.intellij.openapi.ui.DialogBuilder
-import com.intellij.openapi.ui.MessageType
-import com.intellij.openapi.ui.popup.JBPopupFactory
-import com.intellij.openapi.wm.WindowManager
-import com.intellij.psi.PsiDocumentManager
-import com.intellij.psi.PsiWhiteSpace
-import com.intellij.psi.search.DelegatingGlobalSearchScope
-import com.intellij.psi.search.GlobalSearchScope
-import com.intellij.structuralsearch.plugin.util.SmartPsiPointer
-import com.intellij.ui.components.JBLabel
-import com.intellij.ui.components.JBPanel
-import com.intellij.ui.components.JBTextField
-import com.intellij.uiDesigner.core.GridConstraints
-import com.intellij.uiDesigner.core.GridLayoutManager
-import kotlinx.coroutines.experimental.delay
-import kotlinx.coroutines.experimental.launch
-import org.jetbrains.kotlin.idea.caches.resolve.ModuleOrigin
-import org.jetbrains.kotlin.idea.caches.resolve.getNullableModuleInfo
-import org.jetbrains.kotlin.idea.completion.CompletionBenchmarkSink
-import org.jetbrains.kotlin.idea.core.moveCaret
-import org.jetbrains.kotlin.idea.core.util.EDT
-import org.jetbrains.kotlin.idea.refactoring.getLineCount
-import org.jetbrains.kotlin.idea.stubindex.KotlinExactPackagesIndex
-import org.jetbrains.kotlin.idea.util.application.runWriteAction
-import org.jetbrains.kotlin.psi.KtClassOrObject
-import org.jetbrains.kotlin.psi.KtFile
-import org.jetbrains.kotlin.psi.psiUtil.endOffset
-import org.jetbrains.kotlin.psi.psiUtil.getChildrenOfType
-import org.jetbrains.kotlin.psi.psiUtil.nextLeafs
-import java.awt.Robot
-import java.awt.event.KeyEvent
-import java.util.*
-import javax.swing.JFileChooser
-import kotlin.properties.Delegates
-
-class BenchmarkCompletionAction : AnAction() {
-
-    fun showPopup(project: Project, text: String) {
-        val statusBar = WindowManager.getInstance().getStatusBar(project)
-        JBPopupFactory.getInstance()
-                .createHtmlTextBalloonBuilder(text, MessageType.ERROR, null)
-                .setFadeoutTime(5000)
-                .createBalloon().showInCenterOf(statusBar.component)
-    }
-
-    override fun actionPerformed(e: AnActionEvent?) {
-        val project = e?.project!!
-
-
-        val scope = object : DelegatingGlobalSearchScope(GlobalSearchScope.allScope(project)) {
-            override fun isSearchOutsideRootModel(): Boolean {
-                return false
-            }
-        }
-        val ktFiles = mutableListOf<SmartPsiPointer>()
-        val exactPackageIndex = KotlinExactPackagesIndex.getInstance()
-        exactPackageIndex.processAllKeys(project) {
-            exactPackageIndex.get(it, project, scope).forEach {
-                val ptr = SmartPsiPointer(it)
-                ktFiles += ptr
-            }
-            true
-        }
-
-
-        ktFiles.removeAll {
-            val element = it.element as? KtFile ?: return@removeAll true
-            val moduleInfo = element.getNullableModuleInfo() ?: return@removeAll true
-            if (element.isCompiled || !element.isWritable || element.isScript) return@removeAll true
-            return@removeAll moduleInfo.moduleOrigin != ModuleOrigin.MODULE
-        }
-
-        data class Settings(val seed: Long, val attempts: Int, val lines: Int)
-
-        fun showSettingsDialog(): Settings? {
-            var cSeed: JBTextField by Delegates.notNull()
-            var cAttempts: JBTextField by Delegates.notNull()
-            var cLines: JBTextField by Delegates.notNull()
-            val dialogBuilder = DialogBuilder()
-
-
-            val jPanel = JBPanel<JBPanel<*>>(GridLayoutManager(3, 3)).apply {
-                this.add(JBLabel("Random seed: "), GridConstraints().apply { row = 0; column = 0 })
-                this.add(JBTextField().apply {
-                    cSeed = this
-                    text = "0"
-                    toolTipText = "Random seed"
-                }, GridConstraints().apply { row = 0; column = 1; fill = GridConstraints.FILL_HORIZONTAL })
-
-                this.add(JBLabel("Attempts: "), GridConstraints().apply { row = 1; column = 0 })
-                this.add(JBTextField().apply {
-                    cAttempts = this
-                    text = "5"
-                    toolTipText = "Number of files to work with"
-                }, GridConstraints().apply { row = 1; column = 1; fill = GridConstraints.FILL_HORIZONTAL })
-
-                this.add(JBLabel("File lines: "), GridConstraints().apply { row = 2; column = 0 })
-                this.add(JBTextField().apply {
-                    cLines = this
-                    text = "100"
-                    toolTipText = "File lines"
-                }, GridConstraints().apply { row = 2; column = 1; fill = GridConstraints.FILL_HORIZONTAL })
-            }
-            dialogBuilder.centerPanel(jPanel)
-            if (!dialogBuilder.showAndGet()) return null
-
-            return Settings(cSeed.text.toLong(),
-                            cAttempts.text.toInt(),
-                            cLines.text.toInt())
-        }
-
-        val settings = showSettingsDialog() ?: return
-
-        ktFiles.removeAll {
-            val element = it.element as KtFile
-            element.getLineCount() < settings.lines
-        }
-
-        if (ktFiles.size < settings.attempts) return showPopup(project, "Number of attempts > then files in project, ${ktFiles.size}")
-
-
-
-        val random = Random()
-        random.setSeed(settings.seed)
-
-        fun <T> List<T>.randomElement(): T? = if (this.isNotEmpty()) this[random.nextInt(this.size)] else null
-        fun <T> Array<T>.randomElement(): T? = if (this.isNotEmpty()) this[random.nextInt(this.size)] else null
-
-        val robot = Robot()
-
-
-        fun sendKey(keyCode: Int) {
-            robot.keyPress(keyCode)
-            robot.delay(100)
-            robot.keyRelease(keyCode)
-            robot.delay(100)
-        }
-
-        fun type(s: String) {
-            for (c in s.toCharArray()) {
-                val keyCode = KeyEvent.getExtendedKeyCodeForChar(c.toInt())
-                if (KeyEvent.CHAR_UNDEFINED == keyCode.toChar()) {
-                    throw RuntimeException("Key code not found for character '$c'")
-                }
-                if (c.isUpperCase()) {
-                    robot.keyPress(KeyEvent.VK_SHIFT)
-                    robot.delay(10)
-                }
-
-                sendKey(keyCode)
-                if (c.isUpperCase()) {
-                    robot.keyRelease(KeyEvent.VK_SHIFT)
-                    robot.delay(10)
-                }
-
-            }
-        }
-
-        data class Result(val lines: Int, val filePath: String, val first: Long, val full: Long)
-
-        val allResults = mutableListOf<Result>()
-
-        val benchmark = CompletionBenchmarkSink.enableAndGet()
-
-        val typeAtOffsetAndBenchmark: suspend (String, Int, KtFile) -> Unit = {
-            text: String, offset: Int, file: KtFile ->
-            NavigationUtil.openFileWithPsiElement(file.navigationElement, false, true)
-
-            val document = PsiDocumentManager.getInstance(project).getDocument(file)
-            val ourEditor = EditorFactory.getInstance().allEditors.find { it.document == document }
-
-            if (document != null && ourEditor != null) {
-
-                delay(500)
-
-                ourEditor.moveCaret(offset, scrollType = ScrollType.CENTER)
-
-                repeat(2) { sendKey(KeyEvent.VK_ENTER) }
-                delay(500)
-                val rangeMarker = document.createRangeMarker(offset, ourEditor.caretModel.offset)
-                sendKey(KeyEvent.VK_UP)
-
-                val t = text
-                type(t)
-
-                val results = benchmark.channel.receive()
-                println("fsize: ${file.getLineCount()}, ${file.virtualFile.path}")
-                println("first: ${results.firstFlush}, full: ${results.full}")
-
-                allResults += Result(file.getLineCount(), "${file.virtualFile.path}:${document.getLineNumber(offset)}", results.firstFlush, results.full)
-
-                CommandProcessor.getInstance().executeCommand(project, {
-                    runWriteAction {
-                        document.deleteString(rangeMarker.startOffset, rangeMarker.endOffset)
-                        PsiDocumentManager.getInstance(project).commitDocument(document)
-                    }
-                }, "ss", "ss")
-                delay(100)
-            }
-        }
-
-        launch(EDT) {
-            for (i in 0 until settings.attempts) {
-                val file = ktFiles.randomElement()!!.apply { ktFiles.remove(this) }.element as? KtFile ?: continue
-
-                run {
-                    val offset = (file.importList?.nextLeafs?.firstOrNull() as? PsiWhiteSpace)?.endOffset ?: 0
-                    typeAtOffsetAndBenchmark("fun Str", offset, file)
-
-                }
-                val ktClassOrObject = file.getChildrenOfType<KtClassOrObject>()
-                                              .filter { it.getBody() != null }
-                                              .randomElement() ?: continue
-
-                run {
-                    val body = ktClassOrObject.getBody()
-
-                    val offset = body!!.lBrace!!.endOffset
-                    typeAtOffsetAndBenchmark("fun Str", offset, file)
-                }
-
-
-            }
-            CompletionBenchmarkSink.disable()
-            val jfc = JFileChooser()
-            val result = jfc.showSaveDialog(null)
-            if (result == JFileChooser.APPROVE_OPTION) {
-                val file = jfc.selectedFile
-                file.writeText(buildString {
-                    allResults.joinTo(this, separator = "\n") { (l, f, ff, lf) -> "$f, $l, $ff, $lf" }
-                })
-            }
-            showPopup(project, "Done")
-        }
-
-    }
-
-    override fun update(e: AnActionEvent) {
-        e.presentation.isEnabledAndVisible = KotlinInternalMode.enabled
-    }
-}
@@ -0,0 +1,209 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.idea.actions.internal.benchmark
+
+import com.intellij.codeInsight.AutoPopupController
+import com.intellij.codeInsight.completion.CompletionType
+import com.intellij.codeInsight.navigation.NavigationUtil
+import com.intellij.openapi.actionSystem.AnAction
+import com.intellij.openapi.actionSystem.AnActionEvent
+import com.intellij.openapi.command.CommandProcessor
+import com.intellij.openapi.editor.EditorFactory
+import com.intellij.openapi.editor.ScrollType
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.ui.MessageType
+import com.intellij.openapi.ui.popup.JBPopupFactory
+import com.intellij.openapi.wm.WindowManager
+import com.intellij.psi.PsiDocumentManager
+import com.intellij.psi.search.DelegatingGlobalSearchScope
+import com.intellij.psi.search.GlobalSearchScope
+import kotlinx.coroutines.experimental.CancellationException
+import kotlinx.coroutines.experimental.delay
+import kotlinx.coroutines.experimental.launch
+import kotlinx.coroutines.experimental.withTimeout
+import org.jetbrains.kotlin.idea.actions.internal.KotlinInternalMode
+import org.jetbrains.kotlin.idea.caches.resolve.ModuleOrigin
+import org.jetbrains.kotlin.idea.caches.resolve.getNullableModuleInfo
+import org.jetbrains.kotlin.idea.completion.CompletionBenchmarkSink
+import org.jetbrains.kotlin.idea.core.moveCaret
+import org.jetbrains.kotlin.idea.core.util.EDT
+import org.jetbrains.kotlin.idea.refactoring.getLineCount
+import org.jetbrains.kotlin.idea.stubindex.KotlinExactPackagesIndex
+import org.jetbrains.kotlin.idea.util.application.runWriteAction
+import org.jetbrains.kotlin.psi.KtFile
+import java.util.*
+import javax.swing.JFileChooser
+
+abstract class AbstractCompletionBenchmarkAction : AnAction() {
+    override fun actionPerformed(e: AnActionEvent?) {
+        val project = e?.project ?: return
+
+        val benchmarkSink = CompletionBenchmarkSink.enableAndGet()
+        val scenario = createBenchmarkScenario(project, benchmarkSink) ?: return
+
+        launch(EDT) {
+            scenario.doBenchmark()
+            CompletionBenchmarkSink.disable()
+        }
+    }
+
+    internal abstract fun createBenchmarkScenario(project: Project, benchmarkSink: CompletionBenchmarkSink.Impl): AbstractCompletionBenchmarkScenario?
+
+    companion object {
+        fun showPopup(project: Project, text: String) {
+            val statusBar = WindowManager.getInstance().getStatusBar(project)
+            JBPopupFactory.getInstance()
+                    .createHtmlTextBalloonBuilder(text, MessageType.ERROR, null)
+                    .setFadeoutTime(5000)
+                    .createBalloon().showInCenterOf(statusBar.component)
+        }
+
+        internal fun <T> List<T>.randomElement(random: Random): T? = if (this.isNotEmpty()) this[random.nextInt(this.size)] else null
+        internal fun <T> Array<T>.randomElement(random: Random): T? = if (this.isNotEmpty()) this[random.nextInt(this.size)] else null
+
+    }
+
+    fun collectSuitableKotlinFiles(project: Project, filter: (KtFile) -> Boolean): MutableList<KtFile> {
+        val scope = object : DelegatingGlobalSearchScope(GlobalSearchScope.allScope(project)) {
+            override fun isSearchOutsideRootModel(): Boolean = false
+        }
+        val exactPackageIndex = KotlinExactPackagesIndex.getInstance()
+
+        fun KtFile.isUsableForBenchmark(): Boolean {
+            val moduleInfo = this.getNullableModuleInfo() ?: return false
+            if (this.isCompiled || !this.isWritable || this.isScript) return false
+            return moduleInfo.moduleOrigin == ModuleOrigin.MODULE
+        }
+
+        val ktFiles = mutableListOf<KtFile>()
+        exactPackageIndex.processAllKeys(project) {
+            exactPackageIndex.get(it, project, scope).forEach {
+                if (it.isUsableForBenchmark() && filter(it)) {
+                    ktFiles += it
+                }
+            }
+            true
+        }
+        return ktFiles
+    }
+
+    override fun update(e: AnActionEvent) {
+        e.presentation.isEnabledAndVisible = KotlinInternalMode.enabled
+    }
+}
+
+internal abstract class AbstractCompletionBenchmarkScenario(
+        val project: Project, val benchmarkSink: CompletionBenchmarkSink.Impl,
+        val random: Random = Random(), val timeout: Long = 15000) {
+
+
+    sealed class Result {
+        abstract fun toCSV(stringBuilder: StringBuilder)
+
+        open class SuccessResult(val lines: Int, val filePath: String, val first: Long, val full: Long) : Result() {
+            override fun toCSV(stringBuilder: StringBuilder): Unit = with(stringBuilder) {
+                append(filePath)
+                append(", ")
+                append(lines)
+                append(", ")
+                append(first)
+                append(", ")
+                append(full)
+            }
+        }
+
+        class ErrorResult(val filePath: String) : Result() {
+            override fun toCSV(stringBuilder: StringBuilder): Unit = with(stringBuilder) {
+                append(filePath)
+                append(", ")
+                append(", ")
+                append(", ")
+            }
+        }
+    }
+
+
+    protected suspend fun typeAtOffsetAndGetResult(text: String, offset: Int, file: KtFile): Result {
+        NavigationUtil.openFileWithPsiElement(file.navigationElement, false, true)
+
+        val document = PsiDocumentManager.getInstance(project).getDocument(file) ?:
+                       return Result.ErrorResult("${file.virtualFile.path}:O$offset")
+
+        val location = "${file.virtualFile.path}:${document.getLineNumber(offset)}"
+
+        val editor = EditorFactory.getInstance().getEditors(document, project).firstOrNull() ?: return Result.ErrorResult(location)
+
+
+        delay(500)
+
+        editor.moveCaret(offset, scrollType = ScrollType.CENTER)
+
+        delay(500)
+
+        CommandProcessor.getInstance().executeCommand(project, {
+            runWriteAction {
+                document.insertString(editor.caretModel.offset, "\n$text\n")
+                PsiDocumentManager.getInstance(project).commitDocument(document)
+            }
+            editor.moveCaret(editor.caretModel.offset + text.length + 1)
+            AutoPopupController.getInstance(project).scheduleAutoPopup(editor, CompletionType.BASIC, null)
+        }, "insertTextAndInvokeCompletion", "completionBenchmark")
+
+        val result = try {
+            withTimeout(timeout) { collectResult(file, location) }
+        }
+        catch (_: CancellationException) {
+            Result.ErrorResult(location)
+        }
+
+        CommandProcessor.getInstance().executeCommand(project, {
+            runWriteAction {
+                document.deleteString(offset, offset + text.length + 2)
+                PsiDocumentManager.getInstance(project).commitDocument(document)
+            }
+        }, "revertToOriginal", "completionBenchmark")
+
+        delay(100)
+        return result
+    }
+
+    protected suspend fun collectResult(file: KtFile, location: String): Result {
+        val results = benchmarkSink.channel.receive()
+        return Result.SuccessResult(file.getLineCount(), location, results.firstFlush, results.full)
+    }
+
+    protected fun saveResults(allResults: List<Result>) {
+        val jfc = JFileChooser()
+        val result = jfc.showSaveDialog(null)
+        if (result == JFileChooser.APPROVE_OPTION) {
+            val file = jfc.selectedFile
+            file.writeText(buildString {
+                appendln("n, file, lines, ff, full")
+                var i = 0
+                allResults.forEach {
+                    append(i++)
+                    append(", ")
+                    it.toCSV(this)
+                    appendln()
+                }
+            })
+        }
+        AbstractCompletionBenchmarkAction.showPopup(project, "Done")
+    }
+
+    abstract suspend fun doBenchmark()
+}
@@ -0,0 +1,126 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.idea.actions.internal.benchmark
+
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.ui.DialogBuilder
+import com.intellij.ui.components.JBLabel
+import com.intellij.ui.components.JBPanel
+import com.intellij.ui.components.JBTextField
+import com.intellij.uiDesigner.core.GridConstraints
+import com.intellij.uiDesigner.core.GridLayoutManager
+import org.jetbrains.kotlin.idea.actions.internal.benchmark.AbstractCompletionBenchmarkAction.Companion.randomElement
+import org.jetbrains.kotlin.idea.completion.CompletionBenchmarkSink
+import org.jetbrains.kotlin.idea.refactoring.getLineCount
+import org.jetbrains.kotlin.psi.KtFile
+import org.jetbrains.kotlin.psi.KtFunction
+import org.jetbrains.kotlin.psi.psiUtil.collectDescendantsOfType
+import org.jetbrains.kotlin.psi.psiUtil.endOffset
+import java.util.*
+import kotlin.properties.Delegates
+
+
+class LocalCompletionBenchmarkAction : AbstractCompletionBenchmarkAction() {
+
+    override fun createBenchmarkScenario(project: Project, benchmarkSink: CompletionBenchmarkSink.Impl): AbstractCompletionBenchmarkScenario? {
+
+        val settings = showSettingsDialog() ?: return null
+
+        val random = Random(settings.seed)
+
+        fun collectFiles(): List<KtFile>? {
+            val ktFiles = collectSuitableKotlinFiles(project) {
+                it.getLineCount() >= settings.lines
+            }
+
+            if (ktFiles.size < settings.files) {
+                showPopup(project, "Number of attempts > then files in project, ${ktFiles.size}")
+                return null
+            }
+
+            val result = mutableListOf<KtFile>()
+            repeat(settings.files) {
+                result += ktFiles.randomElement(random)!!.also { ktFiles.remove(it) }
+            }
+            return result
+        }
+
+        val ktFiles = collectFiles() ?: return null
+
+        return LocalCompletionBenchmarkScenario(ktFiles, settings, project, benchmarkSink, random)
+    }
+
+
+    data class Settings(val seed: Long, val lines: Int, val files: Int, val functions: Int)
+
+    private fun showSettingsDialog(): Settings? {
+        var cSeed: JBTextField by Delegates.notNull()
+        var cLines: JBTextField by Delegates.notNull()
+        var cFiles: JBTextField by Delegates.notNull()
+        var cFunctions: JBTextField by Delegates.notNull()
+        val dialogBuilder = DialogBuilder()
+
+
+        val jPanel = JBPanel<JBPanel<*>>(GridLayoutManager(4, 2)).apply {
+
+            var i = 0
+            fun addBoxWithLabel(tooltip: String, label: String = tooltip + ":", default: String): JBTextField {
+                this.add(JBLabel(label), GridConstraints().apply { row = i; column = 0 })
+                val textField = JBTextField().apply {
+                    text = default
+                    toolTipText = tooltip
+                }
+                this.add(textField, GridConstraints().apply { row = i++; column = 1; fill = GridConstraints.FILL_HORIZONTAL })
+                return textField
+            }
+            cSeed = addBoxWithLabel("Random seed", default = "0")
+            cFiles = addBoxWithLabel("Files to visit", default = "20")
+            cFunctions = addBoxWithLabel("Max functions to visit", default = "20")
+            cLines = addBoxWithLabel("File lines", default = "100")
+
+        }
+        dialogBuilder.centerPanel(jPanel)
+        if (!dialogBuilder.showAndGet()) return null
+
+        return Settings(cSeed.text.toLong(),
+                        cLines.text.toInt(),
+                        cFiles.text.toInt(),
+                        cFunctions.text.toInt())
+    }
+
+}
+
+internal class LocalCompletionBenchmarkScenario(
+        val files: List<KtFile>,
+        val settings: LocalCompletionBenchmarkAction.Settings,
+        project: Project, benchmarkSink: CompletionBenchmarkSink.Impl,
+        random: Random) : AbstractCompletionBenchmarkScenario(project, benchmarkSink, random) {
+    suspend override fun doBenchmark() {
+
+        val allResults = mutableListOf<Result>()
+        files.forEach { file ->
+            val functions = file.collectDescendantsOfType<KtFunction> { it.hasBlockBody() && it.bodyExpression != null }.toMutableList()
+            val randomFunctions = generateSequence { functions.randomElement(random).also { functions.remove(it) } }
+            randomFunctions.take(settings.functions).forEach {
+                function ->
+                val offset = function.bodyExpression!!.endOffset - 1
+                allResults += typeAtOffsetAndGetResult("listOf(1).", offset, file)
+            }
+        }
+        saveResults(allResults)
+    }
+}
@@ -0,0 +1,129 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.idea.actions.internal.benchmark
+
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.ui.DialogBuilder
+import com.intellij.psi.PsiWhiteSpace
+import com.intellij.ui.components.JBLabel
+import com.intellij.ui.components.JBPanel
+import com.intellij.ui.components.JBTextField
+import com.intellij.uiDesigner.core.GridConstraints
+import com.intellij.uiDesigner.core.GridLayoutManager
+import org.jetbrains.kotlin.idea.actions.internal.benchmark.AbstractCompletionBenchmarkAction.Companion.randomElement
+import org.jetbrains.kotlin.idea.completion.CompletionBenchmarkSink
+import org.jetbrains.kotlin.idea.refactoring.getLineCount
+import org.jetbrains.kotlin.psi.KtClassOrObject
+import org.jetbrains.kotlin.psi.KtFile
+import org.jetbrains.kotlin.psi.psiUtil.collectDescendantsOfType
+import org.jetbrains.kotlin.psi.psiUtil.endOffset
+import org.jetbrains.kotlin.psi.psiUtil.nextLeafs
+import java.util.*
+import kotlin.properties.Delegates
+
+
+class TopLevelCompletionBenchmarkAction : AbstractCompletionBenchmarkAction() {
+
+    override fun createBenchmarkScenario(project: Project, benchmarkSink: CompletionBenchmarkSink.Impl): AbstractCompletionBenchmarkScenario? {
+
+        val settings = showSettingsDialog() ?: return null
+
+        val random = Random(settings.seed)
+
+        fun collectFiles(): List<KtFile>? {
+            val ktFiles = collectSuitableKotlinFiles(project) {
+                it.getLineCount() >= settings.lines
+            }
+
+            if (ktFiles.size < settings.files) {
+                showPopup(project, "Number of attempts > then files in project, ${ktFiles.size}")
+                return null
+            }
+
+            val result = mutableListOf<KtFile>()
+            repeat(settings.files) {
+                result += ktFiles.randomElement(random)!!.also { ktFiles.remove(it) }
+            }
+            return result
+        }
+
+        val ktFiles = collectFiles() ?: return null
+
+        return TopLevelCompletionBenchmarkScenario(ktFiles, settings, project, benchmarkSink, random)
+    }
+
+
+    data class Settings(val seed: Long, val lines: Int, val files: Int)
+
+    private fun showSettingsDialog(): Settings? {
+        var cSeed: JBTextField by Delegates.notNull()
+        var cLines: JBTextField by Delegates.notNull()
+        var cFiles: JBTextField by Delegates.notNull()
+        val dialogBuilder = DialogBuilder()
+
+
+        val jPanel = JBPanel<JBPanel<*>>(GridLayoutManager(3, 2)).apply {
+
+            var i = 0
+            fun addBoxWithLabel(tooltip: String, label: String = tooltip + ":", default: String): JBTextField {
+                this.add(JBLabel(label), GridConstraints().apply { row = i; column = 0 })
+                val textField = JBTextField().apply {
+                    text = default
+                    toolTipText = tooltip
+                }
+                this.add(textField, GridConstraints().apply { row = i++; column = 1; fill = GridConstraints.FILL_HORIZONTAL })
+                return textField
+            }
+            cSeed = addBoxWithLabel("Random seed", default = "0")
+            cFiles = addBoxWithLabel("Files to visit", default = "20")
+            cLines = addBoxWithLabel("File lines", default = "100")
+        }
+        dialogBuilder.centerPanel(jPanel)
+        if (!dialogBuilder.showAndGet()) return null
+
+        return Settings(cSeed.text.toLong(),
+                        cLines.text.toInt(),
+                        cFiles.text.toInt())
+    }
+
+}
+
+internal class TopLevelCompletionBenchmarkScenario(
+        val files: List<KtFile>,
+        val settings: TopLevelCompletionBenchmarkAction.Settings,
+        project: Project, benchmarkSink: CompletionBenchmarkSink.Impl,
+        random: Random) : AbstractCompletionBenchmarkScenario(project, benchmarkSink, random) {
+    suspend override fun doBenchmark() {
+
+        val allResults = mutableListOf<Result>()
+        files.forEach { file ->
+
+            run {
+                val offset = (file.importList?.nextLeafs?.firstOrNull() as? PsiWhiteSpace)?.endOffset ?: 0
+                allResults += typeAtOffsetAndGetResult("fun Str", offset, file)
+            }
+
+            run {
+                val classes = file.collectDescendantsOfType<KtClassOrObject> { it.getBody() != null }
+                val body = classes.randomElement(random)?.getBody() ?: return@run
+                val offset = body.endOffset - 1
+                allResults += typeAtOffsetAndGetResult("fun Str", offset, file)
+            }
+        }
+        saveResults(allResults)
+    }
+}