From ccfcfd8721f6fc2aa3a95685963fc6206642dbe5 Mon Sep 17 00:00:00 2001 From: Simon Ogorodnik Date: Wed, 19 Jul 2017 04:43:49 +0300 Subject: [PATCH] Add benchmark for local completion and improve benchmarking --- .../completion/CompletionBenchmarkSink.kt | 50 ++-- .../idea/completion/CompletionSession.kt | 12 +- .../completion/LookupElementsCollector.kt | 4 +- idea/src/META-INF/plugin.xml | 13 +- .../internal/BenchmarkCompletionAction.kt | 266 ------------------ .../AbstractCompletionBenchmarkAction.kt | 209 ++++++++++++++ .../benchmark/LocalCompletionBenchmark.kt | 126 +++++++++ .../benchmark/TopLevelCompletionBenchmark.kt | 129 +++++++++ 8 files changed, 516 insertions(+), 293 deletions(-) delete mode 100644 idea/src/org/jetbrains/kotlin/idea/actions/internal/BenchmarkCompletionAction.kt create mode 100644 idea/src/org/jetbrains/kotlin/idea/actions/internal/benchmark/AbstractCompletionBenchmarkAction.kt create mode 100644 idea/src/org/jetbrains/kotlin/idea/actions/internal/benchmark/LocalCompletionBenchmark.kt create mode 100644 idea/src/org/jetbrains/kotlin/idea/actions/internal/benchmark/TopLevelCompletionBenchmark.kt diff --git a/idea/idea-completion/src/org/jetbrains/kotlin/idea/completion/CompletionBenchmarkSink.kt b/idea/idea-completion/src/org/jetbrains/kotlin/idea/completion/CompletionBenchmarkSink.kt index bfc19a47f87..599a8ed0932 100644 --- a/idea/idea-completion/src/org/jetbrains/kotlin/idea/completion/CompletionBenchmarkSink.kt +++ b/idea/idea-completion/src/org/jetbrains/kotlin/idea/completion/CompletionBenchmarkSink.kt @@ -17,12 +17,13 @@ package org.jetbrains.kotlin.idea.completion import kotlinx.coroutines.experimental.channels.ConflatedChannel +import java.lang.System.currentTimeMillis interface CompletionBenchmarkSink { fun onCompletionStarted(completionSession: CompletionSession) - fun onCompletionEnded(completionSession: CompletionSession) - fun onFirstFlush(completionSession: CompletionSession) + fun onCompletionEnded(completionSession: CompletionSession, canceled: Boolean) + fun onFlush(completionSession: CompletionSession) companion object { @@ -40,49 +41,60 @@ interface CompletionBenchmarkSink { private object Empty : CompletionBenchmarkSink { override fun onCompletionStarted(completionSession: CompletionSession) {} - override fun onCompletionEnded(completionSession: CompletionSession) {} + override fun onCompletionEnded(completionSession: CompletionSession, canceled: Boolean) {} - override fun onFirstFlush(completionSession: CompletionSession) {} + override fun onFlush(completionSession: CompletionSession) {} } class Impl : CompletionBenchmarkSink { private val pendingSessions = mutableListOf() - private lateinit var results: CompletionBenchmarkResults val channel = ConflatedChannel() + private val perSessionResults = LinkedHashMap() + private var start: Long = 0 + override fun onCompletionStarted(completionSession: CompletionSession) = synchronized(this) { if (pendingSessions.isEmpty()) - results = CompletionBenchmarkResults() + start = currentTimeMillis() pendingSessions += completionSession + perSessionResults[completionSession] = PerSessionResults() } - override fun onCompletionEnded(completionSession: CompletionSession) = synchronized(this) { + override fun onCompletionEnded(completionSession: CompletionSession, canceled: Boolean) = synchronized(this) { pendingSessions -= completionSession + perSessionResults[completionSession]?.onEnd(canceled) if (pendingSessions.isEmpty()) { - results.onEnd() - channel.offer(results) + val firstFlush = perSessionResults.values.filterNot { results -> results.canceled }.map { it.firstFlush }.min() ?: 0 + val full = perSessionResults.values.map { it.full }.max() ?: 0 + channel.offer(CompletionBenchmarkResults(firstFlush, full)) + reset() } } - override fun onFirstFlush(completionSession: CompletionSession) = synchronized(this) { - results.onFirstFlush() + override fun onFlush(completionSession: CompletionSession) = synchronized(this) { + perSessionResults[completionSession]?.onFirstFlush() + Unit } fun reset() = synchronized(this) { pendingSessions.clear() + perSessionResults.clear() } - class CompletionBenchmarkResults { - var start: Long = System.currentTimeMillis() - var firstFlush: Long = 0 - var full: Long = 0 + data class CompletionBenchmarkResults(var firstFlush: Long = 0, var full: Long = 0) + + private inner class PerSessionResults { + var firstFlush = 0L + var full = 0L + var canceled = false + fun onFirstFlush() { - if (firstFlush == 0L) - firstFlush = System.currentTimeMillis() - start + firstFlush = currentTimeMillis() - start } - fun onEnd() { - full = System.currentTimeMillis() - start + fun onEnd(canceled: Boolean) { + full = currentTimeMillis() - start + this.canceled = canceled } } } diff --git a/idea/idea-completion/src/org/jetbrains/kotlin/idea/completion/CompletionSession.kt b/idea/idea-completion/src/org/jetbrains/kotlin/idea/completion/CompletionSession.kt index 6d042b80fd3..323fbff6109 100644 --- a/idea/idea-completion/src/org/jetbrains/kotlin/idea/completion/CompletionSession.kt +++ b/idea/idea-completion/src/org/jetbrains/kotlin/idea/completion/CompletionSession.kt @@ -23,6 +23,7 @@ import com.intellij.codeInsight.completion.CompletionUtil import com.intellij.codeInsight.completion.impl.CamelHumpMatcher import com.intellij.codeInsight.completion.impl.RealPrefixMatchingWeigher import com.intellij.codeInsight.lookup.LookupElement +import com.intellij.openapi.progress.ProcessCanceledException import com.intellij.patterns.PatternCondition import com.intellij.patterns.StandardPatterns import com.intellij.psi.search.GlobalSearchScope @@ -144,7 +145,7 @@ abstract class CompletionSession( // LookupElementsCollector instantiation is deferred because virtual call to createSorter uses data from derived classes protected val collector: LookupElementsCollector by lazy(LazyThreadSafetyMode.NONE) { - LookupElementsCollector(this, prefixMatcher, parameters, resultSet, createSorter(), (file as? KtCodeFragment)?.extraCompletionFilter) + LookupElementsCollector({ CompletionBenchmarkSink.instance.onFlush(this) }, prefixMatcher, parameters, resultSet, createSorter(), (file as? KtCodeFragment)?.extraCompletionFilter) } protected val searchScope: GlobalSearchScope = getResolveScope(parameters.originalFile as KtFile) @@ -204,9 +205,12 @@ abstract class CompletionSession( fun complete(): Boolean { return try { - _complete() - } finally { - CompletionBenchmarkSink.instance.onCompletionEnded(this) + _complete().also { + CompletionBenchmarkSink.instance.onCompletionEnded(this, false) + } + } catch (pce: ProcessCanceledException) { + CompletionBenchmarkSink.instance.onCompletionEnded(this, true) + throw pce } } diff --git a/idea/idea-completion/src/org/jetbrains/kotlin/idea/completion/LookupElementsCollector.kt b/idea/idea-completion/src/org/jetbrains/kotlin/idea/completion/LookupElementsCollector.kt index 667366c6ddf..88fa35a63ee 100644 --- a/idea/idea-completion/src/org/jetbrains/kotlin/idea/completion/LookupElementsCollector.kt +++ b/idea/idea-completion/src/org/jetbrains/kotlin/idea/completion/LookupElementsCollector.kt @@ -29,7 +29,7 @@ import org.jetbrains.kotlin.idea.core.completion.DeclarationLookupObject import java.util.* class LookupElementsCollector( - private val session: CompletionSession, + private val onFlush: () -> Unit, private val prefixMatcher: PrefixMatcher, private val completionParameters: CompletionParameters, resultSet: CompletionResultSet, @@ -54,7 +54,7 @@ class LookupElementsCollector( fun flushToResultSet() { if (!elements.isEmpty()) { - CompletionBenchmarkSink.instance.onFirstFlush(session) + onFlush() resultSet.addAllElements(elements) elements.clear() diff --git a/idea/src/META-INF/plugin.xml b/idea/src/META-INF/plugin.xml index 7744a836b5b..dea2ec615ea 100644 --- a/idea/src/META-INF/plugin.xml +++ b/idea/src/META-INF/plugin.xml @@ -134,8 +134,17 @@ - + + + + + + diff --git a/idea/src/org/jetbrains/kotlin/idea/actions/internal/BenchmarkCompletionAction.kt b/idea/src/org/jetbrains/kotlin/idea/actions/internal/BenchmarkCompletionAction.kt deleted file mode 100644 index 50fe0644b03..00000000000 --- a/idea/src/org/jetbrains/kotlin/idea/actions/internal/BenchmarkCompletionAction.kt +++ /dev/null @@ -1,266 +0,0 @@ -/* - * Copyright 2010-2017 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.jetbrains.kotlin.idea.actions.internal - -import com.intellij.codeInsight.navigation.NavigationUtil -import com.intellij.openapi.actionSystem.AnAction -import com.intellij.openapi.actionSystem.AnActionEvent -import com.intellij.openapi.command.CommandProcessor -import com.intellij.openapi.editor.EditorFactory -import com.intellij.openapi.editor.ScrollType -import com.intellij.openapi.project.Project -import com.intellij.openapi.ui.DialogBuilder -import com.intellij.openapi.ui.MessageType -import com.intellij.openapi.ui.popup.JBPopupFactory -import com.intellij.openapi.wm.WindowManager -import com.intellij.psi.PsiDocumentManager -import com.intellij.psi.PsiWhiteSpace -import com.intellij.psi.search.DelegatingGlobalSearchScope -import com.intellij.psi.search.GlobalSearchScope -import com.intellij.structuralsearch.plugin.util.SmartPsiPointer -import com.intellij.ui.components.JBLabel -import com.intellij.ui.components.JBPanel -import com.intellij.ui.components.JBTextField -import com.intellij.uiDesigner.core.GridConstraints -import com.intellij.uiDesigner.core.GridLayoutManager -import kotlinx.coroutines.experimental.delay -import kotlinx.coroutines.experimental.launch -import org.jetbrains.kotlin.idea.caches.resolve.ModuleOrigin -import org.jetbrains.kotlin.idea.caches.resolve.getNullableModuleInfo -import org.jetbrains.kotlin.idea.completion.CompletionBenchmarkSink -import org.jetbrains.kotlin.idea.core.moveCaret -import org.jetbrains.kotlin.idea.core.util.EDT -import org.jetbrains.kotlin.idea.refactoring.getLineCount -import org.jetbrains.kotlin.idea.stubindex.KotlinExactPackagesIndex -import org.jetbrains.kotlin.idea.util.application.runWriteAction -import org.jetbrains.kotlin.psi.KtClassOrObject -import org.jetbrains.kotlin.psi.KtFile -import org.jetbrains.kotlin.psi.psiUtil.endOffset -import org.jetbrains.kotlin.psi.psiUtil.getChildrenOfType -import org.jetbrains.kotlin.psi.psiUtil.nextLeafs -import java.awt.Robot -import java.awt.event.KeyEvent -import java.util.* -import javax.swing.JFileChooser -import kotlin.properties.Delegates - -class BenchmarkCompletionAction : AnAction() { - - fun showPopup(project: Project, text: String) { - val statusBar = WindowManager.getInstance().getStatusBar(project) - JBPopupFactory.getInstance() - .createHtmlTextBalloonBuilder(text, MessageType.ERROR, null) - .setFadeoutTime(5000) - .createBalloon().showInCenterOf(statusBar.component) - } - - override fun actionPerformed(e: AnActionEvent?) { - val project = e?.project!! - - - val scope = object : DelegatingGlobalSearchScope(GlobalSearchScope.allScope(project)) { - override fun isSearchOutsideRootModel(): Boolean { - return false - } - } - val ktFiles = mutableListOf() - val exactPackageIndex = KotlinExactPackagesIndex.getInstance() - exactPackageIndex.processAllKeys(project) { - exactPackageIndex.get(it, project, scope).forEach { - val ptr = SmartPsiPointer(it) - ktFiles += ptr - } - true - } - - - ktFiles.removeAll { - val element = it.element as? KtFile ?: return@removeAll true - val moduleInfo = element.getNullableModuleInfo() ?: return@removeAll true - if (element.isCompiled || !element.isWritable || element.isScript) return@removeAll true - return@removeAll moduleInfo.moduleOrigin != ModuleOrigin.MODULE - } - - data class Settings(val seed: Long, val attempts: Int, val lines: Int) - - fun showSettingsDialog(): Settings? { - var cSeed: JBTextField by Delegates.notNull() - var cAttempts: JBTextField by Delegates.notNull() - var cLines: JBTextField by Delegates.notNull() - val dialogBuilder = DialogBuilder() - - - val jPanel = JBPanel>(GridLayoutManager(3, 3)).apply { - this.add(JBLabel("Random seed: "), GridConstraints().apply { row = 0; column = 0 }) - this.add(JBTextField().apply { - cSeed = this - text = "0" - toolTipText = "Random seed" - }, GridConstraints().apply { row = 0; column = 1; fill = GridConstraints.FILL_HORIZONTAL }) - - this.add(JBLabel("Attempts: "), GridConstraints().apply { row = 1; column = 0 }) - this.add(JBTextField().apply { - cAttempts = this - text = "5" - toolTipText = "Number of files to work with" - }, GridConstraints().apply { row = 1; column = 1; fill = GridConstraints.FILL_HORIZONTAL }) - - this.add(JBLabel("File lines: "), GridConstraints().apply { row = 2; column = 0 }) - this.add(JBTextField().apply { - cLines = this - text = "100" - toolTipText = "File lines" - }, GridConstraints().apply { row = 2; column = 1; fill = GridConstraints.FILL_HORIZONTAL }) - } - dialogBuilder.centerPanel(jPanel) - if (!dialogBuilder.showAndGet()) return null - - return Settings(cSeed.text.toLong(), - cAttempts.text.toInt(), - cLines.text.toInt()) - } - - val settings = showSettingsDialog() ?: return - - ktFiles.removeAll { - val element = it.element as KtFile - element.getLineCount() < settings.lines - } - - if (ktFiles.size < settings.attempts) return showPopup(project, "Number of attempts > then files in project, ${ktFiles.size}") - - - - val random = Random() - random.setSeed(settings.seed) - - fun List.randomElement(): T? = if (this.isNotEmpty()) this[random.nextInt(this.size)] else null - fun Array.randomElement(): T? = if (this.isNotEmpty()) this[random.nextInt(this.size)] else null - - val robot = Robot() - - - fun sendKey(keyCode: Int) { - robot.keyPress(keyCode) - robot.delay(100) - robot.keyRelease(keyCode) - robot.delay(100) - } - - fun type(s: String) { - for (c in s.toCharArray()) { - val keyCode = KeyEvent.getExtendedKeyCodeForChar(c.toInt()) - if (KeyEvent.CHAR_UNDEFINED == keyCode.toChar()) { - throw RuntimeException("Key code not found for character '$c'") - } - if (c.isUpperCase()) { - robot.keyPress(KeyEvent.VK_SHIFT) - robot.delay(10) - } - - sendKey(keyCode) - if (c.isUpperCase()) { - robot.keyRelease(KeyEvent.VK_SHIFT) - robot.delay(10) - } - - } - } - - data class Result(val lines: Int, val filePath: String, val first: Long, val full: Long) - - val allResults = mutableListOf() - - val benchmark = CompletionBenchmarkSink.enableAndGet() - - val typeAtOffsetAndBenchmark: suspend (String, Int, KtFile) -> Unit = { - text: String, offset: Int, file: KtFile -> - NavigationUtil.openFileWithPsiElement(file.navigationElement, false, true) - - val document = PsiDocumentManager.getInstance(project).getDocument(file) - val ourEditor = EditorFactory.getInstance().allEditors.find { it.document == document } - - if (document != null && ourEditor != null) { - - delay(500) - - ourEditor.moveCaret(offset, scrollType = ScrollType.CENTER) - - repeat(2) { sendKey(KeyEvent.VK_ENTER) } - delay(500) - val rangeMarker = document.createRangeMarker(offset, ourEditor.caretModel.offset) - sendKey(KeyEvent.VK_UP) - - val t = text - type(t) - - val results = benchmark.channel.receive() - println("fsize: ${file.getLineCount()}, ${file.virtualFile.path}") - println("first: ${results.firstFlush}, full: ${results.full}") - - allResults += Result(file.getLineCount(), "${file.virtualFile.path}:${document.getLineNumber(offset)}", results.firstFlush, results.full) - - CommandProcessor.getInstance().executeCommand(project, { - runWriteAction { - document.deleteString(rangeMarker.startOffset, rangeMarker.endOffset) - PsiDocumentManager.getInstance(project).commitDocument(document) - } - }, "ss", "ss") - delay(100) - } - } - - launch(EDT) { - for (i in 0 until settings.attempts) { - val file = ktFiles.randomElement()!!.apply { ktFiles.remove(this) }.element as? KtFile ?: continue - - run { - val offset = (file.importList?.nextLeafs?.firstOrNull() as? PsiWhiteSpace)?.endOffset ?: 0 - typeAtOffsetAndBenchmark("fun Str", offset, file) - - } - val ktClassOrObject = file.getChildrenOfType() - .filter { it.getBody() != null } - .randomElement() ?: continue - - run { - val body = ktClassOrObject.getBody() - - val offset = body!!.lBrace!!.endOffset - typeAtOffsetAndBenchmark("fun Str", offset, file) - } - - - } - CompletionBenchmarkSink.disable() - val jfc = JFileChooser() - val result = jfc.showSaveDialog(null) - if (result == JFileChooser.APPROVE_OPTION) { - val file = jfc.selectedFile - file.writeText(buildString { - allResults.joinTo(this, separator = "\n") { (l, f, ff, lf) -> "$f, $l, $ff, $lf" } - }) - } - showPopup(project, "Done") - } - - } - - override fun update(e: AnActionEvent) { - e.presentation.isEnabledAndVisible = KotlinInternalMode.enabled - } -} diff --git a/idea/src/org/jetbrains/kotlin/idea/actions/internal/benchmark/AbstractCompletionBenchmarkAction.kt b/idea/src/org/jetbrains/kotlin/idea/actions/internal/benchmark/AbstractCompletionBenchmarkAction.kt new file mode 100644 index 00000000000..308e42566c8 --- /dev/null +++ b/idea/src/org/jetbrains/kotlin/idea/actions/internal/benchmark/AbstractCompletionBenchmarkAction.kt @@ -0,0 +1,209 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.idea.actions.internal.benchmark + +import com.intellij.codeInsight.AutoPopupController +import com.intellij.codeInsight.completion.CompletionType +import com.intellij.codeInsight.navigation.NavigationUtil +import com.intellij.openapi.actionSystem.AnAction +import com.intellij.openapi.actionSystem.AnActionEvent +import com.intellij.openapi.command.CommandProcessor +import com.intellij.openapi.editor.EditorFactory +import com.intellij.openapi.editor.ScrollType +import com.intellij.openapi.project.Project +import com.intellij.openapi.ui.MessageType +import com.intellij.openapi.ui.popup.JBPopupFactory +import com.intellij.openapi.wm.WindowManager +import com.intellij.psi.PsiDocumentManager +import com.intellij.psi.search.DelegatingGlobalSearchScope +import com.intellij.psi.search.GlobalSearchScope +import kotlinx.coroutines.experimental.CancellationException +import kotlinx.coroutines.experimental.delay +import kotlinx.coroutines.experimental.launch +import kotlinx.coroutines.experimental.withTimeout +import org.jetbrains.kotlin.idea.actions.internal.KotlinInternalMode +import org.jetbrains.kotlin.idea.caches.resolve.ModuleOrigin +import org.jetbrains.kotlin.idea.caches.resolve.getNullableModuleInfo +import org.jetbrains.kotlin.idea.completion.CompletionBenchmarkSink +import org.jetbrains.kotlin.idea.core.moveCaret +import org.jetbrains.kotlin.idea.core.util.EDT +import org.jetbrains.kotlin.idea.refactoring.getLineCount +import org.jetbrains.kotlin.idea.stubindex.KotlinExactPackagesIndex +import org.jetbrains.kotlin.idea.util.application.runWriteAction +import org.jetbrains.kotlin.psi.KtFile +import java.util.* +import javax.swing.JFileChooser + +abstract class AbstractCompletionBenchmarkAction : AnAction() { + override fun actionPerformed(e: AnActionEvent?) { + val project = e?.project ?: return + + val benchmarkSink = CompletionBenchmarkSink.enableAndGet() + val scenario = createBenchmarkScenario(project, benchmarkSink) ?: return + + launch(EDT) { + scenario.doBenchmark() + CompletionBenchmarkSink.disable() + } + } + + internal abstract fun createBenchmarkScenario(project: Project, benchmarkSink: CompletionBenchmarkSink.Impl): AbstractCompletionBenchmarkScenario? + + companion object { + fun showPopup(project: Project, text: String) { + val statusBar = WindowManager.getInstance().getStatusBar(project) + JBPopupFactory.getInstance() + .createHtmlTextBalloonBuilder(text, MessageType.ERROR, null) + .setFadeoutTime(5000) + .createBalloon().showInCenterOf(statusBar.component) + } + + internal fun List.randomElement(random: Random): T? = if (this.isNotEmpty()) this[random.nextInt(this.size)] else null + internal fun Array.randomElement(random: Random): T? = if (this.isNotEmpty()) this[random.nextInt(this.size)] else null + + } + + fun collectSuitableKotlinFiles(project: Project, filter: (KtFile) -> Boolean): MutableList { + val scope = object : DelegatingGlobalSearchScope(GlobalSearchScope.allScope(project)) { + override fun isSearchOutsideRootModel(): Boolean = false + } + val exactPackageIndex = KotlinExactPackagesIndex.getInstance() + + fun KtFile.isUsableForBenchmark(): Boolean { + val moduleInfo = this.getNullableModuleInfo() ?: return false + if (this.isCompiled || !this.isWritable || this.isScript) return false + return moduleInfo.moduleOrigin == ModuleOrigin.MODULE + } + + val ktFiles = mutableListOf() + exactPackageIndex.processAllKeys(project) { + exactPackageIndex.get(it, project, scope).forEach { + if (it.isUsableForBenchmark() && filter(it)) { + ktFiles += it + } + } + true + } + return ktFiles + } + + override fun update(e: AnActionEvent) { + e.presentation.isEnabledAndVisible = KotlinInternalMode.enabled + } +} + +internal abstract class AbstractCompletionBenchmarkScenario( + val project: Project, val benchmarkSink: CompletionBenchmarkSink.Impl, + val random: Random = Random(), val timeout: Long = 15000) { + + + sealed class Result { + abstract fun toCSV(stringBuilder: StringBuilder) + + open class SuccessResult(val lines: Int, val filePath: String, val first: Long, val full: Long) : Result() { + override fun toCSV(stringBuilder: StringBuilder): Unit = with(stringBuilder) { + append(filePath) + append(", ") + append(lines) + append(", ") + append(first) + append(", ") + append(full) + } + } + + class ErrorResult(val filePath: String) : Result() { + override fun toCSV(stringBuilder: StringBuilder): Unit = with(stringBuilder) { + append(filePath) + append(", ") + append(", ") + append(", ") + } + } + } + + + protected suspend fun typeAtOffsetAndGetResult(text: String, offset: Int, file: KtFile): Result { + NavigationUtil.openFileWithPsiElement(file.navigationElement, false, true) + + val document = PsiDocumentManager.getInstance(project).getDocument(file) ?: + return Result.ErrorResult("${file.virtualFile.path}:O$offset") + + val location = "${file.virtualFile.path}:${document.getLineNumber(offset)}" + + val editor = EditorFactory.getInstance().getEditors(document, project).firstOrNull() ?: return Result.ErrorResult(location) + + + delay(500) + + editor.moveCaret(offset, scrollType = ScrollType.CENTER) + + delay(500) + + CommandProcessor.getInstance().executeCommand(project, { + runWriteAction { + document.insertString(editor.caretModel.offset, "\n$text\n") + PsiDocumentManager.getInstance(project).commitDocument(document) + } + editor.moveCaret(editor.caretModel.offset + text.length + 1) + AutoPopupController.getInstance(project).scheduleAutoPopup(editor, CompletionType.BASIC, null) + }, "insertTextAndInvokeCompletion", "completionBenchmark") + + val result = try { + withTimeout(timeout) { collectResult(file, location) } + } + catch (_: CancellationException) { + Result.ErrorResult(location) + } + + CommandProcessor.getInstance().executeCommand(project, { + runWriteAction { + document.deleteString(offset, offset + text.length + 2) + PsiDocumentManager.getInstance(project).commitDocument(document) + } + }, "revertToOriginal", "completionBenchmark") + + delay(100) + return result + } + + protected suspend fun collectResult(file: KtFile, location: String): Result { + val results = benchmarkSink.channel.receive() + return Result.SuccessResult(file.getLineCount(), location, results.firstFlush, results.full) + } + + protected fun saveResults(allResults: List) { + val jfc = JFileChooser() + val result = jfc.showSaveDialog(null) + if (result == JFileChooser.APPROVE_OPTION) { + val file = jfc.selectedFile + file.writeText(buildString { + appendln("n, file, lines, ff, full") + var i = 0 + allResults.forEach { + append(i++) + append(", ") + it.toCSV(this) + appendln() + } + }) + } + AbstractCompletionBenchmarkAction.showPopup(project, "Done") + } + + abstract suspend fun doBenchmark() +} \ No newline at end of file diff --git a/idea/src/org/jetbrains/kotlin/idea/actions/internal/benchmark/LocalCompletionBenchmark.kt b/idea/src/org/jetbrains/kotlin/idea/actions/internal/benchmark/LocalCompletionBenchmark.kt new file mode 100644 index 00000000000..32ed8aaa523 --- /dev/null +++ b/idea/src/org/jetbrains/kotlin/idea/actions/internal/benchmark/LocalCompletionBenchmark.kt @@ -0,0 +1,126 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.idea.actions.internal.benchmark + +import com.intellij.openapi.project.Project +import com.intellij.openapi.ui.DialogBuilder +import com.intellij.ui.components.JBLabel +import com.intellij.ui.components.JBPanel +import com.intellij.ui.components.JBTextField +import com.intellij.uiDesigner.core.GridConstraints +import com.intellij.uiDesigner.core.GridLayoutManager +import org.jetbrains.kotlin.idea.actions.internal.benchmark.AbstractCompletionBenchmarkAction.Companion.randomElement +import org.jetbrains.kotlin.idea.completion.CompletionBenchmarkSink +import org.jetbrains.kotlin.idea.refactoring.getLineCount +import org.jetbrains.kotlin.psi.KtFile +import org.jetbrains.kotlin.psi.KtFunction +import org.jetbrains.kotlin.psi.psiUtil.collectDescendantsOfType +import org.jetbrains.kotlin.psi.psiUtil.endOffset +import java.util.* +import kotlin.properties.Delegates + + +class LocalCompletionBenchmarkAction : AbstractCompletionBenchmarkAction() { + + override fun createBenchmarkScenario(project: Project, benchmarkSink: CompletionBenchmarkSink.Impl): AbstractCompletionBenchmarkScenario? { + + val settings = showSettingsDialog() ?: return null + + val random = Random(settings.seed) + + fun collectFiles(): List? { + val ktFiles = collectSuitableKotlinFiles(project) { + it.getLineCount() >= settings.lines + } + + if (ktFiles.size < settings.files) { + showPopup(project, "Number of attempts > then files in project, ${ktFiles.size}") + return null + } + + val result = mutableListOf() + repeat(settings.files) { + result += ktFiles.randomElement(random)!!.also { ktFiles.remove(it) } + } + return result + } + + val ktFiles = collectFiles() ?: return null + + return LocalCompletionBenchmarkScenario(ktFiles, settings, project, benchmarkSink, random) + } + + + data class Settings(val seed: Long, val lines: Int, val files: Int, val functions: Int) + + private fun showSettingsDialog(): Settings? { + var cSeed: JBTextField by Delegates.notNull() + var cLines: JBTextField by Delegates.notNull() + var cFiles: JBTextField by Delegates.notNull() + var cFunctions: JBTextField by Delegates.notNull() + val dialogBuilder = DialogBuilder() + + + val jPanel = JBPanel>(GridLayoutManager(4, 2)).apply { + + var i = 0 + fun addBoxWithLabel(tooltip: String, label: String = tooltip + ":", default: String): JBTextField { + this.add(JBLabel(label), GridConstraints().apply { row = i; column = 0 }) + val textField = JBTextField().apply { + text = default + toolTipText = tooltip + } + this.add(textField, GridConstraints().apply { row = i++; column = 1; fill = GridConstraints.FILL_HORIZONTAL }) + return textField + } + cSeed = addBoxWithLabel("Random seed", default = "0") + cFiles = addBoxWithLabel("Files to visit", default = "20") + cFunctions = addBoxWithLabel("Max functions to visit", default = "20") + cLines = addBoxWithLabel("File lines", default = "100") + + } + dialogBuilder.centerPanel(jPanel) + if (!dialogBuilder.showAndGet()) return null + + return Settings(cSeed.text.toLong(), + cLines.text.toInt(), + cFiles.text.toInt(), + cFunctions.text.toInt()) + } + +} + +internal class LocalCompletionBenchmarkScenario( + val files: List, + val settings: LocalCompletionBenchmarkAction.Settings, + project: Project, benchmarkSink: CompletionBenchmarkSink.Impl, + random: Random) : AbstractCompletionBenchmarkScenario(project, benchmarkSink, random) { + suspend override fun doBenchmark() { + + val allResults = mutableListOf() + files.forEach { file -> + val functions = file.collectDescendantsOfType { it.hasBlockBody() && it.bodyExpression != null }.toMutableList() + val randomFunctions = generateSequence { functions.randomElement(random).also { functions.remove(it) } } + randomFunctions.take(settings.functions).forEach { + function -> + val offset = function.bodyExpression!!.endOffset - 1 + allResults += typeAtOffsetAndGetResult("listOf(1).", offset, file) + } + } + saveResults(allResults) + } +} \ No newline at end of file diff --git a/idea/src/org/jetbrains/kotlin/idea/actions/internal/benchmark/TopLevelCompletionBenchmark.kt b/idea/src/org/jetbrains/kotlin/idea/actions/internal/benchmark/TopLevelCompletionBenchmark.kt new file mode 100644 index 00000000000..43f1bea6f04 --- /dev/null +++ b/idea/src/org/jetbrains/kotlin/idea/actions/internal/benchmark/TopLevelCompletionBenchmark.kt @@ -0,0 +1,129 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.idea.actions.internal.benchmark + +import com.intellij.openapi.project.Project +import com.intellij.openapi.ui.DialogBuilder +import com.intellij.psi.PsiWhiteSpace +import com.intellij.ui.components.JBLabel +import com.intellij.ui.components.JBPanel +import com.intellij.ui.components.JBTextField +import com.intellij.uiDesigner.core.GridConstraints +import com.intellij.uiDesigner.core.GridLayoutManager +import org.jetbrains.kotlin.idea.actions.internal.benchmark.AbstractCompletionBenchmarkAction.Companion.randomElement +import org.jetbrains.kotlin.idea.completion.CompletionBenchmarkSink +import org.jetbrains.kotlin.idea.refactoring.getLineCount +import org.jetbrains.kotlin.psi.KtClassOrObject +import org.jetbrains.kotlin.psi.KtFile +import org.jetbrains.kotlin.psi.psiUtil.collectDescendantsOfType +import org.jetbrains.kotlin.psi.psiUtil.endOffset +import org.jetbrains.kotlin.psi.psiUtil.nextLeafs +import java.util.* +import kotlin.properties.Delegates + + +class TopLevelCompletionBenchmarkAction : AbstractCompletionBenchmarkAction() { + + override fun createBenchmarkScenario(project: Project, benchmarkSink: CompletionBenchmarkSink.Impl): AbstractCompletionBenchmarkScenario? { + + val settings = showSettingsDialog() ?: return null + + val random = Random(settings.seed) + + fun collectFiles(): List? { + val ktFiles = collectSuitableKotlinFiles(project) { + it.getLineCount() >= settings.lines + } + + if (ktFiles.size < settings.files) { + showPopup(project, "Number of attempts > then files in project, ${ktFiles.size}") + return null + } + + val result = mutableListOf() + repeat(settings.files) { + result += ktFiles.randomElement(random)!!.also { ktFiles.remove(it) } + } + return result + } + + val ktFiles = collectFiles() ?: return null + + return TopLevelCompletionBenchmarkScenario(ktFiles, settings, project, benchmarkSink, random) + } + + + data class Settings(val seed: Long, val lines: Int, val files: Int) + + private fun showSettingsDialog(): Settings? { + var cSeed: JBTextField by Delegates.notNull() + var cLines: JBTextField by Delegates.notNull() + var cFiles: JBTextField by Delegates.notNull() + val dialogBuilder = DialogBuilder() + + + val jPanel = JBPanel>(GridLayoutManager(3, 2)).apply { + + var i = 0 + fun addBoxWithLabel(tooltip: String, label: String = tooltip + ":", default: String): JBTextField { + this.add(JBLabel(label), GridConstraints().apply { row = i; column = 0 }) + val textField = JBTextField().apply { + text = default + toolTipText = tooltip + } + this.add(textField, GridConstraints().apply { row = i++; column = 1; fill = GridConstraints.FILL_HORIZONTAL }) + return textField + } + cSeed = addBoxWithLabel("Random seed", default = "0") + cFiles = addBoxWithLabel("Files to visit", default = "20") + cLines = addBoxWithLabel("File lines", default = "100") + } + dialogBuilder.centerPanel(jPanel) + if (!dialogBuilder.showAndGet()) return null + + return Settings(cSeed.text.toLong(), + cLines.text.toInt(), + cFiles.text.toInt()) + } + +} + +internal class TopLevelCompletionBenchmarkScenario( + val files: List, + val settings: TopLevelCompletionBenchmarkAction.Settings, + project: Project, benchmarkSink: CompletionBenchmarkSink.Impl, + random: Random) : AbstractCompletionBenchmarkScenario(project, benchmarkSink, random) { + suspend override fun doBenchmark() { + + val allResults = mutableListOf() + files.forEach { file -> + + run { + val offset = (file.importList?.nextLeafs?.firstOrNull() as? PsiWhiteSpace)?.endOffset ?: 0 + allResults += typeAtOffsetAndGetResult("fun Str", offset, file) + } + + run { + val classes = file.collectDescendantsOfType { it.getBody() != null } + val body = classes.randomElement(random)?.getBody() ?: return@run + val offset = body.endOffset - 1 + allResults += typeAtOffsetAndGetResult("fun Str", offset, file) + } + } + saveResults(allResults) + } +} \ No newline at end of file