Added gradle.kts and annotator gradle.kts autocompletion performance benchmark

idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceProjectsTest.kt

@@ -5,12 +5,13 @@
 
 package org.jetbrains.kotlin.idea.perf
 
+import com.intellij.codeInsight.completion.CompletionType
 import com.intellij.codeInsight.daemon.*
 import com.intellij.codeInsight.daemon.impl.DaemonCodeAnalyzerImpl
 import com.intellij.codeInsight.daemon.impl.HighlightInfo
 import com.intellij.codeInsight.daemon.impl.IdentifierHighlighterPassFactory
+import com.intellij.codeInsight.lookup.LookupElement
 import com.intellij.ide.highlighter.ModuleFileType
-import com.intellij.ide.impl.ProjectUtil
 import com.intellij.ide.startup.impl.StartupManagerImpl
 import com.intellij.idea.IdeaTestApplication
 import com.intellij.lang.ExternalAnnotatorsFilter
@@ -19,17 +20,22 @@ import com.intellij.lang.StdLanguages
 import com.intellij.lang.injection.InjectedLanguageManager
 import com.intellij.lang.java.JavaLanguage
 import com.intellij.openapi.application.ApplicationManager
+import com.intellij.openapi.application.runWriteAction
 import com.intellij.openapi.editor.Document
 import com.intellij.openapi.editor.EditorFactory
+import com.intellij.openapi.externalSystem.importing.ImportSpecBuilder
+import com.intellij.openapi.externalSystem.service.execution.ProgressExecutionMode
+import com.intellij.openapi.externalSystem.service.project.manage.ExternalProjectsManagerImpl
+import com.intellij.openapi.externalSystem.util.ExternalSystemUtil
 import com.intellij.openapi.fileEditor.FileDocumentManager
 import com.intellij.openapi.fileEditor.FileEditorManager
 import com.intellij.openapi.fileEditor.impl.FileEditorManagerImpl
-import com.intellij.openapi.module.Module
 import com.intellij.openapi.module.ModuleManager
 import com.intellij.openapi.module.ModuleTypeId
 import com.intellij.openapi.project.DumbService
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.project.ex.ProjectManagerEx
+import com.intellij.openapi.project.guessProjectDir
 import com.intellij.openapi.project.impl.ProjectImpl
 import com.intellij.openapi.projectRoots.JavaSdk
 import com.intellij.openapi.projectRoots.ProjectJdkTable
@@ -38,34 +44,39 @@ import com.intellij.openapi.projectRoots.impl.JavaAwareProjectJdkTableImpl
 import com.intellij.openapi.roots.FileIndexFacade
 import com.intellij.openapi.roots.ProjectRootManager
 import com.intellij.openapi.startup.StartupManager
-import com.intellij.openapi.util.Computable
-import com.intellij.openapi.util.Disposer
 import com.intellij.openapi.vcs.changes.ChangeListManager
 import com.intellij.openapi.vcs.changes.ChangeListManagerImpl
 import com.intellij.openapi.vfs.LocalFileSystem
+import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.openapi.vfs.VirtualFileManager
 import com.intellij.psi.PsiDocumentManager
 import com.intellij.psi.PsiFile
 import com.intellij.psi.PsiManager
 import com.intellij.psi.impl.search.IndexPatternBuilder
+import com.intellij.psi.impl.source.resolve.reference.ReferenceProvidersRegistry
+import com.intellij.psi.search.FilenameIndex
+import com.intellij.psi.search.GlobalSearchScope
 import com.intellij.psi.xml.XmlFileNSInfoProvider
 import com.intellij.testFramework.*
+import com.intellij.testFramework.fixtures.EditorTestFixture
 import com.intellij.testFramework.fixtures.impl.CodeInsightTestFixtureImpl
-import com.intellij.testFramework.propertyBased.MadTestingUtil
 import com.intellij.util.ArrayUtilRt
 import com.intellij.util.ThrowableRunnable
 import com.intellij.util.indexing.UnindexedFilesUpdater
 import com.intellij.util.io.exists
 import com.intellij.xml.XmlSchemaProvider
 import org.jetbrains.kotlin.idea.KotlinLanguage
+import org.jetbrains.kotlin.idea.configuration.getModulesWithKotlinFiles
 import org.jetbrains.kotlin.idea.core.script.ScriptDefinitionsManager
 import org.jetbrains.kotlin.idea.core.script.ScriptDependenciesManager
 import org.jetbrains.kotlin.idea.core.script.settings.KotlinScriptingSettings
 import org.jetbrains.kotlin.idea.core.util.toPsiFile
 import org.jetbrains.kotlin.idea.framework.KotlinSdkType
+import org.jetbrains.kotlin.idea.project.getAndCacheLanguageLevelByDependencies
 import org.jetbrains.kotlin.idea.test.ConfigLibraryUtil
 import org.jetbrains.kotlin.idea.test.invalidateLibraryCache
 import org.jetbrains.plugins.gradle.service.project.GradleProjectOpenProcessor
+import org.jetbrains.plugins.gradle.util.GradleConstants
 import java.io.File
 import java.nio.file.Paths
 
@@ -84,7 +95,7 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
         super.setUp()
 
         myApplication = IdeaTestApplication.getInstance()
-        ApplicationManager.getApplication().runWriteAction {
+        runWriteAction {
             val jdkTableImpl = JavaAwareProjectJdkTableImpl.getInstanceEx()
             val homePath = if (jdkTableImpl.internalJdk.homeDirectory!!.name == "jre") {
                 jdkTableImpl.internalJdk.homeDirectory!!.parent.path
@@ -104,7 +115,7 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
     }
 
     protected fun warmUpProject(stats: Stats) {
-        val project = innerPerfOpenProject("helloKotlin", stats, "warm-up")
+        val project = perfOpenHelloWorld(stats, "warm-up")
         try {
             val perfHighlightFile = perfHighlightFile(project, "src/HelloMain.kt", stats, "warm-up")
             assertTrue("kotlin project has been not imported properly", perfHighlightFile.isNotEmpty())
@@ -114,6 +125,7 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
     }
 
     override fun tearDown() {
+        commitAllDocuments()
         RunAll(
             ThrowableRunnable { super.tearDown() },
             ThrowableRunnable {
@@ -127,41 +139,47 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
             }).run()
     }
 
-    private fun simpleFilename(fileName: String): String {
+    fun simpleFilename(fileName: String): String {
         val lastIndexOf = fileName.lastIndexOf('/')
         return if (lastIndexOf >= 0) fileName.substring(lastIndexOf + 1) else fileName
     }
 
-    protected fun perfOpenKotlinProject(stats: Stats) =
+    protected fun perfOpenKotlinProjectFast(stats: Stats) =
-        perfOpenProject("perfTestProject", stats = stats, path = "..")
+        perfOpenKotlinProject(stats, fast = true)
 
-    protected fun perfOpenProject(name: String, stats: Stats, path: String = "idea/testData/perfTest") {
+    protected fun perfOpenKotlinProject(stats: Stats, fast: Boolean = false) {
-        myProject = innerPerfOpenProject(name, stats, path = path, note = "")
+        myProject = innerPerfOpenProject("kotlin", stats = stats, path = "../perfTestProject", note = "", fast = fast)
     }
 
+    protected fun perfOpenHelloWorld(stats: Stats, note: String = ""): Project =
+        innerPerfOpenProject("helloKotlin", stats, note, path = "idea/testData/perfTest/helloKotlin", simpleModule = true)
+
     protected fun innerPerfOpenProject(
         name: String,
         stats: Stats,
         note: String,
-        path: String = "idea/testData/perfTest"
+        path: String,
+        simpleModule: Boolean = false,
+        fast: Boolean = false
     ): Project {
-        val projectPath = File("$path/$name").canonicalPath
+        val projectPath = File("$path").canonicalPath
 
-        val warmUpIterations = 1
+        val warmUpIterations = if (fast) 0 else 1
-        val iterations = 3
+        val iterations = if (fast) 1 else 3
         val projectManagerEx = ProjectManagerEx.getInstanceEx()
 
         var lastProject: Project? = null
         var counter = 0
 
-        stats.perfTest<Unit, Pair<Project, Boolean>>(
+        stats.perfTest<Unit, Project>(
             warmUpIterations = warmUpIterations,
             iterations = iterations,
             testName = "open project${if (note.isNotEmpty()) " $note" else ""}",
             test = {
-                val projectPathExists = Paths.get(projectPath, ".idea").exists()
+                val project = if (!simpleModule) {
-                val project = if (projectPathExists) {
+                    val project = projectManagerEx.loadProject(name, path)
-                    val project = ProjectUtil.openProject(projectPath, null, false)!!
+                    assertNotNull(project)
+                    //projectManagerEx.openTestProject(project!!)
                     project
                 } else {
                     val project = projectManagerEx.loadAndOpenProject(projectPath)!!
@@ -174,8 +192,6 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
                     FileEditorManagerImpl::class.java
                 )
 
-                projectManagerEx.openTestProject(project)
-
                 dispatchAllInvocationEvents()
 
                 with(StartupManager.getInstance(project) as StartupManagerImpl) {
@@ -187,30 +203,23 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
                     waitUntilRefreshed()
                 }
 
-                it.value = Pair(project, projectPathExists)
+                it.value = project
             },
             tearDown = {
-                it.value?.let { pair ->
+                it.value?.let { project ->
-                    val project = pair.first
 
-                    // import gradle project if `$project/.idea` is present but modules are not imported
+                    runAndMeasure("refresh gradle project $name") {
-                    // it is a temporary dirty hack as it is fixed in latest IC:
+                        refreshGradleProjectIfNeeded(projectPath, project)
-                    // ProjectUtil.openProject picks up gradle import via extension point
-                    if (pair.second && ModuleManager.getInstance(project).modules.isEmpty()) {
-                        dispatchAllInvocationEvents()
-
-                        val virtualFile = LocalFileSystem.getInstance().refreshAndFindFileByPath(projectPath)!!
-
-                        FileDocumentManager.getInstance().saveAllDocuments()
-
-                        GradleProjectOpenProcessor.openGradleProject(project, null, Paths.get(virtualFile.path))
-
-                        dispatchAllInvocationEvents()
-                        runInEdtAndWait {
-                            PlatformTestUtil.saveProject(project)
-                        }
                     }
 
+                    ApplicationManager.getApplication().executeOnPooledThread {
+                        DumbService.getInstance(project).waitForSmartMode()
+
+                        for (module in getModulesWithKotlinFiles(project)) {
+                            module.getAndCacheLanguageLevelByDependencies()
+                        }
+                    }.get()
+
                     assertTrue(
                         "project has to have at least one module",
                         ModuleManager.getInstance(project).modules.isNotEmpty()
@@ -219,6 +228,12 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
                     lastProject = project
                     VirtualFileManager.getInstance().syncRefresh()
 
+                    runWriteAction {
+                        project.save()
+                    }
+
+                    println("# project '$name' successfully opened")
+
                     // close all project but last - we're going to return and use it further
                     if (counter < warmUpIterations + iterations - 1) {
                         closeProject(project)
@@ -248,13 +263,218 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
         return lastProject!!
     }
 
+    fun openGradleProject(projectPath: String, project: Project) {
+        dispatchAllInvocationEvents()
+
+        val virtualFile = LocalFileSystem.getInstance().refreshAndFindFileByPath(projectPath)!!
+
+        FileDocumentManager.getInstance().saveAllDocuments()
+
+        val path = Paths.get(virtualFile.path)
+        GradleProjectOpenProcessor.openGradleProject(project, null, path)
+
+        dispatchAllInvocationEvents()
+        runInEdtAndWait {
+            PlatformTestUtil.saveProject(project)
+        }
+    }
+
+    private fun refreshGradleProjectIfNeeded(projectPath: String, project: Project) {
+        if (listOf("build.gradle.kts", "build.gradle").map { name -> Paths.get(projectPath, name).exists() }.find { e -> e } != true) return
+
+        ExternalProjectsManagerImpl.getInstance(project).setStoreExternally(false)
+
+        GradleProjectOpenProcessor.openGradleProject(project, null, Paths.get(projectPath))
+
+        ExternalSystemUtil.refreshProjects(
+            ImportSpecBuilder(project, GradleConstants.SYSTEM_ID)
+                .forceWhenUptodate()
+                .useDefaultCallback()
+                .use(ProgressExecutionMode.MODAL_SYNC)
+        )
+
+        dispatchAllInvocationEvents()
+        ExternalProjectsManagerImpl.getInstance(project).setStoreExternally(false)
+        dispatchAllInvocationEvents()
+
+        // WARNING: [VD] DO NOT SAVE PROJECT AS IT COULD PERSIST WRONG MODULES INFO
+
+//        runInEdtAndWait {
+//            PlatformTestUtil.saveProject(project)
+//        }
+    }
+
+    /**
+     * @param lookupElements perform basic autocompletion and check presence of suggestion if elements are not empty
+     */
+    fun typeAndCheckLookup(
+        project: Project,
+        fileName: String,
+        marker: String,
+        insertString: String,
+        surroundItems: String = "\n",
+        lookupElements: List<String>,
+        revertChangesAtTheEnd: Boolean = true
+    ) {
+        val fileInEditor = openFileInEditor(project, fileName)
+        val virtualFile = fileInEditor.psiFile.virtualFile
+        val editor = EditorFactory.getInstance().getEditors(fileInEditor.document, project)[0]
+        val editorFixture = EditorTestFixture(project, editor, virtualFile)
+
+        val initialText = editor.document.text
+        try {
+            if (isAKotlinScriptFile(fileName)) {
+                ScriptDependenciesManager.updateScriptDependenciesSynchronously(virtualFile, project)
+            }
+
+            val tasksIdx = fileInEditor.document.text.indexOf(marker)
+            assertTrue(tasksIdx > 0)
+            editor.caretModel.moveToOffset(tasksIdx + marker.length + 1)
+
+            for (surroundItem in surroundItems) {
+                EditorTestUtil.performTypingAction(editor, surroundItem)
+            }
+
+            editor.caretModel.moveToOffset(editor.caretModel.offset - 1)
+            editorFixture.type(insertString)
+
+            if (lookupElements.isNotEmpty()) {
+                val elements = editorFixture.complete(CompletionType.BASIC, 1) ?: emptyArray()
+                val items = elements.map { it.lookupString }.toList()
+                for (lookupElement in lookupElements) {
+                    assertTrue("'$lookupElement' has to be present in items", items.contains(lookupElement))
+                }
+            }
+        } finally {
+            // TODO: [VD] revert ?
+            //editorFixture.performEditorAction(IdeActions.SELECTED_CHANGES_ROLLBACK)
+            if (revertChangesAtTheEnd) {
+                runWriteAction {
+                    editor.document.setText(initialText)
+                    commitDocument(project, editor.document)
+                }
+            }
+        }
+    }
+
+    fun perfTypeAndAutocomplete(
+        stats: Stats,
+        fileName: String,
+        marker: String,
+        insertString: String,
+        surroundItems: String = "\n",
+        lookupElements: List<String>,
+        typeAfterMarker: Boolean = true,
+        revertChangesAtTheEnd: Boolean = true,
+        note: String = ""
+    ) = perfTypeAndAutocomplete(
+        myProject!!, stats, fileName, marker, insertString, surroundItems,
+        lookupElements = lookupElements, typeAfterMarker = typeAfterMarker,
+        revertChangesAtTheEnd = revertChangesAtTheEnd, note = note
+    )
+
+    fun perfTypeAndAutocomplete(
+        project: Project,
+        stats: Stats,
+        fileName: String,
+        marker: String,
+        insertString: String,
+        surroundItems: String = "\n",
+        lookupElements: List<String>,
+        typeAfterMarker: Boolean = true,
+        revertChangesAtTheEnd: Boolean = true,
+        note: String = ""
+    ) {
+        assertTrue("lookupElements has to be not empty", lookupElements.isNotEmpty())
+        stats.perfTest<Pair<String, FixtureEditorFile>, Array<LookupElement>>(
+            warmUpIterations = 8,
+            iterations = 15,
+            testName = "typeAndAutocomplete ${notePrefix(note)}$fileName",
+            setUp = {
+                val fileInEditor = openFileInEditor(project, fileName)
+                val file = fileInEditor.psiFile
+                val virtualFile = file.virtualFile
+                val editor = EditorFactory.getInstance().getEditors(fileInEditor.document, project)[0]
+                val fixture = EditorTestFixture(project, editor, virtualFile)
+                val initialText = editor.document.text
+                if (isAKotlinScriptFile(fileName)) {
+                    runAndMeasure("update script dependencies for $fileName") {
+                        ScriptDependenciesManager.updateScriptDependenciesSynchronously(virtualFile, project)
+                    }
+                }
+
+                val tasksIdx = fileInEditor.document.text.indexOf(marker)
+                assertTrue("marker '$marker' not found in $fileName", tasksIdx > 0)
+                if (typeAfterMarker) {
+                    editor.caretModel.moveToOffset(tasksIdx + marker.length + 1)
+                } else {
+                    editor.caretModel.moveToOffset(tasksIdx - 1)
+                }
+
+                for (surroundItem in surroundItems) {
+                    EditorTestUtil.performTypingAction(editor, surroundItem)
+                }
+
+                editor.caretModel.moveToOffset(editor.caretModel.offset - if (typeAfterMarker) 1 else 2)
+
+                if (!typeAfterMarker) {
+                    for (surroundItem in surroundItems) {
+                        EditorTestUtil.performTypingAction(editor, surroundItem)
+                    }
+                    editor.caretModel.moveToOffset(editor.caretModel.offset - 2)
+                }
+
+                fixture.type(insertString)
+
+                it.setUpValue = Pair(initialText, FixtureEditorFile(file, editor.document, fixture))
+            },
+            test = {
+                val fixture = it.setUpValue!!.second.fixture
+                it.value = fixture.complete(CompletionType.BASIC, 1) ?: emptyArray()
+            },
+            tearDown = {
+                val items = it.value?.map { e -> e.lookupString }?.toList() ?: emptyList()
+                try {
+                    for (lookupElement in lookupElements) {
+                        assertTrue("'$lookupElement' has to be present in items", items.contains(lookupElement))
+                    }
+                } finally {
+                    it.setUpValue?.let { pair ->
+                        val document = pair.second.document
+                        val file = pair.second.psiFile
+                        val text = pair.first
+
+                        try {
+                            if (revertChangesAtTheEnd) {
+                                runWriteAction {
+                                    // TODO: [VD] revert ?
+                                    //editorFixture.performEditorAction(IdeActions.SELECTED_CHANGES_ROLLBACK)
+                                    document.setText(text)
+                                    saveDocument(document)
+                                    commitDocument(project, document)
+                                }
+                                dispatchAllInvocationEvents()
+                            }
+                        } finally {
+                            cleanupCaches(project, file.virtualFile)
+                        }
+                    }
+                    commitAllDocuments()
+                }
+            }
+        )
+    }
+
     protected fun enableAnnotatorsAndLoadDefinitions() = enableAnnotatorsAndLoadDefinitions(myProject!!)
 
     protected fun enableAnnotatorsAndLoadDefinitions(project: Project) {
+        ReferenceProvidersRegistry.getInstance() // pre-load tons of classes
         InjectedLanguageManager.getInstance(project) // zillion of Dom Sem classes
-        LanguageAnnotators.INSTANCE.allForLanguage(JavaLanguage.INSTANCE) // pile of annotator classes loads
+        with(LanguageAnnotators.INSTANCE) {
-        LanguageAnnotators.INSTANCE.allForLanguage(StdLanguages.XML)
+            allForLanguage(JavaLanguage.INSTANCE) // pile of annotator classes loads
-        LanguageAnnotators.INSTANCE.allForLanguage(KotlinLanguage.INSTANCE)
+            allForLanguage(StdLanguages.XML)
+            allForLanguage(KotlinLanguage.INSTANCE)
+        }
         DaemonAnalyzerTestCase.assertTrue(
             "side effect: to load extensions",
             ProblemHighlightFilter.EP_NAME.extensions.toMutableList()
@@ -264,6 +484,7 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
                 .plus(ExternalAnnotatorsFilter.EXTENSION_POINT_NAME.extensions)
                 .plus(IndexPatternBuilder.EP_NAME.extensions).isNotEmpty()
         )
+
         // side effect: to load script definitions"
         val scriptDefinitionsManager = ScriptDefinitionsManager.getInstance(project)
         scriptDefinitionsManager.getAllDefinitions()
@@ -282,7 +503,7 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
         val modulePath = "$projectPath/$name${ModuleFileType.DOT_DEFAULT_EXTENSION}"
         val projectFile = LocalFileSystem.getInstance().refreshAndFindFileByIoFile(File(projectPath))!!
         val srcFile = projectFile.findChild("src")!!
-        val module = ApplicationManager.getApplication().runWriteAction(Computable<Module> {
+        val module = runWriteAction {
             val projectRootManager = ProjectRootManager.getInstance(project)
             with(projectRootManager) {
                 projectSdk = jdk18
@@ -291,7 +512,7 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
             val module = moduleManager.newModule(modulePath, ModuleTypeId.JAVA_MODULE)
             PsiTestUtil.addSourceRoot(module, srcFile)
             module
-        })
+        }
         ConfigLibraryUtil.configureKotlinRuntimeAndSdk(module, jdk18)
     }
 
@@ -305,9 +526,12 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
         note: String = ""
     ): List<HighlightInfo> {
         return highlightFile {
+            val isWarmUp = note == "warm-up"
             var highlightInfos: List<HighlightInfo> = emptyList()
             stats.perfTest<EditorFile, List<HighlightInfo>>(
-                testName = "highlighting ${if (note.isNotEmpty()) "$note " else ""}${simpleFilename(fileName)}",
+                warmUpIterations = if (isWarmUp) 1 else 3,
+                iterations = if (isWarmUp) 2 else 10,
+                testName = "highlighting ${notePrefix(note)}${simpleFilename(fileName)}",
                 setUp = {
                     it.setUpValue = openFileInEditor(project, fileName)
                 },
@@ -332,12 +556,12 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
         }
     }
 
-    private fun highlightFile(block: () -> List<HighlightInfo>): List<HighlightInfo> {
+    fun <T> highlightFile(block: () -> T): T {
-        var highlightInfos: List<HighlightInfo> = emptyList()
+        var value: T? = null
         IdentifierHighlighterPassFactory.doWithHighlightingEnabled {
-            highlightInfos = block()
+            value = block()
         }
-        return highlightInfos
+        return value!!
     }
 
     private fun highlightFile(project: Project, psiFile: PsiFile): List<HighlightInfo> {
@@ -347,74 +571,41 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
         return CodeInsightTestFixtureImpl.instantiateAndRun(psiFile, editor, ArrayUtilRt.EMPTY_INT_ARRAY, true)
     }
 
-    protected fun perfFileAnalysis(name: String, stats: Stats, note: String = "") =
+    protected fun perfScriptDependencies(name: String, stats: Stats, note: String = "") =
-        perfFileAnalysis(myProject!!, name, stats, note = note)
+        perfScriptDependencies(myProject!!, name, stats, note = note)
 
-    private fun perfFileAnalysis(
+    private fun perfScriptDependencies(
         project: Project,
         fileName: String,
         stats: Stats,
         note: String = ""
     ) {
-        val disposable = Disposer.newDisposable("perfFileAnalysis $fileName")
+        if (!isAKotlinScriptFile(fileName)) return
-
+        stats.perfTest<EditorFile, EditorFile>(
-        MadTestingUtil.enableAllInspections(project, disposable)
+            testName = "updateScriptDependencies ${notePrefix(note)}${simpleFilename(fileName)}",
-
+            setUp = { it.setUpValue = openFileInEditor(project, fileName) },
-        try {
+            test = {
-            IdentifierHighlighterPassFactory.doWithHighlightingEnabled {
+                ScriptDependenciesManager.updateScriptDependenciesSynchronously(it.setUpValue!!.psiFile.virtualFile, project)
-                stats.perfTest(
+                it.value = it.setUpValue
-                    testName = "fileAnalysis ${if (note.isNotEmpty()) "$note " else ""}${simpleFilename(fileName)}",
+            },
-                    setUp = perfFileAnalysisSetUp(project, fileName),
+            tearDown = {
-                    test = perfFileAnalysisTest(project),
+                it.setUpValue?.let { ef -> cleanupCaches(project, ef.psiFile.virtualFile) }
-                    tearDown = perfFileAnalysisTearDown(fileName, project)
+                it.value?.let { v -> assertNotNull(v) }
-                )
             }
-        } finally {
+        )
-            Disposer.dispose(disposable)
-        }
     }
 
-    private fun perfFileAnalysisSetUp(
+    fun notePrefix(note: String) = if (note.isNotEmpty()) {
-        project: Project,
+        if (note.endsWith("/")) note else "$note "
-        fileName: String
+    } else ""
-    ): (TestData<EditorFile, List<HighlightInfo>>) -> Unit {
-        return {
-            val fileInEditor = openFileInEditor(project, fileName)
-
-            // Note: Kotlin scripts require dependencies to be loaded
-            if (isAKotlinScriptFile(fileName)) {
-                val vFile = fileInEditor.psiFile.virtualFile
-                ScriptDependenciesManager.updateScriptDependenciesSynchronously(vFile, project)
-            }
-
-            //enableHints(false)
-
-            println("fileAnalysis -> $fileName\n")
-            it.setUpValue = fileInEditor
-        }
-    }
 
     // quite simple impl - good so far
     fun isAKotlinScriptFile(fileName: String) = fileName.endsWith(".kts")
 
-    private fun perfFileAnalysisTest(project: Project): (TestData<EditorFile, List<HighlightInfo>>) -> Unit {
+    fun cleanupCaches(project: Project, vFile: VirtualFile) {
-        return {
+        commitAllDocuments()
-            val fileInEditor = it.setUpValue!!
+        FileEditorManager.getInstance(project).closeFile(vFile)
-            it.value = highlightFile(project, fileInEditor.psiFile)
+        PsiManager.getInstance(project).dropPsiCaches()
-        }
-    }
-
-    private fun perfFileAnalysisTearDown(
-        fileName: String,
-        project: Project
-    ): (TestData<EditorFile, List<HighlightInfo>>) -> Unit {
-        return {
-            commitAllDocuments()
-            //println("fileAnalysis <- $fileName:\n${it.value?.joinToString("\n")}\n")
-            println("fileAnalysis <- $fileName:\n${it.value?.size ?: 0} highlightInfos\n")
-            FileEditorManager.getInstance(project).closeFile(it.setUpValue!!.psiFile.virtualFile)
-            PsiManager.getInstance(project).dropPsiCaches()
-        }
     }
 
     fun openFileInEditor(project: Project, name: String): EditorFile {
@@ -434,18 +625,47 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
         assertNotNull("doc not found for $vFile", EditorFactory.getInstance().getEditors(document))
         assertTrue("expected non empty doc", document.text.isNotEmpty())
 
+        val offset = psiFile.textOffset
+        assertTrue("side effect: to load the text", offset >= 0)
+
         waitForAllEditorsFinallyLoaded(project)
 
         return EditorFile(psiFile = psiFile, document = document)
     }
 
+    private fun baseName(name: String): String {
+        val index = name.lastIndexOf("/")
+        return if (index > 0) name.substring(index + 1) else name
+    }
+
     private fun projectFileByName(project: Project, name: String): PsiFile {
         val fileManager = VirtualFileManager.getInstance()
-        val url = "file://${File("${project.basePath}/$name").absolutePath}"
+        val url = "${project.guessProjectDir()}/$name"
         val virtualFile = fileManager.refreshAndFindFileByUrl(url)
-        return virtualFile!!.toPsiFile(project)!!
+        if (virtualFile != null) {
+            return virtualFile!!.toPsiFile(project)!!
+        }
+
+        val baseFileName = baseName(name)
+        val projectBaseName = baseName(project.name)
+
+        val virtualFiles = FilenameIndex.getVirtualFilesByName(
+            project,
+            baseFileName, true,
+            GlobalSearchScope.projectScope(project)
+        )
+            .filter { it.canonicalPath?.contains("/$projectBaseName/$name") ?: false }.toList()
+
+        assertEquals(
+            "expected the only file with name '$name'\n, it were: [${virtualFiles.map { it.canonicalPath }.joinToString("\n")}]",
+            1,
+            virtualFiles.size
+        )
+        return virtualFiles.iterator().next().toPsiFile(project)!!
     }
 
+    data class FixtureEditorFile(val psiFile: PsiFile, val document: Document, val fixture: EditorTestFixture)
+
     data class EditorFile(val psiFile: PsiFile, val document: Document)
 
 }

idea/performanceTests/org/jetbrains/kotlin/idea/perf/PerformanceProjectsTest.kt

@@ -5,6 +5,25 @@
 
 package org.jetbrains.kotlin.idea.perf
 
+import com.intellij.codeInsight.daemon.impl.HighlightInfo
+import com.intellij.lang.LanguageAnnotators
+import com.intellij.lang.LanguageExtensionPoint
+import com.intellij.lang.annotation.AnnotationHolder
+import com.intellij.lang.annotation.Annotator
+import com.intellij.openapi.editor.EditorFactory
+import com.intellij.openapi.extensions.ExtensionPointName
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.util.Disposer
+import com.intellij.psi.PsiElement
+import com.intellij.testFramework.PlatformTestUtil
+import com.intellij.testFramework.fixtures.EditorTestFixture
+import com.intellij.testFramework.propertyBased.MadTestingUtil
+import org.jetbrains.kotlin.idea.core.script.ScriptDependenciesManager
+import org.jetbrains.kotlin.idea.highlighter.KotlinPsiChecker
+import org.jetbrains.kotlin.idea.highlighter.KotlinPsiCheckerAndHighlightingUpdater
+import java.util.concurrent.atomic.AtomicLong
+import kotlin.test.assertNotEquals
+
 class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
 
     companion object {
@@ -15,11 +34,21 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
         @JvmStatic
         val hwStats: Stats = Stats("helloWorld project")
 
+        @JvmStatic
+        val timer: AtomicLong = AtomicLong()
+
         init {
             // there is no @AfterClass for junit3.8
             Runtime.getRuntime().addShutdownHook(Thread(Runnable { hwStats.close() }))
         }
 
+        fun resetTimestamp() {
+            timer.set(0)
+        }
+
+        fun markTimestamp() {
+            timer.set(System.nanoTime())
+        }
     }
 
     override fun setUp() {
@@ -34,7 +63,7 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
 
     fun testHelloWorldProject() {
         tcSuite("Hello world project") {
-            perfOpenProject("helloKotlin", hwStats)
+            myProject = perfOpenHelloWorld(hwStats)
 
             // highlight
             perfHighlightFile("src/HelloMain.kt", hwStats)
@@ -50,41 +79,243 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
 
                 perfHighlightFile("compiler/psi/src/org/jetbrains/kotlin/psi/KtFile.kt", stats = it)
 
+                perfHighlightFile("idea/idea-analysis/src/org/jetbrains/kotlin/idea/util/PsiPrecedences.kt", stats = it)
+
                 perfHighlightFile("compiler/psi/src/org/jetbrains/kotlin/psi/KtElement.kt", stats = it)
             }
         }
     }
 
-    fun testKotlinProjectHighlightBuildGradle() {
+    fun testKotlinProjectCompletionKtFile() {
-        tcSuite("Kotlin project highlight build gradle") {
+        tcSuite("Kotlin completion ktFile") {
-            val stats = Stats("kotlin project highlight build gradle")
+            val stats = Stats("Kotlin completion ktFile")
-            stats.use {
+            stats.use { stat ->
-                perfOpenKotlinProject(it)
+                perfOpenKotlinProjectFast(stat)
 
-                enableAnnotatorsAndLoadDefinitions()
+                perfTypeAndAutocomplete(
+                    stat,
+                    "compiler/psi/src/org/jetbrains/kotlin/psi/KtFile.kt",
+                    "override fun getDeclarations(): List<KtDeclaration> {",
+                    "val q = import",
+                    lookupElements = listOf("importDirectives"),
+                    note = "in-method getDeclarations-import"
+                )
 
-                perfFileAnalysisBuildGradleKts(it)
+                perfTypeAndAutocomplete(
-                perfFileAnalysisIdeaBuildGradleKts(it)
+                    stat,
-                perfFileAnalysisJpsGradleKts(it)
+                    "compiler/psi/src/org/jetbrains/kotlin/psi/KtFile.kt",
-                perfFileAnalysisVersionGradleKts(it)
+                    "override fun getDeclarations(): List<KtDeclaration> {",
+                    "val q = import",
+                    typeAfterMarker = false,
+                    lookupElements = listOf("importDirectives"),
+                    note = "out-of-method import"
+                )
             }
         }
     }
 
+    fun testKotlinProjectCompletionBuildGradle() {
+        tcSuite("Kotlin completion gradle.kts") {
+            val stats = Stats("kotlin completion gradle.kts")
+            stats.use { stat ->
+                runAndMeasure("open kotlin project") {
+                    perfOpenKotlinProjectFast(stat)
+                }
+
+                runAndMeasure("type and autocomplete") {
+                    perfTypeAndAutocomplete(
+                        stat,
+                        "build.gradle.kts",
+                        "tasks {",
+                        "crea",
+                        lookupElements = listOf("create"),
+                        note = "tasks-create"
+                    )
+                }
+            }
+        }
+    }
+
+    fun testKotlinProjectScriptDependenciesBuildGradle() {
+        tcSuite("Kotlin scriptDependencies gradle.kts") {
+            val stats = Stats("kotlin scriptDependencies gradle.kts")
+            stats.use { stat ->
+                perfOpenKotlinProjectFast(stat)
+
+                perfScriptDependenciesBuildGradleKts(stat)
+                perfScriptDependenciesIdeaBuildGradleKts(stat)
+                perfScriptDependenciesJpsGradleKts(stat)
+                perfScriptDependenciesVersionGradleKts(stat)
+            }
+        }
+    }
+
+    fun testKotlinProjectBuildGradle() {
+        tcSuite("Kotlin gradle.kts") {
+            val stats = Stats("kotlin gradle.kts")
+            stats.use { stat ->
+                perfOpenKotlinProjectFast(stat)
+
+                perfFileAnalysisBuildGradleKts(stat)
+                perfFileAnalysisIdeaBuildGradleKts(stat)
+                perfFileAnalysisJpsGradleKts(stat)
+                perfFileAnalysisVersionGradleKts(stat)
+            }
+        }
+    }
+
+    private fun perfScriptDependenciesBuildGradleKts(it: Stats) {
+        perfScriptDependencies("build.gradle.kts", stats = it)
+    }
+
+    private fun perfScriptDependenciesIdeaBuildGradleKts(it: Stats) {
+        perfScriptDependencies("idea/build.gradle.kts", stats = it, note = "idea/")
+    }
+
+    private fun perfScriptDependenciesJpsGradleKts(it: Stats) {
+        perfScriptDependencies("gradle/jps.gradle.kts", stats = it, note = "gradle/")
+    }
+
+    private fun perfScriptDependenciesVersionGradleKts(it: Stats) {
+        perfScriptDependencies("gradle/versions.gradle.kts", stats = it, note = "gradle/")
+    }
+
     private fun perfFileAnalysisBuildGradleKts(it: Stats) {
-        perfFileAnalysis("build.gradle.kts", stats = it)
+        perfKtsFileAnalysis("build.gradle.kts", stats = it)
     }
 
     private fun perfFileAnalysisIdeaBuildGradleKts(it: Stats) {
-        perfFileAnalysis("idea/build.gradle.kts", stats = it, note = "idea/")
+        perfKtsFileAnalysis("idea/build.gradle.kts", stats = it, note = "idea/")
     }
 
     private fun perfFileAnalysisJpsGradleKts(it: Stats) {
-        perfFileAnalysis("gradle/jps.gradle.kts", stats = it, note = "gradle/")
+        perfKtsFileAnalysis("gradle/jps.gradle.kts", stats = it, note = "gradle/")
     }
 
     private fun perfFileAnalysisVersionGradleKts(it: Stats) {
-        perfFileAnalysis("gradle/versions.gradle.kts", stats = it, note = "gradle/")
+        perfKtsFileAnalysis("gradle/versions.gradle.kts", stats = it, note = "gradle/")
     }
 
+    private fun perfKtsFileAnalysis(
+        fileName: String,
+        stats: Stats,
+        note: String = ""
+    ) {
+        val project = myProject!!
+        val disposable = Disposer.newDisposable("perfKtsFileAnalysis $fileName")
+
+        MadTestingUtil.enableAllInspections(project, disposable)
+
+        replaceWithCustomHighlighter()
+
+        try {
+            highlightFile {
+                val testName = "fileAnalysis ${notePrefix(note)}${simpleFilename(fileName)}"
+                val extraStats = Stats("${stats.name} $testName")
+                val extraTimingsNs = mutableListOf<Long>()
+
+                val warmUpIterations = 3
+                val iterations = 10
+
+                stats.perfTest(
+                    warmUpIterations = warmUpIterations,
+                    iterations = iterations,
+                    testName = testName,
+                    setUp = perfKtsFileAnalysisSetUp(project, fileName),
+                    test = perfKtsFileAnalysisTest(),
+                    tearDown = perfKtsFileAnalysisTearDown(extraTimingsNs, project)
+                )
+
+                extraStats.printWarmUpTimings(
+                    "annotator",
+                    Array(warmUpIterations, init = { null }),
+                    extraTimingsNs.take(warmUpIterations).toLongArray()
+                )
+
+                extraStats.appendTimings(
+                    "annotator",
+                    Array(iterations, init = { null }),
+                    extraTimingsNs.drop(warmUpIterations).toLongArray()
+                )
+            }
+        } finally {
+            Disposer.dispose(disposable)
+        }
+    }
+
+    private fun replaceWithCustomHighlighter() {
+        val pointName = ExtensionPointName.create<LanguageExtensionPoint<Annotator>>(LanguageAnnotators.EP_NAME)
+        val extensionPoint = pointName.getPoint(null)
+
+        val point = LanguageExtensionPoint<Annotator>()
+        point.language = "kotlin"
+        point.implementationClass = TestKotlinPsiChecker::class.java.name
+
+        val extensions = extensionPoint.extensions
+        val filteredExtensions =
+            extensions.filter { it.language != "kotlin" || it.implementationClass != KotlinPsiCheckerAndHighlightingUpdater::class.java.name }
+                .toList()
+        // custom highlighter is already registered if filteredExtensions has the same size as extensions
+        if (filteredExtensions.size < extensions.size) {
+            PlatformTestUtil.maskExtensions(pointName, filteredExtensions + listOf(point), testRootDisposable)
+        }
+    }
+
+    fun perfKtsFileAnalysisSetUp(
+        project: Project,
+        fileName: String
+    ): (TestData<FixtureEditorFile, Pair<Long, List<HighlightInfo>>>) -> Unit {
+        return {
+            val fileInEditor = openFileInEditor(project, fileName)
+
+            val file = fileInEditor.psiFile
+            val virtualFile = file.virtualFile
+            val editor = EditorFactory.getInstance().getEditors(fileInEditor.document, project)[0]
+            val fixture = EditorTestFixture(project, editor, virtualFile)
+
+            // Note: Kotlin scripts require dependencies to be loaded
+            if (isAKotlinScriptFile(fileName)) {
+                ScriptDependenciesManager.updateScriptDependenciesSynchronously(fileInEditor.psiFile.virtualFile, project)
+            }
+
+            resetTimestamp()
+            it.setUpValue = FixtureEditorFile(file, editor.document, fixture)
+        }
+    }
+
+    fun perfKtsFileAnalysisTest(): (TestData<FixtureEditorFile, Pair<Long, List<HighlightInfo>>>) -> Unit {
+        return {
+            it.value = it.setUpValue?.let { fef ->
+                Pair(System.nanoTime(), fef.fixture.doHighlighting())
+            }
+        }
+    }
+
+    fun perfKtsFileAnalysisTearDown(
+        extraTimingsNs: MutableList<Long>,
+        project: Project
+    ): (TestData<FixtureEditorFile, Pair<Long, List<HighlightInfo>>>) -> Unit {
+        return {
+            it.setUpValue?.let { fef ->
+                it.value?.let { v ->
+                    assertTrue(v.second.isNotEmpty())
+                    assertNotEquals(0, timer.get())
+
+                    extraTimingsNs.add(timer.get() - v.first)
+
+                }
+                cleanupCaches(project, fef.psiFile.virtualFile)
+            }
+        }
+    }
+
+
+    class TestKotlinPsiChecker : KotlinPsiChecker() {
+        override fun annotate(
+            element: PsiElement, holder: AnnotationHolder
+        ) {
+            super.annotate(element, holder)
+            markTimestamp()
+        }
+    }
 }

idea/performanceTests/org/jetbrains/kotlin/idea/perf/Stats.kt

@@ -13,13 +13,14 @@ import kotlin.system.measureNanoTime
 import java.lang.ref.WeakReference
 import kotlin.math.exp
 import kotlin.math.ln
+import kotlin.system.measureTimeMillis
+import kotlin.test.assertEquals
 
 class Stats(val name: String = "", val header: Array<String> = arrayOf("Name", "ValueMS", "StdDev")) : Closeable {
     private val perfTestRawDataMs = mutableListOf<Long>()
 
-    private val statsFile: File =
+    private val statsFile: File = File("build/stats${statFilePrefix()}.csv").absoluteFile
-        File("build/stats${if (name.isNotEmpty()) "-${name.toLowerCase().replace(' ', '-')}" else ""}.csv")
+
-            .absoluteFile
     private val statsOutput: BufferedWriter
 
     init {
@@ -28,20 +29,32 @@ class Stats(val name: String = "", val header: Array<String> = arrayOf("Name", "
         statsOutput.appendln(header.joinToString())
     }
 
+    private fun statFilePrefix() = if (name.isNotEmpty()) "-${name.toLowerCase().replace(' ', '-').replace('/', '_')}" else ""
+
     private fun append(id: String, timingsNs: LongArray) {
         val meanNs = timingsNs.average()
         val meanMs = meanNs.toLong().nsToMs
 
-        val stdDivMs = (sqrt(
+        val stdDevMs = if (timingsNs.size > 1) (sqrt(
             timingsNs.fold(0.0,
-                           { accumulator, next -> accumulator + (1.0 * (next - meanMs)).pow(2.0) })
+                           { accumulator, next -> accumulator + (1.0 * (next - meanNs)).pow(2) })
-        ) / timingsNs.size).toLong().nsToMs
+        ) / (timingsNs.size - 1)).toLong().nsToMs
+        else 0
 
-        println("##teamcity[buildStatisticValue key='$id' value='$meanMs']")
+        val geomMeanMs = geomMean(timingsNs.toList()).nsToMs
-        println("##teamcity[buildStatisticValue key='$id stdDev' value='$stdDivMs']")
+
+        for (v in listOf(
+            Triple("mean", "", meanMs),
+            Triple("stdDev", " stdDev", stdDevMs),
+            Triple("geomMean", "geomMean", geomMeanMs)
+        )) {
+            println("##teamcity[testStarted name='$id : ${v.first}' captureStandardOutput='true']")
+            println("##teamcity[buildStatisticValue key='$id${v.second}' value='${v.third}']")
+            println("##teamcity[testFinished name='$id : ${v.first}' duration='${v.third}']")
+        }
 
         perfTestRawDataMs.addAll(timingsNs.map { it.nsToMs }.toList())
-        append(arrayOf(id, meanMs, stdDivMs))
+        append(arrayOf(id, meanMs, stdDevMs))
     }
 
     private fun append(values: Array<Any>) {
@@ -67,7 +80,7 @@ class Stats(val name: String = "", val header: Array<String> = arrayOf("Name", "
         test: (TestData<K, T>) -> Unit,
         tearDown: (TestData<K, T>) -> Unit = { null }
     ) {
-        val namePrefix = "$name: $testName"
+        val namePrefix = "$testName"
         val timingsNs = LongArray(iterations)
         val errors = Array<Throwable?>(iterations, init = { null })
 
@@ -76,22 +89,46 @@ class Stats(val name: String = "", val header: Array<String> = arrayOf("Name", "
 
             mainPhase(iterations, setUp, test, tearDown, timingsNs, namePrefix, errors)
 
-            for (attempt in 0 until iterations) {
+            assertEquals(iterations, timingsNs.size)
-                for (n in listOf("$namePrefix #$attempt", "performance test: $namePrefix #$attempt")) {
+            appendTimings(namePrefix, errors, timingsNs)
-                    println("##teamcity[testStarted name='$n' captureStandardOutput='true']")
-                    if (errors[attempt] != null) {
-                        tcPrintErrors(n, listOf(errors[attempt]!!))
-                    }
-                    val spentMs = timingsNs[attempt].nsToMs
-                    println("##teamcity[buildStatisticValue key='$n' value='$spentMs']")
-                    println("##teamcity[testFinished name='$n' duration='$spentMs']")
-                }
-            }
-
-            append(namePrefix, timingsNs)
         }
     }
 
+    fun printWarmUpTimings(
+        prefix: String,
+        errors: Array<Throwable?>,
+        warmUpTimingsNs: LongArray
+    ) {
+        assertEquals(warmUpTimingsNs.size, errors.size)
+        for (timing in warmUpTimingsNs.withIndex()) {
+            val attempt = timing.index
+            val n = "$name: $prefix warm-up #$attempt"
+            printTestStarted(n)
+            val t = errors[attempt]
+            if (t != null) printTestFinished(n, t!!) else printTestFinished(n, timing.value.nsToMs)
+        }
+    }
+
+    fun appendTimings(
+        prefix: String,
+        errors: Array<Throwable?>,
+        timingsNs: LongArray
+    ) {
+        assertEquals(timingsNs.size, errors.size)
+        val namePrefix = "$name: $prefix"
+        for (attempt in 0 until timingsNs.size) {
+            val n = "$namePrefix #$attempt"
+            printTestStarted(n)
+            if (errors[attempt] != null) {
+                tcPrintErrors(n, listOf(errors[attempt]!!))
+            }
+            val spentMs = timingsNs[attempt].nsToMs
+            printTestFinished(n, spentMs)
+        }
+
+        append(namePrefix, timingsNs)
+    }
+
     private fun <K, T> mainPhase(
         iterations: Int,
         setUp: (TestData<K, T>) -> Unit,
@@ -107,18 +144,24 @@ class Stats(val name: String = "", val header: Array<String> = arrayOf("Name", "
                 testData.reset()
                 triggerGC(attempt)
 
-                setUp(testData)
+                val setUpMillis = measureTimeMillis {
+                    setUp(testData)
+                }
+                println("-- setup took $setUpMillis ms")
                 try {
                     val spentNs = measureNanoTime {
                         test(testData)
                     }
                     timingsNs[attempt] = spentNs
                 } catch (t: Throwable) {
-                    println("error at $namePrefix #$attempt:")
+                    println("# error at $namePrefix #$attempt:")
                     t.printStackTrace()
                     errors[attempt] = t
                 } finally {
-                    tearDown(testData)
+                    val tearDownMillis = measureTimeMillis {
+                        tearDown(testData)
+                    }
+                    println("-- tearDown took $tearDownMillis ms")
                 }
             }
         } catch (t: Throwable) {
@@ -135,43 +178,55 @@ class Stats(val name: String = "", val header: Array<String> = arrayOf("Name", "
         tearDown: (TestData<K, T>) -> Unit
     ) {
         val testData = TestData<K, T>(null, null)
+        val warmUpTimingsNs = LongArray(warmUpIterations)
+        val errors: Array<Throwable?> = Array(warmUpIterations, init = { null })
         for (attempt in 0 until warmUpIterations) {
             testData.reset()
 
             triggerGC(attempt)
 
-            val n = "$namePrefix warm-up #$attempt"
-            println("##teamcity[testStarted name='$n' captureStandardOutput='true']")
-
             try {
-                setUp(testData)
+                val setUpMillis = measureTimeMillis {
+                    setUp(testData)
+                }
+                println("-- setup took $setUpMillis ms")
                 var spentNs: Long = 0
                 try {
                     spentNs = measureNanoTime {
                         test(testData)
                     }
-                } catch (t: Throwable) {
-                    println("error at $n:\n")
-
-                    t.printStackTrace()
-
-                    println("\n")
-
-                    tcPrintErrors(n, listOf(t))
-                    throw t
                 } finally {
-                    tearDown(testData)
+                    val tearDownMillis = measureTimeMillis {
+                        tearDown(testData)
+                    }
+                    println("-- tearDown took $tearDownMillis ms")
                 }
-                val spentMs = spentNs.nsToMs
+                warmUpTimingsNs[attempt] = spentNs
-                println("##teamcity[buildStatisticValue key='$n' value='$spentMs']")
-                println("##teamcity[testFinished name='$n' duration='$spentMs']")
             } catch (t: Throwable) {
-                println("##teamcity[testFinished name='$n']")
+                errors[attempt] = t
-                println("error at $n:")
-                tcPrintErrors(n, listOf(t))
-                throw t
             }
         }
+
+        printWarmUpTimings(namePrefix, errors, warmUpTimingsNs)
+
+        errors.find { it != null }?.let { throw it }
+    }
+
+    fun printTestStarted(testName: String) {
+        println("##teamcity[testStarted name='$testName' captureStandardOutput='true']")
+    }
+
+    fun printTestFinished(testName: String, spentMs: Long) {
+        println("##teamcity[buildStatisticValue key='$testName' value='$spentMs']")
+        println("##teamcity[testFinished name='$testName' duration='$spentMs']")
+    }
+
+    fun printTestFinished(testName: String, error: Throwable) {
+        println("error at $testName:")
+        tcPrintErrors(testName, listOf(error))
+
+        println("##teamcity[buildStatisticValue key='$testName' value='-1']")
+        println("##teamcity[testFinished name='$testName']")
     }
 
     private fun triggerGC(attempt: Int) {
@@ -204,6 +259,13 @@ data class TestData<SV, V>(var setUpValue: SV?, var value: V?) {
     }
 }
 
+inline fun runAndMeasure(note: String, block: () -> Unit) {
+    val openProjectMillis = measureTimeMillis {
+        block()
+    }
+    println("-- $note took $openProjectMillis ms")
+}
+
 inline fun tcSuite(name: String, block: () -> Unit) {
     println("##teamcity[testSuiteStarted name='$name']")
     block()

idea/performanceTests/org/jetbrains/kotlin/idea/perf/projectRoutines.kt

@@ -5,6 +5,12 @@
 
 package org.jetbrains.kotlin.idea.perf
 
+import com.intellij.openapi.application.ApplicationManager
+import com.intellij.openapi.application.runWriteAction
+import com.intellij.openapi.editor.Document
+import com.intellij.openapi.fileEditor.FileDocumentManager
+import com.intellij.openapi.fileEditor.FileEditorManager
+import com.intellij.openapi.fileEditor.impl.text.TextEditorImpl
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.project.ex.ProjectManagerEx
 import com.intellij.psi.PsiDocumentManager
@@ -13,15 +19,37 @@ import com.intellij.testFramework.EdtTestUtil
 import com.intellij.util.ThrowableRunnable
 import com.intellij.util.ui.UIUtil
 import org.jetbrains.kotlin.idea.parameterInfo.HintType
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.TimeoutException
 
 fun commitAllDocuments() {
+    val fileDocumentManager = FileDocumentManager.getInstance()
+    runInEdtAndWait {
+        fileDocumentManager.saveAllDocuments()
+    }
+
     ProjectManagerEx.getInstanceEx().openProjects.forEach { project ->
         val psiDocumentManagerBase = PsiDocumentManager.getInstance(project) as PsiDocumentManagerBase
 
-        EdtTestUtil.runInEdtAndWait(ThrowableRunnable {
+        runInEdtAndWait {
             psiDocumentManagerBase.clearUncommittedDocuments()
             psiDocumentManagerBase.commitAllDocuments()
-        })
+        }
+    }
+}
+
+fun commitDocument(project: Project, document: Document) {
+    val psiDocumentManagerBase = PsiDocumentManager.getInstance(project) as PsiDocumentManagerBase
+    runInEdtAndWait {
+        psiDocumentManagerBase.commitDocument(document)
+    }
+}
+
+fun saveDocument(document: Document) {
+    val fileDocumentManager = FileDocumentManager.getInstance()
+
+    runInEdtAndWait {
+        fileDocumentManager.saveDocument(document)
     }
 }
 
@@ -47,5 +75,29 @@ fun closeProject(project: Project) {
 }
 
 fun waitForAllEditorsFinallyLoaded(project: Project) {
-    // nothing
+    waitForAllEditorsFinallyLoaded(project, 5, TimeUnit.MINUTES)
+}
+
+fun waitForAllEditorsFinallyLoaded(project: Project, timeout: Long, unit: TimeUnit) {
+    ApplicationManager.getApplication().assertIsDispatchThread()
+    val deadline = unit.toMillis(timeout) + System.currentTimeMillis()
+    while (true) {
+        if (System.currentTimeMillis() > deadline) throw TimeoutException()
+        if (waitABitForEditorLoading(project)) break
+        UIUtil.dispatchAllInvocationEvents()
+    }
+}
+
+private fun waitABitForEditorLoading(project: Project): Boolean {
+    for (editor in FileEditorManager.getInstance(project).allEditors) {
+        if (editor is TextEditorImpl) {
+            try {
+                editor.waitForLoaded(100, TimeUnit.MILLISECONDS)
+            } catch (ignored: TimeoutException) {
+                return false
+            }
+
+        }
+    }
+    return true
 }

idea/performanceTests/org/jetbrains/kotlin/idea/perf/projectRoutines.kt.182

@@ -25,6 +25,14 @@ fun commitAllDocuments() {
     }
 }
 
+fun commitDocument(project: Project, document: Document) {
+    val psiDocumentManagerBase = PsiDocumentManager.getInstance(project) as PsiDocumentManagerBase
+
+    EdtTestUtil.runInEdtAndWait(ThrowableRunnable {
+        psiDocumentManagerBase.commitDocument(document)
+    })
+}
+
 fun enableHints(enable: Boolean) =
     HintType.values().forEach { it.option.set(enable) }