Fix performanceTests for 192; Add perfCounters

compiler/frontend/src/org/jetbrains/kotlin/util/PerformanceCounter.kt

@@ -41,6 +41,16 @@ abstract class PerformanceCounter protected constructor(val name: String) {
             countersCopy.forEach { it.report(consumer) }
         }
 
+        fun report(consumer: (String, Int, Long) -> Unit) {
+            val countersCopy = synchronized(allCounters) {
+                allCounters.toTypedArray()
+            }
+            countersCopy.forEach { it.report(consumer) }
+        }
+
+        val numberOfCounters: Int
+            get() = synchronized(allCounters) { allCounters.size }
+
         fun setTimeCounterEnabled(enable: Boolean) {
             enabled = enable
         }
@@ -117,6 +127,9 @@ abstract class PerformanceCounter protected constructor(val name: String) {
             consumer("$name performed $count times, total time $millis ms")
         }
     }
+
+    fun report(consumer: (String, Int, Long) -> Unit) =
+        consumer(name, count, totalTimeNanos)
 }
 
 private class SimpleCounter(name: String) : PerformanceCounter(name) {

idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceProjectsTest.kt

@@ -13,19 +13,10 @@ import com.intellij.codeInsight.lookup.LookupElement
 import com.intellij.ide.highlighter.ModuleFileType
 import com.intellij.ide.startup.impl.StartupManagerImpl
 import com.intellij.idea.IdeaTestApplication
-import com.intellij.lang.ExternalAnnotatorsFilter
-import com.intellij.lang.LanguageAnnotators
-import com.intellij.lang.StdLanguages
-import com.intellij.lang.injection.InjectedLanguageManager
-import com.intellij.lang.java.JavaLanguage
 import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.application.runWriteAction
-import com.intellij.openapi.editor.Document
 import com.intellij.openapi.editor.EditorFactory
-import com.intellij.openapi.externalSystem.importing.ImportSpecBuilder
-import com.intellij.openapi.externalSystem.service.execution.ProgressExecutionMode
 import com.intellij.openapi.externalSystem.service.project.manage.ExternalProjectsManagerImpl
-import com.intellij.openapi.externalSystem.util.ExternalSystemUtil
 import com.intellij.openapi.fileEditor.FileDocumentManager
 import com.intellij.openapi.fileEditor.FileEditorManager
 import com.intellij.openapi.fileEditor.impl.FileEditorManagerImpl
@@ -34,48 +25,38 @@ import com.intellij.openapi.module.ModuleTypeId
 import com.intellij.openapi.project.DumbService
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.project.ex.ProjectManagerEx
-import com.intellij.openapi.project.guessProjectDir
 import com.intellij.openapi.project.impl.ProjectImpl
 import com.intellij.openapi.projectRoots.JavaSdk
 import com.intellij.openapi.projectRoots.ProjectJdkTable
 import com.intellij.openapi.projectRoots.Sdk
 import com.intellij.openapi.projectRoots.impl.JavaAwareProjectJdkTableImpl
-import com.intellij.openapi.roots.FileIndexFacade
 import com.intellij.openapi.roots.ProjectRootManager
 import com.intellij.openapi.startup.StartupManager
 import com.intellij.openapi.vcs.changes.ChangeListManager
 import com.intellij.openapi.vcs.changes.ChangeListManagerImpl
 import com.intellij.openapi.vfs.LocalFileSystem
-import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.openapi.vfs.VirtualFileManager
 import com.intellij.psi.PsiDocumentManager
 import com.intellij.psi.PsiFile
 import com.intellij.psi.PsiManager
-import com.intellij.psi.impl.search.IndexPatternBuilder
-import com.intellij.psi.impl.source.resolve.reference.ReferenceProvidersRegistry
-import com.intellij.psi.search.FilenameIndex
-import com.intellij.psi.search.GlobalSearchScope
-import com.intellij.psi.xml.XmlFileNSInfoProvider
 import com.intellij.testFramework.*
 import com.intellij.testFramework.fixtures.impl.CodeInsightTestFixtureImpl
 import com.intellij.util.ArrayUtilRt
 import com.intellij.util.ThrowableRunnable
 import com.intellij.util.indexing.UnindexedFilesUpdater
 import com.intellij.util.io.exists
-import com.intellij.xml.XmlSchemaProvider
-import org.jetbrains.kotlin.idea.KotlinLanguage
 import org.jetbrains.kotlin.idea.configuration.getModulesWithKotlinFiles
 import org.jetbrains.kotlin.idea.core.script.ScriptConfigurationManager
-import org.jetbrains.kotlin.idea.core.script.ScriptDefinitionsManager
-import org.jetbrains.kotlin.idea.core.script.settings.KotlinScriptingSettings
-import org.jetbrains.kotlin.idea.core.util.toPsiFile
 import org.jetbrains.kotlin.idea.framework.KotlinSdkType
+import org.jetbrains.kotlin.idea.perf.Stats.Companion.runAndMeasure
 import org.jetbrains.kotlin.idea.project.getAndCacheLanguageLevelByDependencies
 import org.jetbrains.kotlin.idea.test.ConfigLibraryUtil
 import org.jetbrains.kotlin.idea.test.invalidateLibraryCache
 import org.jetbrains.kotlin.idea.testFramework.*
-import org.jetbrains.plugins.gradle.service.project.GradleProjectOpenProcessor
+import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.cleanupCaches
-import org.jetbrains.plugins.gradle.util.GradleConstants
+import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.isAKotlinScriptFile
+import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.openFileInEditor
+import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.openFixture
 import java.io.File
 import java.nio.file.Paths
 
@@ -161,7 +142,9 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
         simpleModule: Boolean = false,
         fast: Boolean = false
     ): Project {
-        val projectPath = File("$path").canonicalPath
+        val projectPath = File(path).canonicalPath
+
+        assertTrue("path $path does not exist, check README.md", File(projectPath).exists())
 
         val warmUpIterations = if (fast) 0 else 1
         val iterations = if (fast) 1 else 3
@@ -177,8 +160,15 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
             test = {
                 val project = if (!simpleModule) {
                     val project = projectManagerEx.loadProject(name, path)
-                    assertNotNull(project)
+                    assertNotNull("project $name at $path is not loaded", project)
-                    //projectManagerEx.openTestProject(project!!)
+                    val projectRootManager = ProjectRootManager.getInstance(project!!)
+
+                    runWriteAction {
+                        with(projectRootManager) {
+                            projectSdk = jdk18
+                        }
+                    }
+                    assertTrue("project $name at $path is not opened", projectManagerEx.openProject(project))
                     project
                 } else {
                     val project = projectManagerEx.loadAndOpenProject(projectPath)!!
@@ -195,9 +185,12 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
 
                 with(StartupManager.getInstance(project) as StartupManagerImpl) {
                     scheduleInitialVfsRefresh()
+                    runStartupActivities()
                     runPostStartupActivities()
                 }
 
+                logMessage { "project $name is ${if (project.isInitialized) "initialized" else "not initialized"}" }
+
                 with(ChangeListManager.getInstance(project) as ChangeListManagerImpl) {
                     waitUntilRefreshed()
                 }
@@ -219,10 +212,10 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
                         }
                     }.get()
 
-                    assertTrue(
+                    val modules = ModuleManager.getInstance(project).modules
-                        "project has to have at least one module",
+                    assertTrue("project has to have at least one module", modules.isNotEmpty())
-                        ModuleManager.getInstance(project).modules.isNotEmpty()
+
-                    )
+                    logMessage { "modules of $name: ${modules.map { m -> m.name }}" }
 
                     lastProject = project
                     VirtualFileManager.getInstance().syncRefresh()
@@ -231,7 +224,7 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
                         project.save()
                     }
 
-                    println("# project '$name' successfully opened")
+                    logMessage { "project '$name' successfully opened" }
 
                     // close all project but last - we're going to return and use it further
                     if (counter < warmUpIterations + iterations - 1) {
@@ -250,54 +243,28 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
 
             dispatchAllInvocationEvents()
 
+            logMessage { "project $name is ${if (project.isInitialized) "initialized" else "not initialized"}" }
+
             with(DumbService.getInstance(project)) {
                 queueTask(UnindexedFilesUpdater(project))
                 completeJustSubmittedTasks()
             }
             dispatchAllInvocationEvents()
 
-            enableAnnotatorsAndLoadDefinitions(project)
+            Fixture.enableAnnotatorsAndLoadDefinitions(project)
         }
 
         return lastProject!!
     }
 
-    fun openGradleProject(projectPath: String, project: Project) {
-        dispatchAllInvocationEvents()
-
-        val virtualFile = LocalFileSystem.getInstance().refreshAndFindFileByPath(projectPath)!!
-
-        FileDocumentManager.getInstance().saveAllDocuments()
-
-        val path = Paths.get(virtualFile.path)
-        GradleProjectOpenProcessor.openGradleProject(project, null, path)
-
-        dispatchAllInvocationEvents()
-        runInEdtAndWait {
-            PlatformTestUtil.saveProject(project)
-        }
-    }
-
     private fun refreshGradleProjectIfNeeded(projectPath: String, project: Project) {
         if (listOf("build.gradle.kts", "build.gradle").map { name -> Paths.get(projectPath, name).exists() }.find { e -> e } != true) return
 
-        ExternalProjectsManagerImpl.getInstance(project).setStoreExternally(false)
+        ExternalProjectsManagerImpl.getInstance(project).setStoreExternally(true)
 
-        GradleProjectOpenProcessor.openGradleProject(project, null, Paths.get(projectPath))
+        refreshGradleProject(projectPath, project)
 
-        val gradleArguments = System.getProperty("kotlin.test.gradle.import.arguments")
+        //ExternalProjectsManagerImpl.getInstance(project).setStoreExternally(false)
-        ExternalSystemUtil.refreshProjects(
-            ImportSpecBuilder(project, GradleConstants.SYSTEM_ID)
-                .forceWhenUptodate()
-                .useDefaultCallback()
-                .use(ProgressExecutionMode.MODAL_SYNC)
-                .also {
-                    gradleArguments?.run(it::withArguments)
-                }
-        )
-
-        dispatchAllInvocationEvents()
-        ExternalProjectsManagerImpl.getInstance(project).setStoreExternally(false)
         dispatchAllInvocationEvents()
 
         // WARNING: [VD] DO NOT SAVE PROJECT AS IT COULD PERSIST WRONG MODULES INFO
@@ -307,59 +274,6 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
 //        }
     }
 
-    /**
-     * @param lookupElements perform basic autocompletion and check presence of suggestion if elements are not empty
-     */
-    fun typeAndCheckLookup(
-        project: Project,
-        fileName: String,
-        marker: String,
-        insertString: String,
-        surroundItems: String = "\n",
-        lookupElements: List<String>,
-        revertChangesAtTheEnd: Boolean = true
-    ) {
-        val fileInEditor = openFileInEditor(project, fileName)
-        val virtualFile = fileInEditor.psiFile.virtualFile
-        val editor = EditorFactory.getInstance().getEditors(fileInEditor.document, project)[0]
-        val fixture = Fixture(project, editor, virtualFile)
-
-        val initialText = editor.document.text
-        try {
-            if (isAKotlinScriptFile(fileName)) {
-                ScriptConfigurationManager.updateScriptDependenciesSynchronously(fileInEditor.psiFile, project)
-            }
-
-            val tasksIdx = fileInEditor.document.text.indexOf(marker)
-            assertTrue(tasksIdx > 0)
-            editor.caretModel.moveToOffset(tasksIdx + marker.length + 1)
-
-            for (surroundItem in surroundItems) {
-                EditorTestUtil.performTypingAction(editor, surroundItem)
-            }
-
-            editor.caretModel.moveToOffset(editor.caretModel.offset - 1)
-            fixture.type(insertString)
-
-            if (lookupElements.isNotEmpty()) {
-                val elements = fixture.complete()
-                val items = elements.map { it.lookupString }.toList()
-                for (lookupElement in lookupElements) {
-                    assertTrue("'$lookupElement' has to be present in items", items.contains(lookupElement))
-                }
-            }
-        } finally {
-            // TODO: [VD] revert ?
-            //fixture.performEditorAction(IdeActions.SELECTED_CHANGES_ROLLBACK)
-            if (revertChangesAtTheEnd) {
-                runWriteAction {
-                    editor.document.setText(initialText)
-                    commitDocument(project, editor.document)
-                }
-            }
-        }
-    }
-
     fun perfTypeAndAutocomplete(
         stats: Stats,
         fileName: String,
@@ -394,19 +308,17 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
             iterations = 15,
             testName = "typeAndAutocomplete ${notePrefix(note)}$fileName",
             setUp = {
-                val fileInEditor = openFileInEditor(project, fileName)
+                val fixture = openFixture(project, fileName)
-                val file = fileInEditor.psiFile
+                val editor = fixture.editor
-                val virtualFile = file.virtualFile
+
-                val editor = EditorFactory.getInstance().getEditors(fileInEditor.document, project)[0]
-                val fixture = Fixture(project, editor, virtualFile)
                 val initialText = editor.document.text
                 if (isAKotlinScriptFile(fileName)) {
                     runAndMeasure("update script dependencies for $fileName") {
-                        ScriptConfigurationManager.updateScriptDependenciesSynchronously(file, project)
+                        ScriptConfigurationManager.updateScriptDependenciesSynchronously(fixture.psiFile, project)
                     }
                 }
 
-                val tasksIdx = fileInEditor.document.text.indexOf(marker)
+                val tasksIdx = editor.document.text.indexOf(marker)
                 assertTrue("marker '$marker' not found in $fileName", tasksIdx > 0)
                 if (typeAfterMarker) {
                     editor.caretModel.moveToOffset(tasksIdx + marker.length + 1)
@@ -443,24 +355,7 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
                     }
                 } finally {
                     it.setUpValue?.let { pair ->
-                        val document = pair.second.document
+                        pair.second.revertChanges(revertChangesAtTheEnd, pair.first)
-                        val file = pair.second.vFile
-                        val text = pair.first
-
-                        try {
-                            if (revertChangesAtTheEnd) {
-                                runWriteAction {
-                                    // TODO: [VD] revert ?
-                                    //editorFixture.performEditorAction(IdeActions.SELECTED_CHANGES_ROLLBACK)
-                                    document.setText(text)
-                                    saveDocument(document)
-                                    commitDocument(project, document)
-                                }
-                                dispatchAllInvocationEvents()
-                            }
-                        } finally {
-                            cleanupCaches(project, file)
-                        }
                     }
                     commitAllDocuments()
                 }
@@ -468,36 +363,88 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
         )
     }
 
-    protected fun enableAnnotatorsAndLoadDefinitions() = enableAnnotatorsAndLoadDefinitions(myProject!!)
+    fun perfTypeAndHighlight(
+        stats: Stats,
+        fileName: String,
+        marker: String,
+        insertString: String,
+        surroundItems: String = "\n",
+        typeAfterMarker: Boolean = true,
+        revertChangesAtTheEnd: Boolean = true,
+        note: String = ""
+    ) = perfTypeAndHighlight(
+        myProject!!, stats, fileName, marker, insertString, surroundItems,
+        typeAfterMarker = typeAfterMarker,
+        revertChangesAtTheEnd = revertChangesAtTheEnd, note = note
+    )
 
-    protected fun enableAnnotatorsAndLoadDefinitions(project: Project) {
+    fun perfTypeAndHighlight(
-        ReferenceProvidersRegistry.getInstance() // pre-load tons of classes
+        project: Project,
-        InjectedLanguageManager.getInstance(project) // zillion of Dom Sem classes
+        stats: Stats,
-        with(LanguageAnnotators.INSTANCE) {
+        fileName: String,
-            allForLanguage(JavaLanguage.INSTANCE) // pile of annotator classes loads
+        marker: String,
-            allForLanguage(StdLanguages.XML)
+        insertString: String,
-            allForLanguage(KotlinLanguage.INSTANCE)
+        surroundItems: String = "\n",
-        }
+        typeAfterMarker: Boolean = true,
-        DaemonAnalyzerTestCase.assertTrue(
+        revertChangesAtTheEnd: Boolean = true,
-            "side effect: to load extensions",
+        note: String = ""
-            ProblemHighlightFilter.EP_NAME.extensions.toMutableList()
+    ) {
-                .plus(ImplicitUsageProvider.EP_NAME.extensions)
+        stats.perfTest<Pair<String, Fixture>, List<HighlightInfo>>(
-                .plus(XmlSchemaProvider.EP_NAME.extensions)
+            warmUpIterations = 8,
-                .plus(XmlFileNSInfoProvider.EP_NAME.extensions)
+            iterations = 15,
-                .plus(ExternalAnnotatorsFilter.EXTENSION_POINT_NAME.extensions)
+            testName = "typeAndHighlight ${notePrefix(note)}$fileName",
-                .plus(IndexPatternBuilder.EP_NAME.extensions).isNotEmpty()
+            setUp = {
+                val fixture = openFixture(project, fileName)
+                val editor = fixture.editor
+
+                val initialText = editor.document.text
+                if (isAKotlinScriptFile(fileName)) {
+                    runAndMeasure("update script dependencies for $fileName") {
+                        ScriptConfigurationManager.updateScriptDependenciesSynchronously(fixture.psiFile, project)
+                    }
+                }
+
+                val tasksIdx = editor.document.text.indexOf(marker)
+                assertTrue("marker '$marker' not found in $fileName", tasksIdx > 0)
+                if (typeAfterMarker) {
+                    editor.caretModel.moveToOffset(tasksIdx + marker.length + 1)
+                } else {
+                    editor.caretModel.moveToOffset(tasksIdx - 1)
+                }
+
+                for (surroundItem in surroundItems) {
+                    EditorTestUtil.performTypingAction(editor, surroundItem)
+                }
+
+                editor.caretModel.moveToOffset(editor.caretModel.offset - if (typeAfterMarker) 1 else 2)
+
+                if (!typeAfterMarker) {
+                    for (surroundItem in surroundItems) {
+                        EditorTestUtil.performTypingAction(editor, surroundItem)
+                    }
+                    editor.caretModel.moveToOffset(editor.caretModel.offset - 2)
+                }
+
+                fixture.type(insertString)
+
+                it.setUpValue = Pair(initialText, fixture)
+            },
+            test = {
+                val fixture = it.setUpValue!!.second
+                it.value = fixture.doHighlighting()
+            },
+            tearDown = {
+                it.value?.let { list ->
+                    assertNotEmpty(list)
+                }
+                it.setUpValue?.let { pair ->
+                    pair.second.revertChanges(revertChangesAtTheEnd, pair.first)
+                }
+                commitAllDocuments()
+            }
         )
-
-        // side effect: to load script definitions"
-        val scriptDefinitionsManager = ScriptDefinitionsManager.getInstance(project)
-        scriptDefinitionsManager.getAllDefinitions()
-        dispatchAllInvocationEvents()
-
-        assertTrue(scriptDefinitionsManager.isReady())
-        assertFalse(KotlinScriptingSettings.getInstance(project).isAutoReloadEnabled)
     }
 
-
     private fun initKotlinProject(
         project: Project,
         projectPath: String,
@@ -553,12 +500,6 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
         }
     }
 
-    fun highlightFile(psiFile: PsiFile): List<HighlightInfo> {
-        return highlightFile {
-            highlightFile(myProject!!, psiFile)
-        }
-    }
-
     fun <T> highlightFile(block: () -> T): T {
         var value: T? = null
         IdentifierHighlighterPassFactory.doWithHighlightingEnabled {
@@ -602,71 +543,5 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
         if (note.endsWith("/")) note else "$note "
     } else ""
 
-    // quite simple impl - good so far
-    fun isAKotlinScriptFile(fileName: String) = fileName.endsWith(".kts")
-
-    fun cleanupCaches(project: Project, vFile: VirtualFile) {
-        commitAllDocuments()
-        FileEditorManager.getInstance(project).closeFile(vFile)
-        PsiManager.getInstance(project).dropPsiCaches()
-    }
-
-    fun openFileInEditor(project: Project, name: String): EditorFile {
-        val fileDocumentManager = FileDocumentManager.getInstance()
-        val fileEditorManager = FileEditorManager.getInstance(project)
-
-        val psiFile = projectFileByName(project, name)
-        val vFile = psiFile.virtualFile
-
-        assertTrue("file $vFile is not indexed yet", FileIndexFacade.getInstance(project).isInContent(vFile))
-
-        runInEdtAndWait {
-            fileEditorManager.openFile(vFile, true)
-        }
-        val document = fileDocumentManager.getDocument(vFile)!!
-
-        assertNotNull("doc not found for $vFile", EditorFactory.getInstance().getEditors(document))
-        assertTrue("expected non empty doc", document.text.isNotEmpty())
-
-        val offset = psiFile.textOffset
-        assertTrue("side effect: to load the text", offset >= 0)
-
-        waitForAllEditorsFinallyLoaded(project)
-
-        return EditorFile(psiFile = psiFile, document = document)
-    }
-
-    private fun baseName(name: String): String {
-        val index = name.lastIndexOf("/")
-        return if (index > 0) name.substring(index + 1) else name
-    }
-
-    private fun projectFileByName(project: Project, name: String): PsiFile {
-        val fileManager = VirtualFileManager.getInstance()
-        val url = "${project.guessProjectDir()}/$name"
-        val virtualFile = fileManager.refreshAndFindFileByUrl(url)
-        if (virtualFile != null) {
-            return virtualFile!!.toPsiFile(project)!!
-        }
-
-        val baseFileName = baseName(name)
-        val projectBaseName = baseName(project.name)
-
-        val virtualFiles = FilenameIndex.getVirtualFilesByName(
-            project,
-            baseFileName, true,
-            GlobalSearchScope.projectScope(project)
-        )
-            .filter { it.canonicalPath?.contains("/$projectBaseName/$name") ?: false }.toList()
-
-        assertEquals(
-            "expected the only file with name '$name'\n, it were: [${virtualFiles.map { it.canonicalPath }.joinToString("\n")}]",
-            1,
-            virtualFiles.size
-        )
-        return virtualFiles.iterator().next().toPsiFile(project)!!
-    }
-
-    data class EditorFile(val psiFile: PsiFile, val document: Document)
 
 }

idea/performanceTests/org/jetbrains/kotlin/idea/perf/PerformanceProjectsTest.kt

@@ -7,7 +7,6 @@ package org.jetbrains.kotlin.idea.perf
 
 import com.intellij.codeInsight.daemon.impl.HighlightInfo
 import com.intellij.lang.annotation.AnnotationHolder
-import com.intellij.openapi.editor.EditorFactory
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.util.Disposer
 import com.intellij.psi.PsiElement
@@ -15,7 +14,12 @@ import com.intellij.testFramework.propertyBased.MadTestingUtil
 import org.jetbrains.kotlin.idea.core.script.ScriptConfigurationManager
 import org.jetbrains.kotlin.idea.highlighter.KotlinPsiChecker
 import org.jetbrains.kotlin.idea.highlighter.KotlinPsiCheckerAndHighlightingUpdater
+import org.jetbrains.kotlin.idea.perf.Stats.Companion.TEST_KEY
+import org.jetbrains.kotlin.idea.perf.Stats.Companion.runAndMeasure
+import org.jetbrains.kotlin.idea.perf.Stats.Companion.tcSuite
 import org.jetbrains.kotlin.idea.testFramework.Fixture
+import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.cleanupCaches
+import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.isAKotlinScriptFile
 import java.util.concurrent.atomic.AtomicLong
 import kotlin.test.assertNotEquals
 
@@ -72,11 +76,28 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
             stats.use {
                 perfOpenKotlinProject(it)
 
-                perfHighlightFile("compiler/psi/src/org/jetbrains/kotlin/psi/KtFile.kt", stats = it)
-
                 perfHighlightFile("idea/idea-analysis/src/org/jetbrains/kotlin/idea/util/PsiPrecedences.kt", stats = it)
 
                 perfHighlightFile("compiler/psi/src/org/jetbrains/kotlin/psi/KtElement.kt", stats = it)
+
+                perfHighlightFile("compiler/psi/src/org/jetbrains/kotlin/psi/KtFile.kt", stats = it)
+
+                perfTypeAndHighlight(
+                    it,
+                    "compiler/psi/src/org/jetbrains/kotlin/psi/KtFile.kt",
+                    "override fun getDeclarations(): List<KtDeclaration> {",
+                    "val q = import",
+                    note = "in-method getDeclarations-import"
+                )
+
+                perfTypeAndHighlight(
+                    it,
+                    "compiler/psi/src/org/jetbrains/kotlin/psi/KtFile.kt",
+                    "override fun getDeclarations(): List<KtDeclaration> {",
+                    "val q = import",
+                    typeAfterMarker = false,
+                    note = "out-of-method import"
+                )
             }
         }
     }
@@ -207,7 +228,7 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
             highlightFile {
                 val testName = "fileAnalysis ${notePrefix(note)}${simpleFilename(fileName)}"
                 val extraStats = Stats("${stats.name} $testName")
-                val extraTimingsNs = mutableListOf<Long>()
+                val extraTimingsNs = mutableListOf<Map<String, Any>?>()
 
                 val warmUpIterations = 3
                 val iterations = 10
@@ -223,14 +244,12 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
 
                 extraStats.printWarmUpTimings(
                     "annotator",
-                    Array(warmUpIterations, init = { null }),
+                    extraTimingsNs.take(warmUpIterations).toTypedArray()
-                    extraTimingsNs.take(warmUpIterations).toLongArray()
                 )
 
                 extraStats.appendTimings(
                     "annotator",
-                    Array(iterations, init = { null }),
+                    extraTimingsNs.drop(warmUpIterations).toTypedArray()
-                    extraTimingsNs.drop(warmUpIterations).toLongArray()
                 )
             }
         } finally {
@@ -251,16 +270,11 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
         fileName: String
     ): (TestData<Fixture, Pair<Long, List<HighlightInfo>>>) -> Unit {
         return {
-            val fileInEditor = openFileInEditor(project, fileName)
+            val fixture = Fixture.openFixture(project, fileName)
-
-            val file = fileInEditor.psiFile
-            val virtualFile = file.virtualFile
-            val editor = EditorFactory.getInstance().getEditors(fileInEditor.document, project)[0]
-            val fixture = Fixture(project, editor, virtualFile)
 
             // Note: Kotlin scripts require dependencies to be loaded
             if (isAKotlinScriptFile(fileName)) {
-                ScriptConfigurationManager.updateScriptDependenciesSynchronously(fileInEditor.psiFile, project)
+                ScriptConfigurationManager.updateScriptDependenciesSynchronously(fixture.psiFile, project)
             }
 
             resetTimestamp()
@@ -277,7 +291,7 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
     }
 
     fun perfKtsFileAnalysisTearDown(
-        extraTimingsNs: MutableList<Long>,
+        extraTimingsNs: MutableList<Map<String, Any>?>,
         project: Project
     ): (TestData<Fixture, Pair<Long, List<HighlightInfo>>>) -> Unit {
         return {
@@ -286,7 +300,7 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
                     assertTrue(v.second.isNotEmpty())
                     assertNotEquals(0, timer.get())
 
-                    extraTimingsNs.add(timer.get() - v.first)
+                    extraTimingsNs.add(mapOf(TEST_KEY to (timer.get() - v.first)))
 
                 }
                 cleanupCaches(project, fixture.vFile)

idea/performanceTests/org/jetbrains/kotlin/idea/perf/Stats.kt

@@ -6,6 +6,8 @@
 package org.jetbrains.kotlin.idea.perf
 
 import org.jetbrains.kotlin.idea.perf.WholeProjectPerformanceTest.Companion.nsToMs
+import org.jetbrains.kotlin.idea.testFramework.logMessage
+import org.jetbrains.kotlin.util.PerformanceCounter
 import java.io.*
 import kotlin.math.pow
 import kotlin.math.sqrt
@@ -16,7 +18,10 @@ import kotlin.math.ln
 import kotlin.system.measureTimeMillis
 import kotlin.test.assertEquals
 
+typealias StatInfos = Map<String, Any>?
+
 class Stats(val name: String = "", val header: Array<String> = arrayOf("Name", "ValueMS", "StdDev")) : Closeable {
+
     private val perfTestRawDataMs = mutableListOf<Long>()
 
     private val statsFile: File = File("build/stats${statFilePrefix()}.csv").absoluteFile
@@ -27,40 +32,69 @@ class Stats(val name: String = "", val header: Array<String> = arrayOf("Name", "
         statsOutput = statsFile.bufferedWriter()
 
         statsOutput.appendln(header.joinToString())
+
+        PerformanceCounter.setTimeCounterEnabled(true)
     }
 
     private fun statFilePrefix() = if (name.isNotEmpty()) "-${name.toLowerCase().replace(' ', '-').replace('/', '_')}" else ""
 
-    private fun append(id: String, timingsNs: LongArray) {
+    private fun append(id: String, statInfosArray: Array<StatInfos>) {
-        val meanNs = timingsNs.average()
+        val timingsMs = statInfosArray.map { info -> info?.let { it[TEST_KEY] as Long }?.nsToMs ?: 0L }.toLongArray()
-        val meanMs = meanNs.toLong().nsToMs
 
-        val stdDevMs = if (timingsNs.size > 1) (sqrt(
+        val calcMean = calcMean(timingsMs)
-            timingsNs.fold(0.0,
-                           { accumulator, next -> accumulator + (1.0 * (next - meanNs)).pow(2) })
-        ) / (timingsNs.size - 1)).toLong().nsToMs
-        else 0
-
-        val geomMeanMs = geomMean(timingsNs.toList()).nsToMs
 
         for (v in listOf(
-            Triple("mean", "", meanMs),
+            Triple("mean", "", calcMean.mean.toLong()),
-            Triple("stdDev", " stdDev", stdDevMs),
+            Triple("stdDev", " stdDev", calcMean.stdDev.toLong()),
-            Triple("geomMean", "geomMean", geomMeanMs)
+            Triple("geomMean", "geomMean", calcMean.geomMean.toLong())
         )) {
-            println("##teamcity[testStarted name='$id : ${v.first}' captureStandardOutput='true']")
+            val n = "$id : ${v.first}"
-            println("##teamcity[buildStatisticValue key='$id${v.second}' value='${v.third}']")
+
-            println("##teamcity[testFinished name='$id : ${v.first}' duration='${v.third}']")
+            printTestStarted(n)
+            printStatValue("$id${v.second}", v.third)
+            printTestFinished(n, v.third, includeStatValue = false)
         }
 
-        perfTestRawDataMs.addAll(timingsNs.map { it.nsToMs }.toList())
+        statInfosArray.filterNotNull()
-        append(arrayOf(id, meanMs, stdDevMs))
+            .map { it.keys }
+            .fold(mutableSetOf<String>(), { acc, keys -> acc.addAll(keys); acc })
+            .filter { it != TEST_KEY && it != ERROR_KEY }
+            .sorted().forEach { perfCounterName ->
+                val values = statInfosArray.map { (it?.get(perfCounterName) as Long) }.toLongArray()
+                val statInfoMean = calcMean(values)
+
+                val n = "$id : $perfCounterName"
+                val mean = statInfoMean.mean.toLong()
+
+                val shortName = if (perfCounterName.endsWith(": time")) n.removeSuffix(": time") else null
+
+                shortName?.let { printTestStarted(it) }
+                printStatValue(n, mean)
+                shortName?.let { printTestFinished(it, mean, includeStatValue = false) }
+            }
+
+        perfTestRawDataMs.addAll(timingsMs.toList())
+        append(arrayOf(id, calcMean.mean, calcMean.stdDev))
     }
 
+    private fun calcMean(values: LongArray): Mean {
+        val mean = values.average()
+
+        val stdDev = if (values.size > 1) (sqrt(
+            values.fold(0.0,
+                        { accumulator, next -> accumulator + (1.0 * (next - mean)).pow(2) })
+        ) / (values.size - 1))
+        else 0.0
+
+        val geomMean = geomMean(values.toList())
+
+        return Mean(mean, stdDev, geomMean)
+    }
+
+    data class Mean(val mean: Double, val stdDev: Double, val geomMean: Double)
+
     private fun append(values: Array<Any>) {
-        if (values.size != header.size) {
+        require(values.size == header.size) { "Expected ${header.size} values, actual ${values.size} values" }
-            throw IllegalArgumentException("Expected ${header.size} values, actual ${values.size} values")
-        }
         with(statsOutput) {
             appendln(values.joinToString { it.toString() })
             flush()
@@ -76,67 +110,92 @@ class Stats(val name: String = "", val header: Array<String> = arrayOf("Name", "
         testName: String,
         warmUpIterations: Int = 3,
         iterations: Int = 10,
-        setUp: (TestData<K, T>) -> Unit = { null },
+        setUp: (TestData<K, T>) -> Unit = { },
         test: (TestData<K, T>) -> Unit,
-        tearDown: (TestData<K, T>) -> Unit = { null }
+        tearDown: (TestData<K, T>) -> Unit = { }
     ) {
-        val namePrefix = "$testName"
-        val timingsNs = LongArray(iterations)
-        val errors = Array<Throwable?>(iterations, init = { null })
 
-        tcSuite(namePrefix) {
+        tcSuite(testName) {
-            warmUpPhase(warmUpIterations, namePrefix, setUp, test, tearDown)
+            warmUpPhase(warmUpIterations, testName, setUp, test, tearDown)
+            val statInfoArray = mainPhase(iterations, testName, setUp, test, tearDown)
 
-            mainPhase(iterations, setUp, test, tearDown, timingsNs, namePrefix, errors)
+            assertEquals(iterations, statInfoArray.size)
+            appendTimings(testName, statInfoArray)
+        }
+    }
 
-            assertEquals(iterations, timingsNs.size)
+    private fun _printTimings(
-            appendTimings(namePrefix, errors, timingsNs)
+        prefix: String,
+        statInfoArray: Array<StatInfos>,
+        attemptFn: (Int) -> String = { attempt -> "#$attempt" }
+    ) {
+        for (statInfoIndex in statInfoArray.withIndex()) {
+            val attempt = statInfoIndex.index
+            val statInfo = statInfoIndex.value!!
+            val n = "$name: $prefix ${attemptFn(attempt)}"
+            printTestStarted(n)
+            val t = statInfo[ERROR_KEY] as? Throwable
+            if (t != null) printTestFinished(n, t) else {
+                for ((k, v) in statInfo) {
+                    if (k == TEST_KEY) continue
+                    printStatValue("$n $k", v)
+                }
+
+                printTestFinished(n, (statInfo[TEST_KEY] as Long).nsToMs)
+            }
         }
     }
 
     fun printWarmUpTimings(
         prefix: String,
-        errors: Array<Throwable?>,
+        warmUpStatInfosArray: Array<StatInfos>
-        warmUpTimingsNs: LongArray
     ) {
-        assertEquals(warmUpTimingsNs.size, errors.size)
+        _printTimings(prefix, warmUpStatInfosArray) { attempt -> "warm-up #$attempt" }
-        for (timing in warmUpTimingsNs.withIndex()) {
-            val attempt = timing.index
-            val n = "$name: $prefix warm-up #$attempt"
-            printTestStarted(n)
-            val t = errors[attempt]
-            if (t != null) printTestFinished(n, t!!) else printTestFinished(n, timing.value.nsToMs)
-        }
     }
 
     fun appendTimings(
         prefix: String,
-        errors: Array<Throwable?>,
+        statInfosArray: Array<StatInfos>
-        timingsNs: LongArray
     ) {
-        assertEquals(timingsNs.size, errors.size)
+        _printTimings(prefix, statInfosArray)
         val namePrefix = "$name: $prefix"
-        for (attempt in 0 until timingsNs.size) {
+        append(namePrefix, statInfosArray)
-            val n = "$namePrefix #$attempt"
-            printTestStarted(n)
-            if (errors[attempt] != null) {
-                tcPrintErrors(n, listOf(errors[attempt]!!))
-            }
-            val spentMs = timingsNs[attempt].nsToMs
-            printTestFinished(n, spentMs)
-        }
-
-        append(namePrefix, timingsNs)
     }
 
     private fun <K, T> mainPhase(
         iterations: Int,
+        namePrefix: String,
         setUp: (TestData<K, T>) -> Unit,
         test: (TestData<K, T>) -> Unit,
-        tearDown: (TestData<K, T>) -> Unit,
+        tearDown: (TestData<K, T>) -> Unit
-        timingsNs: LongArray,
+    ): Array<StatInfos> {
+        val statInfosArray = Array<StatInfos>(iterations) { null }
+        _phase(iterations, setUp, statInfosArray, test, namePrefix, tearDown)
+        return statInfosArray
+    }
+
+    private fun <K, T> warmUpPhase(
+        warmUpIterations: Int,
         namePrefix: String,
-        errors: Array<Throwable?>
+        setUp: (TestData<K, T>) -> Unit,
+        test: (TestData<K, T>) -> Unit,
+        tearDown: (TestData<K, T>) -> Unit
+    ) {
+        val warmUpStatInfosArray = Array<StatInfos>(warmUpIterations) { null }
+        _phase(warmUpIterations, setUp, warmUpStatInfosArray, test, namePrefix, tearDown)
+
+        printWarmUpTimings(namePrefix, warmUpStatInfosArray)
+
+        warmUpStatInfosArray.filterNotNull().map { it[ERROR_KEY] as? Exception }.firstOrNull()?.let { throw it }
+    }
+
+    private fun <K, T> _phase(
+        iterations: Int,
+        setUp: (TestData<K, T>) -> Unit,
+        statInfosArray: Array<StatInfos>,
+        test: (TestData<K, T>) -> Unit,
+        namePrefix: String,
+        tearDown: (TestData<K, T>) -> Unit
     ) {
         val testData = TestData<K, T>(null, null)
         try {
@@ -147,21 +206,31 @@ class Stats(val name: String = "", val header: Array<String> = arrayOf("Name", "
                 val setUpMillis = measureTimeMillis {
                     setUp(testData)
                 }
-                println("-- setup took $setUpMillis ms")
+                logMessage { "setup took $setUpMillis ms" }
+                val valueMap = HashMap<String, Any>(2 * PerformanceCounter.numberOfCounters + 1)
+                statInfosArray[attempt] = valueMap
                 try {
-                    val spentNs = measureNanoTime {
+                    valueMap[TEST_KEY] = measureNanoTime {
                         test(testData)
                     }
-                    timingsNs[attempt] = spentNs
+                    PerformanceCounter.report { name, counter, nanos ->
+                        valueMap["counter \"$name\": count"] = counter.toLong()
+                        valueMap["counter \"$name\": time"] = nanos.nsToMs
+                    }
+
                 } catch (t: Throwable) {
                     println("# error at $namePrefix #$attempt:")
                     t.printStackTrace()
-                    errors[attempt] = t
+                    valueMap[ERROR_KEY] = t
                 } finally {
-                    val tearDownMillis = measureTimeMillis {
+                    try {
-                        tearDown(testData)
+                        val tearDownMillis = measureTimeMillis {
+                            tearDown(testData)
+                        }
+                        logMessage { "tearDown took $tearDownMillis ms" }
+                    } finally {
+                        PerformanceCounter.resetAllCounters()
                     }
-                    println("-- tearDown took $tearDownMillis ms")
                 }
             }
         } catch (t: Throwable) {
@@ -170,65 +239,6 @@ class Stats(val name: String = "", val header: Array<String> = arrayOf("Name", "
         }
     }
 
-    private fun <K, T> warmUpPhase(
-        warmUpIterations: Int,
-        namePrefix: String,
-        setUp: (TestData<K, T>) -> Unit,
-        test: (TestData<K, T>) -> Unit,
-        tearDown: (TestData<K, T>) -> Unit
-    ) {
-        val testData = TestData<K, T>(null, null)
-        val warmUpTimingsNs = LongArray(warmUpIterations)
-        val errors: Array<Throwable?> = Array(warmUpIterations, init = { null })
-        for (attempt in 0 until warmUpIterations) {
-            testData.reset()
-
-            triggerGC(attempt)
-
-            try {
-                val setUpMillis = measureTimeMillis {
-                    setUp(testData)
-                }
-                println("-- setup took $setUpMillis ms")
-                var spentNs: Long = 0
-                try {
-                    spentNs = measureNanoTime {
-                        test(testData)
-                    }
-                } finally {
-                    val tearDownMillis = measureTimeMillis {
-                        tearDown(testData)
-                    }
-                    println("-- tearDown took $tearDownMillis ms")
-                }
-                warmUpTimingsNs[attempt] = spentNs
-            } catch (t: Throwable) {
-                errors[attempt] = t
-            }
-        }
-
-        printWarmUpTimings(namePrefix, errors, warmUpTimingsNs)
-
-        errors.find { it != null }?.let { throw it }
-    }
-
-    fun printTestStarted(testName: String) {
-        println("##teamcity[testStarted name='$testName' captureStandardOutput='true']")
-    }
-
-    fun printTestFinished(testName: String, spentMs: Long) {
-        println("##teamcity[buildStatisticValue key='$testName' value='$spentMs']")
-        println("##teamcity[testFinished name='$testName' duration='$spentMs']")
-    }
-
-    fun printTestFinished(testName: String, error: Throwable) {
-        println("error at $testName:")
-        tcPrintErrors(testName, listOf(error))
-
-        println("##teamcity[buildStatisticValue key='$testName' value='-1']")
-        println("##teamcity[testFinished name='$testName']")
-    }
-
     private fun triggerGC(attempt: Int) {
         if (attempt > 0) {
             val ref = WeakReference(IntArray(32 * 1024))
@@ -239,17 +249,89 @@ class Stats(val name: String = "", val header: Array<String> = arrayOf("Name", "
         }
     }
 
-    private fun geomMean(data: List<Long>) = exp(data.fold(0.0, { mul, next -> mul + ln(1.0 * next) }) / data.size).toLong()
+    private fun geomMean(data: List<Long>) = exp(data.fold(0.0, { mul, next -> mul + ln(1.0 * next) }) / data.size)
 
     override fun close() {
         if (perfTestRawDataMs.isNotEmpty()) {
-            val geomMeanMs = geomMean(perfTestRawDataMs.toList())
+            val geomMeanMs = geomMean(perfTestRawDataMs.toList()).toLong()
-            println("##teamcity[buildStatisticValue key='$name geomMean' value='$geomMeanMs']")
+            printStatValue("$name geomMean", geomMeanMs)
             append(arrayOf("$name geomMean", geomMeanMs, 0))
         }
         statsOutput.flush()
         statsOutput.close()
     }
+
+    companion object {
+        const val TEST_KEY = "test"
+        const val ERROR_KEY = "error"
+
+        inline fun runAndMeasure(note: String, block: () -> Unit) {
+            val openProjectMillis = measureTimeMillis {
+                block()
+            }
+            logMessage { "$note took $openProjectMillis ms" }
+        }
+
+        fun printTestStarted(testName: String) {
+            println("##teamcity[testStarted name='$testName' captureStandardOutput='true']")
+        }
+
+        fun printTestFinished(testName: String, spentMs: Long, includeStatValue: Boolean = true) {
+            if (includeStatValue) {
+                printStatValue(testName, spentMs)
+            }
+            println("##teamcity[testFinished name='$testName' duration='$spentMs']")
+        }
+
+        fun printStatValue(name: String, value: Any) {
+            println("##teamcity[buildStatisticValue key='$name' value='$value']")
+        }
+
+        fun printTestFinished(testName: String, error: Throwable) {
+            println("error at $testName:")
+            tcPrintErrors(testName, listOf(error))
+
+            printStatValue(testName, -1)
+            println("##teamcity[testFinished name='$testName']")
+        }
+
+        inline fun tcSuite(name: String, block: () -> Unit) {
+            println("##teamcity[testSuiteStarted name='$name']")
+            try {
+                block()
+            } finally {
+                println("##teamcity[testSuiteFinished name='$name']")
+            }
+        }
+
+        inline fun tcTest(name: String, block: () -> Pair<Long, List<Throwable>>) {
+            printTestStarted(name)
+            val (time, errors) = block()
+            tcPrintErrors(name, errors)
+            printTestFinished(name, time, includeStatValue = false)
+        }
+
+        private fun tcEscape(s: String) = s.replace("|", "||")
+            .replace("[", "|[")
+            .replace("]", "|]")
+            .replace("\r", "|r")
+            .replace("\n", "|n")
+            .replace("'", "|'")
+
+        fun tcPrintErrors(name: String, errors: List<Throwable>) {
+            if (errors.isNotEmpty()) {
+                val detailsWriter = StringWriter()
+                val errorDetailsPrintWriter = PrintWriter(detailsWriter)
+                errors.forEach {
+                    it.printStackTrace(errorDetailsPrintWriter)
+                    errorDetailsPrintWriter.println()
+                }
+                errorDetailsPrintWriter.close()
+                val details = detailsWriter.toString()
+                println("##teamcity[testFailed name='$name' message='Exceptions reported' details='${tcEscape(details)}']")
+            }
+        }
+    }
 }
 
 data class TestData<SV, V>(var setUpValue: SV?, var value: V?) {
@@ -257,48 +339,4 @@ data class TestData<SV, V>(var setUpValue: SV?, var value: V?) {
         setUpValue = null
         value = null
     }
-}
+}
-
-inline fun runAndMeasure(note: String, block: () -> Unit) {
-    val openProjectMillis = measureTimeMillis {
-        block()
-    }
-    println("-- $note took $openProjectMillis ms")
-}
-
-inline fun tcSuite(name: String, block: () -> Unit) {
-    println("##teamcity[testSuiteStarted name='$name']")
-    block()
-    println("##teamcity[testSuiteFinished name='$name']")
-}
-
-inline fun tcTest(name: String, block: () -> Pair<Long, List<Throwable>>) {
-    println("##teamcity[testStarted name='$name' captureStandardOutput='true']")
-    val (time, errors) = block()
-    tcPrintErrors(name, errors)
-    println("##teamcity[testFinished name='$name' duration='$time']")
-}
-
-fun tcPrintErrors(name: String, errors: List<Throwable>) {
-    if (errors.isNotEmpty()) {
-        val detailsWriter = StringWriter()
-        val errorDetailsPrintWriter = PrintWriter(detailsWriter)
-        errors.forEach {
-            it.printStackTrace(errorDetailsPrintWriter)
-            errorDetailsPrintWriter.println()
-        }
-        errorDetailsPrintWriter.close()
-        val details = detailsWriter.toString()
-        println("##teamcity[testFailed name='$name' message='Exceptions reported' details='${details.tcEscape()}']")
-    }
-}
-
-fun String.tcEscape(): String {
-    return this
-        .replace("|", "||")
-        .replace("[", "|[")
-        .replace("]", "|]")
-        .replace("\r", "|r")
-        .replace("\n", "|n")
-        .replace("'", "|'")
-}

idea/performanceTests/org/jetbrains/kotlin/idea/perf/WholeProjectPerformanceTest.kt

@@ -30,6 +30,8 @@ import com.intellij.psi.impl.source.resolve.reference.ReferenceProvidersRegistry
 import com.intellij.psi.xml.XmlFileNSInfoProvider
 import com.intellij.xml.XmlSchemaProvider
 import org.jetbrains.kotlin.idea.framework.KotlinSdkType
+import org.jetbrains.kotlin.idea.perf.Stats.Companion.tcSuite
+import org.jetbrains.kotlin.idea.perf.Stats.Companion.tcTest
 import java.io.*
 
 abstract class WholeProjectPerformanceTest : DaemonAnalyzerTestCase(), WholeProjectFileProvider {

idea/performanceTests/org/jetbrains/kotlin/idea/testFramework/Fixture.kt

@@ -6,16 +6,48 @@
 package org.jetbrains.kotlin.idea.testFramework
 
 import com.intellij.codeInsight.completion.CompletionType
+import com.intellij.codeInsight.daemon.DaemonAnalyzerTestCase
+import com.intellij.codeInsight.daemon.ImplicitUsageProvider
+import com.intellij.codeInsight.daemon.ProblemHighlightFilter
 import com.intellij.codeInsight.daemon.impl.HighlightInfo
 import com.intellij.codeInsight.lookup.LookupElement
+import com.intellij.lang.ExternalAnnotatorsFilter
+import com.intellij.lang.LanguageAnnotators
+import com.intellij.lang.StdLanguages
+import com.intellij.lang.injection.InjectedLanguageManager
+import com.intellij.lang.java.JavaLanguage
+import com.intellij.openapi.application.runWriteAction
 import com.intellij.openapi.editor.Document
 import com.intellij.openapi.editor.Editor
+import com.intellij.openapi.editor.EditorFactory
+import com.intellij.openapi.fileEditor.FileDocumentManager
+import com.intellij.openapi.fileEditor.FileEditorManager
 import com.intellij.openapi.project.Project
+import com.intellij.openapi.project.guessProjectDir
+import com.intellij.openapi.roots.FileIndexFacade
 import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.openapi.vfs.VirtualFileManager
+import com.intellij.psi.PsiFile
+import com.intellij.psi.PsiManager
+import com.intellij.psi.impl.search.IndexPatternBuilder
+import com.intellij.psi.impl.source.resolve.reference.ReferenceProvidersRegistry
+import com.intellij.psi.search.FilenameIndex
+import com.intellij.psi.search.GlobalSearchScope
+import com.intellij.psi.xml.XmlFileNSInfoProvider
+import com.intellij.testFramework.EditorTestUtil
+import com.intellij.testFramework.UsefulTestCase
 import com.intellij.testFramework.fixtures.EditorTestFixture
+import com.intellij.testFramework.runInEdtAndWait
+import com.intellij.xml.XmlSchemaProvider
+import org.jetbrains.kotlin.idea.KotlinLanguage
+import org.jetbrains.kotlin.idea.core.script.ScriptConfigurationManager
+import org.jetbrains.kotlin.idea.core.script.ScriptDefinitionsManager
+import org.jetbrains.kotlin.idea.core.script.settings.KotlinScriptingSettings
+import org.jetbrains.kotlin.idea.core.util.toPsiFile
+import org.jetbrains.kotlin.parsing.KotlinParserDefinition
 
 
-class Fixture(val project: Project, val editor: Editor, val vFile: VirtualFile) {
+class Fixture(val project: Project, val editor: Editor, val psiFile: PsiFile, val vFile: VirtualFile = psiFile.virtualFile) {
     private val delegate = EditorTestFixture(project, editor, vFile)
 
     val document: Document
@@ -29,4 +61,179 @@ class Fixture(val project: Project, val editor: Editor, val vFile: VirtualFile)
 
     fun complete(type: CompletionType = CompletionType.BASIC, invocationCount: Int = 1): Array<LookupElement> =
         delegate.complete(type, invocationCount) ?: emptyArray()
+
+    fun revertChanges(revertChangesAtTheEnd: Boolean, text: String) {
+        try {
+            if (revertChangesAtTheEnd) {
+                runWriteAction {
+                    // TODO: [VD] revert ?
+                    //editorFixture.performEditorAction(IdeActions.SELECTED_CHANGES_ROLLBACK)
+                    document.setText(text)
+                    saveDocument(document)
+                    commitDocument(project, document)
+                }
+                dispatchAllInvocationEvents()
+            }
+        } finally {
+            cleanupCaches(project, vFile)
+        }
+    }
+
+    companion object {
+        // quite simple impl - good so far
+        fun isAKotlinScriptFile(fileName: String) = fileName.endsWith(KotlinParserDefinition.STD_SCRIPT_EXT)
+
+        fun cleanupCaches(project: Project, vFile: VirtualFile) {
+            commitAllDocuments()
+            FileEditorManager.getInstance(project).closeFile(vFile)
+            PsiManager.getInstance(project).dropPsiCaches()
+        }
+
+        fun enableAnnotatorsAndLoadDefinitions(project: Project) {
+            ReferenceProvidersRegistry.getInstance() // pre-load tons of classes
+            InjectedLanguageManager.getInstance(project) // zillion of Dom Sem classes
+            with(LanguageAnnotators.INSTANCE) {
+                allForLanguage(JavaLanguage.INSTANCE) // pile of annotator classes loads
+                allForLanguage(StdLanguages.XML)
+                allForLanguage(KotlinLanguage.INSTANCE)
+            }
+            DaemonAnalyzerTestCase.assertTrue(
+                "side effect: to load extensions",
+                ProblemHighlightFilter.EP_NAME.extensions.toMutableList()
+                    .plus(ImplicitUsageProvider.EP_NAME.extensions)
+                    .plus(XmlSchemaProvider.EP_NAME.extensions)
+                    .plus(XmlFileNSInfoProvider.EP_NAME.extensions)
+                    .plus(ExternalAnnotatorsFilter.EXTENSION_POINT_NAME.extensions)
+                    .plus(IndexPatternBuilder.EP_NAME.extensions).isNotEmpty()
+            )
+
+            // side effect: to load script definitions"
+            val scriptDefinitionsManager = ScriptDefinitionsManager.getInstance(project)
+            scriptDefinitionsManager.getAllDefinitions()
+            dispatchAllInvocationEvents()
+
+            UsefulTestCase.assertTrue(scriptDefinitionsManager.isReady())
+            UsefulTestCase.assertFalse(KotlinScriptingSettings.getInstance(project).isAutoReloadEnabled)
+        }
+
+
+        fun openFixture(project: Project, fileName: String): Fixture {
+            val fileInEditor = openFileInEditor(project, fileName)
+            val file = fileInEditor.psiFile
+            val editor = EditorFactory.getInstance().getEditors(fileInEditor.document, project)[0]
+
+            return Fixture(project, editor, file)
+        }
+
+        private fun baseName(name: String): String {
+            val index = name.lastIndexOf("/")
+            return if (index > 0) name.substring(index + 1) else name
+        }
+
+        private fun projectFileByName(project: Project, name: String): PsiFile {
+            val fileManager = VirtualFileManager.getInstance()
+            val url = "${project.guessProjectDir()}/$name"
+            val virtualFile = fileManager.refreshAndFindFileByUrl(url)
+            if (virtualFile != null) {
+                return virtualFile!!.toPsiFile(project)!!
+            }
+
+            val baseFileName = baseName(name)
+            val projectBaseName = baseName(project.name)
+
+            val virtualFiles = FilenameIndex.getVirtualFilesByName(
+                project,
+                baseFileName, true,
+                GlobalSearchScope.projectScope(project)
+            )
+                .filter { it.canonicalPath?.contains("/$projectBaseName/$name") ?: false }.toList()
+
+            UsefulTestCase.assertEquals(
+                "expected the only file with name '$name'\n, it were: [${virtualFiles.map { it.canonicalPath }.joinToString("\n")}]",
+                1,
+                virtualFiles.size
+            )
+            return virtualFiles.iterator().next().toPsiFile(project)!!
+        }
+
+        fun openFileInEditor(project: Project, name: String): EditorFile {
+            val fileDocumentManager = FileDocumentManager.getInstance()
+            val fileEditorManager = FileEditorManager.getInstance(project)
+
+            val psiFile = projectFileByName(project, name)
+            val vFile = psiFile.virtualFile
+
+            UsefulTestCase.assertTrue("file $vFile is not indexed yet", FileIndexFacade.getInstance(project).isInContent(vFile))
+
+            runInEdtAndWait {
+                fileEditorManager.openFile(vFile, true)
+            }
+            val document = fileDocumentManager.getDocument(vFile)!!
+
+            UsefulTestCase.assertNotNull("doc not found for $vFile", EditorFactory.getInstance().getEditors(document))
+            UsefulTestCase.assertTrue("expected non empty doc", document.text.isNotEmpty())
+
+            val offset = psiFile.textOffset
+            UsefulTestCase.assertTrue("side effect: to load the text", offset >= 0)
+
+            waitForAllEditorsFinallyLoaded(project)
+
+            return EditorFile(psiFile = psiFile, document = document)
+        }
+
+        /**
+         * @param lookupElements perform basic autocompletion and check presence of suggestion if elements are not empty
+         */
+        fun typeAndCheckLookup(
+            project: Project,
+            fileName: String,
+            marker: String,
+            insertString: String,
+            surroundItems: String = "\n",
+            lookupElements: List<String>,
+            revertChangesAtTheEnd: Boolean = true
+        ) {
+            val fileInEditor = openFileInEditor(project, fileName)
+            val editor = EditorFactory.getInstance().getEditors(fileInEditor.document, project)[0]
+            val fixture = Fixture(project, editor, fileInEditor.psiFile)
+
+            val initialText = editor.document.text
+            try {
+                if (isAKotlinScriptFile(fileName)) {
+                    ScriptConfigurationManager.updateScriptDependenciesSynchronously(fileInEditor.psiFile, project)
+                }
+
+                val tasksIdx = fileInEditor.document.text.indexOf(marker)
+                UsefulTestCase.assertTrue(tasksIdx > 0)
+                editor.caretModel.moveToOffset(tasksIdx + marker.length + 1)
+
+                for (surroundItem in surroundItems) {
+                    EditorTestUtil.performTypingAction(editor, surroundItem)
+                }
+
+                editor.caretModel.moveToOffset(editor.caretModel.offset - 1)
+                fixture.type(insertString)
+
+                if (lookupElements.isNotEmpty()) {
+                    val elements = fixture.complete()
+                    val items = elements.map { it.lookupString }.toList()
+                    for (lookupElement in lookupElements) {
+                        UsefulTestCase.assertTrue("'$lookupElement' has to be present in items", items.contains(lookupElement))
+                    }
+                }
+            } finally {
+                // TODO: [VD] revert ?
+                //fixture.performEditorAction(IdeActions.SELECTED_CHANGES_ROLLBACK)
+                if (revertChangesAtTheEnd) {
+                    runWriteAction {
+                        editor.document.setText(initialText)
+                        commitDocument(project, editor.document)
+                    }
+                }
+            }
+        }
+
+    }
 }
+
+data class EditorFile(val psiFile: PsiFile, val document: Document)

idea/performanceTests/org/jetbrains/kotlin/idea/testFramework/gradleRoutines.kt

@@ -0,0 +1,78 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.idea.testFramework
+
+import com.intellij.ide.impl.ProjectUtil
+import com.intellij.openapi.externalSystem.importing.ImportSpecBuilder
+import com.intellij.openapi.externalSystem.service.execution.ProgressExecutionMode
+import com.intellij.openapi.externalSystem.service.project.manage.ExternalProjectsManagerImpl
+import com.intellij.openapi.externalSystem.util.ExternalSystemApiUtil
+import com.intellij.openapi.externalSystem.util.ExternalSystemUtil
+import com.intellij.openapi.project.DumbService
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.roots.ProjectRootManager
+import org.jetbrains.plugins.gradle.service.project.open.setupGradleSettings
+import org.jetbrains.plugins.gradle.settings.GradleProjectSettings
+import org.jetbrains.plugins.gradle.util.GradleConstants
+import org.jetbrains.plugins.gradle.util.GradleLog
+import kotlin.test.assertNotNull
+
+fun refreshGradleProject(projectPath: String, project: Project) {
+    _importProject(projectPath, project)
+
+//    val gradleArguments = System.getProperty("kotlin.test.gradle.import.arguments")
+//    ExternalSystemUtil.refreshProjects(
+//        ImportSpecBuilder(project, GradleConstants.SYSTEM_ID)
+//            .forceWhenUptodate()
+//            .useDefaultCallback()
+//            .use(ProgressExecutionMode.MODAL_SYNC)
+//            .also {
+//                gradleArguments?.run(it::withArguments)
+//            }
+//    )
+
+    //ProjectUtil.updateLastProjectLocation(projectPath)
+
+    dispatchAllInvocationEvents()
+}
+
+/**
+ * inspired by org.jetbrains.plugins.gradle.service.project.open.importProject(projectDirectory, project)
+ */
+private fun _importProject(projectPath: String, project: Project) {
+    GradleLog.LOG.info("Import project at $projectPath")
+    val projectSdk = ProjectRootManager.getInstance(project).projectSdk
+    assertNotNull(projectSdk, "project SDK not found for ${project.name} at $projectPath")
+    val gradleProjectSettings = GradleProjectSettings()
+    setupGradleSettings(gradleProjectSettings, projectPath, project, projectSdk)
+
+    _attachGradleProjectAndRefresh(gradleProjectSettings, project)
+}
+
+/**
+ * inspired by org.jetbrains.plugins.gradle.service.project.open.attachGradleProjectAndRefresh(gradleProjectSettings, project)
+ * except everything is MODAL_SYNC
+ */
+private fun _attachGradleProjectAndRefresh(
+    gradleProjectSettings: GradleProjectSettings,
+    project: Project
+) {
+    val externalProjectPath = gradleProjectSettings.externalProjectPath
+    ExternalProjectsManagerImpl.getInstance(project).runWhenInitialized {
+        DumbService.getInstance(project).runWhenSmart {
+            ExternalSystemUtil.ensureToolWindowInitialized(project, GradleConstants.SYSTEM_ID)
+        }
+    }
+    //ExternalProjectsManagerImpl.getInstance(project).setStoreExternally(false)
+
+    ExternalProjectsManagerImpl.disableProjectWatcherAutoUpdate(project)
+    val settings = ExternalSystemApiUtil.getSettings(project, GradleConstants.SYSTEM_ID)
+    if (settings.getLinkedProjectSettings(externalProjectPath) == null) {
+        settings.linkProject(gradleProjectSettings)
+    }
+    //ExternalSystemUtil.refreshProject(project, GradleConstants.SYSTEM_ID, externalProjectPath, true, ProgressExecutionMode.MODAL_SYNC)
+    ExternalSystemUtil.refreshProject(project, GradleConstants.SYSTEM_ID, externalProjectPath, false, ProgressExecutionMode.MODAL_SYNC)
+}

idea/performanceTests/org/jetbrains/kotlin/idea/testFramework/gradleRoutines.kt.191

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.idea.testFramework
+
+import com.intellij.ide.impl.ProjectUtil
+import com.intellij.openapi.externalSystem.importing.ImportSpecBuilder
+import com.intellij.openapi.externalSystem.service.execution.ProgressExecutionMode
+import com.intellij.openapi.externalSystem.util.ExternalSystemUtil
+import com.intellij.openapi.fileEditor.FileDocumentManager
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.vfs.LocalFileSystem
+import com.intellij.testFramework.PlatformTestUtil
+import com.intellij.testFramework.runInEdtAndWait
+import com.intellij.openapi.project.ex.ProjectManagerEx
+import org.jetbrains.plugins.gradle.service.project.GradleProjectOpenProcessor
+import org.jetbrains.plugins.gradle.util.GradleConstants
+import java.io.File
+import java.nio.file.Paths
+
+fun refreshGradleProject(projectPath: String, project: Project) {
+    GradleProjectOpenProcessor.openGradleProject(project, null, Paths.get(projectPath))
+
+    val gradleArguments = System.getProperty("kotlin.test.gradle.import.arguments")
+    ExternalSystemUtil.refreshProjects(
+        ImportSpecBuilder(project, GradleConstants.SYSTEM_ID)
+            .forceWhenUptodate()
+            .useDefaultCallback()
+            .use(ProgressExecutionMode.MODAL_SYNC)
+            .also {
+                gradleArguments?.run(it::withArguments)
+            }
+    )
+
+    dispatchAllInvocationEvents()
+}
+
+fun openGradleProject(projectPath: String, project: Project) {
+        dispatchAllInvocationEvents()
+
+        val virtualFile = LocalFileSystem.getInstance().refreshAndFindFileByPath(projectPath)!!
+
+        FileDocumentManager.getInstance().saveAllDocuments()
+
+        val path = Paths.get(virtualFile.path)
+        GradleProjectOpenProcessor.openGradleProject(project, null, path)
+
+        dispatchAllInvocationEvents()
+        runInEdtAndWait {
+            PlatformTestUtil.saveProject(project)
+        }
+    }

idea/performanceTests/org/jetbrains/kotlin/idea/testFramework/projectRoutines.kt

@@ -9,26 +9,17 @@ import com.intellij.lang.LanguageAnnotators
 import com.intellij.lang.LanguageExtensionPoint
 import com.intellij.lang.annotation.Annotator
 import com.intellij.openapi.Disposable
-import com.intellij.openapi.application.ApplicationManager
-import com.intellij.openapi.application.runWriteAction
 import com.intellij.openapi.editor.Document
 import com.intellij.openapi.extensions.ExtensionPointName
 import com.intellij.openapi.fileEditor.FileDocumentManager
-import com.intellij.openapi.fileEditor.FileEditorManager
-import com.intellij.openapi.fileEditor.impl.text.TextEditorImpl
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.project.ex.ProjectManagerEx
 import com.intellij.psi.PsiDocumentManager
 import com.intellij.psi.impl.PsiDocumentManagerBase
-import com.intellij.testFramework.EdtTestUtil
 import com.intellij.testFramework.PlatformTestUtil
-import com.intellij.util.ThrowableRunnable
 import com.intellij.testFramework.runInEdtAndWait
 import com.intellij.util.ui.UIUtil
-import org.jetbrains.kotlin.idea.highlighter.KotlinPsiCheckerAndHighlightingUpdater
 import org.jetbrains.kotlin.idea.parameterInfo.HintType
-import java.util.concurrent.TimeUnit
-import java.util.concurrent.TimeoutException
 
 fun commitAllDocuments() {
     val fileDocumentManager = FileDocumentManager.getInstance()
@@ -97,3 +88,7 @@ fun replaceWithCustomHighlighter(parentDisposable: Disposable, fromImplementatio
         PlatformTestUtil.maskExtensions(pointName, filteredExtensions + listOf(point), parentDisposable)
     }
 }
+
+fun logMessage(message: () -> String) {
+    println("-- ${message()}")
+}

idea/performanceTests/org/jetbrains/kotlin/idea/testFramework/projectRoutines.kt.183

@@ -70,3 +70,7 @@ fun waitForAllEditorsFinallyLoaded(project: Project) {
 fun replaceWithCustomHighlighter(parentDisposable: Disposable, fromImplementationClass: String, toImplementationClass: String) {
  // 183 does not have this public api
 }
+
+fun logMessage(message: () -> String) {
+    println("-- ${message()}")
+}

idea/performanceTests/org/jetbrains/kotlin/idea/testFramework/projectRoutines.kt.191

@@ -117,3 +117,7 @@ fun replaceWithCustomHighlighter(parentDisposable: Disposable, fromImplementatio
         PlatformTestUtil.maskExtensions(pointName, filteredExtensions + listOf(point), parentDisposable)
     }
 }
+
+fun logMessage(message: () -> String) {
+    println("-- ${message()}")
+}