Nicer performance tests suite dsl

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceProjectsTest.kt

@@ -5,25 +5,17 @@
 
 package org.jetbrains.kotlin.idea.perf
 
-import com.intellij.codeInsight.daemon.DaemonCodeAnalyzer
-import com.intellij.codeInsight.daemon.DaemonCodeAnalyzerSettings
-import com.intellij.codeInsight.daemon.impl.DaemonCodeAnalyzerImpl
 import com.intellij.codeInsight.daemon.impl.HighlightInfo
 import com.intellij.codeInsight.daemon.impl.IdentifierHighlighterPassFactory
 import com.intellij.codeInsight.lookup.LookupElement
 import com.intellij.codeInspection.InspectionProfileEntry
-import com.intellij.ide.startup.impl.StartupManagerImpl
 import com.intellij.openapi.actionSystem.IdeActions
-import com.intellij.openapi.application.runWriteAction
 import com.intellij.openapi.editor.EditorFactory
 import com.intellij.openapi.fileEditor.FileDocumentManager
 import com.intellij.openapi.fileEditor.FileEditorManager
 import com.intellij.openapi.project.DumbService
 import com.intellij.openapi.project.Project
-import com.intellij.openapi.projectRoots.JavaSdk
 import com.intellij.openapi.projectRoots.Sdk
-import com.intellij.openapi.projectRoots.impl.JavaAwareProjectJdkTableImpl
-import com.intellij.openapi.startup.StartupManager
 import com.intellij.profile.codeInspection.ProjectInspectionProfileManager
 import com.intellij.psi.PsiDocumentManager
 import com.intellij.psi.PsiFile
@@ -35,18 +27,20 @@ import com.intellij.util.ThrowableRunnable
 import com.intellij.util.indexing.UnindexedFilesUpdater
 import com.intellij.util.ui.UIUtil
 import org.jetbrains.kotlin.idea.core.script.ScriptConfigurationManager
-import org.jetbrains.kotlin.idea.framework.KotlinSdkType
 import org.jetbrains.kotlin.idea.perf.Stats.Companion.WARM_UP
 import org.jetbrains.kotlin.idea.perf.Stats.Companion.runAndMeasure
+import org.jetbrains.kotlin.idea.perf.util.PerformanceSuite.ApplicationScope.Companion.initApp
+import org.jetbrains.kotlin.idea.perf.util.PerformanceSuite.ApplicationScope.Companion.initSdk
+import org.jetbrains.kotlin.idea.perf.util.ProfileTools.Companion.initDefaultProfile
+import org.jetbrains.kotlin.idea.perf.util.logMessage
 import org.jetbrains.kotlin.idea.test.invalidateLibraryCache
 import org.jetbrains.kotlin.idea.testFramework.*
+import org.jetbrains.kotlin.idea.testFramework.TestApplicationManager
 import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.cleanupCaches
 import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.close
 import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.isAKotlinScriptFile
 import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.openFileInEditor
 import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.openFixture
-import org.jetbrains.kotlin.idea.testFramework.ProjectOpenAction.SIMPLE_JAVA_MODULE
-import org.jetbrains.kotlin.idea.util.getProjectJdkTableSafe
 import java.io.File
 
 abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
@@ -63,29 +57,11 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
     override fun setUp() {
         super.setUp()
 
-        myApplication = TestApplicationManager.getInstance()
-        runWriteAction {
-            val jdkTableImpl = JavaAwareProjectJdkTableImpl.getInstanceEx()
-            val homePath = if (jdkTableImpl.internalJdk.homeDirectory!!.name == "jre") {
-                jdkTableImpl.internalJdk.homeDirectory!!.parent.path
-            } else {
-                jdkTableImpl.internalJdk.homePath!!
-            }
-
-            val javaSdk = JavaSdk.getInstance()
-            jdk18 = javaSdk.createJdk("1.8", homePath)
-            val internal = javaSdk.createJdk("IDEA jdk", homePath)
-
-            val jdkTable = getProjectJdkTableSafe()
-            jdkTable.addJdk(jdk18, testRootDisposable)
-            jdkTable.addJdk(internal, testRootDisposable)
-            KotlinSdkType.setUpIfNeeded()
-        }
-
-        GradleProcessOutputInterceptor.install(testRootDisposable)
+        myApplication = initApp(testRootDisposable)
+        initSdk(testRootDisposable)
     }
 
-    protected fun warmUpProject(stats: Stats, vararg filesToHighlight: String, openProject: () -> Project) {
+    internal fun warmUpProject(stats: Stats, vararg filesToHighlight: String, openProject: () -> Project) {
         assertTrue(filesToHighlight.isNotEmpty())
 
         val project = openProject()
@@ -96,7 +72,6 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
             }
         } finally {
             closeProject(project)
-            myApplication.setDataProvider(null)
         }
     }
 
@@ -105,15 +80,9 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
         RunAll(
             ThrowableRunnable { super.tearDown() },
             ThrowableRunnable {
-                if (myProject != null) {
-                    logMessage { "myProject is about to be closed" }
-                    DaemonCodeAnalyzerSettings.getInstance().isImportHintEnabled = true // return default value to avoid unnecessary save
-                    (StartupManager.getInstance(project()) as StartupManagerImpl).checkCleared()
-                    (DaemonCodeAnalyzer.getInstance(project()) as DaemonCodeAnalyzerImpl).cleanupAfterTest()
-                    closeProject(project())
-                    myApplication.setDataProvider(null)
+                myProject?.let { project ->
+                    closeProject(project)
                     myProject = null
-                    logMessage { "myProject is closed" }
                 }
             }).run()
     }
@@ -123,14 +92,72 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
         return if (lastIndexOf >= 0) fileName.substring(lastIndexOf + 1) else fileName
     }
 
-    protected fun perfOpenHelloWorld(stats: Stats, note: String = ""): Project =
-        perfOpenProject(
-            name = "helloKotlin",
-            stats = stats,
-            note = note,
-            path = "idea/testData/perfTest/helloKotlin",
-            openAction = SIMPLE_JAVA_MODULE
-        )
+    private fun closeProject(project: Project) = myApplication.closeProject(project)
+
+    protected fun perfOpenProject(
+        stats: Stats,
+        note: String = "",
+        fast: Boolean = false,
+        initializer: ProjectBuilder.() -> Unit,
+    ): Project {
+        val projectBuilder = ProjectBuilder().apply(initializer)
+        val name = projectBuilder.name
+        val openProjectOperation = projectBuilder.openProjectOperation()
+
+        val warmUpIterations = if (fast) 0 else 5
+        val iterations = if (fast) 1 else 5
+
+        var lastProject: Project? = null
+        var counter = 0
+
+        performanceTest<Unit, Project> {
+            name("open project $name${if (note.isNotEmpty()) " $note" else ""}")
+            stats(stats)
+            warmUpIterations(warmUpIterations)
+            iterations(iterations)
+            checkStability(!fast)
+            test {
+                it.value = openProjectOperation.openProject()
+            }
+            tearDown {
+                it.value?.let { project ->
+                    lastProject = project
+                    openProjectOperation.postOpenProject(project)
+
+                    logMessage { "project '$name' successfully opened" }
+
+                    // close all project but last - we're going to return and use it further
+                    if (counter < warmUpIterations + iterations - 1) {
+                        closeProject(project)
+                    }
+                    counter++
+                }
+            }
+        }
+
+        // indexing
+        lastProject?.let { project ->
+            invalidateLibraryCache(project)
+
+            CodeInsightTestFixtureImpl.ensureIndexesUpToDate(project)
+
+            dispatchAllInvocationEvents()
+
+            logMessage { "project $name is ${if (project.isInitialized) "initialized" else "not initialized"}" }
+
+            with(DumbService.getInstance(project)) {
+                queueTask(UnindexedFilesUpdater(project))
+                completeJustSubmittedTasks()
+            }
+            dispatchAllInvocationEvents()
+
+            Fixture.enableAnnotatorsAndLoadDefinitions(project)
+
+            myApplication.setDataProvider(TestDataProvider(project))
+        }
+
+        return lastProject ?: error("unable to open project $name")
+    }
 
     protected fun perfOpenProject(
         name: String,
@@ -170,16 +197,13 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
                 it.value?.let { project ->
                     lastProject = project
                     openAction.postOpenProject(openProject = openProject, project = project)
-                    openAction.initDefaultProfile(project)
+                    project.initDefaultProfile()
 
                     logMessage { "project '$name' successfully opened" }
 
                     // close all project but last - we're going to return and use it further
                     if (counter < warmUpIterations + iterations - 1) {
-                        myApplication.setDataProvider(null)
-                        logMessage { "project '$name' is about to be closed" }
-                        closeProject(project)
-                        logMessage { "project '$name' successfully closed" }
+                        myApplication.closeProject(project)
                     }
                     counter++
                 }
@@ -448,92 +472,6 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
         }
     }
 
-    fun perfType(
-        stats: Stats,
-        fileName: String,
-        marker: String,
-        insertString: String,
-        surroundItems: String = "\n",
-        typeAfterMarker: Boolean = true,
-        revertChangesAtTheEnd: Boolean = true,
-        note: String = "",
-        delay: Long? = null
-    ) = perfType(
-        project(), stats, fileName, marker, insertString, surroundItems,
-        typeAfterMarker = typeAfterMarker,
-        revertChangesAtTheEnd = revertChangesAtTheEnd,
-        note = note,
-        delay = delay
-    )
-
-    fun perfType(
-        project: Project,
-        stats: Stats,
-        fileName: String,
-        marker: String,
-        insertString: String,
-        surroundItems: String = "\n",
-        typeAfterMarker: Boolean = true,
-        revertChangesAtTheEnd: Boolean = true,
-        note: String = "",
-        delay: Long? = null
-    ) {
-        performanceTest<Pair<String, Fixture>, List<HighlightInfo>> {
-            name("type ${notePrefix(note)}$fileName")
-            stats(stats)
-            warmUpIterations(8)
-            iterations(15)
-            setUp {
-                val fixture = openFixture(project, fileName)
-                val editor = fixture.editor
-
-                val initialText = editor.document.text
-                updateScriptDependenciesIfNeeded(fileName, fixture)
-
-                val tasksIdx = editor.document.text.indexOf(marker)
-                assertTrue("marker '$marker' not found in $fileName", tasksIdx > 0)
-                if (typeAfterMarker) {
-                    editor.caretModel.moveToOffset(tasksIdx + marker.length + 1)
-                } else {
-                    editor.caretModel.moveToOffset(tasksIdx - 1)
-                }
-
-                for (surroundItem in surroundItems) {
-                    EditorTestUtil.performTypingAction(editor, surroundItem)
-                }
-
-                editor.caretModel.moveToOffset(editor.caretModel.offset - if (typeAfterMarker) 1 else 2)
-
-                if (!typeAfterMarker) {
-                    for (surroundItem in surroundItems) {
-                        EditorTestUtil.performTypingAction(editor, surroundItem)
-                    }
-                    editor.caretModel.moveToOffset(editor.caretModel.offset - 2)
-                }
-
-                it.setUpValue = Pair(initialText, fixture)
-            }
-            test {
-                val fixture = it.setUpValue!!.second
-                for (i in insertString.indices) {
-                    fixture.type(insertString[i])
-                    delay?.let { d -> Thread.sleep(d) }
-                }
-                it.value = fixture.doHighlighting()
-            }
-            tearDown {
-                it.value?.let { list ->
-                    assertNotEmpty(list)
-                }
-                it.setUpValue?.let { pair ->
-                    pair.second.revertChanges(revertChangesAtTheEnd, pair.first)
-                }
-                commitAllDocuments()
-            }
-            profilerEnabled(true)
-        }
-    }
-
     fun perfCopyAndPaste(
         stats: Stats,
         sourceFileName: String,
@@ -630,12 +568,16 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
     protected fun perfHighlightFileEmptyProfile(name: String, stats: Stats): List<HighlightInfo> =
         perfHighlightFile(project(), name, stats, tools = emptyArray(), note = "empty profile")
 
-    private fun perfHighlightFile(
+    protected fun perfHighlightFile(
         project: Project,
         fileName: String,
         stats: Stats,
         tools: Array<InspectionProfileEntry>? = null,
-        note: String = ""
+        note: String = "",
+        warmUpIterations: Int = 3,
+        iterations: Int = 10,
+        checkStability: Boolean = true,
+        filenameSimplifier: (String) -> String = ::simpleFilename
     ): List<HighlightInfo> {
         val profileManager = ProjectInspectionProfileManager.getInstance(project)
         val currentProfile = profileManager.currentProfile
@@ -647,10 +589,11 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
                 val isWarmUp = note == WARM_UP
                 var highlightInfos: List<HighlightInfo> = emptyList()
                 performanceTest<EditorFile, List<HighlightInfo>> {
-                    name("highlighting ${notePrefix(note)}${simpleFilename(fileName)}")
+                    name("highlighting ${notePrefix(note)}${filenameSimplifier(fileName)}")
                     stats(stats)
-                    warmUpIterations(if (isWarmUp) 1 else 3)
-                    iterations(if (isWarmUp) 2 else 10)
+                    warmUpIterations(if (isWarmUp) 1 else warmUpIterations)
+                    iterations(if (isWarmUp) 2 else iterations)
+                    checkStability(checkStability)
                     setUp {
                         it.setUpValue = openFileInEditor(project, fileName)
                     }
@@ -661,7 +604,10 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
                     tearDown {
                         highlightInfos = it.value ?: emptyList()
                         commitAllDocuments()
-                        FileEditorManager.getInstance(project).closeFile(it.setUpValue!!.psiFile.virtualFile)
+                        it.setUpValue?.let { editorFile ->
+                            val fileEditorManager = FileEditorManager.getInstance(project)
+                            fileEditorManager.closeFile(editorFile.psiFile.virtualFile)
+                        }
                         PsiManager.getInstance(project).dropPsiCaches()
                     }
                     profilerEnabled(!isWarmUp)

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/FileUtils.kt

@@ -1,14 +0,0 @@
-/*
- * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.idea.perf
-
-import java.nio.file.*
-
-fun Path.copyRecursively(targetDirectory: Path) {
-    val src = this
-    Files.walk(src)
-        .forEach { source -> Files.copy(source, targetDirectory.resolve(src.relativize(source)), StandardCopyOption.REPLACE_EXISTING) }
-}

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerformanceNativeProjectsTest.kt

@@ -25,9 +25,9 @@ import org.jetbrains.kotlin.idea.framework.detectLibraryKind
 import org.jetbrains.kotlin.idea.perf.PerformanceNativeProjectsTest.TestProject.*
 import org.jetbrains.kotlin.idea.perf.PerformanceNativeProjectsTest.TestTarget.*
 import org.jetbrains.kotlin.idea.perf.Stats.Companion.WARM_UP
-import org.jetbrains.kotlin.idea.perf.Stats.Companion.tcSuite
+import org.jetbrains.kotlin.idea.perf.util.TeamCity.suite
+import org.jetbrains.kotlin.idea.perf.util.logMessage
 import org.jetbrains.kotlin.idea.testFramework.ProjectOpenAction.GRADLE_PROJECT
-import org.jetbrains.kotlin.idea.testFramework.logMessage
 import org.jetbrains.kotlin.idea.testFramework.suggestOsNeutralFileName
 import org.jetbrains.kotlin.idea.util.projectStructure.allModules
 import org.jetbrains.kotlin.library.KOTLIN_STDLIB_NAME
@@ -157,7 +157,7 @@ class PerformanceNativeProjectsTest : AbstractPerformanceProjectsTest() {
         assertTrue("Target $testTarget is not allowed on your host OS", testTarget.enabled)
 
         val projectName = projectName(testTarget, testProject, enableCommonizer)
-        tcSuite(projectName) {
+        suite(projectName) {
             Stats(projectName).use { stats ->
                 myProject = perfOpenTemplateGradleProject(stats, testTarget, testProject, enableCommonizer)
 

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerformanceProjectsTest.kt

@@ -8,15 +8,13 @@ package org.jetbrains.kotlin.idea.perf
 import com.intellij.codeInsight.daemon.impl.HighlightInfo
 import com.intellij.lang.annotation.AnnotationHolder
 import com.intellij.openapi.project.Project
-import com.intellij.openapi.util.Disposer
 import com.intellij.psi.PsiElement
 import org.jetbrains.kotlin.idea.core.script.ScriptConfigurationManager
 import org.jetbrains.kotlin.idea.highlighter.KotlinPsiChecker
 import org.jetbrains.kotlin.idea.highlighter.KotlinPsiCheckerAndHighlightingUpdater
 import org.jetbrains.kotlin.idea.perf.Stats.Companion.TEST_KEY
-import org.jetbrains.kotlin.idea.perf.Stats.Companion.WARM_UP
 import org.jetbrains.kotlin.idea.perf.Stats.Companion.runAndMeasure
-import org.jetbrains.kotlin.idea.perf.Stats.Companion.tcSuite
+import org.jetbrains.kotlin.idea.perf.util.TeamCity.suite
 import org.jetbrains.kotlin.idea.testFramework.Fixture
 import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.cleanupCaches
 import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.isAKotlinScriptFile
@@ -29,10 +27,10 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
     companion object {
 
         @JvmStatic
-        var warmedUp: Boolean = false
+        val hwStats: Stats = Stats("helloWorld project")
 
         @JvmStatic
-        val hwStats: Stats = Stats("helloWorld project")
+        val warmUp = WarmUpProject(hwStats)
 
         @JvmStatic
         val timer: AtomicLong = AtomicLong()
@@ -53,17 +51,28 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
 
     override fun setUp() {
         super.setUp()
-        // warm up: open simple small project
-        if (!warmedUp) {
-            warmUpProject(hwStats, "src/HelloMain.kt") { perfOpenHelloWorld(hwStats, WARM_UP) }
-            warmedUp = true
-        }
+        warmUp.warmUp(this)
     }
 
     fun testHelloWorldProject() {
+        suite("Hello world project") {
+            myProject = perfOpenProject(stats = hwStats) {
+                name("helloKotlin")
 
-        tcSuite("Hello world project") {
-            myProject = perfOpenHelloWorld(hwStats)
+                kotlinFile("HelloMain") {
+                    topFunction("main") {
+                        param("args", "Array<String>")
+                        body("""println("Hello World!")""")
+                    }
+                }
+
+                kotlinFile("HelloMain2") {
+                    topFunction("main") {
+                        param("args", "Array<String>")
+                        body("""println("Hello World!")""")
+                    }
+                }
+            }
 
             // highlight
             perfHighlightFile("src/HelloMain.kt", hwStats)
@@ -72,9 +81,8 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
     }
 
     fun testKotlinProject() {
-        tcSuite("Kotlin project") {
-            val stats = Stats("kotlin project")
-            stats.use {
+        suite("Kotlin project") {
+            Stats("kotlin project").use {
                 perfOpenKotlinProject(it)
 
                 val filesToHighlight = arrayOf(
@@ -114,9 +122,8 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
     }
 
     fun testKotlinProjectCopyAndPaste() {
-        tcSuite("Kotlin copy-and-paste") {
-            val stats = Stats("Kotlin copy-and-paste")
-            stats.use { stat ->
+        suite("Kotlin copy-and-paste") {
+            Stats("Kotlin copy-and-paste").use { stat ->
                 perfOpenKotlinProjectFast(stat)
 
                 perfCopyAndPaste(
@@ -129,9 +136,8 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
     }
 
     fun testKotlinProjectCompletionKtFile() {
-        tcSuite("Kotlin completion ktFile") {
-            val stats = Stats("Kotlin completion ktFile")
-            stats.use { stat ->
+        suite("Kotlin completion ktFile") {
+            Stats("Kotlin completion ktFile").use { stat ->
                 perfOpenKotlinProjectFast(stat)
 
                 perfTypeAndAutocomplete(
@@ -157,9 +163,8 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
     }
 
     fun testKotlinProjectCompletionBuildGradle() {
-        tcSuite("Kotlin completion gradle.kts") {
-            val stats = Stats("kotlin completion gradle.kts")
-            stats.use { stat ->
+        suite("Kotlin completion gradle.kts") {
+            Stats("kotlin completion gradle.kts").use { stat ->
                 runAndMeasure("open kotlin project") {
                     perfOpenKotlinProjectFast(stat)
                 }
@@ -191,9 +196,8 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
     }
 
     fun testKotlinProjectScriptDependenciesBuildGradle() {
-        tcSuite("Kotlin scriptDependencies gradle.kts") {
-            val stats = Stats("kotlin scriptDependencies gradle.kts")
-            stats.use { stat ->
+        suite("Kotlin scriptDependencies gradle.kts") {
+            Stats("kotlin scriptDependencies gradle.kts").use { stat ->
                 perfOpenKotlinProjectFast(stat)
 
                 perfScriptDependenciesBuildGradleKts(stat)
@@ -205,9 +209,8 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
     }
 
     fun testKotlinProjectBuildGradle() {
-        tcSuite("Kotlin gradle.kts") {
-            val stats = Stats("kotlin gradle.kts")
-            stats.use { stat ->
+        suite("Kotlin gradle.kts") {
+            Stats("kotlin gradle.kts").use { stat ->
                 perfOpenKotlinProjectFast(stat)
 
                 perfFileAnalysisBuildGradleKts(stat)
@@ -280,14 +283,14 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
             val extraStats = Stats("${stats.name} $testName")
             val extraTimingsNs = mutableListOf<Map<String, Any>?>()
 
-            val warmUpIterations = 20
-            val iterations = 30
+            val warmUpIterations = 30
+            val iterations = 50
 
             performanceTest<Fixture, Pair<Long, List<HighlightInfo>>> {
                 name(testName)
                 stats(stats)
-                warmUpIterations(30)
-                iterations(50)
+                warmUpIterations(warmUpIterations)
+                iterations(iterations)
                 setUp(perfKtsFileAnalysisSetUp(project, fileName))
                 test(perfKtsFileAnalysisTest())
                 tearDown(perfKtsFileAnalysisTearDown(extraTimingsNs, project))

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerformanceStressTest.kt

@@ -5,95 +5,91 @@
 
 package org.jetbrains.kotlin.idea.perf
 
-import org.jetbrains.kotlin.idea.perf.Stats.Companion.tcSuite
+import com.intellij.codeInsight.daemon.impl.HighlightInfo
+import com.intellij.testFramework.UsefulTestCase
+import org.jetbrains.kotlin.idea.perf.util.DefaultProfile
+import org.jetbrains.kotlin.idea.perf.util.PerformanceSuite
+import org.jetbrains.kotlin.idea.perf.util.PerformanceSuite.TypingConfig
+import org.jetbrains.kotlin.idea.perf.util.ProjectProfile
+import org.jetbrains.kotlin.idea.perf.util.suite
+import org.jetbrains.kotlin.idea.testFramework.commitAllDocuments
+import org.jetbrains.kotlin.test.JUnit3RunnerWithInners
+import org.junit.runner.RunWith
 
-class PerformanceStressTest : AbstractPerformanceProjectsTest() {
-
-    companion object {
-
-        @JvmStatic
-        var warmedUp: Boolean = false
-
-        @JvmStatic
-        val hwStats: Stats = Stats("helloWorld project")
-
-        init {
-            // there is no @AfterClass for junit3.8
-            Runtime.getRuntime().addShutdownHook(Thread { hwStats.close() })
-        }
-
-    }
-
-    override fun setUp() {
-        super.setUp()
-        // warm up: open simple small project
-        if (!warmedUp) {
-            warmUpProject(hwStats, "src/HelloMain.kt") {
-                openProject {
-                    name("helloWorld")
-
-                    kotlinFile("HelloMain") {
-                        topFunction("main") {
-                            param("args", "Array<String>")
-                            body("""println("Hello World!")""")
-                        }
-                    }
-                }
-            }
-            warmedUp = true
-        }
-    }
+@RunWith(JUnit3RunnerWithInners::class)
+class PerformanceStressTest : UsefulTestCase() {
 
     fun testLotsOfOverloadedMethods() {
         // KT-35135
         val generatedTypes = mutableListOf(listOf<String>())
         generateTypes(arrayOf("Int", "String", "Long", "List<Int>", "Array<Int>"), generatedTypes)
 
-        tcSuite("Lots of overloaded method project") {
-            myProject = openProject {
-                name("kt-35135")
-                buildGradle("idea/testData/perfTest/simpleTemplate/")
+        suite(
+            suiteName = "Lots of overloaded method project",
+            config = PerformanceSuite.StatsScopeConfig(name = "kt-35135 project", warmup = 8, iterations = 15)
+        ) {
+            app {
+                warmUpProject()
 
-                kotlinFile("OverloadX") {
-                    pkg("pkg")
+                project {
+                    descriptor {
+                        name("kt-35135")
+                        buildGradle("idea/testData/perfTest/simpleTemplate/")
 
-                    topClass("OverloadX") {
-                        openClass()
+                        kotlinFile("OverloadX") {
+                            pkg("pkg")
 
-                        for (types in generatedTypes) {
-                            function("foo") {
-                                openFunction()
-                                returnType("String")
-                                for ((index, type) in types.withIndex()) {
-                                    param("arg$index", type)
+                            topClass("OverloadX") {
+                                openClass()
+
+                                for (types in generatedTypes) {
+                                    function("foo") {
+                                        openFunction()
+                                        returnType("String")
+                                        for ((index, type) in types.withIndex()) {
+                                            param("arg$index", type)
+                                        }
+                                        body("TODO()")
+                                    }
                                 }
-                                body("TODO()")
+                            }
+                        }
+
+                        kotlinFile("SomeClass") {
+                            pkg("pkg")
+
+                            topClass("SomeClass") {
+                                superClass("OverloadX")
+
+                                body("ov")
+                            }
+                        }
+                    }
+
+                    profile(DefaultProfile)
+
+                    fixture("src/main/java/pkg/SomeClass.kt").use { fixture ->
+                        val typingConfig = TypingConfig(
+                            fixture,
+                            marker = "ov",
+                            insertString = "override fun foo(): String = TODO()",
+                            delayMs = 50
+                        )
+
+                        measure<List<HighlightInfo>>("type override fun foo()", fixture = fixture) {
+                            before = {
+                                moveCursor(typingConfig)
+                            }
+                            test = {
+                                typeAndHighlight(typingConfig)
+                            }
+                            after = {
+                                fixture.restoreText()
+                                commitAllDocuments()
                             }
                         }
                     }
                 }
-
-                kotlinFile("SomeClass") {
-                    pkg("pkg")
-
-                    topClass("SomeClass") {
-                        superClass("OverloadX")
-
-                        body("ov")
-                    }
-                }
-            }
-
-            val stats = Stats("kt-35135 project")
-            stats.use { stats ->
-                perfType(
-                    stats,
-                    "src/main/java/pkg/SomeClass.kt",
-                    "ov",
-                    "override fun foo(): String = TODO()",
-                    note = "override fun foo()",
-                    delay = 50
-                )
             }
         }
     }

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/ProjectBuilder.kt

@@ -9,6 +9,8 @@ import com.intellij.openapi.project.Project
 import com.intellij.openapi.projectRoots.JavaSdk
 import com.intellij.openapi.projectRoots.impl.JavaAwareProjectJdkTableImpl
 import com.intellij.util.io.*
+import org.jetbrains.kotlin.idea.perf.util.ProfileTools.Companion.initDefaultProfile
+import org.jetbrains.kotlin.idea.testFramework.GRADLE_JDK_NAME
 import org.jetbrains.kotlin.idea.testFramework.OpenProject
 import org.jetbrains.kotlin.idea.testFramework.ProjectOpenAction
 import java.nio.file.Files
@@ -209,31 +211,54 @@ class ProjectBuilder {
         //
         return targetDirectory.toAbsolutePath().toFile().absolutePath
     }
+
+    fun openProjectOperation(): OpenProjectOperation {
+        val builder = this
+        val projectPath = generateFiles()
+
+        val jdkTableImpl = JavaAwareProjectJdkTableImpl.getInstanceEx()
+        val homePath = if (jdkTableImpl.internalJdk.homeDirectory!!.name == "jre") {
+            jdkTableImpl.internalJdk.homeDirectory!!.parent.path
+        } else {
+            jdkTableImpl.internalJdk.homePath!!
+        }
+        val javaSdk = JavaSdk.getInstance()
+        val jdk18 = javaSdk.createJdk("1.8", homePath)
+
+        val openAction = if (buildGradle != null) ProjectOpenAction.GRADLE_PROJECT else ProjectOpenAction.SIMPLE_JAVA_MODULE
+        val openProject = OpenProject(
+            projectPath = projectPath,
+            projectName = name,
+            jdk = jdk18,
+            projectOpenAction = openAction
+        )
+
+        return object : OpenProjectOperation {
+
+            override fun openProject() = ProjectOpenAction.openProject(openProject)
+
+            override fun postOpenProject(project: Project) {
+                openAction.postOpenProject(project = project, openProject = openProject)
+                if (builder.initDefaultProfile) {
+                    project.initDefaultProfile()
+                }
+            }
+
+        }
+    }
+}
+
+interface OpenProjectOperation {
+    fun openProject(): Project
+
+    fun postOpenProject(project: Project)
 }
 
 fun openProject(initializer: ProjectBuilder.() -> Unit): Project {
     val projectBuilder = ProjectBuilder().apply(initializer)
-    val projectPath = projectBuilder.generateFiles()
+    val openProject = projectBuilder.openProjectOperation()
 
-    val jdkTableImpl = JavaAwareProjectJdkTableImpl.getInstanceEx()
-    val homePath = if (jdkTableImpl.internalJdk.homeDirectory!!.name == "jre") {
-        jdkTableImpl.internalJdk.homeDirectory!!.parent.path
-    } else {
-        jdkTableImpl.internalJdk.homePath!!
+    return openProject.openProject().also {
+        openProject.postOpenProject(it)
     }
-    val javaSdk = JavaSdk.getInstance()
-    val jdk18 = javaSdk.createJdk("1.8", homePath)
-    val openAction = if (projectBuilder.buildGradle != null) ProjectOpenAction.GRADLE_PROJECT else ProjectOpenAction.SIMPLE_JAVA_MODULE
-    val openProject = OpenProject(
-        projectPath = projectPath,
-        projectName = projectBuilder.name,
-        jdk = jdk18,
-        projectOpenAction = openAction
-    )
-    val project = ProjectOpenAction.openProject(openProject)
-    openAction.postOpenProject(project = project, openProject = openProject)
-    if (projectBuilder.initDefaultProfile) {
-        openAction.initDefaultProfile(project)
-    }
-    return project
 }

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt

@@ -7,7 +7,8 @@ package org.jetbrains.kotlin.idea.perf
 
 import org.jetbrains.kotlin.idea.perf.WholeProjectPerformanceTest.Companion.nsToMs
 import org.jetbrains.kotlin.idea.perf.profilers.*
-import org.jetbrains.kotlin.idea.testFramework.logMessage
+import org.jetbrains.kotlin.idea.perf.util.TeamCity
+import org.jetbrains.kotlin.idea.perf.util.logMessage
 import org.jetbrains.kotlin.idea.testFramework.suggestOsNeutralFileName
 import org.jetbrains.kotlin.util.PerformanceCounter
 import java.io.*
@@ -57,9 +58,9 @@ class Stats(
         )) {
             val n = "$id : ${v.first}"
 
-            printTestStarted(n)
-            printStatValue("$id${v.second}", v.third)
-            printTestFinished(n, v.third, includeStatValue = false)
+            TeamCity.test(n, durationMs = v.third) {
+                TeamCity.statValue("$id${v.second}", v.third)
+            }
         }
 
         statInfosArray.filterNotNull()
@@ -75,9 +76,11 @@ class Stats(
 
                 val shortName = if (perfCounterName.endsWith(": time")) n.removeSuffix(": time") else null
 
-                shortName?.let { printTestStarted(it) }
-                printStatValue(n, mean)
-                shortName?.let { printTestFinished(it, mean, includeStatValue = false) }
+                shortName?.let {
+                    TeamCity.test(it, durationMs = mean) {
+                        TeamCity.statValue(it, mean)
+                    }
+                }
             }
 
         perfTestRawDataMs.addAll(timingsMs.toList())
@@ -161,15 +164,15 @@ class Stats(
                     val stabilityName = "$name: $testName stability"
 
                     val stable = stabilityPercentage <= acceptanceStabilityLevel
-                    printTestStarted(stabilityName)
-                    printStatValue(stabilityName, stabilityPercentage)
-                    if (stable or !checkStability) {
-                        printTestFinished(stabilityName)
+
+                    val error = if (stable or !checkStability) {
+                        null
                     } else {
-                        printTestFailed(
-                            stabilityName,
-                            "$testName stability is $stabilityPercentage %, above accepted level of $acceptanceStabilityLevel %"
-                        )
+                        "$testName stability is $stabilityPercentage %, above accepted level of $acceptanceStabilityLevel %"
+                    }
+
+                    TeamCity.test(stabilityName, errorDetails = error) {
+                        TeamCity.statValue(stabilityName, stabilityPercentage)
                     }
                 }
             } else {
@@ -178,7 +181,7 @@ class Stats(
         }
 
         if (testName != WARM_UP) {
-            tcSuite(testName, block)
+            TeamCity.suite(testName, block)
         } else {
             block()
         }
@@ -197,19 +200,17 @@ class Stats(
 
             val t = statInfo[ERROR_KEY] as? Throwable
             if (t != null) {
-                printTestStarted(n)
-                printTestFinished(n, t)
+                TeamCity.test(n, errors = listOf(t)) {}
             } else if (!printOnlyErrors) {
-                printTestStarted(n)
-                for ((k, v) in statInfo) {
-                    if (k == TEST_KEY) continue
-                    printStatValue("$n $k", v)
-                    (v as? Number)?.let {
-                        printTestMetadata(n, k, it)
+                TeamCity.test(n, durationMs = (statInfo[TEST_KEY] as Long).nsToMs) {
+                    for ((k, v) in statInfo) {
+                        if (k == TEST_KEY) continue
+                        TeamCity.statValue("$n $k", v)
+                        (v as? Number)?.let {
+                            TeamCity.testMetadata(n, k, it)
+                        }
                     }
                 }
-
-                printTestFinished(n, (statInfo[TEST_KEY] as Long).nsToMs)
             }
         }
     }
@@ -245,7 +246,14 @@ class Stats(
 
     private fun <SV, TV> mainPhase(phaseData: PhaseData<SV, TV>): Array<StatInfos> {
         val statInfosArray = phase(phaseData, "")
-        statInfosArray.filterNotNull().map { it[ERROR_KEY] as? Throwable }.firstOrNull()?.let { throw it }
+        statInfosArray.filterNotNull().map { it[ERROR_KEY] as? Throwable }.firstOrNull()?.let {
+            printTimings(
+                phaseData.testName,
+                printOnlyErrors = true,
+                statInfoArray = statInfosArray
+            )
+            throw it
+        }
         return statInfosArray
     }
 
@@ -300,7 +308,7 @@ class Stats(
             }
         } catch (t: Throwable) {
             logMessage(t) { "error at ${phaseData.testName}" }
-            tcPrintErrors(phaseData.testName, listOf(t))
+            TeamCity.testFailed(name, error = t)
         }
         return statInfosArray
     }
@@ -338,7 +346,7 @@ class Stats(
     override fun close() {
         if (perfTestRawDataMs.isNotEmpty()) {
             val geomMeanMs = geomMean(perfTestRawDataMs.toList()).toLong()
-            printStatValue("$name geomMean", geomMeanMs)
+            TeamCity.statValue("$name geomMean", geomMeanMs)
             append(arrayOf("$name geomMean", geomMeanMs, 0))
         }
         statsOutput.flush()
@@ -357,75 +365,6 @@ class Stats(
             }
             logMessage { "$note took $openProjectMillis ms" }
         }
-
-        fun printTestStarted(testName: String) {
-            println("##teamcity[testStarted name='$testName' captureStandardOutput='true']")
-        }
-
-        fun printStatValue(name: String, value: Any) {
-            println("##teamcity[buildStatisticValue key='$name' value='$value']")
-        }
-
-        fun printTestMetadata(testName: String, name: String, value: Number) {
-            println("##teamcity[testMetadata testName='$testName' name='$name' type='number' value='$value']")
-        }
-
-        private fun printTestFinished(testName: String) {
-            println("##teamcity[testFinished name='$testName']")
-        }
-
-        fun printTestFinished(testName: String, spentMs: Long, includeStatValue: Boolean = true) {
-            if (includeStatValue) {
-                printStatValue(testName, spentMs)
-            }
-            println("##teamcity[testFinished name='$testName' duration='$spentMs']")
-        }
-
-        fun printTestFinished(testName: String, error: Throwable) {
-            println("error at $testName:")
-            printStatValue(testName, -1)
-            tcPrintErrors(testName, listOf(error))
-        }
-
-        private fun printTestFailed(testName: String, details: String) {
-            println("##teamcity[testFailed name='$testName' message='Exceptions reported' details='${tcEscape(details)}']")
-        }
-
-        inline fun tcSuite(name: String, block: () -> Unit) {
-            println("##teamcity[testSuiteStarted name='$name']")
-            try {
-                block()
-            } finally {
-                println("##teamcity[testSuiteFinished name='$name']")
-            }
-        }
-
-        inline fun tcTest(name: String, block: () -> Pair<Long, List<Throwable>>) {
-            printTestStarted(name)
-            val (time, errors) = block()
-            tcPrintErrors(name, errors)
-        }
-
-        private fun tcEscape(s: String) = s.replace("|", "||")
-            .replace("[", "|[")
-            .replace("]", "|]")
-            .replace("\r", "|r")
-            .replace("\n", "|n")
-            .replace("'", "|'")
-
-        fun tcPrintErrors(name: String, errors: List<Throwable>) {
-            if (errors.isNotEmpty()) {
-                val detailsWriter = StringWriter()
-                val errorDetailsPrintWriter = PrintWriter(detailsWriter)
-                errors.forEach {
-                    it.printStackTrace(errorDetailsPrintWriter)
-                    errorDetailsPrintWriter.println()
-                }
-                errorDetailsPrintWriter.close()
-                val details = detailsWriter.toString()
-                printTestFailed(name, details)
-            }
-        }
     }
 
 }

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/WarmUpProject.kt

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.idea.perf
+
+/**
+ * warm up: open simple `hello world` project
+ */
+class WarmUpProject(private val stats: Stats) {
+    private var warmedUp: Boolean = false
+
+    fun warmUp(test: AbstractPerformanceProjectsTest) {
+        if (warmedUp) return
+        test.warmUpProject(stats, "src/HelloMain.kt") {
+            openProject {
+                name("helloWorld")
+
+                kotlinFile("HelloMain") {
+                    topFunction("main") {
+                        param("args", "Array<String>")
+                        body("""println("Hello World!")""")
+                    }
+                }
+            }
+        }
+        warmedUp = true
+    }
+}

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/WholeProjectPerformanceTest.kt

@@ -28,8 +28,8 @@ import com.intellij.psi.impl.source.resolve.reference.ReferenceProvidersRegistry
 import com.intellij.psi.xml.XmlFileNSInfoProvider
 import com.intellij.xml.XmlSchemaProvider
 import org.jetbrains.kotlin.idea.framework.KotlinSdkType
-import org.jetbrains.kotlin.idea.perf.Stats.Companion.tcSuite
-import org.jetbrains.kotlin.idea.perf.Stats.Companion.tcTest
+import org.jetbrains.kotlin.idea.perf.util.TeamCity
+import org.jetbrains.kotlin.idea.testFramework.TestApplicationManager
 import org.jetbrains.kotlin.idea.util.getProjectJdkTableSafe
 import java.io.File
 
@@ -95,7 +95,7 @@ abstract class WholeProjectPerformanceTest : DaemonAnalyzerTestCase(), WholeProj
 
     fun testWholeProjectPerformance() {
 
-        tcSuite(this::class.simpleName ?: "Unknown") {
+        TeamCity.suite(this::class.simpleName ?: "Unknown") {
             val totals = mutableMapOf<String, Long>()
 
             fun appendInspectionResult(file: String, id: String, nanoTime: Long) {
@@ -104,7 +104,7 @@ abstract class WholeProjectPerformanceTest : DaemonAnalyzerTestCase(), WholeProj
                 perfStats.append(file, id, nanoTime)
             }
 
-            tcSuite("TotalPerFile") {
+            TeamCity.suite("TotalPerFile") {
                 // TODO: [VD] temp to limit number of files (and total time to run)
                 val files = provideFiles(project).sortedBy { it.name }.take(10)
 
@@ -120,7 +120,7 @@ abstract class WholeProjectPerformanceTest : DaemonAnalyzerTestCase(), WholeProj
                 }
             }
 
-            tcSuite("Total") {
+            TeamCity.suite("Total") {
                 totals.forEach { (k, v) ->
                     tcTest(k) {
                         v.nsToMs to emptyList()
@@ -139,6 +139,11 @@ abstract class WholeProjectPerformanceTest : DaemonAnalyzerTestCase(), WholeProj
 
     companion object {
         val Long.nsToMs get() = (this * 1e-6).toLong()
+
+        inline fun tcTest(name: String, block: () -> Pair<Long, List<Throwable>>) {
+            val (time, errors) = block()
+            TeamCity.test(name, time, errors = errors) {}
+        }
     }
 
 }

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/compat.kt

@@ -1,20 +0,0 @@
-/*
- * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-@file:Suppress("UNUSED_PARAMETER")
-
-package org.jetbrains.kotlin.idea.perf
-
-import com.intellij.openapi.Disposable
-import com.intellij.openapi.project.Project
-import com.intellij.testFramework.propertyBased.MadTestingUtil
-
-// BUNCH: 193
-fun enableAllInspectionsCompat(project: Project, disposable: Disposable) {
-    MadTestingUtil.enableAllInspections(project, disposable)
-}
-
-// BUNCH: 193
-typealias TestApplicationManager = com.intellij.idea.IdeaTestApplication

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/compat.kt.201

@@ -1,20 +0,0 @@
-/*
- * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-@file:Suppress("UNUSED_PARAMETER")
-
-package org.jetbrains.kotlin.idea.perf
-
-import com.intellij.openapi.Disposable
-import com.intellij.openapi.project.Project
-import com.intellij.testFramework.propertyBased.MadTestingUtil
-
-// BUNCH: 193
-fun enableAllInspectionsCompat(project: Project, disposable: Disposable) {
-    MadTestingUtil.enableAllInspections(project)
-}
-
-// BUNCH: 193
-typealias TestApplicationManager = com.intellij.testFramework.TestApplicationManager

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/profilers/ProfilerHandler.kt

@@ -7,7 +7,7 @@ package org.jetbrains.kotlin.idea.perf.profilers
 
 import org.jetbrains.kotlin.idea.perf.profilers.async.AsyncProfilerHandler
 import org.jetbrains.kotlin.idea.perf.profilers.yk.YKProfilerHandler
-import org.jetbrains.kotlin.idea.testFramework.logMessage
+import org.jetbrains.kotlin.idea.perf.util.logMessage
 
 interface ProfilerHandler {
     fun startProfiling(activityName: String, options: List<String> = emptyList())

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/profilers/async/AsyncProfilerHandler.kt

@@ -10,7 +10,7 @@ import com.intellij.openapi.util.io.FileUtil
 import com.intellij.util.containers.ContainerUtil
 import org.jetbrains.kotlin.idea.perf.profilers.ProfilerHandler
 import org.jetbrains.kotlin.idea.perf.profilers.doOrThrow
-import org.jetbrains.kotlin.idea.testFramework.logMessage
+import org.jetbrains.kotlin.idea.perf.util.logMessage
 import java.io.File
 import java.lang.reflect.Method
 import java.nio.file.Files

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/profilers/yk/YKProfilerHandler.kt

@@ -7,7 +7,7 @@ package org.jetbrains.kotlin.idea.perf.profilers.yk
 
 import org.jetbrains.kotlin.idea.perf.profilers.ProfilerHandler
 import org.jetbrains.kotlin.idea.perf.profilers.doOrThrow
-import org.jetbrains.kotlin.idea.testFramework.logMessage
+import org.jetbrains.kotlin.idea.perf.util.logMessage
 import java.lang.management.ManagementFactory
 import java.lang.reflect.Method
 import java.nio.file.Files

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/fileTools.kt

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.idea.perf.util
+
+import com.intellij.util.io.exists
+import java.io.File
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.Paths
+import java.nio.file.StandardCopyOption
+
+fun String.lastPathSegment() =
+    Paths.get(this).last().toString()
+
+fun exists(path: String, vararg paths: String) =
+    Paths.get(path, *paths).toAbsolutePath().exists()
+
+fun Path.copyRecursively(targetDirectory: Path) {
+    val src = this
+    Files.walk(src)
+        .forEach { source -> Files.copy(source, targetDirectory.resolve(src.relativize(source)), StandardCopyOption.REPLACE_EXISTING) }
+}
+
+fun File.allFilesWithExtension(ext: String): Iterable<File> =
+    walk().filter { it.isFile && it.extension.equals(ext, ignoreCase = true) }.toList()

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/logging.kt

@@ -0,0 +1,99 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.idea.perf.util
+
+import java.io.PrintWriter
+import java.io.StringWriter
+
+inline fun gradleMessage(block: () -> String) {
+    print("#gradle ${block()}")
+}
+
+inline fun logMessage(message: () -> String) {
+    println("-- ${message()}")
+}
+
+fun logMessage(t: Throwable, message: () -> String) {
+    val writer = StringWriter()
+    PrintWriter(writer).use {
+        t.printStackTrace(it)
+    }
+    println("-- ${message()}:\n$writer")
+}
+
+object TeamCity {
+    inline fun message(block: () -> String) {
+        println("##teamcity[${block()}]")
+    }
+
+    fun suite(name: String, block: () -> Unit) {
+        message { "testSuiteStarted name='$name'" }
+        try {
+            block()
+        } finally {
+            message { "testSuiteFinished name='$name'" }
+        }
+    }
+
+    fun test(name: String, durationMs: Long? = null, errors: List<Throwable>, block: () -> Unit) {
+        test(name, durationMs, if (errors.isNotEmpty()) toDetails(errors) else null, block)
+    }
+
+    fun test(name: String, durationMs: Long? = null, errorDetails: String? = null, block: () -> Unit) {
+        testStarted(name)
+        try {
+            block()
+        } finally {
+            if (errorDetails != null) {
+                statValue(name, -1)
+                testFailed(name, errorDetails)
+            } else {
+                testFinished(name, durationMs)
+            }
+        }
+    }
+
+    inline fun statValue(name: String, value: Any) {
+        message { "buildStatisticValue key='$name' value='$value'" }
+    }
+
+    inline fun testStarted(testName: String) {
+        message { "testStarted name='$testName'" }
+    }
+
+    inline fun testMetadata(testName: String, name: String, value: Number) {
+        message { "testMetadata testName='$testName' name='$name' type='number' value='$value'" }
+    }
+
+    inline fun testFinished(testName: String, durationMs: Long? = null) {
+        message { "testFinished name='$testName'${durationMs?.let { " duration='$durationMs'" }}" }
+    }
+
+    fun testFailed(testName: String, error: Throwable) = testFailed(testName, toDetails(listOf(error))!!)
+
+    fun testFailed(testName: String, details: String) {
+        message { "testFailed name='$testName' message='Exceptions reported' details='${escape(details)}'" }
+    }
+
+    private fun escape(s: String) = s.replace("|", "||")
+        .replace("[", "|[")
+        .replace("]", "|]")
+        .replace("\r", "|r")
+        .replace("\n", "|n")
+        .replace("'", "|'")
+
+    private fun toDetails(errors: List<Throwable>): String? {
+        if (errors.isEmpty()) return null
+        val detailsWriter = StringWriter()
+        val errorDetailsPrintWriter = PrintWriter(detailsWriter)
+        errors.forEach {
+            it.printStackTrace(errorDetailsPrintWriter)
+            errorDetailsPrintWriter.println()
+        }
+        errorDetailsPrintWriter.close()
+        return detailsWriter.toString()
+    }
+}

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/performanceSuite.kt

@@ -0,0 +1,416 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.idea.perf.util
+
+import com.intellij.codeInsight.daemon.impl.HighlightInfo
+import com.intellij.openapi.Disposable
+import com.intellij.openapi.application.runWriteAction
+import com.intellij.openapi.editor.EditorFactory
+import com.intellij.openapi.fileEditor.FileDocumentManager
+import com.intellij.openapi.fileEditor.FileEditorManager
+import com.intellij.openapi.project.DumbService
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.project.ex.ProjectManagerEx
+import com.intellij.openapi.projectRoots.JavaSdk
+import com.intellij.openapi.projectRoots.Sdk
+import com.intellij.openapi.projectRoots.impl.JavaAwareProjectJdkTableImpl
+import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.psi.PsiDocumentManager
+import com.intellij.psi.PsiFile
+import com.intellij.psi.PsiManager
+import com.intellij.psi.impl.PsiDocumentManagerBase
+import com.intellij.testFramework.*
+import com.intellij.testFramework.fixtures.impl.CodeInsightTestFixtureImpl
+import com.intellij.util.ArrayUtilRt
+import com.intellij.util.ThrowableRunnable
+import com.intellij.util.containers.toArray
+import com.intellij.util.indexing.UnindexedFilesUpdater
+import org.jetbrains.kotlin.idea.core.script.ScriptConfigurationManager
+import org.jetbrains.kotlin.idea.framework.KotlinSdkType
+import org.jetbrains.kotlin.idea.perf.ProjectBuilder
+import org.jetbrains.kotlin.idea.perf.Stats
+import org.jetbrains.kotlin.idea.perf.Stats.Companion.runAndMeasure
+import org.jetbrains.kotlin.idea.perf.performanceTest
+import org.jetbrains.kotlin.idea.perf.util.ProfileTools.Companion.disableAllInspections
+import org.jetbrains.kotlin.idea.perf.util.ProfileTools.Companion.enableAllInspections
+import org.jetbrains.kotlin.idea.perf.util.ProfileTools.Companion.enableInspections
+import org.jetbrains.kotlin.idea.perf.util.ProfileTools.Companion.enableSingleInspection
+import org.jetbrains.kotlin.idea.perf.util.ProfileTools.Companion.initDefaultProfile
+import org.jetbrains.kotlin.idea.test.invalidateLibraryCache
+import org.jetbrains.kotlin.idea.testFramework.*
+import org.jetbrains.kotlin.idea.testFramework.TestApplicationManager
+import org.jetbrains.kotlin.idea.util.getProjectJdkTableSafe
+import java.io.File
+
+/**
+ * @author Vladimir Ilmov
+ */
+class PerformanceSuite {
+    companion object {
+
+        fun suite(
+            name: String,
+            stats: StatsScope,
+            block: (StatsScope) -> Unit
+        ) {
+            TeamCity.suite(name) {
+                stats.stats.use {
+                    block(stats)
+                }
+            }
+        }
+
+        private fun PsiFile.highlightFile(): List<HighlightInfo> {
+            val document = FileDocumentManager.getInstance().getDocument(virtualFile)!!
+            val editor = EditorFactory.getInstance().getEditors(document).first()
+            PsiDocumentManager.getInstance(project).commitAllDocuments()
+            return CodeInsightTestFixtureImpl.instantiateAndRun(this, editor, ArrayUtilRt.EMPTY_INT_ARRAY, true)
+        }
+
+        fun rollbackChanges(vararg file: VirtualFile) {
+            val fileDocumentManager = FileDocumentManager.getInstance()
+            runInEdtAndWait {
+                fileDocumentManager.reloadFiles(*file)
+            }
+
+            ProjectManagerEx.getInstanceEx().openProjects.forEach { project ->
+                val psiDocumentManagerBase = PsiDocumentManager.getInstance(project) as PsiDocumentManagerBase
+
+                runInEdtAndWait {
+                    psiDocumentManagerBase.clearUncommittedDocuments()
+                    psiDocumentManagerBase.commitAllDocuments()
+                }
+            }
+        }
+    }
+
+    class StatsScope(val config: StatsScopeConfig, val stats: Stats, val rootDisposable: Disposable) {
+        fun app(f: ApplicationScope.() -> Unit) = ApplicationScope(rootDisposable, this).use(f)
+
+        fun <T> measure(name: String, f: MeasurementScope<T>.() -> Unit): List<T?> =
+            MeasurementScope<T>(name, stats, config).apply(f).run()
+
+        fun <T> measure(name: String, f: MeasurementScope<T>.() -> Unit, after: (() -> Unit)?): List<T?> =
+            MeasurementScope<T>(name, stats, config, after = after).apply(f).run()
+
+        fun logStatValue(name: String, value: Any) {
+            logMessage { "buildStatisticValue key='${stats.name}: $name' value='$value'" }
+            TeamCity.statValue("${stats.name}: $name", value)
+        }
+    }
+
+    data class TypingConfig(
+        val fixture: Fixture,
+        val marker: String,
+        val insertString: String,
+        val surroundItems: String = "\n",
+        val typeAfterMarker: Boolean = true,
+        val note: String = "",
+        val delayMs: Long? = null
+    ) : AutoCloseable {
+        override fun close() {
+            fixture.close()
+        }
+    }
+
+    class MeasurementScope<T>(
+        val name: String,
+        val stats: Stats,
+        val config: StatsScopeConfig,
+        var before: () -> Unit = {},
+        var test: (() -> T?)? = null,
+        var after: (() -> Unit)? = null
+    ) {
+        fun run(): List<T?> {
+            val t = test ?: error("test procedure isn't set")
+            val value = mutableListOf<T?>()
+            performanceTest<Unit, T> {
+                name(name)
+                stats(stats)
+                warmUpIterations(config.warmup)
+                iterations(config.iterations)
+                setUp {
+                    before()
+                }
+                test {
+                    value.add(t.invoke())
+                }
+                tearDown {
+                    after?.invoke()
+                }
+                profilerEnabled(config.profile)
+            }
+            return value
+        }
+    }
+
+    class ApplicationScope(val rootDisposable: Disposable, val stats: StatsScope) : AutoCloseable {
+        val application = initApp(rootDisposable)
+        val jdk: Sdk = initSdk(rootDisposable)
+
+        fun project(externalProject: ExternalProject, refresh: Boolean = false, block: ProjectScope.() -> Unit) =
+            ProjectScope(ProjectScopeConfig(externalProject, refresh), this).use(block)
+
+        fun project(block: ProjectWithDescriptorScope.() -> Unit) =
+            ProjectWithDescriptorScope(this).use(block)
+
+        fun project(path: String, openWith: ProjectOpenAction = ProjectOpenAction.EXISTING_IDEA_PROJECT, block: ProjectScope.() -> Unit) =
+            ProjectScope(ProjectScopeConfig(path, openWith), this).use(block)
+
+        fun gradleProject(path: String, refresh: Boolean = false, block: ProjectScope.() -> Unit) =
+            ProjectScope(ProjectScopeConfig(path, ProjectOpenAction.GRADLE_PROJECT, refresh), this).use(block)
+
+        fun warmUpProject() = project {
+            descriptor {
+                name("helloWorld")
+
+                kotlinFile("HelloMain") {
+                    topFunction("main") {
+                        param("args", "Array<String>")
+                        body("""println("Hello World!")""")
+                    }
+                }
+            }
+
+            fixture("src/HelloMain.kt").use {
+                highlight(it)
+            }
+        }
+
+        override fun close() {
+            application?.setDataProvider(null)
+        }
+
+        companion object {
+            fun initApp(rootDisposable: Disposable): TestApplicationManager {
+                val application = TestApplicationManager.getInstance()
+                GradleProcessOutputInterceptor.install(rootDisposable)
+                return application
+            }
+
+            fun initSdk(rootDisposable: Disposable): Sdk {
+                return runWriteAction {
+                    val jdkTableImpl = JavaAwareProjectJdkTableImpl.getInstanceEx()
+                    val homePath = if (jdkTableImpl.internalJdk.homeDirectory!!.name == "jre") {
+                        jdkTableImpl.internalJdk.homeDirectory!!.parent.path
+                    } else {
+                        jdkTableImpl.internalJdk.homePath!!
+                    }
+
+                    val javaSdk = JavaSdk.getInstance()
+                    val jdk = javaSdk.createJdk("1.8", homePath)
+                    val internal = javaSdk.createJdk("IDEA jdk", homePath)
+                    val gradle = javaSdk.createJdk(GRADLE_JDK_NAME, homePath)
+
+                    val jdkTable = getProjectJdkTableSafe()
+                    jdkTable.addJdk(jdk, rootDisposable)
+                    jdkTable.addJdk(internal, rootDisposable)
+                    jdkTable.addJdk(gradle, rootDisposable)
+                    KotlinSdkType.setUpIfNeeded()
+                    jdk
+                }
+            }
+        }
+    }
+
+
+    class StatsScopeConfig(var name: String? = null, var warmup: Int = 2, var iterations: Int = 5, var profile: Boolean = false)
+
+    class ProjectScopeConfig(val path: String, val openWith: ProjectOpenAction, val refresh: Boolean = false) {
+        val name: String = path.lastPathSegment()
+
+        constructor(externalProject: ExternalProject, refresh: Boolean) : this(externalProject.path, externalProject.openWith, refresh)
+    }
+
+    abstract class AbstractProjectScope(val app: ApplicationScope) : AutoCloseable {
+        abstract val project: Project
+        val openFiles = mutableListOf<VirtualFile>()
+
+        fun profile(profile: ProjectProfile) {
+            when (profile) {
+                EmptyProfile -> project.disableAllInspections()
+                DefaultProfile -> project.initDefaultProfile()
+                FullProfile -> project.enableAllInspections()
+                is CustomProfile -> project.enableInspections(*profile.inspectionNames.toArray(emptyArray()))
+            }
+        }
+
+        fun highlight(fixture: Fixture) = highlight(fixture.psiFile)
+
+        fun highlight(editorFile: PsiFile?) =
+            editorFile?.let {
+                it.highlightFile()
+            } ?: error("editor isn't ready for highlight")
+
+        fun moveCursor(config: TypingConfig) {
+            val fixture = config.fixture
+            val editor = fixture.editor
+            updateScriptDependenciesIfNeeded(fixture)
+
+            val marker = config.marker
+            val tasksIdx = fixture.text.indexOf(marker)
+            check(tasksIdx > 0) {
+                "marker '$marker' not found in ${fixture.fileName}"
+            }
+            if (config.typeAfterMarker) {
+                editor.caretModel.moveToOffset(tasksIdx + marker.length + 1)
+            } else {
+                editor.caretModel.moveToOffset(tasksIdx - 1)
+            }
+
+            for (surroundItem in config.surroundItems) {
+                EditorTestUtil.performTypingAction(editor, surroundItem)
+            }
+
+            editor.caretModel.moveToOffset(editor.caretModel.offset - if (config.typeAfterMarker) 1 else 2)
+
+            if (!config.typeAfterMarker) {
+                for (surroundItem in config.surroundItems) {
+                    EditorTestUtil.performTypingAction(editor, surroundItem)
+                }
+                editor.caretModel.moveToOffset(editor.caretModel.offset - 2)
+            }
+        }
+
+        fun typeAndHighlight(config: TypingConfig): List<HighlightInfo> {
+            val string = config.insertString
+            for (i in string.indices) {
+                config.fixture.type(string[i])
+                config.delayMs?.let { d -> Thread.sleep(d) }
+            }
+            return config.fixture.doHighlighting()
+        }
+
+        fun updateScriptDependenciesIfNeeded(fixture: Fixture) {
+            val path = fixture.vFile.path
+            if (Fixture.isAKotlinScriptFile(path)) {
+                runAndMeasure("update script dependencies for $path") {
+                    ScriptConfigurationManager.updateScriptDependenciesSynchronously(fixture.psiFile)
+                }
+            }
+        }
+
+        fun enableSingleInspection(inspectionName: String) =
+            this.project.enableSingleInspection(inspectionName)
+
+        fun enableAllInspections() =
+            this.project.enableAllInspections()
+
+        fun editor(path: String) =
+            Fixture.openFileInEditor(project, path).psiFile.also { openFiles.add(it.virtualFile) }
+
+        fun fixture(path: String): Fixture {
+            val fixture = Fixture.openFixture(project, path)
+            openFiles.add(fixture.vFile)
+            if (Fixture.isAKotlinScriptFile(path)) {
+                ScriptConfigurationManager.updateScriptDependenciesSynchronously(fixture.psiFile)
+            }
+            return fixture
+        }
+
+        fun rollbackChanges() =
+            rollbackChanges(*openFiles.toTypedArray())
+
+        fun close(editorFile: PsiFile?) {
+            commitAllDocuments()
+            editorFile?.virtualFile?.let {
+                FileEditorManager.getInstance(project).closeFile(it)
+            }
+        }
+
+        fun <T> measure(vararg name: String, f: MeasurementScope<T>.() -> Unit): List<T?> {
+            val after = { PsiManager.getInstance(project).dropPsiCaches() }
+            return app.stats.measure("${name.joinToString("-")}", f, after)
+        }
+
+        fun <T> measure(vararg name: String, fixture: Fixture, f: MeasurementScope<T>.() -> Unit): List<T?> =
+            measure("${name.joinToString("-")} ${fixture.fileName}", f = f)
+
+        override fun close() {
+            RunAll(
+                ThrowableRunnable {
+                    project?.let { prj ->
+                        app.application?.closeProject(prj)
+                    }
+                }).run()
+        }
+    }
+
+    class ProjectWithDescriptorScope(app: ApplicationScope) : AbstractProjectScope(app) {
+        private var descriptor: ProjectBuilder? = null
+        override val project: Project by lazy {
+            val builder = descriptor ?: error("project is not configured")
+            val openProject = builder.openProjectOperation()
+
+            openProject.openProject().also {
+                openProject.postOpenProject(it)
+            }
+        }
+
+        fun descriptor(descriptor: ProjectBuilder.() -> Unit) {
+            this.descriptor = ProjectBuilder().apply(descriptor)
+        }
+    }
+
+    class ProjectScope(config: ProjectScopeConfig, app: ApplicationScope) : AbstractProjectScope(app) {
+        override val project: Project = initProject(config, app)
+
+        companion object {
+            fun initProject(config: ProjectScopeConfig, app: ApplicationScope): Project {
+                val projectPath = File(config.path).canonicalPath
+
+                UsefulTestCase.assertTrue("path ${config.path} does not exist, check README.md", File(projectPath).exists())
+
+                val openProject = OpenProject(
+                    projectPath = projectPath,
+                    projectName = config.name,
+                    jdk = app.jdk,
+                    projectOpenAction = config.openWith
+                )
+                val project = ProjectOpenAction.openProject(openProject)
+                openProject.projectOpenAction.postOpenProject(project, openProject)
+
+                // indexing
+                if (config.refresh) {
+                    invalidateLibraryCache(project)
+                }
+
+                CodeInsightTestFixtureImpl.ensureIndexesUpToDate(project)
+
+                dispatchAllInvocationEvents()
+                with(DumbService.getInstance(project)) {
+                    queueTask(UnindexedFilesUpdater(project))
+                    completeJustSubmittedTasks()
+                }
+                dispatchAllInvocationEvents()
+
+                Fixture.enableAnnotatorsAndLoadDefinitions(project)
+
+                app.application?.setDataProvider(TestDataProvider(project))
+
+                return project
+            }
+        }
+    }
+}
+
+sealed class ProjectProfile
+object EmptyProfile : ProjectProfile()
+object DefaultProfile : ProjectProfile()
+object FullProfile : ProjectProfile()
+data class CustomProfile(val inspectionNames: List<String>) : ProjectProfile()
+
+fun UsefulTestCase.suite(
+    suiteName: String? = null,
+    config: PerformanceSuite.StatsScopeConfig = PerformanceSuite.StatsScopeConfig(),
+    block: PerformanceSuite.StatsScope.() -> Unit
+) {
+    PerformanceSuite.suite(
+        suiteName ?: this.javaClass.name,
+        PerformanceSuite.StatsScope(config, Stats(config.name ?: suiteName ?: name), testRootDisposable),
+        block
+    )
+}

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/profileTools.kt

@@ -0,0 +1,96 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.idea.perf.util
+
+import com.intellij.codeInspection.ex.InspectionProfileImpl
+import com.intellij.codeInspection.ex.Tools
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.util.Disposer
+import com.intellij.profile.codeInspection.InspectionProjectProfileManager
+import com.intellij.profile.codeInspection.ProjectInspectionProfileManager
+
+class ProfileTools {
+    companion object {
+        internal fun Project.enableAllInspections() {
+            InspectionProfileImpl.INIT_INSPECTIONS = true
+            val profile = InspectionProfileImpl("all-inspections")
+            profile.enableAllTools(this)
+            replaceProfile(profile)
+        }
+
+        internal fun Project.disableAllInspections() {
+            InspectionProfileImpl.INIT_INSPECTIONS = true
+            val profile = InspectionProfileImpl("no-inspections")
+            profile.disableAllTools(this)
+            replaceProfile(profile)
+        }
+
+        internal fun Project.initDefaultProfile() {
+            val projectInspectionProfileManager = ProjectInspectionProfileManager.getInstance(this)
+            projectInspectionProfileManager.forceLoadSchemes()
+
+            val projectProfile = projectInspectionProfileManager.projectProfile ?: error("project has to have non null profile name")
+            val profile = projectInspectionProfileManager.getProfile(projectProfile)
+            InspectionProfileImpl.INIT_INSPECTIONS = true
+            profile.initInspectionTools(this)
+            // disable some known `bad` inspections
+            profile.disableInspection(this, "SSBasedInspection")
+
+            val enabledTools = profile.getAllEnabledInspectionTools(this)
+            check(enabledTools.isNotEmpty()) {
+                "project $name has to have at least one enabled inspection in profile"
+            }
+            InspectionProfileImpl.INIT_INSPECTIONS = false
+        }
+
+        internal fun Project.enableSingleInspection(inspectionName: String) {
+            InspectionProfileImpl.INIT_INSPECTIONS = true
+            val profile = InspectionProfileImpl("$inspectionName-only")
+            profile.disableAllTools(this)
+            profile.enableTool(inspectionName, this)
+
+            replaceProfile(profile)
+        }
+
+        internal fun Project.enableInspections(vararg inspectionNames: String) {
+            InspectionProfileImpl.INIT_INSPECTIONS = true
+            val profile = InspectionProfileImpl("custom")
+            profile.disableAllTools(this)
+            inspectionNames.forEach {
+                profile.enableTool(it, this)
+            }
+
+            replaceProfile(profile)
+        }
+
+        internal fun InspectionProfileImpl.disableInspection(project: Project, vararg shortIds: String) {
+            shortIds.forEach { shortId ->
+                this.getToolsOrNull(shortId, project)?.let { it.isEnabled = false }
+            }
+        }
+
+        private fun Project.replaceProfile(profile: InspectionProfileImpl) {
+            preloadProfileTools(profile, this)
+            val manager = InspectionProjectProfileManager.getInstance(this) as ProjectInspectionProfileManager
+            manager.addProfile(profile)
+            val prev = manager.currentProfile
+            manager.setCurrentProfile(profile)
+            Disposer.register(this, {
+                InspectionProfileImpl.INIT_INSPECTIONS = false
+                manager.setCurrentProfile(prev)
+                manager.deleteProfile(profile)
+            })
+        }
+
+        private fun preloadProfileTools(
+            profile: InspectionProfileImpl,
+            project: Project
+        ) {
+            // instantiate all tools to avoid extension loading in inconvenient moment
+            profile.getAllEnabledInspectionTools(project).forEach { state: Tools -> state.tool.getTool() }
+        }
+    }
+}

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/testProject.kt

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.idea.perf.util
+
+import org.jetbrains.kotlin.idea.testFramework.ProjectOpenAction
+
+/**
+ * @author Vladimir Ilmov
+ */
+class ExternalProject(val path: String, val openWith: ProjectOpenAction) {
+    companion object {
+        const val KOTLIN_PROJECT_PATH = "../perfTestProject"
+
+        val KOTLIN_GRADLE = ExternalProject(KOTLIN_PROJECT_PATH, ProjectOpenAction.GRADLE_PROJECT)
+        val KOTLIN_JPS = ExternalProject(KOTLIN_PROJECT_PATH, ProjectOpenAction.EXISTING_IDEA_PROJECT)
+        val KOTLIN_AUTO = ExternalProject(KOTLIN_PROJECT_PATH, autoOpenAction(KOTLIN_PROJECT_PATH))
+
+        fun autoOpenAction(path: String): ProjectOpenAction {
+            return if (exists(path, ".idea", "modules.xml"))
+                ProjectOpenAction.EXISTING_IDEA_PROJECT.apply { println("Opening $path in iml mode.") }
+            else
+                ProjectOpenAction.GRADLE_PROJECT.apply { println("Opening $path in Gradle mode.") }
+        }
+    }
+}

idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/Fixture.kt

@@ -13,9 +13,9 @@ import com.intellij.codeInsight.daemon.impl.HighlightInfo
 import com.intellij.codeInsight.lookup.LookupElement
 import com.intellij.lang.ExternalAnnotatorsFilter
 import com.intellij.lang.LanguageAnnotators
-import com.intellij.lang.StdLanguages
 import com.intellij.lang.injection.InjectedLanguageManager
 import com.intellij.lang.java.JavaLanguage
+import com.intellij.lang.xml.XMLLanguage
 import com.intellij.openapi.application.runWriteAction
 import com.intellij.openapi.editor.Document
 import com.intellij.openapi.editor.Editor
@@ -43,18 +43,30 @@ import junit.framework.TestCase.*
 import org.jetbrains.kotlin.idea.KotlinLanguage
 import org.jetbrains.kotlin.idea.core.script.ScriptConfigurationManager
 import org.jetbrains.kotlin.idea.core.script.ScriptDefinitionsManager
-import org.jetbrains.kotlin.idea.core.script.settings.KotlinScriptingSettings
 import org.jetbrains.kotlin.idea.core.util.toPsiFile
 import org.jetbrains.kotlin.parsing.KotlinParserDefinition
-import java.io.Closeable
 
-
-class Fixture(val project: Project, val editor: Editor, val psiFile: PsiFile, val vFile: VirtualFile = psiFile.virtualFile) : Closeable {
+class Fixture(
+    val fileName: String,
+    val project: Project,
+    val editor: Editor,
+    val psiFile: PsiFile,
+    val vFile: VirtualFile = psiFile.virtualFile
+) : AutoCloseable {
     private var delegate = EditorTestFixture(project, editor, vFile)
 
+    private var savedText: String? = null
+
     val document: Document
         get() = editor.document
 
+    val text: String
+        get() = document.text
+
+    init {
+        storeText()
+    }
+
     fun doHighlighting(): List<HighlightInfo> = delegate.doHighlighting() ?: emptyList()
 
     fun type(s: String) {
@@ -73,7 +85,21 @@ class Fixture(val project: Project, val editor: Editor, val psiFile: PsiFile, va
     fun complete(type: CompletionType = CompletionType.BASIC, invocationCount: Int = 1): Array<LookupElement> =
         delegate.complete(type, invocationCount) ?: emptyArray()
 
-    fun revertChanges(revertChangesAtTheEnd: Boolean, text: String) {
+    fun storeText() {
+        savedText = text
+    }
+
+    fun restoreText() {
+        savedText?.let {
+            try {
+                applyText(it)
+            } finally {
+                cleanupCaches(project)
+            }
+        }
+    }
+
+    fun revertChanges(revertChangesAtTheEnd: Boolean = true, text: String) {
         try {
             if (revertChangesAtTheEnd) {
                 // TODO: [VD] revert ?
@@ -108,7 +134,10 @@ class Fixture(val project: Project, val editor: Editor, val psiFile: PsiFile, va
         dispatchAllInvocationEvents()
     }
 
-    override fun close() = close(project, vFile)
+    override fun close() {
+        savedText = null
+        close(project, vFile)
+    }
 
     companion object {
         // quite simple impl - good so far
@@ -128,7 +157,7 @@ class Fixture(val project: Project, val editor: Editor, val psiFile: PsiFile, va
             InjectedLanguageManager.getInstance(project) // zillion of Dom Sem classes
             with(LanguageAnnotators.INSTANCE) {
                 allForLanguage(JavaLanguage.INSTANCE) // pile of annotator classes loads
-                allForLanguage(StdLanguages.XML)
+                allForLanguage(XMLLanguage.INSTANCE)
                 allForLanguage(KotlinLanguage.INSTANCE)
             }
             DaemonAnalyzerTestCase.assertTrue(
@@ -147,7 +176,7 @@ class Fixture(val project: Project, val editor: Editor, val psiFile: PsiFile, va
             dispatchAllInvocationEvents()
 
             assertTrue(scriptDefinitionsManager.isReady())
-            assertFalse(KotlinScriptingSettings.getInstance(project).isAutoReloadEnabled)
+            //assertFalse(KotlinScriptingSettings.getInstance(project).isAutoReloadEnabled)
         }
 
 
@@ -157,7 +186,7 @@ class Fixture(val project: Project, val editor: Editor, val psiFile: PsiFile, va
             val editorFactory = EditorFactory.getInstance()
             val editor = editorFactory.getEditors(fileInEditor.document, project)[0]
 
-            return Fixture(project, editor, file)
+            return Fixture(fileName, project, editor, file)
         }
 
         private fun baseName(name: String): String {
@@ -177,10 +206,10 @@ class Fixture(val project: Project, val editor: Editor, val psiFile: PsiFile, va
             val projectBaseName = baseName(project.name)
 
             val virtualFiles = FilenameIndex.getVirtualFilesByName(
-                    project,
-                    baseFileName, true,
-                    GlobalSearchScope.projectScope(project)
-                )
+                project,
+                baseFileName, true,
+                GlobalSearchScope.projectScope(project)
+            )
                 .filter { it.canonicalPath?.contains("/$projectBaseName/$name") ?: false }.toList()
 
             assertEquals(
@@ -230,7 +259,7 @@ class Fixture(val project: Project, val editor: Editor, val psiFile: PsiFile, va
         ) {
             val fileInEditor = openFileInEditor(project, fileName)
             val editor = EditorFactory.getInstance().getEditors(fileInEditor.document, project)[0]
-            val fixture = Fixture(project, editor, fileInEditor.psiFile)
+            val fixture = Fixture(fileName, project, editor, fileInEditor.psiFile)
 
             val initialText = editor.document.text
             try {

idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/GradleProcessOutputInterceptor.kt

@@ -12,6 +12,7 @@ import com.intellij.openapi.externalSystem.model.task.ExternalSystemTaskNotifica
 import com.intellij.openapi.externalSystem.model.task.ExternalSystemTaskNotificationListener.EP_NAME as EP
 import com.intellij.testFramework.ExtensionTestUtil.maskExtensions
 import org.jetbrains.kotlin.idea.framework.GRADLE_SYSTEM_ID
+import org.jetbrains.kotlin.idea.perf.util.gradleMessage
 import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull
 import java.lang.Exception
 import kotlin.test.assertNull
@@ -44,8 +45,10 @@ private class GradleProcessOutputInterceptorImpl : GradleProcessOutputIntercepto
     private val buffer = StringBuilder()
 
     override fun onTaskOutput(id: ExternalSystemTaskId, text: String, stdOut: Boolean) {
-        if (id.projectSystemId == GRADLE_SYSTEM_ID && text.isNotEmpty())
+        if (id.projectSystemId == GRADLE_SYSTEM_ID && text.isNotEmpty()) {
+            gradleMessage { text }
             buffer.append(text)
+        }
     }
 
     override fun reset() = buffer.setLength(0)

idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/ProjectOpenAction.kt

@@ -5,7 +5,6 @@
 
 package org.jetbrains.kotlin.idea.testFramework
 
-import com.intellij.codeInspection.ex.InspectionProfileImpl
 import com.intellij.ide.highlighter.ModuleFileType
 import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.application.runWriteAction
@@ -17,18 +16,16 @@ import com.intellij.openapi.project.Project
 import com.intellij.openapi.project.ex.ProjectManagerEx
 import com.intellij.openapi.projectRoots.Sdk
 import com.intellij.openapi.roots.ProjectRootManager
-import com.intellij.openapi.util.Disposer
 import com.intellij.openapi.vcs.changes.ChangeListManager
 import com.intellij.openapi.vcs.changes.ChangeListManagerImpl
 import com.intellij.openapi.vfs.LocalFileSystem
 import com.intellij.openapi.vfs.VirtualFileManager
-import com.intellij.profile.codeInspection.ProjectInspectionProfileManager
 import com.intellij.testFramework.PsiTestUtil
 import com.intellij.testFramework.UsefulTestCase.assertTrue
 import com.intellij.util.io.exists
 import org.jetbrains.kotlin.idea.configuration.getModulesWithKotlinFiles
 import org.jetbrains.kotlin.idea.perf.Stats.Companion.runAndMeasure
-import org.jetbrains.kotlin.idea.perf.enableAllInspectionsCompat
+import org.jetbrains.kotlin.idea.perf.util.logMessage
 import org.jetbrains.kotlin.idea.project.getAndCacheLanguageLevelByDependencies
 import org.jetbrains.kotlin.idea.test.ConfigLibraryUtil
 import java.io.File
@@ -89,11 +86,6 @@ enum class ProjectOpenAction {
 
             refreshGradleProject(projectPath, project)
 
-            assertTrue(
-                ModuleManager.getInstance(project).modules.isNotEmpty(),
-                "Gradle project $projectName at $projectPath has to have at least one module"
-            )
-
             return project
         }
 
@@ -144,22 +136,6 @@ enum class ProjectOpenAction {
         //runWriteAction { project.save() }
     }
 
-    fun initDefaultProfile(project: Project) {
-        val projectInspectionProfileManager = ProjectInspectionProfileManager.getInstance(project)
-        projectInspectionProfileManager.forceLoadSchemes()
-
-        val projectProfile = projectInspectionProfileManager.projectProfile ?: error("project has to have non null profile name")
-        val profile = projectInspectionProfileManager.getProfile(projectProfile)
-        InspectionProfileImpl.INIT_INSPECTIONS = true
-        profile.initInspectionTools(project)
-        val enabledTools = profile.getAllEnabledInspectionTools(project)
-        assertTrue(
-            "project ${project.name} has to have at least one enabled inspection in profile",
-            enabledTools.isNotEmpty()
-        )
-        InspectionProfileImpl.INIT_INSPECTIONS = false
-    }
-
     companion object {
         fun openProject(openProject: OpenProject): Project {
             val project = openProject.projectOpenAction.openProject(

idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/compat.kt

@@ -12,4 +12,7 @@ import com.intellij.openapi.project.ex.ProjectManagerEx
 fun ProjectManagerEx.forceCloseProjectEx(project: Project, dispose: Boolean): Boolean {
     if (!dispose) error("dispose should be true")
     return this.forceCloseProject(project, true)
-}
+}
+
+// BUNCH: 193
+typealias TestApplicationManager = com.intellij.idea.IdeaTestApplication

idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/compat.kt.201

@@ -12,4 +12,7 @@ import com.intellij.openapi.project.ex.ProjectManagerEx
 fun ProjectManagerEx.forceCloseProjectEx(project: Project, dispose: Boolean): Boolean {
     if (!dispose) error("dispose should be true")
     return this.forceCloseProject(project)
-}
+}
+
+// BUNCH: 193
+typealias TestApplicationManager = com.intellij.testFramework.TestApplicationManager

idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/gradleRoutines.kt

@@ -15,6 +15,7 @@ import com.intellij.openapi.project.Project
 import com.intellij.openapi.roots.ProjectRootManager
 import org.jetbrains.plugins.gradle.service.project.open.setupGradleSettings
 import org.jetbrains.plugins.gradle.settings.GradleProjectSettings
+import org.jetbrains.plugins.gradle.settings.GradleSettings
 import org.jetbrains.plugins.gradle.util.GradleConstants
 import org.jetbrains.plugins.gradle.util.GradleLog
 import java.io.File
@@ -26,6 +27,8 @@ fun refreshGradleProject(projectPath: String, project: Project) {
     dispatchAllInvocationEvents()
 }
 
+const val GRADLE_JDK_NAME = "Gradle JDK"
+
 /**
  * inspired by org.jetbrains.plugins.gradle.service.project.open.importProject(projectDirectory, project)
  */
@@ -34,7 +37,16 @@ private fun _importProject(projectPath: String, project: Project) {
     val projectSdk = ProjectRootManager.getInstance(project).projectSdk
     assertNotNull(projectSdk, "project SDK not found for ${project.name} at $projectPath")
     val gradleProjectSettings = GradleProjectSettings()
+
+    GradleSettings.getInstance(project).gradleVmOptions =
+        "-Xmx2048m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${System.getProperty("user.dir")}"
+
     setupGradleSettings(gradleProjectSettings, projectPath, project, projectSdk)
+    gradleProjectSettings.gradleJvm = GRADLE_JDK_NAME
+
+    GradleSettings.getInstance(project).getLinkedProjectSettings(projectPath)?.let { linkedProjectSettings ->
+        linkedProjectSettings.gradleJvm = GRADLE_JDK_NAME
+    }
 
     _attachGradleProjectAndRefresh(gradleProjectSettings, project)
 }

idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/gradleRoutines.kt.201

@@ -15,6 +15,7 @@ import com.intellij.openapi.project.Project
 import com.intellij.openapi.roots.ProjectRootManager
 import org.jetbrains.plugins.gradle.service.project.open.setupGradleSettings
 import org.jetbrains.plugins.gradle.settings.GradleProjectSettings
+import org.jetbrains.plugins.gradle.settings.GradleSettings
 import org.jetbrains.plugins.gradle.util.GradleConstants
 import org.jetbrains.plugins.gradle.util.GradleLog
 import java.io.File
@@ -26,6 +27,8 @@ fun refreshGradleProject(projectPath: String, project: Project) {
     dispatchAllInvocationEvents()
 }
 
+const val GRADLE_JDK_NAME = "Gradle JDK"
+
 /**
  * inspired by org.jetbrains.plugins.gradle.service.project.open.importProject(projectDirectory, project)
  */
@@ -34,7 +37,16 @@ private fun _importProject(projectPath: String, project: Project) {
     val projectSdk = ProjectRootManager.getInstance(project).projectSdk
     assertNotNull(projectSdk, "project SDK not found for ${project.name} at $projectPath")
     val gradleProjectSettings = GradleProjectSettings()
+
+    GradleSettings.getInstance(project).gradleVmOptions =
+        "-Xmx2048m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${System.getProperty("user.dir")}"
+
     setupGradleSettings(gradleProjectSettings, projectPath, project, projectSdk)
+    gradleProjectSettings.gradleJvm = GRADLE_JDK_NAME
+
+    GradleSettings.getInstance(project).getLinkedProjectSettings(projectPath)?.let { linkedProjectSettings ->
+        linkedProjectSettings.gradleJvm = GRADLE_JDK_NAME
+    }
 
     _attachGradleProjectAndRefresh(gradleProjectSettings, project)
 }

idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/projectRoutines.kt

@@ -5,6 +5,9 @@
 
 package org.jetbrains.kotlin.idea.testFramework
 
+import com.intellij.codeInsight.daemon.DaemonCodeAnalyzer
+import com.intellij.codeInsight.daemon.DaemonCodeAnalyzerSettings
+import com.intellij.codeInsight.daemon.impl.DaemonCodeAnalyzerImpl
 import com.intellij.ide.startup.impl.StartupManagerImpl
 import com.intellij.lang.LanguageAnnotators
 import com.intellij.lang.LanguageExtensionPoint
@@ -22,9 +25,8 @@ import com.intellij.testFramework.ExtensionTestUtil
 import com.intellij.testFramework.runInEdtAndWait
 import com.intellij.util.ui.UIUtil
 import org.jetbrains.kotlin.idea.parameterInfo.HintType
+import org.jetbrains.kotlin.idea.perf.util.logMessage
 import org.jetbrains.kotlin.idea.test.runPostStartupActivitiesOnce
-import java.io.PrintWriter
-import java.io.StringWriter
 import java.nio.file.Paths
 
 fun commitAllDocuments() {
@@ -70,10 +72,23 @@ fun dispatchAllInvocationEvents() {
 fun loadProjectWithName(path: String, name: String): Project? =
     ProjectManagerEx.getInstanceEx().loadProject(Paths.get(path), name)
 
-fun closeProject(project: Project) {
+fun TestApplicationManager.closeProject(project: Project) {
+    val name = project.name
+    val startupManagerImpl = StartupManager.getInstance(project) as StartupManagerImpl
+    val daemonCodeAnalyzerSettings = DaemonCodeAnalyzerSettings.getInstance()
+    val daemonCodeAnalyzerImpl = DaemonCodeAnalyzer.getInstance(project) as DaemonCodeAnalyzerImpl
+
+    setDataProvider(null)
+    daemonCodeAnalyzerSettings.isImportHintEnabled = true // return default value to avoid unnecessary save
+    startupManagerImpl.checkCleared()
+    daemonCodeAnalyzerImpl.cleanupAfterTest()
+
+    logMessage { "project '$name' is about to be closed" }
     dispatchAllInvocationEvents()
     val projectManagerEx = ProjectManagerEx.getInstanceEx()
     projectManagerEx.forceCloseProjectEx(project, true)
+
+    logMessage { "project '$name' successfully closed" }
 }
 
 fun runStartupActivities(project: Project) {
@@ -104,16 +119,4 @@ fun replaceWithCustomHighlighter(parentDisposable: Disposable, fromImplementatio
     if (filteredExtensions.size < extensions.size) {
         ExtensionTestUtil.maskExtensions(pointName, filteredExtensions + listOf(point), parentDisposable)
     }
-}
-
-fun logMessage(message: () -> String) {
-    println("-- ${message()}")
-}
-
-fun logMessage(t: Throwable, message: () -> String) {
-    val writer = StringWriter()
-    PrintWriter(writer).use {
-        t.printStackTrace(it)
-    }
-    println("-- ${message()}:\n$writer")
 }

idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/projectRoutines.kt.191

@@ -25,6 +25,7 @@ import com.intellij.testFramework.PlatformTestUtil
 import com.intellij.testFramework.runInEdtAndWait
 import com.intellij.util.ui.UIUtil
 import org.jetbrains.kotlin.idea.parameterInfo.HintType
+import org.jetbrains.kotlin.idea.perf.TestApplicationManager
 import org.jetbrains.kotlin.idea.test.runPostStartupActivitiesOnce
 import java.util.concurrent.TimeUnit
 import java.util.concurrent.TimeoutException
@@ -74,7 +75,7 @@ fun dispatchAllInvocationEvents() {
 fun loadProjectWithName(path: String, name: String): Project? =
     ProjectManagerEx.getInstanceEx().loadProject(name, path)
 
-fun closeProject(project: Project) {
+fun TestApplicationManager.closeProject(project: Project) {
     dispatchAllInvocationEvents()
     val projectManagerEx = ProjectManagerEx.getInstanceEx()
     projectManagerEx.forceCloseProjectEx(project, true)
@@ -133,15 +134,3 @@ fun replaceWithCustomHighlighter(parentDisposable: Disposable, fromImplementatio
         PlatformTestUtil.maskExtensions(pointName, filteredExtensions + listOf(point), parentDisposable)
     }
 }
-
-fun logMessage(message: () -> String) {
-    println("-- ${message()}")
-}
-
-fun logMessage(t: Throwable, message: () -> String) {
-    val writer = StringWriter()
-    PrintWriter(writer).use {
-        t.printStackTrace(it)
-    }
-    println("-- ${message()}:\n$writer")
-}

idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/projectRoutines.kt.192

@@ -23,6 +23,7 @@ import com.intellij.testFramework.runInEdtAndWait
 import com.intellij.util.ui.UIUtil
 import org.jetbrains.kotlin.idea.parameterInfo.HintType
 import org.jetbrains.kotlin.idea.test.runPostStartupActivitiesOnce
+import org.jetbrains.kotlin.idea.perf.TestApplicationManager
 import java.io.PrintWriter
 import java.io.StringWriter
 
@@ -69,7 +70,7 @@ fun dispatchAllInvocationEvents() {
 fun loadProjectWithName(path: String, name: String): Project? =
     ProjectManagerEx.getInstanceEx().loadProject(name, path)
 
-fun closeProject(project: Project) {
+fun TestApplicationManager.closeProject(project: Project) {
     dispatchAllInvocationEvents()
     val projectManagerEx = ProjectManagerEx.getInstanceEx()
     projectManagerEx.forceCloseProjectEx(project, true)
@@ -104,15 +105,3 @@ fun replaceWithCustomHighlighter(parentDisposable: Disposable, fromImplementatio
         PlatformTestUtil.maskExtensions(pointName, filteredExtensions + listOf(point), parentDisposable)
     }
 }
-
-fun logMessage(message: () -> String) {
-    println("-- ${message()}")
-}
-
-fun logMessage(t: Throwable, message: () -> String) {
-    val writer = StringWriter()
-    PrintWriter(writer).use {
-        t.printStackTrace(it)
-    }
-    println("-- ${message()}:\n$writer")
-}

idea/testData/perfTest/helloKotlin/src/HelloMain.kt

@@ -1,4 +0,0 @@
-
-fun main(args: Array<String>) {
-    println("Hello World!")
-}

idea/testData/perfTest/helloKotlin/src/HelloMain2.kt

@@ -1,4 +0,0 @@
-
-fun main(args: Array<String>) {
-    println("Hello World!")
-}