Modularization of performance tests

build.gradle.kts

@@ -660,7 +660,7 @@ tasks {
     register("idea-plugin-performance-tests") {
         dependsOn("dist")
         dependsOn(
-            ":idea:performanceTest"
+            ":idea:performanceTests:performanceTest"
         )
     }
 

idea/build.gradle.kts

@@ -28,25 +28,6 @@ sourceSets {
             "idea-live-templates/tests"
         )
     }
-
-    "performanceTest" {
-        java.srcDirs("performanceTests")
-    }
-}
-
-val performanceTestCompile by configurations
-performanceTestCompile.apply {
-    extendsFrom(configurations["testCompile"])
-}
-
-val performanceTestCompileOnly by configurations
-performanceTestCompileOnly.apply {
-    extendsFrom(configurations["testCompileOnly"])
-}
-
-val performanceTestRuntime by configurations
-performanceTestRuntime.apply {
-    extendsFrom(configurations["testRuntime"])
 }
 
 dependencies {
@@ -189,13 +170,6 @@ dependencies {
         testRuntime(intellijPluginDep("google-cloud-tools-core-as"))
         testRuntime(intellijPluginDep("google-login-as"))
     }
-
-    performanceTestCompile(sourceSets["test"].output)
-    performanceTestCompile(sourceSets["main"].output)
-    performanceTestCompile(project(":nj2k"))
-    performanceTestCompile(project(":idea:idea-gradle-tooling-api"))
-    performanceTestCompile(intellijPluginDep("gradle"))
-    performanceTestRuntime(sourceSets["performanceTest"].output)
 }
 
 tasks.named<Copy>("processResources") {
@@ -209,35 +183,6 @@ projectTest(parallel = true) {
     workingDir = rootDir
 }
 
-projectTest(taskName = "performanceTest") {
-    dependsOn(":dist")
-    dependsOn(performanceTestRuntime)
-
-    testClassesDirs = sourceSets["performanceTest"].output.classesDirs
-    classpath = performanceTestRuntime + files("${System.getenv("ASYNC_PROFILER_HOME")}/build/async-profiler.jar")
-    workingDir = rootDir
-
-    jvmArgs?.removeAll { it.startsWith("-Xmx") }
-
-    maxHeapSize = "3g"
-    jvmArgs("-Didea.debug.mode=true")
-    jvmArgs("-XX:SoftRefLRUPolicyMSPerMB=50")
-    jvmArgs(
-        "-XX:ReservedCodeCacheSize=240m",
-        "-XX:+UseCompressedOops",
-        "-XX:+UseConcMarkSweepGC"
-    )
-
-    doFirst {
-        systemProperty("idea.home.path", intellijRootDir().canonicalPath)
-        project.findProperty("cacheRedirectorEnabled")?.let {
-            systemProperty("kotlin.test.gradle.import.arguments", "-PcacheRedirectorEnabled=$it")
-        }
-    }
-}
-
-testsJar {
-    from(sourceSets["performanceTest"].output)
-}
-
 configureFormInstrumentation()
+
+testsJar()

idea/performanceTests/build.gradle.kts

@@ -0,0 +1,78 @@
+
+description = "Kotlin IDE Performance Tests"
+
+plugins {
+    kotlin("jvm")
+    id("jps-compatible")
+}
+
+dependencies {
+    testRuntimeOnly(intellijDep())
+    testRuntimeOnly(intellijRuntimeAnnotations())
+    testRuntimeOnly(project(":kotlin-reflect"))
+    testRuntimeOnly(project(":plugins:android-extensions-ide"))
+    testRuntimeOnly(project(":plugins:kapt3-idea"))
+    testRuntimeOnly(project(":sam-with-receiver-ide-plugin"))
+    testRuntimeOnly(project(":noarg-ide-plugin"))
+    testRuntimeOnly(project(":allopen-ide-plugin"))
+    testRuntimeOnly(project(":kotlin-scripting-idea"))
+    testRuntimeOnly(project(":kotlinx-serialization-ide-plugin"))
+    testRuntimeOnly(project(":nj2k:nj2k-services"))
+    testRuntimeOnly(project(":kotlin-reflect"))
+    testRuntimeOnly(project(":idea:kotlin-gradle-tooling"))
+    testRuntimeOnly(project(":idea:idea-gradle-tooling-api"))
+
+    testImplementation(intellijPluginDep("gradle"))
+    testImplementation(project(":compiler:backend"))
+    testImplementation(project(":idea:idea-jvm"))
+    testImplementation(project(":kotlin-test:kotlin-test-junit"))
+    testImplementation(projectTests(":compiler:tests-common"))
+    testImplementation(projectTests(":idea:idea-test-framework")) { isTransitive = false }
+    testImplementation(projectTests(":idea"))
+    testImplementation(commonDep("junit:junit"))
+
+    Platform[192].orHigher {
+        testCompileOnly(intellijPluginDep("java"))
+        testRuntimeOnly(intellijPluginDep("java"))
+    }
+
+    testCompileOnly(intellijDep())
+    testCompileOnly(project(":nj2k"))
+    testCompileOnly(project(":idea:idea-gradle-tooling-api"))
+    testCompileOnly(intellijDep()) { includeJars("slf4j-api-1.7.25") }
+}
+
+sourceSets {
+    "test" { projectDefault() }
+}
+
+runtimeJar()
+
+sourcesJar()
+
+javadocJar()
+
+apply(from = "$rootDir/gradle/kotlinPluginPublication.gradle.kts")
+
+projectTest(taskName = "performanceTest") {
+    classpath += files("${System.getenv("ASYNC_PROFILER_HOME")}/build/async-profiler.jar")
+    workingDir = rootDir
+
+    jvmArgs?.removeAll { it.startsWith("-Xmx") }
+
+    maxHeapSize = "3g"
+    jvmArgs("-Didea.debug.mode=true")
+    jvmArgs("-XX:SoftRefLRUPolicyMSPerMB=50")
+    jvmArgs(
+        "-XX:ReservedCodeCacheSize=240m",
+        "-XX:+UseCompressedOops",
+        "-XX:+UseConcMarkSweepGC"
+    )
+
+    doFirst {
+        systemProperty("idea.home.path", intellijRootDir().canonicalPath)
+        project.findProperty("cacheRedirectorEnabled")?.let {
+            systemProperty("kotlin.test.gradle.import.arguments", "-PcacheRedirectorEnabled=$it")
+        }
+    }
+}

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceProjectsTest.kt

@@ -55,10 +55,12 @@ import org.jetbrains.kotlin.idea.test.ConfigLibraryUtil
 import org.jetbrains.kotlin.idea.test.invalidateLibraryCache
 import org.jetbrains.kotlin.idea.testFramework.*
 import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.cleanupCaches
+import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.close
 import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.isAKotlinScriptFile
 import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.openFileInEditor
 import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.openFixture
 import org.jetbrains.kotlin.idea.util.getProjectJdkTableSafe
+import org.jetbrains.kotlin.psi.KtFile
 import java.io.File
 import java.nio.file.Paths
 
@@ -229,8 +231,8 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
 
                     // close all project but last - we're going to return and use it further
                     if (counter < warmUpIterations + iterations - 1) {
-                        closeProject(project)
                         myApplication.setDataProvider(null)
+                        closeProject(project)
                     }
                     counter++
                 }
@@ -306,61 +308,64 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
         note: String = ""
     ) {
         assertTrue("lookupElements has to be not empty", lookupElements.isNotEmpty())
-        performanceTest<Pair<String, Fixture>, Array<LookupElement>> {
+        openFixture(project, fileName).use { fixture ->
-            name("typeAndAutocomplete ${notePrefix(note)}$fileName")
+            val editor = fixture.editor
-            stats(stats)
-            warmUpIterations(8)
-            iterations(15)
-            setUp {
-                val fixture = openFixture(project, fileName)
-                val editor = fixture.editor
 
-                val initialText = editor.document.text
+            val initialText = editor.document.text
-                updateScriptDependenciesIfNeeded(fileName, fixture, project)
+            updateScriptDependenciesIfNeeded(fileName, fixture, project)
+            val scriptConfigurationManager = ScriptConfigurationManager.getInstance(fixture.project)
+            val configuration = scriptConfigurationManager.getConfiguration(fixture.psiFile as KtFile)
+            //scriptConfigurationManager.reloadScriptDefinitions()
+            //scriptConfigurationManager.forceReloadConfiguration(fixture.vFile, loaderForOutOfProjectScripts)
 
-                val tasksIdx = editor.document.text.indexOf(marker)
+            performanceTest<Pair<String, Fixture>, Array<LookupElement>> {
-                assertTrue("marker '$marker' not found in $fileName", tasksIdx > 0)
+                name("typeAndAutocomplete ${notePrefix(note)}$fileName")
-                if (typeAfterMarker) {
+                stats(stats)
-                    editor.caretModel.moveToOffset(tasksIdx + marker.length + 1)
+                warmUpIterations(8)
-                } else {
+                iterations(15)
-                    editor.caretModel.moveToOffset(tasksIdx - 1)
+                profileEnabled(true)
-                }
+                setUp {
+                    val tasksIdx = editor.document.text.indexOf(marker)
+                    assertTrue("marker '$marker' not found in $fileName", tasksIdx > 0)
+                    if (typeAfterMarker) {
+                        editor.caretModel.moveToOffset(tasksIdx + marker.length + 1)
+                    } else {
+                        editor.caretModel.moveToOffset(tasksIdx - 1)
+                    }
 
-                for (surroundItem in surroundItems) {
-                    EditorTestUtil.performTypingAction(editor, surroundItem)
-                }
-
-                editor.caretModel.moveToOffset(editor.caretModel.offset - if (typeAfterMarker) 1 else 2)
-
-                if (!typeAfterMarker) {
                     for (surroundItem in surroundItems) {
                         EditorTestUtil.performTypingAction(editor, surroundItem)
                     }
-                    editor.caretModel.moveToOffset(editor.caretModel.offset - 2)
+
+                    editor.caretModel.moveToOffset(editor.caretModel.offset - if (typeAfterMarker) 1 else 2)
+
+                    if (!typeAfterMarker) {
+                        for (surroundItem in surroundItems) {
+                            EditorTestUtil.performTypingAction(editor, surroundItem)
+                        }
+                        editor.caretModel.moveToOffset(editor.caretModel.offset - 2)
+                    }
+
+                    fixture.type(insertString)
+
+                    it.setUpValue = Pair(initialText, fixture)
                 }
-
+                test {
-                fixture.type(insertString)
+                    val fixture = it.setUpValue!!.second
-
+                    it.value = fixture.complete()
-                it.setUpValue = Pair(initialText, fixture)
+                }
-            }
+                tearDown {
-            test {
+                    val items = it.value?.map { e -> e.lookupString }?.toList() ?: emptyList()
-                val fixture = it.setUpValue!!.second
+                    try {
-                it.value = fixture.complete()
+                        for (lookupElement in lookupElements) {
-            }
+                            assertTrue("'$lookupElement' has to be present in items $items", items.contains(lookupElement))
-            tearDown {
+                        }
-                val items = it.value?.map { e -> e.lookupString }?.toList() ?: emptyList()
+                    } finally {
-                try {
+                        fixture.revertChanges(revertChangesAtTheEnd, initialText)
-                    for (lookupElement in lookupElements) {
+                        commitAllDocuments()
-                        assertTrue("'$lookupElement' has to be present in items", items.contains(lookupElement))
                     }
-                } finally {
-                    it.setUpValue?.let { pair ->
-                        pair.second.revertChanges(revertChangesAtTheEnd, pair.first)
-                    }
-                    commitAllDocuments()
                 }
             }
-            profileEnabled(true)
         }
     }
 
@@ -624,7 +629,10 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
                 it.value = it.setUpValue
             }
             tearDown {
-                it.setUpValue?.let { ef -> cleanupCaches(project, ef.psiFile.virtualFile) }
+                it.setUpValue?.let { ef ->
+                    cleanupCaches(project)
+                    close(project, ef.psiFile.virtualFile)
+                }
                 it.value?.let { v -> assertNotNull(v) }
             }
             profileEnabled(true)

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerformanceProjectsTest.kt

@@ -152,16 +152,29 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
                     perfOpenKotlinProjectFast(stat)
                 }
 
+//                runAndMeasure("type and autocomplete") {
+//                    perfTypeAndAutocomplete(
+//                        stat,
+//                        "build.gradle.kts",
+//                        "tasks {",
+//                        "default",
+//                        lookupElements = listOf("defaultJvmTarget"),
+//                        note = "tasks-create"
+//                    )
+//                }
+
                 runAndMeasure("type and autocomplete") {
                     perfTypeAndAutocomplete(
                         stat,
                         "build.gradle.kts",
                         "tasks {",
-                        "crea",
+                        "reg",
-                        lookupElements = listOf("create"),
+                        lookupElements = listOf("register"),
                         note = "tasks-create"
                     )
                 }
+
+
             }
         }
     }
@@ -319,7 +332,9 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
                     extraTimingsNs.add(mapOf(TEST_KEY to (timer.get() - v.first)))
 
                 }
-                cleanupCaches(project, fixture.vFile)
+                fixture.use {
+                    cleanupCaches(project)
+                }
             }
         }
     }

idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/Fixture.kt

@@ -46,9 +46,10 @@ import org.jetbrains.kotlin.idea.core.script.ScriptDefinitionsManager
 import org.jetbrains.kotlin.idea.core.script.settings.KotlinScriptingSettings
 import org.jetbrains.kotlin.idea.core.util.toPsiFile
 import org.jetbrains.kotlin.parsing.KotlinParserDefinition
+import java.io.Closeable
 
 
-class Fixture(val project: Project, val editor: Editor, val psiFile: PsiFile, val vFile: VirtualFile = psiFile.virtualFile) {
+class Fixture(val project: Project, val editor: Editor, val psiFile: PsiFile, val vFile: VirtualFile = psiFile.virtualFile) : Closeable {
     private var delegate = EditorTestFixture(project, editor, vFile)
 
     val document: Document
@@ -76,7 +77,7 @@ class Fixture(val project: Project, val editor: Editor, val psiFile: PsiFile, va
                 applyText(text)
             }
         } finally {
-            cleanupCaches(project, vFile)
+            cleanupCaches(project)
         }
     }
 
@@ -103,16 +104,21 @@ class Fixture(val project: Project, val editor: Editor, val psiFile: PsiFile, va
         dispatchAllInvocationEvents()
     }
 
+    override fun close() = close(project, vFile)
+
     companion object {
         // quite simple impl - good so far
         fun isAKotlinScriptFile(fileName: String) = fileName.endsWith(KotlinParserDefinition.STD_SCRIPT_EXT)
 
-        fun cleanupCaches(project: Project, vFile: VirtualFile) {
+        fun cleanupCaches(project: Project) {
             commitAllDocuments()
-            FileEditorManager.getInstance(project).closeFile(vFile)
             PsiManager.getInstance(project).dropPsiCaches()
         }
 
+        fun close(project: Project, file: VirtualFile) {
+            FileEditorManager.getInstance(project).closeFile(file)
+        }
+
         fun enableAnnotatorsAndLoadDefinitions(project: Project) {
             ReferenceProvidersRegistry.getInstance() // pre-load tons of classes
             InjectedLanguageManager.getInstance(project) // zillion of Dom Sem classes

settings.gradle

@@ -154,6 +154,7 @@ include ":kotlin-build-common",
         ":idea:idea-test-framework",
         ":idea:idea-native",
         ":idea:idea-gradle-native",
+        ":idea:performanceTests",
         ":idea",
         ":idea-runner",
         ":idea:jvm-debugger:eval4j",