Fixed and extended performance tests

idea/performanceTests/README.md

@@ -0,0 +1,42 @@
+# Goal
+
+The main goal of these performance tests is to collect statistics over
+routines (open project, highlighting, inspection, autocompletion etc) for
+further analysis like anomalies, degradations and reference for optimizations.
+
+## Resources
+- Clone another copy of kotlin project to `${KOTLIN_PROJECT}/../perfTestProject`
+- `${KOTLIN_PROJECT}/idea/testData/perfTest/helloKotlin` as _hello world_ project in Kotlin
+
+## Performance test types
+
+### AbstractKotlinProjectsPerformanceTest
+
+Subclasses of `AbstractKotlinProjectsPerformanceTest` provide statistics
+(execution time in **ms**) over some **particular** (semi-synthetic) actions.
+
+The output is provided to console in TeamCity format like
+
+```
+##teamcity[testSuiteStarted name='Kotlin']
+##teamcity[testStarted name='Project opening perfTestProject' captureStandardOutput='true']
+##teamcity[testMetadata key='Project opening perfTestProject' type='number' value='63']
+##teamcity[testFinished name='Project opening perfTestProject' duration='63']
+##teamcity[testSuiteFinished name='Kotlin']
+```
+
+as well in `build/stats.csv`
+
+ File          | ProcessID                              | Time
+--- | --- | ---
+  HelloMain.kt | Highlighting file warm-up HelloMain.kt | 114
+  KtFile.kt    | Highlighting file KtFile.kt            | 1255
+
+### WholeProjectPerformanceTest
+
+**TBD**: drop `@Ignore` from some test cases
+
+Subclasses of `WholeProjectPerformanceTest` collect performance metrics to
+figure out some significant anomalies (e.g. inspection takes ages or
+even **fails**) over huge number of files (like `kotlin` project itself)
+on various **real** use cases.

idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractKotlinProjectsPerformanceTest.kt

@@ -0,0 +1,279 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.idea.perf
+
+import com.intellij.codeInsight.completion.CompletionType
+import com.intellij.codeInsight.daemon.impl.HighlightInfo
+import com.intellij.codeInsight.lookup.LookupElement
+import com.intellij.codeInspection.ex.InspectionProfileImpl
+import com.intellij.ide.highlighter.ModuleFileType
+import com.intellij.idea.IdeaTestApplication
+import com.intellij.openapi.Disposable
+import com.intellij.openapi.application.ApplicationManager
+import com.intellij.openapi.command.CommandProcessor
+import com.intellij.openapi.editor.Document
+import com.intellij.openapi.editor.EditorFactory
+import com.intellij.openapi.fileEditor.FileDocumentManager
+import com.intellij.openapi.fileEditor.FileEditorManager
+import com.intellij.openapi.module.Module
+import com.intellij.openapi.module.ModuleManager
+import com.intellij.openapi.module.ModuleTypeId
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.project.ProjectManager
+import com.intellij.openapi.project.ex.ProjectManagerEx
+import com.intellij.openapi.projectRoots.JavaSdk
+import com.intellij.openapi.projectRoots.ProjectJdkTable
+import com.intellij.openapi.projectRoots.Sdk
+import com.intellij.openapi.projectRoots.impl.JavaAwareProjectJdkTableImpl
+import com.intellij.openapi.roots.ProjectRootManager
+import com.intellij.openapi.util.Computable
+import com.intellij.openapi.vcs.changes.ChangeListManager
+import com.intellij.openapi.vcs.changes.ChangeListManagerImpl
+import com.intellij.openapi.vfs.LocalFileSystem
+import com.intellij.openapi.vfs.VirtualFileManager
+import com.intellij.psi.PsiDocumentManager
+import com.intellij.psi.PsiFile
+import com.intellij.testFramework.LightPlatformTestCase
+import com.intellij.testFramework.PsiTestUtil
+import com.intellij.testFramework.RunAll
+import com.intellij.testFramework.UsefulTestCase
+import com.intellij.testFramework.fixtures.IdeaTestExecutionPolicy
+import com.intellij.testFramework.fixtures.IdeaTestFixtureFactory
+import com.intellij.testFramework.fixtures.TempDirTestFixture
+import com.intellij.testFramework.fixtures.impl.CodeInsightTestFixtureImpl
+import com.intellij.testFramework.fixtures.impl.LightTempDirTestFixtureImpl
+import com.intellij.util.ThrowableRunnable
+import com.intellij.util.io.exists
+import org.jetbrains.kotlin.idea.KotlinFileType
+import org.jetbrains.kotlin.idea.core.util.toPsiFile
+import org.jetbrains.kotlin.idea.framework.KotlinSdkType
+import org.jetbrains.kotlin.idea.test.ConfigLibraryUtil
+import org.jetbrains.kotlin.idea.test.KotlinWithJdkAndRuntimeLightProjectDescriptor
+import java.io.File
+import java.nio.file.Paths
+
+abstract class AbstractKotlinProjectsPerformanceTest : UsefulTestCase() {
+
+    // myProject is not required for all potential perf test cases
+    private var myProject: Project? = null
+    private lateinit var jdk18: Sdk
+    private lateinit var myApplication: IdeaTestApplication
+
+    private val stats: Stats = Stats("-perf")
+
+    override fun setUp() {
+        super.setUp()
+
+        myApplication = IdeaTestApplication.getInstance()
+        ApplicationManager.getApplication().runWriteAction {
+            val jdkTableImpl = JavaAwareProjectJdkTableImpl.getInstanceEx()
+            val homePath = if (jdkTableImpl.internalJdk.homeDirectory!!.name == "jre") {
+                jdkTableImpl.internalJdk.homeDirectory!!.parent.path
+            } else {
+                jdkTableImpl.internalJdk.homePath!!
+            }
+
+            val javaSdk = JavaSdk.getInstance()
+            jdk18 = javaSdk.createJdk("1.8", homePath)
+            val internal = javaSdk.createJdk("IDEA jdk", homePath)
+
+            val jdkTable = ProjectJdkTable.getInstance()
+            jdkTable.addJdk(jdk18, testRootDisposable)
+            jdkTable.addJdk(internal, testRootDisposable)
+            KotlinSdkType.setUpIfNeeded()
+        }
+        InspectionProfileImpl.INIT_INSPECTIONS = true
+
+        // warm up: open simple small project
+        val project = innerPerfOpenProject("helloKotlin", "warm-up ")
+        val perfHighlightFile = perfHighlightFile(project, "src/HelloMain.kt", "warm-up ")
+        assertTrue("kotlin project has been not imported properly", perfHighlightFile.isNotEmpty())
+
+        ProjectManagerEx.getInstanceEx().forceCloseProject(project, true)
+    }
+
+    override fun tearDown() {
+        var runAll = RunAll()
+
+        if (myProject != null) {
+            runAll = runAll
+                .append(ThrowableRunnable { LightPlatformTestCase.doTearDown(myProject!!, myApplication) })
+        }
+
+        runAll.append(ThrowableRunnable { super.tearDown() })
+            .run()
+    }
+
+    protected fun getTempDirFixture(): TempDirTestFixture {
+        val policy = IdeaTestExecutionPolicy.current()
+        return if (policy != null)
+            policy.createTempDirTestFixture()
+        else
+            LightTempDirTestFixtureImpl(true)
+    }
+
+    protected fun perfChangeDocument(fileName: String, note: String = "", block: (document: Document) -> Unit) =
+        perfChangeDocument(myProject!!, fileName, note, block)
+
+    private fun perfChangeDocument(
+        project: Project,
+        fileName: String,
+        nameOfChange: String,
+        block: (document: Document) -> Unit
+    ) {
+        val openFileInEditor = openFileInEditor(project, fileName)
+        val document = openFileInEditor.document
+        val manager = PsiDocumentManager.getInstance(project)
+        CommandProcessor.getInstance().executeCommand(project, {
+            ApplicationManager.getApplication().runWriteAction {
+                tcSimplePerfTest(fileName, "Changing doc $nameOfChange", stats) {
+                    block(document)
+
+                    manager.commitDocument(document)
+                }
+            }
+        }, "change doc $fileName $nameOfChange", "")
+    }
+
+    protected fun perfOpenProject(name: String, path: String = "idea/testData/perfTest") {
+        myProject = innerPerfOpenProject(name, path = path, note = "")
+    }
+
+    private fun innerPerfOpenProject(
+        name: String,
+        note: String,
+        path: String = "idea/testData/perfTest"
+    ): Project {
+        lateinit var project: Project
+        val projectPath = "$path/$name"
+
+        tcSimplePerfTest("", "Project ${note}opening $name", stats) {
+            project = ProjectManager.getInstance().loadAndOpenProject(projectPath)!!
+            if (!Paths.get(projectPath, ".idea").exists()) {
+                initKotlinProject(project, projectPath, name)
+            }
+
+            ProjectManagerEx.getInstanceEx().openTestProject(project)
+
+            disposeOnTearDown(Disposable { ProjectManagerEx.getInstanceEx().forceCloseProject(project, true) })
+        }
+
+        val changeListManagerImpl = ChangeListManager.getInstance(project) as ChangeListManagerImpl
+        changeListManagerImpl.waitUntilRefreshed()
+
+        return project
+    }
+
+    private fun initKotlinProject(
+        project: Project,
+        projectPath: String,
+        name: String
+    ) {
+        val modulePath = "$projectPath/$name${ModuleFileType.DOT_DEFAULT_EXTENSION}"
+        val projectFile = LocalFileSystem.getInstance().refreshAndFindFileByIoFile(File(projectPath))!!
+        val srcFile = projectFile.findChild("src")!!
+        val module = ApplicationManager.getApplication().runWriteAction(Computable<Module> {
+            val projectRootManager = ProjectRootManager.getInstance(project)
+            with(projectRootManager) {
+                projectSdk = jdk18
+            }
+            val moduleManager = ModuleManager.getInstance(project)
+            val module = moduleManager.newModule(modulePath, ModuleTypeId.JAVA_MODULE)
+            PsiTestUtil.addSourceRoot(module, srcFile)
+            module!!
+        })
+        ConfigLibraryUtil.configureKotlinRuntimeAndSdk(module, jdk18)
+    }
+
+    protected fun perfHighlightFile(name: String): List<HighlightInfo> =
+        perfHighlightFile(myProject!!, name)
+
+
+    private fun perfHighlightFile(
+        project: Project,
+        name: String,
+        note: String = ""
+    ): List<HighlightInfo> {
+        val fileInEditor = openFileInEditor(project, name)
+        val file = fileInEditor.psiFile
+
+        var highlightFile: List<HighlightInfo> = emptyList()
+        tcSimplePerfTest(file.name, "Highlighting file $note${file.name}", stats) {
+            highlightFile = highlightFile(file)
+        }
+        return highlightFile
+    }
+
+    inline fun attempts(block: (v: Int) -> Unit) {
+        attempts(3, block)
+    }
+
+    inline fun attempts(count: Int, block: (v: Int) -> Unit) {
+        for (attempt in 0..count) {
+            block(attempt)
+        }
+    }
+
+    fun perfAutoCompletion(
+        name: String,
+        before: String,
+        suggestions: Array<String>,
+        type: String,
+        after: String
+    ) {
+        val factory = IdeaTestFixtureFactory.getFixtureFactory()
+        val fixtureBuilder = factory.createLightFixtureBuilder(KotlinWithJdkAndRuntimeLightProjectDescriptor.INSTANCE)
+        val tempDirFixture = getTempDirFixture()
+        val fixture = factory.createCodeInsightFixture(fixtureBuilder.fixture, tempDirFixture)
+
+        with(fixture) {
+            setUp()
+            configureByText(KotlinFileType.INSTANCE, before)
+        }
+
+        var complete: Array<LookupElement>? = null
+        tcSimplePerfTest("", "Auto completion $name", stats) {
+            with(fixture) {
+                complete = complete(CompletionType.BASIC)
+            }
+        }
+
+        val actualSuggestions = complete?.map { it.lookupString }?.toList() ?: emptyList()
+        assertTrue(actualSuggestions.containsAll(suggestions.toList()))
+
+        with(fixture) {
+            type(type)
+            checkResult(after)
+            tearDown()
+        }
+    }
+
+    private fun highlightFile(psiFile: PsiFile): List<HighlightInfo> {
+        val document = FileDocumentManager.getInstance().getDocument(psiFile.virtualFile)!!
+        val editor = EditorFactory.getInstance().getEditors(document).first()
+        return CodeInsightTestFixtureImpl.instantiateAndRun(psiFile, editor, intArrayOf(), false)
+    }
+
+    data class EditorFile(val psiFile: PsiFile, val document: Document)
+
+    private fun openFileInEditor(project: Project, name: String): EditorFile {
+        val psiFile = projectFileByName(project, name)
+        val vFile = psiFile.virtualFile
+        val fileEditorManager = FileEditorManager.getInstance(project)
+        fileEditorManager.openFile(vFile, true)
+        val document = FileDocumentManager.getInstance().getDocument(vFile)!!
+        assertNotNull(EditorFactory.getInstance().getEditors(document))
+        disposeOnTearDown(Disposable { fileEditorManager.closeFile(vFile) })
+        return EditorFile(psiFile = psiFile, document = document)
+    }
+
+    private fun projectFileByName(project: Project, name: String): PsiFile {
+        val fileManager = VirtualFileManager.getInstance()
+        val url = "file://${File("${project.basePath}/$name").absolutePath}"
+        val virtualFile = fileManager.refreshAndFindFileByUrl(url)
+        return virtualFile!!.toPsiFile(project)!!
+    }
+}

idea/performanceTests/org/jetbrains/kotlin/idea/perf/FirTotalKotlinResolveInIdeTest.kt

@@ -34,7 +34,9 @@ import org.jetbrains.kotlin.idea.caches.project.*
 import org.jetbrains.kotlin.idea.fir.IdeFirDependenciesSymbolProvider
 import org.jetbrains.kotlin.psi.KtFile
 import java.io.File
+import kotlin.test.Ignore
 
+@Ignore(value = "[Simon Ogorodnik] turned off as it is suppose to be run manually")
 class FirTotalKotlinResolveInIdeTest : ModuleTestCase() {
     private val projectRootFile = File(".")
 

idea/performanceTests/org/jetbrains/kotlin/idea/perf/KotlinProjectsPerformanceTest.kt

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.idea.perf
+
+class KotlinProjectsPerformanceTest : AbstractKotlinProjectsPerformanceTest() {
+
+    fun testHelloWorldProject() {
+        tcSuite("HelloWorld") {
+            perfOpenProject("helloKotlin")
+
+            // highlight
+            perfHighlightFile("src/HelloMain.kt")
+            perfHighlightFile("src/HelloMain2.kt")
+
+            // change document
+            perfChangeDocument("src/HelloMain2.kt", "type a single char") { doc ->
+                val text = doc.text
+                val offset = text.indexOf("println")
+
+                doc.insertString(offset, "p\n")
+            }
+
+            perfChangeDocument("src/HelloMain.kt", "type val expression") { doc ->
+                val text = doc.text
+                val offset = text.indexOf("println")
+
+                doc.insertString(offset, "val s =\n")
+            }
+        }
+    }
+
+    fun testAutoCompletion() {
+        tcSuite("AutoCompletion") {
+
+            attempts {
+                perfAutoCompletion(
+                    "inside of method: println$it",
+                    """
+                    fun bar() {
+                        print<caret>
+                    }
+                    """,
+                    suggestions = arrayOf("println", "print"),
+                    type = "l\r",
+                    after = """
+                    fun bar() {
+                        println()
+                    }
+                    """
+                )
+            }
+
+            attempts {
+                perfAutoCompletion(
+                    "outside of method: arrayListOf$it",
+                    before = """
+                    val f: List<String> = array<caret>
+
+                    fun bar(){ }
+                    """,
+                    type = "\n",
+                    suggestions = arrayOf("arrayOf", "arrayOfNulls", "emptyArray"),
+                    after = """
+                    val f: List<String> = arrayListOf()
+
+                    fun bar(){ }
+                    """
+                )
+            }
+        }
+    }
+
+    fun testKotlinProject() {
+        tcSuite("Kotlin") {
+            perfOpenProject("perfTestProject", "..")
+
+            perfHighlightFile("compiler/psi/src/org/jetbrains/kotlin/psi/KtFile.kt")
+
+            perfHighlightFile("compiler/psi/src/org/jetbrains/kotlin/psi/KtElement.kt")
+        }
+    }
+
+}

idea/performanceTests/org/jetbrains/kotlin/idea/perf/Stats.kt

@@ -0,0 +1,88 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.idea.perf
+
+import org.jetbrains.kotlin.idea.perf.WholeProjectPerformanceTest.Companion.nsToMs
+import java.io.*
+import kotlin.system.measureNanoTime
+
+class Stats(name: String = "") : Closeable {
+    private val statsFile: File = File("build/stats$name.csv").absoluteFile
+    private val statsOutput: BufferedWriter
+
+    init {
+        statsOutput = statsFile.bufferedWriter()
+
+        statsOutput.appendln("File, ProcessID, Time")
+    }
+
+    fun append(file: String, id: String, nanoTime: Long) {
+        statsOutput.appendln(buildString {
+            append(file)
+            append(", ")
+            append(id)
+            append(", ")
+            append(nanoTime.nsToMs)
+        })
+        statsOutput.flush()
+    }
+
+    override fun close() {
+        statsOutput.flush()
+        statsOutput.close()
+    }
+}
+
+inline fun tcSuite(name: String, block: () -> Unit) {
+    println("##teamcity[testSuiteStarted name='$name']")
+    block()
+    println("##teamcity[testSuiteFinished name='$name']")
+}
+
+inline fun tcTest(name: String, block: () -> Pair<Long, List<Throwable>>) {
+    println("##teamcity[testStarted name='$name' captureStandardOutput='true']")
+    val (time, errors) = block()
+    if (errors.isNotEmpty()) {
+        val detailsWriter = StringWriter()
+        val errorDetailsPrintWriter = PrintWriter(detailsWriter)
+        errors.forEach {
+            it.printStackTrace(errorDetailsPrintWriter)
+            errorDetailsPrintWriter.println()
+        }
+        errorDetailsPrintWriter.close()
+        val details = detailsWriter.toString()
+        println("##teamcity[testFailed name='$name' message='Exceptions reported' details='${details.tcEscape()}']")
+    }
+    println("##teamcity[testFinished name='$name' duration='$time']")
+}
+
+inline fun tcSimplePerfTest(file: String, name: String, stats: Stats, block: () -> Unit) {
+    var spentMs = 0L
+    tcTest(name) {
+        val errors = mutableListOf<Throwable>()
+        val spentNs = measureNanoTime {
+            try {
+                block()
+            } catch (t: Throwable) {
+                errors += t
+            }
+        }
+        stats.append(file, name, spentNs)
+        spentMs = spentNs.nsToMs
+        spentMs to errors
+    }
+    println("##teamcity[buildStatisticValue key='$name' value='$spentMs']")
+}
+
+fun String.tcEscape(): String {
+    return this
+        .replace("|", "||")
+        .replace("[", "|[")
+        .replace("]", "|]")
+        .replace("\r", "|r")
+        .replace("\n", "|n")
+        .replace("'", "|'")
+}

idea/performanceTests/org/jetbrains/kotlin/idea/perf/WholeProjectJavaInspectionTest.kt

@@ -15,7 +15,9 @@ import com.intellij.psi.search.DelegatingGlobalSearchScope
 import com.intellij.psi.search.FileTypeIndex
 import com.intellij.psi.search.ProjectScope
 import org.jetbrains.kotlin.idea.util.ProjectRootsUtil
+import kotlin.test.Ignore
 
+@Ignore(value = "[VD] disabled temporary for further investigation: too much noise, have no clue how to handle it")
 class WholeProjectJavaInspectionTest : WholeProjectInspectionTest() {
 
     override fun provideFiles(project: Project): Collection<VirtualFile> {

idea/performanceTests/org/jetbrains/kotlin/idea/perf/WholeProjectKotlinInspectionTest.kt

@@ -6,7 +6,9 @@
 package org.jetbrains.kotlin.idea.perf
 
 import com.intellij.codeInspection.ex.Tools
+import kotlin.test.Ignore
 
+@Ignore(value = "[VD] disabled temporary for further investigation: too much noise, have no clue how to handle it")
 class WholeProjectKotlinInspectionTest : WholeProjectInspectionTest(), WholeProjectKotlinFileProvider {
     override fun isEnabledInspection(tools: Tools) = tools.tool.language in setOf(null, "kotlin", "UAST")
 }

idea/performanceTests/org/jetbrains/kotlin/idea/perf/WholeProjectLightClassTest.kt

@@ -15,7 +15,9 @@ import org.jetbrains.kotlin.psi.KtClassOrObject
 import org.jetbrains.kotlin.psi.KtVisitorVoid
 import java.util.*
 import kotlin.system.measureNanoTime
+import kotlin.test.Ignore
 
+@Ignore(value = "[VD] disabled temporary for further investigation: too much noise, have no clue how to handle it")
 class WholeProjectLightClassTest : WholeProjectPerformanceTest(), WholeProjectKotlinFileProvider {
 
     override fun doTest(file: VirtualFile): PerFileTestResult {

idea/performanceTests/org/jetbrains/kotlin/idea/perf/WholeProjectPerformanceTest.kt

@@ -6,28 +6,80 @@
 package org.jetbrains.kotlin.idea.perf
 
 import com.intellij.codeInsight.daemon.DaemonAnalyzerTestCase
+import com.intellij.codeInsight.daemon.ImplicitUsageProvider
+import com.intellij.codeInsight.daemon.ProblemHighlightFilter
+import com.intellij.codeInsight.intention.IntentionManager
 import com.intellij.ide.impl.ProjectUtil
+import com.intellij.idea.IdeaTestApplication
+import com.intellij.lang.ExternalAnnotatorsFilter
+import com.intellij.lang.LanguageAnnotators
+import com.intellij.lang.StdLanguages
+import com.intellij.lang.injection.InjectedLanguageManager
+import com.intellij.lang.java.JavaLanguage
 import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.application.ex.ApplicationEx
+import com.intellij.openapi.projectRoots.JavaSdk
+import com.intellij.openapi.projectRoots.ProjectJdkTable
+import com.intellij.openapi.projectRoots.impl.JavaAwareProjectJdkTableImpl
 import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.psi.PsiElement
 import com.intellij.psi.PsiElementVisitor
-import java.io.File
+import com.intellij.psi.impl.search.IndexPatternBuilder
-import java.io.PrintWriter
+import com.intellij.psi.impl.source.resolve.reference.ReferenceProvidersRegistry
-import java.io.StringWriter
+import com.intellij.psi.xml.XmlFileNSInfoProvider
+import com.intellij.xml.XmlSchemaProvider
+import org.jetbrains.kotlin.idea.framework.KotlinSdkType
+import java.io.*
 
 abstract class WholeProjectPerformanceTest : DaemonAnalyzerTestCase(), WholeProjectFileProvider {
 
     private val rootProjectFile: File = File("../perfTestProject").absoluteFile
-    private val statsFile: File = File("build/stats.csv").absoluteFile
+    private val perfStats: Stats = Stats()
     private val tmp = rootProjectFile
 
+    override fun isStressTest(): Boolean = false
+
+    override fun setUp() {
+
+        IdeaTestApplication.getInstance()
+        // to prevent leaked SDKs: 1.8 and Kotlin SDK
+        ApplicationManager.getApplication().runWriteAction {
+            val jdkTableImpl = JavaAwareProjectJdkTableImpl.getInstanceEx()
+            val homePath = if (jdkTableImpl.internalJdk.homeDirectory!!.name == "jre") {
+                jdkTableImpl.internalJdk.homeDirectory!!.parent.path
+            } else {
+                jdkTableImpl.internalJdk.homePath!!
+            }
+
+            val javaSdk = JavaSdk.getInstance()
+            val j8 = javaSdk.createJdk("1.8", homePath)
+
+            val jdkTable = ProjectJdkTable.getInstance()
+            jdkTable.addJdk(j8, testRootDisposable)
+            KotlinSdkType.setUpIfNeeded()
+        }
+
+        super.setUp()
+
+        // fix due to stress test disabled
+        IntentionManager.getInstance().availableIntentionActions  // hack to avoid slowdowns in PyExtensionFactory
+        // don't need test data
+        //PathManagerEx.getTestDataPath() // to cache stuff
+        ReferenceProvidersRegistry.getInstance() // pre-load tons of classes
+        InjectedLanguageManager.getInstance(project) // zillion of Dom Sem classes
+        LanguageAnnotators.INSTANCE.allForLanguage(JavaLanguage.INSTANCE) // pile of annotator classes loads
+        LanguageAnnotators.INSTANCE.allForLanguage(StdLanguages.XML)
+        ProblemHighlightFilter.EP_NAME.extensions
+        ImplicitUsageProvider.EP_NAME.extensionList
+        XmlSchemaProvider.EP_NAME.extensionList
+        XmlFileNSInfoProvider.EP_NAME.extensionList
+        ExternalAnnotatorsFilter.EXTENSION_POINT_NAME.extensionList
+        IndexPatternBuilder.EP_NAME.extensionList
+    }
+
     override fun setUpProject() {
-
         println("Using project in $tmp")
 
-        tmp.resolve(".idea").deleteRecursively()
-
         (ApplicationManager.getApplication() as ApplicationEx).doNotSave()
         myProject = ProjectUtil.openOrImport(tmp.path, null, false)
     }
@@ -41,24 +93,15 @@ abstract class WholeProjectPerformanceTest : DaemonAnalyzerTestCase(), WholeProj
         tcSuite(this::class.simpleName ?: "Unknown") {
             val totals = mutableMapOf<String, Long>()
 
-            val statsOutput = statsFile.bufferedWriter()
-
-            statsOutput.appendln("File, ProcessID, Time")
-
             fun appendInspectionResult(file: String, id: String, nanoTime: Long) {
                 totals.merge(id, nanoTime) { a, b -> a + b }
 
-                statsOutput.appendln(buildString {
+                perfStats.append(file, id, nanoTime)
-                    append(file)
-                    append(", ")
-                    append(id)
-                    append(", ")
-                    append(nanoTime.nsToMs)
-                })
             }
 
             tcSuite("TotalPerFile") {
-                val files = provideFiles(project)
+                // TODO: [VD] temp to limit number of files (and total time to run)
+                val files = provideFiles(project).sortedBy { it.name }.take(10)
 
                 files.forEach {
                     val filePath = File(it.path).relativeTo(tmp).path.replace(File.separatorChar, '/')
@@ -72,10 +115,6 @@ abstract class WholeProjectPerformanceTest : DaemonAnalyzerTestCase(), WholeProj
                 }
             }
 
-            statsOutput.flush()
-            statsOutput.close()
-
-
             tcSuite("Total") {
                 totals.forEach { (k, v) ->
                     tcTest(k) {
@@ -86,39 +125,6 @@ abstract class WholeProjectPerformanceTest : DaemonAnalyzerTestCase(), WholeProj
         }
     }
 
-    inline fun tcSuite(name: String, block: () -> Unit) {
-        println("##teamcity[testSuiteStarted name='$name']")
-        block()
-        println("##teamcity[testSuiteFinished name='$name']")
-    }
-
-    inline fun tcTest(name: String, block: () -> Pair<Long, List<Throwable>>) {
-        println("##teamcity[testStarted name='$name' captureStandardOutput='true']")
-        val (time, errors) = block()
-        if (errors.isNotEmpty()) {
-            val detailsWriter = StringWriter()
-            val errorDetailsPrintWriter = PrintWriter(detailsWriter)
-            errors.forEach {
-                it.printStackTrace(errorDetailsPrintWriter)
-                errorDetailsPrintWriter.println()
-            }
-            errorDetailsPrintWriter.close()
-            val details = detailsWriter.toString()
-            println("##teamcity[testFailed name='$name' message='Exceptions reported' details='${details.tcEscape()}']")
-        }
-        println("##teamcity[testFinished name='$name' duration='$time']")
-    }
-
-    fun String.tcEscape(): String {
-        return this
-            .replace("|", "||")
-            .replace("[", "|[")
-            .replace("]", "|]")
-            .replace("\r", "|r")
-            .replace("\n", "|n")
-            .replace("'", "|'")
-    }
-
     protected fun PsiElement.acceptRecursively(visitor: PsiElementVisitor) {
         this.accept(visitor)
         for (child in this.children) {

idea/testData/perfTest/helloKotlin/src/HelloMain.kt

@@ -0,0 +1,4 @@
+
+fun main(args: Array<String>) {
+    println("Hello World!")
+}

idea/testData/perfTest/helloKotlin/src/HelloMain2.kt

@@ -0,0 +1,4 @@
+
+fun main(args: Array<String>) {
+    println("Hello World!")
+}