(PerformanceTest) improvements in profiler snapshots

support for typing per inspection tests gradle arguments fixed to allow running tracing from cli Profiler disabled for warmup phases. relates to #KT-38653
2020-05-30 01:33:15 +02:00
parent 0d2552b0b6
commit 9719391c82
13 changed files with 176 additions and 100 deletions
@@ -79,21 +79,20 @@ fun Project.projectTest(
        }
    }

-    doFirst {
-        val agent = tasks.findByPath(":test-instrumenter:jar")!!.outputs.files.singleFile
-
-        val args = project.findProperty("kotlin.test.instrumentation.args")?.let { "=$it" }.orEmpty()
-
-        jvmArgs("-javaagent:$agent$args")
+    if (project.findProperty("kotlin.test.instrumentation.disable")?.toString()?.toBoolean() != true) {
+        doFirst {
+            val agent = tasks.findByPath(":test-instrumenter:jar")!!.outputs.files.singleFile
+            val args = project.findProperty("kotlin.test.instrumentation.args")?.let { "=$it" }.orEmpty()
+            jvmArgs("-javaagent:$agent$args")
+        }
+        dependsOn(":test-instrumenter:jar")
    }

-    dependsOn(":test-instrumenter:jar")
-
    jvmArgs(
        "-ea",
        "-XX:+HeapDumpOnOutOfMemoryError",
        "-XX:+UseCodeCacheFlushing",
-        "-XX:ReservedCodeCacheSize=128m",
+        "-XX:ReservedCodeCacheSize=256m",
        "-Djna.nosys=true"
    )

@@ -14,6 +14,10 @@ Run all Kotlin IDE plugin performance tests with

 `$ gradle idea-plugin-performance-tests`

+## Run with profiler
+
+`YOURKIT_PROFILER_HOME=/Applications/YourKit-Java-Profiler-2019.8.app ./gradlew -Pkotlin.test.instrumentation.disable=true  :idea:performanceTests:performanceTest --tests "<test-filter>"`
+
 ## Performance test

 Output is provided to console in TeamCity format like
@@ -84,9 +84,10 @@ projectTest(taskName = "performanceTest") {
    maxHeapSize = "3g"
    jvmArgs("-Didea.debug.mode=true")
    jvmArgs("-XX:SoftRefLRUPolicyMSPerMB=50")
+
    jvmArgs(
-        "-XX:ReservedCodeCacheSize=240m",
        "-XX:+UseCompressedOops",
+        "-Didea.ProcessCanceledException=disabled",
        "-XX:+UseConcMarkSweepGC"
    )

@@ -97,7 +98,7 @@ projectTest(taskName = "performanceTest") {
                classpath += files("$yourKitHome/lib/yjp-controller-api-redist.jar")
            }
            currentOs.isMacOsX -> {
-                jvmArgs("-agentpath:$yourKitHome/Contents/Resources/bin/mac/libyjpagent.dylib")
+                jvmArgs("-agentpath:$yourKitHome/Contents/Resources/bin/mac/libyjpagent.dylib=delay=5000,_socket_timeout_ms=120000,disablealloc,disable_async_sampling,disablenatives")
                classpath += files("$yourKitHome/Contents/Resources/lib/yjp-controller-api-redist.jar")
            }
        }
@@ -130,7 +131,6 @@ projectTest(taskName = "wholeProjectsPerformanceTest") {
    jvmArgs("-DemptyProfile=${System.getProperty("emptyProfile")}")
    jvmArgs("-XX:SoftRefLRUPolicyMSPerMB=50")
    jvmArgs(
-        "-XX:ReservedCodeCacheSize=240m",
        "-XX:+UseCompressedOops",
        "-XX:+UseConcMarkSweepGC"
    )
@@ -5,12 +5,14 @@

 package org.jetbrains.kotlin.idea.perf

+import com.intellij.codeHighlighting.Pass
 import com.intellij.codeInsight.daemon.impl.HighlightInfo
 import com.intellij.testFramework.UsefulTestCase
 import org.jetbrains.kotlin.idea.perf.util.ExternalProject
 import org.jetbrains.kotlin.idea.perf.util.lastPathSegment
 import org.jetbrains.kotlin.idea.perf.util.suite
 import org.jetbrains.kotlin.test.JUnit3RunnerWithInners
+import org.junit.Ignore
 import org.junit.runner.RunWith

 /**
@@ -31,6 +33,7 @@ class AHeavyInspectionsPerformanceTest : UsefulTestCase() {
        "UnusedSymbol",
        "MemberVisibilityCanBePrivate"
    )
+    val passesToIgnore = ((1 until 100) - Pass.LOCAL_INSPECTIONS - Pass.WHOLE_FILE_LOCAL_INSPECTIONS).toIntArray()

    fun testLocalInspection() {
        suite {
@@ -41,6 +44,65 @@ class AHeavyInspectionsPerformanceTest : UsefulTestCase() {
                        for (file in listOfFiles) {
                            val editorFile = editor(file)

+                            measure<List<HighlightInfo>>(inspection, file.lastPathSegment()) {
+                                test = {
+                                    highlight(editorFile, passesToIgnore)
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    @Ignore
+    fun testUnusedSymbolLocalInspection() {
+        suite {
+            config.warmup = 1
+            config.iterations = 2
+            config.profilerConfig.enabled = true
+            config.profilerConfig.tracing = true
+            app {
+                project(ExternalProject.KOTLIN_AUTO) {
+                    for (inspection in listOfInspections.sliceArray(0 until 1)) {
+                        enableSingleInspection(inspection)
+                        for (file in listOfFiles.sliceArray(0 until 1)) {
+                            val editorFile = editor(file)
+
+                            measure<List<HighlightInfo>>(inspection, file.lastPathSegment()) {
+                                test = {
+                                    highlight(editorFile, passesToIgnore)
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    // used locally to test
+    val UNUSED_AUTO = ExternalProject(
+        "../unused",
+        ExternalProject.autoOpenAction("../unused")
+    )
+
+    @Ignore
+    fun testSingleInspection1() {
+        suite {
+            config.warmup = 0
+            config.iterations = 1
+            config.profilerConfig.enabled = true
+            config.profilerConfig.tracing = true
+            app {
+                project(UNUSED_AUTO) {
+                    for (inspection in listOfInspections.sliceArray(0..1)) {
+                        enableSingleInspection(inspection)
+                        val listOfFiles = listOf("src/main/kotlin/org/test/test.kt")
+                        for (file in listOfFiles) {
+                            val editorFile = editor(file)
+
                            measure<List<HighlightInfo>>(inspection, file.lastPathSegment()) {
                                test = {
                                    highlight(editorFile)
@@ -350,7 +350,7 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
                stats(stats)
                warmUpIterations(8)
                iterations(15)
-                profilerEnabled(true)
+                profilerConfig.enabled = true
                setUp {
                    val markerOffset = editor.document.text.indexOf(marker)
                    assertTrue("marker '$marker' not found in $fileName", markerOffset > 0)
@@ -468,7 +468,7 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
                }
                commitAllDocuments()
            }
-            profilerEnabled(true)
+            profilerConfig.enabled = true
        }
    }

@@ -543,7 +543,7 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
                    commitAllDocuments()
                }
            }
-            profilerEnabled(true)
+            profilerConfig.enabled = true
        }
    }

@@ -610,7 +610,7 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
                        }
                        PsiManager.getInstance(project).dropPsiCaches()
                    }
-                    profilerEnabled(!isWarmUp)
+                    profilerConfig.enabled = true
                }
                highlightInfos
            }
@@ -663,7 +663,7 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
                }
                it.value?.let { v -> assertNotNull(v) }
            }
-            profilerEnabled(true)
+            profilerConfig.enabled = true
            checkStability(false)
        }
    }
@@ -15,9 +15,8 @@ class PerfTestBuilder<SV, TV> {
    private var setUp: (TestData<SV, TV>) -> Unit = { }
    private lateinit var test: (TestData<SV, TV>) -> Unit
    private var tearDown: (TestData<SV, TV>) -> Unit = { }
-    private var profilerEnabled: Boolean = false
    private var checkStability: Boolean = true
-    private var profilerConfig: ProfilerConfig = ProfilerConfig()
+    internal var profilerConfig: ProfilerConfig = ProfilerConfig()

    internal fun run() {
        stats.perfTest(
@@ -27,7 +26,6 @@ class PerfTestBuilder<SV, TV> {
            setUp = setUp,
            test = test,
            tearDown = tearDown,
-            profilerEnabled = profilerEnabled,
            checkStability = checkStability
        )
    }
@@ -60,10 +58,6 @@ class PerfTestBuilder<SV, TV> {
        this.tearDown = tearDown
    }

-    fun profilerEnabled(profilerEnabled: Boolean) {
-        this.profilerEnabled = profilerEnabled
-    }
-
    fun profilerConfig(profilerConfig: ProfilerConfig) {
        this.profilerConfig = profilerConfig
    }
@@ -294,7 +294,7 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
                setUp(perfKtsFileAnalysisSetUp(project, fileName))
                test(perfKtsFileAnalysisTest())
                tearDown(perfKtsFileAnalysisTearDown(extraTimingsNs, project))
-                profilerEnabled(true)
+                profilerConfig.enabled = true
            }

            extraStats.printWarmUpTimings(
@@ -127,7 +127,6 @@ class Stats(
        setUp: (TestData<SV, TV>) -> Unit = { },
        test: (TestData<SV, TV>) -> Unit,
        tearDown: (TestData<SV, TV>) -> Unit = { },
-        profilerEnabled: Boolean = false,
        checkStability: Boolean = true
    ) {

@@ -137,15 +136,13 @@ class Stats(
            setUp = setUp,
            test = test,
            tearDown = tearDown,
-            profilerEnabled = profilerEnabled
        )
        val mainPhaseData = PhaseData(
            iterations = iterations,
            testName = testName,
            setUp = setUp,
            test = test,
-            tearDown = tearDown,
-            profilerEnabled = profilerEnabled
+            tearDown = tearDown
        )
        val block = {
            warmUpPhase(warmPhaseData)
@@ -228,7 +225,7 @@ class Stats(
    }

    private fun <SV, TV> warmUpPhase(phaseData: PhaseData<SV, TV>) {
-        val warmUpStatInfosArray = phase(phaseData, WARM_UP)
+        val warmUpStatInfosArray = phase(phaseData, WARM_UP, true)

        if (phaseData.testName != WARM_UP) {
            printWarmUpTimings(phaseData.testName, warmUpStatInfosArray)
@@ -256,17 +253,18 @@ class Stats(
        return statInfosArray
    }

-    private fun <SV, TV> phase(phaseData: PhaseData<SV, TV>, phaseName: String): Array<StatInfos> {
+    private fun <SV, TV> phase(phaseData: PhaseData<SV, TV>, phaseName: String, warmup: Boolean = false): Array<StatInfos> {
        val statInfosArray = Array<StatInfos>(phaseData.iterations) { null }
        val testData = TestData<SV, TV>(null, null)

        try {
+            val phaseProfiler =
+                createPhaseProfiler(phaseData.testName, phaseName, profilerConfig.copy(warmup = warmup))
+
            for (attempt in 0 until phaseData.iterations) {
                testData.reset()
                triggerGC(attempt)

-                val phaseProfiler = createPhaseProfiler(phaseData, phaseName, attempt, profilerConfig)
-
                val setUpMillis = measureTimeMillis { phaseData.setUp(testData) }
                val attemptName = "${phaseData.testName} #$attempt"
                logMessage { "$attemptName setup took $setUpMillis ms" }
@@ -274,12 +272,10 @@ class Stats(
                val valueMap = HashMap<String, Any>(2 * PerformanceCounter.numberOfCounters + 1)
                statInfosArray[attempt] = valueMap
                try {
-
                    phaseProfiler.start()
                    valueMap[TEST_KEY] = measureNanoTime {
                        phaseData.test(testData)
                    }
-                    phaseProfiler.stop()

                    PerformanceCounter.report { name, counter, nanos ->
                        valueMap["counter \"$name\": count"] = counter.toLong()
@@ -291,6 +287,7 @@ class Stats(
                    valueMap[ERROR_KEY] = t
                    break
                } finally {
+                    phaseProfiler.stop()
                    try {
                        val tearDownMillis = measureTimeMillis {
                            phaseData.tearDown(testData)
@@ -312,20 +309,20 @@ class Stats(
        return statInfosArray
    }

-    private fun <K, T> createPhaseProfiler(
-        phaseData: PhaseData<K, T>,
+    private fun createPhaseProfiler(
+        testName: String,
        phaseName: String,
-        attempt: Int,
        profilerConfig: ProfilerConfig
    ): PhaseProfiler {
-        val profilerHandler = if (phaseData.profilerEnabled) ProfilerHandler.getInstance() else DummyProfilerHandler
+        profilerConfig.name = "$testName${if (phaseName.isEmpty()) "" else "-"+phaseName}"
+        profilerConfig.path = pathToResource("profile/${plainname()}")
+        val profilerHandler = if (profilerConfig.enabled && !profilerConfig.warmup)
+            ProfilerHandler.getInstance(profilerConfig)
+        else
+            DummyProfilerHandler

        return if (profilerHandler != DummyProfilerHandler) {
-            val profilerPath = pathToResource("profile/${plainname()}")
-            check(with(File(profilerPath)) { exists() || mkdirs() }) { "unable to mkdirs $profilerPath for ${phaseData.testName}" }
-            val activityName = "${phaseData.testName}-${if (phaseName.isEmpty()) "" else "$phaseName-"}$attempt"
-
-            ActualPhaseProfiler(activityName, profilerPath, profilerHandler, profilerConfig)
+            ActualPhaseProfiler(profilerHandler)
        } else {
            DummyPhaseProfiler
        }
@@ -374,8 +371,7 @@ data class PhaseData<SV, TV>(
    val testName: String,
    val setUp: (TestData<SV, TV>) -> Unit,
    val test: (TestData<SV, TV>) -> Unit,
-    val tearDown: (TestData<SV, TV>) -> Unit,
-    val profilerEnabled: Boolean = false
+    val tearDown: (TestData<SV, TV>) -> Unit
 )

 data class TestData<SV, TV>(var setUpValue: SV?, var value: TV?) {
@@ -17,19 +17,18 @@ object DummyPhaseProfiler : PhaseProfiler {
 }

 class ActualPhaseProfiler(
-    private val activityName: String,
-    private val profilerPath: String,
-    private val profilerHandler: ProfilerHandler,
-    private val config: ProfilerConfig = ProfilerConfig()
-) :
-    PhaseProfiler {
+    private val profilerHandler: ProfilerHandler
+) : PhaseProfiler {
+    private var attempt: Int = 1
+
    override fun start() {
-        profilerHandler.startProfiling(activityName, config)
+        profilerHandler.startProfiling()
    }

    override fun stop() {
-        profilerHandler.stopProfiling(profilerPath, activityName, config)
+        profilerHandler.stopProfiling(attempt)
+        attempt++
    }
 }

-data class ProfilerConfig(var tracing: Boolean = false, var options: List<String> = emptyList())
+data class ProfilerConfig(var enabled: Boolean = false, var path: String = "", var name: String = "", var tracing: Boolean = false, var options: List<String> = emptyList(), var warmup: Boolean = false)
@@ -8,33 +8,44 @@ package org.jetbrains.kotlin.idea.perf.profilers
 import org.jetbrains.kotlin.idea.perf.profilers.async.AsyncProfilerHandler
 import org.jetbrains.kotlin.idea.perf.profilers.yk.YKProfilerHandler
 import org.jetbrains.kotlin.idea.perf.util.logMessage
+import java.nio.file.Path

 interface ProfilerHandler {
-    fun startProfiling(activityName: String, config: ProfilerConfig)
+    fun startProfiling()

-    fun stopProfiling(snapshotsPath: String, activityName: String, config: ProfilerConfig)
+    fun stopProfiling(attempt: Int)

    companion object {
        private var instance: ProfilerHandler? = null

        @Synchronized
-        fun getInstance(): ProfilerHandler {
+        fun getInstance(profilerConfig: ProfilerConfig): ProfilerHandler {
            return instance ?: run {
                val handler =
-                    doOrLog("asyncProfiler not found") { AsyncProfilerHandler() }
-                        ?: doOrLog("yourKit not found") { YKProfilerHandler() } ?: DummyProfilerHandler
+                    doOrLog("asyncProfiler not found") { AsyncProfilerHandler(profilerConfig) }
+                        ?: doOrLog("yourKit not found") { YKProfilerHandler(profilerConfig) } ?: DummyProfilerHandler
                instance = handler
                return handler
            }
        }
-    }

+        fun determinePhasePath(dumpPath: Path, profilerConfig: ProfilerConfig): Path {
+            val activityPath = dumpPath.parent.resolve(profilerConfig.path)
+            val runNumber =
+                (activityPath.toFile().listFiles()?.maxBy { it.name.toIntOrNull() ?: 0 }?.name?.toIntOrNull()
+                    ?: 0) + 1
+            val runPath = activityPath.resolve("$runNumber")
+            runPath.toFile().mkdirs()
+            logMessage { "profiler's run path found $runPath" }
+            return runPath
+        }
+    }
 }

 object DummyProfilerHandler : ProfilerHandler {
-    override fun startProfiling(activityName: String, config: ProfilerConfig) {}
+    override fun startProfiling() {}

-    override fun stopProfiling(snapshotsPath: String, activityName: String, config: ProfilerConfig) {}
+    override fun stopProfiling(attempt: Int) {}
 }

 internal fun <T> doOrLog(message: String, block: () -> T?): T? {
@@ -26,7 +26,7 @@ import java.util.*
 *
 * AsyncProfiler could be downloaded from https://github.com/jvm-profiling-tools/async-profiler/releases/
 */
-internal class AsyncProfilerHandler : ProfilerHandler {
+internal class AsyncProfilerHandler(val profilerConfig: ProfilerConfig) : ProfilerHandler {

    private val asyncProfiler: Any

@@ -48,20 +48,20 @@ internal class AsyncProfilerHandler : ProfilerHandler {
        executeMethod.invoke(asyncProfiler, command)
    }

-    override fun startProfiling(activityName: String, config: ProfilerConfig) {
+    override fun startProfiling() {
        try {
-            profilingOptions = config.options
-            execute(AsyncProfilerCommandBuilder.buildStartCommand(config.options))
+            profilingOptions = profilerConfig.options
+            execute(AsyncProfilerCommandBuilder.buildStartCommand(profilerConfig.options))
            profilingStarted = true
        } catch (e: Exception) {
            throw RuntimeException(e)
        }
    }

-    override fun stopProfiling(snapshotsPath: String, activityName: String, config: ProfilerConfig) {
-        val combinedOptions = ArrayList(config.options)
-        val commandBuilder = AsyncProfilerCommandBuilder(snapshotsPath)
-        val name = activityName.replace(' ', '_').replace('/', '_')
+    override fun stopProfiling(attempt: Int) {
+        val combinedOptions = ArrayList(profilerConfig.options)
+        val commandBuilder = AsyncProfilerCommandBuilder(profilerConfig.path)
+        val name = "${profilerConfig.name}-$attempt".replace(' ', '_').replace('/', '_')
        val stopAndDumpCommands = commandBuilder.buildStopAndDumpCommands(name, combinedOptions)
        for (stopCommand in stopAndDumpCommands) {
            execute(stopCommand)
@@ -7,11 +7,15 @@ package org.jetbrains.kotlin.idea.perf.profilers.yk

 import org.jetbrains.kotlin.idea.perf.profilers.ProfilerConfig
 import org.jetbrains.kotlin.idea.perf.profilers.ProfilerHandler
+import org.jetbrains.kotlin.idea.perf.profilers.ProfilerHandler.Companion.determinePhasePath
 import org.jetbrains.kotlin.idea.perf.profilers.doOrThrow
 import org.jetbrains.kotlin.idea.perf.util.logMessage
+import java.io.PrintWriter
+import java.io.StringWriter
 import java.lang.management.ManagementFactory
 import java.lang.reflect.Method
 import java.nio.file.Files
+import java.nio.file.Path
 import java.nio.file.Paths

 /**
@@ -19,9 +23,10 @@ import java.nio.file.Paths
 * - ${YOURKIT_PROFILER_HOME}/lib/yjp-controller-api-redist.jar has to be in a classpath
 * - add agentpath vm paramter like `-agentpath:${YOURKIT_PROFILER_HOME}/Resources/bin/mac/libyjpagent.dylib`
 */
-class YKProfilerHandler : ProfilerHandler {
+class YKProfilerHandler(val profilerConfig: ProfilerConfig) : ProfilerHandler {

    private var controller: Any
+    lateinit var phasePath: Path

    init {
        check(ManagementFactory.getRuntimeMXBean().inputArguments.any { it.contains("yjpagent") }) {
@@ -33,20 +38,28 @@ class YKProfilerHandler : ProfilerHandler {
        }
    }

-    override fun startProfiling(activityName: String, config: ProfilerConfig) {
-        if (config.tracing)
-            startTracingMethod.invoke(controller, null)
-        else
-            startCPUSamplingMethod.invoke(controller, null)
+    override fun startProfiling() {
+        try {
+            if (profilerConfig.tracing) {
+                startTracingMethod.invoke(controller, null)
+            } else {
+                startSamplingMethod.invoke(controller, null)
+            }
+        } catch (e: Exception) {
+            val stringWriter = StringWriter().also { e.printStackTrace(PrintWriter(it)) }
+            logMessage { "exception while starting profile ${e.localizedMessage} $stringWriter" }
+        }
    }

-    override fun stopProfiling(snapshotsPath: String, activityName: String, config: ProfilerConfig) {
-        val dumpPath = captureSnapshotMethod.invoke(controller, SNAPSHOT_WITHOUT_HEAP) as String
-        stopCPUProfilingMethod.invoke(controller)
-        val path = Paths.get(dumpPath)
-        val target = path.parent.resolve(snapshotsPath).resolve(activityName)
-        logMessage { "dump is moved to $target" }
-        Files.move(path, target)
+    override fun stopProfiling(attempt: Int) {
+        val pathToSnapshot = captureSnapshotMethod.invoke(controller, SNAPSHOT_WITHOUT_HEAP) as String
+        stopCpuProfilingMethod.invoke(controller)
+        val dumpPath = Paths.get(pathToSnapshot)
+        if (!this::phasePath.isInitialized)
+            phasePath = determinePhasePath(dumpPath, profilerConfig)
+        val targetFile = phasePath.resolve("$attempt-${profilerConfig.name}.snapshot")
+        logMessage { "dump is moved to $targetFile" }
+        Files.move(dumpPath, targetFile)
    }

    companion object {
@@ -55,9 +68,9 @@ class YKProfilerHandler : ProfilerHandler {
        private val ykLibClass: Class<*> = doOrThrow("yjp-controller-api-redist.jar is not in a classpath") {
            Class.forName("com.yourkit.api.Controller")
        }
-        private val startCPUSamplingMethod: Method = doOrThrow("com.yourkit.api.Controller#startCPUSampling(String) not found") {
+        private val startSamplingMethod: Method = doOrThrow("com.yourkit.api.Controller#startSampling(String) not found") {
            ykLibClass.getMethod(
-                "startCPUSampling",
+                "startSampling",
                String::class.java
            )
        }
@@ -74,12 +87,13 @@ class YKProfilerHandler : ProfilerHandler {
            )
        }

-        private val capturePerformanceSnapshotMethod: Method = doOrThrow("com.yourkit.api.Controller#capturePerformanceSnapshot() not found") {
-            ykLibClass.getMethod("capturePerformanceSnapshot")
-        }
+        private val capturePerformanceSnapshotMethod: Method =
+            doOrThrow("com.yourkit.api.Controller#capturePerformanceSnapshot() not found") {
+                ykLibClass.getMethod("capturePerformanceSnapshot")
+            }

-        private val stopCPUProfilingMethod: Method = doOrThrow("com.yourkit.api.Controller#stopCPUProfiling() not found") {
-            ykLibClass.getMethod("stopCPUProfiling")
+        private val stopCpuProfilingMethod: Method = doOrThrow("com.yourkit.api.Controller#stopCPUProfiling() not found") {
+            ykLibClass.getMethod("stopCpuProfiling")
        }
    }
 }
@@ -61,11 +61,11 @@ class PerformanceSuite {
            }
        }

-        private fun PsiFile.highlightFile(): List<HighlightInfo> {
+        private fun PsiFile.highlightFile(toIgnore: IntArray = ArrayUtilRt.EMPTY_INT_ARRAY): List<HighlightInfo> {
            val document = FileDocumentManager.getInstance().getDocument(virtualFile)!!
            val editor = EditorFactory.getInstance().getEditors(document).first()
            PsiDocumentManager.getInstance(project).commitAllDocuments()
-            return CodeInsightTestFixtureImpl.instantiateAndRun(this, editor, ArrayUtilRt.EMPTY_INT_ARRAY, true)
+            return CodeInsightTestFixtureImpl.instantiateAndRun(this, editor, toIgnore, true)
        }

        fun rollbackChanges(vararg file: VirtualFile) {
@@ -139,7 +139,6 @@ class PerformanceSuite {
                tearDown {
                    after?.invoke()
                }
-                profilerEnabled(config.profile)
                profilerConfig(config.profilerConfig)
            }
            return value
@@ -216,7 +215,7 @@ class PerformanceSuite {
    }


-    class StatsScopeConfig(var name: String? = null, var warmup: Int = 2, var iterations: Int = 5, var profile: Boolean = false, var profilerConfig: ProfilerConfig = ProfilerConfig())
+    class StatsScopeConfig(var name: String? = null, var warmup: Int = 2, var iterations: Int = 5, var profilerConfig: ProfilerConfig = ProfilerConfig())

    class ProjectScopeConfig(val path: String, val openWith: ProjectOpenAction, val refresh: Boolean = false) {
        val name: String = path.lastPathSegment()
@@ -239,10 +238,8 @@ class PerformanceSuite {

        fun highlight(fixture: Fixture) = highlight(fixture.psiFile)

-        fun highlight(editorFile: PsiFile?) =
-            editorFile?.let {
-                it.highlightFile()
-            } ?: error("editor isn't ready for highlight")
+        fun highlight(editorFile: PsiFile?, toIgnore: IntArray = ArrayUtilRt.EMPTY_INT_ARRAY) =
+            editorFile?.highlightFile(toIgnore) ?: error("editor isn't ready for highlight")

        fun moveCursor(config: TypingConfig) {
            val fixture = config.fixture
@@ -322,7 +319,7 @@ class PerformanceSuite {

        fun <T> measure(vararg name: String, f: MeasurementScope<T>.() -> Unit): List<T?> {
            val after = { PsiManager.getInstance(project).dropPsiCaches() }
-            return app.stats.measure("${name.joinToString("-")}", f, after)
+            return app.stats.measure(name.joinToString("-"), f, after)
        }

        fun <T> measure(vararg name: String, fixture: Fixture, f: MeasurementScope<T>.() -> Unit): List<T?> =