Add checkStability to perf tests

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceProjectsTest.kt

@@ -152,8 +152,8 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
 
         assertTrue("path $path does not exist, check README.md", File(projectPath).exists())
 
-        val warmUpIterations = if (fast) 0 else 1
-        val iterations = if (fast) 1 else 3
+        val warmUpIterations = if (fast) 0 else 5
+        val iterations = if (fast) 1 else 5
         val projectManagerEx = ProjectManagerEx.getInstanceEx()
 
         var lastProject: Project? = null
@@ -164,6 +164,7 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
             stats(stats)
             warmUpIterations(warmUpIterations)
             iterations(iterations)
+            checkStability(!fast)
             test {
                 val project = if (!simpleModule) {
                     val project = loadProjectWithName(name = name, path = path)

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerfTestBuilder.kt

@@ -14,6 +14,7 @@ class PerfTestBuilder<SV, TV> {
     private lateinit var test: (TestData<SV, TV>) -> Unit
     private var tearDown: (TestData<SV, TV>) -> Unit = { }
     private var profileEnabled: Boolean = false
+    private var checkStability: Boolean = true
 
     fun run() {
         stats.perfTest(
@@ -23,7 +24,8 @@ class PerfTestBuilder<SV, TV> {
             setUp = setUp,
             test = test,
             tearDown = tearDown,
-            profileEnabled = profileEnabled
+            profileEnabled = profileEnabled,
+            checkStability = checkStability
         )
     }
 
@@ -58,6 +60,10 @@ class PerfTestBuilder<SV, TV> {
     fun profileEnabled(profileEnabled: Boolean) {
         this.profileEnabled = profileEnabled
     }
+
+    fun checkStability(checkStability: Boolean) {
+        this.checkStability = checkStability
+    }
 }
 
 fun <SV, TV> performanceTest(initializer: PerfTestBuilder<SV, TV>.() -> Unit) {

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt

@@ -20,8 +20,8 @@ typealias StatInfos = Map<String, Any>?
 
 class Stats(
     val name: String = "",
-    val header: Array<String> = arrayOf("Name", "ValueMS", "StdDev"),
-    val acceptanceStabilityLevel: Int = 25
+    private val header: Array<String> = arrayOf("Name", "ValueMS", "StdDev"),
+    private val acceptanceStabilityLevel: Int = 25
 ) : Closeable {
 
     private val perfTestRawDataMs = mutableListOf<Long>()
@@ -124,7 +124,8 @@ class Stats(
         setUp: (TestData<SV, TV>) -> Unit = { },
         test: (TestData<SV, TV>) -> Unit,
         tearDown: (TestData<SV, TV>) -> Unit = { },
-        profileEnabled: Boolean = false
+        profileEnabled: Boolean = false,
+        checkStability: Boolean = true
     ) {
 
         val warmPhaseData = PhaseData(
@@ -161,10 +162,13 @@ class Stats(
                     val stable = stabilityPercentage <= acceptanceStabilityLevel
                     printTestStarted(stabilityName)
                     printStatValue(stabilityName, stabilityPercentage)
-                    if (stable) {
+                    if (stable or !checkStability) {
                         printTestFinished(stabilityName)
                     } else {
-                        printTestFailed(stabilityName, "stability above $stabilityPercentage %")
+                        printTestFailed(
+                            stabilityName,
+                            "$testName stability is $stabilityPercentage %, above accepted level of $acceptanceStabilityLevel %"
+                        )
                     }
                 }
             } else {