FIR IDE: stop on fist exception in project perf tests

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceProjectsTest.kt

@@ -243,7 +243,8 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
         lookupElements: List<String>,
         typeAfterMarker: Boolean = true,
         revertChangesAtTheEnd: Boolean = true,
-        note: String = ""
+        note: String = "",
+        stopAtException: Boolean = false,
     ) = perfTypeAndAutocomplete(
         project = project(),
         stats = stats,
@@ -255,7 +256,8 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
         lookupElements = lookupElements,
         typeAfterMarker = typeAfterMarker,
         revertChangesAtTheEnd = revertChangesAtTheEnd,
-        note = note
+        note = note,
+        stopAtException = stopAtException,
     )
 
     fun perfTypeAndAutocomplete(
@@ -269,7 +271,8 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
         lookupElements: List<String>,
         typeAfterMarker: Boolean = true,
         revertChangesAtTheEnd: Boolean = true,
-        note: String = ""
+        note: String = "",
+        stopAtException: Boolean = false,
     ) {
         assertTrue("lookupElements has to be not empty", lookupElements.isNotEmpty())
         perfTypeAndDo(
@@ -297,7 +300,8 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
                     assertTrue("'$lookupElement' has to be present in items $items", items.contains(lookupElement))
                 }
             },
-            revertChangesAtTheEnd = revertChangesAtTheEnd
+            revertChangesAtTheEnd = revertChangesAtTheEnd,
+            stopAtException = stopAtException,
         )
     }
 
@@ -353,7 +357,8 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
         setupAfterTypingBlock: (Fixture) -> Unit,
         testBlock: (Fixture) -> V,
         tearDownCheck: (Fixture, V?) -> Unit,
-        revertChangesAtTheEnd: Boolean
+        revertChangesAtTheEnd: Boolean,
+        stopAtException: Boolean = false,
     ) {
         openFixture(project, fileName).use { fixture ->
             val editor = fixture.editor
@@ -403,6 +408,7 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
                         commitAllDocuments()
                     }
                 }
+                stopAtException(stopAtException)
             }
         }
     }
@@ -583,8 +589,8 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
         note: String = ""
     ): List<HighlightInfo> = perfHighlightFile(project(), name, stats, tools = tools, note = note)
 
-    protected fun perfHighlightFileEmptyProfile(name: String, stats: Stats): List<HighlightInfo> =
+    protected fun perfHighlightFileEmptyProfile(name: String, stats: Stats, stopAtException: Boolean = false): List<HighlightInfo> =
-        perfHighlightFile(project(), name, stats, tools = emptyArray(), note = "empty profile")
+        perfHighlightFile(project(), name, stats, tools = emptyArray(), note = "empty profile", stopAtException = stopAtException)
 
     protected fun perfHighlightFile(
         project: Project,
@@ -595,6 +601,7 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
         warmUpIterations: Int = 3,
         iterations: Int = 10,
         checkStability: Boolean = true,
+        stopAtException: Boolean = false,
         filenameSimplifier: (String) -> String = ::simpleFilename
     ): List<HighlightInfo> {
         val profileManager = ProjectInspectionProfileManager.getInstance(project)
@@ -629,6 +636,7 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
                         PsiManager.getInstance(project).dropPsiCaches()
                     }
                     profilerConfig.enabled = true
+                    stopAtException(stopAtException)
                 }
                 highlightInfos
             }

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerfTestBuilder.kt

@@ -17,6 +17,7 @@ class PerfTestBuilder<SV, TV> {
     private var tearDown: (TestData<SV, TV>) -> Unit = { }
     private var checkStability: Boolean = true
     internal var profilerConfig: ProfilerConfig = ProfilerConfig()
+    private var stopAtException: Boolean = false
 
     internal fun run() {
         stats.perfTest(
@@ -26,7 +27,8 @@ class PerfTestBuilder<SV, TV> {
             setUp = setUp,
             test = test,
             tearDown = tearDown,
-            checkStability = checkStability
+            checkStability = checkStability,
+            stopAtException = stopAtException,
         )
     }
 
@@ -65,6 +67,10 @@ class PerfTestBuilder<SV, TV> {
     fun checkStability(checkStability: Boolean) {
         this.checkStability = checkStability
     }
+
+    fun stopAtException(stopAtException: Boolean) {
+        this.stopAtException = stopAtException
+    }
 }
 
 fun <SV, TV> performanceTest(initializer: PerfTestBuilder<SV, TV>.() -> Unit) {

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt

@@ -95,7 +95,8 @@ class Stats(
         setUp: (TestData<SV, TV>) -> Unit = { },
         test: (TestData<SV, TV>) -> Unit,
         tearDown: (TestData<SV, TV>) -> Unit = { },
-        checkStability: Boolean = true
+        checkStability: Boolean = true,
+        stopAtException: Boolean = false,
     ) {
         val warmPhaseData = PhaseData(
             iterations = warmUpIterations,
@@ -114,8 +115,8 @@ class Stats(
         val block = {
             val metricChildren = mutableListOf<Metric>()
             try {
-                warmUpPhase(warmPhaseData, metricChildren)
+                warmUpPhase(warmPhaseData, metricChildren, stopAtException)
-                val statInfoArray = mainPhase(mainPhaseData, metricChildren)
+                val statInfoArray = mainPhase(mainPhaseData, metricChildren, stopAtException)
 
                 assertEquals(iterations, statInfoArray.size)
                 if (testName != WARM_UP) {
@@ -150,6 +151,9 @@ class Stats(
                 }
             } catch (e: Throwable) {
                 processTimings(testName, emptyArray(), metricChildren)
+                if (stopAtException) {
+                    throw e
+                }
             }
         }
 
@@ -225,8 +229,12 @@ class Stats(
         }
     }
 
-    private fun <SV, TV> warmUpPhase(phaseData: PhaseData<SV, TV>, metricChildren: MutableList<Metric>) {
+    private fun <SV, TV> warmUpPhase(
-        val warmUpStatInfosArray = phase(phaseData, WARM_UP, true)
+        phaseData: PhaseData<SV, TV>,
+        metricChildren: MutableList<Metric>,
+        stopAtException: Boolean,
+    ) {
+        val warmUpStatInfosArray = phase(phaseData, WARM_UP, true, stopAtException)
 
         if (phaseData.testName != WARM_UP) {
             printWarmUpTimings(phaseData.testName, warmUpStatInfosArray, metricChildren)
@@ -243,8 +251,12 @@ class Stats(
         warmUpStatInfosArray.filterNotNull().map { it[ERROR_KEY] as? Throwable }.firstOrNull()?.let { throw it }
     }
 
-    private fun <SV, TV> mainPhase(phaseData: PhaseData<SV, TV>, metricChildren: MutableList<Metric>): Array<StatInfos> {
+    private fun <SV, TV> mainPhase(
-        val statInfosArray = phase(phaseData, "")
+        phaseData: PhaseData<SV, TV>,
+        metricChildren: MutableList<Metric>,
+        stopAtException: Boolean,
+    ): Array<StatInfos> {
+        val statInfosArray = phase(phaseData, "", stopAtException = stopAtException)
         statInfosArray.filterNotNull().map { it[ERROR_KEY] as? Throwable }.firstOrNull()?.let {
             convertStatInfoIntoMetrics(
                 phaseData.testName,
@@ -257,7 +269,12 @@ class Stats(
         return statInfosArray
     }
 
-    private fun <SV, TV> phase(phaseData: PhaseData<SV, TV>, phaseName: String, warmup: Boolean = false): Array<StatInfos> {
+    private fun <SV, TV> phase(
+        phaseData: PhaseData<SV, TV>,
+        phaseName: String,
+        warmup: Boolean = false,
+        stopAtException: Boolean,
+    ): Array<StatInfos> {
         val statInfosArray = Array<StatInfos>(phaseData.iterations) { null }
         val testData = TestData<SV, TV>(null, null)
 
@@ -311,6 +328,9 @@ class Stats(
         } catch (t: Throwable) {
             logMessage(t) { "error at ${phaseData.testName}" }
             TeamCity.testFailed(name, error = t)
+            if (stopAtException) {
+                throw t
+            }
         }
         return statInfosArray
     }

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/common/AbstractWholeProjectPerformanceComparisonTest.kt

@@ -5,7 +5,9 @@
 
 package org.jetbrains.kotlin.idea.perf.common
 
-import org.jetbrains.kotlin.idea.perf.*
+import org.jetbrains.kotlin.idea.perf.AbstractPerformanceProjectsTest
+import org.jetbrains.kotlin.idea.perf.Stats
+import org.jetbrains.kotlin.idea.perf.WarmUpProject
 import org.jetbrains.kotlin.idea.perf.util.TeamCity
 import org.jetbrains.kotlin.idea.testFramework.ProjectOpenAction
 
@@ -33,7 +35,7 @@ abstract class AbstractWholeProjectPerformanceComparisonTest : AbstractPerforman
                     FILE_NAMES.NAME_RESOLUTION
                 )
 
-                filesToHighlight.forEach { file -> perfHighlightFileEmptyProfile(file, stats = stat) }
+                filesToHighlight.forEach { file -> perfHighlightFileEmptyProfile(file, stats = stat, stopAtException = true) }
             }
         }
     }
@@ -50,7 +52,8 @@ abstract class AbstractWholeProjectPerformanceComparisonTest : AbstractPerforman
                     insertString = "val a = l",
                     highlightFileBeforeStartTyping = true,
                     lookupElements = listOf("line"),
-                    note = "in-method completion"
+                    note = "in-method completion",
+                    stopAtException = true,
                 )
 
                 perfTypeAndAutocomplete(
@@ -60,7 +63,8 @@ abstract class AbstractWholeProjectPerformanceComparisonTest : AbstractPerforman
                     insertString = "\nval a = ",
                     highlightFileBeforeStartTyping = true,
                     lookupElements = listOf("processAssocTypeVariants"),
-                    note = "top-level completion"
+                    note = "top-level completion",
+                    stopAtException = true,
                 )
 
                 perfTypeAndAutocomplete(
@@ -70,7 +74,8 @@ abstract class AbstractWholeProjectPerformanceComparisonTest : AbstractPerforman
                     highlightFileBeforeStartTyping = true,
                     insertString = "\nval a = s",
                     lookupElements = listOf("scope"),
-                    note = "in big method in big file completion"
+                    note = "in big method in big file completion",
+                    stopAtException = true,
                 )
             }
         }