From 5485c372e8d2b37779c8d50aae53936fc0907e98 Mon Sep 17 00:00:00 2001 From: Ilya Kirillov Date: Thu, 25 Feb 2021 17:46:12 +0100 Subject: [PATCH] FIR IDE: stop on fist exception in project perf tests --- .../perf/AbstractPerformanceProjectsTest.kt | 22 ++++++++---- .../kotlin/idea/perf/PerfTestBuilder.kt | 8 ++++- .../org/jetbrains/kotlin/idea/perf/Stats.kt | 36 ++++++++++++++----- ...ctWholeProjectPerformanceComparisonTest.kt | 15 +++++--- 4 files changed, 60 insertions(+), 21 deletions(-) diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceProjectsTest.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceProjectsTest.kt index a19a0ee38c6..ee31af82f81 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceProjectsTest.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceProjectsTest.kt @@ -243,7 +243,8 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() { lookupElements: List, typeAfterMarker: Boolean = true, revertChangesAtTheEnd: Boolean = true, - note: String = "" + note: String = "", + stopAtException: Boolean = false, ) = perfTypeAndAutocomplete( project = project(), stats = stats, @@ -255,7 +256,8 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() { lookupElements = lookupElements, typeAfterMarker = typeAfterMarker, revertChangesAtTheEnd = revertChangesAtTheEnd, - note = note + note = note, + stopAtException = stopAtException, ) fun perfTypeAndAutocomplete( @@ -269,7 +271,8 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() { lookupElements: List, typeAfterMarker: Boolean = true, revertChangesAtTheEnd: Boolean = true, - note: String = "" + note: String = "", + stopAtException: Boolean = false, ) { assertTrue("lookupElements has to be not empty", lookupElements.isNotEmpty()) perfTypeAndDo( @@ -297,7 +300,8 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() { assertTrue("'$lookupElement' has to be present in items $items", items.contains(lookupElement)) } }, - revertChangesAtTheEnd = revertChangesAtTheEnd + revertChangesAtTheEnd = revertChangesAtTheEnd, + stopAtException = stopAtException, ) } @@ -353,7 +357,8 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() { setupAfterTypingBlock: (Fixture) -> Unit, testBlock: (Fixture) -> V, tearDownCheck: (Fixture, V?) -> Unit, - revertChangesAtTheEnd: Boolean + revertChangesAtTheEnd: Boolean, + stopAtException: Boolean = false, ) { openFixture(project, fileName).use { fixture -> val editor = fixture.editor @@ -403,6 +408,7 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() { commitAllDocuments() } } + stopAtException(stopAtException) } } } @@ -583,8 +589,8 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() { note: String = "" ): List = perfHighlightFile(project(), name, stats, tools = tools, note = note) - protected fun perfHighlightFileEmptyProfile(name: String, stats: Stats): List = - perfHighlightFile(project(), name, stats, tools = emptyArray(), note = "empty profile") + protected fun perfHighlightFileEmptyProfile(name: String, stats: Stats, stopAtException: Boolean = false): List = + perfHighlightFile(project(), name, stats, tools = emptyArray(), note = "empty profile", stopAtException = stopAtException) protected fun perfHighlightFile( project: Project, @@ -595,6 +601,7 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() { warmUpIterations: Int = 3, iterations: Int = 10, checkStability: Boolean = true, + stopAtException: Boolean = false, filenameSimplifier: (String) -> String = ::simpleFilename ): List { val profileManager = ProjectInspectionProfileManager.getInstance(project) @@ -629,6 +636,7 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() { PsiManager.getInstance(project).dropPsiCaches() } profilerConfig.enabled = true + stopAtException(stopAtException) } highlightInfos } diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerfTestBuilder.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerfTestBuilder.kt index 0002c0aa866..ed4057ec89a 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerfTestBuilder.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerfTestBuilder.kt @@ -17,6 +17,7 @@ class PerfTestBuilder { private var tearDown: (TestData) -> Unit = { } private var checkStability: Boolean = true internal var profilerConfig: ProfilerConfig = ProfilerConfig() + private var stopAtException: Boolean = false internal fun run() { stats.perfTest( @@ -26,7 +27,8 @@ class PerfTestBuilder { setUp = setUp, test = test, tearDown = tearDown, - checkStability = checkStability + checkStability = checkStability, + stopAtException = stopAtException, ) } @@ -65,6 +67,10 @@ class PerfTestBuilder { fun checkStability(checkStability: Boolean) { this.checkStability = checkStability } + + fun stopAtException(stopAtException: Boolean) { + this.stopAtException = stopAtException + } } fun performanceTest(initializer: PerfTestBuilder.() -> Unit) { diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt index 01d9dda1b44..026981e3059 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt @@ -95,7 +95,8 @@ class Stats( setUp: (TestData) -> Unit = { }, test: (TestData) -> Unit, tearDown: (TestData) -> Unit = { }, - checkStability: Boolean = true + checkStability: Boolean = true, + stopAtException: Boolean = false, ) { val warmPhaseData = PhaseData( iterations = warmUpIterations, @@ -114,8 +115,8 @@ class Stats( val block = { val metricChildren = mutableListOf() try { - warmUpPhase(warmPhaseData, metricChildren) - val statInfoArray = mainPhase(mainPhaseData, metricChildren) + warmUpPhase(warmPhaseData, metricChildren, stopAtException) + val statInfoArray = mainPhase(mainPhaseData, metricChildren, stopAtException) assertEquals(iterations, statInfoArray.size) if (testName != WARM_UP) { @@ -150,6 +151,9 @@ class Stats( } } catch (e: Throwable) { processTimings(testName, emptyArray(), metricChildren) + if (stopAtException) { + throw e + } } } @@ -225,8 +229,12 @@ class Stats( } } - private fun warmUpPhase(phaseData: PhaseData, metricChildren: MutableList) { - val warmUpStatInfosArray = phase(phaseData, WARM_UP, true) + private fun warmUpPhase( + phaseData: PhaseData, + metricChildren: MutableList, + stopAtException: Boolean, + ) { + val warmUpStatInfosArray = phase(phaseData, WARM_UP, true, stopAtException) if (phaseData.testName != WARM_UP) { printWarmUpTimings(phaseData.testName, warmUpStatInfosArray, metricChildren) @@ -243,8 +251,12 @@ class Stats( warmUpStatInfosArray.filterNotNull().map { it[ERROR_KEY] as? Throwable }.firstOrNull()?.let { throw it } } - private fun mainPhase(phaseData: PhaseData, metricChildren: MutableList): Array { - val statInfosArray = phase(phaseData, "") + private fun mainPhase( + phaseData: PhaseData, + metricChildren: MutableList, + stopAtException: Boolean, + ): Array { + val statInfosArray = phase(phaseData, "", stopAtException = stopAtException) statInfosArray.filterNotNull().map { it[ERROR_KEY] as? Throwable }.firstOrNull()?.let { convertStatInfoIntoMetrics( phaseData.testName, @@ -257,7 +269,12 @@ class Stats( return statInfosArray } - private fun phase(phaseData: PhaseData, phaseName: String, warmup: Boolean = false): Array { + private fun phase( + phaseData: PhaseData, + phaseName: String, + warmup: Boolean = false, + stopAtException: Boolean, + ): Array { val statInfosArray = Array(phaseData.iterations) { null } val testData = TestData(null, null) @@ -311,6 +328,9 @@ class Stats( } catch (t: Throwable) { logMessage(t) { "error at ${phaseData.testName}" } TeamCity.testFailed(name, error = t) + if (stopAtException) { + throw t + } } return statInfosArray } diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/common/AbstractWholeProjectPerformanceComparisonTest.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/common/AbstractWholeProjectPerformanceComparisonTest.kt index 06595c919ab..b2c1e03e5e7 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/common/AbstractWholeProjectPerformanceComparisonTest.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/common/AbstractWholeProjectPerformanceComparisonTest.kt @@ -5,7 +5,9 @@ package org.jetbrains.kotlin.idea.perf.common -import org.jetbrains.kotlin.idea.perf.* +import org.jetbrains.kotlin.idea.perf.AbstractPerformanceProjectsTest +import org.jetbrains.kotlin.idea.perf.Stats +import org.jetbrains.kotlin.idea.perf.WarmUpProject import org.jetbrains.kotlin.idea.perf.util.TeamCity import org.jetbrains.kotlin.idea.testFramework.ProjectOpenAction @@ -33,7 +35,7 @@ abstract class AbstractWholeProjectPerformanceComparisonTest : AbstractPerforman FILE_NAMES.NAME_RESOLUTION ) - filesToHighlight.forEach { file -> perfHighlightFileEmptyProfile(file, stats = stat) } + filesToHighlight.forEach { file -> perfHighlightFileEmptyProfile(file, stats = stat, stopAtException = true) } } } } @@ -50,7 +52,8 @@ abstract class AbstractWholeProjectPerformanceComparisonTest : AbstractPerforman insertString = "val a = l", highlightFileBeforeStartTyping = true, lookupElements = listOf("line"), - note = "in-method completion" + note = "in-method completion", + stopAtException = true, ) perfTypeAndAutocomplete( @@ -60,7 +63,8 @@ abstract class AbstractWholeProjectPerformanceComparisonTest : AbstractPerforman insertString = "\nval a = ", highlightFileBeforeStartTyping = true, lookupElements = listOf("processAssocTypeVariants"), - note = "top-level completion" + note = "top-level completion", + stopAtException = true, ) perfTypeAndAutocomplete( @@ -70,7 +74,8 @@ abstract class AbstractWholeProjectPerformanceComparisonTest : AbstractPerforman highlightFileBeforeStartTyping = true, insertString = "\nval a = s", lookupElements = listOf("scope"), - note = "in big method in big file completion" + note = "in big method in big file completion", + stopAtException = true, ) } }