azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

FIR IDE: stop on fist exception in project perf tests

Browse Source
This commit is contained in:
Ilya Kirillov
2021-02-25 17:46:12 +01:00
parent 211ddd307c
commit 5485c372e8
4 changed files with 60 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceProjectsTest.kt
+15 -7
View File
@@ -243,7 +243,8 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
lookupElements: List<String>,
typeAfterMarker: Boolean = true,
revertChangesAtTheEnd: Boolean = true,
note: String = ""
note: String = "",
stopAtException: Boolean = false,
) = perfTypeAndAutocomplete(
project = project(),
stats = stats,
@@ -255,7 +256,8 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
lookupElements = lookupElements,
typeAfterMarker = typeAfterMarker,
revertChangesAtTheEnd = revertChangesAtTheEnd,
note = note
note = note,
stopAtException = stopAtException,
)
fun perfTypeAndAutocomplete(
@@ -269,7 +271,8 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
lookupElements: List<String>,
typeAfterMarker: Boolean = true,
revertChangesAtTheEnd: Boolean = true,
note: String = ""
note: String = "",
stopAtException: Boolean = false,
) {
assertTrue("lookupElements has to be not empty", lookupElements.isNotEmpty())
perfTypeAndDo(
@@ -297,7 +300,8 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
assertTrue("'$lookupElement' has to be present in items $items", items.contains(lookupElement))
}
},
revertChangesAtTheEnd = revertChangesAtTheEnd
revertChangesAtTheEnd = revertChangesAtTheEnd,
stopAtException = stopAtException,
)
}
@@ -353,7 +357,8 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
setupAfterTypingBlock: (Fixture) -> Unit,
testBlock: (Fixture) -> V,
tearDownCheck: (Fixture, V?) -> Unit,
revertChangesAtTheEnd: Boolean
revertChangesAtTheEnd: Boolean,
stopAtException: Boolean = false,
) {
openFixture(project, fileName).use { fixture ->
val editor = fixture.editor
@@ -403,6 +408,7 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
commitAllDocuments()
}
}
stopAtException(stopAtException)
}
}
}
@@ -583,8 +589,8 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
note: String = ""
): List<HighlightInfo> = perfHighlightFile(project(), name, stats, tools = tools, note = note)
protected fun perfHighlightFileEmptyProfile(name: String, stats: Stats): List<HighlightInfo> =
perfHighlightFile(project(), name, stats, tools = emptyArray(), note = "empty profile")
protected fun perfHighlightFileEmptyProfile(name: String, stats: Stats, stopAtException: Boolean = false): List<HighlightInfo> =
perfHighlightFile(project(), name, stats, tools = emptyArray(), note = "empty profile", stopAtException = stopAtException)
protected fun perfHighlightFile(
project: Project,
@@ -595,6 +601,7 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
warmUpIterations: Int = 3,
iterations: Int = 10,
checkStability: Boolean = true,
stopAtException: Boolean = false,
filenameSimplifier: (String) -> String = ::simpleFilename
): List<HighlightInfo> {
val profileManager = ProjectInspectionProfileManager.getInstance(project)
@@ -629,6 +636,7 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() {
PsiManager.getInstance(project).dropPsiCaches()
}
profilerConfig.enabled = true
stopAtException(stopAtException)
}
highlightInfos
}
idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerfTestBuilder.kt
+7 -1
View File
@@ -17,6 +17,7 @@ class PerfTestBuilder<SV, TV> {
private var tearDown: (TestData<SV, TV>) -> Unit = { }
private var checkStability: Boolean = true
internal var profilerConfig: ProfilerConfig = ProfilerConfig()
private var stopAtException: Boolean = false
internal fun run() {
stats.perfTest(
@@ -26,7 +27,8 @@ class PerfTestBuilder<SV, TV> {
setUp = setUp,
test = test,
tearDown = tearDown,
checkStability = checkStability
checkStability = checkStability,
stopAtException = stopAtException,
)
}
@@ -65,6 +67,10 @@ class PerfTestBuilder<SV, TV> {
fun checkStability(checkStability: Boolean) {
this.checkStability = checkStability
}
fun stopAtException(stopAtException: Boolean) {
this.stopAtException = stopAtException
}
}
fun <SV, TV> performanceTest(initializer: PerfTestBuilder<SV, TV>.() -> Unit) {
idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt
+28 -8
View File
@@ -95,7 +95,8 @@ class Stats(
setUp: (TestData<SV, TV>) -> Unit = { },
test: (TestData<SV, TV>) -> Unit,
tearDown: (TestData<SV, TV>) -> Unit = { },
checkStability: Boolean = true
checkStability: Boolean = true,
stopAtException: Boolean = false,
) {
val warmPhaseData = PhaseData(
iterations = warmUpIterations,
@@ -114,8 +115,8 @@ class Stats(
val block = {
val metricChildren = mutableListOf<Metric>()
try {
warmUpPhase(warmPhaseData, metricChildren)
val statInfoArray = mainPhase(mainPhaseData, metricChildren)
warmUpPhase(warmPhaseData, metricChildren, stopAtException)
val statInfoArray = mainPhase(mainPhaseData, metricChildren, stopAtException)
assertEquals(iterations, statInfoArray.size)
if (testName != WARM_UP) {
@@ -150,6 +151,9 @@ class Stats(
}
} catch (e: Throwable) {
processTimings(testName, emptyArray(), metricChildren)
if (stopAtException) {
throw e
}
}
}
@@ -225,8 +229,12 @@ class Stats(
}
}
private fun <SV, TV> warmUpPhase(phaseData: PhaseData<SV, TV>, metricChildren: MutableList<Metric>) {
val warmUpStatInfosArray = phase(phaseData, WARM_UP, true)
private fun <SV, TV> warmUpPhase(
phaseData: PhaseData<SV, TV>,
metricChildren: MutableList<Metric>,
stopAtException: Boolean,
) {
val warmUpStatInfosArray = phase(phaseData, WARM_UP, true, stopAtException)
if (phaseData.testName != WARM_UP) {
printWarmUpTimings(phaseData.testName, warmUpStatInfosArray, metricChildren)
@@ -243,8 +251,12 @@ class Stats(
warmUpStatInfosArray.filterNotNull().map { it[ERROR_KEY] as? Throwable }.firstOrNull()?.let { throw it }
}
private fun <SV, TV> mainPhase(phaseData: PhaseData<SV, TV>, metricChildren: MutableList<Metric>): Array<StatInfos> {
val statInfosArray = phase(phaseData, "")
private fun <SV, TV> mainPhase(
phaseData: PhaseData<SV, TV>,
metricChildren: MutableList<Metric>,
stopAtException: Boolean,
): Array<StatInfos> {
val statInfosArray = phase(phaseData, "", stopAtException = stopAtException)
statInfosArray.filterNotNull().map { it[ERROR_KEY] as? Throwable }.firstOrNull()?.let {
convertStatInfoIntoMetrics(
phaseData.testName,
@@ -257,7 +269,12 @@ class Stats(
return statInfosArray
}
private fun <SV, TV> phase(phaseData: PhaseData<SV, TV>, phaseName: String, warmup: Boolean = false): Array<StatInfos> {
private fun <SV, TV> phase(
phaseData: PhaseData<SV, TV>,
phaseName: String,
warmup: Boolean = false,
stopAtException: Boolean,
): Array<StatInfos> {
val statInfosArray = Array<StatInfos>(phaseData.iterations) { null }
val testData = TestData<SV, TV>(null, null)
@@ -311,6 +328,9 @@ class Stats(
} catch (t: Throwable) {
logMessage(t) { "error at ${phaseData.testName}" }
TeamCity.testFailed(name, error = t)
if (stopAtException) {
throw t
}
}
return statInfosArray
}
idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/common/AbstractWholeProjectPerformanceComparisonTest.kt
+10 -5
View File
@@ -5,7 +5,9 @@
package org.jetbrains.kotlin.idea.perf.common
import org.jetbrains.kotlin.idea.perf.*
import org.jetbrains.kotlin.idea.perf.AbstractPerformanceProjectsTest
import org.jetbrains.kotlin.idea.perf.Stats
import org.jetbrains.kotlin.idea.perf.WarmUpProject
import org.jetbrains.kotlin.idea.perf.util.TeamCity
import org.jetbrains.kotlin.idea.testFramework.ProjectOpenAction
@@ -33,7 +35,7 @@ abstract class AbstractWholeProjectPerformanceComparisonTest : AbstractPerforman
FILE_NAMES.NAME_RESOLUTION
)
filesToHighlight.forEach { file -> perfHighlightFileEmptyProfile(file, stats = stat) }
filesToHighlight.forEach { file -> perfHighlightFileEmptyProfile(file, stats = stat, stopAtException = true) }
}
}
}
@@ -50,7 +52,8 @@ abstract class AbstractWholeProjectPerformanceComparisonTest : AbstractPerforman
insertString = "val a = l",
highlightFileBeforeStartTyping = true,
lookupElements = listOf("line"),
note = "in-method completion"
note = "in-method completion",
stopAtException = true,
)
perfTypeAndAutocomplete(
@@ -60,7 +63,8 @@ abstract class AbstractWholeProjectPerformanceComparisonTest : AbstractPerforman
insertString = "\nval a = ",
highlightFileBeforeStartTyping = true,
lookupElements = listOf("processAssocTypeVariants"),
note = "top-level completion"
note = "top-level completion",
stopAtException = true,
)
perfTypeAndAutocomplete(
@@ -70,7 +74,8 @@ abstract class AbstractWholeProjectPerformanceComparisonTest : AbstractPerforman
highlightFileBeforeStartTyping = true,
insertString = "\nval a = s",
lookupElements = listOf("scope"),
note = "in big method in big file completion"
note = "in big method in big file completion",
stopAtException = true,
)
}
}