diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceAddImportTest.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceAddImportTest.kt index aca8224d51d..c86c66d2877 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceAddImportTest.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceAddImportTest.kt @@ -22,12 +22,6 @@ abstract class AbstractPerformanceAddImportTest : AbstractPerformanceImportTest( @JvmStatic val stats: Stats = Stats("add-import") - - init { - // there is no @AfterClass for junit3.8 - Runtime.getRuntime().addShutdownHook(Thread(Runnable { stats.close() })) - } - } override fun stats(): Stats = stats diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionHandlerTests.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionHandlerTests.kt index 06b76cdd0ac..37c814e4d66 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionHandlerTests.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionHandlerTests.kt @@ -9,6 +9,8 @@ import com.intellij.application.options.CodeStyle import com.intellij.codeInsight.completion.CompletionType import com.intellij.openapi.application.runWriteAction import com.intellij.openapi.util.io.FileUtil +import com.intellij.testFramework.RunAll +import com.intellij.util.ThrowableRunnable import org.jetbrains.kotlin.idea.completion.test.ExpectedCompletionUtils import org.jetbrains.kotlin.idea.completion.test.configureWithExtraFile import org.jetbrains.kotlin.idea.completion.test.handlers.AbstractCompletionHandlerTest.Companion.CODE_STYLE_SETTING_PREFIX @@ -36,11 +38,6 @@ abstract class AbstractPerformanceCompletionHandlerTests( companion object { @JvmStatic val statsMap: MutableMap = mutableMapOf() - - init { - // there is no @AfterClass for junit3.8 - Runtime.getRuntime().addShutdownHook(Thread(Runnable { statsMap.values.forEach { it.close() } })) - } } private fun stats(): Stats { @@ -52,7 +49,10 @@ abstract class AbstractPerformanceCompletionHandlerTests( override fun tearDown() { commitAllDocuments() - super.tearDown() + RunAll( + ThrowableRunnable { super.tearDown() }, + ThrowableRunnable { statsMap.values.forEach(Stats::flush) } + ).run() } protected open fun doPerfTest(unused: String) { diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionIncrementalResolveTest.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionIncrementalResolveTest.kt index 994ff2d608c..ea2d39b6c49 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionIncrementalResolveTest.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionIncrementalResolveTest.kt @@ -9,6 +9,8 @@ import com.intellij.codeInsight.completion.CompletionType import com.intellij.codeInsight.lookup.LookupElement import com.intellij.openapi.application.runWriteAction import com.intellij.openapi.util.text.StringUtil +import com.intellij.testFramework.RunAll +import com.intellij.util.ThrowableRunnable import org.jetbrains.kotlin.idea.KotlinFileType import org.jetbrains.kotlin.idea.completion.CompletionBindingContextProvider import org.jetbrains.kotlin.idea.perf.Stats.Companion.WARM_UP @@ -33,11 +35,6 @@ abstract class AbstractPerformanceCompletionIncrementalResolveTest : KotlinLight @JvmStatic val stats: Stats = Stats("completion-incremental") - - init { - // there is no @AfterClass for junit3.8 - Runtime.getRuntime().addShutdownHook(Thread(Runnable { stats.close() })) - } } override fun getProjectDescriptor() = KotlinWithJdkAndRuntimeLightProjectDescriptor.INSTANCE @@ -53,7 +50,10 @@ abstract class AbstractPerformanceCompletionIncrementalResolveTest : KotlinLight override fun tearDown() { commitAllDocuments() - super.tearDown() + RunAll( + ThrowableRunnable { super.tearDown() }, + ThrowableRunnable { stats.flush() } + ).run() } private fun doWarmUpPerfTest() { diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceHighlightingTest.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceHighlightingTest.kt index 06ff83ac535..75dbdac6f17 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceHighlightingTest.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceHighlightingTest.kt @@ -7,7 +7,9 @@ package org.jetbrains.kotlin.idea.perf import com.intellij.codeInsight.daemon.impl.HighlightInfo import com.intellij.openapi.application.runWriteAction +import com.intellij.testFramework.RunAll import com.intellij.testFramework.fixtures.impl.CodeInsightTestFixtureImpl.ensureIndexesUpToDate +import com.intellij.util.ThrowableRunnable import org.jetbrains.kotlin.idea.KotlinFileType import org.jetbrains.kotlin.idea.perf.Stats.Companion.WARM_UP import org.jetbrains.kotlin.idea.test.KotlinLightCodeInsightFixtureTestCase @@ -25,10 +27,6 @@ abstract class AbstractPerformanceHighlightingTest : KotlinLightCodeInsightFixtu @JvmStatic val stats: Stats = Stats("highlight") - init { - // there is no @AfterClass for junit3.8 - Runtime.getRuntime().addShutdownHook(Thread(Runnable { stats.close() })) - } } override fun setUp() { @@ -42,7 +40,10 @@ abstract class AbstractPerformanceHighlightingTest : KotlinLightCodeInsightFixtu override fun tearDown() { commitAllDocuments() - super.tearDown() + RunAll( + ThrowableRunnable { super.tearDown() }, + ThrowableRunnable { stats.flush() } + ).run() } private fun doWarmUpPerfTest() { diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceImportTest.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceImportTest.kt index c83de055439..298b5913e8e 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceImportTest.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceImportTest.kt @@ -9,6 +9,8 @@ import com.intellij.application.options.CodeStyle import com.intellij.openapi.application.runWriteAction import com.intellij.psi.PsiDocumentManager import com.intellij.testFramework.LightProjectDescriptor +import com.intellij.testFramework.RunAll +import com.intellij.util.ThrowableRunnable import junit.framework.TestCase import org.jetbrains.kotlin.descriptors.ClassDescriptor import org.jetbrains.kotlin.descriptors.DeclarationDescriptor @@ -30,6 +32,13 @@ abstract class AbstractPerformanceImportTest : KotlinLightCodeInsightFixtureTest protected abstract fun stats(): Stats + override fun tearDown() { + RunAll( + ThrowableRunnable { super.tearDown() }, + ThrowableRunnable { stats().flush() } + ).run() + } + protected fun doPerfTest(unused: String) { val testName = getTestName(false) diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceJavaToKotlinCopyPasteConversionTest.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceJavaToKotlinCopyPasteConversionTest.kt index a84994df334..b1842bceb58 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceJavaToKotlinCopyPasteConversionTest.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceJavaToKotlinCopyPasteConversionTest.kt @@ -9,6 +9,8 @@ import com.intellij.ide.highlighter.JavaFileType import com.intellij.openapi.actionSystem.IdeActions import com.intellij.openapi.application.runWriteAction import com.intellij.psi.PsiDocumentManager +import com.intellij.testFramework.RunAll +import com.intellij.util.ThrowableRunnable import org.jetbrains.kotlin.idea.KotlinFileType import org.jetbrains.kotlin.idea.configuration.ExperimentalFeatures import org.jetbrains.kotlin.idea.conversion.copy.AbstractJavaToKotlinCopyPasteConversionTest @@ -28,11 +30,6 @@ abstract class AbstractPerformanceJavaToKotlinCopyPasteConversionTest(private va val warmedUp: Array = arrayOf(false, false) val stats: Array = arrayOf(Stats("old j2k"), Stats("new j2k")) - - init { - // there is no @AfterClass for junit3.8 - Runtime.getRuntime().addShutdownHook(Thread(Runnable { stats.forEach { it.close() } })) - } } override fun setUp() { @@ -47,6 +44,14 @@ abstract class AbstractPerformanceJavaToKotlinCopyPasteConversionTest(private va } } + override fun tearDown() { + RunAll( + ThrowableRunnable { super.tearDown() }, + ThrowableRunnable { stats.forEach(Stats::flush) } + ).run() + + } + private fun doWarmUpPerfTest() { performanceTest { name(WARM_UP) diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceLiteralKotlinToKotlinCopyPasteTest.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceLiteralKotlinToKotlinCopyPasteTest.kt index d239eabc522..a3ab7d07df4 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceLiteralKotlinToKotlinCopyPasteTest.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceLiteralKotlinToKotlinCopyPasteTest.kt @@ -9,6 +9,8 @@ import com.intellij.openapi.actionSystem.IdeActions import com.intellij.openapi.application.runWriteAction import com.intellij.openapi.fileEditor.ex.FileEditorManagerEx import com.intellij.psi.PsiFile +import com.intellij.testFramework.RunAll +import com.intellij.util.ThrowableRunnable import org.jetbrains.kotlin.idea.AbstractCopyPasteTest import org.jetbrains.kotlin.idea.perf.Stats.Companion.WARM_UP import org.jetbrains.kotlin.idea.test.KotlinWithJdkAndRuntimeLightProjectDescriptor @@ -27,11 +29,6 @@ abstract class AbstractPerformanceLiteralKotlinToKotlinCopyPasteTest : AbstractC @JvmStatic val stats: Stats = Stats("Literal-k2k-CopyPaste") - - init { - // there is no @AfterClass for junit3.8 - Runtime.getRuntime().addShutdownHook(Thread(Runnable { stats.close() })) - } } override fun getProjectDescriptor() = KotlinWithJdkAndRuntimeLightProjectDescriptor.INSTANCE @@ -45,6 +42,13 @@ abstract class AbstractPerformanceLiteralKotlinToKotlinCopyPasteTest : AbstractC } } + override fun tearDown() { + RunAll( + ThrowableRunnable { super.tearDown() }, + ThrowableRunnable { stats.flush() } + ).run() + } + private fun doWarmUpPerfTest() { val fileEditorManager = FileEditorManagerEx.getInstance(project) performanceTest, Unit> { diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceTypingIndentationTest.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceTypingIndentationTest.kt index 0b65c16f9d2..17f0e1ae88e 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceTypingIndentationTest.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceTypingIndentationTest.kt @@ -7,10 +7,10 @@ package org.jetbrains.kotlin.idea.perf import com.intellij.application.options.CodeStyle import com.intellij.openapi.actionSystem.IdeActions -import com.intellij.openapi.application.impl.NonBlockingReadActionImpl import com.intellij.openapi.util.io.FileUtil import com.intellij.testFramework.EditorTestUtil -import com.intellij.util.ui.UIUtil +import com.intellij.testFramework.RunAll +import com.intellij.util.ThrowableRunnable import org.jetbrains.kotlin.formatter.FormatSettingsUtil import org.jetbrains.kotlin.idea.test.KotlinLightCodeInsightFixtureTestCase import org.jetbrains.kotlin.idea.testFramework.dispatchAllInvocationEvents @@ -27,6 +27,13 @@ abstract class AbstractPerformanceTypingIndentationTest : KotlinLightCodeInsight val stats: Stats = Stats("typing-indentation") } + override fun tearDown() { + RunAll( + ThrowableRunnable { super.tearDown() }, + ThrowableRunnable { stats.flush() } + ).run() + } + protected fun doPerfTest(unused: String) { val testName = getTestName(false) diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerformanceProjectsTest.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerformanceProjectsTest.kt index b3e31a8f778..994073910db 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerformanceProjectsTest.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerformanceProjectsTest.kt @@ -9,6 +9,8 @@ import com.intellij.codeInsight.daemon.impl.HighlightInfo import com.intellij.lang.annotation.AnnotationHolder import com.intellij.openapi.project.Project import com.intellij.psi.PsiElement +import com.intellij.testFramework.RunAll +import com.intellij.util.ThrowableRunnable import org.jetbrains.kotlin.idea.core.script.ScriptConfigurationManager import org.jetbrains.kotlin.idea.highlighter.KotlinPsiChecker import org.jetbrains.kotlin.idea.highlighter.KotlinPsiCheckerAndHighlightingUpdater @@ -35,11 +37,6 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() { @JvmStatic val timer: AtomicLong = AtomicLong() - init { - // there is no @AfterClass for junit3.8 - Runtime.getRuntime().addShutdownHook(Thread { hwStats.close() }) - } - fun resetTimestamp() { timer.set(0) } @@ -54,6 +51,13 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() { warmUp.warmUp(this) } + override fun tearDown() { + RunAll( + ThrowableRunnable { super.tearDown() }, + ThrowableRunnable { hwStats.flush() } + ).run() + } + fun testHelloWorldProject() { suite("Hello world project") { myProject = perfOpenProject(stats = hwStats) { @@ -337,14 +341,18 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() { profilerConfig.enabled = true } + val metricChildren = mutableListOf() + extraStats.printWarmUpTimings( "annotator", - extraTimingsNs.take(warmUpIterations).toTypedArray() + extraTimingsNs.take(warmUpIterations).toTypedArray(), + metricChildren ) - extraStats.appendTimings( + extraStats.processTimings( "annotator", - extraTimingsNs.drop(warmUpIterations).toTypedArray() + extraTimingsNs.drop(warmUpIterations).toTypedArray(), + metricChildren ) } } diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt index 184abdbf817..cd041b0fe66 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt @@ -9,12 +9,14 @@ import org.jetbrains.kotlin.idea.perf.WholeProjectPerformanceTest.Companion.nsTo import org.jetbrains.kotlin.idea.perf.profilers.* import org.jetbrains.kotlin.idea.perf.util.TeamCity import org.jetbrains.kotlin.idea.perf.util.logMessage -import org.jetbrains.kotlin.idea.testFramework.suggestOsNeutralFileName import org.jetbrains.kotlin.util.PerformanceCounter import java.io.* -import kotlin.system.measureNanoTime import java.lang.ref.WeakReference +import java.text.SimpleDateFormat +import java.util.* +import kotlin.collections.HashMap import kotlin.math.* +import kotlin.system.measureNanoTime import kotlin.system.measureTimeMillis import kotlin.test.assertEquals @@ -25,44 +27,36 @@ class Stats( private val profilerConfig: ProfilerConfig = ProfilerConfig(), private val header: Array = arrayOf("Name", "ValueMS", "StdDev"), private val acceptanceStabilityLevel: Int = 25 -) : Closeable { +) : AutoCloseable { private val perfTestRawDataMs = mutableListOf() - private val statsFile: File = File(pathToResource("stats${statFilePrefix()}.csv")).absoluteFile - - private val statsOutput: BufferedWriter + private val metrics = mutableListOf() init { - statsOutput = statsFile.bufferedWriter() - - statsOutput.appendLine(header.joinToString()) - PerformanceCounter.setTimeCounterEnabled(true) } - private fun statFilePrefix() = if (name.isNotEmpty()) "-${plainname()}" else "" - - private fun plainname() = suggestOsNeutralFileName(name) - - private fun pathToResource(resource: String) = "build/$resource" - - private fun append(id: String, statInfosArray: Array) { + private fun calcAndProcessMetrics(id: String, statInfosArray: Array, rawMetricChildren: MutableList) { val timingsMs = toTimingsMs(statInfosArray) val calcMean = calcMean(timingsMs) - for (v in listOf( - Triple("mean", "", calcMean.mean.toLong()), - Triple("stdDev", " stdDev", calcMean.stdDev.toLong()), - Triple("geomMean", " geomMean", calcMean.geomMean.toLong()) - )) { - val n = "$id : ${v.first}" + val metricChildren = mutableListOf() + val metric = Metric(id, calcMean.mean.toLong(), measurementError = calcMean.stdDev.toLong(), children = metricChildren) + metrics.add(metric) - TeamCity.test(n, durationMs = v.third, includeStats = false) { - TeamCity.statValue("$id${v.second}", v.third) - } - } + metricChildren.add( + Metric( + "", calcMean.mean.toLong(), + measurementError = calcMean.stdDev.toLong(), + childrenName = "raw_metrics", children = rawMetricChildren + ) + ) + metricChildren.add(Metric("mean", calcMean.mean.toLong())) + // keep geomMean for bwc + metricChildren.add(Metric(GEOM_MEAN, calcMean.geomMean.toLong())) + metricChildren.add(Metric("std_dev", calcMean.stdDev.toLong())) statInfosArray.filterNotNull() .map { it.keys } @@ -76,14 +70,15 @@ class Stats( val mean = statInfoMean.mean.toLong() val shortName = if (perfCounterName.endsWith(": time")) n.removeSuffix(": time") else null + val metricShortName = if (perfCounterName.endsWith(": time")) perfCounterName.removeSuffix(": time") else perfCounterName - TeamCity.test(shortName, durationMs = mean) { - TeamCity.statValue(n, mean) - } + metricChildren.add(Metric(": $metricShortName", mean)) + + TeamCity.test(shortName, durationMs = mean) {} } perfTestRawDataMs.addAll(timingsMs.toList()) - append(arrayOf(id, calcMean.mean, calcMean.stdDev)) + metric.writeTeamCityStats(name) } private fun toTimingsMs(statInfosArray: Array) = @@ -107,19 +102,6 @@ class Stats( data class Mean(val mean: Double, val stdDev: Double, val geomMean: Double) - private fun append(values: Array) { - require(values.size == header.size) { "Expected ${header.size} values, actual ${values.size} values" } - with(statsOutput) { - appendLine(values.joinToString { it.toString() }) - flush() - } - } - - fun append(file: String, id: String, nanoTime: Long) { - val ms = nanoTime.nsToMs - append(arrayOf(file, id, ms)) - } - fun perfTest( testName: String, warmUpIterations: Int = 5, @@ -145,13 +127,12 @@ class Stats( tearDown = tearDown ) val block = { - warmUpPhase(warmPhaseData) - val statInfoArray = mainPhase(mainPhaseData) + val metricChildren = mutableListOf() + warmUpPhase(warmPhaseData, metricChildren) + val statInfoArray = mainPhase(mainPhaseData, metricChildren) assertEquals(iterations, statInfoArray.size) if (testName != WARM_UP) { - appendTimings(testName, statInfoArray) - // do not estimate stability for warm-up if (!testName.contains(WARM_UP)) { val calcMean = calcMean(statInfoArray) @@ -168,11 +149,13 @@ class Stats( } TeamCity.test(stabilityName, errorDetails = error, includeStats = false) { - TeamCity.statValue(stabilityName, stabilityPercentage) + metricChildren.add(Metric("stability", stabilityPercentage)) } } + + processTimings(testName, statInfoArray, metricChildren) } else { - printTimings(testName, printOnlyErrors = true, statInfoArray = statInfoArray) + convertStatInfoIntoMetrics(testName, printOnlyErrors = true, statInfoArray = statInfoArray, metricChildren = metricChildren) } } @@ -183,70 +166,80 @@ class Stats( } } - private fun printTimings( + private fun convertStatInfoIntoMetrics( prefix: String, statInfoArray: Array, printOnlyErrors: Boolean = false, + metricChildren: MutableList, attemptFn: (Int) -> String = { attempt -> "#$attempt" } ) { for (statInfoIndex in statInfoArray.withIndex()) { val attempt = statInfoIndex.index val statInfo = statInfoIndex.value ?: continue - val n = "$name: $prefix ${attemptFn(attempt)}" + val attemptString = attemptFn(attempt) + val s = "$prefix $attemptString" + val n = "$name: $s" + val childrenMetrics = mutableListOf() val t = statInfo[ERROR_KEY] as? Throwable if (t != null) { TeamCity.test(n, errors = listOf(t)) {} } else if (!printOnlyErrors) { - TeamCity.test(n, durationMs = (statInfo[TEST_KEY] as Long).nsToMs) { + val durationMs = (statInfo[TEST_KEY] as Long).nsToMs + TeamCity.test(n, durationMs = durationMs) { for ((k, v) in statInfo) { if (k == TEST_KEY) continue - TeamCity.statValue("$n $k", v) (v as? Number)?.let { + childrenMetrics.add(Metric(k, v)) TeamCity.metadata(n, k, it) } } } + metricChildren.add(Metric(attemptString, durationMs, children = childrenMetrics)) } } } fun printWarmUpTimings( prefix: String, - warmUpStatInfosArray: Array - ) = printTimings(prefix, warmUpStatInfosArray) { attempt -> "warm-up #$attempt" } + warmUpStatInfosArray: Array, + metricChildren: MutableList + ) = convertStatInfoIntoMetrics(prefix, warmUpStatInfosArray, metricChildren = metricChildren) { attempt -> "warm-up #$attempt" } - fun appendTimings( + fun processTimings( prefix: String, - statInfosArray: Array + statInfosArray: Array, + metricChildren: MutableList ) { - printTimings(prefix, statInfosArray) - append("$name: $prefix", statInfosArray) + convertStatInfoIntoMetrics(prefix, statInfosArray, metricChildren = metricChildren) + calcAndProcessMetrics(prefix, statInfosArray, metricChildren) } - private fun warmUpPhase(phaseData: PhaseData) { + private fun warmUpPhase(phaseData: PhaseData, metricChildren: MutableList) { val warmUpStatInfosArray = phase(phaseData, WARM_UP, true) if (phaseData.testName != WARM_UP) { - printWarmUpTimings(phaseData.testName, warmUpStatInfosArray) + printWarmUpTimings(phaseData.testName, warmUpStatInfosArray, metricChildren) } else { - printTimings( + convertStatInfoIntoMetrics( phaseData.testName, printOnlyErrors = true, - statInfoArray = warmUpStatInfosArray + statInfoArray = warmUpStatInfosArray, + metricChildren = metricChildren ) { attempt -> "warm-up #$attempt" } } warmUpStatInfosArray.filterNotNull().map { it[ERROR_KEY] as? Throwable }.firstOrNull()?.let { throw it } } - private fun mainPhase(phaseData: PhaseData): Array { + private fun mainPhase(phaseData: PhaseData, metricChildren: MutableList): Array { val statInfosArray = phase(phaseData, "") statInfosArray.filterNotNull().map { it[ERROR_KEY] as? Throwable }.firstOrNull()?.let { - printTimings( + convertStatInfoIntoMetrics( phaseData.testName, printOnlyErrors = true, - statInfoArray = statInfosArray + statInfoArray = statInfosArray, + metricChildren = metricChildren ) throw it } @@ -265,7 +258,9 @@ class Stats( testData.reset() triggerGC(attempt) - val setUpMillis = measureTimeMillis { phaseData.setUp(testData) } + val setUpMillis = measureTimeMillis { + phaseData.setUp(testData) + } val attemptName = "${phaseData.testName} #$attempt" logMessage { "$attemptName setup took $setUpMillis ms" } @@ -315,7 +310,7 @@ class Stats( profilerConfig: ProfilerConfig ): PhaseProfiler { profilerConfig.name = "$testName${if (phaseName.isEmpty()) "" else "-"+phaseName}" - profilerConfig.path = pathToResource("profile/${plainname()}") + profilerConfig.path = pathToResource("profile/${plainname(name)}") val profilerHandler = if (profilerConfig.enabled && !profilerConfig.warmup) ProfilerHandler.getInstance(profilerConfig) else @@ -341,13 +336,47 @@ class Stats( private fun geomMean(data: List) = exp(data.fold(0.0, { mul, next -> mul + ln(1.0 * next) }) / data.size) override fun close() { + flush() + } + + fun flush() { + val children = metrics.toMutableList() + + val properties = mutableMapOf() + properties[BENCHMARK] = name + + val simpleDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ") + simpleDateFormat.timeZone = TimeZone.getTimeZone("UTC") +// properties["build_timestamp"] = simpleDateFormat.format(Date()) +// properties["build_id"] = 87015694 +// properties["build_branch"] = "rr/perf/json-output" +// properties["agent_name"] = "kotlin-linux-perf-unit879" + + System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE")?.let { teamcityConfig -> + val buildProperties = Properties() + buildProperties.load(FileInputStream(teamcityConfig)) + + properties["build.timestamp"] = simpleDateFormat.format(Date()) + for ((name, key) in + mapOf( + "build_id" to "teamcity.build.id", + "build_branch" to "teamcity.build.branch", + "agent_name" to "agent.name", + )) { + val property = buildProperties.getProperty(key) + properties[name] = if (name == "build_id") property.toLong() else property + } + } if (perfTestRawDataMs.isNotEmpty()) { val geomMeanMs = geomMean(perfTestRawDataMs.toList()).toLong() - TeamCity.statValue("$name geomMean", geomMeanMs) - append(arrayOf("$name geomMean", geomMeanMs, 0)) + Metric(GEOM_MEAN, geomMeanMs).writeTeamCityStats(name) + properties[GEOM_MEAN] = geomMeanMs } - statsOutput.flush() - statsOutput.close() + + val metric = Metric(name, null, children = children, properties = properties) + + metric.writeJson() + metrics.writeCSV(name, header) } companion object { @@ -355,6 +384,8 @@ class Stats( const val ERROR_KEY = "error" const val WARM_UP = "warm-up" + const val GEOM_MEAN = "geomMean" + const val BENCHMARK = "benchmark" inline fun runAndMeasure(note: String, block: () -> Unit) { val openProjectMillis = measureTimeMillis { @@ -366,6 +397,15 @@ class Stats( } +data class Metric( + val name: String, + val value: Number?, + val measurementError: Number? = null, + val childrenName: String = "metrics", + val children: MutableList = mutableListOf(), + val properties: Map? = null +) + data class PhaseData( val iterations: Int, val testName: String, diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/StatsOutput.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/StatsOutput.kt new file mode 100644 index 00000000000..03dbbe5f897 --- /dev/null +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/StatsOutput.kt @@ -0,0 +1,113 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.idea.perf + +import org.jetbrains.kotlin.idea.perf.util.TeamCity +import org.jetbrains.kotlin.idea.testFramework.suggestOsNeutralFileName +import java.io.BufferedWriter +import java.io.File + +internal fun List.writeCSV(name: String, header: Array) { + fun Metric.append(prefix: String, output: BufferedWriter) { + val s = "$prefix ${this.name}".trim() + output.appendLine("$s,${value ?: ""},") + children.forEach { + it.append(s, output) + } + } + + val statsFile = statsFile(name, "csv") + statsFile.bufferedWriter().use { output -> + output.appendLine(header.joinToString()) + forEach { it.append("$name:", output) } + output.flush() + } +} + +internal fun Metric.writeTeamCityStats(name: String, rawMeasurementName: String = "raw_metrics", rawMetrics: Boolean = false) { + fun Metric.append(prefix: String, depth: Int) { + val s = if (this.name.isEmpty()) { + prefix + } else { + if (depth == 0 && this.name != Stats.GEOM_MEAN) "$prefix: ${this.name}" else "$prefix ${this.name}" + }.trim() + value?.let { + TeamCity.statValue(s, it) + } + for (childIndex in children.withIndex()) { + if (!rawMetrics && childrenName == rawMeasurementName && childIndex.index > 0) break + childIndex.value.append(s, depth + 1) + } + } + + append(name, 0) +} + +internal fun Metric.writeJson() { + val statsFile = statsFile(name, "json") + statsFile.bufferedWriter().use { output -> + output.appendLine(toJson("")) + output.flush() + } +} + +private fun statsFile(name: String, extension: String) = + File(pathToResource("stats${statFilePrefix(name)}.$extension")).absoluteFile + +private fun List.toJson(prefix: String) = joinToString(separator = ",", prefix = "[", postfix = "]") { it.toJson(prefix) } + +private fun String.jsonValue() = replace("\"", "\\\"") + +private fun Metric.toJson(prefix: String): String = + buildString { + append("{") + val nam = if (name.isEmpty()) "_value" else name + val nm = nam.jsonValue() + val s = if (name == Stats.GEOM_MEAN) { + "${prefix.substring(0, prefix.length - 1)} $name".trim() + } else { + "$prefix $name${if (prefix.isEmpty()) ":" else ""}".trim() + } + + var commaRequired = false + if (value != null) { + append("\"metric_name\":\"").append(nm).append("\"") + append(",\"metric_value\":").append(value) + measurementError?.let { + append(",\"metric_error\":").append(it) + } + append(",\"legacy_name\":").append("\"${s.jsonValue()}\"") + commaRequired = true + } + + if (properties != null) { + for ((k, v) in properties) { + if (commaRequired) append(",") + append("\"").append(k).append("\":") + if (v is Number) { + append(v) + } else { + append("\"").append(v).append("\"") + } + commaRequired = true + } + } + + if (children.isNotEmpty()) { + val cnm = childrenName.jsonValue() + if (commaRequired) append(",") + append("\"").append(cnm).append("\"") + append(":").append(children.toJson(s)) + } + append("}") + } + + +internal fun pathToResource(resource: String) = "build/$resource" + +internal fun statFilePrefix(name: String) = if (name.isNotEmpty()) "-${plainname(name)}" else "" + +internal fun plainname(name: String) = suggestOsNeutralFileName(name) diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/WholeProjectPerformanceTest.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/WholeProjectPerformanceTest.kt index 3f82ff8440a..f13edda8757 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/WholeProjectPerformanceTest.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/WholeProjectPerformanceTest.kt @@ -101,7 +101,7 @@ abstract class WholeProjectPerformanceTest : DaemonAnalyzerTestCase(), WholeProj fun appendInspectionResult(file: String, id: String, nanoTime: Long) { totals.merge(id, nanoTime) { a, b -> a + b } - perfStats.append(file, id, nanoTime) + //perfStats.append(file, id, nanoTime) } TeamCity.suite("TotalPerFile") { diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/logging.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/logging.kt index eefc0d19bed..4d66ac089cb 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/logging.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/logging.kt @@ -63,7 +63,7 @@ object TeamCity { } inline fun testStarted(testName: String) { - message { "testStarted name='$testName'" } + message { "testStarted name='$testName' captureStandardOutput='true'" } } inline fun metadata(testName: String, name: String, value: Number) {