diff --git a/build.gradle.kts b/build.gradle.kts index cca0a2be505..a589ec76fa4 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -781,9 +781,10 @@ tasks { } register("idea-plugin-performance-tests") { - dependsOn("dist") dependsOn( - ":idea:performanceTests:performanceTest" + "dist", + ":idea:performanceTests:performanceTest", + ":idea:performanceTests:aggregateResults" ) } diff --git a/idea/performanceTests/build.gradle.kts b/idea/performanceTests/build.gradle.kts index 4d528ad9c7d..a2270e1b3e0 100644 --- a/idea/performanceTests/build.gradle.kts +++ b/idea/performanceTests/build.gradle.kts @@ -147,5 +147,12 @@ projectTest(taskName = "wholeProjectsPerformanceTest") { } } +task("aggregateResults", JavaExec::class) { + main = "org.jetbrains.kotlin.idea.perf.util.AggregateResultsKt" + classpath = sourceSets["test"].runtimeClasspath + workingDir = rootDir + args(listOf(File(rootDir, "build"))) +} + testsJar() \ No newline at end of file diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt index 05f5bdb0f65..d5df7367d17 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt @@ -165,6 +165,7 @@ class Stats( statInfoArray: Array, printOnlyErrors: Boolean = false, metricChildren: MutableList, + warmUp: Boolean = false, attemptFn: (Int) -> String = { attempt -> "#$attempt" } ) { for (statInfoIndex in statInfoArray.withIndex()) { @@ -191,7 +192,15 @@ class Stats( } } } - metricChildren.add(Metric(attemptString, metricValue = durationMs, metrics = childrenMetrics)) + metricChildren.add( + Metric( + metricName = attemptString, + index = attempt, + warmUp = if (warmUp) true else null, + metricValue = durationMs, + metrics = childrenMetrics + ) + ) } } } @@ -200,7 +209,7 @@ class Stats( prefix: String, warmUpStatInfosArray: Array, metricChildren: MutableList - ) = convertStatInfoIntoMetrics(prefix, warmUpStatInfosArray, metricChildren = metricChildren) { attempt -> "warm-up #$attempt" } + ) = convertStatInfoIntoMetrics(prefix, warmUpStatInfosArray, warmUp = true, metricChildren = metricChildren) { attempt -> "warm-up #$attempt" } fun processTimings( prefix: String, @@ -224,6 +233,7 @@ class Stats( phaseData.testName, printOnlyErrors = true, statInfoArray = warmUpStatInfosArray, + warmUp = true, metricChildren = metricChildren ) { attempt -> "warm-up #$attempt" } } @@ -376,31 +386,9 @@ class Stats( metrics = it.metrics ?: emptyList() ) - if (benchmark.name == "ParameterNameAndType - TypeParameter") { - println(benchmark) - } - benchmark.writeJson() - with( - Benchmark( - agentName = agentName, - buildBranch = buildBranch, - buildId = buildId, - benchmark = name, - synthetic = true, - name = "geomMean", - buildTimestamp = simpleDateFormat.format(Date()) - ) - ) { - loadJson() - merge(benchmark) - - if (benchmark.name == "ParameterNameAndType - TypeParameter") { - println(this) - } - writeJson() - } + //rebuildGeomMean(agentName, buildBranch, buildId, simpleDateFormat, benchmark) } } finally { metric = null @@ -408,6 +396,31 @@ class Stats( //metrics.writeCSV(name, header) } + private fun rebuildGeomMean( + agentName: String?, + buildBranch: String?, + buildId: Int?, + simpleDateFormat: SimpleDateFormat, + benchmark: Benchmark + ) { + with( + Benchmark( + agentName = agentName, + buildBranch = buildBranch, + buildId = buildId, + benchmark = name, + synthetic = true, + name = "geomMean", + buildTimestamp = simpleDateFormat.format(Date()) + ) + ) { + loadJson() + merge(benchmark) + + writeJson() + } + } + companion object { const val TEST_KEY = "test" const val ERROR_KEY = "error" diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/Metric.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/Metric.kt index aaeb3296092..603cd1df1f8 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/Metric.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/Metric.kt @@ -13,6 +13,7 @@ import java.util.ArrayList @JsonInclude(JsonInclude.Include.NON_NULL) data class Benchmark( + val version: Int = 3, @set:JsonProperty("agentName") var agentName: String?, @set:JsonProperty("benchmark") @@ -21,6 +22,10 @@ data class Benchmark( var name: String? = null, @set:JsonProperty("synthetic") var synthetic: Boolean? = null, + @set:JsonProperty("index") + var index: Int? = null, + @set:JsonProperty("warmUp") + var warmUp: Boolean? = null, @set:JsonProperty("buildTimestamp") var buildTimestamp: String, @set:JsonProperty("buildBranch") @@ -51,8 +56,14 @@ data class Benchmark( private fun String?.escapeName() = this?.replace(Regex("[^A-Za-z0-9_]"), "_") - fun fileName(): String = - listOfNotNull(benchmark?.escapeName(), name?.escapeName(), buildId?.toString()).joinToString(separator = "_") + fun id(): String = + listOfNotNull( + benchmark?.escapeName(), + name?.escapeName(), + buildId?.toString(), + warmUp?.let { "warmUp" } ?: null, + index?.toString() + ).joinToString(separator = "_") fun cleanUp() { metrics?.forEach { it.cleanUp() } @@ -86,6 +97,10 @@ data class Benchmark( data class Metric( @set:JsonProperty("metricName") var metricName: String, + @set:JsonProperty("index") + var index: Int? = null, + @set:JsonProperty("warmUp") + var warmUp: Boolean? = null, @set:JsonProperty("legacyName") var legacyName: String? = null, @set:JsonProperty("metricValue") diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/StatsOutput.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/StatsOutput.kt index 1fa25a6d5db..a3efa8d535b 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/StatsOutput.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/StatsOutput.kt @@ -62,7 +62,7 @@ internal val kotlinJsonMapper = jacksonObjectMapper() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) .configure(SerializationFeature.INDENT_OUTPUT, true) -internal fun Benchmark.statsFile() = statsFile(fileName(), "json") +internal fun Benchmark.statsFile() = statsFile(id(), "json") internal fun Benchmark.writeJson() { val json = kotlinJsonMapper.writeValueAsString(this) @@ -74,6 +74,8 @@ internal fun Benchmark.writeJson() { } } +internal fun File.loadBenchmark(): Benchmark = kotlinJsonMapper.readValue(this, object : TypeReference() {}) + internal fun Benchmark.loadJson() { val statsFile = statsFile() if (statsFile.exists()) { diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/aggregateResults.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/aggregateResults.kt new file mode 100644 index 00000000000..fefce6700e6 --- /dev/null +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/aggregateResults.kt @@ -0,0 +1,60 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.idea.perf.util + +import java.io.File +import java.util.* + +fun main(args: Array) { + val argFile = File(args[0]) + val groupBy = argFile.listFiles() + .filter { it.length() > 0 && it.name.startsWith("stats-") && it.extension == "json" } + .groupBy { it.name.replace("stats-", "").split("_")[0] } + + groupBy.forEach { (k, v) -> + if (v.isEmpty()) return@forEach + + val benchmarks = v.map { it.loadBenchmark() } + + benchmarks.forEach { benchmark -> + benchmark.metrics.firstOrNull { it.metricName == "" }?.let { metric -> + metric.rawMetrics?.firstOrNull { it.warmUp == true && it.index == 0 }?.let { + val warmUpBenchmark = Benchmark( + agentName = benchmark.agentName, + buildBranch = benchmark.buildBranch, + buildId = benchmark.buildId, + benchmark = benchmark.benchmark, + name = benchmark.name, + warmUp = it.warmUp, + index = it.index, + hasError = it.hasError, + buildTimestamp = benchmark.buildTimestamp, + metrics = it.metrics ?: emptyList() + ) + warmUpBenchmark.writeJson() + } + } + } + + // build geom mean benchmark + val first = v.first() + val loadBenchmark = first.loadBenchmark() + val geomMeanBenchmark = Benchmark( + agentName = loadBenchmark.agentName, + buildBranch = loadBenchmark.buildBranch, + buildId = loadBenchmark.buildId, + benchmark = loadBenchmark.benchmark, + synthetic = true, + name = "geomMean", + buildTimestamp = loadBenchmark.buildTimestamp + ) + + benchmarks + .filter { it.synthetic != true && it.warmUp != true } + .forEach { geomMeanBenchmark.merge(it) } + geomMeanBenchmark.writeJson() + } +} \ No newline at end of file