Rebuild IDE performance tests output

format changes: added warmUp tag, index, benchmark format version
perform post processing: geom mean calc, extract warm-up benchmark

build.gradle.kts

@@ -781,9 +781,10 @@ tasks {
     }
 
     register("idea-plugin-performance-tests") {
-        dependsOn("dist")
         dependsOn(
-            ":idea:performanceTests:performanceTest"
+            "dist",
+            ":idea:performanceTests:performanceTest",
+            ":idea:performanceTests:aggregateResults"
         )
     }
 

idea/performanceTests/build.gradle.kts

@@ -147,5 +147,12 @@ projectTest(taskName = "wholeProjectsPerformanceTest") {
     }
 }
 
+task("aggregateResults", JavaExec::class) {
+    main = "org.jetbrains.kotlin.idea.perf.util.AggregateResultsKt"
+    classpath = sourceSets["test"].runtimeClasspath
+    workingDir = rootDir
+    args(listOf(File(rootDir, "build")))
+}
+
 
 testsJar()

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt

@@ -165,6 +165,7 @@ class Stats(
         statInfoArray: Array<StatInfos>,
         printOnlyErrors: Boolean = false,
         metricChildren: MutableList<Metric>,
+        warmUp: Boolean = false,
         attemptFn: (Int) -> String = { attempt -> "#$attempt" }
     ) {
         for (statInfoIndex in statInfoArray.withIndex()) {
@@ -191,7 +192,15 @@ class Stats(
                         }
                     }
                 }
-                metricChildren.add(Metric(attemptString, metricValue = durationMs, metrics = childrenMetrics))
+                metricChildren.add(
+                    Metric(
+                        metricName = attemptString,
+                        index = attempt,
+                        warmUp = if (warmUp) true else null,
+                        metricValue = durationMs,
+                        metrics = childrenMetrics
+                    )
+                )
             }
         }
     }
@@ -200,7 +209,7 @@ class Stats(
         prefix: String,
         warmUpStatInfosArray: Array<StatInfos>,
         metricChildren: MutableList<Metric>
-    ) = convertStatInfoIntoMetrics(prefix, warmUpStatInfosArray, metricChildren = metricChildren) { attempt -> "warm-up #$attempt" }
+    ) = convertStatInfoIntoMetrics(prefix, warmUpStatInfosArray, warmUp = true, metricChildren = metricChildren) { attempt -> "warm-up #$attempt" }
 
     fun processTimings(
         prefix: String,
@@ -224,6 +233,7 @@ class Stats(
                 phaseData.testName,
                 printOnlyErrors = true,
                 statInfoArray = warmUpStatInfosArray,
+                warmUp = true,
                 metricChildren = metricChildren
             ) { attempt -> "warm-up #$attempt" }
         }
@@ -376,31 +386,9 @@ class Stats(
                     metrics = it.metrics ?: emptyList()
                 )
 
-                if (benchmark.name == "ParameterNameAndType - TypeParameter") {
-                    println(benchmark)
-                }
-
                 benchmark.writeJson()
 
-                with(
+                //rebuildGeomMean(agentName, buildBranch, buildId, simpleDateFormat, benchmark)
-                    Benchmark(
-                        agentName = agentName,
-                        buildBranch = buildBranch,
-                        buildId = buildId,
-                        benchmark = name,
-                        synthetic = true,
-                        name = "geomMean",
-                        buildTimestamp = simpleDateFormat.format(Date())
-                    )
-                ) {
-                    loadJson()
-                    merge(benchmark)
-
-                    if (benchmark.name == "ParameterNameAndType - TypeParameter") {
-                        println(this)
-                    }
-                    writeJson()
-                }
             }
         } finally {
             metric = null
@@ -408,6 +396,31 @@ class Stats(
         //metrics.writeCSV(name, header)
     }
 
+    private fun rebuildGeomMean(
+        agentName: String?,
+        buildBranch: String?,
+        buildId: Int?,
+        simpleDateFormat: SimpleDateFormat,
+        benchmark: Benchmark
+    ) {
+        with(
+            Benchmark(
+                agentName = agentName,
+                buildBranch = buildBranch,
+                buildId = buildId,
+                benchmark = name,
+                synthetic = true,
+                name = "geomMean",
+                buildTimestamp = simpleDateFormat.format(Date())
+            )
+        ) {
+            loadJson()
+            merge(benchmark)
+
+            writeJson()
+        }
+    }
+
     companion object {
         const val TEST_KEY = "test"
         const val ERROR_KEY = "error"

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/Metric.kt

@@ -13,6 +13,7 @@ import java.util.ArrayList
 
 @JsonInclude(JsonInclude.Include.NON_NULL)
 data class Benchmark(
+    val version: Int = 3,
     @set:JsonProperty("agentName")
     var agentName: String?,
     @set:JsonProperty("benchmark")
@@ -21,6 +22,10 @@ data class Benchmark(
     var name: String? = null,
     @set:JsonProperty("synthetic")
     var synthetic: Boolean? = null,
+    @set:JsonProperty("index")
+    var index: Int? = null,
+    @set:JsonProperty("warmUp")
+    var warmUp: Boolean? = null,
     @set:JsonProperty("buildTimestamp")
     var buildTimestamp: String,
     @set:JsonProperty("buildBranch")
@@ -51,8 +56,14 @@ data class Benchmark(
 
     private fun String?.escapeName() = this?.replace(Regex("[^A-Za-z0-9_]"), "_")
 
-    fun fileName(): String =
+    fun id(): String =
-        listOfNotNull(benchmark?.escapeName(), name?.escapeName(), buildId?.toString()).joinToString(separator = "_")
+        listOfNotNull(
+            benchmark?.escapeName(),
+            name?.escapeName(),
+            buildId?.toString(),
+            warmUp?.let { "warmUp" } ?: null,
+            index?.toString()
+        ).joinToString(separator = "_")
 
     fun cleanUp() {
         metrics?.forEach { it.cleanUp() }
@@ -86,6 +97,10 @@ data class Benchmark(
 data class Metric(
     @set:JsonProperty("metricName")
     var metricName: String,
+    @set:JsonProperty("index")
+    var index: Int? = null,
+    @set:JsonProperty("warmUp")
+    var warmUp: Boolean? = null,
     @set:JsonProperty("legacyName")
     var legacyName: String? = null,
     @set:JsonProperty("metricValue")

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/StatsOutput.kt

@@ -62,7 +62,7 @@ internal val kotlinJsonMapper = jacksonObjectMapper()
     .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
     .configure(SerializationFeature.INDENT_OUTPUT, true)
 
-internal fun Benchmark.statsFile() = statsFile(fileName(), "json")
+internal fun Benchmark.statsFile() = statsFile(id(), "json")
 
 internal fun Benchmark.writeJson() {
     val json = kotlinJsonMapper.writeValueAsString(this)
@@ -74,6 +74,8 @@ internal fun Benchmark.writeJson() {
     }
 }
 
+internal fun File.loadBenchmark(): Benchmark = kotlinJsonMapper.readValue(this, object : TypeReference<Benchmark>() {})
+
 internal fun Benchmark.loadJson() {
     val statsFile = statsFile()
     if (statsFile.exists()) {

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/aggregateResults.kt

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.idea.perf.util
+
+import java.io.File
+import java.util.*
+
+fun main(args: Array<String>) {
+    val argFile = File(args[0])
+    val groupBy = argFile.listFiles()
+        .filter { it.length() > 0 && it.name.startsWith("stats-") && it.extension == "json" }
+        .groupBy { it.name.replace("stats-", "").split("_")[0] }
+
+    groupBy.forEach { (k, v) ->
+        if (v.isEmpty()) return@forEach
+
+        val benchmarks = v.map { it.loadBenchmark() }
+
+        benchmarks.forEach { benchmark ->
+            benchmark.metrics.firstOrNull { it.metricName == "" }?.let { metric ->
+                metric.rawMetrics?.firstOrNull { it.warmUp == true && it.index == 0 }?.let {
+                    val warmUpBenchmark = Benchmark(
+                        agentName = benchmark.agentName,
+                        buildBranch = benchmark.buildBranch,
+                        buildId = benchmark.buildId,
+                        benchmark = benchmark.benchmark,
+                        name = benchmark.name,
+                        warmUp = it.warmUp,
+                        index = it.index,
+                        hasError = it.hasError,
+                        buildTimestamp = benchmark.buildTimestamp,
+                        metrics = it.metrics ?: emptyList()
+                    )
+                    warmUpBenchmark.writeJson()
+                }
+            }
+        }
+
+        // build geom mean benchmark
+        val first = v.first()
+        val loadBenchmark = first.loadBenchmark()
+        val geomMeanBenchmark = Benchmark(
+            agentName = loadBenchmark.agentName,
+            buildBranch = loadBenchmark.buildBranch,
+            buildId = loadBenchmark.buildId,
+            benchmark = loadBenchmark.benchmark,
+            synthetic = true,
+            name = "geomMean",
+            buildTimestamp = loadBenchmark.buildTimestamp
+        )
+
+        benchmarks
+            .filter { it.synthetic != true && it.warmUp != true }
+            .forEach { geomMeanBenchmark.merge(it) }
+        geomMeanBenchmark.writeJson()
+    }
+}