Run benchmarks in the same suite

This should reduce variation in results, especially when overall time is
 quite low.

libraries/tools/gradle/regression-benchmark-templates/src/main/kotlin/org/jetbrains/kotlin/gradle/benchmark/BenchmarkTemplate.kt

@@ -36,9 +36,11 @@ abstract class BenchmarkTemplate(
     private val projectName: String,
     private val projectGitUrl: String,
     private val gitCommitSha: String,
+    private val stableKotlinVersions: String
 ) {
     private val workingDir = File(args.first())
     val currentKotlinVersion: String = args[1]
+    private val kotlinVersions = setOf(stableKotlinVersions, currentKotlinVersion)
     private val gradleProfilerDir = workingDir.resolve("gradle-profiler")
     private val projectRepoDir = workingDir.resolve(projectName)
     private val scenariosDir = workingDir.resolve("scenarios")
@@ -46,23 +48,21 @@ abstract class BenchmarkTemplate(
 
     private val gitOperationsPrinter = TextProgressMonitor()
 
-    fun <T : InputStream> runAllBenchmarks(
+    fun <T : InputStream> runBenchmarks(
+        repoPatch: (() -> Pair<String, T>)?,
         suite: ScenarioSuite,
-        benchmarks: Map<BenchmarkName, (() -> Pair<String, T>)?>
     ) {
         printStartingMessage()
         downloadGradleProfilerIfNotAvailable()
         checkoutRepository()
 
-        val results = benchmarks.map { (name, patchFile) ->
+        repoReset()
-            repoReset()
+        repoPatch?.let {
-            patchFile?.let {
+            val (patchName, patch) = it()
-                val (patchName, patch) = it()
+            repoApplyPatch(patchName, patch)
-                repoApplyPatch(patchName, patch)
-            }
-            runBenchmark(suite, name)
         }
-        aggregateBenchmarkResults(*results.toTypedArray())
+        val result = runBenchmark(suite)
+        aggregateBenchmarkResults(result)
 
         printEndingMessage()
     }
@@ -138,15 +138,14 @@ abstract class BenchmarkTemplate(
             .call()
     }
 
-    fun runBenchmark(
+    private fun runBenchmark(
         scenarioSuite: ScenarioSuite,
-        benchmarkName: BenchmarkName,
+        @Suppress("UNUSED_PARAMETER") dryRun: Boolean = false
-        dryRun: Boolean = false
     ): BenchmarkResult {
-        println("Staring benchmark $benchmarkName")
+        println("Staring benchmark $projectName")
-        val normalizedBenchmarkName = benchmarkName.normalizeTitle
+        val normalizedBenchmarkName = projectName.normalizeTitle
         if (!scenariosDir.exists()) scenariosDir.mkdirs()
-        val scenarioFile = scenariosDir.resolve("${projectName}_$normalizedBenchmarkName.scenario")
+        val scenarioFile = scenariosDir.resolve("$normalizedBenchmarkName.scenario")
         scenarioSuite.writeTo(scenarioFile)
         val benchmarkOutputDir = benchmarkOutputsDir
             .resolve(projectName)
@@ -186,60 +185,73 @@ abstract class BenchmarkTemplate(
             throw BenchmarkFailedException(profilerProcess.exitValue())
         }
 
-        println("Benchmark $benchmarkName has ended")
+        println("Benchmark $projectName has ended")
         return BenchmarkResult(
-            benchmarkName,
+            projectName,
             benchmarkOutputDir.resolve("benchmark.csv")
         )
     }
 
-    fun aggregateBenchmarkResults(vararg benchmarkResults: BenchmarkResult) {
+    // Not working as intended due to this bug: https://github.com/gradle/gradle-profiler/issues/317
+    fun aggregateBenchmarkResults(benchmarkResult: BenchmarkResult) {
         println("Aggregating benchmark results...")
-        val sortedResults = benchmarkResults.sortedBy { it.name }
+        val results = DataFrame
-        val results = sortedResults
+            .readCSV(benchmarkResult.result, duplicate = true)
-            .map { result ->
+            .drop {
-                DataFrame
+                // Removing unused rows
-                    .readCSV(result.result)
+                it["scenario"] in listOf("version", "tasks") ||
-                    .flipColumnsWithRows()
+                        it["scenario"].toString().startsWith("warm-up build")
-                    .remove("version", "tasks") // removing unused columns
-                    .remove { nameContains("warm-up build") } // removing warm-up times
-                    .add("median build time") { // squashing build times into median build time
-                        valuesOf<Int>().median()
-                    }
-                    .remove { nameContains("measured build") } // removing iterations results
-                    .groupBy("scenario").aggregate { // merging configuration and build times into one row
-                        first { it.values().contains("task start") }["median build time"] into "tasks start median time"
-                        first { it.values().contains("total execution time") }["median build time"] into "execution median time"
-                    }
-                    .add("benchmark") { result.name }
             }
-            .concat()
+            .flipColumnsWithRows()
-            .groupBy("scenario").aggregate { // Merging scenarios from different benchmarks into one row
+            .add("median build time") { // squashing build times into median build time
-                forEach { row ->
+                valuesOf<Int>().median()
-                    row["tasks start median time"] into "Configuration: ${row["benchmark"]}"
+            }
-                    row["execution median time"] into "Execution: ${row["benchmark"]}"
+            .remove { nameContains("measured build") } // removing iterations results
+            .groupBy("scenario").aggregate { // merging configuration and build times into one row
+                first { it.values().contains("task start") }["median build time"] into "tasks start median time"
+                first { it.values().contains("total execution time") }["median build time"] into "execution median time"
+            }
+            .groupBy {
+                expr {
+                    val scenarioName = get("scenario").toString()
+                    if (scenarioName.endsWith(currentKotlinVersion)) {
+                        scenarioName.substringBefore(currentKotlinVersion)
+                    } else {
+                        scenarioName.substringBefore(stableKotlinVersions)
+                    }
                 }
             }
-            .sortBy("scenario")
+            .aggregate {
-            .rename("scenario" to "Scenario")
+                forEach { row ->
+                    val version = if (row["scenario"].toString().endsWith(stableKotlinVersions)) {
+                        stableKotlinVersions
+                    } else {
+                        currentKotlinVersion
+                    }
+                    row["tasks start median time"] into "Configuration: $version"
+                    row["execution median time"] into "Execution: $version"
+                }
+            }
+            .rename { col(0) }.into("Scenario")
+            .sortBy("Scenario")
             .reorderColumnsBy {
                 // "Scenario" column should always be in the first place
                 if (it.name() == "Scenario") "0000Scenario" else it.name()
             }
             .insert("Configuration diff from stable release") {
-                val stableReleaseConfiguration = column<Int>("Configuration: ${sortedResults.first().name}").getValue(this)
+                val stableReleaseConfiguration = column<Int>("Configuration: $stableKotlinVersions").getValue(this)
-                val currentReleaseConfiguration = column<Int>("Configuration: ${sortedResults.last().name}").getValue(this)
+                val currentReleaseConfiguration = column<Int>("Configuration: $currentKotlinVersion").getValue(this)
                 val percent = currentReleaseConfiguration * 100 / stableReleaseConfiguration
                 "${percent}%"
             }
-            .after("Configuration: ${sortedResults.last().name}")
+            .after("Configuration: $currentKotlinVersion")
             .insert("Execution diff from stable release") {
-                val stableReleaseConfiguration = column<Int>("Execution: ${sortedResults.first().name}").getValue(this)
+                val stableReleaseConfiguration = column<Int>("Execution: $stableKotlinVersions").getValue(this)
-                val currentReleaseConfiguration = column<Int>("Execution: ${sortedResults.last().name}").getValue(this)
+                val currentReleaseConfiguration = column<Int>("Execution: $currentKotlinVersion").getValue(this)
                 val percent = currentReleaseConfiguration * 100 / stableReleaseConfiguration
                 "${percent}%"
             }
-            .after("Execution: ${sortedResults.last().name}")
+            .after("Execution: $currentKotlinVersion")
 
         println("Benchmark results:")
         println(results.print(borders = true))
@@ -320,28 +332,41 @@ abstract class BenchmarkTemplate(
     }
 
     private fun ScenarioSuite.writeTo(output: File) {
+        val allScenarios = scenarios
+            .map {scenario ->
+                kotlinVersions.map { scenario to it }
+            }
+            .flatten()
+            .map { (scenario, version) ->
+                "${scenario.title} $version".normalizeTitle
+            }
         output.writeText(
             """
-            |default-scenarios = [${scenarios.joinToString { "\"${it.title.normalizeTitle}\"" }}]
+            |default-scenarios = [${allScenarios.joinToString { "\"$it\"" }}]
             |
             |${scenarios.joinToString(separator = "\n") { it.write() }}
             """.trimMargin()
         )
     }
 
-    private fun Scenario.write(): String =
+    private fun Scenario.write(): String = kotlinVersions.joinToString(separator = "\n") { kotlinVersion ->
+        val finalGradleArgs = gradleArgs
+            .plus("-PkotlinVersion=$kotlinVersion")
+            .joinToString { "\"$it\"" }
         """
-        |${title.normalizeTitle} {
+        |${"$title $kotlinVersion".normalizeTitle} {
-        |    title = "$title"
+        |    title = "$title $kotlinVersion"
         |    warm-ups = $warmups
         |    iterations = $iterations
         |    tasks = [${tasks.joinToString { "\"$it\"" }}]
-        |    ${if (gradleArgs.isNotEmpty()) "gradle-args = [${gradleArgs.joinToString { "\"$it\"" }}]" else ""}
+        |    gradle-args = [$finalGradleArgs]
         |    ${if (cleanupTasks.isNotEmpty()) "cleanup-tasks = [${cleanupTasks.joinToString { "\"$it\"" }}]" else ""}
         |    ${if (applyAbiChange.isNotEmpty()) "apply-abi-change-to = [${applyAbiChange.joinToString { "\"$it\"" }}]" else ""}
         |    ${if (applyAndroidResourceValueChange.isNotEmpty()) "apply-android-resource-value-change-to = [${applyAndroidResourceValueChange.joinToString { "\"$it\"" }}]" else ""}
         |}
+        |
         """.trimMargin()
+    }
 
     private val String.dropLeadingDir: String get() = substringAfter('/')
     private val String.normalizeTitle: String get() = lowercase().replace(" ", "_")
@@ -355,7 +380,11 @@ abstract class BenchmarkTemplate(
                 columnName == "scenario" -> DataColumn.createValueColumn(
                     columnName,
                     columns().map {
-                        it.name.dropLastWhile { it.isDigit() }
+                        if (it.name.contains(currentKotlinVersion)) {
+                            it.name.substringBefore(currentKotlinVersion) + currentKotlinVersion
+                        } else {
+                            it.name.substringBefore(stableKotlinVersions) + stableKotlinVersions
+                        }
                     }.drop(1)
                 )
                 columnName == "version" -> DataColumn.createValueColumn(
@@ -411,6 +440,7 @@ abstract class BenchmarkTemplate(
 
 typealias BenchmarkName = String
 
+@Suppress("Unused")
 class BenchmarkFailedException(exitCode: Int) : Exception(
     "Benchmark process was not finished successfully with $exitCode exit code."
 )
@@ -425,7 +455,8 @@ class BenchmarkResult(
 annotation class BenchmarkProject(
     val name: String,
     val gitUrl: String,
-    val gitCommitSha: String
+    val gitCommitSha: String,
+    val stableKotlinVersion: String,
 )
 
 class BenchmarkScriptDefinition : ScriptCompilationConfiguration(
@@ -451,9 +482,10 @@ class BenchmarkEvaluationConfiguration : ScriptEvaluationConfiguration(
             val name: String = compileConf[benchmarkProjectName]!!
             val gitUrl: String = compileConf[benchmarkGitUrl]!!
             val gitCommitSha: String = compileConf[benchmarkGitCommitSha]!!
+            val stableKotlinVersion: String = compileConf[benchmarkStableKotlinVersion]!!
 
             context.evaluationConfiguration.with evalConf@{
-                constructorArgs(name, gitUrl, gitCommitSha)
+                constructorArgs(name, gitUrl, gitCommitSha, stableKotlinVersion)
             }.asSuccess()
         }
     }
@@ -474,10 +506,12 @@ internal class BenchmarkScriptConfigurator : RefineScriptCompilationConfiguratio
             data[benchmarkProjectName] = benchmarkProject.name
             data[benchmarkGitUrl] = benchmarkProject.gitUrl
             data[benchmarkGitCommitSha] = benchmarkProject.gitCommitSha
+            data[benchmarkStableKotlinVersion] = benchmarkProject.stableKotlinVersion
         }.asSuccess()
     }
 }
 
 private val benchmarkProjectName by PropertiesCollection.key<String>()
 private val benchmarkGitUrl by PropertiesCollection.key<String>()
-private val benchmarkGitCommitSha by PropertiesCollection.key<String>()
+private val benchmarkGitCommitSha by PropertiesCollection.key<String>()
+private val benchmarkStableKotlinVersion by PropertiesCollection.key<String>()

libraries/tools/gradle/regression-benchmarks/benchmarkScripts/duckduckgo.benchmark.kts

@@ -4,25 +4,21 @@
 @file:BenchmarkProject(
     name = "duckduckgo",
     gitUrl = "https://github.com/duckduckgo/Android.git",
-    gitCommitSha = "18e230fcefbefb4317c1fe128b4539a2315e7c0a"
+    gitCommitSha = "18e230fcefbefb4317c1fe128b4539a2315e7c0a",
+    stableKotlinVersion = "1.7.20",
 )
 
 import java.io.File
 
-val stableReleasePatch = {
+val repoPatch = {
-    "duckduckgo-kotlin-1.7.20.patch" to File("benchmarkScripts/files/duckduckgo-kotlin-1.7.20.patch")
-        .readText()
-        .byteInputStream()
-}
-
-val currentReleasePatch = {
     "duckduckgo-kotlin-current.patch" to File("benchmarkScripts/files/duckduckgo-kotlin-current.patch")
         .readText()
         .run { replace("<kotlin_version>", currentKotlinVersion) }
         .byteInputStream()
 }
 
-runAllBenchmarks(
+runBenchmarks(
+    repoPatch,
     suite {
         scenario {
             title = "Clean build"
@@ -86,9 +82,5 @@ runAllBenchmarks(
             iterations = 20
             runTasks(":app:assemblePlayDebug")
         }
-    },
+    }
-    mapOf(
-        "1.7.20" to stableReleasePatch,
-        "1.8.0" to currentReleasePatch
-    )
 )

libraries/tools/gradle/regression-benchmarks/benchmarkScripts/files/graphql-kotlin-1.7.20.patch

@@ -1,13 +0,0 @@
-diff --git a/gradle.properties b/gradle.properties
-index a1796af97..43609273a 100644
---- a/gradle.properties
-+++ b/gradle.properties
-@@ -15,7 +15,7 @@ org.gradle.jvmargs=-Xmx2g -XX:+HeapDumpOnOutOfMemoryError
-
- # dependencies
- kotlinJvmVersion = 1.8
--kotlinVersion = 1.6.21
-+kotlinVersion = 1.7.20
- kotlinCoroutinesVersion = 1.6.4
- # kotlinx-serialization 1.3.3 calls Kotlin 1.7 API
- kotlinxSerializationVersion = 1.3.2

libraries/tools/gradle/regression-benchmarks/benchmarkScripts/graphql-kotlin.benchmark.kts

@@ -9,24 +9,21 @@
 @file:BenchmarkProject(
     name = "graphql-kotlin",
     gitUrl = "https://github.com/ExpediaGroup/graphql-kotlin.git",
-    gitCommitSha = "fd1e9063f3aae144e099cdcfa69a4416fa434fb2"
+    gitCommitSha = "fd1e9063f3aae144e099cdcfa69a4416fa434fb2",
+    stableKotlinVersion = "1.7.20",
 )
 
 import java.io.File
 
-val stableReleasePatch = {
+val repoPatch = {
-    "graphql-kotlin-1.7.20.patch" to File("benchmarkScripts/files/graphql-kotlin-1.7.20.patch")
+    "graphql-kotlin-current.patch" to File("benchmarkScripts/files/graphql-kotlin-repo.patch")
-        .readText()
-        .byteInputStream()
-}
-val currentReleasePatch = {
-    "graphql-kotlin-current.patch" to File("benchmarkScripts/files/graphql-kotlin-current.patch")
         .readText()
         .run { replace("<kotlin_version>", currentKotlinVersion) }
         .byteInputStream()
 }
 
-runAllBenchmarks(
+runBenchmarks(
+    repoPatch,
     suite {
         scenario {
             title = "Spring server clean build"
@@ -91,9 +88,5 @@ runAllBenchmarks(
             iterations = 20
             runTasks("assemble")
         }
-    },
+    }
-    mapOf(
-        "1.7.20" to stableReleasePatch,
-        "1.8.0" to currentReleasePatch
-    )
 )

libraries/tools/gradle/regression-benchmarks/benchmarkScripts/kvision.benchmark.kts

@@ -9,19 +9,21 @@
 @file:BenchmarkProject(
     name = "kvision",
     gitUrl = "https://github.com/rjaros/kvision.git",
-    gitCommitSha = "3fe69bf6db9a3650b026630d857862f3cee6485b"
+    gitCommitSha = "3fe69bf6db9a3650b026630d857862f3cee6485b",
+    stableKotlinVersion = "1.7.20",
 )
 
 import java.io.File
 
-val currentReleasePatch = {
+val repoPatch = {
     "kvision-kotlin-current.patch" to File("benchmarkScripts/files/kvision-kotlin-current.patch")
         .readText()
         .run { replace("<kotlin_version>", currentKotlinVersion) }
         .byteInputStream()
 }
 
-runAllBenchmarks(
+runBenchmarks(
+    repoPatch,
     suite {
         scenario {
             title = "Build Js IR clean build"
@@ -86,9 +88,5 @@ runAllBenchmarks(
             iterations = 20
             runTasks("jsIrJar")
         }
-    },
+    }
-    mapOf(
-        "1.7.20" to null,
-        "1.8.0" to currentReleasePatch
-    )
 )