Run benchmarks in the same suite

This should reduce variation in results, especially when overall time is quite low.
2023-01-26 19:02:29 +01:00
parent ace1bc76a1
commit 3f4c824a23
6 changed files with 111 additions and 107 deletions
@@ -36,9 +36,11 @@ abstract class BenchmarkTemplate(
    private val projectName: String,
    private val projectGitUrl: String,
    private val gitCommitSha: String,
+    private val stableKotlinVersions: String
 ) {
    private val workingDir = File(args.first())
    val currentKotlinVersion: String = args[1]
+    private val kotlinVersions = setOf(stableKotlinVersions, currentKotlinVersion)
    private val gradleProfilerDir = workingDir.resolve("gradle-profiler")
    private val projectRepoDir = workingDir.resolve(projectName)
    private val scenariosDir = workingDir.resolve("scenarios")
@@ -46,23 +48,21 @@ abstract class BenchmarkTemplate(

    private val gitOperationsPrinter = TextProgressMonitor()

-    fun <T : InputStream> runAllBenchmarks(
+    fun <T : InputStream> runBenchmarks(
+        repoPatch: (() -> Pair<String, T>)?,
        suite: ScenarioSuite,
-        benchmarks: Map<BenchmarkName, (() -> Pair<String, T>)?>
    ) {
        printStartingMessage()
        downloadGradleProfilerIfNotAvailable()
        checkoutRepository()

-        val results = benchmarks.map { (name, patchFile) ->
-            repoReset()
-            patchFile?.let {
-                val (patchName, patch) = it()
-                repoApplyPatch(patchName, patch)
-            }
-            runBenchmark(suite, name)
+        repoReset()
+        repoPatch?.let {
+            val (patchName, patch) = it()
+            repoApplyPatch(patchName, patch)
        }
-        aggregateBenchmarkResults(*results.toTypedArray())
+        val result = runBenchmark(suite)
+        aggregateBenchmarkResults(result)

        printEndingMessage()
    }
@@ -138,15 +138,14 @@ abstract class BenchmarkTemplate(
            .call()
    }

-    fun runBenchmark(
+    private fun runBenchmark(
        scenarioSuite: ScenarioSuite,
-        benchmarkName: BenchmarkName,
-        dryRun: Boolean = false
+        @Suppress("UNUSED_PARAMETER") dryRun: Boolean = false
    ): BenchmarkResult {
-        println("Staring benchmark $benchmarkName")
-        val normalizedBenchmarkName = benchmarkName.normalizeTitle
+        println("Staring benchmark $projectName")
+        val normalizedBenchmarkName = projectName.normalizeTitle
        if (!scenariosDir.exists()) scenariosDir.mkdirs()
-        val scenarioFile = scenariosDir.resolve("${projectName}_$normalizedBenchmarkName.scenario")
+        val scenarioFile = scenariosDir.resolve("$normalizedBenchmarkName.scenario")
        scenarioSuite.writeTo(scenarioFile)
        val benchmarkOutputDir = benchmarkOutputsDir
            .resolve(projectName)
@@ -186,60 +185,73 @@ abstract class BenchmarkTemplate(
            throw BenchmarkFailedException(profilerProcess.exitValue())
        }

-        println("Benchmark $benchmarkName has ended")
+        println("Benchmark $projectName has ended")
        return BenchmarkResult(
-            benchmarkName,
+            projectName,
            benchmarkOutputDir.resolve("benchmark.csv")
        )
    }

-    fun aggregateBenchmarkResults(vararg benchmarkResults: BenchmarkResult) {
+    // Not working as intended due to this bug: https://github.com/gradle/gradle-profiler/issues/317
+    fun aggregateBenchmarkResults(benchmarkResult: BenchmarkResult) {
        println("Aggregating benchmark results...")
-        val sortedResults = benchmarkResults.sortedBy { it.name }
-        val results = sortedResults
-            .map { result ->
-                DataFrame
-                    .readCSV(result.result)
-                    .flipColumnsWithRows()
-                    .remove("version", "tasks") // removing unused columns
-                    .remove { nameContains("warm-up build") } // removing warm-up times
-                    .add("median build time") { // squashing build times into median build time
-                        valuesOf<Int>().median()
-                    }
-                    .remove { nameContains("measured build") } // removing iterations results
-                    .groupBy("scenario").aggregate { // merging configuration and build times into one row
-                        first { it.values().contains("task start") }["median build time"] into "tasks start median time"
-                        first { it.values().contains("total execution time") }["median build time"] into "execution median time"
-                    }
-                    .add("benchmark") { result.name }
+        val results = DataFrame
+            .readCSV(benchmarkResult.result, duplicate = true)
+            .drop {
+                // Removing unused rows
+                it["scenario"] in listOf("version", "tasks") ||
+                        it["scenario"].toString().startsWith("warm-up build")
            }
-            .concat()
-            .groupBy("scenario").aggregate { // Merging scenarios from different benchmarks into one row
-                forEach { row ->
-                    row["tasks start median time"] into "Configuration: ${row["benchmark"]}"
-                    row["execution median time"] into "Execution: ${row["benchmark"]}"
+            .flipColumnsWithRows()
+            .add("median build time") { // squashing build times into median build time
+                valuesOf<Int>().median()
+            }
+            .remove { nameContains("measured build") } // removing iterations results
+            .groupBy("scenario").aggregate { // merging configuration and build times into one row
+                first { it.values().contains("task start") }["median build time"] into "tasks start median time"
+                first { it.values().contains("total execution time") }["median build time"] into "execution median time"
+            }
+            .groupBy {
+                expr {
+                    val scenarioName = get("scenario").toString()
+                    if (scenarioName.endsWith(currentKotlinVersion)) {
+                        scenarioName.substringBefore(currentKotlinVersion)
+                    } else {
+                        scenarioName.substringBefore(stableKotlinVersions)
+                    }
                }
            }
-            .sortBy("scenario")
-            .rename("scenario" to "Scenario")
+            .aggregate {
+                forEach { row ->
+                    val version = if (row["scenario"].toString().endsWith(stableKotlinVersions)) {
+                        stableKotlinVersions
+                    } else {
+                        currentKotlinVersion
+                    }
+                    row["tasks start median time"] into "Configuration: $version"
+                    row["execution median time"] into "Execution: $version"
+                }
+            }
+            .rename { col(0) }.into("Scenario")
+            .sortBy("Scenario")
            .reorderColumnsBy {
                // "Scenario" column should always be in the first place
                if (it.name() == "Scenario") "0000Scenario" else it.name()
            }
            .insert("Configuration diff from stable release") {
-                val stableReleaseConfiguration = column<Int>("Configuration: ${sortedResults.first().name}").getValue(this)
-                val currentReleaseConfiguration = column<Int>("Configuration: ${sortedResults.last().name}").getValue(this)
+                val stableReleaseConfiguration = column<Int>("Configuration: $stableKotlinVersions").getValue(this)
+                val currentReleaseConfiguration = column<Int>("Configuration: $currentKotlinVersion").getValue(this)
                val percent = currentReleaseConfiguration * 100 / stableReleaseConfiguration
                "${percent}%"
            }
-            .after("Configuration: ${sortedResults.last().name}")
+            .after("Configuration: $currentKotlinVersion")
            .insert("Execution diff from stable release") {
-                val stableReleaseConfiguration = column<Int>("Execution: ${sortedResults.first().name}").getValue(this)
-                val currentReleaseConfiguration = column<Int>("Execution: ${sortedResults.last().name}").getValue(this)
+                val stableReleaseConfiguration = column<Int>("Execution: $stableKotlinVersions").getValue(this)
+                val currentReleaseConfiguration = column<Int>("Execution: $currentKotlinVersion").getValue(this)
                val percent = currentReleaseConfiguration * 100 / stableReleaseConfiguration
                "${percent}%"
            }
-            .after("Execution: ${sortedResults.last().name}")
+            .after("Execution: $currentKotlinVersion")

        println("Benchmark results:")
        println(results.print(borders = true))
@@ -320,28 +332,41 @@ abstract class BenchmarkTemplate(
    }

    private fun ScenarioSuite.writeTo(output: File) {
+        val allScenarios = scenarios
+            .map {scenario ->
+                kotlinVersions.map { scenario to it }
+            }
+            .flatten()
+            .map { (scenario, version) ->
+                "${scenario.title} $version".normalizeTitle
+            }
        output.writeText(
            """
-            |default-scenarios = [${scenarios.joinToString { "\"${it.title.normalizeTitle}\"" }}]
+            |default-scenarios = [${allScenarios.joinToString { "\"$it\"" }}]
            |
            |${scenarios.joinToString(separator = "\n") { it.write() }}
            """.trimMargin()
        )
    }

-    private fun Scenario.write(): String =
+    private fun Scenario.write(): String = kotlinVersions.joinToString(separator = "\n") { kotlinVersion ->
+        val finalGradleArgs = gradleArgs
+            .plus("-PkotlinVersion=$kotlinVersion")
+            .joinToString { "\"$it\"" }
        """
-        |${title.normalizeTitle} {
-        |    title = "$title"
+        |${"$title $kotlinVersion".normalizeTitle} {
+        |    title = "$title $kotlinVersion"
        |    warm-ups = $warmups
        |    iterations = $iterations
        |    tasks = [${tasks.joinToString { "\"$it\"" }}]
-        |    ${if (gradleArgs.isNotEmpty()) "gradle-args = [${gradleArgs.joinToString { "\"$it\"" }}]" else ""}
+        |    gradle-args = [$finalGradleArgs]
        |    ${if (cleanupTasks.isNotEmpty()) "cleanup-tasks = [${cleanupTasks.joinToString { "\"$it\"" }}]" else ""}
        |    ${if (applyAbiChange.isNotEmpty()) "apply-abi-change-to = [${applyAbiChange.joinToString { "\"$it\"" }}]" else ""}
        |    ${if (applyAndroidResourceValueChange.isNotEmpty()) "apply-android-resource-value-change-to = [${applyAndroidResourceValueChange.joinToString { "\"$it\"" }}]" else ""}
        |}
+        |
        """.trimMargin()
+    }

    private val String.dropLeadingDir: String get() = substringAfter('/')
    private val String.normalizeTitle: String get() = lowercase().replace(" ", "_")
@@ -355,7 +380,11 @@ abstract class BenchmarkTemplate(
                columnName == "scenario" -> DataColumn.createValueColumn(
                    columnName,
                    columns().map {
-                        it.name.dropLastWhile { it.isDigit() }
+                        if (it.name.contains(currentKotlinVersion)) {
+                            it.name.substringBefore(currentKotlinVersion) + currentKotlinVersion
+                        } else {
+                            it.name.substringBefore(stableKotlinVersions) + stableKotlinVersions
+                        }
                    }.drop(1)
                )
                columnName == "version" -> DataColumn.createValueColumn(
@@ -411,6 +440,7 @@ abstract class BenchmarkTemplate(

 typealias BenchmarkName = String

+@Suppress("Unused")
 class BenchmarkFailedException(exitCode: Int) : Exception(
    "Benchmark process was not finished successfully with $exitCode exit code."
 )
@@ -425,7 +455,8 @@ class BenchmarkResult(
 annotation class BenchmarkProject(
    val name: String,
    val gitUrl: String,
-    val gitCommitSha: String
+    val gitCommitSha: String,
+    val stableKotlinVersion: String,
 )

 class BenchmarkScriptDefinition : ScriptCompilationConfiguration(
@@ -451,9 +482,10 @@ class BenchmarkEvaluationConfiguration : ScriptEvaluationConfiguration(
            val name: String = compileConf[benchmarkProjectName]!!
            val gitUrl: String = compileConf[benchmarkGitUrl]!!
            val gitCommitSha: String = compileConf[benchmarkGitCommitSha]!!
+            val stableKotlinVersion: String = compileConf[benchmarkStableKotlinVersion]!!

            context.evaluationConfiguration.with evalConf@{
-                constructorArgs(name, gitUrl, gitCommitSha)
+                constructorArgs(name, gitUrl, gitCommitSha, stableKotlinVersion)
            }.asSuccess()
        }
    }
@@ -474,10 +506,12 @@ internal class BenchmarkScriptConfigurator : RefineScriptCompilationConfiguratio
            data[benchmarkProjectName] = benchmarkProject.name
            data[benchmarkGitUrl] = benchmarkProject.gitUrl
            data[benchmarkGitCommitSha] = benchmarkProject.gitCommitSha
+            data[benchmarkStableKotlinVersion] = benchmarkProject.stableKotlinVersion
        }.asSuccess()
    }
 }

 private val benchmarkProjectName by PropertiesCollection.key<String>()
 private val benchmarkGitUrl by PropertiesCollection.key<String>()
-private val benchmarkGitCommitSha by PropertiesCollection.key<String>()
+private val benchmarkGitCommitSha by PropertiesCollection.key<String>()
+private val benchmarkStableKotlinVersion by PropertiesCollection.key<String>()
@@ -4,25 +4,21 @@
@file:BenchmarkProject(
    name = "duckduckgo",
    gitUrl = "https://github.com/duckduckgo/Android.git",
-    gitCommitSha = "18e230fcefbefb4317c1fe128b4539a2315e7c0a"
+    gitCommitSha = "18e230fcefbefb4317c1fe128b4539a2315e7c0a",
+    stableKotlinVersion = "1.7.20",
 )

 import java.io.File

-val stableReleasePatch = {
-    "duckduckgo-kotlin-1.7.20.patch" to File("benchmarkScripts/files/duckduckgo-kotlin-1.7.20.patch")
-        .readText()
-        .byteInputStream()
-}
-
-val currentReleasePatch = {
+val repoPatch = {
    "duckduckgo-kotlin-current.patch" to File("benchmarkScripts/files/duckduckgo-kotlin-current.patch")
        .readText()
        .run { replace("<kotlin_version>", currentKotlinVersion) }
        .byteInputStream()
 }

-runAllBenchmarks(
+runBenchmarks(
+    repoPatch,
    suite {
        scenario {
            title = "Clean build"
@@ -86,9 +82,5 @@ runAllBenchmarks(
            iterations = 20
            runTasks(":app:assemblePlayDebug")
        }
-    },
-    mapOf(
-        "1.7.20" to stableReleasePatch,
-        "1.8.0" to currentReleasePatch
-    )
+    }
 )
@@ -1,13 +0,0 @@
-diff --git a/gradle.properties b/gradle.properties
-index a1796af97..43609273a 100644
--- a/gradle.properties
-+++ b/gradle.properties
-@@ -15,7 +15,7 @@ org.gradle.jvmargs=-Xmx2g -XX:+HeapDumpOnOutOfMemoryError
-
- # dependencies
- kotlinJvmVersion = 1.8
-kotlinVersion = 1.6.21
-+kotlinVersion = 1.7.20
- kotlinCoroutinesVersion = 1.6.4
- # kotlinx-serialization 1.3.3 calls Kotlin 1.7 API
- kotlinxSerializationVersion = 1.3.2
@@ -9,24 +9,21 @@
@file:BenchmarkProject(
    name = "graphql-kotlin",
    gitUrl = "https://github.com/ExpediaGroup/graphql-kotlin.git",
-    gitCommitSha = "fd1e9063f3aae144e099cdcfa69a4416fa434fb2"
+    gitCommitSha = "fd1e9063f3aae144e099cdcfa69a4416fa434fb2",
+    stableKotlinVersion = "1.7.20",
 )

 import java.io.File

-val stableReleasePatch = {
-    "graphql-kotlin-1.7.20.patch" to File("benchmarkScripts/files/graphql-kotlin-1.7.20.patch")
-        .readText()
-        .byteInputStream()
-}
-val currentReleasePatch = {
-    "graphql-kotlin-current.patch" to File("benchmarkScripts/files/graphql-kotlin-current.patch")
+val repoPatch = {
+    "graphql-kotlin-current.patch" to File("benchmarkScripts/files/graphql-kotlin-repo.patch")
        .readText()
        .run { replace("<kotlin_version>", currentKotlinVersion) }
        .byteInputStream()
 }

-runAllBenchmarks(
+runBenchmarks(
+    repoPatch,
    suite {
        scenario {
            title = "Spring server clean build"
@@ -91,9 +88,5 @@ runAllBenchmarks(
            iterations = 20
            runTasks("assemble")
        }
-    },
-    mapOf(
-        "1.7.20" to stableReleasePatch,
-        "1.8.0" to currentReleasePatch
-    )
+    }
 )
@@ -9,19 +9,21 @@
@file:BenchmarkProject(
    name = "kvision",
    gitUrl = "https://github.com/rjaros/kvision.git",
-    gitCommitSha = "3fe69bf6db9a3650b026630d857862f3cee6485b"
+    gitCommitSha = "3fe69bf6db9a3650b026630d857862f3cee6485b",
+    stableKotlinVersion = "1.7.20",
 )

 import java.io.File

-val currentReleasePatch = {
+val repoPatch = {
    "kvision-kotlin-current.patch" to File("benchmarkScripts/files/kvision-kotlin-current.patch")
        .readText()
        .run { replace("<kotlin_version>", currentKotlinVersion) }
        .byteInputStream()
 }

-runAllBenchmarks(
+runBenchmarks(
+    repoPatch,
    suite {
        scenario {
            title = "Build Js IR clean build"
@@ -86,9 +88,5 @@ runAllBenchmarks(
            iterations = 20
            runTasks("jsIrJar")
        }
-    },
-    mapOf(
-        "1.7.20" to null,
-        "1.8.0" to currentReleasePatch
-    )
+    }
 )