diff --git a/libraries/tools/gradle/regression-benchmark-templates/src/main/kotlin/org/jetbrains/kotlin/gradle/benchmark/BenchmarkTemplate.kt b/libraries/tools/gradle/regression-benchmark-templates/src/main/kotlin/org/jetbrains/kotlin/gradle/benchmark/BenchmarkTemplate.kt index db8d1211560..bf501414dee 100644 --- a/libraries/tools/gradle/regression-benchmark-templates/src/main/kotlin/org/jetbrains/kotlin/gradle/benchmark/BenchmarkTemplate.kt +++ b/libraries/tools/gradle/regression-benchmark-templates/src/main/kotlin/org/jetbrains/kotlin/gradle/benchmark/BenchmarkTemplate.kt @@ -36,9 +36,11 @@ abstract class BenchmarkTemplate( private val projectName: String, private val projectGitUrl: String, private val gitCommitSha: String, + private val stableKotlinVersions: String ) { private val workingDir = File(args.first()) val currentKotlinVersion: String = args[1] + private val kotlinVersions = setOf(stableKotlinVersions, currentKotlinVersion) private val gradleProfilerDir = workingDir.resolve("gradle-profiler") private val projectRepoDir = workingDir.resolve(projectName) private val scenariosDir = workingDir.resolve("scenarios") @@ -46,23 +48,21 @@ abstract class BenchmarkTemplate( private val gitOperationsPrinter = TextProgressMonitor() - fun runAllBenchmarks( + fun runBenchmarks( + repoPatch: (() -> Pair)?, suite: ScenarioSuite, - benchmarks: Map Pair)?> ) { printStartingMessage() downloadGradleProfilerIfNotAvailable() checkoutRepository() - val results = benchmarks.map { (name, patchFile) -> - repoReset() - patchFile?.let { - val (patchName, patch) = it() - repoApplyPatch(patchName, patch) - } - runBenchmark(suite, name) + repoReset() + repoPatch?.let { + val (patchName, patch) = it() + repoApplyPatch(patchName, patch) } - aggregateBenchmarkResults(*results.toTypedArray()) + val result = runBenchmark(suite) + aggregateBenchmarkResults(result) printEndingMessage() } @@ -138,15 +138,14 @@ abstract class BenchmarkTemplate( .call() } - fun runBenchmark( + private fun runBenchmark( scenarioSuite: ScenarioSuite, - benchmarkName: BenchmarkName, - dryRun: Boolean = false + @Suppress("UNUSED_PARAMETER") dryRun: Boolean = false ): BenchmarkResult { - println("Staring benchmark $benchmarkName") - val normalizedBenchmarkName = benchmarkName.normalizeTitle + println("Staring benchmark $projectName") + val normalizedBenchmarkName = projectName.normalizeTitle if (!scenariosDir.exists()) scenariosDir.mkdirs() - val scenarioFile = scenariosDir.resolve("${projectName}_$normalizedBenchmarkName.scenario") + val scenarioFile = scenariosDir.resolve("$normalizedBenchmarkName.scenario") scenarioSuite.writeTo(scenarioFile) val benchmarkOutputDir = benchmarkOutputsDir .resolve(projectName) @@ -186,60 +185,73 @@ abstract class BenchmarkTemplate( throw BenchmarkFailedException(profilerProcess.exitValue()) } - println("Benchmark $benchmarkName has ended") + println("Benchmark $projectName has ended") return BenchmarkResult( - benchmarkName, + projectName, benchmarkOutputDir.resolve("benchmark.csv") ) } - fun aggregateBenchmarkResults(vararg benchmarkResults: BenchmarkResult) { + // Not working as intended due to this bug: https://github.com/gradle/gradle-profiler/issues/317 + fun aggregateBenchmarkResults(benchmarkResult: BenchmarkResult) { println("Aggregating benchmark results...") - val sortedResults = benchmarkResults.sortedBy { it.name } - val results = sortedResults - .map { result -> - DataFrame - .readCSV(result.result) - .flipColumnsWithRows() - .remove("version", "tasks") // removing unused columns - .remove { nameContains("warm-up build") } // removing warm-up times - .add("median build time") { // squashing build times into median build time - valuesOf().median() - } - .remove { nameContains("measured build") } // removing iterations results - .groupBy("scenario").aggregate { // merging configuration and build times into one row - first { it.values().contains("task start") }["median build time"] into "tasks start median time" - first { it.values().contains("total execution time") }["median build time"] into "execution median time" - } - .add("benchmark") { result.name } + val results = DataFrame + .readCSV(benchmarkResult.result, duplicate = true) + .drop { + // Removing unused rows + it["scenario"] in listOf("version", "tasks") || + it["scenario"].toString().startsWith("warm-up build") } - .concat() - .groupBy("scenario").aggregate { // Merging scenarios from different benchmarks into one row - forEach { row -> - row["tasks start median time"] into "Configuration: ${row["benchmark"]}" - row["execution median time"] into "Execution: ${row["benchmark"]}" + .flipColumnsWithRows() + .add("median build time") { // squashing build times into median build time + valuesOf().median() + } + .remove { nameContains("measured build") } // removing iterations results + .groupBy("scenario").aggregate { // merging configuration and build times into one row + first { it.values().contains("task start") }["median build time"] into "tasks start median time" + first { it.values().contains("total execution time") }["median build time"] into "execution median time" + } + .groupBy { + expr { + val scenarioName = get("scenario").toString() + if (scenarioName.endsWith(currentKotlinVersion)) { + scenarioName.substringBefore(currentKotlinVersion) + } else { + scenarioName.substringBefore(stableKotlinVersions) + } } } - .sortBy("scenario") - .rename("scenario" to "Scenario") + .aggregate { + forEach { row -> + val version = if (row["scenario"].toString().endsWith(stableKotlinVersions)) { + stableKotlinVersions + } else { + currentKotlinVersion + } + row["tasks start median time"] into "Configuration: $version" + row["execution median time"] into "Execution: $version" + } + } + .rename { col(0) }.into("Scenario") + .sortBy("Scenario") .reorderColumnsBy { // "Scenario" column should always be in the first place if (it.name() == "Scenario") "0000Scenario" else it.name() } .insert("Configuration diff from stable release") { - val stableReleaseConfiguration = column("Configuration: ${sortedResults.first().name}").getValue(this) - val currentReleaseConfiguration = column("Configuration: ${sortedResults.last().name}").getValue(this) + val stableReleaseConfiguration = column("Configuration: $stableKotlinVersions").getValue(this) + val currentReleaseConfiguration = column("Configuration: $currentKotlinVersion").getValue(this) val percent = currentReleaseConfiguration * 100 / stableReleaseConfiguration "${percent}%" } - .after("Configuration: ${sortedResults.last().name}") + .after("Configuration: $currentKotlinVersion") .insert("Execution diff from stable release") { - val stableReleaseConfiguration = column("Execution: ${sortedResults.first().name}").getValue(this) - val currentReleaseConfiguration = column("Execution: ${sortedResults.last().name}").getValue(this) + val stableReleaseConfiguration = column("Execution: $stableKotlinVersions").getValue(this) + val currentReleaseConfiguration = column("Execution: $currentKotlinVersion").getValue(this) val percent = currentReleaseConfiguration * 100 / stableReleaseConfiguration "${percent}%" } - .after("Execution: ${sortedResults.last().name}") + .after("Execution: $currentKotlinVersion") println("Benchmark results:") println(results.print(borders = true)) @@ -320,28 +332,41 @@ abstract class BenchmarkTemplate( } private fun ScenarioSuite.writeTo(output: File) { + val allScenarios = scenarios + .map {scenario -> + kotlinVersions.map { scenario to it } + } + .flatten() + .map { (scenario, version) -> + "${scenario.title} $version".normalizeTitle + } output.writeText( """ - |default-scenarios = [${scenarios.joinToString { "\"${it.title.normalizeTitle}\"" }}] + |default-scenarios = [${allScenarios.joinToString { "\"$it\"" }}] | |${scenarios.joinToString(separator = "\n") { it.write() }} """.trimMargin() ) } - private fun Scenario.write(): String = + private fun Scenario.write(): String = kotlinVersions.joinToString(separator = "\n") { kotlinVersion -> + val finalGradleArgs = gradleArgs + .plus("-PkotlinVersion=$kotlinVersion") + .joinToString { "\"$it\"" } """ - |${title.normalizeTitle} { - | title = "$title" + |${"$title $kotlinVersion".normalizeTitle} { + | title = "$title $kotlinVersion" | warm-ups = $warmups | iterations = $iterations | tasks = [${tasks.joinToString { "\"$it\"" }}] - | ${if (gradleArgs.isNotEmpty()) "gradle-args = [${gradleArgs.joinToString { "\"$it\"" }}]" else ""} + | gradle-args = [$finalGradleArgs] | ${if (cleanupTasks.isNotEmpty()) "cleanup-tasks = [${cleanupTasks.joinToString { "\"$it\"" }}]" else ""} | ${if (applyAbiChange.isNotEmpty()) "apply-abi-change-to = [${applyAbiChange.joinToString { "\"$it\"" }}]" else ""} | ${if (applyAndroidResourceValueChange.isNotEmpty()) "apply-android-resource-value-change-to = [${applyAndroidResourceValueChange.joinToString { "\"$it\"" }}]" else ""} |} + | """.trimMargin() + } private val String.dropLeadingDir: String get() = substringAfter('/') private val String.normalizeTitle: String get() = lowercase().replace(" ", "_") @@ -355,7 +380,11 @@ abstract class BenchmarkTemplate( columnName == "scenario" -> DataColumn.createValueColumn( columnName, columns().map { - it.name.dropLastWhile { it.isDigit() } + if (it.name.contains(currentKotlinVersion)) { + it.name.substringBefore(currentKotlinVersion) + currentKotlinVersion + } else { + it.name.substringBefore(stableKotlinVersions) + stableKotlinVersions + } }.drop(1) ) columnName == "version" -> DataColumn.createValueColumn( @@ -411,6 +440,7 @@ abstract class BenchmarkTemplate( typealias BenchmarkName = String +@Suppress("Unused") class BenchmarkFailedException(exitCode: Int) : Exception( "Benchmark process was not finished successfully with $exitCode exit code." ) @@ -425,7 +455,8 @@ class BenchmarkResult( annotation class BenchmarkProject( val name: String, val gitUrl: String, - val gitCommitSha: String + val gitCommitSha: String, + val stableKotlinVersion: String, ) class BenchmarkScriptDefinition : ScriptCompilationConfiguration( @@ -451,9 +482,10 @@ class BenchmarkEvaluationConfiguration : ScriptEvaluationConfiguration( val name: String = compileConf[benchmarkProjectName]!! val gitUrl: String = compileConf[benchmarkGitUrl]!! val gitCommitSha: String = compileConf[benchmarkGitCommitSha]!! + val stableKotlinVersion: String = compileConf[benchmarkStableKotlinVersion]!! context.evaluationConfiguration.with evalConf@{ - constructorArgs(name, gitUrl, gitCommitSha) + constructorArgs(name, gitUrl, gitCommitSha, stableKotlinVersion) }.asSuccess() } } @@ -474,10 +506,12 @@ internal class BenchmarkScriptConfigurator : RefineScriptCompilationConfiguratio data[benchmarkProjectName] = benchmarkProject.name data[benchmarkGitUrl] = benchmarkProject.gitUrl data[benchmarkGitCommitSha] = benchmarkProject.gitCommitSha + data[benchmarkStableKotlinVersion] = benchmarkProject.stableKotlinVersion }.asSuccess() } } private val benchmarkProjectName by PropertiesCollection.key() private val benchmarkGitUrl by PropertiesCollection.key() -private val benchmarkGitCommitSha by PropertiesCollection.key() \ No newline at end of file +private val benchmarkGitCommitSha by PropertiesCollection.key() +private val benchmarkStableKotlinVersion by PropertiesCollection.key() diff --git a/libraries/tools/gradle/regression-benchmarks/benchmarkScripts/duckduckgo.benchmark.kts b/libraries/tools/gradle/regression-benchmarks/benchmarkScripts/duckduckgo.benchmark.kts index ea50a2096b5..002286ab324 100644 --- a/libraries/tools/gradle/regression-benchmarks/benchmarkScripts/duckduckgo.benchmark.kts +++ b/libraries/tools/gradle/regression-benchmarks/benchmarkScripts/duckduckgo.benchmark.kts @@ -4,25 +4,21 @@ @file:BenchmarkProject( name = "duckduckgo", gitUrl = "https://github.com/duckduckgo/Android.git", - gitCommitSha = "18e230fcefbefb4317c1fe128b4539a2315e7c0a" + gitCommitSha = "18e230fcefbefb4317c1fe128b4539a2315e7c0a", + stableKotlinVersion = "1.7.20", ) import java.io.File -val stableReleasePatch = { - "duckduckgo-kotlin-1.7.20.patch" to File("benchmarkScripts/files/duckduckgo-kotlin-1.7.20.patch") - .readText() - .byteInputStream() -} - -val currentReleasePatch = { +val repoPatch = { "duckduckgo-kotlin-current.patch" to File("benchmarkScripts/files/duckduckgo-kotlin-current.patch") .readText() .run { replace("", currentKotlinVersion) } .byteInputStream() } -runAllBenchmarks( +runBenchmarks( + repoPatch, suite { scenario { title = "Clean build" @@ -86,9 +82,5 @@ runAllBenchmarks( iterations = 20 runTasks(":app:assemblePlayDebug") } - }, - mapOf( - "1.7.20" to stableReleasePatch, - "1.8.0" to currentReleasePatch - ) + } ) diff --git a/libraries/tools/gradle/regression-benchmarks/benchmarkScripts/files/graphql-kotlin-1.7.20.patch b/libraries/tools/gradle/regression-benchmarks/benchmarkScripts/files/graphql-kotlin-1.7.20.patch deleted file mode 100644 index c81829ba0f9..00000000000 --- a/libraries/tools/gradle/regression-benchmarks/benchmarkScripts/files/graphql-kotlin-1.7.20.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/gradle.properties b/gradle.properties -index a1796af97..43609273a 100644 ---- a/gradle.properties -+++ b/gradle.properties -@@ -15,7 +15,7 @@ org.gradle.jvmargs=-Xmx2g -XX:+HeapDumpOnOutOfMemoryError - - # dependencies - kotlinJvmVersion = 1.8 --kotlinVersion = 1.6.21 -+kotlinVersion = 1.7.20 - kotlinCoroutinesVersion = 1.6.4 - # kotlinx-serialization 1.3.3 calls Kotlin 1.7 API - kotlinxSerializationVersion = 1.3.2 \ No newline at end of file diff --git a/libraries/tools/gradle/regression-benchmarks/benchmarkScripts/files/graphql-kotlin-current.patch b/libraries/tools/gradle/regression-benchmarks/benchmarkScripts/files/graphql-kotlin-repo.patch similarity index 100% rename from libraries/tools/gradle/regression-benchmarks/benchmarkScripts/files/graphql-kotlin-current.patch rename to libraries/tools/gradle/regression-benchmarks/benchmarkScripts/files/graphql-kotlin-repo.patch diff --git a/libraries/tools/gradle/regression-benchmarks/benchmarkScripts/graphql-kotlin.benchmark.kts b/libraries/tools/gradle/regression-benchmarks/benchmarkScripts/graphql-kotlin.benchmark.kts index 671edc73eed..21a36564133 100644 --- a/libraries/tools/gradle/regression-benchmarks/benchmarkScripts/graphql-kotlin.benchmark.kts +++ b/libraries/tools/gradle/regression-benchmarks/benchmarkScripts/graphql-kotlin.benchmark.kts @@ -9,24 +9,21 @@ @file:BenchmarkProject( name = "graphql-kotlin", gitUrl = "https://github.com/ExpediaGroup/graphql-kotlin.git", - gitCommitSha = "fd1e9063f3aae144e099cdcfa69a4416fa434fb2" + gitCommitSha = "fd1e9063f3aae144e099cdcfa69a4416fa434fb2", + stableKotlinVersion = "1.7.20", ) import java.io.File -val stableReleasePatch = { - "graphql-kotlin-1.7.20.patch" to File("benchmarkScripts/files/graphql-kotlin-1.7.20.patch") - .readText() - .byteInputStream() -} -val currentReleasePatch = { - "graphql-kotlin-current.patch" to File("benchmarkScripts/files/graphql-kotlin-current.patch") +val repoPatch = { + "graphql-kotlin-current.patch" to File("benchmarkScripts/files/graphql-kotlin-repo.patch") .readText() .run { replace("", currentKotlinVersion) } .byteInputStream() } -runAllBenchmarks( +runBenchmarks( + repoPatch, suite { scenario { title = "Spring server clean build" @@ -91,9 +88,5 @@ runAllBenchmarks( iterations = 20 runTasks("assemble") } - }, - mapOf( - "1.7.20" to stableReleasePatch, - "1.8.0" to currentReleasePatch - ) + } ) diff --git a/libraries/tools/gradle/regression-benchmarks/benchmarkScripts/kvision.benchmark.kts b/libraries/tools/gradle/regression-benchmarks/benchmarkScripts/kvision.benchmark.kts index df530afbfec..2a96934b274 100644 --- a/libraries/tools/gradle/regression-benchmarks/benchmarkScripts/kvision.benchmark.kts +++ b/libraries/tools/gradle/regression-benchmarks/benchmarkScripts/kvision.benchmark.kts @@ -9,19 +9,21 @@ @file:BenchmarkProject( name = "kvision", gitUrl = "https://github.com/rjaros/kvision.git", - gitCommitSha = "3fe69bf6db9a3650b026630d857862f3cee6485b" + gitCommitSha = "3fe69bf6db9a3650b026630d857862f3cee6485b", + stableKotlinVersion = "1.7.20", ) import java.io.File -val currentReleasePatch = { +val repoPatch = { "kvision-kotlin-current.patch" to File("benchmarkScripts/files/kvision-kotlin-current.patch") .readText() .run { replace("", currentKotlinVersion) } .byteInputStream() } -runAllBenchmarks( +runBenchmarks( + repoPatch, suite { scenario { title = "Build Js IR clean build" @@ -86,9 +88,5 @@ runAllBenchmarks( iterations = 20 runTasks("jsIrJar") } - }, - mapOf( - "1.7.20" to null, - "1.8.0" to currentReleasePatch - ) + } ) \ No newline at end of file