diff --git a/libraries/tools/gradle/regression-benchmark-templates/src/main/kotlin/org/jetbrains/kotlin/gradle/benchmark/BenchmarkTemplate.kt b/libraries/tools/gradle/regression-benchmark-templates/src/main/kotlin/org/jetbrains/kotlin/gradle/benchmark/BenchmarkTemplate.kt index 02b555630bf..0df9a310204 100644 --- a/libraries/tools/gradle/regression-benchmark-templates/src/main/kotlin/org/jetbrains/kotlin/gradle/benchmark/BenchmarkTemplate.kt +++ b/libraries/tools/gradle/regression-benchmark-templates/src/main/kotlin/org/jetbrains/kotlin/gradle/benchmark/BenchmarkTemplate.kt @@ -18,6 +18,7 @@ import org.jetbrains.kotlinx.dataframe.io.toHTML import org.jetbrains.kotlinx.dataframe.io.writeCSV import org.jetbrains.kotlinx.dataframe.math.median import java.io.File +import java.io.InputStream import java.util.zip.ZipInputStream import kotlin.reflect.typeOf import kotlin.script.experimental.annotations.KotlinScript @@ -37,6 +38,7 @@ abstract class BenchmarkTemplate( private val gitCommitSha: String, ) { private val workingDir = File(args.first()) + val currentKotlinVersion: String = args[1] private val gradleProfilerDir = workingDir.resolve("gradle-profiler") private val projectRepoDir = workingDir.resolve(projectName) private val scenariosDir = workingDir.resolve("scenarios") @@ -44,9 +46,9 @@ abstract class BenchmarkTemplate( private val gitOperationsPrinter = TextProgressMonitor() - fun runAllBenchmarks( + fun runAllBenchmarks( suite: ScenarioSuite, - benchmarks: Map + benchmarks: Map Pair)?> ) { printStartingMessage() downloadGradleProfilerIfNotAvailable() @@ -54,7 +56,10 @@ abstract class BenchmarkTemplate( val results = benchmarks.map { (name, patchFile) -> repoReset() - patchFile?.let { repoApplyPatch(it) } + patchFile?.let { + val (patchName, patch) = it() + repoApplyPatch(patchName, patch) + } runBenchmark(suite, name) } aggregateBenchmarkResults(*results.toTypedArray()) @@ -106,12 +111,21 @@ abstract class BenchmarkTemplate( return projectRepoDir } - fun repoApplyPatch(patchFile: PatchFile) { - println("Applying patch $patchFile to repository") + fun repoApplyPatchFromFile( + patchFile: String + ) { val patch = File(patchFile) + repoApplyPatch(patch.name, patch.inputStream()) + } + + fun repoApplyPatch( + patchName: String, + patch: InputStream + ) { + println("Applying patch $patchName to repository") Git.open(projectRepoDir) .apply() - .setPatch(patch.inputStream()) + .setPatch(patch) .call() } @@ -359,7 +373,6 @@ abstract class BenchmarkTemplate( } typealias BenchmarkName = String -typealias PatchFile = String class BenchmarkFailedException(exitCode: Int) : Exception( "Benchmark process was not finished successfully with $exitCode exit code." diff --git a/libraries/tools/gradle/regression-benchmarks/Readme.md b/libraries/tools/gradle/regression-benchmarks/Readme.md index 4ec84a3755d..d84502d6c4f 100644 --- a/libraries/tools/gradle/regression-benchmarks/Readme.md +++ b/libraries/tools/gradle/regression-benchmarks/Readme.md @@ -2,7 +2,8 @@ Contains build regression benchmark scripts for different user projects. -Such benchmarks comparing different build scenarios between last stable Kotlin release and current-in-progress release helping to identify build speed regressions. +Such benchmarks comparing different build scenarios between last stable Kotlin release and current-in-progress release helping +to identify build speed regressions with minimal user-project modifications. All scripts should run via related Gradle task which could be found in "Gradle Regression Benchmark tasks" task group. @@ -15,5 +16,6 @@ All scripts are using infrastructure provided by [template](../regression-benchm - Add required `@file:BenchmarkProject` annotation and few steps that will download profiler plus project itself - Inspect user-project and create required git patches to change Kotlin version in the project - add changes, test it and use `git diff --no-color > name.patch` command. Put created patches into `benchmarkScripts/files` directory. +- Update current Kotlin version patch to use Kotlin version passed as argument to script and available as `currentKotlinVersion` - Write benchmark scenarios and run benchmark with `dryRun = true` flag - Add final changes to script, probably convert it to use `runAllBenchmarks()` function diff --git a/libraries/tools/gradle/regression-benchmarks/benchmarkScripts/duckduckgo.benchmark.kts b/libraries/tools/gradle/regression-benchmarks/benchmarkScripts/duckduckgo.benchmark.kts index 3d9911dd1c2..7cc1e022513 100644 --- a/libraries/tools/gradle/regression-benchmarks/benchmarkScripts/duckduckgo.benchmark.kts +++ b/libraries/tools/gradle/regression-benchmarks/benchmarkScripts/duckduckgo.benchmark.kts @@ -1,13 +1,25 @@ // Android application written in Kotlin: // - kind of big codebase // - uses Kotlin compiler plugins and kapt - @file:BenchmarkProject( name = "duckduckgo", gitUrl = "https://github.com/duckduckgo/Android.git", gitCommitSha = "659bd5aaac1fba922a6df9053daa2b7bcd610375" ) +import java.io.File + +val stableReleasePatch = { + "duckduckgo-kotlin-1.6.10.patch" to File("benchmarkScripts/files/duckduckgo-kotlin-1.6.10.patch").inputStream() +} + +val currentReleasePatch = { + "duckduckgo-kotlin-current.patch" to File("benchmarkScripts/files/duckduckgo-kotlin-current.patch") + .readText() + .run { replace("", currentKotlinVersion) } + .byteInputStream() +} + runAllBenchmarks( suite { scenario { @@ -31,7 +43,7 @@ runAllBenchmarks( } }, mapOf( - "1.6.10" to "benchmarkScripts/files/duckduckgo-kotlin-1.6.10.patch", - "1.6.20" to "benchmarkScripts/files/duckduckgo-kotlin-current.patch" + "1.6.10" to stableReleasePatch, + "1.6.20" to currentReleasePatch ) ) diff --git a/libraries/tools/gradle/regression-benchmarks/benchmarkScripts/files/duckduckgo-kotlin-current.patch b/libraries/tools/gradle/regression-benchmarks/benchmarkScripts/files/duckduckgo-kotlin-current.patch index d729ed5ddfe..34c871ac3ee 100644 --- a/libraries/tools/gradle/regression-benchmarks/benchmarkScripts/files/duckduckgo-kotlin-current.patch +++ b/libraries/tools/gradle/regression-benchmarks/benchmarkScripts/files/duckduckgo-kotlin-current.patch @@ -27,7 +27,7 @@ index 4497fd12..730df20a 100644 ext { - kotlin_version = '1.5.10' -+ kotlin_version = '1.6.255-SNAPSHOT' ++ kotlin_version = '' } ext.spotless = "5.14.0" - ext.anvil_version = "2.3.3" @@ -67,7 +67,7 @@ index fae6ae6e..81cb692c 100644 version.io.reactivex.rxjava2..rxjava=2.2.9 -version.kotlin=1.5.31 -+version.kotlin=1.6.255-SNAPSHOT ++version.kotlin= version.kotlinx.coroutines=1.5.2 diff --git a/libraries/tools/gradle/regression-benchmarks/build.gradle.kts b/libraries/tools/gradle/regression-benchmarks/build.gradle.kts index fc422a1706a..ad674e45f1d 100644 --- a/libraries/tools/gradle/regression-benchmarks/build.gradle.kts +++ b/libraries/tools/gradle/regression-benchmarks/build.gradle.kts @@ -23,7 +23,8 @@ dependencies { val service = project.extensions.getByType() abstract class ScriptArgumentProvider @Inject constructor( - layout: ProjectLayout + layout: ProjectLayout, + private val releaseKotlinVersion: String ) : CommandLineArgumentProvider { @get:Classpath abstract val scriptClasspath: ConfigurableFileCollection @@ -40,7 +41,8 @@ abstract class ScriptArgumentProvider @Inject constructor( "-no-stdlib", "-classpath", scriptClasspath.asFileTree.files.joinToString(separator = File.pathSeparator), "-script", "benchmarkScripts/${script.get()}", - scriptOutputDirectories.get().asFile.absolutePath + scriptOutputDirectories.get().asFile.absolutePath, + releaseKotlinVersion ) } } @@ -62,7 +64,7 @@ fun addBenchmarkTask( classpath = compilerClasspath mainClass.set("org.jetbrains.kotlin.cli.jvm.K2JVMCompiler") - val scriptArgs = objects.newInstance() + val scriptArgs = objects.newInstance(version) scriptArgs.script.set(script) scriptArgs.scriptClasspath.from(scriptsClasspath) argumentProviders.add(scriptArgs)