Add Gradle project to run regression benchmarks

This project will contain all required Gradle infrastructure to run regression benchmark scripts. ^KT-49921 In Progress
2021-12-20 14:41:09 +01:00
parent 7418d3c898
commit d1f6faf236
3 changed files with 91 additions and 0 deletions
@@ -0,0 +1,19 @@
+## Description
+
+Contains build regression benchmark scripts for different user projects. 
+
+Such benchmarks comparing different build scenarios between last stable Kotlin release and current-in-progress release helping to identify build speed regressions.
+
+All scripts should run via related Gradle task which could be found in "Gradle Regression Benchmark tasks" task group.
+
+### Adding benchmark for new user-project
+
+All scripts are using infrastructure provided by [template](../regression-benchmark-templates/Readme.md).
+
+- Add new script file in `benchmarkScripts/` directory
+- Add new Gradle task to run the script
+- Add required `@file:BenchmarkProject` annotation and few steps that will download profiler plus project itself
+- Inspect user-project and create required git patches to change Kotlin version in the project - add changes, test it 
+and use `git diff --no-color > name.patch` command. Put created patches into `benchmarkScripts/files` directory.
+- Write benchmark scenarios and run benchmark with `dryRun = true` flag
+- Add final changes to script, probably convert it to use `runAllBenchmarks()` function
@@ -0,0 +1,70 @@
+plugins {
+    `java-base`
+}
+
+val compilerClasspath = configurations.create("compilerClasspath") {
+    isCanBeResolved = true
+    isCanBeConsumed = false
+}
+val scriptsClasspath = configurations.create("scriptsClasspath") {
+    isCanBeResolved = true
+    isCanBeConsumed = false
+}
+
+dependencies {
+    compilerClasspath.name(project(":kotlin-compiler-embeddable"))
+    compilerClasspath.name(project(":kotlin-scripting-compiler-embeddable"))
+
+    scriptsClasspath.name(project(":gradle:regression-benchmark-templates"))
+    scriptsClasspath.name(project(":kotlin-script-runtime"))
+    scriptsClasspath.name(kotlinStdlib())
+}
+
+val service = project.extensions.getByType<JavaToolchainService>()
+
+abstract class ScriptArgumentProvider @Inject constructor(
+    layout: ProjectLayout
+) : CommandLineArgumentProvider {
+    @get:Classpath
+    abstract val scriptClasspath: ConfigurableFileCollection
+
+    @get:Input
+    abstract val script: Property<String>
+
+    @get:OutputDirectory
+    val scriptOutputDirectories: Provider<Directory> = layout.buildDirectory.dir("benchmark-script")
+
+    override fun asArguments(): Iterable<String> {
+        return listOf(
+            "-no-reflect",
+            "-no-stdlib",
+            "-classpath", scriptClasspath.asFileTree.files.joinToString(separator = File.pathSeparator),
+            "-script", "benchmarkScripts/${script.get()}",
+            scriptOutputDirectories.get().asFile.absolutePath
+        )
+    }
+}
+
+fun addBenchmarkTask(
+    taskName: String,
+    script: String,
+    jdkVersion: JavaLanguageVersion = JavaLanguageVersion.of(8)
+): TaskProvider<JavaExec> {
+    return tasks.register<JavaExec>(taskName) {
+        group = "Gradle Regression Benchmark"
+        description = "Runs regression benchmark from $script"
+
+        dependsOn(":kotlin-gradle-plugin:install")
+
+        javaLauncher.set(service.launcherFor {
+            languageVersion.set(jdkVersion)
+        })
+        classpath = compilerClasspath
+        mainClass.set("org.jetbrains.kotlin.cli.jvm.K2JVMCompiler")
+
+        val scriptArgs = objects.newInstance<ScriptArgumentProvider>()
+        scriptArgs.script.set(script)
+        scriptArgs.scriptClasspath.from(scriptsClasspath)
+        argumentProviders.add(scriptArgs)
+    }
+}