Add build regression benchmark script template
This template provides all infrastructure to write Kotlin scripts for build regression benchmarks. These benchmarks will use gradle-profiler to run provided scenarios on user projects. All benchmark results in the script then will be aggregated into single one showing difference between them. Generally such kind of benchmarks should be used to track early Gradle build regressions between releases. ^KT-49921 In Progress
This commit is contained in:
Yahor Berdnikau
@@ -0,0 +1,18 @@
|
||||
## Description
|
||||
|
||||
Provides Kotlin script template to define and run [gradle-profiler](https://github.com/gradle/gradle-profiler) benchmarks.
|
||||
|
||||
### Writing benchmark script
|
||||
|
||||
This template is automatically applied to all script files with `.benchmark.kts` file extension.
|
||||
|
||||
The script itself should have following:
|
||||
- A `@file:BenchmarkProject` annotation providing name of the project-under-benchmark,
|
||||
git url to the project repo and git commit sha which should be used to run benchmarks.
|
||||
- definition of gradle-profiler scenarios is written in `suite { .. }` DSL.
|
||||
- provide optional git patch files that will be applied to the project before running benchmarks.
|
||||
|
||||
To run the actual benchmarks script either could call specific steps or just call `runAllBenchmarks()` function.
|
||||
This function calls required steps in order, runs provided benchmarks and then shows aggregated result.
|
||||
|
||||
Aggregated results are available in form of CSV or HTML files.
|
||||
@@ -0,0 +1,11 @@
|
||||
plugins {
|
||||
id("org.jetbrains.kotlin.jvm")
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation("org.jetbrains.kotlin:kotlin-scripting-common")
|
||||
implementation("com.squareup.okhttp3:okhttp:4.9.0")
|
||||
implementation("org.eclipse.jgit:org.eclipse.jgit:5.13.0.202109080827-r")
|
||||
implementation("org.slf4j:slf4j-nop:1.7.32")
|
||||
implementation("org.jetbrains.kotlinx:dataframe:0.8.0-rc-3")
|
||||
}
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
@file:Suppress("Unused", "Unused_Variable")
|
||||
/*
|
||||
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package org.jetbrains.kotlin.gradle.benchmark
|
||||
|
||||
class Scenario {
|
||||
lateinit var title: String
|
||||
var warmups = 3
|
||||
var iterations = 10
|
||||
val tasks = mutableListOf<String>()
|
||||
val gradleArgs = mutableListOf<String>()
|
||||
val cleanupTasks = mutableListOf<String>()
|
||||
val applyAbiChange = mutableListOf<String>()
|
||||
val applyNonAbiChange = mutableListOf<String>()
|
||||
|
||||
fun runTasks(vararg tasks: String) {
|
||||
this.tasks.addAll(tasks)
|
||||
}
|
||||
|
||||
fun useGradleArgs(vararg args: String) {
|
||||
gradleArgs.addAll(args)
|
||||
}
|
||||
|
||||
fun runCleanupTasks(vararg tasks: String) {
|
||||
cleanupTasks.addAll(tasks)
|
||||
}
|
||||
|
||||
fun applyAbiChangeTo(pathToClassFile: String) {
|
||||
applyAbiChange.add(pathToClassFile)
|
||||
}
|
||||
|
||||
/**
|
||||
* Currently not-working: https://github.com/gradle/gradle-profiler/issues/378
|
||||
*/
|
||||
fun applyNonAbiChangeTo(pathToClassFile: String) {
|
||||
applyNonAbiChange.add(pathToClassFile)
|
||||
}
|
||||
}
|
||||
|
||||
class ScenarioSuite {
|
||||
private val _scenarios = mutableListOf<Scenario>()
|
||||
val scenarios: List<Scenario> get() = _scenarios.toList()
|
||||
|
||||
fun scenario(init: Scenario.() -> Unit) {
|
||||
val scenario = Scenario()
|
||||
scenario.init()
|
||||
_scenarios.add(scenario)
|
||||
}
|
||||
}
|
||||
|
||||
fun suite(init: ScenarioSuite.() -> Unit): ScenarioSuite {
|
||||
val suite = ScenarioSuite()
|
||||
suite.init()
|
||||
return suite
|
||||
}
|
||||
+433
@@ -0,0 +1,433 @@
|
||||
/*
|
||||
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package org.jetbrains.kotlin.gradle.benchmark
|
||||
|
||||
import okhttp3.OkHttpClient
|
||||
import okhttp3.Request
|
||||
import okio.IOException
|
||||
import org.eclipse.jgit.api.Git
|
||||
import org.eclipse.jgit.api.ResetCommand
|
||||
import org.eclipse.jgit.lib.TextProgressMonitor
|
||||
import org.jetbrains.kotlinx.dataframe.*
|
||||
import org.jetbrains.kotlinx.dataframe.api.*
|
||||
import org.jetbrains.kotlinx.dataframe.io.readCSV
|
||||
import org.jetbrains.kotlinx.dataframe.io.toHTML
|
||||
import org.jetbrains.kotlinx.dataframe.io.writeCSV
|
||||
import org.jetbrains.kotlinx.dataframe.math.median
|
||||
import java.io.File
|
||||
import java.util.zip.ZipInputStream
|
||||
import kotlin.reflect.typeOf
|
||||
import kotlin.script.experimental.annotations.KotlinScript
|
||||
import kotlin.script.experimental.api.*
|
||||
import kotlin.script.experimental.util.PropertiesCollection
|
||||
|
||||
@Suppress("unused")
|
||||
@KotlinScript(
|
||||
fileExtension = "benchmark.kts",
|
||||
compilationConfiguration = BenchmarkScriptDefinition::class,
|
||||
evaluationConfiguration = BenchmarkEvaluationConfiguration::class
|
||||
)
|
||||
abstract class BenchmarkTemplate(
|
||||
vararg args: String,
|
||||
private val projectName: String,
|
||||
private val projectGitUrl: String,
|
||||
private val gitCommitSha: String,
|
||||
) {
|
||||
private val workingDir = File(args.first())
|
||||
private val gradleProfilerDir = workingDir.resolve("gradle-profiler")
|
||||
private val projectRepoDir = workingDir.resolve(projectName)
|
||||
private val scenariosDir = workingDir.resolve("scenarios")
|
||||
private val benchmarkOutputsDir = workingDir.resolve("outputs")
|
||||
|
||||
private val gitOperationsPrinter = TextProgressMonitor()
|
||||
|
||||
fun runAllBenchmarks(
|
||||
suite: ScenarioSuite,
|
||||
benchmarks: Map<BenchmarkName, PatchFile?>
|
||||
) {
|
||||
printStartingMessage()
|
||||
downloadGradleProfilerIfNotAvailable()
|
||||
checkoutRepository()
|
||||
|
||||
val results = benchmarks.map { (name, patchFile) ->
|
||||
repoReset()
|
||||
patchFile?.let { repoApplyPatch(it) }
|
||||
runBenchmark(suite, name)
|
||||
}
|
||||
aggregateBenchmarkResults(*results.toTypedArray())
|
||||
|
||||
printEndingMessage()
|
||||
}
|
||||
|
||||
fun printStartingMessage() {
|
||||
println("$STEP_SEPARATOR Starting Gradle regression benchmark for $projectName $STEP_SEPARATOR")
|
||||
}
|
||||
|
||||
fun printEndingMessage() {
|
||||
println("$STEP_SEPARATOR Gradle regression benchmark for $projectName has ended $STEP_SEPARATOR")
|
||||
}
|
||||
|
||||
fun downloadGradleProfilerIfNotAvailable() {
|
||||
if (gradleProfilerDir.exists()) {
|
||||
println("Gradle profiler has been already downloaded")
|
||||
} else {
|
||||
downloadAndExtractGradleProfiler()
|
||||
}
|
||||
}
|
||||
|
||||
fun checkoutRepository(): File {
|
||||
val git = if (projectRepoDir.exists()) {
|
||||
println("Repository is available, resetting it state")
|
||||
Git.open(projectRepoDir).also {
|
||||
it.reset()
|
||||
.setMode(ResetCommand.ResetType.HARD)
|
||||
.setProgressMonitor(gitOperationsPrinter)
|
||||
.call()
|
||||
}
|
||||
} else {
|
||||
println("Running git checkout for $projectGitUrl")
|
||||
projectRepoDir.mkdirs()
|
||||
Git.cloneRepository()
|
||||
.setDirectory(projectRepoDir)
|
||||
.setCloneSubmodules(true)
|
||||
.setProgressMonitor(gitOperationsPrinter)
|
||||
.setURI(projectGitUrl)
|
||||
.call()
|
||||
}
|
||||
|
||||
git.checkout()
|
||||
.setName(gitCommitSha)
|
||||
.setProgressMonitor(gitOperationsPrinter)
|
||||
.call()
|
||||
|
||||
return projectRepoDir
|
||||
}
|
||||
|
||||
fun repoApplyPatch(patchFile: PatchFile) {
|
||||
println("Applying patch $patchFile to repository")
|
||||
val patch = File(patchFile)
|
||||
Git.open(projectRepoDir)
|
||||
.apply()
|
||||
.setPatch(patch.inputStream())
|
||||
.call()
|
||||
}
|
||||
|
||||
fun repoReset() {
|
||||
println("Hard resetting project repo")
|
||||
Git.open(projectRepoDir)
|
||||
.reset()
|
||||
.setMode(ResetCommand.ResetType.HARD)
|
||||
.setProgressMonitor(gitOperationsPrinter)
|
||||
.call()
|
||||
}
|
||||
|
||||
fun runBenchmark(
|
||||
scenarioSuite: ScenarioSuite,
|
||||
benchmarkName: BenchmarkName,
|
||||
dryRun: Boolean = false
|
||||
): BenchmarkResult {
|
||||
println("Staring benchmark $benchmarkName")
|
||||
val normalizedBenchmarkName = benchmarkName.normalizeTitle
|
||||
if (!scenariosDir.exists()) scenariosDir.mkdirs()
|
||||
val scenarioFile = scenariosDir.resolve("${projectName}_$normalizedBenchmarkName.scenario")
|
||||
scenarioSuite.writeTo(scenarioFile)
|
||||
val benchmarkOutputDir = benchmarkOutputsDir
|
||||
.resolve(projectName)
|
||||
.resolve(normalizedBenchmarkName)
|
||||
.also {
|
||||
if (it.exists()) it.deleteRecursively()
|
||||
it.mkdirs()
|
||||
}
|
||||
|
||||
val profilerProcessBuilder = ProcessBuilder()
|
||||
.directory(workingDir)
|
||||
.inheritIO()
|
||||
.command(
|
||||
gradleProfilerBin.absolutePath,
|
||||
"--benchmark",
|
||||
"--measure-config-time",
|
||||
"--project-dir",
|
||||
projectRepoDir.absolutePath,
|
||||
"--scenario-file",
|
||||
scenarioFile.absolutePath,
|
||||
"--output-dir",
|
||||
benchmarkOutputDir.absolutePath
|
||||
).also {
|
||||
// Required, so 'gradle-profiler' will use toolchain JDK instead of current user one
|
||||
it.environment()["JAVA_HOME"] = System.getProperty("java.home")
|
||||
if (dryRun) it.command().add("--dry-run")
|
||||
}
|
||||
|
||||
val profilerProcess = profilerProcessBuilder.start()
|
||||
// Stop profiler on script stop
|
||||
Runtime.getRuntime().addShutdownHook(Thread {
|
||||
profilerProcess.destroy()
|
||||
})
|
||||
profilerProcess.waitFor()
|
||||
|
||||
if (profilerProcess.exitValue() != 0) {
|
||||
throw BenchmarkFailedException(profilerProcess.exitValue())
|
||||
}
|
||||
|
||||
println("Benchmark $benchmarkName has ended")
|
||||
return BenchmarkResult(
|
||||
benchmarkName,
|
||||
benchmarkOutputDir.resolve("benchmark.csv")
|
||||
)
|
||||
}
|
||||
|
||||
fun aggregateBenchmarkResults(vararg benchmarkResults: BenchmarkResult) {
|
||||
println("Aggregating benchmark results...")
|
||||
val results = benchmarkResults
|
||||
.map { result ->
|
||||
DataFrame
|
||||
.readCSV(result.result)
|
||||
.flipColumnsWithRows()
|
||||
.remove("version", "tasks") // removing unused columns
|
||||
.remove { nameContains("warm-up build") } // removing warm-up times
|
||||
.add("median build time") { // squashing build times into median build time
|
||||
valuesOf<Int>().median(typeOf<Int>())
|
||||
}
|
||||
.remove { nameContains("measured build") } // removing iterations results
|
||||
.groupBy("scenario").aggregate { // merging configuration and build times into one row
|
||||
first { it.values().contains("task start") }["median build time"] into "tasks start median time"
|
||||
first { it.values().contains("execution") }["median build time"] into "execution median time"
|
||||
}
|
||||
.add("benchmark") { result.name }
|
||||
}
|
||||
.concat()
|
||||
.groupBy("scenario").aggregate { // Merging scenarios from different benchmarks into one row
|
||||
forEachRow { row ->
|
||||
row["tasks start median time"] into "Configuration: ${row["benchmark"]}"
|
||||
row["execution median time"] into "Execution: ${row["benchmark"]}"
|
||||
}
|
||||
}
|
||||
.sortBy("scenario")
|
||||
.rename("scenario" to "Scenario")
|
||||
.reorderColumnsBy {
|
||||
// "Scenario" column should always be in the first place
|
||||
if (it.name() == "Scenario") "0000Scenario" else it.name()
|
||||
}
|
||||
|
||||
println("Benchmark results:")
|
||||
println(results.print(borders = true))
|
||||
|
||||
val benchmarkOutputDir = benchmarkOutputsDir.resolve(projectName)
|
||||
val benchmarkCsv = benchmarkOutputDir.resolve("$projectName.csv")
|
||||
results.writeCSV(benchmarkCsv)
|
||||
val benchmarkHtml = benchmarkOutputDir.resolve("$projectName.html")
|
||||
benchmarkHtml.writeText(
|
||||
results.toHTML(
|
||||
includeInit = true
|
||||
).toString()
|
||||
)
|
||||
println("Result in csv format: ${benchmarkCsv.absolutePath}")
|
||||
println("Result in html format: ${benchmarkHtml.absolutePath}")
|
||||
}
|
||||
|
||||
private fun downloadAndExtractGradleProfiler() {
|
||||
println("Downloading gradle-profiler into ${gradleProfilerDir.absolutePath}")
|
||||
|
||||
gradleProfilerDir.mkdirs()
|
||||
|
||||
val okHttpClient = OkHttpClient()
|
||||
val request = Request.Builder()
|
||||
.get()
|
||||
.url(GRADLE_PROFILER_URL)
|
||||
.build()
|
||||
val response = okHttpClient.newCall(request).execute()
|
||||
if (!response.isSuccessful) {
|
||||
throw IOException("Failed to download gradle-profiler, error code: ${response.code}")
|
||||
}
|
||||
|
||||
val contentLength = response.body!!.contentLength()
|
||||
var downloadedLength = 0L
|
||||
print("Downloading: ")
|
||||
response.body!!.byteStream().buffered().use { responseContent ->
|
||||
ZipInputStream(responseContent).use { zip ->
|
||||
var zipEntry = zip.nextEntry
|
||||
while (zipEntry != null) {
|
||||
if (zipEntry.isDirectory) {
|
||||
gradleProfilerDir.resolve(zipEntry.name.dropLeadingDir).also { it.mkdirs() }
|
||||
} else {
|
||||
gradleProfilerDir.resolve(zipEntry.name.dropLeadingDir).outputStream().buffered().use {
|
||||
zip.copyTo(it)
|
||||
}
|
||||
}
|
||||
downloadedLength += zipEntry.compressedSize
|
||||
print("..${downloadedLength * 100 / contentLength}%")
|
||||
zip.closeEntry()
|
||||
zipEntry = zip.nextEntry
|
||||
}
|
||||
}
|
||||
print("\n")
|
||||
}
|
||||
gradleProfilerBin.setExecutable(true)
|
||||
println("Finished downloading gradle-profiler")
|
||||
}
|
||||
|
||||
private fun ScenarioSuite.writeTo(output: File) {
|
||||
output.writeText(
|
||||
"""
|
||||
|default-scenarios = [${scenarios.joinToString { "\"${it.title.normalizeTitle}\"" }}]
|
||||
|
|
||||
|${scenarios.joinToString(separator = "\n") { it.write() }}
|
||||
""".trimMargin()
|
||||
)
|
||||
}
|
||||
|
||||
private fun Scenario.write(): String =
|
||||
"""
|
||||
|${title.normalizeTitle} {
|
||||
| title = "$title"
|
||||
| warm-ups = $warmups
|
||||
| iterations = $iterations
|
||||
| tasks = [${tasks.joinToString { "\"$it\"" }}]
|
||||
| ${if (gradleArgs.isNotEmpty()) "gradle-args = [${gradleArgs.joinToString { "\"$it\"" }}]" else ""}
|
||||
| ${if (cleanupTasks.isNotEmpty()) "cleanup-tasks = [${cleanupTasks.joinToString { "\"$it\"" }}]" else ""}
|
||||
| ${if (applyAbiChange.isNotEmpty()) "apply-abi-change-to = [${applyAbiChange.joinToString { "\"$it\"" }}]" else ""}
|
||||
|}
|
||||
""".trimMargin()
|
||||
|
||||
private val String.dropLeadingDir: String get() = substringAfter('/')
|
||||
private val String.normalizeTitle: String get() = lowercase().replace(" ", "_")
|
||||
|
||||
private fun DataFrame<*>.flipColumnsWithRows(): DataFrame<*> {
|
||||
val firstColumn = columns().first()
|
||||
return DataFrameBuilder(
|
||||
listOf(firstColumn.name) + firstColumn.values.map { it.toString() }
|
||||
).withColumns { columnName ->
|
||||
when {
|
||||
columnName == "scenario" -> DataColumn.createValueColumn(
|
||||
columnName,
|
||||
columns().map {
|
||||
it.name.dropLastWhile { it.isDigit() }
|
||||
}.drop(1)
|
||||
)
|
||||
columnName == "version" -> DataColumn.createValueColumn(
|
||||
columnName,
|
||||
rowToColumn(columnName) { it.toString() }
|
||||
)
|
||||
columnName == "tasks" -> DataColumn.createValueColumn(
|
||||
columnName,
|
||||
rowToColumn(columnName) { it.toString() }
|
||||
)
|
||||
columnName == "value" -> DataColumn.createValueColumn(
|
||||
columnName,
|
||||
rowToColumn(columnName) { it.toString() }
|
||||
)
|
||||
columnName.startsWith("warm-up build") -> DataColumn.createValueColumn(
|
||||
columnName,
|
||||
rowToColumn(columnName) { it.toString().toInt() }
|
||||
)
|
||||
columnName.startsWith("measured build") -> DataColumn.createValueColumn(
|
||||
columnName,
|
||||
rowToColumn(columnName) { it.toString().toInt() }
|
||||
)
|
||||
else -> throw IllegalArgumentException("Unknown column name: $columnName")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun <T : Any> DataFrame<*>.rowToColumn(
|
||||
rowName: String,
|
||||
typeConversion: (Any?) -> T
|
||||
): List<T> =
|
||||
rows().first { it.values().first() == rowName }.values().drop(1).map { typeConversion(it) }
|
||||
|
||||
private val gradleProfilerBin: File
|
||||
get() = gradleProfilerDir
|
||||
.resolve("bin")
|
||||
.run {
|
||||
if (System.getProperty("os.name").contains("windows", ignoreCase = true)) {
|
||||
resolve("gradle-profiler.bat")
|
||||
} else {
|
||||
resolve("gradle-profiler").also { it.setExecutable(true) }
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val STEP_SEPARATOR = "###############"
|
||||
private const val GRADLE_PROFILER_VERSION = "0.16.0"
|
||||
private const val GRADLE_PROFILER_URL: String =
|
||||
"https://repo.gradle.org/gradle/ext-releases-local/org/gradle/profiler/gradle-profiler/$GRADLE_PROFILER_VERSION/gradle-profiler-$GRADLE_PROFILER_VERSION.zip"
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
typealias BenchmarkName = String
|
||||
typealias PatchFile = String
|
||||
|
||||
class BenchmarkFailedException(exitCode: Int) : Exception(
|
||||
"Benchmark process was not finished successfully with $exitCode exit code."
|
||||
)
|
||||
|
||||
class BenchmarkResult(
|
||||
val name: BenchmarkName,
|
||||
val result: File
|
||||
)
|
||||
|
||||
@Target(AnnotationTarget.FILE)
|
||||
@Retention(AnnotationRetention.SOURCE)
|
||||
annotation class BenchmarkProject(
|
||||
val name: String,
|
||||
val gitUrl: String,
|
||||
val gitCommitSha: String
|
||||
)
|
||||
|
||||
class BenchmarkScriptDefinition : ScriptCompilationConfiguration(
|
||||
{
|
||||
defaultImports(
|
||||
listOf(
|
||||
BenchmarkProject::class.qualifiedName!!,
|
||||
"org.jetbrains.kotlin.gradle.benchmark.suite"
|
||||
)
|
||||
)
|
||||
refineConfiguration {
|
||||
onAnnotations(BenchmarkProject::class, handler = BenchmarkScriptConfigurator())
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
class BenchmarkEvaluationConfiguration : ScriptEvaluationConfiguration(
|
||||
{
|
||||
refineConfigurationBeforeEvaluate { context ->
|
||||
val compileConf = context.evaluationConfiguration[compilationConfiguration]
|
||||
?: return@refineConfigurationBeforeEvaluate makeFailureResult("Script compilation configuration is not available")
|
||||
|
||||
val name: String = compileConf[benchmarkProjectName]!!
|
||||
val gitUrl: String = compileConf[benchmarkGitUrl]!!
|
||||
val gitCommitSha: String = compileConf[benchmarkGitCommitSha]!!
|
||||
|
||||
context.evaluationConfiguration.with evalConf@{
|
||||
constructorArgs(name, gitUrl, gitCommitSha)
|
||||
}.asSuccess()
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
internal class BenchmarkScriptConfigurator : RefineScriptCompilationConfigurationHandler {
|
||||
override fun invoke(
|
||||
context: ScriptConfigurationRefinementContext
|
||||
): ResultWithDiagnostics<ScriptCompilationConfiguration> {
|
||||
val benchmarkProject = context.collectedData
|
||||
?.get(ScriptCollectedData.foundAnnotations)
|
||||
?.find { it is BenchmarkProject } as? BenchmarkProject
|
||||
?: return run {
|
||||
makeFailureResult("Script does not contain ${BenchmarkProject::name} annotation!")
|
||||
}
|
||||
|
||||
return ScriptCompilationConfiguration(context.compilationConfiguration) {
|
||||
data[benchmarkProjectName] = benchmarkProject.name
|
||||
data[benchmarkGitUrl] = benchmarkProject.gitUrl
|
||||
data[benchmarkGitCommitSha] = benchmarkProject.gitCommitSha
|
||||
}.asSuccess()
|
||||
}
|
||||
}
|
||||
|
||||
private val benchmarkProjectName by PropertiesCollection.key<String>()
|
||||
private val benchmarkGitUrl by PropertiesCollection.key<String>()
|
||||
private val benchmarkGitCommitSha by PropertiesCollection.key<String>()
|
||||
Reference in New Issue
Block a user