From 6d544cd6677a51a74a37b199d61fd5b8386216f7 Mon Sep 17 00:00:00 2001 From: Mark Punzalan Date: Wed, 24 Apr 2019 00:18:22 -0700 Subject: [PATCH] Add option to use regexes for filtering and add property for passing compiler args when running benchmarks. (#2894) --- HACKING.md | 19 ++++++++++--- .../buildSrc/src/main/kotlin/RunJvmTask.kt | 14 ++++++---- .../src/main/kotlin/RunKotlinNativeTask.kt | 15 ++++++---- performance/buildSrc/src/main/kotlin/Utils.kt | 5 ++++ performance/cinterop/src/main/kotlin/main.kt | 5 ++-- performance/gradle/benchmark.gradle | 6 ++-- performance/ring/src/main/kotlin/main.kt | 5 ++-- .../jetbrains/benchmarksLauncher/launcher.kt | 28 +++++++++++-------- 8 files changed, 64 insertions(+), 33 deletions(-) diff --git a/HACKING.md b/HACKING.md index 61a3c278498..ba1adb20ed9 100644 --- a/HACKING.md +++ b/HACKING.md @@ -110,16 +110,27 @@ To update the blackbox compiler tests set TeamCity build number in `gradle.prope ../gradlew :cinterop:konanRun - **konanRun** task has parameter `filter` which allows to run only some subset of benchmarks : + **konanRun** task has parameter `filter` which allows to run only some subset of benchmarks: ../gradlew :cinterop:konanRun --filter=struct,macros + Or you can use `filterRegex` if you want to specify the filter as regexes: + + ../gradlew :ring:konanRun --filterRegex=String.*,Loop.* + There are also tasks for running benchmarks on JVM (pay attention, some benchmarks e.g. cinterop benchmarks can't be run on JVM) ../gradlew jvmRun - Files with results of benchmarks run are saved in `performance/build`. **nativeReport.json** - for konanRun and **jvmReport.json** for jvmRun. - + Files with results of benchmarks run are saved in `performance/build/nativeReport.json` for konanRun and `jvmReport.json` for jvmRun. + You can change the output filename by setting the `nativeJson` property for konanRun and `jvmJson` for jvmRun: + + ../gradlew :ring:konanRun --filter=String.*,Loop.* -PnativeJson=stringsAndLoops.json + + You can use the `compilerArgs` property to pass flags to the compiler used to compile the benchmarks: + + ../gradlew konanRun -PcompilerArgs="--time -g" + To compare different results run benchmarksAnalyzer tool: cd tools/benchmarksAnalyzer/build/bin//benchmarksAnalyzerReleaseExecutable/ @@ -148,4 +159,4 @@ To update the blackbox compiler tests set TeamCity build number in `gradle.prope teamcity:id:42491947:nativeReport.json Pay attention, user and password information(with flag `-u :`) should be provided to get data from TeamCity. - \ No newline at end of file + diff --git a/performance/buildSrc/src/main/kotlin/RunJvmTask.kt b/performance/buildSrc/src/main/kotlin/RunJvmTask.kt index 60b92bfce85..463feb0762f 100644 --- a/performance/buildSrc/src/main/kotlin/RunJvmTask.kt +++ b/performance/buildSrc/src/main/kotlin/RunJvmTask.kt @@ -13,27 +13,31 @@ import org.jetbrains.kotlin.gradle.plugin.KotlinTarget import javax.inject.Inject import java.io.File -open class RunJvmTask: JavaExec() { +open class RunJvmTask : JavaExec() { var outputFileName: String? = null @Input - @Option(option = "filter", description = "filter") + @Option(option = "filter", description = "Benchmarks to run (comma-separated)") var filter: String = "" + @Input + @Option(option = "filterRegex", description = "Benchmarks to run, described by regular expressions (comma-separated)") + var filterRegex: String = "" override fun configure(configureClosure: Closure): Task { return super.configure(configureClosure) } private fun executeTask(output: java.io.OutputStream? = null) { - val filterArgs = filter.split("\\s*,\\s*".toRegex()) - .map{ if (it.isNotEmpty()) listOf("-f", it) else listOf(null) }.flatten().filterNotNull() + val filterArgs = filter.splitCommaSeparatedOption("-f") + val filterRegexArgs = filterRegex.splitCommaSeparatedOption("-fr") args(filterArgs) + args(filterRegexArgs) exec() } @TaskAction fun run() { if (outputFileName != null) - File(outputFileName).outputStream().use { output -> executeTask(output)} + File(outputFileName).outputStream().use { output -> executeTask(output) } else executeTask() } diff --git a/performance/buildSrc/src/main/kotlin/RunKotlinNativeTask.kt b/performance/buildSrc/src/main/kotlin/RunKotlinNativeTask.kt index b85cd592fab..c9d719a33f5 100644 --- a/performance/buildSrc/src/main/kotlin/RunKotlinNativeTask.kt +++ b/performance/buildSrc/src/main/kotlin/RunKotlinNativeTask.kt @@ -15,14 +15,17 @@ import java.io.File open class RunKotlinNativeTask @Inject constructor( private val curTarget: KotlinTarget -): DefaultTask() { +) : DefaultTask() { var buildType = "RELEASE" var workingDir: Any = project.projectDir var outputFileName: String? = null @Input - @Option(option = "filter", description = "filter") + @Option(option = "filter", description = "Benchmarks to run (comma-separated)") var filter: String = "" + @Input + @Option(option = "filterRegex", description = "Benchmarks to run, described by regular expressions (comma-separated)") + var filterRegex: String = "" private var curArgs: List = emptyList() private val curEnvironment: MutableMap = mutableMapOf() @@ -46,11 +49,11 @@ open class RunKotlinNativeTask @Inject constructor( } private fun executeTask(output: java.io.OutputStream? = null) { - val filterArgs = filter.split("\\s*,\\s*".toRegex()) - .map{ if (it.isNotEmpty()) listOf("-f", it) else listOf(null) }.flatten().filterNotNull() + val filterArgs = filter.splitCommaSeparatedOption("-f") + val filterRegexArgs = filterRegex.splitCommaSeparatedOption("-fr") project.exec { it.executable = curTarget.compilations.main.getBinary("EXECUTABLE", buildType).toString() - it.args = curArgs + filterArgs + it.args = curArgs + filterArgs + filterRegexArgs it.environment = curEnvironment it.workingDir(workingDir) if (output != null) @@ -61,7 +64,7 @@ open class RunKotlinNativeTask @Inject constructor( @TaskAction fun run() { if (outputFileName != null) - File(outputFileName).outputStream().use { output -> executeTask(output)} + File(outputFileName).outputStream().use { output -> executeTask(output) } else executeTask() } diff --git a/performance/buildSrc/src/main/kotlin/Utils.kt b/performance/buildSrc/src/main/kotlin/Utils.kt index 94a3250518b..b81354377c3 100644 --- a/performance/buildSrc/src/main/kotlin/Utils.kt +++ b/performance/buildSrc/src/main/kotlin/Utils.kt @@ -29,6 +29,11 @@ fun Array.runCommand(workingDir: File = File("."), } } +fun String.splitCommaSeparatedOption(optionName: String) = + split("\\s*,\\s*".toRegex()).map { + if (it.isNotEmpty()) listOf(optionName, it) else listOf(null) + }.flatten().filterNotNull() + data class Commit(val revision: String, val developer: String, val webUrlWithDescription: String) val teamCityUrl = "http://buildserver.labs.intellij.net" diff --git a/performance/cinterop/src/main/kotlin/main.kt b/performance/cinterop/src/main/kotlin/main.kt index da884e7aff3..24ec3728847 100644 --- a/performance/cinterop/src/main/kotlin/main.kt +++ b/performance/cinterop/src/main/kotlin/main.kt @@ -21,7 +21,7 @@ import org.jetbrains.structsBenchmarks.* import org.jetbrains.typesBenchmarks.* import org.jetbrains.kliopt.* -class CinteropLauncher(numWarmIterations: Int, numberOfAttempts: Int, prefix: String): Launcher(numWarmIterations, numberOfAttempts, prefix) { +class CinteropLauncher(numWarmIterations: Int, numberOfAttempts: Int, prefix: String) : Launcher(numWarmIterations, numberOfAttempts, prefix) { val stringBenchmark = StringBenchmark() val intMatrixBenchmark = IntMatrixBenchmark() val intBenchmark = IntBenchmark() @@ -43,6 +43,7 @@ class CinteropLauncher(numWarmIterations: Int, numberOfAttempts: Int, prefix: St fun main(args: Array) { BenchmarksRunner.runBenchmarks(args, { parser: ArgParser -> - CinteropLauncher(parser.get("warmup")!!, parser.get("repeat")!!, parser.get("prefix")!!).launch(parser.getAll("filter")) + CinteropLauncher(parser.get("warmup")!!, parser.get("repeat")!!, parser.get("prefix")!!) + .launch(parser.getAll("filter"), parser.getAll("filterRegex")) }) } \ No newline at end of file diff --git a/performance/gradle/benchmark.gradle b/performance/gradle/benchmark.gradle index 00616a0ea46..4872321d1b4 100644 --- a/performance/gradle/benchmark.gradle +++ b/performance/gradle/benchmark.gradle @@ -13,7 +13,6 @@ repositories { } - private def determinePreset() { def preset = MPPTools.defaultHostPreset(project) println("$project has been configured for ${preset.name} platform.") @@ -53,14 +52,15 @@ kotlin { compilations.all { tasks[compileKotlinTaskName].kotlinOptions { jvmTarget = '1.8' + suppressWarnings = true + freeCompilerArgs = (project.hasProperty('compilerArgs') ? compilerArgs.split() : []) } - tasks[compileKotlinTaskName].kotlinOptions.suppressWarnings = true } } fromPreset(hostPreset, 'native') { compilations.main.outputKinds('EXECUTABLE') - compilations.main.extraOpts '-opt' + compilations.main.extraOpts = (project.hasProperty('compilerArgs') ? compilerArgs.split() : []) + '-opt' compilations.main.buildTypes = [RELEASE] } } diff --git a/performance/ring/src/main/kotlin/main.kt b/performance/ring/src/main/kotlin/main.kt index 633be0ffe58..2f966448581 100644 --- a/performance/ring/src/main/kotlin/main.kt +++ b/performance/ring/src/main/kotlin/main.kt @@ -20,7 +20,7 @@ import octoTest import org.jetbrains.benchmarksLauncher.* import org.jetbrains.kliopt.* -class RingLauncher(numWarmIterations: Int, numberOfAttempts: Int, prefix: String): Launcher(numWarmIterations, numberOfAttempts, prefix) { +class RingLauncher(numWarmIterations: Int, numberOfAttempts: Int, prefix: String) : Launcher(numWarmIterations, numberOfAttempts, prefix) { val abstractMethodBenchmark = AbstractMethodBenchmark() val classArrayBenchmark = ClassArrayBenchmark() val classBaselineBenchmark = ClassBaselineBenchmark() @@ -232,6 +232,7 @@ class RingLauncher(numWarmIterations: Int, numberOfAttempts: Int, prefix: String fun main(args: Array) { BenchmarksRunner.runBenchmarks(args, { parser: ArgParser -> - RingLauncher(parser.get("warmup")!!, parser.get("repeat")!!, parser.get("prefix")!!).launch(parser.getAll("filter")) + RingLauncher(parser.get("warmup")!!, parser.get("repeat")!!, parser.get("prefix")!!) + .launch(parser.getAll("filter"), parser.getAll("filterRegex")) }) } \ No newline at end of file diff --git a/performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/launcher.kt b/performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/launcher.kt index e486bd9ed06..6c656fbf4ba 100644 --- a/performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/launcher.kt +++ b/performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/launcher.kt @@ -25,13 +25,18 @@ abstract class Launcher(val numWarmIterations: Int, val numberOfAttempts: Int, v abstract val benchmarks: BenchmarksCollection - protected val benchmarkResults = mutableListOf() - - fun launch(benchmarksToRun: Collection? = null): List { - val runningBenchmarks = benchmarksToRun ?: benchmarks.keys - runningBenchmarks.forEach { - val benchmark = benchmarks[it] - benchmark ?: error("Benchmark $it wasn't found!") + fun launch(filters: Collection? = null, + filterRegexes: Collection? = null): List { + val regexes = filterRegexes?.map { it.toRegex() } ?: listOf() + val filterSet = filters?.toHashSet() ?: hashSetOf() + // Filter benchmarks using given filters, or run all benchmarks if none were given. + val runningBenchmarks = if (filterSet.isNotEmpty() || regexes.isNotEmpty()) { + benchmarks.filterKeys { benchmark -> benchmark in filterSet || regexes.any { it.matches(benchmark) } } + } else benchmarks + if (runningBenchmarks.isEmpty()) + error("No matching benchmarks found") + val benchmarkResults = mutableListOf() + for ((name, benchmark) in runningBenchmarks) { var i = numWarmIterations while (i-- > 0) benchmark() cleanup() @@ -60,7 +65,7 @@ abstract class Launcher(val numWarmIterations: Int, val numberOfAttempts: Int, v val scaledTime = time * 1.0 / autoEvaluatedNumberOfMeasureIteration samples[k] = scaledTime // Save benchmark object - benchmarkResults.add(BenchmarkResult("$prefix$it", BenchmarkResult.Status.PASSED, + benchmarkResults.add(BenchmarkResult("$prefix$name", BenchmarkResult.Status.PASSED, scaledTime / 1000, BenchmarkResult.Metric.EXECUTION_TIME, scaledTime / 1000, k + 1, numWarmIterations)) } @@ -73,10 +78,11 @@ object BenchmarksRunner { fun parse(args: Array): ArgParser { val options = listOf( OptionDescriptor(ArgType.Int(), "warmup", "w", "Number of warm up iterations", "20"), - OptionDescriptor(ArgType.Int(), "repeat", "r", "Number of each becnhmark run", "60"), + OptionDescriptor(ArgType.Int(), "repeat", "r", "Number of each benchmark run", "60"), OptionDescriptor(ArgType.String(), "prefix", "p", "Prefix added to benchmark name", ""), OptionDescriptor(ArgType.String(), "output", "o", "Output file"), - OptionDescriptor(ArgType.String(), "filter", "f", "Benchmark to run", isMultiple = true) + OptionDescriptor(ArgType.String(), "filter", "f", "Benchmark to run", isMultiple = true), + OptionDescriptor(ArgType.String(), "filterRegex", "fr", "Benchmark to run, described by a regular expression", isMultiple = true) ) // Parse args. @@ -99,4 +105,4 @@ object BenchmarksRunner { val results = run(parser) collect(results, parser) } -} \ No newline at end of file +}