Add option to use regexes for filtering and add property for passing compiler args when running benchmarks. (#2894)

HACKING.md

@@ -110,16 +110,27 @@ To update the blackbox compiler tests set TeamCity build number in `gradle.prope
  
     ../gradlew :cinterop:konanRun
     
- **konanRun** task has parameter `filter` which allows to run only some subset of benchmarks :
+ **konanRun** task has parameter `filter` which allows to run only some subset of benchmarks:
  
     ../gradlew :cinterop:konanRun --filter=struct,macros
     
+ Or you can use `filterRegex` if you want to specify the filter as regexes:
+ 
+    ../gradlew :ring:konanRun --filterRegex=String.*,Loop.*
+    
  There are also tasks for running benchmarks on JVM (pay attention, some benchmarks e.g. cinterop benchmarks can't be run on JVM)
  
     ../gradlew jvmRun
     
- Files with results of benchmarks run are saved in `performance/build`. **nativeReport.json** - for konanRun and **jvmReport.json** for jvmRun.
+ Files with results of benchmarks run are saved in `performance/build/nativeReport.json` for konanRun and `jvmReport.json` for jvmRun.
-    
+ You can change the output filename by setting the `nativeJson` property for konanRun and `jvmJson` for jvmRun:
+
+    ../gradlew :ring:konanRun --filter=String.*,Loop.* -PnativeJson=stringsAndLoops.json
+
+ You can use the `compilerArgs` property to pass flags to the compiler used to compile the benchmarks:
+
+    ../gradlew konanRun -PcompilerArgs="--time -g"
+
  To compare different results run benchmarksAnalyzer tool:
  
     cd tools/benchmarksAnalyzer/build/bin/<target>/benchmarksAnalyzerReleaseExecutable/
@@ -148,4 +159,4 @@ To update the blackbox compiler tests set TeamCity build number in `gradle.prope
      teamcity:id:42491947:nativeReport.json
      
  Pay attention, user and password information(with flag `-u <username>:<password>`) should be provided to get data from TeamCity.
-    
+    

performance/buildSrc/src/main/kotlin/RunJvmTask.kt

@@ -13,27 +13,31 @@ import org.jetbrains.kotlin.gradle.plugin.KotlinTarget
 import javax.inject.Inject
 import java.io.File
 
-open class RunJvmTask: JavaExec() {
+open class RunJvmTask : JavaExec() {
     var outputFileName: String? = null
     @Input
-    @Option(option = "filter", description = "filter")
+    @Option(option = "filter", description = "Benchmarks to run (comma-separated)")
     var filter: String = ""
+    @Input
+    @Option(option = "filterRegex", description = "Benchmarks to run, described by regular expressions (comma-separated)")
+    var filterRegex: String = ""
 
     override fun configure(configureClosure: Closure<Any>): Task {
         return super.configure(configureClosure)
     }
 
     private fun executeTask(output: java.io.OutputStream? = null) {
-        val filterArgs = filter.split("\\s*,\\s*".toRegex())
+        val filterArgs = filter.splitCommaSeparatedOption("-f")
-                .map{ if (it.isNotEmpty()) listOf("-f", it) else listOf(null) }.flatten().filterNotNull()
+        val filterRegexArgs = filterRegex.splitCommaSeparatedOption("-fr")
         args(filterArgs)
+        args(filterRegexArgs)
         exec()
     }
 
     @TaskAction
     fun run() {
         if (outputFileName != null)
-            File(outputFileName).outputStream().use { output ->  executeTask(output)}
+            File(outputFileName).outputStream().use { output -> executeTask(output) }
         else
             executeTask()
     }

performance/buildSrc/src/main/kotlin/RunKotlinNativeTask.kt

@@ -15,14 +15,17 @@ import java.io.File
 
 open class RunKotlinNativeTask @Inject constructor(
         private val curTarget: KotlinTarget
-): DefaultTask() {
+) : DefaultTask() {
 
     var buildType = "RELEASE"
     var workingDir: Any = project.projectDir
     var outputFileName: String? = null
     @Input
-    @Option(option = "filter", description = "filter")
+    @Option(option = "filter", description = "Benchmarks to run (comma-separated)")
     var filter: String = ""
+    @Input
+    @Option(option = "filterRegex", description = "Benchmarks to run, described by regular expressions (comma-separated)")
+    var filterRegex: String = ""
 
     private var curArgs: List<String> = emptyList()
     private val curEnvironment: MutableMap<String, Any> = mutableMapOf()
@@ -46,11 +49,11 @@ open class RunKotlinNativeTask @Inject constructor(
     }
 
     private fun executeTask(output: java.io.OutputStream? = null) {
-        val filterArgs = filter.split("\\s*,\\s*".toRegex())
+        val filterArgs = filter.splitCommaSeparatedOption("-f")
-                .map{ if (it.isNotEmpty()) listOf("-f", it) else listOf(null) }.flatten().filterNotNull()
+        val filterRegexArgs = filterRegex.splitCommaSeparatedOption("-fr")
         project.exec {
             it.executable = curTarget.compilations.main.getBinary("EXECUTABLE", buildType).toString()
-            it.args = curArgs + filterArgs
+            it.args = curArgs + filterArgs + filterRegexArgs
             it.environment = curEnvironment
             it.workingDir(workingDir)
             if (output != null)
@@ -61,7 +64,7 @@ open class RunKotlinNativeTask @Inject constructor(
     @TaskAction
     fun run() {
         if (outputFileName != null)
-            File(outputFileName).outputStream().use { output ->  executeTask(output)}
+            File(outputFileName).outputStream().use { output -> executeTask(output) }
         else
             executeTask()
     }

performance/buildSrc/src/main/kotlin/Utils.kt

@@ -29,6 +29,11 @@ fun Array<String>.runCommand(workingDir: File = File("."),
     }
 }
 
+fun String.splitCommaSeparatedOption(optionName: String) =
+        split("\\s*,\\s*".toRegex()).map {
+            if (it.isNotEmpty()) listOf(optionName, it) else listOf(null)
+        }.flatten().filterNotNull()
+
 data class Commit(val revision: String, val developer: String, val webUrlWithDescription: String)
 
 val teamCityUrl = "http://buildserver.labs.intellij.net"

performance/cinterop/src/main/kotlin/main.kt

@@ -21,7 +21,7 @@ import org.jetbrains.structsBenchmarks.*
 import org.jetbrains.typesBenchmarks.*
 import org.jetbrains.kliopt.*
 
-class CinteropLauncher(numWarmIterations: Int, numberOfAttempts: Int, prefix: String): Launcher(numWarmIterations, numberOfAttempts, prefix) {
+class CinteropLauncher(numWarmIterations: Int, numberOfAttempts: Int, prefix: String) : Launcher(numWarmIterations, numberOfAttempts, prefix) {
     val stringBenchmark = StringBenchmark()
     val intMatrixBenchmark = IntMatrixBenchmark()
     val intBenchmark = IntBenchmark()
@@ -43,6 +43,7 @@ class CinteropLauncher(numWarmIterations: Int, numberOfAttempts: Int, prefix: St
 
 fun main(args: Array<String>) {
     BenchmarksRunner.runBenchmarks(args, { parser: ArgParser ->
-        CinteropLauncher(parser.get<Int>("warmup")!!, parser.get<Int>("repeat")!!, parser.get<String>("prefix")!!).launch(parser.getAll<String>("filter"))
+        CinteropLauncher(parser.get<Int>("warmup")!!, parser.get<Int>("repeat")!!, parser.get<String>("prefix")!!)
+                .launch(parser.getAll<String>("filter"), parser.getAll<String>("filterRegex"))
     })
 }

performance/gradle/benchmark.gradle

@@ -13,7 +13,6 @@ repositories {
 
 }
 
-
 private def determinePreset() {
     def preset = MPPTools.defaultHostPreset(project)
     println("$project has been configured for ${preset.name} platform.")
@@ -53,14 +52,15 @@ kotlin {
             compilations.all {
                 tasks[compileKotlinTaskName].kotlinOptions {
                     jvmTarget = '1.8'
+                    suppressWarnings = true
+                    freeCompilerArgs = (project.hasProperty('compilerArgs') ? compilerArgs.split() : [])
                 }
-                tasks[compileKotlinTaskName].kotlinOptions.suppressWarnings = true
             }
         }
 
         fromPreset(hostPreset, 'native') {
             compilations.main.outputKinds('EXECUTABLE')
-            compilations.main.extraOpts '-opt'
+            compilations.main.extraOpts = (project.hasProperty('compilerArgs') ? compilerArgs.split() : []) + '-opt'
             compilations.main.buildTypes = [RELEASE]
         }
     }

performance/ring/src/main/kotlin/main.kt

@@ -20,7 +20,7 @@ import octoTest
 import org.jetbrains.benchmarksLauncher.*
 import org.jetbrains.kliopt.*
 
-class RingLauncher(numWarmIterations: Int, numberOfAttempts: Int, prefix: String): Launcher(numWarmIterations, numberOfAttempts, prefix) {
+class RingLauncher(numWarmIterations: Int, numberOfAttempts: Int, prefix: String) : Launcher(numWarmIterations, numberOfAttempts, prefix) {
     val abstractMethodBenchmark = AbstractMethodBenchmark()
     val classArrayBenchmark = ClassArrayBenchmark()
     val classBaselineBenchmark = ClassBaselineBenchmark()
@@ -232,6 +232,7 @@ class RingLauncher(numWarmIterations: Int, numberOfAttempts: Int, prefix: String
 
 fun main(args: Array<String>) {
     BenchmarksRunner.runBenchmarks(args, { parser: ArgParser ->
-        RingLauncher(parser.get<Int>("warmup")!!, parser.get<Int>("repeat")!!, parser.get<String>("prefix")!!).launch(parser.getAll<String>("filter"))
+        RingLauncher(parser.get<Int>("warmup")!!, parser.get<Int>("repeat")!!, parser.get<String>("prefix")!!)
+                .launch(parser.getAll<String>("filter"), parser.getAll<String>("filterRegex"))
     })
 }

performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/launcher.kt

@@ -25,13 +25,18 @@ abstract class Launcher(val numWarmIterations: Int, val numberOfAttempts: Int, v
 
     abstract val benchmarks: BenchmarksCollection
 
-    protected val benchmarkResults = mutableListOf<BenchmarkResult>()
+    fun launch(filters: Collection<String>? = null,
-
+               filterRegexes: Collection<String>? = null): List<BenchmarkResult> {
-    fun launch(benchmarksToRun: Collection<String>? = null): List<BenchmarkResult> {
+        val regexes = filterRegexes?.map { it.toRegex() } ?: listOf()
-        val runningBenchmarks = benchmarksToRun ?: benchmarks.keys
+        val filterSet = filters?.toHashSet() ?: hashSetOf()
-        runningBenchmarks.forEach {
+        // Filter benchmarks using given filters, or run all benchmarks if none were given.
-            val benchmark = benchmarks[it]
+        val runningBenchmarks = if (filterSet.isNotEmpty() || regexes.isNotEmpty()) {
-            benchmark ?: error("Benchmark $it wasn't found!")
+            benchmarks.filterKeys { benchmark -> benchmark in filterSet || regexes.any { it.matches(benchmark) } }
+        } else benchmarks
+        if (runningBenchmarks.isEmpty())
+            error("No matching benchmarks found")
+        val benchmarkResults = mutableListOf<BenchmarkResult>()
+        for ((name, benchmark) in runningBenchmarks) {
             var i = numWarmIterations
             while (i-- > 0) benchmark()
             cleanup()
@@ -60,7 +65,7 @@ abstract class Launcher(val numWarmIterations: Int, val numberOfAttempts: Int, v
                 val scaledTime = time * 1.0 / autoEvaluatedNumberOfMeasureIteration
                 samples[k] = scaledTime
                 // Save benchmark object
-                benchmarkResults.add(BenchmarkResult("$prefix$it", BenchmarkResult.Status.PASSED,
+                benchmarkResults.add(BenchmarkResult("$prefix$name", BenchmarkResult.Status.PASSED,
                         scaledTime / 1000, BenchmarkResult.Metric.EXECUTION_TIME, scaledTime / 1000,
                         k + 1, numWarmIterations))
             }
@@ -73,10 +78,11 @@ object BenchmarksRunner {
     fun parse(args: Array<String>): ArgParser {
         val options = listOf(
                 OptionDescriptor(ArgType.Int(), "warmup", "w", "Number of warm up iterations", "20"),
-                OptionDescriptor(ArgType.Int(), "repeat", "r", "Number of each becnhmark run", "60"),
+                OptionDescriptor(ArgType.Int(), "repeat", "r", "Number of each benchmark run", "60"),
                 OptionDescriptor(ArgType.String(), "prefix", "p", "Prefix added to benchmark name", ""),
                 OptionDescriptor(ArgType.String(), "output", "o", "Output file"),
-                OptionDescriptor(ArgType.String(), "filter", "f", "Benchmark to run", isMultiple = true)
+                OptionDescriptor(ArgType.String(), "filter", "f", "Benchmark to run", isMultiple = true),
+                OptionDescriptor(ArgType.String(), "filterRegex", "fr", "Benchmark to run, described by a regular expression", isMultiple = true)
         )
 
         // Parse args.
@@ -99,4 +105,4 @@ object BenchmarksRunner {
         val results = run(parser)
         collect(results, parser)
     }
-}
+}