azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add option to use regexes for filtering and add property for passing compiler args when running benchmarks. (#2894)

Browse Source
This commit is contained in:
Mark Punzalan
2019-04-24 00:18:22 -07:00
committed by LepilkinaElena
parent a764723e4f
commit 6d544cd667
8 changed files with 64 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
HACKING.md
+15 -4
View File
@@ -110,16 +110,27 @@ To update the blackbox compiler tests set TeamCity build number in `gradle.prope
../gradlew :cinterop:konanRun
**konanRun** task has parameter `filter` which allows to run only some subset of benchmarks :
**konanRun** task has parameter `filter` which allows to run only some subset of benchmarks:
../gradlew :cinterop:konanRun --filter=struct,macros
Or you can use `filterRegex` if you want to specify the filter as regexes:
../gradlew :ring:konanRun --filterRegex=String.*,Loop.*
There are also tasks for running benchmarks on JVM (pay attention, some benchmarks e.g. cinterop benchmarks can't be run on JVM)
../gradlew jvmRun
Files with results of benchmarks run are saved in `performance/build`. **nativeReport.json** - for konanRun and **jvmReport.json** for jvmRun.
Files with results of benchmarks run are saved in `performance/build/nativeReport.json` for konanRun and `jvmReport.json` for jvmRun.
You can change the output filename by setting the `nativeJson` property for konanRun and `jvmJson` for jvmRun:
../gradlew :ring:konanRun --filter=String.*,Loop.* -PnativeJson=stringsAndLoops.json
You can use the `compilerArgs` property to pass flags to the compiler used to compile the benchmarks:
../gradlew konanRun -PcompilerArgs="--time -g"
To compare different results run benchmarksAnalyzer tool:
cd tools/benchmarksAnalyzer/build/bin/<target>/benchmarksAnalyzerReleaseExecutable/
@@ -148,4 +159,4 @@ To update the blackbox compiler tests set TeamCity build number in `gradle.prope
teamcity:id:42491947:nativeReport.json
Pay attention, user and password information(with flag `-u <username>:<password>`) should be provided to get data from TeamCity.
performance/buildSrc/src/main/kotlin/RunJvmTask.kt
+9 -5
View File
@@ -13,27 +13,31 @@ import org.jetbrains.kotlin.gradle.plugin.KotlinTarget
import javax.inject.Inject
import java.io.File
open class RunJvmTask: JavaExec() {
open class RunJvmTask : JavaExec() {
var outputFileName: String? = null
@Input
@Option(option = "filter", description = "filter")
@Option(option = "filter", description = "Benchmarks to run (comma-separated)")
var filter: String = ""
@Input
@Option(option = "filterRegex", description = "Benchmarks to run, described by regular expressions (comma-separated)")
var filterRegex: String = ""
override fun configure(configureClosure: Closure<Any>): Task {
return super.configure(configureClosure)
}
private fun executeTask(output: java.io.OutputStream? = null) {
val filterArgs = filter.split("\\s*,\\s*".toRegex())
.map{ if (it.isNotEmpty()) listOf("-f", it) else listOf(null) }.flatten().filterNotNull()
val filterArgs = filter.splitCommaSeparatedOption("-f")
val filterRegexArgs = filterRegex.splitCommaSeparatedOption("-fr")
args(filterArgs)
args(filterRegexArgs)
exec()
}
@TaskAction
fun run() {
if (outputFileName != null)
File(outputFileName).outputStream().use { output -> executeTask(output)}
File(outputFileName).outputStream().use { output -> executeTask(output) }
else
executeTask()
}
performance/buildSrc/src/main/kotlin/RunKotlinNativeTask.kt
+9 -6
View File
@@ -15,14 +15,17 @@ import java.io.File
open class RunKotlinNativeTask @Inject constructor(
private val curTarget: KotlinTarget
): DefaultTask() {
) : DefaultTask() {
var buildType = "RELEASE"
var workingDir: Any = project.projectDir
var outputFileName: String? = null
@Input
@Option(option = "filter", description = "filter")
@Option(option = "filter", description = "Benchmarks to run (comma-separated)")
var filter: String = ""
@Input
@Option(option = "filterRegex", description = "Benchmarks to run, described by regular expressions (comma-separated)")
var filterRegex: String = ""
private var curArgs: List<String> = emptyList()
private val curEnvironment: MutableMap<String, Any> = mutableMapOf()
@@ -46,11 +49,11 @@ open class RunKotlinNativeTask @Inject constructor(
}
private fun executeTask(output: java.io.OutputStream? = null) {
val filterArgs = filter.split("\\s*,\\s*".toRegex())
.map{ if (it.isNotEmpty()) listOf("-f", it) else listOf(null) }.flatten().filterNotNull()
val filterArgs = filter.splitCommaSeparatedOption("-f")
val filterRegexArgs = filterRegex.splitCommaSeparatedOption("-fr")
project.exec {
it.executable = curTarget.compilations.main.getBinary("EXECUTABLE", buildType).toString()
it.args = curArgs + filterArgs
it.args = curArgs + filterArgs + filterRegexArgs
it.environment = curEnvironment
it.workingDir(workingDir)
if (output != null)
@@ -61,7 +64,7 @@ open class RunKotlinNativeTask @Inject constructor(
@TaskAction
fun run() {
if (outputFileName != null)
File(outputFileName).outputStream().use { output -> executeTask(output)}
File(outputFileName).outputStream().use { output -> executeTask(output) }
else
executeTask()
}
performance/buildSrc/src/main/kotlin/Utils.kt
+5
View File
@@ -29,6 +29,11 @@ fun Array<String>.runCommand(workingDir: File = File("."),
}
}
fun String.splitCommaSeparatedOption(optionName: String) =
split("\\s*,\\s*".toRegex()).map {
if (it.isNotEmpty()) listOf(optionName, it) else listOf(null)
}.flatten().filterNotNull()
data class Commit(val revision: String, val developer: String, val webUrlWithDescription: String)
val teamCityUrl = "http://buildserver.labs.intellij.net"
performance/cinterop/src/main/kotlin/main.kt
+3 -2
View File
@@ -21,7 +21,7 @@ import org.jetbrains.structsBenchmarks.*
import org.jetbrains.typesBenchmarks.*
import org.jetbrains.kliopt.*
class CinteropLauncher(numWarmIterations: Int, numberOfAttempts: Int, prefix: String): Launcher(numWarmIterations, numberOfAttempts, prefix) {
class CinteropLauncher(numWarmIterations: Int, numberOfAttempts: Int, prefix: String) : Launcher(numWarmIterations, numberOfAttempts, prefix) {
val stringBenchmark = StringBenchmark()
val intMatrixBenchmark = IntMatrixBenchmark()
val intBenchmark = IntBenchmark()
@@ -43,6 +43,7 @@ class CinteropLauncher(numWarmIterations: Int, numberOfAttempts: Int, prefix: St
fun main(args: Array<String>) {
BenchmarksRunner.runBenchmarks(args, { parser: ArgParser ->
CinteropLauncher(parser.get<Int>("warmup")!!, parser.get<Int>("repeat")!!, parser.get<String>("prefix")!!).launch(parser.getAll<String>("filter"))
CinteropLauncher(parser.get<Int>("warmup")!!, parser.get<Int>("repeat")!!, parser.get<String>("prefix")!!)
.launch(parser.getAll<String>("filter"), parser.getAll<String>("filterRegex"))
})
}
performance/gradle/benchmark.gradle
+3 -3
View File
@@ -13,7 +13,6 @@ repositories {
}
private def determinePreset() {
def preset = MPPTools.defaultHostPreset(project)
println("$project has been configured for ${preset.name} platform.")
@@ -53,14 +52,15 @@ kotlin {
compilations.all {
tasks[compileKotlinTaskName].kotlinOptions {
jvmTarget = '1.8'
suppressWarnings = true
freeCompilerArgs = (project.hasProperty('compilerArgs') ? compilerArgs.split() : [])
}
tasks[compileKotlinTaskName].kotlinOptions.suppressWarnings = true
}
}
fromPreset(hostPreset, 'native') {
compilations.main.outputKinds('EXECUTABLE')
compilations.main.extraOpts '-opt'
compilations.main.extraOpts = (project.hasProperty('compilerArgs') ? compilerArgs.split() : []) + '-opt'
compilations.main.buildTypes = [RELEASE]
}
}
performance/ring/src/main/kotlin/main.kt
+3 -2
View File
@@ -20,7 +20,7 @@ import octoTest
import org.jetbrains.benchmarksLauncher.*
import org.jetbrains.kliopt.*
class RingLauncher(numWarmIterations: Int, numberOfAttempts: Int, prefix: String): Launcher(numWarmIterations, numberOfAttempts, prefix) {
class RingLauncher(numWarmIterations: Int, numberOfAttempts: Int, prefix: String) : Launcher(numWarmIterations, numberOfAttempts, prefix) {
val abstractMethodBenchmark = AbstractMethodBenchmark()
val classArrayBenchmark = ClassArrayBenchmark()
val classBaselineBenchmark = ClassBaselineBenchmark()
@@ -232,6 +232,7 @@ class RingLauncher(numWarmIterations: Int, numberOfAttempts: Int, prefix: String
fun main(args: Array<String>) {
BenchmarksRunner.runBenchmarks(args, { parser: ArgParser ->
RingLauncher(parser.get<Int>("warmup")!!, parser.get<Int>("repeat")!!, parser.get<String>("prefix")!!).launch(parser.getAll<String>("filter"))
RingLauncher(parser.get<Int>("warmup")!!, parser.get<Int>("repeat")!!, parser.get<String>("prefix")!!)
.launch(parser.getAll<String>("filter"), parser.getAll<String>("filterRegex"))
})
}
performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/launcher.kt
+17 -11
View File
@@ -25,13 +25,18 @@ abstract class Launcher(val numWarmIterations: Int, val numberOfAttempts: Int, v
abstract val benchmarks: BenchmarksCollection
protected val benchmarkResults = mutableListOf<BenchmarkResult>()
fun launch(benchmarksToRun: Collection<String>? = null): List<BenchmarkResult> {
val runningBenchmarks = benchmarksToRun ?: benchmarks.keys
runningBenchmarks.forEach {
val benchmark = benchmarks[it]
benchmark ?: error("Benchmark $it wasn't found!")
fun launch(filters: Collection<String>? = null,
filterRegexes: Collection<String>? = null): List<BenchmarkResult> {
val regexes = filterRegexes?.map { it.toRegex() } ?: listOf()
val filterSet = filters?.toHashSet() ?: hashSetOf()
// Filter benchmarks using given filters, or run all benchmarks if none were given.
val runningBenchmarks = if (filterSet.isNotEmpty() || regexes.isNotEmpty()) {
benchmarks.filterKeys { benchmark -> benchmark in filterSet || regexes.any { it.matches(benchmark) } }
} else benchmarks
if (runningBenchmarks.isEmpty())
error("No matching benchmarks found")
val benchmarkResults = mutableListOf<BenchmarkResult>()
for ((name, benchmark) in runningBenchmarks) {
var i = numWarmIterations
while (i-- > 0) benchmark()
cleanup()
@@ -60,7 +65,7 @@ abstract class Launcher(val numWarmIterations: Int, val numberOfAttempts: Int, v
val scaledTime = time * 1.0 / autoEvaluatedNumberOfMeasureIteration
samples[k] = scaledTime
// Save benchmark object
benchmarkResults.add(BenchmarkResult("$prefix$it", BenchmarkResult.Status.PASSED,
benchmarkResults.add(BenchmarkResult("$prefix$name", BenchmarkResult.Status.PASSED,
scaledTime / 1000, BenchmarkResult.Metric.EXECUTION_TIME, scaledTime / 1000,
k + 1, numWarmIterations))
}
@@ -73,10 +78,11 @@ object BenchmarksRunner {
fun parse(args: Array<String>): ArgParser {
val options = listOf(
OptionDescriptor(ArgType.Int(), "warmup", "w", "Number of warm up iterations", "20"),
OptionDescriptor(ArgType.Int(), "repeat", "r", "Number of each becnhmark run", "60"),
OptionDescriptor(ArgType.Int(), "repeat", "r", "Number of each benchmark run", "60"),
OptionDescriptor(ArgType.String(), "prefix", "p", "Prefix added to benchmark name", ""),
OptionDescriptor(ArgType.String(), "output", "o", "Output file"),
OptionDescriptor(ArgType.String(), "filter", "f", "Benchmark to run", isMultiple = true)
OptionDescriptor(ArgType.String(), "filter", "f", "Benchmark to run", isMultiple = true),
OptionDescriptor(ArgType.String(), "filterRegex", "fr", "Benchmark to run, described by a regular expression", isMultiple = true)
)
// Parse args.
@@ -99,4 +105,4 @@ object BenchmarksRunner {
val results = run(parser)
collect(results, parser)
}
}
}