diff --git a/HACKING.md b/HACKING.md index 439b86c27e2..12b8457b63f 100644 --- a/HACKING.md +++ b/HACKING.md @@ -117,6 +117,15 @@ To update the blackbox compiler tests set TeamCity build number in `gradle.prope ./gradlew :performance:ring:konanRun --filterRegex=String.*,Loop.* + There us also verbose mode to follow progress of running benchmarks + + ./gradlew :performance:cinterop:konanRun --verbose + + > Task :performance:cinterop:konanRun + [DEBUG] Warm up iterations for benchmark macros + [DEBUG] Running benchmark macros + ... + There are also tasks for running benchmarks on JVM (pay attention, some benchmarks e.g. cinterop benchmarks can't be run on JVM) ./gradlew :performance:jvmRun diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/RunBenchmarksExecutableTask.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/RunBenchmarksExecutableTask.kt index 57f77e8b1a2..621d6fb6614 100644 --- a/build-tools/src/main/kotlin/org/jetbrains/kotlin/RunBenchmarksExecutableTask.kt +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/RunBenchmarksExecutableTask.kt @@ -24,6 +24,9 @@ open class RunBenchmarksExecutableTask @Inject constructor() : DefaultTask() { @Input @Option(option = "filterRegex", description = "Benchmarks to run, described by regular expressions (comma-separated)") var filterRegex: String = "" + @Input + @Option(option = "verbose", description = "Verbose mode of running benchmarks") + var verbose: Boolean = false private var curArgs: List = emptyList() private val curEnvironment: MutableMap = mutableMapOf() @@ -53,6 +56,9 @@ open class RunBenchmarksExecutableTask @Inject constructor() : DefaultTask() { it.args = curArgs + filterArgs + filterRegexArgs it.environment = curEnvironment it.workingDir(workingDir) + if (verbose) { + it.args("-v") + } if (output != null) it.standardOutput = output } diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/RunJvmTask.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/RunJvmTask.kt index 372ebbe3a07..3b48416cb77 100644 --- a/build-tools/src/main/kotlin/org/jetbrains/kotlin/RunJvmTask.kt +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/RunJvmTask.kt @@ -23,6 +23,9 @@ open class RunJvmTask : JavaExec() { @Input @Option(option = "filterRegex", description = "Benchmarks to run, described by regular expressions (comma-separated)") var filterRegex: String = "" + @Input + @Option(option = "verbose", description = "Verbose mode of running benchmarks") + var verbose: Boolean = false override fun configure(configureClosure: Closure): Task { return super.configure(configureClosure) @@ -33,6 +36,9 @@ open class RunJvmTask : JavaExec() { val filterRegexArgs = filterRegex.splitCommaSeparatedOption("-fr") args(filterArgs) args(filterRegexArgs) + if (verbose) { + args("-v") + } exec() } diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/RunKotlinNativeTask.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/RunKotlinNativeTask.kt index 1b1db814e1e..20ef315f751 100644 --- a/build-tools/src/main/kotlin/org/jetbrains/kotlin/RunKotlinNativeTask.kt +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/RunKotlinNativeTask.kt @@ -25,6 +25,9 @@ open class RunKotlinNativeTask @Inject constructor(private val linkTask: KotlinN @Input @Option(option = "filterRegex", description = "Benchmarks to run, described by regular expressions (comma-separated)") var filterRegex: String = "" + @Input + @Option(option = "verbose", description = "Verbose mode of running benchmarks") + var verbose: Boolean = false private val argumentsList = mutableListOf() @@ -63,6 +66,9 @@ open class RunKotlinNativeTask @Inject constructor(private val linkTask: KotlinN it.executable = linkTask.binary.outputFile.absolutePath it.args(argumentsList) it.args("-f", benchmark) + if (verbose) { + it.args("-v") + } it.standardOutput = output } output.toString().removePrefix("[").removeSuffix("]") diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/benchmark/BenchmarkingPlugin.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/benchmark/BenchmarkingPlugin.kt index 20ef3226eb7..80c341db02d 100644 --- a/build-tools/src/main/kotlin/org/jetbrains/kotlin/benchmark/BenchmarkingPlugin.kt +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/benchmark/BenchmarkingPlugin.kt @@ -74,6 +74,8 @@ open class BenchmarkExtension @Inject constructor(val project: Project) { var commonSrcDirs: Collection = emptyList() var jvmSrcDirs: Collection = emptyList() var nativeSrcDirs: Collection = emptyList() + var mingwSrcDirs: Collection = emptyList() + var posixSrcDirs: Collection = emptyList() var linkerOpts: Collection = emptyList() } @@ -107,7 +109,11 @@ open class BenchmarkingPlugin: Plugin { afterEvaluate { benchmark.let { commonMain.kotlin.srcDirs(*it.commonSrcDirs.toTypedArray()) - nativeMain.kotlin.srcDirs(*it.nativeSrcDirs.toTypedArray()) + if (HostManager.hostIsMingw) { + nativeMain.kotlin.srcDirs(*(it.nativeSrcDirs + it.mingwSrcDirs).toTypedArray()) + } else { + nativeMain.kotlin.srcDirs(*(it.nativeSrcDirs + it.posixSrcDirs).toTypedArray()) + } jvmMain.kotlin.srcDirs(*it.jvmSrcDirs.toTypedArray()) } } @@ -128,7 +134,7 @@ open class BenchmarkingPlugin: Plugin { private fun Project.configureNativeTarget(hostPreset: KotlinNativeTargetPreset) { kotlin.targetFromPreset(hostPreset, NATIVE_TARGET_NAME) { - compilations.getByName("main").kotlinOptions.freeCompilerArgs = project.compilerArgs + "-opt" + compilations.getByName("main").kotlinOptions.freeCompilerArgs = project.compilerArgs binaries.executable(NATIVE_EXECUTABLE_NAME, listOf(RELEASE)) { if (HostManager.hostIsMingw) { linkerOpts.add("-L${mingwPath}/lib") diff --git a/performance/cinterop/build.gradle.kts b/performance/cinterop/build.gradle.kts index f7799d6f34c..736662771b3 100644 --- a/performance/cinterop/build.gradle.kts +++ b/performance/cinterop/build.gradle.kts @@ -13,7 +13,9 @@ benchmark { applicationName = "Cinterop" commonSrcDirs = listOf("../../tools/benchmarks/shared/src", "src/main/kotlin", "../shared/src/main/kotlin") jvmSrcDirs = listOf("src/main/kotlin-jvm", "../shared/src/main/kotlin-jvm") - nativeSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native") + nativeSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/common") + mingwSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/mingw") + posixSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/posix") } val native = kotlin.targets.getByName("native") as KotlinNativeTarget diff --git a/performance/cinterop/src/main/kotlin/main.kt b/performance/cinterop/src/main/kotlin/main.kt index 80671c45eb0..45969a7b4f1 100644 --- a/performance/cinterop/src/main/kotlin/main.kt +++ b/performance/cinterop/src/main/kotlin/main.kt @@ -42,7 +42,7 @@ fun main(args: Array) { BenchmarksRunner.runBenchmarks(args, { arguments: BenchmarkArguments -> if (arguments is BaseBenchmarkArguments) { launcher.launch(arguments.warmup, arguments.repeat, arguments.prefix, - arguments.filter, arguments.filterRegex) + arguments.filter, arguments.filterRegex, arguments.verbose) } else emptyList() }, benchmarksListAction = launcher::benchmarksListAction) } \ No newline at end of file diff --git a/performance/objcinterop/build.gradle.kts b/performance/objcinterop/build.gradle.kts index 54ba83cf82c..1af47bdeb9a 100644 --- a/performance/objcinterop/build.gradle.kts +++ b/performance/objcinterop/build.gradle.kts @@ -16,7 +16,9 @@ benchmark { applicationName = "ObjCInterop" commonSrcDirs = listOf("../../tools/benchmarks/shared/src", "src/main/kotlin", "../shared/src/main/kotlin") jvmSrcDirs = listOf("src/main/kotlin-jvm", "../shared/src/main/kotlin-jvm") - nativeSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native") + nativeSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/common") + mingwSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/mingw") + posixSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/posix") linkerOpts = listOf("-L$buildDir", "-lcomplexnumbers") } diff --git a/performance/objcinterop/src/main/kotlin/main.kt b/performance/objcinterop/src/main/kotlin/main.kt index 444361da9d8..d14709486cd 100644 --- a/performance/objcinterop/src/main/kotlin/main.kt +++ b/performance/objcinterop/src/main/kotlin/main.kt @@ -29,7 +29,7 @@ fun main(args: Array) { BenchmarksRunner.runBenchmarks(args, { arguments: BenchmarkArguments -> if (arguments is BaseBenchmarkArguments) { launcher.launch(arguments.warmup, arguments.repeat, arguments.prefix, - arguments.filter, arguments.filterRegex) + arguments.filter, arguments.filterRegex, arguments.verbose) } else emptyList() }, benchmarksListAction = launcher::benchmarksListAction) } \ No newline at end of file diff --git a/performance/ring/build.gradle.kts b/performance/ring/build.gradle.kts index 01b905285ff..e87ec9f2bf9 100644 --- a/performance/ring/build.gradle.kts +++ b/performance/ring/build.gradle.kts @@ -11,5 +11,7 @@ benchmark { applicationName = "Ring" commonSrcDirs = listOf("../../tools/benchmarks/shared/src", "src/main/kotlin", "../shared/src/main/kotlin", "../../endorsedLibraries/kliopt/src/main/kotlin") jvmSrcDirs = listOf("src/main/kotlin-jvm", "../shared/src/main/kotlin-jvm", "../../endorsedLibraries/kliopt/src/main/kotlin-jvm") - nativeSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native", "../../endorsedLibraries/kliopt/src/main/kotlin-native") + nativeSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/common", "../../endorsedLibraries/kliopt/src/main/kotlin-native") + mingwSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/mingw") + posixSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/posix") } diff --git a/performance/ring/src/main/kotlin/main.kt b/performance/ring/src/main/kotlin/main.kt index 2c79113d49f..95e7377ef12 100644 --- a/performance/ring/src/main/kotlin/main.kt +++ b/performance/ring/src/main/kotlin/main.kt @@ -212,7 +212,7 @@ fun main(args: Array) { BenchmarksRunner.runBenchmarks(args, { arguments: BenchmarkArguments -> if (arguments is BaseBenchmarkArguments) { launcher.launch(arguments.warmup, arguments.repeat, arguments.prefix, - arguments.filter, arguments.filterRegex) + arguments.filter, arguments.filterRegex, arguments.verbose) } else emptyList() }, benchmarksListAction = launcher::benchmarksListAction) } \ No newline at end of file diff --git a/performance/shared/src/main/kotlin-jvm/org/jetbrains/benchmarksLauncher/Utils.kt b/performance/shared/src/main/kotlin-jvm/org/jetbrains/benchmarksLauncher/Utils.kt index 9b1c015ae75..89bd0e0834a 100644 --- a/performance/shared/src/main/kotlin-jvm/org/jetbrains/benchmarksLauncher/Utils.kt +++ b/performance/shared/src/main/kotlin-jvm/org/jetbrains/benchmarksLauncher/Utils.kt @@ -17,6 +17,8 @@ package org.jetbrains.benchmarksLauncher import java.io.File +import java.text.SimpleDateFormat +import java.util.Date actual fun writeToFile(fileName: String, text: String) { File(fileName).printWriter().use { out -> @@ -36,3 +38,10 @@ actual inline fun measureNanoTime(block: () -> Unit): Long { actual fun cleanup() {} +actual fun printStderr(message: String) { + System.err.print(message) +} + +actual fun currentTime(): String = + SimpleDateFormat("HH:mm:ss").format(Date()) + diff --git a/performance/shared/src/main/kotlin-native/org/jetbrains/benchmarksLauncher/Utils.kt b/performance/shared/src/main/kotlin-native/common/org/jetbrains/benchmarksLauncher/Utils.kt similarity index 89% rename from performance/shared/src/main/kotlin-native/org/jetbrains/benchmarksLauncher/Utils.kt rename to performance/shared/src/main/kotlin-native/common/org/jetbrains/benchmarksLauncher/Utils.kt index fde5df07bf3..0c2d2e71f97 100644 --- a/performance/shared/src/main/kotlin-native/org/jetbrains/benchmarksLauncher/Utils.kt +++ b/performance/shared/src/main/kotlin-native/common/org/jetbrains/benchmarksLauncher/Utils.kt @@ -18,6 +18,7 @@ package org.jetbrains.benchmarksLauncher import kotlin.native.internal.GC import platform.posix.* +import kotlinx.cinterop.* actual fun writeToFile(fileName: String, text: String) { val file = fopen(fileName, "wt") ?: error("Cannot write file '$fileName'") @@ -40,4 +41,11 @@ actual inline fun measureNanoTime(block: () -> Unit): Long { actual fun cleanup() { GC.collect() -} \ No newline at end of file +} + +actual fun printStderr(message: String) { + val STDERR = fdopen(2, "w") + fprintf(STDERR, message) + fflush(STDERR) +} + diff --git a/performance/shared/src/main/kotlin-native/mingw/org/jetbrains/benchmarksLauncher/Utils.kt b/performance/shared/src/main/kotlin-native/mingw/org/jetbrains/benchmarksLauncher/Utils.kt new file mode 100644 index 00000000000..73ee1a49f31 --- /dev/null +++ b/performance/shared/src/main/kotlin-native/mingw/org/jetbrains/benchmarksLauncher/Utils.kt @@ -0,0 +1,34 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.benchmarksLauncher + +import platform.posix.* +import platform.windows.* +import kotlinx.cinterop.* + +actual fun currentTime() = + memScoped { + val timeVal = alloc() + mingw_gettimeofday(timeVal.ptr, null) + val sec = alloc() + sec.value = timeVal.tv_sec.convert() + val nowtm = localtime(sec.ptr) + var timeBuffer = ByteArray(1024) + strftime(timeBuffer.refTo(0), timeBuffer.size.toULong(), "%H:%M:%S", nowtm) + + timeBuffer.toKString() + } diff --git a/performance/shared/src/main/kotlin-native/posix/org/jetbrains/benchmarksLauncher/Utils.kt b/performance/shared/src/main/kotlin-native/posix/org/jetbrains/benchmarksLauncher/Utils.kt new file mode 100644 index 00000000000..3960886916d --- /dev/null +++ b/performance/shared/src/main/kotlin-native/posix/org/jetbrains/benchmarksLauncher/Utils.kt @@ -0,0 +1,33 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.benchmarksLauncher + +import platform.posix.* +import kotlinx.cinterop.* + +actual fun currentTime() = + memScoped { + val timeVal = alloc() + gettimeofday(timeVal.ptr, null) + val sec = alloc() + sec.value = timeVal.tv_sec + val nowtm = localtime(sec.ptr) + var timeBuffer = ByteArray(1024) + strftime(timeBuffer.refTo(0), timeBuffer.size.toULong(), "%H:%M:%S", nowtm) + + timeBuffer.toKString() + } diff --git a/performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/Utils.kt b/performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/Utils.kt index b08daffa7b2..33431b16eb7 100644 --- a/performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/Utils.kt +++ b/performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/Utils.kt @@ -22,4 +22,8 @@ expect fun assert(value: Boolean) expect inline fun measureNanoTime(block: () -> Unit): Long -expect fun cleanup() \ No newline at end of file +expect fun cleanup() + +expect fun printStderr(message: String) + +expect fun currentTime(): String \ No newline at end of file diff --git a/performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/launcher.kt b/performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/launcher.kt index a39e2ccde8d..e1c4d74d34f 100644 --- a/performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/launcher.kt +++ b/performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/launcher.kt @@ -46,11 +46,28 @@ abstract class Launcher { } } + enum class LogLevel { DEBUG, OFF } + + class Logger(val level: LogLevel = LogLevel.OFF) { + fun log(message: String, messageLevel: LogLevel = LogLevel.DEBUG, usePrefix: Boolean = true) { + if (messageLevel == level) { + if (usePrefix) { + printStderr("[$level][${currentTime()}] $message") + } else { + printStderr("$message") + } + + } + } + } + fun launch(numWarmIterations: Int, numberOfAttempts: Int, prefix: String = "", filters: Collection? = null, - filterRegexes: Collection? = null): List { + filterRegexes: Collection? = null, + verbose: Boolean): List { + val logger = if (verbose) Logger(LogLevel.DEBUG) else Logger() val regexes = filterRegexes?.map { it.toRegex() } ?: listOf() val filterSet = filters?.toHashSet() ?: hashSetOf() // Filter benchmarks using given filters, or run all benchmarks if none were given. @@ -63,6 +80,7 @@ abstract class Launcher { for ((name, benchmark) in runningBenchmarks) { val benchmarkInstance = (benchmark as? BenchmarkEntryWithInit)?.ctor?.invoke() var i = numWarmIterations + logger.log("Warm up iterations for benchmark $name\n") runBenchmark(benchmarkInstance, benchmark, i) var autoEvaluatedNumberOfMeasureIteration = 1 while (true) { @@ -72,8 +90,10 @@ abstract class Launcher { break autoEvaluatedNumberOfMeasureIteration *= 2 } + logger.log("Running benchmark $name ") val samples = DoubleArray(numberOfAttempts) for (k in samples.indices) { + logger.log(".", usePrefix = false) i = autoEvaluatedNumberOfMeasureIteration val time = runBenchmark(benchmarkInstance, benchmark, i) val scaledTime = time * 1.0 / autoEvaluatedNumberOfMeasureIteration @@ -83,6 +103,7 @@ abstract class Launcher { scaledTime / 1000, BenchmarkResult.Metric.EXECUTION_TIME, scaledTime / 1000, k + 1, numWarmIterations)) } + logger.log("\n", usePrefix = false) } return benchmarkResults } @@ -107,6 +128,7 @@ class BaseBenchmarkArguments(argParser: ArgParser): BenchmarkArguments(argParser val filter by argParser.options(ArgType.String, shortName = "f", description = "Benchmark to run", multiple = true) val filterRegex by argParser.options(ArgType.String, shortName = "fr", description = "Benchmark to run, described by a regular expression", multiple = true) + val verbose by argParser.option(ArgType.Boolean, shortName = "v", description = "Verbose mode of running", defaultValue = false) } object BenchmarksRunner { diff --git a/performance/swiftinterop/build.gradle b/performance/swiftinterop/build.gradle index 2845be072ae..022780bd89f 100644 --- a/performance/swiftinterop/build.gradle +++ b/performance/swiftinterop/build.gradle @@ -44,7 +44,8 @@ kotlin { } macosMain { dependsOn commonMain - kotlin.srcDir "../shared/src/main/kotlin-native" + kotlin.srcDir "../shared/src/main/kotlin-native/common" + kotlin.srcDir "../shared/src/main/kotlin-native/posix" kotlin.srcDir "$rootProject.projectDir/endorsedLibraries/kliopt/src/main/kotlin-native" } } diff --git a/performance/swiftinterop/swiftSrc/main.swift b/performance/swiftinterop/swiftSrc/main.swift index 849c4c706f5..7daa559c756 100644 --- a/performance/swiftinterop/swiftSrc/main.swift +++ b/performance/swiftinterop/swiftSrc/main.swift @@ -43,7 +43,8 @@ runner.runBenchmarks(args: args, run: { (arguments: BenchmarkArguments) -> [Benc return swiftLauncher.launch(numWarmIterations: argumentsList.warmup, numberOfAttempts: argumentsList.repeat, prefix: argumentsList.prefix, filters: argumentsList.filter, - filterRegexes: argumentsList.filterRegex) + filterRegexes: argumentsList.filterRegex, + verbose: argumentsList.verbose) } return [BenchmarkResult]() }, parseArgs: { (args: KotlinArray, benchmarksListAction: (() -> KotlinUnit)) -> BenchmarkArguments? in