Added verbose mode to follow benchmarks process (#3218)

HACKING.md

@@ -117,6 +117,15 @@ To update the blackbox compiler tests set TeamCity build number in `gradle.prope
  
     ./gradlew :performance:ring:konanRun --filterRegex=String.*,Loop.*
     
+ There us also verbose mode to follow progress of running benchmarks
+ 
+    ./gradlew :performance:cinterop:konanRun --verbose
+    
+    > Task :performance:cinterop:konanRun
+    [DEBUG] Warm up iterations for benchmark macros
+    [DEBUG] Running benchmark macros
+    ...
+    
  There are also tasks for running benchmarks on JVM (pay attention, some benchmarks e.g. cinterop benchmarks can't be run on JVM)
  
     ./gradlew :performance:jvmRun

build-tools/src/main/kotlin/org/jetbrains/kotlin/RunBenchmarksExecutableTask.kt

@@ -24,6 +24,9 @@ open class RunBenchmarksExecutableTask @Inject constructor() : DefaultTask() {
     @Input
     @Option(option = "filterRegex", description = "Benchmarks to run, described by regular expressions (comma-separated)")
     var filterRegex: String = ""
+    @Input
+    @Option(option = "verbose", description = "Verbose mode of running benchmarks")
+    var verbose: Boolean = false
 
     private var curArgs: List<String> = emptyList()
     private val curEnvironment: MutableMap<String, Any> = mutableMapOf()
@@ -53,6 +56,9 @@ open class RunBenchmarksExecutableTask @Inject constructor() : DefaultTask() {
             it.args = curArgs + filterArgs + filterRegexArgs
             it.environment = curEnvironment
             it.workingDir(workingDir)
+            if (verbose) {
+                it.args("-v")
+            }
             if (output != null)
                 it.standardOutput = output
         }

build-tools/src/main/kotlin/org/jetbrains/kotlin/RunJvmTask.kt

@@ -23,6 +23,9 @@ open class RunJvmTask : JavaExec() {
     @Input
     @Option(option = "filterRegex", description = "Benchmarks to run, described by regular expressions (comma-separated)")
     var filterRegex: String = ""
+    @Input
+    @Option(option = "verbose", description = "Verbose mode of running benchmarks")
+    var verbose: Boolean = false
 
     override fun configure(configureClosure: Closure<Any>): Task {
         return super.configure(configureClosure)
@@ -33,6 +36,9 @@ open class RunJvmTask : JavaExec() {
         val filterRegexArgs = filterRegex.splitCommaSeparatedOption("-fr")
         args(filterArgs)
         args(filterRegexArgs)
+        if (verbose) {
+            args("-v")
+        }
         exec()
     }
 

build-tools/src/main/kotlin/org/jetbrains/kotlin/RunKotlinNativeTask.kt

@@ -25,6 +25,9 @@ open class RunKotlinNativeTask @Inject constructor(private val linkTask: KotlinN
     @Input
     @Option(option = "filterRegex", description = "Benchmarks to run, described by regular expressions (comma-separated)")
     var filterRegex: String = ""
+    @Input
+    @Option(option = "verbose", description = "Verbose mode of running benchmarks")
+    var verbose: Boolean = false
 
     private val argumentsList = mutableListOf<String>()
 
@@ -63,6 +66,9 @@ open class RunKotlinNativeTask @Inject constructor(private val linkTask: KotlinN
                 it.executable = linkTask.binary.outputFile.absolutePath
                 it.args(argumentsList)
                 it.args("-f", benchmark)
+                if (verbose) {
+                    it.args("-v")
+                }
                 it.standardOutput = output
             }
             output.toString().removePrefix("[").removeSuffix("]")

build-tools/src/main/kotlin/org/jetbrains/kotlin/benchmark/BenchmarkingPlugin.kt

@@ -74,6 +74,8 @@ open class BenchmarkExtension @Inject constructor(val project: Project) {
     var commonSrcDirs: Collection<Any> = emptyList()
     var jvmSrcDirs: Collection<Any> = emptyList()
     var nativeSrcDirs: Collection<Any> = emptyList()
+    var mingwSrcDirs: Collection<Any> = emptyList()
+    var posixSrcDirs: Collection<Any> = emptyList()
     var linkerOpts: Collection<String> = emptyList()
 }
 
@@ -107,7 +109,11 @@ open class BenchmarkingPlugin: Plugin<Project> {
             afterEvaluate {
                 benchmark.let {
                     commonMain.kotlin.srcDirs(*it.commonSrcDirs.toTypedArray())
-                    nativeMain.kotlin.srcDirs(*it.nativeSrcDirs.toTypedArray())
+                    if (HostManager.hostIsMingw) {
+                        nativeMain.kotlin.srcDirs(*(it.nativeSrcDirs + it.mingwSrcDirs).toTypedArray())
+                    } else {
+                       nativeMain.kotlin.srcDirs(*(it.nativeSrcDirs + it.posixSrcDirs).toTypedArray())
+                    }
                     jvmMain.kotlin.srcDirs(*it.jvmSrcDirs.toTypedArray())
                 }
             }
@@ -128,7 +134,7 @@ open class BenchmarkingPlugin: Plugin<Project> {
 
     private fun Project.configureNativeTarget(hostPreset: KotlinNativeTargetPreset) {
         kotlin.targetFromPreset(hostPreset, NATIVE_TARGET_NAME) {
-            compilations.getByName("main").kotlinOptions.freeCompilerArgs = project.compilerArgs + "-opt"
+            compilations.getByName("main").kotlinOptions.freeCompilerArgs = project.compilerArgs
             binaries.executable(NATIVE_EXECUTABLE_NAME, listOf(RELEASE)) {
                 if (HostManager.hostIsMingw) {
                     linkerOpts.add("-L${mingwPath}/lib")

performance/cinterop/build.gradle.kts

@@ -13,7 +13,9 @@ benchmark {
     applicationName = "Cinterop"
     commonSrcDirs = listOf("../../tools/benchmarks/shared/src", "src/main/kotlin", "../shared/src/main/kotlin")
     jvmSrcDirs = listOf("src/main/kotlin-jvm", "../shared/src/main/kotlin-jvm")
-    nativeSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native")
+    nativeSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/common")
+    mingwSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/mingw")
+    posixSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/posix")
 }
 
 val native = kotlin.targets.getByName("native") as KotlinNativeTarget

performance/cinterop/src/main/kotlin/main.kt

@@ -42,7 +42,7 @@ fun main(args: Array<String>) {
     BenchmarksRunner.runBenchmarks(args, { arguments: BenchmarkArguments ->
         if (arguments is BaseBenchmarkArguments) {
             launcher.launch(arguments.warmup, arguments.repeat, arguments.prefix,
-                    arguments.filter, arguments.filterRegex)
+                    arguments.filter, arguments.filterRegex, arguments.verbose)
         } else emptyList()
     }, benchmarksListAction = launcher::benchmarksListAction)
 }

performance/objcinterop/build.gradle.kts

@@ -16,7 +16,9 @@ benchmark {
     applicationName = "ObjCInterop"
     commonSrcDirs = listOf("../../tools/benchmarks/shared/src", "src/main/kotlin", "../shared/src/main/kotlin")
     jvmSrcDirs = listOf("src/main/kotlin-jvm", "../shared/src/main/kotlin-jvm")
-    nativeSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native")
+    nativeSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/common")
+    mingwSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/mingw")
+    posixSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/posix")
     linkerOpts = listOf("-L$buildDir", "-lcomplexnumbers")
 }
 

performance/objcinterop/src/main/kotlin/main.kt

@@ -29,7 +29,7 @@ fun main(args: Array<String>) {
     BenchmarksRunner.runBenchmarks(args, { arguments: BenchmarkArguments ->
         if (arguments is BaseBenchmarkArguments) {
             launcher.launch(arguments.warmup, arguments.repeat, arguments.prefix,
-                    arguments.filter, arguments.filterRegex)
+                    arguments.filter, arguments.filterRegex, arguments.verbose)
         } else emptyList()
     }, benchmarksListAction = launcher::benchmarksListAction)
 }

performance/ring/build.gradle.kts

@@ -11,5 +11,7 @@ benchmark {
     applicationName = "Ring"
     commonSrcDirs = listOf("../../tools/benchmarks/shared/src", "src/main/kotlin", "../shared/src/main/kotlin", "../../endorsedLibraries/kliopt/src/main/kotlin")
     jvmSrcDirs = listOf("src/main/kotlin-jvm", "../shared/src/main/kotlin-jvm", "../../endorsedLibraries/kliopt/src/main/kotlin-jvm")
-    nativeSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native", "../../endorsedLibraries/kliopt/src/main/kotlin-native")
+    nativeSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/common", "../../endorsedLibraries/kliopt/src/main/kotlin-native")
+    mingwSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/mingw")
+    posixSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/posix")
 }

performance/ring/src/main/kotlin/main.kt

@@ -212,7 +212,7 @@ fun main(args: Array<String>) {
     BenchmarksRunner.runBenchmarks(args, { arguments: BenchmarkArguments ->
         if (arguments is BaseBenchmarkArguments) {
             launcher.launch(arguments.warmup, arguments.repeat, arguments.prefix,
-                    arguments.filter, arguments.filterRegex)
+                    arguments.filter, arguments.filterRegex, arguments.verbose)
         } else emptyList()
     }, benchmarksListAction = launcher::benchmarksListAction)
 }

performance/shared/src/main/kotlin-jvm/org/jetbrains/benchmarksLauncher/Utils.kt

@@ -17,6 +17,8 @@
 package org.jetbrains.benchmarksLauncher
 
 import java.io.File
+import java.text.SimpleDateFormat
+import java.util.Date
 
 actual fun writeToFile(fileName: String, text: String) {
     File(fileName).printWriter().use { out ->
@@ -36,3 +38,10 @@ actual inline fun measureNanoTime(block: () -> Unit): Long {
 
 actual fun cleanup() {}
 
+actual fun printStderr(message: String) {
+    System.err.print(message)
+}
+
+actual fun currentTime(): String =
+        SimpleDateFormat("HH:mm:ss").format(Date())
+

performance/shared/src/main/kotlin-native/common/org/jetbrains/benchmarksLauncher/Utils.kt

@@ -18,6 +18,7 @@ package org.jetbrains.benchmarksLauncher
 
 import kotlin.native.internal.GC
 import platform.posix.*
+import kotlinx.cinterop.*
 
 actual fun writeToFile(fileName: String, text: String) {
     val file = fopen(fileName, "wt") ?: error("Cannot write file '$fileName'")
@@ -40,4 +41,11 @@ actual inline fun measureNanoTime(block: () -> Unit): Long {
 
 actual fun cleanup() {
     GC.collect()
-}
+}
+
+actual fun printStderr(message: String) {
+    val STDERR = fdopen(2, "w")
+    fprintf(STDERR, message)
+    fflush(STDERR)
+}
+

performance/shared/src/main/kotlin-native/mingw/org/jetbrains/benchmarksLauncher/Utils.kt

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.benchmarksLauncher
+
+import platform.posix.*
+import platform.windows.*
+import kotlinx.cinterop.*
+
+actual fun currentTime() =
+        memScoped {
+            val timeVal = alloc<timeval>()
+            mingw_gettimeofday(timeVal.ptr, null)
+            val sec = alloc<LongVar>()
+            sec.value = timeVal.tv_sec.convert()
+            val nowtm = localtime(sec.ptr)
+            var timeBuffer = ByteArray(1024)
+            strftime(timeBuffer.refTo(0), timeBuffer.size.toULong(), "%H:%M:%S", nowtm)
+
+            timeBuffer.toKString()
+        }

performance/shared/src/main/kotlin-native/posix/org/jetbrains/benchmarksLauncher/Utils.kt

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.benchmarksLauncher
+
+import platform.posix.*
+import kotlinx.cinterop.*
+
+actual fun currentTime() =
+        memScoped {
+            val timeVal = alloc<timeval>()
+            gettimeofday(timeVal.ptr, null)
+            val sec = alloc<LongVar>()
+            sec.value = timeVal.tv_sec
+            val nowtm = localtime(sec.ptr)
+            var timeBuffer = ByteArray(1024)
+            strftime(timeBuffer.refTo(0), timeBuffer.size.toULong(), "%H:%M:%S", nowtm)
+
+            timeBuffer.toKString()
+        }

performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/Utils.kt

@@ -22,4 +22,8 @@ expect fun assert(value: Boolean)
 
 expect inline fun measureNanoTime(block: () -> Unit): Long
 
-expect fun cleanup()
+expect fun cleanup()
+
+expect fun printStderr(message: String)
+
+expect fun currentTime(): String

performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/launcher.kt

@@ -46,11 +46,28 @@ abstract class Launcher {
         }
     }
 
+    enum class LogLevel { DEBUG, OFF }
+
+    class Logger(val level: LogLevel = LogLevel.OFF) {
+         fun log(message: String, messageLevel: LogLevel = LogLevel.DEBUG, usePrefix: Boolean = true) {
+            if (messageLevel == level) {
+                if (usePrefix) {
+                    printStderr("[$level][${currentTime()}] $message")
+                } else {
+                    printStderr("$message")
+                }
+
+            }
+        }
+    }
+
     fun launch(numWarmIterations: Int,
                numberOfAttempts: Int,
                prefix: String = "",
                filters: Collection<String>? = null,
-               filterRegexes: Collection<String>? = null): List<BenchmarkResult> {
+               filterRegexes: Collection<String>? = null,
+               verbose: Boolean): List<BenchmarkResult> {
+        val logger = if (verbose) Logger(LogLevel.DEBUG) else Logger()
         val regexes = filterRegexes?.map { it.toRegex() } ?: listOf()
         val filterSet = filters?.toHashSet() ?: hashSetOf()
         // Filter benchmarks using given filters, or run all benchmarks if none were given.
@@ -63,6 +80,7 @@ abstract class Launcher {
         for ((name, benchmark) in runningBenchmarks) {
             val benchmarkInstance = (benchmark as? BenchmarkEntryWithInit)?.ctor?.invoke()
             var i = numWarmIterations
+            logger.log("Warm up iterations for benchmark $name\n")
             runBenchmark(benchmarkInstance, benchmark, i)
             var autoEvaluatedNumberOfMeasureIteration = 1
             while (true) {
@@ -72,8 +90,10 @@ abstract class Launcher {
                     break
                 autoEvaluatedNumberOfMeasureIteration *= 2
             }
+            logger.log("Running benchmark $name ")
             val samples = DoubleArray(numberOfAttempts)
             for (k in samples.indices) {
+                logger.log(".", usePrefix = false)
                 i = autoEvaluatedNumberOfMeasureIteration
                 val time = runBenchmark(benchmarkInstance, benchmark, i)
                 val scaledTime = time * 1.0 / autoEvaluatedNumberOfMeasureIteration
@@ -83,6 +103,7 @@ abstract class Launcher {
                         scaledTime / 1000, BenchmarkResult.Metric.EXECUTION_TIME, scaledTime / 1000,
                         k + 1, numWarmIterations))
             }
+            logger.log("\n", usePrefix = false)
         }
         return benchmarkResults
     }
@@ -107,6 +128,7 @@ class BaseBenchmarkArguments(argParser: ArgParser): BenchmarkArguments(argParser
     val filter by argParser.options(ArgType.String, shortName = "f", description = "Benchmark to run", multiple = true)
     val filterRegex by argParser.options(ArgType.String, shortName = "fr",
             description = "Benchmark to run, described by a regular expression", multiple = true)
+    val verbose by argParser.option(ArgType.Boolean, shortName = "v", description = "Verbose mode of running", defaultValue = false)
 }
 
 object BenchmarksRunner {

performance/swiftinterop/build.gradle

@@ -44,7 +44,8 @@ kotlin {
         }
         macosMain {
             dependsOn commonMain
-            kotlin.srcDir "../shared/src/main/kotlin-native"
+            kotlin.srcDir "../shared/src/main/kotlin-native/common"
+            kotlin.srcDir "../shared/src/main/kotlin-native/posix"
             kotlin.srcDir "$rootProject.projectDir/endorsedLibraries/kliopt/src/main/kotlin-native"
         }
     }

performance/swiftinterop/swiftSrc/main.swift

@@ -43,7 +43,8 @@ runner.runBenchmarks(args: args, run: { (arguments: BenchmarkArguments) -> [Benc
         return swiftLauncher.launch(numWarmIterations: argumentsList.warmup,
             numberOfAttempts: argumentsList.repeat,
             prefix: argumentsList.prefix, filters: argumentsList.filter,
-            filterRegexes: argumentsList.filterRegex)
+            filterRegexes: argumentsList.filterRegex,
+            verbose: argumentsList.verbose)
     }
     return [BenchmarkResult]()
 }, parseArgs: { (args: KotlinArray,  benchmarksListAction: (() -> KotlinUnit)) -> BenchmarkArguments? in