Add performance tests on singletons (#4018)

HACKING.md

@@ -109,6 +109,12 @@ and then a final native binary is produced from this klibrary using the -Xinclud
  To measure performance of Kotlin/Native compiler on existing benchmarks:
  
     ./gradlew :performance:konanRun
+
+ **NOTE**: **konanRun** task needs built compiler and libs. To test against working tree make sure to run
+
+    ./gradlew dist distPlatformLibs
+
+ before **konanRun**
     
  **konanRun** task can be run separately for one/several benchmark applications:
  
@@ -257,4 +263,4 @@ konanc main.kt -Xtemporary-files-dir=<PATH> -o <OUTPUT_NAME>
 #### Example 3. Replace predefined LLVM pipeline with Clang options.
 ```shell script
 konanc main.kt -Xdisable-phases=BitcodeOptimization -Xoverride-clang-options=-c,-O2
-```
+```

build-tools/src/main/kotlin/org/jetbrains/kotlin/RunKotlinNativeTask.kt

@@ -8,6 +8,7 @@ package org.jetbrains.kotlin
 import org.gradle.api.DefaultTask
 import org.gradle.api.Task
 import org.jetbrains.kotlin.gradle.tasks.KotlinNativeLink
+import org.jetbrains.report.json.*
 import org.gradle.api.tasks.TaskAction
 import org.gradle.api.tasks.options.Option
 import org.gradle.api.tasks.Input
@@ -19,6 +20,11 @@ open class RunKotlinNativeTask @Inject constructor(private val linkTask: Task,
                                                    private val executable: String,
                                                    private val outputFileName: String
 ) : DefaultTask() {
+    enum class RepeatingType {
+        INTERNAL,  // Let the benchmark perform warmups and repeats.
+        EXTERNAL,  // Repeat by relaunching benchmark
+    }
+
     @Input
     @Option(option = "filter", description = "Benchmarks to run (comma-separated)")
     var filter: String = ""
@@ -28,6 +34,12 @@ open class RunKotlinNativeTask @Inject constructor(private val linkTask: Task,
     @Input
     @Option(option = "verbose", description = "Verbose mode of running benchmarks")
     var verbose: Boolean = false
+    @Input
+    var warmupCount: Int = 0
+    @Input
+    var repeatCount: Int = 0
+    @Input
+    var repeatingType = RepeatingType.INTERNAL
 
     private val argumentsList = mutableListOf<String>()
 
@@ -44,9 +56,41 @@ open class RunKotlinNativeTask @Inject constructor(private val linkTask: Task,
         argumentsList.addAll(arguments.toList())
     }
 
+    private fun execBenchmarkOnce(benchmark: String, warmupCount: Int, repeatCount: Int) : String {
+        val output = ByteArrayOutputStream()
+        project.exec {
+            it.executable = executable
+            it.args(argumentsList)
+            it.args("-f", benchmark)
+            if (verbose) {
+                it.args("-v")
+            }
+            it.args("-w", warmupCount.toString())
+            it.args("-r", repeatCount.toString())
+            it.standardOutput = output
+        }
+        return output.toString().removePrefix("[").removeSuffix("]")
+    }
+
+    private fun execBenchmarkRepeatedly(benchmark: String, warmupCount: Int, repeatCount: Int) : List<String> {
+        for (i in 0..warmupCount) {
+            execBenchmarkOnce(benchmark, 0, 1)
+        }
+        val result = mutableListOf<String>()
+        for (i in 0..repeatCount) {
+            val benchmarkReport = JsonTreeParser.parse(execBenchmarkOnce(benchmark, 0, 1)).jsonObject
+            val modifiedBenchmarkReport = JsonObject(HashMap(benchmarkReport.content).apply {
+                put("repeat", JsonLiteral(i))
+                put("warmup", JsonLiteral(warmupCount))
+            })
+            result.add(modifiedBenchmarkReport.toString())
+        }
+        return result
+    }
+
     @TaskAction
     fun run() {
-        var output = ByteArrayOutputStream()
+        val output = ByteArrayOutputStream()
         project.exec {
             it.executable = executable
             it.args("list")
@@ -60,18 +104,11 @@ open class RunKotlinNativeTask @Inject constructor(private val linkTask: Task,
             benchmarks.filter { benchmark -> benchmark in filterArgs || regexes.any { it.matches(benchmark) } }
         } else benchmarks.filter { !it.isEmpty() }
 
-        val results = benchmarksToRun.map { benchmark ->
+        val results = benchmarksToRun.flatMap { benchmark ->
-            output = ByteArrayOutputStream()
+            when (repeatingType) {
-            project.exec {
+                RepeatingType.INTERNAL -> listOf(execBenchmarkOnce(benchmark, warmupCount, repeatCount))
-                it.executable = executable
+                RepeatingType.EXTERNAL -> execBenchmarkRepeatedly(benchmark, warmupCount, repeatCount)
-                it.args(argumentsList)
-                it.args("-f", benchmark)
-                if (verbose) {
-                    it.args("-v")
-                }
-                it.standardOutput = output
             }
-            output.toString().removePrefix("[").removeSuffix("]")
         }
 
         File(outputFileName).printWriter().use { out ->

build-tools/src/main/kotlin/org/jetbrains/kotlin/benchmark/BenchmarkingPlugin.kt

@@ -70,6 +70,7 @@ open class BenchmarkExtension @Inject constructor(val project: Project) {
     var linkerOpts: Collection<String> = emptyList()
     var compilerOpts: List<String> = emptyList()
     var buildType: NativeBuildType = NativeBuildType.RELEASE
+    var repeatingType: RunKotlinNativeTask.RepeatingType = RunKotlinNativeTask.RepeatingType.INTERNAL
 
     val dependencies: BenchmarkDependencies = BenchmarkDependencies()
 
@@ -183,11 +184,11 @@ abstract class BenchmarkingPlugin: Plugin<Project> {
             description = "Runs the benchmark for Kotlin/Native."
         }
         afterEvaluate {
-            (konanRun as RunKotlinNativeTask).args(
+            val task = konanRun as RunKotlinNativeTask
-                    "-w", nativeWarmup.toString(),
+            task.args("-p", "${benchmark.applicationName}::")
-                    "-r", attempts.toString(),
+            task.warmupCount = nativeWarmup
-                    "-p", "${benchmark.applicationName}::"
+            task.repeatCount = attempts
-            )
+            task.repeatingType = benchmark.repeatingType
         }
         return konanRun
     }
@@ -274,4 +275,4 @@ abstract class BenchmarkingPlugin: Plugin<Project> {
         const val NATIVE_EXECUTABLE_NAME = "benchmark"
         const val BENCHMARKING_GROUP = "benchmarking"
     }
-}
+}

performance/build.gradle

@@ -233,6 +233,11 @@ task numerical {
     dependsOn 'numerical:konanRun'
 }
 
+task startup {
+    dependsOn 'clean'
+    dependsOn 'startup:konanRun'
+}
+
 task swiftinterop {
     dependsOn 'clean'
     dependsOn 'swiftinterop:konanRun'
@@ -246,4 +251,4 @@ task videoplayer {
 task KotlinVsSwift {
     dependsOn 'clean'
     dependsOn 'KotlinVsSwift:konanRun'
-}
+}

performance/ring/src/main/kotlin/main.kt

@@ -174,6 +174,7 @@ class RingLauncher : Launcher() {
                     "ParameterNotNull.invokeEightArgsWithoutNullCheck" to BenchmarkEntryWithInit.create(::ParameterNotNullAssertionBenchmark, { invokeEightArgsWithoutNullCheck() }),
                     "PrimeList.calcDirect" to BenchmarkEntryWithInit.create(::PrimeListBenchmark, { calcDirect() }),
                     "PrimeList.calcEratosthenes" to BenchmarkEntryWithInit.create(::PrimeListBenchmark, { calcEratosthenes() }),
+                    "Singleton.access" to BenchmarkEntryWithInit.create(::SingletonBenchmark, { access() }),
                     "String.stringConcat" to BenchmarkEntryWithInit.create(::StringBenchmark, { stringConcat() }),
                     "String.stringConcatNullable" to BenchmarkEntryWithInit.create(::StringBenchmark, { stringConcatNullable() }),
                     "String.stringBuilderConcat" to BenchmarkEntryWithInit.create(::StringBenchmark, { stringBuilderConcat() }),
@@ -219,4 +220,4 @@ fun main(args: Array<String>) {
                     arguments.filter, arguments.filterRegex, arguments.verbose)
         } else emptyList()
     }, benchmarksListAction = launcher::benchmarksListAction)
-}
+}

performance/ring/src/main/kotlin/org/jetbrains/ring/SingletonBenchmark.kt

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+package org.jetbrains.ring
+
+import org.jetbrains.benchmarksLauncher.Blackhole
+
+private object A {
+    val a = Random.nextInt(100)
+}
+
+open class SingletonBenchmark {
+    init {
+        // Make sure A is initialized.
+        Blackhole.consume(A.a)
+    }
+
+    // Benchmark
+    fun access() {
+        for (i in 0 until BENCHMARK_SIZE) {
+            Blackhole.consume(A.a)
+        }
+    }
+}

performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/BenchmarksCollection.kt

@@ -16,6 +16,8 @@
 
 package org.jetbrains.benchmarksLauncher
 
+import org.jetbrains.report.BenchmarkResult
+
 interface AbstractBenchmarkEntry
 
 class BenchmarkEntryWithInit(val ctor: ()->Any, val lambda: (Any) -> Any?): AbstractBenchmarkEntry {
@@ -26,5 +28,13 @@ class BenchmarkEntryWithInit(val ctor: ()->Any, val lambda: (Any) -> Any?): Abst
 
 class BenchmarkEntry(val lambda: () -> Any?) : AbstractBenchmarkEntry
 
+// Controls warmup and repeats manually.
+data class BenchmarkManualResult(
+    val status: BenchmarkResult.Status,
+    val value: Any?,
+    val warmupCount: Int,
+    val durationsNs: List<Double>)
+class BenchmarkEntryManual(val lambda: () -> BenchmarkManualResult) : AbstractBenchmarkEntry
+
 class BenchmarksCollection(private val benchmarks: MutableMap<String, AbstractBenchmarkEntry> = mutableMapOf()) :
         MutableMap<String, AbstractBenchmarkEntry> by benchmarks

performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/launcher.kt

@@ -19,6 +19,11 @@ package org.jetbrains.benchmarksLauncher
 import org.jetbrains.report.BenchmarkResult
 import kotlinx.cli.*
 
+data class RecordTimeMeasurement(
+    val status: BenchmarkResult.Status,
+    val iteration: Int,
+    val warmupCount: Int,
+    val durationNs: Double)
 
 abstract class Launcher {
     abstract val benchmarks: BenchmarksCollection
@@ -35,14 +40,16 @@ abstract class Launcher {
                 while (i-- > 0) benchmark.lambda(benchmarkInstance!!)
                 cleanup()
             }
-        } else {
+        } else if (benchmark is BenchmarkEntry) {
             cleanup()
             measureNanoTime {
-                if (benchmark is BenchmarkEntry) {
+                while (i-- > 0) benchmark.lambda()
-                    while (i-- > 0) benchmark.lambda()
+                cleanup()
-                    cleanup()
-                }
             }
+        } else if (benchmark is BenchmarkEntryManual) {
+            error("runBenchmark cannot run manual benchmark")
+        } else {
+            error("Unknown benchmark type $benchmark")
         }
     }
 
@@ -60,6 +67,72 @@ abstract class Launcher {
         }
     }
 
+    fun runBenchmarkRepeatedly(logger: Logger,
+                               numWarmIterations: Int,
+                               numberOfAttempts: Int,
+                               name: String,
+                               recordMeasurement: (RecordTimeMeasurement) -> Unit,
+                               benchmarkInstance: Any?,
+                               benchmark: AbstractBenchmarkEntry) {
+        logger.log("Warm up iterations for benchmark $name\n")
+        runBenchmark(benchmarkInstance, benchmark, numWarmIterations)
+        var autoEvaluatedNumberOfMeasureIteration = 1
+        while (true) {
+            var j = autoEvaluatedNumberOfMeasureIteration
+            val time = runBenchmark(benchmarkInstance, benchmark, j)
+            if (time >= 100L * 1_000_000) // 100ms
+                break
+            autoEvaluatedNumberOfMeasureIteration *= 2
+        }
+        logger.log("Running benchmark $name ")
+        for (k in 0..numberOfAttempts) {
+            logger.log(".", usePrefix = false)
+            var i = autoEvaluatedNumberOfMeasureIteration
+            val time = runBenchmark(benchmarkInstance, benchmark, i)
+            val scaledTime = time * 1.0 / autoEvaluatedNumberOfMeasureIteration
+            // Save benchmark object
+            recordMeasurement(RecordTimeMeasurement(BenchmarkResult.Status.PASSED, k, numWarmIterations, scaledTime))
+        }
+        logger.log("\n", usePrefix = false)
+    }
+
+    fun runBenchmark(logger: Logger,
+                     numWarmIterations: Int,
+                     numberOfAttempts: Int,
+                     name: String,
+                     recordMeasurement: (RecordTimeMeasurement) -> Unit,
+                     benchmark: AbstractBenchmarkEntry) {
+        when (benchmark) {
+            is BenchmarkEntryWithInit -> {
+                val benchmarkInstance = benchmark.ctor?.invoke()
+                runBenchmarkRepeatedly(logger,
+                                       numWarmIterations,
+                                       numberOfAttempts,
+                                       name,
+                                       recordMeasurement,
+                                       benchmarkInstance,
+                                       benchmark)
+            }
+            is BenchmarkEntry -> {
+                runBenchmarkRepeatedly(logger,
+                                       numWarmIterations,
+                                       numberOfAttempts,
+                                       name,
+                                       recordMeasurement,
+                                       null,
+                                       benchmark)
+            }
+            is BenchmarkEntryManual -> {
+                logger.log("Running manual benchmark $name")
+                val result = benchmark.lambda()
+                for ((i, durationNs) in result.durationsNs.withIndex()) {
+                    recordMeasurement(RecordTimeMeasurement(result.status, i, result.warmupCount, durationNs))
+                }
+            }
+            else -> error("unknown benchmark type $benchmark")
+        }
+    }
+
     fun launch(numWarmIterations: Int,
                numberOfAttempts: Int,
                prefix: String = "",
@@ -73,36 +146,28 @@ abstract class Launcher {
         val runningBenchmarks = if (filterSet.isNotEmpty() || regexes.isNotEmpty()) {
             benchmarks.filterKeys { benchmark -> benchmark in filterSet || regexes.any { it.matches(benchmark) } }
         } else benchmarks
-        if (runningBenchmarks.isEmpty())
+        if (runningBenchmarks.isEmpty()) {
+            printStderr("No matching benchmarks found\n")
             error("No matching benchmarks found")
+        }
         val benchmarkResults = mutableListOf<BenchmarkResult>()
         for ((name, benchmark) in runningBenchmarks) {
-            val benchmarkInstance = (benchmark as? BenchmarkEntryWithInit)?.ctor?.invoke()
+            val recordMeasurement : (RecordTimeMeasurement) -> Unit = {
-            var i = numWarmIterations
+                benchmarkResults.add(BenchmarkResult(
-            logger.log("Warm up iterations for benchmark $name\n")
+                    "$prefix$name",
-            runBenchmark(benchmarkInstance, benchmark, i)
+                    it.status,
-            var autoEvaluatedNumberOfMeasureIteration = 1
+                    it.durationNs / 1000,
-            while (true) {
+                    BenchmarkResult.Metric.EXECUTION_TIME,
-                var j = autoEvaluatedNumberOfMeasureIteration
+                    it.durationNs / 1000,
-                val time = runBenchmark(benchmarkInstance, benchmark, j)
+                    it.iteration + 1,
-                if (time >= 100L * 1_000_000) // 100ms
+                    it.warmupCount))
-                    break
-                autoEvaluatedNumberOfMeasureIteration *= 2
             }
-            logger.log("Running benchmark $name ")
+            try {
-            val samples = DoubleArray(numberOfAttempts)
+                runBenchmark(logger, numWarmIterations, numberOfAttempts, name, recordMeasurement, benchmark)
-            for (k in samples.indices) {
+            } catch (e: Throwable) {
-                logger.log(".", usePrefix = false)
+                printStderr("Failure while running benchmark $name: $e\n")
-                i = autoEvaluatedNumberOfMeasureIteration
+                throw e
-                val time = runBenchmark(benchmarkInstance, benchmark, i)
-                val scaledTime = time * 1.0 / autoEvaluatedNumberOfMeasureIteration
-                samples[k] = scaledTime
-                // Save benchmark object
-                benchmarkResults.add(BenchmarkResult("$prefix$name", BenchmarkResult.Status.PASSED,
-                        scaledTime / 1000, BenchmarkResult.Metric.EXECUTION_TIME, scaledTime / 1000,
-                        k + 1, numWarmIterations))
             }
-            logger.log("\n", usePrefix = false)
         }
         return benchmarkResults
     }

performance/startup/build.gradle.kts

@@ -0,0 +1,26 @@
+import org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType
+import org.jetbrains.kotlin.RunKotlinNativeTask
+
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+plugins {
+    id("benchmarking")
+}
+
+val defaultBuildType = NativeBuildType.RELEASE
+
+benchmark {
+    applicationName = "Startup"
+    commonSrcDirs = listOf("../../tools/benchmarks/shared/src", "src/main/kotlin", "../shared/src/main/kotlin")
+    jvmSrcDirs = listOf("src/main/kotlin-jvm", "../shared/src/main/kotlin-jvm")
+    nativeSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/common")
+    mingwSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/mingw")
+    posixSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/posix")
+    buildType = (findProperty("nativeBuildType") as String?)?.let { NativeBuildType.valueOf(it) } ?: defaultBuildType
+    repeatingType = RunKotlinNativeTask.RepeatingType.EXTERNAL
+
+    dependencies.common(project(":endorsedLibraries:kotlinx.cli"))
+}

performance/startup/gradle.properties

@@ -0,0 +1 @@
+kotlin.native.home=../../dist

performance/startup/src/main/kotlin-jvm/org/jetbrains/startup/Utils.kt

@@ -0,0 +1,16 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+package org.jetbrains.startup
+
+actual class Random actual constructor() {
+    actual companion object {
+        actual var seedInt = 0
+        actual fun nextInt(boundary: Int): Int {
+            seedInt = (3 * seedInt + 11) % boundary
+            return seedInt
+        }
+    }
+}

performance/startup/src/main/kotlin-native/org/jetbrains/startup/Utils.kt

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+package org.jetbrains.startup
+
+actual class Random actual constructor() {
+    @kotlin.native.ThreadLocal
+    actual companion object {
+        actual var seedInt = 0
+        actual fun nextInt(boundary: Int): Int {
+            seedInt = (3 * seedInt + 11) % boundary
+            return seedInt
+        }
+    }
+}

performance/startup/src/main/kotlin/main.kt

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+import org.jetbrains.startup.*
+import org.jetbrains.benchmarksLauncher.*
+import kotlinx.cli.*
+
+class StartupLauncher : Launcher() {
+    override val benchmarks = BenchmarksCollection(
+      mutableMapOf(
+          "Singleton.initialize" to BenchmarkEntryManual(::singletonInitialize),
+          "Singleton.initializeNested" to BenchmarkEntryManual(::singletonInitializeNested),
+      )
+    )
+}
+
+fun main(args: Array<String>) {
+    val launcher = StartupLauncher()
+    BenchmarksRunner.runBenchmarks(args, { arguments: BenchmarkArguments ->
+        if (arguments is BaseBenchmarkArguments) {
+            launcher.launch(arguments.warmup, arguments.repeat, arguments.prefix,
+                    arguments.filter, arguments.filterRegex, arguments.verbose)
+        } else emptyList()
+    }, benchmarksListAction = launcher::benchmarksListAction)
+}

performance/startup/src/main/kotlin/org/jetbrains/startup/Utils.kt

@@ -0,0 +1,13 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+package org.jetbrains.startup
+
+expect class Random() {
+    companion object {
+        var seedInt: Int
+        fun nextInt(boundary: Int = 100): Int
+    }
+}

settings.gradle

@@ -39,6 +39,7 @@ include ':performance:helloworld'
 include ':performance:numerical'
 include ':performance:videoplayer'
 include ':performance:framework'
+include ':performance:startup'
 if (System.getProperty("os.name") == "Mac OS X") {
     include ':performance:objcinterop'
     include ':performance:swiftinterop'