Refactor new type of benchmarks and support running on JVM in new mode (#4041)

build-tools/src/main/kotlin/org/jetbrains/kotlin/BenchmarkRepeatingType.kt

@@ -0,0 +1,6 @@
+package org.jetbrains.kotlin
+
+enum class BenchmarkRepeatingType {
+    INTERNAL,  // Let the benchmark perform warmups and repeats.
+    EXTERNAL,  // Repeat by relaunching benchmark
+}

build-tools/src/main/kotlin/org/jetbrains/kotlin/RunJvmTask.kt

@@ -5,16 +5,21 @@
 
 package org.jetbrains.kotlin
 
-import groovy.lang.Closure
 import org.gradle.api.tasks.JavaExec
-import org.gradle.api.Task
 import org.gradle.api.tasks.TaskAction
 import org.gradle.api.tasks.options.Option
 import org.gradle.api.tasks.Input
+import org.jetbrains.kotlin.benchmark.LogLevel
+import org.jetbrains.kotlin.benchmark.Logger
+import org.jetbrains.report.json.*
+import java.io.ByteArrayOutputStream
 import java.io.File
 
-open class RunJvmTask : JavaExec() {
-    var outputFileName: String? = null
+data class ExecParameters(val warmupCount: Int, val repeatCount: Int,
+                          val filterArgs: List<String>, val filterRegexArgs: List<String>,
+                          val verbose: Boolean, val outputFileName: String?)
+
+open class RunJvmTask: JavaExec() {
     @Input
     @Option(option = "filter", description = "Benchmarks to run (comma-separated)")
     var filter: String = ""
@@ -24,24 +29,97 @@ open class RunJvmTask : JavaExec() {
     @Input
     @Option(option = "verbose", description = "Verbose mode of running benchmarks")
     var verbose: Boolean = false
+    @Input
+    var warmupCount: Int = 0
+    @Input
+    var repeatCount: Int = 0
+    @Input
+    var repeatingType = BenchmarkRepeatingType.INTERNAL
+    @Input
+    var outputFileName: String? = null
 
-    @JvmOverloads
-    private fun executeTask(output: java.io.OutputStream? = null) {
-        val filterArgs = filter.splitCommaSeparatedOption("-f")
-        val filterRegexArgs = filterRegex.splitCommaSeparatedOption("-fr")
-        args(filterArgs)
-        args(filterRegexArgs)
-        if (verbose) {
+    private var predefinedArgs: List<String> = emptyList()
+
+    private fun executeTask(execParameters: ExecParameters): String {
+        // Firstly clean arguments.
+        setArgs(emptyList())
+        args(predefinedArgs)
+        args(execParameters.filterArgs)
+        args(execParameters.filterRegexArgs)
+        args("-w", execParameters.warmupCount)
+        args("-r", execParameters.repeatCount)
+        if (execParameters.verbose) {
             args("-v")
         }
+        execParameters.outputFileName?.let { args("-o", outputFileName) }
+        standardOutput = ByteArrayOutputStream()
         super.exec()
+        return standardOutput.toString()
+    }
+
+    private fun getBenchmarksList(filterArgs: List<String>, filterRegexArgs: List<String>): List<String> {
+        // Firstly clean arguments.
+        setArgs(emptyList())
+        args("list")
+        standardOutput = ByteArrayOutputStream()
+        super.exec()
+        val benchmarks = standardOutput.toString().lines()
+        val regexes = filterRegexArgs.map { it.toRegex() }
+        return if (filterArgs.isNotEmpty() || regexes.isNotEmpty()) {
+            benchmarks.filter { benchmark -> benchmark in filterArgs || regexes.any { it.matches(benchmark) } }
+        } else benchmarks.filter { !it.isEmpty() }
+    }
+
+    private fun execSeparateBenchmarkRepeatedly(benchmark: String): List<String> {
+        // Logging with application should be done only in case it controls running benchmarks itself.
+        // Although it's a responsibility of gradle task.
+        val logger = if (verbose) Logger(LogLevel.DEBUG) else Logger()
+        logger.log("Warm up iterations for benchmark $benchmark\n")
+        for (i in 0.until(warmupCount)) {
+            executeTask(ExecParameters(0, 1, listOf("-f", benchmark),
+                    emptyList(), false, null))
+        }
+        val result = mutableListOf<String>()
+        logger.log("Running benchmark $benchmark ")
+        for (i in 0.until(repeatCount)) {
+            logger.log(".", usePrefix = false)
+            val benchmarkReport = JsonTreeParser.parse(
+                    executeTask(ExecParameters(0, 1, listOf("-f", benchmark),
+                            emptyList(), false, null)
+                    ).removePrefix("[").removeSuffix("]")
+            ).jsonObject
+            val modifiedBenchmarkReport = JsonObject(HashMap(benchmarkReport.content).apply {
+                put("repeat", JsonLiteral(i))
+                put("warmup", JsonLiteral(warmupCount))
+            })
+            result.add(modifiedBenchmarkReport.toString())
+        }
+        logger.log("\n", usePrefix = false)
+        return result
+    }
+
+    private fun execBenchmarksRepeatedly(filterArgs: List<String>, filterRegexArgs: List<String>) {
+        val benchmarksToRun = getBenchmarksList(filterArgs, filterRegexArgs)
+        val results = benchmarksToRun.flatMap { benchmark ->
+            execSeparateBenchmarkRepeatedly(benchmark)
+        }
+        File(outputFileName).printWriter().use { out ->
+            out.println("[${results.joinToString(",")}]")
+        }
     }
 
     @TaskAction
     override fun exec() {
-        if (outputFileName != null)
-            File(outputFileName).outputStream().use { output -> executeTask(output) }
-        else
-            executeTask()
+        assert(outputFileName != null) { "Output file name should be always set" }
+        predefinedArgs = args ?: emptyList()
+        val filterArgs = filter.splitCommaSeparatedOption("-f")
+        val filterRegexArgs = filterRegex.splitCommaSeparatedOption("-fr")
+        when (repeatingType) {
+            BenchmarkRepeatingType.INTERNAL -> executeTask(
+                    ExecParameters(warmupCount, repeatCount, filterArgs, filterRegexArgs, verbose, outputFileName)
+            )
+            BenchmarkRepeatingType.EXTERNAL -> execBenchmarksRepeatedly(filterArgs, filterRegexArgs)
+        }
+
     }
 }

build-tools/src/main/kotlin/org/jetbrains/kotlin/RunKotlinNativeTask.kt

@@ -7,7 +7,8 @@ package org.jetbrains.kotlin
 
 import org.gradle.api.DefaultTask
 import org.gradle.api.Task
-import org.jetbrains.kotlin.gradle.tasks.KotlinNativeLink
+import org.jetbrains.kotlin.benchmark.Logger
+import org.jetbrains.kotlin.benchmark.LogLevel
 import org.jetbrains.report.json.*
 import org.gradle.api.tasks.TaskAction
 import org.gradle.api.tasks.options.Option
@@ -15,15 +16,12 @@ import org.gradle.api.tasks.Input
 import java.io.ByteArrayOutputStream
 import java.io.File
 import javax.inject.Inject
+import kotlin.collections.HashMap
 
 open class RunKotlinNativeTask @Inject constructor(private val linkTask: Task,
                                                    private val executable: String,
                                                    private val outputFileName: String
 ) : DefaultTask() {
-    enum class RepeatingType {
-        INTERNAL,  // Let the benchmark perform warmups and repeats.
-        EXTERNAL,  // Repeat by relaunching benchmark
-    }
 
     @Input
     @Option(option = "filter", description = "Benchmarks to run (comma-separated)")
@@ -39,7 +37,7 @@ open class RunKotlinNativeTask @Inject constructor(private val linkTask: Task,
     @Input
     var repeatCount: Int = 0
     @Input
-    var repeatingType = RepeatingType.INTERNAL
+    var repeatingType = BenchmarkRepeatingType.INTERNAL
 
     private val argumentsList = mutableListOf<String>()
 
@@ -62,7 +60,9 @@ open class RunKotlinNativeTask @Inject constructor(private val linkTask: Task,
             it.executable = executable
             it.args(argumentsList)
             it.args("-f", benchmark)
-            if (verbose) {
+            // Logging with application should be done only in case it controls running benchmarks itself.
+            // Although it's a responsibility of gradle task.
+            if (verbose && repeatingType == BenchmarkRepeatingType.INTERNAL) {
                 it.args("-v")
             }
             it.args("-w", warmupCount.toString())
@@ -73,11 +73,15 @@ open class RunKotlinNativeTask @Inject constructor(private val linkTask: Task,
     }
 
     private fun execBenchmarkRepeatedly(benchmark: String, warmupCount: Int, repeatCount: Int) : List<String> {
-        for (i in 0..warmupCount) {
+        val logger = if (verbose) Logger(LogLevel.DEBUG) else Logger()
+        logger.log("Warm up iterations for benchmark $benchmark\n")
+        for (i in 0.until(warmupCount)) {
             execBenchmarkOnce(benchmark, 0, 1)
         }
         val result = mutableListOf<String>()
-        for (i in 0..repeatCount) {
+        logger.log("Running benchmark $benchmark ")
+        for (i in 0.until(repeatCount)) {
+            logger.log(".", usePrefix = false)
             val benchmarkReport = JsonTreeParser.parse(execBenchmarkOnce(benchmark, 0, 1)).jsonObject
             val modifiedBenchmarkReport = JsonObject(HashMap(benchmarkReport.content).apply {
                 put("repeat", JsonLiteral(i))
@@ -85,6 +89,7 @@ open class RunKotlinNativeTask @Inject constructor(private val linkTask: Task,
             })
             result.add(modifiedBenchmarkReport.toString())
         }
+        logger.log("\n", usePrefix = false)
         return result
     }
 
@@ -106,8 +111,8 @@ open class RunKotlinNativeTask @Inject constructor(private val linkTask: Task,
 
         val results = benchmarksToRun.flatMap { benchmark ->
             when (repeatingType) {
-                RepeatingType.INTERNAL -> listOf(execBenchmarkOnce(benchmark, warmupCount, repeatCount))
-                RepeatingType.EXTERNAL -> execBenchmarkRepeatedly(benchmark, warmupCount, repeatCount)
+                BenchmarkRepeatingType.INTERNAL -> listOf(execBenchmarkOnce(benchmark, warmupCount, repeatCount))
+                BenchmarkRepeatingType.EXTERNAL -> execBenchmarkRepeatedly(benchmark, warmupCount, repeatCount)
             }
         }
 

build-tools/src/main/kotlin/org/jetbrains/kotlin/benchmark/BenchmarkLogger.kt

@@ -0,0 +1,25 @@
+package org.jetbrains.kotlin.benchmark
+
+import java.text.SimpleDateFormat
+import java.util.*
+
+enum class LogLevel { DEBUG, OFF }
+
+class Logger(val level: LogLevel = LogLevel.OFF) {
+    private fun printStderr(message: String) {
+        System.err.print(message)
+    }
+
+    private fun currentTime(): String =
+            SimpleDateFormat("HH:mm:ss").format(Date())
+
+    fun log(message: String, messageLevel: LogLevel = LogLevel.DEBUG, usePrefix: Boolean = true) {
+        if (messageLevel == level) {
+            if (usePrefix) {
+                printStderr("[$level][${currentTime()}] $message")
+            } else {
+                printStderr("$message")
+            }
+        }
+    }
+}

build-tools/src/main/kotlin/org/jetbrains/kotlin/benchmark/BenchmarkingPlugin.kt

@@ -70,7 +70,7 @@ open class BenchmarkExtension @Inject constructor(val project: Project) {
     var linkerOpts: Collection<String> = emptyList()
     var compilerOpts: List<String> = emptyList()
     var buildType: NativeBuildType = NativeBuildType.RELEASE
-    var repeatingType: RunKotlinNativeTask.RepeatingType = RunKotlinNativeTask.RepeatingType.INTERNAL
+    var repeatingType: BenchmarkRepeatingType = BenchmarkRepeatingType.INTERNAL
 
     val dependencies: BenchmarkDependencies = BenchmarkDependencies()
 

build-tools/src/main/kotlin/org/jetbrains/kotlin/benchmark/KotlinNativeBenchmarkingPlugin.kt

@@ -8,7 +8,6 @@ import org.jetbrains.kotlin.*
 import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet
 import org.jetbrains.kotlin.gradle.plugin.mpp.Executable
 import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget
-import org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType
 import org.jetbrains.kotlin.konan.target.HostManager
 import javax.inject.Inject
 import kotlin.reflect.KClass
@@ -76,12 +75,11 @@ open class KotlinNativeBenchmarkingPlugin: BenchmarkingPlugin() {
 
             // Specify settings configured by a user in the benchmark extension.
             afterEvaluate {
-                task.args(
-                        "-w", jvmWarmup,
-                        "-r", attempts,
-                        "-o", buildDir.resolve(jvmBenchResults),
-                        "-p", "${benchmark.applicationName}::"
-                )
+                task.args("-p", "${benchmark.applicationName}::")
+                task.warmupCount = jvmWarmup
+                task.repeatCount = attempts
+                task.outputFileName = buildDir.resolve(jvmBenchResults).absolutePath
+                task.repeatingType = benchmark.repeatingType
             }
         }
     }

performance/ring/src/main/kotlin-jvm/org/jetbrains/ring/UtilsJVM.kt

@@ -19,24 +19,6 @@ package org.jetbrains.ring
 import java.util.concurrent.atomic.AtomicReferenceFieldUpdater
 import java.util.concurrent.locks.ReentrantLock
 
-//-----------------------------------------------------------------------------//
-
-actual class Random actual constructor() {
-    actual companion object {
-        actual var seedInt = 0
-        actual fun nextInt(boundary: Int): Int {
-            seedInt = (3 * seedInt + 11) % boundary
-            return seedInt
-        }
-
-        actual var seedDouble: Double = 0.1
-        actual fun nextDouble(boundary: Double): Double {
-            seedDouble = (7.0 * seedDouble + 7.0) % boundary
-            return seedDouble
-        }
-    }
-}
-
 internal var interceptor: AtomicOperationInterceptor = DefaultInterceptor
     private set
 private val interceptorLock = ReentrantLock()

performance/ring/src/main/kotlin-native/org/jetbrains/ring/Utils.kt

@@ -20,25 +20,6 @@ import kotlin.native.concurrent.FreezableAtomicReference as KAtomicRef
 import kotlin.native.concurrent.isFrozen
 import kotlin.native.concurrent.freeze
 
-//-----------------------------------------------------------------------------//
-
-actual class Random actual constructor() {
-    @kotlin.native.ThreadLocal
-    actual companion object {
-        actual var seedInt = 0
-        actual fun nextInt(boundary: Int): Int {
-            seedInt = (3 * seedInt + 11) % boundary
-            return seedInt
-        }
-
-        actual var seedDouble: Double = 0.1
-        actual fun nextDouble(boundary: Double): Double {
-            seedDouble = (7.0 * seedDouble + 7.0) % boundary
-            return seedDouble
-        }
-    }
-}
-
 public actual class AtomicRef<T> constructor(@PublishedApi internal val a: KAtomicRef<T>) {
     public actual inline var value: T
         get() = a.value

performance/ring/src/main/kotlin/org/jetbrains/ring/DefaultArgumentBenchmark.kt

@@ -15,6 +15,7 @@
  */
 
 package org.jetbrains.ring
+import org.jetbrains.benchmarksLauncher.Random
 
 /**
  * Created by Mikhail.Glukhikh on 10/03/2015.

performance/ring/src/main/kotlin/org/jetbrains/ring/ElvisBenchmark.kt

@@ -17,6 +17,7 @@
 package org.jetbrains.ring
 
 import org.jetbrains.benchmarksLauncher.Blackhole
+import org.jetbrains.benchmarksLauncher.Random
 
 open class ElvisBenchmark {
 

performance/ring/src/main/kotlin/org/jetbrains/ring/LambdaBenchmark.kt

@@ -16,6 +16,8 @@
 
 package org.jetbrains.ring
 
+import org.jetbrains.benchmarksLauncher.Random
+
 var globalAddendum = 0
 
 open class LambdaBenchmark {

performance/ring/src/main/kotlin/org/jetbrains/ring/LinkedListWithAtomicsBenchmark.kt

@@ -5,6 +5,8 @@
 
 package org.jetbrains.ring
 
+import org.jetbrains.benchmarksLauncher.Random
+
 class ChunkBuffer(var readPosition: Int, var writePosition: Int = readPosition + Random.nextInt(50)) {
     private val nextRef: AtomicRef<ChunkBuffer?> = atomic(null)
 

performance/ring/src/main/kotlin/org/jetbrains/ring/MatrixMapBenchmark.kt

@@ -16,7 +16,8 @@
 
 package org.jetbrains.ring
 
-import org.jetbrains.ring.BENCHMARK_SIZE
+import org.jetbrains.benchmarksLauncher.Blackhole
+import org.jetbrains.benchmarksLauncher.Random
 
 /**
  * This class emulates matrix behaviour using a hash map as its implementation

performance/ring/src/main/kotlin/org/jetbrains/ring/SingletonBenchmark.kt

@@ -6,6 +6,7 @@
 package org.jetbrains.ring
 
 import org.jetbrains.benchmarksLauncher.Blackhole
+import org.jetbrains.benchmarksLauncher.Random
 
 private object A {
     val a = Random.nextInt(100)

performance/ring/src/main/kotlin/org/jetbrains/ring/StringBenchmark.kt

@@ -16,6 +16,8 @@
 
 package org.jetbrains.ring
 
+import org.jetbrains.benchmarksLauncher.Random
+
 open class StringBenchmark {
     private var _data: ArrayList<String>? = null
     val data: ArrayList<String>

performance/ring/src/main/kotlin/org/jetbrains/ring/SwitchBenchmark.kt

@@ -17,6 +17,7 @@
 package org.jetbrains.ring
 
 import org.jetbrains.benchmarksLauncher.Blackhole
+import org.jetbrains.benchmarksLauncher.Random
 
 val SPARSE_SWITCH_CASES = intArrayOf(11, 29, 47, 71, 103,
                                      149, 175, 227, 263, 307,

performance/ring/src/main/kotlin/org/jetbrains/ring/Utils.kt

@@ -18,16 +18,6 @@ package org.jetbrains.ring
 
 const val BENCHMARK_SIZE = 10000
 
-expect class Random() {
-    companion object {
-        var seedInt: Int
-        fun nextInt(boundary: Int = 100): Int
-
-        var seedDouble: Double
-        fun nextDouble(boundary: Double = 100.0): Double
-    }
-}
-
 expect class AtomicRef<T> {
     /**
      * Reading/writing this property maps to read/write of volatile variable.

performance/shared/src/main/kotlin-jvm/org/jetbrains/benchmarksLauncher/Utils.kt

@@ -56,3 +56,19 @@ actual class Blackhole {
     }
 }
 
+actual class Random actual constructor() {
+    actual companion object {
+        actual var seedInt = 0
+        actual fun nextInt(boundary: Int): Int {
+            seedInt = (3 * seedInt + 11) % boundary
+            return seedInt
+        }
+
+        actual var seedDouble: Double = 0.1
+        actual fun nextDouble(boundary: Double): Double {
+            seedDouble = (7.0 * seedDouble + 7.0) % boundary
+            return seedDouble
+        }
+    }
+}
+

performance/shared/src/main/kotlin-native/common/org/jetbrains/benchmarksLauncher/Utils.kt

@@ -59,4 +59,21 @@ actual class Blackhole {
             consumer += value.hashCode()
         }
     }
+}
+
+actual class Random actual constructor() {
+    @kotlin.native.ThreadLocal
+    actual companion object {
+        actual var seedInt = 0
+        actual fun nextInt(boundary: Int): Int {
+            seedInt = (3 * seedInt + 11) % boundary
+            return seedInt
+        }
+
+        actual var seedDouble: Double = 0.1
+        actual fun nextDouble(boundary: Double): Double {
+            seedDouble = (7.0 * seedDouble + 7.0) % boundary
+            return seedDouble
+        }
+    }
 }

performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/BenchmarksCollection.kt

@@ -18,23 +18,25 @@ package org.jetbrains.benchmarksLauncher
 
 import org.jetbrains.report.BenchmarkResult
 
-interface AbstractBenchmarkEntry
+interface AbstractBenchmarkEntry {
+    open val useAutoEvaluatedNumberOfMeasure: Boolean
+}
 
 class BenchmarkEntryWithInit(val ctor: ()->Any, val lambda: (Any) -> Any?): AbstractBenchmarkEntry {
     companion object {
         inline fun <T: Any> create(noinline ctor: ()->T, crossinline lambda: T.() -> Any?) = BenchmarkEntryWithInit(ctor) { (it as T).lambda() }
     }
+
+    override val useAutoEvaluatedNumberOfMeasure: Boolean = true
 }
 
-class BenchmarkEntry(val lambda: () -> Any?) : AbstractBenchmarkEntry
+open class BenchmarkEntry(val lambda: () -> Any?) : AbstractBenchmarkEntry {
+    override val useAutoEvaluatedNumberOfMeasure: Boolean = true
+}
 
-// Controls warmup and repeats manually.
-data class BenchmarkManualResult(
-    val status: BenchmarkResult.Status,
-    val value: Any?,
-    val warmupCount: Int,
-    val durationsNs: List<Double>)
-class BenchmarkEntryManual(val lambda: () -> BenchmarkManualResult) : AbstractBenchmarkEntry
+class BenchmarkEntryManual(lambda: () -> Any?) : BenchmarkEntry(lambda) {
+    override val useAutoEvaluatedNumberOfMeasure: Boolean = false
+}
 
 class BenchmarksCollection(private val benchmarks: MutableMap<String, AbstractBenchmarkEntry> = mutableMapOf()) :
         MutableMap<String, AbstractBenchmarkEntry> by benchmarks

performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/Utils.kt

@@ -35,4 +35,14 @@ expect class Blackhole {
         var consumer: Int
         fun consume(value: Any)
     }
+}
+
+expect class Random() {
+    companion object {
+        var seedInt: Int
+        fun nextInt(boundary: Int = 100): Int
+
+        var seedDouble: Double
+        fun nextDouble(boundary: Double = 100.0): Double
+    }
 }

performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/launcher.kt

@@ -67,17 +67,17 @@ abstract class Launcher {
         }
     }
 
-    fun runBenchmarkRepeatedly(logger: Logger,
-                               numWarmIterations: Int,
-                               numberOfAttempts: Int,
-                               name: String,
-                               recordMeasurement: (RecordTimeMeasurement) -> Unit,
-                               benchmarkInstance: Any?,
-                               benchmark: AbstractBenchmarkEntry) {
+    fun runBenchmark(logger: Logger,
+                     numWarmIterations: Int,
+                     numberOfAttempts: Int,
+                     name: String,
+                     recordMeasurement: (RecordTimeMeasurement) -> Unit,
+                     benchmark: AbstractBenchmarkEntry) {
+        val benchmarkInstance = (benchmark as? BenchmarkEntryWithInit)?.ctor?.invoke()
         logger.log("Warm up iterations for benchmark $name\n")
         runBenchmark(benchmarkInstance, benchmark, numWarmIterations)
         var autoEvaluatedNumberOfMeasureIteration = 1
-        while (true) {
+        while (true && benchmark.useAutoEvaluatedNumberOfMeasure) {
             var j = autoEvaluatedNumberOfMeasureIteration
             val time = runBenchmark(benchmarkInstance, benchmark, j)
             if (time >= 100L * 1_000_000) // 100ms
@@ -85,7 +85,7 @@ abstract class Launcher {
             autoEvaluatedNumberOfMeasureIteration *= 2
         }
         logger.log("Running benchmark $name ")
-        for (k in 0..numberOfAttempts) {
+        for (k in 0.until(numberOfAttempts)) {
             logger.log(".", usePrefix = false)
             var i = autoEvaluatedNumberOfMeasureIteration
             val time = runBenchmark(benchmarkInstance, benchmark, i)
@@ -96,43 +96,6 @@ abstract class Launcher {
         logger.log("\n", usePrefix = false)
     }
 
-    fun runBenchmark(logger: Logger,
-                     numWarmIterations: Int,
-                     numberOfAttempts: Int,
-                     name: String,
-                     recordMeasurement: (RecordTimeMeasurement) -> Unit,
-                     benchmark: AbstractBenchmarkEntry) {
-        when (benchmark) {
-            is BenchmarkEntryWithInit -> {
-                val benchmarkInstance = benchmark.ctor?.invoke()
-                runBenchmarkRepeatedly(logger,
-                                       numWarmIterations,
-                                       numberOfAttempts,
-                                       name,
-                                       recordMeasurement,
-                                       benchmarkInstance,
-                                       benchmark)
-            }
-            is BenchmarkEntry -> {
-                runBenchmarkRepeatedly(logger,
-                                       numWarmIterations,
-                                       numberOfAttempts,
-                                       name,
-                                       recordMeasurement,
-                                       null,
-                                       benchmark)
-            }
-            is BenchmarkEntryManual -> {
-                logger.log("Running manual benchmark $name")
-                val result = benchmark.lambda()
-                for ((i, durationNs) in result.durationsNs.withIndex()) {
-                    recordMeasurement(RecordTimeMeasurement(result.status, i, result.warmupCount, durationNs))
-                }
-            }
-            else -> error("unknown benchmark type $benchmark")
-        }
-    }
-
     fun launch(numWarmIterations: Int,
                numberOfAttempts: Int,
                prefix: String = "",
@@ -166,7 +129,10 @@ abstract class Launcher {
                 runBenchmark(logger, numWarmIterations, numberOfAttempts, name, recordMeasurement, benchmark)
             } catch (e: Throwable) {
                 printStderr("Failure while running benchmark $name: $e\n")
-                throw e
+                benchmarkResults.add(BenchmarkResult(
+                        "$prefix$name", BenchmarkResult.Status.FAILED, 0.0,
+                        BenchmarkResult.Metric.EXECUTION_TIME, 0.0, numberOfAttempts, numWarmIterations)
+                )
             }
         }
         return benchmarkResults

performance/startup/build.gradle.kts

@@ -1,5 +1,6 @@
 import org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType
 import org.jetbrains.kotlin.RunKotlinNativeTask
+import org.jetbrains.kotlin.BenchmarkRepeatingType
 
 /*
  * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
@@ -15,12 +16,12 @@ val defaultBuildType = NativeBuildType.RELEASE
 benchmark {
     applicationName = "Startup"
     commonSrcDirs = listOf("../../tools/benchmarks/shared/src", "src/main/kotlin", "../shared/src/main/kotlin")
-    jvmSrcDirs = listOf("src/main/kotlin-jvm", "../shared/src/main/kotlin-jvm")
-    nativeSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/common")
-    mingwSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/mingw")
-    posixSrcDirs = listOf("src/main/kotlin-native", "../shared/src/main/kotlin-native/posix")
+    jvmSrcDirs = listOf("../shared/src/main/kotlin-jvm")
+    nativeSrcDirs = listOf("../shared/src/main/kotlin-native/common")
+    mingwSrcDirs = listOf("../shared/src/main/kotlin-native/mingw")
+    posixSrcDirs = listOf("../shared/src/main/kotlin-native/posix")
     buildType = (findProperty("nativeBuildType") as String?)?.let { NativeBuildType.valueOf(it) } ?: defaultBuildType
-    repeatingType = RunKotlinNativeTask.RepeatingType.EXTERNAL
+    repeatingType = BenchmarkRepeatingType.EXTERNAL
 
     dependencies.common(project(":endorsedLibraries:kotlinx.cli"))
 }

performance/startup/src/main/kotlin-jvm/org/jetbrains/startup/Utils.kt

@@ -1,16 +0,0 @@
-/*
- * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
- * that can be found in the LICENSE file.
- */
-
-package org.jetbrains.startup
-
-actual class Random actual constructor() {
-    actual companion object {
-        actual var seedInt = 0
-        actual fun nextInt(boundary: Int): Int {
-            seedInt = (3 * seedInt + 11) % boundary
-            return seedInt
-        }
-    }
-}

performance/startup/src/main/kotlin-native/org/jetbrains/startup/Utils.kt

@@ -1,17 +0,0 @@
-/*
- * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
- * that can be found in the LICENSE file.
- */
-
-package org.jetbrains.startup
-
-actual class Random actual constructor() {
-    @kotlin.native.ThreadLocal
-    actual companion object {
-        actual var seedInt = 0
-        actual fun nextInt(boundary: Int): Int {
-            seedInt = (3 * seedInt + 11) % boundary
-            return seedInt
-        }
-    }
-}

performance/startup/src/main/kotlin/org/jetbrains/startup/Utils.kt

@@ -1,13 +0,0 @@
-/*
- * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
- * that can be found in the LICENSE file.
- */
-
-package org.jetbrains.startup
-
-expect class Random() {
-    companion object {
-        var seedInt: Int
-        fun nextInt(boundary: Int = 100): Int
-    }
-}