Made benchmark application flexible to run separate benchmarks (#2649)

INTEROP.md

@@ -257,7 +257,7 @@ All the supported C types have corresponding representations in Kotlin:
 *   Pointers and arrays are mapped to `CPointer<T>?`.
 *   Enums can be mapped to either Kotlin enum or integral values, depending on
     heuristics and the [definition file hints](#definition-file-hints).
-*   Structs are mapped to types having fields available via the dot notation,
+*   Structs / unions are mapped to types having fields available via the dot notation,
     i.e. `someStructInstance.field1`.
 *   `typedef` are represented as `typealias`.
 
@@ -527,7 +527,7 @@ it belongs to. Once the control flow leaves the `memScoped` scope the C pointers
 
 ### Passing and receiving structs by value ###
 
-When a C function takes or returns a struct `T` by value, the corresponding
+When a C function takes or returns a struct / union `T` by value, the corresponding
 argument type or return type is represented as `CValue<T>`.
 
 `CValue<T>` is an opaque type, so the structure fields cannot be accessed with

Interop/Runtime/src/main/kotlin/kotlinx/cinterop/Utils.kt

@@ -327,6 +327,9 @@ public inline fun <reified T : CVariable> createValues(count: Int, initializer:
 }
 
 // TODO: optimize other [cValuesOf] methods:
+/**
+ * Returns sequence of immutable values [CValues] to pass them to C code.
+ */
 fun cValuesOf(vararg elements: Byte): CValues<ByteVar> = object : CValues<ByteVar>() {
     // Optimization to avoid unneeded virtual calls in base class implementation.
     override fun getPointer(scope: AutofreeScope): CPointer<ByteVar> {

performance/build.gradle

@@ -17,141 +17,47 @@ buildscript {
     }
 }
 
-apply plugin: 'kotlin-multiplatform'
+task konanRun {
-
+    subprojects.each {
-repositories {
+       dependsOn it.getTasksByName('konanRun', true)[0]
-    maven {
-        url 'https://cache-redirector.jetbrains.com/jcenter'
-    }
-    maven {
-        url kotlinCompilerRepo
-    }
-    maven {
-        url buildKotlinCompilerRepo
-    }
-
-}
-
-defaultTasks 'bench'
-
-private def determinePreset() {
-    def preset = MPPTools.defaultHostPreset(project)
-    println("$project has been configured for ${preset.name} platform.")
-    preset
-}
-
-def hostPreset = determinePreset()
-def applicationName = 'Ring'
-
-kotlin {
-    sourceSets {
-        commonMain {
-            dependencies {
-                implementation "org.jetbrains.kotlin:kotlin-stdlib-common:$kotlinVersion"
-            }
-            kotlin.srcDir '../tools/benchmarks/shared/src'
-            kotlin.srcDir 'src/main/kotlin'
-
-        }
-        nativeMain {
-            kotlin.srcDir 'src/main/kotlin-native'
-        }
-        jvmMain {
-            dependencies {
-                implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion"
-            }
-            kotlin.srcDir 'src/main/kotlin-jvm'
-        }
-    }
-
-    targets {
-        fromPreset(presets.jvm, 'jvm') {
-            compilations.all {
-                tasks[compileKotlinTaskName].kotlinOptions {
-                    jvmTarget = '1.8'
-                }
-                tasks[compileKotlinTaskName].kotlinOptions.suppressWarnings = true
-            }
-        }
-
-        fromPreset(hostPreset, 'native') {
-            compilations.main.outputKinds('EXECUTABLE')
-            compilations.main.extraOpts '-opt'
-            compilations.main.buildTypes = [RELEASE]
-        }
     }
 }
 
-MPPTools.addTimeListener(project)
+task jvmRun {
-
+    subprojects.each {
-MPPTools.createRunTask(project, 'konanRun', kotlin.targets.native) {
+        dependsOn it.getTasksByName('jvmRun', true)[0]
-    workingDir = project.provider {
-        kotlin.targets.native.compilations.main.getBinary('EXECUTABLE', buildType).parentFile
-    }
-    depends("build")
-    args("$nativeWarmup", "$attempts", "${buildDir.absolutePath}/${nativeBenchResults}")
-}
-
-task jvmRun(type: JavaExec)  {
-    dependsOn 'build'
-    def output = new ByteArrayOutputStream()
-    def runtimeClasspath = files(
-            kotlin.targets.jvm.compilations.main.output.allOutputs,
-            project.configurations.getByName(kotlin.targets.jvm.compilations.main.runtimeDependencyConfigurationName)
-    )
-    classpath runtimeClasspath
-    main = "MainKt"
-    args "$jvmWarmup", "$attempts",  "${buildDir.absolutePath}/${jvmBenchResults}"
-}
-
-task konanJsonReport {
-    doLast {
-        def nativeExecutable = MPPTools.getKotlinNativeExecutable(kotlin.targets.native, "RELEASE")
-        def nativeCompileTime = MPPTools.getNativeCompileTime(applicationName)
-        String benchContents = new File("${buildDir.absolutePath}/${nativeBenchResults}").text
-        def properties = getCommonProperties() + ['type'           : 'native',
-                                                  'compilerVersion': "${konanVersion}".toString(),
-                                                  'flags'          : kotlin.targets.native.compilations.main.extraOpts.collect{ "\"$it\"" },
-                                                  'benchmarks'     : benchContents,
-                                                  'compileTime'    : nativeCompileTime,
-                                                  'codeSize'       : MPPTools.getCodeSizeBenchmark(applicationName, nativeExecutable) ]
-        def output = MPPTools.createJsonReport(properties)
-        new File("${buildDir.absolutePath}/${nativeJson}").write(output)
-        uploadBenchmarkResultToBintray(nativeJson)
     }
 }
 
-task jvmJsonReport {
+task clean {
-    doLast {
+    subprojects.each {
-        def jarPath = project.getTasks().getByName("jvmJar").archivePath
+        dependsOn it.getTasksByName('clean', true)[0]
-        def jvmCompileTime = MPPTools.getJvmCompileTime(applicationName)
-        String benchContents = new File("${buildDir.absolutePath}/${jvmBenchResults}").text
-        def properties = getCommonProperties() + ['type'           : 'jvm',
-                                                  'compilerVersion': "${buildKotlinVersion}".toString(),
-                                                  'benchmarks'     : benchContents,
-                                                  'compileTime'    : jvmCompileTime,
-                                                  'codeSize'       : MPPTools.getCodeSizeBenchmark(applicationName, "${jarPath}") ]
-        def output = MPPTools.createJsonReport(properties)
-        new File("${buildDir.absolutePath}/${jvmJson}").write(output)
-        uploadBenchmarkResultToBintray(jvmJson)
     }
+    delete "${buildDir.absolutePath}"
 }
 
-jvmRun.finalizedBy jvmJsonReport
+defaultTasks 'konanRun'
-konanRun.finalizedBy konanJsonReport
 
-private void dumpReport(String name, ByteArrayOutputStream output) {
+// Produce and send slack report.
-    new File("${name}").withOutputStream {
+task slackReport(type: RegressionsReporter) {
-        it.write(output.toByteArray())
+    // Create folder for report (root Kotlin project settings make create report in separate folder).
-    }
+    def reportDirectory = new File(outputReport).parentFile
+    mkdir reportDirectory
+    def targetsResults = new File(new File("${rootBuildDirectory}"), "targetsResults").toString()
+    mkdir targetsResults
+    currentBenchmarksReportFile = "${buildDir.absolutePath}/${nativeJson}"
+    analyzer = MPPTools.findFile("${analyzerTool}${MPPTools.getNativeProgramExtension()}",
+            "${rootBuildDirectory}/${analyzerToolDirectory}")
+    htmlReport = outputReport
+    defaultBranch = project.findProperty('kotlin.native.default.branch') ?: "master"
+    def target = System.getProperty("os.name")
+    summaryFile = "${targetsResults}/${target}.txt"
 }
 
-private def getCommonProperties() {
+task slackSummary(type: RegressionsSummaryReporter) {
-    return ['cpu': System.getProperty("os.arch"),
+    targetsResultFiles = ["Linux": "${rootBuildDirectory}/targetsResults/Linux.txt",
-            'os': System.getProperty("os.name"), // OperatingSystem.current().getName()
+                          "MacOSX": "${rootBuildDirectory}/targetsResults/Mac OS X.txt",
-            'jdkVersion': System.getProperty("java.version"), // org.gradle.internal.jvm.Jvm.current().javaVersion
+                          "Windows": "${rootBuildDirectory}/targetsResults/Windows 10.txt"]
-            'jdkVendor': System.getProperty("java.vendor"),
-            'kotlinVersion': "${kotlinVersion}".toString()]
 }
 
 private def uploadBenchmarkResultToBintray(String fileName) {
@@ -169,35 +75,45 @@ private def uploadBenchmarkResultToBintray(String fileName) {
     }
 }
 
-task bench(type:Exec) {
+def mergeReports(String fileName) {
-    dependsOn jvmRun
+    def reports = []
-    dependsOn konanRun
+    subprojects.each {
-    def extension = MPPTools.getNativeProgramExtension()
+        def reportFile = new File("${it.buildDir.absolutePath}/${fileName}")
-    def analyzer = MPPTools.findFile("${analyzerTool}${extension}", "${rootBuildDirectory}/${analyzerToolDirectory}")
+        if (reportFile.exists()) {
-    if (analyzer != null) {
+            reports.add(reportFile)
-        commandLine "${analyzer}", "-r", "text", "-r", "teamcity", "${buildDir.absolutePath}/${nativeJson}", "${buildDir.absolutePath}/${jvmJson}"
+        }
-    } else {
+    depends("build")
-        println("No analyzer $analyzerTool found in subdirectories of ${rootBuildDirectory}/${analyzerToolDirectory}")
+    }
+    def output = MPPTools.mergeReports(reports)
+    mkdir buildDir.absolutePath
+    new File("${buildDir.absolutePath}/${fileName}").write(output)
+}
+
+task mergeNativeReports {
+    doLast {
+        mergeReports(nativeJson)
+        uploadBenchmarkResultToBintray(nativeJson)
     }
 }
 
-// Produce and send slack report.
+task mergeJvmReports {
-task slackReport(type: RegressionsReporter) {
+    doLast {
-    // Create folder for report (root Kotlin project settings make create report in separate folder).
+        mergeReports(jvmJson)
-    def reportDirectory = outputReport.substring(0, outputReport.lastIndexOf("/"))
+        uploadBenchmarkResultToBintray(jvmJson)
-    mkdir reportDirectory
+    }
-    mkdir "../targetsResults"
-    currentBenchmarksReportFile = "${buildDir.absolutePath}/${nativeJson}"
-    analyzer = MPPTools.findFile("${analyzerTool}${MPPTools.getNativeProgramExtension()}",
-            "${rootBuildDirectory}/${analyzerToolDirectory}")
-    htmlReport = outputReport
-    defaultBranch = project.findProperty('kotlin.native.default.branch') ?: "master"
-    def target = System.getProperty("os.name")
-    summaryFile = "../targetsResults/${target}.txt"
 }
 
-task slackSummary(type: RegressionsSummaryReporter) {
+subprojects.each {
-    targetsResultFiles = ["Linux": "../targetsResults/Linux.txt",
+    it.getTasksByName('jvmJsonReport', true)[0].finalizedBy mergeJvmReports
-                          "MacOSX": "../targetsResults/Mac OS X.txt",
+    it.getTasksByName('konanJsonReport', true)[0].finalizedBy mergeNativeReports
-                          "Windows": "../targetsResults/Windows 10.txt"]
+}
+
+task teamCityStat(type:Exec) {
+    def extension = MPPTools.getNativeProgramExtension()
+    def analyzer = MPPTools.findFile("${analyzerTool}${extension}", "${rootBuildDirectory}/${analyzerToolDirectory}")
+    if (analyzer != null) {
+        commandLine "${analyzer}", "-r", "teamcity", "${buildDir.absolutePath}/${nativeJson}"
+    } else {
+        println("No analyzer $analyzerTool found in subdirectories of ${rootBuildDirectory}/${analyzerToolDirectory}")
+    }
 }

performance/buildSrc/src/main/kotlin/MPPTools.kt

@@ -110,6 +110,16 @@ fun createJsonReport(projectProperties: Map<String, Any>): String {
     return report.toJson()
 }
 
+fun mergeReports(reports: List<File>): String {
+    val reportsToMerge = reports.map {
+        val json = it.inputStream().bufferedReader().use { it.readText() }
+        val reportElement = JsonTreeParser.parse(json)
+        BenchmarksReport.create(reportElement)
+
+    }
+    return reportsToMerge.reduce { result, it -> result + it }.toJson()
+}
+
 // Find file with set name in directory.
 fun findFile(fileName: String, directory: String): String? =
     File(directory).walkBottomUp().find { it.name == fileName }?.getAbsolutePath()

performance/buildSrc/src/main/kotlin/RunJvmTask.kt

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+import groovy.lang.Closure
+import org.gradle.api.tasks.JavaExec
+import org.gradle.api.Task
+import org.gradle.api.tasks.TaskAction
+import org.gradle.api.tasks.options.Option
+import org.gradle.api.tasks.Input
+import org.jetbrains.kotlin.gradle.plugin.KotlinTarget
+import javax.inject.Inject
+import java.io.File
+
+open class RunJvmTask: JavaExec() {
+    var outputFileName: String? = null
+    @Input
+    @Option(option = "filter", description = "filter")
+    var filter: String = ""
+
+    override fun configure(configureClosure: Closure<Any>): Task {
+        return super.configure(configureClosure)
+    }
+
+    private fun executeTask(output: java.io.OutputStream? = null) {
+        val filterArgs = filter.split("\\s*,\\s*".toRegex())
+                .map{ if (it.isNotEmpty()) listOf("-f", it) else listOf(null) }.flatten().filterNotNull()
+        args(filterArgs)
+        exec()
+    }
+
+    @TaskAction
+    fun run() {
+        if (outputFileName != null)
+            File(outputFileName).outputStream().use { output ->  executeTask(output)}
+        else
+            executeTask()
+    }
+}

performance/buildSrc/src/main/kotlin/RunKotlinNativeTask.kt

@@ -7,6 +7,8 @@ import groovy.lang.Closure
 import org.gradle.api.DefaultTask
 import org.gradle.api.Task
 import org.gradle.api.tasks.TaskAction
+import org.gradle.api.tasks.options.Option
+import org.gradle.api.tasks.Input
 import org.jetbrains.kotlin.gradle.plugin.KotlinTarget
 import javax.inject.Inject
 import java.io.File
@@ -18,6 +20,10 @@ open class RunKotlinNativeTask @Inject constructor(
     var buildType = "RELEASE"
     var workingDir: Any = project.projectDir
     var outputFileName: String? = null
+    @Input
+    @Option(option = "filter", description = "filter")
+    var filter: String = ""
+
     private var curArgs: List<String> = emptyList()
     private val curEnvironment: MutableMap<String, Any> = mutableMapOf()
 
@@ -40,9 +46,11 @@ open class RunKotlinNativeTask @Inject constructor(
     }
 
     private fun executeTask(output: java.io.OutputStream? = null) {
+        val filterArgs = filter.split("\\s*,\\s*".toRegex())
+                .map{ if (it.isNotEmpty()) listOf("-f", it) else listOf(null) }.flatten().filterNotNull()
         project.exec {
             it.executable = curTarget.compilations.main.getBinary("EXECUTABLE", buildType).toString()
-            it.args = curArgs
+            it.args = curArgs + filterArgs
             it.environment = curEnvironment
             it.workingDir(workingDir)
             if (output != null)

performance/gradle/benchmark.gradle

@@ -0,0 +1,131 @@
+apply plugin: 'kotlin-multiplatform'
+
+repositories {
+    maven {
+        url 'https://cache-redirector.jetbrains.com/jcenter'
+    }
+    maven {
+        url kotlinCompilerRepo
+    }
+    maven {
+        url buildKotlinCompilerRepo
+    }
+
+}
+
+
+private def determinePreset() {
+    def preset = MPPTools.defaultHostPreset(project)
+    println("$project has been configured for ${preset.name} platform.")
+    preset
+}
+
+def hostPreset = determinePreset()
+
+kotlin {
+    sourceSets {
+        commonMain {
+            dependencies {
+                implementation "org.jetbrains.kotlin:kotlin-stdlib-common:$kotlinVersion"
+            }
+            project.ext.commonSrcDirs.forEach {
+                kotlin.srcDir it
+            }
+
+        }
+        nativeMain {
+            project.ext.nativeSrcDirs.forEach {
+                kotlin.srcDir it
+            }
+        }
+        jvmMain {
+            dependencies {
+                implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion"
+            }
+            project.ext.jvmSrcDirs.forEach {
+                kotlin.srcDir it
+            }
+        }
+    }
+
+    targets {
+        fromPreset(presets.jvm, 'jvm') {
+            compilations.all {
+                tasks[compileKotlinTaskName].kotlinOptions {
+                    jvmTarget = '1.8'
+                }
+                tasks[compileKotlinTaskName].kotlinOptions.suppressWarnings = true
+            }
+        }
+
+        fromPreset(hostPreset, 'native') {
+            compilations.main.outputKinds('EXECUTABLE')
+            compilations.main.extraOpts '-opt'
+            compilations.main.buildTypes = [RELEASE]
+        }
+    }
+}
+
+MPPTools.addTimeListener(project)
+
+MPPTools.createRunTask(project, 'konanRun', kotlin.targets.native) {
+    workingDir = project.provider {
+        kotlin.targets.native.compilations.main.getBinary('EXECUTABLE', buildType).parentFile
+    }
+    depends("build")
+    args("-w", "$nativeWarmup", "-r", "$attempts", "-o", "${buildDir.absolutePath}/${nativeBenchResults}", "-p", "${project.ext.applicationName}::")
+}
+
+task jvmRun(type: RunJvmTask)  {
+    dependsOn 'build'
+    def runtimeClasspath = files(
+            kotlin.targets.jvm.compilations.main.output.allOutputs,
+            project.configurations.getByName(kotlin.targets.jvm.compilations.main.runtimeDependencyConfigurationName)
+    )
+    classpath runtimeClasspath
+    main = "MainKt"
+    args "-w", "$jvmWarmup", "-r", "$attempts", "-o", "${buildDir.absolutePath}/${jvmBenchResults}", "-p", "${project.ext.applicationName}::"
+}
+
+task konanJsonReport {
+    doLast {
+        def nativeExecutable = MPPTools.getKotlinNativeExecutable(kotlin.targets.native, "RELEASE")
+        def nativeCompileTime = MPPTools.getNativeCompileTime(project.ext.applicationName)
+        String benchContents = new File("${buildDir.absolutePath}/${nativeBenchResults}").text
+        def properties = getCommonProperties() + ['type': 'native',
+                                                  'compilerVersion': "${konanVersion}".toString(),
+                                                  'flags': kotlin.targets.native.compilations.main.extraOpts.collect{ "\"$it\"" },
+                                                  'benchmarks': benchContents,
+                                                  'compileTime': nativeCompileTime,
+                                                  'codeSize': MPPTools.getCodeSizeBenchmark(project.ext.applicationName, nativeExecutable) ]
+        def output = MPPTools.createJsonReport(properties)
+        new File("${buildDir.absolutePath}/${nativeJson}").write(output)
+    }
+}
+
+task jvmJsonReport {
+    doLast {
+        def jarPath = project.getTasks().getByName("jvmJar").archivePath
+        def jvmCompileTime = MPPTools.getJvmCompileTime(project.ext.applicationName)
+        String benchContents = new File("${buildDir.absolutePath}/${jvmBenchResults}").text
+        def properties = getCommonProperties() + ['type': 'jvm',
+                                                  'compilerVersion': "${buildKotlinVersion}".toString(),
+                                                  'benchmarks': benchContents,
+                                                  'compileTime': jvmCompileTime,
+                                                  'codeSize': MPPTools.getCodeSizeBenchmark(project.ext.applicationName, "${jarPath}") ]
+        def output = MPPTools.createJsonReport(properties)
+        new File("${buildDir.absolutePath}/${jvmJson}").write(output)
+    }
+}
+
+jvmRun.finalizedBy jvmJsonReport
+konanRun.finalizedBy konanJsonReport
+
+private def getCommonProperties() {
+    return ['cpu': System.getProperty("os.arch"),
+            'os': System.getProperty("os.name"), // OperatingSystem.current().getName()
+            'jdkVersion': System.getProperty("java.version"), // org.gradle.internal.jvm.Jvm.current().javaVersion
+            'jdkVendor': System.getProperty("java.vendor"),
+            'kotlinVersion': "${kotlinVersion}".toString()]
+}
+

performance/ring/build.gradle

@@ -0,0 +1,7 @@
+project.ext {
+    applicationName = 'Ring'
+    commonSrcDirs = ['../../tools/benchmarks/shared/src', 'src/main/kotlin', '../shared/src/main/kotlin', '../../tools/kliopt']
+    jvmSrcDirs = ['src/main/kotlin-jvm', '../shared/src/main/kotlin-jvm']
+    nativeSrcDirs = ['src/main/kotlin-native', '../shared/src/main/kotlin-native']
+}
+apply from: rootProject.file('gradle/benchmark.gradle')

performance/ring/gradle.properties

@@ -0,0 +1 @@
+org.jetbrains.kotlin.native.home=../../dist

performance/ring/src/main/kotlin-jvm/org/jetbrains/ring/Utils.kt

@@ -45,23 +45,4 @@ actual class Random actual constructor() {
             return seedDouble
         }
     }
-}
+}
-
-//-----------------------------------------------------------------------------//
-
-actual fun writeToFile(fileName: String, text: String) {
-    File(fileName).printWriter().use { out ->
-        out.println(text)
-    }
-}
-
-// Wrapper for assert funtion in stdlib
-actual fun assert(value: Boolean) {
-    kotlin.assert(value)
-}
-
-// Wrapper for measureNanoTime funtion in stdlib
-actual inline fun measureNanoTime(block: () -> Unit): Long {
-    return kotlin.system.measureNanoTime(block)
-}
-

performance/ring/src/main/kotlin-native/org/jetbrains/ring/Utils.kt

@@ -47,26 +47,4 @@ actual class Random actual constructor() {
             return seedDouble
         }
     }
-}
+}
-
-//-----------------------------------------------------------------------------//
-
-
-actual fun writeToFile(fileName: String, text: String) {
-    val file = fopen(fileName, "wt") ?: error("Cannot write file '$fileName'")
-    try {
-        if (fputs(text, file) == EOF) throw Error("File write error")
-    } finally {
-        fclose(file)
-    }
-}
-
-// Wrapper for assert funtion in stdlib
-actual fun assert(value: Boolean) {
-    kotlin.assert(value)
-}
-
-// Wrapper for measureNanoTime funtion in stdlib
-actual inline fun measureNanoTime(block: () -> Unit): Long {
-    return kotlin.system.measureNanoTime(block)
-}

performance/ring/src/main/kotlin/main.kt

@@ -0,0 +1,227 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import org.jetbrains.ring.*
+import octoTest
+import org.jetbrains.benchmarksLauncher.*
+import org.jetbrains.kliopt.*
+
+class RingLauncher(numWarmIterations: Int, numberOfAttempts: Int, prefix: String): Launcher(numWarmIterations, numberOfAttempts, prefix) {
+    val abstractMethodBenchmark = AbstractMethodBenchmark()
+    val classArrayBenchmark = ClassArrayBenchmark()
+    val classBaselineBenchmark = ClassBaselineBenchmark()
+    val classListBenchmark = ClassListBenchmark()
+    val classStreamBenchmark = ClassStreamBenchmark()
+    val companionObjectBenchmark = CompanionObjectBenchmark()
+    val defaultArgumentBenchmark = DefaultArgumentBenchmark()
+    val elvisBenchmark = ElvisBenchmark()
+    val eulerBenchmark = EulerBenchmark()
+    val fibonacciBenchmark = FibonacciBenchmark()
+    val forLoopsBenchmark = ForLoopsBenchmark()
+    val inlineBenchmark = InlineBenchmark()
+    val intArrayBenchmark = IntArrayBenchmark()
+    val intBaselineBenchmark = IntBaselineBenchmark()
+    val intListBenchmark = IntListBenchmark()
+    val intStreamBenchmark = IntStreamBenchmark()
+    val lambdaBenchmark = LambdaBenchmark()
+    val loopBenchmark = LoopBenchmark()
+    val matrixMapBenchmark = MatrixMapBenchmark()
+    val parameterNotNullAssertionBenchmark = ParameterNotNullAssertionBenchmark()
+    val primeListBenchmark = PrimeListBenchmark()
+    val stringBenchmark = StringBenchmark()
+    val switchBenchmark = SwitchBenchmark()
+    val withIndiciesBenchmark = WithIndiciesBenchmark()
+
+    override val benchmarks = BenchmarksCollection(
+            mutableMapOf(
+                    "AbstractMethod.sortStrings" to abstractMethodBenchmark::sortStrings,
+                    "AbstractMethod.sortStringsWithComparator" to abstractMethodBenchmark::sortStringsWithComparator,
+                    "ClassArray.copy" to classArrayBenchmark::copy,
+                    "ClassArray.copyManual" to classArrayBenchmark::copyManual,
+                    "ClassArray.filterAndCount" to classArrayBenchmark::filterAndCount,
+                    "ClassArray.filterAndMap" to classArrayBenchmark::filterAndMap,
+                    "ClassArray.filterAndMapManual" to classArrayBenchmark::filterAndMapManual,
+                    "ClassArray.filter" to classArrayBenchmark::filter,
+                    "ClassArray.filterManual" to classArrayBenchmark::filterManual,
+                    "ClassArray.countFilteredManual" to classArrayBenchmark::countFilteredManual,
+                    "ClassArray.countFiltered" to classArrayBenchmark::countFiltered,
+                    "ClassArray.countFilteredLocal" to classArrayBenchmark::countFilteredLocal,
+                    "ClassBaseline.consume" to classBaselineBenchmark::consume,
+                    "ClassBaseline.consumeField" to classBaselineBenchmark::consumeField,
+                    "ClassBaseline.allocateList" to classBaselineBenchmark::allocateList,
+                    "ClassBaseline.allocateArray" to classBaselineBenchmark::allocateArray,
+                    "ClassBaseline.allocateListAndFill" to classBaselineBenchmark::allocateListAndFill,
+                    "ClassBaseline.allocateListAndWrite" to classBaselineBenchmark::allocateListAndWrite,
+                    "ClassBaseline.allocateArrayAndFill" to classBaselineBenchmark::allocateArrayAndFill,
+                    "ClassList.copy" to classListBenchmark::copy,
+                    "ClassList.copyManual" to classListBenchmark::copyManual,
+                    "ClassList.filterAndCount" to classListBenchmark::filterAndCount,
+                    "ClassList.filterAndCountWithLambda" to classListBenchmark::filterAndCountWithLambda,
+                    "ClassList.filterWithLambda" to classListBenchmark::filterWithLambda,
+                    "ClassList.mapWithLambda" to classListBenchmark::mapWithLambda,
+                    "ClassList.countWithLambda" to classListBenchmark::countWithLambda,
+                    "ClassList.filterAndMapWithLambda" to classListBenchmark::filterAndMapWithLambda,
+                    "ClassList.filterAndMapWithLambdaAsSequence" to classListBenchmark::filterAndMapWithLambdaAsSequence,
+                    "ClassList.filterAndMap" to classListBenchmark::filterAndMap,
+                    "ClassList.filterAndMapManual" to classListBenchmark::filterAndMapManual,
+                    "ClassList.filter" to classListBenchmark::filter,
+                    "ClassList.filterManual" to classListBenchmark::filterManual,
+                    "ClassList.countFilteredManual" to classListBenchmark::countFilteredManual,
+                    "ClassList.countFiltered" to classListBenchmark::countFiltered,
+                    "ClassList.reduce" to classListBenchmark::reduce,
+                    "ClassStream.copy" to classStreamBenchmark::copy,
+                    "ClassStream.copyManual" to classStreamBenchmark::copyManual,
+                    "ClassStream.filterAndCount" to classStreamBenchmark::filterAndCount,
+                    "ClassStream.filterAndMap" to classStreamBenchmark::filterAndMap,
+                    "ClassStream.filterAndMapManual" to classStreamBenchmark::filterAndMapManual,
+                    "ClassStream.filter" to classStreamBenchmark::filter,
+                    "ClassStream.filterManual" to classStreamBenchmark::filterManual,
+                    "ClassStream.countFilteredManual" to classStreamBenchmark::countFilteredManual,
+                    "ClassStream.countFiltered" to classStreamBenchmark::countFiltered,
+                    "ClassStream.reduce" to classStreamBenchmark::reduce,
+                    "CompanionObject.invokeRegularFunction" to companionObjectBenchmark::invokeRegularFunction,
+                    "CompanionObject.invokeJvmStaticFunction" to companionObjectBenchmark::invokeJvmStaticFunction,
+                    "DefaultArgument.testOneOfTwo" to defaultArgumentBenchmark::testOneOfTwo,
+                    "DefaultArgument.testTwoOfTwo" to defaultArgumentBenchmark::testTwoOfTwo,
+                    "DefaultArgument.testOneOfFour" to defaultArgumentBenchmark::testOneOfFour,
+                    "DefaultArgument.testFourOfFour" to defaultArgumentBenchmark::testFourOfFour,
+                    "DefaultArgument.testOneOfEight" to defaultArgumentBenchmark::testOneOfEight,
+                    "DefaultArgument.testEightOfEight" to defaultArgumentBenchmark::testEightOfEight,
+                    "Elvis.testElvis" to elvisBenchmark::testElvis,
+                    "Euler.problem1bySequence" to eulerBenchmark::problem1bySequence,
+                    "Euler.problem1" to eulerBenchmark::problem1,
+                    "Euler.problem2" to eulerBenchmark::problem2,
+                    "Euler.problem4" to eulerBenchmark::problem4,
+                    "Euler.problem8" to eulerBenchmark::problem8,
+                    "Euler.problem9" to eulerBenchmark::problem9,
+                    "Euler.problem14" to eulerBenchmark::problem14,
+                    "Euler.problem14full" to eulerBenchmark::problem14full,
+                    "Fibonacci.calcClassic" to fibonacciBenchmark::calcClassic,
+                    "Fibonacci.calc" to fibonacciBenchmark::calc,
+                    "Fibonacci.calcWithProgression" to fibonacciBenchmark::calcWithProgression,
+                    "Fibonacci.calcSquare" to fibonacciBenchmark::calcSquare,
+                    "ForLoops.arrayLoop" to forLoopsBenchmark::arrayLoop,
+                    "ForLoops.intArrayLoop" to forLoopsBenchmark::intArrayLoop,
+                    "ForLoops.floatArrayLoop" to forLoopsBenchmark::floatArrayLoop,
+                    "ForLoops.charArrayLoop" to forLoopsBenchmark::charArrayLoop,
+                    "ForLoops.stringLoop" to forLoopsBenchmark::stringLoop,
+                    "ForLoops.arrayIndicesLoop" to forLoopsBenchmark::arrayIndicesLoop,
+                    "ForLoops.intArrayIndicesLoop" to forLoopsBenchmark::intArrayIndicesLoop,
+                    "ForLoops.floatArrayIndicesLoop" to forLoopsBenchmark::floatArrayIndicesLoop,
+                    "ForLoops.charArrayIndicesLoop" to forLoopsBenchmark::charArrayIndicesLoop,
+                    "ForLoops.stringIndicesLoop" to forLoopsBenchmark::stringIndicesLoop,
+                    "Inline.calculate" to inlineBenchmark::calculate,
+                    "Inline.calculateInline" to inlineBenchmark::calculateInline,
+                    "Inline.calculateGeneric" to inlineBenchmark::calculateGeneric,
+                    "Inline.calculateGenericInline" to inlineBenchmark::calculateGenericInline,
+                    "IntArray.copy" to intArrayBenchmark::copy,
+                    "IntArray.copyManual" to intArrayBenchmark::copyManual,
+                    "IntArray.filterAndCount" to intArrayBenchmark::filterAndCount,
+                    "IntArray.filterSomeAndCount" to intArrayBenchmark::filterSomeAndCount,
+                    "IntArray.filterAndMap" to intArrayBenchmark::filterAndMap,
+                    "IntArray.filterAndMapManual" to intArrayBenchmark::filterAndMapManual,
+                    "IntArray.filter" to intArrayBenchmark::filter,
+                    "IntArray.filterSome" to intArrayBenchmark::filterSome,
+                    "IntArray.filterPrime" to intArrayBenchmark::filterPrime,
+                    "IntArray.filterManual" to intArrayBenchmark::filterManual,
+                    "IntArray.filterSomeManual" to intArrayBenchmark::filterSomeManual,
+                    "IntArray.countFilteredManual" to intArrayBenchmark::countFilteredManual,
+                    "IntArray.countFilteredSomeManual" to intArrayBenchmark::countFilteredSomeManual,
+                    "IntArray.countFilteredPrimeManual" to intArrayBenchmark::countFilteredPrimeManual,
+                    "IntArray.countFiltered" to intArrayBenchmark::countFiltered,
+                    "IntArray.countFilteredSome" to intArrayBenchmark::countFilteredSome,
+                    "IntArray.countFilteredPrime" to intArrayBenchmark::countFilteredPrime,
+                    "IntArray.countFilteredLocal" to intArrayBenchmark::countFilteredLocal,
+                    "IntArray.countFilteredSomeLocal" to intArrayBenchmark::countFilteredSomeLocal,
+                    "IntArray.reduce" to intArrayBenchmark::reduce,
+                    "IntBaseline.consume" to intBaselineBenchmark::consume,
+                    "IntBaseline.allocateList" to intBaselineBenchmark::allocateList,
+                    "IntBaseline.allocateArray" to intBaselineBenchmark::allocateArray,
+                    "IntBaseline.allocateListAndFill" to intBaselineBenchmark::allocateListAndFill,
+                    "IntBaseline.allocateArrayAndFill" to intBaselineBenchmark::allocateArrayAndFill,
+                    "IntList.copy" to intListBenchmark::copy,
+                    "IntList.copyManual" to intListBenchmark::copyManual,
+                    "IntList.filterAndCount" to intListBenchmark::filterAndCount,
+                    "IntList.filterAndMap" to intListBenchmark::filterAndMap,
+                    "IntList.filterAndMapManual" to intListBenchmark::filterAndMapManual,
+                    "IntList.filter" to intListBenchmark::filter,
+                    "IntList.filterManual" to intListBenchmark::filterManual,
+                    "IntList.countFilteredManual" to intListBenchmark::countFilteredManual,
+                    "IntList.countFiltered" to intListBenchmark::countFiltered,
+                    "IntList.countFilteredLocal" to intListBenchmark::countFilteredLocal,
+                    "IntList.reduce" to intListBenchmark::reduce,
+                    "IntStream.copy" to intStreamBenchmark::copy,
+                    "IntStream.copyManual" to intStreamBenchmark::copyManual,
+                    "IntStream.filterAndCount" to intStreamBenchmark::filterAndCount,
+                    "IntStream.filterAndMap" to intStreamBenchmark::filterAndMap,
+                    "IntStream.filterAndMapManual" to intStreamBenchmark::filterAndMapManual,
+                    "IntStream.filter" to intStreamBenchmark::filter,
+                    "IntStream.filterManual" to intStreamBenchmark::filterManual,
+                    "IntStream.countFilteredManual" to intStreamBenchmark::countFilteredManual,
+                    "IntStream.countFiltered" to intStreamBenchmark::countFiltered,
+                    "IntStream.countFilteredLocal" to intStreamBenchmark::countFilteredLocal,
+                    "IntStream.reduce" to intStreamBenchmark::reduce,
+                    "Lambda.noncapturingLambda" to lambdaBenchmark::noncapturingLambda,
+                    "Lambda.noncapturingLambdaNoInline" to lambdaBenchmark::noncapturingLambdaNoInline,
+                    "Lambda.capturingLambda" to lambdaBenchmark::capturingLambda,
+                    "Lambda.capturingLambdaNoInline" to lambdaBenchmark::capturingLambdaNoInline,
+                    "Lambda.mutatingLambda" to lambdaBenchmark::mutatingLambda,
+                    "Lambda.mutatingLambdaNoInline" to lambdaBenchmark::mutatingLambdaNoInline,
+                    "Lambda.methodReference" to lambdaBenchmark::methodReference,
+                    "Lambda.methodReferenceNoInline" to lambdaBenchmark::methodReferenceNoInline,
+                    "Loop.arrayLoop" to loopBenchmark::arrayLoop,
+                    "Loop.arrayIndexLoop" to loopBenchmark::arrayIndexLoop,
+                    "Loop.rangeLoop" to loopBenchmark::rangeLoop,
+                    "Loop.arrayListLoop" to loopBenchmark::arrayListLoop,
+                    "Loop.arrayWhileLoop" to loopBenchmark::arrayWhileLoop,
+                    "Loop.arrayForeachLoop" to loopBenchmark::arrayForeachLoop,
+                    "Loop.arrayListForeachLoop" to loopBenchmark::arrayListForeachLoop,
+                    "MatrixMap.add" to matrixMapBenchmark::add,
+                    "ParameterNotNull.invokeOneArgWithNullCheck" to parameterNotNullAssertionBenchmark::invokeOneArgWithNullCheck,
+                    "ParameterNotNull.invokeOneArgWithoutNullCheck" to parameterNotNullAssertionBenchmark::invokeOneArgWithoutNullCheck,
+                    "ParameterNotNull.invokeTwoArgsWithNullCheck" to parameterNotNullAssertionBenchmark::invokeTwoArgsWithNullCheck,
+                    "ParameterNotNull.invokeTwoArgsWithoutNullCheck" to parameterNotNullAssertionBenchmark::invokeTwoArgsWithoutNullCheck,
+                    "ParameterNotNull.invokeEightArgsWithNullCheck" to parameterNotNullAssertionBenchmark::invokeEightArgsWithNullCheck,
+                    "ParameterNotNull.invokeEightArgsWithoutNullCheck" to parameterNotNullAssertionBenchmark::invokeEightArgsWithoutNullCheck,
+                    "PrimeList.calcDirect" to primeListBenchmark::calcDirect,
+                    "PrimeList.calcEratosthenes" to primeListBenchmark::calcEratosthenes,
+                    "String.stringConcat" to stringBenchmark::stringConcat,
+                    "String.stringConcatNullable" to stringBenchmark::stringConcatNullable,
+                    "String.stringBuilderConcat" to stringBenchmark::stringBuilderConcat,
+                    "String.stringBuilderConcatNullable" to stringBenchmark::stringBuilderConcatNullable,
+                    "String.summarizeSplittedCsv" to stringBenchmark::summarizeSplittedCsv,
+                    "Switch.testSparseIntSwitch" to switchBenchmark::testSparseIntSwitch,
+                    "Switch.testDenseIntSwitch" to switchBenchmark::testDenseIntSwitch,
+                    "Switch.testConstSwitch" to switchBenchmark::testConstSwitch,
+                    "Switch.testObjConstSwitch" to switchBenchmark::testObjConstSwitch,
+                    "Switch.testVarSwitch" to switchBenchmark::testVarSwitch,
+                    "Switch.testStringsSwitch" to switchBenchmark::testStringsSwitch,
+                    "Switch.testEnumsSwitch" to switchBenchmark::testEnumsSwitch,
+                    "Switch.testDenseEnumsSwitch" to switchBenchmark::testDenseEnumsSwitch,
+                    "Switch.testSealedWhenSwitch" to switchBenchmark::testSealedWhenSwitch,
+                    "WithIndicies.withIndicies" to withIndiciesBenchmark::withIndicies,
+                    "WithIndicies.withIndiciesManual" to withIndiciesBenchmark::withIndiciesManual,
+                    "OctoTest" to ::octoTest
+            )
+    )
+}
+
+fun main(args: Array<String>) {
+    BenchmarksRunner.runBenchmarks(args, { parser: ArgParser ->
+        RingLauncher(parser.get<Int>("warmup")!!, parser.get<Int>("repeat")!!, parser.get<String>("prefix")!!).launch(parser.getAll<String>("filter"))
+    })
+}

performance/ring/src/main/kotlin/org/jetbrains/ring/ClassArrayBenchmark.kt

@@ -21,7 +21,7 @@ open class ClassArrayBenchmark {
     val data: Array<Value>
         get() = _data!!
 
-    fun setup() {
+    init {
         val list = ArrayList<Value>(BENCHMARK_SIZE)
         for (n in classValues(BENCHMARK_SIZE))
             list.add(n)

performance/ring/src/main/kotlin/org/jetbrains/ring/ClassListBenchmark.kt

@@ -21,7 +21,7 @@ open class ClassListBenchmark {
     val data: ArrayList<Value>
         get() = _data!!
 
-    fun setup() {
+    init {
         val list = ArrayList<Value>(BENCHMARK_SIZE)
         for (n in classValues(BENCHMARK_SIZE))
             list.add(n)

performance/ring/src/main/kotlin/org/jetbrains/ring/ClassStreamBenchmark.kt

@@ -21,7 +21,7 @@ open class ClassStreamBenchmark {
     val data: Iterable<Value>
         get() = _data!!
 
-    fun setup() {
+    init {
         _data = classValues(BENCHMARK_SIZE)
     }
 

performance/ring/src/main/kotlin/org/jetbrains/ring/DefaultArgumentBenchmark.kt

@@ -24,7 +24,7 @@ package org.jetbrains.ring
 open class DefaultArgumentBenchmark {
     private var arg = 0
 
-    fun setup() {
+    init {
         arg = Random.nextInt()
     }
 

performance/ring/src/main/kotlin/org/jetbrains/ring/ElvisBenchmark.kt

@@ -22,7 +22,7 @@ open class ElvisBenchmark {
 
     var array : Array<Value?> = arrayOf()
 
-    fun setup() {
+    init {
         array = Array(BENCHMARK_SIZE) {
             if (Random.nextInt(BENCHMARK_SIZE) < BENCHMARK_SIZE / 10) null else Value(Random.nextInt())
         }

performance/ring/src/main/kotlin/org/jetbrains/ring/IntArrayBenchmark.kt

@@ -21,7 +21,7 @@ open class IntArrayBenchmark {
     val data: IntArray
         get() = _data!!
 
-    fun setup() {
+    init {
         val list = IntArray(BENCHMARK_SIZE)
         var index = 0
         for (n in intValues(BENCHMARK_SIZE))

performance/ring/src/main/kotlin/org/jetbrains/ring/IntListBenchmark.kt

@@ -21,7 +21,7 @@ open class IntListBenchmark {
     val data: List<Int>
         get() = _data!!
 
-    fun setup() {
+    init {
         val list = ArrayList<Int>(BENCHMARK_SIZE)
         for (n in intValues(BENCHMARK_SIZE))
             list.add(n)

performance/ring/src/main/kotlin/org/jetbrains/ring/IntStreamBenchmark.kt

@@ -21,7 +21,7 @@ open class IntStreamBenchmark {
     val data: Iterable<Int>
         get() = _data!!
 
-    fun setup() {
+    init {
         _data = intValues(BENCHMARK_SIZE)
     }
     

performance/ring/src/main/kotlin/org/jetbrains/ring/LambdaBenchmark.kt

@@ -22,7 +22,7 @@ open class LambdaBenchmark {
     private inline fun <T> runLambda(x: () -> T): T = x()
     private fun <T> runLambdaNoInline(x: () -> T): T = x()
 
-    fun setup() {
+    init {
         globalAddendum = Random.nextInt(20)
     }
 

performance/ring/src/main/kotlin/org/jetbrains/ring/LoopBenchmark.kt

@@ -20,7 +20,7 @@ open class LoopBenchmark {
     lateinit var arrayList: List<Value>
     lateinit var array: Array<Value>
 
-    fun setup() {
+    init {
         val list = ArrayList<Value>(BENCHMARK_SIZE)
         for (n in classValues(BENCHMARK_SIZE))
             list.add(n)

performance/ring/src/main/kotlin/org/jetbrains/ring/OctoTest/basicTest.kt

@@ -2,7 +2,7 @@
  * Created by semoro on 07.07.17.
  */
 
-import org.jetbrains.ring.assert
+import org.jetbrains.benchmarksLauncher.assert
 
 fun octoTest() {
     val tree = OctoTree<Boolean>(4)

performance/ring/src/main/kotlin/org/jetbrains/ring/StringBenchmark.kt

@@ -22,7 +22,7 @@ open class StringBenchmark {
         get() = _data!!
     var csv: String = ""
 
-    fun setup() {
+    init {
         val list = ArrayList<String>(BENCHMARK_SIZE)
         for (n in stringValues(BENCHMARK_SIZE))
             list.add(n)

performance/ring/src/main/kotlin/org/jetbrains/ring/SwitchBenchmark.kt

@@ -474,10 +474,7 @@ open class SwitchBenchmark {
     lateinit var denseIntData: IntArray
     lateinit var sparseIntData: IntArray
 
-    fun setupInts() {
+
-        denseIntData = IntArray(BENCHMARK_SIZE) { Random.nextInt(25) - 1 }
-        sparseIntData= IntArray(BENCHMARK_SIZE) { SPARSE_SWITCH_CASES[Random.nextInt(20)] }
-    }
 
     //Benchmark 
     fun testSparseIntSwitch() {
@@ -516,11 +513,7 @@ open class SwitchBenchmark {
 
     var data : Array<String> = arrayOf()
 
-    fun setupStrings() {
+
-        data = Array(BENCHMARK_SIZE) {
-            "ABCDEFG" + Random.nextInt(22)
-        }
-    }
 
     //Benchmark 
     fun testStringsSwitch() {
@@ -589,14 +582,7 @@ open class SwitchBenchmark {
     lateinit var enumData : Array<MyEnum>
     lateinit var denseEnumData : Array<MyEnum>
 
-    fun setupEnums() {
+
-        enumData = Array(BENCHMARK_SIZE) {
-            MyEnum.values()[it % MyEnum.values().size]
-        }
-        denseEnumData = Array(BENCHMARK_SIZE) {
-            MyEnum.values()[it % 20]
-        }
-    }
 
     //Benchmark 
     fun testEnumsSwitch() {
@@ -631,7 +617,18 @@ open class SwitchBenchmark {
 
     lateinit var sealedClassData: Array<MySealedClass>
 
-    fun setupSealedClassses() {
+    init {
+        data = Array(BENCHMARK_SIZE) {
+            "ABCDEFG" + Random.nextInt(22)
+        }
+        enumData = Array(BENCHMARK_SIZE) {
+            MyEnum.values()[it % MyEnum.values().size]
+        }
+        denseEnumData = Array(BENCHMARK_SIZE) {
+            MyEnum.values()[it % 20]
+        }
+        denseIntData = IntArray(BENCHMARK_SIZE) { Random.nextInt(25) - 1 }
+        sparseIntData = IntArray(BENCHMARK_SIZE) { SPARSE_SWITCH_CASES[Random.nextInt(20)] }
         sealedClassData = Array(BENCHMARK_SIZE) {
             when(Random.nextInt(10)) {
                 0 -> MySealedClass.MySealedClass1()

performance/ring/src/main/kotlin/org/jetbrains/ring/Utils.kt

@@ -16,7 +16,7 @@
 
 package org.jetbrains.ring
 
-expect fun writeToFile(fileName: String, text: String)
+const val BENCHMARK_SIZE = 10000
 
 expect class Blackhole {
     companion object {
@@ -35,7 +35,3 @@ expect class Random() {
     }
 }
 
-expect fun assert(value: Boolean)
-
-expect inline fun measureNanoTime(block: () -> Unit): Long
-

performance/ring/src/main/kotlin/org/jetbrains/ring/WithIndiciesBenchmark.kt

@@ -21,7 +21,7 @@ open class WithIndiciesBenchmark {
     val data: ArrayList<Value>
         get() = _data!!
 
-    fun setup() {
+    init {
         val list = ArrayList<Value>(BENCHMARK_SIZE)
         for (n in classValues(BENCHMARK_SIZE))
             list.add(n)

performance/settings.gradle

@@ -0,0 +1 @@
+include ':ring'

performance/shared/src/main/kotlin-jvm/org/jetbrains/benchmarksLauncher/Utils.kt

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.benchmarksLauncher
+
+import java.io.File
+
+actual fun writeToFile(fileName: String, text: String) {
+    File(fileName).printWriter().use { out ->
+        out.println(text)
+    }
+}
+
+// Wrapper for assert funtion in stdlib.
+actual fun assert(value: Boolean) {
+    kotlin.assert(value)
+}
+
+// Wrapper for measureNanoTime funtion in stdlib.
+actual inline fun measureNanoTime(block: () -> Unit): Long {
+    return kotlin.system.measureNanoTime(block)
+}
+
+actual fun cleanup() {}
+

performance/shared/src/main/kotlin-native/org/jetbrains/benchmarksLauncher/Utils.kt

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.benchmarksLauncher
+
+import kotlin.native.internal.GC
+import platform.posix.*
+
+actual fun writeToFile(fileName: String, text: String) {
+    val file = fopen(fileName, "wt") ?: error("Cannot write file '$fileName'")
+    try {
+        if (fputs(text, file) == EOF) throw Error("File write error")
+    } finally {
+        fclose(file)
+    }
+}
+
+// Wrapper for assert funtion in stdlib
+actual fun assert(value: Boolean) {
+    kotlin.assert(value)
+}
+
+// Wrapper for measureNanoTime funtion in stdlib
+actual inline fun measureNanoTime(block: () -> Unit): Long {
+    return kotlin.system.measureNanoTime(block)
+}
+
+actual fun cleanup() {
+    GC.collect()
+}

performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/BenchmarksCollection.kt

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.benchmarksLauncher
+
+import kotlin.reflect.KFunction0
+
+class BenchmarksCollection(private val benchmarks: MutableMap<String, KFunction0<Any?>> = mutableMapOf()) :
+        MutableMap<String, KFunction0<Any?>> by benchmarks {
+}

performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/JsonReportCreator.kt

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.jetbrains.ring
+package org.jetbrains.benchmarksLauncher
 
 import org.jetbrains.report.BenchmarkResult
 

performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/Utils.kt

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.benchmarksLauncher
+
+expect fun writeToFile(fileName: String, text: String)
+
+expect fun assert(value: Boolean)
+
+expect inline fun measureNanoTime(block: () -> Unit): Long
+
+expect fun cleanup()

performance/shared/src/main/kotlin/org/jetbrains/benchmarksLauncher/launcher.kt

@@ -0,0 +1,102 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.benchmarksLauncher
+
+import kotlin.math.sqrt
+import org.jetbrains.report.BenchmarkResult
+import org.jetbrains.kliopt.*
+
+abstract class Launcher(val numWarmIterations: Int, val numberOfAttempts: Int, val prefix: String = "") {
+    class Results(val mean: Double, val variance: Double)
+
+    abstract val benchmarks: BenchmarksCollection
+
+    protected val benchmarkResults = mutableListOf<BenchmarkResult>()
+
+    fun launch(benchmarksToRun: Collection<String>? = null): List<BenchmarkResult> {
+        val runningBenchmarks = benchmarksToRun ?: benchmarks.keys
+        runningBenchmarks.forEach {
+            val benchmark = benchmarks[it]
+            benchmark ?: error("Benchmark $it wasn't found!")
+            var i = numWarmIterations
+            while (i-- > 0) benchmark()
+            cleanup()
+            var autoEvaluatedNumberOfMeasureIteration = 1
+            while (true) {
+                var j = autoEvaluatedNumberOfMeasureIteration
+                val time = measureNanoTime {
+                    while (j-- > 0) {
+                        benchmark()
+                    }
+                    cleanup()
+                }
+                if (time >= 100L * 1_000_000) // 100ms
+                    break
+                autoEvaluatedNumberOfMeasureIteration *= 2
+            }
+            val samples = DoubleArray(numberOfAttempts)
+            for (k in samples.indices) {
+                i = autoEvaluatedNumberOfMeasureIteration
+                val time = measureNanoTime {
+                    while (i-- > 0) {
+                        benchmark()
+                    }
+                    cleanup()
+                }
+                val scaledTime = time * 1.0 / autoEvaluatedNumberOfMeasureIteration
+                samples[k] = scaledTime
+                // Save benchmark object
+                benchmarkResults.add(BenchmarkResult("$prefix$it", BenchmarkResult.Status.PASSED,
+                        scaledTime / 1000, scaledTime / 1000,
+                        k + 1, numWarmIterations))
+            }
+        }
+        return benchmarkResults
+    }
+}
+
+object BenchmarksRunner {
+    fun parse(args: Array<String>): ArgParser {
+        val options = listOf(
+                OptionDescriptor(ArgType.Int(), "warmup", "w", "Number of warm up iterations", "0"),
+                OptionDescriptor(ArgType.Int(), "repeat", "r", "Number of each becnhmark run", "10"),
+                OptionDescriptor(ArgType.String(), "prefix", "p", "Prefix added to benchmark name", ""),
+                OptionDescriptor(ArgType.String(), "output", "o", "Output file"),
+                OptionDescriptor(ArgType.String(), "filter", "f", "Benchmark to run", isMultiple = true)
+        )
+
+        // Parse args.
+        val argParser = ArgParser(options)
+        argParser.parse(args)
+        return argParser
+    }
+
+    fun collect(results: List<BenchmarkResult>, parser: ArgParser) {
+        parser.get<String>("output")?.let {
+            JsonReportCreator(results).printJsonReport(it)
+        }
+    }
+
+    fun runBenchmarks(args: Array<String>,
+                      run: (parser: ArgParser) -> List<BenchmarkResult>,
+                      parseArgs: (args: Array<String>) -> ArgParser = this::parse,
+                      collect: (results: List<BenchmarkResult>, parser: ArgParser) -> Unit = this::collect) {
+        val parser = parseArgs(args)
+        val results = run(parser)
+        collect(results, parser)
+    }
+}

performance/src/main/kotlin/main.kt

@@ -1,48 +0,0 @@
-/*
- * Copyright 2010-2017 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.jetbrains.ring.Launcher
-import org.jetbrains.ring.JsonReportCreator
-
-fun main(args: Array<String>) {
-    var numWarmIterations = 0       // Should be 100000 for jdk based run
-    var numberOfAttempts = 10
-    var jsonReport: String? = null
-
-    when (args.size) {
-        0 -> { }
-        1 -> numWarmIterations = args[0].toInt()
-        2 -> {
-            numWarmIterations = args[0].toInt()
-            numberOfAttempts = args[1].toInt()
-        }
-        3 -> {
-            numWarmIterations = args[0].toInt()
-            numberOfAttempts = args[1].toInt()
-            jsonReport = args[2].toString()
-        }
-        else -> {
-            println("Usage: perf [# warmup iterations] [# attempts] [# path of json report]")
-            return
-        }
-    }
-
-    println("Ring starting")
-    println("  warmup  iterations count: $numWarmIterations")
-    val results = Launcher(numWarmIterations, numberOfAttempts).runBenchmarks()
-    if (jsonReport != null)
-        JsonReportCreator(results).printJsonReport(jsonReport)
-}

performance/src/main/kotlin/org/jetbrains/ring/launcher.kt

@@ -1,458 +0,0 @@
-/*
- * Copyright 2010-2017 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.jetbrains.ring
-
-import octoTest
-import kotlin.math.sqrt
-import org.jetbrains.report.BenchmarkResult
-
-const val BENCHMARK_SIZE = 10000
-
-//-----------------------------------------------------------------------------//
-
-class Launcher(val numWarmIterations: Int, val numberOfAttempts: Int) {
-    class Results(val mean: Double, val variance: Double)
-
-    val benchmarkResults = mutableListOf<BenchmarkResult>()
-
-    fun launch(benchmark: () -> Any?, name: String) {                          // If benchmark runs too long - use coeff to speed it up.
-        var i = numWarmIterations
-
-        while (i-- > 0) benchmark()
-        cleanup()
-
-        var autoEvaluatedNumberOfMeasureIteration = 1
-        while (true) {
-            var j = autoEvaluatedNumberOfMeasureIteration
-            val time = measureNanoTime {
-                while (j-- > 0) {
-                    benchmark()
-                }
-                cleanup()
-            }
-            if (time >= 100L * 1_000_000) // 100ms
-                break
-            autoEvaluatedNumberOfMeasureIteration *= 2
-        }
-
-        val samples = DoubleArray(numberOfAttempts)
-        for (k in samples.indices) {
-            i = autoEvaluatedNumberOfMeasureIteration
-            val time = measureNanoTime {
-                while (i-- > 0) {
-                    benchmark()
-                }
-                cleanup()
-            }
-            val scaledTime = time * 1.0 / autoEvaluatedNumberOfMeasureIteration
-            samples[k] = scaledTime
-            // Save benchmark object
-            benchmarkResults.add(BenchmarkResult(name, BenchmarkResult.Status.PASSED,
-                                scaledTime / 1000, scaledTime / 1000,
-                                k + 1, numWarmIterations))
-        }
-    }
-
-    //-------------------------------------------------------------------------//
-
-    fun runBenchmarks(): List<BenchmarkResult> {
-        runAbstractMethodBenchmark()
-        runClassArrayBenchmark()
-        runClassBaselineBenchmark()
-        runClassListBenchmark()
-        runClassStreamBenchmark()
-        runCompanionObjectBenchmark()
-        runDefaultArgumentBenchmark()
-        runElvisBenchmark()
-        runEulerBenchmark()
-        runFibonacciBenchmark()
-        runForLoopBenchmark()
-        runInlineBenchmark()
-        runIntArrayBenchmark()
-        runIntBaselineBenchmark()
-        runIntListBenchmark()
-        runIntStreamBenchmark()
-        runLambdaBenchmark()
-        runLoopBenchmark()
-        runMatrixMapBenchmark()
-        runParameterNotNullAssertionBenchmark()
-        runPrimeListBenchmark()
-        runStringBenchmark()
-        runSwitchBenchmark()
-        runWithIndiciesBenchmark()
-        runOctoTest()
-
-        return benchmarkResults
-    }
-
-    //-------------------------------------------------------------------------//
-
-    fun runAbstractMethodBenchmark() {
-        val benchmark = AbstractMethodBenchmark()
-
-        launch(benchmark::sortStrings, "AbstractMethod.sortStrings")
-        launch(benchmark::sortStringsWithComparator, "AbstractMethod.sortStringsWithComparator")
-    }
-
-    //-------------------------------------------------------------------------//
-
-    fun runClassArrayBenchmark() {
-        val benchmark = ClassArrayBenchmark()
-        benchmark.setup()
-
-        launch(benchmark::copy,"ClassArray.copy")
-        launch(benchmark::copyManual, "ClassArray.copyManual")
-        launch(benchmark::filterAndCount, "ClassArray.filterAndCount")
-        launch(benchmark::filterAndMap, "ClassArray.filterAndMap")
-        launch(benchmark::filterAndMapManual, "ClassArray.filterAndMapManual")
-        launch(benchmark::filter, "ClassArray.filter")
-        launch(benchmark::filterManual, "ClassArray.filterManual")
-        launch(benchmark::countFilteredManual, "ClassArray.countFilteredManual")
-        launch(benchmark::countFiltered, "ClassArray.countFiltered")
-        launch(benchmark::countFilteredLocal, "ClassArray.countFilteredLocal")
-    }
-
-    //-------------------------------------------------------------------------//
-
-    fun runClassBaselineBenchmark() {
-        val benchmark = ClassBaselineBenchmark()
-
-        launch(benchmark::consume, "ClassBaseline.consume")
-        launch(benchmark::consumeField, "ClassBaseline.consumeField")
-        launch(benchmark::allocateList, "ClassBaseline.allocateList")
-        launch(benchmark::allocateArray, "ClassBaseline.allocateArray")
-        launch(benchmark::allocateListAndFill, "ClassBaseline.allocateListAndFill")
-        launch(benchmark::allocateListAndWrite, "ClassBaseline.allocateListAndWrite")
-        launch(benchmark::allocateArrayAndFill, "ClassBaseline.allocateArrayAndFill")
-    }
-
-    //-------------------------------------------------------------------------//
-
-    fun runClassListBenchmark() {
-        val benchmark = ClassListBenchmark()
-        benchmark.setup()
-
-        launch(benchmark::copy, "ClassList.copy")
-        launch(benchmark::copyManual, "ClassList.copyManual")
-        launch(benchmark::filterAndCount, "ClassList.filterAndCount")
-        launch(benchmark::filterAndCountWithLambda, "ClassList.filterAndCountWithLambda")
-        launch(benchmark::filterWithLambda, "ClassList.filterWithLambda")
-        launch(benchmark::mapWithLambda, "ClassList.mapWithLambda")
-        launch(benchmark::countWithLambda, "ClassList.countWithLambda")
-        launch(benchmark::filterAndMapWithLambda, "ClassList.filterAndMapWithLambda")
-        launch(benchmark::filterAndMapWithLambdaAsSequence, "ClassList.filterAndMapWithLambdaAsSequence")
-        launch(benchmark::filterAndMap, "ClassList.filterAndMap")
-        launch(benchmark::filterAndMapManual, "ClassList.filterAndMapManual")
-        launch(benchmark::filter, "ClassList.filter")
-        launch(benchmark::filterManual, "ClassList.filterManual")
-        launch(benchmark::countFilteredManual, "ClassList.countFilteredManual")
-        launch(benchmark::countFiltered, "ClassList.countFiltered")
-        launch(benchmark::reduce, "ClassList.reduce")
-    }
-
-    //-------------------------------------------------------------------------//
-
-    fun runClassStreamBenchmark() {
-        val benchmark = ClassStreamBenchmark()
-        benchmark.setup()
-
-        launch(benchmark::copy, "ClassStream.copy")
-        launch(benchmark::copyManual, "ClassStream.copyManual")
-        launch(benchmark::filterAndCount, "ClassStream.filterAndCount")
-        launch(benchmark::filterAndMap, "ClassStream.filterAndMap")
-        launch(benchmark::filterAndMapManual, "ClassStream.filterAndMapManual")
-        launch(benchmark::filter, "ClassStream.filter")
-        launch(benchmark::filterManual, "ClassStream.filterManual")
-        launch(benchmark::countFilteredManual, "ClassStream.countFilteredManual")
-        launch(benchmark::countFiltered, "ClassStream.countFiltered")
-        launch(benchmark::reduce, "ClassStream.reduce")
-    }
-
-    //-------------------------------------------------------------------------//
-
-    fun runCompanionObjectBenchmark() {
-        val benchmark = CompanionObjectBenchmark()
-
-        launch(benchmark::invokeRegularFunction, "CompanionObject.invokeRegularFunction")
-        launch(benchmark::invokeJvmStaticFunction, "CompanionObject.invokeJvmStaticFunction")
-    }
-
-    //-------------------------------------------------------------------------//
-
-    fun runDefaultArgumentBenchmark() {
-        val benchmark = DefaultArgumentBenchmark()
-        benchmark.setup()
-
-        launch(benchmark::testOneOfTwo, "DefaultArgument.testOneOfTwo")
-        launch(benchmark::testTwoOfTwo, "DefaultArgument.testTwoOfTwo")
-        launch(benchmark::testOneOfFour, "DefaultArgument.testOneOfFour")
-        launch(benchmark::testFourOfFour, "DefaultArgument.testFourOfFour")
-        launch(benchmark::testOneOfEight, "DefaultArgument.testOneOfEight")
-        launch(benchmark::testEightOfEight, "DefaultArgument.testEightOfEight")
-    }
-
-    //-------------------------------------------------------------------------//
-
-    fun runElvisBenchmark() {
-        val benchmark = ElvisBenchmark()
-        benchmark.setup()
-
-        launch(benchmark::testElvis, "Elvis.testElvis")
-    }
-
-    //-------------------------------------------------------------------------//
-
-    fun runEulerBenchmark() {
-        val benchmark = EulerBenchmark()
-
-        launch(benchmark::problem1bySequence, "Euler.problem1bySequence")
-        launch(benchmark::problem1, "Euler.problem1")
-        launch(benchmark::problem2, "Euler.problem2")
-        launch(benchmark::problem4, "Euler.problem4")
-        launch(benchmark::problem8, "Euler.problem8")
-        launch(benchmark::problem9, "Euler.problem9")
-        launch(benchmark::problem14, "Euler.problem14")
-        launch(benchmark::problem14full, "Euler.problem14full")
-    }
-
-
-    //-------------------------------------------------------------------------//
-
-    fun runFibonacciBenchmark() {
-        val benchmark = FibonacciBenchmark()
-        launch(benchmark::calcClassic, "Fibonacci.calcClassic")
-        launch(benchmark::calc, "Fibonacci.calc")
-        launch(benchmark::calcWithProgression, "Fibonacci.calcWithProgression")
-        launch(benchmark::calcSquare, "Fibonacci.calcSquare")
-    }
-
-    //-------------------------------------------------------------------------//
-
-    fun runForLoopBenchmark() {
-        val benchmark = ForLoopsBenchmark()
-
-        launch(benchmark::arrayLoop, "ForLoops.arrayLoop")
-        launch(benchmark::intArrayLoop, "ForLoops.intArrayLoop")
-        launch(benchmark::floatArrayLoop, "ForLoops.floatArrayLoop")
-        launch(benchmark::charArrayLoop, "ForLoops.charArrayLoop")
-        launch(benchmark::stringLoop, "ForLoops.stringLoop")
-
-        launch(benchmark::arrayIndicesLoop, "ForLoops.arrayIndicesLoop")
-        launch(benchmark::intArrayIndicesLoop, "ForLoops.intArrayIndicesLoop")
-        launch(benchmark::floatArrayIndicesLoop, "ForLoops.floatArrayIndicesLoop")
-        launch(benchmark::charArrayIndicesLoop, "ForLoops.charArrayIndicesLoop")
-        launch(benchmark::stringIndicesLoop, "ForLoops.stringIndicesLoop")
-    }
-
-    //-------------------------------------------------------------------------//
-
-    fun runInlineBenchmark() {
-        val benchmark = InlineBenchmark()
-        launch(benchmark::calculate, "Inline.calculate")
-        launch(benchmark::calculateInline, "Inline.calculateInline")
-        launch(benchmark::calculateGeneric, "Inline.calculateGeneric")
-        launch(benchmark::calculateGenericInline, "Inline.calculateGenericInline")
-    }
-
-    //-------------------------------------------------------------------------//
-
-    fun runIntArrayBenchmark() {
-        val benchmark = IntArrayBenchmark()
-        benchmark.setup()
-
-        launch(benchmark::copy, "IntArray.copy")
-        launch(benchmark::copyManual, "IntArray.copyManual")
-        launch(benchmark::filterAndCount, "IntArray.filterAndCount")
-        launch(benchmark::filterSomeAndCount, "IntArray.filterSomeAndCount")
-        launch(benchmark::filterAndMap, "IntArray.filterAndMap")
-        launch(benchmark::filterAndMapManual, "IntArray.filterAndMapManual")
-        launch(benchmark::filter, "IntArray.filter")
-        launch(benchmark::filterSome, "IntArray.filterSome")
-        launch(benchmark::filterPrime, "IntArray.filterPrime")
-        launch(benchmark::filterManual, "IntArray.filterManual")
-        launch(benchmark::filterSomeManual, "IntArray.filterSomeManual")
-        launch(benchmark::countFilteredManual, "IntArray.countFilteredManual")
-        launch(benchmark::countFilteredSomeManual, "IntArray.countFilteredSomeManual")
-        launch(benchmark::countFilteredPrimeManual, "IntArray.countFilteredPrimeManual")
-        launch(benchmark::countFiltered, "IntArray.countFiltered")
-        launch(benchmark::countFilteredSome, "IntArray.countFilteredSome")
-        launch(benchmark::countFilteredPrime, "IntArray.countFilteredPrime")
-        launch(benchmark::countFilteredLocal, "IntArray.countFilteredLocal")
-        launch(benchmark::countFilteredSomeLocal, "IntArray.countFilteredSomeLocal")
-        launch(benchmark::reduce, "IntArray.reduce")
-    }
-
-    //-------------------------------------------------------------------------//
-
-    fun runIntBaselineBenchmark() {
-        val benchmark = IntBaselineBenchmark()
-
-        launch(benchmark::consume, "IntBaseline.consume")
-        launch(benchmark::allocateList, "IntBaseline.allocateList")
-        launch(benchmark::allocateArray, "IntBaseline.allocateArray")
-        launch(benchmark::allocateListAndFill, "IntBaseline.allocateListAndFill")
-        launch(benchmark::allocateArrayAndFill, "IntBaseline.allocateArrayAndFill")
-    }
-
-    //-------------------------------------------------------------------------//
-
-    fun runIntListBenchmark() {
-        val benchmark = IntListBenchmark()
-        benchmark.setup()
-
-        launch(benchmark::copy, "IntList.copy")
-        launch(benchmark::copyManual, "IntList.copyManual")
-        launch(benchmark::filterAndCount, "IntList.filterAndCount")
-        launch(benchmark::filterAndMap, "IntList.filterAndMap")
-        launch(benchmark::filterAndMapManual, "IntList.filterAndMapManual")
-        launch(benchmark::filter, "IntList.filter")
-        launch(benchmark::filterManual, "IntList.filterManual")
-        launch(benchmark::countFilteredManual, "IntList.countFilteredManual")
-        launch(benchmark::countFiltered, "IntList.countFiltered")
-        launch(benchmark::countFilteredLocal, "IntList.countFilteredLocal")
-        launch(benchmark::reduce, "IntList.reduce")
-    }
-
-    //-------------------------------------------------------------------------//
-
-    fun runIntStreamBenchmark() {
-        val benchmark = IntStreamBenchmark()
-        benchmark.setup()
-
-        launch(benchmark::copy, "IntStream.copy")
-        launch(benchmark::copyManual, "IntStream.copyManual")
-        launch(benchmark::filterAndCount, "IntStream.filterAndCount")
-        launch(benchmark::filterAndMap, "IntStream.filterAndMap")
-        launch(benchmark::filterAndMapManual, "IntStream.filterAndMapManual")
-        launch(benchmark::filter, "IntStream.filter")
-        launch(benchmark::filterManual, "IntStream.filterManual")
-        launch(benchmark::countFilteredManual, "IntStream.countFilteredManual")
-        launch(benchmark::countFiltered, "IntStream.countFiltered")
-        launch(benchmark::countFilteredLocal, "IntStream.countFilteredLocal")
-        launch(benchmark::reduce, "IntStream.reduce")
-    }
-
-    //-------------------------------------------------------------------------//
-
-    fun runLambdaBenchmark() {
-        val benchmark = LambdaBenchmark()
-        benchmark.setup()
-
-        launch(benchmark::noncapturingLambda, "Lambda.noncapturingLambda")
-        launch(benchmark::noncapturingLambdaNoInline, "Lambda.noncapturingLambdaNoInline")
-        launch(benchmark::capturingLambda, "Lambda.capturingLambda")
-        launch(benchmark::capturingLambdaNoInline, "Lambda.capturingLambdaNoInline")
-        launch(benchmark::mutatingLambda, "Lambda.mutatingLambda")
-        launch(benchmark::mutatingLambdaNoInline, "Lambda.mutatingLambdaNoInline")
-        launch(benchmark::methodReference, "Lambda.methodReference")
-        launch(benchmark::methodReferenceNoInline, "Lambda.methodReferenceNoInline")
-    }
-
-    //-------------------------------------------------------------------------//
-
-    fun runLoopBenchmark() {
-        val benchmark = LoopBenchmark()
-        benchmark.setup()
-
-        launch(benchmark::arrayLoop, "Loop.arrayLoop")
-        launch(benchmark::arrayIndexLoop, "Loop.arrayIndexLoop")
-        launch(benchmark::rangeLoop, "Loop.rangeLoop")
-        launch(benchmark::arrayListLoop, "Loop.arrayListLoop")
-        launch(benchmark::arrayWhileLoop, "Loop.arrayWhileLoop")
-        launch(benchmark::arrayForeachLoop, "Loop.arrayForeachLoop")
-        launch(benchmark::arrayListForeachLoop, "Loop.arrayListForeachLoop")
-    }
-
-    //-------------------------------------------------------------------------//
-
-    fun runMatrixMapBenchmark() {
-        val benchmark = MatrixMapBenchmark()
-
-        launch(benchmark::add, "MatrixMap.add")
-    }
-
-    //-------------------------------------------------------------------------//
-
-    fun runParameterNotNullAssertionBenchmark() {
-        val benchmark = ParameterNotNullAssertionBenchmark()
-
-        launch(benchmark::invokeOneArgWithNullCheck, "ParameterNotNull.invokeOneArgWithNullCheck")
-        launch(benchmark::invokeOneArgWithoutNullCheck, "ParameterNotNull.invokeOneArgWithoutNullCheck")
-        launch(benchmark::invokeTwoArgsWithNullCheck, "ParameterNotNull.invokeTwoArgsWithNullCheck")
-        launch(benchmark::invokeTwoArgsWithoutNullCheck, "ParameterNotNull.invokeTwoArgsWithoutNullCheck")
-        launch(benchmark::invokeEightArgsWithNullCheck, "ParameterNotNull.invokeEightArgsWithNullCheck")
-        launch(benchmark::invokeEightArgsWithoutNullCheck, "ParameterNotNull.invokeEightArgsWithoutNullCheck")
-    }
-
-    //-------------------------------------------------------------------------//
-
-    fun runPrimeListBenchmark() {
-        val benchmark = PrimeListBenchmark()
-
-        launch(benchmark::calcDirect, "PrimeList.calcDirect")
-        launch(benchmark::calcEratosthenes, "PrimeList.calcEratosthenes")
-    }
-
-    //-------------------------------------------------------------------------//
-
-    fun runStringBenchmark() {
-        val benchmark = StringBenchmark()
-        benchmark.setup()
-
-        launch(benchmark::stringConcat, "String.stringConcat")
-        launch(benchmark::stringConcatNullable, "String.stringConcatNullable")
-        launch(benchmark::stringBuilderConcat, "String.stringBuilderConcat")
-        launch(benchmark::stringBuilderConcatNullable, "String.stringBuilderConcatNullable")
-        launch(benchmark::summarizeSplittedCsv, "String.summarizeSplittedCsv")
-    }
-
-    //-------------------------------------------------------------------------//
-
-    fun runSwitchBenchmark() {
-        val benchmark = SwitchBenchmark()
-        benchmark.setupInts()
-        benchmark.setupStrings()
-        benchmark.setupEnums()
-        benchmark.setupSealedClassses()
-
-        launch(benchmark::testSparseIntSwitch, "Switch.testSparseIntSwitch")
-        launch(benchmark::testDenseIntSwitch, "Switch.testDenseIntSwitch")
-        launch(benchmark::testConstSwitch, "Switch.testConstSwitch")
-        launch(benchmark::testObjConstSwitch, "Switch.testObjConstSwitch")
-        launch(benchmark::testVarSwitch, "Switch.testVarSwitch")
-        launch(benchmark::testStringsSwitch, "Switch.testStringsSwitch")
-        launch(benchmark::testEnumsSwitch, "Switch.testEnumsSwitch")
-        launch(benchmark::testDenseEnumsSwitch, "Switch.testDenseEnumsSwitch")
-        launch(benchmark::testSealedWhenSwitch, "Switch.testSealedWhenSwitch")
-    }
-
-    //-------------------------------------------------------------------------//
-
-    fun runWithIndiciesBenchmark() {
-        val benchmark = WithIndiciesBenchmark()
-        benchmark.setup()
-
-        launch(benchmark::withIndicies, "WithIndicies.withIndicies")
-        launch(benchmark::withIndiciesManual, "WithIndicies.withIndiciesManual")
-    }
-
-    //-------------------------------------------------------------------------//
-
-    fun runOctoTest() {
-        launch(::octoTest, "OctoTest")
-    }
-}

tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/BenchmarksReport.kt

@@ -76,6 +76,16 @@ class BenchmarksReport(val env: Environment, benchmarksList: List<BenchmarkResul
         }
         """
     }
+
+    // Concatenate benchmarks report if they have same environment and compiler.
+    operator fun plus(other: BenchmarksReport): BenchmarksReport {
+        if (compiler != other.compiler && env != other.env) {
+            error ("It's impossible to concat reports from different machines!")
+        }
+        val mergedBenchmarks = HashMap<String, List<BenchmarkResult>>(benchmarks)
+        mergedBenchmarks.putAll(other.benchmarks)
+        return BenchmarksReport(env, mergedBenchmarks.flatMap{it.value}, compiler)
+    }
 }
 
 // Class for kotlin compiler

tools/benchmarksAnalyzer/build.gradle

@@ -40,6 +40,7 @@ kotlin {
             }
             kotlin.srcDir '../benchmarks/shared/src'
             kotlin.srcDir 'src/main/kotlin'
+            kotlin.srcDir '../kliopt'
 
         }
         commonTest {

tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt

@@ -221,7 +221,7 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport,
                 // Calculate metrics for showing difference.
                 val percent = current.calcPercentageDiff(previous)
                 val ratio = current.calcRatio(previous)
-                if (abs(percent.mean) >= meaningfulChangesValue) {
+                if (abs(percent.mean) - percent.variance >= meaningfulChangesValue) {
                     return Pair(name, Pair(percent, ratio))
                 }
             }