Write benchmarks analyzer to compare benchmarks runs. (#2493)

build.gradle

@@ -462,10 +462,11 @@ task uploadBundle {
     }
 }
 
-task performance {
+// TODO return when problems with HostManagers will be solved
+/*task performance {
     dependsOn 'dist'
     dependsOn ':performance:bench'
-}
+}*/
 
 task teamcityKonanVersion {
     doLast {

settings.gradle

@@ -29,6 +29,7 @@ include ':backend.native:tests'
 include ':backend.native:debugger-tests'
 include ':utilities'
 //include ':performance'    TODO: return when there is one HostManager class version
+//include ':tools:benchmarksAnalyzer'
 include ':platformLibs'
 
 includeBuild 'shared'
@@ -36,6 +37,7 @@ includeBuild 'extracted/konan.metadata'
 includeBuild 'extracted/konan.serializer'
 includeBuild 'tools/kotlin-native-gradle-plugin'
 
+
 if (hasProperty("kotlinProjectPath")) {
     includeBuild(kotlinProjectPath) {
         dependencySubstitution {

tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/BenchmarksReport.kt

@@ -79,7 +79,7 @@ interface EntityFromJsonFactory<T>: ConvertedFromJson {
 }
 
 // Class for benchcmarks report with all information of run.
-data class BenchmarksReport(val env: Environment, val benchmarks: List<BenchmarkResult>, val compiler: Compiler):
+class BenchmarksReport(val env: Environment, benchmarksList: List<BenchmarkResult>, val compiler: Compiler):
         JsonSerializable {
 
     companion object: EntityFromJsonFactory<BenchmarksReport> {
@@ -88,8 +88,8 @@ data class BenchmarksReport(val env: Environment, val benchmarks: List<Benchmark
                 val env = Environment.create(getRequiredField(data, "env"))
                 val benchmarksObj = getRequiredField(data, "benchmarks")
                 val compiler = Compiler.create(getRequiredField(data, "kotlin"))
-                val benchmarks = parseBenchmarksArray(benchmarksObj)
-                return BenchmarksReport(env, benchmarks, compiler)
+                val benchmarksList = parseBenchmarksArray(benchmarksObj)
+                return BenchmarksReport(env, benchmarksList, compiler)
             } else {
                 error("Top level entity is expected to be an object. Please, check origin files.")
             }
@@ -107,14 +107,20 @@ data class BenchmarksReport(val env: Environment, val benchmarks: List<Benchmark
                 error("Benchmarks field is expected to be an array. Please, check origin files.")
             }
         }
+
+        // Made a map of becnhmarks with name as key from list.
+        private fun structBenchmarks(benchmarksList: List<BenchmarkResult>) =
+                benchmarksList.groupBy{ it.name }
     }
 
+    val benchmarks: Map<String, List<BenchmarkResult>> = structBenchmarks(benchmarksList)
+
     override fun toJson(): String {
         return """
         {
             "env": ${env.toJson()},
             "kotlin": ${compiler.toJson()},
-            "benchmarks": ${arrayToJson(benchmarks)}
+            "benchmarks": ${arrayToJson(benchmarks.flatMap{it.value})}
         }
         """
     }
@@ -279,6 +285,8 @@ class BenchmarkResult(val name: String, val status: Status,
                       val score: Double, val runtimeInUs: Double,
                       val repeat: Int, val warmup: Int): JsonSerializable {
 
+    constructor(name: String, score: Double) : this(name, Status.PASSED, score, 0.0, 0, 0)
+
     companion object: EntityFromJsonFactory<BenchmarkResult> {
 
         override fun create(data: JsonElement): BenchmarkResult {

tools/benchmarksAnalyzer/build.gradle

@@ -0,0 +1,94 @@
+buildscript {
+    ext.rootBuildDirectory = file('../..')
+
+    apply from: "$rootBuildDirectory/gradle/loadRootProperties.gradle"
+    apply from: "$rootBuildDirectory/gradle/kotlinGradlePlugin.gradle"
+
+    repositories {
+        maven {
+            url 'https://cache-redirector.jetbrains.com/jcenter'
+        }
+        maven {
+            url kotlinCompilerRepo
+        }
+    }
+
+    dependencies {
+        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion"
+    }
+}
+
+apply plugin: 'kotlin-multiplatform'
+
+repositories {
+    maven {
+        url 'https://cache-redirector.jetbrains.com/jcenter'
+    }
+    maven {
+        url kotlinCompilerRepo
+    }
+    maven {
+        url buildKotlinCompilerRepo
+    }
+}
+
+kotlin {
+    sourceSets {
+        commonMain {
+            dependencies {
+                implementation "org.jetbrains.kotlin:kotlin-stdlib-common:$kotlinVersion"
+            }
+            kotlin.srcDir '../benchmarks/shared/src'
+            kotlin.srcDir 'src/main/kotlin'
+
+        }
+        commonTest {
+            dependencies {
+                implementation "org.jetbrains.kotlin:kotlin-test-common:$kotlinVersion"
+                implementation "org.jetbrains.kotlin:kotlin-test-annotations-common:$kotlinVersion"
+            }
+            kotlin.srcDir 'src/tests'
+        }
+        jvmTest {
+            dependencies {
+                implementation "org.jetbrains.kotlin:kotlin-test:$kotlinVersion"
+                implementation "org.jetbrains.kotlin:kotlin-test-junit:$kotlinVersion"
+            }
+        }
+        nativeMain {
+            dependsOn commonMain
+            kotlin.srcDir 'src/main/kotlin-native'
+        }
+        jvmMain {
+            dependencies {
+                implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion"
+            }
+            kotlin.srcDir 'src/main/kotlin-jvm'
+        }
+        linuxMain { dependsOn nativeMain }
+        windowsMain { dependsOn nativeMain }
+        macosMain {dependsOn nativeMain }
+    }
+
+    targets {
+        fromPreset(presets.jvm, 'jvm') {
+            def mainOutput = compilations.main.output
+            compilations.all {
+                tasks[compileKotlinTaskName].kotlinOptions {
+                    jvmTarget = '1.8'
+                }
+                tasks[compileKotlinTaskName].kotlinOptions.suppressWarnings = true
+            }
+        }
+
+        fromPreset(presets.mingwX64, 'windows')
+        fromPreset(presets.linuxX64, 'linux')
+        fromPreset(presets.macosX64, 'macos')
+
+        configure([windows, linux, macos]) {
+            compilations.main.outputKinds('EXECUTABLE')
+            compilations.main.extraOpts '-opt'
+            compilations.main.buildTypes = [RELEASE]
+        }
+    }
+}

tools/benchmarksAnalyzer/gradle.properties

@@ -0,0 +1 @@
+org.jetbrains.kotlin.native.home=../../dist

tools/benchmarksAnalyzer/src/main/kotlin-jvm/org/jetbrains/analyzer/Utils.kt

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.analyzer
+
+import java.io.File
+import java.io.InputStream
+
+actual fun readFile(fileName: String): String {
+    val inputStream = File(fileName).inputStream()
+    val inputString = inputStream.bufferedReader().use { it.readText() }
+    return inputString
+}
+
+actual fun format(number: Double, decimalNumber: Int): String =
+    "%.${decimalNumber}f".format(number)
+
+actual fun writeToFile(fileName: String, text: String) {
+    File(fileName).printWriter().use { out ->
+        out.println(text)
+    }
+}
+
+actual fun assert(value: Boolean, lazyMessage: () -> Any) {
+    kotlin.assert(value, lazyMessage)
+}
+
+actual fun getEnv(variableName:String): String? =
+        System.getenv(variableName)

tools/benchmarksAnalyzer/src/main/kotlin-native/org/jetbrains/analyzer/Utils.kt

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.analyzer
+
+import platform.posix.*
+import kotlinx.cinterop.CValuesRef
+import kotlinx.cinterop.*
+
+actual fun readFile(fileName: String): String {
+    val file = fopen(fileName, "r") ?: error("Cannot write file '$fileName'")
+    var buffer = ByteArray(1024)
+    var text = StringBuilder()
+    try {
+        while (true) {
+            val nextLine = fgets(buffer.refTo(0), buffer.size, file)?.toKString()
+            if (nextLine == null) break
+            text.append(nextLine)
+        }
+    } finally {
+        fclose(file)
+    }
+    return text.toString()
+}
+
+actual fun format(number: Double, decimalNumber: Int): String {
+    var buffer = ByteArray(1024)
+    snprintf(buffer.refTo(0), buffer.size.toULong(), "%.${decimalNumber}f", number)
+    return buffer.stringFromUtf8()
+}
+
+actual fun writeToFile(fileName: String, text: String) {
+    val file = fopen(fileName, "wt") ?: error("Cannot write file '$fileName'")
+    try {
+        if (fputs(text, file) == EOF) throw Error("File write error")
+    } finally {
+        fclose(file)
+    }
+}
+
+actual fun assert(value: Boolean, lazyMessage: () -> Any) {
+    kotlin.assert(value, lazyMessage)
+}
+
+actual fun getEnv(variableName:String): String? =
+        getenv(variableName)?.toKString()

tools/benchmarksAnalyzer/src/main/kotlin/main.kt

@@ -0,0 +1,63 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.jetbrains.analyzer.getEnv
+import org.jetbrains.analyzer.readFile
+import org.jetbrains.analyzer.SummaryBenchmarksReport
+import org.jetbrains.kliopt.*
+import org.jetbrains.renders.TextRender
+import org.jetbrains.renders.TeamCityStatisticsRender
+import org.jetbrains.report.BenchmarksReport
+import org.jetbrains.report.json.JsonTreeParser
+
+fun main(args: Array<String>) {
+
+    val options = listOf(
+            OptionDescriptor(ArgType.STRING, "output", "o", "Output file"),
+            OptionDescriptor(ArgType.DOUBLE, "eps", "e", "Meaningful performance changes", "0.5"),
+            OptionDescriptor(ArgType.BOOLEAN, "short", "s", "Show short version of report", "false")
+    )
+
+    val arguments = listOf(
+            ArgDescriptor(ArgType.STRING, "mainReport", "Main report for analysis"),
+            ArgDescriptor(ArgType.STRING, "compareToReport", "Report to compare to", isRequired = false)
+    )
+
+    // Parse args.
+    val argParser = ArgParser(options, arguments)
+    if (argParser.parse(args)) {
+        // Read contents of file.
+        val mainBenchsResults = readFile(argParser.get("mainReport")!!.stringValue)
+        val mainReportElement = JsonTreeParser.parse(mainBenchsResults)
+        val mainBenchsReport = BenchmarksReport.create(mainReportElement)
+        var compareToBenchsReport = argParser.get("compareToReport")?.stringValue?.let {
+            val compareToResults = readFile(it)
+            val compareToReportElement = JsonTreeParser.parse(compareToResults)
+            BenchmarksReport.create(compareToReportElement)
+        }
+
+        // Generate comparasion report
+        val summaryReport = SummaryBenchmarksReport(mainBenchsReport,
+                compareToBenchsReport,
+                argParser.get("eps")!!.doubleValue)
+        TextRender().print(summaryReport, argParser.get("short")!!.booleanValue,
+                argParser.get("output")?.stringValue)
+        // Produce information for TeamCity if needed.
+        getEnv("TEAMCITY_BUILD_PROPERTIES_FILE")?.let {
+            TeamCityStatisticsRender().print(summaryReport, argParser.get("short")!!.booleanValue)
+        }
+    }
+}

tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/FieldChange.kt

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.jetbrains.analyzer
+
+// Report with changes of different fields.
+class ChangeReport<T>(val entityName: String, val changes: List<FieldChange<T>>) {
+    fun renderAsTextReport(): String {
+        var content = ""
+        if (!changes.isEmpty()) {
+            content = "$content$entityName changes\n"
+            content = "$content====================\n"
+            changes.forEach {
+                content = "$content${it.renderAsText()}"
+            }
+        }
+        return content
+    }
+}
+
+// Change of report field.
+class FieldChange<T>(val field: String, val previous: T, val current: T) {
+    companion object {
+        fun <T> getFieldChangeOrNull(field: String, previous: T, current: T): FieldChange<T>? {
+            if (previous != current) {
+                return FieldChange(field, previous, current)
+            }
+            return null
+        }
+    }
+
+    fun renderAsText(): String {
+        return "$field: $previous -> $current\n"
+    }
+}

tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt

@@ -0,0 +1,125 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.jetbrains.analyzer
+import org.jetbrains.report.BenchmarkResult
+import kotlin.math.abs
+import kotlin.math.max
+import kotlin.math.min
+import kotlin.math.pow
+import kotlin.math.sqrt
+
+// Entity to describe avarage values which conssists of mean and variance values.
+data class MeanVariance(val mean: Double, val variance: Double) {
+    override fun toString(): String {
+        val format = { number: Double -> format(number, 2)}
+        return "${format(mean)} ± ${format(variance)}"
+    }
+}
+
+// Composite benchmark which descibes avarage result for several runs and contains mean and variance value.
+data class MeanVarianceBenchmark(val meanBenchmark: BenchmarkResult, val varianceBenchmark: BenchmarkResult) {
+
+    // Calculate difference in percentage compare to another.
+    fun calcPercentageDiff(other: MeanVarianceBenchmark): MeanVariance {
+        assert(other.meanBenchmark.score > 0 &&
+                other.varianceBenchmark.score > 0 &&
+                other.meanBenchmark.score - other.varianceBenchmark.score != 0.0,
+                { "Mean and variance should be positive and not equal!" })
+        val mean = (meanBenchmark.score - other.meanBenchmark.score) / other.meanBenchmark.score
+        val maxValueChange = abs(meanBenchmark.score + varianceBenchmark.score -
+                        other.meanBenchmark.score + other.varianceBenchmark.score) /
+                        abs(other.meanBenchmark.score + other.varianceBenchmark.score)
+
+        val minValueChange = abs(meanBenchmark.score - varianceBenchmark.score -
+                        other.meanBenchmark.score - other.varianceBenchmark.score) /
+                        abs(other.meanBenchmark.score - other.varianceBenchmark.score)
+
+        val variance = abs(abs(mean) - max(minValueChange, maxValueChange))
+        return MeanVariance(mean * 100, variance * 100)
+    }
+
+    // Calculate ratio value compare to another.
+    fun calcRatio(other: MeanVarianceBenchmark): MeanVariance {
+        assert(other.meanBenchmark.score > 0 &&
+                other.varianceBenchmark.score > 0 &&
+                other.meanBenchmark.score - other.varianceBenchmark.score != 0.0,
+                { "Mean and variance should be positive and not equal!" })
+        val mean = meanBenchmark.score / other.meanBenchmark.score
+        val minRatio = (meanBenchmark.score - varianceBenchmark.score) / (other.meanBenchmark.score + other.varianceBenchmark.score)
+        val maxRatio = (meanBenchmark.score + varianceBenchmark.score) / (other.meanBenchmark.score - other.varianceBenchmark.score)
+        val ratioConfInt = min(abs(minRatio - mean), abs(maxRatio - mean))
+        return MeanVariance(mean, ratioConfInt)
+    }
+
+    override fun toString(): String {
+        val format = { number: Double -> format(number)}
+        return "${format(meanBenchmark.score)} ± ${format(varianceBenchmark.score)}"
+    }
+}
+
+fun geometricMean(values: List<Double>) = values.map { it.pow(1.0 / values.size) }.reduce { a, b -> a * b }
+
+fun computeMeanVariance(samples: List<Double>): MeanVariance {
+    val zStar = 1.96    // Critical point for 90% confidence of normal distribution.
+    val mean = samples.sum() / samples.size
+    val variance = samples.indices.sumByDouble { (samples[it] - mean) * (samples[it] - mean) } / samples.size
+    val confidenceInterval = sqrt(variance / samples.size) * zStar
+    return MeanVariance(mean, confidenceInterval)
+}
+
+// Calculate avarage results for bencmarks (each becnhmark can be run several times).
+fun collectMeanResults(benchmarks: Map<String, List<BenchmarkResult>>): BenchmarksTable {
+    return benchmarks.map {(name, resultsSet) ->
+        val repeatedSequence = IntArray(resultsSet.size)
+        var currentStatus = BenchmarkResult.Status.PASSED
+        var currentWarmup = -1
+
+        // Collect common becnhmark values and check them.
+        resultsSet.forEachIndexed { index, result ->
+            // If there was at least one failure, summary is marked as failure.
+            if (result.status == BenchmarkResult.Status.FAILED) {
+                currentStatus = result.status
+            }
+            repeatedSequence[index] = result.repeat
+            if (currentWarmup != -1)
+                if (result.warmup != currentWarmup)
+                    println("Check data consistency. Warmup value for benchmark '${result.name}' differs.")
+            currentWarmup = result.warmup
+        }
+
+        repeatedSequence.sort()
+        // Check if there are missed loop during running benchmarks.
+        repeatedSequence.forEachIndexed { index, element ->
+            if (index != 0)
+                if ((element - repeatedSequence[index - 1]) != 1)
+                    println("Check data consistency. For benchmark '$name' there is no run" +
+                            " between ${repeatedSequence[index - 1]} and $element.")
+        }
+
+        // Create mean and variance benchmarks result.
+        val scoreMeanVariance = computeMeanVariance(resultsSet.map { it.score })
+        val runtimeInUsMeanVariance = computeMeanVariance(resultsSet.map { it.runtimeInUs })
+        val meanBenchmark = BenchmarkResult(name, currentStatus, scoreMeanVariance.mean,
+                runtimeInUsMeanVariance.mean, repeatedSequence[resultsSet.size - 1],
+                currentWarmup)
+        val varianceBenchmark = BenchmarkResult(name, currentStatus, scoreMeanVariance.variance,
+                runtimeInUsMeanVariance.variance, repeatedSequence[resultsSet.size - 1],
+                currentWarmup)
+        name to MeanVarianceBenchmark(meanBenchmark, varianceBenchmark)
+    }.toMap()
+}

tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt

@@ -0,0 +1,209 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.jetbrains.analyzer
+
+import kotlin.math.abs
+import org.jetbrains.report.BenchmarkResult
+import org.jetbrains.report.Environment
+import org.jetbrains.report.Compiler
+import org.jetbrains.report.BenchmarksReport
+
+typealias SummaryBenchmark = Pair<MeanVarianceBenchmark?, MeanVarianceBenchmark?>
+typealias BenchmarksTable = Map<String, MeanVarianceBenchmark>
+typealias SummaryBenchmarksTable = Map<String, SummaryBenchmark>
+typealias ScoreChange = Pair<MeanVariance, MeanVariance>
+
+// Summary report with comparasion of separate benchmarks results.
+class SummaryBenchmarksReport (val currentReport: BenchmarksReport,
+                               val previousReport: BenchmarksReport? = null,
+                               val meaningfulChangesValue: Double = 0.5) {
+    // Report created by joining comparing reports.
+    val mergedReport: Map<String, SummaryBenchmark>
+
+    // Lists of benchmarks in different status.
+    private val benchmarksWithChangedStatus = mutableListOf<FieldChange<BenchmarkResult.Status>>()
+
+    // Maps with changes of performance.
+    var regressions = mapOf<String, ScoreChange>()
+        private set
+    var improvements = mapOf<String, ScoreChange>()
+        private set
+
+    // Summary value of report - geometric mean.
+    val geoMeanBenchmark: SummaryBenchmark
+    var geoMeanScoreChange: ScoreChange? = null
+        private set
+
+    // Changes in environment and tools.
+    val envChanges: List<FieldChange<String>>
+    val kotlinChanges: List<FieldChange<String>>
+
+    // Countable properties.
+    val failedBenchmarks: List<String>
+        get() = mergedReport.filter { it.value.first?.meanBenchmark?.status == BenchmarkResult.Status.FAILED }
+                            .map { it.key }
+
+    val addedBenchmarks: List<String>
+        get() = mergedReport.filter { it.value.second == null }.map { it.key }
+
+    val removedBenchmarks: List<String>
+        get() = mergedReport.filter { it.value.first == null }.map { it.key }
+
+    val benchmarksNumber: Int
+        get() = mergedReport.keys.size
+
+    val currentMeanVarianceBenchmarks: List<MeanVarianceBenchmark>
+        get() = mergedReport.filter { it.value.first != null }.map { it.value.first!! }
+
+    init {
+        // Count avarage values for each benchmark.
+        val currentBenchmarksTable = collectMeanResults(currentReport.benchmarks)
+        val previousBenchmarksTable = previousReport?.let {
+            collectMeanResults(previousReport.benchmarks)
+        }
+        mergedReport = createMergedReport(currentBenchmarksTable, previousBenchmarksTable)
+        geoMeanBenchmark = calculateGeoMeanBenchmark(currentBenchmarksTable, previousBenchmarksTable)
+        if (previousReport != null) {
+            // Check changes in environment and tools.
+            envChanges = analyzeEnvChanges(currentReport.env, previousReport.env)
+            kotlinChanges = analyzeKotlinChanges(currentReport.compiler, previousReport.compiler)
+            analyzePerformanceChanges()
+        } else {
+            envChanges = listOf<FieldChange<String>>()
+            kotlinChanges = listOf<FieldChange<String>>()
+        }
+    }
+
+    fun getBenchmarksWithChangedStatus(): List<FieldChange<BenchmarkResult.Status>> = benchmarksWithChangedStatus
+
+    // Create geometric mean.
+    private fun createGeoMeanBenchmark(benchTable: BenchmarksTable): MeanVarianceBenchmark {
+        val geoMeanBenchmarkName = "Geometric mean"
+        val geoMean = geometricMean(benchTable.toList().map { (_, value) -> value.meanBenchmark.score })
+        val varianceGeoMean = geometricMean(benchTable.toList().map { (_, value) -> value.varianceBenchmark.score })
+        val meanBenchmark = BenchmarkResult(geoMeanBenchmarkName, geoMean)
+        val varianceBenchmark = BenchmarkResult(geoMeanBenchmarkName, varianceGeoMean)
+        return MeanVarianceBenchmark(meanBenchmark, varianceBenchmark)
+    }
+
+    // Merge current and compare to report.
+    private fun createMergedReport(currentBenchmarks: BenchmarksTable, previousBenchmarks: BenchmarksTable?):
+            Map<String, SummaryBenchmark> {
+        val mergedTable = mutableMapOf<String, SummaryBenchmark>()
+        mergedTable.apply {
+            currentBenchmarks.forEach { (name, current) ->
+                val currentBenchmark = current.meanBenchmark
+                // Check existance of benchmark in previous results.
+                if (previousBenchmarks == null || name !in previousBenchmarks) {
+                    getOrPut(name) { SummaryBenchmark(current, null) }
+                } else {
+                    val previousBenchmark = previousBenchmarks[name]!!.meanBenchmark
+                    getOrPut(name) { SummaryBenchmark(current, previousBenchmarks[name]) }
+                    // Explore change of status.
+                    if (previousBenchmark.status != currentBenchmark.status) {
+                        val statusChange = FieldChange("$name", previousBenchmark.status, currentBenchmark.status)
+                        benchmarksWithChangedStatus.add(statusChange)
+                    }
+                }
+            }
+        }
+
+        // Add removed benchmarks to merged report.
+        mergedTable.apply {
+            previousBenchmarks?.filter { (key, _) -> key !in currentBenchmarks }?.forEach { (key, value) ->
+                getOrPut(key) { SummaryBenchmark(null, value) }
+            }
+        }
+
+        return mergedTable
+    }
+
+    // Calculate geometric mean.
+    private fun calculateGeoMeanBenchmark(currentBenchmarks: BenchmarksTable, previousBenchmarks: BenchmarksTable?):
+            SummaryBenchmark {
+        // Calculate geometric mean.
+        val currentGeoMean = createGeoMeanBenchmark(currentBenchmarks)
+        val previousGeoMean = previousBenchmarks?. let { createGeoMeanBenchmark(previousBenchmarks) }
+        return SummaryBenchmark(currentGeoMean, previousGeoMean)
+    }
+
+    private fun getBenchmarkPerfomanceChange(name: String, benchmark: SummaryBenchmark): Pair<String, ScoreChange>? {
+        val (current, previous) = benchmark
+        current?.let {
+            previous?.let {
+                // Calculate metrics for showing difference.
+                val percent = current.calcPercentageDiff(previous)
+                val ratio = current.calcRatio(previous)
+                if (abs(percent.mean) >= meaningfulChangesValue) {
+                    return Pair(name, Pair(percent, ratio))
+                }
+            }
+        }
+        return null
+    }
+
+    // Analyze and collect changes in performance between same becnhmarks.
+    private fun analyzePerformanceChanges() {
+        val performanceChanges = mergedReport.asSequence().map {(name, element) ->
+            getBenchmarkPerfomanceChange(name, element)
+        }.filterNotNull().groupBy {
+            if (it.second.first.mean > 0) "regressions" else "improvements"
+        }
+
+        // Sort regressions and improvements.
+        regressions = performanceChanges["regressions"]
+                ?.sortedByDescending { it.second.first.mean }?.map { it.first to it.second }
+                ?.toMap() ?: mapOf<String, ScoreChange>()
+        improvements = performanceChanges["improvements"]
+                ?.sortedBy { it.second.first.mean }?.map { it.first to it.second }
+                ?.toMap() ?: mapOf<String, ScoreChange>()
+
+        // Calculate change for geometric mean.
+        val (current, previous) = geoMeanBenchmark
+        geoMeanScoreChange = current?. let {
+            previous?. let {
+                Pair(current.calcPercentageDiff(previous), current.calcRatio(previous))
+            }
+        }
+    }
+
+    private fun <T> MutableList<FieldChange<T>>.addFieldChange(field: String, previous: T, current: T) {
+        FieldChange.getFieldChangeOrNull(field, previous, current)?.let {
+            add(it)
+        }
+    }
+
+    private fun analyzeEnvChanges(currentEnv: Environment, previousEnv: Environment): List<FieldChange<String>> {
+        return mutableListOf<FieldChange<String>>().apply {
+            addFieldChange("Machine CPU", previousEnv.machine.cpu, currentEnv.machine.cpu)
+            addFieldChange("Machine OS", previousEnv.machine.os, currentEnv.machine.os)
+            addFieldChange("JDK version", previousEnv.jdk.version, currentEnv.jdk.version)
+            addFieldChange("JDK vendor", previousEnv.jdk.vendor, currentEnv.jdk.vendor)
+        }
+    }
+
+    private fun analyzeKotlinChanges(currentCompiler: Compiler, previousCompiler: Compiler): List<FieldChange<String>> {
+        return mutableListOf<FieldChange<String>>().apply {
+            addFieldChange("Backend type", previousCompiler.backend.type.type, currentCompiler.backend.type.type)
+            addFieldChange("Backend version", previousCompiler.backend.version, currentCompiler.backend.version)
+            addFieldChange("Backend flags", previousCompiler.backend.flags.toString(),
+                    currentCompiler.backend.flags.toString())
+            addFieldChange( "Kotlin version", previousCompiler.kotlinVersion, currentCompiler.kotlinVersion)
+        }
+    }
+}

tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Utils.kt

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.analyzer
+
+expect fun readFile(fileName: String): String
+expect fun format(number: Double, decimalNumber: Int = 4): String
+expect fun writeToFile(fileName: String, text: String)
+expect fun assert(value: Boolean, lazyMessage: () -> Any)
+expect fun getEnv(variableName:String): String?

tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/kliopt/KliOpt.kt

@@ -0,0 +1,224 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.jetbrains.kliopt
+
+// Possible types of arguments.
+enum class ArgType(val hasParameter: Boolean) {
+    BOOLEAN(false),
+    STRING(true),
+    INT(true),
+    DOUBLE(true)
+}
+
+// Common descriptor both for options and positional arguments.
+abstract class Descriptor(val type: ArgType,
+                          val longName: String,
+                          val description: String? = null,
+                          val defaultValue: String? = null,
+                          val isRequired: Boolean = false) {
+    abstract val textDescription: String
+    abstract val helpMessage: String
+}
+
+class OptionDescriptor(
+        type: ArgType,
+        longName: String,
+        val shortName: String ? = null,
+        description: String? = null,
+        defaultValue: String? = null,
+        isRequired: Boolean = false) : Descriptor (type, longName, description, defaultValue, isRequired) {
+    override val textDescription: String
+        get() = "option -$longName"
+
+    override val helpMessage: String
+        get() {
+            val result = StringBuilder()
+            result.append("    -${longName}")
+            shortName?.let { result.append(", -$it") }
+            defaultValue?.let { result.append(" [$it]") }
+            description?.let {result.append(" -> ${it}")}
+            if (!isRequired) result.append(" (optional)")
+            result.append("\n")
+            return result.toString()
+        }
+}
+
+class ArgDescriptor(
+        type: ArgType,
+        longName: String,
+        description: String? = null,
+        defaultValue: String? = null,
+        isRequired: Boolean = true) : Descriptor (type, longName, description, defaultValue, isRequired) {
+    override val textDescription: String
+        get() = "argument $longName"
+
+    override val helpMessage: String
+        get() {
+            val result = StringBuilder()
+            result.append("    ${longName}")
+            defaultValue?.let { result.append(" [$it]") }
+            description?.let {result.append(" -> ${it}")}
+            if (!isRequired) result.append(" (optional)")
+            result.append("\n")
+            return result.toString()
+        }
+}
+
+// Arguments parser.
+class ArgParser(optionsList: List<OptionDescriptor>, argsList: List<ArgDescriptor> = listOf<ArgDescriptor>()) {
+    private val options = optionsList.union(listOf(OptionDescriptor(ArgType.BOOLEAN, "help",
+                                            "h", "Usage info")))
+                            .toList()
+    private val arguments = argsList
+    private lateinit var parsedValues: MutableMap<String, ParsedArg?>
+
+    inner class ParsedArg(val descriptor: Descriptor, val value: String) {
+        val intValue: Int
+            get() {
+                if (descriptor.type != ArgType.INT)
+                    error("Incorrect value for ${descriptor.textDescription}, must be an int")
+                return value.toInt()
+            }
+
+        val stringValue: String
+            get() {
+                if (descriptor.type != ArgType.STRING)
+                    printError("Incorrect value for ${descriptor.textDescription}, must be a string")
+                return value
+            }
+
+        val booleanValue: Boolean
+            get() {
+                if (descriptor.type != ArgType.BOOLEAN)
+                    printError("Incorrect value for ${descriptor.textDescription}, must be a boolean")
+                return value == "true"
+            }
+
+        val doubleValue: Double
+            get() {
+                if (descriptor.type != ArgType.DOUBLE)
+                    printError("Incorrect option value for ${descriptor.textDescription}, must be a double")
+                return value.toDouble()
+            }
+    }
+
+    // Output error. Also adds help usage information for easy understanding of problem.
+    private fun printError(message: String) {
+        error("$message\n${makeUsage()}")
+    }
+
+    private fun saveAsArg(argDescriptors: Map<String, ArgDescriptor>, arg: String): Boolean {
+        // Find uninitialized arguments.
+        val nullArgs = argDescriptors.keys.filter { parsedValues[it] == null }
+        val name = nullArgs.firstOrNull()
+        name?. let {
+            parsedValues[name] = ParsedArg(argDescriptors[name]!!, arg)
+            return true
+        }
+        return false
+    }
+
+    // Parse arguments.
+    // Returns true if all arguments were parsed, otherwise return false and print help message.
+    fun parse(args: Array<String>): Boolean {
+        var index = 0
+        val optDescriptors = options.map { it.longName to it }.toMap()
+        val shortNames = options.filter { it.shortName != null }.map { it.shortName!! to it.longName }.toMap()
+        val argDescriptors = arguments.map { it.longName to it }.toMap()
+        parsedValues = optDescriptors.keys.union(argDescriptors.keys).toList().map { it to null }.toMap().toMutableMap()
+        while (index < args.size) {
+            val arg = args[index]
+            if (arg.startsWith('-')) {
+                // Option is found.
+                val name = shortNames[arg.substring(1)] ?: arg.substring(1)
+                val descriptor = optDescriptors[name]
+                descriptor?. let {
+                    if (descriptor.type.hasParameter) {
+                        if (index < args.size - 1) {
+                            parsedValues[name] = ParsedArg(descriptor, args[index + 1])
+                            index++
+                        } else {
+                            // An error, option with value without value.
+                            printError("No value for ${descriptor.textDescription}")
+                        }
+                    } else {
+                        if (name == "help") {
+                            println(makeUsage())
+                            return false
+                        }
+                        parsedValues[name] = ParsedArg(descriptor, "true")
+                    }
+                } ?: run {
+                    // Try save as argument.
+                    if (!saveAsArg(argDescriptors, arg)) {
+                        printError("Unknown option $arg")
+                    }
+                }
+            } else {
+                // Argument is found.
+                if (!saveAsArg(argDescriptors, arg)) {
+                    printError("Too many arguments!")
+                }
+            }
+            index++
+        }
+
+        parsedValues.forEach { (key, value) ->
+            val descriptor = optDescriptors[key] ?: argDescriptors[key]!!
+            // Not inited, append default value if needed.
+            parsedValues[key] = value ?: run {
+                descriptor.defaultValue?. let {
+                    ParsedArg(descriptor, descriptor.defaultValue)
+                }
+            }
+            // Check if arg is always required.
+            parsedValues[key] ?: run {
+                if (descriptor.isRequired) {
+                    printError("Please, provide value for ${descriptor.textDescription}. It should be always set")
+                }
+            }
+        }
+
+        return true
+    }
+
+    fun get(name: String): ParsedArg? {
+        if (::parsedValues.isInitialized) {
+            return parsedValues[name]
+        } else {
+            println("Method parse() of ArgParser class should be called before getting arguments and options.")
+            return null
+        }
+    }
+
+    private fun makeUsage(): String {
+        val result = StringBuilder()
+        result.append("Usage: \n")
+        if (!arguments.isEmpty()) {
+            result.append("Arguments: \n")
+            arguments.forEach {
+                result.append(it.helpMessage)
+            }
+        }
+        result.append("Options: \n")
+        options.forEach {
+            result.append(it.helpMessage)
+        }
+        return result.toString()
+    }
+}

tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt

@@ -0,0 +1,213 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.jetbrains.renders
+
+import org.jetbrains.analyzer.*
+import org.jetbrains.report.BenchmarkResult
+
+import kotlin.math.abs
+
+// Common interface for printing report in different formats.
+interface Render {
+    fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean = false): String
+    // Print report using render.
+    fun print(report: SummaryBenchmarksReport, onlyChanges: Boolean = false, outputFile: String? = null) {
+        val content = render(report, onlyChanges)
+        outputFile?.let {
+            writeToFile(outputFile, content)
+        } ?: println(content)
+    }
+}
+
+// Report render to text format.
+class TextRender: Render {
+    private val content = StringBuilder()
+    private val headerSeparator = "================="
+    private val wideColumnWidth = 50
+    private val standardColumnWidth = 25
+
+    private fun append(text: String = "") {
+        content.append("$text\n")
+    }
+
+    override fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean): String {
+        renderEnvChanges(report.envChanges, "Environment")
+        renderEnvChanges(report.kotlinChanges, "Compiler")
+        renderStatusSummary(report)
+        renderStatusChangesDetails(report.getBenchmarksWithChangedStatus())
+        renderPerformanceSummary(report.regressions, report.improvements)
+        renderPerformanceDetails(report, onlyChanges)
+        return content.toString()
+    }
+
+    private fun printBucketInfo(bucket: Collection<Any>, name: String) {
+        if (!bucket.isEmpty()) {
+            append("$name: ${bucket.size}")
+        }
+    }
+
+    private fun <T> printStatusChangeInfo(bucket: List<FieldChange<T>>, name: String) {
+        if (!bucket.isEmpty()) {
+            append("$name:")
+            for (change in bucket) {
+                append(change.renderAsText())
+            }
+        }
+    }
+
+    fun renderEnvChanges(envChanges: List<FieldChange<String>>, bucketName: String) {
+        if (!envChanges.isEmpty()) {
+            append(ChangeReport(bucketName, envChanges).renderAsTextReport())
+        }
+    }
+
+    private fun printPerformanceBucket(bucket: Map<String, ScoreChange>, bucketName: String) {
+        if (!bucket.isEmpty()) {
+            var percentsList = bucket.values.map{ it.first.mean }
+            // Maps of regressions and improvements are sorted.
+            val maxValue = percentsList.first()
+            var geomeanValue: Double
+            if (percentsList.first() > 0.0) {
+                geomeanValue = geometricMean(percentsList)
+            } else {
+                // Geometric mean can be counted on positive numbers.
+                val precision = abs(maxValue) + 1
+                percentsList = percentsList.map{it + precision}
+                geomeanValue = geometricMean(percentsList) - precision
+            }
+
+            append("$bucketName: Maximum = ${formatValue(maxValue, true)}," +
+                    " Geometric mean = ${formatValue(geomeanValue, true)}")
+        }
+    }
+
+    fun renderStatusChangesDetails(benchmarksWithChangedStatus: List<FieldChange<BenchmarkResult.Status>>) {
+        if (!benchmarksWithChangedStatus.isEmpty()) {
+            append("Changes in status")
+            append(headerSeparator)
+            printStatusChangeInfo(benchmarksWithChangedStatus
+                    .filter { it.current == BenchmarkResult.Status.FAILED }, "New failures")
+            printStatusChangeInfo(benchmarksWithChangedStatus
+                    .filter { it.current == BenchmarkResult.Status.PASSED }, "New passes")
+            append()
+        }
+    }
+
+    fun renderStatusSummary(report: SummaryBenchmarksReport) {
+        append("Status summary")
+        append(headerSeparator)
+
+        val failedBenchmarks = report.failedBenchmarks
+        val addedBenchmarks = report.addedBenchmarks
+        val removedBenchmarks = report.removedBenchmarks
+        if (failedBenchmarks.isEmpty()) {
+            append("All benchmarks passed!")
+        }
+        if (!failedBenchmarks.isEmpty() || !addedBenchmarks.isEmpty() || !removedBenchmarks.isEmpty()) {
+            printBucketInfo(failedBenchmarks, "Failed benchmarks")
+            printBucketInfo(addedBenchmarks, "Added benchmarks")
+            printBucketInfo(removedBenchmarks, "Removed benchmarks")
+        }
+        append("Total becnhmarks number: ${report.benchmarksNumber}")
+        append()
+    }
+
+    fun renderPerformanceSummary(regressions: Map<String, ScoreChange>,
+                                          improvements: Map<String, ScoreChange>) {
+        if (!regressions.isEmpty() || !improvements.isEmpty()) {
+            append("Performance summary")
+            append(headerSeparator)
+            printPerformanceBucket(regressions, "Regressions")
+            printPerformanceBucket(improvements, "Improvements")
+            append()
+        }
+    }
+
+    private fun formatValue(number: Double, isPercent: Boolean = false): String =
+            if (isPercent) format(number, 2) + "%" else format(number)
+
+    private fun formatColumn(content:String, isWide: Boolean = false): String =
+            content.padEnd(if (isWide) wideColumnWidth else standardColumnWidth, ' ')
+
+    private fun printBenchmarksDetails(fullSet: Map<String, SummaryBenchmark>,
+                                       bucket: Map<String, ScoreChange>? = null) {
+        if (bucket != null) {
+            // There are changes in performance.
+            // Output changed benchmarks.
+            for ((name, change) in bucket) {
+                append(formatColumn(name, true) +
+                        formatColumn(fullSet[name]!!.first!!.toString()) +
+                        formatColumn(fullSet[name]!!.second!!.toString()) +
+                        formatColumn(change.first.toString() + " %") +
+                        formatColumn(change.second.toString()))
+            }
+        } else {
+            // Output all values without performance changes.
+            val placeholder = "-"
+            for ((name, value) in fullSet) {
+                append(formatColumn(name, true) +
+                        formatColumn(value.first?.toString() ?: placeholder) +
+                        formatColumn(value.second?.toString() ?: placeholder) +
+                        formatColumn(placeholder) +
+                        formatColumn(placeholder))
+            }
+        }
+    }
+
+    private fun printTableLineSeparator(tableWidth: Int) =
+            append("${"-".padEnd(tableWidth, '-')}")
+
+    private fun printPerformanceTableHeader(): Int {
+        val wideColumns = listOf(formatColumn("Benchmark", true))
+        val standardColumns = listOf(formatColumn("Current score"),
+                formatColumn("Previous score"),
+                formatColumn("Percent"),
+                formatColumn("Ratio"))
+        val tableWidth = wideColumnWidth * wideColumns.size + standardColumnWidth * standardColumns.size
+        append("${wideColumns.joinToString(separator = "")}${standardColumns.joinToString(separator = "")}")
+        printTableLineSeparator(tableWidth)
+        return tableWidth
+    }
+
+    fun renderPerformanceDetails(report: SummaryBenchmarksReport, onlyChanges: Boolean = false) {
+        append("Performance details")
+        append(headerSeparator)
+
+        if (onlyChanges) {
+            if (report.regressions.isEmpty() && report.improvements.isEmpty()) {
+                append("All becnhmarks are stable.")
+            }
+        }
+
+        val tableWidth = printPerformanceTableHeader()
+        // Print geometric mean.
+        val geoMeanChangeMap = report.geoMeanScoreChange?.
+                let { mapOf(report.geoMeanBenchmark.first!!.meanBenchmark.name to report.geoMeanScoreChange!!) }
+        printBenchmarksDetails(
+                mutableMapOf(report.geoMeanBenchmark.first!!.meanBenchmark.name to report.geoMeanBenchmark),
+                geoMeanChangeMap)
+        printTableLineSeparator(tableWidth)
+        printBenchmarksDetails(report.mergedReport, report.regressions)
+        printBenchmarksDetails(report.mergedReport, report.improvements)
+        if (!onlyChanges) {
+            // Print all remaining results.
+            printBenchmarksDetails(report.mergedReport.filter { it.key !in report.regressions.keys &&
+                    it.key !in report.improvements.keys })
+        }
+    }
+}

tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/TeamCityStatisticsRender.kt

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.jetbrains.renders
+
+import org.jetbrains.analyzer.*
+import org.jetbrains.report.BenchmarkResult
+
+// Report render to text format.
+class TeamCityStatisticsRender: Render {
+    private var content = StringBuilder()
+
+    override fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean): String {
+        // For current benchmarks print score as TeamCity Test Metadata
+        report.currentMeanVarianceBenchmarks.forEach { benchmark ->
+            content.append(renderBenchmark(benchmark.meanBenchmark, "Mean"))
+            content.append(renderBenchmark(benchmark.varianceBenchmark, "Variance"))
+        }
+        // Report geometric mean as build statistic value
+        content.append(renderGeometricMean(report.geoMeanBenchmark.first!!))
+        return content.toString()
+    }
+
+    private fun renderBenchmark(benchmark: BenchmarkResult, metric: String): String =
+        "##teamcity[testMetadata testName='${benchmark.name}' name='$metric' type='number' value='${benchmark.score}']\n"
+
+    private fun renderGeometricMean(geoMeanBenchmark: MeanVarianceBenchmark): String =
+        "##teamcity[buildStatisticValue key='Geometric mean' value='${geoMeanBenchmark.meanBenchmark.score}']\n" +
+                "##teamcity[buildStatisticValue key='Geometric mean variance' value='${geoMeanBenchmark.varianceBenchmark.score}']\n"
+}

tools/benchmarksAnalyzer/src/tests/AnalyzerTests.kt

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.jetbrains.analyzer
+
+import kotlin.test.*
+import kotlin.math.abs
+import org.jetbrains.report.BenchmarkResult
+
+class AnalyzerTests {
+    private val eps = 0.000001
+
+    private fun createMeanVarianceBenchmarks(): Pair<MeanVarianceBenchmark, MeanVarianceBenchmark> {
+        val firstMean = BenchmarkResult("testBenchmark", BenchmarkResult.Status.PASSED, 9.0, 9.0, 10, 10)
+        val firstVariance = BenchmarkResult("testBenchmark", BenchmarkResult.Status.PASSED, 0.0001, 0.0001, 10, 10)
+        val first = MeanVarianceBenchmark(firstMean, firstVariance)
+
+        val secondMean = BenchmarkResult("testBenchmark", BenchmarkResult.Status.PASSED, 10.0, 10.0, 10, 10)
+        val secondVariance = BenchmarkResult("testBenchmark", BenchmarkResult.Status.PASSED, 0.0001, 0.0001, 10, 10)
+        val second = MeanVarianceBenchmark(secondMean, secondVariance)
+
+        return Pair(first, second)
+    }
+
+    @Test
+    fun testGeoMean() {
+        val numbers = listOf(4.0, 6.0, 9.0)
+        val value = geometricMean(numbers)
+        val expected = 6.0
+        assertTrue(abs(value - expected) < eps)
+    }
+
+    @Test
+    fun testComputeMeanVariance() {
+        val numbers = listOf(10.1, 10.2, 10.3)
+        val value = computeMeanVariance(numbers)
+        val expectedMean = 10.2
+        val expectedVariance = 0.092395
+        assertTrue(abs(value.mean - expectedMean) < eps)
+        assertTrue(abs(value.variance - expectedVariance) < eps)
+    }
+
+    @Test
+    fun calcPercentageDiff() {
+        val inputs = createMeanVarianceBenchmarks()
+
+        val percent = inputs.first.calcPercentageDiff(inputs.second)
+        val expectedMean = -10.0
+        val expectedVariance = 0.0021
+        assertTrue(abs(percent.mean - expectedMean) < eps)
+        assertTrue(abs(percent.variance - expectedVariance) < eps)
+    }
+
+    @Test
+    fun calcRatio() {
+        val inputs = createMeanVarianceBenchmarks()
+
+        val ratio = inputs.first.calcRatio(inputs.second)
+        val expectedMean = 0.9
+        val expectedVariance = 0.00001899
+
+        assertTrue(abs(ratio.mean - expectedMean) < eps)
+        assertTrue(abs(ratio.variance - expectedVariance) < eps)
+    }
+}