Support for stable and unstable benchmarks. Refactoring to get separate tables for metrics. (#4641)

kotlin-native/tools/benchmarks/shared/src/main/kotlin/analyzer/Statistics.kt

@@ -25,10 +25,13 @@ val MeanVarianceBenchmark.description: String
 
 // Calculate difference in percentage compare to another.
 fun MeanVarianceBenchmark.calcPercentageDiff(other: MeanVarianceBenchmark): MeanVariance {
+    if (score == 0.0 && variance == 0.0 && other.score == 0.0 && other.variance == 0.0)
+        return MeanVariance(score, variance)
     assert(other.score >= 0 &&
             other.variance >= 0 &&
-            other.score - other.variance != 0.0,
+            (other.score - other.variance != 0.0 || other.score == 0.0),
             { "Mean and variance should be positive and not equal!" })
+
     // Analyze intervals. Calculate difference between border points.
     val (bigValue, smallValue) = if (score > other.score) Pair(this, other) else Pair(other, this)
     val bigValueIntervalStart = bigValue.score - bigValue.variance
@@ -50,11 +53,13 @@ fun MeanVarianceBenchmark.calcPercentageDiff(other: MeanVarianceBenchmark): Mean
 
 // Calculate ratio value compare to another.
 fun MeanVarianceBenchmark.calcRatio(other: MeanVarianceBenchmark): MeanVariance {
+    if (other.score == 0.0 && other.variance == 0.0)
+        return MeanVariance(1.0, 0.0)
     assert(other.score >= 0 &&
             other.variance >= 0 &&
-            other.score - other.variance != 0.0,
+            (other.score - other.variance != 0.0 || other.score == 0.0),
             { "Mean and variance should be positive and not equal!" })
-    val mean = score / other.score
+    val mean = if (other.score != 0.0) (score / other.score) else 0.0
     val minRatio = (score - variance) / (other.score + other.variance)
     val maxRatio = (score + variance) / (other.score - other.variance)
     val ratioConfInt = min(abs(minRatio - mean), abs(maxRatio - mean))
@@ -63,7 +68,7 @@ fun MeanVarianceBenchmark.calcRatio(other: MeanVarianceBenchmark): MeanVariance
 
 fun geometricMean(values: Collection<Double>, totalNumber: Int = values.size) =
         with(values.asSequence().filter { it != 0.0 }) {
-            if (count() == 0) {
+            if (count() == 0 || totalNumber == 0) {
                 0.0
             } else {
                 map { it.pow(1.0 / totalNumber) }.reduce { a, b -> a * b }

kotlin-native/tools/benchmarks/shared/src/main/kotlin/analyzer/SummaryBenchmarksReport.kt

@@ -18,13 +18,11 @@ typealias BenchmarksTable = Map<String, MeanVarianceBenchmark>
 typealias SummaryBenchmarksTable = Map<String, SummaryBenchmark>
 typealias ScoreChange = Pair<MeanVariance, MeanVariance>
 
-// Summary report with comparasion of separate benchmarks results.
+class DetailedBenchmarksReport(currentBenchmarks: Map<String, List<BenchmarkResult>>,
-class SummaryBenchmarksReport(val currentReport: BenchmarksReport,
+                               previousBenchmarks: Map<String, List<BenchmarkResult>>? = null,
-                              val previousReport: BenchmarksReport? = null,
+                               val meaningfulChangesValue: Double = 0.5) {
-                              val meaningfulChangesValue: Double = 0.5) {
     // Report created by joining comparing reports.
     val mergedReport: Map<String, SummaryBenchmark>
-    private val benchmarksDurations: Map<String, Pair<Double?, Double?>>
 
     // Lists of benchmarks in different status.
     private val benchmarksWithChangedStatus = mutableListOf<FieldChange<BenchmarkResult.Status>>()
@@ -40,30 +38,6 @@ class SummaryBenchmarksReport(val currentReport: BenchmarksReport,
     var geoMeanScoreChange: ScoreChange? = null
         private set
 
-    // Environment and tools.
-    val environments: Pair<Environment, Environment?>
-    val compilers: Pair<Compiler, Compiler?>
-
-    // Countable properties.
-    val failedBenchmarks: List<String>
-        get() = mergedReport.filter { it.value.first?.status == BenchmarkResult.Status.FAILED }
-                .map { it.key }
-
-    val addedBenchmarks: List<String>
-        get() = mergedReport.filter { it.value.second == null }.map { it.key }
-
-    val removedBenchmarks: List<String>
-        get() = mergedReport.filter { it.value.first == null }.map { it.key }
-
-    val benchmarksNumber: Int
-        get() = mergedReport.keys.size
-
-    val currentMeanVarianceBenchmarks: List<MeanVarianceBenchmark>
-        get() = mergedReport.filter { it.value.first != null }.map { it.value.first!! }
-
-    val currentBenchmarksDuration: Map<String, Double>
-        get() = benchmarksDurations.filter { it.value.first != null }.map { it.key to it.value.first!! }.toMap()
-
     val maximumRegression: Double
         get() = getMaximumChange(regressions)
 
@@ -76,90 +50,53 @@ class SummaryBenchmarksReport(val currentReport: BenchmarksReport,
     val improvementsGeometricMean: Double
         get() = getGeometricMeanOfChanges(improvements)
 
-    val envChanges: List<FieldChange<String>>
+    val benchmarksNumber: Int
-        get() {
+        get() = mergedReport.keys.size
-            val previousEnvironment = environments.second
-            val currentEnvironment = environments.first
-            return previousEnvironment?.let {
-                mutableListOf<FieldChange<String>>().apply {
-                    addFieldChange("Machine CPU", previousEnvironment.machine.cpu, currentEnvironment.machine.cpu)
-                    addFieldChange("Machine OS", previousEnvironment.machine.os, currentEnvironment.machine.os)
-                    addFieldChange("JDK version", previousEnvironment.jdk.version, currentEnvironment.jdk.version)
-                    addFieldChange("JDK vendor", previousEnvironment.jdk.vendor, currentEnvironment.jdk.vendor)
-                }
-            } ?: listOf<FieldChange<String>>()
-        }
-
-    val kotlinChanges: List<FieldChange<String>>
-        get() {
-            val previousCompiler = compilers.second
-            val currentCompiler = compilers.first
-            return previousCompiler?.let {
-                mutableListOf<FieldChange<String>>().apply {
-                    addFieldChange("Backend type", previousCompiler.backend.type.type, currentCompiler.backend.type.type)
-                    addFieldChange("Backend version", previousCompiler.backend.version, currentCompiler.backend.version)
-                    addFieldChange("Backend flags", previousCompiler.backend.flags.toString(),
-                            currentCompiler.backend.flags.toString())
-                    addFieldChange("Kotlin version", previousCompiler.kotlinVersion, currentCompiler.kotlinVersion)
-                }
-            } ?: listOf<FieldChange<String>>()
-        }
 
     init {
         // Count avarage values for each benchmark.
-        val currentBenchmarksTable = collectMeanResults(currentReport.benchmarks)
+        val currentBenchmarksTable = collectMeanResults(currentBenchmarks)
-        val previousBenchmarksTable = previousReport?.let {
+        val previousBenchmarksTable = previousBenchmarks?.let {
-            collectMeanResults(previousReport.benchmarks)
+            collectMeanResults(previousBenchmarks)
         }
         mergedReport = createMergedReport(currentBenchmarksTable, previousBenchmarksTable)
-        benchmarksDurations = calculateBenchmarksDuration(currentReport, previousReport)
         geoMeanBenchmark = calculateGeoMeanBenchmark(currentBenchmarksTable, previousBenchmarksTable)
-        environments = Pair(currentReport.env, previousReport?.env)
-        compilers = Pair(currentReport.compiler, previousReport?.compiler)
 
-        if (previousReport != null) {
+        if (previousBenchmarks != null) {
             // Check changes in environment and tools.
             analyzePerformanceChanges()
         }
     }
 
-    // Get benchmark report.
-    fun getBenchmarksReport(takeMainReport: Boolean = true) =
-            if (takeMainReport)
-                BenchmarksReport(environments.first, mergedReport.map { (_, value) -> value.first!! }, compilers.first)
-            else
-                BenchmarksReport(environments.second!!, mergedReport.map { (_, value) -> value.second!! }, compilers.second!!)
-
-    fun getResultsByMetric(metric: BenchmarkResult.Metric, getGeoMean: Boolean = true, filter: List<String>? = null,
-                           normalizeData: Map<String, Map<String, Double>>? = null): List<Double?> {
-        val benchmarks = filter?.let {
-            mergedReport.filter { entry ->
-                filter.find {
-                    entry.key.startsWith(it)
-                } != null
-            }
-        } ?: mergedReport
-        val results = benchmarks.map { entry ->
-            val name = entry.key.removeSuffix(metric.suffix)
-            if (entry.value.first!!.metric == metric) {
-                val score = entry.value.first!!.score
-                val value = normalizeData?.let {
-                    it.get(name)?.get("$metric")?.let { score / it }
-                            ?: error("No normalization data for benchmark $name and metric $metric")
-                } ?: score
-                name to value
-            } else name to null
-        }.toMap()
-        if (getGeoMean) {
-            return listOf(geometricMean(results.values.filterNotNull()))
-        }
-        return filter?.let { it.map { results[it] }.toList() } ?: results.values.toList()
-    }
-
     private fun getMaximumChange(bucket: Map<String, ScoreChange>): Double =
             // Maps of regressions and improvements are sorted.
             if (bucket.isEmpty()) 0.0 else bucket.values.map { it.first.mean }.first()
 
+    // Analyze and collect changes in performance between same becnhmarks.
+    private fun analyzePerformanceChanges() {
+        val performanceChanges = mergedReport.asSequence().map { (name, element) ->
+            getBenchmarkPerfomanceChange(name, element)
+        }.filterNotNull().groupBy {
+            if (it.second.first.mean > 0) "regressions" else "improvements"
+        }
+
+        // Sort regressions and improvements.
+        regressions = performanceChanges["regressions"]
+                ?.sortedByDescending { it.second.first.mean }?.map { it.first to it.second }
+                ?.toMap() ?: mapOf<String, ScoreChange>()
+        improvements = performanceChanges["improvements"]
+                ?.sortedBy { it.second.first.mean }?.map { it.first to it.second }
+                ?.toMap() ?: mapOf<String, ScoreChange>()
+
+        // Calculate change for geometric mean.
+        val (current, previous) = geoMeanBenchmark
+        geoMeanScoreChange = current?.let {
+            previous?.let {
+                Pair(current.calcPercentageDiff(previous), current.calcRatio(previous))
+            }
+        }
+    }
+
     private fun getGeometricMeanOfChanges(bucket: Map<String, ScoreChange>): Double {
         if (bucket.isEmpty())
             return 0.0
@@ -175,25 +112,6 @@ class SummaryBenchmarksReport(val currentReport: BenchmarksReport,
 
     fun getBenchmarksWithChangedStatus(): List<FieldChange<BenchmarkResult.Status>> = benchmarksWithChangedStatus
 
-    // Create geometric mean.
-    private fun createGeoMeanBenchmark(benchTable: BenchmarksTable): MeanVarianceBenchmark {
-        val geoMeanBenchmarkName = "Geometric mean"
-        val geoMean = geometricMean(benchTable.toList().map { (_, value) -> value.score })
-        val varianceGeoMean = geometricMean(benchTable.toList().map { (_, value) -> value.variance })
-        return MeanVarianceBenchmark(geoMeanBenchmarkName, geoMean, varianceGeoMean)
-    }
-
-    // Generate map with summary durations of each benchmark.
-    private fun calculateBenchmarksDuration(currentReport: BenchmarksReport, previousReport: BenchmarksReport?):
-            Map<String, Pair<Double?, Double?>> {
-        val currentDurations = collectBenchmarksDurations(currentReport.benchmarks)
-        val previousDurations = previousReport?.let {
-            collectBenchmarksDurations(previousReport.benchmarks)
-        } ?: mapOf<String, Double>()
-        return currentDurations.keys.union(previousDurations.keys)
-                .map { it to Pair(currentDurations[it], previousDurations[it]) }.toMap()
-    }
-
     // Merge current and compare to report.
     private fun createMergedReport(currentBenchmarks: BenchmarksTable, previousBenchmarks: BenchmarksTable?):
             Map<String, SummaryBenchmark> {
@@ -249,29 +167,144 @@ class SummaryBenchmarksReport(val currentReport: BenchmarksReport,
         return null
     }
 
-    // Analyze and collect changes in performance between same becnhmarks.
+    // Create geometric mean.
-    private fun analyzePerformanceChanges() {
+    private fun createGeoMeanBenchmark(benchTable: BenchmarksTable): MeanVarianceBenchmark {
-        val performanceChanges = mergedReport.asSequence().map { (name, element) ->
+        val geoMeanBenchmarkName = "Geometric mean"
-            getBenchmarkPerfomanceChange(name, element)
+        val geoMean = geometricMean(benchTable.toList().map { (_, value) -> value.score })
-        }.filterNotNull().groupBy {
+        val varianceGeoMean = geometricMean(benchTable.toList().map { (_, value) -> value.variance })
-            if (it.second.first.mean > 0) "regressions" else "improvements"
+        return MeanVarianceBenchmark(geoMeanBenchmarkName, geoMean, varianceGeoMean)
+    }
+}
+
+// Summary report with comparasion of separate benchmarks results.
+class SummaryBenchmarksReport(val currentReport: BenchmarksReport,
+                              val previousReport: BenchmarksReport? = null,
+                              val meaningfulChangesValue: Double = 0.5,
+                              private val unstableBenchmarks: List<String> = emptyList()) {
+
+    val detailedMetricReports: Map<BenchmarkResult.Metric, DetailedBenchmarksReport>
+
+    private val benchmarksDurations: Map<String, Pair<Double?, Double?>>
+
+    // Lists of benchmarks in different status.
+    val benchmarksWithChangedStatus
+        get() = getReducedResult { report ->
+            report.getBenchmarksWithChangedStatus()
         }
 
-        // Sort regressions and improvements.
+    // Environment and tools.
-        regressions = performanceChanges["regressions"]
+    val environments: Pair<Environment, Environment?>
-                ?.sortedByDescending { it.second.first.mean }?.map { it.first to it.second }
+    val compilers: Pair<Compiler, Compiler?>
-                ?.toMap() ?: mapOf<String, ScoreChange>()
-        improvements = performanceChanges["improvements"]
-                ?.sortedBy { it.second.first.mean }?.map { it.first to it.second }
-                ?.toMap() ?: mapOf<String, ScoreChange>()
 
-        // Calculate change for geometric mean.
+    private fun <T> getReducedResult(convertor: (DetailedBenchmarksReport) -> List<T>): List<T> {
-        val (current, previous) = geoMeanBenchmark
+        return detailedMetricReports.values.map {
-        geoMeanScoreChange = current?.let {
+            convertor(it)
-            previous?.let {
+        }.flatten()
-                Pair(current.calcPercentageDiff(previous), current.calcRatio(previous))
+    }
-            }
+
+    // Countable properties.
+    val failedBenchmarks: List<String>
+        get() = getReducedResult { report ->
+            report.mergedReport.filter { it.value.first?.status == BenchmarkResult.Status.FAILED }.map { it.key }
         }
+
+    val addedBenchmarks: List<String>
+        get() = getReducedResult { report ->
+            report.mergedReport.filter { it.value.second == null }.map { it.key }
+        }
+
+    val removedBenchmarks: List<String>
+        get() = getReducedResult { report ->
+            report.mergedReport.filter { it.value.first == null }.map { it.key }
+        }
+
+    val currentMeanVarianceBenchmarks: List<MeanVarianceBenchmark>
+        get() = getReducedResult { report ->
+            report.mergedReport.filter { it.value.first != null }.map { it.value.first!! }
+        }
+
+    val benchmarksNumber: Int
+        get() = detailedMetricReports.values.fold(0) { acc, it -> acc + it.benchmarksNumber }
+
+    val currentBenchmarksDuration: Map<String, Double>
+        get() = benchmarksDurations.filter { it.value.first != null }.map { it.key to it.value.first!! }.toMap()
+
+    val envChanges: List<FieldChange<String>>
+        get() {
+            val previousEnvironment = environments.second
+            val currentEnvironment = environments.first
+            return previousEnvironment?.let {
+                mutableListOf<FieldChange<String>>().apply {
+                    addFieldChange("Machine CPU", previousEnvironment.machine.cpu, currentEnvironment.machine.cpu)
+                    addFieldChange("Machine OS", previousEnvironment.machine.os, currentEnvironment.machine.os)
+                    addFieldChange("JDK version", previousEnvironment.jdk.version, currentEnvironment.jdk.version)
+                    addFieldChange("JDK vendor", previousEnvironment.jdk.vendor, currentEnvironment.jdk.vendor)
+                }
+            } ?: listOf<FieldChange<String>>()
+        }
+
+    val kotlinChanges: List<FieldChange<String>>
+        get() {
+            val previousCompiler = compilers.second
+            val currentCompiler = compilers.first
+            return previousCompiler?.let {
+                mutableListOf<FieldChange<String>>().apply {
+                    addFieldChange("Backend type", previousCompiler.backend.type.type, currentCompiler.backend.type.type)
+                    addFieldChange("Backend version", previousCompiler.backend.version, currentCompiler.backend.version)
+                    addFieldChange("Backend flags", previousCompiler.backend.flags.toString(),
+                            currentCompiler.backend.flags.toString())
+                    addFieldChange("Kotlin version", previousCompiler.kotlinVersion, currentCompiler.kotlinVersion)
+                }
+            } ?: listOf<FieldChange<String>>()
+        }
+
+    init {
+        // Count avarage values for each benchmark.
+        detailedMetricReports = BenchmarkResult.Metric.values().map { metric ->
+            val currentBenchmarks = currentReport.benchmarks.map { (name, benchmarks) ->
+                name to benchmarks.filter { it.metric == metric }
+            }.filter { it.second.isNotEmpty() }.toMap()
+            val previousBenchmarks = previousReport?.benchmarks?.map { (name, benchmarks) ->
+                name to benchmarks.filter { it.metric == metric }
+            }?.filter { it.second.isNotEmpty() }?.toMap()
+            metric to DetailedBenchmarksReport(
+                    currentReport.benchmarks.map { (name, benchmarks) ->
+                        name to benchmarks.filter { it.metric == metric }
+                    }.filter { it.second.isNotEmpty() }.toMap(),
+                    previousReport?.benchmarks?.map { (name, benchmarks) ->
+                        name to benchmarks.filter { it.metric == metric }
+                    }?.filter { it.second.isNotEmpty() }?.toMap(),
+                    meaningfulChangesValue
+            )
+        }.toMap()
+        benchmarksDurations = calculateBenchmarksDuration(currentReport, previousReport)
+        environments = Pair(currentReport.env, previousReport?.env)
+        compilers = Pair(currentReport.compiler, previousReport?.compiler)
+    }
+
+    // Get benchmark report.
+    fun getBenchmarksReport(takeMainReport: Boolean = true) =
+            if (takeMainReport)
+                BenchmarksReport(environments.first, getReducedResult { report ->
+                    report.mergedReport.map { (_, value) -> value.first!! }
+                }, compilers.first)
+            else
+                BenchmarksReport(environments.second!!, getReducedResult { report ->
+                    report.mergedReport.map { (_, value) -> value.second!! }
+                }, compilers.second!!)
+
+    fun getUnstableBenchmarksForMetric(metric: BenchmarkResult.Metric) =
+            if (metric == BenchmarkResult.Metric.EXECUTION_TIME) unstableBenchmarks else emptyList()
+
+    // Generate map with summary durations of each benchmark.
+    private fun calculateBenchmarksDuration(currentReport: BenchmarksReport, previousReport: BenchmarksReport?):
+            Map<String, Pair<Double?, Double?>> {
+        val currentDurations = collectBenchmarksDurations(currentReport.benchmarks)
+        val previousDurations = previousReport?.let {
+            collectBenchmarksDurations(previousReport.benchmarks)
+        } ?: mapOf<String, Double>()
+        return currentDurations.keys.union(previousDurations.keys)
+                .map { it to Pair(currentDurations[it], previousDurations[it]) }.toMap()
     }
 
     private fun <T> MutableList<FieldChange<T>>.addFieldChange(field: String, previous: T, current: T) {

kotlin-native/tools/benchmarks/shared/src/main/kotlin/report/BenchmarksReport.kt

@@ -270,7 +270,6 @@ open class BenchmarkResult(val name: String, val status: Status,
                 val metric = if (metricElement != null && metricElement is JsonLiteral)
                     metricFromString(metricElement.unquoted()) ?: Metric.EXECUTION_TIME
                 else Metric.EXECUTION_TIME
-                name += metric.suffix
                 val statusElement = data.getRequiredField("status")
                 if (statusElement is JsonLiteral) {
                     val status = statusFromString(statusElement.unquoted())
@@ -349,3 +348,34 @@ open class MeanVarianceBenchmark(name: String, status: BenchmarkResult.Status, s
             """
     }
 }
+
+// Benchmark with set results stability state.
+open class BenchmarkWithStabilityState(name: String, status: BenchmarkResult.Status, score: Double, metric: BenchmarkResult.Metric,
+                                 runtimeInUs: Double, repeat: Int, warmup: Int, val unstable: Boolean) :
+        BenchmarkResult(name, status, score, metric, runtimeInUs, repeat, warmup) {
+
+    constructor(benchmarkResult: BenchmarkResult, unstable: Boolean) : this(benchmarkResult.name,
+            benchmarkResult.status, benchmarkResult.score, benchmarkResult.metric,
+            benchmarkResult.runtimeInUs, benchmarkResult.repeat, benchmarkResult.warmup, unstable)
+
+    override fun serializeFields(): String {
+        return """
+            ${super.serializeFields()},
+            "unstable": $unstable
+            """
+    }
+
+    companion object : EntityFromJsonFactory<BenchmarkResult> {
+        override fun create(data: JsonElement): BenchmarkWithStabilityState {
+            val parsedObject = BenchmarkResult.create(data)
+            if (data is JsonObject) {
+                val unstableElement = data.getOptionalField("unstable")
+                val unstableFlag = if (unstableElement != null && unstableElement is JsonPrimitive)
+                    unstableElement.boolean else false
+                return  BenchmarkWithStabilityState(parsedObject, unstableFlag)
+            } else {
+                error("Benchmark entity is expected to be an object. Please, check origin files.")
+            }
+        }
+    }
+}

kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/main.kt

@@ -79,6 +79,21 @@ object DBServerConnector : Connector() {
         val accessFileUrl = "$serverUrl/report/$target/$buildNumber"
         return sendGetRequest(accessFileUrl)
     }
+
+    fun getUnstableBenchmarks(): List<String>? {
+        try {
+            val unstableList = sendGetRequest("$serverUrl/unstable")
+            val data = JsonTreeParser.parse(unstableList)
+            if (data !is JsonArray) {
+                return null
+            }
+            return data.jsonArray.map {
+                (it as JsonPrimitive).content
+            }
+        } catch (e: Exception) {
+            return null
+        }
+    }
 }
 
 fun getFileContent(fileName: String, user: String? = null): String {
@@ -155,6 +170,11 @@ fun main(args: Array<String>) {
     val user by argParser.option(ArgType.String, shortName = "u", description = "User access information for authorization")
 
     argParser.parse(args)
+    // Get unstable benchmarks.
+    val unstableBenchmarks = DBServerConnector.getUnstableBenchmarks()
+
+    unstableBenchmarks ?: println("Failed to get access to server and get unstable benchmarks list!")
+
     // Read contents of file.
     val mainBenchsReport = mergeReportsWithDetailedFlags(getBenchmarkReport(mainReport, user))
 
@@ -164,8 +184,8 @@ fun main(args: Array<String>) {
 
     // Generate comparasion report.
     val summaryReport = SummaryBenchmarksReport(mainBenchsReport,
-            compareToBenchsReport,
+            compareToBenchsReport, epsValue,
-            epsValue)
+            unstableBenchmarks ?: emptyList())
 
     var outputFile = output
     renders.forEach {

kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/HTMLRender.kt

@@ -405,7 +405,7 @@ class HTMLRender: Render() {
             }
         }
 
-        val benchmarksWithChangedStatus = report.getBenchmarksWithChangedStatus()
+        val benchmarksWithChangedStatus = report.benchmarksWithChangedStatus
         val newFailures = benchmarksWithChangedStatus
                 .filter { it.current == BenchmarkResult.Status.FAILED }
         val newPasses = benchmarksWithChangedStatus
@@ -479,59 +479,86 @@ class HTMLRender: Render() {
     }
 
     private fun BodyTag.renderPerformanceSummary(report: SummaryBenchmarksReport) {
-        if (!report.improvements.isEmpty() || !report.regressions.isEmpty()) {
+        if (report.detailedMetricReports.values.any { it.improvements.isNotEmpty() } ||
+                report.detailedMetricReports.values.any { it.regressions.isNotEmpty() }) {
             h4 { +"Performance Summary" }
             table {
                 attributes["class"] = "table table-sm table-striped table-hover"
                 attributes["style"] = "width:initial;"
                 thead {
                     tr {
-                        th { +"Change" }
+                        th(rowspan = Natural(2)) { +"Change" }
-                        th { +"#" }
+                        report.detailedMetricReports.forEach { (metric, _) ->
-                        th { +"Maximum" }
+                            th(colspan = Natural(3)) { +metric.value }
-                        th { +"Geometric mean" }
-                    }
-                }
-                val maximumRegression = report.maximumRegression
-                val maximumImprovement = report.maximumImprovement
-                val regressionsGeometricMean = report.regressionsGeometricMean
-                val improvementsGeometricMean = report.improvementsGeometricMean
-                tbody {
-                    if (!report.regressions.isEmpty()) {
-                        tr {
-                            th { +"Regressions" }
-                            td { +"${report.regressions.size}" }
-                            td {
-                                attributes["bgcolor"] = ColoredCell(
-                                        maximumRegression/maxOf(maximumRegression, abs(maximumImprovement)))
-                                        .backgroundStyle
-                                +formatValue(maximumRegression, true)
-                            }
-                            td {
-                                attributes["bgcolor"] = ColoredCell(
-                                        regressionsGeometricMean/maxOf(regressionsGeometricMean,
-                                                            abs(improvementsGeometricMean)))
-                                        .backgroundStyle
-                                +formatValue(report.regressionsGeometricMean, true)
-                            }
                         }
                     }
-                    if (!report.improvements.isEmpty()) {
+                    tr {
+                        report.detailedMetricReports.forEach { _ ->
+                            th { +"#" }
+                            th { +"Maximum" }
+                            th { +"Geometric mean" }
+                        }
+                    }
+                }
+
+                tbody {
+                    tr {
+                        th { +"Regressions" }
+                        report.detailedMetricReports.values.forEach { report ->
+                            val maximumRegression = report.maximumRegression
+                            val regressionsGeometricMean = report.regressionsGeometricMean
+                            val maximumImprovement = report.maximumImprovement
+                            val improvementsGeometricMean = report.improvementsGeometricMean
+                            val maximumChange = maxOf(maximumRegression, abs(maximumImprovement))
+                            val maximumChangeGeoMean = maxOf(regressionsGeometricMean,
+                                    abs(improvementsGeometricMean))
+                            if (!report.regressions.isEmpty()) {
+                                td { +"${report.regressions.size}" }
+                                td {
+                                    attributes["bgcolor"] = ColoredCell(
+                                                (maximumRegression/maximumChange).takeIf{ maximumChange > 0.0 }
+                                            ).backgroundStyle
+                                    +formatValue(maximumRegression, true)
+                                }
+                                td {
+                                    attributes["bgcolor"] = ColoredCell(
+                                            (regressionsGeometricMean/maximumChangeGeoMean).takeIf{ maximumChangeGeoMean > 0.0 }
+                                    ).backgroundStyle
+                                    +formatValue(report.regressionsGeometricMean, true)
+                                }
+                            } else {
+                                repeat(3) { td { +"-" } }
+                            }
+                        }
+
                         tr {
                             th { +"Improvements" }
-                            td { +"${report.improvements.size}" }
+                            report.detailedMetricReports.values.forEach { report ->
-                            td {
+                                val maximumRegression = report.maximumRegression
-                                attributes["bgcolor"] = ColoredCell(
+                                val regressionsGeometricMean = report.regressionsGeometricMean
-                                        maximumImprovement/maxOf(maximumRegression, abs(maximumImprovement)))
+                                val maximumImprovement = report.maximumImprovement
-                                        .backgroundStyle
+                                val improvementsGeometricMean = report.improvementsGeometricMean
-                                +formatValue(report.maximumImprovement, true)
+                                val maximumChange = maxOf(maximumRegression, abs(maximumImprovement))
-                            }
+                                val maximumChangeGeoMean = maxOf(regressionsGeometricMean,
-                            td {
+                                        abs(improvementsGeometricMean))
-                                attributes["bgcolor"] = ColoredCell(
+                                if (!report.improvements.isEmpty()) {
-                                        improvementsGeometricMean/maxOf(regressionsGeometricMean,
+                                    td { +"${report.improvements.size}" }
-                                                        abs(improvementsGeometricMean)))
+                                    td {
-                                        .backgroundStyle
+                                        attributes["bgcolor"] = ColoredCell(
-                                +formatValue(report.improvementsGeometricMean, true)
+                                                (maximumImprovement / maximumChange).takeIf{ maximumChange > 0.0 }
+                                        ).backgroundStyle
+                                        +formatValue(report.maximumImprovement, true)
+                                    }
+                                    td {
+                                        attributes["bgcolor"] = ColoredCell(
+                                                (improvementsGeometricMean / maximumChangeGeoMean)
+                                                        .takeIf{ maximumChangeGeoMean > 0.0 }
+                                        ).backgroundStyle
+                                        +formatValue(report.improvementsGeometricMean, true)
+                                    }
+                                } else {
+                                    repeat(3) { td { +"-" } }
+                                }
                             }
                         }
                     }
@@ -584,43 +611,84 @@ class HTMLRender: Render() {
         }
     }
 
+    private fun TableBlock.renderFilteredBenchmarks(detailedReport: DetailedBenchmarksReport,
+                                                    onlyChanges: Boolean, unstableBenchmarks: List<String>,
+                                                    filterUnstable: Boolean) {
+        fun <T> filterBenchmarks(bucket: Map<String, T>) =
+            bucket.filter { (name, _) ->
+                if (filterUnstable) name in unstableBenchmarks else name !in unstableBenchmarks
+            }
+
+        val filteredRegressions = filterBenchmarks(detailedReport.regressions)
+        val filteredImprovements = filterBenchmarks(detailedReport.improvements)
+        renderBenchmarksDetails(detailedReport.mergedReport, filteredRegressions)
+        renderBenchmarksDetails(detailedReport.mergedReport, filteredImprovements)
+        if (!onlyChanges) {
+            // Print all remaining results.
+            renderBenchmarksDetails(filterBenchmarks(detailedReport.mergedReport).filter {
+                it.key !in detailedReport.regressions.keys &&
+                        it.key !in detailedReport.improvements.keys
+            })
+        }
+    }
+
     private fun BodyTag.renderPerformanceDetails(report: SummaryBenchmarksReport, onlyChanges: Boolean) {
         if (onlyChanges) {
-            if (report.regressions.isEmpty() && report.improvements.isEmpty()) {
+            if (report.detailedMetricReports.values.all { it.improvements.isEmpty() } &&
+                    report.detailedMetricReports.values.all { it.regressions.isEmpty() }) {
                 div("alert alert-success") {
                     attributes["role"] = "alert"
                     +"All becnhmarks are stable!"
                 }
             }
         }
+        report.detailedMetricReports.forEach { (metric, detailedReport) ->
+            renderCollapsedData(metric.value, false) {
+                table {
+                    attributes["id"] = "result"
+                    attributes["class"] = "table table-striped table-bordered"
+                    thead {
+                        tr {
+                            th { +"Benchmark" }
+                            th { +"First score" }
+                            th { +"Second score" }
+                            th { +"Percent" }
+                            th { +"Ratio" }
+                        }
+                    }
+                    val geoMeanChangeMap = detailedReport.geoMeanScoreChange?.let {
+                        mapOf(detailedReport.geoMeanBenchmark.first!!.name to detailedReport.geoMeanScoreChange!!)
+                    }
 
-        table {
+                    tbody {
-            attributes["id"] = "result"
+                        val boldRowStyle = "border-bottom: 2.3pt solid black; border-top: 2.3pt solid black"
-            attributes["class"] = "table table-striped table-bordered"
+                        renderBenchmarksDetails(
-            thead {
+                                mutableMapOf(detailedReport.geoMeanBenchmark.first!!.name to detailedReport.geoMeanBenchmark),
-                tr {
+                                geoMeanChangeMap, boldRowStyle)
-                    th { +"Benchmark" }
+
-                    th { +"First score" }
+                        val unstableBenchmarks = report.getUnstableBenchmarksForMetric(metric)
-                    th { +"Second score" }
+
-                    th { +"Percent" }
+                        if (unstableBenchmarks.isNotEmpty()) {
-                    th { +"Ratio" }
+                            tr {
-                }
+                                attributes["style"] = boldRowStyle
-            }
+                                th(colspan = Natural(5)) { +"Stable" }
-            val geoMeanChangeMap = report.geoMeanScoreChange?.
+                            }
-                    let { mapOf(report.geoMeanBenchmark.first!!.name to report.geoMeanScoreChange!!) }
+                        }
-
+
-            tbody {
+                        renderFilteredBenchmarks(detailedReport, onlyChanges, unstableBenchmarks, false)
-                renderBenchmarksDetails(
+
-                        mutableMapOf(report.geoMeanBenchmark.first!!.name to report.geoMeanBenchmark),
+                        if (unstableBenchmarks.isNotEmpty()) {
-                        geoMeanChangeMap, "border-bottom: 2.3pt solid black; border-top: 2.3pt solid black")
+                            tr {
-                renderBenchmarksDetails(report.mergedReport, report.regressions)
+                                attributes["style"] = boldRowStyle
-                renderBenchmarksDetails(report.mergedReport, report.improvements)
+                                th(colspan = Natural(5)) { +"Unstable" }
-                if (!onlyChanges) {
+                            }
-                    // Print all remaining results.
+                        }
-                    renderBenchmarksDetails(report.mergedReport.filter { it.key !in report.regressions.keys &&
+
-                            it.key !in report.improvements.keys })
+                        renderFilteredBenchmarks(detailedReport, onlyChanges, unstableBenchmarks, true)
+                    }
                 }
             }
+            hr {}
         }
     }
 

kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/MetricResultsRender.kt

@@ -14,14 +14,16 @@ class MetricResultsRender: Render() {
         get() = "metrics"
 
     override fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean): String {
-        val results = report.mergedReport.map { entry ->
+        val results = report.detailedMetricReports.values.map { it.mergedReport }.map { report ->
-            buildString {
+            report.map { entry ->
-                val metric = entry.value.first!!.metric
+                buildString {
-                append("{ \"benchmarkName\": \"${entry.key.removeSuffix(metric.suffix)}\",")
+                    val metric = entry.value.first!!.metric
-                append("\"metric\": \"${metric}\",")
+                    append("{ \"benchmarkName\": \"${entry.key.removeSuffix(metric.suffix)}\",")
-                append("\"value\": \"${entry.value.first!!.score}\" }")
+                    append("\"metric\": \"${metric}\",")
+                    append("\"value\": \"${entry.value.first!!.score}\" }")
+                }
             }
-        }.joinToString(", ")
+        }.flatten().joinToString(", ")
         return "[ $results ]"
 
     }

kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt

@@ -54,7 +54,7 @@ class TextRender: Render() {
         renderEnvChanges(report.envChanges, "Environment")
         renderEnvChanges(report.kotlinChanges, "Compiler")
         renderStatusSummary(report)
-        renderStatusChangesDetails(report.getBenchmarksWithChangedStatus())
+        renderStatusChangesDetails(report.benchmarksWithChangedStatus)
         renderPerformanceSummary(report)
         renderPerformanceDetails(report, onlyChanges)
         return content.toString()
@@ -113,18 +113,27 @@ class TextRender: Render() {
     }
 
     fun renderPerformanceSummary(report: SummaryBenchmarksReport) {
-        if (!report.regressions.isEmpty() || !report.improvements.isEmpty()) {
+        if (report.detailedMetricReports.values.any { it.improvements.isNotEmpty() } ||
+                report.detailedMetricReports.values.any { it.regressions.isNotEmpty() }) {
             append("Performance summary")
             append(headerSeparator)
-            if (!report.regressions.isEmpty()) {
-                append("Regressions: Maximum = ${formatValue(report.maximumRegression, true)}," +
-                        " Geometric mean = ${formatValue(report.regressionsGeometricMean, true)}")
-            }
-            if (!report.improvements.isEmpty()) {
-                append("Improvements: Maximum = ${formatValue(report.maximumImprovement, true)}," +
-                        " Geometric mean = ${formatValue(report.improvementsGeometricMean, true)}")
-            }
             append()
+            report.detailedMetricReports.forEach { (metric, detailedReport) ->
+                if (detailedReport.regressions.isNotEmpty() || detailedReport.improvements.isNotEmpty()) {
+                    append(metric.value)
+                    append(headerSeparator)
+                    if (!detailedReport.regressions.isEmpty()) {
+                        append("Regressions: Maximum = ${formatValue(detailedReport.maximumRegression, true)}," +
+                                " Geometric mean = ${formatValue(detailedReport.regressionsGeometricMean, true)}")
+                    }
+                    if (!detailedReport.improvements.isEmpty()) {
+                        append("Improvements: Maximum = ${formatValue(detailedReport.maximumImprovement, true)}," +
+                                " Geometric mean = ${formatValue(detailedReport.improvementsGeometricMean, true)}")
+                    }
+                    append()
+                }
+            }
+
         }
     }
 
@@ -176,25 +185,58 @@ class TextRender: Render() {
         append(headerSeparator)
 
         if (onlyChanges) {
-            if (report.regressions.isEmpty() && report.improvements.isEmpty()) {
+            if (report.detailedMetricReports.values.all { it.improvements.isEmpty() } &&
+                    report.detailedMetricReports.values.all { it.regressions.isEmpty() }) {
                 append("All becnhmarks are stable.")
             }
         }
 
-        val tableWidth = printPerformanceTableHeader()
+        report.detailedMetricReports.forEach { (metric, detailedReport) ->
-        // Print geometric mean.
+            append()
-        val geoMeanChangeMap = report.geoMeanScoreChange?.
+            append(metric.value)
-                let { mapOf(report.geoMeanBenchmark.first!!.name to report.geoMeanScoreChange!!) }
+            append(headerSeparator)
-        printBenchmarksDetails(
+            val tableWidth = printPerformanceTableHeader()
-                mutableMapOf(report.geoMeanBenchmark.first!!.name to report.geoMeanBenchmark),
+            // Print geometric mean.
-                geoMeanChangeMap)
+            val geoMeanChangeMap = detailedReport.geoMeanScoreChange?.let {
-        printTableLineSeparator(tableWidth)
+                mapOf(detailedReport.geoMeanBenchmark.first!!.name to detailedReport.geoMeanScoreChange!!)
-        printBenchmarksDetails(report.mergedReport, report.regressions)
+            }
-        printBenchmarksDetails(report.mergedReport, report.improvements)
+            printBenchmarksDetails(
+                    mutableMapOf(detailedReport.geoMeanBenchmark.first!!.name to detailedReport.geoMeanBenchmark),
+                    geoMeanChangeMap)
+            printTableLineSeparator(tableWidth)
+            val unstableBenchmarks = report.getUnstableBenchmarksForMetric(metric)
+            if (unstableBenchmarks.isNotEmpty()) {
+                append("Stable")
+                printTableLineSeparator(tableWidth)
+            }
+            renderFilteredPerformanceDetails(detailedReport, onlyChanges, unstableBenchmarks, false)
+            if (unstableBenchmarks.isNotEmpty()) {
+                printTableLineSeparator(tableWidth)
+                append("Unstable")
+                printTableLineSeparator(tableWidth)
+            }
+            renderFilteredPerformanceDetails(detailedReport, onlyChanges, unstableBenchmarks,true)
+        }
+    }
+
+    fun renderFilteredPerformanceDetails(detailedReport: DetailedBenchmarksReport,
+                                         onlyChanges: Boolean, unstableBenchmarks: List<String>,
+                                         filterUnstable: Boolean) {
+        fun <T> filterBenchmarks(bucket: Map<String, T>) =
+                bucket.filter { (name, _) ->
+                    if (filterUnstable) name in unstableBenchmarks else name !in unstableBenchmarks
+                }
+
+        val filteredRegressions = filterBenchmarks(detailedReport.regressions)
+        val filteredImprovements = filterBenchmarks(detailedReport.improvements)
+        printBenchmarksDetails(detailedReport.mergedReport, filteredRegressions)
+        printBenchmarksDetails(detailedReport.mergedReport, filteredImprovements)
         if (!onlyChanges) {
             // Print all remaining results.
-            printBenchmarksDetails(report.mergedReport.filter { it.key !in report.regressions.keys &&
+            printBenchmarksDetails(filterBenchmarks(detailedReport.mergedReport).filter {
-                    it.key !in report.improvements.keys })
+                it.key !in detailedReport.regressions.keys &&
+                        it.key !in detailedReport.improvements.keys
+            })
         }
     }
 }

kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/StatisticsRender.kt

@@ -20,7 +20,7 @@ class StatisticsRender: Render() {
     private var content = StringBuilder()
 
     override fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean): String {
-        val benchmarksWithChangedStatus = report.getBenchmarksWithChangedStatus()
+        val benchmarksWithChangedStatus = report.benchmarksWithChangedStatus
         val newPasses = benchmarksWithChangedStatus
                 .filter { it.current == BenchmarkResult.Status.PASSED }
         val newFailures = benchmarksWithChangedStatus
@@ -28,6 +28,12 @@ class StatisticsRender: Render() {
         if (report.failedBenchmarks.isNotEmpty()) {
             content.append("failed: ${report.failedBenchmarks.size}\n")
         }
+        val regressionsSize = report.detailedMetricReports.values.fold(0) { acc, it ->
+            acc + it.regressions.size
+        }
+        val improvementsSize = report.detailedMetricReports.values.fold(0) { acc, it ->
+            acc + it.improvements.size
+        }
         val status = when {
             newFailures.isNotEmpty() -> {
                 content.append("new failures: ${newFailures.size}\n")
@@ -37,16 +43,16 @@ class StatisticsRender: Render() {
                 content.append("new passes: ${newPasses.size}\n")
                 Status.FIXED
             }
-            report.improvements.isNotEmpty() && report.regressions.isNotEmpty() -> {
+            regressionsSize != 0 && improvementsSize != 0 -> {
-                content.append("regressions: ${report.regressions.size}\nimprovements: ${report.improvements.size}")
+                content.append("regressions: $regressionsSize\nimprovements: $improvementsSize")
                 Status.UNSTABLE
             }
-            report.improvements.isNotEmpty() && report.regressions.isEmpty() ->  {
+            improvementsSize != 0 && regressionsSize == 0 ->  {
-                content.append("improvements: ${report.improvements.size}")
+                content.append("improvements: $improvementsSize")
                 Status.IMPROVED
             }
-            report.improvements.isEmpty() && report.regressions.isNotEmpty() -> {
+            improvementsSize == 0 && regressionsSize != 0 -> {
-                content.append("regressions: ${report.regressions.size}")
+                content.append("regressions: $regressionsSize")
                 Status.REGRESSED
             }
             else -> Status.STABLE

kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/TeamCityStatisticsRender.kt

@@ -29,7 +29,9 @@ class TeamCityStatisticsRender: Render() {
         content.append("##teamcity[testSuiteFinished name='Benchmarks']\n")
 
         // Report geometric mean as build statistic value
-        renderGeometricMean(report.geoMeanBenchmark.first!!)
+        report.detailedMetricReports.forEach { (metric, detailedReport) ->
+            renderGeometricMean(metric.value, detailedReport.geoMeanBenchmark.first!!)
+        }
 
         return content.toString()
     }
@@ -51,8 +53,8 @@ class TeamCityStatisticsRender: Render() {
         content.append("##teamcity[testFinished name='${benchmark.name}' duration='${(duration / 1000).toInt()}']\n")
     }
 
-    private fun renderGeometricMean(geoMeanBenchmark: MeanVarianceBenchmark) {
+    private fun renderGeometricMean(metricName: String, geoMeanBenchmark: MeanVarianceBenchmark) {
-        content.append("##teamcity[buildStatisticValue key='Geometric mean' value='${geoMeanBenchmark.score}']\n")
+        content.append("##teamcity[buildStatisticValue key='$metricName Geometric mean' value='${geoMeanBenchmark.score}']\n")
-        content.append("##teamcity[buildStatisticValue key='Geometric mean variance' value='${geoMeanBenchmark.variance}']\n")
+        content.append("##teamcity[buildStatisticValue key='$metricName Geometric mean variance' value='${geoMeanBenchmark.variance}']\n")
     }
 }

kotlin-native/tools/performance-server/src/main/kotlin/database/BenchmarksIndexesDispatcher.kt

@@ -256,6 +256,7 @@ class BenchmarksIndexesDispatcher(connector: ElasticSearchConnector, val feature
     fun getGeometricMean(metricName: String, featureValue: String = "",
                          buildNumbers: Iterable<String>? = null, normalize: Boolean = false,
                          excludeNames: List<String> = emptyList()): Promise<List<Pair<String, List<Double?>>>> {
+
         // Filter only with metric or also with names.
         val filterBenchmarks = if (excludeNames.isEmpty())
             """
@@ -264,7 +265,7 @@ class BenchmarksIndexesDispatcher(connector: ElasticSearchConnector, val feature
         else """
             "bool": { 
                 "must": { "match": { "benchmarks.metric": "$metricName" } },
-                "must_not": { "terms" : { "benchmarks.name" : [${excludeNames.map { "\"$it\"" }.joinToString()}] } }
+                "must_not": [ ${excludeNames.map { """{ "match_phrase" : { "benchmarks.name" : "$it" } }"""}.joinToString() } ]
             }
         """.trimIndent()
         val queryDescription = """

kotlin-native/tools/performance-server/src/main/kotlin/database/DatabaseRequests.kt

@@ -131,6 +131,42 @@ fun getGoldenResults(goldenResultsIndex: GoldenResultsIndex): Promise<Map<String
     }
 }
 
+// Get list of unstable benchmarks from database.
+fun getUnstableResults(goldenResultsIndex: GoldenResultsIndex): Promise<List<String>> {
+    val queryDescription = """
+        {
+          "_source": ["env"],
+          "query": {
+            "nested" : {
+              "path" : "benchmarks",
+              "query" : {
+                "match": { "benchmarks.unstable": true } 
+              },
+              "inner_hits": {
+                "size": 100, 
+                "_source": ["benchmarks.name"]
+              }    
+            }                      
+          } 
+        }
+    """.trimIndent()
+
+    return goldenResultsIndex.search(queryDescription, listOf("hits.hits.inner_hits")).then { responseString ->
+        val dbResponse = JsonTreeParser.parse(responseString).jsonObject
+        val results = dbResponse.getObjectOrNull("hits")?.getArrayOrNull("hits")
+                ?: error("Wrong response:\n$responseString")
+        results.getObjectOrNull(0)?.let {
+            it
+                    .getObject("inner_hits")
+                    .getObject("benchmarks")
+                    .getObject("hits")
+                    .getArray("hits").map {
+                        (it as JsonObject).getObject("_source").getPrimitive("name").content
+                    }
+        } ?: listOf<String>()
+    }
+}
+
 // Get distinct values for needed field from database.
 fun distinctValues(field: String, index: ElasticSearchIndex): Promise<List<String>> {
     val queryDescription = """

kotlin-native/tools/performance-server/src/main/kotlin/routes/route.kt

@@ -588,6 +588,18 @@ fun router() {
         }
     })
 
+    // Get builds description with additional information.
+    router.get("/unstable", { request, response ->
+        CachableResponseDispatcher.getResponse(request, response) { success, reject ->
+            getUnstableResults(goldenIndex).then { unstableBenchmarks ->
+                success(unstableBenchmarks)
+            }.catch {
+                println("Error during getting unstable benchmarks")
+                reject()
+            }
+        }
+    })
+
     router.get("/report/:target/:buildNumber", { request, response ->
         val target = urlParameterToBaseFormat(request.params.target)
         val buildNumber = request.params.buildNumber.toString()

kotlin-native/tools/performance-server/ui/src/main/kotlin/main.kt

@@ -206,8 +206,8 @@ fun main(args: Array<String>) {
     }
 
     buildsNumberToShow = parameters["count"]?.toInt() ?: buildsNumberToShow
-    beforeDate = parameters["before"]?.let{ decodeURIComponent(it)}
+    beforeDate = parameters["before"]?.let { decodeURIComponent(it) }
-    afterDate = parameters["after"]?.let{ decodeURIComponent(it)}
+    afterDate = parameters["after"]?.let { decodeURIComponent(it) }
 
     // Get branches.
     val branchesUrl = "$serverUrl/branches"
@@ -285,30 +285,6 @@ fun main(args: Array<String>) {
     val platformSpecificBenchs = if (parameters["target"] == "Mac_OS_X") ",FrameworkBenchmarksAnalyzer,SpaceFramework_iosX64" else
         if (parameters["target"] == "Linux") ",kotlinx.coroutines" else ""
 
-    // Collect information for charts library.
-    val valuesToShow = mapOf("EXECUTION_TIME" to listOf(mapOf(
-            "normalize" to "true"
-            )),
-            "COMPILE_TIME" to listOf(mapOf(
-                    "samples" to "HelloWorld,Videoplayer$platformSpecificBenchs",
-                    "agr" to "samples"
-            )),
-            "CODE_SIZE" to listOf(mapOf(
-                    "normalize" to "true",
-                    "exclude" to if (parameters["target"] == "Linux")
-                        "kotlinx.coroutines"
-                    else if (parameters["target"] == "Mac_OS_X")
-                        "SpaceFramework_iosX64"
-                    else ""
-            ), if (platformSpecificBenchs.isNotEmpty()) mapOf(
-                    "normalize" to "true",
-                    "agr" to "samples",
-                    "samples" to platformSpecificBenchs.removePrefix(",")
-            ) else null).filterNotNull(),
-            "BUNDLE_SIZE" to listOf(mapOf("samples" to "KotlinNative",
-                    "agr" to "samples"))
-    )
-
     var execData = listOf<String>() to listOf<List<Double?>>()
     var compileData = listOf<String>() to listOf<List<Double?>>()
     var codeSizeData = listOf<String>() to listOf<List<Double?>>()
@@ -328,6 +304,17 @@ fun main(args: Array<String>) {
 
     val metricUrl = "$serverUrl/metricValue/${parameters["target"]}/"
 
+    val unstableBenchmarksPromise = sendGetRequest("$serverUrl/unstable").then { response ->
+        val unstableList = response as String
+        val data = JsonTreeParser.parse(unstableList)
+        if (data !is JsonArray) {
+            error("Response is expected to be an array.")
+        }
+        data.jsonArray.map {
+            (it as JsonPrimitive).content
+        }
+    }
+
     // Get builds description.
     val buildsInfoPromise = sendGetRequest(descriptionUrl).then { response ->
         val buildsInfo = response as String
@@ -341,100 +328,131 @@ fun main(args: Array<String>) {
         }
     }
 
-    // Send requests to get all needed metric values.
+    unstableBenchmarksPromise.then { unstableBenchmarks ->
-    valuesToShow.map { (metric, listOfSettings) ->
+        // Collect information for charts library.
-        val resultValues = listOfSettings.map { settings ->
+        val valuesToShow = mapOf("EXECUTION_TIME" to listOf(mapOf(
-            val getParameters = with(StringBuilder()) {
+                        "normalize" to "true"
-                if (settings.isNotEmpty()) {
+                ),
-                    append("?")
+                mapOf(
-                }
+                        "normalize" to "true",
-                var prefix = ""
+                        "exclude" to unstableBenchmarks.joinToString(",")
-                settings.forEach { (key, value) ->
+                )),
-                    if (value.isNotEmpty()) {
+                "COMPILE_TIME" to listOf(mapOf(
-                        append("$prefix$key=$value")
+                        "samples" to "HelloWorld,Videoplayer$platformSpecificBenchs",
-                        prefix = "&"
+                        "agr" to "samples"
+                )),
+                "CODE_SIZE" to listOf(mapOf(
+                        "normalize" to "true",
+                        "exclude" to if (parameters["target"] == "Linux")
+                            "kotlinx.coroutines"
+                        else if (parameters["target"] == "Mac_OS_X")
+                            "SpaceFramework_iosX64"
+                        else ""
+                ), if (platformSpecificBenchs.isNotEmpty()) mapOf(
+                        "normalize" to "true",
+                        "agr" to "samples",
+                        "samples" to platformSpecificBenchs.removePrefix(",")
+                ) else null).filterNotNull(),
+                "BUNDLE_SIZE" to listOf(mapOf("samples" to "KotlinNative",
+                        "agr" to "samples"))
+        )
+        // Send requests to get all needed metric values.
+        valuesToShow.map { (metric, listOfSettings) ->
+            val resultValues = listOfSettings.map { settings ->
+                val getParameters = with(StringBuilder()) {
+                    if (settings.isNotEmpty()) {
+                        append("?")
                     }
+                    var prefix = ""
+                    settings.forEach { (key, value) ->
+                        if (value.isNotEmpty()) {
+                            append("$prefix$key=$value")
+                            prefix = "&"
+                        }
+                    }
+                    toString()
                 }
-                toString()
+                val branchParameter = if (parameters["branch"] != "all")
+                    (if (getParameters.isEmpty()) "?" else "&") + "branch=${parameters["branch"]}"
+                else ""
+
+                val url = "$metricUrl$metric$getParameters$branchParameter${
+                    if (parameters["type"] != "all")
+                        (if (getParameters.isEmpty() && branchParameter.isEmpty()) "?" else "&") + "type=${parameters["type"]}"
+                    else ""
+                }&count=$buildsNumberToShow${getDatesComponents()}"
+                sendGetRequest(url)
+            }.toTypedArray()
+
+            // Get metrics values for charts.
+            Promise.all(resultValues).then { responses ->
+                val valuesList = responses.map { response ->
+                    val results = (JsonTreeParser.parse(response) as JsonArray).map {
+                        (it as JsonObject).getPrimitive("first").content to
+                                it.getArray("second").map { (it as JsonPrimitive).doubleOrNull }
+                    }
+
+                    val labels = results.map { it.first }
+                    val values = results[0]?.second?.size?.let { (0..it - 1).map { i -> results.map { it.second[i] } } }
+                            ?: emptyList()
+                    labels to values
+                }
+                val labels = valuesList[0].first
+
+                val values = valuesList.map { it.second }.reduce { acc, valuesPart -> acc + valuesPart }
+
+                when (metric) {
+                    // Update chart with gotten data.
+                    "COMPILE_TIME" -> {
+                        compileData = labels to values.map { it.map { it?.let { it / 1000 } } }
+                        compileChart = Chartist.Line("#compile_chart",
+                                getChartData(labels, compileData.second),
+                                getChartOptions(valuesToShow["COMPILE_TIME"]!![0]!!["samples"]!!.split(',').toTypedArray(),
+                                        "Time, milliseconds"))
+                        buildsInfoPromise.then { builds ->
+                            customizeChart(compileChart, "compile_chart", js("$(\"#compile_chart\")"), builds, parameters)
+                            compileChart.update(getChartData(compileData.first, compileData.second))
+                        }
+                    }
+                    "EXECUTION_TIME" -> {
+                        execData = labels to values
+                        execChart = Chartist.Line("#exec_chart",
+                                getChartData(labels, execData.second),
+                                getChartOptions(arrayOf("Geometric Mean (All)", "Geometric mean (Stable)"),
+                                        "Normalized time"))
+                        buildsInfoPromise.then { builds ->
+                            customizeChart(execChart, "exec_chart", js("$(\"#exec_chart\")"), builds, parameters)
+                            execChart.update(getChartData(execData.first, execData.second))
+                        }
+                    }
+                    "CODE_SIZE" -> {
+                        codeSizeData = labels to values
+                        codeSizeChart = Chartist.Line("#codesize_chart",
+                                getChartData(labels, codeSizeData.second),
+                                getChartOptions(arrayOf("Geometric Mean") + platformSpecificBenchs.split(',')
+                                        .filter { it.isNotEmpty() },
+                                        "Normalized size",
+                                        arrayOf("ct-series-4", "ct-series-5", "ct-series-6")))
+                        buildsInfoPromise.then { builds ->
+                            customizeChart(codeSizeChart, "codesize_chart", js("$(\"#codesize_chart\")"), builds, parameters)
+                            codeSizeChart.update(getChartData(codeSizeData.first, codeSizeData.second, sizeClassNames))
+                        }
+                    }
+                    "BUNDLE_SIZE" -> {
+                        bundleSizeData = labels to values.map { it.map { it?.let { it.toInt() / 1024 / 1024 } } }
+                        bundleSizeChart = Chartist.Line("#bundlesize_chart",
+                                getChartData(labels,
+                                        bundleSizeData.second, sizeClassNames),
+                                getChartOptions(arrayOf("Bundle size"), "Size, MB", arrayOf("ct-series-4")))
+                        buildsInfoPromise.then { builds ->
+                            customizeChart(bundleSizeChart, "bundlesize_chart", js("$(\"#bundlesize_chart\")"), builds, parameters)
+                            bundleSizeChart.update(getChartData(bundleSizeData.first, bundleSizeData.second, sizeClassNames))
+                        }
+                    }
+                    else -> error("No chart for metric $metric")
+                }
+                true
             }
-            val branchParameter = if (parameters["branch"] != "all")
-                (if (getParameters.isEmpty()) "?" else "&") + "branch=${parameters["branch"]}"
-            else ""
-
-            val url = "$metricUrl$metric$getParameters$branchParameter${
-            if (parameters["type"] != "all")
-                (if (getParameters.isEmpty() && branchParameter.isEmpty()) "?" else "&") + "type=${parameters["type"]}"
-            else ""
-            }&count=$buildsNumberToShow${getDatesComponents()}"
-            sendGetRequest(url)
-        }.toTypedArray()
-
-        // Get metrics values for charts.
-        Promise.all(resultValues).then { responses ->
-            val valuesList = responses.map { response ->
-                val results = (JsonTreeParser.parse(response) as JsonArray).map {
-                    (it as JsonObject).getPrimitive("first").content to
-                            it.getArray("second").map { (it as JsonPrimitive).doubleOrNull }
-                }
-
-                val labels = results.map { it.first }
-                val values = results[0]?.second?.size?.let { (0..it - 1).map { i -> results.map { it.second[i] } } }
-                        ?: emptyList()
-                labels to values
-            }
-            val labels = valuesList[0].first
-            val values = valuesList.map { it.second }.reduce { acc, valuesPart -> acc + valuesPart }
-
-            when (metric) {
-                // Update chart with gotten data.
-                "COMPILE_TIME" -> {
-                    compileData = labels to values.map { it.map { it?.let { it / 1000 } } }
-                    compileChart = Chartist.Line("#compile_chart",
-                            getChartData(labels, compileData.second),
-                            getChartOptions(valuesToShow["COMPILE_TIME"]!![0]!!["samples"]!!.split(',').toTypedArray(),
-                                    "Time, milliseconds"))
-                    buildsInfoPromise.then { builds ->
-                        customizeChart(compileChart, "compile_chart", js("$(\"#compile_chart\")"), builds, parameters)
-                        compileChart.update(getChartData(compileData.first, compileData.second))
-                    }
-                }
-                "EXECUTION_TIME" -> {
-                    execData = labels to values
-                    execChart = Chartist.Line("#exec_chart",
-                            getChartData(labels, execData.second),
-                            getChartOptions(arrayOf("Geometric Mean"), "Normalized time"))
-                    buildsInfoPromise.then { builds ->
-                        customizeChart(execChart, "exec_chart", js("$(\"#exec_chart\")"), builds, parameters)
-                        execChart.update(getChartData(execData.first, execData.second))
-                    }
-                }
-                "CODE_SIZE" -> {
-                    codeSizeData = labels to values
-                    codeSizeChart = Chartist.Line("#codesize_chart",
-                            getChartData(labels, codeSizeData.second),
-                            getChartOptions(arrayOf("Geometric Mean") + platformSpecificBenchs.split(',')
-                                    .filter { it.isNotEmpty() },
-                                    "Normalized size",
-                                    arrayOf("ct-series-4", "ct-series-5", "ct-series-6")))
-                    buildsInfoPromise.then { builds ->
-                        customizeChart(codeSizeChart, "codesize_chart", js("$(\"#codesize_chart\")"), builds, parameters)
-                        codeSizeChart.update(getChartData(codeSizeData.first, codeSizeData.second, sizeClassNames))
-                    }
-                }
-                "BUNDLE_SIZE" -> {
-                    bundleSizeData = labels to values.map { it.map { it?.let { it.toInt() / 1024 / 1024 } } }
-                    bundleSizeChart = Chartist.Line("#bundlesize_chart",
-                            getChartData(labels,
-                                    bundleSizeData.second, sizeClassNames),
-                            getChartOptions(arrayOf("Bundle size"), "Size, MB", arrayOf("ct-series-4")))
-                    buildsInfoPromise.then { builds ->
-                        customizeChart(bundleSizeChart, "bundlesize_chart", js("$(\"#bundlesize_chart\")"), builds, parameters)
-                        bundleSizeChart.update(getChartData(bundleSizeData.first, bundleSizeData.second, sizeClassNames))
-                    }
-                }
-                else -> error("No chart for metric $metric")
-            }
-            true
         }
     }