Normalize benchmarks and changes in charts (#2820)

tools/benchmarksAnalyzer/src/main/kotlin/main.kt

@@ -40,6 +40,13 @@ object BintrayConnector : Connector() {
         val fileParametersSize = 3
         val fileDescription = fileLocation.substringAfter(connectorPrefix)
         val fileParameters = fileDescription.split(':', limit = fileParametersSize)
+
+        // Right link to bintray file.
+        if (fileParameters.size == 1) {
+            val accessFileUrl = "$bintrayUrl/${fileParameters[0]}"
+            return sendGetRequest(accessFileUrl, followLocation = true)
+        }
+        // Used builds description format.
         if (fileParameters.size != fileParametersSize) {
             error("To get file from bintray, please, specify, build number from TeamCity and target" +
                     " in format bintray:build_number:target:filename")
@@ -81,26 +88,50 @@ fun getFileContent(fileName: String, user: String? = null): String {
 fun getBenchmarkReport(fileName: String, user: String? = null) =
         BenchmarksReport.create(JsonTreeParser.parse(getFileContent(fileName, user)))
 
+fun parseNormalizeResults(results: String): Map<String, Map<String, Double>> {
+    val parsedNormalizeResults = mutableMapOf<String, MutableMap<String, Double>>()
+    val tokensNumber = 3
+    results.lines().forEach {
+        if (!it.isEmpty()) {
+            val tokens = it.split(",").map { it.trim() }
+            if (tokens.size != tokensNumber) {
+                error("Data for normalization should include benchmark name, metric name and value. Got $it")
+            }
+            parsedNormalizeResults.getOrPut(tokens[0], { mutableMapOf<String, Double>() })[tokens[1]] = tokens[2].toDouble()
+        }
+    }
+    return parsedNormalizeResults
+}
+
 // Prints text summary by users request.
 fun summaryAction(argParser: ArgParser) {
     val benchsReport = SummaryBenchmarksReport(getBenchmarkReport(argParser.get<String>("mainReport")!!, argParser.get<String>("user")))
     val results = mutableListOf<String>()
+    val executionNormalize = argParser.get<String>("exec-normalize")?.let {
+        parseNormalizeResults(getFileContent(it))
+    }
+    val compileNormalize = argParser.get<String>("compile-normalize")?.let {
+        parseNormalizeResults(getFileContent(it))
+    }
+    val codesizeNormalize = argParser.get<String>("codesize-normalize")?.let {
+        parseNormalizeResults(getFileContent(it))
+    }
     results.apply {
         add(benchsReport.failedBenchmarks.size.toString())
         argParser.getAll<String>("exec-samples")?. let {
             val filter = if (it.first() == "all") null else it
             add(benchsReport.getResultsByMetric(BenchmarkResult.Metric.EXECUTION_TIME,
-                    argParser.get<String>("exec")!! == "geomean", filter).joinToString(";"))
+                    argParser.get<String>("exec")!! == "geomean", filter, executionNormalize).joinToString(";"))
         }
         argParser.getAll<String>("compile-samples")?. let {
             val filter = if (it.first() == "all") null else it
             add(benchsReport.getResultsByMetric(BenchmarkResult.Metric.COMPILE_TIME,
-                    argParser.get<String>("compile")!! == "geomean", filter).joinToString(";"))
+                    argParser.get<String>("compile")!! == "geomean", filter, compileNormalize).joinToString(";"))
         }
         argParser.getAll<String>("codesize-samples")?. let {
             val filter = if (it.first() == "all") null else it
             add(benchsReport.getResultsByMetric(BenchmarkResult.Metric.CODE_SIZE,
-                    argParser.get<String>("codesize")!! == "geomean", filter).joinToString(";"))
+                    argParser.get<String>("codesize")!! == "geomean", filter, codesizeNormalize).joinToString(";"))
         }
     }
     println(results.joinToString())
@@ -117,16 +148,22 @@ fun main(args: Array<String>) {
                         OptionDescriptor(ArgType.String(), "exec-samples",
                                 description = "Samples used for execution time metric (value 'all' allows use all samples)",
                                 delimiter = ","),
+                        OptionDescriptor(ArgType.String(), "exec-normalize",
+                                description = "File with golden results which should be used for normalization"),
                         OptionDescriptor(ArgType.Choice(listOf("samples", "geomean")), "compile",
                                 description = "Compile time way of calculation", defaultValue = "geomean"),
                         OptionDescriptor(ArgType.String(), "compile-samples",
                                 description = "Samples used for compile time metric (value 'all' allows use all samples)",
                                 delimiter = ","),
+                        OptionDescriptor(ArgType.String(), "compile-normalize",
+                                description = "File with golden results which should be used for normalization"),
                         OptionDescriptor(ArgType.Choice(listOf("samples", "geomean")), "codesize",
                                 description = "Code size way of calculation", defaultValue = "geomean"),
                         OptionDescriptor(ArgType.String(), "codesize-samples",
                                 description = "Samples used for code size metric (value 'all' allows use all samples)",
                                 delimiter = ","),
+                        OptionDescriptor(ArgType.String(), "codesize-normalize",
+                                description = "File with golden results which should be used for normalization"),
                         OptionDescriptor(ArgType.String(), "user", "u", "User access information for authorization")
                 ), listOf(ArgDescriptor(ArgType.String(), "mainReport", "Main report for analysis"))
             )
@@ -137,7 +174,7 @@ fun main(args: Array<String>) {
             OptionDescriptor(ArgType.String(), "output", "o", "Output file"),
             OptionDescriptor(ArgType.Double(), "eps", "e", "Meaningful performance changes", "1.0"),
             OptionDescriptor(ArgType.Boolean(), "short", "s", "Show short version of report", "false"),
-            OptionDescriptor(ArgType.Choice(listOf("text", "html", "teamcity", "statistics")),
+            OptionDescriptor(ArgType.Choice(listOf("text", "html", "teamcity", "statistics", "metrics")),
                     "renders", "r", "Renders for showing information", "text", isMultiple = true),
             OptionDescriptor(ArgType.String(), "user", "u", "User access information for authorization")
     )

tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt

@@ -133,7 +133,8 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport,
         }
     }
 
-    fun getResultsByMetric(metric: BenchmarkResult.Metric, getGeoMean: Boolean = true, filter: List<String>? = null): List<Double>  {
+    fun getResultsByMetric(metric: BenchmarkResult.Metric, getGeoMean: Boolean = true, filter: List<String>? = null,
+                           normalizeData: Map<String, Map<String, Double>>? = null): List<Double>  {
         val benchmarks = filter?.let {
             mergedReport.filter { entry ->
                 filter.find {
@@ -148,7 +149,14 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport,
         if (filteredBenchmarks.isEmpty()) {
             error("There is no benchmarks for metric $metric")
         }
-        val results = filteredBenchmarks.map { entry -> entry.key.removeSuffix(metric.suffix) to entry.value.first!!.meanBenchmark.score }.toMap()
+        val results = filteredBenchmarks.map { entry ->
+            val score = entry.value.first!!.meanBenchmark.score
+            val name = entry.key.removeSuffix(metric.suffix)
+            val value = normalizeData?.let {
+                it.get(name)?.get("$metric")?.let { score / it }
+                        ?: error("No normalization data for benchmark $name and metric $metric")
+            } ?: score
+            name to value }.toMap()
         if (getGeoMean) {
             return listOf(geometricMean(results.values))
         }

tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/MetricResultsRender.kt

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.jetbrains.renders
+
+import org.jetbrains.analyzer.*
+import org.jetbrains.report.BenchmarkResult
+
+// Report render to text format.
+class MetricResultsRender: Render() {
+    override val name: String
+        get() = "metrics"
+
+    override fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean): String {
+        val results = report.mergedReport.map { entry ->
+            buildString {
+                val metric = entry.value.first!!.meanBenchmark.metric
+                append("{ \"benchmarkName\": \"${entry.key.removeSuffix(metric.suffix)}\",")
+                append("\"metric\": \"${metric}\",")
+                append("\"value\": \"${entry.value.first!!.meanBenchmark.score}\" }")
+            }
+        }.joinToString(", ")
+        return "[ $results ]"
+
+    }
+}

tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt

@@ -32,6 +32,7 @@ abstract class Render {
                 "html" -> HTMLRender()
                 "teamcity" -> TeamCityStatisticsRender()
                 "statistics" -> StatisticsRender()
+                "metrics" -> MetricResultsRender()
                 else -> error("Unknown render $name")
             }
     }

tools/kliopt/org/jetbrains/kliopt/KliOpt.kt

@@ -50,7 +50,7 @@ sealed class ArgType(val hasParameter: kotlin.Boolean) {
             get() = "{ Value should be one of $values }"
 
         override fun check(value: kotlin.String, name: kotlin.String) {
-            if (value !in values) error("Option $name is expected to be obe of $values. $value is provided.")
+            if (value !in values) error("Option $name is expected to be one of $values. $value is provided.")
         }
     }
 }

tools/performance-server/src/main/kotlin/routes/route.kt

@@ -24,6 +24,7 @@ const val teamCityUrl = "https://buildserver.labs.intellij.net/app/rest"
 const val downloadBintrayUrl = "https://dl.bintray.com/content/lepilkinaelena/KotlinNativePerformance"
 const val uploadBintrayUrl = "https://api.bintray.com/content/lepilkinaelena/KotlinNativePerformance"
 const val buildsFileName = "buildsSummary.csv"
+const val goldenResultsFileName = "goldenResults.csv"
 const val bintrayPackage = "builds"
 const val buildsInfoPartsNumber = 11
 
@@ -109,6 +110,9 @@ object LocalCache {
     }
 }
 
+data class GoldenResult(val benchmarkName: String, val metric: String, val value: Double)
+data class GoldenResultsInfo(val bintrayUser: String, val bintrayPassword: String, val goldenResults: Array<GoldenResult>)
+
 // Build information provided from request.
 data class BuildInfo(val buildNumber: String, val branch: String, val startTime: String,
                      val finishTime: String)
@@ -197,11 +201,7 @@ fun checkBuildType(currentType: String, targetType: String): Boolean {
 fun prepareBuildsResponse(builds: Collection<String>, type: String, branch: String, buildNumber: String? = null): List<Build> {
     val buildsObjects = mutableListOf<Build>()
     builds.forEach {
-        val tokens = it.split(",").map { it.trim() }
+        val tokens = buildDescriptionToTokens(it)
-        if (tokens.size != buildsInfoPartsNumber) {
-            error("Build description $it doesn't contain all necessary information. " +
-                    "File with data could be corrupted.")
-        }
         if ((checkBuildType(tokens[5], type) || type == "day") && (branch == tokens[3] || branch == "all")
                 || tokens[0] == buildNumber) {
             buildsObjects.add(Build(tokens[0], tokens[1], tokens[2], tokens[3],
@@ -212,6 +212,15 @@ fun prepareBuildsResponse(builds: Collection<String>, type: String, branch: Stri
     return buildsObjects
 }
 
+fun buildDescriptionToTokens(buildDescription: String): List<String> {
+    val tokens = buildDescription.split(",").map { it.trim() }
+    if (tokens.size != buildsInfoPartsNumber) {
+        error("Build description $buildDescription doesn't contain all necessary information. " +
+                "File with data could be corrupted.")
+    }
+    return tokens
+}
+
 // Routing of requests to current server.
 fun router() {
     val express = require("express")
@@ -262,6 +271,20 @@ fun router() {
         response.sendStatus(200)
     })
 
+    // Register golden results to normalize on Bintray.
+    router.post("/registerGolden", { request, response ->
+        val goldenResultsInfo = JSON.parse<GoldenResultsInfo>(JSON.stringify(request.body))
+        val buildsDescription = StringBuilder(sendGetRequest("$downloadBintrayUrl/$goldenResultsFileName"))
+        goldenResultsInfo.goldenResults.forEach {
+            buildsDescription.append("${it.benchmarkName}, ${it.metric}, ${it.value}\n")
+        }
+        // Upload new version of file.
+        val uploadUrl = "$uploadBintrayUrl/$bintrayPackage/latest/$goldenResultsFileName?publish=1&override=1"
+        sendUploadRequest(uploadUrl, buildsDescription.toString(), goldenResultsInfo.bintrayUser, goldenResultsInfo.bintrayPassword)
+        // Send response.
+        response.sendStatus(200)
+    })
+
     // Get list of builds.
     router.get("/builds/:target/:type/:branch/:id", { request, response ->
         val builds = LocalCache[request.params.target, request.params.id]
@@ -273,6 +296,16 @@ fun router() {
         response.json(prepareBuildsResponse(builds, request.params.type, request.params.branch))
     })
 
+    router.get("/branches/:target", { request, response ->
+        val builds = LocalCache[request.params.target]
+        response.json(builds.map { buildDescriptionToTokens(it)[3] }.distinct())
+    })
+
+    router.get("/buildsNumbers/:target", { request, response ->
+        val builds = LocalCache[request.params.target]
+        response.json(builds.map { buildDescriptionToTokens(it)[0] }.distinct())
+    })
+
     router.get("/clean", { _, response ->
         LocalCache.clean()
         response.sendStatus(200)

tools/performance-server/ui/src/main/kotlin/main.kt

@@ -19,6 +19,7 @@ import org.w3c.xhr.*
 import org.jetbrains.report.json.*
 import org.jetbrains.build.Build
 import kotlin.js.*
+import kotlin.math.ceil
 import org.w3c.dom.*
 
 // API for interop with JS library Chartist.
@@ -30,6 +31,7 @@ external class ChartistPlugins {
 external object Chartist {
     class Svg(form: String, parameters: dynamic, chartArea: String)
     val plugins: ChartistPlugins
+    val Interpolation: dynamic
     fun Line(query: String, data: dynamic, options: dynamic): dynamic
 }
 
@@ -87,6 +89,15 @@ fun getChartOptions(samples: Array<String>, yTitle: String, classNames: Array<St
     chartOptions["chartPadding"] = paddingObject
     val axisXObject: dynamic = object{}
     axisXObject["offset"] = 40
+    axisXObject["labelInterpolationFnc"] = { value, index, labels ->
+        val labelsCount = 30
+        val skipNumber = ceil((labels.length as Int).toDouble() / labelsCount).toInt()
+        if (skipNumber > 1) {
+            if (index % skipNumber  == 0) value else null
+        } else {
+            value
+        }
+    }
     chartOptions["axisX"] = axisXObject
     val axisYObject: dynamic = object{}
     axisYObject["offset"] = 90
@@ -105,6 +116,9 @@ fun getChartOptions(samples: Array<String>, yTitle: String, classNames: Array<St
     axisYTitle["textAnchor"] = "middle"
     axisYTitle["flipTitle"] = true
     titleObject["axisY"] = axisYTitle
+    val interpolationObject: dynamic = {}
+    interpolationObject["fillHoles"] = true
+    chartOptions["lineSmooth"] = Chartist.Interpolation.simple(interpolationObject)
     chartOptions["plugins"] = arrayOf(Chartist.plugins.legend(legendObject), Chartist.plugins.ctAxisTitle(titleObject))
     return chartOptions
 }
@@ -148,6 +162,7 @@ fun customizeChart(chart: dynamic, chartContainer: String, jquerySelector: dynam
                     else ""}"
             val information = buildString {
                 append("<a href=\"$linkToDetailedInfo\">${currentBuild.buildNumber}</a><br>")
+                append("Value: ${data.value.y.toFixed(4)}<br>")
                 if (currentBuild.failuresNumber > 0) {
                     append("failures: ${currentBuild.failuresNumber}<br>")
                 }
@@ -180,7 +195,7 @@ fun customizeChart(chart: dynamic, chartContainer: String, jquerySelector: dynam
 }
 
 fun main(args: Array<String>) {
-    val serverUrl = "https://kotlin-native-perf-summary.labs.jb.gg/builds"
+    val serverUrl = "https://kotlin-native-perf-summary.labs.jb.gg"
 
     // Get parameters from request.
     val url = window.location.href
@@ -196,7 +211,7 @@ fun main(args: Array<String>) {
 
     // Get builds.
     val buildsUrl = buildString {
-        append("$serverUrl")
+        append("$serverUrl/builds")
         append("/${parameters["target"]}")
         append("/${parameters["type"]}")
         append("/${parameters["branch"]}")
@@ -208,10 +223,16 @@ fun main(args: Array<String>) {
     if (data !is JsonArray) {
         error("Response is expected to be an array.")
     }
-    val builds = data.jsonArray.map { Build.create(it as JsonObject) }
+    val builds = data.jsonArray.map { Build.create(it as JsonObject) }.sortedBy { it.buildNumber.substringAfterLast("-") }
+
+    val branchesUrl = "$serverUrl/branches/${parameters["target"]}"
+
+    val branches: Array<String> = JSON.parse(sendGetRequest(branchesUrl))
+    val releaseBranches = branches.filter { "v\\d+\\.\\d+\\.\\d+-fixes".toRegex().find(it) != null }
 
     // Fill autocomplete list.
-    val buildsNumbers = builds.map { json("value" to it.buildNumber, "data" to it.buildNumber) }.toTypedArray()
+    val buildsNumbersUrl = "$serverUrl/buildsNumbers/${parameters["target"]}"
+    val buildsNumbers: Array<String> = JSON.parse(sendGetRequest(buildsNumbersUrl))
 
     // Change inputs values connected with parameters and add events listeners.
     document.querySelector("#inputGroupTarget [value=\"${parameters["target"]}\"]")?.setAttribute("selected", "true")
@@ -245,6 +266,12 @@ fun main(args: Array<String>) {
         }
     })
 
+    // Add release branches to selector.
+    releaseBranches.forEach {
+        val option = Option(it, it)
+        js("$('#inputGroupBranch')").append(js("$(option)"))
+    }
+
     val autocompleteParameters: dynamic = object{}
     autocompleteParameters["lookup"] = buildsNumbers
     autocompleteParameters["onSelect"] = { suggestion ->
@@ -294,7 +321,7 @@ fun main(args: Array<String>) {
             getChartOptions(compileTime.keys.toTypedArray(), "Time, milliseconds"))
     val codeSizeChart = Chartist.Line("#codesize_chart", getChartData(labels, codeSize.values, sizeClassName),
             getChartOptions(codeSize.keys.toTypedArray(), "Size, KB", arrayOf("ct-series-2")))
-    val bundleSizeChart = Chartist.Line("#bundlesize_chart", getChartData(labels, listOf(bundleSize), "ct-series-d"),
+    val bundleSizeChart = Chartist.Line("#bundlesize_chart", getChartData(labels, listOf(bundleSize), sizeClassName),
             getChartOptions(arrayOf("Bundle size"), "Size, MB", arrayOf("ct-series-2")))
 
     // Tooltips and higlights.