Slack performance reporter (#2527)

performance/build.gradle

@@ -113,6 +113,7 @@ task konanJsonReport {
                                                   'compileTime'    : nativeCompileTime]
         def output = MPPTools.createJsonReport(properties)
         new File("${buildDir.absolutePath}/${nativeJson}").write(output)
+        uploadBenchmarkResultToBintray(nativeJson)
     }
 }
 
@@ -126,6 +127,7 @@ task jvmJsonReport {
                                                   'compileTime'    : jvmCompileTime]
         def output = MPPTools.createJsonReport(properties)
         new File("${buildDir.absolutePath}/${jvmJson}").write(output)
+        uploadBenchmarkResultToBintray(jvmJson)
     }
 }
 
@@ -146,14 +148,41 @@ private def getCommonProperties() {
             'kotlinVersion': "${kotlinVersion}".toString()]
 }
 
+private def uploadBenchmarkResultToBintray(String fileName) {
+    def teamcityConfig = System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE")
+    if (teamcityConfig) {
+        def buildProperties = new Properties()
+        buildProperties.load(new FileInputStream(teamcityConfig))
+        def user = buildProperties.getProperty("bintray.user")
+        def password = buildProperties.getProperty("bintray.apikey")
+        def buildNumber = buildProperties.getProperty("build.number")
+        def target = System.getProperty("os.name").replaceAll("\\s", "")
+        MPPTools.uploadFileToBintray("${bintrayUrl}", "${bintrayRepo}",
+                buildNumber, "${bintrayPackage}", "${target}/${buildNumber}/${fileName}",
+                "${buildDir.absolutePath}/${fileName}", user, password)
+    }
+}
+
 task bench(type:Exec) {
     dependsOn jvmRun
     dependsOn konanRun
     def extension = MPPTools.getNativeProgramExtension()
     def analyzer = MPPTools.findFile("${analyzerTool}${extension}", "${rootBuildDirectory}/${analyzerToolDirectory}")
     if (analyzer != null) {
-        commandLine "${analyzer}", "${buildDir.absolutePath}/${nativeJson}", "${buildDir.absolutePath}/${jvmJson}"
+        commandLine "${analyzer}", "-r", "text", "-r", "teamcity", "${buildDir.absolutePath}/${nativeJson}", "${buildDir.absolutePath}/${jvmJson}"
     } else {
         println("No analyzer $analyzerTool found in subdirectories of ${rootBuildDirectory}/${analyzerToolDirectory}")
     }
+}
+
+// Produce and send slack report.
+task slackReport(type: RegressionsReporter) {
+    // Create folder for report (root Kotlin project settings make create report in separate folder).
+    def reportDirectory = outputReport.substring(0, outputReport.lastIndexOf("/"))
+    mkdir reportDirectory
+    currentBenchmarksReportFile = "${buildDir.absolutePath}/${nativeJson}"
+    analyzer = MPPTools.findFile("${analyzerTool}${MPPTools.getNativeProgramExtension()}",
+            "${rootBuildDirectory}/${analyzerToolDirectory}")
+    htmlReport = outputReport
+    defaultBranch = project.findProperty('kotlin.native.default.branch') ?: "master"
 }

performance/buildSrc/build.gradle

@@ -30,6 +30,7 @@ dependencies {
     compileOnly gradleApi()
     implementation "org.jetbrains.kotlin:kotlin-gradle-plugin"
     implementation "org.jetbrains.kotlin:kotlin-stdlib-common:$kotlinVersion"
+    compile group: 'com.ullink.slack', name: 'simpleslackapi', version: '1.2.0'
 }
 
 sourceSets.main.kotlin.srcDirs = ["src", "$projectDir/../../tools/benchmarks/shared/src"]

performance/buildSrc/src/main/kotlin/MPPTools.kt

@@ -16,6 +16,12 @@ import org.jetbrains.report.*
 import org.jetbrains.report.json.*
 import java.nio.file.Paths
 import java.io.File
+import java.io.FileInputStream
+import java.io.BufferedOutputStream
+import java.io.BufferedInputStream
+import java.net.HttpURLConnection
+import java.net.URL
+import java.util.Base64
 
 /*
  * This file includes short-cuts that may potentially be implemented in Kotlin MPP Gradle plugin in the future.
@@ -92,6 +98,38 @@ fun createJsonReport(projectProperties: Map<String, Any>): String {
 fun findFile(fileName: String, directory: String): String? =
     File(directory).walkBottomUp().find { it.name == fileName }?.getAbsolutePath()
 
+fun uploadFileToBintray(url: String, project: String, version: String, packageName: String, bintrayFilePath: String,
+                        filePath: String, username: String? = null, password: String? = null) {
+    val uploadUrl = "$url/$project/$packageName/$version/$bintrayFilePath?publish=1"
+    sendUploadRequest(uploadUrl, filePath, username, password)
+}
+
+fun sendUploadRequest(url: String, fileName: String, username: String? = null, password: String? = null) {
+    val uploadingFile = File(fileName)
+    val connection = URL(url).openConnection() as HttpURLConnection
+    connection.doOutput = true
+    connection.doInput = true
+    connection.requestMethod = "PUT"
+    connection.setRequestProperty("Content-type", "text/plain")
+    if (username != null && password != null) {
+        val auth = Base64.getEncoder().encode((username + ":" + password).toByteArray()).toString(Charsets.UTF_8)
+        connection.addRequestProperty("Authorization", "Basic $auth")
+    }
+
+    try {
+        connection.connect()
+        BufferedOutputStream(connection.outputStream).use { output ->
+            BufferedInputStream(FileInputStream(uploadingFile)).use { input ->
+                input.copyTo(output)
+            }
+        }
+        val response = connection.responseMessage
+        println("Upload request ended with ${connection.responseCode} - $response")
+    } catch (t: Throwable) {
+        error("Couldn't upload file $fileName to $url")
+    }
+}
+
 // A short-cut to add a Kotlin/Native run task.
 @JvmOverloads
 fun createRunTask(
@@ -109,7 +147,7 @@ fun getJvmCompileTime(programName: String): BenchmarkResult =
         TaskTimerListener.getBenchmarkResult(programName, listOf("compileKotlinMetadata", "jvmJar"))
 
 fun getNativeCompileTime(programName: String): BenchmarkResult =
-        TaskTimerListener.getBenchmarkResult(programName, listOf("compileKotlinNative", "linkReleaseExecutableNative"))
+        TaskTimerListener.getBenchmarkResult(programName, listOf("compileKotlinNative", "linkMainReleaseExecutableNative"))
 
 // Class time tracker for all tasks.
 class TaskTimerListener: TaskExecutionListener {

performance/buildSrc/src/main/kotlin/RegressionsReporter.kt

@@ -0,0 +1,243 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+import groovy.lang.Closure
+import org.gradle.api.Action
+import org.gradle.api.DefaultTask
+import org.gradle.api.Project
+import org.gradle.api.Task
+import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.TaskAction
+
+import com.ullink.slack.simpleslackapi.impl.SlackSessionFactory
+
+import org.jetbrains.report.json.*
+
+import java.io.FileInputStream
+import java.io.IOException
+import java.io.File
+import java.util.concurrent.TimeUnit
+import java.net.HttpURLConnection
+import java.net.URL
+import java.util.Base64
+import java.util.Properties
+
+// Run command line from string.
+fun String.runCommand(workingDir: File = File("."),
+                      timeoutAmount: Long = 60,
+                      timeoutUnit: TimeUnit = TimeUnit.SECONDS): String {
+    return try {
+        ProcessBuilder(*this.split("\\s".toRegex()).toTypedArray())
+                .directory(workingDir)
+                .redirectOutput(ProcessBuilder.Redirect.PIPE)
+                .redirectError(ProcessBuilder.Redirect.PIPE)
+                .start().apply {
+                    waitFor(timeoutAmount, timeoutUnit)
+                }.inputStream.bufferedReader().readText()
+    } catch (e: IOException) {
+        error("Couldn't run command $this")
+    }
+}
+
+data class Commit(val revision: String, val developer: String, val webUrlWithDescription: String)
+
+// List of commits.
+class CommitsList(data: JsonElement): ConvertedFromJson {
+
+    val commits: List<Commit>
+
+    init {
+        if (data !is JsonObject) {
+            error("Commits description is expected to be a json object!")
+        }
+        val changesElement = data.getOptionalField("change")
+        commits = changesElement?.let {
+            if (changesElement !is JsonArray) {
+                error("Change field is expected to be an array. Please, check source.")
+            }
+           changesElement.jsonArray.map {
+                with(it as JsonObject) {
+                    Commit(elementToString(getRequiredField("version"), "version"),
+                            elementToString(getRequiredField("username"), "username"),
+                            elementToString(getRequiredField("webUrl"), "webUrl")
+                    )
+                }
+           }
+        } ?: listOf<Commit>()
+    }
+}
+
+/**
+ * Task to produce regressions report and send it to slack. Requires a report with current benchmarks result
+ * and path to analyzer tool
+ *
+ * @property currentBenchmarksReportFile  path to file with becnhmarks result
+ * @property analyzer path to analyzer tool
+ * @property htmlReport name of result html report
+ * @property defaultBranch name of defaukt branch
+ */
+open class RegressionsReporter : DefaultTask() {
+
+    val teamCityUrl = "http://buildserver.labs.intellij.net"
+    val slackUsers = mapOf(
+            "olonho" to "nikolay.igotti",
+            "nikolay.igotti" to "nikolay.igotti",
+            "ilya.matveev" to "ilya.matveev",
+            "ilmat192" to "ilya.matveev",
+            "vasily.v.levchenko" to "minamoto",
+            "vasily.levchenko" to "minamoto",
+            "alexander.gorshenev" to "alexander.gorshenev",
+            "igor.chevdar" to "igor.chevdar",
+            "pavel.punegov" to "Pavel Punegov",
+            "dmitriy.dolovov" to "dmitriy.dolovov",
+            "svyatoslav.scherbina" to "svyatoslav.scherbina",
+            "sbogolepov" to "sergey.bogolepov",
+            "Alexey.Zubakov" to "Alexey.Zubakov",
+            "kirill.shmakov" to "kirill.shmakov",
+            "elena.lepilkina" to "elena.lepilkina")
+
+    @Input
+    lateinit var currentBenchmarksReportFile: String
+
+    @Input
+    lateinit var analyzer: String
+
+    @Input
+    lateinit var htmlReport: String
+
+    @Input
+    lateinit var defaultBranch: String
+
+    private fun tabUrl(buildId: String, buildTypeId: String, tab: String) =
+            "$teamCityUrl/viewLog.html?buildId=$buildId&buildTypeId=$buildTypeId&tab=$tab"
+
+    private fun testReportUrl(buildId: String, buildTypeId: String) =
+            tabUrl(buildId, buildTypeId, "testsInfo")
+
+    private fun buildsUrl(buildLocator: String) =
+            "$teamCityUrl/app/rest/builds/?locator=$buildLocator"
+
+    private fun previousBuildLocator(buildTypeId: String, branchName: String) =
+            "buildType:id:$buildTypeId,branch:name:$branchName,status:SUCCESS,state:finished,count:1"
+
+    private fun changesListUrl(buildLocator: String) =
+            "$teamCityUrl/app/rest/changes/?locator=build:$buildLocator"
+
+    private fun sendGetRequest(url: String, username: String? = null, password: String? = null) : String {
+        val connection = URL(url).openConnection() as HttpURLConnection
+        if (username != null && password != null) {
+            val auth = Base64.getEncoder().encode((username + ":" + password).toByteArray()).toString(Charsets.UTF_8)
+            connection.addRequestProperty("Authorization", "Basic $auth")
+        }
+        connection.setRequestProperty("Accept", "application/json");
+        connection.connect()
+        return connection.inputStream.use { it.reader().use { reader -> reader.readText() } }
+    }
+
+    private fun getBuild(buildLocator: String, user: String, password: String) =
+            try {
+                sendGetRequest(buildsUrl(buildLocator), user, password)
+            } catch (t: Throwable) {
+                error("Try to get build! TeamCity is unreachable!")
+            }
+
+    private fun getBuildProperty(buildJsonDescription: String, property: String) =
+            with(JsonTreeParser.parse(buildJsonDescription) as JsonObject) {
+                if (getPrimitive("count").int == 0) {
+                    error("No build information on TeamCity for $buildJsonDescription!")
+                }
+                (getArray("build").getObject(0).getPrimitive(property) as JsonLiteral).unquoted()
+            }
+
+    private fun getCommits(buildLocator: String, user: String, password: String): CommitsList {
+        val changes = try {
+            sendGetRequest(changesListUrl(buildLocator), user, password)
+        } catch (t: Throwable) {
+            error("Try to get commits! TeamCity is unreachable!")
+        }
+        return CommitsList(JsonTreeParser.parse(changes))
+    }
+
+    @TaskAction
+    fun run() {
+        // Get TeamCity properties.
+        val teamcityConfig = System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE") ?:
+            error("Can't load teamcity config!")
+
+        val buildProperties = Properties()
+        buildProperties.load(FileInputStream(teamcityConfig))
+        val buildId = buildProperties.getProperty("teamcity.build.id")
+        val buildTypeId = buildProperties.getProperty("teamcity.buildType.id")
+        val buildNumber = buildProperties.getProperty("build.number")
+        val user = buildProperties.getProperty("teamcity.auth.userId")
+        val password = buildProperties.getProperty("teamcity.auth.password")
+
+        // Get branch.
+        val currentBuild = getBuild("id:$buildId", user, password)
+        val branch = getBuildProperty(currentBuild,"branchName")
+
+        val testReportUrl = testReportUrl(buildId, buildTypeId)
+
+        // Get previous build on branch.
+        val builds = getBuild(previousBuildLocator(buildTypeId,branch), user, password)
+        val previousBuildsExist = (JsonTreeParser.parse(builds) as JsonObject).getPrimitive("count").int != 0
+
+        // Get changes description.
+        val changesList = getCommits("id:$buildId", user, password)
+        val changesInfo = "*Changes* in branch *$branch:*\n" + buildString {
+            changesList.commits.forEach {
+                append("        - Change ${it.revision} by <@${it.developer}> (details: ${it.webUrlWithDescription})\n")
+            }
+        }
+
+        println("defaul Branch: $defaultBranch")
+
+        // If branch differs from default and it's first build compare to master, otherwise compare to previous build on branch.
+        val compareToBranch = if (previousBuildsExist) branch else defaultBranch
+
+        // File name on bintray is the same as current.
+        val bintrayFileName = currentBenchmarksReportFile.substringAfterLast("/")
+
+        // Get compare to build.
+        val compareToBuild = getBuild(previousBuildLocator(buildTypeId, compareToBranch), user, password)
+        val compareToBuildLink = getBuildProperty(compareToBuild,"webUrl")
+        val compareToBuildNumber = getBuildProperty(compareToBuild,"number")
+        val target = System.getProperty("os.name").replace("\\s".toRegex(), "")
+
+        // Generate comparison report.
+        val output = "$analyzer -r html $currentBenchmarksReportFile bintray:$compareToBuildNumber:$target:$bintrayFileName -o $htmlReport"
+                .runCommand()
+
+        if (output.contains("Uncaught exception")) {
+            error("Error during comparasion of $currentBenchmarksReportFile and " +
+                    "bintray:$compareToBuildNumber:$target:$bintrayFileName with $analyzer! " +
+                    "Please check files existance and their correctness.")
+        }
+
+        val reportLink = tabUrl(buildId, buildTypeId, "report_project170_Benchmarks")
+
+        val title = "\n*Performance report for target $target (build $buildNumber)* - $reportLink\n"
+        val header = "$title\n$changesInfo\n\nCompare to build $compareToBuildNumber: $compareToBuildLink\n\n"
+        val footer = "*Benchmarks statistics:* $testReportUrl"
+        val message = "$header\n$footer\n"
+
+        // Send to channel or user directly.
+        val session = SlackSessionFactory.createWebSocketSlackSession(buildProperties.getProperty("konan-reporter-token"))
+        session.connect()
+
+        if (branch == defaultBranch) {
+            val channel = session.findChannelByName(buildProperties.getProperty("konan-channel-name"))
+            session.sendMessage(channel, message)
+        } else {
+            changesList.commits.filter { it.developer in slackUsers }. map { it.developer }
+                    .toSet().forEach {
+                        val slackUser = session.findUserByUserName(slackUsers[it])
+                        session.sendMessageToUser(slackUser, message, null)
+
+                    }
+        }
+        session.disconnect()
+    }
+}

performance/gradle.properties

@@ -9,4 +9,8 @@ jvmTextReport = jvmReport.txt
 nativeJson = nativeReport.json
 jvmJson = jvmReport.json
 analyzerTool = benchmarksAnalyzer
-analyzerToolDirectory = tools/benchmarksAnalyzer/build/bin
+analyzerToolDirectory = tools/benchmarksAnalyzer/build/bin
+outputReport = ../report/report.html
+bintrayUrl = https://api.bintray.com/content/lepilkinaelena
+bintrayRepo = KotlinNativePerformance
+bintrayPackage = jsonReports

tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/BenchmarksReport.kt

@@ -30,50 +30,6 @@ interface JsonSerializable {
     }
 }
 
-// Entity can be created from json description.
-interface ConvertedFromJson {
-    fun getRequiredField(data: JsonObject, fieldName: String): JsonElement {
-        return data.getOrNull(fieldName) ?: error("Field '$fieldName' doesn't exist in '$data'. Please, check origin files.")
-    }
-
-    fun getOptionalField(data: JsonObject, fieldName: String): JsonElement? {
-        return data.getOrNull(fieldName)
-    }
-
-    // Parse json array to list.
-    // Takes function to convert elements in array to expected type.
-    fun <T> arrayToList(array: JsonArray, convert: JsonArray.(Int) -> T?): List<T> {
-        var results = mutableListOf<T>()
-        var index = 0
-        var current: T? = array.convert(index)
-        while (current != null) {
-            results.add(current)
-            index++
-            current = array.convert(index)
-        }
-        return results
-    }
-
-    // Methods for conversion to expected type with checks of possibility of such conversions.
-    fun elementToDouble(element: JsonElement, name: String): Double =
-            if (element is JsonPrimitive)
-                element.double
-            else
-                error("Field '$name' in '$element' is expected to be a double number. Please, check origin files.")
-
-    fun elementToInt(element: JsonElement, name: String): Int =
-        if (element is JsonPrimitive)
-            element.int
-        else
-            error("Field '$name' in '$element' is expected to be an integer number. Please, check origin files.")
-
-    fun elementToString(element: JsonElement, name:String): String =
-            if (element is JsonLiteral)
-                element.unquoted()
-            else
-                error("Field '$name' in '$element' is expected to be a string. Please, check origin files.")
-}
-
 interface EntityFromJsonFactory<T>: ConvertedFromJson {
     fun create(data: JsonElement): T
 }
@@ -85,9 +41,9 @@ class BenchmarksReport(val env: Environment, benchmarksList: List<BenchmarkResul
     companion object: EntityFromJsonFactory<BenchmarksReport> {
         override fun create(data: JsonElement): BenchmarksReport {
             if (data is JsonObject) {
-                val env = Environment.create(getRequiredField(data, "env"))
-                val benchmarksObj = getRequiredField(data, "benchmarks")
-                val compiler = Compiler.create(getRequiredField(data, "kotlin"))
+                val env = Environment.create(data.getRequiredField("env"))
+                val benchmarksObj = data.getRequiredField("benchmarks")
+                val compiler = Compiler.create(data.getRequiredField("kotlin"))
                 val benchmarksList = parseBenchmarksArray(benchmarksObj)
                 return BenchmarksReport(env, benchmarksList, compiler)
             } else {
@@ -98,11 +54,7 @@ class BenchmarksReport(val env: Environment, benchmarksList: List<BenchmarkResul
         // Parse array with benchmarks to list
         fun parseBenchmarksArray(data: JsonElement): List<BenchmarkResult> {
             if (data is JsonArray) {
-                return arrayToList(data.jsonArray, { index ->
-                    if (this.getObjectOrNull(index) != null)
-                        BenchmarkResult.create(this.getObjectOrNull(index) as JsonObject)
-                    else null
-                })
+                return data.jsonArray.map { BenchmarkResult.create(it as JsonObject) }
             } else {
                 error("Benchmarks field is expected to be an array. Please, check origin files.")
             }
@@ -137,8 +89,8 @@ data class Compiler(val backend: Backend, val kotlinVersion: String): JsonSerial
     companion object: EntityFromJsonFactory<Compiler> {
         override fun create(data: JsonElement): Compiler {
             if (data is JsonObject) {
-                val backend = Backend.create(getRequiredField(data, "backend"))
-                val kotlinVersion = elementToString(getRequiredField(data, "kotlinVersion"), "kotlinVersion")
+                val backend = Backend.create(data.getRequiredField("backend"))
+                val kotlinVersion = elementToString(data.getRequiredField("kotlinVersion"), "kotlinVersion")
 
                 return Compiler(backend, kotlinVersion)
             } else {
@@ -154,16 +106,14 @@ data class Compiler(val backend: Backend, val kotlinVersion: String): JsonSerial
         companion object: EntityFromJsonFactory<Backend> {
             override fun create(data: JsonElement): Backend {
                 if (data is JsonObject) {
-                    val typeElement = getRequiredField(data, "type")
+                    val typeElement = data.getRequiredField("type")
                     if (typeElement is JsonLiteral) {
                         val type = backendTypeFromString(typeElement.unquoted()) ?: error("Backend type should be 'jvm' or 'native'")
-                        val version = elementToString(getRequiredField(data, "version"), "version")
-                        val flagsArray = getOptionalField(data, "flags")
+                        val version = elementToString(data.getRequiredField("version"), "version")
+                        val flagsArray = data.getOptionalField("flags")
                         var flags: List<String> = emptyList()
                         if (flagsArray != null && flagsArray is JsonArray) {
-                            flags = arrayToList(flagsArray.jsonArray, { index ->
-                                this.getPrimitiveOrNull(index)?.toString()
-                            })
+                            flags = flagsArray.jsonArray.map { it.toString() }
                         }
                         return Backend(type, version, flags)
                     } else {
@@ -211,8 +161,8 @@ data class Environment(val machine: Machine, val jdk: JDKInstance): JsonSerializ
     companion object: EntityFromJsonFactory<Environment> {
         override fun create(data: JsonElement): Environment {
             if (data is JsonObject) {
-                val machine = Machine.create(getRequiredField(data, "machine"))
-                val jdk = JDKInstance.create(getRequiredField(data, "jdk"))
+                val machine = Machine.create(data.getRequiredField("machine"))
+                val jdk = JDKInstance.create(data.getRequiredField("jdk"))
 
                 return Environment(machine, jdk)
             } else {
@@ -226,8 +176,8 @@ data class Environment(val machine: Machine, val jdk: JDKInstance): JsonSerializ
         companion object: EntityFromJsonFactory<Machine> {
             override fun create(data: JsonElement): Machine {
                 if (data is JsonObject) {
-                    val cpu = elementToString(getRequiredField(data, "cpu"), "cpu")
-                    val os = elementToString(getRequiredField(data, "os"), "os")
+                    val cpu = elementToString(data.getRequiredField("cpu"), "cpu")
+                    val os = elementToString(data.getRequiredField("os"), "os")
 
                     return Machine(cpu, os)
                 } else {
@@ -251,8 +201,8 @@ data class Environment(val machine: Machine, val jdk: JDKInstance): JsonSerializ
         companion object: EntityFromJsonFactory<JDKInstance> {
             override fun create(data: JsonElement): JDKInstance {
                 if (data is JsonObject) {
-                    val version = elementToString(getRequiredField(data, "version"), "version")
-                    val vendor = elementToString(getRequiredField(data, "vendor"), "vendor")
+                    val version = elementToString(data.getRequiredField("version"), "version")
+                    val vendor = elementToString(data.getRequiredField("vendor"), "vendor")
 
                     return JDKInstance(version, vendor)
                 } else {
@@ -291,15 +241,15 @@ class BenchmarkResult(val name: String, val status: Status,
 
         override fun create(data: JsonElement): BenchmarkResult {
             if (data is JsonObject) {
-                val name = elementToString(getRequiredField(data, "name"), "name")
-                val statusElement = getRequiredField(data, "status")
+                val name = elementToString(data.getRequiredField("name"), "name")
+                val statusElement = data.getRequiredField("status")
                 if (statusElement is JsonLiteral) {
                     val status = statusFromString(statusElement.unquoted())
                             ?: error("Status should be PASSED or FAILED")
-                    val score = elementToDouble(getRequiredField(data, "score"), "score")
-                    val runtimeInUs = elementToDouble(getRequiredField(data, "runtimeInUs"), "runtimeInUs")
-                    val repeat = elementToInt(getRequiredField(data, "repeat"), "repeat")
-                    val warmup = elementToInt(getRequiredField(data, "warmup"), "warmup")
+                    val score = elementToDouble(data.getRequiredField("score"), "score")
+                    val runtimeInUs = elementToDouble(data.getRequiredField("runtimeInUs"), "runtimeInUs")
+                    val repeat = elementToInt(data.getRequiredField("repeat"), "repeat")
+                    val warmup = elementToInt(data.getRequiredField("warmup"), "warmup")
 
                     return BenchmarkResult(name, status, score, runtimeInUs, repeat, warmup)
                 } else {

tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/json/ConvertedFromJson.kt

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.report.json
+
+// Entity can be created from json description.
+interface ConvertedFromJson {
+    // Methods for conversion to expected type with checks of possibility of such conversions.
+    fun elementToDouble(element: JsonElement, name: String): Double =
+            if (element is JsonPrimitive)
+                element.double
+            else
+                error("Field '$name' in '$element' is expected to be a double number. Please, check origin files.")
+
+    fun elementToInt(element: JsonElement, name: String): Int =
+            if (element is JsonPrimitive)
+                element.int
+            else
+                error("Field '$name' in '$element' is expected to be an integer number. Please, check origin files.")
+
+    fun elementToString(element: JsonElement, name:String): String =
+            if (element is JsonLiteral)
+                element.unquoted()
+            else
+                error("Field '$name' in '$element' is expected to be a string. Please, check origin files.")
+}
+
+fun JsonObject.getRequiredField(fieldName: String): JsonElement {
+    return getOrNull(fieldName) ?: error("Field '$fieldName' doesn't exist in '$this'. Please, check origin files.")
+}
+
+fun JsonObject.getOptionalField(fieldName: String): JsonElement? {
+    return getOrNull(fieldName)
+}

tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/json/JsonElement.kt

@@ -324,6 +324,5 @@ data class JsonArray(val content: List<JsonElement>) : JsonElement(), List<JsonE
     override fun toString() = content.joinToString(prefix = "[", postfix = "]")
 }
 
-@PublishedApi
-internal fun unexpectedJson(key: String, expected: String): Nothing =
+fun unexpectedJson(key: String, expected: String): Nothing =
         throw JsonElementTypeMismatchException(key, expected)

tools/benchmarksAnalyzer/build.gradle

@@ -72,7 +72,6 @@ kotlin {
 
     targets {
         fromPreset(presets.jvm, 'jvm') {
-            def mainOutput = compilations.main.output
             compilations.all {
                 tasks[compileKotlinTaskName].kotlinOptions {
                     jvmTarget = '1.8'
@@ -81,14 +80,47 @@ kotlin {
             }
         }
 
-        fromPreset(presets.mingwX64, 'windows')
-        fromPreset(presets.linuxX64, 'linux')
-        fromPreset(presets.macosX64, 'macos')
+        fromPreset(presets.mingwX64, 'windows') {
+            compilations.test.linkerOpts "-L${getMingwPath()}/lib"
+            compilations.main.cinterops {
+                libcurl {
+                    includeDirs.headerFilterOnly "${getMingwPath()}/include"
+                }
+            }
+        }
+
+        fromPreset(presets.linuxX64, 'linux') {
+            compilations.main.cinterops {
+                libcurl {
+                    includeDirs.headerFilterOnly '/usr/include', '/usr/include/x86_64-linux-gnu'
+                }
+            }
+            compilations.test.linkerOpts "-L${getMingwPath()}/lib"
+        }
+        fromPreset(presets.macosX64, 'macos') {
+            compilations.main.cinterops {
+                libcurl {
+                    includeDirs.headerFilterOnly '/opt/local/include', '/usr/local/include'
+                }
+            }
+        }
 
         configure([windows, linux, macos]) {
-            compilations.main.outputKinds('EXECUTABLE')
-            compilations.main.extraOpts '-opt'
-            compilations.main.buildTypes = [RELEASE]
+            binaries {
+                executable('benchmarksAnalyzer', [RELEASE]) {
+                    if (org.gradle.internal.os.OperatingSystem.current().isWindows()) {
+                        println("-L${getMingwPath()}/lib")
+                        linkerOpts("-L${getMingwPath()}/lib")
+                    }
+                }
+            }
         }
     }
+}
+
+def getMingwPath() {
+    def directory = System.getenv("MINGW64_DIR")
+    if (directory == null)
+        directory = "c:/msys64/mingw64"
+    return directory
 }

tools/benchmarksAnalyzer/src/main/kotlin-jvm/org/jetbrains/analyzer/Utils.kt

@@ -18,6 +18,9 @@ package org.jetbrains.analyzer
 
 import java.io.File
 import java.io.InputStream
+import java.net.HttpURLConnection
+import java.net.URL
+import java.util.Base64
 
 actual fun readFile(fileName: String): String {
     val inputStream = File(fileName).inputStream()
@@ -25,8 +28,8 @@ actual fun readFile(fileName: String): String {
     return inputString
 }
 
-actual fun format(number: Double, decimalNumber: Int): String =
-    "%.${decimalNumber}f".format(number)
+actual fun Double.format(decimalNumber: Int): String =
+    "%.${decimalNumber}f".format(this)
 
 actual fun writeToFile(fileName: String, text: String) {
     File(fileName).printWriter().use { out ->
@@ -34,9 +37,39 @@ actual fun writeToFile(fileName: String, text: String) {
     }
 }
 
-actual fun assert(value: Boolean, lazyMessage: () -> Any) {
+actual fun assert(value: Boolean, lazyMessage: () -> Any) =
     kotlin.assert(value, lazyMessage)
+
+// Create http(-s) request.
+fun getHttpRequest(url: String, user: String?, password: String?): HttpURLConnection {
+    val connection = URL(url).openConnection() as HttpURLConnection
+    if (user != null && password != null) {
+        val auth = Base64.getEncoder().encode((user + ":" + password).toByteArray()).toString(Charsets.UTF_8)
+        connection.addRequestProperty("Authorization", "Basic $auth")
+    }
+    connection.setRequestProperty("Accept", "application/json")
+    return connection
 }
 
-actual fun getEnv(variableName:String): String? =
-        System.getenv(variableName)
+actual fun sendGetRequest(url: String, user: String?, password: String?, followLocation: Boolean) : String {
+    val connection = getHttpRequest(url, user, password)
+    connection.connect()
+    val responseCode = connection.responseCode
+    if (!followLocation) {
+        connection.connect()
+        return connection.inputStream.use { it.reader().use { reader -> reader.readText() } }
+    }
+
+    // Request with redirect.
+    if (responseCode != HttpURLConnection.HTTP_MOVED_TEMP &&
+            responseCode != HttpURLConnection.HTTP_MOVED_PERM &&
+            responseCode != HttpURLConnection.HTTP_SEE_OTHER) {
+        error("No opportunity to redirect, but flag for redirecting to location was provided!")
+    }
+    val newUrl = connection.getHeaderField("Location")
+    val cookies = connection.getHeaderField("Set-Cookie")
+    val redirect = getHttpRequest(newUrl, user, password)
+    redirect.setRequestProperty("Cookie", cookies)
+    redirect.connect()
+    return redirect.inputStream.use { it.reader().use { reader -> reader.readText() } }
+}

tools/benchmarksAnalyzer/src/main/kotlin-native/org/jetbrains/analyzer/Utils.kt

@@ -17,8 +17,8 @@
 package org.jetbrains.analyzer
 
 import platform.posix.*
-import kotlinx.cinterop.CValuesRef
 import kotlinx.cinterop.*
+import libcurl.*
 
 actual fun readFile(fileName: String): String {
     val file = fopen(fileName, "r") ?: error("Cannot write file '$fileName'")
@@ -36,9 +36,9 @@ actual fun readFile(fileName: String): String {
     return text.toString()
 }
 
-actual fun format(number: Double, decimalNumber: Int): String {
+actual fun Double.format(decimalNumber: Int): String {
     var buffer = ByteArray(1024)
-    snprintf(buffer.refTo(0), buffer.size.toULong(), "%.${decimalNumber}f", number)
+    snprintf(buffer.refTo(0), buffer.size.toULong(), "%.${decimalNumber}f", this)
     return buffer.stringFromUtf8()
 }
 
@@ -51,9 +51,62 @@ actual fun writeToFile(fileName: String, text: String) {
     }
 }
 
-actual fun assert(value: Boolean, lazyMessage: () -> Any) {
+actual fun assert(value: Boolean, lazyMessage: () -> Any) =
     kotlin.assert(value, lazyMessage)
+
+class CUrl(url: String, user: String? = null, password: String? = null, followLocation: Boolean = false)  {
+    private val stableRef = StableRef.create(this)
+
+    private val curl = curl_easy_init()
+
+    init {
+        curl_easy_setopt(curl, CURLOPT_URL, url)
+        val writeData = staticCFunction(::collectResponse)
+        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeData)
+        curl_easy_setopt(curl, CURLOPT_WRITEDATA, stableRef.asCPointer())
+        if (followLocation) {
+            curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L)
+        }
+        user ?.let {
+            curl_easy_setopt(curl, CURLOPT_USERNAME, it)
+        }
+        password ?.let {
+            curl_easy_setopt(curl, CURLOPT_PASSWORD, it)
+        }
+    }
+
+    val body = StringBuilder()
+
+    fun fetch() {
+        val res = curl_easy_perform(curl)
+        if (res != CURLE_OK)
+            println("curl_easy_perform() failed: ${curl_easy_strerror(res)?.toKString()}")
+    }
+
+    fun close() {
+        curl_easy_cleanup(curl)
+        stableRef.dispose()
+    }
 }
 
-actual fun getEnv(variableName:String): String? =
-        getenv(variableName)?.toKString()
+fun CPointer<ByteVar>.toKString(length: Int): String {
+    val bytes = this.readBytes(length)
+    return bytes.stringFromUtf8()
+}
+
+fun collectResponse(buffer: CPointer<ByteVar>?, size: size_t, nitems: size_t, userdata: COpaquePointer?): size_t {
+    buffer ?: return 0u
+    userdata ?. let {
+        val data = buffer.toKString((size * nitems).toInt()).trim()
+        val curl = userdata.asStableRef<CUrl>().get()
+        curl.body.append(data)
+    }
+    return size * nitems
+}
+
+actual fun sendGetRequest(url: String, user: String?, password: String?, followLocation: Boolean) : String {
+    val curl = CUrl(url, user, password, followLocation)
+    curl.fetch()
+    curl.close()
+    return curl.body.toString()
+}

tools/benchmarksAnalyzer/src/main/kotlin/main.kt

@@ -14,50 +14,110 @@
  * limitations under the License.
  */
 
-import org.jetbrains.analyzer.getEnv
+import org.jetbrains.analyzer.sendGetRequest
 import org.jetbrains.analyzer.readFile
 import org.jetbrains.analyzer.SummaryBenchmarksReport
 import org.jetbrains.kliopt.*
-import org.jetbrains.renders.TextRender
-import org.jetbrains.renders.TeamCityStatisticsRender
+import org.jetbrains.renders.*
 import org.jetbrains.report.BenchmarksReport
 import org.jetbrains.report.json.JsonTreeParser
 
+abstract class Connector {
+    abstract val connectorPrefix: String
+
+    fun isCompatible(fileName: String) =
+            fileName.startsWith(connectorPrefix)
+
+    abstract fun getFileContent(fileLocation: String, user: String? = null): String
+}
+
+object BintrayConnector : Connector() {
+    override val connectorPrefix = "bintray:"
+    val bintrayUrl = "https://dl.bintray.com/content/lepilkinaelena/KotlinNativePerformance"
+
+    override fun getFileContent(fileLocation: String, user: String?): String {
+        val fileParametersSize = 3
+        val fileDescription = fileLocation.substringAfter(connectorPrefix)
+        val fileParameters = fileDescription.split(':', limit = fileParametersSize)
+        if (fileParameters.size != fileParametersSize) {
+            error("To get file from bintray, please, specify, build number from TeamCity and target" +
+                    " in format bintray:build_number:target:filename")
+        }
+        val (buildNumber, target, fileName) = fileParameters
+        val accessFileUrl = "$bintrayUrl/$target/$buildNumber/$fileName"
+        return sendGetRequest(accessFileUrl, followLocation = true)
+    }
+}
+
+object TeamCityConnector: Connector() {
+    override val connectorPrefix = "teamcity:"
+    val teamCityUrl = "http://buildserver.labs.intellij.net"
+
+    override fun getFileContent(fileLocation: String, user: String?): String {
+        val fileDescription = fileLocation.substringAfter(connectorPrefix)
+        val buildLocator = fileDescription.substringBeforeLast(':')
+        val fileName = fileDescription.substringAfterLast(':')
+        if (fileDescription == fileLocation ||
+                fileDescription == buildLocator || fileName == fileDescription) {
+            error("To get file from bintray, please, specify, build number from TeamCity and target" +
+                    " in format teamcity:build_locator:filename")
+        }
+        val accessFileUrl = "$teamCityUrl/app/rest/builds/$buildLocator/artifacts/content/$fileName"
+        val userName = user?.substringBefore(':')
+        val password = user?.substringAfter(':')
+        return sendGetRequest(accessFileUrl, userName, password)
+    }
+}
+
+fun getFileContent(fileName: String, user: String? = null): String {
+    return when {
+        BintrayConnector.isCompatible(fileName) -> BintrayConnector.getFileContent(fileName, user)
+        TeamCityConnector.isCompatible(fileName) -> TeamCityConnector.getFileContent(fileName, user)
+        else -> readFile(fileName)
+    }
+}
+
 fun main(args: Array<String>) {
 
     val options = listOf(
-            OptionDescriptor(ArgType.STRING, "output", "o", "Output file"),
-            OptionDescriptor(ArgType.DOUBLE, "eps", "e", "Meaningful performance changes", "0.5"),
-            OptionDescriptor(ArgType.BOOLEAN, "short", "s", "Show short version of report", "false")
+            OptionDescriptor(ArgType.String(), "output", "o", "Output file"),
+            OptionDescriptor(ArgType.Double(), "eps", "e", "Meaningful performance changes", "0.5"),
+            OptionDescriptor(ArgType.Boolean(), "short", "s", "Show short version of report", "false"),
+            OptionDescriptor(ArgType.Choice(listOf("text", "html", "teamcity")),
+                    "renders", "r", "Renders for showing information", "text", isMultiple = true),
+            OptionDescriptor(ArgType.String(), "user", "u", "User access information for authorization")
     )
 
     val arguments = listOf(
-            ArgDescriptor(ArgType.STRING, "mainReport", "Main report for analysis"),
-            ArgDescriptor(ArgType.STRING, "compareToReport", "Report to compare to", isRequired = false)
+            ArgDescriptor(ArgType.String(), "mainReport", "Main report for analysis"),
+            ArgDescriptor(ArgType.String(), "compareToReport", "Report to compare to", isRequired = false)
     )
 
     // Parse args.
     val argParser = ArgParser(options, arguments)
     if (argParser.parse(args)) {
         // Read contents of file.
-        val mainBenchsResults = readFile(argParser.get("mainReport")!!.stringValue)
+        val mainBenchsResults = getFileContent(argParser.get<String>("mainReport")!!, argParser.get<String>("user"))
         val mainReportElement = JsonTreeParser.parse(mainBenchsResults)
         val mainBenchsReport = BenchmarksReport.create(mainReportElement)
-        var compareToBenchsReport = argParser.get("compareToReport")?.stringValue?.let {
-            val compareToResults = readFile(it)
+        var compareToBenchsReport = argParser.get<String>("compareToReport")?.let {
+            val compareToResults = getFileContent(it, argParser.get<String>("user"))
             val compareToReportElement = JsonTreeParser.parse(compareToResults)
             BenchmarksReport.create(compareToReportElement)
         }
 
-        // Generate comparasion report
+        val renders = argParser.getAll<String>("renders")
+
+        // Generate comparasion report.
         val summaryReport = SummaryBenchmarksReport(mainBenchsReport,
                 compareToBenchsReport,
-                argParser.get("eps")!!.doubleValue)
-        TextRender().print(summaryReport, argParser.get("short")!!.booleanValue,
-                argParser.get("output")?.stringValue)
-        // Produce information for TeamCity if needed.
-        getEnv("TEAMCITY_BUILD_PROPERTIES_FILE")?.let {
-            TeamCityStatisticsRender().print(summaryReport, argParser.get("short")!!.booleanValue)
+                argParser.get<Double>("eps")!!)
+
+        var output = argParser.get<String>("output")
+
+        renders?.forEach {
+            Render.getRenderByName(it).print(summaryReport, argParser.get<Boolean>("short")!!, output)
+            output = null
         }
     }
 }

tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt

@@ -26,7 +26,7 @@ import kotlin.math.sqrt
 // Entity to describe avarage values which conssists of mean and variance values.
 data class MeanVariance(val mean: Double, val variance: Double) {
     override fun toString(): String {
-        val format = { number: Double -> format(number, 2)}
+        val format = { number: Double -> number.format(2)}
         return "${format(mean)} ± ${format(variance)}"
     }
 }
@@ -66,10 +66,9 @@ data class MeanVarianceBenchmark(val meanBenchmark: BenchmarkResult, val varianc
         return MeanVariance(mean, ratioConfInt)
     }
 
-    override fun toString(): String {
-        val format = { number: Double -> format(number)}
-        return "${format(meanBenchmark.score)} ± ${format(varianceBenchmark.score)}"
-    }
+    override fun toString(): String =
+        "${meanBenchmark.score.format()} ± ${varianceBenchmark.score.format()}"
+
 }
 
 fun geometricMean(values: List<Double>) = values.map { it.pow(1.0 / values.size) }.reduce { a, b -> a * b }

tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt

@@ -50,9 +50,9 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport,
     var geoMeanScoreChange: ScoreChange? = null
         private set
 
-    // Changes in environment and tools.
-    val envChanges: List<FieldChange<String>>
-    val kotlinChanges: List<FieldChange<String>>
+    // Environment and tools.
+    val environments: Pair<Environment, Environment?>
+    val compilers: Pair<Compiler, Compiler?>
 
     // Countable properties.
     val failedBenchmarks: List<String>
@@ -74,6 +74,47 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport,
     val currentBenchmarksDuration: Map<String, Double>
         get() = benchmarksDurations.filter{ it.value.first != null }.map { it.key to it.value.first!! }.toMap()
 
+    val maximumRegression: Double
+        get() = getMaximumChange(regressions)
+
+    val maximumImprovement: Double
+        get() = getMaximumChange(improvements)
+
+    val regressionsGeometricMean: Double
+        get() = getGeometricMeanOfChanges(regressions)
+
+    val improvementsGeometricMean: Double
+        get() = getGeometricMeanOfChanges(improvements)
+
+    val envChanges: List<FieldChange<String>>
+        get() {
+            val previousEnvironment = environments.second
+            val currentEnvironment = environments.first
+            return previousEnvironment?.let {
+                mutableListOf<FieldChange<String>>().apply {
+                    addFieldChange("Machine CPU", previousEnvironment.machine.cpu, currentEnvironment.machine.cpu)
+                    addFieldChange("Machine OS", previousEnvironment.machine.os, currentEnvironment.machine.os)
+                    addFieldChange("JDK version", previousEnvironment.jdk.version, currentEnvironment.jdk.version)
+                    addFieldChange("JDK vendor", previousEnvironment.jdk.vendor, currentEnvironment.jdk.vendor)
+                }
+            } ?: listOf<FieldChange<String>>()
+        }
+
+    val kotlinChanges: List<FieldChange<String>>
+        get() {
+            val previousCompiler = compilers.second
+            val currentCompiler = compilers.first
+            return previousCompiler?.let {
+                mutableListOf<FieldChange<String>>().apply {
+                    addFieldChange("Backend type", previousCompiler.backend.type.type, currentCompiler.backend.type.type)
+                    addFieldChange("Backend version", previousCompiler.backend.version, currentCompiler.backend.version)
+                    addFieldChange("Backend flags", previousCompiler.backend.flags.toString(),
+                            currentCompiler.backend.flags.toString())
+                    addFieldChange("Kotlin version", previousCompiler.kotlinVersion, currentCompiler.kotlinVersion)
+                }
+            } ?: listOf<FieldChange<String>>()
+        }
+
     init {
         // Count avarage values for each benchmark.
         val currentBenchmarksTable = collectMeanResults(currentReport.benchmarks)
@@ -83,14 +124,30 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport,
         mergedReport = createMergedReport(currentBenchmarksTable, previousBenchmarksTable)
         benchmarksDurations = calculateBenchmarksDuration(currentReport, previousReport)
         geoMeanBenchmark = calculateGeoMeanBenchmark(currentBenchmarksTable, previousBenchmarksTable)
+        environments = Pair(currentReport.env, previousReport?.env)
+        compilers = Pair(currentReport.compiler, previousReport?.compiler)
+
         if (previousReport != null) {
             // Check changes in environment and tools.
-            envChanges = analyzeEnvChanges(currentReport.env, previousReport.env)
-            kotlinChanges = analyzeKotlinChanges(currentReport.compiler, previousReport.compiler)
             analyzePerformanceChanges()
+        }
+    }
+
+    private fun getMaximumChange(bucket: Map<String, ScoreChange>): Double =
+        // Maps of regressions and improvements are sorted.
+        if (bucket.isEmpty()) 0.0 else bucket.values.map { it.first.mean }.first()
+
+    private fun getGeometricMeanOfChanges(bucket: Map<String, ScoreChange>): Double {
+        if (bucket.isEmpty())
+            return 0.0
+        var percentsList = bucket.values.map { it.first.mean }
+        return if (percentsList.first() > 0.0) {
+            geometricMean(percentsList)
         } else {
-            envChanges = listOf<FieldChange<String>>()
-            kotlinChanges = listOf<FieldChange<String>>()
+            // Geometric mean can be counted on positive numbers.
+            val precision = abs(getMaximumChange(bucket)) + 1
+            percentsList = percentsList.map { it + precision }
+            geometricMean(percentsList) - precision
         }
     }
 
@@ -128,7 +185,7 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport,
                 if (previousBenchmarks == null || name !in previousBenchmarks) {
                     getOrPut(name) { SummaryBenchmark(current, null) }
                 } else {
-                    val previousBenchmark = previousBenchmarks[name]!!.meanBenchmark
+                    val previousBenchmark = previousBenchmarks.getValue(name).meanBenchmark
                     getOrPut(name) { SummaryBenchmark(current, previousBenchmarks[name]) }
                     // Explore change of status.
                     if (previousBenchmark.status != currentBenchmark.status) {
@@ -203,23 +260,4 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport,
             add(it)
         }
     }
-
-    private fun analyzeEnvChanges(currentEnv: Environment, previousEnv: Environment): List<FieldChange<String>> {
-        return mutableListOf<FieldChange<String>>().apply {
-            addFieldChange("Machine CPU", previousEnv.machine.cpu, currentEnv.machine.cpu)
-            addFieldChange("Machine OS", previousEnv.machine.os, currentEnv.machine.os)
-            addFieldChange("JDK version", previousEnv.jdk.version, currentEnv.jdk.version)
-            addFieldChange("JDK vendor", previousEnv.jdk.vendor, currentEnv.jdk.vendor)
-        }
-    }
-
-    private fun analyzeKotlinChanges(currentCompiler: Compiler, previousCompiler: Compiler): List<FieldChange<String>> {
-        return mutableListOf<FieldChange<String>>().apply {
-            addFieldChange("Backend type", previousCompiler.backend.type.type, currentCompiler.backend.type.type)
-            addFieldChange("Backend version", previousCompiler.backend.version, currentCompiler.backend.version)
-            addFieldChange("Backend flags", previousCompiler.backend.flags.toString(),
-                    currentCompiler.backend.flags.toString())
-            addFieldChange( "Kotlin version", previousCompiler.kotlinVersion, currentCompiler.kotlinVersion)
-        }
-    }
 }

tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Utils.kt

@@ -17,7 +17,8 @@
 package org.jetbrains.analyzer
 
 expect fun readFile(fileName: String): String
-expect fun format(number: Double, decimalNumber: Int = 4): String
+expect fun Double.format(decimalNumber: Int = 4): String
 expect fun writeToFile(fileName: String, text: String)
 expect fun assert(value: Boolean, lazyMessage: () -> Any)
-expect fun getEnv(variableName:String): String?
+expect fun sendGetRequest(url: String, user: String? = null, password: String? = null,
+                          followLocation: Boolean = false) : String

tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/kliopt/KliOpt.kt

@@ -18,11 +18,41 @@
 package org.jetbrains.kliopt
 
 // Possible types of arguments.
-enum class ArgType(val hasParameter: Boolean) {
-    BOOLEAN(false),
-    STRING(true),
-    INT(true),
-    DOUBLE(true)
+sealed class ArgType(val hasParameter: kotlin.Boolean) {
+    abstract val description: kotlin.String
+    open fun check(value: kotlin.String, name: kotlin.String) {}
+    class Boolean : ArgType(false) {
+        override val description: kotlin.String
+            get() = ""
+    }
+    class String : ArgType(true) {
+        override val description: kotlin.String
+            get() = "{ String }"
+    }
+    class Int : ArgType(true) {
+        override val description: kotlin.String
+            get() = "{ Int }"
+
+        override fun check(value: kotlin.String, name: kotlin.String) {
+            value.toIntOrNull() ?: error("Option $name is expected to be integer number. $value is provided.")
+        }
+    }
+    class Double : ArgType(true) {
+        override val description: kotlin.String
+            get() = "{ Double }"
+
+        override fun check(value: kotlin.String, name: kotlin.String) {
+            value.toDoubleOrNull() ?: error("Option $name is expected to be double number. $value is provided.")
+        }
+    }
+    class Choice(val values: List<kotlin.String>) : ArgType(true) {
+        override val description: kotlin.String
+            get() = "{ Value should be one of $values }"
+
+        override fun check(value: kotlin.String, name: kotlin.String) {
+            if (value !in values) error("Option $name is expected to be obe of $values. $value is provided.")
+        }
+    }
 }
 
 // Common descriptor both for options and positional arguments.
@@ -41,7 +71,8 @@ class OptionDescriptor(
         val shortName: String ? = null,
         description: String? = null,
         defaultValue: String? = null,
-        isRequired: Boolean = false) : Descriptor (type, longName, description, defaultValue, isRequired) {
+        isRequired: Boolean = false,
+        val isMultiple: Boolean = false) : Descriptor (type, longName, description, defaultValue, isRequired) {
     override val textDescription: String
         get() = "option -$longName"
 
@@ -53,6 +84,7 @@ class OptionDescriptor(
             defaultValue?.let { result.append(" [$it]") }
             description?.let {result.append(" -> ${it}")}
             if (!isRequired) result.append(" (optional)")
+            result.append(" ${type.description}")
             result.append("\n")
             return result.toString()
         }
@@ -74,6 +106,7 @@ class ArgDescriptor(
             defaultValue?.let { result.append(" [$it]") }
             description?.let {result.append(" -> ${it}")}
             if (!isRequired) result.append(" (optional)")
+            result.append(" ${type.description}")
             result.append("\n")
             return result.toString()
         }
@@ -81,58 +114,69 @@ class ArgDescriptor(
 
 // Arguments parser.
 class ArgParser(optionsList: List<OptionDescriptor>, argsList: List<ArgDescriptor> = listOf<ArgDescriptor>()) {
-    private val options = optionsList.union(listOf(OptionDescriptor(ArgType.BOOLEAN, "help",
+    private val options = optionsList.union(listOf(OptionDescriptor(ArgType.Boolean(), "help",
                                             "h", "Usage info")))
                             .toList()
     private val arguments = argsList
     private lateinit var parsedValues: MutableMap<String, ParsedArg?>
 
-    inner class ParsedArg(val descriptor: Descriptor, val value: String) {
-        val intValue: Int
-            get() {
-                if (descriptor.type != ArgType.INT)
-                    error("Incorrect value for ${descriptor.textDescription}, must be an int")
-                return value.toInt()
+    inner class ParsedArg(val descriptor: Descriptor, val values: List<String>) {
+        init {
+            // Check correctness of initialization data.
+            if (values.isEmpty()) {
+                printError("Parsed value should be provided!")
             }
+        }
 
-        val stringValue: String
-            get() {
-                if (descriptor.type != ArgType.STRING)
-                    printError("Incorrect value for ${descriptor.textDescription}, must be a string")
-                return value
-            }
+        // Get value of argument converted to expected type.
+        private fun <T : Any> getTyped(value: String): T {
+            val typedValue = when (descriptor.type) {
+                is ArgType.Int -> value.toInt()
+                is ArgType.Double -> value.toDouble()
+                is ArgType.Boolean -> value == "true"
+                else -> value
+            } as? T
+            typedValue ?: printError("Argument ${descriptor.longName} has type ${descriptor.type} which differs from expected!")
+            return typedValue
+        }
 
-        val booleanValue: Boolean
-            get() {
-                if (descriptor.type != ArgType.BOOLEAN)
-                    printError("Incorrect value for ${descriptor.textDescription}, must be a boolean")
-                return value == "true"
-            }
+        // Get value of argument.
+        fun <T : Any> get(): T {
+            return getTyped<T>(values[0])
+        }
 
-        val doubleValue: Double
-            get() {
-                if (descriptor.type != ArgType.DOUBLE)
-                    printError("Incorrect option value for ${descriptor.textDescription}, must be a double")
-                return value.toDouble()
-            }
+        // Get all values of argument.
+        // For options that can be set multiple types.
+        fun <T : Any> getAll(): List<T> {
+            return values.map { getTyped<T>(it) }
+        }
     }
 
     // Output error. Also adds help usage information for easy understanding of problem.
-    private fun printError(message: String) {
+    private fun printError(message: String): Nothing {
         error("$message\n${makeUsage()}")
     }
 
-    private fun saveAsArg(argDescriptors: Map<String, ArgDescriptor>, arg: String): Boolean {
+    private fun saveAsArg(argDescriptors: Map<String, ArgDescriptor>, arg: String, processedValues: Map<String, MutableList<String>>): Boolean {
         // Find uninitialized arguments.
-        val nullArgs = argDescriptors.keys.filter { parsedValues[it] == null }
+        val nullArgs = argDescriptors.keys.filter { processedValues.getValue(it).isEmpty() }
         val name = nullArgs.firstOrNull()
         name?. let {
-            parsedValues[name] = ParsedArg(argDescriptors[name]!!, arg)
+            argDescriptors.getValue(name).type.check(arg, name)
+            processedValues.getValue(name).add(arg)
             return true
         }
         return false
     }
 
+    private fun saveAsOption(descriptor: OptionDescriptor, value: String, processedValues: Map<String, MutableList<String>>) {
+        if (!descriptor.isMultiple && !processedValues.getValue(descriptor.longName).isEmpty()) {
+            printError("Option ${descriptor.longName} is used more than one time!")
+        }
+        descriptor.type.check(value, descriptor.longName)
+        processedValues.getValue(descriptor.longName).add(value)
+    }
+
     // Parse arguments.
     // Returns true if all arguments were parsed, otherwise return false and print help message.
     fun parse(args: Array<String>): Boolean {
@@ -140,7 +184,9 @@ class ArgParser(optionsList: List<OptionDescriptor>, argsList: List<ArgDescripto
         val optDescriptors = options.map { it.longName to it }.toMap()
         val shortNames = options.filter { it.shortName != null }.map { it.shortName!! to it.longName }.toMap()
         val argDescriptors = arguments.map { it.longName to it }.toMap()
-        parsedValues = optDescriptors.keys.union(argDescriptors.keys).toList().map { it to null }.toMap().toMutableMap()
+        val descriptorsKeys = optDescriptors.keys.union(argDescriptors.keys).toList()
+        val processedValues = descriptorsKeys.map { it to mutableListOf<String>() }.toMap().toMutableMap()
+        parsedValues = descriptorsKeys.map { it to null }.toMap().toMutableMap()
         while (index < args.size) {
             val arg = args[index]
             if (arg.startsWith('-')) {
@@ -150,7 +196,7 @@ class ArgParser(optionsList: List<OptionDescriptor>, argsList: List<ArgDescripto
                 descriptor?. let {
                     if (descriptor.type.hasParameter) {
                         if (index < args.size - 1) {
-                            parsedValues[name] = ParsedArg(descriptor, args[index + 1])
+                            saveAsOption(descriptor, args[index + 1], processedValues)
                             index++
                         } else {
                             // An error, option with value without value.
@@ -161,48 +207,61 @@ class ArgParser(optionsList: List<OptionDescriptor>, argsList: List<ArgDescripto
                             println(makeUsage())
                             return false
                         }
-                        parsedValues[name] = ParsedArg(descriptor, "true")
+                        saveAsOption (descriptor, "true", processedValues)
                     }
                 } ?: run {
                     // Try save as argument.
-                    if (!saveAsArg(argDescriptors, arg)) {
+                    if (!saveAsArg(argDescriptors, arg, processedValues)) {
                         printError("Unknown option $arg")
                     }
                 }
             } else {
                 // Argument is found.
-                if (!saveAsArg(argDescriptors, arg)) {
+                if (!saveAsArg(argDescriptors, arg, processedValues)) {
                     printError("Too many arguments!")
                 }
             }
             index++
         }
 
-        parsedValues.forEach { (key, value) ->
-            val descriptor = optDescriptors[key] ?: argDescriptors[key]!!
+        processedValues.forEach { (key, value) ->
+            val descriptor = optDescriptors[key] ?: argDescriptors.getValue(key)
             // Not inited, append default value if needed.
-            parsedValues[key] = value ?: run {
+            if (value.isEmpty()) {
                 descriptor.defaultValue?. let {
-                    ParsedArg(descriptor, descriptor.defaultValue)
-                }
-            }
-            // Check if arg is always required.
-            parsedValues[key] ?: run {
-                if (descriptor.isRequired) {
-                    printError("Please, provide value for ${descriptor.textDescription}. It should be always set")
+                    parsedValues[key] = ParsedArg(descriptor, listOf(descriptor.defaultValue))
+                } ?: run {
+                    if (descriptor.isRequired) {
+                        printError("Please, provide value for ${descriptor.textDescription}. It should be always set")
+                    } else {
+                        parsedValues[key] = null
+                    }
                 }
+            } else {
+                parsedValues[key] = ParsedArg(descriptor, value)
             }
         }
-
         return true
     }
 
-    fun get(name: String): ParsedArg? {
+    // Get value of argument.
+    fun <T : Any> get(name: String): T? {
         if (::parsedValues.isInitialized) {
-            return parsedValues[name]
+            val arg = parsedValues[name]
+            return arg?.get()
         } else {
-            println("Method parse() of ArgParser class should be called before getting arguments and options.")
-            return null
+            printError("Method parse() of ArgParser class should be called before getting arguments and options.")
+        }
+    }
+
+    // Get all values of argument.
+    // For options that can be set multiple types.
+    fun <T : Any> getAll(name: String): List<T>? {
+        if (::parsedValues.isInitialized) {
+            val arg = parsedValues[name]
+            return arg?.getAll()
+        } else {
+            printError("Method parse() of ArgParser class should be called before getting arguments and options.")
         }
     }
 

tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/HTMLRender.kt

@@ -0,0 +1,660 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.jetbrains.renders
+
+import org.jetbrains.analyzer.*
+import org.jetbrains.report.*
+
+import kotlin.math.sin
+import kotlin.math.abs
+import kotlin.math.pow
+
+private fun <T : Comparable<T>> clamp(value: T, minValue: T, maxValue: T): T =
+        minOf(maxOf(value, minValue), maxValue)
+
+// Natural number.
+class Natural(initValue: Int) {
+    val value = if (initValue > 0) initValue else error("Provided value $initValue isn't natural")
+
+    override fun toString(): String {
+        return value.toString()
+    }
+}
+
+interface Element {
+    fun render(builder: StringBuilder, indent: String)
+}
+
+class TextElement(val text: String) : Element {
+    override fun render(builder: StringBuilder, indent: String) {
+        builder.append("$indent$text\n")
+    }
+}
+
+@DslMarker
+annotation class HtmlTagMarker
+
+@HtmlTagMarker
+abstract class Tag(val name: String) : Element {
+    val children = arrayListOf<Element>()
+    val attributes = hashMapOf<String, String>()
+
+    protected fun <T : Element> initTag(tag: T, init: T.() -> Unit): T {
+        tag.init()
+        children.add(tag)
+        return tag
+    }
+
+    override fun render(builder: StringBuilder, indent: String) {
+        builder.append("$indent<$name${renderAttributes()}>\n")
+        for (c in children) {
+            c.render(builder, indent + "  ")
+        }
+        builder.append("$indent</$name>\n")
+    }
+
+    private fun renderAttributes(): String =
+            attributes.map { (attr, value) ->
+                "$attr=\"$value\""
+            }.joinToString(separator = " ", prefix = " ")
+
+    override fun toString(): String {
+        val builder = StringBuilder()
+        render(builder, "")
+        return builder.toString()
+    }
+}
+
+abstract class TagWithText(name: String) : Tag(name) {
+    operator fun String.unaryPlus() {
+        children.add(TextElement(this))
+    }
+}
+
+class HTML : TagWithText("html") {
+    fun head(init: Head.() -> Unit) = initTag(Head(), init)
+    fun body(init: Body.() -> Unit) = initTag(Body(), init)
+}
+
+class Head : TagWithText("head") {
+    fun title(init: Title.() -> Unit) = initTag(Title(), init)
+    fun link(init: Link.() -> Unit) = initTag(Link(), init)
+    fun script(init: Script.() -> Unit) = initTag(Script(), init)
+}
+
+class Title : TagWithText("title")
+class Link : TagWithText("link")
+class Script : TagWithText("script")
+
+abstract class BodyTag(name: String) : TagWithText(name) {
+    fun b(init: B.() -> Unit) = initTag(B(), init)
+    fun p(init: P.() -> Unit) = initTag(P(), init)
+    fun h1(init: H1.() -> Unit) = initTag(H1(), init)
+    fun h2(init: H2.() -> Unit) = initTag(H2(), init)
+    fun h4(init: H4.() -> Unit) = initTag(H4(), init)
+    fun hr(init: HR.() -> Unit) = initTag(HR(), init)
+    fun a(href: String, init: A.() -> Unit) {
+        val a = initTag(A(), init)
+        a.href = href
+    }
+
+    fun img(src: String, init: Image.() -> Unit) {
+        val element = initTag(Image(), init)
+        element.src = src
+    }
+
+    fun table(init: Table.() -> Unit) = initTag(Table(), init)
+    fun div(classAttr: String, init: Div.() -> Unit) = initTag(Div(classAttr), init)
+    fun button(classAttr: String, init: Button.() -> Unit) = initTag(Button(classAttr), init)
+    fun header(classAttr: String, init: Header.() -> Unit) = initTag(Header(classAttr), init)
+    fun span(classAttr: String, init: Span.() -> Unit) = initTag(Span(classAttr), init)
+}
+
+abstract class BodyTagWithClass(name: String, val classAttr: String) : BodyTag(name) {
+    init {
+        attributes["class"] = classAttr
+    }
+}
+
+class Body : BodyTag("body")
+class B : BodyTag("b")
+class P : BodyTag("p")
+class H1 : BodyTag("h1")
+class H2 : BodyTag("h2")
+class H4 : BodyTag("h4")
+class HR : BodyTag("hr")
+class Div(classAttr: String) : BodyTagWithClass("div", classAttr)
+class Header(classAttr: String) : BodyTagWithClass("header", classAttr)
+class Button(classAttr: String) : BodyTagWithClass("button", classAttr)
+class Span(classAttr: String) : BodyTagWithClass("span", classAttr)
+
+class A : BodyTag("a") {
+    var href: String by attributes
+}
+
+class Image : BodyTag("img") {
+    var src: String by attributes
+}
+
+abstract class TableTag(name: String) : BodyTag(name) {
+    fun thead(init: THead.() -> Unit) = initTag(THead(), init)
+    fun tbody(init: TBody.() -> Unit) = initTag(TBody(), init)
+    fun tfoot(init: TFoot.() -> Unit) = initTag(TFoot(), init)
+}
+
+abstract class TableBlock(name: String) : TableTag(name) {
+    fun tr(init: TableRow.() -> Unit) = initTag(TableRow(), init)
+}
+
+class Table : TableTag("table")
+class THead : TableBlock("thead")
+class TFoot : TableBlock("tfoot")
+class TBody : TableBlock("tbody")
+
+abstract class TableRowTag(name: String) : TableBlock(name) {
+    var colspan = Natural(1)
+        set(value) {
+            attributes["colspan"] = value.toString()
+        }
+    var rowspan = Natural(1)
+        set(value) {
+            attributes["rowspan"] = value.toString()
+        }
+
+    fun th(rowspan: Natural = Natural(1), colspan: Natural = Natural(1), init: TableHeadInfo.() -> Unit) {
+        val element = initTag(TableHeadInfo(), init)
+        element.rowspan = rowspan
+        element.colspan = colspan
+    }
+
+    fun td(rowspan: Natural = Natural(1), colspan: Natural = Natural(1), init: TableDataInfo.() -> Unit) {
+        val element = initTag(TableDataInfo(), init)
+        element.rowspan = rowspan
+        element.colspan = colspan
+    }
+}
+
+class TableRow : TableRowTag("tr")
+class TableHeadInfo : TableRowTag("th")
+class TableDataInfo : TableRowTag("td")
+
+fun html(init: HTML.() -> Unit): HTML {
+    val html = HTML()
+    html.init()
+    return html
+}
+
+// Report render to html format.
+class HTMLRender: Render() {
+    override val name: String
+        get() = "html"
+
+    override fun render (report: SummaryBenchmarksReport, onlyChanges: Boolean) =
+        html {
+            head {
+                title { +"Benchmarks report" }
+
+                // Links to bootstrap files.
+                link {
+                    attributes["href"] = "https://stackpath.bootstrapcdn.com/bootstrap/4.2.1/css/bootstrap.min.css"
+                    attributes["rel"] = "stylesheet"
+                }
+                script {
+                    attributes["src"] = "https://code.jquery.com/jquery-3.3.1.slim.min.js"
+                }
+                script {
+                    attributes["src"] = "https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.6/umd/popper.min.js"
+                }
+                script {
+                    attributes["src"] = "https://stackpath.bootstrapcdn.com/bootstrap/4.2.1/js/bootstrap.min.js"
+                }
+            }
+
+            body {
+                header("navbar navbar-expand navbar-dark flex-column flex-md-row bd-navbar") {
+                    attributes["style"] = "background-color:#161616;"
+                    img("https://dashboard.snapcraft.io/site_media/appmedia/2018/04/256px-kotlin-logo-svg.png") {
+                        attributes["style"] = "width:60px;height:60px;"
+                    }
+
+                    span("navbar-brand mb-0 h1") { +"Benchmarks report" }
+                }
+                div("container-fluid") {
+                    p{}
+                    renderEnvironmentTable(report.environments)
+                    renderCompilerTable(report.compilers)
+                    hr {}
+                    renderStatusSummary(report)
+                    hr {}
+                    renderPerformanceSummary(report)
+                    renderPerformanceDetails(report, onlyChanges)
+                }
+            }
+        }.toString()
+
+    private fun TableRowTag.formatComparedTableData(data: String, compareToData: String?) {
+        td {
+            compareToData?. let {
+                // Highlight changed data.
+                if (it != data)
+                    attributes["bgcolor"] = "yellow"
+            }
+            if (data.isEmpty()) {
+                +"-"
+            } else {
+                +data
+            }
+        }
+    }
+
+    private fun TableTag.renderEnvironment(environment: Environment, name: String, compareTo: Environment? = null) {
+        tbody {
+            tr {
+                th {
+                    attributes["scope"] = "row"
+                    +name
+                }
+                formatComparedTableData(environment.machine.os, compareTo?.machine?.os)
+                formatComparedTableData(environment.machine.cpu, compareTo?.machine?.cpu)
+                formatComparedTableData(environment.jdk.version, compareTo?.jdk?.version)
+                formatComparedTableData(environment.jdk.vendor, compareTo?.jdk?.vendor)
+            }
+        }
+    }
+
+    private fun BodyTag.renderEnvironmentTable(environments: Pair<Environment, Environment?>) {
+        h4 { +"Environment" }
+        table {
+            attributes["class"] = "table table-sm table-bordered table-hover"
+            attributes["style"] = "width:initial;"
+            val firstEnvironment = environments.first
+            val secondEnvironment = environments.second
+            // Table header.
+            thead {
+                tr {
+                    th(rowspan = Natural(2)) { +"Run" }
+                    th(colspan = Natural(2)) { +"Machine" }
+                    th(colspan = Natural(2)) { +"JDK" }
+                }
+                tr {
+                    th { + "OS" }
+                    th { + "CPU" }
+                    th { + "Version"}
+                    th { + "Vendor"}
+                }
+            }
+            renderEnvironment(firstEnvironment, "First")
+            secondEnvironment?. let { renderEnvironment(it, "Second", firstEnvironment) }
+        }
+    }
+
+    private fun TableTag.renderCompiler(compiler: Compiler, name: String, compareTo: Compiler? = null) {
+        tbody {
+            tr {
+                th {
+                    attributes["scope"] = "row"
+                    +name
+                }
+                formatComparedTableData(compiler.backend.type.type, compareTo?.backend?.type?.type)
+                formatComparedTableData(compiler.backend.version, compareTo?.backend?.version)
+                formatComparedTableData(compiler.backend.flags.joinToString(), compareTo?.backend?.flags?.joinToString())
+                formatComparedTableData(compiler.kotlinVersion, compareTo?.kotlinVersion)
+            }
+        }
+    }
+
+    private fun BodyTag.renderCompilerTable(compilers: Pair<Compiler, Compiler?>) {
+        h4 { +"Compiler" }
+        table {
+            attributes["class"] = "table table-sm table-bordered table-hover"
+            attributes["style"] = "width:initial;"
+            val firstCompiler = compilers.first
+            val secondCompiler = compilers.second
+
+            // Table header.
+            thead {
+                tr {
+                    th(rowspan = Natural(2)) { +"Run" }
+                    th(colspan = Natural(3)) { +"Backend" }
+                    th(rowspan = Natural(2)) { +"Kotlin" }
+                }
+                tr {
+                    th { + "Type" }
+                    th { + "Version" }
+                    th { + "Flags"}
+                }
+            }
+            renderCompiler(firstCompiler, "First")
+            secondCompiler?. let { renderCompiler(it, "Second", firstCompiler) }
+        }
+    }
+
+    private fun TableBlock.renderBucketInfo(bucket: Collection<Any>, name: String) {
+        if (!bucket.isEmpty()) {
+            tr {
+                th {
+                    attributes["scope"] = "row"
+                    +name
+                }
+                td {
+                    +"${bucket.size}"
+                }
+            }
+        }
+    }
+
+    private fun BodyTag.renderCollapsedData(name: String, isCollapsed: Boolean = false, colorStyle: String = "",
+                                            init: BodyTag.() -> Unit) {
+        val show = if (!isCollapsed) "show" else ""
+        val tagName = name.replace(' ', '_')
+        div("accordion") {
+            div("card") {
+                attributes["style"] = "border-bottom: 1px solid rgba(0,0,0,.125);"
+                div("card-header") {
+                    attributes["id"] = "heading"
+                    attributes["style"] = "padding: 0;$colorStyle"
+                    button("btn btn-link") {
+                        attributes["data-toggle"] = "collapse"
+                        attributes["data-target"] = "#$tagName"
+                        +name
+                    }
+                }
+
+                div("collapse $show") {
+                    attributes["id"] = tagName
+                    div("accordion-inner") {
+                        init()
+                    }
+                }
+            }
+        }
+    }
+
+    private fun TableTag.renderTableFromList(list: List<String>, name: String) {
+        if (!list.isEmpty()) {
+            thead {
+                tr {
+                    th { +name }
+                }
+            }
+            list.forEach {
+                tbody {
+                    tr {
+                        td { +it }
+                    }
+                }
+            }
+        }
+    }
+
+    private fun BodyTag.renderStatusSummary(report: SummaryBenchmarksReport) {
+        h4 { +"Status Summary" }
+        val failedBenchmarks = report.failedBenchmarks
+        if (failedBenchmarks.isEmpty()) {
+            div("alert alert-success") {
+                attributes["role"] = "alert"
+                +"All benchmarks passed!"
+            }
+        } else {
+            div("alert alert-danger") {
+                attributes["role"] = "alert"
+                +"There are failed benchmarks!"
+            }
+        }
+
+        val benchmarksWithChangedStatus = report.getBenchmarksWithChangedStatus()
+        val newFailures = benchmarksWithChangedStatus
+                .filter { it.current == BenchmarkResult.Status.FAILED }
+        val newPasses = benchmarksWithChangedStatus
+                .filter { it.current == BenchmarkResult.Status.PASSED }
+
+        table {
+            attributes["class"] = "table table-sm table-striped table-hover"
+            attributes["style"] = "width:initial; font-size: 11pt;"
+            thead {
+                tr {
+                    th { +"Status Group" }
+                    th { +"#" }
+                }
+            }
+            tbody {
+                renderBucketInfo(failedBenchmarks, "Failed (total)")
+                renderBucketInfo(newFailures, "New Failures")
+                renderBucketInfo(newPasses, "New Passes")
+                renderBucketInfo(report.addedBenchmarks, "Added")
+                renderBucketInfo(report.removedBenchmarks, "Removed")
+            }
+            tfoot {
+                tr {
+                    th { +"Total becnhmarks number" }
+                    th { +"${report.benchmarksNumber}" }
+                }
+            }
+        }
+        if (!failedBenchmarks.isEmpty()) {
+            renderCollapsedData("Failures", colorStyle = "background-color: lightpink") {
+                table {
+                    attributes["class"] = "table table-sm table-striped table-hover"
+                    attributes["style"] = "width:initial; font-size: 11pt;"
+                    val newFailuresList = newFailures.map { it.field }
+                    renderTableFromList(newFailuresList, "New Failures")
+
+                    val existingFailures = failedBenchmarks.filter { it !in newFailuresList }
+                    renderTableFromList(existingFailures, "Existing Failures")
+                }
+            }
+        }
+        if (!newPasses.isEmpty()) {
+            renderCollapsedData("New Passes", colorStyle = "background-color: lightgreen") {
+                table {
+                    attributes["class"] = "table table-sm table-striped table-hover"
+                    attributes["style"] = "width:initial; font-size: 11pt;"
+                    renderTableFromList(newPasses.map { it.field }, "New Passes")
+                }
+            }
+        }
+    }
+
+    private fun BodyTag.renderPerformanceSummary(report: SummaryBenchmarksReport) {
+        if (!report.improvements.isEmpty() || !report.regressions.isEmpty()) {
+            h4 { +"Performance Summary" }
+            table {
+                attributes["class"] = "table table-sm table-striped table-hover"
+                attributes["style"] = "width:initial;"
+                thead {
+                    tr {
+                        th { +"Change" }
+                        th { +"#" }
+                        th { +"Maximum" }
+                        th { +"Geometric mean" }
+                    }
+                }
+                val maximumRegression = report.maximumRegression
+                val maximumImprovement = report.maximumImprovement
+                val regressionsGeometricMean = report.regressionsGeometricMean
+                val improvementsGeometricMean = report.improvementsGeometricMean
+                tbody {
+                    if (!report.regressions.isEmpty()) {
+                        tr {
+                            th { +"Regressions" }
+                            td { +"${report.regressions.size}" }
+                            td {
+                                attributes["bgcolor"] = ColoredCell(
+                                        maximumRegression/maxOf(maximumRegression, abs(maximumImprovement)))
+                                        .backgroundStyle
+                                +formatValue(maximumRegression, true)
+                            }
+                            td {
+                                attributes["bgcolor"] = ColoredCell(
+                                        regressionsGeometricMean/maxOf(regressionsGeometricMean,
+                                                            abs(improvementsGeometricMean)))
+                                        .backgroundStyle
+                                +formatValue(report.regressionsGeometricMean, true)
+                            }
+                        }
+                    }
+                    if (!report.improvements.isEmpty()) {
+                        tr {
+                            th { +"Improvements" }
+                            td { +"${report.improvements.size}" }
+                            td {
+                                attributes["bgcolor"] = ColoredCell(
+                                        maximumImprovement/maxOf(maximumRegression, abs(maximumImprovement)))
+                                        .backgroundStyle
+                                +formatValue(report.maximumImprovement, true)
+                            }
+                            td {
+                                attributes["bgcolor"] = ColoredCell(
+                                        improvementsGeometricMean/maxOf(regressionsGeometricMean,
+                                                        abs(improvementsGeometricMean)))
+                                        .backgroundStyle
+                                +formatValue(report.improvementsGeometricMean, true)
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    private fun TableBlock.renderBenchmarksDetails(fullSet: Map<String, SummaryBenchmark>,
+                                                   bucket: Map<String, ScoreChange>? = null, rowStyle: String? = null) {
+        if (bucket != null) {
+            // There are changes in performance.
+            // Output changed benchmarks.
+            for ((name, change) in bucket) {
+                tr {
+                    rowStyle?. let {
+                        attributes["style"] = rowStyle
+                    }
+                    th { +name }
+                    td { +"${fullSet.getValue(name).first}" }
+                    td { +"${fullSet.getValue(name).second}" }
+                    td {
+                        attributes["bgcolor"] = ColoredCell(change.first.mean / abs(bucket.values.first().first.mean))
+                                .backgroundStyle
+                        +"${change.first.toString() + " %"}"
+                    }
+                    td {
+                        val scaledRatio = if (change.first.mean < 0)
+                                            (1 - change.second.mean) / (-1 + bucket.values.first().second.mean)
+                                            else (change.second.mean / bucket.values.first().second.mean)
+                        attributes["bgcolor"] = ColoredCell(scaledRatio).backgroundStyle
+                        +"${change.second}"
+                    }
+                }
+            }
+        } else {
+            // Output all values without performance changes.
+            val placeholder = "-"
+            for ((name, value) in fullSet) {
+                tr {
+                    th { +name }
+                    td { +"${value.first?.toString() ?: placeholder}" }
+                    td { +"${value.second?.toString() ?: placeholder}" }
+                    td { +placeholder }
+                    td { +placeholder }
+                }
+            }
+        }
+    }
+
+    private fun BodyTag.renderPerformanceDetails(report: SummaryBenchmarksReport, onlyChanges: Boolean) {
+        if (onlyChanges) {
+            if (report.regressions.isEmpty() && report.improvements.isEmpty()) {
+                div("alert alert-success") {
+                    attributes["role"] = "alert"
+                    +"All becnhmarks are stable!"
+                }
+            }
+        }
+
+        table {
+            attributes["id"] = "result"
+            attributes["class"] = "table table-striped table-bordered"
+            thead {
+                tr {
+                    th { +"Benchmark" }
+                    th { +"First score" }
+                    th { +"Second score" }
+                    th { +"Percent" }
+                    th { +"Ratio" }
+                }
+            }
+            val geoMeanChangeMap = report.geoMeanScoreChange?.
+                    let { mapOf(report.geoMeanBenchmark.first!!.meanBenchmark.name to report.geoMeanScoreChange!!) }
+
+            tbody {
+                renderBenchmarksDetails(
+                        mutableMapOf(report.geoMeanBenchmark.first!!.meanBenchmark.name to report.geoMeanBenchmark),
+                        geoMeanChangeMap, "border-bottom: 2.3pt solid black; border-top: 2.3pt solid black")
+                renderBenchmarksDetails(report.mergedReport, report.regressions)
+                renderBenchmarksDetails(report.mergedReport, report.improvements)
+                if (!onlyChanges) {
+                    // Print all remaining results.
+                    renderBenchmarksDetails(report.mergedReport.filter { it.key !in report.regressions.keys &&
+                            it.key !in report.improvements.keys })
+                }
+            }
+        }
+    }
+
+    data class Color(val red: Double, val green: Double, val blue: Double) {
+        operator fun times(coefficient: Double) =
+                Color(red * coefficient, green * coefficient, blue * coefficient)
+
+        operator fun plus(other: Color) =
+                Color(red + other.red, green + other.green, blue + other.blue)
+
+        override fun toString() =
+            "#" + buildString {
+                listOf(red, green, blue).forEach {
+                    append(clamp((it * 255).toInt(), 0, 255).toString(16).padStart(2, '0'))
+                }
+            }
+    }
+
+    class ColoredCell(val scaledValue: Double, val reverse: Boolean = false) {
+        val value: Double
+        val neutralColor = Color(1.0,1.0 , 1.0)
+        val negativeColor = Color(0.0, 1.0, 0.0)
+        val positiveColor = Color(1.0, 0.0, 0.0)
+
+        init {
+            value = if (abs(scaledValue) <= 1.0) scaledValue else error ("Value should be scaled in range [-1.0; 1.0]")
+        }
+
+        val backgroundStyle: String
+            get() = getColor().toString()
+
+        fun getColor(): Color {
+            val currentValue = clamp(value, -1.0, 1.0)
+            val cellValue = if (reverse) -currentValue else currentValue
+            val baseColor = if (cellValue < 0) negativeColor else positiveColor
+            // Smooth mapping to put first 20% of change into 50% of range,
+            // although really we should compensate for luma.
+            val color = sin((abs(cellValue).pow(.477)) * kotlin.math.PI * .5)
+            return linearInterpolation(neutralColor, baseColor, color)
+        }
+
+        private fun linearInterpolation(a: Color, b: Color, coefficient: Double): Color {
+            val reversedCoefficient = 1.0 - coefficient
+            return a * reversedCoefficient + b * coefficient
+        }
+    }
+}

tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt

@@ -22,9 +22,23 @@ import org.jetbrains.report.BenchmarkResult
 
 import kotlin.math.abs
 
-// Common interface for printing report in different formats.
-interface Render {
-    fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean = false): String
+// Base class for printing report in different formats.
+abstract class Render {
+
+    companion object {
+        fun getRenderByName(name: String) =
+            when (name) {
+                "text" -> TextRender()
+                "html" -> HTMLRender()
+                "teamcity" -> TeamCityStatisticsRender()
+                else -> error("Unknown render $name")
+            }
+    }
+
+    abstract val name: String
+
+    abstract fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean = false): String
+
     // Print report using render.
     fun print(report: SummaryBenchmarksReport, onlyChanges: Boolean = false, outputFile: String? = null) {
         val content = render(report, onlyChanges)
@@ -32,10 +46,16 @@ interface Render {
             writeToFile(outputFile, content)
         } ?: println(content)
     }
+
+    protected fun formatValue(number: Double, isPercent: Boolean = false): String =
+            if (isPercent) number.format(2) + "%" else number.format()
 }
 
 // Report render to text format.
-class TextRender: Render {
+class TextRender: Render() {
+    override val name: String
+        get() = "text"
+
     private val content = StringBuilder()
     private val headerSeparator = "================="
     private val wideColumnWidth = 50
@@ -50,7 +70,7 @@ class TextRender: Render {
         renderEnvChanges(report.kotlinChanges, "Compiler")
         renderStatusSummary(report)
         renderStatusChangesDetails(report.getBenchmarksWithChangedStatus())
-        renderPerformanceSummary(report.regressions, report.improvements)
+        renderPerformanceSummary(report)
         renderPerformanceDetails(report, onlyChanges)
         return content.toString()
     }
@@ -76,26 +96,6 @@ class TextRender: Render {
         }
     }
 
-    private fun printPerformanceBucket(bucket: Map<String, ScoreChange>, bucketName: String) {
-        if (!bucket.isEmpty()) {
-            var percentsList = bucket.values.map{ it.first.mean }
-            // Maps of regressions and improvements are sorted.
-            val maxValue = percentsList.first()
-            var geomeanValue: Double
-            if (percentsList.first() > 0.0) {
-                geomeanValue = geometricMean(percentsList)
-            } else {
-                // Geometric mean can be counted on positive numbers.
-                val precision = abs(maxValue) + 1
-                percentsList = percentsList.map{it + precision}
-                geomeanValue = geometricMean(percentsList) - precision
-            }
-
-            append("$bucketName: Maximum = ${formatValue(maxValue, true)}," +
-                    " Geometric mean = ${formatValue(geomeanValue, true)}")
-        }
-    }
-
     fun renderStatusChangesDetails(benchmarksWithChangedStatus: List<FieldChange<BenchmarkResult.Status>>) {
         if (!benchmarksWithChangedStatus.isEmpty()) {
             append("Changes in status")
@@ -127,20 +127,22 @@ class TextRender: Render {
         append()
     }
 
-    fun renderPerformanceSummary(regressions: Map<String, ScoreChange>,
-                                          improvements: Map<String, ScoreChange>) {
-        if (!regressions.isEmpty() || !improvements.isEmpty()) {
+    fun renderPerformanceSummary(report: SummaryBenchmarksReport) {
+        if (!report.regressions.isEmpty() || !report.improvements.isEmpty()) {
             append("Performance summary")
             append(headerSeparator)
-            printPerformanceBucket(regressions, "Regressions")
-            printPerformanceBucket(improvements, "Improvements")
+            if (!report.regressions.isEmpty()) {
+                append("Regressions: Maximum = ${formatValue(report.maximumRegression, true)}," +
+                        " Geometric mean = ${formatValue(report.regressionsGeometricMean, true)}")
+            }
+            if (!report.improvements.isEmpty()) {
+                append("Improvements: Maximum = ${formatValue(report.maximumImprovement, true)}," +
+                        " Geometric mean = ${formatValue(report.improvementsGeometricMean, true)}")
+            }
             append()
         }
     }
 
-    private fun formatValue(number: Double, isPercent: Boolean = false): String =
-            if (isPercent) format(number, 2) + "%" else format(number)
-
     private fun formatColumn(content:String, isWide: Boolean = false): String =
             content.padEnd(if (isWide) wideColumnWidth else standardColumnWidth, ' ')
 
@@ -151,8 +153,8 @@ class TextRender: Render {
             // Output changed benchmarks.
             for ((name, change) in bucket) {
                 append(formatColumn(name, true) +
-                        formatColumn(fullSet[name]!!.first!!.toString()) +
-                        formatColumn(fullSet[name]!!.second!!.toString()) +
+                        formatColumn(fullSet.getValue(name).first.toString()) +
+                        formatColumn(fullSet.getValue(name).second.toString()) +
                         formatColumn(change.first.toString() + " %") +
                         formatColumn(change.second.toString()))
             }
@@ -174,8 +176,8 @@ class TextRender: Render {
 
     private fun printPerformanceTableHeader(): Int {
         val wideColumns = listOf(formatColumn("Benchmark", true))
-        val standardColumns = listOf(formatColumn("Current score"),
-                formatColumn("Previous score"),
+        val standardColumns = listOf(formatColumn("First score"),
+                formatColumn("Second score"),
                 formatColumn("Percent"),
                 formatColumn("Ratio"))
         val tableWidth = wideColumnWidth * wideColumns.size + standardColumnWidth * standardColumns.size

tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/TeamCityStatisticsRender.kt

@@ -21,7 +21,10 @@ import org.jetbrains.analyzer.*
 import org.jetbrains.report.BenchmarkResult
 
 // Report render to text format.
-class TeamCityStatisticsRender: Render {
+class TeamCityStatisticsRender: Render() {
+    override val name: String
+        get() = "teamcity"
+
     private var content = StringBuilder()
 
     override fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean): String {

tools/benchmarksAnalyzer/src/nativeInterop/cinterop/libcurl.def

@@ -0,0 +1,5 @@
+headers = curl/curl.h
+headerFilter = curl/*
+linkerOpts.osx = -L/opt/local/lib -L/usr/local/opt/curl/lib -lcurl
+linkerOpts.linux = -L/usr/lib64 -L/usr/lib/x86_64-linux-gnu -lcurl
+linkerOpts.mingw = -lcurl