From e2ad488f284303db39983eff8382c805c7d5112e Mon Sep 17 00:00:00 2001 From: LepilkinaElena Date: Tue, 29 Jan 2019 14:16:59 +0300 Subject: [PATCH] Slack performance reporter (#2527) --- performance/build.gradle | 31 +- performance/buildSrc/build.gradle | 1 + .../buildSrc/src/main/kotlin/MPPTools.kt | 40 +- .../src/main/kotlin/RegressionsReporter.kt | 243 +++++++ performance/gradle.properties | 6 +- .../org/jetbrains/report/BenchmarksReport.kt | 94 +-- .../report/json/ConvertedFromJson.kt | 47 ++ .../org/jetbrains/report/json/JsonElement.kt | 3 +- tools/benchmarksAnalyzer/build.gradle | 46 +- .../org/jetbrains/analyzer/Utils.kt | 43 +- .../org/jetbrains/analyzer/Utils.kt | 65 +- .../src/main/kotlin/main.kt | 96 ++- .../org/jetbrains/analyzer/Statistics.kt | 9 +- .../analyzer/SummaryBenchmarksReport.kt | 92 ++- .../kotlin/org/jetbrains/analyzer/Utils.kt | 5 +- .../kotlin/org/jetbrains/kliopt/KliOpt.kt | 169 +++-- .../org/jetbrains/renders/HTMLRender.kt | 660 ++++++++++++++++++ .../kotlin/org/jetbrains/renders/Render.kt | 76 +- .../renders/TeamCityStatisticsRender.kt | 5 +- .../src/nativeInterop/cinterop/libcurl.def | 5 + 20 files changed, 1496 insertions(+), 240 deletions(-) create mode 100644 performance/buildSrc/src/main/kotlin/RegressionsReporter.kt create mode 100644 tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/json/ConvertedFromJson.kt create mode 100644 tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/HTMLRender.kt create mode 100644 tools/benchmarksAnalyzer/src/nativeInterop/cinterop/libcurl.def diff --git a/performance/build.gradle b/performance/build.gradle index 45bc74c10d3..b1db3080952 100644 --- a/performance/build.gradle +++ b/performance/build.gradle @@ -113,6 +113,7 @@ task konanJsonReport { 'compileTime' : nativeCompileTime] def output = MPPTools.createJsonReport(properties) new File("${buildDir.absolutePath}/${nativeJson}").write(output) + uploadBenchmarkResultToBintray(nativeJson) } } @@ -126,6 +127,7 @@ task jvmJsonReport { 'compileTime' : jvmCompileTime] def output = MPPTools.createJsonReport(properties) new File("${buildDir.absolutePath}/${jvmJson}").write(output) + uploadBenchmarkResultToBintray(jvmJson) } } @@ -146,14 +148,41 @@ private def getCommonProperties() { 'kotlinVersion': "${kotlinVersion}".toString()] } +private def uploadBenchmarkResultToBintray(String fileName) { + def teamcityConfig = System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE") + if (teamcityConfig) { + def buildProperties = new Properties() + buildProperties.load(new FileInputStream(teamcityConfig)) + def user = buildProperties.getProperty("bintray.user") + def password = buildProperties.getProperty("bintray.apikey") + def buildNumber = buildProperties.getProperty("build.number") + def target = System.getProperty("os.name").replaceAll("\\s", "") + MPPTools.uploadFileToBintray("${bintrayUrl}", "${bintrayRepo}", + buildNumber, "${bintrayPackage}", "${target}/${buildNumber}/${fileName}", + "${buildDir.absolutePath}/${fileName}", user, password) + } +} + task bench(type:Exec) { dependsOn jvmRun dependsOn konanRun def extension = MPPTools.getNativeProgramExtension() def analyzer = MPPTools.findFile("${analyzerTool}${extension}", "${rootBuildDirectory}/${analyzerToolDirectory}") if (analyzer != null) { - commandLine "${analyzer}", "${buildDir.absolutePath}/${nativeJson}", "${buildDir.absolutePath}/${jvmJson}" + commandLine "${analyzer}", "-r", "text", "-r", "teamcity", "${buildDir.absolutePath}/${nativeJson}", "${buildDir.absolutePath}/${jvmJson}" } else { println("No analyzer $analyzerTool found in subdirectories of ${rootBuildDirectory}/${analyzerToolDirectory}") } +} + +// Produce and send slack report. +task slackReport(type: RegressionsReporter) { + // Create folder for report (root Kotlin project settings make create report in separate folder). + def reportDirectory = outputReport.substring(0, outputReport.lastIndexOf("/")) + mkdir reportDirectory + currentBenchmarksReportFile = "${buildDir.absolutePath}/${nativeJson}" + analyzer = MPPTools.findFile("${analyzerTool}${MPPTools.getNativeProgramExtension()}", + "${rootBuildDirectory}/${analyzerToolDirectory}") + htmlReport = outputReport + defaultBranch = project.findProperty('kotlin.native.default.branch') ?: "master" } \ No newline at end of file diff --git a/performance/buildSrc/build.gradle b/performance/buildSrc/build.gradle index ca1a9e7226e..260a959986a 100644 --- a/performance/buildSrc/build.gradle +++ b/performance/buildSrc/build.gradle @@ -30,6 +30,7 @@ dependencies { compileOnly gradleApi() implementation "org.jetbrains.kotlin:kotlin-gradle-plugin" implementation "org.jetbrains.kotlin:kotlin-stdlib-common:$kotlinVersion" + compile group: 'com.ullink.slack', name: 'simpleslackapi', version: '1.2.0' } sourceSets.main.kotlin.srcDirs = ["src", "$projectDir/../../tools/benchmarks/shared/src"] \ No newline at end of file diff --git a/performance/buildSrc/src/main/kotlin/MPPTools.kt b/performance/buildSrc/src/main/kotlin/MPPTools.kt index d0086b73d43..528e499234c 100644 --- a/performance/buildSrc/src/main/kotlin/MPPTools.kt +++ b/performance/buildSrc/src/main/kotlin/MPPTools.kt @@ -16,6 +16,12 @@ import org.jetbrains.report.* import org.jetbrains.report.json.* import java.nio.file.Paths import java.io.File +import java.io.FileInputStream +import java.io.BufferedOutputStream +import java.io.BufferedInputStream +import java.net.HttpURLConnection +import java.net.URL +import java.util.Base64 /* * This file includes short-cuts that may potentially be implemented in Kotlin MPP Gradle plugin in the future. @@ -92,6 +98,38 @@ fun createJsonReport(projectProperties: Map): String { fun findFile(fileName: String, directory: String): String? = File(directory).walkBottomUp().find { it.name == fileName }?.getAbsolutePath() +fun uploadFileToBintray(url: String, project: String, version: String, packageName: String, bintrayFilePath: String, + filePath: String, username: String? = null, password: String? = null) { + val uploadUrl = "$url/$project/$packageName/$version/$bintrayFilePath?publish=1" + sendUploadRequest(uploadUrl, filePath, username, password) +} + +fun sendUploadRequest(url: String, fileName: String, username: String? = null, password: String? = null) { + val uploadingFile = File(fileName) + val connection = URL(url).openConnection() as HttpURLConnection + connection.doOutput = true + connection.doInput = true + connection.requestMethod = "PUT" + connection.setRequestProperty("Content-type", "text/plain") + if (username != null && password != null) { + val auth = Base64.getEncoder().encode((username + ":" + password).toByteArray()).toString(Charsets.UTF_8) + connection.addRequestProperty("Authorization", "Basic $auth") + } + + try { + connection.connect() + BufferedOutputStream(connection.outputStream).use { output -> + BufferedInputStream(FileInputStream(uploadingFile)).use { input -> + input.copyTo(output) + } + } + val response = connection.responseMessage + println("Upload request ended with ${connection.responseCode} - $response") + } catch (t: Throwable) { + error("Couldn't upload file $fileName to $url") + } +} + // A short-cut to add a Kotlin/Native run task. @JvmOverloads fun createRunTask( @@ -109,7 +147,7 @@ fun getJvmCompileTime(programName: String): BenchmarkResult = TaskTimerListener.getBenchmarkResult(programName, listOf("compileKotlinMetadata", "jvmJar")) fun getNativeCompileTime(programName: String): BenchmarkResult = - TaskTimerListener.getBenchmarkResult(programName, listOf("compileKotlinNative", "linkReleaseExecutableNative")) + TaskTimerListener.getBenchmarkResult(programName, listOf("compileKotlinNative", "linkMainReleaseExecutableNative")) // Class time tracker for all tasks. class TaskTimerListener: TaskExecutionListener { diff --git a/performance/buildSrc/src/main/kotlin/RegressionsReporter.kt b/performance/buildSrc/src/main/kotlin/RegressionsReporter.kt new file mode 100644 index 00000000000..8b68bda6f3b --- /dev/null +++ b/performance/buildSrc/src/main/kotlin/RegressionsReporter.kt @@ -0,0 +1,243 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +import groovy.lang.Closure +import org.gradle.api.Action +import org.gradle.api.DefaultTask +import org.gradle.api.Project +import org.gradle.api.Task +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.TaskAction + +import com.ullink.slack.simpleslackapi.impl.SlackSessionFactory + +import org.jetbrains.report.json.* + +import java.io.FileInputStream +import java.io.IOException +import java.io.File +import java.util.concurrent.TimeUnit +import java.net.HttpURLConnection +import java.net.URL +import java.util.Base64 +import java.util.Properties + +// Run command line from string. +fun String.runCommand(workingDir: File = File("."), + timeoutAmount: Long = 60, + timeoutUnit: TimeUnit = TimeUnit.SECONDS): String { + return try { + ProcessBuilder(*this.split("\\s".toRegex()).toTypedArray()) + .directory(workingDir) + .redirectOutput(ProcessBuilder.Redirect.PIPE) + .redirectError(ProcessBuilder.Redirect.PIPE) + .start().apply { + waitFor(timeoutAmount, timeoutUnit) + }.inputStream.bufferedReader().readText() + } catch (e: IOException) { + error("Couldn't run command $this") + } +} + +data class Commit(val revision: String, val developer: String, val webUrlWithDescription: String) + +// List of commits. +class CommitsList(data: JsonElement): ConvertedFromJson { + + val commits: List + + init { + if (data !is JsonObject) { + error("Commits description is expected to be a json object!") + } + val changesElement = data.getOptionalField("change") + commits = changesElement?.let { + if (changesElement !is JsonArray) { + error("Change field is expected to be an array. Please, check source.") + } + changesElement.jsonArray.map { + with(it as JsonObject) { + Commit(elementToString(getRequiredField("version"), "version"), + elementToString(getRequiredField("username"), "username"), + elementToString(getRequiredField("webUrl"), "webUrl") + ) + } + } + } ?: listOf() + } +} + +/** + * Task to produce regressions report and send it to slack. Requires a report with current benchmarks result + * and path to analyzer tool + * + * @property currentBenchmarksReportFile path to file with becnhmarks result + * @property analyzer path to analyzer tool + * @property htmlReport name of result html report + * @property defaultBranch name of defaukt branch + */ +open class RegressionsReporter : DefaultTask() { + + val teamCityUrl = "http://buildserver.labs.intellij.net" + val slackUsers = mapOf( + "olonho" to "nikolay.igotti", + "nikolay.igotti" to "nikolay.igotti", + "ilya.matveev" to "ilya.matveev", + "ilmat192" to "ilya.matveev", + "vasily.v.levchenko" to "minamoto", + "vasily.levchenko" to "minamoto", + "alexander.gorshenev" to "alexander.gorshenev", + "igor.chevdar" to "igor.chevdar", + "pavel.punegov" to "Pavel Punegov", + "dmitriy.dolovov" to "dmitriy.dolovov", + "svyatoslav.scherbina" to "svyatoslav.scherbina", + "sbogolepov" to "sergey.bogolepov", + "Alexey.Zubakov" to "Alexey.Zubakov", + "kirill.shmakov" to "kirill.shmakov", + "elena.lepilkina" to "elena.lepilkina") + + @Input + lateinit var currentBenchmarksReportFile: String + + @Input + lateinit var analyzer: String + + @Input + lateinit var htmlReport: String + + @Input + lateinit var defaultBranch: String + + private fun tabUrl(buildId: String, buildTypeId: String, tab: String) = + "$teamCityUrl/viewLog.html?buildId=$buildId&buildTypeId=$buildTypeId&tab=$tab" + + private fun testReportUrl(buildId: String, buildTypeId: String) = + tabUrl(buildId, buildTypeId, "testsInfo") + + private fun buildsUrl(buildLocator: String) = + "$teamCityUrl/app/rest/builds/?locator=$buildLocator" + + private fun previousBuildLocator(buildTypeId: String, branchName: String) = + "buildType:id:$buildTypeId,branch:name:$branchName,status:SUCCESS,state:finished,count:1" + + private fun changesListUrl(buildLocator: String) = + "$teamCityUrl/app/rest/changes/?locator=build:$buildLocator" + + private fun sendGetRequest(url: String, username: String? = null, password: String? = null) : String { + val connection = URL(url).openConnection() as HttpURLConnection + if (username != null && password != null) { + val auth = Base64.getEncoder().encode((username + ":" + password).toByteArray()).toString(Charsets.UTF_8) + connection.addRequestProperty("Authorization", "Basic $auth") + } + connection.setRequestProperty("Accept", "application/json"); + connection.connect() + return connection.inputStream.use { it.reader().use { reader -> reader.readText() } } + } + + private fun getBuild(buildLocator: String, user: String, password: String) = + try { + sendGetRequest(buildsUrl(buildLocator), user, password) + } catch (t: Throwable) { + error("Try to get build! TeamCity is unreachable!") + } + + private fun getBuildProperty(buildJsonDescription: String, property: String) = + with(JsonTreeParser.parse(buildJsonDescription) as JsonObject) { + if (getPrimitive("count").int == 0) { + error("No build information on TeamCity for $buildJsonDescription!") + } + (getArray("build").getObject(0).getPrimitive(property) as JsonLiteral).unquoted() + } + + private fun getCommits(buildLocator: String, user: String, password: String): CommitsList { + val changes = try { + sendGetRequest(changesListUrl(buildLocator), user, password) + } catch (t: Throwable) { + error("Try to get commits! TeamCity is unreachable!") + } + return CommitsList(JsonTreeParser.parse(changes)) + } + + @TaskAction + fun run() { + // Get TeamCity properties. + val teamcityConfig = System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE") ?: + error("Can't load teamcity config!") + + val buildProperties = Properties() + buildProperties.load(FileInputStream(teamcityConfig)) + val buildId = buildProperties.getProperty("teamcity.build.id") + val buildTypeId = buildProperties.getProperty("teamcity.buildType.id") + val buildNumber = buildProperties.getProperty("build.number") + val user = buildProperties.getProperty("teamcity.auth.userId") + val password = buildProperties.getProperty("teamcity.auth.password") + + // Get branch. + val currentBuild = getBuild("id:$buildId", user, password) + val branch = getBuildProperty(currentBuild,"branchName") + + val testReportUrl = testReportUrl(buildId, buildTypeId) + + // Get previous build on branch. + val builds = getBuild(previousBuildLocator(buildTypeId,branch), user, password) + val previousBuildsExist = (JsonTreeParser.parse(builds) as JsonObject).getPrimitive("count").int != 0 + + // Get changes description. + val changesList = getCommits("id:$buildId", user, password) + val changesInfo = "*Changes* in branch *$branch:*\n" + buildString { + changesList.commits.forEach { + append(" - Change ${it.revision} by <@${it.developer}> (details: ${it.webUrlWithDescription})\n") + } + } + + println("defaul Branch: $defaultBranch") + + // If branch differs from default and it's first build compare to master, otherwise compare to previous build on branch. + val compareToBranch = if (previousBuildsExist) branch else defaultBranch + + // File name on bintray is the same as current. + val bintrayFileName = currentBenchmarksReportFile.substringAfterLast("/") + + // Get compare to build. + val compareToBuild = getBuild(previousBuildLocator(buildTypeId, compareToBranch), user, password) + val compareToBuildLink = getBuildProperty(compareToBuild,"webUrl") + val compareToBuildNumber = getBuildProperty(compareToBuild,"number") + val target = System.getProperty("os.name").replace("\\s".toRegex(), "") + + // Generate comparison report. + val output = "$analyzer -r html $currentBenchmarksReportFile bintray:$compareToBuildNumber:$target:$bintrayFileName -o $htmlReport" + .runCommand() + + if (output.contains("Uncaught exception")) { + error("Error during comparasion of $currentBenchmarksReportFile and " + + "bintray:$compareToBuildNumber:$target:$bintrayFileName with $analyzer! " + + "Please check files existance and their correctness.") + } + + val reportLink = tabUrl(buildId, buildTypeId, "report_project170_Benchmarks") + + val title = "\n*Performance report for target $target (build $buildNumber)* - $reportLink\n" + val header = "$title\n$changesInfo\n\nCompare to build $compareToBuildNumber: $compareToBuildLink\n\n" + val footer = "*Benchmarks statistics:* $testReportUrl" + val message = "$header\n$footer\n" + + // Send to channel or user directly. + val session = SlackSessionFactory.createWebSocketSlackSession(buildProperties.getProperty("konan-reporter-token")) + session.connect() + + if (branch == defaultBranch) { + val channel = session.findChannelByName(buildProperties.getProperty("konan-channel-name")) + session.sendMessage(channel, message) + } else { + changesList.commits.filter { it.developer in slackUsers }. map { it.developer } + .toSet().forEach { + val slackUser = session.findUserByUserName(slackUsers[it]) + session.sendMessageToUser(slackUser, message, null) + + } + } + session.disconnect() + } +} \ No newline at end of file diff --git a/performance/gradle.properties b/performance/gradle.properties index 3a760731811..6b701828f24 100644 --- a/performance/gradle.properties +++ b/performance/gradle.properties @@ -9,4 +9,8 @@ jvmTextReport = jvmReport.txt nativeJson = nativeReport.json jvmJson = jvmReport.json analyzerTool = benchmarksAnalyzer -analyzerToolDirectory = tools/benchmarksAnalyzer/build/bin \ No newline at end of file +analyzerToolDirectory = tools/benchmarksAnalyzer/build/bin +outputReport = ../report/report.html +bintrayUrl = https://api.bintray.com/content/lepilkinaelena +bintrayRepo = KotlinNativePerformance +bintrayPackage = jsonReports \ No newline at end of file diff --git a/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/BenchmarksReport.kt b/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/BenchmarksReport.kt index 161eaf30d23..a807ae5309c 100644 --- a/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/BenchmarksReport.kt +++ b/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/BenchmarksReport.kt @@ -30,50 +30,6 @@ interface JsonSerializable { } } -// Entity can be created from json description. -interface ConvertedFromJson { - fun getRequiredField(data: JsonObject, fieldName: String): JsonElement { - return data.getOrNull(fieldName) ?: error("Field '$fieldName' doesn't exist in '$data'. Please, check origin files.") - } - - fun getOptionalField(data: JsonObject, fieldName: String): JsonElement? { - return data.getOrNull(fieldName) - } - - // Parse json array to list. - // Takes function to convert elements in array to expected type. - fun arrayToList(array: JsonArray, convert: JsonArray.(Int) -> T?): List { - var results = mutableListOf() - var index = 0 - var current: T? = array.convert(index) - while (current != null) { - results.add(current) - index++ - current = array.convert(index) - } - return results - } - - // Methods for conversion to expected type with checks of possibility of such conversions. - fun elementToDouble(element: JsonElement, name: String): Double = - if (element is JsonPrimitive) - element.double - else - error("Field '$name' in '$element' is expected to be a double number. Please, check origin files.") - - fun elementToInt(element: JsonElement, name: String): Int = - if (element is JsonPrimitive) - element.int - else - error("Field '$name' in '$element' is expected to be an integer number. Please, check origin files.") - - fun elementToString(element: JsonElement, name:String): String = - if (element is JsonLiteral) - element.unquoted() - else - error("Field '$name' in '$element' is expected to be a string. Please, check origin files.") -} - interface EntityFromJsonFactory: ConvertedFromJson { fun create(data: JsonElement): T } @@ -85,9 +41,9 @@ class BenchmarksReport(val env: Environment, benchmarksList: List { override fun create(data: JsonElement): BenchmarksReport { if (data is JsonObject) { - val env = Environment.create(getRequiredField(data, "env")) - val benchmarksObj = getRequiredField(data, "benchmarks") - val compiler = Compiler.create(getRequiredField(data, "kotlin")) + val env = Environment.create(data.getRequiredField("env")) + val benchmarksObj = data.getRequiredField("benchmarks") + val compiler = Compiler.create(data.getRequiredField("kotlin")) val benchmarksList = parseBenchmarksArray(benchmarksObj) return BenchmarksReport(env, benchmarksList, compiler) } else { @@ -98,11 +54,7 @@ class BenchmarksReport(val env: Environment, benchmarksList: List { if (data is JsonArray) { - return arrayToList(data.jsonArray, { index -> - if (this.getObjectOrNull(index) != null) - BenchmarkResult.create(this.getObjectOrNull(index) as JsonObject) - else null - }) + return data.jsonArray.map { BenchmarkResult.create(it as JsonObject) } } else { error("Benchmarks field is expected to be an array. Please, check origin files.") } @@ -137,8 +89,8 @@ data class Compiler(val backend: Backend, val kotlinVersion: String): JsonSerial companion object: EntityFromJsonFactory { override fun create(data: JsonElement): Compiler { if (data is JsonObject) { - val backend = Backend.create(getRequiredField(data, "backend")) - val kotlinVersion = elementToString(getRequiredField(data, "kotlinVersion"), "kotlinVersion") + val backend = Backend.create(data.getRequiredField("backend")) + val kotlinVersion = elementToString(data.getRequiredField("kotlinVersion"), "kotlinVersion") return Compiler(backend, kotlinVersion) } else { @@ -154,16 +106,14 @@ data class Compiler(val backend: Backend, val kotlinVersion: String): JsonSerial companion object: EntityFromJsonFactory { override fun create(data: JsonElement): Backend { if (data is JsonObject) { - val typeElement = getRequiredField(data, "type") + val typeElement = data.getRequiredField("type") if (typeElement is JsonLiteral) { val type = backendTypeFromString(typeElement.unquoted()) ?: error("Backend type should be 'jvm' or 'native'") - val version = elementToString(getRequiredField(data, "version"), "version") - val flagsArray = getOptionalField(data, "flags") + val version = elementToString(data.getRequiredField("version"), "version") + val flagsArray = data.getOptionalField("flags") var flags: List = emptyList() if (flagsArray != null && flagsArray is JsonArray) { - flags = arrayToList(flagsArray.jsonArray, { index -> - this.getPrimitiveOrNull(index)?.toString() - }) + flags = flagsArray.jsonArray.map { it.toString() } } return Backend(type, version, flags) } else { @@ -211,8 +161,8 @@ data class Environment(val machine: Machine, val jdk: JDKInstance): JsonSerializ companion object: EntityFromJsonFactory { override fun create(data: JsonElement): Environment { if (data is JsonObject) { - val machine = Machine.create(getRequiredField(data, "machine")) - val jdk = JDKInstance.create(getRequiredField(data, "jdk")) + val machine = Machine.create(data.getRequiredField("machine")) + val jdk = JDKInstance.create(data.getRequiredField("jdk")) return Environment(machine, jdk) } else { @@ -226,8 +176,8 @@ data class Environment(val machine: Machine, val jdk: JDKInstance): JsonSerializ companion object: EntityFromJsonFactory { override fun create(data: JsonElement): Machine { if (data is JsonObject) { - val cpu = elementToString(getRequiredField(data, "cpu"), "cpu") - val os = elementToString(getRequiredField(data, "os"), "os") + val cpu = elementToString(data.getRequiredField("cpu"), "cpu") + val os = elementToString(data.getRequiredField("os"), "os") return Machine(cpu, os) } else { @@ -251,8 +201,8 @@ data class Environment(val machine: Machine, val jdk: JDKInstance): JsonSerializ companion object: EntityFromJsonFactory { override fun create(data: JsonElement): JDKInstance { if (data is JsonObject) { - val version = elementToString(getRequiredField(data, "version"), "version") - val vendor = elementToString(getRequiredField(data, "vendor"), "vendor") + val version = elementToString(data.getRequiredField("version"), "version") + val vendor = elementToString(data.getRequiredField("vendor"), "vendor") return JDKInstance(version, vendor) } else { @@ -291,15 +241,15 @@ class BenchmarkResult(val name: String, val status: Status, override fun create(data: JsonElement): BenchmarkResult { if (data is JsonObject) { - val name = elementToString(getRequiredField(data, "name"), "name") - val statusElement = getRequiredField(data, "status") + val name = elementToString(data.getRequiredField("name"), "name") + val statusElement = data.getRequiredField("status") if (statusElement is JsonLiteral) { val status = statusFromString(statusElement.unquoted()) ?: error("Status should be PASSED or FAILED") - val score = elementToDouble(getRequiredField(data, "score"), "score") - val runtimeInUs = elementToDouble(getRequiredField(data, "runtimeInUs"), "runtimeInUs") - val repeat = elementToInt(getRequiredField(data, "repeat"), "repeat") - val warmup = elementToInt(getRequiredField(data, "warmup"), "warmup") + val score = elementToDouble(data.getRequiredField("score"), "score") + val runtimeInUs = elementToDouble(data.getRequiredField("runtimeInUs"), "runtimeInUs") + val repeat = elementToInt(data.getRequiredField("repeat"), "repeat") + val warmup = elementToInt(data.getRequiredField("warmup"), "warmup") return BenchmarkResult(name, status, score, runtimeInUs, repeat, warmup) } else { diff --git a/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/json/ConvertedFromJson.kt b/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/json/ConvertedFromJson.kt new file mode 100644 index 00000000000..85bbb448c51 --- /dev/null +++ b/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/json/ConvertedFromJson.kt @@ -0,0 +1,47 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.report.json + +// Entity can be created from json description. +interface ConvertedFromJson { + // Methods for conversion to expected type with checks of possibility of such conversions. + fun elementToDouble(element: JsonElement, name: String): Double = + if (element is JsonPrimitive) + element.double + else + error("Field '$name' in '$element' is expected to be a double number. Please, check origin files.") + + fun elementToInt(element: JsonElement, name: String): Int = + if (element is JsonPrimitive) + element.int + else + error("Field '$name' in '$element' is expected to be an integer number. Please, check origin files.") + + fun elementToString(element: JsonElement, name:String): String = + if (element is JsonLiteral) + element.unquoted() + else + error("Field '$name' in '$element' is expected to be a string. Please, check origin files.") +} + +fun JsonObject.getRequiredField(fieldName: String): JsonElement { + return getOrNull(fieldName) ?: error("Field '$fieldName' doesn't exist in '$this'. Please, check origin files.") +} + +fun JsonObject.getOptionalField(fieldName: String): JsonElement? { + return getOrNull(fieldName) +} \ No newline at end of file diff --git a/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/json/JsonElement.kt b/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/json/JsonElement.kt index 18316b41085..20e5de45086 100644 --- a/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/json/JsonElement.kt +++ b/tools/benchmarks/shared/src/main/kotlin/org/jetbrains/report/json/JsonElement.kt @@ -324,6 +324,5 @@ data class JsonArray(val content: List) : JsonElement(), List @@ -34,9 +37,39 @@ actual fun writeToFile(fileName: String, text: String) { } } -actual fun assert(value: Boolean, lazyMessage: () -> Any) { +actual fun assert(value: Boolean, lazyMessage: () -> Any) = kotlin.assert(value, lazyMessage) + +// Create http(-s) request. +fun getHttpRequest(url: String, user: String?, password: String?): HttpURLConnection { + val connection = URL(url).openConnection() as HttpURLConnection + if (user != null && password != null) { + val auth = Base64.getEncoder().encode((user + ":" + password).toByteArray()).toString(Charsets.UTF_8) + connection.addRequestProperty("Authorization", "Basic $auth") + } + connection.setRequestProperty("Accept", "application/json") + return connection } -actual fun getEnv(variableName:String): String? = - System.getenv(variableName) +actual fun sendGetRequest(url: String, user: String?, password: String?, followLocation: Boolean) : String { + val connection = getHttpRequest(url, user, password) + connection.connect() + val responseCode = connection.responseCode + if (!followLocation) { + connection.connect() + return connection.inputStream.use { it.reader().use { reader -> reader.readText() } } + } + + // Request with redirect. + if (responseCode != HttpURLConnection.HTTP_MOVED_TEMP && + responseCode != HttpURLConnection.HTTP_MOVED_PERM && + responseCode != HttpURLConnection.HTTP_SEE_OTHER) { + error("No opportunity to redirect, but flag for redirecting to location was provided!") + } + val newUrl = connection.getHeaderField("Location") + val cookies = connection.getHeaderField("Set-Cookie") + val redirect = getHttpRequest(newUrl, user, password) + redirect.setRequestProperty("Cookie", cookies) + redirect.connect() + return redirect.inputStream.use { it.reader().use { reader -> reader.readText() } } +} \ No newline at end of file diff --git a/tools/benchmarksAnalyzer/src/main/kotlin-native/org/jetbrains/analyzer/Utils.kt b/tools/benchmarksAnalyzer/src/main/kotlin-native/org/jetbrains/analyzer/Utils.kt index fedc84c274c..6d74998d13b 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin-native/org/jetbrains/analyzer/Utils.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin-native/org/jetbrains/analyzer/Utils.kt @@ -17,8 +17,8 @@ package org.jetbrains.analyzer import platform.posix.* -import kotlinx.cinterop.CValuesRef import kotlinx.cinterop.* +import libcurl.* actual fun readFile(fileName: String): String { val file = fopen(fileName, "r") ?: error("Cannot write file '$fileName'") @@ -36,9 +36,9 @@ actual fun readFile(fileName: String): String { return text.toString() } -actual fun format(number: Double, decimalNumber: Int): String { +actual fun Double.format(decimalNumber: Int): String { var buffer = ByteArray(1024) - snprintf(buffer.refTo(0), buffer.size.toULong(), "%.${decimalNumber}f", number) + snprintf(buffer.refTo(0), buffer.size.toULong(), "%.${decimalNumber}f", this) return buffer.stringFromUtf8() } @@ -51,9 +51,62 @@ actual fun writeToFile(fileName: String, text: String) { } } -actual fun assert(value: Boolean, lazyMessage: () -> Any) { +actual fun assert(value: Boolean, lazyMessage: () -> Any) = kotlin.assert(value, lazyMessage) + +class CUrl(url: String, user: String? = null, password: String? = null, followLocation: Boolean = false) { + private val stableRef = StableRef.create(this) + + private val curl = curl_easy_init() + + init { + curl_easy_setopt(curl, CURLOPT_URL, url) + val writeData = staticCFunction(::collectResponse) + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeData) + curl_easy_setopt(curl, CURLOPT_WRITEDATA, stableRef.asCPointer()) + if (followLocation) { + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L) + } + user ?.let { + curl_easy_setopt(curl, CURLOPT_USERNAME, it) + } + password ?.let { + curl_easy_setopt(curl, CURLOPT_PASSWORD, it) + } + } + + val body = StringBuilder() + + fun fetch() { + val res = curl_easy_perform(curl) + if (res != CURLE_OK) + println("curl_easy_perform() failed: ${curl_easy_strerror(res)?.toKString()}") + } + + fun close() { + curl_easy_cleanup(curl) + stableRef.dispose() + } } -actual fun getEnv(variableName:String): String? = - getenv(variableName)?.toKString() +fun CPointer.toKString(length: Int): String { + val bytes = this.readBytes(length) + return bytes.stringFromUtf8() +} + +fun collectResponse(buffer: CPointer?, size: size_t, nitems: size_t, userdata: COpaquePointer?): size_t { + buffer ?: return 0u + userdata ?. let { + val data = buffer.toKString((size * nitems).toInt()).trim() + val curl = userdata.asStableRef().get() + curl.body.append(data) + } + return size * nitems +} + +actual fun sendGetRequest(url: String, user: String?, password: String?, followLocation: Boolean) : String { + val curl = CUrl(url, user, password, followLocation) + curl.fetch() + curl.close() + return curl.body.toString() +} diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/main.kt b/tools/benchmarksAnalyzer/src/main/kotlin/main.kt index 722acd86fcf..3443763d894 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/main.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/main.kt @@ -14,50 +14,110 @@ * limitations under the License. */ -import org.jetbrains.analyzer.getEnv +import org.jetbrains.analyzer.sendGetRequest import org.jetbrains.analyzer.readFile import org.jetbrains.analyzer.SummaryBenchmarksReport import org.jetbrains.kliopt.* -import org.jetbrains.renders.TextRender -import org.jetbrains.renders.TeamCityStatisticsRender +import org.jetbrains.renders.* import org.jetbrains.report.BenchmarksReport import org.jetbrains.report.json.JsonTreeParser +abstract class Connector { + abstract val connectorPrefix: String + + fun isCompatible(fileName: String) = + fileName.startsWith(connectorPrefix) + + abstract fun getFileContent(fileLocation: String, user: String? = null): String +} + +object BintrayConnector : Connector() { + override val connectorPrefix = "bintray:" + val bintrayUrl = "https://dl.bintray.com/content/lepilkinaelena/KotlinNativePerformance" + + override fun getFileContent(fileLocation: String, user: String?): String { + val fileParametersSize = 3 + val fileDescription = fileLocation.substringAfter(connectorPrefix) + val fileParameters = fileDescription.split(':', limit = fileParametersSize) + if (fileParameters.size != fileParametersSize) { + error("To get file from bintray, please, specify, build number from TeamCity and target" + + " in format bintray:build_number:target:filename") + } + val (buildNumber, target, fileName) = fileParameters + val accessFileUrl = "$bintrayUrl/$target/$buildNumber/$fileName" + return sendGetRequest(accessFileUrl, followLocation = true) + } +} + +object TeamCityConnector: Connector() { + override val connectorPrefix = "teamcity:" + val teamCityUrl = "http://buildserver.labs.intellij.net" + + override fun getFileContent(fileLocation: String, user: String?): String { + val fileDescription = fileLocation.substringAfter(connectorPrefix) + val buildLocator = fileDescription.substringBeforeLast(':') + val fileName = fileDescription.substringAfterLast(':') + if (fileDescription == fileLocation || + fileDescription == buildLocator || fileName == fileDescription) { + error("To get file from bintray, please, specify, build number from TeamCity and target" + + " in format teamcity:build_locator:filename") + } + val accessFileUrl = "$teamCityUrl/app/rest/builds/$buildLocator/artifacts/content/$fileName" + val userName = user?.substringBefore(':') + val password = user?.substringAfter(':') + return sendGetRequest(accessFileUrl, userName, password) + } +} + +fun getFileContent(fileName: String, user: String? = null): String { + return when { + BintrayConnector.isCompatible(fileName) -> BintrayConnector.getFileContent(fileName, user) + TeamCityConnector.isCompatible(fileName) -> TeamCityConnector.getFileContent(fileName, user) + else -> readFile(fileName) + } +} + fun main(args: Array) { val options = listOf( - OptionDescriptor(ArgType.STRING, "output", "o", "Output file"), - OptionDescriptor(ArgType.DOUBLE, "eps", "e", "Meaningful performance changes", "0.5"), - OptionDescriptor(ArgType.BOOLEAN, "short", "s", "Show short version of report", "false") + OptionDescriptor(ArgType.String(), "output", "o", "Output file"), + OptionDescriptor(ArgType.Double(), "eps", "e", "Meaningful performance changes", "0.5"), + OptionDescriptor(ArgType.Boolean(), "short", "s", "Show short version of report", "false"), + OptionDescriptor(ArgType.Choice(listOf("text", "html", "teamcity")), + "renders", "r", "Renders for showing information", "text", isMultiple = true), + OptionDescriptor(ArgType.String(), "user", "u", "User access information for authorization") ) val arguments = listOf( - ArgDescriptor(ArgType.STRING, "mainReport", "Main report for analysis"), - ArgDescriptor(ArgType.STRING, "compareToReport", "Report to compare to", isRequired = false) + ArgDescriptor(ArgType.String(), "mainReport", "Main report for analysis"), + ArgDescriptor(ArgType.String(), "compareToReport", "Report to compare to", isRequired = false) ) // Parse args. val argParser = ArgParser(options, arguments) if (argParser.parse(args)) { // Read contents of file. - val mainBenchsResults = readFile(argParser.get("mainReport")!!.stringValue) + val mainBenchsResults = getFileContent(argParser.get("mainReport")!!, argParser.get("user")) val mainReportElement = JsonTreeParser.parse(mainBenchsResults) val mainBenchsReport = BenchmarksReport.create(mainReportElement) - var compareToBenchsReport = argParser.get("compareToReport")?.stringValue?.let { - val compareToResults = readFile(it) + var compareToBenchsReport = argParser.get("compareToReport")?.let { + val compareToResults = getFileContent(it, argParser.get("user")) val compareToReportElement = JsonTreeParser.parse(compareToResults) BenchmarksReport.create(compareToReportElement) } - // Generate comparasion report + val renders = argParser.getAll("renders") + + // Generate comparasion report. val summaryReport = SummaryBenchmarksReport(mainBenchsReport, compareToBenchsReport, - argParser.get("eps")!!.doubleValue) - TextRender().print(summaryReport, argParser.get("short")!!.booleanValue, - argParser.get("output")?.stringValue) - // Produce information for TeamCity if needed. - getEnv("TEAMCITY_BUILD_PROPERTIES_FILE")?.let { - TeamCityStatisticsRender().print(summaryReport, argParser.get("short")!!.booleanValue) + argParser.get("eps")!!) + + var output = argParser.get("output") + + renders?.forEach { + Render.getRenderByName(it).print(summaryReport, argParser.get("short")!!, output) + output = null } } } \ No newline at end of file diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt index 17de976d291..1c425d64d6d 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Statistics.kt @@ -26,7 +26,7 @@ import kotlin.math.sqrt // Entity to describe avarage values which conssists of mean and variance values. data class MeanVariance(val mean: Double, val variance: Double) { override fun toString(): String { - val format = { number: Double -> format(number, 2)} + val format = { number: Double -> number.format(2)} return "${format(mean)} ± ${format(variance)}" } } @@ -66,10 +66,9 @@ data class MeanVarianceBenchmark(val meanBenchmark: BenchmarkResult, val varianc return MeanVariance(mean, ratioConfInt) } - override fun toString(): String { - val format = { number: Double -> format(number)} - return "${format(meanBenchmark.score)} ± ${format(varianceBenchmark.score)}" - } + override fun toString(): String = + "${meanBenchmark.score.format()} ± ${varianceBenchmark.score.format()}" + } fun geometricMean(values: List) = values.map { it.pow(1.0 / values.size) }.reduce { a, b -> a * b } diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt index 83b7553cd28..2f78893e757 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt @@ -50,9 +50,9 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport, var geoMeanScoreChange: ScoreChange? = null private set - // Changes in environment and tools. - val envChanges: List> - val kotlinChanges: List> + // Environment and tools. + val environments: Pair + val compilers: Pair // Countable properties. val failedBenchmarks: List @@ -74,6 +74,47 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport, val currentBenchmarksDuration: Map get() = benchmarksDurations.filter{ it.value.first != null }.map { it.key to it.value.first!! }.toMap() + val maximumRegression: Double + get() = getMaximumChange(regressions) + + val maximumImprovement: Double + get() = getMaximumChange(improvements) + + val regressionsGeometricMean: Double + get() = getGeometricMeanOfChanges(regressions) + + val improvementsGeometricMean: Double + get() = getGeometricMeanOfChanges(improvements) + + val envChanges: List> + get() { + val previousEnvironment = environments.second + val currentEnvironment = environments.first + return previousEnvironment?.let { + mutableListOf>().apply { + addFieldChange("Machine CPU", previousEnvironment.machine.cpu, currentEnvironment.machine.cpu) + addFieldChange("Machine OS", previousEnvironment.machine.os, currentEnvironment.machine.os) + addFieldChange("JDK version", previousEnvironment.jdk.version, currentEnvironment.jdk.version) + addFieldChange("JDK vendor", previousEnvironment.jdk.vendor, currentEnvironment.jdk.vendor) + } + } ?: listOf>() + } + + val kotlinChanges: List> + get() { + val previousCompiler = compilers.second + val currentCompiler = compilers.first + return previousCompiler?.let { + mutableListOf>().apply { + addFieldChange("Backend type", previousCompiler.backend.type.type, currentCompiler.backend.type.type) + addFieldChange("Backend version", previousCompiler.backend.version, currentCompiler.backend.version) + addFieldChange("Backend flags", previousCompiler.backend.flags.toString(), + currentCompiler.backend.flags.toString()) + addFieldChange("Kotlin version", previousCompiler.kotlinVersion, currentCompiler.kotlinVersion) + } + } ?: listOf>() + } + init { // Count avarage values for each benchmark. val currentBenchmarksTable = collectMeanResults(currentReport.benchmarks) @@ -83,14 +124,30 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport, mergedReport = createMergedReport(currentBenchmarksTable, previousBenchmarksTable) benchmarksDurations = calculateBenchmarksDuration(currentReport, previousReport) geoMeanBenchmark = calculateGeoMeanBenchmark(currentBenchmarksTable, previousBenchmarksTable) + environments = Pair(currentReport.env, previousReport?.env) + compilers = Pair(currentReport.compiler, previousReport?.compiler) + if (previousReport != null) { // Check changes in environment and tools. - envChanges = analyzeEnvChanges(currentReport.env, previousReport.env) - kotlinChanges = analyzeKotlinChanges(currentReport.compiler, previousReport.compiler) analyzePerformanceChanges() + } + } + + private fun getMaximumChange(bucket: Map): Double = + // Maps of regressions and improvements are sorted. + if (bucket.isEmpty()) 0.0 else bucket.values.map { it.first.mean }.first() + + private fun getGeometricMeanOfChanges(bucket: Map): Double { + if (bucket.isEmpty()) + return 0.0 + var percentsList = bucket.values.map { it.first.mean } + return if (percentsList.first() > 0.0) { + geometricMean(percentsList) } else { - envChanges = listOf>() - kotlinChanges = listOf>() + // Geometric mean can be counted on positive numbers. + val precision = abs(getMaximumChange(bucket)) + 1 + percentsList = percentsList.map { it + precision } + geometricMean(percentsList) - precision } } @@ -128,7 +185,7 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport, if (previousBenchmarks == null || name !in previousBenchmarks) { getOrPut(name) { SummaryBenchmark(current, null) } } else { - val previousBenchmark = previousBenchmarks[name]!!.meanBenchmark + val previousBenchmark = previousBenchmarks.getValue(name).meanBenchmark getOrPut(name) { SummaryBenchmark(current, previousBenchmarks[name]) } // Explore change of status. if (previousBenchmark.status != currentBenchmark.status) { @@ -203,23 +260,4 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport, add(it) } } - - private fun analyzeEnvChanges(currentEnv: Environment, previousEnv: Environment): List> { - return mutableListOf>().apply { - addFieldChange("Machine CPU", previousEnv.machine.cpu, currentEnv.machine.cpu) - addFieldChange("Machine OS", previousEnv.machine.os, currentEnv.machine.os) - addFieldChange("JDK version", previousEnv.jdk.version, currentEnv.jdk.version) - addFieldChange("JDK vendor", previousEnv.jdk.vendor, currentEnv.jdk.vendor) - } - } - - private fun analyzeKotlinChanges(currentCompiler: Compiler, previousCompiler: Compiler): List> { - return mutableListOf>().apply { - addFieldChange("Backend type", previousCompiler.backend.type.type, currentCompiler.backend.type.type) - addFieldChange("Backend version", previousCompiler.backend.version, currentCompiler.backend.version) - addFieldChange("Backend flags", previousCompiler.backend.flags.toString(), - currentCompiler.backend.flags.toString()) - addFieldChange( "Kotlin version", previousCompiler.kotlinVersion, currentCompiler.kotlinVersion) - } - } } \ No newline at end of file diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Utils.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Utils.kt index e689be5dfc9..dc21d68c94d 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Utils.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/Utils.kt @@ -17,7 +17,8 @@ package org.jetbrains.analyzer expect fun readFile(fileName: String): String -expect fun format(number: Double, decimalNumber: Int = 4): String +expect fun Double.format(decimalNumber: Int = 4): String expect fun writeToFile(fileName: String, text: String) expect fun assert(value: Boolean, lazyMessage: () -> Any) -expect fun getEnv(variableName:String): String? \ No newline at end of file +expect fun sendGetRequest(url: String, user: String? = null, password: String? = null, + followLocation: Boolean = false) : String \ No newline at end of file diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/kliopt/KliOpt.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/kliopt/KliOpt.kt index f1ca367c479..a4eb381d997 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/kliopt/KliOpt.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/kliopt/KliOpt.kt @@ -18,11 +18,41 @@ package org.jetbrains.kliopt // Possible types of arguments. -enum class ArgType(val hasParameter: Boolean) { - BOOLEAN(false), - STRING(true), - INT(true), - DOUBLE(true) +sealed class ArgType(val hasParameter: kotlin.Boolean) { + abstract val description: kotlin.String + open fun check(value: kotlin.String, name: kotlin.String) {} + class Boolean : ArgType(false) { + override val description: kotlin.String + get() = "" + } + class String : ArgType(true) { + override val description: kotlin.String + get() = "{ String }" + } + class Int : ArgType(true) { + override val description: kotlin.String + get() = "{ Int }" + + override fun check(value: kotlin.String, name: kotlin.String) { + value.toIntOrNull() ?: error("Option $name is expected to be integer number. $value is provided.") + } + } + class Double : ArgType(true) { + override val description: kotlin.String + get() = "{ Double }" + + override fun check(value: kotlin.String, name: kotlin.String) { + value.toDoubleOrNull() ?: error("Option $name is expected to be double number. $value is provided.") + } + } + class Choice(val values: List) : ArgType(true) { + override val description: kotlin.String + get() = "{ Value should be one of $values }" + + override fun check(value: kotlin.String, name: kotlin.String) { + if (value !in values) error("Option $name is expected to be obe of $values. $value is provided.") + } + } } // Common descriptor both for options and positional arguments. @@ -41,7 +71,8 @@ class OptionDescriptor( val shortName: String ? = null, description: String? = null, defaultValue: String? = null, - isRequired: Boolean = false) : Descriptor (type, longName, description, defaultValue, isRequired) { + isRequired: Boolean = false, + val isMultiple: Boolean = false) : Descriptor (type, longName, description, defaultValue, isRequired) { override val textDescription: String get() = "option -$longName" @@ -53,6 +84,7 @@ class OptionDescriptor( defaultValue?.let { result.append(" [$it]") } description?.let {result.append(" -> ${it}")} if (!isRequired) result.append(" (optional)") + result.append(" ${type.description}") result.append("\n") return result.toString() } @@ -74,6 +106,7 @@ class ArgDescriptor( defaultValue?.let { result.append(" [$it]") } description?.let {result.append(" -> ${it}")} if (!isRequired) result.append(" (optional)") + result.append(" ${type.description}") result.append("\n") return result.toString() } @@ -81,58 +114,69 @@ class ArgDescriptor( // Arguments parser. class ArgParser(optionsList: List, argsList: List = listOf()) { - private val options = optionsList.union(listOf(OptionDescriptor(ArgType.BOOLEAN, "help", + private val options = optionsList.union(listOf(OptionDescriptor(ArgType.Boolean(), "help", "h", "Usage info"))) .toList() private val arguments = argsList private lateinit var parsedValues: MutableMap - inner class ParsedArg(val descriptor: Descriptor, val value: String) { - val intValue: Int - get() { - if (descriptor.type != ArgType.INT) - error("Incorrect value for ${descriptor.textDescription}, must be an int") - return value.toInt() + inner class ParsedArg(val descriptor: Descriptor, val values: List) { + init { + // Check correctness of initialization data. + if (values.isEmpty()) { + printError("Parsed value should be provided!") } + } - val stringValue: String - get() { - if (descriptor.type != ArgType.STRING) - printError("Incorrect value for ${descriptor.textDescription}, must be a string") - return value - } + // Get value of argument converted to expected type. + private fun getTyped(value: String): T { + val typedValue = when (descriptor.type) { + is ArgType.Int -> value.toInt() + is ArgType.Double -> value.toDouble() + is ArgType.Boolean -> value == "true" + else -> value + } as? T + typedValue ?: printError("Argument ${descriptor.longName} has type ${descriptor.type} which differs from expected!") + return typedValue + } - val booleanValue: Boolean - get() { - if (descriptor.type != ArgType.BOOLEAN) - printError("Incorrect value for ${descriptor.textDescription}, must be a boolean") - return value == "true" - } + // Get value of argument. + fun get(): T { + return getTyped(values[0]) + } - val doubleValue: Double - get() { - if (descriptor.type != ArgType.DOUBLE) - printError("Incorrect option value for ${descriptor.textDescription}, must be a double") - return value.toDouble() - } + // Get all values of argument. + // For options that can be set multiple types. + fun getAll(): List { + return values.map { getTyped(it) } + } } // Output error. Also adds help usage information for easy understanding of problem. - private fun printError(message: String) { + private fun printError(message: String): Nothing { error("$message\n${makeUsage()}") } - private fun saveAsArg(argDescriptors: Map, arg: String): Boolean { + private fun saveAsArg(argDescriptors: Map, arg: String, processedValues: Map>): Boolean { // Find uninitialized arguments. - val nullArgs = argDescriptors.keys.filter { parsedValues[it] == null } + val nullArgs = argDescriptors.keys.filter { processedValues.getValue(it).isEmpty() } val name = nullArgs.firstOrNull() name?. let { - parsedValues[name] = ParsedArg(argDescriptors[name]!!, arg) + argDescriptors.getValue(name).type.check(arg, name) + processedValues.getValue(name).add(arg) return true } return false } + private fun saveAsOption(descriptor: OptionDescriptor, value: String, processedValues: Map>) { + if (!descriptor.isMultiple && !processedValues.getValue(descriptor.longName).isEmpty()) { + printError("Option ${descriptor.longName} is used more than one time!") + } + descriptor.type.check(value, descriptor.longName) + processedValues.getValue(descriptor.longName).add(value) + } + // Parse arguments. // Returns true if all arguments were parsed, otherwise return false and print help message. fun parse(args: Array): Boolean { @@ -140,7 +184,9 @@ class ArgParser(optionsList: List, argsList: List() }.toMap().toMutableMap() + parsedValues = descriptorsKeys.map { it to null }.toMap().toMutableMap() while (index < args.size) { val arg = args[index] if (arg.startsWith('-')) { @@ -150,7 +196,7 @@ class ArgParser(optionsList: List, argsList: List, argsList: List - val descriptor = optDescriptors[key] ?: argDescriptors[key]!! + processedValues.forEach { (key, value) -> + val descriptor = optDescriptors[key] ?: argDescriptors.getValue(key) // Not inited, append default value if needed. - parsedValues[key] = value ?: run { + if (value.isEmpty()) { descriptor.defaultValue?. let { - ParsedArg(descriptor, descriptor.defaultValue) - } - } - // Check if arg is always required. - parsedValues[key] ?: run { - if (descriptor.isRequired) { - printError("Please, provide value for ${descriptor.textDescription}. It should be always set") + parsedValues[key] = ParsedArg(descriptor, listOf(descriptor.defaultValue)) + } ?: run { + if (descriptor.isRequired) { + printError("Please, provide value for ${descriptor.textDescription}. It should be always set") + } else { + parsedValues[key] = null + } } + } else { + parsedValues[key] = ParsedArg(descriptor, value) } } - return true } - fun get(name: String): ParsedArg? { + // Get value of argument. + fun get(name: String): T? { if (::parsedValues.isInitialized) { - return parsedValues[name] + val arg = parsedValues[name] + return arg?.get() } else { - println("Method parse() of ArgParser class should be called before getting arguments and options.") - return null + printError("Method parse() of ArgParser class should be called before getting arguments and options.") + } + } + + // Get all values of argument. + // For options that can be set multiple types. + fun getAll(name: String): List? { + if (::parsedValues.isInitialized) { + val arg = parsedValues[name] + return arg?.getAll() + } else { + printError("Method parse() of ArgParser class should be called before getting arguments and options.") } } diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/HTMLRender.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/HTMLRender.kt new file mode 100644 index 00000000000..4c5617cc436 --- /dev/null +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/HTMLRender.kt @@ -0,0 +1,660 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.jetbrains.renders + +import org.jetbrains.analyzer.* +import org.jetbrains.report.* + +import kotlin.math.sin +import kotlin.math.abs +import kotlin.math.pow + +private fun > clamp(value: T, minValue: T, maxValue: T): T = + minOf(maxOf(value, minValue), maxValue) + +// Natural number. +class Natural(initValue: Int) { + val value = if (initValue > 0) initValue else error("Provided value $initValue isn't natural") + + override fun toString(): String { + return value.toString() + } +} + +interface Element { + fun render(builder: StringBuilder, indent: String) +} + +class TextElement(val text: String) : Element { + override fun render(builder: StringBuilder, indent: String) { + builder.append("$indent$text\n") + } +} + +@DslMarker +annotation class HtmlTagMarker + +@HtmlTagMarker +abstract class Tag(val name: String) : Element { + val children = arrayListOf() + val attributes = hashMapOf() + + protected fun initTag(tag: T, init: T.() -> Unit): T { + tag.init() + children.add(tag) + return tag + } + + override fun render(builder: StringBuilder, indent: String) { + builder.append("$indent<$name${renderAttributes()}>\n") + for (c in children) { + c.render(builder, indent + " ") + } + builder.append("$indent\n") + } + + private fun renderAttributes(): String = + attributes.map { (attr, value) -> + "$attr=\"$value\"" + }.joinToString(separator = " ", prefix = " ") + + override fun toString(): String { + val builder = StringBuilder() + render(builder, "") + return builder.toString() + } +} + +abstract class TagWithText(name: String) : Tag(name) { + operator fun String.unaryPlus() { + children.add(TextElement(this)) + } +} + +class HTML : TagWithText("html") { + fun head(init: Head.() -> Unit) = initTag(Head(), init) + fun body(init: Body.() -> Unit) = initTag(Body(), init) +} + +class Head : TagWithText("head") { + fun title(init: Title.() -> Unit) = initTag(Title(), init) + fun link(init: Link.() -> Unit) = initTag(Link(), init) + fun script(init: Script.() -> Unit) = initTag(Script(), init) +} + +class Title : TagWithText("title") +class Link : TagWithText("link") +class Script : TagWithText("script") + +abstract class BodyTag(name: String) : TagWithText(name) { + fun b(init: B.() -> Unit) = initTag(B(), init) + fun p(init: P.() -> Unit) = initTag(P(), init) + fun h1(init: H1.() -> Unit) = initTag(H1(), init) + fun h2(init: H2.() -> Unit) = initTag(H2(), init) + fun h4(init: H4.() -> Unit) = initTag(H4(), init) + fun hr(init: HR.() -> Unit) = initTag(HR(), init) + fun a(href: String, init: A.() -> Unit) { + val a = initTag(A(), init) + a.href = href + } + + fun img(src: String, init: Image.() -> Unit) { + val element = initTag(Image(), init) + element.src = src + } + + fun table(init: Table.() -> Unit) = initTag(Table(), init) + fun div(classAttr: String, init: Div.() -> Unit) = initTag(Div(classAttr), init) + fun button(classAttr: String, init: Button.() -> Unit) = initTag(Button(classAttr), init) + fun header(classAttr: String, init: Header.() -> Unit) = initTag(Header(classAttr), init) + fun span(classAttr: String, init: Span.() -> Unit) = initTag(Span(classAttr), init) +} + +abstract class BodyTagWithClass(name: String, val classAttr: String) : BodyTag(name) { + init { + attributes["class"] = classAttr + } +} + +class Body : BodyTag("body") +class B : BodyTag("b") +class P : BodyTag("p") +class H1 : BodyTag("h1") +class H2 : BodyTag("h2") +class H4 : BodyTag("h4") +class HR : BodyTag("hr") +class Div(classAttr: String) : BodyTagWithClass("div", classAttr) +class Header(classAttr: String) : BodyTagWithClass("header", classAttr) +class Button(classAttr: String) : BodyTagWithClass("button", classAttr) +class Span(classAttr: String) : BodyTagWithClass("span", classAttr) + +class A : BodyTag("a") { + var href: String by attributes +} + +class Image : BodyTag("img") { + var src: String by attributes +} + +abstract class TableTag(name: String) : BodyTag(name) { + fun thead(init: THead.() -> Unit) = initTag(THead(), init) + fun tbody(init: TBody.() -> Unit) = initTag(TBody(), init) + fun tfoot(init: TFoot.() -> Unit) = initTag(TFoot(), init) +} + +abstract class TableBlock(name: String) : TableTag(name) { + fun tr(init: TableRow.() -> Unit) = initTag(TableRow(), init) +} + +class Table : TableTag("table") +class THead : TableBlock("thead") +class TFoot : TableBlock("tfoot") +class TBody : TableBlock("tbody") + +abstract class TableRowTag(name: String) : TableBlock(name) { + var colspan = Natural(1) + set(value) { + attributes["colspan"] = value.toString() + } + var rowspan = Natural(1) + set(value) { + attributes["rowspan"] = value.toString() + } + + fun th(rowspan: Natural = Natural(1), colspan: Natural = Natural(1), init: TableHeadInfo.() -> Unit) { + val element = initTag(TableHeadInfo(), init) + element.rowspan = rowspan + element.colspan = colspan + } + + fun td(rowspan: Natural = Natural(1), colspan: Natural = Natural(1), init: TableDataInfo.() -> Unit) { + val element = initTag(TableDataInfo(), init) + element.rowspan = rowspan + element.colspan = colspan + } +} + +class TableRow : TableRowTag("tr") +class TableHeadInfo : TableRowTag("th") +class TableDataInfo : TableRowTag("td") + +fun html(init: HTML.() -> Unit): HTML { + val html = HTML() + html.init() + return html +} + +// Report render to html format. +class HTMLRender: Render() { + override val name: String + get() = "html" + + override fun render (report: SummaryBenchmarksReport, onlyChanges: Boolean) = + html { + head { + title { +"Benchmarks report" } + + // Links to bootstrap files. + link { + attributes["href"] = "https://stackpath.bootstrapcdn.com/bootstrap/4.2.1/css/bootstrap.min.css" + attributes["rel"] = "stylesheet" + } + script { + attributes["src"] = "https://code.jquery.com/jquery-3.3.1.slim.min.js" + } + script { + attributes["src"] = "https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.6/umd/popper.min.js" + } + script { + attributes["src"] = "https://stackpath.bootstrapcdn.com/bootstrap/4.2.1/js/bootstrap.min.js" + } + } + + body { + header("navbar navbar-expand navbar-dark flex-column flex-md-row bd-navbar") { + attributes["style"] = "background-color:#161616;" + img("https://dashboard.snapcraft.io/site_media/appmedia/2018/04/256px-kotlin-logo-svg.png") { + attributes["style"] = "width:60px;height:60px;" + } + + span("navbar-brand mb-0 h1") { +"Benchmarks report" } + } + div("container-fluid") { + p{} + renderEnvironmentTable(report.environments) + renderCompilerTable(report.compilers) + hr {} + renderStatusSummary(report) + hr {} + renderPerformanceSummary(report) + renderPerformanceDetails(report, onlyChanges) + } + } + }.toString() + + private fun TableRowTag.formatComparedTableData(data: String, compareToData: String?) { + td { + compareToData?. let { + // Highlight changed data. + if (it != data) + attributes["bgcolor"] = "yellow" + } + if (data.isEmpty()) { + +"-" + } else { + +data + } + } + } + + private fun TableTag.renderEnvironment(environment: Environment, name: String, compareTo: Environment? = null) { + tbody { + tr { + th { + attributes["scope"] = "row" + +name + } + formatComparedTableData(environment.machine.os, compareTo?.machine?.os) + formatComparedTableData(environment.machine.cpu, compareTo?.machine?.cpu) + formatComparedTableData(environment.jdk.version, compareTo?.jdk?.version) + formatComparedTableData(environment.jdk.vendor, compareTo?.jdk?.vendor) + } + } + } + + private fun BodyTag.renderEnvironmentTable(environments: Pair) { + h4 { +"Environment" } + table { + attributes["class"] = "table table-sm table-bordered table-hover" + attributes["style"] = "width:initial;" + val firstEnvironment = environments.first + val secondEnvironment = environments.second + // Table header. + thead { + tr { + th(rowspan = Natural(2)) { +"Run" } + th(colspan = Natural(2)) { +"Machine" } + th(colspan = Natural(2)) { +"JDK" } + } + tr { + th { + "OS" } + th { + "CPU" } + th { + "Version"} + th { + "Vendor"} + } + } + renderEnvironment(firstEnvironment, "First") + secondEnvironment?. let { renderEnvironment(it, "Second", firstEnvironment) } + } + } + + private fun TableTag.renderCompiler(compiler: Compiler, name: String, compareTo: Compiler? = null) { + tbody { + tr { + th { + attributes["scope"] = "row" + +name + } + formatComparedTableData(compiler.backend.type.type, compareTo?.backend?.type?.type) + formatComparedTableData(compiler.backend.version, compareTo?.backend?.version) + formatComparedTableData(compiler.backend.flags.joinToString(), compareTo?.backend?.flags?.joinToString()) + formatComparedTableData(compiler.kotlinVersion, compareTo?.kotlinVersion) + } + } + } + + private fun BodyTag.renderCompilerTable(compilers: Pair) { + h4 { +"Compiler" } + table { + attributes["class"] = "table table-sm table-bordered table-hover" + attributes["style"] = "width:initial;" + val firstCompiler = compilers.first + val secondCompiler = compilers.second + + // Table header. + thead { + tr { + th(rowspan = Natural(2)) { +"Run" } + th(colspan = Natural(3)) { +"Backend" } + th(rowspan = Natural(2)) { +"Kotlin" } + } + tr { + th { + "Type" } + th { + "Version" } + th { + "Flags"} + } + } + renderCompiler(firstCompiler, "First") + secondCompiler?. let { renderCompiler(it, "Second", firstCompiler) } + } + } + + private fun TableBlock.renderBucketInfo(bucket: Collection, name: String) { + if (!bucket.isEmpty()) { + tr { + th { + attributes["scope"] = "row" + +name + } + td { + +"${bucket.size}" + } + } + } + } + + private fun BodyTag.renderCollapsedData(name: String, isCollapsed: Boolean = false, colorStyle: String = "", + init: BodyTag.() -> Unit) { + val show = if (!isCollapsed) "show" else "" + val tagName = name.replace(' ', '_') + div("accordion") { + div("card") { + attributes["style"] = "border-bottom: 1px solid rgba(0,0,0,.125);" + div("card-header") { + attributes["id"] = "heading" + attributes["style"] = "padding: 0;$colorStyle" + button("btn btn-link") { + attributes["data-toggle"] = "collapse" + attributes["data-target"] = "#$tagName" + +name + } + } + + div("collapse $show") { + attributes["id"] = tagName + div("accordion-inner") { + init() + } + } + } + } + } + + private fun TableTag.renderTableFromList(list: List, name: String) { + if (!list.isEmpty()) { + thead { + tr { + th { +name } + } + } + list.forEach { + tbody { + tr { + td { +it } + } + } + } + } + } + + private fun BodyTag.renderStatusSummary(report: SummaryBenchmarksReport) { + h4 { +"Status Summary" } + val failedBenchmarks = report.failedBenchmarks + if (failedBenchmarks.isEmpty()) { + div("alert alert-success") { + attributes["role"] = "alert" + +"All benchmarks passed!" + } + } else { + div("alert alert-danger") { + attributes["role"] = "alert" + +"There are failed benchmarks!" + } + } + + val benchmarksWithChangedStatus = report.getBenchmarksWithChangedStatus() + val newFailures = benchmarksWithChangedStatus + .filter { it.current == BenchmarkResult.Status.FAILED } + val newPasses = benchmarksWithChangedStatus + .filter { it.current == BenchmarkResult.Status.PASSED } + + table { + attributes["class"] = "table table-sm table-striped table-hover" + attributes["style"] = "width:initial; font-size: 11pt;" + thead { + tr { + th { +"Status Group" } + th { +"#" } + } + } + tbody { + renderBucketInfo(failedBenchmarks, "Failed (total)") + renderBucketInfo(newFailures, "New Failures") + renderBucketInfo(newPasses, "New Passes") + renderBucketInfo(report.addedBenchmarks, "Added") + renderBucketInfo(report.removedBenchmarks, "Removed") + } + tfoot { + tr { + th { +"Total becnhmarks number" } + th { +"${report.benchmarksNumber}" } + } + } + } + if (!failedBenchmarks.isEmpty()) { + renderCollapsedData("Failures", colorStyle = "background-color: lightpink") { + table { + attributes["class"] = "table table-sm table-striped table-hover" + attributes["style"] = "width:initial; font-size: 11pt;" + val newFailuresList = newFailures.map { it.field } + renderTableFromList(newFailuresList, "New Failures") + + val existingFailures = failedBenchmarks.filter { it !in newFailuresList } + renderTableFromList(existingFailures, "Existing Failures") + } + } + } + if (!newPasses.isEmpty()) { + renderCollapsedData("New Passes", colorStyle = "background-color: lightgreen") { + table { + attributes["class"] = "table table-sm table-striped table-hover" + attributes["style"] = "width:initial; font-size: 11pt;" + renderTableFromList(newPasses.map { it.field }, "New Passes") + } + } + } + } + + private fun BodyTag.renderPerformanceSummary(report: SummaryBenchmarksReport) { + if (!report.improvements.isEmpty() || !report.regressions.isEmpty()) { + h4 { +"Performance Summary" } + table { + attributes["class"] = "table table-sm table-striped table-hover" + attributes["style"] = "width:initial;" + thead { + tr { + th { +"Change" } + th { +"#" } + th { +"Maximum" } + th { +"Geometric mean" } + } + } + val maximumRegression = report.maximumRegression + val maximumImprovement = report.maximumImprovement + val regressionsGeometricMean = report.regressionsGeometricMean + val improvementsGeometricMean = report.improvementsGeometricMean + tbody { + if (!report.regressions.isEmpty()) { + tr { + th { +"Regressions" } + td { +"${report.regressions.size}" } + td { + attributes["bgcolor"] = ColoredCell( + maximumRegression/maxOf(maximumRegression, abs(maximumImprovement))) + .backgroundStyle + +formatValue(maximumRegression, true) + } + td { + attributes["bgcolor"] = ColoredCell( + regressionsGeometricMean/maxOf(regressionsGeometricMean, + abs(improvementsGeometricMean))) + .backgroundStyle + +formatValue(report.regressionsGeometricMean, true) + } + } + } + if (!report.improvements.isEmpty()) { + tr { + th { +"Improvements" } + td { +"${report.improvements.size}" } + td { + attributes["bgcolor"] = ColoredCell( + maximumImprovement/maxOf(maximumRegression, abs(maximumImprovement))) + .backgroundStyle + +formatValue(report.maximumImprovement, true) + } + td { + attributes["bgcolor"] = ColoredCell( + improvementsGeometricMean/maxOf(regressionsGeometricMean, + abs(improvementsGeometricMean))) + .backgroundStyle + +formatValue(report.improvementsGeometricMean, true) + } + } + } + } + } + } + } + + private fun TableBlock.renderBenchmarksDetails(fullSet: Map, + bucket: Map? = null, rowStyle: String? = null) { + if (bucket != null) { + // There are changes in performance. + // Output changed benchmarks. + for ((name, change) in bucket) { + tr { + rowStyle?. let { + attributes["style"] = rowStyle + } + th { +name } + td { +"${fullSet.getValue(name).first}" } + td { +"${fullSet.getValue(name).second}" } + td { + attributes["bgcolor"] = ColoredCell(change.first.mean / abs(bucket.values.first().first.mean)) + .backgroundStyle + +"${change.first.toString() + " %"}" + } + td { + val scaledRatio = if (change.first.mean < 0) + (1 - change.second.mean) / (-1 + bucket.values.first().second.mean) + else (change.second.mean / bucket.values.first().second.mean) + attributes["bgcolor"] = ColoredCell(scaledRatio).backgroundStyle + +"${change.second}" + } + } + } + } else { + // Output all values without performance changes. + val placeholder = "-" + for ((name, value) in fullSet) { + tr { + th { +name } + td { +"${value.first?.toString() ?: placeholder}" } + td { +"${value.second?.toString() ?: placeholder}" } + td { +placeholder } + td { +placeholder } + } + } + } + } + + private fun BodyTag.renderPerformanceDetails(report: SummaryBenchmarksReport, onlyChanges: Boolean) { + if (onlyChanges) { + if (report.regressions.isEmpty() && report.improvements.isEmpty()) { + div("alert alert-success") { + attributes["role"] = "alert" + +"All becnhmarks are stable!" + } + } + } + + table { + attributes["id"] = "result" + attributes["class"] = "table table-striped table-bordered" + thead { + tr { + th { +"Benchmark" } + th { +"First score" } + th { +"Second score" } + th { +"Percent" } + th { +"Ratio" } + } + } + val geoMeanChangeMap = report.geoMeanScoreChange?. + let { mapOf(report.geoMeanBenchmark.first!!.meanBenchmark.name to report.geoMeanScoreChange!!) } + + tbody { + renderBenchmarksDetails( + mutableMapOf(report.geoMeanBenchmark.first!!.meanBenchmark.name to report.geoMeanBenchmark), + geoMeanChangeMap, "border-bottom: 2.3pt solid black; border-top: 2.3pt solid black") + renderBenchmarksDetails(report.mergedReport, report.regressions) + renderBenchmarksDetails(report.mergedReport, report.improvements) + if (!onlyChanges) { + // Print all remaining results. + renderBenchmarksDetails(report.mergedReport.filter { it.key !in report.regressions.keys && + it.key !in report.improvements.keys }) + } + } + } + } + + data class Color(val red: Double, val green: Double, val blue: Double) { + operator fun times(coefficient: Double) = + Color(red * coefficient, green * coefficient, blue * coefficient) + + operator fun plus(other: Color) = + Color(red + other.red, green + other.green, blue + other.blue) + + override fun toString() = + "#" + buildString { + listOf(red, green, blue).forEach { + append(clamp((it * 255).toInt(), 0, 255).toString(16).padStart(2, '0')) + } + } + } + + class ColoredCell(val scaledValue: Double, val reverse: Boolean = false) { + val value: Double + val neutralColor = Color(1.0,1.0 , 1.0) + val negativeColor = Color(0.0, 1.0, 0.0) + val positiveColor = Color(1.0, 0.0, 0.0) + + init { + value = if (abs(scaledValue) <= 1.0) scaledValue else error ("Value should be scaled in range [-1.0; 1.0]") + } + + val backgroundStyle: String + get() = getColor().toString() + + fun getColor(): Color { + val currentValue = clamp(value, -1.0, 1.0) + val cellValue = if (reverse) -currentValue else currentValue + val baseColor = if (cellValue < 0) negativeColor else positiveColor + // Smooth mapping to put first 20% of change into 50% of range, + // although really we should compensate for luma. + val color = sin((abs(cellValue).pow(.477)) * kotlin.math.PI * .5) + return linearInterpolation(neutralColor, baseColor, color) + } + + private fun linearInterpolation(a: Color, b: Color, coefficient: Double): Color { + val reversedCoefficient = 1.0 - coefficient + return a * reversedCoefficient + b * coefficient + } + } +} \ No newline at end of file diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt index e5542cef17f..0ed2d4ddead 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt @@ -22,9 +22,23 @@ import org.jetbrains.report.BenchmarkResult import kotlin.math.abs -// Common interface for printing report in different formats. -interface Render { - fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean = false): String +// Base class for printing report in different formats. +abstract class Render { + + companion object { + fun getRenderByName(name: String) = + when (name) { + "text" -> TextRender() + "html" -> HTMLRender() + "teamcity" -> TeamCityStatisticsRender() + else -> error("Unknown render $name") + } + } + + abstract val name: String + + abstract fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean = false): String + // Print report using render. fun print(report: SummaryBenchmarksReport, onlyChanges: Boolean = false, outputFile: String? = null) { val content = render(report, onlyChanges) @@ -32,10 +46,16 @@ interface Render { writeToFile(outputFile, content) } ?: println(content) } + + protected fun formatValue(number: Double, isPercent: Boolean = false): String = + if (isPercent) number.format(2) + "%" else number.format() } // Report render to text format. -class TextRender: Render { +class TextRender: Render() { + override val name: String + get() = "text" + private val content = StringBuilder() private val headerSeparator = "=================" private val wideColumnWidth = 50 @@ -50,7 +70,7 @@ class TextRender: Render { renderEnvChanges(report.kotlinChanges, "Compiler") renderStatusSummary(report) renderStatusChangesDetails(report.getBenchmarksWithChangedStatus()) - renderPerformanceSummary(report.regressions, report.improvements) + renderPerformanceSummary(report) renderPerformanceDetails(report, onlyChanges) return content.toString() } @@ -76,26 +96,6 @@ class TextRender: Render { } } - private fun printPerformanceBucket(bucket: Map, bucketName: String) { - if (!bucket.isEmpty()) { - var percentsList = bucket.values.map{ it.first.mean } - // Maps of regressions and improvements are sorted. - val maxValue = percentsList.first() - var geomeanValue: Double - if (percentsList.first() > 0.0) { - geomeanValue = geometricMean(percentsList) - } else { - // Geometric mean can be counted on positive numbers. - val precision = abs(maxValue) + 1 - percentsList = percentsList.map{it + precision} - geomeanValue = geometricMean(percentsList) - precision - } - - append("$bucketName: Maximum = ${formatValue(maxValue, true)}," + - " Geometric mean = ${formatValue(geomeanValue, true)}") - } - } - fun renderStatusChangesDetails(benchmarksWithChangedStatus: List>) { if (!benchmarksWithChangedStatus.isEmpty()) { append("Changes in status") @@ -127,20 +127,22 @@ class TextRender: Render { append() } - fun renderPerformanceSummary(regressions: Map, - improvements: Map) { - if (!regressions.isEmpty() || !improvements.isEmpty()) { + fun renderPerformanceSummary(report: SummaryBenchmarksReport) { + if (!report.regressions.isEmpty() || !report.improvements.isEmpty()) { append("Performance summary") append(headerSeparator) - printPerformanceBucket(regressions, "Regressions") - printPerformanceBucket(improvements, "Improvements") + if (!report.regressions.isEmpty()) { + append("Regressions: Maximum = ${formatValue(report.maximumRegression, true)}," + + " Geometric mean = ${formatValue(report.regressionsGeometricMean, true)}") + } + if (!report.improvements.isEmpty()) { + append("Improvements: Maximum = ${formatValue(report.maximumImprovement, true)}," + + " Geometric mean = ${formatValue(report.improvementsGeometricMean, true)}") + } append() } } - private fun formatValue(number: Double, isPercent: Boolean = false): String = - if (isPercent) format(number, 2) + "%" else format(number) - private fun formatColumn(content:String, isWide: Boolean = false): String = content.padEnd(if (isWide) wideColumnWidth else standardColumnWidth, ' ') @@ -151,8 +153,8 @@ class TextRender: Render { // Output changed benchmarks. for ((name, change) in bucket) { append(formatColumn(name, true) + - formatColumn(fullSet[name]!!.first!!.toString()) + - formatColumn(fullSet[name]!!.second!!.toString()) + + formatColumn(fullSet.getValue(name).first.toString()) + + formatColumn(fullSet.getValue(name).second.toString()) + formatColumn(change.first.toString() + " %") + formatColumn(change.second.toString())) } @@ -174,8 +176,8 @@ class TextRender: Render { private fun printPerformanceTableHeader(): Int { val wideColumns = listOf(formatColumn("Benchmark", true)) - val standardColumns = listOf(formatColumn("Current score"), - formatColumn("Previous score"), + val standardColumns = listOf(formatColumn("First score"), + formatColumn("Second score"), formatColumn("Percent"), formatColumn("Ratio")) val tableWidth = wideColumnWidth * wideColumns.size + standardColumnWidth * standardColumns.size diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/TeamCityStatisticsRender.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/TeamCityStatisticsRender.kt index 8e6bcabbf3d..d6f343002dc 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/TeamCityStatisticsRender.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/TeamCityStatisticsRender.kt @@ -21,7 +21,10 @@ import org.jetbrains.analyzer.* import org.jetbrains.report.BenchmarkResult // Report render to text format. -class TeamCityStatisticsRender: Render { +class TeamCityStatisticsRender: Render() { + override val name: String + get() = "teamcity" + private var content = StringBuilder() override fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean): String { diff --git a/tools/benchmarksAnalyzer/src/nativeInterop/cinterop/libcurl.def b/tools/benchmarksAnalyzer/src/nativeInterop/cinterop/libcurl.def new file mode 100644 index 00000000000..79775cad0c7 --- /dev/null +++ b/tools/benchmarksAnalyzer/src/nativeInterop/cinterop/libcurl.def @@ -0,0 +1,5 @@ +headers = curl/curl.h +headerFilter = curl/* +linkerOpts.osx = -L/opt/local/lib -L/usr/local/opt/curl/lib -lcurl +linkerOpts.linux = -L/usr/lib64 -L/usr/lib/x86_64-linux-gnu -lcurl +linkerOpts.mingw = -lcurl