diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/main.kt b/tools/benchmarksAnalyzer/src/main/kotlin/main.kt index 03e4dd71d98..72c0bd17692 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/main.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/main.kt @@ -40,6 +40,13 @@ object BintrayConnector : Connector() { val fileParametersSize = 3 val fileDescription = fileLocation.substringAfter(connectorPrefix) val fileParameters = fileDescription.split(':', limit = fileParametersSize) + + // Right link to bintray file. + if (fileParameters.size == 1) { + val accessFileUrl = "$bintrayUrl/${fileParameters[0]}" + return sendGetRequest(accessFileUrl, followLocation = true) + } + // Used builds description format. if (fileParameters.size != fileParametersSize) { error("To get file from bintray, please, specify, build number from TeamCity and target" + " in format bintray:build_number:target:filename") @@ -81,26 +88,50 @@ fun getFileContent(fileName: String, user: String? = null): String { fun getBenchmarkReport(fileName: String, user: String? = null) = BenchmarksReport.create(JsonTreeParser.parse(getFileContent(fileName, user))) +fun parseNormalizeResults(results: String): Map> { + val parsedNormalizeResults = mutableMapOf>() + val tokensNumber = 3 + results.lines().forEach { + if (!it.isEmpty()) { + val tokens = it.split(",").map { it.trim() } + if (tokens.size != tokensNumber) { + error("Data for normalization should include benchmark name, metric name and value. Got $it") + } + parsedNormalizeResults.getOrPut(tokens[0], { mutableMapOf() })[tokens[1]] = tokens[2].toDouble() + } + } + return parsedNormalizeResults +} + // Prints text summary by users request. fun summaryAction(argParser: ArgParser) { val benchsReport = SummaryBenchmarksReport(getBenchmarkReport(argParser.get("mainReport")!!, argParser.get("user"))) val results = mutableListOf() + val executionNormalize = argParser.get("exec-normalize")?.let { + parseNormalizeResults(getFileContent(it)) + } + val compileNormalize = argParser.get("compile-normalize")?.let { + parseNormalizeResults(getFileContent(it)) + } + val codesizeNormalize = argParser.get("codesize-normalize")?.let { + parseNormalizeResults(getFileContent(it)) + } results.apply { add(benchsReport.failedBenchmarks.size.toString()) argParser.getAll("exec-samples")?. let { val filter = if (it.first() == "all") null else it add(benchsReport.getResultsByMetric(BenchmarkResult.Metric.EXECUTION_TIME, - argParser.get("exec")!! == "geomean", filter).joinToString(";")) + argParser.get("exec")!! == "geomean", filter, executionNormalize).joinToString(";")) } argParser.getAll("compile-samples")?. let { val filter = if (it.first() == "all") null else it add(benchsReport.getResultsByMetric(BenchmarkResult.Metric.COMPILE_TIME, - argParser.get("compile")!! == "geomean", filter).joinToString(";")) + argParser.get("compile")!! == "geomean", filter, compileNormalize).joinToString(";")) } argParser.getAll("codesize-samples")?. let { val filter = if (it.first() == "all") null else it add(benchsReport.getResultsByMetric(BenchmarkResult.Metric.CODE_SIZE, - argParser.get("codesize")!! == "geomean", filter).joinToString(";")) + argParser.get("codesize")!! == "geomean", filter, codesizeNormalize).joinToString(";")) } } println(results.joinToString()) @@ -117,16 +148,22 @@ fun main(args: Array) { OptionDescriptor(ArgType.String(), "exec-samples", description = "Samples used for execution time metric (value 'all' allows use all samples)", delimiter = ","), + OptionDescriptor(ArgType.String(), "exec-normalize", + description = "File with golden results which should be used for normalization"), OptionDescriptor(ArgType.Choice(listOf("samples", "geomean")), "compile", description = "Compile time way of calculation", defaultValue = "geomean"), OptionDescriptor(ArgType.String(), "compile-samples", description = "Samples used for compile time metric (value 'all' allows use all samples)", delimiter = ","), + OptionDescriptor(ArgType.String(), "compile-normalize", + description = "File with golden results which should be used for normalization"), OptionDescriptor(ArgType.Choice(listOf("samples", "geomean")), "codesize", description = "Code size way of calculation", defaultValue = "geomean"), OptionDescriptor(ArgType.String(), "codesize-samples", description = "Samples used for code size metric (value 'all' allows use all samples)", delimiter = ","), + OptionDescriptor(ArgType.String(), "codesize-normalize", + description = "File with golden results which should be used for normalization"), OptionDescriptor(ArgType.String(), "user", "u", "User access information for authorization") ), listOf(ArgDescriptor(ArgType.String(), "mainReport", "Main report for analysis")) ) @@ -137,7 +174,7 @@ fun main(args: Array) { OptionDescriptor(ArgType.String(), "output", "o", "Output file"), OptionDescriptor(ArgType.Double(), "eps", "e", "Meaningful performance changes", "1.0"), OptionDescriptor(ArgType.Boolean(), "short", "s", "Show short version of report", "false"), - OptionDescriptor(ArgType.Choice(listOf("text", "html", "teamcity", "statistics")), + OptionDescriptor(ArgType.Choice(listOf("text", "html", "teamcity", "statistics", "metrics")), "renders", "r", "Renders for showing information", "text", isMultiple = true), OptionDescriptor(ArgType.String(), "user", "u", "User access information for authorization") ) diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt index f47d4570a56..2725a07c006 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt @@ -133,7 +133,8 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport, } } - fun getResultsByMetric(metric: BenchmarkResult.Metric, getGeoMean: Boolean = true, filter: List? = null): List { + fun getResultsByMetric(metric: BenchmarkResult.Metric, getGeoMean: Boolean = true, filter: List? = null, + normalizeData: Map>? = null): List { val benchmarks = filter?.let { mergedReport.filter { entry -> filter.find { @@ -148,7 +149,14 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport, if (filteredBenchmarks.isEmpty()) { error("There is no benchmarks for metric $metric") } - val results = filteredBenchmarks.map { entry -> entry.key.removeSuffix(metric.suffix) to entry.value.first!!.meanBenchmark.score }.toMap() + val results = filteredBenchmarks.map { entry -> + val score = entry.value.first!!.meanBenchmark.score + val name = entry.key.removeSuffix(metric.suffix) + val value = normalizeData?.let { + it.get(name)?.get("$metric")?.let { score / it } + ?: error("No normalization data for benchmark $name and metric $metric") + } ?: score + name to value }.toMap() if (getGeoMean) { return listOf(geometricMean(results.values)) } diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/MetricResultsRender.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/MetricResultsRender.kt new file mode 100644 index 00000000000..a0f8b4620cf --- /dev/null +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/MetricResultsRender.kt @@ -0,0 +1,40 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.jetbrains.renders + +import org.jetbrains.analyzer.* +import org.jetbrains.report.BenchmarkResult + +// Report render to text format. +class MetricResultsRender: Render() { + override val name: String + get() = "metrics" + + override fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean): String { + val results = report.mergedReport.map { entry -> + buildString { + val metric = entry.value.first!!.meanBenchmark.metric + append("{ \"benchmarkName\": \"${entry.key.removeSuffix(metric.suffix)}\",") + append("\"metric\": \"${metric}\",") + append("\"value\": \"${entry.value.first!!.meanBenchmark.score}\" }") + } + }.joinToString(", ") + return "[ $results ]" + + } +} \ No newline at end of file diff --git a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt index 91eecea433b..6d02346eda1 100644 --- a/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt +++ b/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt @@ -32,6 +32,7 @@ abstract class Render { "html" -> HTMLRender() "teamcity" -> TeamCityStatisticsRender() "statistics" -> StatisticsRender() + "metrics" -> MetricResultsRender() else -> error("Unknown render $name") } } diff --git a/tools/kliopt/org/jetbrains/kliopt/KliOpt.kt b/tools/kliopt/org/jetbrains/kliopt/KliOpt.kt index f5539d534d7..ba488770257 100644 --- a/tools/kliopt/org/jetbrains/kliopt/KliOpt.kt +++ b/tools/kliopt/org/jetbrains/kliopt/KliOpt.kt @@ -50,7 +50,7 @@ sealed class ArgType(val hasParameter: kotlin.Boolean) { get() = "{ Value should be one of $values }" override fun check(value: kotlin.String, name: kotlin.String) { - if (value !in values) error("Option $name is expected to be obe of $values. $value is provided.") + if (value !in values) error("Option $name is expected to be one of $values. $value is provided.") } } } diff --git a/tools/performance-server/src/main/kotlin/routes/route.kt b/tools/performance-server/src/main/kotlin/routes/route.kt index 5eedd7c7780..442b4dcd39e 100644 --- a/tools/performance-server/src/main/kotlin/routes/route.kt +++ b/tools/performance-server/src/main/kotlin/routes/route.kt @@ -24,6 +24,7 @@ const val teamCityUrl = "https://buildserver.labs.intellij.net/app/rest" const val downloadBintrayUrl = "https://dl.bintray.com/content/lepilkinaelena/KotlinNativePerformance" const val uploadBintrayUrl = "https://api.bintray.com/content/lepilkinaelena/KotlinNativePerformance" const val buildsFileName = "buildsSummary.csv" +const val goldenResultsFileName = "goldenResults.csv" const val bintrayPackage = "builds" const val buildsInfoPartsNumber = 11 @@ -109,6 +110,9 @@ object LocalCache { } } +data class GoldenResult(val benchmarkName: String, val metric: String, val value: Double) +data class GoldenResultsInfo(val bintrayUser: String, val bintrayPassword: String, val goldenResults: Array) + // Build information provided from request. data class BuildInfo(val buildNumber: String, val branch: String, val startTime: String, val finishTime: String) @@ -197,11 +201,7 @@ fun checkBuildType(currentType: String, targetType: String): Boolean { fun prepareBuildsResponse(builds: Collection, type: String, branch: String, buildNumber: String? = null): List { val buildsObjects = mutableListOf() builds.forEach { - val tokens = it.split(",").map { it.trim() } - if (tokens.size != buildsInfoPartsNumber) { - error("Build description $it doesn't contain all necessary information. " + - "File with data could be corrupted.") - } + val tokens = buildDescriptionToTokens(it) if ((checkBuildType(tokens[5], type) || type == "day") && (branch == tokens[3] || branch == "all") || tokens[0] == buildNumber) { buildsObjects.add(Build(tokens[0], tokens[1], tokens[2], tokens[3], @@ -212,6 +212,15 @@ fun prepareBuildsResponse(builds: Collection, type: String, branch: Stri return buildsObjects } +fun buildDescriptionToTokens(buildDescription: String): List { + val tokens = buildDescription.split(",").map { it.trim() } + if (tokens.size != buildsInfoPartsNumber) { + error("Build description $buildDescription doesn't contain all necessary information. " + + "File with data could be corrupted.") + } + return tokens +} + // Routing of requests to current server. fun router() { val express = require("express") @@ -262,6 +271,20 @@ fun router() { response.sendStatus(200) }) + // Register golden results to normalize on Bintray. + router.post("/registerGolden", { request, response -> + val goldenResultsInfo = JSON.parse(JSON.stringify(request.body)) + val buildsDescription = StringBuilder(sendGetRequest("$downloadBintrayUrl/$goldenResultsFileName")) + goldenResultsInfo.goldenResults.forEach { + buildsDescription.append("${it.benchmarkName}, ${it.metric}, ${it.value}\n") + } + // Upload new version of file. + val uploadUrl = "$uploadBintrayUrl/$bintrayPackage/latest/$goldenResultsFileName?publish=1&override=1" + sendUploadRequest(uploadUrl, buildsDescription.toString(), goldenResultsInfo.bintrayUser, goldenResultsInfo.bintrayPassword) + // Send response. + response.sendStatus(200) + }) + // Get list of builds. router.get("/builds/:target/:type/:branch/:id", { request, response -> val builds = LocalCache[request.params.target, request.params.id] @@ -273,6 +296,16 @@ fun router() { response.json(prepareBuildsResponse(builds, request.params.type, request.params.branch)) }) + router.get("/branches/:target", { request, response -> + val builds = LocalCache[request.params.target] + response.json(builds.map { buildDescriptionToTokens(it)[3] }.distinct()) + }) + + router.get("/buildsNumbers/:target", { request, response -> + val builds = LocalCache[request.params.target] + response.json(builds.map { buildDescriptionToTokens(it)[0] }.distinct()) + }) + router.get("/clean", { _, response -> LocalCache.clean() response.sendStatus(200) diff --git a/tools/performance-server/ui/src/main/kotlin/main.kt b/tools/performance-server/ui/src/main/kotlin/main.kt index fa77619a76a..1aaa7e2fdd1 100644 --- a/tools/performance-server/ui/src/main/kotlin/main.kt +++ b/tools/performance-server/ui/src/main/kotlin/main.kt @@ -19,6 +19,7 @@ import org.w3c.xhr.* import org.jetbrains.report.json.* import org.jetbrains.build.Build import kotlin.js.* +import kotlin.math.ceil import org.w3c.dom.* // API for interop with JS library Chartist. @@ -30,6 +31,7 @@ external class ChartistPlugins { external object Chartist { class Svg(form: String, parameters: dynamic, chartArea: String) val plugins: ChartistPlugins + val Interpolation: dynamic fun Line(query: String, data: dynamic, options: dynamic): dynamic } @@ -87,6 +89,15 @@ fun getChartOptions(samples: Array, yTitle: String, classNames: Array + val labelsCount = 30 + val skipNumber = ceil((labels.length as Int).toDouble() / labelsCount).toInt() + if (skipNumber > 1) { + if (index % skipNumber == 0) value else null + } else { + value + } + } chartOptions["axisX"] = axisXObject val axisYObject: dynamic = object{} axisYObject["offset"] = 90 @@ -105,6 +116,9 @@ fun getChartOptions(samples: Array, yTitle: String, classNames: Array${currentBuild.buildNumber}
") + append("Value: ${data.value.y.toFixed(4)}
") if (currentBuild.failuresNumber > 0) { append("failures: ${currentBuild.failuresNumber}
") } @@ -180,7 +195,7 @@ fun customizeChart(chart: dynamic, chartContainer: String, jquerySelector: dynam } fun main(args: Array) { - val serverUrl = "https://kotlin-native-perf-summary.labs.jb.gg/builds" + val serverUrl = "https://kotlin-native-perf-summary.labs.jb.gg" // Get parameters from request. val url = window.location.href @@ -196,7 +211,7 @@ fun main(args: Array) { // Get builds. val buildsUrl = buildString { - append("$serverUrl") + append("$serverUrl/builds") append("/${parameters["target"]}") append("/${parameters["type"]}") append("/${parameters["branch"]}") @@ -208,10 +223,16 @@ fun main(args: Array) { if (data !is JsonArray) { error("Response is expected to be an array.") } - val builds = data.jsonArray.map { Build.create(it as JsonObject) } + val builds = data.jsonArray.map { Build.create(it as JsonObject) }.sortedBy { it.buildNumber.substringAfterLast("-") } + + val branchesUrl = "$serverUrl/branches/${parameters["target"]}" + + val branches: Array = JSON.parse(sendGetRequest(branchesUrl)) + val releaseBranches = branches.filter { "v\\d+\\.\\d+\\.\\d+-fixes".toRegex().find(it) != null } // Fill autocomplete list. - val buildsNumbers = builds.map { json("value" to it.buildNumber, "data" to it.buildNumber) }.toTypedArray() + val buildsNumbersUrl = "$serverUrl/buildsNumbers/${parameters["target"]}" + val buildsNumbers: Array = JSON.parse(sendGetRequest(buildsNumbersUrl)) // Change inputs values connected with parameters and add events listeners. document.querySelector("#inputGroupTarget [value=\"${parameters["target"]}\"]")?.setAttribute("selected", "true") @@ -245,6 +266,12 @@ fun main(args: Array) { } }) + // Add release branches to selector. + releaseBranches.forEach { + val option = Option(it, it) + js("$('#inputGroupBranch')").append(js("$(option)")) + } + val autocompleteParameters: dynamic = object{} autocompleteParameters["lookup"] = buildsNumbers autocompleteParameters["onSelect"] = { suggestion -> @@ -294,7 +321,7 @@ fun main(args: Array) { getChartOptions(compileTime.keys.toTypedArray(), "Time, milliseconds")) val codeSizeChart = Chartist.Line("#codesize_chart", getChartData(labels, codeSize.values, sizeClassName), getChartOptions(codeSize.keys.toTypedArray(), "Size, KB", arrayOf("ct-series-2"))) - val bundleSizeChart = Chartist.Line("#bundlesize_chart", getChartData(labels, listOf(bundleSize), "ct-series-d"), + val bundleSizeChart = Chartist.Line("#bundlesize_chart", getChartData(labels, listOf(bundleSize), sizeClassName), getChartOptions(arrayOf("Bundle size"), "Size, MB", arrayOf("ct-series-2"))) // Tooltips and higlights.