azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Normalize benchmarks and changes in charts (#2820)

Browse Source
This commit is contained in:
LepilkinaElena
2019-03-28 10:40:31 +03:00
committed by GitHub
parent 31892bd82e
commit 7971090461
7 changed files with 163 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/benchmarksAnalyzer/src/main/kotlin/main.kt
+41 -4
View File
@@ -40,6 +40,13 @@ object BintrayConnector : Connector() {
val fileParametersSize = 3
val fileDescription = fileLocation.substringAfter(connectorPrefix)
val fileParameters = fileDescription.split(':', limit = fileParametersSize)
// Right link to bintray file.
if (fileParameters.size == 1) {
val accessFileUrl = "$bintrayUrl/${fileParameters[0]}"
return sendGetRequest(accessFileUrl, followLocation = true)
}
// Used builds description format.
if (fileParameters.size != fileParametersSize) {
error("To get file from bintray, please, specify, build number from TeamCity and target" +
" in format bintray:build_number:target:filename")
@@ -81,26 +88,50 @@ fun getFileContent(fileName: String, user: String? = null): String {
fun getBenchmarkReport(fileName: String, user: String? = null) =
BenchmarksReport.create(JsonTreeParser.parse(getFileContent(fileName, user)))
fun parseNormalizeResults(results: String): Map<String, Map<String, Double>> {
val parsedNormalizeResults = mutableMapOf<String, MutableMap<String, Double>>()
val tokensNumber = 3
results.lines().forEach {
if (!it.isEmpty()) {
val tokens = it.split(",").map { it.trim() }
if (tokens.size != tokensNumber) {
error("Data for normalization should include benchmark name, metric name and value. Got $it")
}
parsedNormalizeResults.getOrPut(tokens[0], { mutableMapOf<String, Double>() })[tokens[1]] = tokens[2].toDouble()
}
}
return parsedNormalizeResults
}
// Prints text summary by users request.
fun summaryAction(argParser: ArgParser) {
val benchsReport = SummaryBenchmarksReport(getBenchmarkReport(argParser.get<String>("mainReport")!!, argParser.get<String>("user")))
val results = mutableListOf<String>()
val executionNormalize = argParser.get<String>("exec-normalize")?.let {
parseNormalizeResults(getFileContent(it))
}
val compileNormalize = argParser.get<String>("compile-normalize")?.let {
parseNormalizeResults(getFileContent(it))
}
val codesizeNormalize = argParser.get<String>("codesize-normalize")?.let {
parseNormalizeResults(getFileContent(it))
}
results.apply {
add(benchsReport.failedBenchmarks.size.toString())
argParser.getAll<String>("exec-samples")?. let {
val filter = if (it.first() == "all") null else it
add(benchsReport.getResultsByMetric(BenchmarkResult.Metric.EXECUTION_TIME,
argParser.get<String>("exec")!! == "geomean", filter).joinToString(";"))
argParser.get<String>("exec")!! == "geomean", filter, executionNormalize).joinToString(";"))
}
argParser.getAll<String>("compile-samples")?. let {
val filter = if (it.first() == "all") null else it
add(benchsReport.getResultsByMetric(BenchmarkResult.Metric.COMPILE_TIME,
argParser.get<String>("compile")!! == "geomean", filter).joinToString(";"))
argParser.get<String>("compile")!! == "geomean", filter, compileNormalize).joinToString(";"))
}
argParser.getAll<String>("codesize-samples")?. let {
val filter = if (it.first() == "all") null else it
add(benchsReport.getResultsByMetric(BenchmarkResult.Metric.CODE_SIZE,
argParser.get<String>("codesize")!! == "geomean", filter).joinToString(";"))
argParser.get<String>("codesize")!! == "geomean", filter, codesizeNormalize).joinToString(";"))
}
}
println(results.joinToString())
@@ -117,16 +148,22 @@ fun main(args: Array<String>) {
OptionDescriptor(ArgType.String(), "exec-samples",
description = "Samples used for execution time metric (value 'all' allows use all samples)",
delimiter = ","),
OptionDescriptor(ArgType.String(), "exec-normalize",
description = "File with golden results which should be used for normalization"),
OptionDescriptor(ArgType.Choice(listOf("samples", "geomean")), "compile",
description = "Compile time way of calculation", defaultValue = "geomean"),
OptionDescriptor(ArgType.String(), "compile-samples",
description = "Samples used for compile time metric (value 'all' allows use all samples)",
delimiter = ","),
OptionDescriptor(ArgType.String(), "compile-normalize",
description = "File with golden results which should be used for normalization"),
OptionDescriptor(ArgType.Choice(listOf("samples", "geomean")), "codesize",
description = "Code size way of calculation", defaultValue = "geomean"),
OptionDescriptor(ArgType.String(), "codesize-samples",
description = "Samples used for code size metric (value 'all' allows use all samples)",
delimiter = ","),
OptionDescriptor(ArgType.String(), "codesize-normalize",
description = "File with golden results which should be used for normalization"),
OptionDescriptor(ArgType.String(), "user", "u", "User access information for authorization")
), listOf(ArgDescriptor(ArgType.String(), "mainReport", "Main report for analysis"))
)
@@ -137,7 +174,7 @@ fun main(args: Array<String>) {
OptionDescriptor(ArgType.String(), "output", "o", "Output file"),
OptionDescriptor(ArgType.Double(), "eps", "e", "Meaningful performance changes", "1.0"),
OptionDescriptor(ArgType.Boolean(), "short", "s", "Show short version of report", "false"),
OptionDescriptor(ArgType.Choice(listOf("text", "html", "teamcity", "statistics")),
OptionDescriptor(ArgType.Choice(listOf("text", "html", "teamcity", "statistics", "metrics")),
"renders", "r", "Renders for showing information", "text", isMultiple = true),
OptionDescriptor(ArgType.String(), "user", "u", "User access information for authorization")
)
tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/analyzer/SummaryBenchmarksReport.kt
+10 -2
View File
@@ -133,7 +133,8 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport,
}
}
fun getResultsByMetric(metric: BenchmarkResult.Metric, getGeoMean: Boolean = true, filter: List<String>? = null): List<Double> {
fun getResultsByMetric(metric: BenchmarkResult.Metric, getGeoMean: Boolean = true, filter: List<String>? = null,
normalizeData: Map<String, Map<String, Double>>? = null): List<Double> {
val benchmarks = filter?.let {
mergedReport.filter { entry ->
filter.find {
@@ -148,7 +149,14 @@ class SummaryBenchmarksReport (val currentReport: BenchmarksReport,
if (filteredBenchmarks.isEmpty()) {
error("There is no benchmarks for metric $metric")
}
val results = filteredBenchmarks.map { entry -> entry.key.removeSuffix(metric.suffix) to entry.value.first!!.meanBenchmark.score }.toMap()
val results = filteredBenchmarks.map { entry ->
val score = entry.value.first!!.meanBenchmark.score
val name = entry.key.removeSuffix(metric.suffix)
val value = normalizeData?.let {
it.get(name)?.get("$metric")?.let { score / it }
?: error("No normalization data for benchmark $name and metric $metric")
} ?: score
name to value }.toMap()
if (getGeoMean) {
return listOf(geometricMean(results.values))
}
tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/MetricResultsRender.kt
+40
View File
@@ -0,0 +1,40 @@
/*
* Copyright 2010-2018 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.renders
import org.jetbrains.analyzer.*
import org.jetbrains.report.BenchmarkResult
// Report render to text format.
class MetricResultsRender: Render() {
override val name: String
get() = "metrics"
override fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean): String {
val results = report.mergedReport.map { entry ->
buildString {
val metric = entry.value.first!!.meanBenchmark.metric
append("{ \"benchmarkName\": \"${entry.key.removeSuffix(metric.suffix)}\",")
append("\"metric\": \"${metric}\",")
append("\"value\": \"${entry.value.first!!.meanBenchmark.score}\" }")
}
}.joinToString(", ")
return "[ $results ]"
}
}
tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt
+1
View File
@@ -32,6 +32,7 @@ abstract class Render {
"html" -> HTMLRender()
"teamcity" -> TeamCityStatisticsRender()
"statistics" -> StatisticsRender()
"metrics" -> MetricResultsRender()
else -> error("Unknown render $name")
}
}
tools/kliopt/org/jetbrains/kliopt/KliOpt.kt
+1 -1
View File
@@ -50,7 +50,7 @@ sealed class ArgType(val hasParameter: kotlin.Boolean) {
get() = "{ Value should be one of $values }"
override fun check(value: kotlin.String, name: kotlin.String) {
if (value !in values) error("Option $name is expected to be obe of $values. $value is provided.")
if (value !in values) error("Option $name is expected to be one of $values. $value is provided.")
}
}
}
tools/performance-server/src/main/kotlin/routes/route.kt
+38 -5
View File
@@ -24,6 +24,7 @@ const val teamCityUrl = "https://buildserver.labs.intellij.net/app/rest"
const val downloadBintrayUrl = "https://dl.bintray.com/content/lepilkinaelena/KotlinNativePerformance"
const val uploadBintrayUrl = "https://api.bintray.com/content/lepilkinaelena/KotlinNativePerformance"
const val buildsFileName = "buildsSummary.csv"
const val goldenResultsFileName = "goldenResults.csv"
const val bintrayPackage = "builds"
const val buildsInfoPartsNumber = 11
@@ -109,6 +110,9 @@ object LocalCache {
}
}
data class GoldenResult(val benchmarkName: String, val metric: String, val value: Double)
data class GoldenResultsInfo(val bintrayUser: String, val bintrayPassword: String, val goldenResults: Array<GoldenResult>)
// Build information provided from request.
data class BuildInfo(val buildNumber: String, val branch: String, val startTime: String,
val finishTime: String)
@@ -197,11 +201,7 @@ fun checkBuildType(currentType: String, targetType: String): Boolean {
fun prepareBuildsResponse(builds: Collection<String>, type: String, branch: String, buildNumber: String? = null): List<Build> {
val buildsObjects = mutableListOf<Build>()
builds.forEach {
val tokens = it.split(",").map { it.trim() }
if (tokens.size != buildsInfoPartsNumber) {
error("Build description $it doesn't contain all necessary information. " +
"File with data could be corrupted.")
}
val tokens = buildDescriptionToTokens(it)
if ((checkBuildType(tokens[5], type) || type == "day") && (branch == tokens[3] || branch == "all")
|| tokens[0] == buildNumber) {
buildsObjects.add(Build(tokens[0], tokens[1], tokens[2], tokens[3],
@@ -212,6 +212,15 @@ fun prepareBuildsResponse(builds: Collection<String>, type: String, branch: Stri
return buildsObjects
}
fun buildDescriptionToTokens(buildDescription: String): List<String> {
val tokens = buildDescription.split(",").map { it.trim() }
if (tokens.size != buildsInfoPartsNumber) {
error("Build description $buildDescription doesn't contain all necessary information. " +
"File with data could be corrupted.")
}
return tokens
}
// Routing of requests to current server.
fun router() {
val express = require("express")
@@ -262,6 +271,20 @@ fun router() {
response.sendStatus(200)
})
// Register golden results to normalize on Bintray.
router.post("/registerGolden", { request, response ->
val goldenResultsInfo = JSON.parse<GoldenResultsInfo>(JSON.stringify(request.body))
val buildsDescription = StringBuilder(sendGetRequest("$downloadBintrayUrl/$goldenResultsFileName"))
goldenResultsInfo.goldenResults.forEach {
buildsDescription.append("${it.benchmarkName}, ${it.metric}, ${it.value}\n")
}
// Upload new version of file.
val uploadUrl = "$uploadBintrayUrl/$bintrayPackage/latest/$goldenResultsFileName?publish=1&override=1"
sendUploadRequest(uploadUrl, buildsDescription.toString(), goldenResultsInfo.bintrayUser, goldenResultsInfo.bintrayPassword)
// Send response.
response.sendStatus(200)
})
// Get list of builds.
router.get("/builds/:target/:type/:branch/:id", { request, response ->
val builds = LocalCache[request.params.target, request.params.id]
@@ -273,6 +296,16 @@ fun router() {
response.json(prepareBuildsResponse(builds, request.params.type, request.params.branch))
})
router.get("/branches/:target", { request, response ->
val builds = LocalCache[request.params.target]
response.json(builds.map { buildDescriptionToTokens(it)[3] }.distinct())
})
router.get("/buildsNumbers/:target", { request, response ->
val builds = LocalCache[request.params.target]
response.json(builds.map { buildDescriptionToTokens(it)[0] }.distinct())
})
router.get("/clean", { _, response ->
LocalCache.clean()
response.sendStatus(200)
tools/performance-server/ui/src/main/kotlin/main.kt
+32 -5
View File
@@ -19,6 +19,7 @@ import org.w3c.xhr.*
import org.jetbrains.report.json.*
import org.jetbrains.build.Build
import kotlin.js.*
import kotlin.math.ceil
import org.w3c.dom.*
// API for interop with JS library Chartist.
@@ -30,6 +31,7 @@ external class ChartistPlugins {
external object Chartist {
class Svg(form: String, parameters: dynamic, chartArea: String)
val plugins: ChartistPlugins
val Interpolation: dynamic
fun Line(query: String, data: dynamic, options: dynamic): dynamic
}
@@ -87,6 +89,15 @@ fun getChartOptions(samples: Array<String>, yTitle: String, classNames: Array<St
chartOptions["chartPadding"] = paddingObject
val axisXObject: dynamic = object{}
axisXObject["offset"] = 40
axisXObject["labelInterpolationFnc"] = { value, index, labels ->
val labelsCount = 30
val skipNumber = ceil((labels.length as Int).toDouble() / labelsCount).toInt()
if (skipNumber > 1) {
if (index % skipNumber == 0) value else null
} else {
value
}
}
chartOptions["axisX"] = axisXObject
val axisYObject: dynamic = object{}
axisYObject["offset"] = 90
@@ -105,6 +116,9 @@ fun getChartOptions(samples: Array<String>, yTitle: String, classNames: Array<St
axisYTitle["textAnchor"] = "middle"
axisYTitle["flipTitle"] = true
titleObject["axisY"] = axisYTitle
val interpolationObject: dynamic = {}
interpolationObject["fillHoles"] = true
chartOptions["lineSmooth"] = Chartist.Interpolation.simple(interpolationObject)
chartOptions["plugins"] = arrayOf(Chartist.plugins.legend(legendObject), Chartist.plugins.ctAxisTitle(titleObject))
return chartOptions
}
@@ -148,6 +162,7 @@ fun customizeChart(chart: dynamic, chartContainer: String, jquerySelector: dynam
else ""}"
val information = buildString {
append("<a href=\"$linkToDetailedInfo\">${currentBuild.buildNumber}</a><br>")
append("Value: ${data.value.y.toFixed(4)}<br>")
if (currentBuild.failuresNumber > 0) {
append("failures: ${currentBuild.failuresNumber}<br>")
}
@@ -180,7 +195,7 @@ fun customizeChart(chart: dynamic, chartContainer: String, jquerySelector: dynam
}
fun main(args: Array<String>) {
val serverUrl = "https://kotlin-native-perf-summary.labs.jb.gg/builds"
val serverUrl = "https://kotlin-native-perf-summary.labs.jb.gg"
// Get parameters from request.
val url = window.location.href
@@ -196,7 +211,7 @@ fun main(args: Array<String>) {
// Get builds.
val buildsUrl = buildString {
append("$serverUrl")
append("$serverUrl/builds")
append("/${parameters["target"]}")
append("/${parameters["type"]}")
append("/${parameters["branch"]}")
@@ -208,10 +223,16 @@ fun main(args: Array<String>) {
if (data !is JsonArray) {
error("Response is expected to be an array.")
}
val builds = data.jsonArray.map { Build.create(it as JsonObject) }
val builds = data.jsonArray.map { Build.create(it as JsonObject) }.sortedBy { it.buildNumber.substringAfterLast("-") }
val branchesUrl = "$serverUrl/branches/${parameters["target"]}"
val branches: Array<String> = JSON.parse(sendGetRequest(branchesUrl))
val releaseBranches = branches.filter { "v\\d+\\.\\d+\\.\\d+-fixes".toRegex().find(it) != null }
// Fill autocomplete list.
val buildsNumbers = builds.map { json("value" to it.buildNumber, "data" to it.buildNumber) }.toTypedArray()
val buildsNumbersUrl = "$serverUrl/buildsNumbers/${parameters["target"]}"
val buildsNumbers: Array<String> = JSON.parse(sendGetRequest(buildsNumbersUrl))
// Change inputs values connected with parameters and add events listeners.
document.querySelector("#inputGroupTarget [value=\"${parameters["target"]}\"]")?.setAttribute("selected", "true")
@@ -245,6 +266,12 @@ fun main(args: Array<String>) {
}
})
// Add release branches to selector.
releaseBranches.forEach {
val option = Option(it, it)
js("$('#inputGroupBranch')").append(js("$(option)"))
}
val autocompleteParameters: dynamic = object{}
autocompleteParameters["lookup"] = buildsNumbers
autocompleteParameters["onSelect"] = { suggestion ->
@@ -294,7 +321,7 @@ fun main(args: Array<String>) {
getChartOptions(compileTime.keys.toTypedArray(), "Time, milliseconds"))
val codeSizeChart = Chartist.Line("#codesize_chart", getChartData(labels, codeSize.values, sizeClassName),
getChartOptions(codeSize.keys.toTypedArray(), "Size, KB", arrayOf("ct-series-2")))
val bundleSizeChart = Chartist.Line("#bundlesize_chart", getChartData(labels, listOf(bundleSize), "ct-series-d"),
val bundleSizeChart = Chartist.Line("#bundlesize_chart", getChartData(labels, listOf(bundleSize), sizeClassName),
getChartOptions(arrayOf("Bundle size"), "Size, MB", arrayOf("ct-series-2")))
// Tooltips and higlights.