diff --git a/kotlin-native/tools/benchmarks/shared/src/main/kotlin/analyzer/Statistics.kt b/kotlin-native/tools/benchmarks/shared/src/main/kotlin/analyzer/Statistics.kt index fcde19c32d3..79859ecd9d9 100644 --- a/kotlin-native/tools/benchmarks/shared/src/main/kotlin/analyzer/Statistics.kt +++ b/kotlin-native/tools/benchmarks/shared/src/main/kotlin/analyzer/Statistics.kt @@ -25,10 +25,13 @@ val MeanVarianceBenchmark.description: String // Calculate difference in percentage compare to another. fun MeanVarianceBenchmark.calcPercentageDiff(other: MeanVarianceBenchmark): MeanVariance { + if (score == 0.0 && variance == 0.0 && other.score == 0.0 && other.variance == 0.0) + return MeanVariance(score, variance) assert(other.score >= 0 && other.variance >= 0 && - other.score - other.variance != 0.0, + (other.score - other.variance != 0.0 || other.score == 0.0), { "Mean and variance should be positive and not equal!" }) + // Analyze intervals. Calculate difference between border points. val (bigValue, smallValue) = if (score > other.score) Pair(this, other) else Pair(other, this) val bigValueIntervalStart = bigValue.score - bigValue.variance @@ -50,11 +53,13 @@ fun MeanVarianceBenchmark.calcPercentageDiff(other: MeanVarianceBenchmark): Mean // Calculate ratio value compare to another. fun MeanVarianceBenchmark.calcRatio(other: MeanVarianceBenchmark): MeanVariance { + if (other.score == 0.0 && other.variance == 0.0) + return MeanVariance(1.0, 0.0) assert(other.score >= 0 && other.variance >= 0 && - other.score - other.variance != 0.0, + (other.score - other.variance != 0.0 || other.score == 0.0), { "Mean and variance should be positive and not equal!" }) - val mean = score / other.score + val mean = if (other.score != 0.0) (score / other.score) else 0.0 val minRatio = (score - variance) / (other.score + other.variance) val maxRatio = (score + variance) / (other.score - other.variance) val ratioConfInt = min(abs(minRatio - mean), abs(maxRatio - mean)) @@ -63,7 +68,7 @@ fun MeanVarianceBenchmark.calcRatio(other: MeanVarianceBenchmark): MeanVariance fun geometricMean(values: Collection, totalNumber: Int = values.size) = with(values.asSequence().filter { it != 0.0 }) { - if (count() == 0) { + if (count() == 0 || totalNumber == 0) { 0.0 } else { map { it.pow(1.0 / totalNumber) }.reduce { a, b -> a * b } diff --git a/kotlin-native/tools/benchmarks/shared/src/main/kotlin/analyzer/SummaryBenchmarksReport.kt b/kotlin-native/tools/benchmarks/shared/src/main/kotlin/analyzer/SummaryBenchmarksReport.kt index 6048d975197..12c98cd070e 100644 --- a/kotlin-native/tools/benchmarks/shared/src/main/kotlin/analyzer/SummaryBenchmarksReport.kt +++ b/kotlin-native/tools/benchmarks/shared/src/main/kotlin/analyzer/SummaryBenchmarksReport.kt @@ -18,13 +18,11 @@ typealias BenchmarksTable = Map typealias SummaryBenchmarksTable = Map typealias ScoreChange = Pair -// Summary report with comparasion of separate benchmarks results. -class SummaryBenchmarksReport(val currentReport: BenchmarksReport, - val previousReport: BenchmarksReport? = null, - val meaningfulChangesValue: Double = 0.5) { +class DetailedBenchmarksReport(currentBenchmarks: Map>, + previousBenchmarks: Map>? = null, + val meaningfulChangesValue: Double = 0.5) { // Report created by joining comparing reports. val mergedReport: Map - private val benchmarksDurations: Map> // Lists of benchmarks in different status. private val benchmarksWithChangedStatus = mutableListOf>() @@ -40,30 +38,6 @@ class SummaryBenchmarksReport(val currentReport: BenchmarksReport, var geoMeanScoreChange: ScoreChange? = null private set - // Environment and tools. - val environments: Pair - val compilers: Pair - - // Countable properties. - val failedBenchmarks: List - get() = mergedReport.filter { it.value.first?.status == BenchmarkResult.Status.FAILED } - .map { it.key } - - val addedBenchmarks: List - get() = mergedReport.filter { it.value.second == null }.map { it.key } - - val removedBenchmarks: List - get() = mergedReport.filter { it.value.first == null }.map { it.key } - - val benchmarksNumber: Int - get() = mergedReport.keys.size - - val currentMeanVarianceBenchmarks: List - get() = mergedReport.filter { it.value.first != null }.map { it.value.first!! } - - val currentBenchmarksDuration: Map - get() = benchmarksDurations.filter { it.value.first != null }.map { it.key to it.value.first!! }.toMap() - val maximumRegression: Double get() = getMaximumChange(regressions) @@ -76,90 +50,53 @@ class SummaryBenchmarksReport(val currentReport: BenchmarksReport, val improvementsGeometricMean: Double get() = getGeometricMeanOfChanges(improvements) - val envChanges: List> - get() { - val previousEnvironment = environments.second - val currentEnvironment = environments.first - return previousEnvironment?.let { - mutableListOf>().apply { - addFieldChange("Machine CPU", previousEnvironment.machine.cpu, currentEnvironment.machine.cpu) - addFieldChange("Machine OS", previousEnvironment.machine.os, currentEnvironment.machine.os) - addFieldChange("JDK version", previousEnvironment.jdk.version, currentEnvironment.jdk.version) - addFieldChange("JDK vendor", previousEnvironment.jdk.vendor, currentEnvironment.jdk.vendor) - } - } ?: listOf>() - } - - val kotlinChanges: List> - get() { - val previousCompiler = compilers.second - val currentCompiler = compilers.first - return previousCompiler?.let { - mutableListOf>().apply { - addFieldChange("Backend type", previousCompiler.backend.type.type, currentCompiler.backend.type.type) - addFieldChange("Backend version", previousCompiler.backend.version, currentCompiler.backend.version) - addFieldChange("Backend flags", previousCompiler.backend.flags.toString(), - currentCompiler.backend.flags.toString()) - addFieldChange("Kotlin version", previousCompiler.kotlinVersion, currentCompiler.kotlinVersion) - } - } ?: listOf>() - } + val benchmarksNumber: Int + get() = mergedReport.keys.size init { // Count avarage values for each benchmark. - val currentBenchmarksTable = collectMeanResults(currentReport.benchmarks) - val previousBenchmarksTable = previousReport?.let { - collectMeanResults(previousReport.benchmarks) + val currentBenchmarksTable = collectMeanResults(currentBenchmarks) + val previousBenchmarksTable = previousBenchmarks?.let { + collectMeanResults(previousBenchmarks) } mergedReport = createMergedReport(currentBenchmarksTable, previousBenchmarksTable) - benchmarksDurations = calculateBenchmarksDuration(currentReport, previousReport) geoMeanBenchmark = calculateGeoMeanBenchmark(currentBenchmarksTable, previousBenchmarksTable) - environments = Pair(currentReport.env, previousReport?.env) - compilers = Pair(currentReport.compiler, previousReport?.compiler) - if (previousReport != null) { + if (previousBenchmarks != null) { // Check changes in environment and tools. analyzePerformanceChanges() } } - // Get benchmark report. - fun getBenchmarksReport(takeMainReport: Boolean = true) = - if (takeMainReport) - BenchmarksReport(environments.first, mergedReport.map { (_, value) -> value.first!! }, compilers.first) - else - BenchmarksReport(environments.second!!, mergedReport.map { (_, value) -> value.second!! }, compilers.second!!) - - fun getResultsByMetric(metric: BenchmarkResult.Metric, getGeoMean: Boolean = true, filter: List? = null, - normalizeData: Map>? = null): List { - val benchmarks = filter?.let { - mergedReport.filter { entry -> - filter.find { - entry.key.startsWith(it) - } != null - } - } ?: mergedReport - val results = benchmarks.map { entry -> - val name = entry.key.removeSuffix(metric.suffix) - if (entry.value.first!!.metric == metric) { - val score = entry.value.first!!.score - val value = normalizeData?.let { - it.get(name)?.get("$metric")?.let { score / it } - ?: error("No normalization data for benchmark $name and metric $metric") - } ?: score - name to value - } else name to null - }.toMap() - if (getGeoMean) { - return listOf(geometricMean(results.values.filterNotNull())) - } - return filter?.let { it.map { results[it] }.toList() } ?: results.values.toList() - } - private fun getMaximumChange(bucket: Map): Double = // Maps of regressions and improvements are sorted. if (bucket.isEmpty()) 0.0 else bucket.values.map { it.first.mean }.first() + // Analyze and collect changes in performance between same becnhmarks. + private fun analyzePerformanceChanges() { + val performanceChanges = mergedReport.asSequence().map { (name, element) -> + getBenchmarkPerfomanceChange(name, element) + }.filterNotNull().groupBy { + if (it.second.first.mean > 0) "regressions" else "improvements" + } + + // Sort regressions and improvements. + regressions = performanceChanges["regressions"] + ?.sortedByDescending { it.second.first.mean }?.map { it.first to it.second } + ?.toMap() ?: mapOf() + improvements = performanceChanges["improvements"] + ?.sortedBy { it.second.first.mean }?.map { it.first to it.second } + ?.toMap() ?: mapOf() + + // Calculate change for geometric mean. + val (current, previous) = geoMeanBenchmark + geoMeanScoreChange = current?.let { + previous?.let { + Pair(current.calcPercentageDiff(previous), current.calcRatio(previous)) + } + } + } + private fun getGeometricMeanOfChanges(bucket: Map): Double { if (bucket.isEmpty()) return 0.0 @@ -175,25 +112,6 @@ class SummaryBenchmarksReport(val currentReport: BenchmarksReport, fun getBenchmarksWithChangedStatus(): List> = benchmarksWithChangedStatus - // Create geometric mean. - private fun createGeoMeanBenchmark(benchTable: BenchmarksTable): MeanVarianceBenchmark { - val geoMeanBenchmarkName = "Geometric mean" - val geoMean = geometricMean(benchTable.toList().map { (_, value) -> value.score }) - val varianceGeoMean = geometricMean(benchTable.toList().map { (_, value) -> value.variance }) - return MeanVarianceBenchmark(geoMeanBenchmarkName, geoMean, varianceGeoMean) - } - - // Generate map with summary durations of each benchmark. - private fun calculateBenchmarksDuration(currentReport: BenchmarksReport, previousReport: BenchmarksReport?): - Map> { - val currentDurations = collectBenchmarksDurations(currentReport.benchmarks) - val previousDurations = previousReport?.let { - collectBenchmarksDurations(previousReport.benchmarks) - } ?: mapOf() - return currentDurations.keys.union(previousDurations.keys) - .map { it to Pair(currentDurations[it], previousDurations[it]) }.toMap() - } - // Merge current and compare to report. private fun createMergedReport(currentBenchmarks: BenchmarksTable, previousBenchmarks: BenchmarksTable?): Map { @@ -249,29 +167,144 @@ class SummaryBenchmarksReport(val currentReport: BenchmarksReport, return null } - // Analyze and collect changes in performance between same becnhmarks. - private fun analyzePerformanceChanges() { - val performanceChanges = mergedReport.asSequence().map { (name, element) -> - getBenchmarkPerfomanceChange(name, element) - }.filterNotNull().groupBy { - if (it.second.first.mean > 0) "regressions" else "improvements" + // Create geometric mean. + private fun createGeoMeanBenchmark(benchTable: BenchmarksTable): MeanVarianceBenchmark { + val geoMeanBenchmarkName = "Geometric mean" + val geoMean = geometricMean(benchTable.toList().map { (_, value) -> value.score }) + val varianceGeoMean = geometricMean(benchTable.toList().map { (_, value) -> value.variance }) + return MeanVarianceBenchmark(geoMeanBenchmarkName, geoMean, varianceGeoMean) + } +} + +// Summary report with comparasion of separate benchmarks results. +class SummaryBenchmarksReport(val currentReport: BenchmarksReport, + val previousReport: BenchmarksReport? = null, + val meaningfulChangesValue: Double = 0.5, + private val unstableBenchmarks: List = emptyList()) { + + val detailedMetricReports: Map + + private val benchmarksDurations: Map> + + // Lists of benchmarks in different status. + val benchmarksWithChangedStatus + get() = getReducedResult { report -> + report.getBenchmarksWithChangedStatus() } - // Sort regressions and improvements. - regressions = performanceChanges["regressions"] - ?.sortedByDescending { it.second.first.mean }?.map { it.first to it.second } - ?.toMap() ?: mapOf() - improvements = performanceChanges["improvements"] - ?.sortedBy { it.second.first.mean }?.map { it.first to it.second } - ?.toMap() ?: mapOf() + // Environment and tools. + val environments: Pair + val compilers: Pair - // Calculate change for geometric mean. - val (current, previous) = geoMeanBenchmark - geoMeanScoreChange = current?.let { - previous?.let { - Pair(current.calcPercentageDiff(previous), current.calcRatio(previous)) - } + private fun getReducedResult(convertor: (DetailedBenchmarksReport) -> List): List { + return detailedMetricReports.values.map { + convertor(it) + }.flatten() + } + + // Countable properties. + val failedBenchmarks: List + get() = getReducedResult { report -> + report.mergedReport.filter { it.value.first?.status == BenchmarkResult.Status.FAILED }.map { it.key } } + + val addedBenchmarks: List + get() = getReducedResult { report -> + report.mergedReport.filter { it.value.second == null }.map { it.key } + } + + val removedBenchmarks: List + get() = getReducedResult { report -> + report.mergedReport.filter { it.value.first == null }.map { it.key } + } + + val currentMeanVarianceBenchmarks: List + get() = getReducedResult { report -> + report.mergedReport.filter { it.value.first != null }.map { it.value.first!! } + } + + val benchmarksNumber: Int + get() = detailedMetricReports.values.fold(0) { acc, it -> acc + it.benchmarksNumber } + + val currentBenchmarksDuration: Map + get() = benchmarksDurations.filter { it.value.first != null }.map { it.key to it.value.first!! }.toMap() + + val envChanges: List> + get() { + val previousEnvironment = environments.second + val currentEnvironment = environments.first + return previousEnvironment?.let { + mutableListOf>().apply { + addFieldChange("Machine CPU", previousEnvironment.machine.cpu, currentEnvironment.machine.cpu) + addFieldChange("Machine OS", previousEnvironment.machine.os, currentEnvironment.machine.os) + addFieldChange("JDK version", previousEnvironment.jdk.version, currentEnvironment.jdk.version) + addFieldChange("JDK vendor", previousEnvironment.jdk.vendor, currentEnvironment.jdk.vendor) + } + } ?: listOf>() + } + + val kotlinChanges: List> + get() { + val previousCompiler = compilers.second + val currentCompiler = compilers.first + return previousCompiler?.let { + mutableListOf>().apply { + addFieldChange("Backend type", previousCompiler.backend.type.type, currentCompiler.backend.type.type) + addFieldChange("Backend version", previousCompiler.backend.version, currentCompiler.backend.version) + addFieldChange("Backend flags", previousCompiler.backend.flags.toString(), + currentCompiler.backend.flags.toString()) + addFieldChange("Kotlin version", previousCompiler.kotlinVersion, currentCompiler.kotlinVersion) + } + } ?: listOf>() + } + + init { + // Count avarage values for each benchmark. + detailedMetricReports = BenchmarkResult.Metric.values().map { metric -> + val currentBenchmarks = currentReport.benchmarks.map { (name, benchmarks) -> + name to benchmarks.filter { it.metric == metric } + }.filter { it.second.isNotEmpty() }.toMap() + val previousBenchmarks = previousReport?.benchmarks?.map { (name, benchmarks) -> + name to benchmarks.filter { it.metric == metric } + }?.filter { it.second.isNotEmpty() }?.toMap() + metric to DetailedBenchmarksReport( + currentReport.benchmarks.map { (name, benchmarks) -> + name to benchmarks.filter { it.metric == metric } + }.filter { it.second.isNotEmpty() }.toMap(), + previousReport?.benchmarks?.map { (name, benchmarks) -> + name to benchmarks.filter { it.metric == metric } + }?.filter { it.second.isNotEmpty() }?.toMap(), + meaningfulChangesValue + ) + }.toMap() + benchmarksDurations = calculateBenchmarksDuration(currentReport, previousReport) + environments = Pair(currentReport.env, previousReport?.env) + compilers = Pair(currentReport.compiler, previousReport?.compiler) + } + + // Get benchmark report. + fun getBenchmarksReport(takeMainReport: Boolean = true) = + if (takeMainReport) + BenchmarksReport(environments.first, getReducedResult { report -> + report.mergedReport.map { (_, value) -> value.first!! } + }, compilers.first) + else + BenchmarksReport(environments.second!!, getReducedResult { report -> + report.mergedReport.map { (_, value) -> value.second!! } + }, compilers.second!!) + + fun getUnstableBenchmarksForMetric(metric: BenchmarkResult.Metric) = + if (metric == BenchmarkResult.Metric.EXECUTION_TIME) unstableBenchmarks else emptyList() + + // Generate map with summary durations of each benchmark. + private fun calculateBenchmarksDuration(currentReport: BenchmarksReport, previousReport: BenchmarksReport?): + Map> { + val currentDurations = collectBenchmarksDurations(currentReport.benchmarks) + val previousDurations = previousReport?.let { + collectBenchmarksDurations(previousReport.benchmarks) + } ?: mapOf() + return currentDurations.keys.union(previousDurations.keys) + .map { it to Pair(currentDurations[it], previousDurations[it]) }.toMap() } private fun MutableList>.addFieldChange(field: String, previous: T, current: T) { diff --git a/kotlin-native/tools/benchmarks/shared/src/main/kotlin/report/BenchmarksReport.kt b/kotlin-native/tools/benchmarks/shared/src/main/kotlin/report/BenchmarksReport.kt index 7448d89117b..7323ab9cafe 100644 --- a/kotlin-native/tools/benchmarks/shared/src/main/kotlin/report/BenchmarksReport.kt +++ b/kotlin-native/tools/benchmarks/shared/src/main/kotlin/report/BenchmarksReport.kt @@ -270,7 +270,6 @@ open class BenchmarkResult(val name: String, val status: Status, val metric = if (metricElement != null && metricElement is JsonLiteral) metricFromString(metricElement.unquoted()) ?: Metric.EXECUTION_TIME else Metric.EXECUTION_TIME - name += metric.suffix val statusElement = data.getRequiredField("status") if (statusElement is JsonLiteral) { val status = statusFromString(statusElement.unquoted()) @@ -348,4 +347,35 @@ open class MeanVarianceBenchmark(name: String, status: BenchmarkResult.Status, s "variance": $variance """ } +} + +// Benchmark with set results stability state. +open class BenchmarkWithStabilityState(name: String, status: BenchmarkResult.Status, score: Double, metric: BenchmarkResult.Metric, + runtimeInUs: Double, repeat: Int, warmup: Int, val unstable: Boolean) : + BenchmarkResult(name, status, score, metric, runtimeInUs, repeat, warmup) { + + constructor(benchmarkResult: BenchmarkResult, unstable: Boolean) : this(benchmarkResult.name, + benchmarkResult.status, benchmarkResult.score, benchmarkResult.metric, + benchmarkResult.runtimeInUs, benchmarkResult.repeat, benchmarkResult.warmup, unstable) + + override fun serializeFields(): String { + return """ + ${super.serializeFields()}, + "unstable": $unstable + """ + } + + companion object : EntityFromJsonFactory { + override fun create(data: JsonElement): BenchmarkWithStabilityState { + val parsedObject = BenchmarkResult.create(data) + if (data is JsonObject) { + val unstableElement = data.getOptionalField("unstable") + val unstableFlag = if (unstableElement != null && unstableElement is JsonPrimitive) + unstableElement.boolean else false + return BenchmarkWithStabilityState(parsedObject, unstableFlag) + } else { + error("Benchmark entity is expected to be an object. Please, check origin files.") + } + } + } } \ No newline at end of file diff --git a/kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/main.kt b/kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/main.kt index 250ee8283b2..0e8b7e9b15a 100644 --- a/kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/main.kt +++ b/kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/main.kt @@ -79,6 +79,21 @@ object DBServerConnector : Connector() { val accessFileUrl = "$serverUrl/report/$target/$buildNumber" return sendGetRequest(accessFileUrl) } + + fun getUnstableBenchmarks(): List? { + try { + val unstableList = sendGetRequest("$serverUrl/unstable") + val data = JsonTreeParser.parse(unstableList) + if (data !is JsonArray) { + return null + } + return data.jsonArray.map { + (it as JsonPrimitive).content + } + } catch (e: Exception) { + return null + } + } } fun getFileContent(fileName: String, user: String? = null): String { @@ -155,6 +170,11 @@ fun main(args: Array) { val user by argParser.option(ArgType.String, shortName = "u", description = "User access information for authorization") argParser.parse(args) + // Get unstable benchmarks. + val unstableBenchmarks = DBServerConnector.getUnstableBenchmarks() + + unstableBenchmarks ?: println("Failed to get access to server and get unstable benchmarks list!") + // Read contents of file. val mainBenchsReport = mergeReportsWithDetailedFlags(getBenchmarkReport(mainReport, user)) @@ -164,8 +184,8 @@ fun main(args: Array) { // Generate comparasion report. val summaryReport = SummaryBenchmarksReport(mainBenchsReport, - compareToBenchsReport, - epsValue) + compareToBenchsReport, epsValue, + unstableBenchmarks ?: emptyList()) var outputFile = output renders.forEach { diff --git a/kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/HTMLRender.kt b/kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/HTMLRender.kt index d9423e3fc7d..a44d1f514ec 100644 --- a/kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/HTMLRender.kt +++ b/kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/HTMLRender.kt @@ -405,7 +405,7 @@ class HTMLRender: Render() { } } - val benchmarksWithChangedStatus = report.getBenchmarksWithChangedStatus() + val benchmarksWithChangedStatus = report.benchmarksWithChangedStatus val newFailures = benchmarksWithChangedStatus .filter { it.current == BenchmarkResult.Status.FAILED } val newPasses = benchmarksWithChangedStatus @@ -479,59 +479,86 @@ class HTMLRender: Render() { } private fun BodyTag.renderPerformanceSummary(report: SummaryBenchmarksReport) { - if (!report.improvements.isEmpty() || !report.regressions.isEmpty()) { + if (report.detailedMetricReports.values.any { it.improvements.isNotEmpty() } || + report.detailedMetricReports.values.any { it.regressions.isNotEmpty() }) { h4 { +"Performance Summary" } table { attributes["class"] = "table table-sm table-striped table-hover" attributes["style"] = "width:initial;" thead { tr { - th { +"Change" } - th { +"#" } - th { +"Maximum" } - th { +"Geometric mean" } - } - } - val maximumRegression = report.maximumRegression - val maximumImprovement = report.maximumImprovement - val regressionsGeometricMean = report.regressionsGeometricMean - val improvementsGeometricMean = report.improvementsGeometricMean - tbody { - if (!report.regressions.isEmpty()) { - tr { - th { +"Regressions" } - td { +"${report.regressions.size}" } - td { - attributes["bgcolor"] = ColoredCell( - maximumRegression/maxOf(maximumRegression, abs(maximumImprovement))) - .backgroundStyle - +formatValue(maximumRegression, true) - } - td { - attributes["bgcolor"] = ColoredCell( - regressionsGeometricMean/maxOf(regressionsGeometricMean, - abs(improvementsGeometricMean))) - .backgroundStyle - +formatValue(report.regressionsGeometricMean, true) - } + th(rowspan = Natural(2)) { +"Change" } + report.detailedMetricReports.forEach { (metric, _) -> + th(colspan = Natural(3)) { +metric.value } } } - if (!report.improvements.isEmpty()) { + tr { + report.detailedMetricReports.forEach { _ -> + th { +"#" } + th { +"Maximum" } + th { +"Geometric mean" } + } + } + } + + tbody { + tr { + th { +"Regressions" } + report.detailedMetricReports.values.forEach { report -> + val maximumRegression = report.maximumRegression + val regressionsGeometricMean = report.regressionsGeometricMean + val maximumImprovement = report.maximumImprovement + val improvementsGeometricMean = report.improvementsGeometricMean + val maximumChange = maxOf(maximumRegression, abs(maximumImprovement)) + val maximumChangeGeoMean = maxOf(regressionsGeometricMean, + abs(improvementsGeometricMean)) + if (!report.regressions.isEmpty()) { + td { +"${report.regressions.size}" } + td { + attributes["bgcolor"] = ColoredCell( + (maximumRegression/maximumChange).takeIf{ maximumChange > 0.0 } + ).backgroundStyle + +formatValue(maximumRegression, true) + } + td { + attributes["bgcolor"] = ColoredCell( + (regressionsGeometricMean/maximumChangeGeoMean).takeIf{ maximumChangeGeoMean > 0.0 } + ).backgroundStyle + +formatValue(report.regressionsGeometricMean, true) + } + } else { + repeat(3) { td { +"-" } } + } + } + tr { th { +"Improvements" } - td { +"${report.improvements.size}" } - td { - attributes["bgcolor"] = ColoredCell( - maximumImprovement/maxOf(maximumRegression, abs(maximumImprovement))) - .backgroundStyle - +formatValue(report.maximumImprovement, true) - } - td { - attributes["bgcolor"] = ColoredCell( - improvementsGeometricMean/maxOf(regressionsGeometricMean, - abs(improvementsGeometricMean))) - .backgroundStyle - +formatValue(report.improvementsGeometricMean, true) + report.detailedMetricReports.values.forEach { report -> + val maximumRegression = report.maximumRegression + val regressionsGeometricMean = report.regressionsGeometricMean + val maximumImprovement = report.maximumImprovement + val improvementsGeometricMean = report.improvementsGeometricMean + val maximumChange = maxOf(maximumRegression, abs(maximumImprovement)) + val maximumChangeGeoMean = maxOf(regressionsGeometricMean, + abs(improvementsGeometricMean)) + if (!report.improvements.isEmpty()) { + td { +"${report.improvements.size}" } + td { + attributes["bgcolor"] = ColoredCell( + (maximumImprovement / maximumChange).takeIf{ maximumChange > 0.0 } + ).backgroundStyle + +formatValue(report.maximumImprovement, true) + } + td { + attributes["bgcolor"] = ColoredCell( + (improvementsGeometricMean / maximumChangeGeoMean) + .takeIf{ maximumChangeGeoMean > 0.0 } + ).backgroundStyle + +formatValue(report.improvementsGeometricMean, true) + } + } else { + repeat(3) { td { +"-" } } + } } } } @@ -584,43 +611,84 @@ class HTMLRender: Render() { } } + private fun TableBlock.renderFilteredBenchmarks(detailedReport: DetailedBenchmarksReport, + onlyChanges: Boolean, unstableBenchmarks: List, + filterUnstable: Boolean) { + fun filterBenchmarks(bucket: Map) = + bucket.filter { (name, _) -> + if (filterUnstable) name in unstableBenchmarks else name !in unstableBenchmarks + } + + val filteredRegressions = filterBenchmarks(detailedReport.regressions) + val filteredImprovements = filterBenchmarks(detailedReport.improvements) + renderBenchmarksDetails(detailedReport.mergedReport, filteredRegressions) + renderBenchmarksDetails(detailedReport.mergedReport, filteredImprovements) + if (!onlyChanges) { + // Print all remaining results. + renderBenchmarksDetails(filterBenchmarks(detailedReport.mergedReport).filter { + it.key !in detailedReport.regressions.keys && + it.key !in detailedReport.improvements.keys + }) + } + } + private fun BodyTag.renderPerformanceDetails(report: SummaryBenchmarksReport, onlyChanges: Boolean) { if (onlyChanges) { - if (report.regressions.isEmpty() && report.improvements.isEmpty()) { + if (report.detailedMetricReports.values.all { it.improvements.isEmpty() } && + report.detailedMetricReports.values.all { it.regressions.isEmpty() }) { div("alert alert-success") { attributes["role"] = "alert" +"All becnhmarks are stable!" } } } + report.detailedMetricReports.forEach { (metric, detailedReport) -> + renderCollapsedData(metric.value, false) { + table { + attributes["id"] = "result" + attributes["class"] = "table table-striped table-bordered" + thead { + tr { + th { +"Benchmark" } + th { +"First score" } + th { +"Second score" } + th { +"Percent" } + th { +"Ratio" } + } + } + val geoMeanChangeMap = detailedReport.geoMeanScoreChange?.let { + mapOf(detailedReport.geoMeanBenchmark.first!!.name to detailedReport.geoMeanScoreChange!!) + } - table { - attributes["id"] = "result" - attributes["class"] = "table table-striped table-bordered" - thead { - tr { - th { +"Benchmark" } - th { +"First score" } - th { +"Second score" } - th { +"Percent" } - th { +"Ratio" } - } - } - val geoMeanChangeMap = report.geoMeanScoreChange?. - let { mapOf(report.geoMeanBenchmark.first!!.name to report.geoMeanScoreChange!!) } - - tbody { - renderBenchmarksDetails( - mutableMapOf(report.geoMeanBenchmark.first!!.name to report.geoMeanBenchmark), - geoMeanChangeMap, "border-bottom: 2.3pt solid black; border-top: 2.3pt solid black") - renderBenchmarksDetails(report.mergedReport, report.regressions) - renderBenchmarksDetails(report.mergedReport, report.improvements) - if (!onlyChanges) { - // Print all remaining results. - renderBenchmarksDetails(report.mergedReport.filter { it.key !in report.regressions.keys && - it.key !in report.improvements.keys }) + tbody { + val boldRowStyle = "border-bottom: 2.3pt solid black; border-top: 2.3pt solid black" + renderBenchmarksDetails( + mutableMapOf(detailedReport.geoMeanBenchmark.first!!.name to detailedReport.geoMeanBenchmark), + geoMeanChangeMap, boldRowStyle) + + val unstableBenchmarks = report.getUnstableBenchmarksForMetric(metric) + + if (unstableBenchmarks.isNotEmpty()) { + tr { + attributes["style"] = boldRowStyle + th(colspan = Natural(5)) { +"Stable" } + } + } + + renderFilteredBenchmarks(detailedReport, onlyChanges, unstableBenchmarks, false) + + if (unstableBenchmarks.isNotEmpty()) { + tr { + attributes["style"] = boldRowStyle + th(colspan = Natural(5)) { +"Unstable" } + } + } + + renderFilteredBenchmarks(detailedReport, onlyChanges, unstableBenchmarks, true) + } } } + hr {} } } diff --git a/kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/MetricResultsRender.kt b/kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/MetricResultsRender.kt index bab962287c6..b73a302d4dc 100644 --- a/kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/MetricResultsRender.kt +++ b/kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/MetricResultsRender.kt @@ -14,14 +14,16 @@ class MetricResultsRender: Render() { get() = "metrics" override fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean): String { - val results = report.mergedReport.map { entry -> - buildString { - val metric = entry.value.first!!.metric - append("{ \"benchmarkName\": \"${entry.key.removeSuffix(metric.suffix)}\",") - append("\"metric\": \"${metric}\",") - append("\"value\": \"${entry.value.first!!.score}\" }") + val results = report.detailedMetricReports.values.map { it.mergedReport }.map { report -> + report.map { entry -> + buildString { + val metric = entry.value.first!!.metric + append("{ \"benchmarkName\": \"${entry.key.removeSuffix(metric.suffix)}\",") + append("\"metric\": \"${metric}\",") + append("\"value\": \"${entry.value.first!!.score}\" }") + } } - }.joinToString(", ") + }.flatten().joinToString(", ") return "[ $results ]" } diff --git a/kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt b/kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt index eda8ddb085e..e1920fa4ca3 100644 --- a/kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt +++ b/kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt @@ -54,7 +54,7 @@ class TextRender: Render() { renderEnvChanges(report.envChanges, "Environment") renderEnvChanges(report.kotlinChanges, "Compiler") renderStatusSummary(report) - renderStatusChangesDetails(report.getBenchmarksWithChangedStatus()) + renderStatusChangesDetails(report.benchmarksWithChangedStatus) renderPerformanceSummary(report) renderPerformanceDetails(report, onlyChanges) return content.toString() @@ -113,18 +113,27 @@ class TextRender: Render() { } fun renderPerformanceSummary(report: SummaryBenchmarksReport) { - if (!report.regressions.isEmpty() || !report.improvements.isEmpty()) { + if (report.detailedMetricReports.values.any { it.improvements.isNotEmpty() } || + report.detailedMetricReports.values.any { it.regressions.isNotEmpty() }) { append("Performance summary") append(headerSeparator) - if (!report.regressions.isEmpty()) { - append("Regressions: Maximum = ${formatValue(report.maximumRegression, true)}," + - " Geometric mean = ${formatValue(report.regressionsGeometricMean, true)}") - } - if (!report.improvements.isEmpty()) { - append("Improvements: Maximum = ${formatValue(report.maximumImprovement, true)}," + - " Geometric mean = ${formatValue(report.improvementsGeometricMean, true)}") - } append() + report.detailedMetricReports.forEach { (metric, detailedReport) -> + if (detailedReport.regressions.isNotEmpty() || detailedReport.improvements.isNotEmpty()) { + append(metric.value) + append(headerSeparator) + if (!detailedReport.regressions.isEmpty()) { + append("Regressions: Maximum = ${formatValue(detailedReport.maximumRegression, true)}," + + " Geometric mean = ${formatValue(detailedReport.regressionsGeometricMean, true)}") + } + if (!detailedReport.improvements.isEmpty()) { + append("Improvements: Maximum = ${formatValue(detailedReport.maximumImprovement, true)}," + + " Geometric mean = ${formatValue(detailedReport.improvementsGeometricMean, true)}") + } + append() + } + } + } } @@ -176,25 +185,58 @@ class TextRender: Render() { append(headerSeparator) if (onlyChanges) { - if (report.regressions.isEmpty() && report.improvements.isEmpty()) { + if (report.detailedMetricReports.values.all { it.improvements.isEmpty() } && + report.detailedMetricReports.values.all { it.regressions.isEmpty() }) { append("All becnhmarks are stable.") } } - val tableWidth = printPerformanceTableHeader() - // Print geometric mean. - val geoMeanChangeMap = report.geoMeanScoreChange?. - let { mapOf(report.geoMeanBenchmark.first!!.name to report.geoMeanScoreChange!!) } - printBenchmarksDetails( - mutableMapOf(report.geoMeanBenchmark.first!!.name to report.geoMeanBenchmark), - geoMeanChangeMap) - printTableLineSeparator(tableWidth) - printBenchmarksDetails(report.mergedReport, report.regressions) - printBenchmarksDetails(report.mergedReport, report.improvements) + report.detailedMetricReports.forEach { (metric, detailedReport) -> + append() + append(metric.value) + append(headerSeparator) + val tableWidth = printPerformanceTableHeader() + // Print geometric mean. + val geoMeanChangeMap = detailedReport.geoMeanScoreChange?.let { + mapOf(detailedReport.geoMeanBenchmark.first!!.name to detailedReport.geoMeanScoreChange!!) + } + printBenchmarksDetails( + mutableMapOf(detailedReport.geoMeanBenchmark.first!!.name to detailedReport.geoMeanBenchmark), + geoMeanChangeMap) + printTableLineSeparator(tableWidth) + val unstableBenchmarks = report.getUnstableBenchmarksForMetric(metric) + if (unstableBenchmarks.isNotEmpty()) { + append("Stable") + printTableLineSeparator(tableWidth) + } + renderFilteredPerformanceDetails(detailedReport, onlyChanges, unstableBenchmarks, false) + if (unstableBenchmarks.isNotEmpty()) { + printTableLineSeparator(tableWidth) + append("Unstable") + printTableLineSeparator(tableWidth) + } + renderFilteredPerformanceDetails(detailedReport, onlyChanges, unstableBenchmarks,true) + } + } + + fun renderFilteredPerformanceDetails(detailedReport: DetailedBenchmarksReport, + onlyChanges: Boolean, unstableBenchmarks: List, + filterUnstable: Boolean) { + fun filterBenchmarks(bucket: Map) = + bucket.filter { (name, _) -> + if (filterUnstable) name in unstableBenchmarks else name !in unstableBenchmarks + } + + val filteredRegressions = filterBenchmarks(detailedReport.regressions) + val filteredImprovements = filterBenchmarks(detailedReport.improvements) + printBenchmarksDetails(detailedReport.mergedReport, filteredRegressions) + printBenchmarksDetails(detailedReport.mergedReport, filteredImprovements) if (!onlyChanges) { // Print all remaining results. - printBenchmarksDetails(report.mergedReport.filter { it.key !in report.regressions.keys && - it.key !in report.improvements.keys }) + printBenchmarksDetails(filterBenchmarks(detailedReport.mergedReport).filter { + it.key !in detailedReport.regressions.keys && + it.key !in detailedReport.improvements.keys + }) } } } diff --git a/kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/StatisticsRender.kt b/kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/StatisticsRender.kt index abccbb5808c..b8da8626516 100644 --- a/kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/StatisticsRender.kt +++ b/kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/StatisticsRender.kt @@ -20,7 +20,7 @@ class StatisticsRender: Render() { private var content = StringBuilder() override fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean): String { - val benchmarksWithChangedStatus = report.getBenchmarksWithChangedStatus() + val benchmarksWithChangedStatus = report.benchmarksWithChangedStatus val newPasses = benchmarksWithChangedStatus .filter { it.current == BenchmarkResult.Status.PASSED } val newFailures = benchmarksWithChangedStatus @@ -28,6 +28,12 @@ class StatisticsRender: Render() { if (report.failedBenchmarks.isNotEmpty()) { content.append("failed: ${report.failedBenchmarks.size}\n") } + val regressionsSize = report.detailedMetricReports.values.fold(0) { acc, it -> + acc + it.regressions.size + } + val improvementsSize = report.detailedMetricReports.values.fold(0) { acc, it -> + acc + it.improvements.size + } val status = when { newFailures.isNotEmpty() -> { content.append("new failures: ${newFailures.size}\n") @@ -37,16 +43,16 @@ class StatisticsRender: Render() { content.append("new passes: ${newPasses.size}\n") Status.FIXED } - report.improvements.isNotEmpty() && report.regressions.isNotEmpty() -> { - content.append("regressions: ${report.regressions.size}\nimprovements: ${report.improvements.size}") + regressionsSize != 0 && improvementsSize != 0 -> { + content.append("regressions: $regressionsSize\nimprovements: $improvementsSize") Status.UNSTABLE } - report.improvements.isNotEmpty() && report.regressions.isEmpty() -> { - content.append("improvements: ${report.improvements.size}") + improvementsSize != 0 && regressionsSize == 0 -> { + content.append("improvements: $improvementsSize") Status.IMPROVED } - report.improvements.isEmpty() && report.regressions.isNotEmpty() -> { - content.append("regressions: ${report.regressions.size}") + improvementsSize == 0 && regressionsSize != 0 -> { + content.append("regressions: $regressionsSize") Status.REGRESSED } else -> Status.STABLE diff --git a/kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/TeamCityStatisticsRender.kt b/kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/TeamCityStatisticsRender.kt index 548a421a65d..81d44d782a7 100644 --- a/kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/TeamCityStatisticsRender.kt +++ b/kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/TeamCityStatisticsRender.kt @@ -29,7 +29,9 @@ class TeamCityStatisticsRender: Render() { content.append("##teamcity[testSuiteFinished name='Benchmarks']\n") // Report geometric mean as build statistic value - renderGeometricMean(report.geoMeanBenchmark.first!!) + report.detailedMetricReports.forEach { (metric, detailedReport) -> + renderGeometricMean(metric.value, detailedReport.geoMeanBenchmark.first!!) + } return content.toString() } @@ -51,8 +53,8 @@ class TeamCityStatisticsRender: Render() { content.append("##teamcity[testFinished name='${benchmark.name}' duration='${(duration / 1000).toInt()}']\n") } - private fun renderGeometricMean(geoMeanBenchmark: MeanVarianceBenchmark) { - content.append("##teamcity[buildStatisticValue key='Geometric mean' value='${geoMeanBenchmark.score}']\n") - content.append("##teamcity[buildStatisticValue key='Geometric mean variance' value='${geoMeanBenchmark.variance}']\n") + private fun renderGeometricMean(metricName: String, geoMeanBenchmark: MeanVarianceBenchmark) { + content.append("##teamcity[buildStatisticValue key='$metricName Geometric mean' value='${geoMeanBenchmark.score}']\n") + content.append("##teamcity[buildStatisticValue key='$metricName Geometric mean variance' value='${geoMeanBenchmark.variance}']\n") } } \ No newline at end of file diff --git a/kotlin-native/tools/performance-server/src/main/kotlin/database/BenchmarksIndexesDispatcher.kt b/kotlin-native/tools/performance-server/src/main/kotlin/database/BenchmarksIndexesDispatcher.kt index 2ae964db6ba..d5927313b9c 100644 --- a/kotlin-native/tools/performance-server/src/main/kotlin/database/BenchmarksIndexesDispatcher.kt +++ b/kotlin-native/tools/performance-server/src/main/kotlin/database/BenchmarksIndexesDispatcher.kt @@ -256,6 +256,7 @@ class BenchmarksIndexesDispatcher(connector: ElasticSearchConnector, val feature fun getGeometricMean(metricName: String, featureValue: String = "", buildNumbers: Iterable? = null, normalize: Boolean = false, excludeNames: List = emptyList()): Promise>>> { + // Filter only with metric or also with names. val filterBenchmarks = if (excludeNames.isEmpty()) """ @@ -264,7 +265,7 @@ class BenchmarksIndexesDispatcher(connector: ElasticSearchConnector, val feature else """ "bool": { "must": { "match": { "benchmarks.metric": "$metricName" } }, - "must_not": { "terms" : { "benchmarks.name" : [${excludeNames.map { "\"$it\"" }.joinToString()}] } } + "must_not": [ ${excludeNames.map { """{ "match_phrase" : { "benchmarks.name" : "$it" } }"""}.joinToString() } ] } """.trimIndent() val queryDescription = """ diff --git a/kotlin-native/tools/performance-server/src/main/kotlin/database/DatabaseRequests.kt b/kotlin-native/tools/performance-server/src/main/kotlin/database/DatabaseRequests.kt index 8d247aa221b..2d74ee2a81f 100644 --- a/kotlin-native/tools/performance-server/src/main/kotlin/database/DatabaseRequests.kt +++ b/kotlin-native/tools/performance-server/src/main/kotlin/database/DatabaseRequests.kt @@ -131,6 +131,42 @@ fun getGoldenResults(goldenResultsIndex: GoldenResultsIndex): Promise> { + val queryDescription = """ + { + "_source": ["env"], + "query": { + "nested" : { + "path" : "benchmarks", + "query" : { + "match": { "benchmarks.unstable": true } + }, + "inner_hits": { + "size": 100, + "_source": ["benchmarks.name"] + } + } + } + } + """.trimIndent() + + return goldenResultsIndex.search(queryDescription, listOf("hits.hits.inner_hits")).then { responseString -> + val dbResponse = JsonTreeParser.parse(responseString).jsonObject + val results = dbResponse.getObjectOrNull("hits")?.getArrayOrNull("hits") + ?: error("Wrong response:\n$responseString") + results.getObjectOrNull(0)?.let { + it + .getObject("inner_hits") + .getObject("benchmarks") + .getObject("hits") + .getArray("hits").map { + (it as JsonObject).getObject("_source").getPrimitive("name").content + } + } ?: listOf() + } +} + // Get distinct values for needed field from database. fun distinctValues(field: String, index: ElasticSearchIndex): Promise> { val queryDescription = """ diff --git a/kotlin-native/tools/performance-server/src/main/kotlin/routes/route.kt b/kotlin-native/tools/performance-server/src/main/kotlin/routes/route.kt index da4e4db3c59..358334e0e38 100644 --- a/kotlin-native/tools/performance-server/src/main/kotlin/routes/route.kt +++ b/kotlin-native/tools/performance-server/src/main/kotlin/routes/route.kt @@ -588,6 +588,18 @@ fun router() { } }) + // Get builds description with additional information. + router.get("/unstable", { request, response -> + CachableResponseDispatcher.getResponse(request, response) { success, reject -> + getUnstableResults(goldenIndex).then { unstableBenchmarks -> + success(unstableBenchmarks) + }.catch { + println("Error during getting unstable benchmarks") + reject() + } + } + }) + router.get("/report/:target/:buildNumber", { request, response -> val target = urlParameterToBaseFormat(request.params.target) val buildNumber = request.params.buildNumber.toString() diff --git a/kotlin-native/tools/performance-server/ui/src/main/kotlin/main.kt b/kotlin-native/tools/performance-server/ui/src/main/kotlin/main.kt index ce26fc3c05b..5cf3c8d58bd 100644 --- a/kotlin-native/tools/performance-server/ui/src/main/kotlin/main.kt +++ b/kotlin-native/tools/performance-server/ui/src/main/kotlin/main.kt @@ -206,8 +206,8 @@ fun main(args: Array) { } buildsNumberToShow = parameters["count"]?.toInt() ?: buildsNumberToShow - beforeDate = parameters["before"]?.let{ decodeURIComponent(it)} - afterDate = parameters["after"]?.let{ decodeURIComponent(it)} + beforeDate = parameters["before"]?.let { decodeURIComponent(it) } + afterDate = parameters["after"]?.let { decodeURIComponent(it) } // Get branches. val branchesUrl = "$serverUrl/branches" @@ -285,30 +285,6 @@ fun main(args: Array) { val platformSpecificBenchs = if (parameters["target"] == "Mac_OS_X") ",FrameworkBenchmarksAnalyzer,SpaceFramework_iosX64" else if (parameters["target"] == "Linux") ",kotlinx.coroutines" else "" - // Collect information for charts library. - val valuesToShow = mapOf("EXECUTION_TIME" to listOf(mapOf( - "normalize" to "true" - )), - "COMPILE_TIME" to listOf(mapOf( - "samples" to "HelloWorld,Videoplayer$platformSpecificBenchs", - "agr" to "samples" - )), - "CODE_SIZE" to listOf(mapOf( - "normalize" to "true", - "exclude" to if (parameters["target"] == "Linux") - "kotlinx.coroutines" - else if (parameters["target"] == "Mac_OS_X") - "SpaceFramework_iosX64" - else "" - ), if (platformSpecificBenchs.isNotEmpty()) mapOf( - "normalize" to "true", - "agr" to "samples", - "samples" to platformSpecificBenchs.removePrefix(",") - ) else null).filterNotNull(), - "BUNDLE_SIZE" to listOf(mapOf("samples" to "KotlinNative", - "agr" to "samples")) - ) - var execData = listOf() to listOf>() var compileData = listOf() to listOf>() var codeSizeData = listOf() to listOf>() @@ -328,6 +304,17 @@ fun main(args: Array) { val metricUrl = "$serverUrl/metricValue/${parameters["target"]}/" + val unstableBenchmarksPromise = sendGetRequest("$serverUrl/unstable").then { response -> + val unstableList = response as String + val data = JsonTreeParser.parse(unstableList) + if (data !is JsonArray) { + error("Response is expected to be an array.") + } + data.jsonArray.map { + (it as JsonPrimitive).content + } + } + // Get builds description. val buildsInfoPromise = sendGetRequest(descriptionUrl).then { response -> val buildsInfo = response as String @@ -341,100 +328,131 @@ fun main(args: Array) { } } - // Send requests to get all needed metric values. - valuesToShow.map { (metric, listOfSettings) -> - val resultValues = listOfSettings.map { settings -> - val getParameters = with(StringBuilder()) { - if (settings.isNotEmpty()) { - append("?") - } - var prefix = "" - settings.forEach { (key, value) -> - if (value.isNotEmpty()) { - append("$prefix$key=$value") - prefix = "&" + unstableBenchmarksPromise.then { unstableBenchmarks -> + // Collect information for charts library. + val valuesToShow = mapOf("EXECUTION_TIME" to listOf(mapOf( + "normalize" to "true" + ), + mapOf( + "normalize" to "true", + "exclude" to unstableBenchmarks.joinToString(",") + )), + "COMPILE_TIME" to listOf(mapOf( + "samples" to "HelloWorld,Videoplayer$platformSpecificBenchs", + "agr" to "samples" + )), + "CODE_SIZE" to listOf(mapOf( + "normalize" to "true", + "exclude" to if (parameters["target"] == "Linux") + "kotlinx.coroutines" + else if (parameters["target"] == "Mac_OS_X") + "SpaceFramework_iosX64" + else "" + ), if (platformSpecificBenchs.isNotEmpty()) mapOf( + "normalize" to "true", + "agr" to "samples", + "samples" to platformSpecificBenchs.removePrefix(",") + ) else null).filterNotNull(), + "BUNDLE_SIZE" to listOf(mapOf("samples" to "KotlinNative", + "agr" to "samples")) + ) + // Send requests to get all needed metric values. + valuesToShow.map { (metric, listOfSettings) -> + val resultValues = listOfSettings.map { settings -> + val getParameters = with(StringBuilder()) { + if (settings.isNotEmpty()) { + append("?") } + var prefix = "" + settings.forEach { (key, value) -> + if (value.isNotEmpty()) { + append("$prefix$key=$value") + prefix = "&" + } + } + toString() } - toString() + val branchParameter = if (parameters["branch"] != "all") + (if (getParameters.isEmpty()) "?" else "&") + "branch=${parameters["branch"]}" + else "" + + val url = "$metricUrl$metric$getParameters$branchParameter${ + if (parameters["type"] != "all") + (if (getParameters.isEmpty() && branchParameter.isEmpty()) "?" else "&") + "type=${parameters["type"]}" + else "" + }&count=$buildsNumberToShow${getDatesComponents()}" + sendGetRequest(url) + }.toTypedArray() + + // Get metrics values for charts. + Promise.all(resultValues).then { responses -> + val valuesList = responses.map { response -> + val results = (JsonTreeParser.parse(response) as JsonArray).map { + (it as JsonObject).getPrimitive("first").content to + it.getArray("second").map { (it as JsonPrimitive).doubleOrNull } + } + + val labels = results.map { it.first } + val values = results[0]?.second?.size?.let { (0..it - 1).map { i -> results.map { it.second[i] } } } + ?: emptyList() + labels to values + } + val labels = valuesList[0].first + + val values = valuesList.map { it.second }.reduce { acc, valuesPart -> acc + valuesPart } + + when (metric) { + // Update chart with gotten data. + "COMPILE_TIME" -> { + compileData = labels to values.map { it.map { it?.let { it / 1000 } } } + compileChart = Chartist.Line("#compile_chart", + getChartData(labels, compileData.second), + getChartOptions(valuesToShow["COMPILE_TIME"]!![0]!!["samples"]!!.split(',').toTypedArray(), + "Time, milliseconds")) + buildsInfoPromise.then { builds -> + customizeChart(compileChart, "compile_chart", js("$(\"#compile_chart\")"), builds, parameters) + compileChart.update(getChartData(compileData.first, compileData.second)) + } + } + "EXECUTION_TIME" -> { + execData = labels to values + execChart = Chartist.Line("#exec_chart", + getChartData(labels, execData.second), + getChartOptions(arrayOf("Geometric Mean (All)", "Geometric mean (Stable)"), + "Normalized time")) + buildsInfoPromise.then { builds -> + customizeChart(execChart, "exec_chart", js("$(\"#exec_chart\")"), builds, parameters) + execChart.update(getChartData(execData.first, execData.second)) + } + } + "CODE_SIZE" -> { + codeSizeData = labels to values + codeSizeChart = Chartist.Line("#codesize_chart", + getChartData(labels, codeSizeData.second), + getChartOptions(arrayOf("Geometric Mean") + platformSpecificBenchs.split(',') + .filter { it.isNotEmpty() }, + "Normalized size", + arrayOf("ct-series-4", "ct-series-5", "ct-series-6"))) + buildsInfoPromise.then { builds -> + customizeChart(codeSizeChart, "codesize_chart", js("$(\"#codesize_chart\")"), builds, parameters) + codeSizeChart.update(getChartData(codeSizeData.first, codeSizeData.second, sizeClassNames)) + } + } + "BUNDLE_SIZE" -> { + bundleSizeData = labels to values.map { it.map { it?.let { it.toInt() / 1024 / 1024 } } } + bundleSizeChart = Chartist.Line("#bundlesize_chart", + getChartData(labels, + bundleSizeData.second, sizeClassNames), + getChartOptions(arrayOf("Bundle size"), "Size, MB", arrayOf("ct-series-4"))) + buildsInfoPromise.then { builds -> + customizeChart(bundleSizeChart, "bundlesize_chart", js("$(\"#bundlesize_chart\")"), builds, parameters) + bundleSizeChart.update(getChartData(bundleSizeData.first, bundleSizeData.second, sizeClassNames)) + } + } + else -> error("No chart for metric $metric") + } + true } - val branchParameter = if (parameters["branch"] != "all") - (if (getParameters.isEmpty()) "?" else "&") + "branch=${parameters["branch"]}" - else "" - - val url = "$metricUrl$metric$getParameters$branchParameter${ - if (parameters["type"] != "all") - (if (getParameters.isEmpty() && branchParameter.isEmpty()) "?" else "&") + "type=${parameters["type"]}" - else "" - }&count=$buildsNumberToShow${getDatesComponents()}" - sendGetRequest(url) - }.toTypedArray() - - // Get metrics values for charts. - Promise.all(resultValues).then { responses -> - val valuesList = responses.map { response -> - val results = (JsonTreeParser.parse(response) as JsonArray).map { - (it as JsonObject).getPrimitive("first").content to - it.getArray("second").map { (it as JsonPrimitive).doubleOrNull } - } - - val labels = results.map { it.first } - val values = results[0]?.second?.size?.let { (0..it - 1).map { i -> results.map { it.second[i] } } } - ?: emptyList() - labels to values - } - val labels = valuesList[0].first - val values = valuesList.map { it.second }.reduce { acc, valuesPart -> acc + valuesPart } - - when (metric) { - // Update chart with gotten data. - "COMPILE_TIME" -> { - compileData = labels to values.map { it.map { it?.let { it / 1000 } } } - compileChart = Chartist.Line("#compile_chart", - getChartData(labels, compileData.second), - getChartOptions(valuesToShow["COMPILE_TIME"]!![0]!!["samples"]!!.split(',').toTypedArray(), - "Time, milliseconds")) - buildsInfoPromise.then { builds -> - customizeChart(compileChart, "compile_chart", js("$(\"#compile_chart\")"), builds, parameters) - compileChart.update(getChartData(compileData.first, compileData.second)) - } - } - "EXECUTION_TIME" -> { - execData = labels to values - execChart = Chartist.Line("#exec_chart", - getChartData(labels, execData.second), - getChartOptions(arrayOf("Geometric Mean"), "Normalized time")) - buildsInfoPromise.then { builds -> - customizeChart(execChart, "exec_chart", js("$(\"#exec_chart\")"), builds, parameters) - execChart.update(getChartData(execData.first, execData.second)) - } - } - "CODE_SIZE" -> { - codeSizeData = labels to values - codeSizeChart = Chartist.Line("#codesize_chart", - getChartData(labels, codeSizeData.second), - getChartOptions(arrayOf("Geometric Mean") + platformSpecificBenchs.split(',') - .filter { it.isNotEmpty() }, - "Normalized size", - arrayOf("ct-series-4", "ct-series-5", "ct-series-6"))) - buildsInfoPromise.then { builds -> - customizeChart(codeSizeChart, "codesize_chart", js("$(\"#codesize_chart\")"), builds, parameters) - codeSizeChart.update(getChartData(codeSizeData.first, codeSizeData.second, sizeClassNames)) - } - } - "BUNDLE_SIZE" -> { - bundleSizeData = labels to values.map { it.map { it?.let { it.toInt() / 1024 / 1024 } } } - bundleSizeChart = Chartist.Line("#bundlesize_chart", - getChartData(labels, - bundleSizeData.second, sizeClassNames), - getChartOptions(arrayOf("Bundle size"), "Size, MB", arrayOf("ct-series-4"))) - buildsInfoPromise.then { builds -> - customizeChart(bundleSizeChart, "bundlesize_chart", js("$(\"#bundlesize_chart\")"), builds, parameters) - bundleSizeChart.update(getChartData(bundleSizeData.first, bundleSizeData.second, sizeClassNames)) - } - } - else -> error("No chart for metric $metric") - } - true } }