azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support for stable and unstable benchmarks. Refactoring to get separate tables for metrics. (#4641)

Browse Source
This commit is contained in:
LepilkinaElena
2021-01-18 17:41:59 +03:00
committed by Nikolay Krasko
parent ece5d9b109
commit a403b9ff05
13 changed files with 646 additions and 371 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/main.kt
+22 -2
View File
@@ -79,6 +79,21 @@ object DBServerConnector : Connector() {
val accessFileUrl = "$serverUrl/report/$target/$buildNumber"
return sendGetRequest(accessFileUrl)
}
fun getUnstableBenchmarks(): List<String>? {
try {
val unstableList = sendGetRequest("$serverUrl/unstable")
val data = JsonTreeParser.parse(unstableList)
if (data !is JsonArray) {
return null
}
return data.jsonArray.map {
(it as JsonPrimitive).content
}
} catch (e: Exception) {
return null
}
}
}
fun getFileContent(fileName: String, user: String? = null): String {
@@ -155,6 +170,11 @@ fun main(args: Array<String>) {
val user by argParser.option(ArgType.String, shortName = "u", description = "User access information for authorization")
argParser.parse(args)
// Get unstable benchmarks.
val unstableBenchmarks = DBServerConnector.getUnstableBenchmarks()
unstableBenchmarks ?: println("Failed to get access to server and get unstable benchmarks list!")
// Read contents of file.
val mainBenchsReport = mergeReportsWithDetailedFlags(getBenchmarkReport(mainReport, user))
@@ -164,8 +184,8 @@ fun main(args: Array<String>) {
// Generate comparasion report.
val summaryReport = SummaryBenchmarksReport(mainBenchsReport,
compareToBenchsReport,
epsValue)
compareToBenchsReport, epsValue,
unstableBenchmarks ?: emptyList())
var outputFile = output
renders.forEach {
kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/HTMLRender.kt
+138 -70
View File
@@ -405,7 +405,7 @@ class HTMLRender: Render() {
}
}
val benchmarksWithChangedStatus = report.getBenchmarksWithChangedStatus()
val benchmarksWithChangedStatus = report.benchmarksWithChangedStatus
val newFailures = benchmarksWithChangedStatus
.filter { it.current == BenchmarkResult.Status.FAILED }
val newPasses = benchmarksWithChangedStatus
@@ -479,59 +479,86 @@ class HTMLRender: Render() {
}
private fun BodyTag.renderPerformanceSummary(report: SummaryBenchmarksReport) {
if (!report.improvements.isEmpty() || !report.regressions.isEmpty()) {
if (report.detailedMetricReports.values.any { it.improvements.isNotEmpty() } ||
report.detailedMetricReports.values.any { it.regressions.isNotEmpty() }) {
h4 { +"Performance Summary" }
table {
attributes["class"] = "table table-sm table-striped table-hover"
attributes["style"] = "width:initial;"
thead {
tr {
th { +"Change" }
th { +"#" }
th { +"Maximum" }
th { +"Geometric mean" }
}
}
val maximumRegression = report.maximumRegression
val maximumImprovement = report.maximumImprovement
val regressionsGeometricMean = report.regressionsGeometricMean
val improvementsGeometricMean = report.improvementsGeometricMean
tbody {
if (!report.regressions.isEmpty()) {
tr {
th { +"Regressions" }
td { +"${report.regressions.size}" }
td {
attributes["bgcolor"] = ColoredCell(
maximumRegression/maxOf(maximumRegression, abs(maximumImprovement)))
.backgroundStyle
+formatValue(maximumRegression, true)
}
td {
attributes["bgcolor"] = ColoredCell(
regressionsGeometricMean/maxOf(regressionsGeometricMean,
abs(improvementsGeometricMean)))
.backgroundStyle
+formatValue(report.regressionsGeometricMean, true)
}
th(rowspan = Natural(2)) { +"Change" }
report.detailedMetricReports.forEach { (metric, _) ->
th(colspan = Natural(3)) { +metric.value }
}
}
if (!report.improvements.isEmpty()) {
tr {
report.detailedMetricReports.forEach { _ ->
th { +"#" }
th { +"Maximum" }
th { +"Geometric mean" }
}
}
}
tbody {
tr {
th { +"Regressions" }
report.detailedMetricReports.values.forEach { report ->
val maximumRegression = report.maximumRegression
val regressionsGeometricMean = report.regressionsGeometricMean
val maximumImprovement = report.maximumImprovement
val improvementsGeometricMean = report.improvementsGeometricMean
val maximumChange = maxOf(maximumRegression, abs(maximumImprovement))
val maximumChangeGeoMean = maxOf(regressionsGeometricMean,
abs(improvementsGeometricMean))
if (!report.regressions.isEmpty()) {
td { +"${report.regressions.size}" }
td {
attributes["bgcolor"] = ColoredCell(
(maximumRegression/maximumChange).takeIf{ maximumChange > 0.0 }
).backgroundStyle
+formatValue(maximumRegression, true)
}
td {
attributes["bgcolor"] = ColoredCell(
(regressionsGeometricMean/maximumChangeGeoMean).takeIf{ maximumChangeGeoMean > 0.0 }
).backgroundStyle
+formatValue(report.regressionsGeometricMean, true)
}
} else {
repeat(3) { td { +"-" } }
}
}
tr {
th { +"Improvements" }
td { +"${report.improvements.size}" }
td {
attributes["bgcolor"] = ColoredCell(
maximumImprovement/maxOf(maximumRegression, abs(maximumImprovement)))
.backgroundStyle
+formatValue(report.maximumImprovement, true)
}
td {
attributes["bgcolor"] = ColoredCell(
improvementsGeometricMean/maxOf(regressionsGeometricMean,
abs(improvementsGeometricMean)))
.backgroundStyle
+formatValue(report.improvementsGeometricMean, true)
report.detailedMetricReports.values.forEach { report ->
val maximumRegression = report.maximumRegression
val regressionsGeometricMean = report.regressionsGeometricMean
val maximumImprovement = report.maximumImprovement
val improvementsGeometricMean = report.improvementsGeometricMean
val maximumChange = maxOf(maximumRegression, abs(maximumImprovement))
val maximumChangeGeoMean = maxOf(regressionsGeometricMean,
abs(improvementsGeometricMean))
if (!report.improvements.isEmpty()) {
td { +"${report.improvements.size}" }
td {
attributes["bgcolor"] = ColoredCell(
(maximumImprovement / maximumChange).takeIf{ maximumChange > 0.0 }
).backgroundStyle
+formatValue(report.maximumImprovement, true)
}
td {
attributes["bgcolor"] = ColoredCell(
(improvementsGeometricMean / maximumChangeGeoMean)
.takeIf{ maximumChangeGeoMean > 0.0 }
).backgroundStyle
+formatValue(report.improvementsGeometricMean, true)
}
} else {
repeat(3) { td { +"-" } }
}
}
}
}
@@ -584,43 +611,84 @@ class HTMLRender: Render() {
}
}
private fun TableBlock.renderFilteredBenchmarks(detailedReport: DetailedBenchmarksReport,
onlyChanges: Boolean, unstableBenchmarks: List<String>,
filterUnstable: Boolean) {
fun <T> filterBenchmarks(bucket: Map<String, T>) =
bucket.filter { (name, _) ->
if (filterUnstable) name in unstableBenchmarks else name !in unstableBenchmarks
}
val filteredRegressions = filterBenchmarks(detailedReport.regressions)
val filteredImprovements = filterBenchmarks(detailedReport.improvements)
renderBenchmarksDetails(detailedReport.mergedReport, filteredRegressions)
renderBenchmarksDetails(detailedReport.mergedReport, filteredImprovements)
if (!onlyChanges) {
// Print all remaining results.
renderBenchmarksDetails(filterBenchmarks(detailedReport.mergedReport).filter {
it.key !in detailedReport.regressions.keys &&
it.key !in detailedReport.improvements.keys
})
}
}
private fun BodyTag.renderPerformanceDetails(report: SummaryBenchmarksReport, onlyChanges: Boolean) {
if (onlyChanges) {
if (report.regressions.isEmpty() && report.improvements.isEmpty()) {
if (report.detailedMetricReports.values.all { it.improvements.isEmpty() } &&
report.detailedMetricReports.values.all { it.regressions.isEmpty() }) {
div("alert alert-success") {
attributes["role"] = "alert"
+"All becnhmarks are stable!"
}
}
}
report.detailedMetricReports.forEach { (metric, detailedReport) ->
renderCollapsedData(metric.value, false) {
table {
attributes["id"] = "result"
attributes["class"] = "table table-striped table-bordered"
thead {
tr {
th { +"Benchmark" }
th { +"First score" }
th { +"Second score" }
th { +"Percent" }
th { +"Ratio" }
}
}
val geoMeanChangeMap = detailedReport.geoMeanScoreChange?.let {
mapOf(detailedReport.geoMeanBenchmark.first!!.name to detailedReport.geoMeanScoreChange!!)
}
table {
attributes["id"] = "result"
attributes["class"] = "table table-striped table-bordered"
thead {
tr {
th { +"Benchmark" }
th { +"First score" }
th { +"Second score" }
th { +"Percent" }
th { +"Ratio" }
}
}
val geoMeanChangeMap = report.geoMeanScoreChange?.
let { mapOf(report.geoMeanBenchmark.first!!.name to report.geoMeanScoreChange!!) }
tbody {
renderBenchmarksDetails(
mutableMapOf(report.geoMeanBenchmark.first!!.name to report.geoMeanBenchmark),
geoMeanChangeMap, "border-bottom: 2.3pt solid black; border-top: 2.3pt solid black")
renderBenchmarksDetails(report.mergedReport, report.regressions)
renderBenchmarksDetails(report.mergedReport, report.improvements)
if (!onlyChanges) {
// Print all remaining results.
renderBenchmarksDetails(report.mergedReport.filter { it.key !in report.regressions.keys &&
it.key !in report.improvements.keys })
tbody {
val boldRowStyle = "border-bottom: 2.3pt solid black; border-top: 2.3pt solid black"
renderBenchmarksDetails(
mutableMapOf(detailedReport.geoMeanBenchmark.first!!.name to detailedReport.geoMeanBenchmark),
geoMeanChangeMap, boldRowStyle)
val unstableBenchmarks = report.getUnstableBenchmarksForMetric(metric)
if (unstableBenchmarks.isNotEmpty()) {
tr {
attributes["style"] = boldRowStyle
th(colspan = Natural(5)) { +"Stable" }
}
}
renderFilteredBenchmarks(detailedReport, onlyChanges, unstableBenchmarks, false)
if (unstableBenchmarks.isNotEmpty()) {
tr {
attributes["style"] = boldRowStyle
th(colspan = Natural(5)) { +"Unstable" }
}
}
renderFilteredBenchmarks(detailedReport, onlyChanges, unstableBenchmarks, true)
}
}
}
hr {}
}
}
kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/MetricResultsRender.kt
+9 -7
View File
@@ -14,14 +14,16 @@ class MetricResultsRender: Render() {
get() = "metrics"
override fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean): String {
val results = report.mergedReport.map { entry ->
buildString {
val metric = entry.value.first!!.metric
append("{ \"benchmarkName\": \"${entry.key.removeSuffix(metric.suffix)}\",")
append("\"metric\": \"${metric}\",")
append("\"value\": \"${entry.value.first!!.score}\" }")
val results = report.detailedMetricReports.values.map { it.mergedReport }.map { report ->
report.map { entry ->
buildString {
val metric = entry.value.first!!.metric
append("{ \"benchmarkName\": \"${entry.key.removeSuffix(metric.suffix)}\",")
append("\"metric\": \"${metric}\",")
append("\"value\": \"${entry.value.first!!.score}\" }")
}
}
}.joinToString(", ")
}.flatten().joinToString(", ")
return "[ $results ]"
}
kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt
+65 -23
View File
@@ -54,7 +54,7 @@ class TextRender: Render() {
renderEnvChanges(report.envChanges, "Environment")
renderEnvChanges(report.kotlinChanges, "Compiler")
renderStatusSummary(report)
renderStatusChangesDetails(report.getBenchmarksWithChangedStatus())
renderStatusChangesDetails(report.benchmarksWithChangedStatus)
renderPerformanceSummary(report)
renderPerformanceDetails(report, onlyChanges)
return content.toString()
@@ -113,18 +113,27 @@ class TextRender: Render() {
}
fun renderPerformanceSummary(report: SummaryBenchmarksReport) {
if (!report.regressions.isEmpty() || !report.improvements.isEmpty()) {
if (report.detailedMetricReports.values.any { it.improvements.isNotEmpty() } ||
report.detailedMetricReports.values.any { it.regressions.isNotEmpty() }) {
append("Performance summary")
append(headerSeparator)
if (!report.regressions.isEmpty()) {
append("Regressions: Maximum = ${formatValue(report.maximumRegression, true)}," +
" Geometric mean = ${formatValue(report.regressionsGeometricMean, true)}")
}
if (!report.improvements.isEmpty()) {
append("Improvements: Maximum = ${formatValue(report.maximumImprovement, true)}," +
" Geometric mean = ${formatValue(report.improvementsGeometricMean, true)}")
}
append()
report.detailedMetricReports.forEach { (metric, detailedReport) ->
if (detailedReport.regressions.isNotEmpty() || detailedReport.improvements.isNotEmpty()) {
append(metric.value)
append(headerSeparator)
if (!detailedReport.regressions.isEmpty()) {
append("Regressions: Maximum = ${formatValue(detailedReport.maximumRegression, true)}," +
" Geometric mean = ${formatValue(detailedReport.regressionsGeometricMean, true)}")
}
if (!detailedReport.improvements.isEmpty()) {
append("Improvements: Maximum = ${formatValue(detailedReport.maximumImprovement, true)}," +
" Geometric mean = ${formatValue(detailedReport.improvementsGeometricMean, true)}")
}
append()
}
}
}
}
@@ -176,25 +185,58 @@ class TextRender: Render() {
append(headerSeparator)
if (onlyChanges) {
if (report.regressions.isEmpty() && report.improvements.isEmpty()) {
if (report.detailedMetricReports.values.all { it.improvements.isEmpty() } &&
report.detailedMetricReports.values.all { it.regressions.isEmpty() }) {
append("All becnhmarks are stable.")
}
}
val tableWidth = printPerformanceTableHeader()
// Print geometric mean.
val geoMeanChangeMap = report.geoMeanScoreChange?.
let { mapOf(report.geoMeanBenchmark.first!!.name to report.geoMeanScoreChange!!) }
printBenchmarksDetails(
mutableMapOf(report.geoMeanBenchmark.first!!.name to report.geoMeanBenchmark),
geoMeanChangeMap)
printTableLineSeparator(tableWidth)
printBenchmarksDetails(report.mergedReport, report.regressions)
printBenchmarksDetails(report.mergedReport, report.improvements)
report.detailedMetricReports.forEach { (metric, detailedReport) ->
append()
append(metric.value)
append(headerSeparator)
val tableWidth = printPerformanceTableHeader()
// Print geometric mean.
val geoMeanChangeMap = detailedReport.geoMeanScoreChange?.let {
mapOf(detailedReport.geoMeanBenchmark.first!!.name to detailedReport.geoMeanScoreChange!!)
}
printBenchmarksDetails(
mutableMapOf(detailedReport.geoMeanBenchmark.first!!.name to detailedReport.geoMeanBenchmark),
geoMeanChangeMap)
printTableLineSeparator(tableWidth)
val unstableBenchmarks = report.getUnstableBenchmarksForMetric(metric)
if (unstableBenchmarks.isNotEmpty()) {
append("Stable")
printTableLineSeparator(tableWidth)
}
renderFilteredPerformanceDetails(detailedReport, onlyChanges, unstableBenchmarks, false)
if (unstableBenchmarks.isNotEmpty()) {
printTableLineSeparator(tableWidth)
append("Unstable")
printTableLineSeparator(tableWidth)
}
renderFilteredPerformanceDetails(detailedReport, onlyChanges, unstableBenchmarks,true)
}
}
fun renderFilteredPerformanceDetails(detailedReport: DetailedBenchmarksReport,
onlyChanges: Boolean, unstableBenchmarks: List<String>,
filterUnstable: Boolean) {
fun <T> filterBenchmarks(bucket: Map<String, T>) =
bucket.filter { (name, _) ->
if (filterUnstable) name in unstableBenchmarks else name !in unstableBenchmarks
}
val filteredRegressions = filterBenchmarks(detailedReport.regressions)
val filteredImprovements = filterBenchmarks(detailedReport.improvements)
printBenchmarksDetails(detailedReport.mergedReport, filteredRegressions)
printBenchmarksDetails(detailedReport.mergedReport, filteredImprovements)
if (!onlyChanges) {
// Print all remaining results.
printBenchmarksDetails(report.mergedReport.filter { it.key !in report.regressions.keys &&
it.key !in report.improvements.keys })
printBenchmarksDetails(filterBenchmarks(detailedReport.mergedReport).filter {
it.key !in detailedReport.regressions.keys &&
it.key !in detailedReport.improvements.keys
})
}
}
}
kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/StatisticsRender.kt
+13 -7
View File
@@ -20,7 +20,7 @@ class StatisticsRender: Render() {
private var content = StringBuilder()
override fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean): String {
val benchmarksWithChangedStatus = report.getBenchmarksWithChangedStatus()
val benchmarksWithChangedStatus = report.benchmarksWithChangedStatus
val newPasses = benchmarksWithChangedStatus
.filter { it.current == BenchmarkResult.Status.PASSED }
val newFailures = benchmarksWithChangedStatus
@@ -28,6 +28,12 @@ class StatisticsRender: Render() {
if (report.failedBenchmarks.isNotEmpty()) {
content.append("failed: ${report.failedBenchmarks.size}\n")
}
val regressionsSize = report.detailedMetricReports.values.fold(0) { acc, it ->
acc + it.regressions.size
}
val improvementsSize = report.detailedMetricReports.values.fold(0) { acc, it ->
acc + it.improvements.size
}
val status = when {
newFailures.isNotEmpty() -> {
content.append("new failures: ${newFailures.size}\n")
@@ -37,16 +43,16 @@ class StatisticsRender: Render() {
content.append("new passes: ${newPasses.size}\n")
Status.FIXED
}
report.improvements.isNotEmpty() && report.regressions.isNotEmpty() -> {
content.append("regressions: ${report.regressions.size}\nimprovements: ${report.improvements.size}")
regressionsSize != 0 && improvementsSize != 0 -> {
content.append("regressions: $regressionsSize\nimprovements: $improvementsSize")
Status.UNSTABLE
}
report.improvements.isNotEmpty() && report.regressions.isEmpty() -> {
content.append("improvements: ${report.improvements.size}")
improvementsSize != 0 && regressionsSize == 0 -> {
content.append("improvements: $improvementsSize")
Status.IMPROVED
}
report.improvements.isEmpty() && report.regressions.isNotEmpty() -> {
content.append("regressions: ${report.regressions.size}")
improvementsSize == 0 && regressionsSize != 0 -> {
content.append("regressions: $regressionsSize")
Status.REGRESSED
}
else -> Status.STABLE
kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/TeamCityStatisticsRender.kt
+6 -4
View File
@@ -29,7 +29,9 @@ class TeamCityStatisticsRender: Render() {
content.append("##teamcity[testSuiteFinished name='Benchmarks']\n")
// Report geometric mean as build statistic value
renderGeometricMean(report.geoMeanBenchmark.first!!)
report.detailedMetricReports.forEach { (metric, detailedReport) ->
renderGeometricMean(metric.value, detailedReport.geoMeanBenchmark.first!!)
}
return content.toString()
}
@@ -51,8 +53,8 @@ class TeamCityStatisticsRender: Render() {
content.append("##teamcity[testFinished name='${benchmark.name}' duration='${(duration / 1000).toInt()}']\n")
}
private fun renderGeometricMean(geoMeanBenchmark: MeanVarianceBenchmark) {
content.append("##teamcity[buildStatisticValue key='Geometric mean' value='${geoMeanBenchmark.score}']\n")
content.append("##teamcity[buildStatisticValue key='Geometric mean variance' value='${geoMeanBenchmark.variance}']\n")
private fun renderGeometricMean(metricName: String, geoMeanBenchmark: MeanVarianceBenchmark) {
content.append("##teamcity[buildStatisticValue key='$metricName Geometric mean' value='${geoMeanBenchmark.score}']\n")
content.append("##teamcity[buildStatisticValue key='$metricName Geometric mean variance' value='${geoMeanBenchmark.variance}']\n")
}
}