azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support for stable and unstable benchmarks. Refactoring to get separate tables for metrics. (#4641)

Browse Source
This commit is contained in:
LepilkinaElena
2021-01-18 17:41:59 +03:00
committed by Nikolay Krasko
parent ece5d9b109
commit a403b9ff05
13 changed files with 646 additions and 371 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kotlin-native/tools/benchmarks/shared/src/main/kotlin/analyzer/Statistics.kt
+9 -4
View File
@@ -25,10 +25,13 @@ val MeanVarianceBenchmark.description: String
// Calculate difference in percentage compare to another.
fun MeanVarianceBenchmark.calcPercentageDiff(other: MeanVarianceBenchmark): MeanVariance {
if (score == 0.0 && variance == 0.0 && other.score == 0.0 && other.variance == 0.0)
return MeanVariance(score, variance)
assert(other.score >= 0 &&
other.variance >= 0 &&
other.score - other.variance != 0.0,
(other.score - other.variance != 0.0 || other.score == 0.0),
{ "Mean and variance should be positive and not equal!" })
// Analyze intervals. Calculate difference between border points.
val (bigValue, smallValue) = if (score > other.score) Pair(this, other) else Pair(other, this)
val bigValueIntervalStart = bigValue.score - bigValue.variance
@@ -50,11 +53,13 @@ fun MeanVarianceBenchmark.calcPercentageDiff(other: MeanVarianceBenchmark): Mean
// Calculate ratio value compare to another.
fun MeanVarianceBenchmark.calcRatio(other: MeanVarianceBenchmark): MeanVariance {
if (other.score == 0.0 && other.variance == 0.0)
return MeanVariance(1.0, 0.0)
assert(other.score >= 0 &&
other.variance >= 0 &&
other.score - other.variance != 0.0,
(other.score - other.variance != 0.0 || other.score == 0.0),
{ "Mean and variance should be positive and not equal!" })
val mean = score / other.score
val mean = if (other.score != 0.0) (score / other.score) else 0.0
val minRatio = (score - variance) / (other.score + other.variance)
val maxRatio = (score + variance) / (other.score - other.variance)
val ratioConfInt = min(abs(minRatio - mean), abs(maxRatio - mean))
@@ -63,7 +68,7 @@ fun MeanVarianceBenchmark.calcRatio(other: MeanVarianceBenchmark): MeanVariance
fun geometricMean(values: Collection<Double>, totalNumber: Int = values.size) =
with(values.asSequence().filter { it != 0.0 }) {
if (count() == 0) {
if (count() == 0 || totalNumber == 0) {
0.0
} else {
map { it.pow(1.0 / totalNumber) }.reduce { a, b -> a * b }
kotlin-native/tools/benchmarks/shared/src/main/kotlin/analyzer/SummaryBenchmarksReport.kt
+168 -135
View File
@@ -18,13 +18,11 @@ typealias BenchmarksTable = Map<String, MeanVarianceBenchmark>
typealias SummaryBenchmarksTable = Map<String, SummaryBenchmark>
typealias ScoreChange = Pair<MeanVariance, MeanVariance>
// Summary report with comparasion of separate benchmarks results.
class SummaryBenchmarksReport(val currentReport: BenchmarksReport,
val previousReport: BenchmarksReport? = null,
val meaningfulChangesValue: Double = 0.5) {
class DetailedBenchmarksReport(currentBenchmarks: Map<String, List<BenchmarkResult>>,
previousBenchmarks: Map<String, List<BenchmarkResult>>? = null,
val meaningfulChangesValue: Double = 0.5) {
// Report created by joining comparing reports.
val mergedReport: Map<String, SummaryBenchmark>
private val benchmarksDurations: Map<String, Pair<Double?, Double?>>
// Lists of benchmarks in different status.
private val benchmarksWithChangedStatus = mutableListOf<FieldChange<BenchmarkResult.Status>>()
@@ -40,30 +38,6 @@ class SummaryBenchmarksReport(val currentReport: BenchmarksReport,
var geoMeanScoreChange: ScoreChange? = null
private set
// Environment and tools.
val environments: Pair<Environment, Environment?>
val compilers: Pair<Compiler, Compiler?>
// Countable properties.
val failedBenchmarks: List<String>
get() = mergedReport.filter { it.value.first?.status == BenchmarkResult.Status.FAILED }
.map { it.key }
val addedBenchmarks: List<String>
get() = mergedReport.filter { it.value.second == null }.map { it.key }
val removedBenchmarks: List<String>
get() = mergedReport.filter { it.value.first == null }.map { it.key }
val benchmarksNumber: Int
get() = mergedReport.keys.size
val currentMeanVarianceBenchmarks: List<MeanVarianceBenchmark>
get() = mergedReport.filter { it.value.first != null }.map { it.value.first!! }
val currentBenchmarksDuration: Map<String, Double>
get() = benchmarksDurations.filter { it.value.first != null }.map { it.key to it.value.first!! }.toMap()
val maximumRegression: Double
get() = getMaximumChange(regressions)
@@ -76,90 +50,53 @@ class SummaryBenchmarksReport(val currentReport: BenchmarksReport,
val improvementsGeometricMean: Double
get() = getGeometricMeanOfChanges(improvements)
val envChanges: List<FieldChange<String>>
get() {
val previousEnvironment = environments.second
val currentEnvironment = environments.first
return previousEnvironment?.let {
mutableListOf<FieldChange<String>>().apply {
addFieldChange("Machine CPU", previousEnvironment.machine.cpu, currentEnvironment.machine.cpu)
addFieldChange("Machine OS", previousEnvironment.machine.os, currentEnvironment.machine.os)
addFieldChange("JDK version", previousEnvironment.jdk.version, currentEnvironment.jdk.version)
addFieldChange("JDK vendor", previousEnvironment.jdk.vendor, currentEnvironment.jdk.vendor)
}
} ?: listOf<FieldChange<String>>()
}
val kotlinChanges: List<FieldChange<String>>
get() {
val previousCompiler = compilers.second
val currentCompiler = compilers.first
return previousCompiler?.let {
mutableListOf<FieldChange<String>>().apply {
addFieldChange("Backend type", previousCompiler.backend.type.type, currentCompiler.backend.type.type)
addFieldChange("Backend version", previousCompiler.backend.version, currentCompiler.backend.version)
addFieldChange("Backend flags", previousCompiler.backend.flags.toString(),
currentCompiler.backend.flags.toString())
addFieldChange("Kotlin version", previousCompiler.kotlinVersion, currentCompiler.kotlinVersion)
}
} ?: listOf<FieldChange<String>>()
}
val benchmarksNumber: Int
get() = mergedReport.keys.size
init {
// Count avarage values for each benchmark.
val currentBenchmarksTable = collectMeanResults(currentReport.benchmarks)
val previousBenchmarksTable = previousReport?.let {
collectMeanResults(previousReport.benchmarks)
val currentBenchmarksTable = collectMeanResults(currentBenchmarks)
val previousBenchmarksTable = previousBenchmarks?.let {
collectMeanResults(previousBenchmarks)
}
mergedReport = createMergedReport(currentBenchmarksTable, previousBenchmarksTable)
benchmarksDurations = calculateBenchmarksDuration(currentReport, previousReport)
geoMeanBenchmark = calculateGeoMeanBenchmark(currentBenchmarksTable, previousBenchmarksTable)
environments = Pair(currentReport.env, previousReport?.env)
compilers = Pair(currentReport.compiler, previousReport?.compiler)
if (previousReport != null) {
if (previousBenchmarks != null) {
// Check changes in environment and tools.
analyzePerformanceChanges()
}
}
// Get benchmark report.
fun getBenchmarksReport(takeMainReport: Boolean = true) =
if (takeMainReport)
BenchmarksReport(environments.first, mergedReport.map { (_, value) -> value.first!! }, compilers.first)
else
BenchmarksReport(environments.second!!, mergedReport.map { (_, value) -> value.second!! }, compilers.second!!)
fun getResultsByMetric(metric: BenchmarkResult.Metric, getGeoMean: Boolean = true, filter: List<String>? = null,
normalizeData: Map<String, Map<String, Double>>? = null): List<Double?> {
val benchmarks = filter?.let {
mergedReport.filter { entry ->
filter.find {
entry.key.startsWith(it)
} != null
}
} ?: mergedReport
val results = benchmarks.map { entry ->
val name = entry.key.removeSuffix(metric.suffix)
if (entry.value.first!!.metric == metric) {
val score = entry.value.first!!.score
val value = normalizeData?.let {
it.get(name)?.get("$metric")?.let { score / it }
?: error("No normalization data for benchmark $name and metric $metric")
} ?: score
name to value
} else name to null
}.toMap()
if (getGeoMean) {
return listOf(geometricMean(results.values.filterNotNull()))
}
return filter?.let { it.map { results[it] }.toList() } ?: results.values.toList()
}
private fun getMaximumChange(bucket: Map<String, ScoreChange>): Double =
// Maps of regressions and improvements are sorted.
if (bucket.isEmpty()) 0.0 else bucket.values.map { it.first.mean }.first()
// Analyze and collect changes in performance between same becnhmarks.
private fun analyzePerformanceChanges() {
val performanceChanges = mergedReport.asSequence().map { (name, element) ->
getBenchmarkPerfomanceChange(name, element)
}.filterNotNull().groupBy {
if (it.second.first.mean > 0) "regressions" else "improvements"
}
// Sort regressions and improvements.
regressions = performanceChanges["regressions"]
?.sortedByDescending { it.second.first.mean }?.map { it.first to it.second }
?.toMap() ?: mapOf<String, ScoreChange>()
improvements = performanceChanges["improvements"]
?.sortedBy { it.second.first.mean }?.map { it.first to it.second }
?.toMap() ?: mapOf<String, ScoreChange>()
// Calculate change for geometric mean.
val (current, previous) = geoMeanBenchmark
geoMeanScoreChange = current?.let {
previous?.let {
Pair(current.calcPercentageDiff(previous), current.calcRatio(previous))
}
}
}
private fun getGeometricMeanOfChanges(bucket: Map<String, ScoreChange>): Double {
if (bucket.isEmpty())
return 0.0
@@ -175,25 +112,6 @@ class SummaryBenchmarksReport(val currentReport: BenchmarksReport,
fun getBenchmarksWithChangedStatus(): List<FieldChange<BenchmarkResult.Status>> = benchmarksWithChangedStatus
// Create geometric mean.
private fun createGeoMeanBenchmark(benchTable: BenchmarksTable): MeanVarianceBenchmark {
val geoMeanBenchmarkName = "Geometric mean"
val geoMean = geometricMean(benchTable.toList().map { (_, value) -> value.score })
val varianceGeoMean = geometricMean(benchTable.toList().map { (_, value) -> value.variance })
return MeanVarianceBenchmark(geoMeanBenchmarkName, geoMean, varianceGeoMean)
}
// Generate map with summary durations of each benchmark.
private fun calculateBenchmarksDuration(currentReport: BenchmarksReport, previousReport: BenchmarksReport?):
Map<String, Pair<Double?, Double?>> {
val currentDurations = collectBenchmarksDurations(currentReport.benchmarks)
val previousDurations = previousReport?.let {
collectBenchmarksDurations(previousReport.benchmarks)
} ?: mapOf<String, Double>()
return currentDurations.keys.union(previousDurations.keys)
.map { it to Pair(currentDurations[it], previousDurations[it]) }.toMap()
}
// Merge current and compare to report.
private fun createMergedReport(currentBenchmarks: BenchmarksTable, previousBenchmarks: BenchmarksTable?):
Map<String, SummaryBenchmark> {
@@ -249,29 +167,144 @@ class SummaryBenchmarksReport(val currentReport: BenchmarksReport,
return null
}
// Analyze and collect changes in performance between same becnhmarks.
private fun analyzePerformanceChanges() {
val performanceChanges = mergedReport.asSequence().map { (name, element) ->
getBenchmarkPerfomanceChange(name, element)
}.filterNotNull().groupBy {
if (it.second.first.mean > 0) "regressions" else "improvements"
// Create geometric mean.
private fun createGeoMeanBenchmark(benchTable: BenchmarksTable): MeanVarianceBenchmark {
val geoMeanBenchmarkName = "Geometric mean"
val geoMean = geometricMean(benchTable.toList().map { (_, value) -> value.score })
val varianceGeoMean = geometricMean(benchTable.toList().map { (_, value) -> value.variance })
return MeanVarianceBenchmark(geoMeanBenchmarkName, geoMean, varianceGeoMean)
}
}
// Summary report with comparasion of separate benchmarks results.
class SummaryBenchmarksReport(val currentReport: BenchmarksReport,
val previousReport: BenchmarksReport? = null,
val meaningfulChangesValue: Double = 0.5,
private val unstableBenchmarks: List<String> = emptyList()) {
val detailedMetricReports: Map<BenchmarkResult.Metric, DetailedBenchmarksReport>
private val benchmarksDurations: Map<String, Pair<Double?, Double?>>
// Lists of benchmarks in different status.
val benchmarksWithChangedStatus
get() = getReducedResult { report ->
report.getBenchmarksWithChangedStatus()
}
// Sort regressions and improvements.
regressions = performanceChanges["regressions"]
?.sortedByDescending { it.second.first.mean }?.map { it.first to it.second }
?.toMap() ?: mapOf<String, ScoreChange>()
improvements = performanceChanges["improvements"]
?.sortedBy { it.second.first.mean }?.map { it.first to it.second }
?.toMap() ?: mapOf<String, ScoreChange>()
// Environment and tools.
val environments: Pair<Environment, Environment?>
val compilers: Pair<Compiler, Compiler?>
// Calculate change for geometric mean.
val (current, previous) = geoMeanBenchmark
geoMeanScoreChange = current?.let {
previous?.let {
Pair(current.calcPercentageDiff(previous), current.calcRatio(previous))
}
private fun <T> getReducedResult(convertor: (DetailedBenchmarksReport) -> List<T>): List<T> {
return detailedMetricReports.values.map {
convertor(it)
}.flatten()
}
// Countable properties.
val failedBenchmarks: List<String>
get() = getReducedResult { report ->
report.mergedReport.filter { it.value.first?.status == BenchmarkResult.Status.FAILED }.map { it.key }
}
val addedBenchmarks: List<String>
get() = getReducedResult { report ->
report.mergedReport.filter { it.value.second == null }.map { it.key }
}
val removedBenchmarks: List<String>
get() = getReducedResult { report ->
report.mergedReport.filter { it.value.first == null }.map { it.key }
}
val currentMeanVarianceBenchmarks: List<MeanVarianceBenchmark>
get() = getReducedResult { report ->
report.mergedReport.filter { it.value.first != null }.map { it.value.first!! }
}
val benchmarksNumber: Int
get() = detailedMetricReports.values.fold(0) { acc, it -> acc + it.benchmarksNumber }
val currentBenchmarksDuration: Map<String, Double>
get() = benchmarksDurations.filter { it.value.first != null }.map { it.key to it.value.first!! }.toMap()
val envChanges: List<FieldChange<String>>
get() {
val previousEnvironment = environments.second
val currentEnvironment = environments.first
return previousEnvironment?.let {
mutableListOf<FieldChange<String>>().apply {
addFieldChange("Machine CPU", previousEnvironment.machine.cpu, currentEnvironment.machine.cpu)
addFieldChange("Machine OS", previousEnvironment.machine.os, currentEnvironment.machine.os)
addFieldChange("JDK version", previousEnvironment.jdk.version, currentEnvironment.jdk.version)
addFieldChange("JDK vendor", previousEnvironment.jdk.vendor, currentEnvironment.jdk.vendor)
}
} ?: listOf<FieldChange<String>>()
}
val kotlinChanges: List<FieldChange<String>>
get() {
val previousCompiler = compilers.second
val currentCompiler = compilers.first
return previousCompiler?.let {
mutableListOf<FieldChange<String>>().apply {
addFieldChange("Backend type", previousCompiler.backend.type.type, currentCompiler.backend.type.type)
addFieldChange("Backend version", previousCompiler.backend.version, currentCompiler.backend.version)
addFieldChange("Backend flags", previousCompiler.backend.flags.toString(),
currentCompiler.backend.flags.toString())
addFieldChange("Kotlin version", previousCompiler.kotlinVersion, currentCompiler.kotlinVersion)
}
} ?: listOf<FieldChange<String>>()
}
init {
// Count avarage values for each benchmark.
detailedMetricReports = BenchmarkResult.Metric.values().map { metric ->
val currentBenchmarks = currentReport.benchmarks.map { (name, benchmarks) ->
name to benchmarks.filter { it.metric == metric }
}.filter { it.second.isNotEmpty() }.toMap()
val previousBenchmarks = previousReport?.benchmarks?.map { (name, benchmarks) ->
name to benchmarks.filter { it.metric == metric }
}?.filter { it.second.isNotEmpty() }?.toMap()
metric to DetailedBenchmarksReport(
currentReport.benchmarks.map { (name, benchmarks) ->
name to benchmarks.filter { it.metric == metric }
}.filter { it.second.isNotEmpty() }.toMap(),
previousReport?.benchmarks?.map { (name, benchmarks) ->
name to benchmarks.filter { it.metric == metric }
}?.filter { it.second.isNotEmpty() }?.toMap(),
meaningfulChangesValue
)
}.toMap()
benchmarksDurations = calculateBenchmarksDuration(currentReport, previousReport)
environments = Pair(currentReport.env, previousReport?.env)
compilers = Pair(currentReport.compiler, previousReport?.compiler)
}
// Get benchmark report.
fun getBenchmarksReport(takeMainReport: Boolean = true) =
if (takeMainReport)
BenchmarksReport(environments.first, getReducedResult { report ->
report.mergedReport.map { (_, value) -> value.first!! }
}, compilers.first)
else
BenchmarksReport(environments.second!!, getReducedResult { report ->
report.mergedReport.map { (_, value) -> value.second!! }
}, compilers.second!!)
fun getUnstableBenchmarksForMetric(metric: BenchmarkResult.Metric) =
if (metric == BenchmarkResult.Metric.EXECUTION_TIME) unstableBenchmarks else emptyList()
// Generate map with summary durations of each benchmark.
private fun calculateBenchmarksDuration(currentReport: BenchmarksReport, previousReport: BenchmarksReport?):
Map<String, Pair<Double?, Double?>> {
val currentDurations = collectBenchmarksDurations(currentReport.benchmarks)
val previousDurations = previousReport?.let {
collectBenchmarksDurations(previousReport.benchmarks)
} ?: mapOf<String, Double>()
return currentDurations.keys.union(previousDurations.keys)
.map { it to Pair(currentDurations[it], previousDurations[it]) }.toMap()
}
private fun <T> MutableList<FieldChange<T>>.addFieldChange(field: String, previous: T, current: T) {
kotlin-native/tools/benchmarks/shared/src/main/kotlin/report/BenchmarksReport.kt
+31 -1
View File
@@ -270,7 +270,6 @@ open class BenchmarkResult(val name: String, val status: Status,
val metric = if (metricElement != null && metricElement is JsonLiteral)
metricFromString(metricElement.unquoted()) ?: Metric.EXECUTION_TIME
else Metric.EXECUTION_TIME
name += metric.suffix
val statusElement = data.getRequiredField("status")
if (statusElement is JsonLiteral) {
val status = statusFromString(statusElement.unquoted())
@@ -348,4 +347,35 @@ open class MeanVarianceBenchmark(name: String, status: BenchmarkResult.Status, s
"variance": $variance
"""
}
}
// Benchmark with set results stability state.
open class BenchmarkWithStabilityState(name: String, status: BenchmarkResult.Status, score: Double, metric: BenchmarkResult.Metric,
runtimeInUs: Double, repeat: Int, warmup: Int, val unstable: Boolean) :
BenchmarkResult(name, status, score, metric, runtimeInUs, repeat, warmup) {
constructor(benchmarkResult: BenchmarkResult, unstable: Boolean) : this(benchmarkResult.name,
benchmarkResult.status, benchmarkResult.score, benchmarkResult.metric,
benchmarkResult.runtimeInUs, benchmarkResult.repeat, benchmarkResult.warmup, unstable)
override fun serializeFields(): String {
return """
${super.serializeFields()},
"unstable": $unstable
"""
}
companion object : EntityFromJsonFactory<BenchmarkResult> {
override fun create(data: JsonElement): BenchmarkWithStabilityState {
val parsedObject = BenchmarkResult.create(data)
if (data is JsonObject) {
val unstableElement = data.getOptionalField("unstable")
val unstableFlag = if (unstableElement != null && unstableElement is JsonPrimitive)
unstableElement.boolean else false
return BenchmarkWithStabilityState(parsedObject, unstableFlag)
} else {
error("Benchmark entity is expected to be an object. Please, check origin files.")
}
}
}
}
kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/main.kt
+22 -2
View File
@@ -79,6 +79,21 @@ object DBServerConnector : Connector() {
val accessFileUrl = "$serverUrl/report/$target/$buildNumber"
return sendGetRequest(accessFileUrl)
}
fun getUnstableBenchmarks(): List<String>? {
try {
val unstableList = sendGetRequest("$serverUrl/unstable")
val data = JsonTreeParser.parse(unstableList)
if (data !is JsonArray) {
return null
}
return data.jsonArray.map {
(it as JsonPrimitive).content
}
} catch (e: Exception) {
return null
}
}
}
fun getFileContent(fileName: String, user: String? = null): String {
@@ -155,6 +170,11 @@ fun main(args: Array<String>) {
val user by argParser.option(ArgType.String, shortName = "u", description = "User access information for authorization")
argParser.parse(args)
// Get unstable benchmarks.
val unstableBenchmarks = DBServerConnector.getUnstableBenchmarks()
unstableBenchmarks ?: println("Failed to get access to server and get unstable benchmarks list!")
// Read contents of file.
val mainBenchsReport = mergeReportsWithDetailedFlags(getBenchmarkReport(mainReport, user))
@@ -164,8 +184,8 @@ fun main(args: Array<String>) {
// Generate comparasion report.
val summaryReport = SummaryBenchmarksReport(mainBenchsReport,
compareToBenchsReport,
epsValue)
compareToBenchsReport, epsValue,
unstableBenchmarks ?: emptyList())
var outputFile = output
renders.forEach {
kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/HTMLRender.kt
+138 -70
View File
@@ -405,7 +405,7 @@ class HTMLRender: Render() {
}
}
val benchmarksWithChangedStatus = report.getBenchmarksWithChangedStatus()
val benchmarksWithChangedStatus = report.benchmarksWithChangedStatus
val newFailures = benchmarksWithChangedStatus
.filter { it.current == BenchmarkResult.Status.FAILED }
val newPasses = benchmarksWithChangedStatus
@@ -479,59 +479,86 @@ class HTMLRender: Render() {
}
private fun BodyTag.renderPerformanceSummary(report: SummaryBenchmarksReport) {
if (!report.improvements.isEmpty() || !report.regressions.isEmpty()) {
if (report.detailedMetricReports.values.any { it.improvements.isNotEmpty() } ||
report.detailedMetricReports.values.any { it.regressions.isNotEmpty() }) {
h4 { +"Performance Summary" }
table {
attributes["class"] = "table table-sm table-striped table-hover"
attributes["style"] = "width:initial;"
thead {
tr {
th { +"Change" }
th { +"#" }
th { +"Maximum" }
th { +"Geometric mean" }
}
}
val maximumRegression = report.maximumRegression
val maximumImprovement = report.maximumImprovement
val regressionsGeometricMean = report.regressionsGeometricMean
val improvementsGeometricMean = report.improvementsGeometricMean
tbody {
if (!report.regressions.isEmpty()) {
tr {
th { +"Regressions" }
td { +"${report.regressions.size}" }
td {
attributes["bgcolor"] = ColoredCell(
maximumRegression/maxOf(maximumRegression, abs(maximumImprovement)))
.backgroundStyle
+formatValue(maximumRegression, true)
}
td {
attributes["bgcolor"] = ColoredCell(
regressionsGeometricMean/maxOf(regressionsGeometricMean,
abs(improvementsGeometricMean)))
.backgroundStyle
+formatValue(report.regressionsGeometricMean, true)
}
th(rowspan = Natural(2)) { +"Change" }
report.detailedMetricReports.forEach { (metric, _) ->
th(colspan = Natural(3)) { +metric.value }
}
}
if (!report.improvements.isEmpty()) {
tr {
report.detailedMetricReports.forEach { _ ->
th { +"#" }
th { +"Maximum" }
th { +"Geometric mean" }
}
}
}
tbody {
tr {
th { +"Regressions" }
report.detailedMetricReports.values.forEach { report ->
val maximumRegression = report.maximumRegression
val regressionsGeometricMean = report.regressionsGeometricMean
val maximumImprovement = report.maximumImprovement
val improvementsGeometricMean = report.improvementsGeometricMean
val maximumChange = maxOf(maximumRegression, abs(maximumImprovement))
val maximumChangeGeoMean = maxOf(regressionsGeometricMean,
abs(improvementsGeometricMean))
if (!report.regressions.isEmpty()) {
td { +"${report.regressions.size}" }
td {
attributes["bgcolor"] = ColoredCell(
(maximumRegression/maximumChange).takeIf{ maximumChange > 0.0 }
).backgroundStyle
+formatValue(maximumRegression, true)
}
td {
attributes["bgcolor"] = ColoredCell(
(regressionsGeometricMean/maximumChangeGeoMean).takeIf{ maximumChangeGeoMean > 0.0 }
).backgroundStyle
+formatValue(report.regressionsGeometricMean, true)
}
} else {
repeat(3) { td { +"-" } }
}
}
tr {
th { +"Improvements" }
td { +"${report.improvements.size}" }
td {
attributes["bgcolor"] = ColoredCell(
maximumImprovement/maxOf(maximumRegression, abs(maximumImprovement)))
.backgroundStyle
+formatValue(report.maximumImprovement, true)
}
td {
attributes["bgcolor"] = ColoredCell(
improvementsGeometricMean/maxOf(regressionsGeometricMean,
abs(improvementsGeometricMean)))
.backgroundStyle
+formatValue(report.improvementsGeometricMean, true)
report.detailedMetricReports.values.forEach { report ->
val maximumRegression = report.maximumRegression
val regressionsGeometricMean = report.regressionsGeometricMean
val maximumImprovement = report.maximumImprovement
val improvementsGeometricMean = report.improvementsGeometricMean
val maximumChange = maxOf(maximumRegression, abs(maximumImprovement))
val maximumChangeGeoMean = maxOf(regressionsGeometricMean,
abs(improvementsGeometricMean))
if (!report.improvements.isEmpty()) {
td { +"${report.improvements.size}" }
td {
attributes["bgcolor"] = ColoredCell(
(maximumImprovement / maximumChange).takeIf{ maximumChange > 0.0 }
).backgroundStyle
+formatValue(report.maximumImprovement, true)
}
td {
attributes["bgcolor"] = ColoredCell(
(improvementsGeometricMean / maximumChangeGeoMean)
.takeIf{ maximumChangeGeoMean > 0.0 }
).backgroundStyle
+formatValue(report.improvementsGeometricMean, true)
}
} else {
repeat(3) { td { +"-" } }
}
}
}
}
@@ -584,43 +611,84 @@ class HTMLRender: Render() {
}
}
private fun TableBlock.renderFilteredBenchmarks(detailedReport: DetailedBenchmarksReport,
onlyChanges: Boolean, unstableBenchmarks: List<String>,
filterUnstable: Boolean) {
fun <T> filterBenchmarks(bucket: Map<String, T>) =
bucket.filter { (name, _) ->
if (filterUnstable) name in unstableBenchmarks else name !in unstableBenchmarks
}
val filteredRegressions = filterBenchmarks(detailedReport.regressions)
val filteredImprovements = filterBenchmarks(detailedReport.improvements)
renderBenchmarksDetails(detailedReport.mergedReport, filteredRegressions)
renderBenchmarksDetails(detailedReport.mergedReport, filteredImprovements)
if (!onlyChanges) {
// Print all remaining results.
renderBenchmarksDetails(filterBenchmarks(detailedReport.mergedReport).filter {
it.key !in detailedReport.regressions.keys &&
it.key !in detailedReport.improvements.keys
})
}
}
private fun BodyTag.renderPerformanceDetails(report: SummaryBenchmarksReport, onlyChanges: Boolean) {
if (onlyChanges) {
if (report.regressions.isEmpty() && report.improvements.isEmpty()) {
if (report.detailedMetricReports.values.all { it.improvements.isEmpty() } &&
report.detailedMetricReports.values.all { it.regressions.isEmpty() }) {
div("alert alert-success") {
attributes["role"] = "alert"
+"All becnhmarks are stable!"
}
}
}
report.detailedMetricReports.forEach { (metric, detailedReport) ->
renderCollapsedData(metric.value, false) {
table {
attributes["id"] = "result"
attributes["class"] = "table table-striped table-bordered"
thead {
tr {
th { +"Benchmark" }
th { +"First score" }
th { +"Second score" }
th { +"Percent" }
th { +"Ratio" }
}
}
val geoMeanChangeMap = detailedReport.geoMeanScoreChange?.let {
mapOf(detailedReport.geoMeanBenchmark.first!!.name to detailedReport.geoMeanScoreChange!!)
}
table {
attributes["id"] = "result"
attributes["class"] = "table table-striped table-bordered"
thead {
tr {
th { +"Benchmark" }
th { +"First score" }
th { +"Second score" }
th { +"Percent" }
th { +"Ratio" }
}
}
val geoMeanChangeMap = report.geoMeanScoreChange?.
let { mapOf(report.geoMeanBenchmark.first!!.name to report.geoMeanScoreChange!!) }
tbody {
renderBenchmarksDetails(
mutableMapOf(report.geoMeanBenchmark.first!!.name to report.geoMeanBenchmark),
geoMeanChangeMap, "border-bottom: 2.3pt solid black; border-top: 2.3pt solid black")
renderBenchmarksDetails(report.mergedReport, report.regressions)
renderBenchmarksDetails(report.mergedReport, report.improvements)
if (!onlyChanges) {
// Print all remaining results.
renderBenchmarksDetails(report.mergedReport.filter { it.key !in report.regressions.keys &&
it.key !in report.improvements.keys })
tbody {
val boldRowStyle = "border-bottom: 2.3pt solid black; border-top: 2.3pt solid black"
renderBenchmarksDetails(
mutableMapOf(detailedReport.geoMeanBenchmark.first!!.name to detailedReport.geoMeanBenchmark),
geoMeanChangeMap, boldRowStyle)
val unstableBenchmarks = report.getUnstableBenchmarksForMetric(metric)
if (unstableBenchmarks.isNotEmpty()) {
tr {
attributes["style"] = boldRowStyle
th(colspan = Natural(5)) { +"Stable" }
}
}
renderFilteredBenchmarks(detailedReport, onlyChanges, unstableBenchmarks, false)
if (unstableBenchmarks.isNotEmpty()) {
tr {
attributes["style"] = boldRowStyle
th(colspan = Natural(5)) { +"Unstable" }
}
}
renderFilteredBenchmarks(detailedReport, onlyChanges, unstableBenchmarks, true)
}
}
}
hr {}
}
}
kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/MetricResultsRender.kt
+9 -7
View File
@@ -14,14 +14,16 @@ class MetricResultsRender: Render() {
get() = "metrics"
override fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean): String {
val results = report.mergedReport.map { entry ->
buildString {
val metric = entry.value.first!!.metric
append("{ \"benchmarkName\": \"${entry.key.removeSuffix(metric.suffix)}\",")
append("\"metric\": \"${metric}\",")
append("\"value\": \"${entry.value.first!!.score}\" }")
val results = report.detailedMetricReports.values.map { it.mergedReport }.map { report ->
report.map { entry ->
buildString {
val metric = entry.value.first!!.metric
append("{ \"benchmarkName\": \"${entry.key.removeSuffix(metric.suffix)}\",")
append("\"metric\": \"${metric}\",")
append("\"value\": \"${entry.value.first!!.score}\" }")
}
}
}.joinToString(", ")
}.flatten().joinToString(", ")
return "[ $results ]"
}
kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/Render.kt
+65 -23
View File
@@ -54,7 +54,7 @@ class TextRender: Render() {
renderEnvChanges(report.envChanges, "Environment")
renderEnvChanges(report.kotlinChanges, "Compiler")
renderStatusSummary(report)
renderStatusChangesDetails(report.getBenchmarksWithChangedStatus())
renderStatusChangesDetails(report.benchmarksWithChangedStatus)
renderPerformanceSummary(report)
renderPerformanceDetails(report, onlyChanges)
return content.toString()
@@ -113,18 +113,27 @@ class TextRender: Render() {
}
fun renderPerformanceSummary(report: SummaryBenchmarksReport) {
if (!report.regressions.isEmpty() || !report.improvements.isEmpty()) {
if (report.detailedMetricReports.values.any { it.improvements.isNotEmpty() } ||
report.detailedMetricReports.values.any { it.regressions.isNotEmpty() }) {
append("Performance summary")
append(headerSeparator)
if (!report.regressions.isEmpty()) {
append("Regressions: Maximum = ${formatValue(report.maximumRegression, true)}," +
" Geometric mean = ${formatValue(report.regressionsGeometricMean, true)}")
}
if (!report.improvements.isEmpty()) {
append("Improvements: Maximum = ${formatValue(report.maximumImprovement, true)}," +
" Geometric mean = ${formatValue(report.improvementsGeometricMean, true)}")
}
append()
report.detailedMetricReports.forEach { (metric, detailedReport) ->
if (detailedReport.regressions.isNotEmpty() || detailedReport.improvements.isNotEmpty()) {
append(metric.value)
append(headerSeparator)
if (!detailedReport.regressions.isEmpty()) {
append("Regressions: Maximum = ${formatValue(detailedReport.maximumRegression, true)}," +
" Geometric mean = ${formatValue(detailedReport.regressionsGeometricMean, true)}")
}
if (!detailedReport.improvements.isEmpty()) {
append("Improvements: Maximum = ${formatValue(detailedReport.maximumImprovement, true)}," +
" Geometric mean = ${formatValue(detailedReport.improvementsGeometricMean, true)}")
}
append()
}
}
}
}
@@ -176,25 +185,58 @@ class TextRender: Render() {
append(headerSeparator)
if (onlyChanges) {
if (report.regressions.isEmpty() && report.improvements.isEmpty()) {
if (report.detailedMetricReports.values.all { it.improvements.isEmpty() } &&
report.detailedMetricReports.values.all { it.regressions.isEmpty() }) {
append("All becnhmarks are stable.")
}
}
val tableWidth = printPerformanceTableHeader()
// Print geometric mean.
val geoMeanChangeMap = report.geoMeanScoreChange?.
let { mapOf(report.geoMeanBenchmark.first!!.name to report.geoMeanScoreChange!!) }
printBenchmarksDetails(
mutableMapOf(report.geoMeanBenchmark.first!!.name to report.geoMeanBenchmark),
geoMeanChangeMap)
printTableLineSeparator(tableWidth)
printBenchmarksDetails(report.mergedReport, report.regressions)
printBenchmarksDetails(report.mergedReport, report.improvements)
report.detailedMetricReports.forEach { (metric, detailedReport) ->
append()
append(metric.value)
append(headerSeparator)
val tableWidth = printPerformanceTableHeader()
// Print geometric mean.
val geoMeanChangeMap = detailedReport.geoMeanScoreChange?.let {
mapOf(detailedReport.geoMeanBenchmark.first!!.name to detailedReport.geoMeanScoreChange!!)
}
printBenchmarksDetails(
mutableMapOf(detailedReport.geoMeanBenchmark.first!!.name to detailedReport.geoMeanBenchmark),
geoMeanChangeMap)
printTableLineSeparator(tableWidth)
val unstableBenchmarks = report.getUnstableBenchmarksForMetric(metric)
if (unstableBenchmarks.isNotEmpty()) {
append("Stable")
printTableLineSeparator(tableWidth)
}
renderFilteredPerformanceDetails(detailedReport, onlyChanges, unstableBenchmarks, false)
if (unstableBenchmarks.isNotEmpty()) {
printTableLineSeparator(tableWidth)
append("Unstable")
printTableLineSeparator(tableWidth)
}
renderFilteredPerformanceDetails(detailedReport, onlyChanges, unstableBenchmarks,true)
}
}
fun renderFilteredPerformanceDetails(detailedReport: DetailedBenchmarksReport,
onlyChanges: Boolean, unstableBenchmarks: List<String>,
filterUnstable: Boolean) {
fun <T> filterBenchmarks(bucket: Map<String, T>) =
bucket.filter { (name, _) ->
if (filterUnstable) name in unstableBenchmarks else name !in unstableBenchmarks
}
val filteredRegressions = filterBenchmarks(detailedReport.regressions)
val filteredImprovements = filterBenchmarks(detailedReport.improvements)
printBenchmarksDetails(detailedReport.mergedReport, filteredRegressions)
printBenchmarksDetails(detailedReport.mergedReport, filteredImprovements)
if (!onlyChanges) {
// Print all remaining results.
printBenchmarksDetails(report.mergedReport.filter { it.key !in report.regressions.keys &&
it.key !in report.improvements.keys })
printBenchmarksDetails(filterBenchmarks(detailedReport.mergedReport).filter {
it.key !in detailedReport.regressions.keys &&
it.key !in detailedReport.improvements.keys
})
}
}
}
kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/StatisticsRender.kt
+13 -7
View File
@@ -20,7 +20,7 @@ class StatisticsRender: Render() {
private var content = StringBuilder()
override fun render(report: SummaryBenchmarksReport, onlyChanges: Boolean): String {
val benchmarksWithChangedStatus = report.getBenchmarksWithChangedStatus()
val benchmarksWithChangedStatus = report.benchmarksWithChangedStatus
val newPasses = benchmarksWithChangedStatus
.filter { it.current == BenchmarkResult.Status.PASSED }
val newFailures = benchmarksWithChangedStatus
@@ -28,6 +28,12 @@ class StatisticsRender: Render() {
if (report.failedBenchmarks.isNotEmpty()) {
content.append("failed: ${report.failedBenchmarks.size}\n")
}
val regressionsSize = report.detailedMetricReports.values.fold(0) { acc, it ->
acc + it.regressions.size
}
val improvementsSize = report.detailedMetricReports.values.fold(0) { acc, it ->
acc + it.improvements.size
}
val status = when {
newFailures.isNotEmpty() -> {
content.append("new failures: ${newFailures.size}\n")
@@ -37,16 +43,16 @@ class StatisticsRender: Render() {
content.append("new passes: ${newPasses.size}\n")
Status.FIXED
}
report.improvements.isNotEmpty() && report.regressions.isNotEmpty() -> {
content.append("regressions: ${report.regressions.size}\nimprovements: ${report.improvements.size}")
regressionsSize != 0 && improvementsSize != 0 -> {
content.append("regressions: $regressionsSize\nimprovements: $improvementsSize")
Status.UNSTABLE
}
report.improvements.isNotEmpty() && report.regressions.isEmpty() -> {
content.append("improvements: ${report.improvements.size}")
improvementsSize != 0 && regressionsSize == 0 -> {
content.append("improvements: $improvementsSize")
Status.IMPROVED
}
report.improvements.isEmpty() && report.regressions.isNotEmpty() -> {
content.append("regressions: ${report.regressions.size}")
improvementsSize == 0 && regressionsSize != 0 -> {
content.append("regressions: $regressionsSize")
Status.REGRESSED
}
else -> Status.STABLE
kotlin-native/tools/benchmarksAnalyzer/src/main/kotlin/org/jetbrains/renders/TeamCityStatisticsRender.kt
+6 -4
View File
@@ -29,7 +29,9 @@ class TeamCityStatisticsRender: Render() {
content.append("##teamcity[testSuiteFinished name='Benchmarks']\n")
// Report geometric mean as build statistic value
renderGeometricMean(report.geoMeanBenchmark.first!!)
report.detailedMetricReports.forEach { (metric, detailedReport) ->
renderGeometricMean(metric.value, detailedReport.geoMeanBenchmark.first!!)
}
return content.toString()
}
@@ -51,8 +53,8 @@ class TeamCityStatisticsRender: Render() {
content.append("##teamcity[testFinished name='${benchmark.name}' duration='${(duration / 1000).toInt()}']\n")
}
private fun renderGeometricMean(geoMeanBenchmark: MeanVarianceBenchmark) {
content.append("##teamcity[buildStatisticValue key='Geometric mean' value='${geoMeanBenchmark.score}']\n")
content.append("##teamcity[buildStatisticValue key='Geometric mean variance' value='${geoMeanBenchmark.variance}']\n")
private fun renderGeometricMean(metricName: String, geoMeanBenchmark: MeanVarianceBenchmark) {
content.append("##teamcity[buildStatisticValue key='$metricName Geometric mean' value='${geoMeanBenchmark.score}']\n")
content.append("##teamcity[buildStatisticValue key='$metricName Geometric mean variance' value='${geoMeanBenchmark.variance}']\n")
}
}
kotlin-native/tools/performance-server/src/main/kotlin/database/BenchmarksIndexesDispatcher.kt
+2 -1
View File
@@ -256,6 +256,7 @@ class BenchmarksIndexesDispatcher(connector: ElasticSearchConnector, val feature
fun getGeometricMean(metricName: String, featureValue: String = "",
buildNumbers: Iterable<String>? = null, normalize: Boolean = false,
excludeNames: List<String> = emptyList()): Promise<List<Pair<String, List<Double?>>>> {
// Filter only with metric or also with names.
val filterBenchmarks = if (excludeNames.isEmpty())
"""
@@ -264,7 +265,7 @@ class BenchmarksIndexesDispatcher(connector: ElasticSearchConnector, val feature
else """
"bool": {
"must": { "match": { "benchmarks.metric": "$metricName" } },
"must_not": { "terms" : { "benchmarks.name" : [${excludeNames.map { "\"$it\"" }.joinToString()}] } }
"must_not": [ ${excludeNames.map { """{ "match_phrase" : { "benchmarks.name" : "$it" } }"""}.joinToString() } ]
}
""".trimIndent()
val queryDescription = """
kotlin-native/tools/performance-server/src/main/kotlin/database/DatabaseRequests.kt
+36
View File
@@ -131,6 +131,42 @@ fun getGoldenResults(goldenResultsIndex: GoldenResultsIndex): Promise<Map<String
}
}
// Get list of unstable benchmarks from database.
fun getUnstableResults(goldenResultsIndex: GoldenResultsIndex): Promise<List<String>> {
val queryDescription = """
{
"_source": ["env"],
"query": {
"nested" : {
"path" : "benchmarks",
"query" : {
"match": { "benchmarks.unstable": true }
},
"inner_hits": {
"size": 100,
"_source": ["benchmarks.name"]
}
}
}
}
""".trimIndent()
return goldenResultsIndex.search(queryDescription, listOf("hits.hits.inner_hits")).then { responseString ->
val dbResponse = JsonTreeParser.parse(responseString).jsonObject
val results = dbResponse.getObjectOrNull("hits")?.getArrayOrNull("hits")
?: error("Wrong response:\n$responseString")
results.getObjectOrNull(0)?.let {
it
.getObject("inner_hits")
.getObject("benchmarks")
.getObject("hits")
.getArray("hits").map {
(it as JsonObject).getObject("_source").getPrimitive("name").content
}
} ?: listOf<String>()
}
}
// Get distinct values for needed field from database.
fun distinctValues(field: String, index: ElasticSearchIndex): Promise<List<String>> {
val queryDescription = """
kotlin-native/tools/performance-server/src/main/kotlin/routes/route.kt
+12
View File
@@ -588,6 +588,18 @@ fun router() {
}
})
// Get builds description with additional information.
router.get("/unstable", { request, response ->
CachableResponseDispatcher.getResponse(request, response) { success, reject ->
getUnstableResults(goldenIndex).then { unstableBenchmarks ->
success(unstableBenchmarks)
}.catch {
println("Error during getting unstable benchmarks")
reject()
}
}
})
router.get("/report/:target/:buildNumber", { request, response ->
val target = urlParameterToBaseFormat(request.params.target)
val buildNumber = request.params.buildNumber.toString()
kotlin-native/tools/performance-server/ui/src/main/kotlin/main.kt
+135 -117
View File
@@ -206,8 +206,8 @@ fun main(args: Array<String>) {
}
buildsNumberToShow = parameters["count"]?.toInt() ?: buildsNumberToShow
beforeDate = parameters["before"]?.let{ decodeURIComponent(it)}
afterDate = parameters["after"]?.let{ decodeURIComponent(it)}
beforeDate = parameters["before"]?.let { decodeURIComponent(it) }
afterDate = parameters["after"]?.let { decodeURIComponent(it) }
// Get branches.
val branchesUrl = "$serverUrl/branches"
@@ -285,30 +285,6 @@ fun main(args: Array<String>) {
val platformSpecificBenchs = if (parameters["target"] == "Mac_OS_X") ",FrameworkBenchmarksAnalyzer,SpaceFramework_iosX64" else
if (parameters["target"] == "Linux") ",kotlinx.coroutines" else ""
// Collect information for charts library.
val valuesToShow = mapOf("EXECUTION_TIME" to listOf(mapOf(
"normalize" to "true"
)),
"COMPILE_TIME" to listOf(mapOf(
"samples" to "HelloWorld,Videoplayer$platformSpecificBenchs",
"agr" to "samples"
)),
"CODE_SIZE" to listOf(mapOf(
"normalize" to "true",
"exclude" to if (parameters["target"] == "Linux")
"kotlinx.coroutines"
else if (parameters["target"] == "Mac_OS_X")
"SpaceFramework_iosX64"
else ""
), if (platformSpecificBenchs.isNotEmpty()) mapOf(
"normalize" to "true",
"agr" to "samples",
"samples" to platformSpecificBenchs.removePrefix(",")
) else null).filterNotNull(),
"BUNDLE_SIZE" to listOf(mapOf("samples" to "KotlinNative",
"agr" to "samples"))
)
var execData = listOf<String>() to listOf<List<Double?>>()
var compileData = listOf<String>() to listOf<List<Double?>>()
var codeSizeData = listOf<String>() to listOf<List<Double?>>()
@@ -328,6 +304,17 @@ fun main(args: Array<String>) {
val metricUrl = "$serverUrl/metricValue/${parameters["target"]}/"
val unstableBenchmarksPromise = sendGetRequest("$serverUrl/unstable").then { response ->
val unstableList = response as String
val data = JsonTreeParser.parse(unstableList)
if (data !is JsonArray) {
error("Response is expected to be an array.")
}
data.jsonArray.map {
(it as JsonPrimitive).content
}
}
// Get builds description.
val buildsInfoPromise = sendGetRequest(descriptionUrl).then { response ->
val buildsInfo = response as String
@@ -341,100 +328,131 @@ fun main(args: Array<String>) {
}
}
// Send requests to get all needed metric values.
valuesToShow.map { (metric, listOfSettings) ->
val resultValues = listOfSettings.map { settings ->
val getParameters = with(StringBuilder()) {
if (settings.isNotEmpty()) {
append("?")
}
var prefix = ""
settings.forEach { (key, value) ->
if (value.isNotEmpty()) {
append("$prefix$key=$value")
prefix = "&"
unstableBenchmarksPromise.then { unstableBenchmarks ->
// Collect information for charts library.
val valuesToShow = mapOf("EXECUTION_TIME" to listOf(mapOf(
"normalize" to "true"
),
mapOf(
"normalize" to "true",
"exclude" to unstableBenchmarks.joinToString(",")
)),
"COMPILE_TIME" to listOf(mapOf(
"samples" to "HelloWorld,Videoplayer$platformSpecificBenchs",
"agr" to "samples"
)),
"CODE_SIZE" to listOf(mapOf(
"normalize" to "true",
"exclude" to if (parameters["target"] == "Linux")
"kotlinx.coroutines"
else if (parameters["target"] == "Mac_OS_X")
"SpaceFramework_iosX64"
else ""
), if (platformSpecificBenchs.isNotEmpty()) mapOf(
"normalize" to "true",
"agr" to "samples",
"samples" to platformSpecificBenchs.removePrefix(",")
) else null).filterNotNull(),
"BUNDLE_SIZE" to listOf(mapOf("samples" to "KotlinNative",
"agr" to "samples"))
)
// Send requests to get all needed metric values.
valuesToShow.map { (metric, listOfSettings) ->
val resultValues = listOfSettings.map { settings ->
val getParameters = with(StringBuilder()) {
if (settings.isNotEmpty()) {
append("?")
}
var prefix = ""
settings.forEach { (key, value) ->
if (value.isNotEmpty()) {
append("$prefix$key=$value")
prefix = "&"
}
}
toString()
}
toString()
val branchParameter = if (parameters["branch"] != "all")
(if (getParameters.isEmpty()) "?" else "&") + "branch=${parameters["branch"]}"
else ""
val url = "$metricUrl$metric$getParameters$branchParameter${
if (parameters["type"] != "all")
(if (getParameters.isEmpty() && branchParameter.isEmpty()) "?" else "&") + "type=${parameters["type"]}"
else ""
}&count=$buildsNumberToShow${getDatesComponents()}"
sendGetRequest(url)
}.toTypedArray()
// Get metrics values for charts.
Promise.all(resultValues).then { responses ->
val valuesList = responses.map { response ->
val results = (JsonTreeParser.parse(response) as JsonArray).map {
(it as JsonObject).getPrimitive("first").content to
it.getArray("second").map { (it as JsonPrimitive).doubleOrNull }
}
val labels = results.map { it.first }
val values = results[0]?.second?.size?.let { (0..it - 1).map { i -> results.map { it.second[i] } } }
?: emptyList()
labels to values
}
val labels = valuesList[0].first
val values = valuesList.map { it.second }.reduce { acc, valuesPart -> acc + valuesPart }
when (metric) {
// Update chart with gotten data.
"COMPILE_TIME" -> {
compileData = labels to values.map { it.map { it?.let { it / 1000 } } }
compileChart = Chartist.Line("#compile_chart",
getChartData(labels, compileData.second),
getChartOptions(valuesToShow["COMPILE_TIME"]!![0]!!["samples"]!!.split(',').toTypedArray(),
"Time, milliseconds"))
buildsInfoPromise.then { builds ->
customizeChart(compileChart, "compile_chart", js("$(\"#compile_chart\")"), builds, parameters)
compileChart.update(getChartData(compileData.first, compileData.second))
}
}
"EXECUTION_TIME" -> {
execData = labels to values
execChart = Chartist.Line("#exec_chart",
getChartData(labels, execData.second),
getChartOptions(arrayOf("Geometric Mean (All)", "Geometric mean (Stable)"),
"Normalized time"))
buildsInfoPromise.then { builds ->
customizeChart(execChart, "exec_chart", js("$(\"#exec_chart\")"), builds, parameters)
execChart.update(getChartData(execData.first, execData.second))
}
}
"CODE_SIZE" -> {
codeSizeData = labels to values
codeSizeChart = Chartist.Line("#codesize_chart",
getChartData(labels, codeSizeData.second),
getChartOptions(arrayOf("Geometric Mean") + platformSpecificBenchs.split(',')
.filter { it.isNotEmpty() },
"Normalized size",
arrayOf("ct-series-4", "ct-series-5", "ct-series-6")))
buildsInfoPromise.then { builds ->
customizeChart(codeSizeChart, "codesize_chart", js("$(\"#codesize_chart\")"), builds, parameters)
codeSizeChart.update(getChartData(codeSizeData.first, codeSizeData.second, sizeClassNames))
}
}
"BUNDLE_SIZE" -> {
bundleSizeData = labels to values.map { it.map { it?.let { it.toInt() / 1024 / 1024 } } }
bundleSizeChart = Chartist.Line("#bundlesize_chart",
getChartData(labels,
bundleSizeData.second, sizeClassNames),
getChartOptions(arrayOf("Bundle size"), "Size, MB", arrayOf("ct-series-4")))
buildsInfoPromise.then { builds ->
customizeChart(bundleSizeChart, "bundlesize_chart", js("$(\"#bundlesize_chart\")"), builds, parameters)
bundleSizeChart.update(getChartData(bundleSizeData.first, bundleSizeData.second, sizeClassNames))
}
}
else -> error("No chart for metric $metric")
}
true
}
val branchParameter = if (parameters["branch"] != "all")
(if (getParameters.isEmpty()) "?" else "&") + "branch=${parameters["branch"]}"
else ""
val url = "$metricUrl$metric$getParameters$branchParameter${
if (parameters["type"] != "all")
(if (getParameters.isEmpty() && branchParameter.isEmpty()) "?" else "&") + "type=${parameters["type"]}"
else ""
}&count=$buildsNumberToShow${getDatesComponents()}"
sendGetRequest(url)
}.toTypedArray()
// Get metrics values for charts.
Promise.all(resultValues).then { responses ->
val valuesList = responses.map { response ->
val results = (JsonTreeParser.parse(response) as JsonArray).map {
(it as JsonObject).getPrimitive("first").content to
it.getArray("second").map { (it as JsonPrimitive).doubleOrNull }
}
val labels = results.map { it.first }
val values = results[0]?.second?.size?.let { (0..it - 1).map { i -> results.map { it.second[i] } } }
?: emptyList()
labels to values
}
val labels = valuesList[0].first
val values = valuesList.map { it.second }.reduce { acc, valuesPart -> acc + valuesPart }
when (metric) {
// Update chart with gotten data.
"COMPILE_TIME" -> {
compileData = labels to values.map { it.map { it?.let { it / 1000 } } }
compileChart = Chartist.Line("#compile_chart",
getChartData(labels, compileData.second),
getChartOptions(valuesToShow["COMPILE_TIME"]!![0]!!["samples"]!!.split(',').toTypedArray(),
"Time, milliseconds"))
buildsInfoPromise.then { builds ->
customizeChart(compileChart, "compile_chart", js("$(\"#compile_chart\")"), builds, parameters)
compileChart.update(getChartData(compileData.first, compileData.second))
}
}
"EXECUTION_TIME" -> {
execData = labels to values
execChart = Chartist.Line("#exec_chart",
getChartData(labels, execData.second),
getChartOptions(arrayOf("Geometric Mean"), "Normalized time"))
buildsInfoPromise.then { builds ->
customizeChart(execChart, "exec_chart", js("$(\"#exec_chart\")"), builds, parameters)
execChart.update(getChartData(execData.first, execData.second))
}
}
"CODE_SIZE" -> {
codeSizeData = labels to values
codeSizeChart = Chartist.Line("#codesize_chart",
getChartData(labels, codeSizeData.second),
getChartOptions(arrayOf("Geometric Mean") + platformSpecificBenchs.split(',')
.filter { it.isNotEmpty() },
"Normalized size",
arrayOf("ct-series-4", "ct-series-5", "ct-series-6")))
buildsInfoPromise.then { builds ->
customizeChart(codeSizeChart, "codesize_chart", js("$(\"#codesize_chart\")"), builds, parameters)
codeSizeChart.update(getChartData(codeSizeData.first, codeSizeData.second, sizeClassNames))
}
}
"BUNDLE_SIZE" -> {
bundleSizeData = labels to values.map { it.map { it?.let { it.toInt() / 1024 / 1024 } } }
bundleSizeChart = Chartist.Line("#bundlesize_chart",
getChartData(labels,
bundleSizeData.second, sizeClassNames),
getChartOptions(arrayOf("Bundle size"), "Size, MB", arrayOf("ct-series-4")))
buildsInfoPromise.then { builds ->
customizeChart(bundleSizeChart, "bundlesize_chart", js("$(\"#bundlesize_chart\")"), builds, parameters)
bundleSizeChart.update(getChartData(bundleSizeData.first, bundleSizeData.second, sizeClassNames))
}
}
else -> error("No chart for metric $metric")
}
true
}
}