azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rebuild IDE performance tests output

Browse Source 
format changes: added warmUp tag, index, benchmark format version
perform post processing: geom mean calc, extract warm-up benchmark
This commit is contained in:
Vladimir Dolzhenko
2020-10-30 21:28:35 +01:00
parent f87e46b599
commit 3c0d43dc0d
6 changed files with 128 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build.gradle.kts
+3 -2
View File
@@ -781,9 +781,10 @@ tasks {
}
register("idea-plugin-performance-tests") {
dependsOn("dist")
dependsOn(
":idea:performanceTests:performanceTest"
"dist",
":idea:performanceTests:performanceTest",
":idea:performanceTests:aggregateResults"
)
}
idea/performanceTests/build.gradle.kts
+7
View File
@@ -147,5 +147,12 @@ projectTest(taskName = "wholeProjectsPerformanceTest") {
}
}
task("aggregateResults", JavaExec::class) {
main = "org.jetbrains.kotlin.idea.perf.util.AggregateResultsKt"
classpath = sourceSets["test"].runtimeClasspath
workingDir = rootDir
args(listOf(File(rootDir, "build")))
}
testsJar()
idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt
+38 -25
View File
@@ -165,6 +165,7 @@ class Stats(
statInfoArray: Array<StatInfos>,
printOnlyErrors: Boolean = false,
metricChildren: MutableList<Metric>,
warmUp: Boolean = false,
attemptFn: (Int) -> String = { attempt -> "#$attempt" }
) {
for (statInfoIndex in statInfoArray.withIndex()) {
@@ -191,7 +192,15 @@ class Stats(
}
}
}
metricChildren.add(Metric(attemptString, metricValue = durationMs, metrics = childrenMetrics))
metricChildren.add(
Metric(
metricName = attemptString,
index = attempt,
warmUp = if (warmUp) true else null,
metricValue = durationMs,
metrics = childrenMetrics
)
)
}
}
}
@@ -200,7 +209,7 @@ class Stats(
prefix: String,
warmUpStatInfosArray: Array<StatInfos>,
metricChildren: MutableList<Metric>
) = convertStatInfoIntoMetrics(prefix, warmUpStatInfosArray, metricChildren = metricChildren) { attempt -> "warm-up #$attempt" }
) = convertStatInfoIntoMetrics(prefix, warmUpStatInfosArray, warmUp = true, metricChildren = metricChildren) { attempt -> "warm-up #$attempt" }
fun processTimings(
prefix: String,
@@ -224,6 +233,7 @@ class Stats(
phaseData.testName,
printOnlyErrors = true,
statInfoArray = warmUpStatInfosArray,
warmUp = true,
metricChildren = metricChildren
) { attempt -> "warm-up #$attempt" }
}
@@ -376,31 +386,9 @@ class Stats(
metrics = it.metrics ?: emptyList()
)
if (benchmark.name == "ParameterNameAndType - TypeParameter") {
println(benchmark)
}
benchmark.writeJson()
with(
Benchmark(
agentName = agentName,
buildBranch = buildBranch,
buildId = buildId,
benchmark = name,
synthetic = true,
name = "geomMean",
buildTimestamp = simpleDateFormat.format(Date())
)
) {
loadJson()
merge(benchmark)
if (benchmark.name == "ParameterNameAndType - TypeParameter") {
println(this)
}
writeJson()
}
//rebuildGeomMean(agentName, buildBranch, buildId, simpleDateFormat, benchmark)
}
} finally {
metric = null
@@ -408,6 +396,31 @@ class Stats(
//metrics.writeCSV(name, header)
}
private fun rebuildGeomMean(
agentName: String?,
buildBranch: String?,
buildId: Int?,
simpleDateFormat: SimpleDateFormat,
benchmark: Benchmark
) {
with(
Benchmark(
agentName = agentName,
buildBranch = buildBranch,
buildId = buildId,
benchmark = name,
synthetic = true,
name = "geomMean",
buildTimestamp = simpleDateFormat.format(Date())
)
) {
loadJson()
merge(benchmark)
writeJson()
}
}
companion object {
const val TEST_KEY = "test"
const val ERROR_KEY = "error"
idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/Metric.kt
+17 -2
View File
@@ -13,6 +13,7 @@ import java.util.ArrayList
@JsonInclude(JsonInclude.Include.NON_NULL)
data class Benchmark(
val version: Int = 3,
@set:JsonProperty("agentName")
var agentName: String?,
@set:JsonProperty("benchmark")
@@ -21,6 +22,10 @@ data class Benchmark(
var name: String? = null,
@set:JsonProperty("synthetic")
var synthetic: Boolean? = null,
@set:JsonProperty("index")
var index: Int? = null,
@set:JsonProperty("warmUp")
var warmUp: Boolean? = null,
@set:JsonProperty("buildTimestamp")
var buildTimestamp: String,
@set:JsonProperty("buildBranch")
@@ -51,8 +56,14 @@ data class Benchmark(
private fun String?.escapeName() = this?.replace(Regex("[^A-Za-z0-9_]"), "_")
fun fileName(): String =
listOfNotNull(benchmark?.escapeName(), name?.escapeName(), buildId?.toString()).joinToString(separator = "_")
fun id(): String =
listOfNotNull(
benchmark?.escapeName(),
name?.escapeName(),
buildId?.toString(),
warmUp?.let { "warmUp" } ?: null,
index?.toString()
).joinToString(separator = "_")
fun cleanUp() {
metrics?.forEach { it.cleanUp() }
@@ -86,6 +97,10 @@ data class Benchmark(
data class Metric(
@set:JsonProperty("metricName")
var metricName: String,
@set:JsonProperty("index")
var index: Int? = null,
@set:JsonProperty("warmUp")
var warmUp: Boolean? = null,
@set:JsonProperty("legacyName")
var legacyName: String? = null,
@set:JsonProperty("metricValue")
idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/StatsOutput.kt
+3 -1
View File
@@ -62,7 +62,7 @@ internal val kotlinJsonMapper = jacksonObjectMapper()
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
.configure(SerializationFeature.INDENT_OUTPUT, true)
internal fun Benchmark.statsFile() = statsFile(fileName(), "json")
internal fun Benchmark.statsFile() = statsFile(id(), "json")
internal fun Benchmark.writeJson() {
val json = kotlinJsonMapper.writeValueAsString(this)
@@ -74,6 +74,8 @@ internal fun Benchmark.writeJson() {
}
}
internal fun File.loadBenchmark(): Benchmark = kotlinJsonMapper.readValue(this, object : TypeReference<Benchmark>() {})
internal fun Benchmark.loadJson() {
val statsFile = statsFile()
if (statsFile.exists()) {
idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/aggregateResults.kt
+60
View File
@@ -0,0 +1,60 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.idea.perf.util
import java.io.File
import java.util.*
fun main(args: Array<String>) {
val argFile = File(args[0])
val groupBy = argFile.listFiles()
.filter { it.length() > 0 && it.name.startsWith("stats-") && it.extension == "json" }
.groupBy { it.name.replace("stats-", "").split("_")[0] }
groupBy.forEach { (k, v) ->
if (v.isEmpty()) return@forEach
val benchmarks = v.map { it.loadBenchmark() }
benchmarks.forEach { benchmark ->
benchmark.metrics.firstOrNull { it.metricName == "" }?.let { metric ->
metric.rawMetrics?.firstOrNull { it.warmUp == true && it.index == 0 }?.let {
val warmUpBenchmark = Benchmark(
agentName = benchmark.agentName,
buildBranch = benchmark.buildBranch,
buildId = benchmark.buildId,
benchmark = benchmark.benchmark,
name = benchmark.name,
warmUp = it.warmUp,
index = it.index,
hasError = it.hasError,
buildTimestamp = benchmark.buildTimestamp,
metrics = it.metrics ?: emptyList()
)
warmUpBenchmark.writeJson()
}
}
}
// build geom mean benchmark
val first = v.first()
val loadBenchmark = first.loadBenchmark()
val geomMeanBenchmark = Benchmark(
agentName = loadBenchmark.agentName,
buildBranch = loadBenchmark.buildBranch,
buildId = loadBenchmark.buildId,
benchmark = loadBenchmark.benchmark,
synthetic = true,
name = "geomMean",
buildTimestamp = loadBenchmark.buildTimestamp
)
benchmarks
.filter { it.synthetic != true && it.warmUp != true }
.forEach { geomMeanBenchmark.merge(it) }
geomMeanBenchmark.writeJson()
}
}