Split perf test json reports in a separate files/docs

2020-10-15 23:23:35 +02:00
parent d7a783e077
commit 2790bc1105
7 changed files with 317 additions and 194 deletions
@@ -36,6 +36,9 @@ dependencies {
    testImplementation(projectTests(":idea"))
    testImplementation(project(":idea:idea-gradle")) { isTransitive = false }
    testImplementation(commonDep("junit:junit"))
+    testImplementation("com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.11.+")
+    testImplementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.11.+")
+

    Platform[192].orHigher {
        testCompileOnly(intellijPluginDep("java"))
@@ -16,6 +16,7 @@ import org.jetbrains.kotlin.idea.highlighter.KotlinPsiChecker
 import org.jetbrains.kotlin.idea.highlighter.KotlinPsiCheckerAndHighlightingUpdater
 import org.jetbrains.kotlin.idea.perf.Stats.Companion.TEST_KEY
 import org.jetbrains.kotlin.idea.perf.Stats.Companion.runAndMeasure
+import org.jetbrains.kotlin.idea.perf.util.Metric
 import org.jetbrains.kotlin.idea.perf.util.TeamCity.suite
 import org.jetbrains.kotlin.idea.testFramework.Fixture
 import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.cleanupCaches
@@ -7,8 +7,7 @@ package org.jetbrains.kotlin.idea.perf

 import org.jetbrains.kotlin.idea.perf.WholeProjectPerformanceTest.Companion.nsToMs
 import org.jetbrains.kotlin.idea.perf.profilers.*
-import org.jetbrains.kotlin.idea.perf.util.TeamCity
-import org.jetbrains.kotlin.idea.perf.util.logMessage
+import org.jetbrains.kotlin.idea.perf.util.*
 import org.jetbrains.kotlin.util.PerformanceCounter
 import java.io.*
 import java.lang.ref.WeakReference
@@ -25,13 +24,11 @@ typealias StatInfos = Map<String, Any>?
 class Stats(
    val name: String = "",
    private val profilerConfig: ProfilerConfig = ProfilerConfig(),
-    private val header: Array<String> = arrayOf("Name", "ValueMS", "StdDev"),
    private val acceptanceStabilityLevel: Int = 25
 ) : AutoCloseable {

    private val perfTestRawDataMs = mutableListOf<Long>()
-
-    private val metrics = mutableListOf<Metric>()
+    private var metric: Metric? = null

    init {
        PerformanceCounter.setTimeCounterEnabled(true)
@@ -43,25 +40,25 @@ class Stats(
        val calcMean = calcMean(timingsMs)

        val metricChildren = mutableListOf<Metric>()
-        val hasError = rawMetricChildren.any { it.hasError }
-        val metric = Metric(
-            id, value = calcMean.mean.toLong(), measurementError = calcMean.stdDev.toLong(),
-            hasError = hasError, children = metricChildren
+        val hasError = if (rawMetricChildren.any { it.hasError == true }) true else null
+        metric = Metric(
+            id, metricValue = calcMean.mean.toLong(), metricError = calcMean.stdDev.toLong(),
+            hasError = hasError, metrics = metricChildren
        )
-        metrics.add(metric)
+        // TODO:

        metricChildren.add(
            Metric(
-                "", calcMean.mean.toLong(),
+                "", metricValue = calcMean.mean.toLong(),
                hasError = hasError,
-                measurementError = calcMean.stdDev.toLong(),
-                childrenName = "rawMetrics", children = rawMetricChildren
+                metricError = calcMean.stdDev.toLong(),
+                rawMetrics = rawMetricChildren
            )
        )
-        metricChildren.add(Metric("mean", calcMean.mean.toLong()))
+        metricChildren.add(Metric("mean", metricValue = calcMean.mean.toLong()))
        // keep geomMean for bwc
-        metricChildren.add(Metric(GEOM_MEAN, calcMean.geomMean.toLong()))
-        metricChildren.add(Metric("stdDev", calcMean.stdDev.toLong()))
+        metricChildren.add(Metric(GEOM_MEAN, metricValue = calcMean.geomMean.toLong()))
+        metricChildren.add(Metric("stdDev", metricValue = calcMean.stdDev.toLong()))

        statInfosArray.filterNotNull()
            .map { it.keys }
@@ -77,13 +74,13 @@ class Stats(
                val shortName = if (perfCounterName.endsWith(": time")) n.removeSuffix(": time") else null
                val metricShortName = if (perfCounterName.endsWith(": time")) perfCounterName.removeSuffix(": time") else perfCounterName

-                metricChildren.add(Metric(": $metricShortName", mean))
+                metricChildren.add(Metric(": $metricShortName", metricValue = mean))

                TeamCity.test(shortName, durationMs = mean) {}
            }

        perfTestRawDataMs.addAll(timingsMs.toList())
-        metric.writeTeamCityStats(name)
+        metric!!.writeTeamCityStats(name)
    }

    private fun toTimingsMs(statInfosArray: Array<StatInfos>) =
@@ -91,22 +88,6 @@ class Stats(

    private fun calcMean(statInfosArray: Array<StatInfos>): Mean = calcMean(toTimingsMs(statInfosArray))

-    private fun calcMean(values: LongArray): Mean {
-        val mean = values.average()
-
-        val stdDev = if (values.size > 1) (sqrt(
-            values.fold(0.0,
-                        { accumulator, next -> accumulator + (1.0 * (next - mean)).pow(2) })
-        ) / (values.size - 1))
-        else 0.0
-
-        val geomMean = geomMean(values.toList())
-
-        return Mean(mean, stdDev, geomMean)
-    }
-
-    data class Mean(val mean: Double, val stdDev: Double, val geomMean: Double)
-
    fun <SV, TV> perfTest(
        testName: String,
        warmUpIterations: Int = 5,
@@ -116,7 +97,6 @@ class Stats(
        tearDown: (TestData<SV, TV>) -> Unit = { },
        checkStability: Boolean = true
    ) {
-
        val warmPhaseData = PhaseData(
            iterations = warmUpIterations,
            testName = testName,
@@ -155,7 +135,7 @@ class Stats(
                        }

                        TeamCity.test(stabilityName, errorDetails = error, includeStats = false) {
-                            metricChildren.add(Metric("stability", stabilityPercentage))
+                            metricChildren.add(Metric("stability", metricValue = stabilityPercentage.toLong()))
                        }
                    }

@@ -198,19 +178,20 @@ class Stats(
            val t = statInfo[ERROR_KEY] as? Throwable
            if (t != null) {
                TeamCity.test(n, errors = listOf(t)) {}
-                metricChildren.add(Metric(attemptString, value = null, hasError = true))
+                //logMessage { "metricChildren.add(Metric('$attemptString', value = null, hasError = true))" }
+                metricChildren.add(Metric(attemptString, metricValue = null, hasError = true))
            } else if (!printOnlyErrors) {
                val durationMs = (statInfo[TEST_KEY] as Long).nsToMs
                TeamCity.test(n, durationMs = durationMs, includeStats = false) {
                    for ((k, v) in statInfo) {
                        if (k == TEST_KEY) continue
                        (v as? Number)?.let {
-                            childrenMetrics.add(Metric(k, v))
+                            childrenMetrics.add(Metric(k, metricValue = v.toLong()))
                            //TeamCity.metadata(n, k, it)
                        }
                    }
                }
-                metricChildren.add(Metric(attemptString, durationMs, children = childrenMetrics))
+                metricChildren.add(Metric(attemptString, metricValue = durationMs, metrics = childrenMetrics))
            }
        }
    }
@@ -351,18 +332,11 @@ class Stats(
        }
    }

-    private fun geomMean(data: List<Long>) = exp(data.fold(0.0, { mul, next -> mul + ln(1.0 * next) }) / data.size)
-
    override fun close() {
        flush()
    }

    fun flush() {
-        val children = metrics.toMutableList()
-
-        val properties = mutableMapOf<String, Any>()
-        properties[BENCHMARK] = name
-
        val simpleDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ")
        simpleDateFormat.timeZone = TimeZone.getTimeZone("UTC")
 //        properties["buildTimestamp"] = simpleDateFormat.format(Date())
@@ -370,31 +344,68 @@ class Stats(
 //        properties["buildBranch"] = "rr/perf/json-output"
 //        properties["agentName"] = "kotlin-linux-perf-unit879"

+        var buildId: Int? = null
+        var agentName: String? = null
+        var buildBranch: String? = null
+
        System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE")?.let { teamcityConfig ->
            val buildProperties = Properties()
            buildProperties.load(FileInputStream(teamcityConfig))

-            properties["buildTimestamp"] = simpleDateFormat.format(Date())
-            for ((name, key) in
-            mapOf(
-                "buildId" to "teamcity.build.id",
-                "buildBranch" to "teamcity.build.branch",
-                "agentName" to "agent.name",
-            )) {
-                val property = buildProperties.getProperty(key)
-                properties[name] = if (name == "buildId") property.toLong() else property
-            }
+            buildId = buildProperties["teamcity.build.id"]?.toString()?.toInt()
+            agentName = buildProperties["agent.name"]?.toString()
+            buildBranch = buildProperties["teamcity.build.branch"]?.toString()
        }
+
        if (perfTestRawDataMs.isNotEmpty()) {
            val geomMeanMs = geomMean(perfTestRawDataMs.toList()).toLong()
-            Metric(GEOM_MEAN, geomMeanMs).writeTeamCityStats(name)
-            properties[GEOM_MEAN] = geomMeanMs
+            Metric(GEOM_MEAN, metricValue = geomMeanMs).writeTeamCityStats(name)
        }

-        val metric = Metric(name, null, children = children, properties = properties)
+        try {
+            metric?.let {
+                val benchmark = Benchmark(
+                    agentName = agentName,
+                    buildBranch = buildBranch,
+                    buildId = buildId,
+                    benchmark = name,
+                    name = it.metricName,
+                    metricValue = it.metricValue,
+                    metricError = it.metricError,
+                    buildTimestamp = simpleDateFormat.format(Date()),
+                    metrics = it.metrics ?: emptyList()
+                )

-        metric.writeJson()
-        metrics.writeCSV(name, header)
+                if (benchmark.name == "ParameterNameAndType - TypeParameter") {
+                    println(benchmark)
+                }
+
+                benchmark.writeJson()
+
+                with(
+                    Benchmark(
+                        agentName = agentName,
+                        buildBranch = buildBranch,
+                        buildId = buildId,
+                        benchmark = name,
+                        synthetic = true,
+                        name = "geomMean",
+                        buildTimestamp = simpleDateFormat.format(Date())
+                    )
+                ) {
+                    loadJson()
+                    merge(benchmark)
+
+                    if (benchmark.name == "ParameterNameAndType - TypeParameter") {
+                        println(this)
+                    }
+                    writeJson()
+                }
+            }
+        } finally {
+            metric = null
+        }
+        //metrics.writeCSV(name, header)
    }

    companion object {
@@ -403,7 +414,8 @@ class Stats(

        const val WARM_UP = "warm-up"
        const val GEOM_MEAN = "geomMean"
-        const val BENCHMARK = "benchmark"
+
+        internal val extraMetricNames = setOf("", GEOM_MEAN, "mean", "stdDev")

        inline fun runAndMeasure(note: String, block: () -> Unit) {
            val openProjectMillis = measureTimeMillis {
@@ -415,15 +427,23 @@ class Stats(

 }

-data class Metric(
-    val name: String,
-    val value: Number?,
-    val hasError: Boolean = false,
-    val measurementError: Number? = null,
-    val childrenName: String = "metrics",
-    val children: MutableList<Metric> = mutableListOf(),
-    val properties: Map<String, Any>? = null
-)
+internal fun calcMean(values: LongArray): Mean {
+    val mean = values.average()
+
+    val stdDev = if (values.size > 1) (sqrt(
+        values.fold(0.0,
+                    { accumulator, next -> accumulator + (1.0 * (next - mean)).pow(2) })
+    ) / (values.size - 1))
+    else 0.0
+
+    val geomMean = geomMean(values.toList())
+
+    return Mean(mean, stdDev, geomMean)
+}
+
+private fun geomMean(data: List<Long>) = exp(data.fold(0.0, { mul, next -> mul + ln(1.0 * next) }) / data.size)
+
+internal data class Mean(val mean: Double, val stdDev: Double, val geomMean: Double)

 data class PhaseData<SV, TV>(
    val iterations: Int,
@@ -1,122 +0,0 @@
-/*
- * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.idea.perf
-
-import org.jetbrains.kotlin.idea.perf.util.TeamCity
-import org.jetbrains.kotlin.idea.testFramework.suggestOsNeutralFileName
-import java.io.BufferedWriter
-import java.io.File
-
-internal fun List<Metric>.writeCSV(name: String, header: Array<String>) {
-    fun Metric.append(prefix: String, output: BufferedWriter) {
-        val s = "$prefix ${this.name}".trim()
-        output.appendLine("$s,${value ?: ""},")
-        children.forEach {
-            it.append(s, output)
-        }
-    }
-
-    val statsFile = statsFile(name, "csv")
-    statsFile.bufferedWriter().use { output ->
-        output.appendLine(header.joinToString())
-        forEach { it.append("$name:", output) }
-        output.flush()
-    }
-}
-
-internal fun Metric.writeTeamCityStats(name: String, rawMeasurementName: String = "rawMetrics", rawMetrics: Boolean = false) {
-    fun Metric.append(prefix: String, depth: Int) {
-        val s = if (this.name.isEmpty()) {
-            prefix
-        } else {
-            if (depth == 0 && this.name != Stats.GEOM_MEAN) "$prefix: ${this.name}" else "$prefix ${this.name}"
-        }.trim()
-        if (s != prefix) {
-            value?.let {
-                TeamCity.statValue(s, it)
-            }
-        }
-        for (childIndex in children.withIndex()) {
-            if (!rawMetrics && childrenName == rawMeasurementName && childIndex.index > 0) break
-            childIndex.value.append(s, depth + 1)
-        }
-    }
-
-    append(name, 0)
-}
-
-internal fun Metric.writeJson() {
-    val statsFile = statsFile(name, "json")
-    statsFile.bufferedWriter().use { output ->
-        output.appendLine(toJson(""))
-        output.flush()
-    }
-}
-
-private fun statsFile(name: String, extension: String) =
-    File(pathToResource("stats${statFilePrefix(name)}.$extension")).absoluteFile
-
-private fun List<Metric>.toJson(prefix: String) = joinToString(separator = ",", prefix = "[", postfix = "]") { it.toJson(prefix) }
-
-private fun String.jsonValue() = replace("\"", "\\\"")
-
-private fun Metric.toJson(prefix: String): String =
-    buildString {
-        append("{")
-        val nam = if (name.isEmpty()) "_value" else name
-        val nm = nam.jsonValue()
-        val s = if (name == Stats.GEOM_MEAN) {
-            "${prefix.substring(0, prefix.length - 1)} $name".trim()
-        } else {
-            "$prefix $name${if (prefix.isEmpty()) ":" else ""}".trim()
-        }
-
-
-        var commaRequired = false
-        if (value != null || hasError) {
-            append("\"metricName\":\"").append(nm).append("\"")
-            commaRequired = true
-        }
-
-        if (value != null) {
-            append(",\"metricValue\":").append(value)
-            measurementError?.let {
-                append(",\"metricError\":").append(it)
-            }
-            //append(",\"legacyName\":").append("\"${s.jsonValue()}\"")
-        }
-        if (hasError) {
-            append(",\"hasError\": $hasError")
-        }
-
-        if (properties != null) {
-            for ((k, v) in properties) {
-                if (commaRequired) append(",")
-                append("\"").append(k).append("\":")
-                if (v is Number) {
-                    append(v)
-                } else {
-                    append("\"").append(v).append("\"")
-                }
-                commaRequired = true
-            }
-        }
-
-        if (children.isNotEmpty()) {
-            val cnm = childrenName.jsonValue()
-            if (commaRequired) append(",")
-            append("\"").append(cnm).append("\"")
-            append(":").append(children.toJson(s))
-        }
-        append("}")
-    }
-
-
-internal fun pathToResource(resource: String) = "build/$resource"
-
-internal fun statFilePrefix(name: String) = if (name.isNotEmpty()) "-${plainname(name)}" else ""
-
-internal fun plainname(name: String) = suggestOsNeutralFileName(name)
@@ -36,7 +36,7 @@ import java.io.File
 abstract class WholeProjectPerformanceTest : DaemonAnalyzerTestCase(), WholeProjectFileProvider {

    private val rootProjectFile: File = File("../perfTestProject").absoluteFile
-    private val perfStats: Stats = Stats(name = "whole", header = arrayOf("File", "ProcessID", "Time"))
+    private val perfStats: Stats = Stats(name = "whole")
    private val tmp = rootProjectFile

    override fun isStressTest(): Boolean = false
@@ -0,0 +1,129 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.idea.perf.util
+
+import com.fasterxml.jackson.annotation.JsonInclude
+import com.fasterxml.jackson.annotation.JsonProperty
+import org.jetbrains.kotlin.idea.perf.Stats
+import org.jetbrains.kotlin.idea.perf.calcMean
+import java.util.ArrayList
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+data class Benchmark(
+    @set:JsonProperty("agentName")
+    var agentName: String?,
+    @set:JsonProperty("benchmark")
+    var benchmark: String,
+    @set:JsonProperty("name")
+    var name: String? = null,
+    @set:JsonProperty("synthetic")
+    var synthetic: Boolean? = null,
+    @set:JsonProperty("buildTimestamp")
+    var buildTimestamp: String,
+    @set:JsonProperty("buildBranch")
+    var buildBranch: String?,
+    @set:JsonProperty("buildId")
+    var buildId: Int?,
+    @set:JsonProperty("metricValue")
+    var metricValue: Long? = null,
+    @set:JsonProperty("metricError")
+    var metricError: Long? = null,
+    @set:JsonProperty("hasError")
+    var hasError: Boolean? = null,
+
+    @set:JsonProperty("metrics")
+    var metrics: List<Metric> = ArrayList()
+) {
+    init {
+        hasError = if (metrics.any { it.ifHasError() == true }) true else null
+    }
+
+    fun resetValue() {
+        buildId = null
+        metricValue = null
+        metricError = null
+        buildBranch = null
+        metrics.forEach { it.resetValue() }
+    }
+
+    fun fileName(): String =
+        listOfNotNull(benchmark, name?.replace("[^A-Za-z0-9_]", ""), buildId?.toString()).joinToString(separator = "_")
+
+    fun cleanUp() {
+        metrics?.forEach { it.cleanUp() }
+        metrics = metrics?.filter { it.metricValue != null || (it.metrics?.isNotEmpty() == true) }
+    }
+
+    fun merge(extraBenchmark: Benchmark) {
+        val benchmarkMetrics = extraBenchmark.metrics.filter {
+            it.metricName !in Stats.extraMetricNames
+        }
+        benchmarkMetrics.forEach { it.rawMetrics = null }
+        val extraBenchmarkName = extraBenchmark.name!!
+        metrics = metrics.filterNot { it.metricName == extraBenchmarkName } + Metric(
+            metricName = extraBenchmarkName,
+            metricValue = extraBenchmark.metricValue,
+            metricError = extraBenchmark.metricError,
+            hasError = extraBenchmark.hasError,
+            metrics = benchmarkMetrics
+        )
+        cleanUp()
+
+        hasError = if (metrics.any { it.ifHasError() == true }) true else null
+        val values = metrics.mapNotNull { it.metricValue }.toLongArray()
+        val calcMean = calcMean(values)
+        metricValue = calcMean.geomMean.toLong()
+        metricError = calcMean.stdDev.toLong()
+    }
+}
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+data class Metric(
+    @set:JsonProperty("metricName")
+    var metricName: String,
+    @set:JsonProperty("legacyName")
+    var legacyName: String? = null,
+    @set:JsonProperty("metricValue")
+    var metricValue: Long?,
+    @set:JsonProperty("metricError")
+    var metricError: Long? = null,
+    @set:JsonProperty("hasError")
+    var hasError: Boolean? = null,
+    @set:JsonProperty("metrics")
+    var metrics: List<Metric>? = null,
+    @set:JsonProperty("rawMetrics")
+    var rawMetrics: List<Metric>? = null
+) {
+    fun ifHasError(): Boolean? {
+        if (hasError == true) return true
+        hasError =
+            if ((metrics?.any { it.ifHasError() == true } == true) || (rawMetrics?.any { it.ifHasError() == true } == true)) {
+                true
+            } else {
+                null
+            }
+        return hasError
+    }
+
+    fun resetValue() {
+        metricValue = null
+        metricError = null
+        metrics?.forEach { it.resetValue() }
+        rawMetrics?.forEach { it.resetValue() }
+    }
+
+    fun cleanUp() {
+        legacyName = null
+        metrics?.forEach { it.cleanUp() }
+        rawMetrics?.forEach { it.cleanUp() }
+
+        metrics = metrics?.filter { it.metricValue != null || (it.metrics?.isNotEmpty() == true) }
+        if (metrics?.isEmpty() == true) metrics = null
+
+        rawMetrics = rawMetrics?.filter { it.metricValue != null || (it.metrics?.isNotEmpty() == true) }
+        if (rawMetrics?.isEmpty() == true) rawMetrics = null
+    }
+}
@@ -0,0 +1,92 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.idea.perf.util
+
+import com.fasterxml.jackson.core.type.TypeReference
+import com.fasterxml.jackson.databind.DeserializationFeature
+import com.fasterxml.jackson.databind.MapperFeature
+import com.fasterxml.jackson.databind.SerializationFeature
+import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
+import com.fasterxml.jackson.module.kotlin.registerKotlinModule
+import org.jetbrains.kotlin.idea.perf.Stats
+import org.jetbrains.kotlin.idea.testFramework.suggestOsNeutralFileName
+import java.io.BufferedWriter
+import java.io.File
+
+internal fun List<Metric>.writeCSV(name: String, header: Array<String>) {
+    fun Metric.append(prefix: String, output: BufferedWriter) {
+        val s = "$prefix ${this.metricName}".trim()
+        output.appendLine("$s,${metricValue ?: ""},")
+        metrics?.forEach {
+            it.append(s, output)
+        }
+    }
+
+    val statsFile = statsFile(name, "csv")
+    statsFile.bufferedWriter().use { output ->
+        output.appendLine(header.joinToString())
+        forEach { it.append("$name:", output) }
+        output.flush()
+    }
+}
+
+internal fun Metric.writeTeamCityStats(name: String, rawMeasurementName: String = "rawMetrics", rawMetrics: Boolean = false) {
+    fun Metric.append(prefix: String, depth: Int) {
+        val s = if (this.metricName.isEmpty()) {
+            prefix
+        } else {
+            if (depth == 0 && this.metricName != Stats.GEOM_MEAN) "$prefix: ${this.metricName}" else "$prefix ${this.metricName}"
+        }.trim()
+        if (s != prefix) {
+            metricValue?.let {
+                TeamCity.statValue(s, it)
+            }
+        }
+        metrics?.let { list ->
+            for (childIndex in list.withIndex()) {
+                if (!rawMetrics && childIndex.index > 0) break
+                childIndex.value.append(s, depth + 1)
+            }
+        }
+    }
+
+    append(name, 0)
+}
+
+internal val kotlinJsonMapper = jacksonObjectMapper()
+    .registerKotlinModule()
+    .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES, true)
+    .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
+    .configure(SerializationFeature.INDENT_OUTPUT, true)
+
+internal fun Benchmark.statsFile() = statsFile(fileName(), "json")
+
+internal fun Benchmark.writeJson() {
+    val json = kotlinJsonMapper.writeValueAsString(this)
+    val statsFile = statsFile()
+    logMessage { "write $statsFile" }
+    statsFile.bufferedWriter().use { output ->
+        output.appendLine(json)
+        output.flush()
+    }
+}
+
+internal fun Benchmark.loadJson() {
+    val statsFile = statsFile()
+    if (statsFile.exists()) {
+        val value = kotlinJsonMapper.readValue(statsFile, object : TypeReference<Benchmark>() {})
+        metrics = value.metrics
+    }
+}
+
+private fun statsFile(name: String, extension: String) =
+    File(pathToResource("stats${statFilePrefix(name)}.$extension")).absoluteFile
+
+internal fun pathToResource(resource: String) = "build/$resource"
+
+internal fun statFilePrefix(name: String) = if (name.isNotEmpty()) "-${plainname(name)}" else ""
+
+internal fun plainname(name: String) = suggestOsNeutralFileName(name)