Add json output for IDE performance tests

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceAddImportTest.kt

@@ -22,12 +22,6 @@ abstract class AbstractPerformanceAddImportTest : AbstractPerformanceImportTest(
 
         @JvmStatic
         val stats: Stats = Stats("add-import")
-
-        init {
-            // there is no @AfterClass for junit3.8
-            Runtime.getRuntime().addShutdownHook(Thread(Runnable { stats.close() }))
-        }
-
     }
 
     override fun stats(): Stats = stats

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionHandlerTests.kt

@@ -9,6 +9,8 @@ import com.intellij.application.options.CodeStyle
 import com.intellij.codeInsight.completion.CompletionType
 import com.intellij.openapi.application.runWriteAction
 import com.intellij.openapi.util.io.FileUtil
+import com.intellij.testFramework.RunAll
+import com.intellij.util.ThrowableRunnable
 import org.jetbrains.kotlin.idea.completion.test.ExpectedCompletionUtils
 import org.jetbrains.kotlin.idea.completion.test.configureWithExtraFile
 import org.jetbrains.kotlin.idea.completion.test.handlers.AbstractCompletionHandlerTest.Companion.CODE_STYLE_SETTING_PREFIX
@@ -36,11 +38,6 @@ abstract class AbstractPerformanceCompletionHandlerTests(
     companion object {
         @JvmStatic
         val statsMap: MutableMap<String, Stats> = mutableMapOf()
-
-        init {
-            // there is no @AfterClass for junit3.8
-            Runtime.getRuntime().addShutdownHook(Thread(Runnable { statsMap.values.forEach { it.close() } }))
-        }
     }
 
     private fun stats(): Stats {
@@ -52,7 +49,10 @@ abstract class AbstractPerformanceCompletionHandlerTests(
 
     override fun tearDown() {
         commitAllDocuments()
-        super.tearDown()
+        RunAll(
+            ThrowableRunnable { super.tearDown() },
+            ThrowableRunnable { statsMap.values.forEach(Stats::flush) }
+        ).run()
     }
 
     protected open fun doPerfTest(unused: String) {

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionIncrementalResolveTest.kt

@@ -9,6 +9,8 @@ import com.intellij.codeInsight.completion.CompletionType
 import com.intellij.codeInsight.lookup.LookupElement
 import com.intellij.openapi.application.runWriteAction
 import com.intellij.openapi.util.text.StringUtil
+import com.intellij.testFramework.RunAll
+import com.intellij.util.ThrowableRunnable
 import org.jetbrains.kotlin.idea.KotlinFileType
 import org.jetbrains.kotlin.idea.completion.CompletionBindingContextProvider
 import org.jetbrains.kotlin.idea.perf.Stats.Companion.WARM_UP
@@ -33,11 +35,6 @@ abstract class AbstractPerformanceCompletionIncrementalResolveTest : KotlinLight
 
         @JvmStatic
         val stats: Stats = Stats("completion-incremental")
-
-        init {
-            // there is no @AfterClass for junit3.8
-            Runtime.getRuntime().addShutdownHook(Thread(Runnable { stats.close() }))
-        }
     }
 
     override fun getProjectDescriptor() = KotlinWithJdkAndRuntimeLightProjectDescriptor.INSTANCE
@@ -53,7 +50,10 @@ abstract class AbstractPerformanceCompletionIncrementalResolveTest : KotlinLight
 
     override fun tearDown() {
         commitAllDocuments()
-        super.tearDown()
+        RunAll(
+            ThrowableRunnable { super.tearDown() },
+            ThrowableRunnable { stats.flush() }
+        ).run()
     }
 
     private fun doWarmUpPerfTest() {

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceHighlightingTest.kt

@@ -7,7 +7,9 @@ package org.jetbrains.kotlin.idea.perf
 
 import com.intellij.codeInsight.daemon.impl.HighlightInfo
 import com.intellij.openapi.application.runWriteAction
+import com.intellij.testFramework.RunAll
 import com.intellij.testFramework.fixtures.impl.CodeInsightTestFixtureImpl.ensureIndexesUpToDate
+import com.intellij.util.ThrowableRunnable
 import org.jetbrains.kotlin.idea.KotlinFileType
 import org.jetbrains.kotlin.idea.perf.Stats.Companion.WARM_UP
 import org.jetbrains.kotlin.idea.test.KotlinLightCodeInsightFixtureTestCase
@@ -25,10 +27,6 @@ abstract class AbstractPerformanceHighlightingTest : KotlinLightCodeInsightFixtu
         @JvmStatic
         val stats: Stats = Stats("highlight")
 
-        init {
-            // there is no @AfterClass for junit3.8
-            Runtime.getRuntime().addShutdownHook(Thread(Runnable { stats.close() }))
-        }
     }
 
     override fun setUp() {
@@ -42,7 +40,10 @@ abstract class AbstractPerformanceHighlightingTest : KotlinLightCodeInsightFixtu
 
     override fun tearDown() {
         commitAllDocuments()
-        super.tearDown()
+        RunAll(
+            ThrowableRunnable { super.tearDown() },
+            ThrowableRunnable { stats.flush() }
+        ).run()
     }
 
     private fun doWarmUpPerfTest() {

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceImportTest.kt

@@ -9,6 +9,8 @@ import com.intellij.application.options.CodeStyle
 import com.intellij.openapi.application.runWriteAction
 import com.intellij.psi.PsiDocumentManager
 import com.intellij.testFramework.LightProjectDescriptor
+import com.intellij.testFramework.RunAll
+import com.intellij.util.ThrowableRunnable
 import junit.framework.TestCase
 import org.jetbrains.kotlin.descriptors.ClassDescriptor
 import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
@@ -30,6 +32,13 @@ abstract class AbstractPerformanceImportTest : KotlinLightCodeInsightFixtureTest
 
     protected abstract fun stats(): Stats
 
+    override fun tearDown() {
+        RunAll(
+            ThrowableRunnable { super.tearDown() },
+            ThrowableRunnable { stats().flush() }
+        ).run()
+    }
+
     protected fun doPerfTest(unused: String) {
         val testName = getTestName(false)
 

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceJavaToKotlinCopyPasteConversionTest.kt

@@ -9,6 +9,8 @@ import com.intellij.ide.highlighter.JavaFileType
 import com.intellij.openapi.actionSystem.IdeActions
 import com.intellij.openapi.application.runWriteAction
 import com.intellij.psi.PsiDocumentManager
+import com.intellij.testFramework.RunAll
+import com.intellij.util.ThrowableRunnable
 import org.jetbrains.kotlin.idea.KotlinFileType
 import org.jetbrains.kotlin.idea.configuration.ExperimentalFeatures
 import org.jetbrains.kotlin.idea.conversion.copy.AbstractJavaToKotlinCopyPasteConversionTest
@@ -28,11 +30,6 @@ abstract class AbstractPerformanceJavaToKotlinCopyPasteConversionTest(private va
         val warmedUp: Array<Boolean> = arrayOf(false, false)
 
         val stats: Array<Stats> = arrayOf(Stats("old j2k"), Stats("new j2k"))
-
-        init {
-            // there is no @AfterClass for junit3.8
-            Runtime.getRuntime().addShutdownHook(Thread(Runnable { stats.forEach { it.close() } }))
-        }
     }
 
     override fun setUp() {
@@ -47,6 +44,14 @@ abstract class AbstractPerformanceJavaToKotlinCopyPasteConversionTest(private va
         }
     }
 
+    override fun tearDown() {
+        RunAll(
+            ThrowableRunnable { super.tearDown() },
+            ThrowableRunnable { stats.forEach(Stats::flush) }
+        ).run()
+
+    }
+
     private fun doWarmUpPerfTest() {
         performanceTest<Unit, Unit> {
             name(WARM_UP)

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceLiteralKotlinToKotlinCopyPasteTest.kt

@@ -9,6 +9,8 @@ import com.intellij.openapi.actionSystem.IdeActions
 import com.intellij.openapi.application.runWriteAction
 import com.intellij.openapi.fileEditor.ex.FileEditorManagerEx
 import com.intellij.psi.PsiFile
+import com.intellij.testFramework.RunAll
+import com.intellij.util.ThrowableRunnable
 import org.jetbrains.kotlin.idea.AbstractCopyPasteTest
 import org.jetbrains.kotlin.idea.perf.Stats.Companion.WARM_UP
 import org.jetbrains.kotlin.idea.test.KotlinWithJdkAndRuntimeLightProjectDescriptor
@@ -27,11 +29,6 @@ abstract class AbstractPerformanceLiteralKotlinToKotlinCopyPasteTest : AbstractC
 
         @JvmStatic
         val stats: Stats = Stats("Literal-k2k-CopyPaste")
-
-        init {
-            // there is no @AfterClass for junit3.8
-            Runtime.getRuntime().addShutdownHook(Thread(Runnable { stats.close() }))
-        }
     }
 
     override fun getProjectDescriptor() = KotlinWithJdkAndRuntimeLightProjectDescriptor.INSTANCE
@@ -45,6 +42,13 @@ abstract class AbstractPerformanceLiteralKotlinToKotlinCopyPasteTest : AbstractC
         }
     }
 
+    override fun tearDown() {
+        RunAll(
+            ThrowableRunnable { super.tearDown() },
+            ThrowableRunnable { stats.flush() }
+        ).run()
+    }
+
     private fun doWarmUpPerfTest() {
         val fileEditorManager = FileEditorManagerEx.getInstance(project)
         performanceTest<Pair<PsiFile, PsiFile>, Unit> {

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceTypingIndentationTest.kt

@@ -7,10 +7,10 @@ package org.jetbrains.kotlin.idea.perf
 
 import com.intellij.application.options.CodeStyle
 import com.intellij.openapi.actionSystem.IdeActions
-import com.intellij.openapi.application.impl.NonBlockingReadActionImpl
 import com.intellij.openapi.util.io.FileUtil
 import com.intellij.testFramework.EditorTestUtil
-import com.intellij.util.ui.UIUtil
+import com.intellij.testFramework.RunAll
+import com.intellij.util.ThrowableRunnable
 import org.jetbrains.kotlin.formatter.FormatSettingsUtil
 import org.jetbrains.kotlin.idea.test.KotlinLightCodeInsightFixtureTestCase
 import org.jetbrains.kotlin.idea.testFramework.dispatchAllInvocationEvents
@@ -27,6 +27,13 @@ abstract class AbstractPerformanceTypingIndentationTest : KotlinLightCodeInsight
         val stats: Stats = Stats("typing-indentation")
     }
 
+    override fun tearDown() {
+        RunAll(
+            ThrowableRunnable { super.tearDown() },
+            ThrowableRunnable { stats.flush() }
+        ).run()
+    }
+
     protected fun doPerfTest(unused: String) {
         val testName = getTestName(false)
 

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerformanceProjectsTest.kt

@@ -9,6 +9,8 @@ import com.intellij.codeInsight.daemon.impl.HighlightInfo
 import com.intellij.lang.annotation.AnnotationHolder
 import com.intellij.openapi.project.Project
 import com.intellij.psi.PsiElement
+import com.intellij.testFramework.RunAll
+import com.intellij.util.ThrowableRunnable
 import org.jetbrains.kotlin.idea.core.script.ScriptConfigurationManager
 import org.jetbrains.kotlin.idea.highlighter.KotlinPsiChecker
 import org.jetbrains.kotlin.idea.highlighter.KotlinPsiCheckerAndHighlightingUpdater
@@ -35,11 +37,6 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
         @JvmStatic
         val timer: AtomicLong = AtomicLong()
 
-        init {
-            // there is no @AfterClass for junit3.8
-            Runtime.getRuntime().addShutdownHook(Thread { hwStats.close() })
-        }
-
         fun resetTimestamp() {
             timer.set(0)
         }
@@ -54,6 +51,13 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
         warmUp.warmUp(this)
     }
 
+    override fun tearDown() {
+        RunAll(
+            ThrowableRunnable { super.tearDown() },
+            ThrowableRunnable { hwStats.flush() }
+        ).run()
+    }
+
     fun testHelloWorldProject() {
         suite("Hello world project") {
             myProject = perfOpenProject(stats = hwStats) {
@@ -337,14 +341,18 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
                 profilerConfig.enabled = true
             }
 
+            val metricChildren = mutableListOf<Metric>()
+
             extraStats.printWarmUpTimings(
                 "annotator",
-                extraTimingsNs.take(warmUpIterations).toTypedArray()
+                extraTimingsNs.take(warmUpIterations).toTypedArray(),
+                metricChildren
             )
 
-            extraStats.appendTimings(
+            extraStats.processTimings(
                 "annotator",
-                extraTimingsNs.drop(warmUpIterations).toTypedArray()
+                extraTimingsNs.drop(warmUpIterations).toTypedArray(),
+                metricChildren
             )
         }
     }

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt

@@ -9,12 +9,14 @@ import org.jetbrains.kotlin.idea.perf.WholeProjectPerformanceTest.Companion.nsTo
 import org.jetbrains.kotlin.idea.perf.profilers.*
 import org.jetbrains.kotlin.idea.perf.util.TeamCity
 import org.jetbrains.kotlin.idea.perf.util.logMessage
-import org.jetbrains.kotlin.idea.testFramework.suggestOsNeutralFileName
 import org.jetbrains.kotlin.util.PerformanceCounter
 import java.io.*
-import kotlin.system.measureNanoTime
 import java.lang.ref.WeakReference
+import java.text.SimpleDateFormat
+import java.util.*
+import kotlin.collections.HashMap
 import kotlin.math.*
+import kotlin.system.measureNanoTime
 import kotlin.system.measureTimeMillis
 import kotlin.test.assertEquals
 
@@ -25,44 +27,36 @@ class Stats(
     private val profilerConfig: ProfilerConfig = ProfilerConfig(),
     private val header: Array<String> = arrayOf("Name", "ValueMS", "StdDev"),
     private val acceptanceStabilityLevel: Int = 25
-) : Closeable {
+) : AutoCloseable {
 
     private val perfTestRawDataMs = mutableListOf<Long>()
 
-    private val statsFile: File = File(pathToResource("stats${statFilePrefix()}.csv")).absoluteFile
+    private val metrics = mutableListOf<Metric>()
-
-    private val statsOutput: BufferedWriter
 
     init {
-        statsOutput = statsFile.bufferedWriter()
-
-        statsOutput.appendLine(header.joinToString())
-
         PerformanceCounter.setTimeCounterEnabled(true)
     }
 
-    private fun statFilePrefix() = if (name.isNotEmpty()) "-${plainname()}" else ""
+    private fun calcAndProcessMetrics(id: String, statInfosArray: Array<StatInfos>, rawMetricChildren: MutableList<Metric>) {
-
-    private fun plainname() = suggestOsNeutralFileName(name)
-
-    private fun pathToResource(resource: String) = "build/$resource"
-
-    private fun append(id: String, statInfosArray: Array<StatInfos>) {
         val timingsMs = toTimingsMs(statInfosArray)
 
         val calcMean = calcMean(timingsMs)
 
-        for (v in listOf(
+        val metricChildren = mutableListOf<Metric>()
-            Triple("mean", "", calcMean.mean.toLong()),
+        val metric = Metric(id, calcMean.mean.toLong(), measurementError = calcMean.stdDev.toLong(), children = metricChildren)
-            Triple("stdDev", " stdDev", calcMean.stdDev.toLong()),
+        metrics.add(metric)
-            Triple("geomMean", " geomMean", calcMean.geomMean.toLong())
-        )) {
-            val n = "$id : ${v.first}"
 
-            TeamCity.test(n, durationMs = v.third, includeStats = false) {
+        metricChildren.add(
-                TeamCity.statValue("$id${v.second}", v.third)
+            Metric(
-            }
+                "", calcMean.mean.toLong(),
-        }
+                measurementError = calcMean.stdDev.toLong(),
+                childrenName = "raw_metrics", children = rawMetricChildren
+            )
+        )
+        metricChildren.add(Metric("mean", calcMean.mean.toLong()))
+        // keep geomMean for bwc
+        metricChildren.add(Metric(GEOM_MEAN, calcMean.geomMean.toLong()))
+        metricChildren.add(Metric("std_dev", calcMean.stdDev.toLong()))
 
         statInfosArray.filterNotNull()
             .map { it.keys }
@@ -76,14 +70,15 @@ class Stats(
                 val mean = statInfoMean.mean.toLong()
 
                 val shortName = if (perfCounterName.endsWith(": time")) n.removeSuffix(": time") else null
+                val metricShortName = if (perfCounterName.endsWith(": time")) perfCounterName.removeSuffix(": time") else perfCounterName
 
-                TeamCity.test(shortName, durationMs = mean) {
+                metricChildren.add(Metric(": $metricShortName", mean))
-                    TeamCity.statValue(n, mean)
+
-                }
+                TeamCity.test(shortName, durationMs = mean) {}
             }
 
         perfTestRawDataMs.addAll(timingsMs.toList())
-        append(arrayOf(id, calcMean.mean, calcMean.stdDev))
+        metric.writeTeamCityStats(name)
     }
 
     private fun toTimingsMs(statInfosArray: Array<StatInfos>) =
@@ -107,19 +102,6 @@ class Stats(
 
     data class Mean(val mean: Double, val stdDev: Double, val geomMean: Double)
 
-    private fun append(values: Array<Any>) {
-        require(values.size == header.size) { "Expected ${header.size} values, actual ${values.size} values" }
-        with(statsOutput) {
-            appendLine(values.joinToString { it.toString() })
-            flush()
-        }
-    }
-
-    fun append(file: String, id: String, nanoTime: Long) {
-        val ms = nanoTime.nsToMs
-        append(arrayOf(file, id, ms))
-    }
-
     fun <SV, TV> perfTest(
         testName: String,
         warmUpIterations: Int = 5,
@@ -145,13 +127,12 @@ class Stats(
             tearDown = tearDown
         )
         val block = {
-            warmUpPhase(warmPhaseData)
+            val metricChildren = mutableListOf<Metric>()
-            val statInfoArray = mainPhase(mainPhaseData)
+            warmUpPhase(warmPhaseData, metricChildren)
+            val statInfoArray = mainPhase(mainPhaseData, metricChildren)
 
             assertEquals(iterations, statInfoArray.size)
             if (testName != WARM_UP) {
-                appendTimings(testName, statInfoArray)
-
                 // do not estimate stability for warm-up
                 if (!testName.contains(WARM_UP)) {
                     val calcMean = calcMean(statInfoArray)
@@ -168,11 +149,13 @@ class Stats(
                     }
 
                     TeamCity.test(stabilityName, errorDetails = error, includeStats = false) {
-                        TeamCity.statValue(stabilityName, stabilityPercentage)
+                        metricChildren.add(Metric("stability", stabilityPercentage))
                     }
                 }
+
+                processTimings(testName, statInfoArray, metricChildren)
             } else {
-                printTimings(testName, printOnlyErrors = true, statInfoArray = statInfoArray)
+                convertStatInfoIntoMetrics(testName, printOnlyErrors = true, statInfoArray = statInfoArray, metricChildren = metricChildren)
             }
         }
 
@@ -183,70 +166,80 @@ class Stats(
         }
     }
 
-    private fun printTimings(
+    private fun convertStatInfoIntoMetrics(
         prefix: String,
         statInfoArray: Array<StatInfos>,
         printOnlyErrors: Boolean = false,
+        metricChildren: MutableList<Metric>,
         attemptFn: (Int) -> String = { attempt -> "#$attempt" }
     ) {
         for (statInfoIndex in statInfoArray.withIndex()) {
             val attempt = statInfoIndex.index
             val statInfo = statInfoIndex.value ?: continue
-            val n = "$name: $prefix ${attemptFn(attempt)}"
+            val attemptString = attemptFn(attempt)
+            val s = "$prefix $attemptString"
+            val n = "$name: $s"
+            val childrenMetrics = mutableListOf<Metric>()
 
             val t = statInfo[ERROR_KEY] as? Throwable
             if (t != null) {
                 TeamCity.test(n, errors = listOf(t)) {}
             } else if (!printOnlyErrors) {
-                TeamCity.test(n, durationMs = (statInfo[TEST_KEY] as Long).nsToMs) {
+                val durationMs = (statInfo[TEST_KEY] as Long).nsToMs
+                TeamCity.test(n, durationMs = durationMs) {
                     for ((k, v) in statInfo) {
                         if (k == TEST_KEY) continue
-                        TeamCity.statValue("$n $k", v)
                         (v as? Number)?.let {
+                            childrenMetrics.add(Metric(k, v))
                             TeamCity.metadata(n, k, it)
                         }
                     }
                 }
+                metricChildren.add(Metric(attemptString, durationMs, children = childrenMetrics))
             }
         }
     }
 
     fun printWarmUpTimings(
         prefix: String,
-        warmUpStatInfosArray: Array<StatInfos>
+        warmUpStatInfosArray: Array<StatInfos>,
-    ) = printTimings(prefix, warmUpStatInfosArray) { attempt -> "warm-up #$attempt" }
+        metricChildren: MutableList<Metric>
+    ) = convertStatInfoIntoMetrics(prefix, warmUpStatInfosArray, metricChildren = metricChildren) { attempt -> "warm-up #$attempt" }
 
-    fun appendTimings(
+    fun processTimings(
         prefix: String,
-        statInfosArray: Array<StatInfos>
+        statInfosArray: Array<StatInfos>,
+        metricChildren: MutableList<Metric>
     ) {
-        printTimings(prefix, statInfosArray)
+        convertStatInfoIntoMetrics(prefix, statInfosArray, metricChildren = metricChildren)
-        append("$name: $prefix", statInfosArray)
+        calcAndProcessMetrics(prefix, statInfosArray, metricChildren)
     }
 
-    private fun <SV, TV> warmUpPhase(phaseData: PhaseData<SV, TV>) {
+    private fun <SV, TV> warmUpPhase(phaseData: PhaseData<SV, TV>, metricChildren: MutableList<Metric>) {
         val warmUpStatInfosArray = phase(phaseData, WARM_UP, true)
 
         if (phaseData.testName != WARM_UP) {
-            printWarmUpTimings(phaseData.testName, warmUpStatInfosArray)
+            printWarmUpTimings(phaseData.testName, warmUpStatInfosArray, metricChildren)
         } else {
-            printTimings(
+            convertStatInfoIntoMetrics(
                 phaseData.testName,
                 printOnlyErrors = true,
-                statInfoArray = warmUpStatInfosArray
+                statInfoArray = warmUpStatInfosArray,
+                metricChildren = metricChildren
             ) { attempt -> "warm-up #$attempt" }
         }
 
         warmUpStatInfosArray.filterNotNull().map { it[ERROR_KEY] as? Throwable }.firstOrNull()?.let { throw it }
     }
 
-    private fun <SV, TV> mainPhase(phaseData: PhaseData<SV, TV>): Array<StatInfos> {
+    private fun <SV, TV> mainPhase(phaseData: PhaseData<SV, TV>, metricChildren: MutableList<Metric>): Array<StatInfos> {
         val statInfosArray = phase(phaseData, "")
         statInfosArray.filterNotNull().map { it[ERROR_KEY] as? Throwable }.firstOrNull()?.let {
-            printTimings(
+            convertStatInfoIntoMetrics(
                 phaseData.testName,
                 printOnlyErrors = true,
-                statInfoArray = statInfosArray
+                statInfoArray = statInfosArray,
+                metricChildren = metricChildren
             )
             throw it
         }
@@ -265,7 +258,9 @@ class Stats(
                 testData.reset()
                 triggerGC(attempt)
 
-                val setUpMillis = measureTimeMillis { phaseData.setUp(testData) }
+                val setUpMillis = measureTimeMillis {
+                    phaseData.setUp(testData)
+                }
                 val attemptName = "${phaseData.testName} #$attempt"
                 logMessage { "$attemptName setup took $setUpMillis ms" }
 
@@ -315,7 +310,7 @@ class Stats(
         profilerConfig: ProfilerConfig
     ): PhaseProfiler {
         profilerConfig.name = "$testName${if (phaseName.isEmpty()) "" else "-"+phaseName}"
-        profilerConfig.path = pathToResource("profile/${plainname()}")
+        profilerConfig.path = pathToResource("profile/${plainname(name)}")
         val profilerHandler = if (profilerConfig.enabled && !profilerConfig.warmup)
             ProfilerHandler.getInstance(profilerConfig)
         else
@@ -341,13 +336,47 @@ class Stats(
     private fun geomMean(data: List<Long>) = exp(data.fold(0.0, { mul, next -> mul + ln(1.0 * next) }) / data.size)
 
     override fun close() {
+        flush()
+    }
+
+    fun flush() {
+        val children = metrics.toMutableList()
+
+        val properties = mutableMapOf<String, Any>()
+        properties[BENCHMARK] = name
+
+        val simpleDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ")
+        simpleDateFormat.timeZone = TimeZone.getTimeZone("UTC")
+//        properties["build_timestamp"] = simpleDateFormat.format(Date())
+//        properties["build_id"] = 87015694
+//        properties["build_branch"] = "rr/perf/json-output"
+//        properties["agent_name"] = "kotlin-linux-perf-unit879"
+
+        System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE")?.let { teamcityConfig ->
+            val buildProperties = Properties()
+            buildProperties.load(FileInputStream(teamcityConfig))
+
+            properties["build.timestamp"] = simpleDateFormat.format(Date())
+            for ((name, key) in
+            mapOf(
+                "build_id" to "teamcity.build.id",
+                "build_branch" to "teamcity.build.branch",
+                "agent_name" to "agent.name",
+            )) {
+                val property = buildProperties.getProperty(key)
+                properties[name] = if (name == "build_id") property.toLong() else property
+            }
+        }
         if (perfTestRawDataMs.isNotEmpty()) {
             val geomMeanMs = geomMean(perfTestRawDataMs.toList()).toLong()
-            TeamCity.statValue("$name geomMean", geomMeanMs)
+            Metric(GEOM_MEAN, geomMeanMs).writeTeamCityStats(name)
-            append(arrayOf("$name geomMean", geomMeanMs, 0))
+            properties[GEOM_MEAN] = geomMeanMs
         }
-        statsOutput.flush()
+
-        statsOutput.close()
+        val metric = Metric(name, null, children = children, properties = properties)
+
+        metric.writeJson()
+        metrics.writeCSV(name, header)
     }
 
     companion object {
@@ -355,6 +384,8 @@ class Stats(
         const val ERROR_KEY = "error"
 
         const val WARM_UP = "warm-up"
+        const val GEOM_MEAN = "geomMean"
+        const val BENCHMARK = "benchmark"
 
         inline fun runAndMeasure(note: String, block: () -> Unit) {
             val openProjectMillis = measureTimeMillis {
@@ -366,6 +397,15 @@ class Stats(
 
 }
 
+data class Metric(
+    val name: String,
+    val value: Number?,
+    val measurementError: Number? = null,
+    val childrenName: String = "metrics",
+    val children: MutableList<Metric> = mutableListOf(),
+    val properties: Map<String, Any>? = null
+)
+
 data class PhaseData<SV, TV>(
     val iterations: Int,
     val testName: String,

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/StatsOutput.kt

@@ -0,0 +1,113 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.idea.perf
+
+import org.jetbrains.kotlin.idea.perf.util.TeamCity
+import org.jetbrains.kotlin.idea.testFramework.suggestOsNeutralFileName
+import java.io.BufferedWriter
+import java.io.File
+
+internal fun List<Metric>.writeCSV(name: String, header: Array<String>) {
+    fun Metric.append(prefix: String, output: BufferedWriter) {
+        val s = "$prefix ${this.name}".trim()
+        output.appendLine("$s,${value ?: ""},")
+        children.forEach {
+            it.append(s, output)
+        }
+    }
+
+    val statsFile = statsFile(name, "csv")
+    statsFile.bufferedWriter().use { output ->
+        output.appendLine(header.joinToString())
+        forEach { it.append("$name:", output) }
+        output.flush()
+    }
+}
+
+internal fun Metric.writeTeamCityStats(name: String, rawMeasurementName: String = "raw_metrics", rawMetrics: Boolean = false) {
+    fun Metric.append(prefix: String, depth: Int) {
+        val s = if (this.name.isEmpty()) {
+            prefix
+        } else {
+            if (depth == 0 && this.name != Stats.GEOM_MEAN) "$prefix: ${this.name}" else "$prefix ${this.name}"
+        }.trim()
+        value?.let {
+            TeamCity.statValue(s, it)
+        }
+        for (childIndex in children.withIndex()) {
+            if (!rawMetrics && childrenName == rawMeasurementName && childIndex.index > 0) break
+            childIndex.value.append(s, depth + 1)
+        }
+    }
+
+    append(name, 0)
+}
+
+internal fun Metric.writeJson() {
+    val statsFile = statsFile(name, "json")
+    statsFile.bufferedWriter().use { output ->
+        output.appendLine(toJson(""))
+        output.flush()
+    }
+}
+
+private fun statsFile(name: String, extension: String) =
+    File(pathToResource("stats${statFilePrefix(name)}.$extension")).absoluteFile
+
+private fun List<Metric>.toJson(prefix: String) = joinToString(separator = ",", prefix = "[", postfix = "]") { it.toJson(prefix) }
+
+private fun String.jsonValue() = replace("\"", "\\\"")
+
+private fun Metric.toJson(prefix: String): String =
+    buildString {
+        append("{")
+        val nam = if (name.isEmpty()) "_value" else name
+        val nm = nam.jsonValue()
+        val s = if (name == Stats.GEOM_MEAN) {
+            "${prefix.substring(0, prefix.length - 1)} $name".trim()
+        } else {
+            "$prefix $name${if (prefix.isEmpty()) ":" else ""}".trim()
+        }
+
+        var commaRequired = false
+        if (value != null) {
+            append("\"metric_name\":\"").append(nm).append("\"")
+            append(",\"metric_value\":").append(value)
+            measurementError?.let {
+                append(",\"metric_error\":").append(it)
+            }
+            append(",\"legacy_name\":").append("\"${s.jsonValue()}\"")
+            commaRequired = true
+        }
+
+        if (properties != null) {
+            for ((k, v) in properties) {
+                if (commaRequired) append(",")
+                append("\"").append(k).append("\":")
+                if (v is Number) {
+                    append(v)
+                } else {
+                    append("\"").append(v).append("\"")
+                }
+                commaRequired = true
+            }
+        }
+
+        if (children.isNotEmpty()) {
+            val cnm = childrenName.jsonValue()
+            if (commaRequired) append(",")
+            append("\"").append(cnm).append("\"")
+            append(":").append(children.toJson(s))
+        }
+        append("}")
+    }
+
+
+internal fun pathToResource(resource: String) = "build/$resource"
+
+internal fun statFilePrefix(name: String) = if (name.isNotEmpty()) "-${plainname(name)}" else ""
+
+internal fun plainname(name: String) = suggestOsNeutralFileName(name)

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/WholeProjectPerformanceTest.kt

@@ -101,7 +101,7 @@ abstract class WholeProjectPerformanceTest : DaemonAnalyzerTestCase(), WholeProj
             fun appendInspectionResult(file: String, id: String, nanoTime: Long) {
                 totals.merge(id, nanoTime) { a, b -> a + b }
 
-                perfStats.append(file, id, nanoTime)
+                //perfStats.append(file, id, nanoTime)
             }
 
             TeamCity.suite("TotalPerFile") {

idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/logging.kt

@@ -63,7 +63,7 @@ object TeamCity {
     }
 
     inline fun testStarted(testName: String) {
-        message { "testStarted name='$testName'" }
+        message { "testStarted name='$testName' captureStandardOutput='true'" }
     }
 
     inline fun metadata(testName: String, name: String, value: Number) {