azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add json output for IDE performance tests

Browse Source
This commit is contained in:
Vladimir Dolzhenko
2020-09-13 00:20:18 +02:00
parent 72dea05854
commit 51d405e950
13 changed files with 299 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceAddImportTest.kt
-6
View File
@@ -22,12 +22,6 @@ abstract class AbstractPerformanceAddImportTest : AbstractPerformanceImportTest(
@JvmStatic
val stats: Stats = Stats("add-import")
init {
// there is no @AfterClass for junit3.8
Runtime.getRuntime().addShutdownHook(Thread(Runnable { stats.close() }))
}
}
override fun stats(): Stats = stats
idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionHandlerTests.kt
+6 -6
View File
@@ -9,6 +9,8 @@ import com.intellij.application.options.CodeStyle
import com.intellij.codeInsight.completion.CompletionType
import com.intellij.openapi.application.runWriteAction
import com.intellij.openapi.util.io.FileUtil
import com.intellij.testFramework.RunAll
import com.intellij.util.ThrowableRunnable
import org.jetbrains.kotlin.idea.completion.test.ExpectedCompletionUtils
import org.jetbrains.kotlin.idea.completion.test.configureWithExtraFile
import org.jetbrains.kotlin.idea.completion.test.handlers.AbstractCompletionHandlerTest.Companion.CODE_STYLE_SETTING_PREFIX
@@ -36,11 +38,6 @@ abstract class AbstractPerformanceCompletionHandlerTests(
companion object {
@JvmStatic
val statsMap: MutableMap<String, Stats> = mutableMapOf()
init {
// there is no @AfterClass for junit3.8
Runtime.getRuntime().addShutdownHook(Thread(Runnable { statsMap.values.forEach { it.close() } }))
}
}
private fun stats(): Stats {
@@ -52,7 +49,10 @@ abstract class AbstractPerformanceCompletionHandlerTests(
override fun tearDown() {
commitAllDocuments()
super.tearDown()
RunAll(
ThrowableRunnable { super.tearDown() },
ThrowableRunnable { statsMap.values.forEach(Stats::flush) }
).run()
}
protected open fun doPerfTest(unused: String) {
idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionIncrementalResolveTest.kt
+6 -6
View File
@@ -9,6 +9,8 @@ import com.intellij.codeInsight.completion.CompletionType
import com.intellij.codeInsight.lookup.LookupElement
import com.intellij.openapi.application.runWriteAction
import com.intellij.openapi.util.text.StringUtil
import com.intellij.testFramework.RunAll
import com.intellij.util.ThrowableRunnable
import org.jetbrains.kotlin.idea.KotlinFileType
import org.jetbrains.kotlin.idea.completion.CompletionBindingContextProvider
import org.jetbrains.kotlin.idea.perf.Stats.Companion.WARM_UP
@@ -33,11 +35,6 @@ abstract class AbstractPerformanceCompletionIncrementalResolveTest : KotlinLight
@JvmStatic
val stats: Stats = Stats("completion-incremental")
init {
// there is no @AfterClass for junit3.8
Runtime.getRuntime().addShutdownHook(Thread(Runnable { stats.close() }))
}
}
override fun getProjectDescriptor() = KotlinWithJdkAndRuntimeLightProjectDescriptor.INSTANCE
@@ -53,7 +50,10 @@ abstract class AbstractPerformanceCompletionIncrementalResolveTest : KotlinLight
override fun tearDown() {
commitAllDocuments()
super.tearDown()
RunAll(
ThrowableRunnable { super.tearDown() },
ThrowableRunnable { stats.flush() }
).run()
}
private fun doWarmUpPerfTest() {
idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceHighlightingTest.kt
+6 -5
View File
@@ -7,7 +7,9 @@ package org.jetbrains.kotlin.idea.perf
import com.intellij.codeInsight.daemon.impl.HighlightInfo
import com.intellij.openapi.application.runWriteAction
import com.intellij.testFramework.RunAll
import com.intellij.testFramework.fixtures.impl.CodeInsightTestFixtureImpl.ensureIndexesUpToDate
import com.intellij.util.ThrowableRunnable
import org.jetbrains.kotlin.idea.KotlinFileType
import org.jetbrains.kotlin.idea.perf.Stats.Companion.WARM_UP
import org.jetbrains.kotlin.idea.test.KotlinLightCodeInsightFixtureTestCase
@@ -25,10 +27,6 @@ abstract class AbstractPerformanceHighlightingTest : KotlinLightCodeInsightFixtu
@JvmStatic
val stats: Stats = Stats("highlight")
init {
// there is no @AfterClass for junit3.8
Runtime.getRuntime().addShutdownHook(Thread(Runnable { stats.close() }))
}
}
override fun setUp() {
@@ -42,7 +40,10 @@ abstract class AbstractPerformanceHighlightingTest : KotlinLightCodeInsightFixtu
override fun tearDown() {
commitAllDocuments()
super.tearDown()
RunAll(
ThrowableRunnable { super.tearDown() },
ThrowableRunnable { stats.flush() }
).run()
}
private fun doWarmUpPerfTest() {
idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceImportTest.kt
+9
View File
@@ -9,6 +9,8 @@ import com.intellij.application.options.CodeStyle
import com.intellij.openapi.application.runWriteAction
import com.intellij.psi.PsiDocumentManager
import com.intellij.testFramework.LightProjectDescriptor
import com.intellij.testFramework.RunAll
import com.intellij.util.ThrowableRunnable
import junit.framework.TestCase
import org.jetbrains.kotlin.descriptors.ClassDescriptor
import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
@@ -30,6 +32,13 @@ abstract class AbstractPerformanceImportTest : KotlinLightCodeInsightFixtureTest
protected abstract fun stats(): Stats
override fun tearDown() {
RunAll(
ThrowableRunnable { super.tearDown() },
ThrowableRunnable { stats().flush() }
).run()
}
protected fun doPerfTest(unused: String) {
val testName = getTestName(false)
idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceJavaToKotlinCopyPasteConversionTest.kt
+10 -5
View File
@@ -9,6 +9,8 @@ import com.intellij.ide.highlighter.JavaFileType
import com.intellij.openapi.actionSystem.IdeActions
import com.intellij.openapi.application.runWriteAction
import com.intellij.psi.PsiDocumentManager
import com.intellij.testFramework.RunAll
import com.intellij.util.ThrowableRunnable
import org.jetbrains.kotlin.idea.KotlinFileType
import org.jetbrains.kotlin.idea.configuration.ExperimentalFeatures
import org.jetbrains.kotlin.idea.conversion.copy.AbstractJavaToKotlinCopyPasteConversionTest
@@ -28,11 +30,6 @@ abstract class AbstractPerformanceJavaToKotlinCopyPasteConversionTest(private va
val warmedUp: Array<Boolean> = arrayOf(false, false)
val stats: Array<Stats> = arrayOf(Stats("old j2k"), Stats("new j2k"))
init {
// there is no @AfterClass for junit3.8
Runtime.getRuntime().addShutdownHook(Thread(Runnable { stats.forEach { it.close() } }))
}
}
override fun setUp() {
@@ -47,6 +44,14 @@ abstract class AbstractPerformanceJavaToKotlinCopyPasteConversionTest(private va
}
}
override fun tearDown() {
RunAll(
ThrowableRunnable { super.tearDown() },
ThrowableRunnable { stats.forEach(Stats::flush) }
).run()
}
private fun doWarmUpPerfTest() {
performanceTest<Unit, Unit> {
name(WARM_UP)
idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceLiteralKotlinToKotlinCopyPasteTest.kt
+9 -5
View File
@@ -9,6 +9,8 @@ import com.intellij.openapi.actionSystem.IdeActions
import com.intellij.openapi.application.runWriteAction
import com.intellij.openapi.fileEditor.ex.FileEditorManagerEx
import com.intellij.psi.PsiFile
import com.intellij.testFramework.RunAll
import com.intellij.util.ThrowableRunnable
import org.jetbrains.kotlin.idea.AbstractCopyPasteTest
import org.jetbrains.kotlin.idea.perf.Stats.Companion.WARM_UP
import org.jetbrains.kotlin.idea.test.KotlinWithJdkAndRuntimeLightProjectDescriptor
@@ -27,11 +29,6 @@ abstract class AbstractPerformanceLiteralKotlinToKotlinCopyPasteTest : AbstractC
@JvmStatic
val stats: Stats = Stats("Literal-k2k-CopyPaste")
init {
// there is no @AfterClass for junit3.8
Runtime.getRuntime().addShutdownHook(Thread(Runnable { stats.close() }))
}
}
override fun getProjectDescriptor() = KotlinWithJdkAndRuntimeLightProjectDescriptor.INSTANCE
@@ -45,6 +42,13 @@ abstract class AbstractPerformanceLiteralKotlinToKotlinCopyPasteTest : AbstractC
}
}
override fun tearDown() {
RunAll(
ThrowableRunnable { super.tearDown() },
ThrowableRunnable { stats.flush() }
).run()
}
private fun doWarmUpPerfTest() {
val fileEditorManager = FileEditorManagerEx.getInstance(project)
performanceTest<Pair<PsiFile, PsiFile>, Unit> {
idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceTypingIndentationTest.kt
+9 -2
View File
@@ -7,10 +7,10 @@ package org.jetbrains.kotlin.idea.perf
import com.intellij.application.options.CodeStyle
import com.intellij.openapi.actionSystem.IdeActions
import com.intellij.openapi.application.impl.NonBlockingReadActionImpl
import com.intellij.openapi.util.io.FileUtil
import com.intellij.testFramework.EditorTestUtil
import com.intellij.util.ui.UIUtil
import com.intellij.testFramework.RunAll
import com.intellij.util.ThrowableRunnable
import org.jetbrains.kotlin.formatter.FormatSettingsUtil
import org.jetbrains.kotlin.idea.test.KotlinLightCodeInsightFixtureTestCase
import org.jetbrains.kotlin.idea.testFramework.dispatchAllInvocationEvents
@@ -27,6 +27,13 @@ abstract class AbstractPerformanceTypingIndentationTest : KotlinLightCodeInsight
val stats: Stats = Stats("typing-indentation")
}
override fun tearDown() {
RunAll(
ThrowableRunnable { super.tearDown() },
ThrowableRunnable { stats.flush() }
).run()
}
protected fun doPerfTest(unused: String) {
val testName = getTestName(false)
idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerformanceProjectsTest.kt
+16 -8
View File
@@ -9,6 +9,8 @@ import com.intellij.codeInsight.daemon.impl.HighlightInfo
import com.intellij.lang.annotation.AnnotationHolder
import com.intellij.openapi.project.Project
import com.intellij.psi.PsiElement
import com.intellij.testFramework.RunAll
import com.intellij.util.ThrowableRunnable
import org.jetbrains.kotlin.idea.core.script.ScriptConfigurationManager
import org.jetbrains.kotlin.idea.highlighter.KotlinPsiChecker
import org.jetbrains.kotlin.idea.highlighter.KotlinPsiCheckerAndHighlightingUpdater
@@ -35,11 +37,6 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
@JvmStatic
val timer: AtomicLong = AtomicLong()
init {
// there is no @AfterClass for junit3.8
Runtime.getRuntime().addShutdownHook(Thread { hwStats.close() })
}
fun resetTimestamp() {
timer.set(0)
}
@@ -54,6 +51,13 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
warmUp.warmUp(this)
}
override fun tearDown() {
RunAll(
ThrowableRunnable { super.tearDown() },
ThrowableRunnable { hwStats.flush() }
).run()
}
fun testHelloWorldProject() {
suite("Hello world project") {
myProject = perfOpenProject(stats = hwStats) {
@@ -337,14 +341,18 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
profilerConfig.enabled = true
}
val metricChildren = mutableListOf<Metric>()
extraStats.printWarmUpTimings(
"annotator",
extraTimingsNs.take(warmUpIterations).toTypedArray()
extraTimingsNs.take(warmUpIterations).toTypedArray(),
metricChildren
)
extraStats.appendTimings(
extraStats.processTimings(
"annotator",
extraTimingsNs.drop(warmUpIterations).toTypedArray()
extraTimingsNs.drop(warmUpIterations).toTypedArray(),
metricChildren
)
}
}
idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt
+113 -73
View File
@@ -9,12 +9,14 @@ import org.jetbrains.kotlin.idea.perf.WholeProjectPerformanceTest.Companion.nsTo
import org.jetbrains.kotlin.idea.perf.profilers.*
import org.jetbrains.kotlin.idea.perf.util.TeamCity
import org.jetbrains.kotlin.idea.perf.util.logMessage
import org.jetbrains.kotlin.idea.testFramework.suggestOsNeutralFileName
import org.jetbrains.kotlin.util.PerformanceCounter
import java.io.*
import kotlin.system.measureNanoTime
import java.lang.ref.WeakReference
import java.text.SimpleDateFormat
import java.util.*
import kotlin.collections.HashMap
import kotlin.math.*
import kotlin.system.measureNanoTime
import kotlin.system.measureTimeMillis
import kotlin.test.assertEquals
@@ -25,44 +27,36 @@ class Stats(
private val profilerConfig: ProfilerConfig = ProfilerConfig(),
private val header: Array<String> = arrayOf("Name", "ValueMS", "StdDev"),
private val acceptanceStabilityLevel: Int = 25
) : Closeable {
) : AutoCloseable {
private val perfTestRawDataMs = mutableListOf<Long>()
private val statsFile: File = File(pathToResource("stats${statFilePrefix()}.csv")).absoluteFile
private val statsOutput: BufferedWriter
private val metrics = mutableListOf<Metric>()
init {
statsOutput = statsFile.bufferedWriter()
statsOutput.appendLine(header.joinToString())
PerformanceCounter.setTimeCounterEnabled(true)
}
private fun statFilePrefix() = if (name.isNotEmpty()) "-${plainname()}" else ""
private fun plainname() = suggestOsNeutralFileName(name)
private fun pathToResource(resource: String) = "build/$resource"
private fun append(id: String, statInfosArray: Array<StatInfos>) {
private fun calcAndProcessMetrics(id: String, statInfosArray: Array<StatInfos>, rawMetricChildren: MutableList<Metric>) {
val timingsMs = toTimingsMs(statInfosArray)
val calcMean = calcMean(timingsMs)
for (v in listOf(
Triple("mean", "", calcMean.mean.toLong()),
Triple("stdDev", " stdDev", calcMean.stdDev.toLong()),
Triple("geomMean", " geomMean", calcMean.geomMean.toLong())
)) {
val n = "$id : ${v.first}"
val metricChildren = mutableListOf<Metric>()
val metric = Metric(id, calcMean.mean.toLong(), measurementError = calcMean.stdDev.toLong(), children = metricChildren)
metrics.add(metric)
TeamCity.test(n, durationMs = v.third, includeStats = false) {
TeamCity.statValue("$id${v.second}", v.third)
}
}
metricChildren.add(
Metric(
"", calcMean.mean.toLong(),
measurementError = calcMean.stdDev.toLong(),
childrenName = "raw_metrics", children = rawMetricChildren
)
)
metricChildren.add(Metric("mean", calcMean.mean.toLong()))
// keep geomMean for bwc
metricChildren.add(Metric(GEOM_MEAN, calcMean.geomMean.toLong()))
metricChildren.add(Metric("std_dev", calcMean.stdDev.toLong()))
statInfosArray.filterNotNull()
.map { it.keys }
@@ -76,14 +70,15 @@ class Stats(
val mean = statInfoMean.mean.toLong()
val shortName = if (perfCounterName.endsWith(": time")) n.removeSuffix(": time") else null
val metricShortName = if (perfCounterName.endsWith(": time")) perfCounterName.removeSuffix(": time") else perfCounterName
TeamCity.test(shortName, durationMs = mean) {
TeamCity.statValue(n, mean)
}
metricChildren.add(Metric(": $metricShortName", mean))
TeamCity.test(shortName, durationMs = mean) {}
}
perfTestRawDataMs.addAll(timingsMs.toList())
append(arrayOf(id, calcMean.mean, calcMean.stdDev))
metric.writeTeamCityStats(name)
}
private fun toTimingsMs(statInfosArray: Array<StatInfos>) =
@@ -107,19 +102,6 @@ class Stats(
data class Mean(val mean: Double, val stdDev: Double, val geomMean: Double)
private fun append(values: Array<Any>) {
require(values.size == header.size) { "Expected ${header.size} values, actual ${values.size} values" }
with(statsOutput) {
appendLine(values.joinToString { it.toString() })
flush()
}
}
fun append(file: String, id: String, nanoTime: Long) {
val ms = nanoTime.nsToMs
append(arrayOf(file, id, ms))
}
fun <SV, TV> perfTest(
testName: String,
warmUpIterations: Int = 5,
@@ -145,13 +127,12 @@ class Stats(
tearDown = tearDown
)
val block = {
warmUpPhase(warmPhaseData)
val statInfoArray = mainPhase(mainPhaseData)
val metricChildren = mutableListOf<Metric>()
warmUpPhase(warmPhaseData, metricChildren)
val statInfoArray = mainPhase(mainPhaseData, metricChildren)
assertEquals(iterations, statInfoArray.size)
if (testName != WARM_UP) {
appendTimings(testName, statInfoArray)
// do not estimate stability for warm-up
if (!testName.contains(WARM_UP)) {
val calcMean = calcMean(statInfoArray)
@@ -168,11 +149,13 @@ class Stats(
}
TeamCity.test(stabilityName, errorDetails = error, includeStats = false) {
TeamCity.statValue(stabilityName, stabilityPercentage)
metricChildren.add(Metric("stability", stabilityPercentage))
}
}
processTimings(testName, statInfoArray, metricChildren)
} else {
printTimings(testName, printOnlyErrors = true, statInfoArray = statInfoArray)
convertStatInfoIntoMetrics(testName, printOnlyErrors = true, statInfoArray = statInfoArray, metricChildren = metricChildren)
}
}
@@ -183,70 +166,80 @@ class Stats(
}
}
private fun printTimings(
private fun convertStatInfoIntoMetrics(
prefix: String,
statInfoArray: Array<StatInfos>,
printOnlyErrors: Boolean = false,
metricChildren: MutableList<Metric>,
attemptFn: (Int) -> String = { attempt -> "#$attempt" }
) {
for (statInfoIndex in statInfoArray.withIndex()) {
val attempt = statInfoIndex.index
val statInfo = statInfoIndex.value ?: continue
val n = "$name: $prefix ${attemptFn(attempt)}"
val attemptString = attemptFn(attempt)
val s = "$prefix $attemptString"
val n = "$name: $s"
val childrenMetrics = mutableListOf<Metric>()
val t = statInfo[ERROR_KEY] as? Throwable
if (t != null) {
TeamCity.test(n, errors = listOf(t)) {}
} else if (!printOnlyErrors) {
TeamCity.test(n, durationMs = (statInfo[TEST_KEY] as Long).nsToMs) {
val durationMs = (statInfo[TEST_KEY] as Long).nsToMs
TeamCity.test(n, durationMs = durationMs) {
for ((k, v) in statInfo) {
if (k == TEST_KEY) continue
TeamCity.statValue("$n $k", v)
(v as? Number)?.let {
childrenMetrics.add(Metric(k, v))
TeamCity.metadata(n, k, it)
}
}
}
metricChildren.add(Metric(attemptString, durationMs, children = childrenMetrics))
}
}
}
fun printWarmUpTimings(
prefix: String,
warmUpStatInfosArray: Array<StatInfos>
) = printTimings(prefix, warmUpStatInfosArray) { attempt -> "warm-up #$attempt" }
warmUpStatInfosArray: Array<StatInfos>,
metricChildren: MutableList<Metric>
) = convertStatInfoIntoMetrics(prefix, warmUpStatInfosArray, metricChildren = metricChildren) { attempt -> "warm-up #$attempt" }
fun appendTimings(
fun processTimings(
prefix: String,
statInfosArray: Array<StatInfos>
statInfosArray: Array<StatInfos>,
metricChildren: MutableList<Metric>
) {
printTimings(prefix, statInfosArray)
append("$name: $prefix", statInfosArray)
convertStatInfoIntoMetrics(prefix, statInfosArray, metricChildren = metricChildren)
calcAndProcessMetrics(prefix, statInfosArray, metricChildren)
}
private fun <SV, TV> warmUpPhase(phaseData: PhaseData<SV, TV>) {
private fun <SV, TV> warmUpPhase(phaseData: PhaseData<SV, TV>, metricChildren: MutableList<Metric>) {
val warmUpStatInfosArray = phase(phaseData, WARM_UP, true)
if (phaseData.testName != WARM_UP) {
printWarmUpTimings(phaseData.testName, warmUpStatInfosArray)
printWarmUpTimings(phaseData.testName, warmUpStatInfosArray, metricChildren)
} else {
printTimings(
convertStatInfoIntoMetrics(
phaseData.testName,
printOnlyErrors = true,
statInfoArray = warmUpStatInfosArray
statInfoArray = warmUpStatInfosArray,
metricChildren = metricChildren
) { attempt -> "warm-up #$attempt" }
}
warmUpStatInfosArray.filterNotNull().map { it[ERROR_KEY] as? Throwable }.firstOrNull()?.let { throw it }
}
private fun <SV, TV> mainPhase(phaseData: PhaseData<SV, TV>): Array<StatInfos> {
private fun <SV, TV> mainPhase(phaseData: PhaseData<SV, TV>, metricChildren: MutableList<Metric>): Array<StatInfos> {
val statInfosArray = phase(phaseData, "")
statInfosArray.filterNotNull().map { it[ERROR_KEY] as? Throwable }.firstOrNull()?.let {
printTimings(
convertStatInfoIntoMetrics(
phaseData.testName,
printOnlyErrors = true,
statInfoArray = statInfosArray
statInfoArray = statInfosArray,
metricChildren = metricChildren
)
throw it
}
@@ -265,7 +258,9 @@ class Stats(
testData.reset()
triggerGC(attempt)
val setUpMillis = measureTimeMillis { phaseData.setUp(testData) }
val setUpMillis = measureTimeMillis {
phaseData.setUp(testData)
}
val attemptName = "${phaseData.testName} #$attempt"
logMessage { "$attemptName setup took $setUpMillis ms" }
@@ -315,7 +310,7 @@ class Stats(
profilerConfig: ProfilerConfig
): PhaseProfiler {
profilerConfig.name = "$testName${if (phaseName.isEmpty()) "" else "-"+phaseName}"
profilerConfig.path = pathToResource("profile/${plainname()}")
profilerConfig.path = pathToResource("profile/${plainname(name)}")
val profilerHandler = if (profilerConfig.enabled && !profilerConfig.warmup)
ProfilerHandler.getInstance(profilerConfig)
else
@@ -341,13 +336,47 @@ class Stats(
private fun geomMean(data: List<Long>) = exp(data.fold(0.0, { mul, next -> mul + ln(1.0 * next) }) / data.size)
override fun close() {
flush()
}
fun flush() {
val children = metrics.toMutableList()
val properties = mutableMapOf<String, Any>()
properties[BENCHMARK] = name
val simpleDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ")
simpleDateFormat.timeZone = TimeZone.getTimeZone("UTC")
// properties["build_timestamp"] = simpleDateFormat.format(Date())
// properties["build_id"] = 87015694
// properties["build_branch"] = "rr/perf/json-output"
// properties["agent_name"] = "kotlin-linux-perf-unit879"
System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE")?.let { teamcityConfig ->
val buildProperties = Properties()
buildProperties.load(FileInputStream(teamcityConfig))
properties["build.timestamp"] = simpleDateFormat.format(Date())
for ((name, key) in
mapOf(
"build_id" to "teamcity.build.id",
"build_branch" to "teamcity.build.branch",
"agent_name" to "agent.name",
)) {
val property = buildProperties.getProperty(key)
properties[name] = if (name == "build_id") property.toLong() else property
}
}
if (perfTestRawDataMs.isNotEmpty()) {
val geomMeanMs = geomMean(perfTestRawDataMs.toList()).toLong()
TeamCity.statValue("$name geomMean", geomMeanMs)
append(arrayOf("$name geomMean", geomMeanMs, 0))
Metric(GEOM_MEAN, geomMeanMs).writeTeamCityStats(name)
properties[GEOM_MEAN] = geomMeanMs
}
statsOutput.flush()
statsOutput.close()
val metric = Metric(name, null, children = children, properties = properties)
metric.writeJson()
metrics.writeCSV(name, header)
}
companion object {
@@ -355,6 +384,8 @@ class Stats(
const val ERROR_KEY = "error"
const val WARM_UP = "warm-up"
const val GEOM_MEAN = "geomMean"
const val BENCHMARK = "benchmark"
inline fun runAndMeasure(note: String, block: () -> Unit) {
val openProjectMillis = measureTimeMillis {
@@ -366,6 +397,15 @@ class Stats(
}
data class Metric(
val name: String,
val value: Number?,
val measurementError: Number? = null,
val childrenName: String = "metrics",
val children: MutableList<Metric> = mutableListOf(),
val properties: Map<String, Any>? = null
)
data class PhaseData<SV, TV>(
val iterations: Int,
val testName: String,
idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/StatsOutput.kt
+113
View File
@@ -0,0 +1,113 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.idea.perf
import org.jetbrains.kotlin.idea.perf.util.TeamCity
import org.jetbrains.kotlin.idea.testFramework.suggestOsNeutralFileName
import java.io.BufferedWriter
import java.io.File
internal fun List<Metric>.writeCSV(name: String, header: Array<String>) {
fun Metric.append(prefix: String, output: BufferedWriter) {
val s = "$prefix ${this.name}".trim()
output.appendLine("$s,${value ?: ""},")
children.forEach {
it.append(s, output)
}
}
val statsFile = statsFile(name, "csv")
statsFile.bufferedWriter().use { output ->
output.appendLine(header.joinToString())
forEach { it.append("$name:", output) }
output.flush()
}
}
internal fun Metric.writeTeamCityStats(name: String, rawMeasurementName: String = "raw_metrics", rawMetrics: Boolean = false) {
fun Metric.append(prefix: String, depth: Int) {
val s = if (this.name.isEmpty()) {
prefix
} else {
if (depth == 0 && this.name != Stats.GEOM_MEAN) "$prefix: ${this.name}" else "$prefix ${this.name}"
}.trim()
value?.let {
TeamCity.statValue(s, it)
}
for (childIndex in children.withIndex()) {
if (!rawMetrics && childrenName == rawMeasurementName && childIndex.index > 0) break
childIndex.value.append(s, depth + 1)
}
}
append(name, 0)
}
internal fun Metric.writeJson() {
val statsFile = statsFile(name, "json")
statsFile.bufferedWriter().use { output ->
output.appendLine(toJson(""))
output.flush()
}
}
private fun statsFile(name: String, extension: String) =
File(pathToResource("stats${statFilePrefix(name)}.$extension")).absoluteFile
private fun List<Metric>.toJson(prefix: String) = joinToString(separator = ",", prefix = "[", postfix = "]") { it.toJson(prefix) }
private fun String.jsonValue() = replace("\"", "\\\"")
private fun Metric.toJson(prefix: String): String =
buildString {
append("{")
val nam = if (name.isEmpty()) "_value" else name
val nm = nam.jsonValue()
val s = if (name == Stats.GEOM_MEAN) {
"${prefix.substring(0, prefix.length - 1)} $name".trim()
} else {
"$prefix $name${if (prefix.isEmpty()) ":" else ""}".trim()
}
var commaRequired = false
if (value != null) {
append("\"metric_name\":\"").append(nm).append("\"")
append(",\"metric_value\":").append(value)
measurementError?.let {
append(",\"metric_error\":").append(it)
}
append(",\"legacy_name\":").append("\"${s.jsonValue()}\"")
commaRequired = true
}
if (properties != null) {
for ((k, v) in properties) {
if (commaRequired) append(",")
append("\"").append(k).append("\":")
if (v is Number) {
append(v)
} else {
append("\"").append(v).append("\"")
}
commaRequired = true
}
}
if (children.isNotEmpty()) {
val cnm = childrenName.jsonValue()
if (commaRequired) append(",")
append("\"").append(cnm).append("\"")
append(":").append(children.toJson(s))
}
append("}")
}
internal fun pathToResource(resource: String) = "build/$resource"
internal fun statFilePrefix(name: String) = if (name.isNotEmpty()) "-${plainname(name)}" else ""
internal fun plainname(name: String) = suggestOsNeutralFileName(name)
idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/WholeProjectPerformanceTest.kt
+1 -1
View File
@@ -101,7 +101,7 @@ abstract class WholeProjectPerformanceTest : DaemonAnalyzerTestCase(), WholeProj
fun appendInspectionResult(file: String, id: String, nanoTime: Long) {
totals.merge(id, nanoTime) { a, b -> a + b }
perfStats.append(file, id, nanoTime)
//perfStats.append(file, id, nanoTime)
}
TeamCity.suite("TotalPerFile") {
idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/logging.kt
+1 -1
View File
@@ -63,7 +63,7 @@ object TeamCity {
}
inline fun testStarted(testName: String) {
message { "testStarted name='$testName'" }
message { "testStarted name='$testName' captureStandardOutput='true'" }
}
inline fun metadata(testName: String, name: String, value: Number) {