From 5ca4c2ce1ad1ea4290297be79ffb46aaaa83f57a Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Tue, 13 Oct 2020 15:16:23 +0200 Subject: [PATCH] Use camelCase propNames in ide perf tests json stats; report hasError --- .../org/jetbrains/kotlin/idea/perf/Stats.kt | 91 +++++++++++-------- .../jetbrains/kotlin/idea/perf/StatsOutput.kt | 23 +++-- 2 files changed, 70 insertions(+), 44 deletions(-) diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt index 98dd5de21fb..10729957080 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt @@ -43,20 +43,25 @@ class Stats( val calcMean = calcMean(timingsMs) val metricChildren = mutableListOf() - val metric = Metric(id, calcMean.mean.toLong(), measurementError = calcMean.stdDev.toLong(), children = metricChildren) + val hasError = rawMetricChildren.any { it.hasError } + val metric = Metric( + id, value = calcMean.mean.toLong(), measurementError = calcMean.stdDev.toLong(), + hasError = hasError, children = metricChildren + ) metrics.add(metric) metricChildren.add( Metric( "", calcMean.mean.toLong(), + hasError = hasError, measurementError = calcMean.stdDev.toLong(), - childrenName = "raw_metrics", children = rawMetricChildren + childrenName = "rawMetrics", children = rawMetricChildren ) ) metricChildren.add(Metric("mean", calcMean.mean.toLong())) // keep geomMean for bwc metricChildren.add(Metric(GEOM_MEAN, calcMean.geomMean.toLong())) - metricChildren.add(Metric("std_dev", calcMean.stdDev.toLong())) + metricChildren.add(Metric("stdDev", calcMean.stdDev.toLong())) statInfosArray.filterNotNull() .map { it.keys } @@ -128,34 +133,43 @@ class Stats( ) val block = { val metricChildren = mutableListOf() - warmUpPhase(warmPhaseData, metricChildren) - val statInfoArray = mainPhase(mainPhaseData, metricChildren) + try { + warmUpPhase(warmPhaseData, metricChildren) + val statInfoArray = mainPhase(mainPhaseData, metricChildren) - assertEquals(iterations, statInfoArray.size) - if (testName != WARM_UP) { - // do not estimate stability for warm-up - if (!testName.contains(WARM_UP)) { - val calcMean = calcMean(statInfoArray) - val stabilityPercentage = round(calcMean.stdDev * 100.0 / calcMean.mean).toInt() - logMessage { "$testName stability is $stabilityPercentage %" } - val stabilityName = "$name: $testName stability" + assertEquals(iterations, statInfoArray.size) + if (testName != WARM_UP) { + // do not estimate stability for warm-up + if (!testName.contains(WARM_UP)) { + val calcMean = calcMean(statInfoArray) + val stabilityPercentage = round(calcMean.stdDev * 100.0 / calcMean.mean).toInt() + logMessage { "$testName stability is $stabilityPercentage %" } + val stabilityName = "$name: $testName stability" - val stable = stabilityPercentage <= acceptanceStabilityLevel + val stable = stabilityPercentage <= acceptanceStabilityLevel - val error = if (stable or !checkStability) { - null - } else { - "$testName stability is $stabilityPercentage %, above accepted level of $acceptanceStabilityLevel %" + val error = if (stable or !checkStability) { + null + } else { + "$testName stability is $stabilityPercentage %, above accepted level of $acceptanceStabilityLevel %" + } + + TeamCity.test(stabilityName, errorDetails = error, includeStats = false) { + metricChildren.add(Metric("stability", stabilityPercentage)) + } } - TeamCity.test(stabilityName, errorDetails = error, includeStats = false) { - metricChildren.add(Metric("stability", stabilityPercentage)) - } + processTimings(testName, statInfoArray, metricChildren) + } else { + convertStatInfoIntoMetrics( + testName, + printOnlyErrors = true, + statInfoArray = statInfoArray, + metricChildren = metricChildren + ) } - - processTimings(testName, statInfoArray, metricChildren) - } else { - convertStatInfoIntoMetrics(testName, printOnlyErrors = true, statInfoArray = statInfoArray, metricChildren = metricChildren) + } catch (e: Exception) { + processTimings(testName, emptyArray(), metricChildren) } } @@ -184,6 +198,7 @@ class Stats( val t = statInfo[ERROR_KEY] as? Throwable if (t != null) { TeamCity.test(n, errors = listOf(t)) {} + metricChildren.add(Metric(attemptString, value = null, hasError = true)) } else if (!printOnlyErrors) { val durationMs = (statInfo[TEST_KEY] as Long).nsToMs TeamCity.test(n, durationMs = durationMs, includeStats = false) { @@ -211,8 +226,11 @@ class Stats( statInfosArray: Array, metricChildren: MutableList ) { - convertStatInfoIntoMetrics(prefix, statInfosArray, metricChildren = metricChildren) - calcAndProcessMetrics(prefix, statInfosArray, metricChildren) + try { + convertStatInfoIntoMetrics(prefix, statInfosArray, metricChildren = metricChildren) + } finally { + calcAndProcessMetrics(prefix, statInfosArray, metricChildren) + } } private fun warmUpPhase(phaseData: PhaseData, metricChildren: MutableList) { @@ -347,24 +365,24 @@ class Stats( val simpleDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ") simpleDateFormat.timeZone = TimeZone.getTimeZone("UTC") -// properties["build_timestamp"] = simpleDateFormat.format(Date()) -// properties["build_id"] = 87015694 -// properties["build_branch"] = "rr/perf/json-output" -// properties["agent_name"] = "kotlin-linux-perf-unit879" +// properties["buildTimestamp"] = simpleDateFormat.format(Date()) +// properties["buildId"] = 87015694 +// properties["buildBranch"] = "rr/perf/json-output" +// properties["agentName"] = "kotlin-linux-perf-unit879" System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE")?.let { teamcityConfig -> val buildProperties = Properties() buildProperties.load(FileInputStream(teamcityConfig)) - properties["build.timestamp"] = simpleDateFormat.format(Date()) + properties["buildTimestamp"] = simpleDateFormat.format(Date()) for ((name, key) in mapOf( - "build_id" to "teamcity.build.id", - "build_branch" to "teamcity.build.branch", - "agent_name" to "agent.name", + "buildId" to "teamcity.build.id", + "buildBranch" to "teamcity.build.branch", + "agentName" to "agent.name", )) { val property = buildProperties.getProperty(key) - properties[name] = if (name == "build_id") property.toLong() else property + properties[name] = if (name == "buildId") property.toLong() else property } } if (perfTestRawDataMs.isNotEmpty()) { @@ -400,6 +418,7 @@ class Stats( data class Metric( val name: String, val value: Number?, + val hasError: Boolean = false, val measurementError: Number? = null, val childrenName: String = "metrics", val children: MutableList = mutableListOf(), diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/StatsOutput.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/StatsOutput.kt index 0e7bed6e79a..6780db2a6e0 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/StatsOutput.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/StatsOutput.kt @@ -27,7 +27,7 @@ internal fun List.writeCSV(name: String, header: Array) { } } -internal fun Metric.writeTeamCityStats(name: String, rawMeasurementName: String = "raw_metrics", rawMetrics: Boolean = false) { +internal fun Metric.writeTeamCityStats(name: String, rawMeasurementName: String = "rawMetrics", rawMetrics: Boolean = false) { fun Metric.append(prefix: String, depth: Int) { val s = if (this.name.isEmpty()) { prefix @@ -74,17 +74,24 @@ private fun Metric.toJson(prefix: String): String = "$prefix $name${if (prefix.isEmpty()) ":" else ""}".trim() } + var commaRequired = false - if (value != null) { - append("\"metric_name\":\"").append(nm).append("\"") - append(",\"metric_value\":").append(value) - measurementError?.let { - append(",\"metric_error\":").append(it) - } - //append(",\"legacy_name\":").append("\"${s.jsonValue()}\"") + if (value != null || hasError) { + append("\"metricName\":\"").append(nm).append("\"") commaRequired = true } + if (value != null) { + append(",\"metricValue\":").append(value) + measurementError?.let { + append(",\"metricError\":").append(it) + } + //append(",\"legacyName\":").append("\"${s.jsonValue()}\"") + } + if (hasError) { + append(",\"hasError\": $hasError") + } + if (properties != null) { for ((k, v) in properties) { if (commaRequired) append(",")