diff --git a/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/BuildRegister.kt b/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/BuildRegister.kt index a67b2f1c113..4c4761bdd7a 100644 --- a/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/BuildRegister.kt +++ b/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/BuildRegister.kt @@ -26,6 +26,7 @@ open class BuildRegister : DefaultTask() { var onlyBranch: String? = null var bundleSize: Int? = null + var buildNumberSuffix: String? = null var fileWithResult: String = "nativeReport.json" val performanceServer = "https://kotlin-native-perf-summary.labs.jb.gg" @@ -77,7 +78,8 @@ open class BuildRegister : DefaultTask() { append("\"teamCityUser\":\"$teamCityUser\",") append("\"teamCityPassword\":\"$teamCityPassword\",") append("\"fileWithResult\":\"$fileWithResult\",") - append("\"bundleSize\": ${bundleSize?.let { "\"$bundleSize\"" } ?: bundleSize}}") + append("\"bundleSize\": ${bundleSize?.let { "\"$bundleSize\"" } ?: bundleSize},") + append("\"buildNumberSuffix\": ${buildNumberSuffix?.let { "\"$buildNumberSuffix\"" } ?: buildNumberSuffix}}") } if (onlyBranch == null || onlyBranch == branch) { println(sendPostRequest("$performanceServer/register", requestBody)) diff --git a/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/RunKotlinNativeTask.kt b/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/RunKotlinNativeTask.kt index e4adb0ba719..eac1d7619e2 100644 --- a/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/RunKotlinNativeTask.kt +++ b/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/RunKotlinNativeTask.kt @@ -113,7 +113,7 @@ open class RunKotlinNativeTask @Inject constructor(private val linkTask: Task, val filterRegexArgs = filterRegex.splitCommaSeparatedOption("-fr") val regexes = filterRegexArgs.map { it.toRegex() } val benchmarksToRun = if (filterArgs.isNotEmpty() || regexes.isNotEmpty()) { - benchmarks.filter { benchmark -> benchmark in filterArgs || regexes.any { it.matches(benchmark) } } + benchmarks.filter { benchmark -> benchmark in filterArgs || regexes.any { it.matches(benchmark) } }.filter { it.isNotEmpty() } } else benchmarks.filter { !it.isEmpty() } val results = benchmarksToRun.flatMap { benchmark -> diff --git a/kotlin-native/performance/build.gradle b/kotlin-native/performance/build.gradle index 84e95e8517f..ee157a22fb1 100644 --- a/kotlin-native/performance/build.gradle +++ b/kotlin-native/performance/build.gradle @@ -253,6 +253,7 @@ task registerBuild(type: BuildRegister) { dist = (new File(dist)).isAbsolute() ? dist : "${project.getProjectDir()}/$dist" bundleSize = (new File(dist)).directorySize() } + buildNumberSuffix = project.findProperty('buildNumberSuffix') } task registerExternalBuild(type: BuildRegister) { diff --git a/kotlin-native/tools/performance-server/src/main/kotlin/database/BenchmarksIndexesDispatcher.kt b/kotlin-native/tools/performance-server/src/main/kotlin/database/BenchmarksIndexesDispatcher.kt index 1ee410495e4..5d445c29b17 100644 --- a/kotlin-native/tools/performance-server/src/main/kotlin/database/BenchmarksIndexesDispatcher.kt +++ b/kotlin-native/tools/performance-server/src/main/kotlin/database/BenchmarksIndexesDispatcher.kt @@ -100,7 +100,10 @@ class BenchmarksIndexesDispatcher(connector: ElasticSearchConnector, val feature // Get benchmarks values of needed metric for choosen build number. fun getSamples(metricName: String, featureValue: String = "", samples: List, buildsCountToShow: Int, buildNumbers: Iterable? = null, - normalize: Boolean = false): Promise>>> { + normalize: Boolean = false, withSuffix: String? = null): Promise>>> { + + val filteredBuilds = buildNumbers?.filter { buildNumberIsIncluded(it, withSuffix) } + val queryDescription = """ { "_source": ["buildNumber"], @@ -108,7 +111,7 @@ class BenchmarksIndexesDispatcher(connector: ElasticSearchConnector, val feature "query": { "bool": { "must": [ - ${buildNumbers.str { builds -> + ${filteredBuilds.str { builds -> """ { "terms" : { "buildNumber" : [${builds.map { "\"${it.second}\"" }.joinToString()}] } },""" } } @@ -255,10 +258,18 @@ class BenchmarksIndexesDispatcher(connector: ElasticSearchConnector, val feature } } + private fun buildNumberIsIncluded(build: CompositeBuildNumber, withSuffix: String?) = + withSuffix?.let { + build.second.contains(withSuffix) + } ?: !build.second.contains("(") + + // Get geometric mean for benchmarks values of needed metric. fun getGeometricMean(metricName: String, featureValue: String = "", buildNumbers: Iterable? = null, normalize: Boolean = false, - excludeNames: List = emptyList()): Promise>>> { + excludeNames: List = emptyList(), withSuffix: String? = null): Promise>>> { + + val filteredBuilds = buildNumbers?.filter { buildNumberIsIncluded(it, withSuffix) } // Filter only with metric or also with names. val filterBenchmarks = if (excludeNames.isEmpty()) @@ -282,7 +293,7 @@ class BenchmarksIndexesDispatcher(connector: ElasticSearchConnector, val feature } }, """ } } - ${buildNumbers.str { builds -> + ${filteredBuilds.str { builds -> """ "aggs" : { "builds": { @@ -325,7 +336,7 @@ class BenchmarksIndexesDispatcher(connector: ElasticSearchConnector, val feature } } - ${buildNumbers.str { + ${filteredBuilds.str { """ } }""" } } @@ -343,7 +354,8 @@ class BenchmarksIndexesDispatcher(connector: ElasticSearchConnector, val feature ?.getObjectOrNull("buckets") ?: error("Wrong response:\n$responseString") buildNumbers.map { - it to listOf(buckets + it to if (buildNumberIsIncluded(it, withSuffix)) + listOf(buckets .getObject(it.second) .getObject("metric_build") .getObject("metric_samples") @@ -352,7 +364,9 @@ class BenchmarksIndexesDispatcher(connector: ElasticSearchConnector, val feature .getObjectOrNull("geom_mean") ?.getPrimitive("value") ?.double - ) + ) + else + listOf(null) } } ?: listOf((null to "golden") to listOf(aggregations .getObject("metric_build") diff --git a/kotlin-native/tools/performance-server/src/main/kotlin/routes/route.kt b/kotlin-native/tools/performance-server/src/main/kotlin/routes/route.kt index efadef71e71..8832aa63fcc 100644 --- a/kotlin-native/tools/performance-server/src/main/kotlin/routes/route.kt +++ b/kotlin-native/tools/performance-server/src/main/kotlin/routes/route.kt @@ -104,13 +104,13 @@ data class TCBuildInfo(val buildNumber: String, val branch: String, val startTim val finishTime: String, val buildType: String?) data class BuildRegister(val buildId: String, val teamCityUser: String, val teamCityPassword: String, - val bundleSize: String?, val fileWithResult: String) { + val bundleSize: String?, val fileWithResult: String, val buildNumberSuffix: String?) { companion object { fun create(json: String): BuildRegister { val requestDetails = JSON.parse(json) // Parse method doesn't create real instance with all methods. So create it by hands. return BuildRegister(requestDetails.buildId, requestDetails.teamCityUser, requestDetails.teamCityPassword, - requestDetails.bundleSize, requestDetails.fileWithResult) + requestDetails.bundleSize, requestDetails.fileWithResult, requestDetails.buildNumberSuffix) } } @@ -182,7 +182,8 @@ data class BuildRegister(val buildId: String, val teamCityUser: String, val team "${format(currentTime.getUTCMinutes())}" + "${format(currentTime.getUTCSeconds())}" + "${if (timeZone > 0) "+" else "-"}${format(timeZone)}${format(0)}" - TCBuildInfo(buildNumber!!, branch!!, startTime!!, finishTime, buildType) + val resultBuildNumber = buildNumberSuffix?.let { "$buildNumber($it)" } ?: buildNumber!! + TCBuildInfo(resultBuildNumber, branch!!, startTime!!, finishTime, buildType) } } } @@ -206,7 +207,8 @@ internal fun orderedValues(values: List, buildElement: (T) -> CompositeBu Int.MAX_VALUE }, { buildElement(it).first?.let { if (it == "DEV") 0 else 1 } ?: 0 }, // Develop and release builds - { buildElement(it).second.substringAfterLast("-").toDouble() } // build counter + { buildElement(it).second.substringAfterLast("-").substringBefore("(").toDouble() }, // build counter + { buildElement(it).second.contains("(") } // build suffix ) ) @@ -398,6 +400,7 @@ fun router() { var buildsCountToShow = 200 var beforeDate: String? = null var afterDate: String? = null + var buildSuffix: String? = null // Parse parameters from request if it exists. if (request.query != undefined) { @@ -428,13 +431,16 @@ fun router() { if (request.query.after != undefined) { afterDate = decodeURIComponent(request.query.after) } + if (request.query.buildSuffix != undefined) { + buildSuffix = request.query.buildSuffix + } } getBuildsNumbers(type, branch, target, buildsCountToShow, buildInfoIndex, beforeDate, afterDate).then { buildNumbers -> if (aggregation == "geomean") { // Get geometric mean for samples. benchmarksDispatcher.getGeometricMean(metric, target, buildNumbers, normalize, - excludeNames).then { geoMeansValues -> + excludeNames, buildSuffix).then { geoMeansValues -> success(orderedValues(geoMeansValues, { it -> it.first }, branch == "master") .map { it.first.second to it.second }) }.catch { errorResponse -> @@ -443,7 +449,7 @@ fun router() { reject() } } else { - benchmarksDispatcher.getSamples(metric, target, samples, buildsCountToShow, buildNumbers, normalize) + benchmarksDispatcher.getSamples(metric, target, samples, buildsCountToShow, buildNumbers, normalize, buildSuffix) .then { geoMeansValues -> success(orderedValues(geoMeansValues, { it -> it.first }, branch == "master") .map { it.first.second to it.second }) diff --git a/kotlin-native/tools/performance-server/ui/src/main/kotlin/main.kt b/kotlin-native/tools/performance-server/ui/src/main/kotlin/main.kt index 5cf3c8d58bd..1229b7a9cc2 100644 --- a/kotlin-native/tools/performance-server/ui/src/main/kotlin/main.kt +++ b/kotlin-native/tools/performance-server/ui/src/main/kotlin/main.kt @@ -336,7 +336,12 @@ fun main(args: Array) { mapOf( "normalize" to "true", "exclude" to unstableBenchmarks.joinToString(",") - )), + ), + mapOf( + "normalize" to "true", + "buildSuffix" to "(NewMM)" + ) + ), "COMPILE_TIME" to listOf(mapOf( "samples" to "HelloWorld,Videoplayer$platformSpecificBenchs", "agr" to "samples" @@ -418,7 +423,7 @@ fun main(args: Array) { execData = labels to values execChart = Chartist.Line("#exec_chart", getChartData(labels, execData.second), - getChartOptions(arrayOf("Geometric Mean (All)", "Geometric mean (Stable)"), + getChartOptions(arrayOf("Geometric Mean (All)", "Geometric mean (Stable)", "Geometric mean (New MM)"), "Normalized time")) buildsInfoPromise.then { builds -> customizeChart(execChart, "exec_chart", js("$(\"#exec_chart\")"), builds, parameters)