From 510e38b79143762b62de9557f7d1196f3ad939fb Mon Sep 17 00:00:00 2001 From: Elena Lepilkina Date: Wed, 17 Mar 2021 17:09:22 +0300 Subject: [PATCH] [K/N][performance] Adopted performance service to work with new configurations and build numbers --- .../kotlinx_cli_jvm_1_5_255_SNAPSHOT.xml | 4 +- .../jetbrains/elastic/ElasticSearchIndex.kt | 12 +++- .../database/BenchmarksIndexesDispatcher.kt | 25 ++++--- .../main/kotlin/database/DatabaseRequests.kt | 28 +++++--- .../src/main/kotlin/routes/route.kt | 72 +++++++++++++------ 5 files changed, 97 insertions(+), 44 deletions(-) diff --git a/.idea/artifacts/kotlinx_cli_jvm_1_5_255_SNAPSHOT.xml b/.idea/artifacts/kotlinx_cli_jvm_1_5_255_SNAPSHOT.xml index 2dd8777a819..5598008f8e9 100644 --- a/.idea/artifacts/kotlinx_cli_jvm_1_5_255_SNAPSHOT.xml +++ b/.idea/artifacts/kotlinx_cli_jvm_1_5_255_SNAPSHOT.xml @@ -1,6 +1,8 @@ $PROJECT_DIR$/kotlin-native/endorsedLibraries/kotlinx.cli/build/libs - + + + \ No newline at end of file diff --git a/kotlin-native/tools/performance-server/shared/src/main/kotlin/org/jetbrains/elastic/ElasticSearchIndex.kt b/kotlin-native/tools/performance-server/shared/src/main/kotlin/org/jetbrains/elastic/ElasticSearchIndex.kt index 5d1c09fb9e6..216cf04c37a 100644 --- a/kotlin-native/tools/performance-server/shared/src/main/kotlin/org/jetbrains/elastic/ElasticSearchIndex.kt +++ b/kotlin-native/tools/performance-server/shared/src/main/kotlin/org/jetbrains/elastic/ElasticSearchIndex.kt @@ -85,7 +85,8 @@ class CommitsList : ConvertedFromJson, JsonSerializable { data class BuildInfo(val buildNumber: String, val startTime: String, val endTime: String, val commitsList: CommitsList, val branch: String, - val agentInfo: String /* Important agent information often used in requests.*/) : JsonSerializable { + val agentInfo: String /* Important agent information often used in requests.*/, + val buildType: String?) : JsonSerializable { override fun serializeFields() = """ "buildNumber": "$buildNumber", "startTime": "$startTime", @@ -93,6 +94,9 @@ data class BuildInfo(val buildNumber: String, val startTime: String, val endTime ${commitsList.serializeFields()}, "branch": "$branch", "agentInfo": "$agentInfo" + ${buildType?.let {""", + "buildType": "$buildType" + """}} """ companion object : EntityFromJsonFactory { @@ -112,7 +116,11 @@ data class BuildInfo(val buildNumber: String, val startTime: String, val endTime val agentInfo = agentInfoElement?.let { elementToString(agentInfoElement, "agentInfo") } ?: "" - return BuildInfo(buildNumber, startTime, endTime, CommitsList(commits), branch, agentInfo) + val buildTypeElement = data.getOptionalField("buildType") + val buildType = buildTypeElement?.let { + elementToString(buildTypeElement, "buildType") + } ?: null + return BuildInfo(buildNumber, startTime, endTime, CommitsList(commits), branch, agentInfo, buildType) } else { error("Top level entity is expected to be an object. Please, check origin files.") } diff --git a/kotlin-native/tools/performance-server/src/main/kotlin/database/BenchmarksIndexesDispatcher.kt b/kotlin-native/tools/performance-server/src/main/kotlin/database/BenchmarksIndexesDispatcher.kt index d5927313b9c..1ee410495e4 100644 --- a/kotlin-native/tools/performance-server/src/main/kotlin/database/BenchmarksIndexesDispatcher.kt +++ b/kotlin-native/tools/performance-server/src/main/kotlin/database/BenchmarksIndexesDispatcher.kt @@ -11,6 +11,7 @@ import org.jetbrains.utils.* import org.jetbrains.report.json.* import org.jetbrains.report.* +typealias CompositeBuildNumber = Pair fun Iterable.isEmpty() = count() == 0 fun Iterable.isNotEmpty() = !isEmpty() @@ -98,8 +99,8 @@ class BenchmarksIndexesDispatcher(connector: ElasticSearchConnector, val feature // Get benchmarks values of needed metric for choosen build number. fun getSamples(metricName: String, featureValue: String = "", samples: List, buildsCountToShow: Int, - buildNumbers: Iterable? = null, - normalize: Boolean = false): Promise>>> { + buildNumbers: Iterable? = null, + normalize: Boolean = false): Promise>>> { val queryDescription = """ { "_source": ["buildNumber"], @@ -109,7 +110,7 @@ class BenchmarksIndexesDispatcher(connector: ElasticSearchConnector, val feature "must": [ ${buildNumbers.str { builds -> """ - { "terms" : { "buildNumber" : [${builds.map { "\"$it\"" }.joinToString()}] } },""" } + { "terms" : { "buildNumber" : [${builds.map { "\"${it.second}\"" }.joinToString()}] } },""" } } ${featureFilter.str { "${it(featureValue)}," } } {"nested" : { @@ -142,19 +143,21 @@ class BenchmarksIndexesDispatcher(connector: ElasticSearchConnector, val feature val indexesMap = samples.mapIndexed { index, it -> it to index }.toMap() val valuesMap = buildNumbers?.map { it to arrayOfNulls(samples.size) - }?.toMap()?.toMutableMap() ?: mutableMapOf>() + }?.toMap()?.toMutableMap() ?: mutableMapOf>() + val buildTypes = buildNumbers?.map { it.second to it.first }?.toMap() ?: emptyMap() // Parse and save values in requested order. results.forEach { val element = it as JsonObject val build = element.getObject("_source").getPrimitive("buildNumber").content - buildNumbers?.let { valuesMap.getOrPut(build) { arrayOfNulls(samples.size) } } + val buildInfo = buildTypes[build] to build + buildNumbers?.let { valuesMap.getOrPut(buildInfo) { arrayOfNulls(samples.size) } } element .getObject("inner_hits") .getObject("benchmarks") .getObject("hits") .getArray("hits").forEach { val source = (it as JsonObject).getObject("_source") - valuesMap[build]!![indexesMap[source.getPrimitive("name").content]!!] = + valuesMap[buildInfo]!![indexesMap[source.getPrimitive("name").content]!!] = source.getPrimitive(if (normalize) "normalizedScore" else "score").double } @@ -254,8 +257,8 @@ class BenchmarksIndexesDispatcher(connector: ElasticSearchConnector, val feature // Get geometric mean for benchmarks values of needed metric. fun getGeometricMean(metricName: String, featureValue: String = "", - buildNumbers: Iterable? = null, normalize: Boolean = false, - excludeNames: List = emptyList()): Promise>>> { + buildNumbers: Iterable? = null, normalize: Boolean = false, + excludeNames: List = emptyList()): Promise>>> { // Filter only with metric or also with names. val filterBenchmarks = if (excludeNames.isEmpty()) @@ -285,7 +288,7 @@ class BenchmarksIndexesDispatcher(connector: ElasticSearchConnector, val feature "builds": { "filters" : { "filters": { - ${builds.map { "\"$it\": { \"match\" : { \"buildNumber\" : \"$it\" }}" } + ${builds.map { "\"${it.second}\": { \"match\" : { \"buildNumber\" : \"${it.second}\" }}" } .joinToString(",\n")} } },""" @@ -341,7 +344,7 @@ class BenchmarksIndexesDispatcher(connector: ElasticSearchConnector, val feature ?: error("Wrong response:\n$responseString") buildNumbers.map { it to listOf(buckets - .getObject(it) + .getObject(it.second) .getObject("metric_build") .getObject("metric_samples") .getObject("buckets") @@ -351,7 +354,7 @@ class BenchmarksIndexesDispatcher(connector: ElasticSearchConnector, val feature ?.double ) } - } ?: listOf("golden" to listOf(aggregations + } ?: listOf((null to "golden") to listOf(aggregations .getObject("metric_build") .getObject("metric_samples") .getObject("buckets") diff --git a/kotlin-native/tools/performance-server/src/main/kotlin/database/DatabaseRequests.kt b/kotlin-native/tools/performance-server/src/main/kotlin/database/DatabaseRequests.kt index 2d74ee2a81f..e126717f771 100644 --- a/kotlin-native/tools/performance-server/src/main/kotlin/database/DatabaseRequests.kt +++ b/kotlin-native/tools/performance-server/src/main/kotlin/database/DatabaseRequests.kt @@ -43,15 +43,21 @@ internal fun getBuildsDescription(type: String?, branch: String?, agentInfo: Str "sort": {"startTime": "desc" }, "query": { "bool": { + ${type?.let {""" + "must": [ + { "bool": { + "should": [ + { "regexp": { "buildNumber": { "value": "${if (it == "release") + ".*eap.*|.*release.*|.*rc.*" else ".*dev.*"}" } } + }, + { "match": { "buildType": "${it.toUpperCase()}" } } + ] + }}, + { "bool": { + + """} ?: ""} "must": [ { "match": { "agentInfo": "$agentInfo" } } - ${type?.let { - """, - { "regexp": { "buildNumber": { "value": "${if (it == "release") - ".*eap.*|.*release.*|.*rc.*" else ".*dev.*"}" } } - } - """ - } ?: ""} ${beforeDate?.let { """, { "range": { "startTime": { "lt": "$it" } } } @@ -68,6 +74,9 @@ internal fun getBuildsDescription(type: String?, branch: String?, agentInfo: Str """ } ?: ""} ] + ${type?.let {""" + }}] + """}} } } } @@ -109,7 +118,10 @@ fun getBuildsNumbers(type: String?, branch: String?, agentInfo: String, buildsCo buildInfoIndex: ElasticSearchIndex, beforeDate: String? = null, afterDate: String? = null) = getBuildsDescription(type, branch, agentInfo, buildInfoIndex, buildsCountToShow, beforeDate, afterDate, true) .then { responseArray -> - responseArray.map { (it as JsonObject).getObject("_source").getPrimitive("buildNumber").content } + responseArray.map { + val build = (it as JsonObject).getObject("_source") + build.getPrimitiveOrNull("buildType")?.content to build.getPrimitive("buildNumber").content + } } // Get full builds information corresponding to machine and branch. diff --git a/kotlin-native/tools/performance-server/src/main/kotlin/routes/route.kt b/kotlin-native/tools/performance-server/src/main/kotlin/routes/route.kt index 358334e0e38..efadef71e71 100644 --- a/kotlin-native/tools/performance-server/src/main/kotlin/routes/route.kt +++ b/kotlin-native/tools/performance-server/src/main/kotlin/routes/route.kt @@ -101,7 +101,7 @@ fun GoldenResultsInfo.toBenchmarksReport(): BenchmarksReport { // Build information provided from request. data class TCBuildInfo(val buildNumber: String, val branch: String, val startTime: String, - val finishTime: String) + val finishTime: String, val buildType: String?) data class BuildRegister(val buildId: String, val teamCityUser: String, val teamCityPassword: String, val bundleSize: String?, val fileWithResult: String) { @@ -125,7 +125,18 @@ data class BuildRegister(val buildId: String, val teamCityUser: String, val team fun sendTeamCityRequest(url: String, json: Boolean = false) = UrlNetworkConnector(teamCityUrl).sendRequest(RequestMethod.GET, url, teamCityUser, teamCityPassword, json) - fun getBranchName(project: String): Promise { + fun sendTeamCityOptionalRequest(url: String, json: Boolean = false) = + UrlNetworkConnector(teamCityUrl).sendOptionalRequest(RequestMethod.GET, url, teamCityUser, teamCityPassword, json) + + private fun List.anyMatches(text: String): Boolean { + this.forEach { + if (it.toRegex().matches(text)) + return true + } + return false + } + + fun getBranchName(projects: List): Promise { val url = "builds?locator=id:$buildId&fields=build(revisions(revision(vcsBranchName,vcs-root-instance)))" var branch: String? = null return sendTeamCityRequest(url, true).then { response -> @@ -134,25 +145,33 @@ data class BuildRegister(val buildId: String, val teamCityUser: String, val team (it as JsonObject).getObject("revisions").getArray("revision").forEach { val currentBranch = (it as JsonObject).getPrimitive("vcsBranchName").content.removePrefix("refs/heads/") val currentProject = (it as JsonObject).getObject("vcs-root-instance").getPrimitive("name").content - if (project == currentProject) { + if (projects.anyMatches(currentProject)) { branch = currentBranch } return@forEach } } - branch ?: error("No project $project can be found in build $buildId") + branch ?: error("No project from list $projects can be found in build $buildId") } } + private fun getBuildType(): Promise { + val url = "$teamCityBuildUrl/resulting-properties/env.BUILD_TYPE" + return sendTeamCityOptionalRequest(url, false).then { response -> + response + } + } private fun format(timeValue: Int): String = if (timeValue < 10) "0$timeValue" else "$timeValue" fun getBuildInformation(): Promise { return Promise.all(arrayOf(sendTeamCityRequest("$teamCityBuildUrl/number"), - getBranchName("Kotlin Native"), - sendTeamCityRequest("$teamCityBuildUrl/startDate"))).then { results -> - val (buildNumber, branch, startTime) = results + getBranchName(listOf("Kotlin Native", "Kotlin_BuildPlayground_Setup202Plugin_Kotlin", "Kotlin_KotlinDev_Kotlin", + "Kotlin_KotlinRelease_(\\d+)_Kotlin")), + sendTeamCityRequest("$teamCityBuildUrl/startDate"), + getBuildType())).then { results -> + val (buildNumber, branch, startTime, buildType) = results val currentTime = Date() val timeZone = currentTime.getTimezoneOffset() / -60 // Convert to hours. // Get finish time as current time, because buid on TeamCity isn't finished. @@ -163,25 +182,31 @@ data class BuildRegister(val buildId: String, val teamCityUser: String, val team "${format(currentTime.getUTCMinutes())}" + "${format(currentTime.getUTCSeconds())}" + "${if (timeZone > 0) "+" else "-"}${format(timeZone)}${format(0)}" - TCBuildInfo(buildNumber, branch, startTime, finishTime) + TCBuildInfo(buildNumber!!, branch!!, startTime!!, finishTime, buildType) } } } // Get builds numbers in right order. -internal fun orderedValues(values: List, buildElement: (T) -> String = { it -> it.toString() }, +internal fun orderedValues(values: List, buildElement: (T) -> CompositeBuildNumber, skipMilestones: Boolean = false) = values.sortedWith( - compareBy({ buildElement(it).substringBefore(".").toInt() }, - { buildElement(it).substringAfter(".").substringBefore("-").toDouble() }, - { - if (skipMilestones) 0 - else if (buildElement(it).substringAfter("-").startsWith("M")) - buildElement(it).substringAfter("M").substringBefore("-").toInt() + compareBy( + { if (buildElement(it).first != null) 1 else 0 }, // Old builds have no set build type. + { buildElement(it).second.substringBefore(".").toInt() }, // Kotlin version + { buildElement(it).second.substringAfter(".").substringBefore("-").toDouble() }, + { // Milestones and release candidates. + val buildNumber = buildElement(it).second + if (skipMilestones && buildElement(it).first == null ) 0 + else if (buildNumber.substringAfter("-").startsWith("M")) + buildNumber.substringAfter("M").substringBefore("-").toInt() + else if (buildNumber.substringAfter("-").startsWith("RC")) + Int.MAX_VALUE / 2 else Int.MAX_VALUE }, - { buildElement(it).substringAfterLast("-").toDouble() } + { buildElement(it).first?.let { if (it == "DEV") 0 else 1 } ?: 0 }, // Develop and release builds + { buildElement(it).second.substringAfterLast("-").toDouble() } // build counter ) ) @@ -226,7 +251,8 @@ fun router() { // Build was rerun. val buildNumber = "${currentBuildInfo.buildNumber}.$rerunNumber" currentBuildInfo = BuildInfo(buildNumber, currentBuildInfo.startTime, currentBuildInfo.endTime, - currentBuildInfo.commitsList, currentBuildInfo.branch, currentBuildInfo.agentInfo) + currentBuildInfo.commitsList, currentBuildInfo.branch, currentBuildInfo.agentInfo, + currentBuildInfo.buildType) return getConsistentBuildInfo(currentBuildInfo, reports, rerunNumber + 1) } } @@ -256,7 +282,7 @@ fun router() { // Register build information. var buildInfoInstance = getConsistentBuildInfo( BuildInfo(buildInfo.buildNumber, buildInfo.startTime, buildInfo.finishTime, - commitsList, buildInfo.branch, reports[0].env.machine.os), + commitsList, buildInfo.branch, reports[0].env.machine.os, buildInfo.buildType), reports ) if (register.bundleSize != null) { @@ -343,7 +369,7 @@ fun router() { val buildNumbers = buildsInfo.map { it.buildNumber } // Get number of failed benchmarks for each build. benchmarksDispatcher.getFailuresNumber(target, buildNumbers).then { failures -> - success(orderedValues(buildsInfo, { it -> it.buildNumber }, branch == "master").map { + success(orderedValues(buildsInfo, { it -> it.buildType to it.buildNumber }, branch == "master").map { Build(it.buildNumber, it.startTime, it.endTime, it.branch, it.commitsList.serializeFields(), failures[it.buildNumber] ?: 0) }) @@ -409,7 +435,8 @@ fun router() { // Get geometric mean for samples. benchmarksDispatcher.getGeometricMean(metric, target, buildNumbers, normalize, excludeNames).then { geoMeansValues -> - success(orderedValues(geoMeansValues, { it -> it.first }, branch == "master")) + success(orderedValues(geoMeansValues, { it -> it.first }, branch == "master") + .map { it.first.second to it.second }) }.catch { errorResponse -> println("Error during getting geometric mean") println(errorResponse) @@ -418,7 +445,8 @@ fun router() { } else { benchmarksDispatcher.getSamples(metric, target, samples, buildsCountToShow, buildNumbers, normalize) .then { geoMeansValues -> - success(orderedValues(geoMeansValues, { it -> it.first }, branch == "master")) + success(orderedValues(geoMeansValues, { it -> it.first }, branch == "master") + .map { it.first.second to it.second }) }.catch { println("Error during getting samples") reject() @@ -498,7 +526,7 @@ fun router() { val jsonReport = artifactoryUrlConnector.sendRequest(RequestMethod.GET, accessFileUrl).await() var reports = convert(jsonReport, currentBuildNumber, target) val buildInfoRecord = BuildInfo(currentBuildNumber, infoParts[1], infoParts[2], - CommitsList.parse(infoParts[4]), infoParts[3], target) + CommitsList.parse(infoParts[4]), infoParts[3], target, null) val externalJsonReport = artifactoryUrlConnector.sendOptionalRequest(RequestMethod.GET, accessExternalFileUrl) .await()