From 3c798502c80299915669ddab36cc413235accf2f Mon Sep 17 00:00:00 2001 From: Yunir Salimzyanov Date: Mon, 13 Jul 2020 14:13:22 +0300 Subject: [PATCH] Synchronize muted TeamCity tests with database for .bunch configurations --- .../jetbrains/kotlin/test/mutes/MutedSet.kt | 3 - .../jetbrains/kotlin/test/mutes/MutedTest.kt | 4 +- .../kotlin/test/mutes/MutedTestToJson.kt | 48 ++++++- .../kotlin/test/mutes/MutedTestsSync.kt | 133 +++++++++++------- .../kotlin/test/mutes/TeamCityInteraction.kt | 14 +- 5 files changed, 133 insertions(+), 69 deletions(-) diff --git a/compiler/tests-mutes/src/org/jetbrains/kotlin/test/mutes/MutedSet.kt b/compiler/tests-mutes/src/org/jetbrains/kotlin/test/mutes/MutedSet.kt index a0cc22849d8..11b36416e3c 100644 --- a/compiler/tests-mutes/src/org/jetbrains/kotlin/test/mutes/MutedSet.kt +++ b/compiler/tests-mutes/src/org/jetbrains/kotlin/test/mutes/MutedSet.kt @@ -14,9 +14,6 @@ class MutedSet(muted: List) { .groupBy { it.methodKey } // Method key -> List of muted tests .mapValues { (_, tests) -> tests.groupBy { it.simpleClassName } } - val flakyTests: List = - muted.filter { it.isFlaky } - fun mutedTest(testClass: Class<*>, methodKey: String): MutedTest? { val mutedTests = cache[methodKey]?.get(testClass.simpleName) ?: return null diff --git a/compiler/tests-mutes/src/org/jetbrains/kotlin/test/mutes/MutedTest.kt b/compiler/tests-mutes/src/org/jetbrains/kotlin/test/mutes/MutedTest.kt index 1d2934fe1d2..995336146c0 100644 --- a/compiler/tests-mutes/src/org/jetbrains/kotlin/test/mutes/MutedTest.kt +++ b/compiler/tests-mutes/src/org/jetbrains/kotlin/test/mutes/MutedTest.kt @@ -90,4 +90,6 @@ private fun parseMutedTest(str: String): MutedTest { return MutedTest(testKey, issue, hasFailFile, isFlaky) } -private class ParseError(message: String, override val cause: Throwable? = null) : IllegalArgumentException(message) \ No newline at end of file +private class ParseError(message: String, override val cause: Throwable? = null) : IllegalArgumentException(message) + +internal fun flakyTests(file: File) = loadMutedTests(file).filter { it.isFlaky } \ No newline at end of file diff --git a/compiler/tests-mutes/src/org/jetbrains/kotlin/test/mutes/MutedTestToJson.kt b/compiler/tests-mutes/src/org/jetbrains/kotlin/test/mutes/MutedTestToJson.kt index f98a3b0d70e..1f937c51d40 100644 --- a/compiler/tests-mutes/src/org/jetbrains/kotlin/test/mutes/MutedTestToJson.kt +++ b/compiler/tests-mutes/src/org/jetbrains/kotlin/test/mutes/MutedTestToJson.kt @@ -18,12 +18,12 @@ data class MuteTestJson( val resolution: JsonNode ) -internal fun createMuteTestJson(testName: String, description: String, scope: Scope): MuteTestJson { +internal fun createMuteTestJson(testName: String, description: String, scopeId: String, isBuildType: Boolean): MuteTestJson { val assignmentJson = """{ "text" : "$TAG $description" }""" - val scopeJson = if (scope.isBuildType) - """{"buildTypes":{"buildType":[{"id":"${scope.id}"}]}}""" + val scopeJson = if (isBuildType) + """{"buildTypes":{"buildType":[{"id":"$scopeId"}]}}""" else - """{"project":{"id":"${scope.id}"}}""" + """{"project":{"id":"$scopeId"}}""" val targetJson = """{ "tests" : { "test" : [ { "name" : "$testName" } ] } }""" val resolutionJson = """{ "type" : "manually" }""" @@ -35,3 +35,43 @@ internal fun createMuteTestJson(testName: String, description: String, scope: Sc jsonObjectMapper.readTree(resolutionJson) ) } + +internal fun filterMutedTestsByScope(muteTestJson: List, scopeId: String, isBuildType: Boolean): Map { + val filterCondition = { testJson: MuteTestJson -> + if (isBuildType) { + val buildTypes = testJson.scope.get("buildTypes") + val buildTypeIds = buildTypes?.get("buildType")?.toList()?.map { + it.get("id").textValue() + } ?: listOf() + buildTypeIds.contains(scopeId) + } else { + testJson.scope.get("project")?.get("id")?.textValue() == scopeId + } + } + + return muteTestJson.filter(filterCondition) + .flatMap { mutedTestJson -> + val testNames = mutedTestJson.target.get("tests").get("test").toList().map { it.get("name").textValue() } + testNames.map { testName -> + testName to mutedTestJson + } + } + .toMap() +} + +internal fun transformMutedTestsToJson(flakyTests: List?, scopeId: String, isBuildType: Boolean): Map { + val mutedMap = mutableMapOf() + if (flakyTests != null) { + for (muted in flakyTests) { + val testName = formatClassnameWithInnerClasses(muted.key) + mutedMap[testName] = createMuteTestJson(testName, muted.issue ?: "", scopeId, isBuildType) + } + } + return mutedMap +} + +private fun formatClassnameWithInnerClasses(classname: String): String { + val classFindRegex = "\\.(?=[A-Z])".toRegex() + val (pkg, name) = classname.split(classFindRegex, limit = 2) + return "$pkg.${name.replace(classFindRegex, "\\$")}" +} \ No newline at end of file diff --git a/compiler/tests-mutes/src/org/jetbrains/kotlin/test/mutes/MutedTestsSync.kt b/compiler/tests-mutes/src/org/jetbrains/kotlin/test/mutes/MutedTestsSync.kt index 995f225c83c..10dbf77f8d1 100644 --- a/compiler/tests-mutes/src/org/jetbrains/kotlin/test/mutes/MutedTestsSync.kt +++ b/compiler/tests-mutes/src/org/jetbrains/kotlin/test/mutes/MutedTestsSync.kt @@ -17,66 +17,91 @@ fun main() { * Purpose: possibility to run flaky tests on teamcity that will not affect on build status */ fun syncMutedTestsOnTeamCityWithDatabase() { - val remotelyMutedTests = getMutedTestsOnTeamcityForRootProject() - for (scope in Scope.values().filter { it.id != null }) { - val remotelyMutedTestsForSpecificScope: Map = filterMutedTestsByScope(remotelyMutedTests, scope) - val locallyMutedTests: Map = getMutedTestsFromDatabase(scope) - val deleteList = remotelyMutedTestsForSpecificScope - locallyMutedTests.keys - val uploadList = locallyMutedTests - remotelyMutedTestsForSpecificScope.keys - deleteMutedTests(deleteList) - uploadMutedTests(uploadList) - } -} + val remotelyMutedTests = RemotelyMutedTests() + val locallyMutedTests = LocallyMutedTests() + val bunches = Bunches.parseRulesToBunches(locallyMutedTests.tests.keys) -private fun getMutedTestsFromDatabase(scope: Scope): Map { - val mutedSet = MutedSet(loadMutedTests(scope.localDBPath)) - val mutedMap = mutableMapOf() - for (muted in mutedSet.flakyTests) { - val testName = formatClassnameWithInnerClasses(muted.key) - mutedMap[testName] = createMuteTestJson(testName, muted.issue ?: "", scope) - } - return mutedMap -} + syncMutedTests(remotelyMutedTests.projectTests, locallyMutedTests.projectTests) -private fun formatClassnameWithInnerClasses(classname: String): String { - val classFindRegex = "\\.(?=[A-Z])".toRegex() - val (pkg, name) = classname.split(classFindRegex, limit = 2) - return "$pkg.${name.replace(classFindRegex, "\\$")}" -} - -private fun filterMutedTestsByScope(muteTestJson: List, scope: Scope): Map { - val filterCondition = { testJson: MuteTestJson -> - if (scope.isBuildType) { - val buildTypes = testJson.scope.get("buildTypes") - val buildTypeIds = buildTypes?.get("buildType")?.toList()?.map { - it.get("id").textValue() - } ?: listOf() - buildTypeIds.contains(scope.id) - } else { - testJson.scope.get("project")?.get("id")?.textValue() == scope.id + for ((originalBunchId, foundBunchId) in bunches) { + getBuildTypeId(originalBunchId)?.let { buildTypeId -> + syncMutedTests(remotelyMutedTests.getTestsJson(buildTypeId), locallyMutedTests.getTestsJson(foundBunchId, buildTypeId)) } } - - return muteTestJson.filter(filterCondition) - .flatMap { mutedTestJson -> - val testNames = mutedTestJson.target.get("tests").get("test").toList().map { it.get("name").textValue() } - testNames.map { testName -> - testName to mutedTestJson - } - } - .toMap() } -private const val databaseDir = "../../tests" +private fun syncMutedTests( + remotelyMutedTests: Map, + locallyMutedTests: Map +) { + val deleteList = remotelyMutedTests - locallyMutedTests.keys + val uploadList = locallyMutedTests - remotelyMutedTests.keys + deleteMutedTests(deleteList) + uploadMutedTests(uploadList) +} + +internal fun getMandatoryProperty(propertyName: String) = (System.getProperty(propertyName) + ?: throw Exception("Property $propertyName must be set")) + +object Bunches { + private val bunchRules: List = readAllRulesFromFile() + internal val baseBunchId = bunchRules.first() + + internal fun parseRulesToBunches(platforms: Set): Map { + return bunchRules + .map { it.split('_') } + .map { rule -> + rule.first() to (rule.find { platforms.contains(it) } ?: baseBunchId) + }.toMap() + } + + private fun readAllRulesFromFile(): List { + val file = File("../..", ".bunch") + if (!file.exists()) { + throw BunchException("Can't build list of rules. File '${file.canonicalPath}' doesn't exist") + } + return file.readLines() + .map { it.trim() } + .filter { it.isNotEmpty() } + } + + private class BunchException(msg: String? = null) : Exception(msg) +} + private const val mutesPackageName = "org.jetbrains.kotlin.test.mutes" +internal val projectId = getMandatoryProperty("$mutesPackageName.tests.project.id") +internal fun getBuildTypeId(bunchId: String) = System.getProperty("$mutesPackageName.$bunchId") -// FIX ME WHEN BUNCH 192 REMOVED -// FIX ME WHEN BUNCH as36 REMOVED -internal enum class Scope(val id: String?, val localDBPath: File, val isBuildType: Boolean) { - COMMON(System.getProperty("$mutesPackageName.tests.project.id"), File("$databaseDir/mute-common.csv"), false), - IJ201(System.getProperty("$mutesPackageName.201"), File("$databaseDir/mute-platform.csv"), true), - IJ193(System.getProperty("$mutesPackageName.193"), File("$databaseDir/mute-platform.csv.193"), true), - IJ192(System.getProperty("$mutesPackageName.192"), File("$databaseDir/mute-platform.csv.192"), true), - AS36(System.getProperty("$mutesPackageName.as36"), File("$databaseDir/mute-platform.csv.as36"), true), - AS40(System.getProperty("$mutesPackageName.as40"), File("$databaseDir/mute-platform.csv.as40"), true); +class RemotelyMutedTests { + val tests = getMutedTestsOnTeamcityForRootProject(projectId) + val projectTests = getTestsJson(projectId, false) + internal fun getTestsJson(scopeId: String, isBuildType: Boolean = true): Map { + return filterMutedTestsByScope(tests, scopeId, isBuildType) + } +} + +class LocallyMutedTests { + private val muteCommonTestKey = "COMMON" + val tests = getMutedTestsFromDatabase() + val projectTests = getTestsJson(muteCommonTestKey, projectId, false) + + internal fun getTestsJson(platformId: String, scopeId: String, isBuildType: Boolean = true): Map { + return transformMutedTestsToJson(tests[platformId], scopeId, isBuildType) + } + + private fun getMutedTestsFromDatabase(): Map> { + val mutedTestsMap = mutableMapOf>() + val databaseDir = "../../tests" + + val commonDatabaseFile = File(databaseDir, "mute-common.csv") + mutedTestsMap[muteCommonTestKey] = flakyTests(commonDatabaseFile) + + val platformDatabaseFile = File(databaseDir, "mute-platform.csv") + File(databaseDir).walkTopDown().filter { f -> f.name.startsWith(platformDatabaseFile.name) }.toList().map { f -> + val key = if (f.extension != "csv") f.extension else Bunches.baseBunchId + mutedTestsMap[key] = flakyTests(f) + } + + return mutedTestsMap + } } \ No newline at end of file diff --git a/compiler/tests-mutes/src/org/jetbrains/kotlin/test/mutes/TeamCityInteraction.kt b/compiler/tests-mutes/src/org/jetbrains/kotlin/test/mutes/TeamCityInteraction.kt index b688278068d..baa70f26cfc 100644 --- a/compiler/tests-mutes/src/org/jetbrains/kotlin/test/mutes/TeamCityInteraction.kt +++ b/compiler/tests-mutes/src/org/jetbrains/kotlin/test/mutes/TeamCityInteraction.kt @@ -1,22 +1,21 @@ package org.jetbrains.kotlin.test.mutes import com.fasterxml.jackson.module.kotlin.treeToValue +import khttp.DEFAULT_TIMEOUT import khttp.responses.Response import khttp.structures.authorization.Authorization internal const val TAG = "[MUTED-BY-CSVFILE]" -private val buildServerUrl = System.getProperty("org.jetbrains.kotlin.test.mutes.teamcity.server.url") +private val buildServerUrl = getMandatoryProperty("org.jetbrains.kotlin.test.mutes.teamcity.server.url") private val headers = mapOf("Content-type" to "application/json", "Accept" to "application/json") private val authUser = object : Authorization { - override val header = "Authorization" to "Bearer ${System.getProperty("org.jetbrains.kotlin.test.mutes.teamcity.server.token")}" + override val header = "Authorization" to "Bearer ${getMandatoryProperty("org.jetbrains.kotlin.test.mutes.teamcity.server.token")}" } -internal fun getMutedTestsOnTeamcityForRootProject(): List { - val projectScopeId = Scope.COMMON.id - +internal fun getMutedTestsOnTeamcityForRootProject(rootScopeId: String): List { val params = mapOf( - "locator" to "project:(id:$projectScopeId)", + "locator" to "project:(id:$rootScopeId)", "fields" to "mute(id,assignment(text),scope(project(id),buildTypes(buildType(id))),target(tests(test(name))),resolution)" ) @@ -46,7 +45,8 @@ internal fun deleteMutedTests(deleteMap: Map) { val response = khttp.delete( "$buildServerUrl/app/rest/mutes/id:${muteTestJson.id}", headers = headers, - auth = authUser + auth = authUser, + timeout = DEFAULT_TIMEOUT * 2 ) try { checkResponseAndLog(response)