Add logic to synchronize muted tests on teamcity with database flaky tests

KTI-239

build.gradle.kts

@@ -513,6 +513,10 @@ val dist = tasks.register("dist") {
     dependsOn(":kotlin-compiler:dist")
 }
 
+val syncMutedTests = tasks.register("syncMutedTests") {
+    dependsOn(":compiler:tests-mutes:run")
+}
+
 val copyCompilerToIdeaPlugin by task<Copy> {
     dependsOn(dist)
     into(ideaPluginDir)

compiler/tests-mutes/build.gradle.kts

@@ -1,10 +1,13 @@
 plugins {
     kotlin("jvm")
     id("jps-compatible")
+    application
 }
 
 dependencies {
     api(kotlinStdlib())
+    implementation("khttp:khttp:1.0.0")
+    implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.11.0")
 }
 
 sourceSets {
@@ -13,3 +16,10 @@ sourceSets {
     }
     "test" {}
 }
+
+val mutesPackageName = "org.jetbrains.kotlin.test.mutes"
+
+application {
+    mainClassName = "$mutesPackageName.MutedTestsSyncKt"
+    applicationDefaultJvmArgs = rootProject.properties.filterKeys { it.startsWith(mutesPackageName) }.map { (k, v) -> "-D$k=$v" }
+}

compiler/tests-mutes/src/org/jetbrains/kotlin/test/mutes/MutedSet.kt

@@ -14,6 +14,9 @@ class MutedSet(muted: List<MutedTest>) {
             .groupBy { it.methodKey } // Method key -> List of muted tests
             .mapValues { (_, tests) -> tests.groupBy { it.simpleClassName } }
 
+    val flakyTests: List<MutedTest> =
+        muted.filter { it.isFlaky }
+
     fun mutedTest(testClass: Class<*>, methodKey: String): MutedTest? {
         val mutedTests = cache[methodKey]?.get(testClass.simpleName) ?: return null
 

compiler/tests-mutes/src/org/jetbrains/kotlin/test/mutes/MutedTestToJson.kt

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.mutes
+
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
+
+internal val jsonObjectMapper = jacksonObjectMapper()
+
+data class MuteTestJson(
+    val id: Int,
+    val assignment: JsonNode,
+    val scope: JsonNode,
+    val target: JsonNode,
+    val resolution: JsonNode
+)
+
+internal fun createMuteTestJson(testName: String, description: String, scope: Scope): MuteTestJson {
+    val assignmentJson = """{ "text" : "$TAG $description" }"""
+    val scopeJson = if (scope.isBuildType)
+        """{"buildTypes":{"buildType":[{"id":"${scope.id}"}]}}"""
+    else
+        """{"project":{"id":"${scope.id}"}}"""
+    val targetJson = """{ "tests" : { "test" : [ { "name" : "$testName" } ] } }"""
+    val resolutionJson = """{ "type" : "manually" }"""
+
+    return MuteTestJson(
+        0,
+        jsonObjectMapper.readTree(assignmentJson),
+        jsonObjectMapper.readTree(scopeJson),
+        jsonObjectMapper.readTree(targetJson),
+        jsonObjectMapper.readTree(resolutionJson)
+    )
+}

compiler/tests-mutes/src/org/jetbrains/kotlin/test/mutes/MutedTestsSync.kt

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.test.mutes
+
+import java.io.File
+
+fun main() {
+    syncMutedTestsOnTeamCityWithDatabase()
+}
+
+/**
+ * Synchronize muted tests on teamcity with flaky tests in database
+ *
+ * Purpose: possibility to run flaky tests on teamcity that will not affect on build status
+ */
+fun syncMutedTestsOnTeamCityWithDatabase() {
+    val remotelyMutedTests = getMutedTestsOnTeamcityForRootProject()
+    for (scope in Scope.values().filter { it.id != null }) {
+        val remotelyMutedTestsForSpecificScope: Map<String, MuteTestJson> = filterMutedTestsByScope(remotelyMutedTests, scope)
+        val locallyMutedTests: Map<String, MuteTestJson> = getMutedTestsFromDatabase(scope)
+        val deleteList = remotelyMutedTestsForSpecificScope - locallyMutedTests.keys
+        val uploadList = locallyMutedTests - remotelyMutedTestsForSpecificScope.keys
+        deleteMutedTests(deleteList)
+        uploadMutedTests(uploadList)
+    }
+}
+
+private fun getMutedTestsFromDatabase(scope: Scope): Map<String, MuteTestJson> {
+    val mutedSet = MutedSet(loadMutedTests(scope.localDBPath))
+    val mutedMap = mutableMapOf<String, MuteTestJson>()
+    for (muted in mutedSet.flakyTests) {
+        val testName = formatClassnameWithInnerClasses(muted.key)
+        mutedMap[testName] = createMuteTestJson(testName, muted.issue ?: "", scope)
+    }
+    return mutedMap
+}
+
+private fun formatClassnameWithInnerClasses(classname: String): String {
+    val classFindRegex = "\\.(?=[A-Z])".toRegex()
+    val (pkg, name) = classname.split(classFindRegex, limit = 2)
+    return "$pkg.${name.replace(classFindRegex, "\\$")}"
+}
+
+private fun filterMutedTestsByScope(muteTestJson: List<MuteTestJson>, scope: Scope): Map<String, MuteTestJson> {
+    val filterCondition = { testJson: MuteTestJson ->
+        if (scope.isBuildType) {
+            val buildTypes = testJson.scope.get("buildTypes")
+            val buildTypeIds = buildTypes?.get("buildType")?.toList()?.map {
+                it.get("id").textValue()
+            } ?: listOf()
+            buildTypeIds.contains(scope.id)
+        } else {
+            testJson.scope.get("project")?.get("id")?.textValue() == scope.id
+        }
+    }
+
+    return muteTestJson.filter(filterCondition)
+        .flatMap { mutedTestJson ->
+            val testNames = mutedTestJson.target.get("tests").get("test").toList().map { it.get("name").textValue() }
+            testNames.map { testName ->
+                testName to mutedTestJson
+            }
+        }
+        .toMap()
+}
+
+private const val databaseDir = "../../tests"
+private const val mutesPackageName = "org.jetbrains.kotlin.test.mutes"
+
+// FIX ME WHEN BUNCH 192 REMOVED
+// FIX ME WHEN BUNCH as36 REMOVED
+internal enum class Scope(val id: String?, val localDBPath: File, val isBuildType: Boolean) {
+    COMMON(System.getProperty("$mutesPackageName.tests.project.id"), File("$databaseDir/mute-common.csv"), false),
+    IJ193(System.getProperty("$mutesPackageName.193"), File("$databaseDir/mute-platform.csv"), true),
+    IJ192(System.getProperty("$mutesPackageName.192"), File("$databaseDir/mute-platform.csv.192"), true),
+    IJ201(System.getProperty("$mutesPackageName.201"), File("$databaseDir/mute-platform.csv.201"), true),
+    AS36(System.getProperty("$mutesPackageName.as36"), File("$databaseDir/mute-platform.csv.as36"), true),
+    AS40(System.getProperty("$mutesPackageName.as40"), File("$databaseDir/mute-platform.csv.as40"), true);
+}

compiler/tests-mutes/src/org/jetbrains/kotlin/test/mutes/TeamCityInteraction.kt

@@ -0,0 +1,68 @@
+package org.jetbrains.kotlin.test.mutes
+
+import com.fasterxml.jackson.module.kotlin.treeToValue
+import khttp.responses.Response
+import khttp.structures.authorization.Authorization
+
+internal const val TAG = "[MUTED-BY-CSVFILE]"
+private val buildServerUrl = System.getProperty("org.jetbrains.kotlin.test.mutes.teamcity.server.url")
+private val headers = mapOf("Content-type" to "application/json", "Accept" to "application/json")
+private val authUser = object : Authorization {
+    override val header = "Authorization" to "Bearer ${System.getProperty("org.jetbrains.kotlin.test.mutes.teamcity.server.token")}"
+}
+
+
+internal fun getMutedTestsOnTeamcityForRootProject(): List<MuteTestJson> {
+    val projectScopeId = Scope.COMMON.id
+
+    val params = mapOf(
+        "locator" to "project:(id:$projectScopeId)",
+        "fields" to "mute(id,assignment(text),scope(project(id),buildTypes(buildType(id))),target(tests(test(name))),resolution)"
+    )
+
+    val response = khttp.get("$buildServerUrl/app/rest/mutes", headers, params, auth = authUser)
+    checkResponseAndLog(response)
+
+    val alreadyMutedTestsOnTeamCity = jsonObjectMapper.readTree(response.text).get("mute")
+        .filter { jn -> jn.get("assignment").get("text").textValue().startsWith(TAG) }
+
+    return alreadyMutedTestsOnTeamCity.mapNotNull { jsonObjectMapper.treeToValue<MuteTestJson>(it) }
+}
+
+internal fun uploadMutedTests(uploadMap: Map<String, MuteTestJson>) {
+    for ((_, muteTestJson) in uploadMap) {
+        val response = khttp.post(
+            "$buildServerUrl/app/rest/mutes",
+            headers = headers,
+            data = jsonObjectMapper.writeValueAsString(muteTestJson),
+            auth = authUser
+        )
+        checkResponseAndLog(response)
+    }
+}
+
+internal fun deleteMutedTests(deleteMap: Map<String, MuteTestJson>) {
+    for ((_, muteTestJson) in deleteMap) {
+        val response = khttp.delete(
+            "$buildServerUrl/app/rest/mutes/id:${muteTestJson.id}",
+            headers = headers,
+            auth = authUser
+        )
+        try {
+            checkResponseAndLog(response)
+        } catch (e: Exception) {
+            System.err.println(e.message)
+        }
+    }
+}
+
+private fun checkResponseAndLog(response: Response) {
+    val isResponseBad = response.connection.responseCode !in 200..299
+    if (isResponseBad) {
+        throw Exception(
+            "${response.request.method}-request to ${response.request.url} failed:\n" +
+                    "${response.text}\n" +
+                    "${response.request.data ?: ""}"
+        )
+    }
+}