Revert "[K/N][build] Cleanup obsolete build tasks"

This reverts commit a9cae3fe87.

kotlin-native/build-tools/build.gradle.kts

@@ -3,6 +3,7 @@
  * that can be found in the license/LICENSE.txt file.
  */
 
+import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 import java.util.Properties
 
@@ -29,6 +30,8 @@ val rootProperties = Properties().apply {
 
 val kotlinVersion = project.bootstrapKotlinVersion
 val slackApiVersion: String by rootProperties
+val ktorVersion: String by rootProperties
+val shadowVersion: String by rootProperties
 val metadataVersion: String by rootProperties
 
 group = "org.jetbrains.kotlin"
@@ -50,6 +53,9 @@ dependencies {
     implementation(commonDependency("org.jetbrains.kotlin:kotlin-reflect")) { isTransitive = false }
     implementation("org.jetbrains.kotlin:kotlin-build-gradle-plugin:${kotlinBuildProperties.buildGradlePluginVersion}")
 
+    implementation("com.ullink.slack:simpleslackapi:$slackApiVersion") {
+        exclude(group = "com.google.code.gson", module = "gson") // Workaround for Gradle dependency resolution error
+    }
     val versionProperties = Properties()
     project.rootProject.projectDir.resolve("gradle/versions.properties").inputStream().use { propInput ->
         versionProperties.load(propInput)
@@ -64,17 +70,18 @@ dependencies {
         }
     }
 
+    implementation("io.ktor:ktor-client-auth:$ktorVersion")
+    implementation("io.ktor:ktor-client-core:$ktorVersion")
+    implementation("io.ktor:ktor-client-cio:$ktorVersion")
+
     api(project(":native:kotlin-native-utils"))
     api(project(":kotlin-native-shared"))
     api(project(":kotlinx-metadata-klib"))
+    implementation("gradle.plugin.com.github.johnrengelman:shadow:${rootProject.extra["versions.shadow"]}")
 }
 
-kotlin {
-    sourceSets {
-        main {
-            kotlin.srcDir("$projectDir/../tools/benchmarks/shared/src/main/kotlin/report")
-        }
-    }
+sourceSets["main"].withConvention(KotlinSourceSet::class) {
+    kotlin.srcDir("$projectDir/../tools/benchmarks/shared/src/main/kotlin/report")
 }
 
 val compileKotlin: KotlinCompile by tasks

kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/CollisionDetector.kt

@@ -0,0 +1,85 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin
+
+import org.gradle.api.DefaultTask
+import org.gradle.api.tasks.TaskAction
+import org.gradle.api.artifacts.Configuration
+import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.InputFiles
+import java.io.File
+import java.util.zip.ZipFile
+
+/**
+ * Task to find out collisions before producing fat jar.
+ *
+ * @property configurations added to fat jar configurations
+ * @property resolvingRules a map containing rules to resolve conflicts. Key - conflicting file, value - a jar to copy the file from
+ * @property resolvingRulesWithRegexes a map containing rules to resolve conflicts. Key - regular expression describing conflicting file, value - a jar to copy the file from
+ * @property librariesWithIgnoredClassCollisions libraries which collision in class files are ignored
+ */
+open class CollisionDetector : DefaultTask() {
+    @InputFiles
+    var configurations = listOf<Configuration>()
+    @Input
+    val resolvingRules = mutableMapOf<String, String>()
+    @Input
+    val resolvingRulesWithRegexes = mutableMapOf<Regex, String>()
+    @Input
+    val librariesWithIgnoredClassCollisions = mutableListOf<String>()
+    val resolvedConflicts = mutableMapOf<String, File>()
+
+    // Key - filename, value - jar file containing it.
+    private val filesInfo = mutableMapOf<String, String>()
+
+    @TaskAction
+    fun run() {
+        configurations.forEach { configuration ->
+            configuration.files.filter { it.name.endsWith(".jar") }.forEach { processedFile ->
+                ZipFile(processedFile).use { zip ->
+                    zip.entries().asSequence().filterNot {
+                        it.isDirectory ||
+                        it.name.equals("META-INF/MANIFEST.MF", ignoreCase = true) ||
+                        it.name.equals("META-INF/versions/9/module-info.class", ignoreCase = true) ||
+                        it.name.startsWith("META-INF/services/", ignoreCase = true)
+                    }.forEach {
+                        val outputPath = it.name
+                        if (outputPath in filesInfo.keys) {
+                            val rule = resolvingRules.getOrElse(outputPath) {
+                                resolvingRulesWithRegexes.entries.firstOrNull { (key, _) -> key.matches(outputPath) }?.value
+                            }
+                            var ignoreJar = false
+                            if (rule != null && processedFile.name.startsWith(rule)) {
+                                resolvedConflicts[outputPath] = processedFile
+                            } else {
+                                // Skip class files from ignored libraries if version of libraries had collision are the same.
+                                val versionRegex = "\\d+\\.\\d+(\\.\\d+)?(-M\\d)?(-\\w+(-\\d+)?)?".toRegex()
+                                val currentVersion = versionRegex.find(processedFile.name)?.groupValues?.get(0)
+                                val collisionLibVersion = versionRegex.find(filesInfo.getValue(outputPath))?.groupValues?.get(0)
+                                if (outputPath.endsWith(".class") && currentVersion == collisionLibVersion) {
+                                    if (processedFile.name == filesInfo[outputPath]) {
+                                        ignoreJar = true
+                                    } else {
+                                        librariesWithIgnoredClassCollisions.forEach {
+                                            if (processedFile.name.startsWith(it)) {
+                                                ignoreJar = true
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                            if (rule == null && !ignoreJar) {
+                                error("Collision is detected. File $outputPath is found in ${filesInfo[outputPath]} and ${processedFile.name}")
+                            }
+                        } else {
+                            filesInfo[outputPath] = processedFile.name
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/CollisionTransformer.kt

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin
+
+import com.github.jengelman.gradle.plugins.shadow.transformers.Transformer
+import com.github.jengelman.gradle.plugins.shadow.transformers.TransformerContext
+import org.gradle.api.file.FileTreeElement
+import shadow.org.apache.tools.zip.ZipOutputStream
+import java.io.File
+import shadow.org.apache.commons.io.IOUtils
+import shadow.org.apache.tools.zip.ZipEntry
+import shadow.org.apache.tools.zip.ZipFile
+
+class CollisionTransformer : Transformer {
+    var resolvedConflicts = mutableMapOf<String, File>()
+    private val foundConflictsFiles = mutableSetOf<String>()
+
+    override fun getName() = "CollisionTransformer"
+
+    override fun canTransformResource(element: FileTreeElement): Boolean  {
+        val result = element.name in resolvedConflicts.keys
+        if (result) {
+            foundConflictsFiles.add(element.name)
+        }
+        return result
+    }
+
+    override fun transform(context: TransformerContext) {}
+
+    override fun hasTransformedResource(): Boolean {
+        return foundConflictsFiles.isNotEmpty()
+    }
+
+    override fun modifyOutputStream(jos: ZipOutputStream, preserveFileTimestamps: Boolean) {
+        foundConflictsFiles.forEach {
+            val entry = ZipEntry(it)
+            entry.time = TransformerContext.getEntryTimestamp(preserveFileTimestamps, entry.time)
+            jos.putNextEntry(entry)
+            val archive = ZipFile(resolvedConflicts[it])
+            archive.getInputStream(archive.getEntry(it)).use {
+                IOUtils.copyLarge(it, jos)
+            }
+            jos.closeEntry()
+        }
+        foundConflictsFiles.clear()
+    }
+}

kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/ExternalReportUtils.kt

@@ -0,0 +1,26 @@
+package org.jetbrains.kotlin
+
+
+import com.google.gson.annotations.*
+import com.google.gson.*
+import com.google.gson.stream.JsonReader
+import java.io.File
+import java.io.FileReader
+import java.io.PrintWriter
+
+
+data class ExternalTestReport(@Expose val statistics: Statistics, @Expose val groups: List<KonanTestGroupReport>)
+
+fun saveReport(reportFileName: String, statistics: Statistics){
+    File(reportFileName).apply {
+        parentFile.mkdirs()
+        PrintWriter(this).use {
+            it.append(gson.toJson(ExternalTestReport(statistics, emptyList())))
+        }
+    }
+}
+
+internal val gson = GsonBuilder().excludeFieldsWithoutExposeAnnotation().create()!!
+fun loadReport(reportFileName: String) : ExternalTestReport = JsonReader(FileReader(reportFileName)).use {
+    gson.fromJson(it, ExternalTestReport::class.java)
+}

kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/KonanTestSuiteReport.kt

@@ -18,6 +18,7 @@ data class Statistics(
         @Expose var error: Int = 0,
         @Expose var skipped: Int = 0) {
 
+
     fun pass(count: Int = 1) { passed += count }
 
     fun skip(count: Int = 1) { skipped += count }
@@ -47,10 +48,8 @@ data class KonanTestSuiteReport(@Expose val name: String, val tests: List<KonanT
 data class KonanTestCaseReport(@Expose val name: String, @Expose val status: TestStatus, @Expose val comment: String? = null)
 
 class KonanTestSuiteReportEnvironment(val name: String, val project: Project, val statistics: Statistics) {
-    private val tc = if ((System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE") != null)) TeamCityTestPrinter(project) else null
-
+    private val tc = if (Tc.enabled) TeamCityTestPrinter(project) else null
     val tests = mutableListOf<KonanTestCaseReport>()
-
     fun executeTest(testName: String, action:() -> Unit) {
         var test: KonanTestCaseReport?
         try {

kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/KotlinBuildPusher.kt

@@ -0,0 +1,106 @@
+package org.jetbrains.kotlin
+
+import io.ktor.client.HttpClient
+import io.ktor.client.request.get
+import io.ktor.client.request.header
+import io.ktor.client.request.post
+import io.ktor.content.TextContent
+import io.ktor.http.ContentType
+import kotlinx.coroutines.runBlocking
+import org.gradle.api.DefaultTask
+import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.TaskAction
+import org.gradle.api.tasks.options.Option
+
+@Suppress("UnstableApiUsage")
+open class KotlinBuildPusher : DefaultTask() {
+    @Input
+    @Option(option = "token", description = "Teamcity Bear token")
+    var token: String? = ""
+
+    @Input
+    @Option(option = "buildServer", description = "Teamcity server")
+    var buildServer: String = ""
+
+    @Input
+    @Option(option = "compilerConfigurationId", description = "Teamcity configuration id")
+    var compilerConfigurationId: String = "Kotlin_dev_Compiler"
+
+    @Input
+    @Option(option = "overrideConfigurationId", description = "Teamcity kotlin override configuration id")
+    var overrideConfigurationId: String = "Kotlin_dev_DeployMavenArtifacts_OverrideNative"
+
+    @Input
+    @Option(option = "kotlinVersion", description = "Kotlin Compiler Version")
+    var kotlinVersion: String = ""
+
+    @Input
+    @Option(option = "konanVersion", description = "Kotlin Native Compiler Version")
+    var konanVersion: String = ""
+
+    @TaskAction
+    fun run() {
+        requireNotNull(token, { "Teamcity Bear token required" })
+        val client = HttpClient()
+        runBlocking {
+            val buildId = client.get<String>(
+                scheme = "https",
+                host = buildServer,
+                path = "/app/rest/builds/buildType:(id:$compilerConfigurationId),number:$kotlinVersion/id"
+            ) {
+                header("Authorization", "Bearer $token")
+            }
+            project.logger.info("pusher: buildId: $buildId")
+
+            /**
+             * <build>
+             * <buildType id="Kotlin_dev_DeployMavenArtifacts_OverrideNative"/>
+             * <lastChanges>
+             * <change locator="build:2330798"/>
+             * </lastChanges>
+             * <properties>
+             * <property name="system.versions.kotlin-native" value="1.3.50-dev-10483"/>
+             * </properties>
+             * </build>
+             */
+            val content = buildString {
+                build {
+                    buildType(overrideConfigurationId)
+                    lastChanges {
+                        change(buildId)
+                    }
+                    properties {
+                        property("system.versions.kotlin-native", konanVersion)
+                    }
+                }
+            }
+            project.logger.info("pusher: content: \"$content\"")
+
+            val res = client.post<String>(
+                scheme = "https",
+                host = buildServer,
+                path = "/app/rest/buildQueue"
+            ) {
+                header("Authorization", "Bearer $token")
+                header("Origin", "https://$buildServer")
+                body = TextContent(content, ContentType.Application.Xml)
+            }
+            project.logger.info("pusher: result: \"$res\"")
+        }
+        client.close()
+    }
+}
+
+internal fun StringBuilder.paired(tag:String, body:StringBuilder.()->Unit) {
+    appendln("<$tag>")
+    body()
+    appendln("</$tag>")
+}
+
+internal fun StringBuilder.build(body: StringBuilder.() -> Unit) = paired("build", body)
+internal fun StringBuilder.buildType(id: String) = appendln("<buildType id=\"$id\"/>")
+internal fun StringBuilder.lastChanges(body: StringBuilder.() -> Unit) = paired("lastChanges", body)
+internal fun StringBuilder.change(build: String) = appendln("<change locator=\"build:$build\"/>")
+internal fun StringBuilder.properties(body: StringBuilder.() -> Unit) = paired("properties", body)
+internal fun StringBuilder.property(key: String, value: String) = appendln("<property name=\"$key\" value=\"$value\"/>")
+

kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/Reporter.kt

@@ -0,0 +1,109 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+package org.jetbrains.kotlin
+
+import com.ullink.slack.simpleslackapi.impl.SlackSessionFactory
+import org.gradle.api.DefaultTask
+import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.TaskAction
+import java.io.FileInputStream
+import java.util.*
+
+
+internal object Tc {
+    private val teamcityConfig = System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE")
+    val enabled:Boolean = (teamcityConfig != null)
+    private val buildConfig by lazy {
+        teamcityConfig ?: return@lazy null
+        val properties = Properties()
+        properties.load(FileInputStream(teamcityConfig))
+        properties
+    }
+    val buildId = buildConfig?.getProperty("teamcity.build.id")
+    val buildTypeId = buildConfig?.getProperty("teamcity.buildType.id")
+    val konanReporterToken = buildConfig?.getProperty("konan-reporter-token")
+    val konanChannelName = buildConfig?.getProperty("konan-channel-name")
+
+}
+
+private fun buildLogUrlTab(buildId: String?, buildTypeId: String?): String = tabUrl(buildId, buildTypeId, "buildLog")
+
+private fun tabUrl(buildId: String?, buildTypeId: String?, tab: String?): String =
+        "http://buildserver.labs.intellij.net/viewLog.html?buildId=$buildId&buildTypeId=$buildTypeId&tab=$tab"
+
+private fun testReportUrl(buildId: String?, buildTypeId: String?): String = tabUrl(buildId, buildTypeId, "testsInfo")
+
+private fun sendTextToSlack(report: String) {
+    with(SlackSessionFactory.createWebSocketSlackSession(Tc.konanReporterToken)) {
+        connect()
+        sendMessage(findChannelByName(Tc.konanChannelName),
+                "Hello, аборигены Котлина!\n текущий статус:\n$report")
+        disconnect()
+    }
+}
+
+private fun reportEpilogue(): String {
+    val logUrl = buildLogUrlTab(Tc.buildId, Tc.buildTypeId)
+    val testReportUrl = testReportUrl(Tc.buildId, Tc.buildTypeId)
+    return "\nlog url: $logUrl\ntest report url: $testReportUrl"
+}
+
+private val Statistics.report
+        get() = "total: $total\npassed: $passed\nfailed: $failed\nerror: $error\nskipped: $skipped"
+
+private val Statistics.oneLineReport
+    get() = "(total: $total, passed: $passed, failed: $failed, error: $error, skipped: $skipped)"
+
+
+open class Reporter : DefaultTask() {
+
+    @Input
+    lateinit var reportHome: String
+
+    @TaskAction
+    fun report() {
+
+        val reportJson = loadReport("$reportHome/external/results.json")
+
+        val report: String =
+            "${reportJson.statistics.report}\n ${reportEpilogue()}"
+
+        project.logger.info(report)
+        if (doSlackSending())
+            sendTextToSlack(report)
+    }
+}
+
+private fun DefaultTask.doSlackSending() = !project.hasProperty("build.reporter.noSlack")
+        || !project.property("build.reporter.noSlack").toString().toBoolean()
+
+open class NightlyReporter: DefaultTask() {
+    @Input
+    lateinit var externalMacosReport:String
+    @Input
+    lateinit var externalLinuxReport:String
+    @Input
+    lateinit var externalWindowsReport:String
+
+    @TaskAction
+    fun report() {
+        val externalMacosJsonReport = loadReport("${project.rootDir.absolutePath}/$externalMacosReport")
+        val externalLinuxJsonReport = loadReport("${project.rootDir.absolutePath}/$externalLinuxReport")
+        val externalWindowsJsonReport = loadReport("${project.rootDir.absolutePath}/$externalWindowsReport")
+        val report = buildString {
+            append("Mac OS ")
+            appendln(externalMacosJsonReport.statistics.oneLineReport)
+            append("Linux ")
+            appendln(externalLinuxJsonReport.statistics.oneLineReport)
+            append("Windows ")
+            appendln(externalWindowsJsonReport.statistics.oneLineReport)
+            appendln(reportEpilogue())
+        }
+        project.logger.info(report)
+        if (doSlackSending())
+            sendTextToSlack(report)
+    }
+}

kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/Utils.kt

@@ -6,7 +6,6 @@
 
 package org.jetbrains.kotlin
 
-import com.google.gson.GsonBuilder
 import org.gradle.api.Project
 import org.gradle.api.Task
 import org.jetbrains.kotlin.konan.properties.loadProperties
@@ -482,5 +481,3 @@ internal val Project.testTargetConfigurables: Configurables
         val testTarget = project.testTarget
         return platformManager.platform(testTarget).configurables
     }
-
-internal val gson = GsonBuilder().excludeFieldsWithoutExposeAnnotation().create()!!

kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/Wrappers.kt

@@ -0,0 +1,53 @@
+package org.jetbrains.kotlin
+
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.Optional
+import org.gradle.api.tasks.wrapper.Wrapper
+
+open class CustomWrapper : Wrapper() {
+    internal lateinit var mainWrapperTask: Wrapper
+
+    val mainWrapperVersion: String
+        @Input get() = mainWrapperTask.gradleVersion
+    val mainWrapperDistrSha256: String?
+        @Optional @Input get() = mainWrapperTask.distributionSha256Sum
+}
+
+open class WrappersExtension {
+    var projects = mutableListOf<Any>()
+    var distributionType = Wrapper.DistributionType.BIN
+}
+
+class GradleWrappers : Plugin<Project> {
+
+    override fun apply(project: Project): Unit = with(project) {
+        val mainWrapperTask = tasks.findByName("wrapper") as? Wrapper ?: return@with
+        val wrappers = extensions.create(
+                WrappersExtension::class.java,
+                "wrappers",
+                WrappersExtension::class.java
+        )
+        afterEvaluate {
+            wrappers.projects.map { file(it) }.forEach {
+                tasks.create("${it.name}Wrapper", CustomWrapper::class.java).apply {
+                    this.mainWrapperTask = mainWrapperTask
+                    jarFile = it.resolve("gradle/wrapper/gradle-wrapper.jar")
+                    scriptFile = it.resolve("gradlew")
+                    distributionType = wrappers.distributionType
+                    mainWrapperTask.dependsOn(this)
+
+                    // Get these parameters from the main wrapper task to support
+                    // command line options like --gradle-version.
+                    // Gradle doesn't provide access to the values passed in command line
+                    // at the configuration phase, so we have to get these values at the execution phase.
+                    doFirst {
+                        gradleVersion = mainWrapperVersion
+                        distributionSha256Sum = mainWrapperDistrSha256
+                    }
+                }
+            }
+        }
+    }
+}