Creating fat jar with shadow plugin and collisions detector (#3466)

Interop/Runtime/build.gradle

@@ -67,7 +67,7 @@ repositories {
 
 dependencies {
     compile "org.jetbrains.kotlin:kotlin-stdlib:$buildKotlinVersion"
-    compile "org.jetbrains.kotlin:kotlin-reflect:$buildKotlinVersion"
+    compile "org.jetbrains.kotlin:kotlin-reflect:$kotlinVersion"
 }
 
 sourceSets.main.kotlin.srcDirs += "src/jvm/kotlin"

backend.native/build.gradle

@@ -114,8 +114,6 @@ configurations {
     kotlin_reflect_jar
     kotlin_script_runtime_jar
     kotlin_native_utils_jar
-    kotlin_util_io_jar
-    kotlin_util_klib_jar
     trove4j_jar
     kotlinCommonSources
 
@@ -136,8 +134,6 @@ dependencies {
     kotlin_reflect_jar "$kotlinReflectModule@jar"
     kotlin_script_runtime_jar "$kotlinScriptRuntimeModule@jar"
     kotlin_native_utils_jar "$kotlinNativeUtilsModule@jar"
-    kotlin_util_io_jar "$kotlinUtilIoModule@jar"
-    kotlin_util_klib_jar "$kotlinUtilKlibModule@jar"
 
     [kotlinCommonStdlibModule, kotlinTestCommonModule, kotlinTestAnnotationsCommonModule].each {
         kotlinCommonSources(it) { transitive = false }
@@ -238,7 +234,7 @@ jar {
     dependsOn ':runtime:hostRuntime', 'external_jars'
 }
 
-def externalJars = ['compiler', 'stdlib', 'reflect', 'script_runtime', 'native_utils', 'util_io', 'util_klib']
+def externalJars = ['compiler', 'stdlib', 'reflect', 'script_runtime', 'native_utils']
 
 task trove4jCopy(type: Copy) {
     from configurations.getByName("trove4j_jar") {

build-tools/build.gradle.kts

@@ -30,6 +30,7 @@ group = "org.jetbrains.kotlin"
 version = konanVersion
 
 repositories {
+    jcenter()
     maven(kotlinCompilerRepo)
     maven(buildKotlinCompilerRepo)
     maven("https://cache-redirector.jetbrains.com/maven-central")
@@ -53,6 +54,7 @@ dependencies {
 
     // Located in <repo root>/shared and always provided by the composite build.
     api("org.jetbrains.kotlin:kotlin-native-shared:$konanVersion")
+    implementation("com.github.jengelman.gradle.plugins:shadow:5.1.0")
 }
 
 sourceSets["main"].withConvention(KotlinSourceSet::class) {

build-tools/src/main/kotlin/org/jetbrains/kotlin/CollisionDetector.kt

@@ -0,0 +1,92 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin
+
+import org.gradle.api.DefaultTask
+import org.gradle.api.tasks.TaskAction
+import org.gradle.api.artifacts.Configuration
+import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.InputFiles
+import java.io.File
+
+fun <K, V> Map<K, V>.firstOrNull(predicate:(K, V) -> Boolean): V? {
+    var foundValue: V? = null
+    forEach {
+        if (predicate(it.key, it.value)) {
+            foundValue = it.value
+            return@forEach
+        }
+    }
+    return foundValue
+}
+
+
+/**
+ * Task to find out collisions before producing fat jar.
+ *
+ * @property configurations added to fat jar configurations
+ * @property ignoredFiles excluded from analysis files
+ * @property resolvingRules a map containing rules to resolve conflicts. Key - conflicting file, value - a jar to copy the file from
+ * @property resolvingRulesWithRegexes a map containing rules to resolve conflicts. Key - regular expression describing conflicting file, value - a jar to copy the file from
+ * @property librariesWithIgnoredClassCollisions libraries which collision in class files are ignored
+ */
+open class CollisionDetector : DefaultTask() {
+    @InputFiles
+    var configurations = listOf<Configuration>()
+    @Input
+    val ignoredFiles = mutableListOf<String>()
+    @Input
+    val resolvingRules = mutableMapOf<String, String>()
+    @Input
+    val resolvingRulesWithRegexes = mutableMapOf<Regex, String>()
+    @Input
+    val librariesWithIgnoredClassCollisions = mutableListOf<String>()
+    val resolvedConflicts = mutableMapOf<String, File>()
+
+    // Key - filename, value - jar file containing it.
+    private val filesInfo = mutableMapOf<String, String>()
+
+    @TaskAction
+    fun run() {
+        configurations.forEach { configuration ->
+            configuration.files.filter { it.name.endsWith(".jar") }.forEach { processedFile ->
+                project.zipTree(processedFile.absolutePath).matching { it.exclude(ignoredFiles) }.forEach {
+                    val outputPath = it.absolutePath.substringAfter(processedFile.name).substringAfter("/")
+                    if (outputPath in filesInfo.keys) {
+                        val rule = resolvingRules.getOrElse(outputPath) {
+                            resolvingRulesWithRegexes.firstOrNull { key, _ -> key.matches(outputPath) }
+                        }
+                        var ignoreJar = false
+                        if (rule != null && processedFile.name.startsWith(rule)) {
+                            resolvedConflicts[outputPath] = processedFile
+                        } else {
+                            // Skip class files from ignored libraries if version of libraries had collision are the same.
+                            val versionRegex = "\\d+\\.\\d+(\\.\\d+)?(-\\w+(-\\d+)?)?".toRegex()
+                            val currentVersion = versionRegex.find(processedFile.name)?.groupValues?.get(0)
+                            val collisionLibVersion = versionRegex.find(filesInfo.getValue(outputPath))?.groupValues?.get(0)
+                            if (outputPath.endsWith(".class") && currentVersion == collisionLibVersion) {
+                                if (processedFile.name == filesInfo[outputPath]) {
+                                    ignoreJar = true
+                                } else {
+                                    librariesWithIgnoredClassCollisions.forEach {
+                                        if (processedFile.name.startsWith(it)) {
+                                            ignoreJar = true
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                        if (rule == null && !ignoreJar) {
+                            error("Collision is detected. File $outputPath is found in ${filesInfo[outputPath]} and ${processedFile.name}")
+                        }
+                    } else {
+                        filesInfo[outputPath] = processedFile.name
+                    }
+                }
+            }
+        }
+    }
+}

build-tools/src/main/kotlin/org/jetbrains/kotlin/CollisionTransformer.kt

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin
+
+import com.github.jengelman.gradle.plugins.shadow.transformers.Transformer
+import com.github.jengelman.gradle.plugins.shadow.transformers.TransformerContext
+import org.gradle.api.file.FileTreeElement
+import shadow.org.apache.tools.zip.ZipOutputStream
+import java.io.File
+import shadow.org.apache.commons.io.IOUtils
+import shadow.org.apache.tools.zip.ZipEntry
+import shadow.org.apache.tools.zip.ZipFile
+
+class CollisionTransformer: Transformer {
+    var resolvedConflicts = mutableMapOf<String, File>()
+    private val foundConflictsFiles = mutableSetOf<String>()
+
+    override fun canTransformResource(element: FileTreeElement): Boolean  {
+        val result = element.name in resolvedConflicts.keys
+        if (result) {
+            foundConflictsFiles.add(element.name)
+        }
+        return result
+    }
+
+    override fun transform(context: TransformerContext) {}
+
+    override fun hasTransformedResource(): Boolean {
+        return foundConflictsFiles.isNotEmpty()
+    }
+
+    override fun modifyOutputStream(jos: ZipOutputStream, preserveFileTimestamps: Boolean) {
+        foundConflictsFiles.forEach {
+            val entry = ZipEntry(it)
+            entry.time = TransformerContext.getEntryTimestamp(preserveFileTimestamps, entry.time)
+            jos.putNextEntry(entry)
+            val archive = ZipFile(resolvedConflicts[it])
+            archive.getInputStream(archive.getEntry(it)).use {
+                IOUtils.copyLarge(it, jos)
+            }
+            jos.closeEntry()
+        }
+        foundConflictsFiles.clear()
+    }
+}

build.gradle

@@ -13,12 +13,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
+import kotlin.text.Regex
 import org.jetbrains.kotlin.konan.target.*
 import org.jetbrains.kotlin.konan.util.*
 import org.jetbrains.kotlin.CopySamples
 import org.jetbrains.kotlin.CopyCommonSources
 import org.jetbrains.kotlin.PlatformInfo
 import org.jetbrains.kotlin.KotlinBuildPusher
+import org.jetbrains.kotlin.CollisionDetector
+import org.jetbrains.kotlin.CollisionTransformer
+import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
 
 buildscript {
     apply from: "gradle/kotlinGradlePlugin.gradle"
@@ -27,12 +32,14 @@ buildscript {
         maven { url kotlinCompilerRepo }
         maven { url "https://kotlin.bintray.com/kotlinx" }
         maven { url "https://cache-redirector.jetbrains.com/maven-central" }
+        jcenter()
     }
 
     dependencies {
         classpath "org.jetbrains.kotlin:kotlin-native-utils:$kotlinVersion"
         classpath "org.jetbrains.kotlin:kotlin-native-shared:$konanVersion"
         classpath "org.jetbrains.kotlin:kotlin-native-build-tools:$konanVersion"
+        classpath 'com.github.jengelman.gradle.plugins:shadow:5.1.0'
     }
 }
 import org.jetbrains.kotlin.konan.*
@@ -63,8 +70,6 @@ ext {
     kotlinReflectModule="org.jetbrains.kotlin:kotlin-reflect:${kotlinVersion}"
     kotlinScriptRuntimeModule="org.jetbrains.kotlin:kotlin-script-runtime:${kotlinVersion}"
     kotlinNativeUtilsModule="org.jetbrains.kotlin:kotlin-native-utils:${kotlinVersion}"
-    kotlinUtilIoModule="org.jetbrains.kotlin:kotlin-util-io:${kotlinVersion}"
-    kotlinUtilKlibModule="org.jetbrains.kotlin:kotlin-util-klib:${kotlinVersion}"
     kotlinUtilKliMetadatabModule="org.jetbrains.kotlin:kotlin-util-klib-metadata:${kotlinVersion}"
 
     konanVersionFull = KonanVersionGeneratedKt.getCurrentKonanVersion()
@@ -72,6 +77,13 @@ ext {
 }
 
 allprojects {
+    buildscript {
+        repositories {
+            maven {
+                url 'https://cache-redirector.jetbrains.com/jcenter'
+            }
+        }
+    }
     if (path != ":dependencies") {
         evaluationDependsOn(":dependencies")
     }
@@ -166,6 +178,7 @@ void loadCommandLineProperties() {
 configurations {
     ftpAntTask
     kotlinCommonSources
+    distPack
 }
 
 dependencies {
@@ -173,6 +186,16 @@ dependencies {
     [kotlinCommonStdlibModule, kotlinTestCommonModule, kotlinTestAnnotationsCommonModule].each {
         kotlinCommonSources(it) { transitive = false }
     }
+    distPack project(':Interop:Runtime')
+    distPack project(':Interop:Indexer')
+    distPack project(':Interop:StubGenerator')
+    distPack project(':backend.native')
+    distPack project(':utilities')
+    distPack project(':klib')
+    distPack project(path: ':endorsedLibraries:kotlinx.cli', configuration: "jvmRuntimeElements")
+    distPack "org.jetbrains.kotlin:kotlin-native-shared:$konanVersion"
+    distPack "org.jetbrains.kotlin:konan.metadata:$konanVersion"
+    distPack  "org.jetbrains.kotlin:konan.serializer:$konanVersion"
 }
 
 task sharedJar {
@@ -226,42 +249,60 @@ task distSources {
     dependsOn(distEndorsedSources)
 }
 
-def jarContent(def jarTask, FileTree baseFileTree, Iterable<String> excludedList = []) {
+task detectJarCollision(type: CollisionDetector) {
-    jarTask.from {
+    ignoredFiles.addAll(["META-INF/MANIFEST.MF", "META-INF/services/**"])
-        baseFileTree
+    configurations = [project.configurations.distPack]
-                .matching { include("*.jar") }
+    resolvingRules["META-INF/kotlin-util-klib.kotlin_module"] = "kotlin-compiler"
-                .collect { zipTree("${it.absolutePath}").matching { exclude(excludedList) } }
+    resolvingRules["META-INF/kotlin-util-io.kotlin_module"] = "kotlin-compiler"
+    resolvingRules["META-INF/descriptors.jvm.kotlin_module"] = "kotlin-compiler"
+    resolvingRules["META-INF/descriptors.kotlin_module"] = "kotlin-compiler"
+    resolvingRules["META-INF/descriptors.runtime.kotlin_module"] = "kotlin-compiler"
+    resolvingRules["META-INF/deserialization.kotlin_module"] = "kotlin-compiler"
+    resolvingRules["META-INF/metadata.jvm.kotlin_module"] = "kotlin-compiler"
+    resolvingRules["META-INF/metadata.kotlin_module"] = "kotlin-compiler"
+    resolvingRules["META-INF/type-system.kotlin_module"] = "kotlin-compiler"
+    resolvingRules["META-INF/util.runtime.kotlin_module"] = "kotlin-compiler"
+    resolvingRules["kotlin/annotation/annotation.kotlin_builtins"] = "kotlin-compiler"
+    resolvingRules["kotlin/collections/collections.kotlin_builtins"] = "kotlin-compiler"
+    resolvingRules["kotlin/coroutines/coroutines.kotlin_builtins"] = "kotlin-compiler"
+    resolvingRules["kotlin/internal/internal.kotlin_builtins"] = "kotlin-compiler"
+    resolvingRules["kotlin/kotlin.kotlin_builtins"] = "kotlin-compiler"
+    resolvingRules["kotlin/ranges/ranges.kotlin_builtins"] = "kotlin-compiler"
+    resolvingRules["kotlin/reflect/reflect.kotlin_builtins"] = "kotlin-compiler"
+    resolvingRules["META-INF/maven/com.google.protobuf/protobuf-java/pom.properties"] = "kotlin-compiler"
+    resolvingRules["META-INF/maven/com.google.protobuf/protobuf-java/pom.xml"] = "kotlin-compiler"
+    librariesWithIgnoredClassCollisions.addAll(["kotlin-util-klib", "kotlin-util-io"])
+    if (project.hasProperty('kotlinProjectPath')) {
+        resolvingRulesWithRegexes[new Regex("META-INF/.+\\.kotlin_module")] = "kotlin-compiler"
+        resolvingRules["META-INF/extensions/compiler.xml"] = "kotlin-compiler"
+        resolvingRules["kotlinManifest.properties"] = "kotlin-compiler"
+        librariesWithIgnoredClassCollisions.addAll(["util", "container", "resolution", "serialization", "psi", "frontend",
+                                                    "frontend.common", "frontend.java", "cli-common", "ir.tree",
+                                                    "ir.psi2ir", "ir.backend.common", "backend.jvm", "backend.js",
+                                                    "backend.wasm", "ir.serialization.common", "ir.serialization.js",
+                                                    "backend-common", "backend", "plugin-api", "light-classes", "cli",
+                                                    "cli-js", "incremental-compilation-impl", "js.ast", "js.serializer",
+                                                    "js.parser", "js.frontend", "js.translator", "js.dce", "metadata",
+                                                    "metadata.jvm", "descriptors", "descriptors.jvm", "descriptors.runtime",
+                                                    "deserialization", "util.runtime", "type-system", "cones", "resolve",
+                                                    "tree", "psi2fir", "fir2ir", "java", "kotlin-build-common"])
     }
 }
 
-task mergeJars(type: Jar) {
+task shadowJar(type: ShadowJar) {
-    dependsOn ':backend.native:jar'
+    dependsOn ':detectJarCollision'
-    dependsOn ':utilities:jar'
+    mergeServiceFiles()
-    dependsOn ':klib:jar'
+    destinationDirectory.set(file("$distDir/konan/lib"))
-    dependsOn ':sharedJar'
+    archiveBaseName.set("kotlin-native")
-    dependsOn ':endorsedLibraries:jvmJar'
+    configurations = [project.configurations.distPack]
-
+    archiveClassifier.set(null)
-    archiveFileName = 'kotlin-native.jar'
+    transform(CollisionTransformer.class) {
-    destinationDirectory = file("$distDir/konan/lib")
+        resolvedConflicts = detectJarCollision.resolvedConflicts
-    
+    }
-    jarContent(it, project(':backend.native').fileTree('build/external_jars'),
-            ['META-INF/services/org.jetbrains.kotlin.resolve.ExternalOverridabilityCondition'])
-    jarContent(it, project(':backend.native').fileTree('build/libs'))
-    jarContent(it, project(':utilities').fileTree('build/libs'))
-    jarContent(it, project('Interop').fileTree('Runtime/build/libs'))
-    jarContent(it, project('Interop').fileTree('Indexer/build/libs'))
-    jarContent(it, project('Interop').fileTree('StubGenerator/build/libs'))
-
-    jarContent(it, project(':klib').fileTree('build/libs'))
-    jarContent(it, project(':endorsedLibraries:kotlinx.cli').fileTree('build/libs'))
-    jarContent(it, fileTree("${gradle.includedBuild('kotlin-native-shared').projectDir}/build/libs"))
-    jarContent(it, fileTree("${gradle.includedBuild('konan.metadata').projectDir}/build/libs"))
-    jarContent(it, fileTree("${gradle.includedBuild('konan.serializer').projectDir}/build/libs"))
-
 }
 
 task distCompiler(type: Copy) {
-    dependsOn ':mergeJars'
+    dependsOn ':shadowJar'
 
     destinationDir distDir
 

settings.gradle

@@ -62,6 +62,7 @@ if (hasProperty("kotlinProjectPath")) {
             substitute module("org.jetbrains.kotlin:kotlin-util-klib:$kotlinVersion") with project(':kotlin-util-klib')
             substitute module("org.jetbrains.kotlin:kotlin-util-klib-metadata:$kotlinVersion") with project(':kotlin-util-klib-metadata')
             substitute module("org.jetbrains.kotlin:kotlin-native-utils:$kotlinVersion") with project(':kotlin-native:kotlin-native-utils')
+            substitute module("org.jetbrains.kotlin:kotlin-reflect:$kotlinVersion") with project(':kotlin-reflect')
         }
     }
 }

utilities/build.gradle

@@ -16,8 +16,6 @@ repositories {
 
 dependencies {
     compile "org.jetbrains.kotlin:kotlin-stdlib:$buildKotlinVersion"
-    compile "org.jetbrains.kotlin:kotlin-util-io:$kotlinVersion"
-    compile "org.jetbrains.kotlin:kotlin-util-klib:$kotlinVersion"
     compile project(':backend.native')
     compile project(':Interop:StubGenerator')
     compile project(':klib')