From e21ab4f1e728e3fe875343d83c98ede9fb3d6bd9 Mon Sep 17 00:00:00 2001 From: LepilkinaElena Date: Mon, 21 Oct 2019 12:32:06 +0300 Subject: [PATCH] Creating fat jar with shadow plugin and collisions detector (#3466) --- Interop/Runtime/build.gradle | 2 +- backend.native/build.gradle | 6 +- build-tools/build.gradle.kts | 2 + .../org/jetbrains/kotlin/CollisionDetector.kt | 92 +++++++++++++++ .../jetbrains/kotlin/CollisionTransformer.kt | 48 ++++++++ build.gradle | 105 ++++++++++++------ settings.gradle | 1 + utilities/build.gradle | 2 - 8 files changed, 218 insertions(+), 40 deletions(-) create mode 100644 build-tools/src/main/kotlin/org/jetbrains/kotlin/CollisionDetector.kt create mode 100644 build-tools/src/main/kotlin/org/jetbrains/kotlin/CollisionTransformer.kt diff --git a/Interop/Runtime/build.gradle b/Interop/Runtime/build.gradle index 94f2418d80f..b1ad103f969 100644 --- a/Interop/Runtime/build.gradle +++ b/Interop/Runtime/build.gradle @@ -67,7 +67,7 @@ repositories { dependencies { compile "org.jetbrains.kotlin:kotlin-stdlib:$buildKotlinVersion" - compile "org.jetbrains.kotlin:kotlin-reflect:$buildKotlinVersion" + compile "org.jetbrains.kotlin:kotlin-reflect:$kotlinVersion" } sourceSets.main.kotlin.srcDirs += "src/jvm/kotlin" diff --git a/backend.native/build.gradle b/backend.native/build.gradle index e9aa3fc4194..cb9a4d7ee1a 100644 --- a/backend.native/build.gradle +++ b/backend.native/build.gradle @@ -114,8 +114,6 @@ configurations { kotlin_reflect_jar kotlin_script_runtime_jar kotlin_native_utils_jar - kotlin_util_io_jar - kotlin_util_klib_jar trove4j_jar kotlinCommonSources @@ -136,8 +134,6 @@ dependencies { kotlin_reflect_jar "$kotlinReflectModule@jar" kotlin_script_runtime_jar "$kotlinScriptRuntimeModule@jar" kotlin_native_utils_jar "$kotlinNativeUtilsModule@jar" - kotlin_util_io_jar "$kotlinUtilIoModule@jar" - kotlin_util_klib_jar "$kotlinUtilKlibModule@jar" [kotlinCommonStdlibModule, kotlinTestCommonModule, kotlinTestAnnotationsCommonModule].each { kotlinCommonSources(it) { transitive = false } @@ -238,7 +234,7 @@ jar { dependsOn ':runtime:hostRuntime', 'external_jars' } -def externalJars = ['compiler', 'stdlib', 'reflect', 'script_runtime', 'native_utils', 'util_io', 'util_klib'] +def externalJars = ['compiler', 'stdlib', 'reflect', 'script_runtime', 'native_utils'] task trove4jCopy(type: Copy) { from configurations.getByName("trove4j_jar") { diff --git a/build-tools/build.gradle.kts b/build-tools/build.gradle.kts index 813f958da21..ee86e29df97 100644 --- a/build-tools/build.gradle.kts +++ b/build-tools/build.gradle.kts @@ -30,6 +30,7 @@ group = "org.jetbrains.kotlin" version = konanVersion repositories { + jcenter() maven(kotlinCompilerRepo) maven(buildKotlinCompilerRepo) maven("https://cache-redirector.jetbrains.com/maven-central") @@ -53,6 +54,7 @@ dependencies { // Located in /shared and always provided by the composite build. api("org.jetbrains.kotlin:kotlin-native-shared:$konanVersion") + implementation("com.github.jengelman.gradle.plugins:shadow:5.1.0") } sourceSets["main"].withConvention(KotlinSourceSet::class) { diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/CollisionDetector.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/CollisionDetector.kt new file mode 100644 index 00000000000..1687def49b8 --- /dev/null +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/CollisionDetector.kt @@ -0,0 +1,92 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin + +import org.gradle.api.DefaultTask +import org.gradle.api.tasks.TaskAction +import org.gradle.api.artifacts.Configuration +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputFiles +import java.io.File + +fun Map.firstOrNull(predicate:(K, V) -> Boolean): V? { + var foundValue: V? = null + forEach { + if (predicate(it.key, it.value)) { + foundValue = it.value + return@forEach + } + } + return foundValue +} + + +/** + * Task to find out collisions before producing fat jar. + * + * @property configurations added to fat jar configurations + * @property ignoredFiles excluded from analysis files + * @property resolvingRules a map containing rules to resolve conflicts. Key - conflicting file, value - a jar to copy the file from + * @property resolvingRulesWithRegexes a map containing rules to resolve conflicts. Key - regular expression describing conflicting file, value - a jar to copy the file from + * @property librariesWithIgnoredClassCollisions libraries which collision in class files are ignored + */ +open class CollisionDetector : DefaultTask() { + @InputFiles + var configurations = listOf() + @Input + val ignoredFiles = mutableListOf() + @Input + val resolvingRules = mutableMapOf() + @Input + val resolvingRulesWithRegexes = mutableMapOf() + @Input + val librariesWithIgnoredClassCollisions = mutableListOf() + val resolvedConflicts = mutableMapOf() + + // Key - filename, value - jar file containing it. + private val filesInfo = mutableMapOf() + + @TaskAction + fun run() { + configurations.forEach { configuration -> + configuration.files.filter { it.name.endsWith(".jar") }.forEach { processedFile -> + project.zipTree(processedFile.absolutePath).matching { it.exclude(ignoredFiles) }.forEach { + val outputPath = it.absolutePath.substringAfter(processedFile.name).substringAfter("/") + if (outputPath in filesInfo.keys) { + val rule = resolvingRules.getOrElse(outputPath) { + resolvingRulesWithRegexes.firstOrNull { key, _ -> key.matches(outputPath) } + } + var ignoreJar = false + if (rule != null && processedFile.name.startsWith(rule)) { + resolvedConflicts[outputPath] = processedFile + } else { + // Skip class files from ignored libraries if version of libraries had collision are the same. + val versionRegex = "\\d+\\.\\d+(\\.\\d+)?(-\\w+(-\\d+)?)?".toRegex() + val currentVersion = versionRegex.find(processedFile.name)?.groupValues?.get(0) + val collisionLibVersion = versionRegex.find(filesInfo.getValue(outputPath))?.groupValues?.get(0) + if (outputPath.endsWith(".class") && currentVersion == collisionLibVersion) { + if (processedFile.name == filesInfo[outputPath]) { + ignoreJar = true + } else { + librariesWithIgnoredClassCollisions.forEach { + if (processedFile.name.startsWith(it)) { + ignoreJar = true + } + } + } + } + } + if (rule == null && !ignoreJar) { + error("Collision is detected. File $outputPath is found in ${filesInfo[outputPath]} and ${processedFile.name}") + } + } else { + filesInfo[outputPath] = processedFile.name + } + } + } + } + } +} diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/CollisionTransformer.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/CollisionTransformer.kt new file mode 100644 index 00000000000..ccf81693bd6 --- /dev/null +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/CollisionTransformer.kt @@ -0,0 +1,48 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin + +import com.github.jengelman.gradle.plugins.shadow.transformers.Transformer +import com.github.jengelman.gradle.plugins.shadow.transformers.TransformerContext +import org.gradle.api.file.FileTreeElement +import shadow.org.apache.tools.zip.ZipOutputStream +import java.io.File +import shadow.org.apache.commons.io.IOUtils +import shadow.org.apache.tools.zip.ZipEntry +import shadow.org.apache.tools.zip.ZipFile + +class CollisionTransformer: Transformer { + var resolvedConflicts = mutableMapOf() + private val foundConflictsFiles = mutableSetOf() + + override fun canTransformResource(element: FileTreeElement): Boolean { + val result = element.name in resolvedConflicts.keys + if (result) { + foundConflictsFiles.add(element.name) + } + return result + } + + override fun transform(context: TransformerContext) {} + + override fun hasTransformedResource(): Boolean { + return foundConflictsFiles.isNotEmpty() + } + + override fun modifyOutputStream(jos: ZipOutputStream, preserveFileTimestamps: Boolean) { + foundConflictsFiles.forEach { + val entry = ZipEntry(it) + entry.time = TransformerContext.getEntryTimestamp(preserveFileTimestamps, entry.time) + jos.putNextEntry(entry) + val archive = ZipFile(resolvedConflicts[it]) + archive.getInputStream(archive.getEntry(it)).use { + IOUtils.copyLarge(it, jos) + } + jos.closeEntry() + } + foundConflictsFiles.clear() + } +} \ No newline at end of file diff --git a/build.gradle b/build.gradle index d4c2a07f7d6..34f6be2e25d 100644 --- a/build.gradle +++ b/build.gradle @@ -13,12 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +import kotlin.text.Regex import org.jetbrains.kotlin.konan.target.* import org.jetbrains.kotlin.konan.util.* import org.jetbrains.kotlin.CopySamples import org.jetbrains.kotlin.CopyCommonSources import org.jetbrains.kotlin.PlatformInfo import org.jetbrains.kotlin.KotlinBuildPusher +import org.jetbrains.kotlin.CollisionDetector +import org.jetbrains.kotlin.CollisionTransformer +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar buildscript { apply from: "gradle/kotlinGradlePlugin.gradle" @@ -27,12 +32,14 @@ buildscript { maven { url kotlinCompilerRepo } maven { url "https://kotlin.bintray.com/kotlinx" } maven { url "https://cache-redirector.jetbrains.com/maven-central" } + jcenter() } dependencies { classpath "org.jetbrains.kotlin:kotlin-native-utils:$kotlinVersion" classpath "org.jetbrains.kotlin:kotlin-native-shared:$konanVersion" classpath "org.jetbrains.kotlin:kotlin-native-build-tools:$konanVersion" + classpath 'com.github.jengelman.gradle.plugins:shadow:5.1.0' } } import org.jetbrains.kotlin.konan.* @@ -63,8 +70,6 @@ ext { kotlinReflectModule="org.jetbrains.kotlin:kotlin-reflect:${kotlinVersion}" kotlinScriptRuntimeModule="org.jetbrains.kotlin:kotlin-script-runtime:${kotlinVersion}" kotlinNativeUtilsModule="org.jetbrains.kotlin:kotlin-native-utils:${kotlinVersion}" - kotlinUtilIoModule="org.jetbrains.kotlin:kotlin-util-io:${kotlinVersion}" - kotlinUtilKlibModule="org.jetbrains.kotlin:kotlin-util-klib:${kotlinVersion}" kotlinUtilKliMetadatabModule="org.jetbrains.kotlin:kotlin-util-klib-metadata:${kotlinVersion}" konanVersionFull = KonanVersionGeneratedKt.getCurrentKonanVersion() @@ -72,6 +77,13 @@ ext { } allprojects { + buildscript { + repositories { + maven { + url 'https://cache-redirector.jetbrains.com/jcenter' + } + } + } if (path != ":dependencies") { evaluationDependsOn(":dependencies") } @@ -166,6 +178,7 @@ void loadCommandLineProperties() { configurations { ftpAntTask kotlinCommonSources + distPack } dependencies { @@ -173,6 +186,16 @@ dependencies { [kotlinCommonStdlibModule, kotlinTestCommonModule, kotlinTestAnnotationsCommonModule].each { kotlinCommonSources(it) { transitive = false } } + distPack project(':Interop:Runtime') + distPack project(':Interop:Indexer') + distPack project(':Interop:StubGenerator') + distPack project(':backend.native') + distPack project(':utilities') + distPack project(':klib') + distPack project(path: ':endorsedLibraries:kotlinx.cli', configuration: "jvmRuntimeElements") + distPack "org.jetbrains.kotlin:kotlin-native-shared:$konanVersion" + distPack "org.jetbrains.kotlin:konan.metadata:$konanVersion" + distPack "org.jetbrains.kotlin:konan.serializer:$konanVersion" } task sharedJar { @@ -226,42 +249,60 @@ task distSources { dependsOn(distEndorsedSources) } -def jarContent(def jarTask, FileTree baseFileTree, Iterable excludedList = []) { - jarTask.from { - baseFileTree - .matching { include("*.jar") } - .collect { zipTree("${it.absolutePath}").matching { exclude(excludedList) } } +task detectJarCollision(type: CollisionDetector) { + ignoredFiles.addAll(["META-INF/MANIFEST.MF", "META-INF/services/**"]) + configurations = [project.configurations.distPack] + resolvingRules["META-INF/kotlin-util-klib.kotlin_module"] = "kotlin-compiler" + resolvingRules["META-INF/kotlin-util-io.kotlin_module"] = "kotlin-compiler" + resolvingRules["META-INF/descriptors.jvm.kotlin_module"] = "kotlin-compiler" + resolvingRules["META-INF/descriptors.kotlin_module"] = "kotlin-compiler" + resolvingRules["META-INF/descriptors.runtime.kotlin_module"] = "kotlin-compiler" + resolvingRules["META-INF/deserialization.kotlin_module"] = "kotlin-compiler" + resolvingRules["META-INF/metadata.jvm.kotlin_module"] = "kotlin-compiler" + resolvingRules["META-INF/metadata.kotlin_module"] = "kotlin-compiler" + resolvingRules["META-INF/type-system.kotlin_module"] = "kotlin-compiler" + resolvingRules["META-INF/util.runtime.kotlin_module"] = "kotlin-compiler" + resolvingRules["kotlin/annotation/annotation.kotlin_builtins"] = "kotlin-compiler" + resolvingRules["kotlin/collections/collections.kotlin_builtins"] = "kotlin-compiler" + resolvingRules["kotlin/coroutines/coroutines.kotlin_builtins"] = "kotlin-compiler" + resolvingRules["kotlin/internal/internal.kotlin_builtins"] = "kotlin-compiler" + resolvingRules["kotlin/kotlin.kotlin_builtins"] = "kotlin-compiler" + resolvingRules["kotlin/ranges/ranges.kotlin_builtins"] = "kotlin-compiler" + resolvingRules["kotlin/reflect/reflect.kotlin_builtins"] = "kotlin-compiler" + resolvingRules["META-INF/maven/com.google.protobuf/protobuf-java/pom.properties"] = "kotlin-compiler" + resolvingRules["META-INF/maven/com.google.protobuf/protobuf-java/pom.xml"] = "kotlin-compiler" + librariesWithIgnoredClassCollisions.addAll(["kotlin-util-klib", "kotlin-util-io"]) + if (project.hasProperty('kotlinProjectPath')) { + resolvingRulesWithRegexes[new Regex("META-INF/.+\\.kotlin_module")] = "kotlin-compiler" + resolvingRules["META-INF/extensions/compiler.xml"] = "kotlin-compiler" + resolvingRules["kotlinManifest.properties"] = "kotlin-compiler" + librariesWithIgnoredClassCollisions.addAll(["util", "container", "resolution", "serialization", "psi", "frontend", + "frontend.common", "frontend.java", "cli-common", "ir.tree", + "ir.psi2ir", "ir.backend.common", "backend.jvm", "backend.js", + "backend.wasm", "ir.serialization.common", "ir.serialization.js", + "backend-common", "backend", "plugin-api", "light-classes", "cli", + "cli-js", "incremental-compilation-impl", "js.ast", "js.serializer", + "js.parser", "js.frontend", "js.translator", "js.dce", "metadata", + "metadata.jvm", "descriptors", "descriptors.jvm", "descriptors.runtime", + "deserialization", "util.runtime", "type-system", "cones", "resolve", + "tree", "psi2fir", "fir2ir", "java", "kotlin-build-common"]) } } -task mergeJars(type: Jar) { - dependsOn ':backend.native:jar' - dependsOn ':utilities:jar' - dependsOn ':klib:jar' - dependsOn ':sharedJar' - dependsOn ':endorsedLibraries:jvmJar' - - archiveFileName = 'kotlin-native.jar' - destinationDirectory = file("$distDir/konan/lib") - - jarContent(it, project(':backend.native').fileTree('build/external_jars'), - ['META-INF/services/org.jetbrains.kotlin.resolve.ExternalOverridabilityCondition']) - jarContent(it, project(':backend.native').fileTree('build/libs')) - jarContent(it, project(':utilities').fileTree('build/libs')) - jarContent(it, project('Interop').fileTree('Runtime/build/libs')) - jarContent(it, project('Interop').fileTree('Indexer/build/libs')) - jarContent(it, project('Interop').fileTree('StubGenerator/build/libs')) - - jarContent(it, project(':klib').fileTree('build/libs')) - jarContent(it, project(':endorsedLibraries:kotlinx.cli').fileTree('build/libs')) - jarContent(it, fileTree("${gradle.includedBuild('kotlin-native-shared').projectDir}/build/libs")) - jarContent(it, fileTree("${gradle.includedBuild('konan.metadata').projectDir}/build/libs")) - jarContent(it, fileTree("${gradle.includedBuild('konan.serializer').projectDir}/build/libs")) - +task shadowJar(type: ShadowJar) { + dependsOn ':detectJarCollision' + mergeServiceFiles() + destinationDirectory.set(file("$distDir/konan/lib")) + archiveBaseName.set("kotlin-native") + configurations = [project.configurations.distPack] + archiveClassifier.set(null) + transform(CollisionTransformer.class) { + resolvedConflicts = detectJarCollision.resolvedConflicts + } } task distCompiler(type: Copy) { - dependsOn ':mergeJars' + dependsOn ':shadowJar' destinationDir distDir diff --git a/settings.gradle b/settings.gradle index 393fe2ed3fb..860ba643b7a 100644 --- a/settings.gradle +++ b/settings.gradle @@ -62,6 +62,7 @@ if (hasProperty("kotlinProjectPath")) { substitute module("org.jetbrains.kotlin:kotlin-util-klib:$kotlinVersion") with project(':kotlin-util-klib') substitute module("org.jetbrains.kotlin:kotlin-util-klib-metadata:$kotlinVersion") with project(':kotlin-util-klib-metadata') substitute module("org.jetbrains.kotlin:kotlin-native-utils:$kotlinVersion") with project(':kotlin-native:kotlin-native-utils') + substitute module("org.jetbrains.kotlin:kotlin-reflect:$kotlinVersion") with project(':kotlin-reflect') } } } diff --git a/utilities/build.gradle b/utilities/build.gradle index 4bc9f16af1c..327a98de986 100644 --- a/utilities/build.gradle +++ b/utilities/build.gradle @@ -16,8 +16,6 @@ repositories { dependencies { compile "org.jetbrains.kotlin:kotlin-stdlib:$buildKotlinVersion" - compile "org.jetbrains.kotlin:kotlin-util-io:$kotlinVersion" - compile "org.jetbrains.kotlin:kotlin-util-klib:$kotlinVersion" compile project(':backend.native') compile project(':Interop:StubGenerator') compile project(':klib')