azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Creating fat jar with shadow plugin and collisions detector (#3466)

Browse Source
This commit is contained in:
LepilkinaElena
2019-10-21 12:32:06 +03:00
committed by GitHub
parent 9874675b09
commit e21ab4f1e7
8 changed files with 218 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Interop/Runtime/build.gradle
+1 -1
View File
@@ -67,7 +67,7 @@ repositories {
dependencies {
compile "org.jetbrains.kotlin:kotlin-stdlib:$buildKotlinVersion"
compile "org.jetbrains.kotlin:kotlin-reflect:$buildKotlinVersion"
compile "org.jetbrains.kotlin:kotlin-reflect:$kotlinVersion"
}
sourceSets.main.kotlin.srcDirs += "src/jvm/kotlin"
backend.native/build.gradle
+1 -5
View File
@@ -114,8 +114,6 @@ configurations {
kotlin_reflect_jar
kotlin_script_runtime_jar
kotlin_native_utils_jar
kotlin_util_io_jar
kotlin_util_klib_jar
trove4j_jar
kotlinCommonSources
@@ -136,8 +134,6 @@ dependencies {
kotlin_reflect_jar "$kotlinReflectModule@jar"
kotlin_script_runtime_jar "$kotlinScriptRuntimeModule@jar"
kotlin_native_utils_jar "$kotlinNativeUtilsModule@jar"
kotlin_util_io_jar "$kotlinUtilIoModule@jar"
kotlin_util_klib_jar "$kotlinUtilKlibModule@jar"
[kotlinCommonStdlibModule, kotlinTestCommonModule, kotlinTestAnnotationsCommonModule].each {
kotlinCommonSources(it) { transitive = false }
@@ -238,7 +234,7 @@ jar {
dependsOn ':runtime:hostRuntime', 'external_jars'
}
def externalJars = ['compiler', 'stdlib', 'reflect', 'script_runtime', 'native_utils', 'util_io', 'util_klib']
def externalJars = ['compiler', 'stdlib', 'reflect', 'script_runtime', 'native_utils']
task trove4jCopy(type: Copy) {
from configurations.getByName("trove4j_jar") {
build-tools/build.gradle.kts
+2
View File
@@ -30,6 +30,7 @@ group = "org.jetbrains.kotlin"
version = konanVersion
repositories {
jcenter()
maven(kotlinCompilerRepo)
maven(buildKotlinCompilerRepo)
maven("https://cache-redirector.jetbrains.com/maven-central")
@@ -53,6 +54,7 @@ dependencies {
// Located in <repo root>/shared and always provided by the composite build.
api("org.jetbrains.kotlin:kotlin-native-shared:$konanVersion")
implementation("com.github.jengelman.gradle.plugins:shadow:5.1.0")
}
sourceSets["main"].withConvention(KotlinSourceSet::class) {
build-tools/src/main/kotlin/org/jetbrains/kotlin/CollisionDetector.kt
+92
View File
@@ -0,0 +1,92 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin
import org.gradle.api.DefaultTask
import org.gradle.api.tasks.TaskAction
import org.gradle.api.artifacts.Configuration
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputFiles
import java.io.File
fun <K, V> Map<K, V>.firstOrNull(predicate:(K, V) -> Boolean): V? {
var foundValue: V? = null
forEach {
if (predicate(it.key, it.value)) {
foundValue = it.value
return@forEach
}
}
return foundValue
}
/**
* Task to find out collisions before producing fat jar.
*
* @property configurations added to fat jar configurations
* @property ignoredFiles excluded from analysis files
* @property resolvingRules a map containing rules to resolve conflicts. Key - conflicting file, value - a jar to copy the file from
* @property resolvingRulesWithRegexes a map containing rules to resolve conflicts. Key - regular expression describing conflicting file, value - a jar to copy the file from
* @property librariesWithIgnoredClassCollisions libraries which collision in class files are ignored
*/
open class CollisionDetector : DefaultTask() {
@InputFiles
var configurations = listOf<Configuration>()
@Input
val ignoredFiles = mutableListOf<String>()
@Input
val resolvingRules = mutableMapOf<String, String>()
@Input
val resolvingRulesWithRegexes = mutableMapOf<Regex, String>()
@Input
val librariesWithIgnoredClassCollisions = mutableListOf<String>()
val resolvedConflicts = mutableMapOf<String, File>()
// Key - filename, value - jar file containing it.
private val filesInfo = mutableMapOf<String, String>()
@TaskAction
fun run() {
configurations.forEach { configuration ->
configuration.files.filter { it.name.endsWith(".jar") }.forEach { processedFile ->
project.zipTree(processedFile.absolutePath).matching { it.exclude(ignoredFiles) }.forEach {
val outputPath = it.absolutePath.substringAfter(processedFile.name).substringAfter("/")
if (outputPath in filesInfo.keys) {
val rule = resolvingRules.getOrElse(outputPath) {
resolvingRulesWithRegexes.firstOrNull { key, _ -> key.matches(outputPath) }
}
var ignoreJar = false
if (rule != null && processedFile.name.startsWith(rule)) {
resolvedConflicts[outputPath] = processedFile
} else {
// Skip class files from ignored libraries if version of libraries had collision are the same.
val versionRegex = "\\d+\\.\\d+(\\.\\d+)?(-\\w+(-\\d+)?)?".toRegex()
val currentVersion = versionRegex.find(processedFile.name)?.groupValues?.get(0)
val collisionLibVersion = versionRegex.find(filesInfo.getValue(outputPath))?.groupValues?.get(0)
if (outputPath.endsWith(".class") && currentVersion == collisionLibVersion) {
if (processedFile.name == filesInfo[outputPath]) {
ignoreJar = true
} else {
librariesWithIgnoredClassCollisions.forEach {
if (processedFile.name.startsWith(it)) {
ignoreJar = true
}
}
}
}
}
if (rule == null && !ignoreJar) {
error("Collision is detected. File $outputPath is found in ${filesInfo[outputPath]} and ${processedFile.name}")
}
} else {
filesInfo[outputPath] = processedFile.name
}
}
}
}
}
}
build-tools/src/main/kotlin/org/jetbrains/kotlin/CollisionTransformer.kt
+48
View File
@@ -0,0 +1,48 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin
import com.github.jengelman.gradle.plugins.shadow.transformers.Transformer
import com.github.jengelman.gradle.plugins.shadow.transformers.TransformerContext
import org.gradle.api.file.FileTreeElement
import shadow.org.apache.tools.zip.ZipOutputStream
import java.io.File
import shadow.org.apache.commons.io.IOUtils
import shadow.org.apache.tools.zip.ZipEntry
import shadow.org.apache.tools.zip.ZipFile
class CollisionTransformer: Transformer {
var resolvedConflicts = mutableMapOf<String, File>()
private val foundConflictsFiles = mutableSetOf<String>()
override fun canTransformResource(element: FileTreeElement): Boolean {
val result = element.name in resolvedConflicts.keys
if (result) {
foundConflictsFiles.add(element.name)
}
return result
}
override fun transform(context: TransformerContext) {}
override fun hasTransformedResource(): Boolean {
return foundConflictsFiles.isNotEmpty()
}
override fun modifyOutputStream(jos: ZipOutputStream, preserveFileTimestamps: Boolean) {
foundConflictsFiles.forEach {
val entry = ZipEntry(it)
entry.time = TransformerContext.getEntryTimestamp(preserveFileTimestamps, entry.time)
jos.putNextEntry(entry)
val archive = ZipFile(resolvedConflicts[it])
archive.getInputStream(archive.getEntry(it)).use {
IOUtils.copyLarge(it, jos)
}
jos.closeEntry()
}
foundConflictsFiles.clear()
}
}
build.gradle
+73 -32
View File
@@ -13,12 +13,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import kotlin.text.Regex
import org.jetbrains.kotlin.konan.target.*
import org.jetbrains.kotlin.konan.util.*
import org.jetbrains.kotlin.CopySamples
import org.jetbrains.kotlin.CopyCommonSources
import org.jetbrains.kotlin.PlatformInfo
import org.jetbrains.kotlin.KotlinBuildPusher
import org.jetbrains.kotlin.CollisionDetector
import org.jetbrains.kotlin.CollisionTransformer
import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
buildscript {
apply from: "gradle/kotlinGradlePlugin.gradle"
@@ -27,12 +32,14 @@ buildscript {
maven { url kotlinCompilerRepo }
maven { url "https://kotlin.bintray.com/kotlinx" }
maven { url "https://cache-redirector.jetbrains.com/maven-central" }
jcenter()
}
dependencies {
classpath "org.jetbrains.kotlin:kotlin-native-utils:$kotlinVersion"
classpath "org.jetbrains.kotlin:kotlin-native-shared:$konanVersion"
classpath "org.jetbrains.kotlin:kotlin-native-build-tools:$konanVersion"
classpath 'com.github.jengelman.gradle.plugins:shadow:5.1.0'
}
}
import org.jetbrains.kotlin.konan.*
@@ -63,8 +70,6 @@ ext {
kotlinReflectModule="org.jetbrains.kotlin:kotlin-reflect:${kotlinVersion}"
kotlinScriptRuntimeModule="org.jetbrains.kotlin:kotlin-script-runtime:${kotlinVersion}"
kotlinNativeUtilsModule="org.jetbrains.kotlin:kotlin-native-utils:${kotlinVersion}"
kotlinUtilIoModule="org.jetbrains.kotlin:kotlin-util-io:${kotlinVersion}"
kotlinUtilKlibModule="org.jetbrains.kotlin:kotlin-util-klib:${kotlinVersion}"
kotlinUtilKliMetadatabModule="org.jetbrains.kotlin:kotlin-util-klib-metadata:${kotlinVersion}"
konanVersionFull = KonanVersionGeneratedKt.getCurrentKonanVersion()
@@ -72,6 +77,13 @@ ext {
}
allprojects {
buildscript {
repositories {
maven {
url 'https://cache-redirector.jetbrains.com/jcenter'
}
}
}
if (path != ":dependencies") {
evaluationDependsOn(":dependencies")
}
@@ -166,6 +178,7 @@ void loadCommandLineProperties() {
configurations {
ftpAntTask
kotlinCommonSources
distPack
}
dependencies {
@@ -173,6 +186,16 @@ dependencies {
[kotlinCommonStdlibModule, kotlinTestCommonModule, kotlinTestAnnotationsCommonModule].each {
kotlinCommonSources(it) { transitive = false }
}
distPack project(':Interop:Runtime')
distPack project(':Interop:Indexer')
distPack project(':Interop:StubGenerator')
distPack project(':backend.native')
distPack project(':utilities')
distPack project(':klib')
distPack project(path: ':endorsedLibraries:kotlinx.cli', configuration: "jvmRuntimeElements")
distPack "org.jetbrains.kotlin:kotlin-native-shared:$konanVersion"
distPack "org.jetbrains.kotlin:konan.metadata:$konanVersion"
distPack "org.jetbrains.kotlin:konan.serializer:$konanVersion"
}
task sharedJar {
@@ -226,42 +249,60 @@ task distSources {
dependsOn(distEndorsedSources)
}
def jarContent(def jarTask, FileTree baseFileTree, Iterable<String> excludedList = []) {
jarTask.from {
baseFileTree
.matching { include("*.jar") }
.collect { zipTree("${it.absolutePath}").matching { exclude(excludedList) } }
task detectJarCollision(type: CollisionDetector) {
ignoredFiles.addAll(["META-INF/MANIFEST.MF", "META-INF/services/**"])
configurations = [project.configurations.distPack]
resolvingRules["META-INF/kotlin-util-klib.kotlin_module"] = "kotlin-compiler"
resolvingRules["META-INF/kotlin-util-io.kotlin_module"] = "kotlin-compiler"
resolvingRules["META-INF/descriptors.jvm.kotlin_module"] = "kotlin-compiler"
resolvingRules["META-INF/descriptors.kotlin_module"] = "kotlin-compiler"
resolvingRules["META-INF/descriptors.runtime.kotlin_module"] = "kotlin-compiler"
resolvingRules["META-INF/deserialization.kotlin_module"] = "kotlin-compiler"
resolvingRules["META-INF/metadata.jvm.kotlin_module"] = "kotlin-compiler"
resolvingRules["META-INF/metadata.kotlin_module"] = "kotlin-compiler"
resolvingRules["META-INF/type-system.kotlin_module"] = "kotlin-compiler"
resolvingRules["META-INF/util.runtime.kotlin_module"] = "kotlin-compiler"
resolvingRules["kotlin/annotation/annotation.kotlin_builtins"] = "kotlin-compiler"
resolvingRules["kotlin/collections/collections.kotlin_builtins"] = "kotlin-compiler"
resolvingRules["kotlin/coroutines/coroutines.kotlin_builtins"] = "kotlin-compiler"
resolvingRules["kotlin/internal/internal.kotlin_builtins"] = "kotlin-compiler"
resolvingRules["kotlin/kotlin.kotlin_builtins"] = "kotlin-compiler"
resolvingRules["kotlin/ranges/ranges.kotlin_builtins"] = "kotlin-compiler"
resolvingRules["kotlin/reflect/reflect.kotlin_builtins"] = "kotlin-compiler"
resolvingRules["META-INF/maven/com.google.protobuf/protobuf-java/pom.properties"] = "kotlin-compiler"
resolvingRules["META-INF/maven/com.google.protobuf/protobuf-java/pom.xml"] = "kotlin-compiler"
librariesWithIgnoredClassCollisions.addAll(["kotlin-util-klib", "kotlin-util-io"])
if (project.hasProperty('kotlinProjectPath')) {
resolvingRulesWithRegexes[new Regex("META-INF/.+\\.kotlin_module")] = "kotlin-compiler"
resolvingRules["META-INF/extensions/compiler.xml"] = "kotlin-compiler"
resolvingRules["kotlinManifest.properties"] = "kotlin-compiler"
librariesWithIgnoredClassCollisions.addAll(["util", "container", "resolution", "serialization", "psi", "frontend",
"frontend.common", "frontend.java", "cli-common", "ir.tree",
"ir.psi2ir", "ir.backend.common", "backend.jvm", "backend.js",
"backend.wasm", "ir.serialization.common", "ir.serialization.js",
"backend-common", "backend", "plugin-api", "light-classes", "cli",
"cli-js", "incremental-compilation-impl", "js.ast", "js.serializer",
"js.parser", "js.frontend", "js.translator", "js.dce", "metadata",
"metadata.jvm", "descriptors", "descriptors.jvm", "descriptors.runtime",
"deserialization", "util.runtime", "type-system", "cones", "resolve",
"tree", "psi2fir", "fir2ir", "java", "kotlin-build-common"])
}
}
task mergeJars(type: Jar) {
dependsOn ':backend.native:jar'
dependsOn ':utilities:jar'
dependsOn ':klib:jar'
dependsOn ':sharedJar'
dependsOn ':endorsedLibraries:jvmJar'
archiveFileName = 'kotlin-native.jar'
destinationDirectory = file("$distDir/konan/lib")
jarContent(it, project(':backend.native').fileTree('build/external_jars'),
['META-INF/services/org.jetbrains.kotlin.resolve.ExternalOverridabilityCondition'])
jarContent(it, project(':backend.native').fileTree('build/libs'))
jarContent(it, project(':utilities').fileTree('build/libs'))
jarContent(it, project('Interop').fileTree('Runtime/build/libs'))
jarContent(it, project('Interop').fileTree('Indexer/build/libs'))
jarContent(it, project('Interop').fileTree('StubGenerator/build/libs'))
jarContent(it, project(':klib').fileTree('build/libs'))
jarContent(it, project(':endorsedLibraries:kotlinx.cli').fileTree('build/libs'))
jarContent(it, fileTree("${gradle.includedBuild('kotlin-native-shared').projectDir}/build/libs"))
jarContent(it, fileTree("${gradle.includedBuild('konan.metadata').projectDir}/build/libs"))
jarContent(it, fileTree("${gradle.includedBuild('konan.serializer').projectDir}/build/libs"))
task shadowJar(type: ShadowJar) {
dependsOn ':detectJarCollision'
mergeServiceFiles()
destinationDirectory.set(file("$distDir/konan/lib"))
archiveBaseName.set("kotlin-native")
configurations = [project.configurations.distPack]
archiveClassifier.set(null)
transform(CollisionTransformer.class) {
resolvedConflicts = detectJarCollision.resolvedConflicts
}
}
task distCompiler(type: Copy) {
dependsOn ':mergeJars'
dependsOn ':shadowJar'
destinationDir distDir
settings.gradle
+1
View File
@@ -62,6 +62,7 @@ if (hasProperty("kotlinProjectPath")) {
substitute module("org.jetbrains.kotlin:kotlin-util-klib:$kotlinVersion") with project(':kotlin-util-klib')
substitute module("org.jetbrains.kotlin:kotlin-util-klib-metadata:$kotlinVersion") with project(':kotlin-util-klib-metadata')
substitute module("org.jetbrains.kotlin:kotlin-native-utils:$kotlinVersion") with project(':kotlin-native:kotlin-native-utils')
substitute module("org.jetbrains.kotlin:kotlin-reflect:$kotlinVersion") with project(':kotlin-reflect')
}
}
}
utilities/build.gradle
-2
View File
@@ -16,8 +16,6 @@ repositories {
dependencies {
compile "org.jetbrains.kotlin:kotlin-stdlib:$buildKotlinVersion"
compile "org.jetbrains.kotlin:kotlin-util-io:$kotlinVersion"
compile "org.jetbrains.kotlin:kotlin-util-klib:$kotlinVersion"
compile project(':backend.native')
compile project(':Interop:StubGenerator')
compile project(':klib')