[Gradle, Native] Pass external dependencies to Kotlin/Native link tasks
^KT-44626
This commit is contained in:
Dmitriy Dolovov
@@ -0,0 +1,182 @@
|
||||
/*
|
||||
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package org.jetbrains.kotlin.utils
|
||||
|
||||
/**
|
||||
* ID of [ResolvedDependency].
|
||||
*/
|
||||
class ResolvedDependencyId private constructor(val uniqueNames: Set<String>) {
|
||||
constructor(vararg uniqueNames: String) : this(uniqueNames.toSortedSet())
|
||||
constructor(uniqueNames: Iterable<String>) : this(uniqueNames.toSortedSet())
|
||||
|
||||
init {
|
||||
check(uniqueNames.isNotEmpty())
|
||||
}
|
||||
|
||||
override fun toString() = buildString {
|
||||
uniqueNames.forEachIndexed { index, uniqueName ->
|
||||
when (index) {
|
||||
0 -> append(uniqueName)
|
||||
else -> {
|
||||
when (index) {
|
||||
1 -> append(" (")
|
||||
/* 2+ */ else -> append(", ")
|
||||
}
|
||||
append(uniqueName)
|
||||
if (index == uniqueNames.size - 1) append(")")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun equals(other: Any?) = (other as? ResolvedDependencyId)?.uniqueNames == uniqueNames
|
||||
override fun hashCode() = uniqueNames.hashCode()
|
||||
|
||||
operator fun contains(other: ResolvedDependencyId): Boolean {
|
||||
return uniqueNames.containsAll(other.uniqueNames)
|
||||
}
|
||||
|
||||
fun withVersion(version: ResolvedDependencyVersion): String = buildString {
|
||||
append(this@ResolvedDependencyId.toString())
|
||||
if (!version.isEmpty()) append(": ").append(version)
|
||||
}
|
||||
|
||||
companion object {
|
||||
val SOURCE_CODE_MODULE_ID = ResolvedDependencyId("/")
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A representation of a particular module (component, dependency) in the project. Keeps the information about the module version and
|
||||
* the place of the module in the project dependencies hierarchy.
|
||||
*
|
||||
* - [id] identifier of the module
|
||||
* - [selectedVersion] actual version of the module that is used in the project
|
||||
* - [requestedVersionsByIncomingDependencies] the mapping between ID of the dependee module (i.e. the module that depends on this module)
|
||||
* and the version (requested version) that the dependee module requires from this module. The requested version can be different
|
||||
* for different dependee modules, which is absolutely legal. The [selectedVersion] is always equal to one of the requested versions
|
||||
* (the one that wins among others, which is typically handled inside of the build system). If the module is the top-level module,
|
||||
* then it has [ResolvedDependencyId.SOURCE_CODE_MODULE_ID] in the mapping.
|
||||
* - [artifactPaths] absolute paths to every artifact represented by this module
|
||||
*/
|
||||
class ResolvedDependency(
|
||||
val id: ResolvedDependencyId,
|
||||
var visibleAsFirstLevelDependency: Boolean = true,
|
||||
var selectedVersion: ResolvedDependencyVersion,
|
||||
val requestedVersionsByIncomingDependencies: MutableMap<ResolvedDependencyId, ResolvedDependencyVersion>,
|
||||
val artifactPaths: MutableSet<ResolvedDependencyArtifactPath>
|
||||
) {
|
||||
val moduleIdWithVersion: String
|
||||
get() = id.withVersion(selectedVersion)
|
||||
|
||||
override fun toString() = moduleIdWithVersion
|
||||
}
|
||||
|
||||
object ResolvedDependenciesSupport {
|
||||
fun serialize(modules: Collection<ResolvedDependency>): String {
|
||||
val moduleIdToIndex = mutableMapOf(ResolvedDependencyId.SOURCE_CODE_MODULE_ID to 0).apply {
|
||||
modules.forEachIndexed { index, module -> this[module.id] = index + 1 }
|
||||
}
|
||||
|
||||
return buildString {
|
||||
modules.forEach { module ->
|
||||
val moduleIndex = moduleIdToIndex.getValue(module.id)
|
||||
append(moduleIndex.toString()).append(' ')
|
||||
module.id.uniqueNames.joinTo(this, separator = ",")
|
||||
append('[').append(module.selectedVersion).append(']')
|
||||
module.requestedVersionsByIncomingDependencies.entries.joinTo(
|
||||
this,
|
||||
separator = ""
|
||||
) { (incomingDependencyId, requestedVersion) ->
|
||||
val incomingDependencyIndex = moduleIdToIndex.getValue(incomingDependencyId)
|
||||
" #$incomingDependencyIndex[$requestedVersion]"
|
||||
}
|
||||
append('\n')
|
||||
module.artifactPaths.joinTo(this, separator = "") { artifactPath -> "\t$artifactPath\n" }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun deserialize(source: String, onMalformedLine: (lineNo: Int, line: String) -> Unit): Collection<ResolvedDependency> {
|
||||
val moduleIndexToId: MutableMap<Int, ResolvedDependencyId> = mutableMapOf(0 to ResolvedDependencyId.SOURCE_CODE_MODULE_ID)
|
||||
val requestedVersionsByIncomingDependenciesIndices: MutableMap<ResolvedDependencyId, Map<Int, ResolvedDependencyVersion>> =
|
||||
mutableMapOf()
|
||||
|
||||
val modules = mutableListOf<ResolvedDependency>()
|
||||
|
||||
source.lines().forEachIndexed { lineNo, line ->
|
||||
fun malformedLine(): Collection<ResolvedDependency> {
|
||||
onMalformedLine(lineNo, line)
|
||||
return emptyList()
|
||||
}
|
||||
|
||||
when {
|
||||
line.isBlank() -> return@forEachIndexed
|
||||
line[0].isWhitespace() -> {
|
||||
val currentModule = modules.lastOrNull()
|
||||
val artifactPath = line.trimStart { it.isWhitespace() }
|
||||
if (currentModule != null && artifactPath.isNotBlank())
|
||||
currentModule.artifactPaths += ResolvedDependencyArtifactPath(artifactPath)
|
||||
else
|
||||
return malformedLine()
|
||||
}
|
||||
else -> {
|
||||
val result = DEPENDENCY_MODULE_REGEX.matchEntire(line) ?: return malformedLine()
|
||||
val moduleIndex = result.groupValues[1].toInt()
|
||||
val moduleId = ResolvedDependencyId(result.groupValues[2].split(','))
|
||||
val selectedVersion = ResolvedDependencyVersion(result.groupValues[3])
|
||||
|
||||
if (result.groupValues.size > 4) {
|
||||
val requestedVersions: Map<Int, ResolvedDependencyVersion> = result.groupValues[4].trimStart()
|
||||
.split(' ')
|
||||
.associate { token ->
|
||||
val tokenResult = REQUESTED_VERSION_BY_INCOMING_DEPENDENCY_REGEX.matchEntire(token)
|
||||
?: return malformedLine()
|
||||
val incomingDependencyIndex = tokenResult.groupValues[1].toInt()
|
||||
val requestedVersion = ResolvedDependencyVersion(tokenResult.groupValues[2])
|
||||
incomingDependencyIndex to requestedVersion
|
||||
}
|
||||
requestedVersionsByIncomingDependenciesIndices[moduleId] = requestedVersions
|
||||
}
|
||||
|
||||
moduleIndexToId[moduleIndex] = moduleId
|
||||
modules += ResolvedDependency(
|
||||
id = moduleId,
|
||||
selectedVersion = selectedVersion,
|
||||
requestedVersionsByIncomingDependencies = mutableMapOf(), // To be filled later.
|
||||
artifactPaths = mutableSetOf() // To be filled on the next iterations.
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Stamp incoming dependencies & requested versions.
|
||||
modules.forEach { module ->
|
||||
requestedVersionsByIncomingDependenciesIndices[module.id]?.forEach { (incomingDependencyIndex, requestedVersion) ->
|
||||
val incomingDependencyId = moduleIndexToId.getValue(incomingDependencyIndex)
|
||||
module.requestedVersionsByIncomingDependencies[incomingDependencyId] = requestedVersion
|
||||
}
|
||||
}
|
||||
|
||||
return modules
|
||||
}
|
||||
|
||||
private val DEPENDENCY_MODULE_REGEX = Regex("^(\\d+) ([^\\[]+)\\[([^]]+)](.*)?$")
|
||||
private val REQUESTED_VERSION_BY_INCOMING_DEPENDENCY_REGEX = Regex("^#(\\d+)\\[(.*)]$")
|
||||
}
|
||||
|
||||
data class ResolvedDependencyVersion(val version: String) {
|
||||
fun isEmpty() = version.isEmpty()
|
||||
override fun toString() = version
|
||||
|
||||
companion object {
|
||||
val EMPTY = ResolvedDependencyVersion("")
|
||||
}
|
||||
}
|
||||
|
||||
data class ResolvedDependencyArtifactPath(val path: String) {
|
||||
override fun toString() = path
|
||||
}
|
||||
+166
-13
@@ -10,10 +10,13 @@ import groovy.lang.Closure
|
||||
import org.gradle.api.DefaultTask
|
||||
import org.gradle.api.Project
|
||||
import org.gradle.api.artifacts.*
|
||||
import org.gradle.api.artifacts.component.ModuleComponentIdentifier
|
||||
import org.gradle.api.artifacts.component.ModuleComponentSelector
|
||||
import org.gradle.api.artifacts.result.DependencyResult
|
||||
import org.gradle.api.artifacts.result.ResolvedDependencyResult
|
||||
import org.gradle.api.file.ConfigurableFileCollection
|
||||
import org.gradle.api.file.FileCollection
|
||||
import org.gradle.api.file.FileTree
|
||||
import org.gradle.api.logging.Logger
|
||||
import org.gradle.api.provider.Provider
|
||||
import org.gradle.api.tasks.*
|
||||
import org.gradle.api.tasks.compile.AbstractCompile
|
||||
@@ -33,7 +36,7 @@ import org.jetbrains.kotlin.gradle.targets.native.internal.isAllowCommonizer
|
||||
import org.jetbrains.kotlin.gradle.utils.*
|
||||
import org.jetbrains.kotlin.gradle.utils.klibModuleName
|
||||
import org.jetbrains.kotlin.gradle.utils.listFilesOrEmpty
|
||||
import org.jetbrains.kotlin.gradle.utils.newProperty
|
||||
import org.jetbrains.kotlin.konan.CompilerVersion
|
||||
import org.jetbrains.kotlin.konan.library.KLIB_INTEROP_IR_PROVIDER_IDENTIFIER
|
||||
import org.jetbrains.kotlin.konan.properties.resolvablePropertyList
|
||||
import org.jetbrains.kotlin.konan.properties.saveToFile
|
||||
@@ -44,12 +47,15 @@ import org.jetbrains.kotlin.konan.target.HostManager
|
||||
import org.jetbrains.kotlin.konan.target.KonanTarget
|
||||
import org.jetbrains.kotlin.library.*
|
||||
import org.jetbrains.kotlin.project.model.LanguageSettings
|
||||
import org.jetbrains.kotlin.utils.ResolvedDependency as KResolvedDependency
|
||||
import org.jetbrains.kotlin.utils.ResolvedDependencyId as KResolvedDependencyId
|
||||
import org.jetbrains.kotlin.utils.ResolvedDependenciesSupport as KResolvedDependenciesSupport
|
||||
import org.jetbrains.kotlin.utils.ResolvedDependencyArtifactPath as KResolvedDependencyArtifactPath
|
||||
import org.jetbrains.kotlin.utils.ResolvedDependencyVersion as KResolvedDependencyVersion
|
||||
import java.io.File
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.security.MessageDigest
|
||||
import java.nio.file.Files
|
||||
import javax.inject.Inject
|
||||
import org.jetbrains.kotlin.konan.file.File as KFile
|
||||
import org.jetbrains.kotlin.util.Logger as KLogger
|
||||
|
||||
// TODO: It's just temporary tasks used while KN isn't integrated with Big Kotlin compilation infrastructure.
|
||||
// region Useful extensions
|
||||
@@ -112,9 +118,10 @@ private fun FileCollection.filterOutPublishableInteropLibs(project: Project): Fi
|
||||
* for it (NO-SOURCE check). So we need to take this case into account
|
||||
* and skip libraries that were not compiled. See also: GH-2617 (K/N repo).
|
||||
*/
|
||||
private fun Collection<File>.filterKlibsPassedToCompiler(): List<File> = filter {
|
||||
(it.extension == "klib" || it.isDirectory) && it.exists()
|
||||
}
|
||||
private val File.isKlibPassedToCompiler: Boolean
|
||||
get() = (extension == "klib" || isDirectory) && exists()
|
||||
|
||||
private fun Collection<File>.filterKlibsPassedToCompiler(): List<File> = filter { it.isKlibPassedToCompiler }
|
||||
|
||||
// endregion
|
||||
abstract class AbstractKotlinNativeCompile<T : KotlinCommonToolOptions, K : KotlinNativeCompilationData<*>> : AbstractCompile() {
|
||||
@@ -604,7 +611,10 @@ constructor(
|
||||
override fun buildCompilerArgs(): List<String> = mutableListOf<String>().apply {
|
||||
addAll(super.buildCompilerArgs())
|
||||
|
||||
addAll(CacheBuilder(project, binary, konanTarget).buildCompilerArgs())
|
||||
val externalDependenciesArgs = ExternalDependenciesBuilder(project, binary).buildCompilerArgs()
|
||||
addAll(externalDependenciesArgs)
|
||||
|
||||
addAll(CacheBuilder(project, binary, konanTarget, externalDependenciesArgs).buildCompilerArgs())
|
||||
|
||||
addKey("-tr", processTests)
|
||||
addArgIfNotNull("-entry", entryPoint)
|
||||
@@ -678,12 +688,154 @@ constructor(
|
||||
validatedExportedLibraries()
|
||||
super.compile()
|
||||
}
|
||||
|
||||
companion object {
|
||||
}
|
||||
}
|
||||
|
||||
internal class CacheBuilder(val project: Project, val binary: NativeBinary, val konanTarget: KonanTarget) {
|
||||
@Suppress("FunctionName")
|
||||
internal class ExternalDependenciesBuilder(val project: Project, val binary: NativeBinary) {
|
||||
private val compilation: KotlinNativeCompilation
|
||||
get() = binary.compilation
|
||||
|
||||
private val compileDependencyConfiguration: Configuration
|
||||
get() = project.configurations.getByName(compilation.compileDependencyConfigurationName)
|
||||
|
||||
fun buildCompilerArgs(): List<String> {
|
||||
val konanVersion = Distribution(project.konanHome).compilerVersion?.let(CompilerVersion.Companion::fromString)
|
||||
?: project.konanVersion
|
||||
|
||||
if (!konanVersion.isAtLeast(1, 6, 0)) return emptyList()
|
||||
|
||||
val modules = buildDependencies()
|
||||
if (modules.isEmpty()) return emptyList()
|
||||
|
||||
val dependenciesFile = Files.createTempFile("kotlin-native-external-dependencies", ".deps").toAbsolutePath().toFile()
|
||||
dependenciesFile.deleteOnExit()
|
||||
dependenciesFile.writeText(KResolvedDependenciesSupport.serialize(modules))
|
||||
|
||||
return listOf("-Xexternal-dependencies=${dependenciesFile.path}")
|
||||
}
|
||||
|
||||
private fun buildDependencies(): Collection<KResolvedDependency> {
|
||||
// Collect all artifacts.
|
||||
val moduleNameToArtifactPaths: MutableMap</* unique name*/ String, MutableSet<KResolvedDependencyArtifactPath>> = mutableMapOf()
|
||||
compileDependencyConfiguration.incoming.artifacts.artifacts.mapNotNull { resolvedArtifact ->
|
||||
val uniqueName = (resolvedArtifact.id.componentIdentifier as? ModuleComponentIdentifier)?.uniqueName ?: return@mapNotNull null
|
||||
val artifactPath = resolvedArtifact.file.absolutePath
|
||||
|
||||
moduleNameToArtifactPaths.getOrPut(uniqueName) { mutableSetOf() } += KResolvedDependencyArtifactPath(artifactPath)
|
||||
}
|
||||
|
||||
// The build system may express the single module as two modules where the first one is a common
|
||||
// module without artifacts and the second one is a platform-specific module with mandatory artifact.
|
||||
// Example: "org.jetbrains.kotlinx:atomicfu" (common) and "org.jetbrains.kotlinx:atomicfu-macosx64" (platform-specific).
|
||||
// Both such modules should be merged into a single module with just two names:
|
||||
// "org.jetbrains.kotlinx:atomicfu (org.jetbrains.kotlinx:atomicfu-macosx64)".
|
||||
val moduleIdsToMerge: MutableMap</* platform-specific */ KResolvedDependencyId, /* common */ KResolvedDependencyId> = mutableMapOf()
|
||||
|
||||
// Collect plain modules.
|
||||
val plainModules: MutableMap<KResolvedDependencyId, KResolvedDependency> = mutableMapOf()
|
||||
fun processModule(resolvedDependency: DependencyResult, incomingDependencyId: KResolvedDependencyId) {
|
||||
if (resolvedDependency !is ResolvedDependencyResult) return
|
||||
|
||||
val requestedModule = resolvedDependency.requested as? ModuleComponentSelector ?: return
|
||||
val selectedModule = resolvedDependency.selected
|
||||
val selectedModuleId = selectedModule.id as? ModuleComponentIdentifier ?: return
|
||||
|
||||
val moduleId = KResolvedDependencyId(selectedModuleId.uniqueName)
|
||||
val module = plainModules.getOrPut(moduleId) {
|
||||
val artifactPaths = moduleId.uniqueNames.asSequence()
|
||||
.mapNotNull { uniqueName -> moduleNameToArtifactPaths[uniqueName] }
|
||||
.firstOrNull()
|
||||
.orEmpty()
|
||||
|
||||
KResolvedDependency(
|
||||
id = moduleId,
|
||||
selectedVersion = KResolvedDependencyVersion(selectedModuleId.version),
|
||||
requestedVersionsByIncomingDependencies = mutableMapOf(), // To be filled in just below.
|
||||
artifactPaths = artifactPaths.toMutableSet()
|
||||
)
|
||||
}
|
||||
|
||||
// Record the requested version of the module by the current incoming dependency.
|
||||
module.requestedVersionsByIncomingDependencies[incomingDependencyId] = KResolvedDependencyVersion(requestedModule.version)
|
||||
|
||||
// TODO: Use [ResolvedDependencyResult.resolvedVariant.externalVariant] to find a connection between platform-specific
|
||||
// and common modules when "resolvedVariant" and "externalVariant" graduate from incubating state.
|
||||
if (module.artifactPaths.isNotEmpty()) {
|
||||
val originModuleId = resolvedDependency.from.id as? ModuleComponentIdentifier
|
||||
if (originModuleId != null
|
||||
&& selectedModuleId.group == originModuleId.group
|
||||
&& selectedModuleId.module.startsWith(originModuleId.module)
|
||||
&& selectedModuleId.version == originModuleId.version
|
||||
) {
|
||||
// These two modules should be merged.
|
||||
moduleIdsToMerge[moduleId] = KResolvedDependencyId(originModuleId.uniqueName)
|
||||
}
|
||||
}
|
||||
|
||||
selectedModule.dependencies.forEach { processModule(it, incomingDependencyId = moduleId) }
|
||||
}
|
||||
|
||||
compileDependencyConfiguration.incoming.resolutionResult.root.dependencies.forEach { dependencyResult ->
|
||||
processModule(dependencyResult, incomingDependencyId = KResolvedDependencyId.SOURCE_CODE_MODULE_ID)
|
||||
}
|
||||
|
||||
if (moduleIdsToMerge.isEmpty())
|
||||
return plainModules.values
|
||||
|
||||
// Do merge.
|
||||
val replacedModules: MutableMap</* old module ID */ KResolvedDependencyId, /* new module */ KResolvedDependency> = mutableMapOf()
|
||||
moduleIdsToMerge.forEach { (platformSpecificModuleId, commonModuleId) ->
|
||||
val platformSpecificModule = plainModules.getValue(platformSpecificModuleId)
|
||||
val commonModule = plainModules.getValue(commonModuleId)
|
||||
|
||||
val replacementModuleId = KResolvedDependencyId(platformSpecificModuleId.uniqueNames + commonModuleId.uniqueNames)
|
||||
val replacementModule = KResolvedDependency(
|
||||
id = replacementModuleId,
|
||||
visibleAsFirstLevelDependency = commonModule.visibleAsFirstLevelDependency,
|
||||
selectedVersion = commonModule.selectedVersion,
|
||||
requestedVersionsByIncomingDependencies = mutableMapOf<KResolvedDependencyId, KResolvedDependencyVersion>().apply {
|
||||
this += commonModule.requestedVersionsByIncomingDependencies
|
||||
this += platformSpecificModule.requestedVersionsByIncomingDependencies - commonModuleId
|
||||
},
|
||||
artifactPaths = mutableSetOf<KResolvedDependencyArtifactPath>().apply {
|
||||
this += commonModule.artifactPaths
|
||||
this += platformSpecificModule.artifactPaths
|
||||
}
|
||||
)
|
||||
|
||||
replacedModules[platformSpecificModuleId] = replacementModule
|
||||
replacedModules[commonModuleId] = replacementModule
|
||||
}
|
||||
|
||||
// Assemble new modules together (without "replaced" and with "replacements").
|
||||
val mergedModules: MutableMap<KResolvedDependencyId, KResolvedDependency> = mutableMapOf()
|
||||
mergedModules += plainModules - replacedModules.keys
|
||||
replacedModules.values.forEach { replacementModule -> mergedModules[replacementModule.id] = replacementModule }
|
||||
|
||||
// Fix references to point to "replacement" modules instead of "replaced" modules.
|
||||
mergedModules.values.forEach { module ->
|
||||
module.requestedVersionsByIncomingDependencies.mapNotNull { (replacedModuleId, requestedVersion) ->
|
||||
val replacementModuleId = replacedModules[replacedModuleId]?.id ?: return@mapNotNull null
|
||||
Triple(replacedModuleId, replacementModuleId, requestedVersion)
|
||||
}.forEach { (replacedModuleId, replacementModuleId, requestedVersion) ->
|
||||
module.requestedVersionsByIncomingDependencies.remove(replacedModuleId)
|
||||
module.requestedVersionsByIncomingDependencies[replacementModuleId] = requestedVersion
|
||||
}
|
||||
}
|
||||
|
||||
return mergedModules.values
|
||||
}
|
||||
|
||||
private val ModuleComponentIdentifier.uniqueName: String
|
||||
get() = "$group:$module"
|
||||
}
|
||||
|
||||
internal class CacheBuilder(
|
||||
val project: Project,
|
||||
val binary: NativeBinary,
|
||||
val konanTarget: KonanTarget,
|
||||
val externalDependenciesArgs: List<String>
|
||||
) {
|
||||
|
||||
private val nativeSingleFileResolveStrategy: SingleFileKlibResolveStrategy
|
||||
get() = CompilerSingleFileKlibResolveAllowingIrProvidersStrategy(
|
||||
@@ -777,6 +929,7 @@ internal class CacheBuilder(val project: Project, val binary: NativeBinary, val
|
||||
)
|
||||
if (debuggable)
|
||||
args += "-g"
|
||||
args += externalDependenciesArgs
|
||||
args += "-Xadd-cache=${library.libraryFile.absolutePath}"
|
||||
args += "-Xcache-directory=${cacheDirectory.absolutePath}"
|
||||
args += "-Xcache-directory=${rootCacheDirectory.absolutePath}"
|
||||
|
||||
Reference in New Issue
Block a user