diff --git a/compiler/util/src/org/jetbrains/kotlin/utils/ResolvedDependencies.kt b/compiler/util/src/org/jetbrains/kotlin/utils/ResolvedDependencies.kt new file mode 100644 index 00000000000..a71a0d8b236 --- /dev/null +++ b/compiler/util/src/org/jetbrains/kotlin/utils/ResolvedDependencies.kt @@ -0,0 +1,182 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.utils + +/** + * ID of [ResolvedDependency]. + */ +class ResolvedDependencyId private constructor(val uniqueNames: Set) { + constructor(vararg uniqueNames: String) : this(uniqueNames.toSortedSet()) + constructor(uniqueNames: Iterable) : this(uniqueNames.toSortedSet()) + + init { + check(uniqueNames.isNotEmpty()) + } + + override fun toString() = buildString { + uniqueNames.forEachIndexed { index, uniqueName -> + when (index) { + 0 -> append(uniqueName) + else -> { + when (index) { + 1 -> append(" (") + /* 2+ */ else -> append(", ") + } + append(uniqueName) + if (index == uniqueNames.size - 1) append(")") + } + } + } + } + + override fun equals(other: Any?) = (other as? ResolvedDependencyId)?.uniqueNames == uniqueNames + override fun hashCode() = uniqueNames.hashCode() + + operator fun contains(other: ResolvedDependencyId): Boolean { + return uniqueNames.containsAll(other.uniqueNames) + } + + fun withVersion(version: ResolvedDependencyVersion): String = buildString { + append(this@ResolvedDependencyId.toString()) + if (!version.isEmpty()) append(": ").append(version) + } + + companion object { + val SOURCE_CODE_MODULE_ID = ResolvedDependencyId("/") + } +} + +/** + * A representation of a particular module (component, dependency) in the project. Keeps the information about the module version and + * the place of the module in the project dependencies hierarchy. + * + * - [id] identifier of the module + * - [selectedVersion] actual version of the module that is used in the project + * - [requestedVersionsByIncomingDependencies] the mapping between ID of the dependee module (i.e. the module that depends on this module) + * and the version (requested version) that the dependee module requires from this module. The requested version can be different + * for different dependee modules, which is absolutely legal. The [selectedVersion] is always equal to one of the requested versions + * (the one that wins among others, which is typically handled inside of the build system). If the module is the top-level module, + * then it has [ResolvedDependencyId.SOURCE_CODE_MODULE_ID] in the mapping. + * - [artifactPaths] absolute paths to every artifact represented by this module + */ +class ResolvedDependency( + val id: ResolvedDependencyId, + var visibleAsFirstLevelDependency: Boolean = true, + var selectedVersion: ResolvedDependencyVersion, + val requestedVersionsByIncomingDependencies: MutableMap, + val artifactPaths: MutableSet +) { + val moduleIdWithVersion: String + get() = id.withVersion(selectedVersion) + + override fun toString() = moduleIdWithVersion +} + +object ResolvedDependenciesSupport { + fun serialize(modules: Collection): String { + val moduleIdToIndex = mutableMapOf(ResolvedDependencyId.SOURCE_CODE_MODULE_ID to 0).apply { + modules.forEachIndexed { index, module -> this[module.id] = index + 1 } + } + + return buildString { + modules.forEach { module -> + val moduleIndex = moduleIdToIndex.getValue(module.id) + append(moduleIndex.toString()).append(' ') + module.id.uniqueNames.joinTo(this, separator = ",") + append('[').append(module.selectedVersion).append(']') + module.requestedVersionsByIncomingDependencies.entries.joinTo( + this, + separator = "" + ) { (incomingDependencyId, requestedVersion) -> + val incomingDependencyIndex = moduleIdToIndex.getValue(incomingDependencyId) + " #$incomingDependencyIndex[$requestedVersion]" + } + append('\n') + module.artifactPaths.joinTo(this, separator = "") { artifactPath -> "\t$artifactPath\n" } + } + } + } + + fun deserialize(source: String, onMalformedLine: (lineNo: Int, line: String) -> Unit): Collection { + val moduleIndexToId: MutableMap = mutableMapOf(0 to ResolvedDependencyId.SOURCE_CODE_MODULE_ID) + val requestedVersionsByIncomingDependenciesIndices: MutableMap> = + mutableMapOf() + + val modules = mutableListOf() + + source.lines().forEachIndexed { lineNo, line -> + fun malformedLine(): Collection { + onMalformedLine(lineNo, line) + return emptyList() + } + + when { + line.isBlank() -> return@forEachIndexed + line[0].isWhitespace() -> { + val currentModule = modules.lastOrNull() + val artifactPath = line.trimStart { it.isWhitespace() } + if (currentModule != null && artifactPath.isNotBlank()) + currentModule.artifactPaths += ResolvedDependencyArtifactPath(artifactPath) + else + return malformedLine() + } + else -> { + val result = DEPENDENCY_MODULE_REGEX.matchEntire(line) ?: return malformedLine() + val moduleIndex = result.groupValues[1].toInt() + val moduleId = ResolvedDependencyId(result.groupValues[2].split(',')) + val selectedVersion = ResolvedDependencyVersion(result.groupValues[3]) + + if (result.groupValues.size > 4) { + val requestedVersions: Map = result.groupValues[4].trimStart() + .split(' ') + .associate { token -> + val tokenResult = REQUESTED_VERSION_BY_INCOMING_DEPENDENCY_REGEX.matchEntire(token) + ?: return malformedLine() + val incomingDependencyIndex = tokenResult.groupValues[1].toInt() + val requestedVersion = ResolvedDependencyVersion(tokenResult.groupValues[2]) + incomingDependencyIndex to requestedVersion + } + requestedVersionsByIncomingDependenciesIndices[moduleId] = requestedVersions + } + + moduleIndexToId[moduleIndex] = moduleId + modules += ResolvedDependency( + id = moduleId, + selectedVersion = selectedVersion, + requestedVersionsByIncomingDependencies = mutableMapOf(), // To be filled later. + artifactPaths = mutableSetOf() // To be filled on the next iterations. + ) + } + } + } + + // Stamp incoming dependencies & requested versions. + modules.forEach { module -> + requestedVersionsByIncomingDependenciesIndices[module.id]?.forEach { (incomingDependencyIndex, requestedVersion) -> + val incomingDependencyId = moduleIndexToId.getValue(incomingDependencyIndex) + module.requestedVersionsByIncomingDependencies[incomingDependencyId] = requestedVersion + } + } + + return modules + } + + private val DEPENDENCY_MODULE_REGEX = Regex("^(\\d+) ([^\\[]+)\\[([^]]+)](.*)?$") + private val REQUESTED_VERSION_BY_INCOMING_DEPENDENCY_REGEX = Regex("^#(\\d+)\\[(.*)]$") +} + +data class ResolvedDependencyVersion(val version: String) { + fun isEmpty() = version.isEmpty() + override fun toString() = version + + companion object { + val EMPTY = ResolvedDependencyVersion("") + } +} + +data class ResolvedDependencyArtifactPath(val path: String) { + override fun toString() = path +} diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/tasks/KotlinNativeTasks.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/tasks/KotlinNativeTasks.kt index 6b3511a93cd..eef3f32a160 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/tasks/KotlinNativeTasks.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/tasks/KotlinNativeTasks.kt @@ -10,10 +10,13 @@ import groovy.lang.Closure import org.gradle.api.DefaultTask import org.gradle.api.Project import org.gradle.api.artifacts.* +import org.gradle.api.artifacts.component.ModuleComponentIdentifier +import org.gradle.api.artifacts.component.ModuleComponentSelector +import org.gradle.api.artifacts.result.DependencyResult +import org.gradle.api.artifacts.result.ResolvedDependencyResult import org.gradle.api.file.ConfigurableFileCollection import org.gradle.api.file.FileCollection import org.gradle.api.file.FileTree -import org.gradle.api.logging.Logger import org.gradle.api.provider.Provider import org.gradle.api.tasks.* import org.gradle.api.tasks.compile.AbstractCompile @@ -33,7 +36,7 @@ import org.jetbrains.kotlin.gradle.targets.native.internal.isAllowCommonizer import org.jetbrains.kotlin.gradle.utils.* import org.jetbrains.kotlin.gradle.utils.klibModuleName import org.jetbrains.kotlin.gradle.utils.listFilesOrEmpty -import org.jetbrains.kotlin.gradle.utils.newProperty +import org.jetbrains.kotlin.konan.CompilerVersion import org.jetbrains.kotlin.konan.library.KLIB_INTEROP_IR_PROVIDER_IDENTIFIER import org.jetbrains.kotlin.konan.properties.resolvablePropertyList import org.jetbrains.kotlin.konan.properties.saveToFile @@ -44,12 +47,15 @@ import org.jetbrains.kotlin.konan.target.HostManager import org.jetbrains.kotlin.konan.target.KonanTarget import org.jetbrains.kotlin.library.* import org.jetbrains.kotlin.project.model.LanguageSettings +import org.jetbrains.kotlin.utils.ResolvedDependency as KResolvedDependency +import org.jetbrains.kotlin.utils.ResolvedDependencyId as KResolvedDependencyId +import org.jetbrains.kotlin.utils.ResolvedDependenciesSupport as KResolvedDependenciesSupport +import org.jetbrains.kotlin.utils.ResolvedDependencyArtifactPath as KResolvedDependencyArtifactPath +import org.jetbrains.kotlin.utils.ResolvedDependencyVersion as KResolvedDependencyVersion import java.io.File -import java.nio.charset.StandardCharsets -import java.security.MessageDigest +import java.nio.file.Files import javax.inject.Inject import org.jetbrains.kotlin.konan.file.File as KFile -import org.jetbrains.kotlin.util.Logger as KLogger // TODO: It's just temporary tasks used while KN isn't integrated with Big Kotlin compilation infrastructure. // region Useful extensions @@ -112,9 +118,10 @@ private fun FileCollection.filterOutPublishableInteropLibs(project: Project): Fi * for it (NO-SOURCE check). So we need to take this case into account * and skip libraries that were not compiled. See also: GH-2617 (K/N repo). */ -private fun Collection.filterKlibsPassedToCompiler(): List = filter { - (it.extension == "klib" || it.isDirectory) && it.exists() -} +private val File.isKlibPassedToCompiler: Boolean + get() = (extension == "klib" || isDirectory) && exists() + +private fun Collection.filterKlibsPassedToCompiler(): List = filter { it.isKlibPassedToCompiler } // endregion abstract class AbstractKotlinNativeCompile> : AbstractCompile() { @@ -604,7 +611,10 @@ constructor( override fun buildCompilerArgs(): List = mutableListOf().apply { addAll(super.buildCompilerArgs()) - addAll(CacheBuilder(project, binary, konanTarget).buildCompilerArgs()) + val externalDependenciesArgs = ExternalDependenciesBuilder(project, binary).buildCompilerArgs() + addAll(externalDependenciesArgs) + + addAll(CacheBuilder(project, binary, konanTarget, externalDependenciesArgs).buildCompilerArgs()) addKey("-tr", processTests) addArgIfNotNull("-entry", entryPoint) @@ -678,12 +688,154 @@ constructor( validatedExportedLibraries() super.compile() } - - companion object { - } } -internal class CacheBuilder(val project: Project, val binary: NativeBinary, val konanTarget: KonanTarget) { +@Suppress("FunctionName") +internal class ExternalDependenciesBuilder(val project: Project, val binary: NativeBinary) { + private val compilation: KotlinNativeCompilation + get() = binary.compilation + + private val compileDependencyConfiguration: Configuration + get() = project.configurations.getByName(compilation.compileDependencyConfigurationName) + + fun buildCompilerArgs(): List { + val konanVersion = Distribution(project.konanHome).compilerVersion?.let(CompilerVersion.Companion::fromString) + ?: project.konanVersion + + if (!konanVersion.isAtLeast(1, 6, 0)) return emptyList() + + val modules = buildDependencies() + if (modules.isEmpty()) return emptyList() + + val dependenciesFile = Files.createTempFile("kotlin-native-external-dependencies", ".deps").toAbsolutePath().toFile() + dependenciesFile.deleteOnExit() + dependenciesFile.writeText(KResolvedDependenciesSupport.serialize(modules)) + + return listOf("-Xexternal-dependencies=${dependenciesFile.path}") + } + + private fun buildDependencies(): Collection { + // Collect all artifacts. + val moduleNameToArtifactPaths: MutableMap> = mutableMapOf() + compileDependencyConfiguration.incoming.artifacts.artifacts.mapNotNull { resolvedArtifact -> + val uniqueName = (resolvedArtifact.id.componentIdentifier as? ModuleComponentIdentifier)?.uniqueName ?: return@mapNotNull null + val artifactPath = resolvedArtifact.file.absolutePath + + moduleNameToArtifactPaths.getOrPut(uniqueName) { mutableSetOf() } += KResolvedDependencyArtifactPath(artifactPath) + } + + // The build system may express the single module as two modules where the first one is a common + // module without artifacts and the second one is a platform-specific module with mandatory artifact. + // Example: "org.jetbrains.kotlinx:atomicfu" (common) and "org.jetbrains.kotlinx:atomicfu-macosx64" (platform-specific). + // Both such modules should be merged into a single module with just two names: + // "org.jetbrains.kotlinx:atomicfu (org.jetbrains.kotlinx:atomicfu-macosx64)". + val moduleIdsToMerge: MutableMap = mutableMapOf() + + // Collect plain modules. + val plainModules: MutableMap = mutableMapOf() + fun processModule(resolvedDependency: DependencyResult, incomingDependencyId: KResolvedDependencyId) { + if (resolvedDependency !is ResolvedDependencyResult) return + + val requestedModule = resolvedDependency.requested as? ModuleComponentSelector ?: return + val selectedModule = resolvedDependency.selected + val selectedModuleId = selectedModule.id as? ModuleComponentIdentifier ?: return + + val moduleId = KResolvedDependencyId(selectedModuleId.uniqueName) + val module = plainModules.getOrPut(moduleId) { + val artifactPaths = moduleId.uniqueNames.asSequence() + .mapNotNull { uniqueName -> moduleNameToArtifactPaths[uniqueName] } + .firstOrNull() + .orEmpty() + + KResolvedDependency( + id = moduleId, + selectedVersion = KResolvedDependencyVersion(selectedModuleId.version), + requestedVersionsByIncomingDependencies = mutableMapOf(), // To be filled in just below. + artifactPaths = artifactPaths.toMutableSet() + ) + } + + // Record the requested version of the module by the current incoming dependency. + module.requestedVersionsByIncomingDependencies[incomingDependencyId] = KResolvedDependencyVersion(requestedModule.version) + + // TODO: Use [ResolvedDependencyResult.resolvedVariant.externalVariant] to find a connection between platform-specific + // and common modules when "resolvedVariant" and "externalVariant" graduate from incubating state. + if (module.artifactPaths.isNotEmpty()) { + val originModuleId = resolvedDependency.from.id as? ModuleComponentIdentifier + if (originModuleId != null + && selectedModuleId.group == originModuleId.group + && selectedModuleId.module.startsWith(originModuleId.module) + && selectedModuleId.version == originModuleId.version + ) { + // These two modules should be merged. + moduleIdsToMerge[moduleId] = KResolvedDependencyId(originModuleId.uniqueName) + } + } + + selectedModule.dependencies.forEach { processModule(it, incomingDependencyId = moduleId) } + } + + compileDependencyConfiguration.incoming.resolutionResult.root.dependencies.forEach { dependencyResult -> + processModule(dependencyResult, incomingDependencyId = KResolvedDependencyId.SOURCE_CODE_MODULE_ID) + } + + if (moduleIdsToMerge.isEmpty()) + return plainModules.values + + // Do merge. + val replacedModules: MutableMap = mutableMapOf() + moduleIdsToMerge.forEach { (platformSpecificModuleId, commonModuleId) -> + val platformSpecificModule = plainModules.getValue(platformSpecificModuleId) + val commonModule = plainModules.getValue(commonModuleId) + + val replacementModuleId = KResolvedDependencyId(platformSpecificModuleId.uniqueNames + commonModuleId.uniqueNames) + val replacementModule = KResolvedDependency( + id = replacementModuleId, + visibleAsFirstLevelDependency = commonModule.visibleAsFirstLevelDependency, + selectedVersion = commonModule.selectedVersion, + requestedVersionsByIncomingDependencies = mutableMapOf().apply { + this += commonModule.requestedVersionsByIncomingDependencies + this += platformSpecificModule.requestedVersionsByIncomingDependencies - commonModuleId + }, + artifactPaths = mutableSetOf().apply { + this += commonModule.artifactPaths + this += platformSpecificModule.artifactPaths + } + ) + + replacedModules[platformSpecificModuleId] = replacementModule + replacedModules[commonModuleId] = replacementModule + } + + // Assemble new modules together (without "replaced" and with "replacements"). + val mergedModules: MutableMap = mutableMapOf() + mergedModules += plainModules - replacedModules.keys + replacedModules.values.forEach { replacementModule -> mergedModules[replacementModule.id] = replacementModule } + + // Fix references to point to "replacement" modules instead of "replaced" modules. + mergedModules.values.forEach { module -> + module.requestedVersionsByIncomingDependencies.mapNotNull { (replacedModuleId, requestedVersion) -> + val replacementModuleId = replacedModules[replacedModuleId]?.id ?: return@mapNotNull null + Triple(replacedModuleId, replacementModuleId, requestedVersion) + }.forEach { (replacedModuleId, replacementModuleId, requestedVersion) -> + module.requestedVersionsByIncomingDependencies.remove(replacedModuleId) + module.requestedVersionsByIncomingDependencies[replacementModuleId] = requestedVersion + } + } + + return mergedModules.values + } + + private val ModuleComponentIdentifier.uniqueName: String + get() = "$group:$module" +} + +internal class CacheBuilder( + val project: Project, + val binary: NativeBinary, + val konanTarget: KonanTarget, + val externalDependenciesArgs: List +) { private val nativeSingleFileResolveStrategy: SingleFileKlibResolveStrategy get() = CompilerSingleFileKlibResolveAllowingIrProvidersStrategy( @@ -777,6 +929,7 @@ internal class CacheBuilder(val project: Project, val binary: NativeBinary, val ) if (debuggable) args += "-g" + args += externalDependenciesArgs args += "-Xadd-cache=${library.libraryFile.absolutePath}" args += "-Xcache-directory=${cacheDirectory.absolutePath}" args += "-Xcache-directory=${rootCacheDirectory.absolutePath}"