From 933c52d4705b6d0685b3e492b16f68485b506b9f Mon Sep 17 00:00:00 2001 From: "sebastian.sellmair" Date: Tue, 6 Apr 2021 15:52:00 +0200 Subject: [PATCH] [Commonizer] Gracefully handle absent targets for hierarchical commonization --- .../native/internal/CInteropCommonizerTask.kt | 36 +++------ .../kotlin/commonizer/CliCommonizer.kt | 4 +- .../jetbrains/kotlin/commonizer/Commonizer.kt | 2 +- .../kotlin/commonizer/CommonizerDependency.kt | 47 +++++++++++ .../isCommonizerTargetIdentityString.kt | 10 +++ .../commonizer/parseCommonizerTarget.kt | 8 ++ .../kotlin/commonizer/CommonizeLibcurlTest.kt | 9 ++- .../commonizer/CommonizerDependencyTest.kt | 34 ++++++++ .../CommonizerTargetIdentityStringTest.kt | 19 ++++- .../kotlin/commonizer/CommonizerParameters.kt | 9 ++- .../kotlin/commonizer/TargetDependent.kt | 6 +- .../cli/DependenciesLibrariesSetOptionType.kt | 26 ++++++ .../cli/DependencyLibrariesOptionType.kt | 2 +- .../kotlin/commonizer/cli/nativeTasks.kt | 6 +- .../org/jetbrains/kotlin/commonizer/facade.kt | 80 ++++++++++++------- .../commonizer/konan/LibraryCommonizer.kt | 36 +++++---- .../kotlin/commonizer/konan/NativeLibrary.kt | 3 - .../kotlin/commonizer/mergedtree/CirNode.kt | 2 + .../commonizer/mergedtree/CirRootNode.kt | 3 - .../mergedtree/mergers/RootMerger.kt | 2 +- .../commonizer/metadata/CirTreeSerializer.kt | 6 -- .../CommonizerDependencyRepository.kt | 32 ++++++++ .../commonizer/repository/FilesRepository.kt | 11 +-- .../repository/KonanDistributionRepository.kt | 12 +-- .../commonizer/repository/Repository.kt | 7 -- .../commonizer/repository/StdlibRepository.kt | 3 +- 26 files changed, 297 insertions(+), 118 deletions(-) create mode 100644 native/commonizer-api/src/org/jetbrains/kotlin/commonizer/CommonizerDependency.kt create mode 100644 native/commonizer-api/src/org/jetbrains/kotlin/commonizer/isCommonizerTargetIdentityString.kt create mode 100644 native/commonizer-api/test/org/jetbrains/kotlin/commonizer/CommonizerDependencyTest.kt create mode 100644 native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/DependenciesLibrariesSetOptionType.kt create mode 100644 native/commonizer/src/org/jetbrains/kotlin/commonizer/repository/CommonizerDependencyRepository.kt diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CInteropCommonizerTask.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CInteropCommonizerTask.kt index 7bd574c92d5..d60909e3682 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CInteropCommonizerTask.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/native/internal/CInteropCommonizerTask.kt @@ -48,22 +48,6 @@ internal open class CInteropCommonizerTask : AbstractCInteropCommonizerTask() { internal var cinterops = setOf() private set - /** - * All library files produced by the [Project.commonizeNativeDistributionTask] that are relevant for commonization - */ - @get:Classpath - internal val nonHierarchicalNativeDistributionLibraries: Set - get() { - val commonizeNativeDistribution = project.commonizeNativeDistributionTask.get() - return getCommonizationParameters().flatMapTo(mutableSetOf()) { parameters -> - parameters.commonizerTarget.withAllAncestors().flatMap { target -> - commonizeNativeDistribution.commonizerTargetOutputDirectories.flatMap { outputDirectory -> - NativeDistributionCommonizerOutputLayout.getTargetDirectory(outputDirectory, target) - .listFiles().orEmpty().toList() - } - } - } - } @OutputDirectories fun getAllOutputDirectories(): Set { @@ -109,23 +93,29 @@ internal open class CInteropCommonizerTask : AbstractCInteropCommonizerTask() { GradleCliCommonizer(project).commonizeLibraries( konanHome = project.file(project.konanHome), outputCommonizerTarget = parameters.commonizerTarget, - inputLibraries = cinteropsForTarget.map { it.libraryFile.get() }.toSet(), - dependencyLibraries = cinteropsForTarget.flatMap { it.dependencies.files }.toSet() + nativeDistributionLibraries(parameters), + inputLibraries = cinteropsForTarget.map { it.libraryFile.get() }.filter { it.exists() }.toSet(), + dependencyLibraries = cinteropsForTarget.flatMap { it.dependencies.files }.map(::NonTargetedCommonizerDependency).toSet() + + nativeDistributionDependencies(parameters), outputDirectory = outputDirectory(parameters) ) } - private fun nativeDistributionLibraries(parameters: CInteropCommonizationParameters): Set { - val task = project.commonizeNativeDistributionHierarchicalTask?.get() ?: return nonHierarchicalNativeDistributionLibraries + private fun nativeDistributionDependencies(parameters: CInteropCommonizationParameters): Set { + val task = project.commonizeNativeDistributionHierarchicalTask?.get() ?: return emptySet() val rootTarget = task.rootCommonizerTargets .firstOrNull { rootTarget -> parameters.commonizerTarget in rootTarget } ?: return emptySet() val rootTargetOutput = task.getRootOutputDirectory(rootTarget) - return parameters.commonizerTarget.withAllAncestors().flatMap { target -> - HierarchicalCommonizerOutputLayout.getTargetDirectory(rootTargetOutput, target).listFiles().orEmpty().toList() - }.toSet() + return parameters.commonizerTarget.withAllAncestors() + .flatMap { target -> createCommonizerDependencies(rootTargetOutput, target) } + .toSet() + } + + private fun createCommonizerDependencies(rootOutput: File, target: CommonizerTarget): List { + return HierarchicalCommonizerOutputLayout.getTargetDirectory(rootOutput, target).listFiles().orEmpty() + .map { file -> TargetedCommonizerDependency(target, file) } } @Nested diff --git a/native/commonizer-api/src/org/jetbrains/kotlin/commonizer/CliCommonizer.kt b/native/commonizer-api/src/org/jetbrains/kotlin/commonizer/CliCommonizer.kt index 686db4d1e58..8dd0042f641 100644 --- a/native/commonizer-api/src/org/jetbrains/kotlin/commonizer/CliCommonizer.kt +++ b/native/commonizer-api/src/org/jetbrains/kotlin/commonizer/CliCommonizer.kt @@ -25,7 +25,7 @@ public class CliCommonizer(private val executor: Executor) : Commonizer { override fun commonizeLibraries( konanHome: File, inputLibraries: Set, - dependencyLibraries: Set, + dependencyLibraries: Set, outputCommonizerTarget: SharedCommonizerTarget, outputDirectory: File ) { @@ -37,7 +37,7 @@ public class CliCommonizer(private val executor: Executor) : Commonizer { add("-output-commonizer-target"); add(outputCommonizerTarget.identityString) add("-output-path"); add(outputDirectory.absolutePath) if (dependencyLibraries.isNotEmpty()) { - add("-dependency-libraries"); add(dependencyLibraries.joinToString(";") { it.absolutePath }) + add("-dependency-libraries"); add(dependencyLibraries.joinToString(";")) } } executor(arguments) diff --git a/native/commonizer-api/src/org/jetbrains/kotlin/commonizer/Commonizer.kt b/native/commonizer-api/src/org/jetbrains/kotlin/commonizer/Commonizer.kt index 8780ad03ad5..fec337d0b42 100644 --- a/native/commonizer-api/src/org/jetbrains/kotlin/commonizer/Commonizer.kt +++ b/native/commonizer-api/src/org/jetbrains/kotlin/commonizer/Commonizer.kt @@ -14,7 +14,7 @@ public interface Commonizer : Serializable { public fun commonizeLibraries( konanHome: File, inputLibraries: Set, - dependencyLibraries: Set, + dependencyLibraries: Set, outputCommonizerTarget: SharedCommonizerTarget, outputDirectory: File ) diff --git a/native/commonizer-api/src/org/jetbrains/kotlin/commonizer/CommonizerDependency.kt b/native/commonizer-api/src/org/jetbrains/kotlin/commonizer/CommonizerDependency.kt new file mode 100644 index 00000000000..3f2bea9ab31 --- /dev/null +++ b/native/commonizer-api/src/org/jetbrains/kotlin/commonizer/CommonizerDependency.kt @@ -0,0 +1,47 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.commonizer + +import java.io.File + +public sealed class CommonizerDependency { + public abstract val file: File + + final override fun toString(): String { + return this.identityString + } +} + +public data class TargetedCommonizerDependency( + public val target: CommonizerTarget, + public override val file: File +) : CommonizerDependency() + +public data class NonTargetedCommonizerDependency( + public override val file: File +) : CommonizerDependency() + +public val CommonizerDependency.identityString: String + get() = when (this) { + is NonTargetedCommonizerDependency -> this.file.canonicalPath + is TargetedCommonizerDependency -> "${target.identityString}::${file.canonicalPath}" + } + + +public fun parseCommonizerDependency(identityString: String): CommonizerDependency { + val split = identityString.split("::", limit = 2) + if (split.size == 2) { + parseTargetedCommonizerDependencyOrNull(split[0], split[1])?.let { return it } + } + return NonTargetedCommonizerDependency(File(identityString)) +} + +private fun parseTargetedCommonizerDependencyOrNull(commonizerTarget: String, canonicalFilePath: String): TargetedCommonizerDependency? { + return TargetedCommonizerDependency( + target = parseCommonizerTargetOrNull(commonizerTarget) ?: return null, + file = File(canonicalFilePath) + ) +} diff --git a/native/commonizer-api/src/org/jetbrains/kotlin/commonizer/isCommonizerTargetIdentityString.kt b/native/commonizer-api/src/org/jetbrains/kotlin/commonizer/isCommonizerTargetIdentityString.kt new file mode 100644 index 00000000000..d9479622bca --- /dev/null +++ b/native/commonizer-api/src/org/jetbrains/kotlin/commonizer/isCommonizerTargetIdentityString.kt @@ -0,0 +1,10 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.commonizer + +public fun isCommonizerTargetIdentityString(value: String): Boolean { + return parseCommonizerTargetOrNull(value) != null +} diff --git a/native/commonizer-api/src/org/jetbrains/kotlin/commonizer/parseCommonizerTarget.kt b/native/commonizer-api/src/org/jetbrains/kotlin/commonizer/parseCommonizerTarget.kt index f411c4aa009..3075551c21a 100644 --- a/native/commonizer-api/src/org/jetbrains/kotlin/commonizer/parseCommonizerTarget.kt +++ b/native/commonizer-api/src/org/jetbrains/kotlin/commonizer/parseCommonizerTarget.kt @@ -9,6 +9,14 @@ import org.jetbrains.kotlin.commonizer.IdentityStringSyntaxNode.LeafTargetSyntax import org.jetbrains.kotlin.commonizer.IdentityStringSyntaxNode.SharedTargetSyntaxNode import org.jetbrains.kotlin.commonizer.IdentityStringToken.* +public fun parseCommonizerTargetOrNull(identityString: String): CommonizerTarget? { + return try { + parseCommonizerTarget(identityString) + } catch (t: IllegalArgumentException) { + null + } +} + public fun parseCommonizerTarget(identityString: String): CommonizerTarget { try { val tokens = tokenizeIdentityString(identityString) diff --git a/native/commonizer-api/test/org/jetbrains/kotlin/commonizer/CommonizeLibcurlTest.kt b/native/commonizer-api/test/org/jetbrains/kotlin/commonizer/CommonizeLibcurlTest.kt index c7298f0362d..4d9cd9de9cc 100644 --- a/native/commonizer-api/test/org/jetbrains/kotlin/commonizer/CommonizeLibcurlTest.kt +++ b/native/commonizer-api/test/org/jetbrains/kotlin/commonizer/CommonizeLibcurlTest.kt @@ -28,8 +28,13 @@ class CommonizeLibcurlTest { commonizer.commonizeLibraries( konanHome = konanHome, inputLibraries = libraries, - dependencyLibraries = KonanDistribution(konanHome).platformLibsDir.resolve(LINUX_X64.name).listFiles().orEmpty().toSet() + - KonanDistribution(konanHome).platformLibsDir.resolve(LINUX_ARM64.name).listFiles().orEmpty().toSet(), + dependencyLibraries = emptySet() + + KonanDistribution(konanHome).platformLibsDir.resolve(LINUX_X64.name).listFiles().orEmpty() + .map { TargetedCommonizerDependency(LeafCommonizerTarget(LINUX_X64), it) }.toSet() + + + KonanDistribution(konanHome).platformLibsDir.resolve(LINUX_ARM64.name).listFiles().orEmpty() + .map { TargetedCommonizerDependency(LeafCommonizerTarget(LINUX_ARM64), it) } + .toSet(), outputCommonizerTarget = CommonizerTarget(LINUX_ARM64, LINUX_X64), outputDirectory = temporaryOutputDirectory.root ) diff --git a/native/commonizer-api/test/org/jetbrains/kotlin/commonizer/CommonizerDependencyTest.kt b/native/commonizer-api/test/org/jetbrains/kotlin/commonizer/CommonizerDependencyTest.kt new file mode 100644 index 00000000000..7e4ade5a51c --- /dev/null +++ b/native/commonizer-api/test/org/jetbrains/kotlin/commonizer/CommonizerDependencyTest.kt @@ -0,0 +1,34 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.commonizer + +import java.io.File +import kotlin.test.Test +import kotlin.test.assertEquals + +class CommonizerDependencyTest { + + @Test + fun `sample identityString`() { + assertEquals( + "((a, b), c)::/hello.txt", + TargetedCommonizerDependency(parseCommonizerTarget("((a,b), c)"), File("/hello.txt")).identityString + ) + } + + @Test + fun `test serialize deserialize`() { + assertEquals( + parseCommonizerDependency(NonTargetedCommonizerDependency(File("hello.txt")).identityString), + NonTargetedCommonizerDependency(File("hello.txt").canonicalFile) + ) + + assertEquals( + parseCommonizerDependency(TargetedCommonizerDependency(parseCommonizerTarget("((a,b), c)"), File("hello.txt")).identityString), + TargetedCommonizerDependency(parseCommonizerTarget("((a,b), c)"), File("hello.txt").canonicalFile) + ) + } +} diff --git a/native/commonizer-api/test/org/jetbrains/kotlin/commonizer/CommonizerTargetIdentityStringTest.kt b/native/commonizer-api/test/org/jetbrains/kotlin/commonizer/CommonizerTargetIdentityStringTest.kt index c5a09954331..e135db92a26 100644 --- a/native/commonizer-api/test/org/jetbrains/kotlin/commonizer/CommonizerTargetIdentityStringTest.kt +++ b/native/commonizer-api/test/org/jetbrains/kotlin/commonizer/CommonizerTargetIdentityStringTest.kt @@ -7,8 +7,7 @@ package org.jetbrains.kotlin.commonizer import org.jetbrains.kotlin.konan.target.KonanTarget import org.jetbrains.kotlin.konan.target.KonanTarget.* -import kotlin.test.Test -import kotlin.test.assertEquals +import kotlin.test.* class CommonizerTargetIdentityStringTest { @@ -134,4 +133,20 @@ class CommonizerTargetIdentityStringTest { fun `fail parsing CommonizerTarget 10`() { parseCommonizerTarget("(x, (x, y)") } + + @Test + fun isCommonizerIdentityString() { + assertFalse(isCommonizerTargetIdentityString("")) + assertTrue(isCommonizerTargetIdentityString("()")) + assertTrue(isCommonizerTargetIdentityString("(a,b)")) + assertFalse(isCommonizerTargetIdentityString("...")) + assertTrue(isCommonizerTargetIdentityString("x")) + assertFalse(isCommonizerTargetIdentityString("((a)")) + } + + @Test + fun parseCommonizerTargetOrNull() { + assertEquals(parseCommonizerTarget("((a,b),c)"), parseCommonizerTargetOrNull("((a,b),c)")) + assertNull(parseCommonizerTargetOrNull("")) + } } diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/CommonizerParameters.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/CommonizerParameters.kt index f080c4a5456..6042c1884fe 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/commonizer/CommonizerParameters.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/CommonizerParameters.kt @@ -14,16 +14,17 @@ class CommonizerParameters( val outputTarget: SharedCommonizerTarget, val manifestProvider: TargetDependent, val dependenciesProvider: TargetDependent, - val targetProviders: TargetDependent, + val targetProviders: TargetDependent, val resultsConsumer: ResultsConsumer, val statsCollector: StatsCollector? = null, - val progressLogger: ((String) -> Unit)? = null + val progressLogger: ((String) -> Unit)? = null, ) fun CommonizerParameters.getCommonModuleNames(): Set { - if (targetProviders.size < 2) return emptySet() // too few targets + val supportedTargets = targetProviders.filterNonNull() + if (supportedTargets.size < 2) return emptySet() // too few targets - val allModuleNames: List> = targetProviders.toList().map { targetProvider -> + val allModuleNames: List> = supportedTargets.toList().map { targetProvider -> targetProvider.modulesProvider.loadModuleInfos().mapTo(HashSet()) { it.name } } diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/TargetDependent.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/TargetDependent.kt index 03eb8ed32b4..23a00ff8602 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/commonizer/TargetDependent.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/TargetDependent.kt @@ -12,7 +12,7 @@ sealed interface TargetDependent : Iterable { val targets: List fun indexOf(target: CommonizerTarget): Int = targets.indexOf(target) - fun map(mapper: (target: CommonizerTarget, T) -> R): TargetDependent { + fun map(mapper: (target: CommonizerTarget, T) -> R): TargetDependent { return TargetDependent(targets) { target -> mapper(target, get(target)) } @@ -54,6 +54,9 @@ internal fun TargetDependent.mapTargets(mapper: (CommonizerTarget) -> return TargetDependent(targets) { target -> mapper(target) } } +internal inline fun TargetDependent.forEachWithTarget(action: (target: CommonizerTarget, T) -> Unit) { + targets.forEach { target -> action(target, this[target]) } +} internal fun TargetDependent(map: Map): TargetDependent { return MapBasedTargetDependent(map.toMap()) @@ -66,6 +69,7 @@ internal fun TargetDependent(keys: Iterable, factory: (tar internal fun EagerTargetDependent(keys: Iterable, factory: (target: CommonizerTarget) -> T): TargetDependent { return keys.associateWith(factory).toTargetDependent() } + private class MapBasedTargetDependent(private val map: Map) : TargetDependent { override val targets: List = map.keys.toList() override fun get(target: CommonizerTarget): T = map.getValue(target) diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/DependenciesLibrariesSetOptionType.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/DependenciesLibrariesSetOptionType.kt new file mode 100644 index 00000000000..10f0e06860e --- /dev/null +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/DependenciesLibrariesSetOptionType.kt @@ -0,0 +1,26 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.commonizer.cli + +import org.jetbrains.kotlin.commonizer.CommonizerDependency +import org.jetbrains.kotlin.commonizer.parseCommonizerDependency + +internal abstract class DependenciesLibrariesSetOptionType( + mandatory: Boolean, + alias: String, + description: String +) : OptionType>( + mandatory = mandatory, + alias = alias, + description = description +) { + override fun parse(rawValue: String, onError: (reason: String) -> Nothing): Option> { + if (rawValue.isBlank()) { + return Option(this, emptyList()) + } + return Option(this, rawValue.split(";").map(::parseCommonizerDependency)) + } +} diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/DependencyLibrariesOptionType.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/DependencyLibrariesOptionType.kt index 4daabacd662..2f77c5c5335 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/DependencyLibrariesOptionType.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/DependencyLibrariesOptionType.kt @@ -5,7 +5,7 @@ package org.jetbrains.kotlin.commonizer.cli -internal object DependencyLibrariesOptionType : LibrariesSetOptionType( +internal object DependencyLibrariesOptionType : DependenciesLibrariesSetOptionType( mandatory = false, alias = "dependency-libraries", description = "';' separated list of klib file paths that can be used as dependency" diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/nativeTasks.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/nativeTasks.kt index ed68d1cb369..f1a240f62c8 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/nativeTasks.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/cli/nativeTasks.kt @@ -47,7 +47,7 @@ internal class NativeKlibCommonize(options: Collection>) : Task(option val distribution = KonanDistribution(getMandatory()) val destination = getMandatory() val targetLibraries = getMandatory, InputLibrariesOptionType>() - val dependencyLibraries = getOptional, DependencyLibrariesOptionType>().orEmpty() + val dependencyLibraries = getOptional, DependencyLibrariesOptionType>().orEmpty() val outputCommonizerTarget = compatGetOutputTarget() val statsType = getOptional { it == "log-stats" } ?: StatsType.NONE @@ -71,7 +71,7 @@ internal class NativeKlibCommonize(options: Collection>) : Task(option outputTarget = outputCommonizerTarget, repository = repository, dependencies = StdlibRepository(distribution, libraryLoader) + - FilesRepository(dependencyLibraries.toSet(), libraryLoader), + CommonizerDependencyRepository(dependencyLibraries.toSet(), libraryLoader), resultsConsumer = resultsConsumer, statsCollector = statsCollector, progressLogger = progressLogger @@ -100,7 +100,7 @@ internal class NativeDistributionCommonize(options: Collection>) : Tas val progressLogger = ProgressLogger(CliLoggerAdapter(2), startImmediately = true) val libraryLoader = DefaultNativeLibraryLoader(progressLogger) - val repository = KonanDistributionRepository(distribution, outputTarget.allLeaves(), libraryLoader) + val repository = KonanDistributionRepository(distribution, outputTarget.konanTargets, libraryLoader) val statsCollector = StatsCollector(statsType, outputTarget.withAllAncestors().toList()) val resultsConsumer = buildResultsConsumer { diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/facade.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/facade.kt index 3d1714e92f7..19dd280c736 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/commonizer/facade.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/facade.kt @@ -7,14 +7,15 @@ package org.jetbrains.kotlin.commonizer import kotlinx.metadata.klib.ChunkedKlibModuleFragmentWriteStrategy import org.jetbrains.kotlin.commonizer.ResultsConsumer.ModuleResult +import org.jetbrains.kotlin.commonizer.ResultsConsumer.ModuleResult.Missing import org.jetbrains.kotlin.commonizer.ResultsConsumer.Status import org.jetbrains.kotlin.commonizer.core.CommonizationVisitor import org.jetbrains.kotlin.commonizer.mergedtree.CirCommonizedClassifierNodes import org.jetbrains.kotlin.commonizer.mergedtree.CirKnownClassifiers import org.jetbrains.kotlin.commonizer.mergedtree.CirNode.Companion.indexOfCommon -import org.jetbrains.kotlin.commonizer.mergedtree.CirNode.Companion.indices +import org.jetbrains.kotlin.commonizer.mergedtree.CirNode.Companion.targetIndices import org.jetbrains.kotlin.commonizer.mergedtree.CirRootNode -import org.jetbrains.kotlin.commonizer.metadata.CirTreeSerializer +import org.jetbrains.kotlin.commonizer.metadata.CirTreeSerializer.serializeSingleTarget import org.jetbrains.kotlin.commonizer.tree.CirTreeRoot import org.jetbrains.kotlin.commonizer.tree.assembleCirTree import org.jetbrains.kotlin.commonizer.tree.deserializeCirTree @@ -38,7 +39,7 @@ private fun commonize( parameters: CommonizerParameters, storageManager: StorageManager, target: SharedCommonizerTarget, -): CirRootNode { +): CirRootNode? { val cirTrees = getCirTree(parameters, storageManager, target) parameters.progressLogger?.invoke("Build cir tree for $target") @@ -48,11 +49,11 @@ private fun commonize( commonDependencies = parameters.dependencyClassifiers(target) ) - val mergedTree = mergeCirTree(storageManager, classifiers, cirTrees) + val mergedTree = merge(storageManager, classifiers, cirTrees) ?: return null mergedTree.accept(CommonizationVisitor(classifiers, mergedTree), Unit) parameters.progressLogger?.invoke("Commonized declarations for $target") - serialize(mergedTree, parameters) + serialize(parameters, mergedTree, target) parameters.progressLogger?.invoke("Commonized target $target") return mergedTree @@ -60,40 +61,61 @@ private fun commonize( private fun getCirTree( parameters: CommonizerParameters, storageManager: StorageManager, target: SharedCommonizerTarget -): TargetDependent { +): TargetDependent { return EagerTargetDependent(target.targets) { childTarget -> when (childTarget) { - is LeafCommonizerTarget -> deserializeCirTree(parameters, parameters.targetProviders[childTarget]) - is SharedCommonizerTarget -> commonize(parameters, storageManager, childTarget).assembleCirTree() + is LeafCommonizerTarget -> deserialize(parameters, childTarget) + is SharedCommonizerTarget -> commonize(parameters, storageManager, childTarget)?.assembleCirTree() } } } -private fun serialize(mergedTree: CirRootNode, parameters: CommonizerParameters) { - for (targetIndex in mergedTree.indices) { - serializeTarget(mergedTree, targetIndex, parameters) - } +private fun deserialize(parameters: CommonizerParameters, target: CommonizerTarget): CirTreeRoot? { + val targetProvider = parameters.targetProviders[target] ?: return null + return deserializeCirTree(parameters, targetProvider) } -private fun serializeTarget(mergedTree: CirRootNode, targetIndex: Int, parameters: CommonizerParameters) { - val target = mergedTree.getTarget(targetIndex) +private fun merge( + storageManager: StorageManager, classifiers: CirKnownClassifiers, cirTrees: TargetDependent, +): CirRootNode? { + val availableTrees = cirTrees.filterNonNull() + /* Nothing to merge */ + if (availableTrees.size == 0) return null - if (target in parameters.targetProviders.targets) { - parameters.targetProviders[target].modulesProvider.loadModuleInfos() + /* No propagation from leaves */ + if (availableTrees.size == 1 && availableTrees.targets.single() is LeafCommonizerTarget) return null + + return mergeCirTree(storageManager, classifiers, availableTrees) +} + +private fun serialize(parameters: CommonizerParameters, mergedTree: CirRootNode, commonTarget: CommonizerTarget) { + for (targetIndex in mergedTree.targetIndices) { + val target = mergedTree.targetDeclarations[targetIndex]?.target ?: continue + serializeMissingModules(parameters, target) + if (target is LeafCommonizerTarget) { + serialize(parameters, mergedTree, target, targetIndex) + } + } + serialize(parameters, mergedTree, commonTarget, mergedTree.indexOfCommon) +} + +private fun serialize(parameters: CommonizerParameters, mergedTree: CirRootNode, target: CommonizerTarget, targetIndex: Int) { + serializeSingleTarget(mergedTree, targetIndex, parameters.statsCollector) { metadataModule -> + val libraryName = metadataModule.name + val serializedMetadata = with(metadataModule.write(KLIB_FRAGMENT_WRITE_STRATEGY)) { + SerializedMetadata(header, fragments, fragmentNames) + } + val manifestData = parameters.manifestProvider[target].getManifest(libraryName) + parameters.resultsConsumer.consume(target, ModuleResult.Commonized(libraryName, serializedMetadata, manifestData)) + } + parameters.resultsConsumer.targetConsumed(target) +} + +private fun serializeMissingModules(parameters: CommonizerParameters, requestedTarget: CommonizerTarget) { + if (requestedTarget in parameters.targetProviders.targets) { + parameters.targetProviders[requestedTarget]?.modulesProvider?.loadModuleInfos().orEmpty() .filter { it.name !in parameters.getCommonModuleNames() } - .forEach { missingModule -> parameters.resultsConsumer.consume(target, ModuleResult.Missing(missingModule.originalLocation)) } - } - - if (targetIndex == mergedTree.indexOfCommon || target is LeafCommonizerTarget) { - CirTreeSerializer.serializeSingleTarget(mergedTree, targetIndex, parameters.statsCollector) { metadataModule -> - val libraryName = metadataModule.name - val serializedMetadata = with(metadataModule.write(KLIB_FRAGMENT_WRITE_STRATEGY)) { - SerializedMetadata(header, fragments, fragmentNames) - } - val manifestData = parameters.manifestProvider[target].getManifest(libraryName) - parameters.resultsConsumer.consume(target, ModuleResult.Commonized(libraryName, serializedMetadata, manifestData)) - } - parameters.resultsConsumer.targetConsumed(target) + .forEach { missingModule -> parameters.resultsConsumer.consume(requestedTarget, Missing(missingModule.originalLocation)) } } } diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/konan/LibraryCommonizer.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/konan/LibraryCommonizer.kt index 9c595cfb85f..78140bf6d1d 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/commonizer/konan/LibraryCommonizer.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/konan/LibraryCommonizer.kt @@ -9,6 +9,7 @@ import org.jetbrains.kotlin.commonizer.* import org.jetbrains.kotlin.commonizer.repository.Repository import org.jetbrains.kotlin.commonizer.stats.StatsCollector import org.jetbrains.kotlin.commonizer.utils.ProgressLogger +import org.jetbrains.kotlin.utils.addToStdlib.ifNotEmpty internal class LibraryCommonizer internal constructor( private val outputTarget: SharedCommonizerTarget, @@ -26,26 +27,27 @@ internal class LibraryCommonizer internal constructor( progressLogger.logTotal() } - private fun loadLibraries(): TargetDependent { - val librariesByTargets = outputTarget.allLeaves().associateWith { target -> - NativeLibrariesToCommonize(repository.getLibraries(target).toList()) + private fun loadLibraries(): TargetDependent { + val libraries = EagerTargetDependent(outputTarget.allLeaves()) { target -> + repository.getLibraries(target).toList().ifNotEmpty(::NativeLibrariesToCommonize) } - librariesByTargets.forEach { (target, librariesToCommonize) -> - if (librariesToCommonize.libraries.isEmpty()) { - progressLogger.warning("No platform libraries found for target ${target.prettyName}. This target will be excluded from commonization.") - } + libraries.forEachWithTarget { target, librariesOrNull -> + if (librariesOrNull == null) + progressLogger.warning( + "No libraries found for target ${target.prettyName}. This target will be excluded from commonization." + ) } + progressLogger.log("Resolved libraries to be commonized") - return TargetDependent(librariesByTargets) + return libraries } - private fun commonizeAndSaveResults(allLibraries: TargetDependent) { + private fun commonizeAndSaveResults(libraries: TargetDependent) { val parameters = CommonizerParameters( outputTarget = outputTarget, - targetProviders = TargetDependent(outputTarget.allLeaves()) { target -> createTargetProvider(target, allLibraries[target]) } - .filterNonNull(), - manifestProvider = createManifestProvider(allLibraries), + targetProviders = libraries.map { target, targetLibraries -> createTargetProvider(target, targetLibraries) }, + manifestProvider = createManifestProvider(libraries), dependenciesProvider = createDependenciesProvider(), resultsConsumer = resultsConsumer, statsCollector = statsCollector, @@ -54,8 +56,8 @@ internal class LibraryCommonizer internal constructor( runCommonization(parameters) } - private fun createTargetProvider(target: CommonizerTarget, libraries: NativeLibrariesToCommonize): TargetProvider? { - if (libraries.libraries.isEmpty()) return null + private fun createTargetProvider(target: CommonizerTarget, libraries: NativeLibrariesToCommonize?): TargetProvider? { + if (libraries == null) return null return TargetProvider( target = target, modulesProvider = DefaultModulesProvider.create(libraries) @@ -69,13 +71,13 @@ internal class LibraryCommonizer internal constructor( } private fun createManifestProvider( - libraries: TargetDependent + libraries: TargetDependent ): TargetDependent { return TargetDependent(outputTarget.withAllAncestors()) { target -> when (target) { - is LeafCommonizerTarget -> libraries[target] + is LeafCommonizerTarget -> libraries[target] ?: error("Can't provide manifest for missing target $target") is SharedCommonizerTarget -> CommonNativeManifestDataProvider( - target.allLeaves().map { leafTarget -> libraries[leafTarget] } + target.allLeaves().mapNotNull { leafTarget -> libraries.getOrNull(leafTarget) } ) } } diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/konan/NativeLibrary.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/konan/NativeLibrary.kt index f659de37a96..6145ae54ccd 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/commonizer/konan/NativeLibrary.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/konan/NativeLibrary.kt @@ -8,9 +8,6 @@ package org.jetbrains.kotlin.commonizer.konan import gnu.trove.THashMap -import org.jetbrains.kotlin.commonizer.LeafCommonizerTarget -import org.jetbrains.kotlin.commonizer.SharedCommonizerTarget -import org.jetbrains.kotlin.commonizer.TargetDependent import org.jetbrains.kotlin.library.KotlinLibrary interface NativeManifestDataProvider { diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/mergedtree/CirNode.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/mergedtree/CirNode.kt index 785eb825c14..e6d1e2cabcf 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/commonizer/mergedtree/CirNode.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/mergedtree/CirNode.kt @@ -22,6 +22,8 @@ interface CirNode { companion object { inline val CirNode<*, *>.indices: IntRange get() = 0..targetDeclarations.size + inline val CirNode<*, *>.targetIndices: List get() = indices - indexOfCommon + inline val CirNode<*, *>.indexOfCommon: Int get() = targetDeclarations.size diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/mergedtree/CirRootNode.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/mergedtree/CirRootNode.kt index 02446432262..f0f117e512e 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/commonizer/mergedtree/CirRootNode.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/mergedtree/CirRootNode.kt @@ -19,9 +19,6 @@ class CirRootNode( ) : CirNode { val modules: MutableMap = THashMap() - fun getTarget(targetIndex: Int): CommonizerTarget = - (if (targetIndex == indexOfCommon) commonDeclaration() else targetDeclarations[targetIndex])!!.target - override fun accept(visitor: CirNodeVisitor, data: T): R = visitor.visitRootNode(this, data) diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/mergedtree/mergers/RootMerger.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/mergedtree/mergers/RootMerger.kt index 9ed291ec21c..b773b74c04c 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/commonizer/mergedtree/mergers/RootMerger.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/mergedtree/mergers/RootMerger.kt @@ -23,7 +23,7 @@ internal class RootMerger( val commonModuleNames = parameters.getCommonModuleNames() val missingModuleInfosByTargets = mutableMapOf>() - parameters.targetProviders.forEachIndexed { targetIndex, targetProvider -> + parameters.targetProviders.filterNotNull().forEachIndexed { targetIndex, targetProvider -> val allModuleInfos = targetProvider.modulesProvider.loadModuleInfos() val (commonModuleInfos, missingModuleInfos) = allModuleInfos.partition { it.name in commonModuleNames } diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/metadata/CirTreeSerializer.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/metadata/CirTreeSerializer.kt index e4f9c102685..de50ce6e569 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/commonizer/metadata/CirTreeSerializer.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/metadata/CirTreeSerializer.kt @@ -293,7 +293,6 @@ private class CirTreeSerializationVisitor( internal data class CirTreeSerializationContext( val targetIndex: Int, - val target: CommonizerTarget, val isCommon: Boolean, val typeParameterIndexOffset: Int, val currentPath: Path @@ -332,7 +331,6 @@ internal data class CirTreeSerializationContext( return CirTreeSerializationContext( targetIndex = targetIndex, - target = target, isCommon = isCommon, typeParameterIndexOffset = 0, currentPath = Path.Module(moduleName) @@ -344,7 +342,6 @@ internal data class CirTreeSerializationContext( return CirTreeSerializationContext( targetIndex = targetIndex, - target = target, isCommon = isCommon, typeParameterIndexOffset = 0, currentPath = Path.Package(packageName) @@ -369,7 +366,6 @@ internal data class CirTreeSerializationContext( return CirTreeSerializationContext( targetIndex = targetIndex, - target = target, isCommon = isCommon, typeParameterIndexOffset = typeParameterIndexOffset + outerClassTypeParametersCount, currentPath = newPath @@ -394,7 +390,6 @@ internal data class CirTreeSerializationContext( return CirTreeSerializationContext( targetIndex = targetIndex, - target = target, isCommon = isCommon, typeParameterIndexOffset = typeParameterIndexOffset + ownerClassTypeParametersCount, currentPath = newPath @@ -417,7 +412,6 @@ internal data class CirTreeSerializationContext( fun rootContext(rootNode: CirRootNode, targetIndex: Int): CirTreeSerializationContext = CirTreeSerializationContext( targetIndex = targetIndex, - target = rootNode.getTarget(targetIndex), isCommon = rootNode.indexOfCommon == targetIndex, typeParameterIndexOffset = 0, currentPath = Path.Empty diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/repository/CommonizerDependencyRepository.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/repository/CommonizerDependencyRepository.kt new file mode 100644 index 00000000000..bd4c87d40e4 --- /dev/null +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/repository/CommonizerDependencyRepository.kt @@ -0,0 +1,32 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.commonizer.repository + +import org.jetbrains.kotlin.commonizer.* +import org.jetbrains.kotlin.commonizer.NativeLibraryLoader +import org.jetbrains.kotlin.commonizer.konan.NativeLibrary + +internal class CommonizerDependencyRepository( + private val dependencies: Set, + private val libraryLoader: NativeLibraryLoader +) : Repository { + + private val nonTargetedDependencyRepository = FilesRepository( + libraryFiles = dependencies.filterIsInstance().map { it.file }.toSet(), + libraryLoader = libraryLoader + ) + + private val targetedDependencies: Map>> by lazy { + dependencies + .filterIsInstance() + .groupBy { it.target } + .mapValues { (_, dependencies) -> lazy { dependencies.map { libraryLoader(it.file) }.toSet() } } + } + + override fun getLibraries(target: CommonizerTarget): Set { + return targetedDependencies[target]?.value.orEmpty() + nonTargetedDependencyRepository.getLibraries(target) + } +} diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/repository/FilesRepository.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/repository/FilesRepository.kt index 58a516f074a..11d2d71775a 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/commonizer/repository/FilesRepository.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/repository/FilesRepository.kt @@ -17,7 +17,7 @@ internal class FilesRepository( private val libraryLoader: NativeLibraryLoader ) : Repository { - private val librariesByTarget: Map, Set> by lazy { + private val librariesByKonanTargets: Map, Set> by lazy { libraryFiles .map(libraryLoader::invoke) .groupBy { library -> library.manifestData.nativeTargets.map(::konanTargetOrThrow).toSet() } @@ -25,10 +25,11 @@ internal class FilesRepository( } override fun getLibraries(target: CommonizerTarget): Set { - return librariesByTarget[target.konanTargets].orEmpty() + return librariesByKonanTargets[target.konanTargets].orEmpty() + } + + private fun konanTargetOrThrow(value: String): KonanTarget { + return KonanTarget.predefinedTargets[value] ?: error("Unexpected KonanTarget $value") } } -private fun konanTargetOrThrow(value: String): KonanTarget { - return KonanTarget.predefinedTargets[value] ?: error("Unexpected KonanTarget $value") -} diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/repository/KonanDistributionRepository.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/repository/KonanDistributionRepository.kt index eee46c33ba9..ae67d589ba0 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/commonizer/repository/KonanDistributionRepository.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/repository/KonanDistributionRepository.kt @@ -6,14 +6,16 @@ package org.jetbrains.kotlin.commonizer.repository import org.jetbrains.kotlin.commonizer.* +import org.jetbrains.kotlin.commonizer.NativeLibraryLoader import org.jetbrains.kotlin.commonizer.konan.NativeLibrary +import org.jetbrains.kotlin.konan.target.KonanTarget internal class KonanDistributionRepository( konanDistribution: KonanDistribution, - targets: Set, + targets: Set, libraryLoader: NativeLibraryLoader, ) : Repository { - private val librariesByTarget: Map>> = + private val librariesByTarget: Map>> = targets.associateWith { target -> lazy { konanDistribution.platformLibsDir @@ -27,9 +29,7 @@ internal class KonanDistributionRepository( } override fun getLibraries(target: CommonizerTarget): Set { - return when (target) { - is LeafCommonizerTarget -> librariesByTarget[target]?.value ?: error("Missing target libraries for: $target") - is SharedCommonizerTarget -> emptySet() - } + val singleTarget = target.konanTargets.singleOrNull() ?: return emptySet() + return librariesByTarget[singleTarget]?.value ?: error("Missing target libraries for $target") } } diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/repository/Repository.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/repository/Repository.kt index fd50c8c28a2..4c70c357abe 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/commonizer/repository/Repository.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/repository/Repository.kt @@ -6,7 +6,6 @@ package org.jetbrains.kotlin.commonizer.repository import org.jetbrains.kotlin.commonizer.CommonizerTarget -import org.jetbrains.kotlin.commonizer.LeafCommonizerTarget import org.jetbrains.kotlin.commonizer.konan.NativeLibrary internal interface Repository { @@ -25,9 +24,3 @@ private class CompositeRepository(val repositories: Iterable) : Repo return repositories.map { it.getLibraries(target) }.flatten().toSet() } } - -internal object EmptyRepository : Repository { - override fun getLibraries(target: CommonizerTarget): Set { - return emptySet() - } -} diff --git a/native/commonizer/src/org/jetbrains/kotlin/commonizer/repository/StdlibRepository.kt b/native/commonizer/src/org/jetbrains/kotlin/commonizer/repository/StdlibRepository.kt index 97e5b015e6d..726fa48dd39 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/commonizer/repository/StdlibRepository.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/commonizer/repository/StdlibRepository.kt @@ -18,7 +18,6 @@ internal class StdlibRepository( } override fun getLibraries(target: CommonizerTarget): Set { - return if (target is SharedCommonizerTarget) setOf(stdlib) - else emptySet() + return if (target is SharedCommonizerTarget) setOf(stdlib) else emptySet() } }