[Gradle] Implement support for hierarchical commonization

2021-03-26 15:53:45 +01:00
parent 68c3e39058
commit 8c941fc203
21 changed files with 269 additions and 59 deletions
@@ -32,8 +32,11 @@ fun CommonizerParameters.getCommonModuleNames(): Set<String> {

 internal fun CommonizerParameters.dependencyClassifiers(target: CommonizerTarget): CirProvidedClassifiers {
    val modules = outputTarget.withAllAncestors()
+        .sortedBy { it.level }
        .filter { it == target || it isAncestorOf target }
        .mapNotNull { compatibleTarget -> dependenciesProvider[compatibleTarget] }

-    return CirProvidedClassifiers.of(modules.map { module -> CirProvidedClassifiers.by(module) } + CirFictitiousFunctionClassifiers)
+    return modules.fold<ModulesProvider, CirProvidedClassifiers>(CirFictitiousFunctionClassifiers) { classifiers, module ->
+        CirProvidedClassifiers.of(classifiers, CirProvidedClassifiers.by(module))
+    }
 }
@@ -63,6 +63,9 @@ internal fun <T> TargetDependent(keys: Iterable<CommonizerTarget>, factory: (tar
    return FactoryBasedTargetDependent(keys.toList(), factory)
 }

+internal fun <T> EagerTargetDependent(keys: Iterable<CommonizerTarget>, factory: (target: CommonizerTarget) -> T): TargetDependent<T> {
+    return keys.associateWith(factory).toTargetDependent()
+}
 private class MapBasedTargetDependent<T>(private val map: Map<CommonizerTarget, T>) : TargetDependent<T> {
    override val targets: List<CommonizerTarget> = map.keys.toList()
    override fun get(target: CommonizerTarget): T = map.getValue(target)
@@ -8,7 +8,8 @@ package org.jetbrains.kotlin.commonizer.cli
 import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.konan.target.KonanTarget.Companion.predefinedTargets

-internal object NativeTargetsOptionType : OptionType<List<KonanTarget>>("targets", "Comma-separated list of hardware targets") {
+internal object NativeTargetsOptionType : OptionType<List<KonanTarget>>(
+    "targets", "Comma-separated list of hardware targets", mandatory = false) {
    override fun parse(rawValue: String, onError: (reason: String) -> Nothing): Option<List<KonanTarget>> {
        val targetNames = rawValue.split(',')
        if (targetNames.isEmpty()) onError("No hardware targets specified: $rawValue")
@@ -11,7 +11,7 @@ import org.jetbrains.kotlin.commonizer.parseCommonizerTarget
 internal object OutputCommonizerTargetOptionType : OptionType<SharedCommonizerTarget>(
    alias = "output-commonizer-target",
    description = "Shared commonizer target representing the commonized output hierarchy",
-    mandatory = true
+    mandatory = false
 ) {
    override fun parse(rawValue: String, onError: (reason: String) -> Nothing): Option<SharedCommonizerTarget> {
        return try {
@@ -35,7 +35,7 @@ internal abstract class Task(private val options: Collection<Option<*>>) : Compa
        return option.value as T
    }

-    protected inline fun <reified T, reified O : OptionType<T>> getOptional(nameFilter: (String) -> Boolean = { true }): T? {
+    internal inline fun <reified T, reified O : OptionType<T>> getOptional(nameFilter: (String) -> Boolean = { true }): T? {
        val option = options.filter { it.type is O }.singleOrNull { nameFilter(it.type.alias) }
        if (option != null) check(!option.type.mandatory)

@@ -18,6 +18,7 @@ internal enum class TaskType(
            NativeDistributionOptionType,
            OutputOptionType,
            NativeTargetsOptionType,
+            OutputCommonizerTargetOptionType,
            BooleanOptionType(
                "copy-stdlib",
                "Boolean (default false);\nwhether to copy Kotlin/Native endorsed libraries to the destination",
@@ -48,7 +48,7 @@ internal class NativeKlibCommonize(options: Collection<Option<*>>) : Task(option
        val destination = getMandatory<File, OutputOptionType>()
        val targetLibraries = getMandatory<List<File>, InputLibrariesOptionType>()
        val dependencyLibraries = getOptional<List<File>, DependencyLibrariesOptionType>().orEmpty()
-        val outputCommonizerTarget = getMandatory<SharedCommonizerTarget, OutputCommonizerTargetOptionType>()
+        val outputCommonizerTarget = compatGetOutputTarget()
        val statsType = getOptional<StatsType, StatsTypeOptionType> { it == "log-stats" } ?: StatsType.NONE

        val konanTargets = outputCommonizerTarget.konanTargets
@@ -87,8 +87,12 @@ internal class NativeDistributionCommonize(options: Collection<Option<*>>) : Tas
    override fun execute(logPrefix: String) {
        val distribution = KonanDistribution(getMandatory<File, NativeDistributionOptionType>())
        val destination = getMandatory<File, OutputOptionType>()
-        val konanTargets = getMandatory<List<KonanTarget>, NativeTargetsOptionType>()
-        val commonizerTargets = konanTargets.map(::CommonizerTarget)
+
+        val outputTarget = compatGetOutputTarget()
+        val outputLayout = if (getOptional<SharedCommonizerTarget, OutputCommonizerTargetOptionType>() != null)
+            HierarchicalCommonizerOutputLayout
+        else NativeDistributionCommonizerOutputLayout
+

        val copyStdlib = getOptional<Boolean, BooleanOptionType> { it == "copy-stdlib" } ?: false
        val copyEndorsedLibs = getOptional<Boolean, BooleanOptionType> { it == "copy-endorsed-libs" } ?: false
@@ -96,30 +100,24 @@ internal class NativeDistributionCommonize(options: Collection<Option<*>>) : Tas

        val progressLogger = ProgressLogger(CliLoggerAdapter(2), startImmediately = true)
        val libraryLoader = DefaultNativeLibraryLoader(progressLogger)
-        val repository = KonanDistributionRepository(distribution, commonizerTargets.toSet(), libraryLoader)
-        val existingTargets = commonizerTargets.filter { repository.getLibraries(it).isNotEmpty() }.toSet()
-        val statsCollector = StatsCollector(statsType, commonizerTargets)
+        val repository = KonanDistributionRepository(distribution, outputTarget.allLeaves(), libraryLoader)
+        val statsCollector = StatsCollector(statsType, outputTarget.withAllAncestors().toList())

        val resultsConsumer = buildResultsConsumer {
-            this add ModuleSerializer(destination, NativeDistributionCommonizerOutputLayout)
-            this add CopyUnconsumedModulesAsIsConsumer(
-                repository, destination, commonizerTargets.toSet(), NativeDistributionCommonizerOutputLayout, progressLogger
-            )
+            this add ModuleSerializer(destination, outputLayout)
+            this add CopyUnconsumedModulesAsIsConsumer(repository, destination, outputTarget.allLeaves(), outputLayout, progressLogger)
            if (copyStdlib) this add CopyStdlibResultsConsumer(distribution, destination, progressLogger)
            if (copyEndorsedLibs) this add CopyEndorsedLibrairesResultsConsumer(distribution, destination, progressLogger)
-
-            SharedCommonizerTarget.ifNotEmpty(existingTargets)?.let { sharedTargetForLogger ->
-                this add LoggingResultsConsumer(sharedTargetForLogger, progressLogger)
-            }
+            this add LoggingResultsConsumer(outputTarget, progressLogger)
        }

-        val targetNames = commonizerTargets.joinToString { it.prettyName }
-        val descriptionSuffix = estimateLibrariesCount(repository, commonizerTargets).let { " ($it items)" }
+        val targetNames = outputTarget.allLeaves().joinToString { it.prettyName }
+        val descriptionSuffix = estimateLibrariesCount(repository, outputTarget.allLeaves()).let { " ($it items)" }
        val description = "${logPrefix}Preparing commonized Kotlin/Native libraries for targets $targetNames$descriptionSuffix"
        println(description)

        LibraryCommonizer(
-            outputTarget = SharedCommonizerTarget(commonizerTargets.toSet()),
+            outputTarget = outputTarget,
            repository = repository,
            dependencies = StdlibRepository(distribution, libraryLoader),
            resultsConsumer = resultsConsumer,
@@ -133,8 +131,16 @@ internal class NativeDistributionCommonize(options: Collection<Option<*>>) : Tas
    }

    companion object {
-        private fun estimateLibrariesCount(repository: Repository, targets: List<LeafCommonizerTarget>): Int {
+        private fun estimateLibrariesCount(repository: Repository, targets: Iterable<LeafCommonizerTarget>): Int {
            return targets.flatMap { repository.getLibraries(it) }.count()
        }
    }
 }
+
+private fun Task.compatGetOutputTarget(): SharedCommonizerTarget {
+    getOptional<SharedCommonizerTarget, OutputCommonizerTargetOptionType>()?.let { return it }
+    val konanTargets = getOptional<List<KonanTarget>, NativeTargetsOptionType>() ?: throw IllegalArgumentException(
+        "Missing ${OutputCommonizerTargetOptionType.alias} or deprecated ${NativeTargetsOptionType.alias} option was specified"
+    )
+    return SharedCommonizerTarget(konanTargets)
+}
@@ -5,23 +5,23 @@

 package org.jetbrains.kotlin.commonizer.core

-import org.jetbrains.kotlin.commonizer.LeafCommonizerTarget
+import org.jetbrains.kotlin.commonizer.CommonizerTarget
 import org.jetbrains.kotlin.commonizer.SharedCommonizerTarget
 import org.jetbrains.kotlin.commonizer.cir.CirRoot

 class RootCommonizer : AbstractStandardCommonizer<CirRoot, CirRoot>() {
-    private val leafTargets = mutableSetOf<LeafCommonizerTarget>()
+    private val targets = mutableSetOf<CommonizerTarget>()

    override fun commonizationResult() = CirRoot.create(
-        target = SharedCommonizerTarget(leafTargets)
+        target = SharedCommonizerTarget(targets)
    )

    override fun initialize(first: CirRoot) {
-        leafTargets += first.target as LeafCommonizerTarget
+        targets += first.target
    }

    override fun doCommonizeWith(next: CirRoot): Boolean {
-        leafTargets += next.target as LeafCommonizerTarget
+        targets += next.target
        return true
    }
 }
@@ -61,7 +61,7 @@ private fun commonize(
 private fun getCirTree(
    parameters: CommonizerParameters, storageManager: StorageManager, target: SharedCommonizerTarget
 ): TargetDependent<CirTreeRoot> {
-    return TargetDependent(target.targets) { childTarget ->
+    return EagerTargetDependent(target.targets) { childTarget ->
        when (childTarget) {
            is LeafCommonizerTarget -> deserializeCirTree(parameters, parameters.targetProviders[childTarget])
            is SharedCommonizerTarget -> commonize(parameters, storageManager, childTarget).assembleCirTree()
@@ -50,10 +50,6 @@ interface CirProvidedClassifiers {
            }
        }

-        fun of(delegates: Iterable<CirProvidedClassifiers?>): CirProvidedClassifiers {
-            return of(*delegates.filterNotNull().toTypedArray())
-        }
-
        fun by(modulesProvider: ModulesProvider?): CirProvidedClassifiers =
            if (modulesProvider != null) CirProvidedClassifiersByModules.load(modulesProvider) else EMPTY
    }
@@ -68,19 +68,23 @@ class CommonizerFacadeTest {
        private fun Map<String, List<String>>.toCommonizerParameters(
            resultsConsumer: ResultsConsumer,
            manifestDataProvider: NativeManifestDataProvider = MockNativeManifestDataProvider()
-        ) = CommonizerParameters(
-            targetProviders = mapKeys { (targetName, _) -> LeafCommonizerTarget(targetName) }.toTargetDependent()
-                .map { target, moduleNames ->
+        ): CommonizerParameters {
+            val targetDependentModuleNames = mapKeys { (targetName, _) -> LeafCommonizerTarget(targetName) }.toTargetDependent()
+            val sharedTarget = SharedCommonizerTarget(targetDependentModuleNames.targets.toSet())
+
+            return CommonizerParameters(
+                outputTarget = sharedTarget,
+                dependenciesProvider = TargetDependent(sharedTarget.withAllAncestors()) { null },
+                manifestProvider = TargetDependent(sharedTarget.withAllAncestors()) { manifestDataProvider },
+                targetProviders = targetDependentModuleNames.map { target, moduleNames ->
                    TargetProvider(
                        target = target,
-                        modulesProvider = MockModulesProvider.create(moduleNames),
-                        dependencyModulesProvider = null,
-                        manifestProvider = manifestDataProvider,
+                        modulesProvider = MockModulesProvider.create(moduleNames)
                    )
                },
-            resultsConsumer = resultsConsumer,
-            commonManifestProvider = manifestDataProvider
-        )
+                resultsConsumer = resultsConsumer,
+            )
+        }

        private fun doTestNothingToCommonize(originalModules: Map<String, List<String>>) {
            val results = MockResultsConsumer()