From b777c89bc88887716527ce298eca0cbc8e7038be Mon Sep 17 00:00:00 2001 From: "sebastian.sellmair" Date: Thu, 11 Mar 2021 12:00:45 +0100 Subject: [PATCH] [Commonizer] NativeKlibCommonize: Only load dependencies from native distribution --- .../commonizer/CommonizerParameters.kt | 10 +--- .../descriptors/commonizer/cli/nativeTasks.kt | 15 ++---- ...sProvider.kt => DefaultModulesProvider.kt} | 9 ++-- .../commonizer/konan/LibraryCommonizer.kt | 46 ++++++++++--------- .../repository/KonanDistributionRepository.kt | 20 ++++++++ .../AbstractCommonizationFromSourcesTest.kt | 5 +- 6 files changed, 60 insertions(+), 45 deletions(-) rename native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/{NativeDistributionModulesProvider.kt => DefaultModulesProvider.kt} (89%) diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizerParameters.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizerParameters.kt index f53a823693d..4ac532173aa 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizerParameters.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizerParameters.kt @@ -11,20 +11,14 @@ import org.jetbrains.kotlin.descriptors.commonizer.stats.StatsCollector class CommonizerParameters( val resultsConsumer: ResultsConsumer, val manifestDataProvider: TargetedNativeManifestDataProvider, + // common module dependencies (ex: Kotlin stdlib) + val dependencyModulesProvider: ModulesProvider? = null, val statsCollector: StatsCollector? = null, val progressLogger: ((String) -> Unit)? = null ) { // use linked hash map to preserve order private val _targetProviders = LinkedHashMap() val targetProviders: List get() = _targetProviders.values.toList() - val sharedTarget: SharedCommonizerTarget get() = SharedCommonizerTarget(_targetProviders.keys) - - // common module dependencies (ex: Kotlin stdlib) - var dependencyModulesProvider: ModulesProvider? = null - set(value) { - check(field == null) - field = value - } fun addTarget(targetProvider: TargetProvider): CommonizerParameters { require(targetProvider.target !in _targetProviders) { "Target ${targetProvider.target} is already added" } diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/nativeTasks.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/nativeTasks.kt index 4215c665aef..8746ce5e318 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/nativeTasks.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/cli/nativeTasks.kt @@ -7,15 +7,7 @@ package org.jetbrains.kotlin.descriptors.commonizer.cli import org.jetbrains.kotlin.descriptors.commonizer.* import org.jetbrains.kotlin.descriptors.commonizer.konan.* -import org.jetbrains.kotlin.descriptors.commonizer.konan.CopyUnconsumedModulesAsIsConsumer -import org.jetbrains.kotlin.descriptors.commonizer.konan.CopyStdlibResultsConsumer -import org.jetbrains.kotlin.descriptors.commonizer.konan.LibraryCommonizer -import org.jetbrains.kotlin.descriptors.commonizer.konan.ModuleSerializer import org.jetbrains.kotlin.descriptors.commonizer.repository.* -import org.jetbrains.kotlin.descriptors.commonizer.repository.EmptyRepository -import org.jetbrains.kotlin.descriptors.commonizer.repository.FilesRepository -import org.jetbrains.kotlin.descriptors.commonizer.repository.KonanDistributionRepository -import org.jetbrains.kotlin.descriptors.commonizer.repository.Repository import org.jetbrains.kotlin.descriptors.commonizer.stats.FileStatsOutput import org.jetbrains.kotlin.descriptors.commonizer.stats.StatsCollector import org.jetbrains.kotlin.descriptors.commonizer.stats.StatsType @@ -78,8 +70,11 @@ internal class NativeKlibCommonize(options: Collection>) : Task(option LibraryCommonizer( konanDistribution = distribution, repository = repository, - dependencies = KonanDistributionRepository(distribution, commonizerTargets.toSet(), libraryLoader) + - FilesRepository(dependencyLibraries.toSet(), libraryLoader), + dependencies = + KonanDistributionRepository( + distribution, commonizerTargets.toSet(), libraryLoader, + filter = KonanDistributionRepository.Filter.onlyDependenciesOf(repository) + ) + FilesRepository(dependencyLibraries.toSet(), libraryLoader), libraryLoader = libraryLoader, targets = commonizerTargets, resultsConsumer = resultsConsumer, diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionModulesProvider.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/DefaultModulesProvider.kt similarity index 89% rename from native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionModulesProvider.kt rename to native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/DefaultModulesProvider.kt index 51e76d1e079..53a9eb715f4 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionModulesProvider.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/DefaultModulesProvider.kt @@ -12,7 +12,7 @@ import org.jetbrains.kotlin.library.SerializedMetadata import org.jetbrains.kotlin.library.metadata.parseModuleHeader import java.io.File -internal class NativeDistributionModulesProvider(libraries: Collection) : ModulesProvider { +internal class DefaultModulesProvider(libraries: Collection) : ModulesProvider { internal class NativeModuleInfo( name: String, originalLocation: File, @@ -69,10 +69,13 @@ internal class NativeDistributionModulesProvider(libraries: Collection): ModulesProvider = + DefaultModulesProvider(libraries.toList()) } } diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/LibraryCommonizer.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/LibraryCommonizer.kt index e8d46eb8e54..4964dedf205 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/LibraryCommonizer.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/LibraryCommonizer.kt @@ -6,11 +6,9 @@ package org.jetbrains.kotlin.descriptors.commonizer.konan import org.jetbrains.kotlin.descriptors.commonizer.* -import org.jetbrains.kotlin.descriptors.commonizer.konan.LibraryCommonizer.* import org.jetbrains.kotlin.descriptors.commonizer.repository.Repository import org.jetbrains.kotlin.descriptors.commonizer.stats.StatsCollector import org.jetbrains.kotlin.descriptors.commonizer.utils.ProgressLogger -import org.jetbrains.kotlin.konan.library.* internal class LibraryCommonizer internal constructor( private val konanDistribution: KonanDistribution, @@ -47,32 +45,36 @@ internal class LibraryCommonizer internal constructor( } private fun commonizeAndSaveResults(allLibraries: AllNativeLibraries) { - val manifestProvider = TargetedNativeManifestDataProvider(allLibraries) + val parameters = CommonizerParameters( + resultsConsumer = resultsConsumer, + manifestDataProvider = TargetedNativeManifestDataProvider(allLibraries), + dependencyModulesProvider = DefaultModulesProvider.forStandardLibrary(allLibraries.stdlib), + statsCollector = statsCollector, + progressLogger = progressLogger::log + ) - val parameters = CommonizerParameters(resultsConsumer, manifestProvider, statsCollector, progressLogger::log).apply { - dependencyModulesProvider = NativeDistributionModulesProvider.forStandardLibrary(allLibraries.stdlib) - - allLibraries.librariesByTargets.forEach { (target, librariesToCommonize) -> - if (librariesToCommonize.libraries.isEmpty()) return@forEach - - val modulesProvider = NativeDistributionModulesProvider.platformLibraries(librariesToCommonize) - val dependencyModuleProvider = NativeDistributionModulesProvider.platformLibraries( - NativeLibrariesToCommonize(dependencies.getLibraries(target).toList()), - ) - - addTarget( - TargetProvider( - target = target, - modulesProvider = modulesProvider, - dependencyModulesProvider = dependencyModuleProvider - ) - ) - } + allLibraries.librariesByTargets.forEach { (target, librariesToCommonize) -> + parameters.addTarget(target, librariesToCommonize) } runCommonization(parameters) } + private fun CommonizerParameters.addTarget(target: LeafCommonizerTarget, libraries: NativeLibrariesToCommonize) { + if (libraries.libraries.isEmpty()) return + + val modulesProvider = DefaultModulesProvider.platformLibraries(libraries) + val dependencyModuleProvider = DefaultModulesProvider.platformLibraries(dependencies.getLibraries(target)) + + addTarget( + TargetProvider( + target = target, + modulesProvider = modulesProvider, + dependencyModulesProvider = dependencyModuleProvider + ) + ) + } + private fun checkPreconditions() { when (targets.size) { 0 -> progressLogger.fatal("No targets specified") diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/repository/KonanDistributionRepository.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/repository/KonanDistributionRepository.kt index 34c848ebc48..49643f934d4 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/repository/KonanDistributionRepository.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/repository/KonanDistributionRepository.kt @@ -10,12 +10,19 @@ import org.jetbrains.kotlin.descriptors.commonizer.LeafCommonizerTarget import org.jetbrains.kotlin.descriptors.commonizer.NativeLibraryLoader import org.jetbrains.kotlin.descriptors.commonizer.konan.NativeLibrary import org.jetbrains.kotlin.descriptors.commonizer.platformLibsDir +import org.jetbrains.kotlin.descriptors.commonizer.repository.KonanDistributionRepository.Filter internal class KonanDistributionRepository( konanDistribution: KonanDistribution, targets: Set, libraryLoader: NativeLibraryLoader, + filter: Filter = Filter.none() ) : Repository { + fun interface Filter { + operator fun invoke(target: LeafCommonizerTarget, library: NativeLibrary): Boolean + + companion object Factory + } private val librariesByTarget: Map>> = targets.associateWith { target -> @@ -26,6 +33,7 @@ internal class KonanDistributionRepository( ?.listFiles() .orEmpty().toList() .map { libraryLoader(it) } + .filter { filter(target, it) } .toSet() } } @@ -34,3 +42,15 @@ internal class KonanDistributionRepository( return librariesByTarget[target]?.value ?: error("Missing target $target") } } + +internal fun Filter.Factory.onlyDependenciesOf(libraries: Repository): Filter { + val requestedDependencies = mutableMapOf>() + return Filter { target, dependency -> + val requestedDependenciesForTarget = requestedDependencies.getOrPut(target) { + libraries.getLibraries(target).flatMap { library -> library.manifestData.dependencies }.toSet() + } + dependency.manifestData.uniqueName in requestedDependenciesForTarget + } +} + +internal fun Filter.Factory.none() = Filter { _, _ -> true } diff --git a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/AbstractCommonizationFromSourcesTest.kt b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/AbstractCommonizationFromSourcesTest.kt index 77ecb0fd806..fab747ad9d7 100644 --- a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/AbstractCommonizationFromSourcesTest.kt +++ b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/AbstractCommonizationFromSourcesTest.kt @@ -207,8 +207,9 @@ private class AnalyzedModules( fun toCommonizerParameters( resultsConsumer: ResultsConsumer, manifestDataProvider: TargetedNativeManifestDataProvider = MockNativeManifestDataProvider() - ) = CommonizerParameters(resultsConsumer, manifestDataProvider).also { parameters -> - parameters.dependencyModulesProvider = dependencyModules[sharedTarget]?.let(MockModulesProvider::create) + ) = CommonizerParameters( + resultsConsumer, manifestDataProvider, dependencyModulesProvider = dependencyModules[sharedTarget]?.let(MockModulesProvider::create) + ).also { parameters -> leafTargets.forEach { leafTarget -> val originalModule = originalModules.getValue(leafTarget)