From b0ff3e7e5ea589db3c61279ee5661012e66ed3e0 Mon Sep 17 00:00:00 2001 From: Dmitriy Dolovov Date: Thu, 26 Nov 2020 17:19:03 +0300 Subject: [PATCH] [Commonizer] More fine-grained control of commonized module dependencies - Reduce usage of 'isUnderStandardKotlinPackages' check in commonizer source code - Rely on common module dependencies supplied via commonizer Parameters which not only Kotlin standard library but may also include common fragments of other libraries --- .../descriptors/commonizer/Parameters.kt | 9 +- .../descriptors/commonizer/TargetProvider.kt | 5 +- .../descriptors/commonizer/builder/context.kt | 161 +++++--- .../konan/NativeDistributionCommonizer.kt | 58 +-- .../NativeDistributionModulesProvider.kt | 53 +-- .../konan/NativeDistributionStdlibProvider.kt | 46 +++ ...istributionLibrary.kt => NativeLibrary.kt} | 32 +- .../commonizer/mergedtree/CirTreeMerger.kt | 45 ++- .../commonizer/mergedtree/collectors.kt | 6 +- .../descriptors/commonizer/utils/fqName.kt | 12 +- .../commonizer/utils/moduleDescriptor.kt | 90 +---- .../linux/package_kotlinx_cinterop.kt | 12 - .../macos/package_kotlinx_cinterop.kt | 12 - .../common/package_kotlinx_cinterop.kt | 0 .../linux/package_kotlinx_cinterop.kt | 12 - .../macos/package_kotlinx_cinterop.kt | 12 - .../macos/package_kotlinx_cinterop.kt | 6 - .../common}/package_kotlinx_cinterop.kt | 0 .../original/ios/package_kotlinx_cinterop.kt | 6 - .../macos/package_kotlinx_cinterop.kt | 6 - .../AbstractCommonizationFromSourcesTest.kt | 358 ++++++++++-------- .../commonizer/CommonizerFacadeTest.kt | 3 +- .../descriptors/commonizer/utils/mocks.kt | 5 +- 23 files changed, 479 insertions(+), 470 deletions(-) create mode 100644 native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionStdlibProvider.kt rename native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/{NativeDistributionLibrary.kt => NativeLibrary.kt} (69%) delete mode 100644 native/commonizer/testData/classifierCommonization/typeAliases/commonized/linux/package_kotlinx_cinterop.kt delete mode 100644 native/commonizer/testData/classifierCommonization/typeAliases/commonized/macos/package_kotlinx_cinterop.kt rename native/commonizer/testData/classifierCommonization/typeAliases/{commonized => dependee}/common/package_kotlinx_cinterop.kt (100%) delete mode 100644 native/commonizer/testData/classifierCommonization/typeAliases/original/linux/package_kotlinx_cinterop.kt delete mode 100644 native/commonizer/testData/classifierCommonization/typeAliases/original/macos/package_kotlinx_cinterop.kt delete mode 100644 native/commonizer/testData/functionCommonization/valueParameters/commonized/macos/package_kotlinx_cinterop.kt rename native/commonizer/testData/functionCommonization/valueParameters/{commonized/ios => dependee/common}/package_kotlinx_cinterop.kt (100%) delete mode 100644 native/commonizer/testData/functionCommonization/valueParameters/original/ios/package_kotlinx_cinterop.kt delete mode 100644 native/commonizer/testData/functionCommonization/valueParameters/original/macos/package_kotlinx_cinterop.kt diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/Parameters.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/Parameters.kt index 6104607ebc1..e86707b0ff3 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/Parameters.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/Parameters.kt @@ -16,16 +16,13 @@ class Parameters( val targetProviders: List get() = _targetProviders.values.toList() - // only for test purposes - internal var extendedLookupForBuiltInsClassifiers: Boolean = false + // common module dependencies (ex: Kotlin stdlib) + var dependeeModulesProvider: ModulesProvider? = null set(value) { - check(!field || value) + check(field == null) field = value } - // only for test purposes - internal var commonModulesProvider: ModulesProvider? = null - fun addTarget(targetProvider: TargetProvider): Parameters { require(targetProvider.target !in _targetProviders) { "Target ${targetProvider.target} is already added" } _targetProviders[targetProvider.target] = targetProvider diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/TargetProvider.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/TargetProvider.kt index b3abce353fd..d2969f68ac0 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/TargetProvider.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/TargetProvider.kt @@ -14,7 +14,8 @@ class TargetProvider( val target: InputTarget, val builtInsClass: Class, val builtInsProvider: BuiltInsProvider, - val modulesProvider: ModulesProvider + val modulesProvider: ModulesProvider, + val dependeeModulesProvider: ModulesProvider? ) interface BuiltInsProvider { @@ -40,5 +41,5 @@ interface ModulesProvider { ) fun loadModuleInfos(): Map - fun loadModules(): Map + fun loadModules(dependencies: Collection): Map } diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/context.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/context.kt index 0cc87156e5e..b1ce4f240a6 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/context.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/context.kt @@ -7,6 +7,7 @@ package org.jetbrains.kotlin.descriptors.commonizer.builder import gnu.trove.THashMap import org.jetbrains.kotlin.builtins.KotlinBuiltIns +import org.jetbrains.kotlin.builtins.StandardNames import org.jetbrains.kotlin.descriptors.* import org.jetbrains.kotlin.descriptors.commonizer.Parameters import org.jetbrains.kotlin.descriptors.commonizer.Target @@ -133,68 +134,84 @@ class TargetDeclarationsBuilderComponents( val storageManager: StorageManager, val target: Target, val builtIns: KotlinBuiltIns, - val lazyModulesLookupTable: NotNullLazyValue>, - val isCommon: Boolean, + val lazyClassifierLookupTable: NotNullLazyValue, val index: Int, private val cache: DeclarationsBuilderCache ) { - // only for test purposes - internal var extendedLookupForBuiltInsClassifiers: Boolean = false - // N.B. this function may create new classifiers for types from Kotlin/Native forward declarations packages - fun findClassOrTypeAlias(classId: ClassId): ClassifierDescriptorWithTypeParameters { - return when { - classId.packageFqName.isUnderStandardKotlinPackages -> { - // look up for classifier in built-ins module: - val builtInsModule = builtIns.builtInsModule - - // TODO: this works fine for Native as far as built-ins module contains full Native stdlib, but this is not enough for JVM and JS - val classifier = builtInsModule.resolveClassOrTypeAlias(classId) - if (classifier != null) - return classifier - - if (extendedLookupForBuiltInsClassifiers) { - return findOriginalClassOrTypeAlias(classId) - ?: error("Classifier ${classId.asString()} not found neither in built-ins module $builtInsModule nor in original modules for $target") - } - - error("Classifier ${classId.asString()} not found in built-ins module $builtInsModule for $target") - } - classId.packageFqName.isUnderKotlinNativeSyntheticPackages -> { - // that's a synthetic Kotlin/Native classifier that was exported as forward declaration in one or more modules, - // but did not match any existing class or typealias - cache.getOrPutForwardDeclarationsModule(index) { - // N.B. forward declarations module is created only on demand - createKotlinNativeForwardDeclarationsModule( - storageManager = storageManager, - builtIns = builtIns - ) - }.resolveClassOrTypeAlias(classId) - ?: error("Classifier ${classId.asString()} not found for $target") - } - else -> { - cache.getCachedClassifier(classId, index) // first, look up in created descriptors cache - ?: findOriginalClassOrTypeAlias(classId) // then, attempt to load the original classifier - ?: error("Classifier ${classId.asString()} not found for $target") - } + fun findClassOrTypeAlias(classifierId: ClassId): ClassifierDescriptorWithTypeParameters { + return if (classifierId.packageFqName.isUnderKotlinNativeSyntheticPackages) { + // that's a synthetic Kotlin/Native classifier that was exported as forward declaration in one or more modules, + // but did not match any existing class or typealias + cache.getOrPutForwardDeclarationsModule(index) { + // N.B. forward declarations module is created only on demand + createKotlinNativeForwardDeclarationsModule( + storageManager = storageManager, + builtIns = builtIns + ) + }.resolveClassOrTypeAlias(classifierId) + ?: error("Classifier ${classifierId.asString()} not found for $target") + } else { + cache.getCachedClassifier(classifierId, index) // first, look up in created descriptors cache + ?: lazyClassifierLookupTable().resolveClassOrTypeAlias(classifierId) // then, attempt to load the original classifier + ?: error("Classifier ${classifierId.asString()} not found for $target") } } +} - private fun findOriginalClassOrTypeAlias(classId: ClassId): ClassifierDescriptorWithTypeParameters? { - if (classId.packageFqName.isRoot) - return null +class LazyClassifierLookupTable(lazyModules: Map>) { + private val table = THashMap>() + private val allModules: Collection - // first, guess containing module and look up in it - val classifier = lazyModulesLookupTable() - .guessModuleByPackageFqName(classId.packageFqName) - ?.resolveClassOrTypeAlias(classId) + init { + // add "module:" prefix for each key representing a module name, not a package name + lazyModules.forEach { (moduleName, modules) -> table[MODULE_NAME_PREFIX + moduleName.toLowerCase()] = modules } + allModules = lazyModules.values.flatten() + } - // if failed, then look up though all modules - return classifier - ?: lazyModulesLookupTable().values - .asSequence() - .mapNotNull { it?.resolveClassOrTypeAlias(classId) } - .firstOrNull() + fun resolveClassOrTypeAlias(classifierId: ClassId): ClassifierDescriptorWithTypeParameters? { + if (table.isEmpty) return null + + val packageFqName = classifierId.packageFqName + if (packageFqName.isRoot) return null + + val packageFqNameRaw = packageFqName.asString() + table[packageFqNameRaw]?.let { modules -> + for (module in modules) + return module.resolveClassOrTypeAlias(classifierId) ?: continue + } + + val packageFqNameFragments = packageFqNameRaw.split('.') + val moduleNameForLookup = when (packageFqNameFragments[0]) { + "kotlin" -> "kotlin" + "platform" -> if (packageFqNameFragments.size == 2) packageFqNameFragments[1].toLowerCase() else null + else -> null + } + + // try to find the classifier by guessing its container module + if (moduleNameForLookup != null) { + table[MODULE_NAME_PREFIX + moduleNameForLookup]?.let { modules -> + for (module in modules) { + val classifier = module.resolveClassOrTypeAlias(classifierId) ?: continue + table[packageFqNameRaw] = modules // cache to speed-up the further look-ups + return classifier + } + } + } + + // last resort: brute force + for (module in allModules) { + val classifier = module.resolveClassOrTypeAlias(classifierId) ?: continue + table[packageFqNameRaw] = listOf(module) // cache to speed-up the further look-ups + return classifier + } + + table[packageFqNameRaw] = null // cache to speed-up the further look-ups + return null + } + + companion object { + private const val MODULE_NAME_PREFIX = "module:" } } @@ -204,6 +221,10 @@ fun CirRootNode.createGlobalBuilderComponents( ): GlobalDeclarationsBuilderComponents { val cache = DeclarationsBuilderCache(dimension) + val lazyCommonDependeeModules = storageManager.createLazyValue { + parameters.dependeeModulesProvider?.loadModules(emptyList()).orEmpty() + } + val targetContexts = (0 until dimension).map { index -> val isCommon = index == indexOfCommon @@ -216,24 +237,38 @@ fun CirRootNode.createGlobalBuilderComponents( } val lazyModulesLookupTable = storageManager.createLazyValue { - val source = if (isCommon) - parameters.commonModulesProvider?.loadModules() ?: emptyMap() - else - parameters.targetProviders[index].modulesProvider.loadModules() - THashMap(source) + val result = mutableMapOf>() + + val commonDependeeModules: Map = lazyCommonDependeeModules() + + if (!isCommon) { + with(parameters.targetProviders[index]) { + val targetDependeeModules: Map = + dependeeModulesProvider?.loadModules(commonDependeeModules.values).orEmpty() + + val targetModules: Map = + modulesProvider.loadModules(targetDependeeModules.values + commonDependeeModules.values) + + targetModules.forEach { (moduleName, module) -> result.getOrPut(moduleName) { mutableListOf() } += module } + targetDependeeModules.forEach { (moduleName, module) -> result.getOrPut(moduleName) { mutableListOf() } += module } + } + } + + commonDependeeModules.forEach { (moduleName, module) -> result.getOrPut(moduleName) { mutableListOf() } += module } + + result.getOrPut(StandardNames.BUILT_INS_PACKAGE_FQ_NAME.asString()) { mutableListOf() } += builtIns.builtInsModule + + LazyClassifierLookupTable(result) } TargetDeclarationsBuilderComponents( storageManager = storageManager, target = root.target, builtIns = builtIns, - lazyModulesLookupTable = lazyModulesLookupTable, - isCommon = isCommon, + lazyClassifierLookupTable = lazyModulesLookupTable, index = index, cache = cache - ).also { - it.extendedLookupForBuiltInsClassifiers = parameters.extendedLookupForBuiltInsClassifiers - } + ) } return GlobalDeclarationsBuilderComponents(storageManager, targetContexts, cache, parameters.statsCollector) diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt index 7255c59f086..992eb4e6d4a 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt @@ -55,13 +55,13 @@ class NativeDistributionCommonizer( clockMark.reset() // 1. load libraries - val librariesByTargets = loadLibraries() + val allLibraries = loadLibraries() // 2. run commonization - val result = commonize(librariesByTargets) + val result = commonize(allLibraries) // 3. write new libraries - saveModules(librariesByTargets, result) + saveModules(allLibraries, result) logTotal() } @@ -86,29 +86,29 @@ class NativeDistributionCommonizer( private fun logTotal() = logger.log("TOTAL: ${clockMark.elapsedSinceStart()}") - private fun loadLibraries(): Map { + private fun loadLibraries(): AllNativeLibraries { val stdlibPath = repository.resolve(konanCommonLibraryPath(KONAN_STDLIB_NAME)) - val stdlib = loadLibrary(stdlibPath) + val stdlib = NativeLibrary(loadLibrary(stdlibPath)) - val result = targets.associate { target -> + val librariesByTargets = targets.associate { target -> val leafTarget = InputTarget(target.name, target) val platformLibs = leafTarget.platformLibrariesSource .takeIf { it.isDirectory } ?.listFiles() ?.takeIf { it.isNotEmpty() } - ?.map { loadLibrary(it) } + ?.map { NativeLibrary(loadLibrary(it)) } .orEmpty() if (platformLibs.isEmpty()) logger.warning("No platform libraries found for target $target. This target will be excluded from commonization.") - leafTarget to NativeDistributionLibraries(stdlib, platformLibs) + leafTarget to NativeLibrariesToCommonize(platformLibs) } logProgress("Read lazy (uninitialized) libraries") - return result + return AllNativeLibraries(stdlib, librariesByTargets) } private fun loadLibrary(location: File): KotlinLibrary { @@ -136,7 +136,7 @@ class NativeDistributionCommonizer( return library } - private fun commonize(librariesByTargets: Map): Result { + private fun commonize(allLibraries: AllNativeLibraries): Result { val statsCollector = when (statsType) { RAW -> RawStatsCollector(targets, FileStatsOutput(destination, "raw")) AGGREGATED -> AggregatedStatsCollector(targets, FileStatsOutput(destination, "aggregated")) @@ -144,20 +144,23 @@ class NativeDistributionCommonizer( } statsCollector.use { val parameters = Parameters(statsCollector, ::logProgress).apply { - librariesByTargets.forEach { (target, libraries) -> - if (libraries.platformLibs.isEmpty()) return@forEach + val storageManager = LockBasedStorageManager("Commonized modules") - val provider = NativeDistributionModulesProvider( - storageManager = LockBasedStorageManager("Target $target"), - libraries = libraries - ) + val stdlibProvider = NativeDistributionStdlibProvider(storageManager, allLibraries.stdlib) + dependeeModulesProvider = stdlibProvider + + allLibraries.librariesByTargets.forEach { (target, librariesToCommonize) -> + if (librariesToCommonize.libraries.isEmpty()) return@forEach + + val modulesProvider = NativeDistributionModulesProvider(storageManager, librariesToCommonize) addTarget( TargetProvider( target = target, builtInsClass = KonanBuiltIns::class.java, - builtInsProvider = provider, - modulesProvider = provider + builtInsProvider = stdlibProvider, + modulesProvider = modulesProvider, + dependeeModulesProvider = null // stdlib is already set as common dependency ) ) } @@ -167,10 +170,7 @@ class NativeDistributionCommonizer( } } - private fun saveModules( - originalLibrariesByTargets: Map, - result: Result - ) { + private fun saveModules(originalLibraries: AllNativeLibraries, result: Result) { // optimization: stdlib and endorsed libraries effectively remain the same across all Kotlin/Native targets, // so they can be just copied to the new destination without running serializer copyCommonStandardLibraries() @@ -180,8 +180,8 @@ class NativeDistributionCommonizer( // It may happen that all targets to be commonized (or at least all but one target) miss platform libraries. // In such case commonizer will do nothing and return a special result value 'NothingToCommonize'. // So, let's just copy platform libraries from the target where they are to the new destination. - originalLibrariesByTargets.forEach { (target, libraries) -> - copyTargetAsIs(target, libraries.platformLibs.size) + originalLibraries.librariesByTargets.forEach { (target, librariesToCommonize) -> + copyTargetAsIs(target, librariesToCommonize.libraries.size) } } @@ -194,11 +194,11 @@ class NativeDistributionCommonizer( ) // 'targetsToCopy' are some targets with empty set of platform libraries - val targetsToCopy = originalLibrariesByTargets.keys - result.leafTargets + val targetsToCopy = originalLibraries.librariesByTargets.keys - result.leafTargets if (targetsToCopy.isNotEmpty()) { targetsToCopy.forEach { target -> - val libraries = originalLibrariesByTargets.getValue(target) - copyTargetAsIs(target, libraries.platformLibs.size) + val librariesToCommonize = originalLibraries.librariesByTargets.getValue(target) + copyTargetAsIs(target, librariesToCommonize.libraries.size) } } @@ -214,11 +214,11 @@ class NativeDistributionCommonizer( val starredTarget: String? when (target) { is InputTarget -> { - manifestProvider = originalLibrariesByTargets.getValue(target) + manifestProvider = originalLibraries.librariesByTargets.getValue(target) starredTarget = target.name } is OutputTarget -> { - manifestProvider = CommonNativeManifestDataProvider(originalLibrariesByTargets.values) + manifestProvider = CommonNativeManifestDataProvider(originalLibraries.librariesByTargets.values) starredTarget = null } } diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionModulesProvider.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionModulesProvider.kt index b2929503f72..f05e72f196e 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionModulesProvider.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionModulesProvider.kt @@ -5,38 +5,26 @@ package org.jetbrains.kotlin.descriptors.commonizer.konan -import org.jetbrains.kotlin.builtins.KotlinBuiltIns import org.jetbrains.kotlin.config.LanguageVersionSettingsImpl import org.jetbrains.kotlin.descriptors.ModuleDescriptor -import org.jetbrains.kotlin.descriptors.commonizer.BuiltInsProvider import org.jetbrains.kotlin.descriptors.commonizer.ModulesProvider import org.jetbrains.kotlin.descriptors.commonizer.ModulesProvider.CInteropModuleAttributes import org.jetbrains.kotlin.descriptors.commonizer.ModulesProvider.ModuleInfo import org.jetbrains.kotlin.descriptors.commonizer.utils.NativeFactories import org.jetbrains.kotlin.descriptors.commonizer.utils.createKotlinNativeForwardDeclarationsModule +import org.jetbrains.kotlin.descriptors.commonizer.utils.strip +import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl import org.jetbrains.kotlin.incremental.components.LookupTracker -import org.jetbrains.kotlin.konan.library.KONAN_STDLIB_NAME import org.jetbrains.kotlin.storage.StorageManager +import org.jetbrains.kotlin.utils.addIfNotNull import java.io.File internal class NativeDistributionModulesProvider( private val storageManager: StorageManager, - private val libraries: NativeDistributionLibraries -) : BuiltInsProvider, ModulesProvider { - override fun loadBuiltIns(): KotlinBuiltIns { - val stdlib = NativeFactories.DefaultDeserializedDescriptorFactory.createDescriptorAndNewBuiltIns( - library = libraries.stdlib.library, - languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT, - storageManager = storageManager, - packageAccessHandler = null - ) - stdlib.setDependencies(listOf(stdlib)) - - return stdlib.builtIns - } - + private val librariesToCommonize: NativeLibrariesToCommonize +) : ModulesProvider { override fun loadModuleInfos(): Map { - return libraries.platformLibs.associate { library -> + return librariesToCommonize.libraries.associate { library -> val manifestData = library.manifestData val name = manifestData.uniqueName @@ -54,11 +42,18 @@ internal class NativeDistributionModulesProvider( } } - override fun loadModules(): Map { - val builtIns = loadBuiltIns() - val stdlib = builtIns.builtInsModule + override fun loadModules(dependencies: Collection): Map { + check(dependencies.isNotEmpty()) { "At least Kotlin/Native stdlib should be provided" } - val platformModulesMap = libraries.platformLibs.associate { library -> + val dependenciesMap = mutableMapOf>() + dependencies.forEach { dependency -> + val name = dependency.name.strip() + dependenciesMap.getOrPut(name) { mutableListOf() } += dependency as ModuleDescriptorImpl + } + + val builtIns = dependencies.first().builtIns + + val platformModulesMap = librariesToCommonize.libraries.associate { library -> val name = library.manifestData.uniqueName val module = NativeFactories.DefaultDeserializedDescriptorFactory.createDescriptorOptionalBuiltIns( library = library.library, @@ -78,11 +73,17 @@ internal class NativeDistributionModulesProvider( ) platformModulesMap.forEach { (name, module) -> - val dependencies = libraries.getManifest(name) - .dependencies - .map { if (it == KONAN_STDLIB_NAME) stdlib else platformModulesMap.getValue(it) } + val moduleDependencies = mutableListOf() + moduleDependencies += module - module.setDependencies(listOf(module) + dependencies + forwardDeclarations) + librariesToCommonize.getManifest(name).dependencies.forEach { + moduleDependencies.addIfNotNull(platformModulesMap[it]) + moduleDependencies += dependenciesMap[it].orEmpty() + } + + moduleDependencies += forwardDeclarations + + module.setDependencies(moduleDependencies) } return platformModulesMap diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionStdlibProvider.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionStdlibProvider.kt new file mode 100644 index 00000000000..02b2b469258 --- /dev/null +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionStdlibProvider.kt @@ -0,0 +1,46 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.descriptors.commonizer.konan + +import org.jetbrains.kotlin.builtins.KotlinBuiltIns +import org.jetbrains.kotlin.config.LanguageVersionSettingsImpl +import org.jetbrains.kotlin.descriptors.ModuleDescriptor +import org.jetbrains.kotlin.descriptors.commonizer.BuiltInsProvider +import org.jetbrains.kotlin.descriptors.commonizer.ModulesProvider +import org.jetbrains.kotlin.descriptors.commonizer.ModulesProvider.ModuleInfo +import org.jetbrains.kotlin.descriptors.commonizer.utils.NativeFactories +import org.jetbrains.kotlin.konan.library.KONAN_STDLIB_NAME +import org.jetbrains.kotlin.storage.StorageManager +import java.io.File + +internal class NativeDistributionStdlibProvider( + private val storageManager: StorageManager, + private val stdlib: NativeLibrary +) : BuiltInsProvider, ModulesProvider { + private val moduleInfo = ModuleInfo( + name = KONAN_STDLIB_NAME, + originalLocation = File(stdlib.library.libraryFile.absolutePath), + cInteropAttributes = null + ) + + override fun loadBuiltIns(): KotlinBuiltIns = loadStdlibModule().builtIns + override fun loadModuleInfos(): Map = mapOf(KONAN_STDLIB_NAME to moduleInfo) + + override fun loadModules(dependencies: Collection): Map { + check(dependencies.isEmpty()) + return mapOf(KONAN_STDLIB_NAME to loadStdlibModule()) + } + + private fun loadStdlibModule() = + NativeFactories.DefaultDeserializedDescriptorFactory.createDescriptorAndNewBuiltIns( + library = stdlib.library, + languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT, + storageManager = storageManager, + packageAccessHandler = null + ).apply { + setDependencies(listOf(this)) + } +} \ No newline at end of file diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionLibrary.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeLibrary.kt similarity index 69% rename from native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionLibrary.kt rename to native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeLibrary.kt index a0ce30ddb91..e9587a195d4 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionLibrary.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeLibrary.kt @@ -6,6 +6,7 @@ package org.jetbrains.kotlin.descriptors.commonizer.konan import gnu.trove.THashMap +import org.jetbrains.kotlin.descriptors.commonizer.InputTarget import org.jetbrains.kotlin.library.KotlinLibrary internal interface NativeManifestDataProvider { @@ -13,33 +14,34 @@ internal interface NativeManifestDataProvider { } /** - * A separate Kotlin/Native library from the distribution. + * A separate Kotlin/Native library. */ -internal class NativeDistributionLibrary( +internal class NativeLibrary( val library: KotlinLibrary ) { val manifestData = NativeSensitiveManifestData.readFrom(library) } /** - * A collection of Kotlin/Native libraries for a certain Native target + stdlib from the distribution. + * A collection of Kotlin/Native libraries for a certain Native target. */ -internal class NativeDistributionLibraries( - val stdlib: NativeDistributionLibrary, - val platformLibs: List -) : NativeManifestDataProvider { - constructor(stdlib: KotlinLibrary, platformLibs: List) : this( - NativeDistributionLibrary(stdlib), - platformLibs.map(::NativeDistributionLibrary) - ) - +internal class NativeLibrariesToCommonize(val libraries: List) : NativeManifestDataProvider { private val manifestIndex: Map = buildManifestIndex() override fun getManifest(libraryName: String) = manifestIndex.getValue(libraryName) + + companion object { + fun create(libraries: List) = NativeLibrariesToCommonize(libraries.map(::NativeLibrary)) + } } +internal class AllNativeLibraries( + val stdlib: NativeLibrary, + val librariesByTargets: Map +) + internal class CommonNativeManifestDataProvider( - libraryGroups: Collection + libraryGroups: Collection ) : NativeManifestDataProvider { private val manifestIndex: Map @@ -66,5 +68,5 @@ internal class CommonNativeManifestDataProvider( override fun getManifest(libraryName: String) = manifestIndex.getValue(libraryName) } -private fun NativeDistributionLibraries.buildManifestIndex(): MutableMap = - (platformLibs + stdlib).map { it.manifestData }.associateByTo(THashMap()) { it.uniqueName } +private fun NativeLibrariesToCommonize.buildManifestIndex(): MutableMap = + libraries.map { it.manifestData }.associateByTo(THashMap()) { it.uniqueName } diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/mergedtree/CirTreeMerger.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/mergedtree/CirTreeMerger.kt index 2164fd37540..4ddb14e5bde 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/mergedtree/CirTreeMerger.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/mergedtree/CirTreeMerger.kt @@ -59,14 +59,26 @@ class CirTreeMerger( private val size = parameters.targetProviders.size fun merge(): CirTreeMergeResult { + val result = processRoot() + System.gc() + return result + } + + private fun processRoot(): CirTreeMergeResult { val rootNode: CirRootNode = buildRootNode(storageManager, size) + // remember any exported forward declarations from common fragments of dependee modules + parameters.dependeeModulesProvider?.loadModuleInfos()?.values?.forEach(::processCInteropModuleAttributes) + + // load common dependencies + val dependeeModules = parameters.dependeeModulesProvider?.loadModules(emptyList())?.values.orEmpty() + val allModuleInfos: List> = parameters.targetProviders.map { it.modulesProvider.loadModuleInfos() } val commonModuleNames = allModuleInfos.map { it.keys }.reduce { a, b -> a intersect b } parameters.targetProviders.forEachIndexed { targetIndex, targetProvider -> val commonModuleInfos = allModuleInfos[targetIndex].filterKeys { it in commonModuleNames } - processTarget(rootNode, targetIndex, targetProvider, commonModuleInfos) + processTarget(rootNode, targetIndex, targetProvider, commonModuleInfos, dependeeModules) parameters.progressLogger?.invoke("Loaded declarations for [${targetProvider.target.name}]") System.gc() } @@ -87,7 +99,8 @@ class CirTreeMerger( rootNode: CirRootNode, targetIndex: Int, targetProvider: TargetProvider, - commonModuleInfos: Map + commonModuleInfos: Map, + dependeeModules: Collection ) { rootNode.targetDeclarations[targetIndex] = CirRootFactory.create( targetProvider.target, @@ -95,7 +108,10 @@ class CirTreeMerger( targetProvider.builtInsProvider ) - val moduleDescriptors: Map = targetProvider.modulesProvider.loadModules() + val targetDependeeModules = targetProvider.dependeeModulesProvider?.loadModules(dependeeModules)?.values.orEmpty() + val allDependeeModules = targetDependeeModules + dependeeModules + + val moduleDescriptors: Map = targetProvider.modulesProvider.loadModules(allDependeeModules) val modules: MutableMap = rootNode.modules moduleDescriptors.forEach { (name, moduleDescriptor) -> @@ -110,16 +126,7 @@ class CirTreeMerger( moduleInfo: ModuleInfo, moduleDescriptor: ModuleDescriptor ) { - moduleInfo.cInteropAttributes?.let { cInteropAttributes -> - val exportForwardDeclarations = cInteropAttributes.exportForwardDeclarations.takeIf { it.isNotEmpty() } ?: return@let - val mainPackageFqName = FqName(cInteropAttributes.mainPackageFqName).intern() - - exportForwardDeclarations.forEach { classFqName -> - // Class has synthetic package FQ name (cnames/objcnames). Need to transfer it to the main package. - val className = Name.identifier(classFqName.substringAfterLast('.')).intern() - cache.addExportedForwardDeclaration(internedClassId(mainPackageFqName, className)) - } - } + processCInteropModuleAttributes(moduleInfo) val moduleName: Name = moduleDescriptor.name.intern() val moduleNode: CirModuleNode = modules.getOrPut(moduleName) { @@ -258,4 +265,16 @@ class CirTreeMerger( } typeAliasNode.targetDeclarations[targetIndex] = CirTypeAliasFactory.create(typeAliasDescriptor) } + + private fun processCInteropModuleAttributes(moduleInfo: ModuleInfo) { + val cInteropAttributes = moduleInfo.cInteropAttributes ?: return + val exportForwardDeclarations = cInteropAttributes.exportForwardDeclarations.takeIf { it.isNotEmpty() } ?: return + val mainPackageFqName = FqName(cInteropAttributes.mainPackageFqName).intern() + + exportForwardDeclarations.forEach { classFqName -> + // Class has synthetic package FQ name (cnames/objcnames). Need to transfer it to the main package. + val className = Name.identifier(classFqName.substringAfterLast('.')).intern() + cache.addExportedForwardDeclaration(internedClassId(mainPackageFqName, className)) + } + } } diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/mergedtree/collectors.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/mergedtree/collectors.kt index ea544306b47..1d1916a2382 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/mergedtree/collectors.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/mergedtree/collectors.kt @@ -5,6 +5,7 @@ package org.jetbrains.kotlin.descriptors.commonizer.mergedtree +import org.jetbrains.kotlin.backend.common.serialization.metadata.impl.ClassifierAliasingPackageFragmentDescriptor import org.jetbrains.kotlin.backend.common.serialization.metadata.impl.ExportedForwardDeclarationsPackageFragmentDescriptor import org.jetbrains.kotlin.descriptors.* import org.jetbrains.kotlin.descriptors.commonizer.utils.* @@ -69,12 +70,9 @@ internal fun ModuleDescriptor.collectNonEmptyPackageMemberScopes(collector: (FqN val packageFragmentProvider = this.packageFragmentProvider fun recurse(packageFqName: FqName) { - if (packageFqName.isUnderStandardKotlinPackages || packageFqName.isUnderKotlinNativeSyntheticPackages) - return - val ownPackageFragments = packageFragmentProvider.packageFragments(packageFqName) val ownPackageMemberScopes = ownPackageFragments.asSequence() - .filter { it !is ExportedForwardDeclarationsPackageFragmentDescriptor } + .filter { it !is ExportedForwardDeclarationsPackageFragmentDescriptor && it !is ClassifierAliasingPackageFragmentDescriptor } .map { it.getMemberScope() } .filter { it != MemberScope.Empty } .toList(ownPackageFragments.size) diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/utils/fqName.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/utils/fqName.kt index 0bb6100c64a..05b3c364828 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/utils/fqName.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/utils/fqName.kt @@ -10,18 +10,17 @@ import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor import org.jetbrains.kotlin.descriptors.annotations.AnnotationDescriptor import org.jetbrains.kotlin.name.ClassId import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.name.Name import org.jetbrains.kotlin.serialization.konan.impl.ForwardDeclarationsFqNames internal val DEPRECATED_ANNOTATION_FQN: FqName = FqName(Deprecated::class.java.name).intern() internal val DEPRECATED_ANNOTATION_CID: ClassId = internedClassId(DEPRECATED_ANNOTATION_FQN) -internal val STANDARD_KOTLIN_PACKAGE_FQNS: List = listOf( - StandardNames.BUILT_INS_PACKAGE_FQ_NAME.intern(), - FqName("kotlinx").intern() +private val STANDARD_KOTLIN_PACKAGES = listOf( + StandardNames.BUILT_INS_PACKAGE_FQ_NAME.asString(), + "kotlinx" ) -private val STANDARD_KOTLIN_PACKAGES = STANDARD_KOTLIN_PACKAGE_FQNS.map { it.asString() } - private val KOTLIN_NATIVE_SYNTHETIC_PACKAGES = ForwardDeclarationsFqNames.syntheticPackages .map { fqName -> check(!fqName.isRoot) @@ -37,6 +36,9 @@ private val OBJC_INTEROP_CALLABLE_ANNOTATIONS = listOf( "ObjCFactory" ) +internal fun Name.strip(): String = + asString().removeSurrounding("<", ">") + internal val FqName.isUnderStandardKotlinPackages: Boolean get() = hasAnyPrefix(STANDARD_KOTLIN_PACKAGES) diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/utils/moduleDescriptor.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/utils/moduleDescriptor.kt index aea590684ca..2e1e7715b17 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/utils/moduleDescriptor.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/utils/moduleDescriptor.kt @@ -5,25 +5,20 @@ package org.jetbrains.kotlin.descriptors.commonizer.utils -import org.jetbrains.kotlin.backend.common.serialization.metadata.impl.ExportedForwardDeclarationsPackageFragmentDescriptor import org.jetbrains.kotlin.builtins.KotlinBuiltIns import org.jetbrains.kotlin.builtins.konan.KonanBuiltIns -import org.jetbrains.kotlin.descriptors.ClassDescriptor -import org.jetbrains.kotlin.descriptors.ClassifierDescriptorWithTypeParameters -import org.jetbrains.kotlin.descriptors.ModuleDescriptor +import org.jetbrains.kotlin.descriptors.* import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl -import org.jetbrains.kotlin.descriptors.packageFragments import org.jetbrains.kotlin.incremental.components.NoLookupLocation import org.jetbrains.kotlin.konan.util.KlibMetadataFactories import org.jetbrains.kotlin.library.metadata.NativeTypeTransformer import org.jetbrains.kotlin.library.metadata.NullFlexibleTypeDeserializer import org.jetbrains.kotlin.name.ClassId import org.jetbrains.kotlin.name.FqName -import org.jetbrains.kotlin.resolve.scopes.MemberScope import org.jetbrains.kotlin.serialization.konan.impl.KlibResolvedModuleDescriptorsFactoryImpl import org.jetbrains.kotlin.storage.StorageManager -internal val ModuleDescriptor.packageFragmentProvider +internal val ModuleDescriptor.packageFragmentProvider: PackageFragmentProvider get() = (this as ModuleDescriptorImpl).packageFragmentProviderForModuleContentWithoutDependencies internal fun createKotlinNativeForwardDeclarationsModule( @@ -42,73 +37,30 @@ internal fun ModuleDescriptor.resolveClassOrTypeAlias(classId: ClassId): Classif if (relativeClassName.isRoot) return null - var memberScope: MemberScope = getPackage(classId.packageFqName).memberScope + return packageFragmentProvider.packageFragments(classId.packageFqName).asSequence().mapNotNull { packageFragment -> + var memberScope = packageFragment.getMemberScope() - val classifierName = if ('.' in relativeClassName.asString()) { - // resolve member scope of the nested class - relativeClassName.pathSegments().reduce { first, second -> - memberScope = (memberScope.getContributedClassifier( - first, - NoLookupLocation.FOR_ALREADY_TRACKED - ) as? ClassDescriptor)?.unsubstitutedMemberScope ?: return null + val classifierName = if ('.' in relativeClassName.asString()) { + // resolve member scope of the nested class + relativeClassName.pathSegments().reduce { first, second -> + memberScope = (memberScope.getContributedClassifier( + first, + NoLookupLocation.FOR_ALREADY_TRACKED + ) as? ClassDescriptor)?.unsubstitutedMemberScope ?: return@mapNotNull null - second + second + } + } else { + relativeClassName.shortName() } - } else { - relativeClassName.shortName() - } - return memberScope.getContributedClassifier( - classifierName, - NoLookupLocation.FOR_ALREADY_TRACKED - ) as? ClassifierDescriptorWithTypeParameters + memberScope.getContributedClassifier( + classifierName, + NoLookupLocation.FOR_ALREADY_TRACKED + ) as? ClassifierDescriptorWithTypeParameters + }.firstOrNull() } -internal fun MutableMap.guessModuleByPackageFqName(packageFqName: FqName): ModuleDescriptor? { - if (isEmpty()) return null - - val packageFqNameRaw = packageFqName.asString() - if (containsKey(packageFqNameRaw)) { - return this[packageFqNameRaw] // might return null if this is a previously cached result - } - - fun guessByEnding(): ModuleDescriptor? { - return entries - .firstOrNull { (name, _) -> name.endsWith(packageFqNameRaw, ignoreCase = true) } - ?.value - } - - fun guessBySmartEnding(): ModuleDescriptor? { - val packageFqNameFragments = packageFqNameRaw.split('.') - if (packageFqNameFragments.size < 2) return null - - return entries.firstOrNull { (name, _) -> - var startIndex = 0 - for (fragment in packageFqNameFragments) { - val index = name.indexOf(fragment, startIndex = startIndex, ignoreCase = true) - if (index < startIndex) - return@firstOrNull false - else - startIndex = index + fragment.length - } - true - }?.value - } - - val candidate = guessByEnding() ?: guessBySmartEnding() - this[packageFqNameRaw] = candidate // cache to speed-up the further look-ups - return candidate -} - -internal val ModuleDescriptor.hasSomethingUnderStandardKotlinPackages: Boolean - get() { - val packageFragmentProvider = packageFragmentProvider - return STANDARD_KOTLIN_PACKAGE_FQNS.any { fqName -> - packageFragmentProvider.packageFragments(fqName).any { packageFragment -> - packageFragment !is ExportedForwardDeclarationsPackageFragmentDescriptor - && packageFragment.getMemberScope() != MemberScope.Empty - } - } - } +internal const val MODULE_NAME_PREFIX = "module:" internal val NativeFactories = KlibMetadataFactories(::KonanBuiltIns, NullFlexibleTypeDeserializer, NativeTypeTransformer()) diff --git a/native/commonizer/testData/classifierCommonization/typeAliases/commonized/linux/package_kotlinx_cinterop.kt b/native/commonizer/testData/classifierCommonization/typeAliases/commonized/linux/package_kotlinx_cinterop.kt deleted file mode 100644 index cf366643a8f..00000000000 --- a/native/commonizer/testData/classifierCommonization/typeAliases/commonized/linux/package_kotlinx_cinterop.kt +++ /dev/null @@ -1,12 +0,0 @@ -// this is to avoid missing Kotlin/Native stdlib -package kotlinx.cinterop - -// fake classes with the default constructor and no member scope -abstract class CStructVar -class CPointer -@Suppress("FINAL_UPPER_BOUND") class UByteVarOf -class UByte - -// fake typealiases -typealias CArrayPointer = CPointer -typealias UByteVar = UByteVarOf diff --git a/native/commonizer/testData/classifierCommonization/typeAliases/commonized/macos/package_kotlinx_cinterop.kt b/native/commonizer/testData/classifierCommonization/typeAliases/commonized/macos/package_kotlinx_cinterop.kt deleted file mode 100644 index cf366643a8f..00000000000 --- a/native/commonizer/testData/classifierCommonization/typeAliases/commonized/macos/package_kotlinx_cinterop.kt +++ /dev/null @@ -1,12 +0,0 @@ -// this is to avoid missing Kotlin/Native stdlib -package kotlinx.cinterop - -// fake classes with the default constructor and no member scope -abstract class CStructVar -class CPointer -@Suppress("FINAL_UPPER_BOUND") class UByteVarOf -class UByte - -// fake typealiases -typealias CArrayPointer = CPointer -typealias UByteVar = UByteVarOf diff --git a/native/commonizer/testData/classifierCommonization/typeAliases/commonized/common/package_kotlinx_cinterop.kt b/native/commonizer/testData/classifierCommonization/typeAliases/dependee/common/package_kotlinx_cinterop.kt similarity index 100% rename from native/commonizer/testData/classifierCommonization/typeAliases/commonized/common/package_kotlinx_cinterop.kt rename to native/commonizer/testData/classifierCommonization/typeAliases/dependee/common/package_kotlinx_cinterop.kt diff --git a/native/commonizer/testData/classifierCommonization/typeAliases/original/linux/package_kotlinx_cinterop.kt b/native/commonizer/testData/classifierCommonization/typeAliases/original/linux/package_kotlinx_cinterop.kt deleted file mode 100644 index cf366643a8f..00000000000 --- a/native/commonizer/testData/classifierCommonization/typeAliases/original/linux/package_kotlinx_cinterop.kt +++ /dev/null @@ -1,12 +0,0 @@ -// this is to avoid missing Kotlin/Native stdlib -package kotlinx.cinterop - -// fake classes with the default constructor and no member scope -abstract class CStructVar -class CPointer -@Suppress("FINAL_UPPER_BOUND") class UByteVarOf -class UByte - -// fake typealiases -typealias CArrayPointer = CPointer -typealias UByteVar = UByteVarOf diff --git a/native/commonizer/testData/classifierCommonization/typeAliases/original/macos/package_kotlinx_cinterop.kt b/native/commonizer/testData/classifierCommonization/typeAliases/original/macos/package_kotlinx_cinterop.kt deleted file mode 100644 index cf366643a8f..00000000000 --- a/native/commonizer/testData/classifierCommonization/typeAliases/original/macos/package_kotlinx_cinterop.kt +++ /dev/null @@ -1,12 +0,0 @@ -// this is to avoid missing Kotlin/Native stdlib -package kotlinx.cinterop - -// fake classes with the default constructor and no member scope -abstract class CStructVar -class CPointer -@Suppress("FINAL_UPPER_BOUND") class UByteVarOf -class UByte - -// fake typealiases -typealias CArrayPointer = CPointer -typealias UByteVar = UByteVarOf diff --git a/native/commonizer/testData/functionCommonization/valueParameters/commonized/macos/package_kotlinx_cinterop.kt b/native/commonizer/testData/functionCommonization/valueParameters/commonized/macos/package_kotlinx_cinterop.kt deleted file mode 100644 index fe5970a0bf4..00000000000 --- a/native/commonizer/testData/functionCommonization/valueParameters/commonized/macos/package_kotlinx_cinterop.kt +++ /dev/null @@ -1,6 +0,0 @@ -// this is to avoid missing Kotlin/Native stdlib -package kotlinx.cinterop - -@Target(AnnotationTarget.FUNCTION, AnnotationTarget.PROPERTY_GETTER, AnnotationTarget.PROPERTY_SETTER) -@Retention(AnnotationRetention.BINARY) -annotation class ObjCMethod() // fake annotation class without properties diff --git a/native/commonizer/testData/functionCommonization/valueParameters/commonized/ios/package_kotlinx_cinterop.kt b/native/commonizer/testData/functionCommonization/valueParameters/dependee/common/package_kotlinx_cinterop.kt similarity index 100% rename from native/commonizer/testData/functionCommonization/valueParameters/commonized/ios/package_kotlinx_cinterop.kt rename to native/commonizer/testData/functionCommonization/valueParameters/dependee/common/package_kotlinx_cinterop.kt diff --git a/native/commonizer/testData/functionCommonization/valueParameters/original/ios/package_kotlinx_cinterop.kt b/native/commonizer/testData/functionCommonization/valueParameters/original/ios/package_kotlinx_cinterop.kt deleted file mode 100644 index fe5970a0bf4..00000000000 --- a/native/commonizer/testData/functionCommonization/valueParameters/original/ios/package_kotlinx_cinterop.kt +++ /dev/null @@ -1,6 +0,0 @@ -// this is to avoid missing Kotlin/Native stdlib -package kotlinx.cinterop - -@Target(AnnotationTarget.FUNCTION, AnnotationTarget.PROPERTY_GETTER, AnnotationTarget.PROPERTY_SETTER) -@Retention(AnnotationRetention.BINARY) -annotation class ObjCMethod() // fake annotation class without properties diff --git a/native/commonizer/testData/functionCommonization/valueParameters/original/macos/package_kotlinx_cinterop.kt b/native/commonizer/testData/functionCommonization/valueParameters/original/macos/package_kotlinx_cinterop.kt deleted file mode 100644 index fe5970a0bf4..00000000000 --- a/native/commonizer/testData/functionCommonization/valueParameters/original/macos/package_kotlinx_cinterop.kt +++ /dev/null @@ -1,6 +0,0 @@ -// this is to avoid missing Kotlin/Native stdlib -package kotlinx.cinterop - -@Target(AnnotationTarget.FUNCTION, AnnotationTarget.PROPERTY_GETTER, AnnotationTarget.PROPERTY_SETTER) -@Retention(AnnotationRetention.BINARY) -annotation class ObjCMethod() // fake annotation class without properties diff --git a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/AbstractCommonizationFromSourcesTest.kt b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/AbstractCommonizationFromSourcesTest.kt index 05993646fef..9f798a872ab 100644 --- a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/AbstractCommonizationFromSourcesTest.kt +++ b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/AbstractCommonizationFromSourcesTest.kt @@ -17,7 +17,7 @@ import org.jetbrains.kotlin.config.CommonConfigurationKeys import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.config.languageVersionSettings import org.jetbrains.kotlin.descriptors.* -import org.jetbrains.kotlin.descriptors.commonizer.SourceModuleRoot.Companion.COMMON_TARGET_NAME +import org.jetbrains.kotlin.descriptors.commonizer.SourceModuleRoot.Companion.SHARED_TARGET_NAME import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.ClassCollector import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.FunctionCollector import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.collectMembers @@ -30,10 +30,8 @@ import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl import org.jetbrains.kotlin.js.resolve.diagnostics.findPsi import org.jetbrains.kotlin.name.Name import org.jetbrains.kotlin.platform.CommonPlatforms -import org.jetbrains.kotlin.platform.TargetPlatform import org.jetbrains.kotlin.psi.KtFile import org.jetbrains.kotlin.psi.KtPsiFactory -import org.jetbrains.kotlin.resolve.PlatformDependentAnalyzerServices import org.jetbrains.kotlin.resolve.scopes.MemberScope import org.jetbrains.kotlin.test.KotlinTestUtils.* import org.jetbrains.kotlin.test.testFramework.KtUsefulTestCase @@ -70,10 +68,10 @@ abstract class AbstractCommonizationFromSourcesTest : KtUsefulTestCase() { val result: Result = runCommonization(analyzedModules.toCommonizationParameters()) assertCommonizationPerformed(result) - val sharedTarget: OutputTarget = analyzedModules.commonizedCommonModule.target + val sharedTarget: OutputTarget = analyzedModules.sharedTarget assertEquals(sharedTarget, result.sharedTarget) - val sharedModuleAsExpected: ModuleDescriptor = analyzedModules.commonizedCommonModule.module + val sharedModuleAsExpected: ModuleDescriptor = analyzedModules.commonizedModules.getValue(sharedTarget) val sharedModuleByCommonizer: ModuleDescriptor = (result.modulesByTargets.getValue(sharedTarget).single() as ModuleResult.Commonized).module @@ -81,11 +79,11 @@ abstract class AbstractCommonizationFromSourcesTest : KtUsefulTestCase() { assertValidModule(sharedModuleByCommonizer) assertModulesAreEqual(sharedModuleAsExpected, sharedModuleByCommonizer, "\"$sharedTarget\" target") - val leafTargets: Set = analyzedModules.commonizedPlatformModules.keys + val leafTargets: Set = analyzedModules.leafTargets assertEquals(leafTargets, result.leafTargets) for (leafTarget in leafTargets) { - val leafTargetModuleAsExpected: ModuleDescriptor = analyzedModules.commonizedPlatformModules.getValue(leafTarget).module + val leafTargetModuleAsExpected: ModuleDescriptor = analyzedModules.commonizedModules.getValue(leafTarget) val leafTargetModuleByCommonizer: ModuleDescriptor = (result.modulesByTargets.getValue(leafTarget).single() as ModuleResult.Commonized).module @@ -98,117 +96,196 @@ abstract class AbstractCommonizationFromSourcesTest : KtUsefulTestCase() { private data class SourceModuleRoot( val targetName: String, - val root: File + val location: File ) { init { - assertIsDirectory(root) + assertIsDirectory(location) } companion object { fun load(directory: File): SourceModuleRoot = SourceModuleRoot( targetName = directory.name, - root = directory + location = directory ) - const val COMMON_TARGET_NAME = "common" + const val SHARED_TARGET_NAME = "common" } } private class SourceModuleRoots( - val originalPlatformRoots: Map, - val commonizedPlatformRoots: Map, - val commonizedCommonRoot: SourceModuleRoot + val originalRoots: Map, + val commonizedRoots: Map, + val dependeeRoots: Map ) { + val leafTargets: Set = originalRoots.keys + val sharedTarget: OutputTarget + init { - check(originalPlatformRoots.isNotEmpty()) - check(COMMON_TARGET_NAME !in originalPlatformRoots) - check(originalPlatformRoots.keys == commonizedPlatformRoots.keys) - check(commonizedCommonRoot.targetName == COMMON_TARGET_NAME) + check(leafTargets.size >= 2) + check(leafTargets.none { it.name == SHARED_TARGET_NAME }) + + val sharedTargets = commonizedRoots.keys.filterIsInstance() + check(sharedTargets.size == 1) + + sharedTarget = sharedTargets.single() + check(sharedTarget.targets == leafTargets) + + val allTargets = leafTargets + sharedTarget + check(commonizedRoots.keys == allTargets) + check(allTargets.containsAll(dependeeRoots.keys)) } companion object { fun load(dataDir: File): SourceModuleRoots = try { - val originalRoots = listRoots(dataDir, ORIGINAL_ROOTS_DIR) - val commonizedRoots = listRoots(dataDir, COMMONIZED_ROOTS_DIR) + val originalRoots = listRoots(dataDir, ORIGINAL_ROOTS_DIR).mapKeys { InputTarget(it.key) } - SourceModuleRoots( - originalPlatformRoots = originalRoots, - commonizedPlatformRoots = commonizedRoots - COMMON_TARGET_NAME, - commonizedCommonRoot = commonizedRoots.getValue(COMMON_TARGET_NAME) - ) + val leafTargets = originalRoots.keys + val sharedTarget = OutputTarget(leafTargets) + + fun getTarget(targetName: String): Target = + if (targetName == SHARED_TARGET_NAME) sharedTarget else leafTargets.first { it.name == targetName } + + val commonizedRoots = listRoots(dataDir, COMMONIZED_ROOTS_DIR).mapKeys { getTarget(it.key) } + val dependeeRoots = listRoots(dataDir, DEPENDEE_ROOTS_DIR).mapKeys { getTarget(it.key) } + + SourceModuleRoots(originalRoots, commonizedRoots, dependeeRoots) } catch (e: Exception) { fail("Source module misconfiguration in $dataDir", cause = e) } private const val ORIGINAL_ROOTS_DIR = "original" private const val COMMONIZED_ROOTS_DIR = "commonized" + private const val DEPENDEE_ROOTS_DIR = "dependee" private fun listRoots(dataDir: File, rootsDirName: String): Map = dataDir.resolve(rootsDirName).listFiles()?.toSet().orEmpty().map(SourceModuleRoot::load).associateBy { it.targetName } } } -private class AnalyzedModule( - val target: T, - val module: ModuleDescriptor +private class AnalyzedModuleDependencies( + val regularDependencies: Map>, + val expectByDependencies: List ) { - companion object { - fun create( - target: T, - sourceModuleRoot: SourceModuleRoot, - commonSourceModuleRoot: SourceModuleRoot? = null, - parentDisposable: Disposable - ): AnalyzedModule { - val moduleName: String = sourceModuleRoot.root.parentFile.parentFile.name - check(Name.isValidIdentifier(moduleName)) + fun withExpectByDependency(dependency: ModuleDescriptor) = + AnalyzedModuleDependencies( + regularDependencies = regularDependencies, + expectByDependencies = expectByDependencies + dependency + ) - return AnalyzedModule( - target = target, - module = analyze( - moduleName = moduleName, - moduleRoot = sourceModuleRoot.root, - commonModuleRoot = commonSourceModuleRoot?.root, - parentDisposable = parentDisposable + companion object { + val EMPTY = AnalyzedModuleDependencies(emptyMap(), emptyList()) + + fun create(regularDependencies: Map, expectByDependencies: List) = + AnalyzedModuleDependencies(regularDependencies.mapValues { listOf(it.value) }, expectByDependencies) + } +} + +private class AnalyzedModules( + val originalModules: Map, + val commonizedModules: Map, + val dependeeModules: Map +) { + val leafTargets: Set + val sharedTarget: OutputTarget + + init { + originalModules.keys.let { targets -> + check(targets.isNotEmpty()) + + leafTargets = targets.filterIsInstance().toSet() + check(targets.size == leafTargets.size) + } + + sharedTarget = OutputTarget(leafTargets) + val allTargets = leafTargets + sharedTarget + + check(commonizedModules.keys == allTargets) + check(allTargets.containsAll(dependeeModules.keys)) + } + + fun toCommonizationParameters(): Parameters { + val parameters = Parameters() + + leafTargets.forEach { leafTarget -> + val originalModule = originalModules.getValue(leafTarget) + + parameters.addTarget( + TargetProvider( + target = leafTarget, + builtInsClass = originalModule.builtIns::class.java, + builtInsProvider = MockBuiltInsProvider(originalModule.builtIns), + modulesProvider = MockModulesProvider(originalModule), + dependeeModulesProvider = dependeeModules[leafTarget]?.let(::MockModulesProvider) ) ) } - private fun analyze( - moduleName: String, - moduleRoot: File, - commonModuleRoot: File?, - parentDisposable: Disposable - ): ModuleDescriptor { - val commonModule: ModuleDescriptor? = if (commonModuleRoot != null) { - analyzeModule( - moduleName = "common" + moduleName.capitalize(), - moduleRoot = commonModuleRoot, - dependencyContainer = null, // common module does not have any specific dependencies - parentDisposable = parentDisposable - ) - } else null + parameters.dependeeModulesProvider = dependeeModules[sharedTarget]?.let(::MockModulesProvider) - val module: ModuleDescriptor = analyzeModule( - moduleName = moduleName, - moduleRoot = moduleRoot, - dependencyContainer = commonModule?.let(::CommonizedCommonDependenciesContainer), // platform module has dependencies to common module - parentDisposable = parentDisposable + return parameters + } + + companion object { + fun create( + sourceModuleRoots: SourceModuleRoots, + parentDisposable: Disposable + ): AnalyzedModules = with(sourceModuleRoots) { + // first, build the modules that are are the dependencies for "original" and "commonized" modules + val dependeeModules = + createModules(sharedTarget, dependeeRoots, AnalyzedModuleDependencies.EMPTY, parentDisposable, isDependeeModule = true) + + val dependencies = AnalyzedModuleDependencies.create( + regularDependencies = dependeeModules, + expectByDependencies = listOfNotNull(dependeeModules[sharedTarget]) ) - if (commonModule != null) { - check(commonModule in module.expectedByModules) - check(commonModule in module.allDependencyModules) + // then, build "original" and "commonized" modules + val originalModules = createModules(sharedTarget, originalRoots, dependencies, parentDisposable) + val commonizedModules = createModules(sharedTarget, commonizedRoots, dependencies, parentDisposable) + + return AnalyzedModules(originalModules, commonizedModules, dependeeModules) + } + + private fun createModules( + sharedTarget: OutputTarget, + moduleRoots: Map, + dependencies: AnalyzedModuleDependencies, + parentDisposable: Disposable, + isDependeeModule: Boolean = false + ): Map { + val result = mutableMapOf() + + var dependenciesForOthers = dependencies + + // first, process the common module + moduleRoots[sharedTarget]?.let { moduleRoot -> + val commonModule = createModule(sharedTarget, sharedTarget, moduleRoot, dependencies, parentDisposable, isDependeeModule) + result[sharedTarget] = commonModule + dependenciesForOthers = dependencies.withExpectByDependency(commonModule) } - return module + // then, all platform modules + moduleRoots.filterKeys { it != sharedTarget }.forEach { (leafTarget, moduleRoot) -> + result[leafTarget] = + createModule(sharedTarget, leafTarget, moduleRoot, dependenciesForOthers, parentDisposable, isDependeeModule) + } + + return result } - private fun analyzeModule( - moduleName: String, - moduleRoot: File, - dependencyContainer: CommonDependenciesContainer?, - parentDisposable: Disposable + private fun createModule( + sharedTarget: OutputTarget, + currentTarget: Target, + moduleRoot: SourceModuleRoot, + dependencies: AnalyzedModuleDependencies, + parentDisposable: Disposable, + isDependeeModule: Boolean ): ModuleDescriptor { + val moduleName: String = moduleRoot.location.parentFile.parentFile.name.let { + if (isDependeeModule) "dependee-$it" else it + } + check(Name.isValidIdentifier(moduleName)) + val configuration: CompilerConfiguration = newConfiguration() configuration.put(CommonConfigurationKeys.MODULE_NAME, moduleName) @@ -220,7 +297,7 @@ private class AnalyzedModule( val psiFactory = KtPsiFactory(environment.project) - val psiFiles: List = moduleRoot.walkTopDown() + val psiFiles: List = moduleRoot.location.walkTopDown() .filter { it.isFile } .map { psiFactory.createFile(it.name, doLoadFile(it)) } .toList() @@ -231,105 +308,68 @@ private class AnalyzedModule( dependOnBuiltIns = true, languageVersionSettings = environment.configuration.languageVersionSettings, targetPlatform = CommonPlatforms.defaultCommonPlatform, - dependenciesContainer = dependencyContainer + dependenciesContainer = DependenciesContainerImpl(sharedTarget, currentTarget, dependencies) ) { content -> environment.createPackagePartProvider(content.moduleContentScope) }.moduleDescriptor - module.accept(PatchingTestDescriptorVisitor, Unit) + if (!isDependeeModule) + module.accept(PatchingTestDescriptorVisitor, Unit) return module } } } -private class AnalyzedModules( - val originalPlatformModules: Map>, - val commonizedPlatformModules: Map>, - val commonizedCommonModule: AnalyzedModule -) { +private class DependenciesContainerImpl( + sharedTarget: OutputTarget, + currentTarget: Target, + dependencies: AnalyzedModuleDependencies +) : CommonDependenciesContainer { + private val moduleInfoToModule = mutableMapOf() + private val expectByModuleInfos = mutableListOf() + private val regularModuleInfos = mutableListOf() + init { - check(originalPlatformModules.isNotEmpty()) - check(originalPlatformModules.keys == commonizedPlatformModules.keys) - } - - fun toCommonizationParameters(): Parameters { - val parameters = originalPlatformModules.mapValues { it.value.module }.toCommonizationParameters() - parameters.commonModulesProvider = MockModulesProvider(commonizedCommonModule.module) - return parameters - } - - companion object { - fun create( - sourceModuleRoots: SourceModuleRoots, - parentDisposable: Disposable - ): AnalyzedModules { - val originalPlatformModules: Map> = createInputTargetModules( - sourceModuleRoots = sourceModuleRoots.originalPlatformRoots, - parentDisposable = parentDisposable - ) - - val commonizedCommonModule: AnalyzedModule = AnalyzedModule.create( - target = OutputTarget(originalPlatformModules.keys), - sourceModuleRoot = sourceModuleRoots.commonizedCommonRoot, - parentDisposable = parentDisposable - ) - - val commonizedPlatformModules: Map> = createInputTargetModules( - sourceModuleRoots = sourceModuleRoots.commonizedPlatformRoots, - commonSourceModuleRoot = sourceModuleRoots.commonizedCommonRoot, - parentDisposable = parentDisposable - ) - - return AnalyzedModules( - originalPlatformModules = originalPlatformModules, - commonizedPlatformModules = commonizedPlatformModules, - commonizedCommonModule = commonizedCommonModule - ) + if (currentTarget != sharedTarget) { + dependencies.expectByDependencies.forEach { expectByDependency -> + val moduleInfo = ModuleInfoImpl(expectByDependency, emptyList()) + moduleInfoToModule[moduleInfo] = expectByDependency + expectByModuleInfos += moduleInfo + } } - private fun createInputTargetModules( - sourceModuleRoots: Map, - commonSourceModuleRoot: SourceModuleRoot? = null, - parentDisposable: Disposable - ): Map> = sourceModuleRoots.map { (targetName, sourceModuleRoot) -> - AnalyzedModule.create( - target = InputTarget(targetName), - sourceModuleRoot = sourceModuleRoot, - commonSourceModuleRoot = commonSourceModuleRoot, - parentDisposable = parentDisposable - ) - }.associateBy { it.target } - } -} + dependencies.regularDependencies[currentTarget]?.forEach { regularDependency -> + val moduleInfo = ModuleInfoImpl(regularDependency, expectByModuleInfos) + moduleInfoToModule[moduleInfo] = regularDependency + regularModuleInfos += moduleInfo + } -private class CommonizedCommonDependenciesContainer( - private val commonModule: ModuleDescriptor -) : CommonDependenciesContainer { - private val commonModuleInfo = object : ModuleInfo { - override val name: Name get() = commonModule.name - - override fun dependencies(): List = listOf(this) - override fun dependencyOnBuiltIns(): ModuleInfo.DependencyOnBuiltIns = ModuleInfo.DependencyOnBuiltIns.LAST - - override val platform: TargetPlatform get() = CommonPlatforms.defaultCommonPlatform - override val analyzerServices: PlatformDependentAnalyzerServices get() = CommonPlatformAnalyzerServices + regularModuleInfos += expectByModuleInfos } - override val moduleInfos: List get() = listOf(commonModuleInfo) + private inner class ModuleInfoImpl( + private val module: ModuleDescriptor, + private val regularDependencies: List + ) : ModuleInfo { + override val name get() = module.name - override fun moduleDescriptorForModuleInfo(moduleInfo: ModuleInfo): ModuleDescriptor { - if (moduleInfo !== commonModuleInfo) - error("Unknown module info $moduleInfo") + override fun dependencies() = listOf(this) + regularDependencies + override fun dependencyOnBuiltIns() = ModuleInfo.DependencyOnBuiltIns.LAST - return commonModule + override val platform get() = CommonPlatforms.defaultCommonPlatform + override val analyzerServices get() = CommonPlatformAnalyzerServices } + override val moduleInfos: List get() = regularModuleInfos + override val friendModuleInfos: List get() = emptyList() + override val refinesModuleInfos: List get() = expectByModuleInfos + + override fun moduleDescriptorForModuleInfo(moduleInfo: ModuleInfo) = + moduleInfoToModule[moduleInfo] ?: error("Unknown module info $moduleInfo") + override fun registerDependencyForAllModules(moduleInfo: ModuleInfo, descriptorForModule: ModuleDescriptorImpl) = Unit override fun packageFragmentProviderForModuleInfo(moduleInfo: ModuleInfo): PackageFragmentProvider? = null - - override val friendModuleInfos: List get() = emptyList() - override val refinesModuleInfos: List get() = listOf(commonModuleInfo) } private object PatchingTestDescriptorVisitor : DeclarationDescriptorVisitorEmptyBodies() { @@ -368,21 +408,3 @@ private object PatchingTestDescriptorVisitor : DeclarationDescriptorVisitorEmpty } } } - -private fun Map.toCommonizationParameters(): Parameters = Parameters().also { parameters -> - forEach { (target, moduleDescriptor) -> - if (!parameters.extendedLookupForBuiltInsClassifiers) { - if (moduleDescriptor.hasSomethingUnderStandardKotlinPackages) - parameters.extendedLookupForBuiltInsClassifiers = true - } - - parameters.addTarget( - TargetProvider( - target = target, - builtInsClass = moduleDescriptor.builtIns::class.java, - builtInsProvider = MockBuiltInsProvider(moduleDescriptor.builtIns), - modulesProvider = MockModulesProvider(moduleDescriptor) - ) - ) - } -} diff --git a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/CommonizerFacadeTest.kt b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/CommonizerFacadeTest.kt index 8664742f33b..38fc7cfb952 100644 --- a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/CommonizerFacadeTest.kt +++ b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/CommonizerFacadeTest.kt @@ -67,7 +67,8 @@ class CommonizerFacadeTest { target = InputTarget(targetName), builtInsClass = DefaultBuiltIns::class.java, builtInsProvider = BuiltInsProvider.defaultBuiltInsProvider, - modulesProvider = MockModulesProvider(moduleNames) + modulesProvider = MockModulesProvider(moduleNames), + dependeeModulesProvider = null ) ) } diff --git a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/utils/mocks.kt b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/utils/mocks.kt index bd5826048e0..dd4ab4c6ca1 100644 --- a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/utils/mocks.kt +++ b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/utils/mocks.kt @@ -46,8 +46,7 @@ internal fun mockClassType( storageManager = LockBasedStorageManager.NO_LOCKS, target = InputTarget("Arbitrary target"), builtIns = DefaultBuiltIns.Instance, - lazyModulesLookupTable = LockBasedStorageManager.NO_LOCKS.createLazyValue { mutableMapOf() }, - isCommon = false, + lazyClassifierLookupTable = LockBasedStorageManager.NO_LOCKS.createLazyValue { LazyClassifierLookupTable(emptyMap()) }, index = 0, cache = DeclarationsBuilderCache(1) ) @@ -173,7 +172,7 @@ internal class MockModulesProvider : ModulesProvider { } override fun loadModuleInfos() = moduleInfos - override fun loadModules() = modules + override fun loadModules(dependencies: Collection): Map = modules private fun fakeModuleInfo(name: String) = ModuleInfo(name, File("/tmp/commonizer/mocks/$name"), null) }