diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/TargetProvider.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/TargetProvider.kt index 6f6e279ec53..fdd2be0ba74 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/TargetProvider.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/TargetProvider.kt @@ -6,6 +6,7 @@ package org.jetbrains.kotlin.descriptors.commonizer import org.jetbrains.kotlin.descriptors.ModuleDescriptor +import org.jetbrains.kotlin.library.SerializedMetadata import java.io.File class TargetProvider( @@ -15,7 +16,7 @@ class TargetProvider( ) interface ModulesProvider { - class ModuleInfo( + open class ModuleInfo( val name: String, val originalLocation: File, val cInteropAttributes: CInteropModuleAttributes? @@ -26,6 +27,17 @@ interface ModulesProvider { val exportForwardDeclarations: Collection ) + /** + * Returns information about all modules that can be loaded by this [ModulesProvider] in the form of [ModuleInfo]s. + * This function is relatively light-weight and does not have significant impact on performance. + */ fun loadModuleInfos(): Collection + + /** + * Loads metadata for the specified module. + */ + fun loadModuleMetadata(name: String): SerializedMetadata + + @Deprecated("To be replaced by loadModuleMetadata()") fun loadModules(dependencies: Collection): Map } diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt index 78f860004f9..4e831df93d2 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt @@ -135,12 +135,12 @@ class NativeDistributionCommonizer( copyEndorsedLibs = copyEndorsedLibs, logProgress = ::logProgress ) - dependeeModulesProvider = NativeDistributionStdlibProvider(storageManager, allLibraries.stdlib) + dependeeModulesProvider = NativeDistributionModulesProvider.forStandardLibrary(storageManager, allLibraries.stdlib) allLibraries.librariesByTargets.forEach { (target, librariesToCommonize) -> if (librariesToCommonize.libraries.isEmpty()) return@forEach - val modulesProvider = NativeDistributionModulesProvider(storageManager, librariesToCommonize) + val modulesProvider = NativeDistributionModulesProvider.platformLibraries(storageManager, librariesToCommonize) addTarget( TargetProvider( diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionModulesProvider.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionModulesProvider.kt index 9d302d72d36..5dbd7c2546d 100644 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionModulesProvider.kt +++ b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionModulesProvider.kt @@ -15,20 +15,36 @@ import org.jetbrains.kotlin.descriptors.commonizer.utils.createKotlinNativeForwa import org.jetbrains.kotlin.descriptors.commonizer.utils.strip import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl import org.jetbrains.kotlin.incremental.components.LookupTracker +import org.jetbrains.kotlin.konan.library.KONAN_STDLIB_NAME +import org.jetbrains.kotlin.library.SerializedMetadata +import org.jetbrains.kotlin.library.metadata.parseModuleHeader import org.jetbrains.kotlin.storage.StorageManager import org.jetbrains.kotlin.utils.addIfNotNull import java.io.File -internal class NativeDistributionModulesProvider( - private val storageManager: StorageManager, - private val librariesToCommonize: NativeLibrariesToCommonize -) : ModulesProvider { - private val moduleInfos by lazy { - librariesToCommonize.libraries.map { library -> +internal abstract class NativeDistributionModulesProvider(libraries: Collection) : ModulesProvider { + internal class NativeModuleInfo( + name: String, + originalLocation: File, + val dependencies: Set, + cInteropAttributes: CInteropModuleAttributes? + ) : ModuleInfo(name, originalLocation, cInteropAttributes) + + protected val libraryMap: Map + protected val moduleInfoMap: Map + + init { + check(libraries.isNotEmpty()) { "No libraries supplied" } + + val libraryMap = mutableMapOf() + val moduleInfoMap = mutableMapOf() + + libraries.forEach { library -> val manifestData = library.manifestData val name = manifestData.uniqueName val location = File(library.library.libraryFile.path) + val dependencies = manifestData.dependencies.toSet() val cInteropAttributes = if (manifestData.isInterop) { val packageFqName = manifestData.packageFqName @@ -38,56 +54,105 @@ internal class NativeDistributionModulesProvider( CInteropModuleAttributes(packageFqName, manifestData.exportForwardDeclarations) } else null - ModuleInfo(name, location, cInteropAttributes) + libraryMap.put(name, library)?.let { error("Duplicated libraries: $name") } + moduleInfoMap[name] = NativeModuleInfo(name, location, dependencies, cInteropAttributes) } + + this.libraryMap = libraryMap + this.moduleInfoMap = moduleInfoMap } - override fun loadModuleInfos() = moduleInfos + final override fun loadModuleInfos(): Collection = moduleInfoMap.values - override fun loadModules(dependencies: Collection): Map { - check(dependencies.isNotEmpty()) { "At least Kotlin/Native stdlib should be provided" } + final override fun loadModuleMetadata(name: String): SerializedMetadata { + val library = libraryMap[name]?.library ?: error("No such library: $name") - val dependenciesMap = mutableMapOf>() - dependencies.forEach { dependency -> - val name = dependency.name.strip() - dependenciesMap.getOrPut(name) { mutableListOf() } += dependency as ModuleDescriptorImpl + val moduleHeader = library.moduleHeaderData + val fragmentNames = parseModuleHeader(moduleHeader).packageFragmentNameList.toSet() + val fragments = fragmentNames.map { fragmentName -> + val partNames = library.packageMetadataParts(fragmentName) + partNames.map { partName -> library.packageMetadata(fragmentName, partName) } } - val builtIns = dependencies.first().builtIns - - val platformModulesMap = librariesToCommonize.libraries.associate { library -> - val name = library.manifestData.uniqueName - val module = NativeFactories.DefaultDeserializedDescriptorFactory.createDescriptorOptionalBuiltIns( - library = library.library, - languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT, - storageManager = storageManager, - builtIns = builtIns, - packageAccessHandler = null, - lookupTracker = LookupTracker.DO_NOTHING - ) - - name to module - } - - val forwardDeclarations = createKotlinNativeForwardDeclarationsModule( - storageManager = storageManager, - builtIns = builtIns + return SerializedMetadata( + module = moduleHeader, + fragments = fragments, + fragmentNames = fragmentNames.toList() ) + } - platformModulesMap.forEach { (name, module) -> - val moduleDependencies = mutableListOf() - moduleDependencies += module + companion object { + fun forStandardLibrary( + storageManager: StorageManager, + stdlib: NativeLibrary + ): ModulesProvider { + check(stdlib.manifestData.uniqueName == KONAN_STDLIB_NAME) - librariesToCommonize.getManifest(name).dependencies.forEach { - moduleDependencies.addIfNotNull(platformModulesMap[it]) - moduleDependencies += dependenciesMap[it].orEmpty() + return object : NativeDistributionModulesProvider(listOf(stdlib)) { + override fun loadModules(dependencies: Collection): Map { + check(dependencies.isEmpty()) + + val stdlibModule = NativeFactories.DefaultDeserializedDescriptorFactory.createDescriptorAndNewBuiltIns( + library = stdlib.library, + languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT, + storageManager = storageManager, + packageAccessHandler = null + ).apply { + setDependencies(listOf(this)) + } + + return mapOf(KONAN_STDLIB_NAME to stdlibModule) + } } - - moduleDependencies += forwardDeclarations - - module.setDependencies(moduleDependencies) } - return platformModulesMap + fun platformLibraries( + storageManager: StorageManager, + librariesToCommonize: NativeLibrariesToCommonize + ): ModulesProvider = object : NativeDistributionModulesProvider(librariesToCommonize.libraries) { + override fun loadModules(dependencies: Collection): Map { + check(dependencies.isNotEmpty()) { "At least Kotlin/Native stdlib should be provided" } + + val dependenciesMap = mutableMapOf>() + dependencies.forEach { dependency -> + val name = dependency.name.strip() + dependenciesMap.getOrPut(name) { mutableListOf() } += dependency as ModuleDescriptorImpl + } + + val builtIns = dependencies.first().builtIns + + val platformModulesMap = libraryMap.mapValues { (_, library) -> + NativeFactories.DefaultDeserializedDescriptorFactory.createDescriptorOptionalBuiltIns( + library = library.library, + languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT, + storageManager = storageManager, + builtIns = builtIns, + packageAccessHandler = null, + lookupTracker = LookupTracker.DO_NOTHING + ) + } + + val forwardDeclarations = createKotlinNativeForwardDeclarationsModule( + storageManager = storageManager, + builtIns = builtIns + ) + + platformModulesMap.forEach { (name, module) -> + val moduleDependencies = mutableListOf() + moduleDependencies += module + + moduleInfoMap.getValue(name).dependencies.forEach { + moduleDependencies.addIfNotNull(platformModulesMap[it]) + moduleDependencies += dependenciesMap[it].orEmpty() + } + + moduleDependencies += forwardDeclarations + + module.setDependencies(moduleDependencies) + } + + return platformModulesMap + } + } } } diff --git a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionStdlibProvider.kt b/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionStdlibProvider.kt deleted file mode 100644 index 5bd6b13deb3..00000000000 --- a/native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionStdlibProvider.kt +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.descriptors.commonizer.konan - -import org.jetbrains.kotlin.config.LanguageVersionSettingsImpl -import org.jetbrains.kotlin.descriptors.ModuleDescriptor -import org.jetbrains.kotlin.descriptors.commonizer.ModulesProvider -import org.jetbrains.kotlin.descriptors.commonizer.ModulesProvider.ModuleInfo -import org.jetbrains.kotlin.descriptors.commonizer.utils.NativeFactories -import org.jetbrains.kotlin.konan.library.KONAN_STDLIB_NAME -import org.jetbrains.kotlin.storage.StorageManager -import java.io.File - -internal class NativeDistributionStdlibProvider( - private val storageManager: StorageManager, - private val stdlib: NativeLibrary -) : ModulesProvider { - private val moduleInfo = ModuleInfo( - name = KONAN_STDLIB_NAME, - originalLocation = File(stdlib.library.libraryFile.absolutePath), - cInteropAttributes = null - ) - - override fun loadModuleInfos() = listOf(moduleInfo) - - override fun loadModules(dependencies: Collection): Map { - check(dependencies.isEmpty()) - return mapOf(KONAN_STDLIB_NAME to loadStdlibModule()) - } - - private fun loadStdlibModule() = - NativeFactories.DefaultDeserializedDescriptorFactory.createDescriptorAndNewBuiltIns( - library = stdlib.library, - languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT, - storageManager = storageManager, - packageAccessHandler = null - ).apply { - setDependencies(listOf(this)) - } -} \ No newline at end of file diff --git a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/AbstractCommonizationFromSourcesTest.kt b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/AbstractCommonizationFromSourcesTest.kt index b9eb1cf3357..dc9aef06856 100644 --- a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/AbstractCommonizationFromSourcesTest.kt +++ b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/AbstractCommonizationFromSourcesTest.kt @@ -11,14 +11,11 @@ import org.jetbrains.kotlin.analyzer.ModuleInfo import org.jetbrains.kotlin.analyzer.common.CommonDependenciesContainer import org.jetbrains.kotlin.analyzer.common.CommonPlatformAnalyzerServices import org.jetbrains.kotlin.analyzer.common.CommonResolverForModuleFactory -import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataMonolithicSerializer -import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataVersion import org.jetbrains.kotlin.builtins.DefaultBuiltIns import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment import org.jetbrains.kotlin.config.CommonConfigurationKeys import org.jetbrains.kotlin.config.CompilerConfiguration -import org.jetbrains.kotlin.config.LanguageVersionSettingsImpl import org.jetbrains.kotlin.config.languageVersionSettings import org.jetbrains.kotlin.descriptors.CallableMemberDescriptor import org.jetbrains.kotlin.descriptors.ModuleDescriptor @@ -239,7 +236,9 @@ private class AnalyzedModules( val commonizedModules: Map = createModules(sharedTarget, commonizedRoots, dependencies, parentDisposable) - .mapValues { (_, moduleDescriptor) -> serializer.serializeModule(moduleDescriptor) } + .mapValues { (_, moduleDescriptor) -> MockModulesProvider.SERIALIZER.serializeModule(moduleDescriptor) } + + return AnalyzedModules(originalModules, commonizedModules, dependeeModules) } @@ -337,13 +336,6 @@ private class AnalyzedModules( return module } - - private val serializer = KlibMetadataMonolithicSerializer( - languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT, - metadataVersion = KlibMetadataVersion.INSTANCE, - skipExpects = false, - project = null - ) } } diff --git a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/utils/mocks.kt b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/utils/mocks.kt index e559ce1187b..62f37b967d5 100644 --- a/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/utils/mocks.kt +++ b/native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/utils/mocks.kt @@ -5,6 +5,9 @@ package org.jetbrains.kotlin.descriptors.commonizer.utils +import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataMonolithicSerializer +import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataVersion +import org.jetbrains.kotlin.config.LanguageVersionSettingsImpl import org.jetbrains.kotlin.descriptors.* import org.jetbrains.kotlin.descriptors.annotations.Annotations import org.jetbrains.kotlin.descriptors.commonizer.* @@ -14,6 +17,7 @@ import org.jetbrains.kotlin.descriptors.commonizer.cir.factory.CirClassFactory import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.* import org.jetbrains.kotlin.descriptors.impl.AbstractTypeAliasDescriptor import org.jetbrains.kotlin.descriptors.impl.ClassDescriptorImpl +import org.jetbrains.kotlin.library.SerializedMetadata import org.jetbrains.kotlin.name.ClassId import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.name.Name @@ -30,6 +34,7 @@ import org.jetbrains.kotlin.storage.getValue internal fun mockEmptyModule(moduleName: String): ModuleDescriptor { val module = KotlinTestUtils.createEmptyModule(moduleName) module.initialize(PackageFragmentProvider.Empty) + module.setDependencies(module) return module } @@ -155,7 +160,13 @@ internal class MockModulesProvider private constructor( private val moduleInfos = modules.keys.map { name -> fakeModuleInfo(name) } override fun loadModuleInfos() = moduleInfos - override fun loadModules(dependencies: Collection): Map = modules + + override fun loadModuleMetadata(name: String): SerializedMetadata { + val module = modules[name] ?: error("No such module: $name") + return SERIALIZER.serializeModule(module) + } + + override fun loadModules(dependencies: Collection) = modules private fun fakeModuleInfo(name: String) = ModuleInfo(name, File("/tmp/commonizer/mocks/$name"), null) @@ -174,6 +185,13 @@ internal class MockModulesProvider private constructor( fun create(module: ModuleDescriptor) = MockModulesProvider( mapOf(module.name.strip() to module) ) + + val SERIALIZER = KlibMetadataMonolithicSerializer( + languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT, + metadataVersion = KlibMetadataVersion.INSTANCE, + skipExpects = false, + project = null + ) } }