[Commonizer] Add ModulesProvider.loadModuleMetadata() to load raw metadata

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/TargetProvider.kt

@@ -6,6 +6,7 @@
 package org.jetbrains.kotlin.descriptors.commonizer
 
 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
+import org.jetbrains.kotlin.library.SerializedMetadata
 import java.io.File
 
 class TargetProvider(
@@ -15,7 +16,7 @@ class TargetProvider(
 )
 
 interface ModulesProvider {
-    class ModuleInfo(
+    open class ModuleInfo(
         val name: String,
         val originalLocation: File,
         val cInteropAttributes: CInteropModuleAttributes?
@@ -26,6 +27,17 @@ interface ModulesProvider {
         val exportForwardDeclarations: Collection<String>
     )
 
+    /**
+     * Returns information about all modules that can be loaded by this [ModulesProvider] in the form of [ModuleInfo]s.
+     * This function is relatively light-weight and does not have significant impact on performance.
+     */
     fun loadModuleInfos(): Collection<ModuleInfo>
+
+    /**
+     * Loads metadata for the specified module.
+     */
+    fun loadModuleMetadata(name: String): SerializedMetadata
+
+    @Deprecated("To be replaced by loadModuleMetadata()")
     fun loadModules(dependencies: Collection<ModuleDescriptor>): Map<String, ModuleDescriptor>
 }

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt

@@ -135,12 +135,12 @@ class NativeDistributionCommonizer(
                 copyEndorsedLibs = copyEndorsedLibs,
                 logProgress = ::logProgress
             )
-            dependeeModulesProvider = NativeDistributionStdlibProvider(storageManager, allLibraries.stdlib)
+            dependeeModulesProvider = NativeDistributionModulesProvider.forStandardLibrary(storageManager, allLibraries.stdlib)
 
             allLibraries.librariesByTargets.forEach { (target, librariesToCommonize) ->
                 if (librariesToCommonize.libraries.isEmpty()) return@forEach
 
-                val modulesProvider = NativeDistributionModulesProvider(storageManager, librariesToCommonize)
+                val modulesProvider = NativeDistributionModulesProvider.platformLibraries(storageManager, librariesToCommonize)
 
                 addTarget(
                     TargetProvider(

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionModulesProvider.kt

@@ -15,20 +15,36 @@ import org.jetbrains.kotlin.descriptors.commonizer.utils.createKotlinNativeForwa
 import org.jetbrains.kotlin.descriptors.commonizer.utils.strip
 import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
 import org.jetbrains.kotlin.incremental.components.LookupTracker
+import org.jetbrains.kotlin.konan.library.KONAN_STDLIB_NAME
+import org.jetbrains.kotlin.library.SerializedMetadata
+import org.jetbrains.kotlin.library.metadata.parseModuleHeader
 import org.jetbrains.kotlin.storage.StorageManager
 import org.jetbrains.kotlin.utils.addIfNotNull
 import java.io.File
 
-internal class NativeDistributionModulesProvider(
+internal abstract class NativeDistributionModulesProvider(libraries: Collection<NativeLibrary>) : ModulesProvider {
-    private val storageManager: StorageManager,
+    internal class NativeModuleInfo(
-    private val librariesToCommonize: NativeLibrariesToCommonize
+        name: String,
-) : ModulesProvider {
+        originalLocation: File,
-    private val moduleInfos by lazy {
+        val dependencies: Set<String>,
-        librariesToCommonize.libraries.map { library ->
+        cInteropAttributes: CInteropModuleAttributes?
+    ) : ModuleInfo(name, originalLocation, cInteropAttributes)
+
+    protected val libraryMap: Map<String, NativeLibrary>
+    protected val moduleInfoMap: Map<String, NativeModuleInfo>
+
+    init {
+        check(libraries.isNotEmpty()) { "No libraries supplied" }
+
+        val libraryMap = mutableMapOf<String, NativeLibrary>()
+        val moduleInfoMap = mutableMapOf<String, NativeModuleInfo>()
+
+        libraries.forEach { library ->
             val manifestData = library.manifestData
 
             val name = manifestData.uniqueName
             val location = File(library.library.libraryFile.path)
+            val dependencies = manifestData.dependencies.toSet()
 
             val cInteropAttributes = if (manifestData.isInterop) {
                 val packageFqName = manifestData.packageFqName
@@ -38,56 +54,105 @@ internal class NativeDistributionModulesProvider(
                 CInteropModuleAttributes(packageFqName, manifestData.exportForwardDeclarations)
             } else null
 
-            ModuleInfo(name, location, cInteropAttributes)
+            libraryMap.put(name, library)?.let { error("Duplicated libraries: $name") }
+            moduleInfoMap[name] = NativeModuleInfo(name, location, dependencies, cInteropAttributes)
         }
+
+        this.libraryMap = libraryMap
+        this.moduleInfoMap = moduleInfoMap
     }
 
-    override fun loadModuleInfos() = moduleInfos
+    final override fun loadModuleInfos(): Collection<ModuleInfo> = moduleInfoMap.values
 
-    override fun loadModules(dependencies: Collection<ModuleDescriptor>): Map<String, ModuleDescriptor> {
+    final override fun loadModuleMetadata(name: String): SerializedMetadata {
-        check(dependencies.isNotEmpty()) { "At least Kotlin/Native stdlib should be provided" }
+        val library = libraryMap[name]?.library ?: error("No such library: $name")
 
-        val dependenciesMap = mutableMapOf<String, MutableList<ModuleDescriptorImpl>>()
+        val moduleHeader = library.moduleHeaderData
-        dependencies.forEach { dependency ->
+        val fragmentNames = parseModuleHeader(moduleHeader).packageFragmentNameList.toSet()
-            val name = dependency.name.strip()
+        val fragments = fragmentNames.map { fragmentName ->
-            dependenciesMap.getOrPut(name) { mutableListOf() } += dependency as ModuleDescriptorImpl
+            val partNames = library.packageMetadataParts(fragmentName)
+            partNames.map { partName -> library.packageMetadata(fragmentName, partName) }
         }
 
-        val builtIns = dependencies.first().builtIns
+        return SerializedMetadata(
-
+            module = moduleHeader,
-        val platformModulesMap = librariesToCommonize.libraries.associate { library ->
+            fragments = fragments,
-            val name = library.manifestData.uniqueName
+            fragmentNames = fragmentNames.toList()
-            val module = NativeFactories.DefaultDeserializedDescriptorFactory.createDescriptorOptionalBuiltIns(
-                library = library.library,
-                languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT,
-                storageManager = storageManager,
-                builtIns = builtIns,
-                packageAccessHandler = null,
-                lookupTracker = LookupTracker.DO_NOTHING
-            )
-
-            name to module
-        }
-
-        val forwardDeclarations = createKotlinNativeForwardDeclarationsModule(
-            storageManager = storageManager,
-            builtIns = builtIns
         )
+    }
 
-        platformModulesMap.forEach { (name, module) ->
+    companion object {
-            val moduleDependencies = mutableListOf<ModuleDescriptorImpl>()
+        fun forStandardLibrary(
-            moduleDependencies += module
+            storageManager: StorageManager,
+            stdlib: NativeLibrary
+        ): ModulesProvider {
+            check(stdlib.manifestData.uniqueName == KONAN_STDLIB_NAME)
 
-            librariesToCommonize.getManifest(name).dependencies.forEach {
+            return object : NativeDistributionModulesProvider(listOf(stdlib)) {
-                moduleDependencies.addIfNotNull(platformModulesMap[it])
+                override fun loadModules(dependencies: Collection<ModuleDescriptor>): Map<String, ModuleDescriptor> {
-                moduleDependencies += dependenciesMap[it].orEmpty()
+                    check(dependencies.isEmpty())
+
+                    val stdlibModule = NativeFactories.DefaultDeserializedDescriptorFactory.createDescriptorAndNewBuiltIns(
+                        library = stdlib.library,
+                        languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT,
+                        storageManager = storageManager,
+                        packageAccessHandler = null
+                    ).apply {
+                        setDependencies(listOf(this))
+                    }
+
+                    return mapOf(KONAN_STDLIB_NAME to stdlibModule)
+                }
             }
-
-            moduleDependencies += forwardDeclarations
-
-            module.setDependencies(moduleDependencies)
         }
 
-        return platformModulesMap
+        fun platformLibraries(
+            storageManager: StorageManager,
+            librariesToCommonize: NativeLibrariesToCommonize
+        ): ModulesProvider = object : NativeDistributionModulesProvider(librariesToCommonize.libraries) {
+            override fun loadModules(dependencies: Collection<ModuleDescriptor>): Map<String, ModuleDescriptor> {
+                check(dependencies.isNotEmpty()) { "At least Kotlin/Native stdlib should be provided" }
+
+                val dependenciesMap = mutableMapOf<String, MutableList<ModuleDescriptorImpl>>()
+                dependencies.forEach { dependency ->
+                    val name = dependency.name.strip()
+                    dependenciesMap.getOrPut(name) { mutableListOf() } += dependency as ModuleDescriptorImpl
+                }
+
+                val builtIns = dependencies.first().builtIns
+
+                val platformModulesMap = libraryMap.mapValues { (_, library) ->
+                    NativeFactories.DefaultDeserializedDescriptorFactory.createDescriptorOptionalBuiltIns(
+                        library = library.library,
+                        languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT,
+                        storageManager = storageManager,
+                        builtIns = builtIns,
+                        packageAccessHandler = null,
+                        lookupTracker = LookupTracker.DO_NOTHING
+                    )
+                }
+
+                val forwardDeclarations = createKotlinNativeForwardDeclarationsModule(
+                    storageManager = storageManager,
+                    builtIns = builtIns
+                )
+
+                platformModulesMap.forEach { (name, module) ->
+                    val moduleDependencies = mutableListOf<ModuleDescriptorImpl>()
+                    moduleDependencies += module
+
+                    moduleInfoMap.getValue(name).dependencies.forEach {
+                        moduleDependencies.addIfNotNull(platformModulesMap[it])
+                        moduleDependencies += dependenciesMap[it].orEmpty()
+                    }
+
+                    moduleDependencies += forwardDeclarations
+
+                    module.setDependencies(moduleDependencies)
+                }
+
+                return platformModulesMap
+            }
+        }
     }
 }

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionStdlibProvider.kt

@@ -1,43 +0,0 @@
-/*
- * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.descriptors.commonizer.konan
-
-import org.jetbrains.kotlin.config.LanguageVersionSettingsImpl
-import org.jetbrains.kotlin.descriptors.ModuleDescriptor
-import org.jetbrains.kotlin.descriptors.commonizer.ModulesProvider
-import org.jetbrains.kotlin.descriptors.commonizer.ModulesProvider.ModuleInfo
-import org.jetbrains.kotlin.descriptors.commonizer.utils.NativeFactories
-import org.jetbrains.kotlin.konan.library.KONAN_STDLIB_NAME
-import org.jetbrains.kotlin.storage.StorageManager
-import java.io.File
-
-internal class NativeDistributionStdlibProvider(
-    private val storageManager: StorageManager,
-    private val stdlib: NativeLibrary
-) : ModulesProvider {
-    private val moduleInfo = ModuleInfo(
-        name = KONAN_STDLIB_NAME,
-        originalLocation = File(stdlib.library.libraryFile.absolutePath),
-        cInteropAttributes = null
-    )
-
-    override fun loadModuleInfos() = listOf(moduleInfo)
-
-    override fun loadModules(dependencies: Collection<ModuleDescriptor>): Map<String, ModuleDescriptor> {
-        check(dependencies.isEmpty())
-        return mapOf(KONAN_STDLIB_NAME to loadStdlibModule())
-    }
-
-    private fun loadStdlibModule() =
-        NativeFactories.DefaultDeserializedDescriptorFactory.createDescriptorAndNewBuiltIns(
-            library = stdlib.library,
-            languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT,
-            storageManager = storageManager,
-            packageAccessHandler = null
-        ).apply {
-            setDependencies(listOf(this))
-        }
-}

native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/AbstractCommonizationFromSourcesTest.kt

@@ -11,14 +11,11 @@ import org.jetbrains.kotlin.analyzer.ModuleInfo
 import org.jetbrains.kotlin.analyzer.common.CommonDependenciesContainer
 import org.jetbrains.kotlin.analyzer.common.CommonPlatformAnalyzerServices
 import org.jetbrains.kotlin.analyzer.common.CommonResolverForModuleFactory
-import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataMonolithicSerializer
-import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataVersion
 import org.jetbrains.kotlin.builtins.DefaultBuiltIns
 import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles
 import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
 import org.jetbrains.kotlin.config.CommonConfigurationKeys
 import org.jetbrains.kotlin.config.CompilerConfiguration
-import org.jetbrains.kotlin.config.LanguageVersionSettingsImpl
 import org.jetbrains.kotlin.config.languageVersionSettings
 import org.jetbrains.kotlin.descriptors.CallableMemberDescriptor
 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
@@ -239,7 +236,9 @@ private class AnalyzedModules(
 
             val commonizedModules: Map<CommonizerTarget, SerializedMetadata> =
                 createModules(sharedTarget, commonizedRoots, dependencies, parentDisposable)
-                    .mapValues { (_, moduleDescriptor) -> serializer.serializeModule(moduleDescriptor) }
+                    .mapValues { (_, moduleDescriptor) -> MockModulesProvider.SERIALIZER.serializeModule(moduleDescriptor) }
+
+
 
             return AnalyzedModules(originalModules, commonizedModules, dependeeModules)
         }
@@ -337,13 +336,6 @@ private class AnalyzedModules(
 
             return module
         }
-
-        private val serializer = KlibMetadataMonolithicSerializer(
-            languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT,
-            metadataVersion = KlibMetadataVersion.INSTANCE,
-            skipExpects = false,
-            project = null
-        )
     }
 }
 

native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/utils/mocks.kt

@@ -5,6 +5,9 @@
 
 package org.jetbrains.kotlin.descriptors.commonizer.utils
 
+import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataMonolithicSerializer
+import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataVersion
+import org.jetbrains.kotlin.config.LanguageVersionSettingsImpl
 import org.jetbrains.kotlin.descriptors.*
 import org.jetbrains.kotlin.descriptors.annotations.Annotations
 import org.jetbrains.kotlin.descriptors.commonizer.*
@@ -14,6 +17,7 @@ import org.jetbrains.kotlin.descriptors.commonizer.cir.factory.CirClassFactory
 import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.*
 import org.jetbrains.kotlin.descriptors.impl.AbstractTypeAliasDescriptor
 import org.jetbrains.kotlin.descriptors.impl.ClassDescriptorImpl
+import org.jetbrains.kotlin.library.SerializedMetadata
 import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.Name
@@ -30,6 +34,7 @@ import org.jetbrains.kotlin.storage.getValue
 internal fun mockEmptyModule(moduleName: String): ModuleDescriptor {
     val module = KotlinTestUtils.createEmptyModule(moduleName)
     module.initialize(PackageFragmentProvider.Empty)
+    module.setDependencies(module)
     return module
 }
 
@@ -155,7 +160,13 @@ internal class MockModulesProvider private constructor(
     private val moduleInfos = modules.keys.map { name -> fakeModuleInfo(name) }
 
     override fun loadModuleInfos() = moduleInfos
-    override fun loadModules(dependencies: Collection<ModuleDescriptor>): Map<String, ModuleDescriptor> = modules
+
+    override fun loadModuleMetadata(name: String): SerializedMetadata {
+        val module = modules[name] ?: error("No such module: $name")
+        return SERIALIZER.serializeModule(module)
+    }
+
+    override fun loadModules(dependencies: Collection<ModuleDescriptor>) = modules
 
     private fun fakeModuleInfo(name: String) = ModuleInfo(name, File("/tmp/commonizer/mocks/$name"), null)
 
@@ -174,6 +185,13 @@ internal class MockModulesProvider private constructor(
         fun create(module: ModuleDescriptor) = MockModulesProvider(
             mapOf(module.name.strip() to module)
         )
+
+        val SERIALIZER = KlibMetadataMonolithicSerializer(
+            languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT,
+            metadataVersion = KlibMetadataVersion.INSTANCE,
+            skipExpects = false,
+            project = null
+        )
     }
 }