[Commonizer] Add ModulesProvider.loadModuleMetadata() to load raw metadata

2021-02-04 15:06:26 +03:00
parent cb226e74a5
commit 6c92ea518b
6 changed files with 146 additions and 102 deletions
@@ -6,6 +6,7 @@
 package org.jetbrains.kotlin.descriptors.commonizer

 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
+import org.jetbrains.kotlin.library.SerializedMetadata
 import java.io.File

 class TargetProvider(
@@ -15,7 +16,7 @@ class TargetProvider(
 )

 interface ModulesProvider {
-    class ModuleInfo(
+    open class ModuleInfo(
        val name: String,
        val originalLocation: File,
        val cInteropAttributes: CInteropModuleAttributes?
@@ -26,6 +27,17 @@ interface ModulesProvider {
        val exportForwardDeclarations: Collection<String>
    )

+    /**
+     * Returns information about all modules that can be loaded by this [ModulesProvider] in the form of [ModuleInfo]s.
+     * This function is relatively light-weight and does not have significant impact on performance.
+     */
    fun loadModuleInfos(): Collection<ModuleInfo>
+
+    /**
+     * Loads metadata for the specified module.
+     */
+    fun loadModuleMetadata(name: String): SerializedMetadata
+
+    @Deprecated("To be replaced by loadModuleMetadata()")
    fun loadModules(dependencies: Collection<ModuleDescriptor>): Map<String, ModuleDescriptor>
 }
@@ -135,12 +135,12 @@ class NativeDistributionCommonizer(
                copyEndorsedLibs = copyEndorsedLibs,
                logProgress = ::logProgress
            )
-            dependeeModulesProvider = NativeDistributionStdlibProvider(storageManager, allLibraries.stdlib)
+            dependeeModulesProvider = NativeDistributionModulesProvider.forStandardLibrary(storageManager, allLibraries.stdlib)

            allLibraries.librariesByTargets.forEach { (target, librariesToCommonize) ->
                if (librariesToCommonize.libraries.isEmpty()) return@forEach

-                val modulesProvider = NativeDistributionModulesProvider(storageManager, librariesToCommonize)
+                val modulesProvider = NativeDistributionModulesProvider.platformLibraries(storageManager, librariesToCommonize)

                addTarget(
                    TargetProvider(
@@ -15,20 +15,36 @@ import org.jetbrains.kotlin.descriptors.commonizer.utils.createKotlinNativeForwa
 import org.jetbrains.kotlin.descriptors.commonizer.utils.strip
 import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
 import org.jetbrains.kotlin.incremental.components.LookupTracker
+import org.jetbrains.kotlin.konan.library.KONAN_STDLIB_NAME
+import org.jetbrains.kotlin.library.SerializedMetadata
+import org.jetbrains.kotlin.library.metadata.parseModuleHeader
 import org.jetbrains.kotlin.storage.StorageManager
 import org.jetbrains.kotlin.utils.addIfNotNull
 import java.io.File

-internal class NativeDistributionModulesProvider(
-    private val storageManager: StorageManager,
-    private val librariesToCommonize: NativeLibrariesToCommonize
-) : ModulesProvider {
-    private val moduleInfos by lazy {
-        librariesToCommonize.libraries.map { library ->
+internal abstract class NativeDistributionModulesProvider(libraries: Collection<NativeLibrary>) : ModulesProvider {
+    internal class NativeModuleInfo(
+        name: String,
+        originalLocation: File,
+        val dependencies: Set<String>,
+        cInteropAttributes: CInteropModuleAttributes?
+    ) : ModuleInfo(name, originalLocation, cInteropAttributes)
+
+    protected val libraryMap: Map<String, NativeLibrary>
+    protected val moduleInfoMap: Map<String, NativeModuleInfo>
+
+    init {
+        check(libraries.isNotEmpty()) { "No libraries supplied" }
+
+        val libraryMap = mutableMapOf<String, NativeLibrary>()
+        val moduleInfoMap = mutableMapOf<String, NativeModuleInfo>()
+
+        libraries.forEach { library ->
            val manifestData = library.manifestData

            val name = manifestData.uniqueName
            val location = File(library.library.libraryFile.path)
+            val dependencies = manifestData.dependencies.toSet()

            val cInteropAttributes = if (manifestData.isInterop) {
                val packageFqName = manifestData.packageFqName
@@ -38,56 +54,105 @@ internal class NativeDistributionModulesProvider(
                CInteropModuleAttributes(packageFqName, manifestData.exportForwardDeclarations)
            } else null

-            ModuleInfo(name, location, cInteropAttributes)
+            libraryMap.put(name, library)?.let { error("Duplicated libraries: $name") }
+            moduleInfoMap[name] = NativeModuleInfo(name, location, dependencies, cInteropAttributes)
        }
+
+        this.libraryMap = libraryMap
+        this.moduleInfoMap = moduleInfoMap
    }

-    override fun loadModuleInfos() = moduleInfos
+    final override fun loadModuleInfos(): Collection<ModuleInfo> = moduleInfoMap.values

-    override fun loadModules(dependencies: Collection<ModuleDescriptor>): Map<String, ModuleDescriptor> {
-        check(dependencies.isNotEmpty()) { "At least Kotlin/Native stdlib should be provided" }
+    final override fun loadModuleMetadata(name: String): SerializedMetadata {
+        val library = libraryMap[name]?.library ?: error("No such library: $name")

-        val dependenciesMap = mutableMapOf<String, MutableList<ModuleDescriptorImpl>>()
-        dependencies.forEach { dependency ->
-            val name = dependency.name.strip()
-            dependenciesMap.getOrPut(name) { mutableListOf() } += dependency as ModuleDescriptorImpl
+        val moduleHeader = library.moduleHeaderData
+        val fragmentNames = parseModuleHeader(moduleHeader).packageFragmentNameList.toSet()
+        val fragments = fragmentNames.map { fragmentName ->
+            val partNames = library.packageMetadataParts(fragmentName)
+            partNames.map { partName -> library.packageMetadata(fragmentName, partName) }
        }

-        val builtIns = dependencies.first().builtIns
-
-        val platformModulesMap = librariesToCommonize.libraries.associate { library ->
-            val name = library.manifestData.uniqueName
-            val module = NativeFactories.DefaultDeserializedDescriptorFactory.createDescriptorOptionalBuiltIns(
-                library = library.library,
-                languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT,
-                storageManager = storageManager,
-                builtIns = builtIns,
-                packageAccessHandler = null,
-                lookupTracker = LookupTracker.DO_NOTHING
-            )
-
-            name to module
-        }
-
-        val forwardDeclarations = createKotlinNativeForwardDeclarationsModule(
-            storageManager = storageManager,
-            builtIns = builtIns
+        return SerializedMetadata(
+            module = moduleHeader,
+            fragments = fragments,
+            fragmentNames = fragmentNames.toList()
        )
+    }

-        platformModulesMap.forEach { (name, module) ->
-            val moduleDependencies = mutableListOf<ModuleDescriptorImpl>()
-            moduleDependencies += module
+    companion object {
+        fun forStandardLibrary(
+            storageManager: StorageManager,
+            stdlib: NativeLibrary
+        ): ModulesProvider {
+            check(stdlib.manifestData.uniqueName == KONAN_STDLIB_NAME)

-            librariesToCommonize.getManifest(name).dependencies.forEach {
-                moduleDependencies.addIfNotNull(platformModulesMap[it])
-                moduleDependencies += dependenciesMap[it].orEmpty()
+            return object : NativeDistributionModulesProvider(listOf(stdlib)) {
+                override fun loadModules(dependencies: Collection<ModuleDescriptor>): Map<String, ModuleDescriptor> {
+                    check(dependencies.isEmpty())
+
+                    val stdlibModule = NativeFactories.DefaultDeserializedDescriptorFactory.createDescriptorAndNewBuiltIns(
+                        library = stdlib.library,
+                        languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT,
+                        storageManager = storageManager,
+                        packageAccessHandler = null
+                    ).apply {
+                        setDependencies(listOf(this))
+                    }
+
+                    return mapOf(KONAN_STDLIB_NAME to stdlibModule)
+                }
            }
-
-            moduleDependencies += forwardDeclarations
-
-            module.setDependencies(moduleDependencies)
        }

-        return platformModulesMap
+        fun platformLibraries(
+            storageManager: StorageManager,
+            librariesToCommonize: NativeLibrariesToCommonize
+        ): ModulesProvider = object : NativeDistributionModulesProvider(librariesToCommonize.libraries) {
+            override fun loadModules(dependencies: Collection<ModuleDescriptor>): Map<String, ModuleDescriptor> {
+                check(dependencies.isNotEmpty()) { "At least Kotlin/Native stdlib should be provided" }
+
+                val dependenciesMap = mutableMapOf<String, MutableList<ModuleDescriptorImpl>>()
+                dependencies.forEach { dependency ->
+                    val name = dependency.name.strip()
+                    dependenciesMap.getOrPut(name) { mutableListOf() } += dependency as ModuleDescriptorImpl
+                }
+
+                val builtIns = dependencies.first().builtIns
+
+                val platformModulesMap = libraryMap.mapValues { (_, library) ->
+                    NativeFactories.DefaultDeserializedDescriptorFactory.createDescriptorOptionalBuiltIns(
+                        library = library.library,
+                        languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT,
+                        storageManager = storageManager,
+                        builtIns = builtIns,
+                        packageAccessHandler = null,
+                        lookupTracker = LookupTracker.DO_NOTHING
+                    )
+                }
+
+                val forwardDeclarations = createKotlinNativeForwardDeclarationsModule(
+                    storageManager = storageManager,
+                    builtIns = builtIns
+                )
+
+                platformModulesMap.forEach { (name, module) ->
+                    val moduleDependencies = mutableListOf<ModuleDescriptorImpl>()
+                    moduleDependencies += module
+
+                    moduleInfoMap.getValue(name).dependencies.forEach {
+                        moduleDependencies.addIfNotNull(platformModulesMap[it])
+                        moduleDependencies += dependenciesMap[it].orEmpty()
+                    }
+
+                    moduleDependencies += forwardDeclarations
+
+                    module.setDependencies(moduleDependencies)
+                }
+
+                return platformModulesMap
+            }
+        }
    }
 }
@@ -1,43 +0,0 @@
-/*
- * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.descriptors.commonizer.konan
-
-import org.jetbrains.kotlin.config.LanguageVersionSettingsImpl
-import org.jetbrains.kotlin.descriptors.ModuleDescriptor
-import org.jetbrains.kotlin.descriptors.commonizer.ModulesProvider
-import org.jetbrains.kotlin.descriptors.commonizer.ModulesProvider.ModuleInfo
-import org.jetbrains.kotlin.descriptors.commonizer.utils.NativeFactories
-import org.jetbrains.kotlin.konan.library.KONAN_STDLIB_NAME
-import org.jetbrains.kotlin.storage.StorageManager
-import java.io.File
-
-internal class NativeDistributionStdlibProvider(
-    private val storageManager: StorageManager,
-    private val stdlib: NativeLibrary
-) : ModulesProvider {
-    private val moduleInfo = ModuleInfo(
-        name = KONAN_STDLIB_NAME,
-        originalLocation = File(stdlib.library.libraryFile.absolutePath),
-        cInteropAttributes = null
-    )
-
-    override fun loadModuleInfos() = listOf(moduleInfo)
-
-    override fun loadModules(dependencies: Collection<ModuleDescriptor>): Map<String, ModuleDescriptor> {
-        check(dependencies.isEmpty())
-        return mapOf(KONAN_STDLIB_NAME to loadStdlibModule())
-    }
-
-    private fun loadStdlibModule() =
-        NativeFactories.DefaultDeserializedDescriptorFactory.createDescriptorAndNewBuiltIns(
-            library = stdlib.library,
-            languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT,
-            storageManager = storageManager,
-            packageAccessHandler = null
-        ).apply {
-            setDependencies(listOf(this))
-        }
-}
@@ -11,14 +11,11 @@ import org.jetbrains.kotlin.analyzer.ModuleInfo
 import org.jetbrains.kotlin.analyzer.common.CommonDependenciesContainer
 import org.jetbrains.kotlin.analyzer.common.CommonPlatformAnalyzerServices
 import org.jetbrains.kotlin.analyzer.common.CommonResolverForModuleFactory
-import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataMonolithicSerializer
-import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataVersion
 import org.jetbrains.kotlin.builtins.DefaultBuiltIns
 import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles
 import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
 import org.jetbrains.kotlin.config.CommonConfigurationKeys
 import org.jetbrains.kotlin.config.CompilerConfiguration
-import org.jetbrains.kotlin.config.LanguageVersionSettingsImpl
 import org.jetbrains.kotlin.config.languageVersionSettings
 import org.jetbrains.kotlin.descriptors.CallableMemberDescriptor
 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
@@ -239,7 +236,9 @@ private class AnalyzedModules(

            val commonizedModules: Map<CommonizerTarget, SerializedMetadata> =
                createModules(sharedTarget, commonizedRoots, dependencies, parentDisposable)
-                    .mapValues { (_, moduleDescriptor) -> serializer.serializeModule(moduleDescriptor) }
+                    .mapValues { (_, moduleDescriptor) -> MockModulesProvider.SERIALIZER.serializeModule(moduleDescriptor) }
+
+

            return AnalyzedModules(originalModules, commonizedModules, dependeeModules)
        }
@@ -337,13 +336,6 @@ private class AnalyzedModules(

            return module
        }
-
-        private val serializer = KlibMetadataMonolithicSerializer(
-            languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT,
-            metadataVersion = KlibMetadataVersion.INSTANCE,
-            skipExpects = false,
-            project = null
-        )
    }
 }

@@ -5,6 +5,9 @@

 package org.jetbrains.kotlin.descriptors.commonizer.utils

+import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataMonolithicSerializer
+import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataVersion
+import org.jetbrains.kotlin.config.LanguageVersionSettingsImpl
 import org.jetbrains.kotlin.descriptors.*
 import org.jetbrains.kotlin.descriptors.annotations.Annotations
 import org.jetbrains.kotlin.descriptors.commonizer.*
@@ -14,6 +17,7 @@ import org.jetbrains.kotlin.descriptors.commonizer.cir.factory.CirClassFactory
 import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.*
 import org.jetbrains.kotlin.descriptors.impl.AbstractTypeAliasDescriptor
 import org.jetbrains.kotlin.descriptors.impl.ClassDescriptorImpl
+import org.jetbrains.kotlin.library.SerializedMetadata
 import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.Name
@@ -30,6 +34,7 @@ import org.jetbrains.kotlin.storage.getValue
 internal fun mockEmptyModule(moduleName: String): ModuleDescriptor {
    val module = KotlinTestUtils.createEmptyModule(moduleName)
    module.initialize(PackageFragmentProvider.Empty)
+    module.setDependencies(module)
    return module
 }

@@ -155,7 +160,13 @@ internal class MockModulesProvider private constructor(
    private val moduleInfos = modules.keys.map { name -> fakeModuleInfo(name) }

    override fun loadModuleInfos() = moduleInfos
-    override fun loadModules(dependencies: Collection<ModuleDescriptor>): Map<String, ModuleDescriptor> = modules
+
+    override fun loadModuleMetadata(name: String): SerializedMetadata {
+        val module = modules[name] ?: error("No such module: $name")
+        return SERIALIZER.serializeModule(module)
+    }
+
+    override fun loadModules(dependencies: Collection<ModuleDescriptor>) = modules

    private fun fakeModuleInfo(name: String) = ModuleInfo(name, File("/tmp/commonizer/mocks/$name"), null)

@@ -174,6 +185,13 @@ internal class MockModulesProvider private constructor(
        fun create(module: ModuleDescriptor) = MockModulesProvider(
            mapOf(module.name.strip() to module)
        )
+
+        val SERIALIZER = KlibMetadataMonolithicSerializer(
+            languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT,
+            metadataVersion = KlibMetadataVersion.INSTANCE,
+            skipExpects = false,
+            project = null
+        )
    }
 }