[Commonizer] Integrate metadata builder with the commonizer environment

2020-10-19 12:31:42 +03:00
parent 2bcaf1fa63
commit d610837caf
7 changed files with 77 additions and 49 deletions
@@ -24,6 +24,9 @@ class CommonizerParameters(
            field = value
        }

+    // To be removed. Used only for tests now.
+    internal var generateDescriptors: Boolean = false
+
    fun addTarget(targetProvider: TargetProvider): CommonizerParameters {
        require(targetProvider.target !in _targetProviders) { "Target ${targetProvider.target} is already added" }
        _targetProviders[targetProvider.target] = targetProvider
@@ -6,6 +6,7 @@
 package org.jetbrains.kotlin.descriptors.commonizer

 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
+import org.jetbrains.kotlin.library.SerializedMetadata
 import java.io.File

 sealed class CommonizerResult {
@@ -21,5 +22,15 @@ sealed class CommonizerResult {

 sealed class ModuleResult {
    class Missing(val originalLocation: File) : ModuleResult()
-    class Commonized(val module: ModuleDescriptor) : ModuleResult()
+
+    class Commonized(
+        @Deprecated("To be removed. Used only for tests now.")
+        internal val module: ModuleDescriptor?,
+        val metadata: LibraryMetadata
+    ) : ModuleResult()
 }
+
+class LibraryMetadata(
+    val libraryName: String,
+    val metadata: SerializedMetadata
+)
@@ -218,7 +218,10 @@ class LazyClassifierLookupTable(lazyModules: Map<String, List<ModuleDescriptor>>
 fun CirRootNode.createGlobalBuilderComponents(
    storageManager: StorageManager,
    parameters: CommonizerParameters
-): GlobalDeclarationsBuilderComponents {
+): GlobalDeclarationsBuilderComponents? {
+    if (!parameters.generateDescriptors)
+        return null
+
    val cache = DeclarationsBuilderCache(dimension)

    val lazyCommonDependeeModules = storageManager.createLazyValue {
@@ -5,12 +5,19 @@

 package org.jetbrains.kotlin.descriptors.commonizer

+import kotlinx.metadata.klib.ChunkedKlibModuleFragmentWriteStrategy
+import org.jetbrains.kotlin.descriptors.ModuleDescriptor
 import org.jetbrains.kotlin.descriptors.commonizer.builder.DeclarationsBuilderVisitor1
 import org.jetbrains.kotlin.descriptors.commonizer.builder.DeclarationsBuilderVisitor2
 import org.jetbrains.kotlin.descriptors.commonizer.builder.createGlobalBuilderComponents
 import org.jetbrains.kotlin.descriptors.commonizer.core.CommonizationVisitor
 import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.*
+import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.CirNode.Companion.dimension
 import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.CirTreeMerger.CirTreeMergeResult
+import org.jetbrains.kotlin.descriptors.commonizer.metadata.MetadataBuilder
+import org.jetbrains.kotlin.descriptors.commonizer.utils.strip
+import org.jetbrains.kotlin.library.SerializedMetadata
+import org.jetbrains.kotlin.serialization.konan.impl.KlibResolvedModuleDescriptorsFactoryImpl.Companion.FORWARD_DECLARATIONS_MODULE_NAME
 import org.jetbrains.kotlin.storage.LockBasedStorageManager
 import org.jetbrains.kotlin.storage.StorageManager

@@ -24,16 +31,46 @@ fun runCommonization(parameters: CommonizerParameters): CommonizerResult {
    val mergedTree = mergeResult.root

    // build resulting descriptors:
-    val components = mergedTree.createGlobalBuilderComponents(storageManager, parameters)
-    mergedTree.accept(DeclarationsBuilderVisitor1(components), emptyList())
-    mergedTree.accept(DeclarationsBuilderVisitor2(components), emptyList())
-
    val modulesByTargets = LinkedHashMap<CommonizerTarget, Collection<ModuleResult>>() // use linked hash map to preserve order
-    components.targetComponents.forEach { component ->
-        val target = component.target
-        check(target !in modulesByTargets)
+    val klibFragmentWriteStrategy = ChunkedKlibModuleFragmentWriteStrategy()

-        val commonizedModules: List<ModuleResult.Commonized> = components.cache.getAllModules(component.index).map(ModuleResult::Commonized)
+    // optional part for generating descriptors: begin
+    val components = mergedTree.createGlobalBuilderComponents(storageManager, parameters)
+    if (components != null) {
+        mergedTree.accept(DeclarationsBuilderVisitor1(components), emptyList())
+        mergedTree.accept(DeclarationsBuilderVisitor2(components), emptyList())
+    }
+    // optional part for generating descriptors: end
+
+    for (targetIndex in 0 until mergedTree.dimension) {
+        val (target, metadataModules) = MetadataBuilder.build(mergedTree, targetIndex)
+
+        // optional part for generating descriptors: begin
+        val moduleDescriptors: Map<String, ModuleDescriptor>? = components?.targetComponents?.get(targetIndex)?.let { component ->
+            check(component.target == target)
+            check(component.index == targetIndex)
+
+            components.cache.getAllModules(targetIndex)
+                .filter { it.name != FORWARD_DECLARATIONS_MODULE_NAME }
+                .associateBy { it.name.strip() }
+        }
+        // optional part for generating descriptors: end
+
+        val commonizedModules: List<ModuleResult.Commonized> = metadataModules.map { metadataModule ->
+            val libraryName = metadataModule.name
+            val serializedMetadata = with(metadataModule.write(klibFragmentWriteStrategy)) {
+                SerializedMetadata(header, fragments, fragmentNames)
+            }
+
+            val libraryMetadata = LibraryMetadata(libraryName, serializedMetadata)
+
+            // optional part for generating descriptors: begin
+            val moduleDescriptor = moduleDescriptors?.get(libraryName)
+            // optional part for generating descriptors: end
+
+            ModuleResult.Commonized(moduleDescriptor, libraryMetadata)
+        }
+        parameters.progressLogger?.invoke("Built metadata for target [$target]")

        val missingModules: List<ModuleResult.Missing> = if (target is LeafTarget)
            mergeResult.missingModuleInfos.getValue(target).map { ModuleResult.Missing(it.originalLocation) }
@@ -42,8 +79,6 @@ fun runCommonization(parameters: CommonizerParameters): CommonizerResult {
        modulesByTargets[target] = commonizedModules + missingModules
    }

-    parameters.progressLogger?.invoke("Prepared new descriptors")
-
    return CommonizerResult.Done(modulesByTargets)
 }

@@ -5,19 +5,15 @@

 package org.jetbrains.kotlin.descriptors.commonizer.konan

-import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataMonolithicSerializer
 import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataVersion
 import org.jetbrains.kotlin.backend.common.serialization.metadata.metadataVersion
 import org.jetbrains.kotlin.builtins.konan.KonanBuiltIns
-import org.jetbrains.kotlin.config.LanguageVersionSettingsImpl
-import org.jetbrains.kotlin.descriptors.ModuleDescriptor
 import org.jetbrains.kotlin.descriptors.commonizer.*
 import org.jetbrains.kotlin.descriptors.commonizer.konan.NativeDistributionCommonizer.StatsType.*
 import org.jetbrains.kotlin.descriptors.commonizer.stats.AggregatedStatsCollector
 import org.jetbrains.kotlin.descriptors.commonizer.stats.FileStatsOutput
 import org.jetbrains.kotlin.descriptors.commonizer.stats.RawStatsCollector
 import org.jetbrains.kotlin.descriptors.commonizer.utils.ResettableClockMark
-import org.jetbrains.kotlin.descriptors.konan.NATIVE_STDLIB_MODULE_NAME
 import org.jetbrains.kotlin.konan.library.*
 import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.library.KotlinLibrary
@@ -28,8 +24,6 @@ import org.jetbrains.kotlin.library.impl.BuiltInsPlatform
 import org.jetbrains.kotlin.library.impl.KotlinLibraryLayoutForWriter
 import org.jetbrains.kotlin.library.impl.KotlinLibraryWriterImpl
 import org.jetbrains.kotlin.library.resolveSingleFileKlib
-import org.jetbrains.kotlin.name.Name
-import org.jetbrains.kotlin.serialization.konan.impl.KlibResolvedModuleDescriptorsFactoryImpl
 import org.jetbrains.kotlin.storage.LockBasedStorageManager
 import org.jetbrains.kotlin.util.Logger
 import java.io.File
@@ -186,13 +180,6 @@ class NativeDistributionCommonizer(
            }

            is CommonizerResult.Done -> {
-                val serializer = KlibMetadataMonolithicSerializer(
-                    languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT,
-                    metadataVersion = KlibMetadataVersion.INSTANCE,
-                    skipExpects = false,
-                    project = null
-                )
-
                // 'targetsToCopy' are some targets with empty set of platform libraries
                val targetsToCopy = originalLibraries.librariesByTargets.keys - result.leafTargets
                if (targetsToCopy.isNotEmpty()) {
@@ -206,7 +193,7 @@ class NativeDistributionCommonizer(
                val targetsToSerialize = result.leafTargets + result.sharedTarget
                targetsToSerialize.forEach { target ->
                    val moduleResults: Collection<ModuleResult> = result.modulesByTargets.getValue(target)
-                    val newModules: Collection<ModuleDescriptor> = moduleResults.mapNotNull { (it as? ModuleResult.Commonized)?.module }
+                    val newLibraries: Collection<LibraryMetadata> = moduleResults.mapNotNull { (it as? ModuleResult.Commonized)?.metadata }
                    val missingModuleLocations: List<File> =
                        moduleResults.mapNotNull { (it as? ModuleResult.Missing)?.originalLocation }

@@ -224,7 +211,7 @@ class NativeDistributionCommonizer(
                    }

                    val targetName = leafTargetNames.joinToString { if (it == starredTarget) "$it(*)" else it }
-                    serializeTarget(target, targetName, newModules, missingModuleLocations, manifestProvider, serializer)
+                    serializeTarget(target, targetName, newLibraries, missingModuleLocations, manifestProvider)
                }
            }
        }
@@ -268,27 +255,20 @@ class NativeDistributionCommonizer(
    private fun serializeTarget(
        target: CommonizerTarget,
        targetName: String,
-        newModules: Collection<ModuleDescriptor>,
+        newLibraries: Collection<LibraryMetadata>,
        missingModuleLocations: List<File>,
-        manifestProvider: NativeManifestDataProvider,
-        serializer: KlibMetadataMonolithicSerializer
+        manifestProvider: NativeManifestDataProvider
    ) {
        val librariesDestination = target.librariesDestination
        librariesDestination.mkdirs() // always create an empty directory even if there is nothing to copy

-        for (newModule in newModules) {
-            val libraryName = newModule.name
+        for (newLibrary in newLibraries) {
+            val libraryName = newLibrary.libraryName

-            if (!shouldBeSerialized(libraryName))
-                continue
+            val manifestData = manifestProvider.getManifest(libraryName)
+            val libraryDestination = librariesDestination.resolve(libraryName)

-            val metadata = serializer.serializeModule(newModule)
-            val plainName = libraryName.asString().removePrefix("<").removeSuffix(">")
-
-            val manifestData = manifestProvider.getManifest(plainName)
-            val libraryDestination = librariesDestination.resolve(plainName)
-
-            writeLibrary(metadata, manifestData, libraryDestination)
+            writeLibrary(newLibrary.metadata, manifestData, libraryDestination)
        }

        for (missingModuleLocation in missingModuleLocations) {
@@ -330,9 +310,4 @@ class NativeDistributionCommonizer(
            is LeafTarget -> destination.resolve(KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR).resolve(name)
            is SharedTarget -> destination.resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR)
        }
-
-    private companion object {
-        fun shouldBeSerialized(libraryName: Name) =
-            libraryName != NATIVE_STDLIB_MODULE_NAME && libraryName != KlibResolvedModuleDescriptorsFactoryImpl.FORWARD_DECLARATIONS_MODULE_NAME
-    }
 }
@@ -76,7 +76,7 @@ abstract class AbstractCommonizationFromSourcesTest : KtUsefulTestCase() {

        val sharedModuleAsExpected: ModuleDescriptor = analyzedModules.commonizedModules.getValue(sharedTarget)
        val sharedModuleByCommonizer: ModuleDescriptor =
-            (result.modulesByTargets.getValue(sharedTarget).single() as ModuleResult.Commonized).module
+            (result.modulesByTargets.getValue(sharedTarget).single() as ModuleResult.Commonized).module!!

        assertValidModule(sharedModuleAsExpected)
        assertValidModule(sharedModuleByCommonizer)
@@ -88,7 +88,7 @@ abstract class AbstractCommonizationFromSourcesTest : KtUsefulTestCase() {
        for (leafTarget in leafTargets) {
            val leafTargetModuleAsExpected: ModuleDescriptor = analyzedModules.commonizedModules.getValue(leafTarget)
            val leafTargetModuleByCommonizer: ModuleDescriptor =
-                (result.modulesByTargets.getValue(leafTarget).single() as ModuleResult.Commonized).module
+                (result.modulesByTargets.getValue(leafTarget).single() as ModuleResult.Commonized).module!!

            assertValidModule(leafTargetModuleAsExpected)
            assertValidModule(leafTargetModuleByCommonizer)
@@ -205,6 +205,7 @@ private class AnalyzedModules(

    fun toCommonizationParameters(): CommonizerParameters {
        val parameters = CommonizerParameters()
+        parameters.generateDescriptors = true

        leafTargets.forEach { leafTarget ->
            val originalModule = originalModules.getValue(leafTarget)
@@ -121,7 +121,7 @@ class CommonizerFacadeTest {
            actualModuleResults.forEach { moduleResult ->
                when (moduleResult) {
                    is ModuleResult.Commonized -> {
-                        actualCommonizedModuleNames += moduleResult.module.name.asString().removeSurrounding("<", ">")
+                        actualCommonizedModuleNames += moduleResult.metadata.libraryName
                    }
                    is ModuleResult.Missing -> {
                        actualMissingModuleNames += moduleResult.originalLocation.name