[Commonizer] Integrate metadata builder with the commonizer environment

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizerParameters.kt

@@ -24,6 +24,9 @@ class CommonizerParameters(
             field = value
         }
 
+    // To be removed. Used only for tests now.
+    internal var generateDescriptors: Boolean = false
+
     fun addTarget(targetProvider: TargetProvider): CommonizerParameters {
         require(targetProvider.target !in _targetProviders) { "Target ${targetProvider.target} is already added" }
         _targetProviders[targetProvider.target] = targetProvider

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/CommonizerResult.kt

@@ -6,6 +6,7 @@
 package org.jetbrains.kotlin.descriptors.commonizer
 
 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
+import org.jetbrains.kotlin.library.SerializedMetadata
 import java.io.File
 
 sealed class CommonizerResult {
@@ -21,5 +22,15 @@ sealed class CommonizerResult {
 
 sealed class ModuleResult {
     class Missing(val originalLocation: File) : ModuleResult()
-    class Commonized(val module: ModuleDescriptor) : ModuleResult()
+
+    class Commonized(
+        @Deprecated("To be removed. Used only for tests now.")
+        internal val module: ModuleDescriptor?,
+        val metadata: LibraryMetadata
+    ) : ModuleResult()
 }
+
+class LibraryMetadata(
+    val libraryName: String,
+    val metadata: SerializedMetadata
+)

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/context.kt

@@ -218,7 +218,10 @@ class LazyClassifierLookupTable(lazyModules: Map<String, List<ModuleDescriptor>>
 fun CirRootNode.createGlobalBuilderComponents(
     storageManager: StorageManager,
     parameters: CommonizerParameters
-): GlobalDeclarationsBuilderComponents {
+): GlobalDeclarationsBuilderComponents? {
+    if (!parameters.generateDescriptors)
+        return null
+
     val cache = DeclarationsBuilderCache(dimension)
 
     val lazyCommonDependeeModules = storageManager.createLazyValue {

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/facade.kt

@@ -5,12 +5,19 @@
 
 package org.jetbrains.kotlin.descriptors.commonizer
 
+import kotlinx.metadata.klib.ChunkedKlibModuleFragmentWriteStrategy
+import org.jetbrains.kotlin.descriptors.ModuleDescriptor
 import org.jetbrains.kotlin.descriptors.commonizer.builder.DeclarationsBuilderVisitor1
 import org.jetbrains.kotlin.descriptors.commonizer.builder.DeclarationsBuilderVisitor2
 import org.jetbrains.kotlin.descriptors.commonizer.builder.createGlobalBuilderComponents
 import org.jetbrains.kotlin.descriptors.commonizer.core.CommonizationVisitor
 import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.*
+import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.CirNode.Companion.dimension
 import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.CirTreeMerger.CirTreeMergeResult
+import org.jetbrains.kotlin.descriptors.commonizer.metadata.MetadataBuilder
+import org.jetbrains.kotlin.descriptors.commonizer.utils.strip
+import org.jetbrains.kotlin.library.SerializedMetadata
+import org.jetbrains.kotlin.serialization.konan.impl.KlibResolvedModuleDescriptorsFactoryImpl.Companion.FORWARD_DECLARATIONS_MODULE_NAME
 import org.jetbrains.kotlin.storage.LockBasedStorageManager
 import org.jetbrains.kotlin.storage.StorageManager
 
@@ -24,16 +31,46 @@ fun runCommonization(parameters: CommonizerParameters): CommonizerResult {
     val mergedTree = mergeResult.root
 
     // build resulting descriptors:
-    val components = mergedTree.createGlobalBuilderComponents(storageManager, parameters)
-    mergedTree.accept(DeclarationsBuilderVisitor1(components), emptyList())
-    mergedTree.accept(DeclarationsBuilderVisitor2(components), emptyList())
-
     val modulesByTargets = LinkedHashMap<CommonizerTarget, Collection<ModuleResult>>() // use linked hash map to preserve order
-    components.targetComponents.forEach { component ->
+    val klibFragmentWriteStrategy = ChunkedKlibModuleFragmentWriteStrategy()
-        val target = component.target
-        check(target !in modulesByTargets)
 
-        val commonizedModules: List<ModuleResult.Commonized> = components.cache.getAllModules(component.index).map(ModuleResult::Commonized)
+    // optional part for generating descriptors: begin
+    val components = mergedTree.createGlobalBuilderComponents(storageManager, parameters)
+    if (components != null) {
+        mergedTree.accept(DeclarationsBuilderVisitor1(components), emptyList())
+        mergedTree.accept(DeclarationsBuilderVisitor2(components), emptyList())
+    }
+    // optional part for generating descriptors: end
+
+    for (targetIndex in 0 until mergedTree.dimension) {
+        val (target, metadataModules) = MetadataBuilder.build(mergedTree, targetIndex)
+
+        // optional part for generating descriptors: begin
+        val moduleDescriptors: Map<String, ModuleDescriptor>? = components?.targetComponents?.get(targetIndex)?.let { component ->
+            check(component.target == target)
+            check(component.index == targetIndex)
+
+            components.cache.getAllModules(targetIndex)
+                .filter { it.name != FORWARD_DECLARATIONS_MODULE_NAME }
+                .associateBy { it.name.strip() }
+        }
+        // optional part for generating descriptors: end
+
+        val commonizedModules: List<ModuleResult.Commonized> = metadataModules.map { metadataModule ->
+            val libraryName = metadataModule.name
+            val serializedMetadata = with(metadataModule.write(klibFragmentWriteStrategy)) {
+                SerializedMetadata(header, fragments, fragmentNames)
+            }
+
+            val libraryMetadata = LibraryMetadata(libraryName, serializedMetadata)
+
+            // optional part for generating descriptors: begin
+            val moduleDescriptor = moduleDescriptors?.get(libraryName)
+            // optional part for generating descriptors: end
+
+            ModuleResult.Commonized(moduleDescriptor, libraryMetadata)
+        }
+        parameters.progressLogger?.invoke("Built metadata for target [$target]")
 
         val missingModules: List<ModuleResult.Missing> = if (target is LeafTarget)
             mergeResult.missingModuleInfos.getValue(target).map { ModuleResult.Missing(it.originalLocation) }
@@ -42,8 +79,6 @@ fun runCommonization(parameters: CommonizerParameters): CommonizerResult {
         modulesByTargets[target] = commonizedModules + missingModules
     }
 
-    parameters.progressLogger?.invoke("Prepared new descriptors")
-
     return CommonizerResult.Done(modulesByTargets)
 }
 

native/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/konan/NativeDistributionCommonizer.kt

@@ -5,19 +5,15 @@
 
 package org.jetbrains.kotlin.descriptors.commonizer.konan
 
-import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataMonolithicSerializer
 import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataVersion
 import org.jetbrains.kotlin.backend.common.serialization.metadata.metadataVersion
 import org.jetbrains.kotlin.builtins.konan.KonanBuiltIns
-import org.jetbrains.kotlin.config.LanguageVersionSettingsImpl
-import org.jetbrains.kotlin.descriptors.ModuleDescriptor
 import org.jetbrains.kotlin.descriptors.commonizer.*
 import org.jetbrains.kotlin.descriptors.commonizer.konan.NativeDistributionCommonizer.StatsType.*
 import org.jetbrains.kotlin.descriptors.commonizer.stats.AggregatedStatsCollector
 import org.jetbrains.kotlin.descriptors.commonizer.stats.FileStatsOutput
 import org.jetbrains.kotlin.descriptors.commonizer.stats.RawStatsCollector
 import org.jetbrains.kotlin.descriptors.commonizer.utils.ResettableClockMark
-import org.jetbrains.kotlin.descriptors.konan.NATIVE_STDLIB_MODULE_NAME
 import org.jetbrains.kotlin.konan.library.*
 import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.library.KotlinLibrary
@@ -28,8 +24,6 @@ import org.jetbrains.kotlin.library.impl.BuiltInsPlatform
 import org.jetbrains.kotlin.library.impl.KotlinLibraryLayoutForWriter
 import org.jetbrains.kotlin.library.impl.KotlinLibraryWriterImpl
 import org.jetbrains.kotlin.library.resolveSingleFileKlib
-import org.jetbrains.kotlin.name.Name
-import org.jetbrains.kotlin.serialization.konan.impl.KlibResolvedModuleDescriptorsFactoryImpl
 import org.jetbrains.kotlin.storage.LockBasedStorageManager
 import org.jetbrains.kotlin.util.Logger
 import java.io.File
@@ -186,13 +180,6 @@ class NativeDistributionCommonizer(
             }
 
             is CommonizerResult.Done -> {
-                val serializer = KlibMetadataMonolithicSerializer(
-                    languageVersionSettings = LanguageVersionSettingsImpl.DEFAULT,
-                    metadataVersion = KlibMetadataVersion.INSTANCE,
-                    skipExpects = false,
-                    project = null
-                )
-
                 // 'targetsToCopy' are some targets with empty set of platform libraries
                 val targetsToCopy = originalLibraries.librariesByTargets.keys - result.leafTargets
                 if (targetsToCopy.isNotEmpty()) {
@@ -206,7 +193,7 @@ class NativeDistributionCommonizer(
                 val targetsToSerialize = result.leafTargets + result.sharedTarget
                 targetsToSerialize.forEach { target ->
                     val moduleResults: Collection<ModuleResult> = result.modulesByTargets.getValue(target)
-                    val newModules: Collection<ModuleDescriptor> = moduleResults.mapNotNull { (it as? ModuleResult.Commonized)?.module }
+                    val newLibraries: Collection<LibraryMetadata> = moduleResults.mapNotNull { (it as? ModuleResult.Commonized)?.metadata }
                     val missingModuleLocations: List<File> =
                         moduleResults.mapNotNull { (it as? ModuleResult.Missing)?.originalLocation }
 
@@ -224,7 +211,7 @@ class NativeDistributionCommonizer(
                     }
 
                     val targetName = leafTargetNames.joinToString { if (it == starredTarget) "$it(*)" else it }
-                    serializeTarget(target, targetName, newModules, missingModuleLocations, manifestProvider, serializer)
+                    serializeTarget(target, targetName, newLibraries, missingModuleLocations, manifestProvider)
                 }
             }
         }
@@ -268,27 +255,20 @@ class NativeDistributionCommonizer(
     private fun serializeTarget(
         target: CommonizerTarget,
         targetName: String,
-        newModules: Collection<ModuleDescriptor>,
+        newLibraries: Collection<LibraryMetadata>,
         missingModuleLocations: List<File>,
-        manifestProvider: NativeManifestDataProvider,
+        manifestProvider: NativeManifestDataProvider
-        serializer: KlibMetadataMonolithicSerializer
     ) {
         val librariesDestination = target.librariesDestination
         librariesDestination.mkdirs() // always create an empty directory even if there is nothing to copy
 
-        for (newModule in newModules) {
+        for (newLibrary in newLibraries) {
-            val libraryName = newModule.name
+            val libraryName = newLibrary.libraryName
 
-            if (!shouldBeSerialized(libraryName))
+            val manifestData = manifestProvider.getManifest(libraryName)
-                continue
+            val libraryDestination = librariesDestination.resolve(libraryName)
 
-            val metadata = serializer.serializeModule(newModule)
+            writeLibrary(newLibrary.metadata, manifestData, libraryDestination)
-            val plainName = libraryName.asString().removePrefix("<").removeSuffix(">")
-
-            val manifestData = manifestProvider.getManifest(plainName)
-            val libraryDestination = librariesDestination.resolve(plainName)
-
-            writeLibrary(metadata, manifestData, libraryDestination)
         }
 
         for (missingModuleLocation in missingModuleLocations) {
@@ -330,9 +310,4 @@ class NativeDistributionCommonizer(
             is LeafTarget -> destination.resolve(KONAN_DISTRIBUTION_PLATFORM_LIBS_DIR).resolve(name)
             is SharedTarget -> destination.resolve(KONAN_DISTRIBUTION_COMMON_LIBS_DIR)
         }
-
-    private companion object {
-        fun shouldBeSerialized(libraryName: Name) =
-            libraryName != NATIVE_STDLIB_MODULE_NAME && libraryName != KlibResolvedModuleDescriptorsFactoryImpl.FORWARD_DECLARATIONS_MODULE_NAME
-    }
 }

native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/AbstractCommonizationFromSourcesTest.kt

@@ -76,7 +76,7 @@ abstract class AbstractCommonizationFromSourcesTest : KtUsefulTestCase() {
 
         val sharedModuleAsExpected: ModuleDescriptor = analyzedModules.commonizedModules.getValue(sharedTarget)
         val sharedModuleByCommonizer: ModuleDescriptor =
-            (result.modulesByTargets.getValue(sharedTarget).single() as ModuleResult.Commonized).module
+            (result.modulesByTargets.getValue(sharedTarget).single() as ModuleResult.Commonized).module!!
 
         assertValidModule(sharedModuleAsExpected)
         assertValidModule(sharedModuleByCommonizer)
@@ -88,7 +88,7 @@ abstract class AbstractCommonizationFromSourcesTest : KtUsefulTestCase() {
         for (leafTarget in leafTargets) {
             val leafTargetModuleAsExpected: ModuleDescriptor = analyzedModules.commonizedModules.getValue(leafTarget)
             val leafTargetModuleByCommonizer: ModuleDescriptor =
-                (result.modulesByTargets.getValue(leafTarget).single() as ModuleResult.Commonized).module
+                (result.modulesByTargets.getValue(leafTarget).single() as ModuleResult.Commonized).module!!
 
             assertValidModule(leafTargetModuleAsExpected)
             assertValidModule(leafTargetModuleByCommonizer)
@@ -205,6 +205,7 @@ private class AnalyzedModules(
 
     fun toCommonizationParameters(): CommonizerParameters {
         val parameters = CommonizerParameters()
+        parameters.generateDescriptors = true
 
         leafTargets.forEach { leafTarget ->
             val originalModule = originalModules.getValue(leafTarget)

native/commonizer/tests/org/jetbrains/kotlin/descriptors/commonizer/CommonizerFacadeTest.kt

@@ -121,7 +121,7 @@ class CommonizerFacadeTest {
             actualModuleResults.forEach { moduleResult ->
                 when (moduleResult) {
                     is ModuleResult.Commonized -> {
-                        actualCommonizedModuleNames += moduleResult.module.name.asString().removeSurrounding("<", ">")
+                        actualCommonizedModuleNames += moduleResult.metadata.libraryName
                     }
                     is ModuleResult.Missing -> {
                         actualMissingModuleNames += moduleResult.originalLocation.name