[klib] Deduplicate KLIB serialization logic in K/JS and K/N

Introduce the new `serializeModuleIntoKlib` function that all KLIB-based backends can use to serialize a module to a KLIB. ^KT-64392 Fixed
2024-01-19 19:19:08 +01:00
parent c335cd0b1e
commit 93a1ceae75
8 changed files with 348 additions and 310 deletions
@@ -1,10 +1,8 @@
 package org.jetbrains.kotlin.backend.konan

-import org.jetbrains.kotlin.ir.KtDiagnosticReporterWithImplicitIrBasedContext
 import org.jetbrains.kotlin.KtSourceFile
 import org.jetbrains.kotlin.backend.common.serialization.CompatibilityMode
-import org.jetbrains.kotlin.backend.common.serialization.metadata.makeSerializedKlibMetadata
-import org.jetbrains.kotlin.backend.common.serialization.metadata.serializeKlibHeader
+import org.jetbrains.kotlin.backend.common.serialization.serializeModuleIntoKlib
 import org.jetbrains.kotlin.backend.konan.driver.PhaseContext
 import org.jetbrains.kotlin.backend.konan.driver.phases.Fir2IrOutput
 import org.jetbrains.kotlin.backend.konan.driver.phases.FirOutput
@@ -14,9 +12,7 @@ import org.jetbrains.kotlin.backend.konan.serialization.KonanIrModuleSerializer
 import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
 import org.jetbrains.kotlin.cli.common.fir.reportToMessageCollector
 import org.jetbrains.kotlin.config.CommonConfigurationKeys
-import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.config.languageVersionSettings
-import org.jetbrains.kotlin.diagnostics.DiagnosticReporter
 import org.jetbrains.kotlin.diagnostics.DiagnosticReporterFactory
 import org.jetbrains.kotlin.fir.FirSession
 import org.jetbrains.kotlin.fir.backend.ConstValueProviderImpl
@@ -27,12 +23,10 @@ import org.jetbrains.kotlin.fir.packageFqName
 import org.jetbrains.kotlin.fir.pipeline.FirResult
 import org.jetbrains.kotlin.fir.resolve.ScopeSession
 import org.jetbrains.kotlin.fir.resolve.providers.firProvider
-import org.jetbrains.kotlin.fir.serialization.*
-import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
+import org.jetbrains.kotlin.fir.serialization.FirKLibSerializerExtension
+import org.jetbrains.kotlin.fir.serialization.serializeSingleFirFile
 import org.jetbrains.kotlin.konan.library.KonanLibrary
-import org.jetbrains.kotlin.library.SerializedIrFile
 import org.jetbrains.kotlin.library.metadata.resolver.TopologicalLibraryOrder
-import org.jetbrains.kotlin.metadata.ProtoBuf
 import org.jetbrains.kotlin.utils.toMetadataVersion

 internal fun PhaseContext.firSerializer(input: FirOutput): SerializerOutput? = when (input) {
@@ -72,39 +66,60 @@ internal fun PhaseContext.firSerializerBase(

    val actualizedFirDeclarations = fir2IrOutput?.irActualizedResult?.actualizedExpectDeclarations?.extractFirDeclarations()
    val diagnosticReporter = DiagnosticReporterFactory.createPendingReporter()
-    val serializerOutput = serializeNativeModule(
-            configuration = configuration,
-            diagnosticReporter = diagnosticReporter,
-            sourceFiles,
-            usedResolvedLibraries?.map { it.library as KonanLibrary },
-            fir2IrOutput?.irModuleFragment,
+    val serializerOutput = serializeModuleIntoKlib(
            moduleName = fir2IrOutput?.irModuleFragment?.descriptor?.name?.asString()
                    ?: firResult.outputs.last().session.moduleData.name.asString(),
-            firFilesAndSessionsBySourceFile,
-            bodiesOnlyForInlines = produceHeaderKlib,
-            skipPrivateApi = produceHeaderKlib,
-    ) { firFile, originalSession, originalScopeSession ->
-        val session = fir2IrOutput?.components?.session ?: originalSession
-        val scopeSession = fir2IrOutput?.components?.scopeSession ?: originalScopeSession
-        val firProvider = fir2IrOutput?.components?.firProvider ?: originalSession.firProvider
-        serializeSingleFirFile(
-                firFile,
-                session,
-                scopeSession,
-                actualizedFirDeclarations,
-                FirKLibSerializerExtension(
-                        session, firProvider, metadataVersion,
-                        fir2IrOutput?.let {
-                            ConstValueProviderImpl(fir2IrOutput.components)
-                        },
-                        allowErrorTypes = false,
-                        exportKDoc = shouldExportKDoc(),
-                        additionalMetadataProvider = fir2IrOutput?.components?.annotationsFromPluginRegistrar?.createAdditionalMetadataProvider()
-                ),
-                configuration.languageVersionSettings,
-                produceHeaderKlib,
-        )
-    }
+            irModuleFragment = fir2IrOutput?.irModuleFragment,
+            configuration = configuration,
+            diagnosticReporter = diagnosticReporter,
+            sourceFiles = sourceFiles,
+            compatibilityMode = CompatibilityMode.CURRENT,
+            cleanFiles = emptyList(),
+            dependencies = usedResolvedLibraries?.map { it.library as KonanLibrary }.orEmpty(),
+            createModuleSerializer = { irDiagnosticReporter,
+                                       irBuiltIns,
+                                       compatibilityMode,
+                                       normalizeAbsolutePaths,
+                                       sourceBaseDirs,
+                                       languageVersionSettings,
+                                       shouldCheckSignaturesOnUniqueness ->
+                KonanIrModuleSerializer(
+                        diagnosticReporter = irDiagnosticReporter,
+                        irBuiltIns = irBuiltIns,
+                        compatibilityMode = compatibilityMode,
+                        normalizeAbsolutePaths = normalizeAbsolutePaths,
+                        sourceBaseDirs = sourceBaseDirs,
+                        languageVersionSettings = languageVersionSettings,
+                        bodiesOnlyForInlines = produceHeaderKlib,
+                        skipPrivateApi = produceHeaderKlib,
+                        shouldCheckSignaturesOnUniqueness = shouldCheckSignaturesOnUniqueness,
+                )
+            },
+            serializeFileMetadata = { ktSourceFile ->
+                val (firFile, originalSession, originalScopeSession) = firFilesAndSessionsBySourceFile[ktSourceFile]
+                        ?: error("cannot find FIR file by source file ${ktSourceFile.name} (${ktSourceFile.path})")
+                val session = fir2IrOutput?.components?.session ?: originalSession
+                val scopeSession = fir2IrOutput?.components?.scopeSession ?: originalScopeSession
+                val firProvider = fir2IrOutput?.components?.firProvider ?: originalSession.firProvider
+                serializeSingleFirFile(
+                        firFile,
+                        session,
+                        scopeSession,
+                        actualizedFirDeclarations,
+                        FirKLibSerializerExtension(
+                                session, firProvider, metadataVersion,
+                                fir2IrOutput?.let {
+                                    ConstValueProviderImpl(fir2IrOutput.components)
+                                },
+                                allowErrorTypes = false,
+                                exportKDoc = shouldExportKDoc(),
+                                additionalMetadataProvider = fir2IrOutput?.components?.annotationsFromPluginRegistrar?.createAdditionalMetadataProvider()
+                        ),
+                        configuration.languageVersionSettings,
+                        produceHeaderKlib,
+                ) to firFile.packageFqName
+            },
+    )
    val renderDiagnosticNames = configuration.getBoolean(CLIConfigurationKeys.RENDER_DIAGNOSTIC_INTERNAL_NAME)
    diagnosticReporter.reportToMessageCollector(messageCollector, renderDiagnosticNames)
    if (diagnosticReporter.hasErrors) {
@@ -112,80 +127,3 @@ internal fun PhaseContext.firSerializerBase(
    }
    return serializerOutput
 }
-
-class KotlinFileSerializedData(
-        val source: KtSourceFile,
-        val firFile: FirFile,
-        val metadata: ByteArray,
-        val irData: SerializedIrFile?,
-) {
-    val fqName: String get() = irData?.fqName ?: firFile.packageFqName.asString()
-    val path: String? get() = irData?.path ?: source.path
-}
-
-private fun serializeNativeModule(
-        configuration: CompilerConfiguration,
-        diagnosticReporter: DiagnosticReporter,
-        files: List<KtSourceFile>,
-        dependencies: List<KonanLibrary>?,
-        moduleFragment: IrModuleFragment?,
-        moduleName: String,
-        firFilesAndSessionsBySourceFile: Map<KtSourceFile, Triple<FirFile, FirSession, ScopeSession>>,
-        bodiesOnlyForInlines: Boolean = false,
-        skipPrivateApi: Boolean = false,
-        serializeSingleFile: (FirFile, FirSession, ScopeSession) -> ProtoBuf.PackageFragment
-): SerializerOutput {
-    if (moduleFragment != null) {
-        assert(files.size == moduleFragment.files.size)
-    }
-
-    val sourceBaseDirs = configuration[CommonConfigurationKeys.KLIB_RELATIVE_PATH_BASES] ?: emptyList()
-    val absolutePathNormalization = configuration[CommonConfigurationKeys.KLIB_NORMALIZE_ABSOLUTE_PATH] ?: false
-    val signatureClashChecks = configuration[CommonConfigurationKeys.PRODUCE_KLIB_SIGNATURES_CLASH_CHECKS] ?: true
-
-    val serializedIr = moduleFragment?.let {
-        KonanIrModuleSerializer(
-                KtDiagnosticReporterWithImplicitIrBasedContext(diagnosticReporter, configuration.languageVersionSettings),
-                moduleFragment.irBuiltins,
-                CompatibilityMode.CURRENT,
-                normalizeAbsolutePaths = absolutePathNormalization,
-                sourceBaseDirs = sourceBaseDirs,
-                languageVersionSettings = configuration.languageVersionSettings,
-                bodiesOnlyForInlines = bodiesOnlyForInlines,
-                skipPrivateApi = skipPrivateApi,
-                shouldCheckSignaturesOnUniqueness = signatureClashChecks,
-        ).serializedIrModule(moduleFragment)
-    }
-
-    val serializedFiles = serializedIr?.files?.toList()
-
-    val compiledKotlinFiles = files.mapIndexed { index, ktSourceFile ->
-        val binaryFile = serializedFiles?.get(index)?.also {
-            assert(ktSourceFile.path == it.path) {
-                """The Kt and Ir files are put in different order
-                Kt: ${ktSourceFile.path}
-                Ir: ${it.path}
-            """.trimMargin()
-            }
-        }
-        val (firFile, session, scopeSession) = firFilesAndSessionsBySourceFile[ktSourceFile]
-                ?: error("cannot find FIR file by source file ${ktSourceFile.name} (${ktSourceFile.path})")
-        val packageFragment = serializeSingleFile(firFile, session, scopeSession)
-        KotlinFileSerializedData(ktSourceFile, firFile, packageFragment.toByteArray(), binaryFile)
-    }
-
-    val header = serializeKlibHeader(
-            configuration.languageVersionSettings, moduleName,
-            compiledKotlinFiles.map { it.fqName }.distinct().sorted(),
-            emptyList()
-    ).toByteArray()
-
-    val serializedMetadata =
-            makeSerializedKlibMetadata(
-                    compiledKotlinFiles.groupBy { it.fqName }
-                            .map { (fqn, data) -> fqn to data.sortedBy { it.path }.map { it.metadata } }.toMap(),
-                    header
-            )
-
-    return SerializerOutput(serializedMetadata, serializedIr, null, dependencies.orEmpty())
-}
@@ -5,7 +5,6 @@

 package org.jetbrains.kotlin.backend.konan.driver.phases

-import org.jetbrains.kotlin.ir.KtDiagnosticReporterWithImplicitIrBasedContext
 import org.jetbrains.kotlin.backend.common.phaser.createSimpleNamedCompilerPhase
 import org.jetbrains.kotlin.backend.common.serialization.CompatibilityMode
 import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataMonolithicSerializer
@@ -16,10 +15,9 @@ import org.jetbrains.kotlin.config.CommonConfigurationKeys
 import org.jetbrains.kotlin.config.languageVersionSettings
 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
 import org.jetbrains.kotlin.diagnostics.DiagnosticReporterFactory
+import org.jetbrains.kotlin.ir.KtDiagnosticReporterWithImplicitIrBasedContext
 import org.jetbrains.kotlin.ir.util.IrMessageLogger
 import org.jetbrains.kotlin.konan.library.KonanLibrary
-import org.jetbrains.kotlin.library.SerializedIrModule
-import org.jetbrains.kotlin.library.SerializedMetadata

 internal data class SerializerInput(
        val moduleDescriptor: ModuleDescriptor,
@@ -27,12 +25,7 @@ internal data class SerializerInput(
        val produceHeaderKlib: Boolean,
 )

-data class SerializerOutput(
-        val serializedMetadata: SerializedMetadata?,
-        val serializedIr: SerializedIrModule?,
-        val dataFlowGraph: ByteArray?,
-        val neededLibraries: List<KonanLibrary>
-)
+typealias SerializerOutput = org.jetbrains.kotlin.backend.common.serialization.SerializerOutput<KonanLibrary>

 internal val SerializerPhase = createSimpleNamedCompilerPhase<PhaseContext, SerializerInput, SerializerOutput>(
        "Serializer", "IR serializer",