[klib] Deduplicate KLIB serialization logic in K/JS and K/N

Introduce the new `serializeModuleIntoKlib` function that all KLIB-based
backends can use to serialize a module to a KLIB.

^KT-64392 Fixed

compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/klib/irForKlib.kt

@@ -19,13 +19,13 @@ import org.jetbrains.kotlin.backend.common.lower.ExpectDeclarationRemover
 import org.jetbrains.kotlin.backend.common.overrides.FakeOverrideChecker
 import org.jetbrains.kotlin.backend.common.serialization.DescriptorByIdSignatureFinderImpl
 import org.jetbrains.kotlin.backend.common.serialization.ICData
+import org.jetbrains.kotlin.backend.common.serialization.KotlinFileSerializedData
 import org.jetbrains.kotlin.backend.common.serialization.mangle.ManglerChecker
 import org.jetbrains.kotlin.backend.common.serialization.mangle.descriptor.Ir2DescriptorManglerAdapter
 import org.jetbrains.kotlin.backend.common.serialization.signature.IdSignatureDescriptor
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.config.languageVersionSettings
 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
-import org.jetbrains.kotlin.ir.backend.js.KotlinFileSerializedData
 import org.jetbrains.kotlin.ir.backend.js.generateModuleFragmentWithPlugins
 import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker
 import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsManglerDesc
@@ -87,7 +87,7 @@ fun generateIrForKlibSerialization(
             messageLogger = messageLogger
         ),
         feContext,
-        ICData(icData.map { it.irData }, errorPolicy.allowErrors),
+        ICData(icData.map { it.irData!! }, errorPolicy.allowErrors),
         stubGenerator = stubGenerator
     )
 

compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/serializeModuleIntoKlib.kt

@@ -0,0 +1,200 @@
+/*
+ * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.backend.common.serialization
+
+import com.intellij.openapi.vfs.VfsUtilCore
+import org.jetbrains.kotlin.KtIoFileSourceFile
+import org.jetbrains.kotlin.KtPsiSourceFile
+import org.jetbrains.kotlin.KtSourceFile
+import org.jetbrains.kotlin.KtVirtualFileSourceFile
+import org.jetbrains.kotlin.backend.common.serialization.metadata.makeSerializedKlibMetadata
+import org.jetbrains.kotlin.backend.common.serialization.metadata.serializeKlibHeader
+import org.jetbrains.kotlin.config.CommonConfigurationKeys
+import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.config.LanguageVersionSettings
+import org.jetbrains.kotlin.config.languageVersionSettings
+import org.jetbrains.kotlin.diagnostics.DiagnosticReporter
+import org.jetbrains.kotlin.ir.IrBuiltIns
+import org.jetbrains.kotlin.ir.IrDiagnosticReporter
+import org.jetbrains.kotlin.ir.KtDiagnosticReporterWithImplicitIrBasedContext
+import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
+import org.jetbrains.kotlin.library.KotlinLibrary
+import org.jetbrains.kotlin.library.SerializedIrFile
+import org.jetbrains.kotlin.library.SerializedIrModule
+import org.jetbrains.kotlin.library.SerializedMetadata
+import org.jetbrains.kotlin.metadata.ProtoBuf
+import org.jetbrains.kotlin.name.FqName
+import java.io.File
+
+/**
+ * Holds the binary data for a single Kotlin file to be written to a KLIB, i.e., its metadata and IR (unless it's a metadata-only KLIB).
+ *
+ * @property metadata  Serialized metadata of the corresponding source file.
+ * @property irData Serialized IR for this file, or `null` if this is a metadata-only KLIB.
+ * @property path The path to the corresponding source file, or `null` if that source file didn't have a path.
+ * @property fqName The fully qualified name of the package containing the serialized file.
+ */
+class KotlinFileSerializedData private constructor(
+    val metadata: ByteArray,
+    val irData: SerializedIrFile?,
+    val path: String?,
+    val fqName: String,
+) {
+
+    /**
+     * Used for creating file serialization data for IR-containing KLIBs.
+     *
+     * @param metadata Serialized metadata of the corresponding source file.
+     * @param irData Serialized IR for this file.
+     */
+    constructor(metadata: ByteArray, irData: SerializedIrFile) : this(metadata, irData, irData.path, irData.fqName)
+
+    /**
+     * Used for creating file serialization data in metadata-only KLIBs.
+     *
+     * @param metadata Serialized metadata of the corresponding source file.
+     * @param path The path of the serialized file.
+     * @param fqName The fully qualified name of the package containing the serialized file.
+     */
+    constructor(metadata: ByteArray, path: String?, fqName: String) : this(metadata, irData = null, path, fqName)
+}
+
+class SerializerOutput<Dependency : KotlinLibrary>(
+    val serializedMetadata: SerializedMetadata?,
+    val serializedIr: SerializedIrModule?,
+    val dataFlowGraph: ByteArray?,
+    val neededLibraries: List<Dependency>,
+)
+
+fun KtSourceFile.toIoFileOrNull(): File? = when (this) {
+    is KtIoFileSourceFile -> file
+    is KtVirtualFileSourceFile -> VfsUtilCore.virtualToIoFile(virtualFile)
+    is KtPsiSourceFile -> VfsUtilCore.virtualToIoFile(psiFile.virtualFile)
+    else -> path?.let(::File)
+}
+
+/**
+ * Produces all the necessary binary data for writing a KLIB for a Kotlin module, including its metadata.
+ *
+ * If [irModuleFragment] is not `null`, serializes the module's IR into binary form by running [IrModuleSerializer].
+ *
+ * For producing a metadata-only KLIB, pass `null` to [irModuleFragment].
+ *
+ * @param moduleName The name of the module being serialized to be written into the KLIB header.
+ * @param irModuleFragment The IR to be serialized into the KLIB being produced, or `null` if this is going to be a metadata-only KLIB.
+ * @param configuration Used to determine certain serialization parameters and enable/disable serialization diagnostics.
+ * @param diagnosticReporter Used for reporting serialization-time diagnostics, for example, about clashing IR signatures.
+ * @param sourceFiles The source files from which the KLIB is being compiled. Used for serializing file metadata.
+ * @param compatibilityMode The information about KLIB ABI.
+ * @param cleanFiles In the case of incremental compilation, the list of files that were not changed and therefore don't need to be
+ *     serialized again.
+ * @param dependencies The list of KLIBs that the KLIB being produced depends on.
+ * @param createModuleSerializer Used for creating a backend-specific instance of [IrModuleSerializer].
+ * @param serializeFileMetadata The name of this argument kind of speaks for itself.
+ * @param runKlibCheckers Additional checks to be run before serializing [irModuleFragment]. Can be used to report serialization-time
+ *     diagnostics.
+ * @param processCompiledFileData Called for each newly serialized file. Useful for incremental compilation.
+ * @param processKlibHeader Called after serializing the KLIB header. Useful for incremental compilation.
+ */
+fun <Dependency : KotlinLibrary> serializeModuleIntoKlib(
+    moduleName: String,
+    irModuleFragment: IrModuleFragment?,
+    configuration: CompilerConfiguration,
+    diagnosticReporter: DiagnosticReporter,
+    sourceFiles: List<KtSourceFile>,
+    compatibilityMode: CompatibilityMode,
+    cleanFiles: List<KotlinFileSerializedData>,
+    dependencies: List<Dependency>,
+    createModuleSerializer: (
+        irDiagnosticReporter: IrDiagnosticReporter,
+        irBuiltins: IrBuiltIns,
+        compatibilityMode: CompatibilityMode,
+        normalizeAbsolutePaths: Boolean,
+        sourceBaseDirs: Collection<String>,
+        languageVersionSettings: LanguageVersionSettings,
+        shouldCheckSignaturesOnUniqueness: Boolean,
+    ) -> IrModuleSerializer<*>,
+    serializeFileMetadata: (KtSourceFile) -> Pair<ProtoBuf.PackageFragment, FqName>,
+    runKlibCheckers: (IrModuleFragment, IrDiagnosticReporter, CompilerConfiguration) -> Unit = { _, _, _ -> },
+    processCompiledFileData: ((File, KotlinFileSerializedData) -> Unit)? = null,
+    processKlibHeader: (ByteArray) -> Unit = {},
+): SerializerOutput<Dependency> {
+    if (irModuleFragment != null) {
+        assert(sourceFiles.size == irModuleFragment.files.size) {
+            "The number of source files (${sourceFiles.size}) does not match the number of IrFiles (${irModuleFragment.files.size})"
+        }
+    }
+
+    val sourceBaseDirs = configuration[CommonConfigurationKeys.KLIB_RELATIVE_PATH_BASES] ?: emptyList()
+    val normalizeAbsolutePath = configuration.getBoolean(CommonConfigurationKeys.KLIB_NORMALIZE_ABSOLUTE_PATH)
+    val signatureClashChecks = configuration[CommonConfigurationKeys.PRODUCE_KLIB_SIGNATURES_CLASH_CHECKS] ?: true
+
+    val serializedIr = irModuleFragment?.let {
+        val irDiagnosticReporter = KtDiagnosticReporterWithImplicitIrBasedContext(diagnosticReporter, configuration.languageVersionSettings)
+        runKlibCheckers(it, irDiagnosticReporter, configuration)
+        createModuleSerializer(
+            irDiagnosticReporter,
+            it.irBuiltins,
+            compatibilityMode,
+            normalizeAbsolutePath,
+            sourceBaseDirs,
+            configuration.languageVersionSettings,
+            signatureClashChecks,
+        ).serializedIrModule(it)
+    }
+
+    val serializedFiles = serializedIr?.files?.toList()
+
+    val compiledKotlinFiles = sourceFiles.mapIndexedTo(cleanFiles.toMutableList()) { i, ktSourceFile ->
+        val binaryFile = serializedFiles?.get(i)?.also {
+            assert(ktSourceFile.path == it.path) {
+                """The Kt and Ir files are put in different order
+                Kt: ${ktSourceFile.path}
+                Ir: ${it.path}
+                """.trimMargin()
+            }
+        }
+        val (packageFragment, fqName) = serializeFileMetadata(ktSourceFile)
+        val metadata = packageFragment.toByteArray()
+        val compiledKotlinFile = if (binaryFile == null)
+            KotlinFileSerializedData(metadata, ktSourceFile.path, fqName.asString())
+        else
+            KotlinFileSerializedData(metadata, binaryFile)
+
+        if (processCompiledFileData != null) {
+            val ioFile = ktSourceFile.toIoFileOrNull() ?: error("No file found for source ${ktSourceFile.path}")
+            processCompiledFileData(ioFile, compiledKotlinFile)
+        }
+
+        compiledKotlinFile
+    }
+
+    val header = serializeKlibHeader(
+        languageVersionSettings = configuration.languageVersionSettings,
+        moduleName = moduleName,
+        fragmentNames = compiledKotlinFiles.map { it.fqName }.distinct().sorted(),
+        emptyPackages = emptyList(),
+    ).toByteArray()
+
+    processKlibHeader(header)
+
+    val serializedMetadata = makeSerializedKlibMetadata(
+        fragments = compiledKotlinFiles
+            .groupBy { it.fqName }
+            .map { (fqn, data) ->
+                fqn to data.sortedBy { it.path }.map { it.metadata }
+            }
+            .toMap(),
+        header = header,
+    )
+
+    return SerializerOutput(
+        serializedMetadata = serializedMetadata,
+        serializedIr = if (serializedIr == null) null else SerializedIrModule(compiledKotlinFiles.mapNotNull { it.irData }),
+        dataFlowGraph = null,
+        neededLibraries = dependencies,
+    )
+}

compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt

@@ -34,7 +34,6 @@ import org.jetbrains.kotlin.diagnostics.DiagnosticReporter
 import org.jetbrains.kotlin.incremental.components.LookupTracker
 import org.jetbrains.kotlin.incremental.js.IncrementalDataProvider
 import org.jetbrains.kotlin.ir.IrBuiltIns
-import org.jetbrains.kotlin.ir.KtDiagnosticReporterWithImplicitIrBasedContext
 import org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI
 import org.jetbrains.kotlin.ir.backend.js.checkers.JsKlibCheckers
 import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.*
@@ -57,6 +56,7 @@ import org.jetbrains.kotlin.library.impl.buildKotlinLibrary
 import org.jetbrains.kotlin.library.metadata.KlibMetadataFactories
 import org.jetbrains.kotlin.library.metadata.KlibMetadataVersion
 import org.jetbrains.kotlin.metadata.ProtoBuf
+import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.progress.IncrementalNextRoundException
 import org.jetbrains.kotlin.progress.ProgressIndicatorAndCompilationCanceledStatus
 import org.jetbrains.kotlin.psi.KtFile
@@ -100,8 +100,6 @@ private val CompilerConfiguration.metadataVersion
 internal val SerializedIrFile.fileMetadata: ByteArray
     get() = backendSpecificMetadata ?: error("Expect file caches to have backendSpecificMetadata, but '$path' doesn't")
 
-class KotlinFileSerializedData(val metadata: ByteArray, val irData: SerializedIrFile)
-
 fun generateKLib(
     depsDescriptors: ModulesStructure,
     outputKlibPath: String,
@@ -624,100 +622,72 @@ fun serializeModuleIntoKlib(
     builtInsPlatform: BuiltInsPlatform = BuiltInsPlatform.JS,
     serializeSingleFile: (KtSourceFile) -> ProtoBuf.PackageFragment
 ) {
-    assert(files.size == moduleFragment.files.size)
-
-    val compatibilityMode = CompatibilityMode(abiVersion)
-    val sourceBaseDirs = configuration[CommonConfigurationKeys.KLIB_RELATIVE_PATH_BASES] ?: emptyList()
-    val absolutePathNormalization = configuration[CommonConfigurationKeys.KLIB_NORMALIZE_ABSOLUTE_PATH] ?: false
-    val signatureClashChecks = configuration[CommonConfigurationKeys.PRODUCE_KLIB_SIGNATURES_CLASH_CHECKS] ?: true
-
     val moduleExportedNames = moduleFragment.collectExportedNames()
-
-    val irDiagnosticReporter = KtDiagnosticReporterWithImplicitIrBasedContext(diagnosticReporter, configuration.languageVersionSettings)
-    if (builtInsPlatform == BuiltInsPlatform.JS) {
-        val cleanFilesIrData = cleanFiles.map { it.irData }
-        JsKlibCheckers.check(cleanFilesIrData, moduleFragment, moduleExportedNames, irDiagnosticReporter, configuration)
-    }
-
-    val serializedIr =
-        JsIrModuleSerializer(
-            irDiagnosticReporter,
-            moduleFragment.irBuiltins,
-            compatibilityMode,
-            normalizeAbsolutePaths = absolutePathNormalization,
-            sourceBaseDirs = sourceBaseDirs,
-            configuration.languageVersionSettings,
-            signatureClashChecks,
-        ) { JsIrFileMetadata(moduleExportedNames[it]?.values?.toSmartList() ?: emptyList()) }.serializedIrModule(moduleFragment)
-
-    val moduleDescriptor = moduleFragment.descriptor
-
     val incrementalResultsConsumer = configuration.get(JSConfigurationKeys.INCREMENTAL_RESULTS_CONSUMER)
     val empty = ByteArray(0)
-
-    fun processCompiledFileData(ioFile: File, compiledFile: KotlinFileSerializedData) {
-        incrementalResultsConsumer?.run {
-            processPackagePart(ioFile, compiledFile.metadata, empty, empty)
-            with(compiledFile.irData) {
-                processIrFile(
-                    ioFile,
-                    fileData,
-                    types,
-                    signatures,
-                    strings,
-                    declarations,
-                    bodies,
-                    fqName.toByteArray(),
-                    fileMetadata,
-                    debugInfo,
-                )
+    val serializerOutput = serializeModuleIntoKlib(
+        moduleName = moduleFragment.name.asString(),
+        irModuleFragment = moduleFragment,
+        configuration = configuration,
+        diagnosticReporter = diagnosticReporter,
+        sourceFiles = files,
+        compatibilityMode = CompatibilityMode(abiVersion),
+        cleanFiles = cleanFiles,
+        dependencies = dependencies,
+        createModuleSerializer = {
+                irDiagnosticReporter,
+                irBuiltins,
+                compatibilityMode,
+                normalizeAbsolutePaths,
+                sourceBaseDirs,
+                languageVersionSettings,
+                shouldCheckSignaturesOnUniqueness,
+            ->
+            JsIrModuleSerializer(
+                irDiagnosticReporter,
+                irBuiltins,
+                compatibilityMode,
+                normalizeAbsolutePaths,
+                sourceBaseDirs,
+                languageVersionSettings,
+                shouldCheckSignaturesOnUniqueness,
+            ) { JsIrFileMetadata(moduleExportedNames[it]?.values?.toSmartList() ?: emptyList()) }
+        },
+        serializeFileMetadata = { serializeSingleFile(it) to FqName.ROOT },
+        runKlibCheckers = { irModuleFragment, irDiagnosticReporter, compilerConfiguration ->
+            if (builtInsPlatform == BuiltInsPlatform.JS) {
+                val cleanFilesIrData = cleanFiles.map { it.irData ?: error("Metadata-only KLIBs are not supported in Kotlin/JS") }
+                JsKlibCheckers.check(cleanFilesIrData, irModuleFragment, moduleExportedNames, irDiagnosticReporter, compilerConfiguration)
             }
-        }
-    }
+        },
+        processCompiledFileData = { ioFile, compiledFile ->
+            incrementalResultsConsumer?.run {
+                processPackagePart(ioFile, compiledFile.metadata, empty, empty)
+                with(compiledFile.irData!!) {
+                    processIrFile(
+                        ioFile,
+                        fileData,
+                        types,
+                        signatures,
+                        strings,
+                        declarations,
+                        bodies,
+                        fqName.toByteArray(),
+                        fileMetadata,
+                        debugInfo,
+                    )
+                }
+            }
+        },
+        processKlibHeader = {
+            incrementalResultsConsumer?.processHeader(it)
+        },
+    )
 
-    val additionalFiles = mutableListOf<KotlinFileSerializedData>()
-
-    for ((ktSourceFile, binaryFile) in files.zip(serializedIr.files)) {
-        assert(ktSourceFile.path == binaryFile.path) {
-            """The Kt and Ir files are put in different order
-                Kt: ${ktSourceFile.path}
-                Ir: ${binaryFile.path}
-            """.trimMargin()
-        }
-        val packageFragment = serializeSingleFile(ktSourceFile)
-        val compiledKotlinFile = KotlinFileSerializedData(packageFragment.toByteArray(), binaryFile)
-
-        additionalFiles += compiledKotlinFile
-        val ioFile = ktSourceFile.toIoFileOrNull()
-        assert(ioFile != null) {
-            "No file found for source ${ktSourceFile.path}"
-        }
-        processCompiledFileData(ioFile!!, compiledKotlinFile)
-    }
-
-    val compiledKotlinFiles = cleanFiles + additionalFiles
-
-    val header = serializeKlibHeader(
-        configuration.languageVersionSettings, moduleDescriptor,
-        compiledKotlinFiles.map { it.irData.fqName }.distinct().sorted(),
-        emptyList()
-    ).toByteArray()
-
-    incrementalResultsConsumer?.run {
-        processHeader(header)
-    }
-
-    val serializedMetadata =
-        makeSerializedKlibMetadata(
-            compiledKotlinFiles.groupBy { it.irData.fqName }
-                .map { (fqn, data) -> fqn to data.sortedBy { it.irData.path }.map { it.metadata } }.toMap(),
-            header
-        )
-
-    val fullSerializedIr = SerializedIrModule(compiledKotlinFiles.map { it.irData })
+    val fullSerializedIr = serializerOutput.serializedIr ?: error("Metadata-only KLIBs are not supported in Kotlin/JS")
 
     val versions = KotlinLibraryVersioning(
-        abiVersion = compatibilityMode.abiVersion,
+        abiVersion = abiVersion,
         libraryVersion = null,
         compilerVersion = KotlinCompilerVersion.VERSION,
         metadataVersion = KlibMetadataVersion.INSTANCE.toString(),
@@ -737,9 +707,9 @@ fun serializeModuleIntoKlib(
     }
 
     buildKotlinLibrary(
-        linkDependencies = dependencies,
+        linkDependencies = serializerOutput.neededLibraries,
         ir = fullSerializedIr,
-        metadata = serializedMetadata,
+        metadata = serializerOutput.serializedMetadata ?: error("expected serialized metadata"),
         dataFlowGraph = null,
         manifestProperties = properties,
         moduleName = moduleName,
@@ -809,13 +779,6 @@ private fun Map<IrModuleFragment, KotlinLibrary>.getUniqueNameForEachFragment():
     }.toMap()
 }
 
-fun KtSourceFile.toIoFileOrNull(): File? = when (this) {
-    is KtIoFileSourceFile -> file
-    is KtVirtualFileSourceFile -> VfsUtilCore.virtualToIoFile(virtualFile)
-    is KtPsiSourceFile -> VfsUtilCore.virtualToIoFile(psiFile.virtualFile)
-    else -> path?.let(::File)
-}
-
 fun IncrementalDataProvider.getSerializedData(newSources: List<KtSourceFile>): List<KotlinFileSerializedData> {
     val nonCompiledSources = newSources.associateBy { it.toIoFileOrNull()!! }
     val compiledIrFiles = serializedIrFiles

compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/IrBackendInput.kt

@@ -6,13 +6,12 @@
 package org.jetbrains.kotlin.test.backend.ir
 
 import org.jetbrains.kotlin.KtSourceFile
-import org.jetbrains.kotlin.backend.common.actualizer.IrActualizedResult
 import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
+import org.jetbrains.kotlin.backend.common.serialization.KotlinFileSerializedData
 import org.jetbrains.kotlin.backend.jvm.JvmIrCodegenFactory
 import org.jetbrains.kotlin.codegen.state.GenerationState
 import org.jetbrains.kotlin.diagnostics.impl.BaseDiagnosticsCollector
 import org.jetbrains.kotlin.fir.backend.FirMangler
-import org.jetbrains.kotlin.ir.backend.js.KotlinFileSerializedData
 import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
 import org.jetbrains.kotlin.ir.util.KotlinMangler
 import org.jetbrains.kotlin.metadata.ProtoBuf

compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/Fir2IrJsWasmResultsConverter.kt

@@ -7,6 +7,7 @@ package org.jetbrains.kotlin.test.frontend.fir
 
 import org.jetbrains.kotlin.KtSourceFile
 import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
+import org.jetbrains.kotlin.backend.common.serialization.KotlinFileSerializedData
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.config.languageVersionSettings
 import org.jetbrains.kotlin.diagnostics.impl.BaseDiagnosticsCollector
@@ -20,7 +21,6 @@ import org.jetbrains.kotlin.fir.pipeline.Fir2IrActualizedResult
 import org.jetbrains.kotlin.fir.serialization.FirKLibSerializerExtension
 import org.jetbrains.kotlin.fir.serialization.serializeSingleFirFile
 import org.jetbrains.kotlin.ir.backend.js.JsFactories
-import org.jetbrains.kotlin.ir.backend.js.KotlinFileSerializedData
 import org.jetbrains.kotlin.ir.backend.js.getSerializedData
 import org.jetbrains.kotlin.ir.backend.js.incrementalDataProvider
 import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsManglerIr

compiler/tests-common/tests/org/jetbrains/kotlin/klib/AbstractKlibIrTextTestCase.kt

@@ -8,20 +8,20 @@ package org.jetbrains.kotlin.klib
 import com.intellij.openapi.project.Project
 import com.intellij.psi.PsiElement
 import junit.framework.TestCase
-import org.jetbrains.kotlin.ir.KtDiagnosticReporterWithImplicitIrBasedContext
+import org.jetbrains.kotlin.KtPsiSourceFile
 import org.jetbrains.kotlin.backend.common.CommonKLibResolver
-import org.jetbrains.kotlin.ir.util.isExpect
 import org.jetbrains.kotlin.backend.common.linkage.issues.checkNoUnboundSymbols
 import org.jetbrains.kotlin.backend.common.linkage.partial.PartialLinkageSupportForLinker
-import org.jetbrains.kotlin.backend.common.serialization.CompatibilityMode
 import org.jetbrains.kotlin.backend.common.serialization.DeserializationStrategy
 import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataIncrementalSerializer
-import org.jetbrains.kotlin.backend.common.serialization.metadata.makeSerializedKlibMetadata
 import org.jetbrains.kotlin.backend.common.serialization.signature.IdSignatureDescriptor
 import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles
 import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
 import org.jetbrains.kotlin.codegen.CodegenTestCase
-import org.jetbrains.kotlin.config.*
+import org.jetbrains.kotlin.config.CommonConfigurationKeys
+import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.config.LanguageFeature
+import org.jetbrains.kotlin.config.languageVersionSettings
 import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
 import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
@@ -31,18 +31,20 @@ import org.jetbrains.kotlin.ir.AbstractIrGeneratorTestCase
 import org.jetbrains.kotlin.ir.IrElement
 import org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI
 import org.jetbrains.kotlin.ir.backend.js.JsFactories
-import org.jetbrains.kotlin.ir.backend.js.KotlinFileSerializedData
 import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker
-import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrModuleSerializer
 import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsManglerDesc
+import org.jetbrains.kotlin.ir.backend.js.serializeModuleIntoKlib
+import org.jetbrains.kotlin.ir.backend.js.serializeScope
 import org.jetbrains.kotlin.ir.declarations.*
 import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl
 import org.jetbrains.kotlin.ir.util.*
-import org.jetbrains.kotlin.ir.visitors.*
+import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid
+import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid
+import org.jetbrains.kotlin.ir.visitors.acceptVoid
 import org.jetbrains.kotlin.js.analyze.TopDownAnalyzerFacadeForJS
-import org.jetbrains.kotlin.library.*
+import org.jetbrains.kotlin.library.KotlinAbiVersion
+import org.jetbrains.kotlin.library.KotlinLibrary
 import org.jetbrains.kotlin.library.impl.BuiltInsPlatform
-import org.jetbrains.kotlin.library.impl.buildKotlinLibrary
 import org.jetbrains.kotlin.library.metadata.KlibMetadataVersion
 import org.jetbrains.kotlin.library.metadata.resolver.TopologicalLibraryOrder
 import org.jetbrains.kotlin.metadata.ProtoBuf
@@ -62,7 +64,6 @@ import org.jetbrains.kotlin.test.util.KtTestUtil
 import org.jetbrains.kotlin.util.DummyLogger
 import org.jetbrains.kotlin.utils.rethrow
 import java.io.File
-import java.util.*
 
 /**
  * Compares compiled and deserialized IR
@@ -147,81 +148,25 @@ abstract class AbstractKlibIrTextTestCase : CodegenTestCase() {
         stdlib: KotlinLibrary,
         containsErrorCode: Boolean,
     ): String {
-        val ktFiles = myFiles.psiFiles
-        val serializedIr = JsIrModuleSerializer(
-            KtDiagnosticReporterWithImplicitIrBasedContext(
-                DiagnosticReporterFactory.createPendingReporter(),
-                myEnvironment.configuration.languageVersionSettings,
-            ),
-            irModuleFragment.irBuiltins,
-            CompatibilityMode.CURRENT,
-            false,
-            emptyList(),
-            myEnvironment.configuration.languageVersionSettings,
-        ).serializedIrModule(irModuleFragment)
-
-        val moduleDescriptor = irModuleFragment.descriptor
         val metadataSerializer = klibMetadataIncrementalSerializer(myEnvironment.configuration, myEnvironment.project, containsErrorCode)
-
-
-        val compiledKotlinFiles = mutableListOf<KotlinFileSerializedData>()
-        for ((ktFile, binaryFile) in ktFiles.zip(serializedIr.files)) {
-            assert(ktFile.virtualFilePath == binaryFile.path) {
-                """The Kt and Ir files are put in different order
-                Kt: ${ktFile.virtualFilePath}
-                Ir: ${binaryFile.path}
-            """.trimMargin()
-            }
-            val packageFragment = metadataSerializer.serializeScope(ktFile, bindingContext, moduleDescriptor)
-            val compiledKotlinFile = KotlinFileSerializedData(packageFragment.toByteArray(), binaryFile)
-
-            compiledKotlinFiles += compiledKotlinFile
-        }
-
-        val header = metadataSerializer.serializeHeader(
-            moduleDescriptor,
-            compiledKotlinFiles.map { it.irData.fqName }.distinct().sorted(),
-            emptyList()
-        ).toByteArray()
-
-
-        val serializedMetadata = makeSerializedKlibMetadata(
-            compiledKotlinFiles.groupBy { it.irData.fqName }
-                .map { (fqn, data) -> fqn to data.sortedBy { it.irData.path }.map { it.metadata } }.toMap(),
-            header
-        )
-
-        val fullSerializedIr = SerializedIrModule(compiledKotlinFiles.map { it.irData })
-
-        val properties = Properties().also { p ->
-            if (containsErrorCode) {
-                p.setProperty(KLIB_PROPERTY_CONTAINS_ERROR_CODE, "true")
-            }
-        }
-
-        val versions = KotlinLibraryVersioning(
-            abiVersion = KotlinAbiVersion.CURRENT,
-            libraryVersion = null,
-            compilerVersion = KotlinCompilerVersion.VERSION,
-            metadataVersion = KlibMetadataVersion.INSTANCE.toString(),
-        )
-
         val klibDir = org.jetbrains.kotlin.konan.file.createTempDir("testKlib")
-
-        buildKotlinLibrary(
-            linkDependencies = listOf(stdlib),
-            ir = fullSerializedIr,
-            metadata = serializedMetadata,
-            dataFlowGraph = null,
-            manifestProperties = properties,
+        serializeModuleIntoKlib(
             moduleName = MODULE_NAME,
+            configuration = myEnvironment.configuration,
+            diagnosticReporter = DiagnosticReporterFactory.createPendingReporter(),
+            files = myFiles.psiFiles.map(::KtPsiSourceFile),
+            klibPath = klibDir.canonicalPath,
+            dependencies = listOf(stdlib),
+            moduleFragment = irModuleFragment,
+            cleanFiles = emptyList(),
             nopack = true,
             perFile = false,
-            output = klibDir.canonicalPath,
-            versions = versions,
-            builtInsPlatform = BuiltInsPlatform.JS
+            containsErrorCode = containsErrorCode,
+            abiVersion = KotlinAbiVersion.CURRENT,
+            jsOutputName = null,
+            builtInsPlatform = BuiltInsPlatform.JS,
+            serializeSingleFile = { metadataSerializer.serializeScope(it, bindingContext, irModuleFragment.descriptor) }
         )
-
         return klibDir.canonicalPath
     }
 

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/FirNativeSerializer.kt

@@ -1,10 +1,8 @@
 package org.jetbrains.kotlin.backend.konan
 
-import org.jetbrains.kotlin.ir.KtDiagnosticReporterWithImplicitIrBasedContext
 import org.jetbrains.kotlin.KtSourceFile
 import org.jetbrains.kotlin.backend.common.serialization.CompatibilityMode
-import org.jetbrains.kotlin.backend.common.serialization.metadata.makeSerializedKlibMetadata
-import org.jetbrains.kotlin.backend.common.serialization.metadata.serializeKlibHeader
+import org.jetbrains.kotlin.backend.common.serialization.serializeModuleIntoKlib
 import org.jetbrains.kotlin.backend.konan.driver.PhaseContext
 import org.jetbrains.kotlin.backend.konan.driver.phases.Fir2IrOutput
 import org.jetbrains.kotlin.backend.konan.driver.phases.FirOutput
@@ -14,9 +12,7 @@ import org.jetbrains.kotlin.backend.konan.serialization.KonanIrModuleSerializer
 import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
 import org.jetbrains.kotlin.cli.common.fir.reportToMessageCollector
 import org.jetbrains.kotlin.config.CommonConfigurationKeys
-import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.config.languageVersionSettings
-import org.jetbrains.kotlin.diagnostics.DiagnosticReporter
 import org.jetbrains.kotlin.diagnostics.DiagnosticReporterFactory
 import org.jetbrains.kotlin.fir.FirSession
 import org.jetbrains.kotlin.fir.backend.ConstValueProviderImpl
@@ -27,12 +23,10 @@ import org.jetbrains.kotlin.fir.packageFqName
 import org.jetbrains.kotlin.fir.pipeline.FirResult
 import org.jetbrains.kotlin.fir.resolve.ScopeSession
 import org.jetbrains.kotlin.fir.resolve.providers.firProvider
-import org.jetbrains.kotlin.fir.serialization.*
-import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
+import org.jetbrains.kotlin.fir.serialization.FirKLibSerializerExtension
+import org.jetbrains.kotlin.fir.serialization.serializeSingleFirFile
 import org.jetbrains.kotlin.konan.library.KonanLibrary
-import org.jetbrains.kotlin.library.SerializedIrFile
 import org.jetbrains.kotlin.library.metadata.resolver.TopologicalLibraryOrder
-import org.jetbrains.kotlin.metadata.ProtoBuf
 import org.jetbrains.kotlin.utils.toMetadataVersion
 
 internal fun PhaseContext.firSerializer(input: FirOutput): SerializerOutput? = when (input) {
@@ -72,39 +66,60 @@ internal fun PhaseContext.firSerializerBase(
 
     val actualizedFirDeclarations = fir2IrOutput?.irActualizedResult?.actualizedExpectDeclarations?.extractFirDeclarations()
     val diagnosticReporter = DiagnosticReporterFactory.createPendingReporter()
-    val serializerOutput = serializeNativeModule(
-            configuration = configuration,
-            diagnosticReporter = diagnosticReporter,
-            sourceFiles,
-            usedResolvedLibraries?.map { it.library as KonanLibrary },
-            fir2IrOutput?.irModuleFragment,
+    val serializerOutput = serializeModuleIntoKlib(
             moduleName = fir2IrOutput?.irModuleFragment?.descriptor?.name?.asString()
                     ?: firResult.outputs.last().session.moduleData.name.asString(),
-            firFilesAndSessionsBySourceFile,
-            bodiesOnlyForInlines = produceHeaderKlib,
-            skipPrivateApi = produceHeaderKlib,
-    ) { firFile, originalSession, originalScopeSession ->
-        val session = fir2IrOutput?.components?.session ?: originalSession
-        val scopeSession = fir2IrOutput?.components?.scopeSession ?: originalScopeSession
-        val firProvider = fir2IrOutput?.components?.firProvider ?: originalSession.firProvider
-        serializeSingleFirFile(
-                firFile,
-                session,
-                scopeSession,
-                actualizedFirDeclarations,
-                FirKLibSerializerExtension(
-                        session, firProvider, metadataVersion,
-                        fir2IrOutput?.let {
-                            ConstValueProviderImpl(fir2IrOutput.components)
-                        },
-                        allowErrorTypes = false,
-                        exportKDoc = shouldExportKDoc(),
-                        additionalMetadataProvider = fir2IrOutput?.components?.annotationsFromPluginRegistrar?.createAdditionalMetadataProvider()
-                ),
-                configuration.languageVersionSettings,
-                produceHeaderKlib,
-        )
-    }
+            irModuleFragment = fir2IrOutput?.irModuleFragment,
+            configuration = configuration,
+            diagnosticReporter = diagnosticReporter,
+            sourceFiles = sourceFiles,
+            compatibilityMode = CompatibilityMode.CURRENT,
+            cleanFiles = emptyList(),
+            dependencies = usedResolvedLibraries?.map { it.library as KonanLibrary }.orEmpty(),
+            createModuleSerializer = { irDiagnosticReporter,
+                                       irBuiltIns,
+                                       compatibilityMode,
+                                       normalizeAbsolutePaths,
+                                       sourceBaseDirs,
+                                       languageVersionSettings,
+                                       shouldCheckSignaturesOnUniqueness ->
+                KonanIrModuleSerializer(
+                        diagnosticReporter = irDiagnosticReporter,
+                        irBuiltIns = irBuiltIns,
+                        compatibilityMode = compatibilityMode,
+                        normalizeAbsolutePaths = normalizeAbsolutePaths,
+                        sourceBaseDirs = sourceBaseDirs,
+                        languageVersionSettings = languageVersionSettings,
+                        bodiesOnlyForInlines = produceHeaderKlib,
+                        skipPrivateApi = produceHeaderKlib,
+                        shouldCheckSignaturesOnUniqueness = shouldCheckSignaturesOnUniqueness,
+                )
+            },
+            serializeFileMetadata = { ktSourceFile ->
+                val (firFile, originalSession, originalScopeSession) = firFilesAndSessionsBySourceFile[ktSourceFile]
+                        ?: error("cannot find FIR file by source file ${ktSourceFile.name} (${ktSourceFile.path})")
+                val session = fir2IrOutput?.components?.session ?: originalSession
+                val scopeSession = fir2IrOutput?.components?.scopeSession ?: originalScopeSession
+                val firProvider = fir2IrOutput?.components?.firProvider ?: originalSession.firProvider
+                serializeSingleFirFile(
+                        firFile,
+                        session,
+                        scopeSession,
+                        actualizedFirDeclarations,
+                        FirKLibSerializerExtension(
+                                session, firProvider, metadataVersion,
+                                fir2IrOutput?.let {
+                                    ConstValueProviderImpl(fir2IrOutput.components)
+                                },
+                                allowErrorTypes = false,
+                                exportKDoc = shouldExportKDoc(),
+                                additionalMetadataProvider = fir2IrOutput?.components?.annotationsFromPluginRegistrar?.createAdditionalMetadataProvider()
+                        ),
+                        configuration.languageVersionSettings,
+                        produceHeaderKlib,
+                ) to firFile.packageFqName
+            },
+    )
     val renderDiagnosticNames = configuration.getBoolean(CLIConfigurationKeys.RENDER_DIAGNOSTIC_INTERNAL_NAME)
     diagnosticReporter.reportToMessageCollector(messageCollector, renderDiagnosticNames)
     if (diagnosticReporter.hasErrors) {
@@ -112,80 +127,3 @@ internal fun PhaseContext.firSerializerBase(
     }
     return serializerOutput
 }
-
-class KotlinFileSerializedData(
-        val source: KtSourceFile,
-        val firFile: FirFile,
-        val metadata: ByteArray,
-        val irData: SerializedIrFile?,
-) {
-    val fqName: String get() = irData?.fqName ?: firFile.packageFqName.asString()
-    val path: String? get() = irData?.path ?: source.path
-}
-
-private fun serializeNativeModule(
-        configuration: CompilerConfiguration,
-        diagnosticReporter: DiagnosticReporter,
-        files: List<KtSourceFile>,
-        dependencies: List<KonanLibrary>?,
-        moduleFragment: IrModuleFragment?,
-        moduleName: String,
-        firFilesAndSessionsBySourceFile: Map<KtSourceFile, Triple<FirFile, FirSession, ScopeSession>>,
-        bodiesOnlyForInlines: Boolean = false,
-        skipPrivateApi: Boolean = false,
-        serializeSingleFile: (FirFile, FirSession, ScopeSession) -> ProtoBuf.PackageFragment
-): SerializerOutput {
-    if (moduleFragment != null) {
-        assert(files.size == moduleFragment.files.size)
-    }
-
-    val sourceBaseDirs = configuration[CommonConfigurationKeys.KLIB_RELATIVE_PATH_BASES] ?: emptyList()
-    val absolutePathNormalization = configuration[CommonConfigurationKeys.KLIB_NORMALIZE_ABSOLUTE_PATH] ?: false
-    val signatureClashChecks = configuration[CommonConfigurationKeys.PRODUCE_KLIB_SIGNATURES_CLASH_CHECKS] ?: true
-
-    val serializedIr = moduleFragment?.let {
-        KonanIrModuleSerializer(
-                KtDiagnosticReporterWithImplicitIrBasedContext(diagnosticReporter, configuration.languageVersionSettings),
-                moduleFragment.irBuiltins,
-                CompatibilityMode.CURRENT,
-                normalizeAbsolutePaths = absolutePathNormalization,
-                sourceBaseDirs = sourceBaseDirs,
-                languageVersionSettings = configuration.languageVersionSettings,
-                bodiesOnlyForInlines = bodiesOnlyForInlines,
-                skipPrivateApi = skipPrivateApi,
-                shouldCheckSignaturesOnUniqueness = signatureClashChecks,
-        ).serializedIrModule(moduleFragment)
-    }
-
-    val serializedFiles = serializedIr?.files?.toList()
-
-    val compiledKotlinFiles = files.mapIndexed { index, ktSourceFile ->
-        val binaryFile = serializedFiles?.get(index)?.also {
-            assert(ktSourceFile.path == it.path) {
-                """The Kt and Ir files are put in different order
-                Kt: ${ktSourceFile.path}
-                Ir: ${it.path}
-            """.trimMargin()
-            }
-        }
-        val (firFile, session, scopeSession) = firFilesAndSessionsBySourceFile[ktSourceFile]
-                ?: error("cannot find FIR file by source file ${ktSourceFile.name} (${ktSourceFile.path})")
-        val packageFragment = serializeSingleFile(firFile, session, scopeSession)
-        KotlinFileSerializedData(ktSourceFile, firFile, packageFragment.toByteArray(), binaryFile)
-    }
-
-    val header = serializeKlibHeader(
-            configuration.languageVersionSettings, moduleName,
-            compiledKotlinFiles.map { it.fqName }.distinct().sorted(),
-            emptyList()
-    ).toByteArray()
-
-    val serializedMetadata =
-            makeSerializedKlibMetadata(
-                    compiledKotlinFiles.groupBy { it.fqName }
-                            .map { (fqn, data) -> fqn to data.sortedBy { it.path }.map { it.metadata } }.toMap(),
-                    header
-            )
-
-    return SerializerOutput(serializedMetadata, serializedIr, null, dependencies.orEmpty())
-}

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/Serializer.kt

@@ -5,7 +5,6 @@
 
 package org.jetbrains.kotlin.backend.konan.driver.phases
 
-import org.jetbrains.kotlin.ir.KtDiagnosticReporterWithImplicitIrBasedContext
 import org.jetbrains.kotlin.backend.common.phaser.createSimpleNamedCompilerPhase
 import org.jetbrains.kotlin.backend.common.serialization.CompatibilityMode
 import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataMonolithicSerializer
@@ -16,10 +15,9 @@ import org.jetbrains.kotlin.config.CommonConfigurationKeys
 import org.jetbrains.kotlin.config.languageVersionSettings
 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
 import org.jetbrains.kotlin.diagnostics.DiagnosticReporterFactory
+import org.jetbrains.kotlin.ir.KtDiagnosticReporterWithImplicitIrBasedContext
 import org.jetbrains.kotlin.ir.util.IrMessageLogger
 import org.jetbrains.kotlin.konan.library.KonanLibrary
-import org.jetbrains.kotlin.library.SerializedIrModule
-import org.jetbrains.kotlin.library.SerializedMetadata
 
 internal data class SerializerInput(
         val moduleDescriptor: ModuleDescriptor,
@@ -27,12 +25,7 @@ internal data class SerializerInput(
         val produceHeaderKlib: Boolean,
 )
 
-data class SerializerOutput(
-        val serializedMetadata: SerializedMetadata?,
-        val serializedIr: SerializedIrModule?,
-        val dataFlowGraph: ByteArray?,
-        val neededLibraries: List<KonanLibrary>
-)
+typealias SerializerOutput = org.jetbrains.kotlin.backend.common.serialization.SerializerOutput<KonanLibrary>
 
 internal val SerializerPhase = createSimpleNamedCompilerPhase<PhaseContext, SerializerInput, SerializerOutput>(
         "Serializer", "IR serializer",