[FIR] KT-58444: Don't run IR generation when compiling native shared

Normally during metadata compilation we use
`K2MetadataCompiler` and it serializes
metadata to klibs without generating IR.

Native shared compilation uses the K2Native
compiler, which generates klibs based on both
the fir and fir2ir information.

It would be nice to reuse `K2MetadataCompiler`
for native shared, but currently it has no
support for the commonizer, and also reads
the stdlib metadata from a jar instead of a
klib, and there's no simple idiomatic way
to pass the `distribution.klib` value down
to `KLibResolverHelper` (native compilation
has some commonized dependencies, and they
explicitly depend on `stdlib`, so we must
resolve it this way or strip it away to be
able to use the metadata jar instead). There
may be other issues, I only digged a bit.

Instead, this change modifies the K2Native
in such a way that it can generate klibs
with metadata without fir2ir info.

Note that this change does not fix KT-58139,
because KT-58139 fails due to performing the
constants evaluation directly during the
metadata serialization when the needed info
does (and, as I was told, should) not exist
(namely, valid property initializers).

^KT-58444 Fixed

compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/metadata/KlibMetadataSerializer.kt

@@ -250,10 +250,22 @@ fun serializeKlibHeader(
     moduleDescriptor: ModuleDescriptor,
     fragmentNames: List<String>,
     emptyPackages: List<String>
+) = serializeKlibHeader(
+    languageVersionSettings,
+    moduleDescriptor.name.asString(),
+    fragmentNames,
+    emptyPackages,
+)
+
+fun serializeKlibHeader(
+    languageVersionSettings: LanguageVersionSettings,
+    moduleName: String,
+    fragmentNames: List<String>,
+    emptyPackages: List<String>
 ): KlibMetadataProtoBuf.Header {
     val header = KlibMetadataProtoBuf.Header.newBuilder()
 
-    header.moduleName = moduleDescriptor.name.asString()
+    header.moduleName = moduleName
 
     if (languageVersionSettings.isPreRelease()) {
         header.flags = KlibMetadataHeaderFlags.PRE_RELEASE

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/FirNativeSerializer.kt

@@ -6,6 +6,7 @@ import org.jetbrains.kotlin.backend.common.serialization.metadata.makeSerialized
 import org.jetbrains.kotlin.backend.common.serialization.metadata.serializeKlibHeader
 import org.jetbrains.kotlin.backend.konan.driver.PhaseContext
 import org.jetbrains.kotlin.backend.konan.driver.phases.Fir2IrOutput
+import org.jetbrains.kotlin.backend.konan.driver.phases.FirOutput
 import org.jetbrains.kotlin.backend.konan.driver.phases.SerializerOutput
 import org.jetbrains.kotlin.backend.konan.serialization.KonanIrModuleSerializer
 import org.jetbrains.kotlin.config.CommonConfigurationKeys
@@ -16,6 +17,9 @@ import org.jetbrains.kotlin.fir.FirSession
 import org.jetbrains.kotlin.fir.backend.ConstValueProviderImpl
 import org.jetbrains.kotlin.fir.backend.extractFirDeclarations
 import org.jetbrains.kotlin.fir.declarations.FirFile
+import org.jetbrains.kotlin.fir.moduleData
+import org.jetbrains.kotlin.fir.packageFqName
+import org.jetbrains.kotlin.fir.pipeline.FirResult
 import org.jetbrains.kotlin.fir.resolve.ScopeSession
 import org.jetbrains.kotlin.fir.serialization.*
 import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
@@ -27,14 +31,22 @@ import org.jetbrains.kotlin.library.metadata.resolver.TopologicalLibraryOrder
 import org.jetbrains.kotlin.metadata.ProtoBuf
 import org.jetbrains.kotlin.utils.toMetadataVersion
 
-internal fun PhaseContext.firSerializer(
+internal fun PhaseContext.firSerializer(input: FirOutput): SerializerOutput? = when (input) {
-        input: Fir2IrOutput
+    !is FirOutput.Full -> null
+    else -> firSerializerBase(input.firResult, null)
+}
+
+internal fun PhaseContext.fir2IrSerializer(input: Fir2IrOutput) = firSerializerBase(input.firResult, input)
+
+internal fun PhaseContext.firSerializerBase(
+        firResult: FirResult,
+        fir2IrInput: Fir2IrOutput?,
 ): SerializerOutput {
     val configuration = config.configuration
     val sourceFiles = mutableListOf<KtSourceFile>()
     val firFilesAndSessionsBySourceFile = mutableMapOf<KtSourceFile, Triple<FirFile, FirSession, ScopeSession>>()
 
-    for (firOutput in input.firResult.outputs) {
+    for (firOutput in firResult.outputs) {
         for (firFile in firOutput.fir) {
             sourceFiles.add(firFile.sourceFile!!)
             firFilesAndSessionsBySourceFile[firFile.sourceFile!!] = Triple(firFile, firOutput.session, firOutput.scopeSession)
@@ -45,21 +57,24 @@ internal fun PhaseContext.firSerializer(
             configuration.get(CommonConfigurationKeys.METADATA_VERSION)
                     ?: configuration.languageVersionSettings.languageVersion.toMetadataVersion()
 
-    val resolvedLibraries = config.resolvedLibraries.getFullResolvedList(TopologicalLibraryOrder)
+    val usedResolvedLibraries = fir2IrInput?.let {
-    val usedResolvedLibraries = resolvedLibraries.filter {
+        config.resolvedLibraries.getFullResolvedList(TopologicalLibraryOrder).filter {
-        (!it.isDefault && !configuration.getBoolean(KonanConfigKeys.PURGE_USER_LIBS)) || it in input.usedLibraries
+            (!it.isDefault && !configuration.getBoolean(KonanConfigKeys.PURGE_USER_LIBS)) || it in fir2IrInput.usedLibraries
+        }
     }
-    val actualizedFirDeclarations = input.irActualizedResult.extractFirDeclarations()
+
+    val actualizedFirDeclarations = fir2IrInput?.irActualizedResult?.extractFirDeclarations()
     return serializeNativeModule(
             configuration = configuration,
             messageLogger = configuration.get(IrMessageLogger.IR_MESSAGE_LOGGER) ?: IrMessageLogger.None,
             sourceFiles,
-            usedResolvedLibraries.map { it.library as KonanLibrary },
+            usedResolvedLibraries?.map { it.library as KonanLibrary },
-            input.irModuleFragment,
+            fir2IrInput?.irModuleFragment,
-            expectDescriptorToSymbol = mutableMapOf() // TODO: expect -> actual mapping
+            moduleName = fir2IrInput?.irModuleFragment?.descriptor?.name?.asString()
-    ) { file ->
+                    ?: firResult.outputs.last().session.moduleData.name.asString(),
-        val (firFile, session, scopeSession) = firFilesAndSessionsBySourceFile[file]
+            expectDescriptorToSymbol = mutableMapOf(), // TODO: expect -> actual mapping
-                ?: error("cannot find FIR file by source file ${file.name} (${file.path})")
+            firFilesAndSessionsBySourceFile,
+    ) { firFile, session, scopeSession ->
         serializeSingleFirFile(
                 firFile,
                 session,
@@ -67,7 +82,9 @@ internal fun PhaseContext.firSerializer(
                 actualizedFirDeclarations,
                 FirKLibSerializerExtension(
                         session, metadataVersion,
-                        ConstValueProviderImpl(input.components),
+                        fir2IrInput?.let {
+                            ConstValueProviderImpl(fir2IrInput.components)
+                        },
                         allowErrorTypes = false, exportKDoc = shouldExportKDoc()
                 ),
                 configuration.languageVersionSettings,
@@ -75,60 +92,77 @@ internal fun PhaseContext.firSerializer(
     }
 }
 
-class KotlinFileSerializedData(val metadata: ByteArray, val irData: SerializedIrFile)
+class KotlinFileSerializedData(
+        val source: KtSourceFile,
+        val firFile: FirFile,
+        val metadata: ByteArray,
+        val irData: SerializedIrFile?,
+) {
+    val fqName: String get() = irData?.fqName ?: firFile.packageFqName.asString()
+    val path: String? get() = irData?.path ?: source.path
+}
 
 internal fun PhaseContext.serializeNativeModule(
         configuration: CompilerConfiguration,
         messageLogger: IrMessageLogger,
         files: List<KtSourceFile>,
-        dependencies: List<KonanLibrary>,
+        dependencies: List<KonanLibrary>?,
-        moduleFragment: IrModuleFragment,
+        moduleFragment: IrModuleFragment?,
+        moduleName: String,
         expectDescriptorToSymbol: MutableMap<DeclarationDescriptor, IrSymbol>,
-        serializeSingleFile: (KtSourceFile) -> ProtoBuf.PackageFragment
+        firFilesAndSessionsBySourceFile: Map<KtSourceFile, Triple<FirFile, FirSession, ScopeSession>>,
+        serializeSingleFile: (FirFile, FirSession, ScopeSession) -> ProtoBuf.PackageFragment
 ): SerializerOutput {
-    assert(files.size == moduleFragment.files.size)
+    if (moduleFragment != null) {
+        assert(files.size == moduleFragment.files.size)
+    }
 
     val sourceBaseDirs = configuration[CommonConfigurationKeys.KLIB_RELATIVE_PATH_BASES] ?: emptyList()
     val absolutePathNormalization = configuration[CommonConfigurationKeys.KLIB_NORMALIZE_ABSOLUTE_PATH] ?: false
     val expectActualLinker = config.configuration.get(CommonConfigurationKeys.EXPECT_ACTUAL_LINKER) ?: false
 
-    val serializedIr =
+    val serializedIr = moduleFragment?.let {
-            KonanIrModuleSerializer(
+        KonanIrModuleSerializer(
-                    messageLogger,
+                messageLogger,
-                    moduleFragment.irBuiltins,
+                moduleFragment.irBuiltins,
-                    expectDescriptorToSymbol,
+                expectDescriptorToSymbol,
-                    skipExpects = !expectActualLinker,
+                skipExpects = !expectActualLinker,
-                    CompatibilityMode.CURRENT,
+                CompatibilityMode.CURRENT,
-                    normalizeAbsolutePaths = absolutePathNormalization,
+                normalizeAbsolutePaths = absolutePathNormalization,
-                    sourceBaseDirs = sourceBaseDirs,
+                sourceBaseDirs = sourceBaseDirs,
-                    languageVersionSettings = configuration.languageVersionSettings,
+                languageVersionSettings = configuration.languageVersionSettings,
-            ).serializedIrModule(moduleFragment)
+        ).serializedIrModule(moduleFragment)
+    }
 
-    val moduleDescriptor = moduleFragment.descriptor
+    val serializedFiles = serializedIr?.files?.toList()
 
-    val compiledKotlinFiles = files.zip(serializedIr.files).map { (ktSourceFile, binaryFile) ->
+    val compiledKotlinFiles = files.mapIndexed { index, ktSourceFile ->
-        assert(ktSourceFile.path == binaryFile.path) {
+        val binaryFile = serializedFiles?.get(index)?.also {
-            """The Kt and Ir files are put in different order
+            assert(ktSourceFile.path == it.path) {
+                """The Kt and Ir files are put in different order
                 Kt: ${ktSourceFile.path}
-                Ir: ${binaryFile.path}
+                Ir: ${it.path}
             """.trimMargin()
+            }
         }
-        val packageFragment = serializeSingleFile(ktSourceFile)
+        val (firFile, session, scopeSession) = firFilesAndSessionsBySourceFile[ktSourceFile]
-        KotlinFileSerializedData(packageFragment.toByteArray(), binaryFile)
+                ?: error("cannot find FIR file by source file ${ktSourceFile.name} (${ktSourceFile.path})")
+        val packageFragment = serializeSingleFile(firFile, session, scopeSession)
+        KotlinFileSerializedData(ktSourceFile, firFile, packageFragment.toByteArray(), binaryFile)
     }
 
     val header = serializeKlibHeader(
-            configuration.languageVersionSettings, moduleDescriptor,
+            configuration.languageVersionSettings, moduleName,
-            compiledKotlinFiles.map { it.irData.fqName }.distinct().sorted(),
+            compiledKotlinFiles.map { it.fqName }.distinct().sorted(),
             emptyList()
     ).toByteArray()
 
     val serializedMetadata =
             makeSerializedKlibMetadata(
-                    compiledKotlinFiles.groupBy { it.irData.fqName }
+                    compiledKotlinFiles.groupBy { it.fqName }
-                            .map { (fqn, data) -> fqn to data.sortedBy { it.irData.path }.map { it.metadata } }.toMap(),
+                            .map { (fqn, data) -> fqn to data.sortedBy { it.path }.map { it.metadata } }.toMap(),
                     header
             )
 
-    return SerializerOutput(serializedMetadata, serializedIr, null, dependencies)
+    return SerializerOutput(serializedMetadata, serializedIr, null, dependencies.orEmpty())
 }

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/DynamicCompilerDriver.kt

@@ -103,9 +103,13 @@ internal class DynamicCompilerDriver : CompilerDriver() {
         if (frontendOutput is FirOutput.ShouldNotGenerateCode) return null
         require(frontendOutput is FirOutput.Full)
 
-        val fir2IrOutput = engine.runFir2Ir(frontendOutput)
+        return if (environment.configuration.getBoolean(KonanConfigKeys.METADATA_KLIB)) {
-        engine.runK2SpecialBackendChecks(fir2IrOutput)
+            engine.runFirSerializer(frontendOutput)
-        return engine.runFirSerializer(fir2IrOutput)
+        } else {
+            val fir2IrOutput = engine.runFir2Ir(frontendOutput)
+            engine.runK2SpecialBackendChecks(fir2IrOutput)
+            engine.runFir2IrSerializer(fir2IrOutput)
+        }
     }
 
     private fun serializeKlibK1(

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/FirSerializer.kt

@@ -8,16 +8,30 @@ package org.jetbrains.kotlin.backend.konan.driver.phases
 import org.jetbrains.kotlin.backend.konan.driver.PhaseContext
 import org.jetbrains.kotlin.backend.konan.driver.PhaseEngine
 import org.jetbrains.kotlin.backend.konan.firSerializer
+import org.jetbrains.kotlin.backend.konan.fir2IrSerializer
 
-internal val FirSerializerPhase = createSimpleNamedCompilerPhase<PhaseContext, Fir2IrOutput, SerializerOutput>(
+internal val FirSerializerPhase = createSimpleNamedCompilerPhase<PhaseContext, FirOutput, SerializerOutput?>(
         "FirSerializer", "Fir serializer",
         outputIfNotEnabled = { _, _, _, _ -> SerializerOutput(null, null, null, listOf()) }
-) { context: PhaseContext, input: Fir2IrOutput ->
+) { context: PhaseContext, input: FirOutput ->
     context.firSerializer(input)
 }
 
+internal val Fir2IrSerializerPhase = createSimpleNamedCompilerPhase<PhaseContext, Fir2IrOutput, SerializerOutput>(
+        "Fir2IrSerializer", "Fir2Ir serializer",
+        outputIfNotEnabled = { _, _, _, _ -> SerializerOutput(null, null, null, listOf()) }
+) { context: PhaseContext, input: Fir2IrOutput ->
+    context.fir2IrSerializer(input)
+}
+
 internal fun <T : PhaseContext> PhaseEngine<T>.runFirSerializer(
+        firOutput: FirOutput
+): SerializerOutput? {
+    return this.runPhase(FirSerializerPhase, firOutput)
+}
+
+internal fun <T : PhaseContext> PhaseEngine<T>.runFir2IrSerializer(
         fir2irOutput: Fir2IrOutput
 ): SerializerOutput {
-    return this.runPhase(FirSerializerPhase, fir2irOutput)
+    return this.runPhase(Fir2IrSerializerPhase, fir2irOutput)
 }