From 00e8dc1c39eda63a3db2bb09504ef45f07e27017 Mon Sep 17 00:00:00 2001 From: Nikolay Lunyak Date: Mon, 22 May 2023 12:34:00 +0300 Subject: [PATCH] [FIR] KT-58444: Don't run IR generation when compiling native shared Normally during metadata compilation we use `K2MetadataCompiler` and it serializes metadata to klibs without generating IR. Native shared compilation uses the K2Native compiler, which generates klibs based on both the fir and fir2ir information. It would be nice to reuse `K2MetadataCompiler` for native shared, but currently it has no support for the commonizer, and also reads the stdlib metadata from a jar instead of a klib, and there's no simple idiomatic way to pass the `distribution.klib` value down to `KLibResolverHelper` (native compilation has some commonized dependencies, and they explicitly depend on `stdlib`, so we must resolve it this way or strip it away to be able to use the metadata jar instead). There may be other issues, I only digged a bit. Instead, this change modifies the K2Native in such a way that it can generate klibs with metadata without fir2ir info. Note that this change does not fix KT-58139, because KT-58139 fails due to performing the constants evaluation directly during the metadata serialization when the needed info does (and, as I was told, should) not exist (namely, valid property initializers). ^KT-58444 Fixed --- .../metadata/KlibMetadataSerializer.kt | 14 ++- .../backend/konan/FirNativeSerializer.kt | 118 +++++++++++------- .../konan/driver/DynamicCompilerDriver.kt | 10 +- .../konan/driver/phases/FirSerializer.kt | 20 ++- 4 files changed, 113 insertions(+), 49 deletions(-) diff --git a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/metadata/KlibMetadataSerializer.kt b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/metadata/KlibMetadataSerializer.kt index a695e7951ee..d5f8d70ca5b 100644 --- a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/metadata/KlibMetadataSerializer.kt +++ b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/metadata/KlibMetadataSerializer.kt @@ -250,10 +250,22 @@ fun serializeKlibHeader( moduleDescriptor: ModuleDescriptor, fragmentNames: List, emptyPackages: List +) = serializeKlibHeader( + languageVersionSettings, + moduleDescriptor.name.asString(), + fragmentNames, + emptyPackages, +) + +fun serializeKlibHeader( + languageVersionSettings: LanguageVersionSettings, + moduleName: String, + fragmentNames: List, + emptyPackages: List ): KlibMetadataProtoBuf.Header { val header = KlibMetadataProtoBuf.Header.newBuilder() - header.moduleName = moduleDescriptor.name.asString() + header.moduleName = moduleName if (languageVersionSettings.isPreRelease()) { header.flags = KlibMetadataHeaderFlags.PRE_RELEASE diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/FirNativeSerializer.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/FirNativeSerializer.kt index f027c9aeda4..82b5d5d2572 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/FirNativeSerializer.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/FirNativeSerializer.kt @@ -6,6 +6,7 @@ import org.jetbrains.kotlin.backend.common.serialization.metadata.makeSerialized import org.jetbrains.kotlin.backend.common.serialization.metadata.serializeKlibHeader import org.jetbrains.kotlin.backend.konan.driver.PhaseContext import org.jetbrains.kotlin.backend.konan.driver.phases.Fir2IrOutput +import org.jetbrains.kotlin.backend.konan.driver.phases.FirOutput import org.jetbrains.kotlin.backend.konan.driver.phases.SerializerOutput import org.jetbrains.kotlin.backend.konan.serialization.KonanIrModuleSerializer import org.jetbrains.kotlin.config.CommonConfigurationKeys @@ -16,6 +17,9 @@ import org.jetbrains.kotlin.fir.FirSession import org.jetbrains.kotlin.fir.backend.ConstValueProviderImpl import org.jetbrains.kotlin.fir.backend.extractFirDeclarations import org.jetbrains.kotlin.fir.declarations.FirFile +import org.jetbrains.kotlin.fir.moduleData +import org.jetbrains.kotlin.fir.packageFqName +import org.jetbrains.kotlin.fir.pipeline.FirResult import org.jetbrains.kotlin.fir.resolve.ScopeSession import org.jetbrains.kotlin.fir.serialization.* import org.jetbrains.kotlin.ir.declarations.IrModuleFragment @@ -27,14 +31,22 @@ import org.jetbrains.kotlin.library.metadata.resolver.TopologicalLibraryOrder import org.jetbrains.kotlin.metadata.ProtoBuf import org.jetbrains.kotlin.utils.toMetadataVersion -internal fun PhaseContext.firSerializer( - input: Fir2IrOutput +internal fun PhaseContext.firSerializer(input: FirOutput): SerializerOutput? = when (input) { + !is FirOutput.Full -> null + else -> firSerializerBase(input.firResult, null) +} + +internal fun PhaseContext.fir2IrSerializer(input: Fir2IrOutput) = firSerializerBase(input.firResult, input) + +internal fun PhaseContext.firSerializerBase( + firResult: FirResult, + fir2IrInput: Fir2IrOutput?, ): SerializerOutput { val configuration = config.configuration val sourceFiles = mutableListOf() val firFilesAndSessionsBySourceFile = mutableMapOf>() - for (firOutput in input.firResult.outputs) { + for (firOutput in firResult.outputs) { for (firFile in firOutput.fir) { sourceFiles.add(firFile.sourceFile!!) firFilesAndSessionsBySourceFile[firFile.sourceFile!!] = Triple(firFile, firOutput.session, firOutput.scopeSession) @@ -45,21 +57,24 @@ internal fun PhaseContext.firSerializer( configuration.get(CommonConfigurationKeys.METADATA_VERSION) ?: configuration.languageVersionSettings.languageVersion.toMetadataVersion() - val resolvedLibraries = config.resolvedLibraries.getFullResolvedList(TopologicalLibraryOrder) - val usedResolvedLibraries = resolvedLibraries.filter { - (!it.isDefault && !configuration.getBoolean(KonanConfigKeys.PURGE_USER_LIBS)) || it in input.usedLibraries + val usedResolvedLibraries = fir2IrInput?.let { + config.resolvedLibraries.getFullResolvedList(TopologicalLibraryOrder).filter { + (!it.isDefault && !configuration.getBoolean(KonanConfigKeys.PURGE_USER_LIBS)) || it in fir2IrInput.usedLibraries + } } - val actualizedFirDeclarations = input.irActualizedResult.extractFirDeclarations() + + val actualizedFirDeclarations = fir2IrInput?.irActualizedResult?.extractFirDeclarations() return serializeNativeModule( configuration = configuration, messageLogger = configuration.get(IrMessageLogger.IR_MESSAGE_LOGGER) ?: IrMessageLogger.None, sourceFiles, - usedResolvedLibraries.map { it.library as KonanLibrary }, - input.irModuleFragment, - expectDescriptorToSymbol = mutableMapOf() // TODO: expect -> actual mapping - ) { file -> - val (firFile, session, scopeSession) = firFilesAndSessionsBySourceFile[file] - ?: error("cannot find FIR file by source file ${file.name} (${file.path})") + usedResolvedLibraries?.map { it.library as KonanLibrary }, + fir2IrInput?.irModuleFragment, + moduleName = fir2IrInput?.irModuleFragment?.descriptor?.name?.asString() + ?: firResult.outputs.last().session.moduleData.name.asString(), + expectDescriptorToSymbol = mutableMapOf(), // TODO: expect -> actual mapping + firFilesAndSessionsBySourceFile, + ) { firFile, session, scopeSession -> serializeSingleFirFile( firFile, session, @@ -67,7 +82,9 @@ internal fun PhaseContext.firSerializer( actualizedFirDeclarations, FirKLibSerializerExtension( session, metadataVersion, - ConstValueProviderImpl(input.components), + fir2IrInput?.let { + ConstValueProviderImpl(fir2IrInput.components) + }, allowErrorTypes = false, exportKDoc = shouldExportKDoc() ), configuration.languageVersionSettings, @@ -75,60 +92,77 @@ internal fun PhaseContext.firSerializer( } } -class KotlinFileSerializedData(val metadata: ByteArray, val irData: SerializedIrFile) +class KotlinFileSerializedData( + val source: KtSourceFile, + val firFile: FirFile, + val metadata: ByteArray, + val irData: SerializedIrFile?, +) { + val fqName: String get() = irData?.fqName ?: firFile.packageFqName.asString() + val path: String? get() = irData?.path ?: source.path +} internal fun PhaseContext.serializeNativeModule( configuration: CompilerConfiguration, messageLogger: IrMessageLogger, files: List, - dependencies: List, - moduleFragment: IrModuleFragment, + dependencies: List?, + moduleFragment: IrModuleFragment?, + moduleName: String, expectDescriptorToSymbol: MutableMap, - serializeSingleFile: (KtSourceFile) -> ProtoBuf.PackageFragment + firFilesAndSessionsBySourceFile: Map>, + serializeSingleFile: (FirFile, FirSession, ScopeSession) -> ProtoBuf.PackageFragment ): SerializerOutput { - assert(files.size == moduleFragment.files.size) + if (moduleFragment != null) { + assert(files.size == moduleFragment.files.size) + } val sourceBaseDirs = configuration[CommonConfigurationKeys.KLIB_RELATIVE_PATH_BASES] ?: emptyList() val absolutePathNormalization = configuration[CommonConfigurationKeys.KLIB_NORMALIZE_ABSOLUTE_PATH] ?: false val expectActualLinker = config.configuration.get(CommonConfigurationKeys.EXPECT_ACTUAL_LINKER) ?: false - val serializedIr = - KonanIrModuleSerializer( - messageLogger, - moduleFragment.irBuiltins, - expectDescriptorToSymbol, - skipExpects = !expectActualLinker, - CompatibilityMode.CURRENT, - normalizeAbsolutePaths = absolutePathNormalization, - sourceBaseDirs = sourceBaseDirs, - languageVersionSettings = configuration.languageVersionSettings, - ).serializedIrModule(moduleFragment) + val serializedIr = moduleFragment?.let { + KonanIrModuleSerializer( + messageLogger, + moduleFragment.irBuiltins, + expectDescriptorToSymbol, + skipExpects = !expectActualLinker, + CompatibilityMode.CURRENT, + normalizeAbsolutePaths = absolutePathNormalization, + sourceBaseDirs = sourceBaseDirs, + languageVersionSettings = configuration.languageVersionSettings, + ).serializedIrModule(moduleFragment) + } - val moduleDescriptor = moduleFragment.descriptor + val serializedFiles = serializedIr?.files?.toList() - val compiledKotlinFiles = files.zip(serializedIr.files).map { (ktSourceFile, binaryFile) -> - assert(ktSourceFile.path == binaryFile.path) { - """The Kt and Ir files are put in different order + val compiledKotlinFiles = files.mapIndexed { index, ktSourceFile -> + val binaryFile = serializedFiles?.get(index)?.also { + assert(ktSourceFile.path == it.path) { + """The Kt and Ir files are put in different order Kt: ${ktSourceFile.path} - Ir: ${binaryFile.path} + Ir: ${it.path} """.trimMargin() + } } - val packageFragment = serializeSingleFile(ktSourceFile) - KotlinFileSerializedData(packageFragment.toByteArray(), binaryFile) + val (firFile, session, scopeSession) = firFilesAndSessionsBySourceFile[ktSourceFile] + ?: error("cannot find FIR file by source file ${ktSourceFile.name} (${ktSourceFile.path})") + val packageFragment = serializeSingleFile(firFile, session, scopeSession) + KotlinFileSerializedData(ktSourceFile, firFile, packageFragment.toByteArray(), binaryFile) } val header = serializeKlibHeader( - configuration.languageVersionSettings, moduleDescriptor, - compiledKotlinFiles.map { it.irData.fqName }.distinct().sorted(), + configuration.languageVersionSettings, moduleName, + compiledKotlinFiles.map { it.fqName }.distinct().sorted(), emptyList() ).toByteArray() val serializedMetadata = makeSerializedKlibMetadata( - compiledKotlinFiles.groupBy { it.irData.fqName } - .map { (fqn, data) -> fqn to data.sortedBy { it.irData.path }.map { it.metadata } }.toMap(), + compiledKotlinFiles.groupBy { it.fqName } + .map { (fqn, data) -> fqn to data.sortedBy { it.path }.map { it.metadata } }.toMap(), header ) - return SerializerOutput(serializedMetadata, serializedIr, null, dependencies) + return SerializerOutput(serializedMetadata, serializedIr, null, dependencies.orEmpty()) } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/DynamicCompilerDriver.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/DynamicCompilerDriver.kt index 981717f10c6..83dc743bec7 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/DynamicCompilerDriver.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/DynamicCompilerDriver.kt @@ -103,9 +103,13 @@ internal class DynamicCompilerDriver : CompilerDriver() { if (frontendOutput is FirOutput.ShouldNotGenerateCode) return null require(frontendOutput is FirOutput.Full) - val fir2IrOutput = engine.runFir2Ir(frontendOutput) - engine.runK2SpecialBackendChecks(fir2IrOutput) - return engine.runFirSerializer(fir2IrOutput) + return if (environment.configuration.getBoolean(KonanConfigKeys.METADATA_KLIB)) { + engine.runFirSerializer(frontendOutput) + } else { + val fir2IrOutput = engine.runFir2Ir(frontendOutput) + engine.runK2SpecialBackendChecks(fir2IrOutput) + engine.runFir2IrSerializer(fir2IrOutput) + } } private fun serializeKlibK1( diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/FirSerializer.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/FirSerializer.kt index 66bd254bfa6..5624604b93a 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/FirSerializer.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/FirSerializer.kt @@ -8,16 +8,30 @@ package org.jetbrains.kotlin.backend.konan.driver.phases import org.jetbrains.kotlin.backend.konan.driver.PhaseContext import org.jetbrains.kotlin.backend.konan.driver.PhaseEngine import org.jetbrains.kotlin.backend.konan.firSerializer +import org.jetbrains.kotlin.backend.konan.fir2IrSerializer -internal val FirSerializerPhase = createSimpleNamedCompilerPhase( +internal val FirSerializerPhase = createSimpleNamedCompilerPhase( "FirSerializer", "Fir serializer", outputIfNotEnabled = { _, _, _, _ -> SerializerOutput(null, null, null, listOf()) } -) { context: PhaseContext, input: Fir2IrOutput -> +) { context: PhaseContext, input: FirOutput -> context.firSerializer(input) } +internal val Fir2IrSerializerPhase = createSimpleNamedCompilerPhase( + "Fir2IrSerializer", "Fir2Ir serializer", + outputIfNotEnabled = { _, _, _, _ -> SerializerOutput(null, null, null, listOf()) } +) { context: PhaseContext, input: Fir2IrOutput -> + context.fir2IrSerializer(input) +} + internal fun PhaseEngine.runFirSerializer( + firOutput: FirOutput +): SerializerOutput? { + return this.runPhase(FirSerializerPhase, firOutput) +} + +internal fun PhaseEngine.runFir2IrSerializer( fir2irOutput: Fir2IrOutput ): SerializerOutput { - return this.runPhase(FirSerializerPhase, fir2irOutput) + return this.runPhase(Fir2IrSerializerPhase, fir2irOutput) } \ No newline at end of file