From 93a1ceae757174f26687138bf03030ecaa80d620 Mon Sep 17 00:00:00 2001 From: Sergej Jaskiewicz Date: Fri, 19 Jan 2024 19:19:08 +0100 Subject: [PATCH] [klib] Deduplicate KLIB serialization logic in K/JS and K/N Introduce the new `serializeModuleIntoKlib` function that all KLIB-based backends can use to serialize a module to a KLIB. ^KT-64392 Fixed --- .../jetbrains/kotlin/cli/js/klib/irForKlib.kt | 4 +- .../serialization/serializeModuleIntoKlib.kt | 200 ++++++++++++++++++ .../jetbrains/kotlin/ir/backend/js/klib.kt | 161 ++++++-------- .../kotlin/test/backend/ir/IrBackendInput.kt | 3 +- .../fir/Fir2IrJsWasmResultsConverter.kt | 2 +- .../kotlin/klib/AbstractKlibIrTextTestCase.kt | 105 +++------ .../backend/konan/FirNativeSerializer.kt | 172 +++++---------- .../backend/konan/driver/phases/Serializer.kt | 11 +- 8 files changed, 348 insertions(+), 310 deletions(-) create mode 100644 compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/serializeModuleIntoKlib.kt diff --git a/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/klib/irForKlib.kt b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/klib/irForKlib.kt index bb7ca1a7469..01d325097a5 100644 --- a/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/klib/irForKlib.kt +++ b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/klib/irForKlib.kt @@ -19,13 +19,13 @@ import org.jetbrains.kotlin.backend.common.lower.ExpectDeclarationRemover import org.jetbrains.kotlin.backend.common.overrides.FakeOverrideChecker import org.jetbrains.kotlin.backend.common.serialization.DescriptorByIdSignatureFinderImpl import org.jetbrains.kotlin.backend.common.serialization.ICData +import org.jetbrains.kotlin.backend.common.serialization.KotlinFileSerializedData import org.jetbrains.kotlin.backend.common.serialization.mangle.ManglerChecker import org.jetbrains.kotlin.backend.common.serialization.mangle.descriptor.Ir2DescriptorManglerAdapter import org.jetbrains.kotlin.backend.common.serialization.signature.IdSignatureDescriptor import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.config.languageVersionSettings import org.jetbrains.kotlin.descriptors.ModuleDescriptor -import org.jetbrains.kotlin.ir.backend.js.KotlinFileSerializedData import org.jetbrains.kotlin.ir.backend.js.generateModuleFragmentWithPlugins import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsManglerDesc @@ -87,7 +87,7 @@ fun generateIrForKlibSerialization( messageLogger = messageLogger ), feContext, - ICData(icData.map { it.irData }, errorPolicy.allowErrors), + ICData(icData.map { it.irData!! }, errorPolicy.allowErrors), stubGenerator = stubGenerator ) diff --git a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/serializeModuleIntoKlib.kt b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/serializeModuleIntoKlib.kt new file mode 100644 index 00000000000..887e68b6d63 --- /dev/null +++ b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/serializeModuleIntoKlib.kt @@ -0,0 +1,200 @@ +/* + * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.backend.common.serialization + +import com.intellij.openapi.vfs.VfsUtilCore +import org.jetbrains.kotlin.KtIoFileSourceFile +import org.jetbrains.kotlin.KtPsiSourceFile +import org.jetbrains.kotlin.KtSourceFile +import org.jetbrains.kotlin.KtVirtualFileSourceFile +import org.jetbrains.kotlin.backend.common.serialization.metadata.makeSerializedKlibMetadata +import org.jetbrains.kotlin.backend.common.serialization.metadata.serializeKlibHeader +import org.jetbrains.kotlin.config.CommonConfigurationKeys +import org.jetbrains.kotlin.config.CompilerConfiguration +import org.jetbrains.kotlin.config.LanguageVersionSettings +import org.jetbrains.kotlin.config.languageVersionSettings +import org.jetbrains.kotlin.diagnostics.DiagnosticReporter +import org.jetbrains.kotlin.ir.IrBuiltIns +import org.jetbrains.kotlin.ir.IrDiagnosticReporter +import org.jetbrains.kotlin.ir.KtDiagnosticReporterWithImplicitIrBasedContext +import org.jetbrains.kotlin.ir.declarations.IrModuleFragment +import org.jetbrains.kotlin.library.KotlinLibrary +import org.jetbrains.kotlin.library.SerializedIrFile +import org.jetbrains.kotlin.library.SerializedIrModule +import org.jetbrains.kotlin.library.SerializedMetadata +import org.jetbrains.kotlin.metadata.ProtoBuf +import org.jetbrains.kotlin.name.FqName +import java.io.File + +/** + * Holds the binary data for a single Kotlin file to be written to a KLIB, i.e., its metadata and IR (unless it's a metadata-only KLIB). + * + * @property metadata Serialized metadata of the corresponding source file. + * @property irData Serialized IR for this file, or `null` if this is a metadata-only KLIB. + * @property path The path to the corresponding source file, or `null` if that source file didn't have a path. + * @property fqName The fully qualified name of the package containing the serialized file. + */ +class KotlinFileSerializedData private constructor( + val metadata: ByteArray, + val irData: SerializedIrFile?, + val path: String?, + val fqName: String, +) { + + /** + * Used for creating file serialization data for IR-containing KLIBs. + * + * @param metadata Serialized metadata of the corresponding source file. + * @param irData Serialized IR for this file. + */ + constructor(metadata: ByteArray, irData: SerializedIrFile) : this(metadata, irData, irData.path, irData.fqName) + + /** + * Used for creating file serialization data in metadata-only KLIBs. + * + * @param metadata Serialized metadata of the corresponding source file. + * @param path The path of the serialized file. + * @param fqName The fully qualified name of the package containing the serialized file. + */ + constructor(metadata: ByteArray, path: String?, fqName: String) : this(metadata, irData = null, path, fqName) +} + +class SerializerOutput( + val serializedMetadata: SerializedMetadata?, + val serializedIr: SerializedIrModule?, + val dataFlowGraph: ByteArray?, + val neededLibraries: List, +) + +fun KtSourceFile.toIoFileOrNull(): File? = when (this) { + is KtIoFileSourceFile -> file + is KtVirtualFileSourceFile -> VfsUtilCore.virtualToIoFile(virtualFile) + is KtPsiSourceFile -> VfsUtilCore.virtualToIoFile(psiFile.virtualFile) + else -> path?.let(::File) +} + +/** + * Produces all the necessary binary data for writing a KLIB for a Kotlin module, including its metadata. + * + * If [irModuleFragment] is not `null`, serializes the module's IR into binary form by running [IrModuleSerializer]. + * + * For producing a metadata-only KLIB, pass `null` to [irModuleFragment]. + * + * @param moduleName The name of the module being serialized to be written into the KLIB header. + * @param irModuleFragment The IR to be serialized into the KLIB being produced, or `null` if this is going to be a metadata-only KLIB. + * @param configuration Used to determine certain serialization parameters and enable/disable serialization diagnostics. + * @param diagnosticReporter Used for reporting serialization-time diagnostics, for example, about clashing IR signatures. + * @param sourceFiles The source files from which the KLIB is being compiled. Used for serializing file metadata. + * @param compatibilityMode The information about KLIB ABI. + * @param cleanFiles In the case of incremental compilation, the list of files that were not changed and therefore don't need to be + * serialized again. + * @param dependencies The list of KLIBs that the KLIB being produced depends on. + * @param createModuleSerializer Used for creating a backend-specific instance of [IrModuleSerializer]. + * @param serializeFileMetadata The name of this argument kind of speaks for itself. + * @param runKlibCheckers Additional checks to be run before serializing [irModuleFragment]. Can be used to report serialization-time + * diagnostics. + * @param processCompiledFileData Called for each newly serialized file. Useful for incremental compilation. + * @param processKlibHeader Called after serializing the KLIB header. Useful for incremental compilation. + */ +fun serializeModuleIntoKlib( + moduleName: String, + irModuleFragment: IrModuleFragment?, + configuration: CompilerConfiguration, + diagnosticReporter: DiagnosticReporter, + sourceFiles: List, + compatibilityMode: CompatibilityMode, + cleanFiles: List, + dependencies: List, + createModuleSerializer: ( + irDiagnosticReporter: IrDiagnosticReporter, + irBuiltins: IrBuiltIns, + compatibilityMode: CompatibilityMode, + normalizeAbsolutePaths: Boolean, + sourceBaseDirs: Collection, + languageVersionSettings: LanguageVersionSettings, + shouldCheckSignaturesOnUniqueness: Boolean, + ) -> IrModuleSerializer<*>, + serializeFileMetadata: (KtSourceFile) -> Pair, + runKlibCheckers: (IrModuleFragment, IrDiagnosticReporter, CompilerConfiguration) -> Unit = { _, _, _ -> }, + processCompiledFileData: ((File, KotlinFileSerializedData) -> Unit)? = null, + processKlibHeader: (ByteArray) -> Unit = {}, +): SerializerOutput { + if (irModuleFragment != null) { + assert(sourceFiles.size == irModuleFragment.files.size) { + "The number of source files (${sourceFiles.size}) does not match the number of IrFiles (${irModuleFragment.files.size})" + } + } + + val sourceBaseDirs = configuration[CommonConfigurationKeys.KLIB_RELATIVE_PATH_BASES] ?: emptyList() + val normalizeAbsolutePath = configuration.getBoolean(CommonConfigurationKeys.KLIB_NORMALIZE_ABSOLUTE_PATH) + val signatureClashChecks = configuration[CommonConfigurationKeys.PRODUCE_KLIB_SIGNATURES_CLASH_CHECKS] ?: true + + val serializedIr = irModuleFragment?.let { + val irDiagnosticReporter = KtDiagnosticReporterWithImplicitIrBasedContext(diagnosticReporter, configuration.languageVersionSettings) + runKlibCheckers(it, irDiagnosticReporter, configuration) + createModuleSerializer( + irDiagnosticReporter, + it.irBuiltins, + compatibilityMode, + normalizeAbsolutePath, + sourceBaseDirs, + configuration.languageVersionSettings, + signatureClashChecks, + ).serializedIrModule(it) + } + + val serializedFiles = serializedIr?.files?.toList() + + val compiledKotlinFiles = sourceFiles.mapIndexedTo(cleanFiles.toMutableList()) { i, ktSourceFile -> + val binaryFile = serializedFiles?.get(i)?.also { + assert(ktSourceFile.path == it.path) { + """The Kt and Ir files are put in different order + Kt: ${ktSourceFile.path} + Ir: ${it.path} + """.trimMargin() + } + } + val (packageFragment, fqName) = serializeFileMetadata(ktSourceFile) + val metadata = packageFragment.toByteArray() + val compiledKotlinFile = if (binaryFile == null) + KotlinFileSerializedData(metadata, ktSourceFile.path, fqName.asString()) + else + KotlinFileSerializedData(metadata, binaryFile) + + if (processCompiledFileData != null) { + val ioFile = ktSourceFile.toIoFileOrNull() ?: error("No file found for source ${ktSourceFile.path}") + processCompiledFileData(ioFile, compiledKotlinFile) + } + + compiledKotlinFile + } + + val header = serializeKlibHeader( + languageVersionSettings = configuration.languageVersionSettings, + moduleName = moduleName, + fragmentNames = compiledKotlinFiles.map { it.fqName }.distinct().sorted(), + emptyPackages = emptyList(), + ).toByteArray() + + processKlibHeader(header) + + val serializedMetadata = makeSerializedKlibMetadata( + fragments = compiledKotlinFiles + .groupBy { it.fqName } + .map { (fqn, data) -> + fqn to data.sortedBy { it.path }.map { it.metadata } + } + .toMap(), + header = header, + ) + + return SerializerOutput( + serializedMetadata = serializedMetadata, + serializedIr = if (serializedIr == null) null else SerializedIrModule(compiledKotlinFiles.mapNotNull { it.irData }), + dataFlowGraph = null, + neededLibraries = dependencies, + ) +} diff --git a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt index ae7612d1eb0..65f160d0c95 100644 --- a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt +++ b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt @@ -34,7 +34,6 @@ import org.jetbrains.kotlin.diagnostics.DiagnosticReporter import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.incremental.js.IncrementalDataProvider import org.jetbrains.kotlin.ir.IrBuiltIns -import org.jetbrains.kotlin.ir.KtDiagnosticReporterWithImplicitIrBasedContext import org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI import org.jetbrains.kotlin.ir.backend.js.checkers.JsKlibCheckers import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.* @@ -57,6 +56,7 @@ import org.jetbrains.kotlin.library.impl.buildKotlinLibrary import org.jetbrains.kotlin.library.metadata.KlibMetadataFactories import org.jetbrains.kotlin.library.metadata.KlibMetadataVersion import org.jetbrains.kotlin.metadata.ProtoBuf +import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.progress.IncrementalNextRoundException import org.jetbrains.kotlin.progress.ProgressIndicatorAndCompilationCanceledStatus import org.jetbrains.kotlin.psi.KtFile @@ -100,8 +100,6 @@ private val CompilerConfiguration.metadataVersion internal val SerializedIrFile.fileMetadata: ByteArray get() = backendSpecificMetadata ?: error("Expect file caches to have backendSpecificMetadata, but '$path' doesn't") -class KotlinFileSerializedData(val metadata: ByteArray, val irData: SerializedIrFile) - fun generateKLib( depsDescriptors: ModulesStructure, outputKlibPath: String, @@ -624,100 +622,72 @@ fun serializeModuleIntoKlib( builtInsPlatform: BuiltInsPlatform = BuiltInsPlatform.JS, serializeSingleFile: (KtSourceFile) -> ProtoBuf.PackageFragment ) { - assert(files.size == moduleFragment.files.size) - - val compatibilityMode = CompatibilityMode(abiVersion) - val sourceBaseDirs = configuration[CommonConfigurationKeys.KLIB_RELATIVE_PATH_BASES] ?: emptyList() - val absolutePathNormalization = configuration[CommonConfigurationKeys.KLIB_NORMALIZE_ABSOLUTE_PATH] ?: false - val signatureClashChecks = configuration[CommonConfigurationKeys.PRODUCE_KLIB_SIGNATURES_CLASH_CHECKS] ?: true - val moduleExportedNames = moduleFragment.collectExportedNames() - - val irDiagnosticReporter = KtDiagnosticReporterWithImplicitIrBasedContext(diagnosticReporter, configuration.languageVersionSettings) - if (builtInsPlatform == BuiltInsPlatform.JS) { - val cleanFilesIrData = cleanFiles.map { it.irData } - JsKlibCheckers.check(cleanFilesIrData, moduleFragment, moduleExportedNames, irDiagnosticReporter, configuration) - } - - val serializedIr = - JsIrModuleSerializer( - irDiagnosticReporter, - moduleFragment.irBuiltins, - compatibilityMode, - normalizeAbsolutePaths = absolutePathNormalization, - sourceBaseDirs = sourceBaseDirs, - configuration.languageVersionSettings, - signatureClashChecks, - ) { JsIrFileMetadata(moduleExportedNames[it]?.values?.toSmartList() ?: emptyList()) }.serializedIrModule(moduleFragment) - - val moduleDescriptor = moduleFragment.descriptor - val incrementalResultsConsumer = configuration.get(JSConfigurationKeys.INCREMENTAL_RESULTS_CONSUMER) val empty = ByteArray(0) - - fun processCompiledFileData(ioFile: File, compiledFile: KotlinFileSerializedData) { - incrementalResultsConsumer?.run { - processPackagePart(ioFile, compiledFile.metadata, empty, empty) - with(compiledFile.irData) { - processIrFile( - ioFile, - fileData, - types, - signatures, - strings, - declarations, - bodies, - fqName.toByteArray(), - fileMetadata, - debugInfo, - ) + val serializerOutput = serializeModuleIntoKlib( + moduleName = moduleFragment.name.asString(), + irModuleFragment = moduleFragment, + configuration = configuration, + diagnosticReporter = diagnosticReporter, + sourceFiles = files, + compatibilityMode = CompatibilityMode(abiVersion), + cleanFiles = cleanFiles, + dependencies = dependencies, + createModuleSerializer = { + irDiagnosticReporter, + irBuiltins, + compatibilityMode, + normalizeAbsolutePaths, + sourceBaseDirs, + languageVersionSettings, + shouldCheckSignaturesOnUniqueness, + -> + JsIrModuleSerializer( + irDiagnosticReporter, + irBuiltins, + compatibilityMode, + normalizeAbsolutePaths, + sourceBaseDirs, + languageVersionSettings, + shouldCheckSignaturesOnUniqueness, + ) { JsIrFileMetadata(moduleExportedNames[it]?.values?.toSmartList() ?: emptyList()) } + }, + serializeFileMetadata = { serializeSingleFile(it) to FqName.ROOT }, + runKlibCheckers = { irModuleFragment, irDiagnosticReporter, compilerConfiguration -> + if (builtInsPlatform == BuiltInsPlatform.JS) { + val cleanFilesIrData = cleanFiles.map { it.irData ?: error("Metadata-only KLIBs are not supported in Kotlin/JS") } + JsKlibCheckers.check(cleanFilesIrData, irModuleFragment, moduleExportedNames, irDiagnosticReporter, compilerConfiguration) } - } - } + }, + processCompiledFileData = { ioFile, compiledFile -> + incrementalResultsConsumer?.run { + processPackagePart(ioFile, compiledFile.metadata, empty, empty) + with(compiledFile.irData!!) { + processIrFile( + ioFile, + fileData, + types, + signatures, + strings, + declarations, + bodies, + fqName.toByteArray(), + fileMetadata, + debugInfo, + ) + } + } + }, + processKlibHeader = { + incrementalResultsConsumer?.processHeader(it) + }, + ) - val additionalFiles = mutableListOf() - - for ((ktSourceFile, binaryFile) in files.zip(serializedIr.files)) { - assert(ktSourceFile.path == binaryFile.path) { - """The Kt and Ir files are put in different order - Kt: ${ktSourceFile.path} - Ir: ${binaryFile.path} - """.trimMargin() - } - val packageFragment = serializeSingleFile(ktSourceFile) - val compiledKotlinFile = KotlinFileSerializedData(packageFragment.toByteArray(), binaryFile) - - additionalFiles += compiledKotlinFile - val ioFile = ktSourceFile.toIoFileOrNull() - assert(ioFile != null) { - "No file found for source ${ktSourceFile.path}" - } - processCompiledFileData(ioFile!!, compiledKotlinFile) - } - - val compiledKotlinFiles = cleanFiles + additionalFiles - - val header = serializeKlibHeader( - configuration.languageVersionSettings, moduleDescriptor, - compiledKotlinFiles.map { it.irData.fqName }.distinct().sorted(), - emptyList() - ).toByteArray() - - incrementalResultsConsumer?.run { - processHeader(header) - } - - val serializedMetadata = - makeSerializedKlibMetadata( - compiledKotlinFiles.groupBy { it.irData.fqName } - .map { (fqn, data) -> fqn to data.sortedBy { it.irData.path }.map { it.metadata } }.toMap(), - header - ) - - val fullSerializedIr = SerializedIrModule(compiledKotlinFiles.map { it.irData }) + val fullSerializedIr = serializerOutput.serializedIr ?: error("Metadata-only KLIBs are not supported in Kotlin/JS") val versions = KotlinLibraryVersioning( - abiVersion = compatibilityMode.abiVersion, + abiVersion = abiVersion, libraryVersion = null, compilerVersion = KotlinCompilerVersion.VERSION, metadataVersion = KlibMetadataVersion.INSTANCE.toString(), @@ -737,9 +707,9 @@ fun serializeModuleIntoKlib( } buildKotlinLibrary( - linkDependencies = dependencies, + linkDependencies = serializerOutput.neededLibraries, ir = fullSerializedIr, - metadata = serializedMetadata, + metadata = serializerOutput.serializedMetadata ?: error("expected serialized metadata"), dataFlowGraph = null, manifestProperties = properties, moduleName = moduleName, @@ -809,13 +779,6 @@ private fun Map.getUniqueNameForEachFragment(): }.toMap() } -fun KtSourceFile.toIoFileOrNull(): File? = when (this) { - is KtIoFileSourceFile -> file - is KtVirtualFileSourceFile -> VfsUtilCore.virtualToIoFile(virtualFile) - is KtPsiSourceFile -> VfsUtilCore.virtualToIoFile(psiFile.virtualFile) - else -> path?.let(::File) -} - fun IncrementalDataProvider.getSerializedData(newSources: List): List { val nonCompiledSources = newSources.associateBy { it.toIoFileOrNull()!! } val compiledIrFiles = serializedIrFiles diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/IrBackendInput.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/IrBackendInput.kt index 75ab3b025d2..7c8cd3e525f 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/IrBackendInput.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/IrBackendInput.kt @@ -6,13 +6,12 @@ package org.jetbrains.kotlin.test.backend.ir import org.jetbrains.kotlin.KtSourceFile -import org.jetbrains.kotlin.backend.common.actualizer.IrActualizedResult import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext +import org.jetbrains.kotlin.backend.common.serialization.KotlinFileSerializedData import org.jetbrains.kotlin.backend.jvm.JvmIrCodegenFactory import org.jetbrains.kotlin.codegen.state.GenerationState import org.jetbrains.kotlin.diagnostics.impl.BaseDiagnosticsCollector import org.jetbrains.kotlin.fir.backend.FirMangler -import org.jetbrains.kotlin.ir.backend.js.KotlinFileSerializedData import org.jetbrains.kotlin.ir.declarations.IrModuleFragment import org.jetbrains.kotlin.ir.util.KotlinMangler import org.jetbrains.kotlin.metadata.ProtoBuf diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/Fir2IrJsWasmResultsConverter.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/Fir2IrJsWasmResultsConverter.kt index d56743709ff..73cdf2f6afb 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/Fir2IrJsWasmResultsConverter.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/Fir2IrJsWasmResultsConverter.kt @@ -7,6 +7,7 @@ package org.jetbrains.kotlin.test.frontend.fir import org.jetbrains.kotlin.KtSourceFile import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext +import org.jetbrains.kotlin.backend.common.serialization.KotlinFileSerializedData import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.config.languageVersionSettings import org.jetbrains.kotlin.diagnostics.impl.BaseDiagnosticsCollector @@ -20,7 +21,6 @@ import org.jetbrains.kotlin.fir.pipeline.Fir2IrActualizedResult import org.jetbrains.kotlin.fir.serialization.FirKLibSerializerExtension import org.jetbrains.kotlin.fir.serialization.serializeSingleFirFile import org.jetbrains.kotlin.ir.backend.js.JsFactories -import org.jetbrains.kotlin.ir.backend.js.KotlinFileSerializedData import org.jetbrains.kotlin.ir.backend.js.getSerializedData import org.jetbrains.kotlin.ir.backend.js.incrementalDataProvider import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsManglerIr diff --git a/compiler/tests-common/tests/org/jetbrains/kotlin/klib/AbstractKlibIrTextTestCase.kt b/compiler/tests-common/tests/org/jetbrains/kotlin/klib/AbstractKlibIrTextTestCase.kt index a1d4372414d..4767b5aaa63 100644 --- a/compiler/tests-common/tests/org/jetbrains/kotlin/klib/AbstractKlibIrTextTestCase.kt +++ b/compiler/tests-common/tests/org/jetbrains/kotlin/klib/AbstractKlibIrTextTestCase.kt @@ -8,20 +8,20 @@ package org.jetbrains.kotlin.klib import com.intellij.openapi.project.Project import com.intellij.psi.PsiElement import junit.framework.TestCase -import org.jetbrains.kotlin.ir.KtDiagnosticReporterWithImplicitIrBasedContext +import org.jetbrains.kotlin.KtPsiSourceFile import org.jetbrains.kotlin.backend.common.CommonKLibResolver -import org.jetbrains.kotlin.ir.util.isExpect import org.jetbrains.kotlin.backend.common.linkage.issues.checkNoUnboundSymbols import org.jetbrains.kotlin.backend.common.linkage.partial.PartialLinkageSupportForLinker -import org.jetbrains.kotlin.backend.common.serialization.CompatibilityMode import org.jetbrains.kotlin.backend.common.serialization.DeserializationStrategy import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataIncrementalSerializer -import org.jetbrains.kotlin.backend.common.serialization.metadata.makeSerializedKlibMetadata import org.jetbrains.kotlin.backend.common.serialization.signature.IdSignatureDescriptor import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment import org.jetbrains.kotlin.codegen.CodegenTestCase -import org.jetbrains.kotlin.config.* +import org.jetbrains.kotlin.config.CommonConfigurationKeys +import org.jetbrains.kotlin.config.CompilerConfiguration +import org.jetbrains.kotlin.config.LanguageFeature +import org.jetbrains.kotlin.config.languageVersionSettings import org.jetbrains.kotlin.descriptors.DeclarationDescriptor import org.jetbrains.kotlin.descriptors.ModuleDescriptor import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl @@ -31,18 +31,20 @@ import org.jetbrains.kotlin.ir.AbstractIrGeneratorTestCase import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI import org.jetbrains.kotlin.ir.backend.js.JsFactories -import org.jetbrains.kotlin.ir.backend.js.KotlinFileSerializedData import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker -import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrModuleSerializer import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsManglerDesc +import org.jetbrains.kotlin.ir.backend.js.serializeModuleIntoKlib +import org.jetbrains.kotlin.ir.backend.js.serializeScope import org.jetbrains.kotlin.ir.declarations.* import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl import org.jetbrains.kotlin.ir.util.* -import org.jetbrains.kotlin.ir.visitors.* +import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid +import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid +import org.jetbrains.kotlin.ir.visitors.acceptVoid import org.jetbrains.kotlin.js.analyze.TopDownAnalyzerFacadeForJS -import org.jetbrains.kotlin.library.* +import org.jetbrains.kotlin.library.KotlinAbiVersion +import org.jetbrains.kotlin.library.KotlinLibrary import org.jetbrains.kotlin.library.impl.BuiltInsPlatform -import org.jetbrains.kotlin.library.impl.buildKotlinLibrary import org.jetbrains.kotlin.library.metadata.KlibMetadataVersion import org.jetbrains.kotlin.library.metadata.resolver.TopologicalLibraryOrder import org.jetbrains.kotlin.metadata.ProtoBuf @@ -62,7 +64,6 @@ import org.jetbrains.kotlin.test.util.KtTestUtil import org.jetbrains.kotlin.util.DummyLogger import org.jetbrains.kotlin.utils.rethrow import java.io.File -import java.util.* /** * Compares compiled and deserialized IR @@ -147,81 +148,25 @@ abstract class AbstractKlibIrTextTestCase : CodegenTestCase() { stdlib: KotlinLibrary, containsErrorCode: Boolean, ): String { - val ktFiles = myFiles.psiFiles - val serializedIr = JsIrModuleSerializer( - KtDiagnosticReporterWithImplicitIrBasedContext( - DiagnosticReporterFactory.createPendingReporter(), - myEnvironment.configuration.languageVersionSettings, - ), - irModuleFragment.irBuiltins, - CompatibilityMode.CURRENT, - false, - emptyList(), - myEnvironment.configuration.languageVersionSettings, - ).serializedIrModule(irModuleFragment) - - val moduleDescriptor = irModuleFragment.descriptor val metadataSerializer = klibMetadataIncrementalSerializer(myEnvironment.configuration, myEnvironment.project, containsErrorCode) - - - val compiledKotlinFiles = mutableListOf() - for ((ktFile, binaryFile) in ktFiles.zip(serializedIr.files)) { - assert(ktFile.virtualFilePath == binaryFile.path) { - """The Kt and Ir files are put in different order - Kt: ${ktFile.virtualFilePath} - Ir: ${binaryFile.path} - """.trimMargin() - } - val packageFragment = metadataSerializer.serializeScope(ktFile, bindingContext, moduleDescriptor) - val compiledKotlinFile = KotlinFileSerializedData(packageFragment.toByteArray(), binaryFile) - - compiledKotlinFiles += compiledKotlinFile - } - - val header = metadataSerializer.serializeHeader( - moduleDescriptor, - compiledKotlinFiles.map { it.irData.fqName }.distinct().sorted(), - emptyList() - ).toByteArray() - - - val serializedMetadata = makeSerializedKlibMetadata( - compiledKotlinFiles.groupBy { it.irData.fqName } - .map { (fqn, data) -> fqn to data.sortedBy { it.irData.path }.map { it.metadata } }.toMap(), - header - ) - - val fullSerializedIr = SerializedIrModule(compiledKotlinFiles.map { it.irData }) - - val properties = Properties().also { p -> - if (containsErrorCode) { - p.setProperty(KLIB_PROPERTY_CONTAINS_ERROR_CODE, "true") - } - } - - val versions = KotlinLibraryVersioning( - abiVersion = KotlinAbiVersion.CURRENT, - libraryVersion = null, - compilerVersion = KotlinCompilerVersion.VERSION, - metadataVersion = KlibMetadataVersion.INSTANCE.toString(), - ) - val klibDir = org.jetbrains.kotlin.konan.file.createTempDir("testKlib") - - buildKotlinLibrary( - linkDependencies = listOf(stdlib), - ir = fullSerializedIr, - metadata = serializedMetadata, - dataFlowGraph = null, - manifestProperties = properties, + serializeModuleIntoKlib( moduleName = MODULE_NAME, + configuration = myEnvironment.configuration, + diagnosticReporter = DiagnosticReporterFactory.createPendingReporter(), + files = myFiles.psiFiles.map(::KtPsiSourceFile), + klibPath = klibDir.canonicalPath, + dependencies = listOf(stdlib), + moduleFragment = irModuleFragment, + cleanFiles = emptyList(), nopack = true, perFile = false, - output = klibDir.canonicalPath, - versions = versions, - builtInsPlatform = BuiltInsPlatform.JS + containsErrorCode = containsErrorCode, + abiVersion = KotlinAbiVersion.CURRENT, + jsOutputName = null, + builtInsPlatform = BuiltInsPlatform.JS, + serializeSingleFile = { metadataSerializer.serializeScope(it, bindingContext, irModuleFragment.descriptor) } ) - return klibDir.canonicalPath } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/FirNativeSerializer.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/FirNativeSerializer.kt index bcbbb2fa96d..5e93bfc7ce7 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/FirNativeSerializer.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/FirNativeSerializer.kt @@ -1,10 +1,8 @@ package org.jetbrains.kotlin.backend.konan -import org.jetbrains.kotlin.ir.KtDiagnosticReporterWithImplicitIrBasedContext import org.jetbrains.kotlin.KtSourceFile import org.jetbrains.kotlin.backend.common.serialization.CompatibilityMode -import org.jetbrains.kotlin.backend.common.serialization.metadata.makeSerializedKlibMetadata -import org.jetbrains.kotlin.backend.common.serialization.metadata.serializeKlibHeader +import org.jetbrains.kotlin.backend.common.serialization.serializeModuleIntoKlib import org.jetbrains.kotlin.backend.konan.driver.PhaseContext import org.jetbrains.kotlin.backend.konan.driver.phases.Fir2IrOutput import org.jetbrains.kotlin.backend.konan.driver.phases.FirOutput @@ -14,9 +12,7 @@ import org.jetbrains.kotlin.backend.konan.serialization.KonanIrModuleSerializer import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys import org.jetbrains.kotlin.cli.common.fir.reportToMessageCollector import org.jetbrains.kotlin.config.CommonConfigurationKeys -import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.config.languageVersionSettings -import org.jetbrains.kotlin.diagnostics.DiagnosticReporter import org.jetbrains.kotlin.diagnostics.DiagnosticReporterFactory import org.jetbrains.kotlin.fir.FirSession import org.jetbrains.kotlin.fir.backend.ConstValueProviderImpl @@ -27,12 +23,10 @@ import org.jetbrains.kotlin.fir.packageFqName import org.jetbrains.kotlin.fir.pipeline.FirResult import org.jetbrains.kotlin.fir.resolve.ScopeSession import org.jetbrains.kotlin.fir.resolve.providers.firProvider -import org.jetbrains.kotlin.fir.serialization.* -import org.jetbrains.kotlin.ir.declarations.IrModuleFragment +import org.jetbrains.kotlin.fir.serialization.FirKLibSerializerExtension +import org.jetbrains.kotlin.fir.serialization.serializeSingleFirFile import org.jetbrains.kotlin.konan.library.KonanLibrary -import org.jetbrains.kotlin.library.SerializedIrFile import org.jetbrains.kotlin.library.metadata.resolver.TopologicalLibraryOrder -import org.jetbrains.kotlin.metadata.ProtoBuf import org.jetbrains.kotlin.utils.toMetadataVersion internal fun PhaseContext.firSerializer(input: FirOutput): SerializerOutput? = when (input) { @@ -72,39 +66,60 @@ internal fun PhaseContext.firSerializerBase( val actualizedFirDeclarations = fir2IrOutput?.irActualizedResult?.actualizedExpectDeclarations?.extractFirDeclarations() val diagnosticReporter = DiagnosticReporterFactory.createPendingReporter() - val serializerOutput = serializeNativeModule( - configuration = configuration, - diagnosticReporter = diagnosticReporter, - sourceFiles, - usedResolvedLibraries?.map { it.library as KonanLibrary }, - fir2IrOutput?.irModuleFragment, + val serializerOutput = serializeModuleIntoKlib( moduleName = fir2IrOutput?.irModuleFragment?.descriptor?.name?.asString() ?: firResult.outputs.last().session.moduleData.name.asString(), - firFilesAndSessionsBySourceFile, - bodiesOnlyForInlines = produceHeaderKlib, - skipPrivateApi = produceHeaderKlib, - ) { firFile, originalSession, originalScopeSession -> - val session = fir2IrOutput?.components?.session ?: originalSession - val scopeSession = fir2IrOutput?.components?.scopeSession ?: originalScopeSession - val firProvider = fir2IrOutput?.components?.firProvider ?: originalSession.firProvider - serializeSingleFirFile( - firFile, - session, - scopeSession, - actualizedFirDeclarations, - FirKLibSerializerExtension( - session, firProvider, metadataVersion, - fir2IrOutput?.let { - ConstValueProviderImpl(fir2IrOutput.components) - }, - allowErrorTypes = false, - exportKDoc = shouldExportKDoc(), - additionalMetadataProvider = fir2IrOutput?.components?.annotationsFromPluginRegistrar?.createAdditionalMetadataProvider() - ), - configuration.languageVersionSettings, - produceHeaderKlib, - ) - } + irModuleFragment = fir2IrOutput?.irModuleFragment, + configuration = configuration, + diagnosticReporter = diagnosticReporter, + sourceFiles = sourceFiles, + compatibilityMode = CompatibilityMode.CURRENT, + cleanFiles = emptyList(), + dependencies = usedResolvedLibraries?.map { it.library as KonanLibrary }.orEmpty(), + createModuleSerializer = { irDiagnosticReporter, + irBuiltIns, + compatibilityMode, + normalizeAbsolutePaths, + sourceBaseDirs, + languageVersionSettings, + shouldCheckSignaturesOnUniqueness -> + KonanIrModuleSerializer( + diagnosticReporter = irDiagnosticReporter, + irBuiltIns = irBuiltIns, + compatibilityMode = compatibilityMode, + normalizeAbsolutePaths = normalizeAbsolutePaths, + sourceBaseDirs = sourceBaseDirs, + languageVersionSettings = languageVersionSettings, + bodiesOnlyForInlines = produceHeaderKlib, + skipPrivateApi = produceHeaderKlib, + shouldCheckSignaturesOnUniqueness = shouldCheckSignaturesOnUniqueness, + ) + }, + serializeFileMetadata = { ktSourceFile -> + val (firFile, originalSession, originalScopeSession) = firFilesAndSessionsBySourceFile[ktSourceFile] + ?: error("cannot find FIR file by source file ${ktSourceFile.name} (${ktSourceFile.path})") + val session = fir2IrOutput?.components?.session ?: originalSession + val scopeSession = fir2IrOutput?.components?.scopeSession ?: originalScopeSession + val firProvider = fir2IrOutput?.components?.firProvider ?: originalSession.firProvider + serializeSingleFirFile( + firFile, + session, + scopeSession, + actualizedFirDeclarations, + FirKLibSerializerExtension( + session, firProvider, metadataVersion, + fir2IrOutput?.let { + ConstValueProviderImpl(fir2IrOutput.components) + }, + allowErrorTypes = false, + exportKDoc = shouldExportKDoc(), + additionalMetadataProvider = fir2IrOutput?.components?.annotationsFromPluginRegistrar?.createAdditionalMetadataProvider() + ), + configuration.languageVersionSettings, + produceHeaderKlib, + ) to firFile.packageFqName + }, + ) val renderDiagnosticNames = configuration.getBoolean(CLIConfigurationKeys.RENDER_DIAGNOSTIC_INTERNAL_NAME) diagnosticReporter.reportToMessageCollector(messageCollector, renderDiagnosticNames) if (diagnosticReporter.hasErrors) { @@ -112,80 +127,3 @@ internal fun PhaseContext.firSerializerBase( } return serializerOutput } - -class KotlinFileSerializedData( - val source: KtSourceFile, - val firFile: FirFile, - val metadata: ByteArray, - val irData: SerializedIrFile?, -) { - val fqName: String get() = irData?.fqName ?: firFile.packageFqName.asString() - val path: String? get() = irData?.path ?: source.path -} - -private fun serializeNativeModule( - configuration: CompilerConfiguration, - diagnosticReporter: DiagnosticReporter, - files: List, - dependencies: List?, - moduleFragment: IrModuleFragment?, - moduleName: String, - firFilesAndSessionsBySourceFile: Map>, - bodiesOnlyForInlines: Boolean = false, - skipPrivateApi: Boolean = false, - serializeSingleFile: (FirFile, FirSession, ScopeSession) -> ProtoBuf.PackageFragment -): SerializerOutput { - if (moduleFragment != null) { - assert(files.size == moduleFragment.files.size) - } - - val sourceBaseDirs = configuration[CommonConfigurationKeys.KLIB_RELATIVE_PATH_BASES] ?: emptyList() - val absolutePathNormalization = configuration[CommonConfigurationKeys.KLIB_NORMALIZE_ABSOLUTE_PATH] ?: false - val signatureClashChecks = configuration[CommonConfigurationKeys.PRODUCE_KLIB_SIGNATURES_CLASH_CHECKS] ?: true - - val serializedIr = moduleFragment?.let { - KonanIrModuleSerializer( - KtDiagnosticReporterWithImplicitIrBasedContext(diagnosticReporter, configuration.languageVersionSettings), - moduleFragment.irBuiltins, - CompatibilityMode.CURRENT, - normalizeAbsolutePaths = absolutePathNormalization, - sourceBaseDirs = sourceBaseDirs, - languageVersionSettings = configuration.languageVersionSettings, - bodiesOnlyForInlines = bodiesOnlyForInlines, - skipPrivateApi = skipPrivateApi, - shouldCheckSignaturesOnUniqueness = signatureClashChecks, - ).serializedIrModule(moduleFragment) - } - - val serializedFiles = serializedIr?.files?.toList() - - val compiledKotlinFiles = files.mapIndexed { index, ktSourceFile -> - val binaryFile = serializedFiles?.get(index)?.also { - assert(ktSourceFile.path == it.path) { - """The Kt and Ir files are put in different order - Kt: ${ktSourceFile.path} - Ir: ${it.path} - """.trimMargin() - } - } - val (firFile, session, scopeSession) = firFilesAndSessionsBySourceFile[ktSourceFile] - ?: error("cannot find FIR file by source file ${ktSourceFile.name} (${ktSourceFile.path})") - val packageFragment = serializeSingleFile(firFile, session, scopeSession) - KotlinFileSerializedData(ktSourceFile, firFile, packageFragment.toByteArray(), binaryFile) - } - - val header = serializeKlibHeader( - configuration.languageVersionSettings, moduleName, - compiledKotlinFiles.map { it.fqName }.distinct().sorted(), - emptyList() - ).toByteArray() - - val serializedMetadata = - makeSerializedKlibMetadata( - compiledKotlinFiles.groupBy { it.fqName } - .map { (fqn, data) -> fqn to data.sortedBy { it.path }.map { it.metadata } }.toMap(), - header - ) - - return SerializerOutput(serializedMetadata, serializedIr, null, dependencies.orEmpty()) -} diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/Serializer.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/Serializer.kt index d7c1d9221ee..cc5ff8cabf2 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/Serializer.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/driver/phases/Serializer.kt @@ -5,7 +5,6 @@ package org.jetbrains.kotlin.backend.konan.driver.phases -import org.jetbrains.kotlin.ir.KtDiagnosticReporterWithImplicitIrBasedContext import org.jetbrains.kotlin.backend.common.phaser.createSimpleNamedCompilerPhase import org.jetbrains.kotlin.backend.common.serialization.CompatibilityMode import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataMonolithicSerializer @@ -16,10 +15,9 @@ import org.jetbrains.kotlin.config.CommonConfigurationKeys import org.jetbrains.kotlin.config.languageVersionSettings import org.jetbrains.kotlin.descriptors.ModuleDescriptor import org.jetbrains.kotlin.diagnostics.DiagnosticReporterFactory +import org.jetbrains.kotlin.ir.KtDiagnosticReporterWithImplicitIrBasedContext import org.jetbrains.kotlin.ir.util.IrMessageLogger import org.jetbrains.kotlin.konan.library.KonanLibrary -import org.jetbrains.kotlin.library.SerializedIrModule -import org.jetbrains.kotlin.library.SerializedMetadata internal data class SerializerInput( val moduleDescriptor: ModuleDescriptor, @@ -27,12 +25,7 @@ internal data class SerializerInput( val produceHeaderKlib: Boolean, ) -data class SerializerOutput( - val serializedMetadata: SerializedMetadata?, - val serializedIr: SerializedIrModule?, - val dataFlowGraph: ByteArray?, - val neededLibraries: List -) +typealias SerializerOutput = org.jetbrains.kotlin.backend.common.serialization.SerializerOutput internal val SerializerPhase = createSimpleNamedCompilerPhase( "Serializer", "IR serializer",