[klib] Deduplicate KLIB serialization logic in K/JS and K/N
Introduce the new `serializeModuleIntoKlib` function that all KLIB-based backends can use to serialize a module to a KLIB. ^KT-64392 Fixed
This commit is contained in:
Sergej Jaskiewicz
committed by
Space Team
Space Team
parent
c335cd0b1e
commit
93a1ceae75
+200
@@ -0,0 +1,200 @@
|
||||
/*
|
||||
* Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package org.jetbrains.kotlin.backend.common.serialization
|
||||
|
||||
import com.intellij.openapi.vfs.VfsUtilCore
|
||||
import org.jetbrains.kotlin.KtIoFileSourceFile
|
||||
import org.jetbrains.kotlin.KtPsiSourceFile
|
||||
import org.jetbrains.kotlin.KtSourceFile
|
||||
import org.jetbrains.kotlin.KtVirtualFileSourceFile
|
||||
import org.jetbrains.kotlin.backend.common.serialization.metadata.makeSerializedKlibMetadata
|
||||
import org.jetbrains.kotlin.backend.common.serialization.metadata.serializeKlibHeader
|
||||
import org.jetbrains.kotlin.config.CommonConfigurationKeys
|
||||
import org.jetbrains.kotlin.config.CompilerConfiguration
|
||||
import org.jetbrains.kotlin.config.LanguageVersionSettings
|
||||
import org.jetbrains.kotlin.config.languageVersionSettings
|
||||
import org.jetbrains.kotlin.diagnostics.DiagnosticReporter
|
||||
import org.jetbrains.kotlin.ir.IrBuiltIns
|
||||
import org.jetbrains.kotlin.ir.IrDiagnosticReporter
|
||||
import org.jetbrains.kotlin.ir.KtDiagnosticReporterWithImplicitIrBasedContext
|
||||
import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
|
||||
import org.jetbrains.kotlin.library.KotlinLibrary
|
||||
import org.jetbrains.kotlin.library.SerializedIrFile
|
||||
import org.jetbrains.kotlin.library.SerializedIrModule
|
||||
import org.jetbrains.kotlin.library.SerializedMetadata
|
||||
import org.jetbrains.kotlin.metadata.ProtoBuf
|
||||
import org.jetbrains.kotlin.name.FqName
|
||||
import java.io.File
|
||||
|
||||
/**
|
||||
* Holds the binary data for a single Kotlin file to be written to a KLIB, i.e., its metadata and IR (unless it's a metadata-only KLIB).
|
||||
*
|
||||
* @property metadata Serialized metadata of the corresponding source file.
|
||||
* @property irData Serialized IR for this file, or `null` if this is a metadata-only KLIB.
|
||||
* @property path The path to the corresponding source file, or `null` if that source file didn't have a path.
|
||||
* @property fqName The fully qualified name of the package containing the serialized file.
|
||||
*/
|
||||
class KotlinFileSerializedData private constructor(
|
||||
val metadata: ByteArray,
|
||||
val irData: SerializedIrFile?,
|
||||
val path: String?,
|
||||
val fqName: String,
|
||||
) {
|
||||
|
||||
/**
|
||||
* Used for creating file serialization data for IR-containing KLIBs.
|
||||
*
|
||||
* @param metadata Serialized metadata of the corresponding source file.
|
||||
* @param irData Serialized IR for this file.
|
||||
*/
|
||||
constructor(metadata: ByteArray, irData: SerializedIrFile) : this(metadata, irData, irData.path, irData.fqName)
|
||||
|
||||
/**
|
||||
* Used for creating file serialization data in metadata-only KLIBs.
|
||||
*
|
||||
* @param metadata Serialized metadata of the corresponding source file.
|
||||
* @param path The path of the serialized file.
|
||||
* @param fqName The fully qualified name of the package containing the serialized file.
|
||||
*/
|
||||
constructor(metadata: ByteArray, path: String?, fqName: String) : this(metadata, irData = null, path, fqName)
|
||||
}
|
||||
|
||||
class SerializerOutput<Dependency : KotlinLibrary>(
|
||||
val serializedMetadata: SerializedMetadata?,
|
||||
val serializedIr: SerializedIrModule?,
|
||||
val dataFlowGraph: ByteArray?,
|
||||
val neededLibraries: List<Dependency>,
|
||||
)
|
||||
|
||||
fun KtSourceFile.toIoFileOrNull(): File? = when (this) {
|
||||
is KtIoFileSourceFile -> file
|
||||
is KtVirtualFileSourceFile -> VfsUtilCore.virtualToIoFile(virtualFile)
|
||||
is KtPsiSourceFile -> VfsUtilCore.virtualToIoFile(psiFile.virtualFile)
|
||||
else -> path?.let(::File)
|
||||
}
|
||||
|
||||
/**
|
||||
* Produces all the necessary binary data for writing a KLIB for a Kotlin module, including its metadata.
|
||||
*
|
||||
* If [irModuleFragment] is not `null`, serializes the module's IR into binary form by running [IrModuleSerializer].
|
||||
*
|
||||
* For producing a metadata-only KLIB, pass `null` to [irModuleFragment].
|
||||
*
|
||||
* @param moduleName The name of the module being serialized to be written into the KLIB header.
|
||||
* @param irModuleFragment The IR to be serialized into the KLIB being produced, or `null` if this is going to be a metadata-only KLIB.
|
||||
* @param configuration Used to determine certain serialization parameters and enable/disable serialization diagnostics.
|
||||
* @param diagnosticReporter Used for reporting serialization-time diagnostics, for example, about clashing IR signatures.
|
||||
* @param sourceFiles The source files from which the KLIB is being compiled. Used for serializing file metadata.
|
||||
* @param compatibilityMode The information about KLIB ABI.
|
||||
* @param cleanFiles In the case of incremental compilation, the list of files that were not changed and therefore don't need to be
|
||||
* serialized again.
|
||||
* @param dependencies The list of KLIBs that the KLIB being produced depends on.
|
||||
* @param createModuleSerializer Used for creating a backend-specific instance of [IrModuleSerializer].
|
||||
* @param serializeFileMetadata The name of this argument kind of speaks for itself.
|
||||
* @param runKlibCheckers Additional checks to be run before serializing [irModuleFragment]. Can be used to report serialization-time
|
||||
* diagnostics.
|
||||
* @param processCompiledFileData Called for each newly serialized file. Useful for incremental compilation.
|
||||
* @param processKlibHeader Called after serializing the KLIB header. Useful for incremental compilation.
|
||||
*/
|
||||
fun <Dependency : KotlinLibrary> serializeModuleIntoKlib(
|
||||
moduleName: String,
|
||||
irModuleFragment: IrModuleFragment?,
|
||||
configuration: CompilerConfiguration,
|
||||
diagnosticReporter: DiagnosticReporter,
|
||||
sourceFiles: List<KtSourceFile>,
|
||||
compatibilityMode: CompatibilityMode,
|
||||
cleanFiles: List<KotlinFileSerializedData>,
|
||||
dependencies: List<Dependency>,
|
||||
createModuleSerializer: (
|
||||
irDiagnosticReporter: IrDiagnosticReporter,
|
||||
irBuiltins: IrBuiltIns,
|
||||
compatibilityMode: CompatibilityMode,
|
||||
normalizeAbsolutePaths: Boolean,
|
||||
sourceBaseDirs: Collection<String>,
|
||||
languageVersionSettings: LanguageVersionSettings,
|
||||
shouldCheckSignaturesOnUniqueness: Boolean,
|
||||
) -> IrModuleSerializer<*>,
|
||||
serializeFileMetadata: (KtSourceFile) -> Pair<ProtoBuf.PackageFragment, FqName>,
|
||||
runKlibCheckers: (IrModuleFragment, IrDiagnosticReporter, CompilerConfiguration) -> Unit = { _, _, _ -> },
|
||||
processCompiledFileData: ((File, KotlinFileSerializedData) -> Unit)? = null,
|
||||
processKlibHeader: (ByteArray) -> Unit = {},
|
||||
): SerializerOutput<Dependency> {
|
||||
if (irModuleFragment != null) {
|
||||
assert(sourceFiles.size == irModuleFragment.files.size) {
|
||||
"The number of source files (${sourceFiles.size}) does not match the number of IrFiles (${irModuleFragment.files.size})"
|
||||
}
|
||||
}
|
||||
|
||||
val sourceBaseDirs = configuration[CommonConfigurationKeys.KLIB_RELATIVE_PATH_BASES] ?: emptyList()
|
||||
val normalizeAbsolutePath = configuration.getBoolean(CommonConfigurationKeys.KLIB_NORMALIZE_ABSOLUTE_PATH)
|
||||
val signatureClashChecks = configuration[CommonConfigurationKeys.PRODUCE_KLIB_SIGNATURES_CLASH_CHECKS] ?: true
|
||||
|
||||
val serializedIr = irModuleFragment?.let {
|
||||
val irDiagnosticReporter = KtDiagnosticReporterWithImplicitIrBasedContext(diagnosticReporter, configuration.languageVersionSettings)
|
||||
runKlibCheckers(it, irDiagnosticReporter, configuration)
|
||||
createModuleSerializer(
|
||||
irDiagnosticReporter,
|
||||
it.irBuiltins,
|
||||
compatibilityMode,
|
||||
normalizeAbsolutePath,
|
||||
sourceBaseDirs,
|
||||
configuration.languageVersionSettings,
|
||||
signatureClashChecks,
|
||||
).serializedIrModule(it)
|
||||
}
|
||||
|
||||
val serializedFiles = serializedIr?.files?.toList()
|
||||
|
||||
val compiledKotlinFiles = sourceFiles.mapIndexedTo(cleanFiles.toMutableList()) { i, ktSourceFile ->
|
||||
val binaryFile = serializedFiles?.get(i)?.also {
|
||||
assert(ktSourceFile.path == it.path) {
|
||||
"""The Kt and Ir files are put in different order
|
||||
Kt: ${ktSourceFile.path}
|
||||
Ir: ${it.path}
|
||||
""".trimMargin()
|
||||
}
|
||||
}
|
||||
val (packageFragment, fqName) = serializeFileMetadata(ktSourceFile)
|
||||
val metadata = packageFragment.toByteArray()
|
||||
val compiledKotlinFile = if (binaryFile == null)
|
||||
KotlinFileSerializedData(metadata, ktSourceFile.path, fqName.asString())
|
||||
else
|
||||
KotlinFileSerializedData(metadata, binaryFile)
|
||||
|
||||
if (processCompiledFileData != null) {
|
||||
val ioFile = ktSourceFile.toIoFileOrNull() ?: error("No file found for source ${ktSourceFile.path}")
|
||||
processCompiledFileData(ioFile, compiledKotlinFile)
|
||||
}
|
||||
|
||||
compiledKotlinFile
|
||||
}
|
||||
|
||||
val header = serializeKlibHeader(
|
||||
languageVersionSettings = configuration.languageVersionSettings,
|
||||
moduleName = moduleName,
|
||||
fragmentNames = compiledKotlinFiles.map { it.fqName }.distinct().sorted(),
|
||||
emptyPackages = emptyList(),
|
||||
).toByteArray()
|
||||
|
||||
processKlibHeader(header)
|
||||
|
||||
val serializedMetadata = makeSerializedKlibMetadata(
|
||||
fragments = compiledKotlinFiles
|
||||
.groupBy { it.fqName }
|
||||
.map { (fqn, data) ->
|
||||
fqn to data.sortedBy { it.path }.map { it.metadata }
|
||||
}
|
||||
.toMap(),
|
||||
header = header,
|
||||
)
|
||||
|
||||
return SerializerOutput(
|
||||
serializedMetadata = serializedMetadata,
|
||||
serializedIr = if (serializedIr == null) null else SerializedIrModule(compiledKotlinFiles.mapNotNull { it.irData }),
|
||||
dataFlowGraph = null,
|
||||
neededLibraries = dependencies,
|
||||
)
|
||||
}
|
||||
@@ -34,7 +34,6 @@ import org.jetbrains.kotlin.diagnostics.DiagnosticReporter
|
||||
import org.jetbrains.kotlin.incremental.components.LookupTracker
|
||||
import org.jetbrains.kotlin.incremental.js.IncrementalDataProvider
|
||||
import org.jetbrains.kotlin.ir.IrBuiltIns
|
||||
import org.jetbrains.kotlin.ir.KtDiagnosticReporterWithImplicitIrBasedContext
|
||||
import org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI
|
||||
import org.jetbrains.kotlin.ir.backend.js.checkers.JsKlibCheckers
|
||||
import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.*
|
||||
@@ -57,6 +56,7 @@ import org.jetbrains.kotlin.library.impl.buildKotlinLibrary
|
||||
import org.jetbrains.kotlin.library.metadata.KlibMetadataFactories
|
||||
import org.jetbrains.kotlin.library.metadata.KlibMetadataVersion
|
||||
import org.jetbrains.kotlin.metadata.ProtoBuf
|
||||
import org.jetbrains.kotlin.name.FqName
|
||||
import org.jetbrains.kotlin.progress.IncrementalNextRoundException
|
||||
import org.jetbrains.kotlin.progress.ProgressIndicatorAndCompilationCanceledStatus
|
||||
import org.jetbrains.kotlin.psi.KtFile
|
||||
@@ -100,8 +100,6 @@ private val CompilerConfiguration.metadataVersion
|
||||
internal val SerializedIrFile.fileMetadata: ByteArray
|
||||
get() = backendSpecificMetadata ?: error("Expect file caches to have backendSpecificMetadata, but '$path' doesn't")
|
||||
|
||||
class KotlinFileSerializedData(val metadata: ByteArray, val irData: SerializedIrFile)
|
||||
|
||||
fun generateKLib(
|
||||
depsDescriptors: ModulesStructure,
|
||||
outputKlibPath: String,
|
||||
@@ -624,100 +622,72 @@ fun serializeModuleIntoKlib(
|
||||
builtInsPlatform: BuiltInsPlatform = BuiltInsPlatform.JS,
|
||||
serializeSingleFile: (KtSourceFile) -> ProtoBuf.PackageFragment
|
||||
) {
|
||||
assert(files.size == moduleFragment.files.size)
|
||||
|
||||
val compatibilityMode = CompatibilityMode(abiVersion)
|
||||
val sourceBaseDirs = configuration[CommonConfigurationKeys.KLIB_RELATIVE_PATH_BASES] ?: emptyList()
|
||||
val absolutePathNormalization = configuration[CommonConfigurationKeys.KLIB_NORMALIZE_ABSOLUTE_PATH] ?: false
|
||||
val signatureClashChecks = configuration[CommonConfigurationKeys.PRODUCE_KLIB_SIGNATURES_CLASH_CHECKS] ?: true
|
||||
|
||||
val moduleExportedNames = moduleFragment.collectExportedNames()
|
||||
|
||||
val irDiagnosticReporter = KtDiagnosticReporterWithImplicitIrBasedContext(diagnosticReporter, configuration.languageVersionSettings)
|
||||
if (builtInsPlatform == BuiltInsPlatform.JS) {
|
||||
val cleanFilesIrData = cleanFiles.map { it.irData }
|
||||
JsKlibCheckers.check(cleanFilesIrData, moduleFragment, moduleExportedNames, irDiagnosticReporter, configuration)
|
||||
}
|
||||
|
||||
val serializedIr =
|
||||
JsIrModuleSerializer(
|
||||
irDiagnosticReporter,
|
||||
moduleFragment.irBuiltins,
|
||||
compatibilityMode,
|
||||
normalizeAbsolutePaths = absolutePathNormalization,
|
||||
sourceBaseDirs = sourceBaseDirs,
|
||||
configuration.languageVersionSettings,
|
||||
signatureClashChecks,
|
||||
) { JsIrFileMetadata(moduleExportedNames[it]?.values?.toSmartList() ?: emptyList()) }.serializedIrModule(moduleFragment)
|
||||
|
||||
val moduleDescriptor = moduleFragment.descriptor
|
||||
|
||||
val incrementalResultsConsumer = configuration.get(JSConfigurationKeys.INCREMENTAL_RESULTS_CONSUMER)
|
||||
val empty = ByteArray(0)
|
||||
|
||||
fun processCompiledFileData(ioFile: File, compiledFile: KotlinFileSerializedData) {
|
||||
incrementalResultsConsumer?.run {
|
||||
processPackagePart(ioFile, compiledFile.metadata, empty, empty)
|
||||
with(compiledFile.irData) {
|
||||
processIrFile(
|
||||
ioFile,
|
||||
fileData,
|
||||
types,
|
||||
signatures,
|
||||
strings,
|
||||
declarations,
|
||||
bodies,
|
||||
fqName.toByteArray(),
|
||||
fileMetadata,
|
||||
debugInfo,
|
||||
)
|
||||
val serializerOutput = serializeModuleIntoKlib(
|
||||
moduleName = moduleFragment.name.asString(),
|
||||
irModuleFragment = moduleFragment,
|
||||
configuration = configuration,
|
||||
diagnosticReporter = diagnosticReporter,
|
||||
sourceFiles = files,
|
||||
compatibilityMode = CompatibilityMode(abiVersion),
|
||||
cleanFiles = cleanFiles,
|
||||
dependencies = dependencies,
|
||||
createModuleSerializer = {
|
||||
irDiagnosticReporter,
|
||||
irBuiltins,
|
||||
compatibilityMode,
|
||||
normalizeAbsolutePaths,
|
||||
sourceBaseDirs,
|
||||
languageVersionSettings,
|
||||
shouldCheckSignaturesOnUniqueness,
|
||||
->
|
||||
JsIrModuleSerializer(
|
||||
irDiagnosticReporter,
|
||||
irBuiltins,
|
||||
compatibilityMode,
|
||||
normalizeAbsolutePaths,
|
||||
sourceBaseDirs,
|
||||
languageVersionSettings,
|
||||
shouldCheckSignaturesOnUniqueness,
|
||||
) { JsIrFileMetadata(moduleExportedNames[it]?.values?.toSmartList() ?: emptyList()) }
|
||||
},
|
||||
serializeFileMetadata = { serializeSingleFile(it) to FqName.ROOT },
|
||||
runKlibCheckers = { irModuleFragment, irDiagnosticReporter, compilerConfiguration ->
|
||||
if (builtInsPlatform == BuiltInsPlatform.JS) {
|
||||
val cleanFilesIrData = cleanFiles.map { it.irData ?: error("Metadata-only KLIBs are not supported in Kotlin/JS") }
|
||||
JsKlibCheckers.check(cleanFilesIrData, irModuleFragment, moduleExportedNames, irDiagnosticReporter, compilerConfiguration)
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
processCompiledFileData = { ioFile, compiledFile ->
|
||||
incrementalResultsConsumer?.run {
|
||||
processPackagePart(ioFile, compiledFile.metadata, empty, empty)
|
||||
with(compiledFile.irData!!) {
|
||||
processIrFile(
|
||||
ioFile,
|
||||
fileData,
|
||||
types,
|
||||
signatures,
|
||||
strings,
|
||||
declarations,
|
||||
bodies,
|
||||
fqName.toByteArray(),
|
||||
fileMetadata,
|
||||
debugInfo,
|
||||
)
|
||||
}
|
||||
}
|
||||
},
|
||||
processKlibHeader = {
|
||||
incrementalResultsConsumer?.processHeader(it)
|
||||
},
|
||||
)
|
||||
|
||||
val additionalFiles = mutableListOf<KotlinFileSerializedData>()
|
||||
|
||||
for ((ktSourceFile, binaryFile) in files.zip(serializedIr.files)) {
|
||||
assert(ktSourceFile.path == binaryFile.path) {
|
||||
"""The Kt and Ir files are put in different order
|
||||
Kt: ${ktSourceFile.path}
|
||||
Ir: ${binaryFile.path}
|
||||
""".trimMargin()
|
||||
}
|
||||
val packageFragment = serializeSingleFile(ktSourceFile)
|
||||
val compiledKotlinFile = KotlinFileSerializedData(packageFragment.toByteArray(), binaryFile)
|
||||
|
||||
additionalFiles += compiledKotlinFile
|
||||
val ioFile = ktSourceFile.toIoFileOrNull()
|
||||
assert(ioFile != null) {
|
||||
"No file found for source ${ktSourceFile.path}"
|
||||
}
|
||||
processCompiledFileData(ioFile!!, compiledKotlinFile)
|
||||
}
|
||||
|
||||
val compiledKotlinFiles = cleanFiles + additionalFiles
|
||||
|
||||
val header = serializeKlibHeader(
|
||||
configuration.languageVersionSettings, moduleDescriptor,
|
||||
compiledKotlinFiles.map { it.irData.fqName }.distinct().sorted(),
|
||||
emptyList()
|
||||
).toByteArray()
|
||||
|
||||
incrementalResultsConsumer?.run {
|
||||
processHeader(header)
|
||||
}
|
||||
|
||||
val serializedMetadata =
|
||||
makeSerializedKlibMetadata(
|
||||
compiledKotlinFiles.groupBy { it.irData.fqName }
|
||||
.map { (fqn, data) -> fqn to data.sortedBy { it.irData.path }.map { it.metadata } }.toMap(),
|
||||
header
|
||||
)
|
||||
|
||||
val fullSerializedIr = SerializedIrModule(compiledKotlinFiles.map { it.irData })
|
||||
val fullSerializedIr = serializerOutput.serializedIr ?: error("Metadata-only KLIBs are not supported in Kotlin/JS")
|
||||
|
||||
val versions = KotlinLibraryVersioning(
|
||||
abiVersion = compatibilityMode.abiVersion,
|
||||
abiVersion = abiVersion,
|
||||
libraryVersion = null,
|
||||
compilerVersion = KotlinCompilerVersion.VERSION,
|
||||
metadataVersion = KlibMetadataVersion.INSTANCE.toString(),
|
||||
@@ -737,9 +707,9 @@ fun serializeModuleIntoKlib(
|
||||
}
|
||||
|
||||
buildKotlinLibrary(
|
||||
linkDependencies = dependencies,
|
||||
linkDependencies = serializerOutput.neededLibraries,
|
||||
ir = fullSerializedIr,
|
||||
metadata = serializedMetadata,
|
||||
metadata = serializerOutput.serializedMetadata ?: error("expected serialized metadata"),
|
||||
dataFlowGraph = null,
|
||||
manifestProperties = properties,
|
||||
moduleName = moduleName,
|
||||
@@ -809,13 +779,6 @@ private fun Map<IrModuleFragment, KotlinLibrary>.getUniqueNameForEachFragment():
|
||||
}.toMap()
|
||||
}
|
||||
|
||||
fun KtSourceFile.toIoFileOrNull(): File? = when (this) {
|
||||
is KtIoFileSourceFile -> file
|
||||
is KtVirtualFileSourceFile -> VfsUtilCore.virtualToIoFile(virtualFile)
|
||||
is KtPsiSourceFile -> VfsUtilCore.virtualToIoFile(psiFile.virtualFile)
|
||||
else -> path?.let(::File)
|
||||
}
|
||||
|
||||
fun IncrementalDataProvider.getSerializedData(newSources: List<KtSourceFile>): List<KotlinFileSerializedData> {
|
||||
val nonCompiledSources = newSources.associateBy { it.toIoFileOrNull()!! }
|
||||
val compiledIrFiles = serializedIrFiles
|
||||
|
||||
Reference in New Issue
Block a user