[klib] Don't pass around metadata serialization closure

Pass the metadata serializer instance instead. This allows to further
reduce code duplication by introducing the common interface
`KlibSingleFileMetadataSerializer` for abstracting away K1 and K2
representation of a source file, as well as reusing
`Fir2KlibMetadataSerializer` across different backends.

KT-64392
This commit is contained in:
Sergej Jaskiewicz
2024-01-22 14:21:20 +01:00
committed by Space Team
parent 5a6f51e474
commit f39335b3f7
23 changed files with 388 additions and 406 deletions
@@ -0,0 +1,35 @@
/*
* Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.backend.common.serialization.metadata
import org.jetbrains.kotlin.KtSourceFile
import org.jetbrains.kotlin.metadata.ProtoBuf
import org.jetbrains.kotlin.name.FqName
/**
* Something capable of serializing the metadata of a source file to a protobuf message, one file at a time.
*/
interface KlibSingleFileMetadataSerializer<SourceFile> {
/**
* The number of source files whose metadata is to be serialized.
*/
val numberOfSourceFiles: Int
/**
* Serializes the metadata of a single source file to a protobuf message and returns the message.
*/
fun serializeSingleFileMetadata(file: SourceFile): ProtoBuf.PackageFragment
/**
* Iterates through each file whose metadata is to be serialized, providing an opportunity to call [serializeSingleFileMetadata]
* and perform additional processing of the serialized data.
*
* @param block A closure that accepts the index of the file in the list of source files, the source file, its corresponding
* [KtSourceFile], and the fully qualified name of the package containing the file.
*/
fun forEachFile(block: (Int, SourceFile, KtSourceFile, FqName) -> Unit)
}
@@ -10,6 +10,7 @@ import org.jetbrains.kotlin.KtIoFileSourceFile
import org.jetbrains.kotlin.KtPsiSourceFile
import org.jetbrains.kotlin.KtSourceFile
import org.jetbrains.kotlin.KtVirtualFileSourceFile
import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibSingleFileMetadataSerializer
import org.jetbrains.kotlin.backend.common.serialization.metadata.serializeKlibHeader
import org.jetbrains.kotlin.config.CommonConfigurationKeys
import org.jetbrains.kotlin.config.CompilerConfiguration
@@ -24,8 +25,6 @@ import org.jetbrains.kotlin.library.KotlinLibrary
import org.jetbrains.kotlin.library.SerializedIrFile
import org.jetbrains.kotlin.library.SerializedIrModule
import org.jetbrains.kotlin.library.SerializedMetadata
import org.jetbrains.kotlin.metadata.ProtoBuf
import org.jetbrains.kotlin.name.FqName
import java.io.File
/**
@@ -86,24 +85,22 @@ fun KtSourceFile.toIoFileOrNull(): File? = when (this) {
* @param irModuleFragment The IR to be serialized into the KLIB being produced, or `null` if this is going to be a metadata-only KLIB.
* @param configuration Used to determine certain serialization parameters and enable/disable serialization diagnostics.
* @param diagnosticReporter Used for reporting serialization-time diagnostics, for example, about clashing IR signatures.
* @param sourceFiles The source files from which the KLIB is being compiled. Used for serializing file metadata.
* @param compatibilityMode The information about KLIB ABI.
* @param cleanFiles In the case of incremental compilation, the list of files that were not changed and therefore don't need to be
* serialized again.
* @param dependencies The list of KLIBs that the KLIB being produced depends on.
* @param createModuleSerializer Used for creating a backend-specific instance of [IrModuleSerializer].
* @param serializeFileMetadata The name of this argument kind of speaks for itself.
* @param metadataSerializer Something capable of serializing the metadata of the source files. See the corresponding interface KDoc.
* @param runKlibCheckers Additional checks to be run before serializing [irModuleFragment]. Can be used to report serialization-time
* diagnostics.
* @param processCompiledFileData Called for each newly serialized file. Useful for incremental compilation.
* @param processKlibHeader Called after serializing the KLIB header. Useful for incremental compilation.
*/
fun <Dependency : KotlinLibrary> serializeModuleIntoKlib(
fun <Dependency : KotlinLibrary, SourceFile> serializeModuleIntoKlib(
moduleName: String,
irModuleFragment: IrModuleFragment?,
configuration: CompilerConfiguration,
diagnosticReporter: DiagnosticReporter,
sourceFiles: List<KtSourceFile>,
compatibilityMode: CompatibilityMode,
cleanFiles: List<KotlinFileSerializedData>,
dependencies: List<Dependency>,
@@ -116,14 +113,14 @@ fun <Dependency : KotlinLibrary> serializeModuleIntoKlib(
languageVersionSettings: LanguageVersionSettings,
shouldCheckSignaturesOnUniqueness: Boolean,
) -> IrModuleSerializer<*>,
serializeFileMetadata: (KtSourceFile) -> Pair<ProtoBuf.PackageFragment, FqName>,
metadataSerializer: KlibSingleFileMetadataSerializer<SourceFile>,
runKlibCheckers: (IrModuleFragment, IrDiagnosticReporter, CompilerConfiguration) -> Unit = { _, _, _ -> },
processCompiledFileData: ((File, KotlinFileSerializedData) -> Unit)? = null,
processKlibHeader: (ByteArray) -> Unit = {},
): SerializerOutput<Dependency> {
if (irModuleFragment != null) {
assert(sourceFiles.size == irModuleFragment.files.size) {
"The number of source files (${sourceFiles.size}) does not match the number of IrFiles (${irModuleFragment.files.size})"
assert(metadataSerializer.numberOfSourceFiles == irModuleFragment.files.size) {
"The number of source files (${metadataSerializer.numberOfSourceFiles}) does not match the number of IrFiles (${irModuleFragment.files.size})"
}
}
@@ -147,28 +144,31 @@ fun <Dependency : KotlinLibrary> serializeModuleIntoKlib(
val serializedFiles = serializedIr?.files?.toList()
val compiledKotlinFiles = sourceFiles.mapIndexedTo(cleanFiles.toMutableList()) { i, ktSourceFile ->
val binaryFile = serializedFiles?.get(i)?.also {
assert(ktSourceFile.path == it.path) {
"""The Kt and Ir files are put in different order
Kt: ${ktSourceFile.path}
Ir: ${it.path}
""".trimMargin()
val compiledKotlinFiles = buildList {
addAll(cleanFiles)
metadataSerializer.forEachFile { i, sourceFile, ktSourceFile, packageFqName ->
val binaryFile = serializedFiles?.get(i)?.also {
assert(ktSourceFile.path == it.path) {
"""The Kt and Ir files are put in different order
Kt: ${ktSourceFile.path}
Ir: ${it.path}
""".trimMargin()
}
}
}
val (packageFragment, fqName) = serializeFileMetadata(ktSourceFile)
val metadata = packageFragment.toByteArray()
val compiledKotlinFile = if (binaryFile == null)
KotlinFileSerializedData(metadata, ktSourceFile.path, fqName.asString())
else
KotlinFileSerializedData(metadata, binaryFile)
val protoBuf = metadataSerializer.serializeSingleFileMetadata(sourceFile)
val metadata = protoBuf.toByteArray()
val compiledKotlinFile = if (binaryFile == null)
KotlinFileSerializedData(metadata, ktSourceFile.path, packageFqName.asString())
else
KotlinFileSerializedData(metadata, binaryFile)
if (processCompiledFileData != null) {
val ioFile = ktSourceFile.toIoFileOrNull() ?: error("No file found for source ${ktSourceFile.path}")
processCompiledFileData(ioFile, compiledKotlinFile)
}
if (processCompiledFileData != null) {
val ioFile = ktSourceFile.toIoFileOrNull() ?: error("No file found for source ${ktSourceFile.path}")
processCompiledFileData(ioFile, compiledKotlinFile)
}
compiledKotlinFile
add(compiledKotlinFile)
}
}
val header = serializeKlibHeader(
@@ -6,21 +6,35 @@
package org.jetbrains.kotlin.ir.backend.js
import com.intellij.openapi.project.Project
import com.intellij.psi.PsiElement
import org.jetbrains.kotlin.KtPsiSourceFile
import org.jetbrains.kotlin.KtSourceFile
import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataSerializer
import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibSingleFileMetadataSerializer
import org.jetbrains.kotlin.config.CommonConfigurationKeys
import org.jetbrains.kotlin.config.CompilerConfiguration
import org.jetbrains.kotlin.config.LanguageVersionSettings
import org.jetbrains.kotlin.config.languageVersionSettings
import org.jetbrains.kotlin.descriptors.CallableDescriptor
import org.jetbrains.kotlin.descriptors.ClassifierDescriptor
import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
import org.jetbrains.kotlin.library.metadata.KlibMetadataVersion
import org.jetbrains.kotlin.metadata.ProtoBuf
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.psi.KtFile
import org.jetbrains.kotlin.resolve.BindingContext
import org.jetbrains.kotlin.resolve.BindingContextUtils
import org.jetbrains.kotlin.resolve.descriptorUtil.fqNameSafe
import org.jetbrains.kotlin.resolve.descriptorUtil.module
import org.jetbrains.kotlin.serialization.DescriptorSerializer
// TODO: need a refactoring between IncrementalSerializer and MonolithicSerializer.
class KlibMetadataIncrementalSerializer(
private val ktFiles: List<KtFile>,
private val bindingContext: BindingContext,
private val moduleDescriptor: ModuleDescriptor,
languageVersionSettings: LanguageVersionSettings,
metadataVersion: KlibMetadataVersion,
project: Project,
@@ -33,9 +47,54 @@ class KlibMetadataIncrementalSerializer(
exportKDoc = exportKDoc,
skipExpects = true, // Incremental compilation is not supposed to work when producing pure metadata (IR-less) KLIBs.
allowErrorTypes = allowErrorTypes
) {
), KlibSingleFileMetadataSerializer<KtFile> {
fun serializePackageFragment(
constructor(
files: List<KtFile>,
configuration: CompilerConfiguration,
project: Project,
bindingContext: BindingContext,
moduleDescriptor: ModuleDescriptor,
allowErrorTypes: Boolean,
) : this(
ktFiles = files,
bindingContext = bindingContext,
moduleDescriptor = moduleDescriptor,
languageVersionSettings = configuration.languageVersionSettings,
metadataVersion = configuration.get(CommonConfigurationKeys.METADATA_VERSION) as? KlibMetadataVersion
?: KlibMetadataVersion.INSTANCE,
project = project,
exportKDoc = false,
allowErrorTypes = allowErrorTypes,
)
constructor(modulesStructure: ModulesStructure, moduleFragment: IrModuleFragment) : this(
(modulesStructure.mainModule as MainModule.SourceFiles).files,
modulesStructure.compilerConfiguration,
modulesStructure.project,
modulesStructure.jsFrontEndResult.bindingContext,
moduleFragment.descriptor,
modulesStructure.jsFrontEndResult.hasErrors,
)
override fun serializeSingleFileMetadata(file: KtFile): ProtoBuf.PackageFragment {
val memberScope = file.declarations.map { getDescriptorForElement(bindingContext, it) }
return serializePackageFragment(moduleDescriptor, memberScope, file.packageFqName)
}
override val numberOfSourceFiles: Int
get() = ktFiles.size
override fun forEachFile(block: (Int, KtFile, KtSourceFile, FqName) -> Unit) {
ktFiles.forEachIndexed { i, ktFile ->
block(i, ktFile, KtPsiSourceFile(ktFile), ktFile.packageFqName)
}
}
private fun getDescriptorForElement(context: BindingContext, element: PsiElement): DeclarationDescriptor =
BindingContextUtils.getNotNull(context, BindingContext.DECLARATION_TO_DESCRIPTOR, element)
private fun serializePackageFragment(
module: ModuleDescriptor,
scope: Collection<DeclarationDescriptor>,
fqName: FqName
@@ -6,9 +6,8 @@
package org.jetbrains.kotlin.ir.backend.js
import com.intellij.openapi.project.Project
import com.intellij.openapi.vfs.VfsUtilCore
import com.intellij.psi.PsiElement
import org.jetbrains.kotlin.*
import org.jetbrains.kotlin.KtPsiSourceFile
import org.jetbrains.kotlin.KtSourceFile
import org.jetbrains.kotlin.analyzer.AbstractAnalyzerWithCompilerReport
import org.jetbrains.kotlin.analyzer.AnalysisResult
import org.jetbrains.kotlin.analyzer.CompilationErrorException
@@ -22,12 +21,12 @@ import org.jetbrains.kotlin.backend.common.overrides.FakeOverrideChecker
import org.jetbrains.kotlin.backend.common.serialization.*
import org.jetbrains.kotlin.backend.common.serialization.mangle.ManglerChecker
import org.jetbrains.kotlin.backend.common.serialization.mangle.descriptor.Ir2DescriptorManglerAdapter
import org.jetbrains.kotlin.backend.common.serialization.metadata.*
import org.jetbrains.kotlin.backend.common.serialization.metadata.DynamicTypeDeserializer
import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibSingleFileMetadataSerializer
import org.jetbrains.kotlin.backend.common.serialization.signature.IdSignatureDescriptor
import org.jetbrains.kotlin.backend.common.toLogger
import org.jetbrains.kotlin.builtins.KotlinBuiltIns
import org.jetbrains.kotlin.config.*
import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
import org.jetbrains.kotlin.diagnostics.DiagnosticReporter
@@ -55,8 +54,6 @@ import org.jetbrains.kotlin.library.impl.BuiltInsPlatform
import org.jetbrains.kotlin.library.impl.buildKotlinLibrary
import org.jetbrains.kotlin.library.metadata.KlibMetadataFactories
import org.jetbrains.kotlin.library.metadata.KlibMetadataVersion
import org.jetbrains.kotlin.metadata.ProtoBuf
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.progress.IncrementalNextRoundException
import org.jetbrains.kotlin.progress.ProgressIndicatorAndCompilationCanceledStatus
import org.jetbrains.kotlin.psi.KtFile
@@ -66,7 +63,6 @@ import org.jetbrains.kotlin.psi2ir.descriptors.IrBuiltInsOverDescriptors
import org.jetbrains.kotlin.psi2ir.generators.GeneratorContext
import org.jetbrains.kotlin.psi2ir.generators.TypeTranslatorImpl
import org.jetbrains.kotlin.resolve.BindingContext
import org.jetbrains.kotlin.resolve.BindingContextUtils
import org.jetbrains.kotlin.storage.LockBasedStorageManager
import org.jetbrains.kotlin.storage.StorageManager
import org.jetbrains.kotlin.utils.DFS
@@ -94,9 +90,6 @@ val KotlinLibrary.serializedIrFileFingerprints: List<SerializedIrFileFingerprint
val KotlinLibrary.serializedKlibFingerprint: SerializedKlibFingerprint?
get() = manifestProperties.getProperty(KLIB_PROPERTY_SERIALIZED_KLIB_FINGERPRINT)?.let { SerializedKlibFingerprint.fromString(it) }
private val CompilerConfiguration.metadataVersion
get() = get(CommonConfigurationKeys.METADATA_VERSION) as? KlibMetadataVersion ?: KlibMetadataVersion.INSTANCE
internal val SerializedIrFile.fileMetadata: ByteArray
get() = backendSpecificMetadata ?: error("Expect file caches to have backendSpecificMetadata, but '$path' doesn't")
@@ -110,9 +103,7 @@ fun generateKLib(
moduleFragment: IrModuleFragment,
diagnosticReporter: DiagnosticReporter,
builtInsPlatform: BuiltInsPlatform = BuiltInsPlatform.JS,
serializeSingleFile: (KtSourceFile) -> ProtoBuf.PackageFragment
) {
val files = (depsDescriptors.mainModule as MainModule.SourceFiles).files.map(::KtPsiSourceFile)
val configuration = depsDescriptors.compilerConfiguration
val allDependencies = depsDescriptors.allDependencies
@@ -120,7 +111,7 @@ fun generateKLib(
configuration[CommonConfigurationKeys.MODULE_NAME]!!,
configuration,
diagnosticReporter,
files,
KlibMetadataIncrementalSerializer(depsDescriptors, moduleFragment),
outputKlibPath,
allDependencies,
moduleFragment,
@@ -131,7 +122,6 @@ fun generateKLib(
abiVersion,
jsOutputName,
builtInsPlatform,
serializeSingleFile
)
}
@@ -521,8 +511,19 @@ class ModulesStructure(
val analysisResult = analyzer.analysisResult
if (compilerConfiguration.getBoolean(CommonConfigurationKeys.INCREMENTAL_COMPILATION)) {
/** can throw [IncrementalNextRoundException] */
compareMetadataAndGoToNextICRoundIfNeeded(analysisResult, compilerConfiguration, project, files, errorPolicy.allowErrors)
val shouldGoToNextIcRound = shouldGoToNextIcRound(compilerConfiguration) {
KlibMetadataIncrementalSerializer(
files,
compilerConfiguration,
project,
analysisResult.bindingContext,
analysisResult.moduleDescriptor,
errorPolicy.allowErrors,
)
}
if (shouldGoToNextIcRound) {
throw IncrementalNextRoundException()
}
}
var hasErrors = false
@@ -589,12 +590,6 @@ class ModulesStructure(
null // null in case compiling builtInModule itself
}
private fun getDescriptorForElement(
context: BindingContext,
element: PsiElement
): DeclarationDescriptor = BindingContextUtils.getNotNull(context, BindingContext.DECLARATION_TO_DESCRIPTOR, element)
private const val FILE_FINGERPRINTS_SEPARATOR = " "
private fun List<SerializedIrFileFingerprint>.joinIrFileFingerprints(): String {
@@ -609,7 +604,7 @@ fun serializeModuleIntoKlib(
moduleName: String,
configuration: CompilerConfiguration,
diagnosticReporter: DiagnosticReporter,
files: List<KtSourceFile>,
metadataSerializer: KlibSingleFileMetadataSerializer<*>,
klibPath: String,
dependencies: List<KotlinLibrary>,
moduleFragment: IrModuleFragment,
@@ -620,7 +615,6 @@ fun serializeModuleIntoKlib(
abiVersion: KotlinAbiVersion,
jsOutputName: String?,
builtInsPlatform: BuiltInsPlatform = BuiltInsPlatform.JS,
serializeSingleFile: (KtSourceFile) -> ProtoBuf.PackageFragment
) {
val moduleExportedNames = moduleFragment.collectExportedNames()
val incrementalResultsConsumer = configuration.get(JSConfigurationKeys.INCREMENTAL_RESULTS_CONSUMER)
@@ -630,7 +624,6 @@ fun serializeModuleIntoKlib(
irModuleFragment = moduleFragment,
configuration = configuration,
diagnosticReporter = diagnosticReporter,
sourceFiles = files,
compatibilityMode = CompatibilityMode(abiVersion),
cleanFiles = cleanFiles,
dependencies = dependencies,
@@ -653,7 +646,7 @@ fun serializeModuleIntoKlib(
shouldCheckSignaturesOnUniqueness,
) { JsIrFileMetadata(moduleExportedNames[it]?.values?.toSmartList() ?: emptyList()) }
},
serializeFileMetadata = { serializeSingleFile(it) to FqName.ROOT },
metadataSerializer = metadataSerializer,
runKlibCheckers = { irModuleFragment, irDiagnosticReporter, compilerConfiguration ->
if (builtInsPlatform == BuiltInsPlatform.JS) {
val cleanFilesIrData = cleanFiles.map { it.irData ?: error("Metadata-only KLIBs are not supported in Kotlin/JS") }
@@ -725,54 +718,22 @@ const val KLIB_PROPERTY_JS_OUTPUT_NAME = "jsOutputName"
const val KLIB_PROPERTY_SERIALIZED_IR_FILE_FINGERPRINTS = "serializedIrFileFingerprints"
const val KLIB_PROPERTY_SERIALIZED_KLIB_FINGERPRINT = "serializedKlibFingerprint"
fun KlibMetadataIncrementalSerializer.serializeScope(
ktFile: KtFile,
bindingContext: BindingContext,
moduleDescriptor: ModuleDescriptor
): ProtoBuf.PackageFragment {
val memberScope = ktFile.declarations.map { getDescriptorForElement(bindingContext, it) }
return serializePackageFragment(moduleDescriptor, memberScope, ktFile.packageFqName)
}
fun KlibMetadataIncrementalSerializer.serializeScope(
ktSourceFile: KtSourceFile,
bindingContext: BindingContext,
moduleDescriptor: ModuleDescriptor
): ProtoBuf.PackageFragment {
val ktFile = (ktSourceFile as KtPsiSourceFile).psiFile as KtFile
val memberScope = ktFile.declarations.map { getDescriptorForElement(bindingContext, it) }
return serializePackageFragment(moduleDescriptor, memberScope, ktFile.packageFqName)
}
private fun compareMetadataAndGoToNextICRoundIfNeeded(
analysisResult: AnalysisResult,
config: CompilerConfiguration,
project: Project,
files: List<KtFile>,
allowErrors: Boolean
) {
val nextRoundChecker = config.get(JSConfigurationKeys.INCREMENTAL_NEXT_ROUND_CHECKER) ?: return
val bindingContext = analysisResult.bindingContext
val serializer = KlibMetadataIncrementalSerializer(config, project, allowErrors)
for (ktFile in files) {
val packageFragment = serializer.serializeScope(ktFile, bindingContext, analysisResult.moduleDescriptor)
// to minimize a number of IC rounds, we should inspect all proto for changes first,
// then go to a next round if needed, with all new dirty files
nextRoundChecker.checkProtoChanges(VfsUtilCore.virtualToIoFile(ktFile.virtualFile), packageFragment.toByteArray())
fun <SourceFile> shouldGoToNextIcRound(
compilerConfiguration: CompilerConfiguration,
createMetadataSerializer: () -> KlibSingleFileMetadataSerializer<SourceFile>,
): Boolean {
val nextRoundChecker = compilerConfiguration.get(JSConfigurationKeys.INCREMENTAL_NEXT_ROUND_CHECKER) ?: return false
createMetadataSerializer().run {
forEachFile { _, sourceFile, ktSourceFile, _ ->
val protoBuf = serializeSingleFileMetadata(sourceFile)
// to minimize the number of IC rounds, we should inspect all proto for changes first,
// then go to the next round if needed, with all new dirty files
nextRoundChecker.checkProtoChanges(ktSourceFile.toIoFileOrNull()!!, protoBuf.toByteArray())
}
}
if (nextRoundChecker.shouldGoToNextRound()) throw IncrementalNextRoundException()
return nextRoundChecker.shouldGoToNextRound()
}
fun KlibMetadataIncrementalSerializer(configuration: CompilerConfiguration, project: Project, allowErrors: Boolean) =
KlibMetadataIncrementalSerializer(
languageVersionSettings = configuration.languageVersionSettings,
metadataVersion = configuration.metadataVersion,
project = project,
exportKDoc = false,
allowErrorTypes = allowErrors
)
private fun Map<IrModuleFragment, KotlinLibrary>.getUniqueNameForEachFragment(): Map<IrModuleFragment, String> {
return this.entries.mapNotNull { (moduleFragment, klib) ->
klib.jsOutputName?.let { moduleFragment to it }