[KLIB] Implement new linker based on IdSignature

- Remove klib dependency on metadata and uniqID
 - Refactored proto format to make it more effective and compact
  -- Use special encoding for some types of data (coordinates, flags, types)
  -- Remove symbols table
  -- Use packed proto list if it is possible
 - Remove extension from metadata
 - Remove special ids for function interfaces
 - Fix klib IO
 - Fix incremental cache
 - General code clean up
This commit is contained in:
Roman Artemev
2020-01-27 15:58:52 +03:00
committed by romanart
parent 31d73c5d79
commit 6a37955a36
211 changed files with 6616 additions and 78716 deletions
@@ -1,25 +0,0 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.backend.js
import org.jetbrains.kotlin.descriptors.PropertyDescriptor
import org.jetbrains.kotlin.descriptors.Visibilities
import org.jetbrains.kotlin.descriptors.Visibility
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.psi2ir.generators.GeneratorExtensions
class JsGeneratorExtensions : GeneratorExtensions() {
override fun computeFieldVisibility(descriptor: PropertyDescriptor): Visibility =
if (descriptor.annotations.hasAnnotation(JS_EXPORT_FQ_NAME))
descriptor.visibility
else
Visibilities.PRIVATE
companion object {
val JS_EXPORT_FQ_NAME = FqName("kotlin.js.JsExport")
}
}
@@ -13,19 +13,19 @@ import org.jetbrains.kotlin.analyzer.AnalysisResult
import org.jetbrains.kotlin.backend.common.LoggingContext
import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension
import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext
import org.jetbrains.kotlin.backend.common.serialization.DescriptorTable
import org.jetbrains.kotlin.backend.common.serialization.DeserializationStrategy
import org.jetbrains.kotlin.backend.common.serialization.KlibIrVersion
import org.jetbrains.kotlin.backend.common.serialization.mangle.ManglerChecker
import org.jetbrains.kotlin.backend.common.serialization.mangle.descriptor.Ir2DescriptorManglerAdapter
import org.jetbrains.kotlin.backend.common.serialization.metadata.DynamicTypeDeserializer
import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataVersion
import org.jetbrains.kotlin.backend.common.serialization.signature.IdSignatureDescriptor
import org.jetbrains.kotlin.builtins.KotlinBuiltIns
import org.jetbrains.kotlin.config.*
import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker
import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrModuleSerializer
import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsMangler
import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.*
import org.jetbrains.kotlin.ir.backend.js.lower.serialization.metadata.KlibMetadataIncrementalSerializer
import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
import org.jetbrains.kotlin.ir.descriptors.IrBuiltIns
@@ -47,6 +47,7 @@ import org.jetbrains.kotlin.psi.KtFile
import org.jetbrains.kotlin.psi2ir.Psi2IrConfiguration
import org.jetbrains.kotlin.psi2ir.Psi2IrTranslator
import org.jetbrains.kotlin.psi2ir.generators.GeneratorContext
import org.jetbrains.kotlin.psi2ir.generators.GeneratorExtensions
import org.jetbrains.kotlin.resolve.BindingContext
import org.jetbrains.kotlin.resolve.BindingContextUtils
import org.jetbrains.kotlin.storage.LockBasedStorageManager
@@ -113,7 +114,7 @@ fun generateKLib(
val irData = compiledIrFiles[f] ?: error("No Ir Data found for file $f")
val metaFile = compiledMetaFiles[f] ?: error("No Meta Data found for file $f")
val irFile = with(irData) {
SerializedIrFile(fileData, String(fqn), f.path.replace('\\', '/'), symbols, types, strings, bodies, declarations)
SerializedIrFile(fileData, String(fqn), f.path.replace('\\', '/'), types, signatures, strings, bodies, declarations)
}
storage.add(KotlinFileSerializedData(metaFile.metadata, irFile))
}
@@ -133,6 +134,8 @@ fun generateKLib(
val moduleFragment = psi2IrContext.generateModuleFragmentWithPlugins(project, files,
deserializer = null, expectDescriptorToSymbol = expectDescriptorToSymbol)
moduleFragment.acceptVoid(ManglerChecker(JsManglerIr, Ir2DescriptorManglerAdapter(JsManglerDesc)))
val moduleName = configuration[CommonConfigurationKeys.MODULE_NAME]!!
if (!configuration.klibMpp) {
@@ -185,9 +188,8 @@ fun loadIr(
val psi2IrContext: GeneratorContext = runAnalysisAndPreparePsi2Ir(depsDescriptors)
val irBuiltIns = psi2IrContext.irBuiltIns
val symbolTable = psi2IrContext.symbolTable
val moduleDescriptor = psi2IrContext.moduleDescriptor
val deserializer = JsIrLinker(moduleDescriptor, JsMangler, emptyLoggingContext, irBuiltIns, symbolTable)
val deserializer = JsIrLinker(emptyLoggingContext, irBuiltIns, symbolTable)
val deserializedModuleFragments = sortDependencies(allDependencies.getFullList(), depsDescriptors.descriptors).map {
deserializer.deserializeIrModuleHeader(depsDescriptors.getModuleDescriptor(it))!!
@@ -426,12 +428,16 @@ fun serializeModuleIntoKlib(
) {
assert(files.size == moduleFragment.files.size)
val descriptorTable = DescriptorTable.createDefault()
val serializedIr =
JsIrModuleSerializer(emptyLoggingContext, moduleFragment.irBuiltins, descriptorTable, skipExpects = !configuration.klibMpp, expectDescriptorToSymbol = expectDescriptorToSymbol).serializedIrModule(moduleFragment)
JsIrModuleSerializer(
emptyLoggingContext,
moduleFragment.irBuiltins,
expectDescriptorToSymbol = expectDescriptorToSymbol,
skipExpects = !configuration.klibMpp
).serializedIrModule(moduleFragment)
val moduleDescriptor = moduleFragment.descriptor
val metadataSerializer = KlibMetadataIncrementalSerializer(configuration, descriptorTable)
val metadataSerializer = KlibMetadataIncrementalSerializer(configuration)
val incrementalResultsConsumer = configuration.get(JSConfigurationKeys.INCREMENTAL_RESULTS_CONSUMER)
val empty = ByteArray(0)
@@ -440,7 +446,7 @@ fun serializeModuleIntoKlib(
incrementalResultsConsumer?.run {
processPackagePart(ioFile, compiledFile.metadata, empty, empty)
with(compiledFile.irData) {
processIrFile(ioFile, fileData, symbols, types, strings, declarations, bodies, fqName.toByteArray())
processIrFile(ioFile, fileData, types, signatures, strings, declarations, bodies, fqName.toByteArray())
}
}
}
@@ -517,7 +523,7 @@ private fun compareMetadataAndGoToNextICRoundIfNeeded(
) {
val nextRoundChecker = config.get(JSConfigurationKeys.INCREMENTAL_NEXT_ROUND_CHECKER) ?: return
val bindingContext = analysisResult.bindingContext
val serializer = KlibMetadataIncrementalSerializer(config, FakeDescriptorTable())
val serializer = KlibMetadataIncrementalSerializer(config)
for (ktFile in files) {
val packageFragment = serializer.serializeScope(ktFile, bindingContext, analysisResult.moduleDescriptor)
// to minimize a number of IC rounds, we should inspect all proto for changes first,
@@ -528,27 +534,8 @@ private fun compareMetadataAndGoToNextICRoundIfNeeded(
if (nextRoundChecker.shouldGoToNextRound()) throw IncrementalNextRoundException()
}
/**
* A hack to serialize metadata to compare it with cached proto before frontend errors are reported.
* DescriptorUniqId is not used during comparison, so fake IDs can be used to satisfy [KlibMetadataIncrementalSerializer] interface.
*/
private class FakeDescriptorTable : DescriptorTable {
private val descriptors = mutableMapOf<DeclarationDescriptor, Long>()
override fun put(descriptor: DeclarationDescriptor, uniqId: UniqId) {
throw NotImplementedError("FakeDescriptorTable#put is not expected to be called!")
}
override fun get(descriptor: DeclarationDescriptor): Long =
descriptors.getOrPut(descriptor) { descriptors.size.toLong() }
}
private fun KlibMetadataIncrementalSerializer(
configuration: CompilerConfiguration,
descriptorTable: DescriptorTable
) = KlibMetadataIncrementalSerializer(
private fun KlibMetadataIncrementalSerializer(configuration: CompilerConfiguration) = KlibMetadataIncrementalSerializer(
languageVersionSettings = configuration.languageVersionSettings,
metadataVersion = configuration.metadataVersion,
descriptorTable = descriptorTable,
skipExpects = !configuration.klibMpp
)
@@ -1,30 +1,32 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir
import org.jetbrains.kotlin.backend.common.serialization.*
import org.jetbrains.kotlin.backend.common.serialization.GlobalDeclarationTable
import org.jetbrains.kotlin.backend.common.serialization.IdSignatureClashTracker
import org.jetbrains.kotlin.backend.common.serialization.signature.IdSignatureSerializer
import org.jetbrains.kotlin.descriptors.PropertyDescriptor
import org.jetbrains.kotlin.ir.declarations.IrDeclaration
import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction
import org.jetbrains.kotlin.ir.declarations.IrTypeParameter
import org.jetbrains.kotlin.ir.descriptors.IrBuiltIns
import org.jetbrains.kotlin.ir.util.UniqId
import org.jetbrains.kotlin.ir.util.IdSignature
import org.jetbrains.kotlin.ir.util.render
class JsUniqIdClashTracker : UniqIdClashTracker {
private val committedUniqIds = mutableMapOf<UniqId, IrDeclaration>()
class JsUniqIdClashTracker : IdSignatureClashTracker {
private val committedIdSignatures = mutableMapOf<IdSignature, IrDeclaration>()
override fun commit(declaration: IrDeclaration, uniqId: UniqId) {
if (uniqId.isLocal) return // don't track local ids
override fun commit(declaration: IrDeclaration, signature: IdSignature) {
if (!signature.isPublic) return // don't track local ids
if (uniqId in committedUniqIds) {
val clashedDeclaration = committedUniqIds[uniqId]!!
if (signature in committedIdSignatures) {
val clashedDeclaration = committedIdSignatures[signature]!!
if (declaration !is IrTypeParameter && declaration.descriptor.containingDeclaration !is PropertyDescriptor) {
// TODO: handle clashes properly
error("UniqId clash: $uniqId; Existed declaration ${clashedDeclaration.render()} clashed with new ${declaration.render()}")
error("IdSignature clash: $signature; Existed declaration ${clashedDeclaration.render()} clashed with new ${declaration.render()}")
} else {
// Check whether they are type parameters of the same extension property but different accessors
val parent = declaration.parent
@@ -36,11 +38,12 @@ class JsUniqIdClashTracker : UniqIdClashTracker {
}
}
committedUniqIds[uniqId] = declaration
committedIdSignatures[signature] = declaration
}
}
class JsGlobalDeclarationTable(builtIns: IrBuiltIns) : GlobalDeclarationTable(JsMangler, JsUniqIdClashTracker()) {
class JsGlobalDeclarationTable(signatureSerializer: IdSignatureSerializer, builtIns: IrBuiltIns) :
GlobalDeclarationTable(signatureSerializer, JsManglerIr, JsUniqIdClashTracker()) {
init {
loadKnownBuiltins(builtIns)
}
@@ -1,21 +0,0 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir
import org.jetbrains.kotlin.backend.common.serialization.*
import org.jetbrains.kotlin.backend.common.serialization.DescriptorReferenceDeserializer
import org.jetbrains.kotlin.backend.common.serialization.DescriptorUniqIdAware
import org.jetbrains.kotlin.backend.common.serialization.DeserializedDescriptorUniqIdAware
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.ir.descriptors.IrBuiltIns
import org.jetbrains.kotlin.ir.util.KotlinMangler
class JsDescriptorReferenceDeserializer(
currentModule: ModuleDescriptor,
mangler: KotlinMangler,
builtIns: IrBuiltIns
) : DescriptorReferenceDeserializer(currentModule, mangler, builtIns, mutableMapOf()),
DescriptorUniqIdAware by DeserializedDescriptorUniqIdAware
@@ -8,18 +8,19 @@ package org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir
import org.jetbrains.kotlin.backend.common.LoggingContext
import org.jetbrains.kotlin.backend.common.serialization.IrFileSerializer
import org.jetbrains.kotlin.backend.common.serialization.DeclarationTable
import org.jetbrains.kotlin.ir.declarations.IrProperty
import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
import org.jetbrains.kotlin.ir.symbols.IrSymbol
class JsIrFileSerializer(
logger: LoggingContext,
declarationTable: DeclarationTable,
private val expectDescriptorToSymbol: MutableMap<DeclarationDescriptor, IrSymbol>,
expectDescriptorToSymbol: MutableMap<DeclarationDescriptor, IrSymbol>,
skipExpects: Boolean,
bodiesOnlyForInlines: Boolean = false
) : IrFileSerializer(logger, declarationTable, expectDescriptorToSymbol, bodiesOnlyForInlines = bodiesOnlyForInlines, skipExpects = skipExpects) {
// Temporary keep order of any property, even of constants
override fun keepOrderOfProperties(property: IrProperty): Boolean = true
}
) : IrFileSerializer(
logger,
declarationTable,
expectDescriptorToSymbol,
bodiesOnlyForInlines = bodiesOnlyForInlines,
skipExpects = skipExpects
)
@@ -6,39 +6,25 @@
package org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir
import org.jetbrains.kotlin.backend.common.LoggingContext
import org.jetbrains.kotlin.backend.common.serialization.*
import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
import org.jetbrains.kotlin.descriptors.DeclarationDescriptorWithVisibility
import org.jetbrains.kotlin.backend.common.serialization.DeserializationStrategy
import org.jetbrains.kotlin.backend.common.serialization.KotlinIrLinker
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.descriptors.Visibilities
import org.jetbrains.kotlin.descriptors.konan.kotlinLibrary
import org.jetbrains.kotlin.ir.descriptors.IrBuiltIns
import org.jetbrains.kotlin.ir.util.KotlinMangler
import org.jetbrains.kotlin.ir.util.SymbolTable
import org.jetbrains.kotlin.ir.util.UniqId
import org.jetbrains.kotlin.resolve.descriptorUtil.isPublishedApi
class JsIrLinker(
currentModule: ModuleDescriptor,
mangler: KotlinMangler,
logger: LoggingContext,
builtIns: IrBuiltIns,
symbolTable: SymbolTable
) : KotlinIrLinker(logger, builtIns, symbolTable, emptyList(), null, mangler),
DescriptorUniqIdAware by DeserializedDescriptorUniqIdAware {
class JsIrLinker(logger: LoggingContext, builtIns: IrBuiltIns, symbolTable: SymbolTable) :
KotlinIrLinker(logger, builtIns, symbolTable, emptyList(), null) {
override val descriptorReferenceDeserializer =
JsDescriptorReferenceDeserializer(currentModule, mangler, builtIns)
override fun reader(moduleDescriptor: ModuleDescriptor, fileIndex: Int, uniqId: UniqId) =
moduleDescriptor.kotlinLibrary.irDeclaration(uniqId.index, fileIndex)
override fun readSymbol(moduleDescriptor: ModuleDescriptor, fileIndex: Int, symbolIndex: Int) =
moduleDescriptor.kotlinLibrary.symbol(symbolIndex, fileIndex)
override fun reader(moduleDescriptor: ModuleDescriptor, fileIndex: Int, idSigIndex: Int) =
moduleDescriptor.kotlinLibrary.irDeclaration(idSigIndex, fileIndex)
override fun readType(moduleDescriptor: ModuleDescriptor, fileIndex: Int, typeIndex: Int) =
moduleDescriptor.kotlinLibrary.type(typeIndex, fileIndex)
override fun readSignature(moduleDescriptor: ModuleDescriptor, fileIndex: Int, signatureIndex: Int) =
moduleDescriptor.kotlinLibrary.signature(signatureIndex, fileIndex)
override fun readString(moduleDescriptor: ModuleDescriptor, fileIndex: Int, stringIndex: Int) =
moduleDescriptor.kotlinLibrary.string(stringIndex, fileIndex)
@@ -51,5 +37,9 @@ class JsIrLinker(
override fun readFileCount(moduleDescriptor: ModuleDescriptor) =
moduleDescriptor.kotlinLibrary.fileCount()
private val ModuleDescriptor.userName get() = kotlinLibrary.libraryFile.absolutePath
override fun createModuleDeserializer(moduleDescriptor: ModuleDescriptor, strategy: DeserializationStrategy): IrModuleDeserializer =
JsModuleDeserializer(moduleDescriptor, strategy)
private inner class JsModuleDeserializer(moduleDescriptor: ModuleDescriptor, strategy: DeserializationStrategy) :
KotlinIrLinker.IrModuleDeserializer(moduleDescriptor, strategy)
}
@@ -7,8 +7,8 @@ package org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir
import org.jetbrains.kotlin.backend.common.LoggingContext
import org.jetbrains.kotlin.backend.common.serialization.DeclarationTable
import org.jetbrains.kotlin.backend.common.serialization.DescriptorTable
import org.jetbrains.kotlin.backend.common.serialization.IrModuleSerializer
import org.jetbrains.kotlin.backend.common.serialization.signature.IdSignatureSerializer
import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
import org.jetbrains.kotlin.ir.declarations.IrFile
import org.jetbrains.kotlin.ir.descriptors.IrBuiltIns
@@ -17,13 +17,13 @@ import org.jetbrains.kotlin.ir.symbols.IrSymbol
class JsIrModuleSerializer(
logger: LoggingContext,
irBuiltIns: IrBuiltIns,
private val descriptorTable: DescriptorTable,
private val expectDescriptorToSymbol: MutableMap<DeclarationDescriptor, IrSymbol>,
val skipExpects: Boolean
) : IrModuleSerializer<JsIrFileSerializer>(logger) {
private val globalDeclarationTable = JsGlobalDeclarationTable(irBuiltIns)
private val signaturer = IdSignatureSerializer(JsManglerIr)
private val globalDeclarationTable = JsGlobalDeclarationTable(signaturer, irBuiltIns)
override fun createSerializerForFile(file: IrFile): JsIrFileSerializer =
JsIrFileSerializer(logger, DeclarationTable(descriptorTable, globalDeclarationTable, 0), expectDescriptorToSymbol, skipExpects = skipExpects)
JsIrFileSerializer(logger, DeclarationTable(globalDeclarationTable), expectDescriptorToSymbol, skipExpects = skipExpects)
}
@@ -7,9 +7,7 @@ package org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir
import org.jetbrains.kotlin.backend.common.serialization.mangle.KotlinExportChecker
import org.jetbrains.kotlin.backend.common.serialization.mangle.KotlinMangleComputer
import org.jetbrains.kotlin.backend.common.serialization.mangle.classic.ClassicExportChecker
import org.jetbrains.kotlin.backend.common.serialization.mangle.classic.ClassicKotlinManglerImpl
import org.jetbrains.kotlin.backend.common.serialization.mangle.classic.ClassicMangleComputer
import org.jetbrains.kotlin.backend.common.serialization.mangle.MangleMode
import org.jetbrains.kotlin.backend.common.serialization.mangle.descriptor.DescriptorBasedKotlinManglerImpl
import org.jetbrains.kotlin.backend.common.serialization.mangle.descriptor.DescriptorExportCheckerVisitor
import org.jetbrains.kotlin.backend.common.serialization.mangle.descriptor.DescriptorMangleComputer
@@ -29,14 +27,14 @@ abstract class AbstractJsManglerIr : IrBasedKotlinManglerImpl() {
override fun IrDeclaration.isPlatformSpecificExported() = false
}
private class JsIrManglerComputer(builder: StringBuilder) : IrMangleComputer(builder) {
override fun copy(): IrMangleComputer = JsIrManglerComputer(builder)
private class JsIrManglerComputer(builder: StringBuilder, mode: MangleMode) : IrMangleComputer(builder, mode) {
override fun copy(newMode: MangleMode): IrMangleComputer = JsIrManglerComputer(builder, newMode)
}
override fun getExportChecker(): KotlinExportChecker<IrDeclaration> = exportChecker
override fun getMangleComputer(prefix: String): KotlinMangleComputer<IrDeclaration> {
return JsIrManglerComputer(StringBuilder(256))
override fun getMangleComputer(mode: MangleMode): KotlinMangleComputer<IrDeclaration> {
return JsIrManglerComputer(StringBuilder(256), mode)
}
}
@@ -52,14 +50,15 @@ abstract class AbstractJsDescriptorMangler : DescriptorBasedKotlinManglerImpl()
override fun DeclarationDescriptor.isPlatformSpecificExported() = false
}
private class JsDescriptorManglerComputer(builder: StringBuilder, prefix: String) : DescriptorMangleComputer(builder, prefix) {
override fun copy(): DescriptorMangleComputer = JsDescriptorManglerComputer(builder, specialPrefix)
private class JsDescriptorManglerComputer(builder: StringBuilder, mode: MangleMode) :
DescriptorMangleComputer(builder, mode) {
override fun copy(newMode: MangleMode): DescriptorMangleComputer = JsDescriptorManglerComputer(builder, newMode)
}
override fun getExportChecker(): KotlinExportChecker<DeclarationDescriptor> = exportChecker
override fun getMangleComputer(prefix: String): KotlinMangleComputer<DeclarationDescriptor> {
return JsDescriptorManglerComputer(StringBuilder(256), prefix)
override fun getMangleComputer(mode: MangleMode): KotlinMangleComputer<DeclarationDescriptor> {
return JsDescriptorManglerComputer(StringBuilder(256), mode)
}
}