azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[IR SERIALIZATION] Make [de]serialization processes incremental

Browse Source 
- Use local uniqID local indexation file-local
 - Rethink the way declaration is looked for
 - Store in public file index only publicly accessible symbols
 - refactor API
 - code clean up
This commit is contained in:
Roman Artemev
2019-08-08 14:16:19 +03:00
committed by romanart
parent 96a2cc1f17
commit d50eb5ce52
7 changed files with 225 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/DeserializeDescriptorReference.kt
+3
View File
@@ -30,6 +30,9 @@ abstract class DescriptorReferenceDeserializer(
protected open fun resolveSpecialDescriptor(fqn: FqName) = builtIns.builtIns.getBuiltInClassByFqName(fqn)
protected abstract fun resolveSpecialDescriptor(fqn: FqName): DeclarationDescriptor
abstract fun checkIfSpecialDescriptorId(id: Long): Boolean
protected abstract fun getDescriptorIdOrNull(descriptor: DeclarationDescriptor): Long?
protected open fun checkIfSpecialDescriptorId(id: Long) = with(mangler) { id.isSpecial }
protected open fun getDescriptorIdOrNull(descriptor: DeclarationDescriptor) =
compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/IrFileSerializer.kt
+62 -68
View File
@@ -5,6 +5,7 @@
package org.jetbrains.kotlin.backend.common.serialization
//import org.jetbrains.kotlin.library.SerializedIrFile
import org.jetbrains.kotlin.backend.common.LoggingContext
import org.jetbrains.kotlin.backend.common.ir.ir2string
import org.jetbrains.kotlin.descriptors.*
@@ -12,7 +13,6 @@ import org.jetbrains.kotlin.builtins.FunctionInterfacePackageFragment
import org.jetbrains.kotlin.descriptors.ClassKind
import org.jetbrains.kotlin.descriptors.ClassKind.*
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.IrStatement
import org.jetbrains.kotlin.ir.SourceManager
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.expressions.*
@@ -24,9 +24,9 @@ import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid
import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid
import org.jetbrains.kotlin.ir.visitors.acceptVoid
import org.jetbrains.kotlin.library.IrIrSerializedIrFile
import org.jetbrains.kotlin.library.SerializedIrModule
//import org.jetbrains.kotlin.library.SerializedIrFile
import org.jetbrains.kotlin.library.impl.*
import org.jetbrains.kotlin.library.impl.SerializedDeclaration
import org.jetbrains.kotlin.library.impl.SkippedDeclaration
import org.jetbrains.kotlin.library.impl.TopLevelDeclaration
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.name.Name
import org.jetbrains.kotlin.types.Variance
@@ -35,9 +35,9 @@ import org.jetbrains.kotlin.backend.common.serialization.proto.ClassKind as Prot
import org.jetbrains.kotlin.backend.common.serialization.proto.Coordinates as ProtoCoordinates
import org.jetbrains.kotlin.backend.common.serialization.proto.FieldAccessCommon as ProtoFieldAccessCommon
import org.jetbrains.kotlin.backend.common.serialization.proto.FileEntry as ProtoFileEntry
import org.jetbrains.kotlin.backend.common.serialization.proto.FqName as ProtoFqName
import org.jetbrains.kotlin.backend.common.serialization.proto.IrAnonymousInit as ProtoAnonymousInit
import org.jetbrains.kotlin.backend.common.serialization.proto.IrBlock as ProtoBlock
import org.jetbrains.kotlin.backend.common.serialization.proto.IrDataIndex as ProtoBodyIndex
import org.jetbrains.kotlin.backend.common.serialization.proto.IrBlockBody as ProtoBlockBody
import org.jetbrains.kotlin.backend.common.serialization.proto.IrBranch as ProtoBranch
import org.jetbrains.kotlin.backend.common.serialization.proto.IrBreak as ProtoBreak
@@ -50,6 +50,10 @@ import org.jetbrains.kotlin.backend.common.serialization.proto.IrConst as ProtoC
import org.jetbrains.kotlin.backend.common.serialization.proto.IrConstructor as ProtoConstructor
import org.jetbrains.kotlin.backend.common.serialization.proto.IrConstructorCall as ProtoConstructorCall
import org.jetbrains.kotlin.backend.common.serialization.proto.IrContinue as ProtoContinue
import org.jetbrains.kotlin.backend.common.serialization.proto.IrDataIndex as ProtoBodyIndex
import org.jetbrains.kotlin.backend.common.serialization.proto.IrDataIndex as ProtoStringIndex
import org.jetbrains.kotlin.backend.common.serialization.proto.IrDataIndex as ProtoSymbolIndex
import org.jetbrains.kotlin.backend.common.serialization.proto.IrDataIndex as ProtoTypeIndex
import org.jetbrains.kotlin.backend.common.serialization.proto.IrDeclaration as ProtoDeclaration
import org.jetbrains.kotlin.backend.common.serialization.proto.IrDeclarationBase as ProtoDeclarationBase
import org.jetbrains.kotlin.backend.common.serialization.proto.IrDeclarationContainer as ProtoDeclarationContainer
@@ -77,7 +81,6 @@ import org.jetbrains.kotlin.backend.common.serialization.proto.IrGetValue as Pro
import org.jetbrains.kotlin.backend.common.serialization.proto.IrInstanceInitializerCall as ProtoInstanceInitializerCall
import org.jetbrains.kotlin.backend.common.serialization.proto.IrLocalDelegatedProperty as ProtoLocalDelegatedProperty
import org.jetbrains.kotlin.backend.common.serialization.proto.IrLocalDelegatedPropertyReference as ProtoLocalDelegatedPropertyReference
import org.jetbrains.kotlin.backend.common.serialization.proto.IrModule as ProtoModule
import org.jetbrains.kotlin.backend.common.serialization.proto.IrOperation as ProtoOperation
import org.jetbrains.kotlin.backend.common.serialization.proto.IrProperty as ProtoProperty
import org.jetbrains.kotlin.backend.common.serialization.proto.IrPropertyReference as ProtoPropertyReference
@@ -90,7 +93,6 @@ import org.jetbrains.kotlin.backend.common.serialization.proto.IrStarProjection
import org.jetbrains.kotlin.backend.common.serialization.proto.IrStatement as ProtoStatement
import org.jetbrains.kotlin.backend.common.serialization.proto.IrStatementOrigin as ProtoStatementOrigin
import org.jetbrains.kotlin.backend.common.serialization.proto.IrStringConcat as ProtoStringConcat
import org.jetbrains.kotlin.backend.common.serialization.proto.IrDataIndex as ProtoSymbolIndex
import org.jetbrains.kotlin.backend.common.serialization.proto.IrSymbolData as ProtoSymbolData
import org.jetbrains.kotlin.backend.common.serialization.proto.IrSymbolKind as ProtoSymbolKind
import org.jetbrains.kotlin.backend.common.serialization.proto.IrSyntheticBody as ProtoSyntheticBody
@@ -101,7 +103,6 @@ import org.jetbrains.kotlin.backend.common.serialization.proto.IrType as ProtoTy
import org.jetbrains.kotlin.backend.common.serialization.proto.IrTypeAbbreviation as ProtoTypeAbbreviation
import org.jetbrains.kotlin.backend.common.serialization.proto.IrTypeAlias as ProtoTypeAlias
import org.jetbrains.kotlin.backend.common.serialization.proto.IrTypeArgument as ProtoTypeArgument
import org.jetbrains.kotlin.backend.common.serialization.proto.IrDataIndex as ProtoTypeIndex
import org.jetbrains.kotlin.backend.common.serialization.proto.IrTypeOp as ProtoTypeOp
import org.jetbrains.kotlin.backend.common.serialization.proto.IrTypeOperator as ProtoTypeOperator
import org.jetbrains.kotlin.backend.common.serialization.proto.IrTypeParameter as ProtoTypeParameter
@@ -118,50 +119,55 @@ import org.jetbrains.kotlin.backend.common.serialization.proto.Loop as ProtoLoop
import org.jetbrains.kotlin.backend.common.serialization.proto.MemberAccessCommon as ProtoMemberAccessCommon
import org.jetbrains.kotlin.backend.common.serialization.proto.ModalityKind as ProtoModalityKind
import org.jetbrains.kotlin.backend.common.serialization.proto.NullableIrExpression as ProtoNullableIrExpression
import org.jetbrains.kotlin.backend.common.serialization.proto.IrDataIndex as ProtoStringIndex
import org.jetbrains.kotlin.backend.common.serialization.proto.TypeArguments as ProtoTypeArguments
import org.jetbrains.kotlin.backend.common.serialization.proto.Visibility as ProtoVisibility
import org.jetbrains.kotlin.backend.common.serialization.proto.FqName as ProtoFqName
open class IrFileSerializer(
val logger: LoggingContext,
val declarationTable: DeclarationTable,
val mangler: KotlinMangler,
val bodiesOnlyForInlines: Boolean = false
private val declarationTable: DeclarationTable,
mangler: KotlinMangler,
private val bodiesOnlyForInlines: Boolean = false
) {
private val loopIndex = mutableMapOf<IrLoop, Int>()
private var currentLoopIndex = 0
val descriptorReferenceSerializer = DescriptorReferenceSerializer(declarationTable, { serializeString(it) }, { serializeFqName(it) }, mangler)
// The same symbol can be used multiple times in a module
// The same symbol can be used multiple times in a file
// so use this index to store symbol data only once.
var protoSymbolMap = mutableMapOf<IrSymbol, Int>()
var protoSymbolArray = arrayListOf<ProtoSymbolData>()
private val protoSymbolMap = mutableMapOf<IrSymbol, Int>()
private val protoSymbolArray = arrayListOf<ProtoSymbolData>()
// The same type can be used multiple times in a module
// The same type can be used multiple times in a file
// so use this index to store type data only once.
var protoTypeMap = mutableMapOf<IrTypeKey, Int>()
var protoTypeArray = arrayListOf<ProtoType>()
private val protoTypeMap = mutableMapOf<IrTypeKey, Int>()
private val protoTypeArray = arrayListOf<ProtoType>()
var protoStringMap = mutableMapOf<String, Int>()
var protoStringArray = arrayListOf<String>()
private val protoStringMap = mutableMapOf<String, Int>()
private val protoStringArray = arrayListOf<String>()
var protoBodyArray = mutableListOf<XStatementOrExpression>()
private val protoBodyArray = mutableListOf<XStatementOrExpression>()
sealed class XStatementOrExpression() {
private val descriptorReferenceSerializer =
DescriptorReferenceSerializer(declarationTable, { serializeString(it) }, { serializeFqName(it) }, mangler)
sealed class XStatementOrExpression {
abstract fun toByteArray(): ByteArray
open fun toProtoStatement(): ProtoStatement { error("It is not a ProtoStatement") }
open fun toProtoExpression(): ProtoExpression { error("It is not a ProtoExpression") }
open fun toProtoStatement(): ProtoStatement {
error("It is not a ProtoStatement")
}
class XStatement(private val proto: ProtoStatement): XStatementOrExpression() {
override fun toByteArray() = proto.toByteArray()
open fun toProtoExpression(): ProtoExpression {
error("It is not a ProtoExpression")
}
class XStatement(private val proto: ProtoStatement) : XStatementOrExpression() {
override fun toByteArray(): ByteArray = proto.toByteArray()
override fun toProtoStatement() = proto
}
class XExpression(private val proto: ProtoExpression): XStatementOrExpression() {
override fun toByteArray() = proto.toByteArray()
class XExpression(private val proto: ProtoExpression) : XStatementOrExpression() {
override fun toByteArray(): ByteArray = proto.toByteArray()
override fun toProtoExpression() = proto
}
}
@@ -215,11 +221,11 @@ open class IrFileSerializer(
return proto.build()
}
fun serializeName(name: Name): ProtoStringIndex = serializeString(name.toString())
private fun serializeName(name: Name): ProtoStringIndex = serializeString(name.toString())
/* ------- IrSymbols -------------------------------------------------------- */
fun protoSymbolKind(symbol: IrSymbol): ProtoSymbolKind = when (symbol) {
private fun protoSymbolKind(symbol: IrSymbol): ProtoSymbolKind = when (symbol) {
is IrAnonymousInitializerSymbol ->
ProtoSymbolKind.ANONYMOUS_INIT_SYMBOL
is IrClassSymbol ->
@@ -256,7 +262,7 @@ open class IrFileSerializer(
TODO("Unexpected symbol kind: $symbol")
}
fun serializeIrSymbolData(symbol: IrSymbol): ProtoSymbolData {
private fun serializeIrSymbolData(symbol: IrSymbol): ProtoSymbolData {
val declaration = symbol.owner as? IrDeclaration ?: error("Expected IrDeclaration: ${symbol.owner}")
@@ -284,10 +290,6 @@ open class IrFileSerializer(
val symbolData = serializeIrSymbolData(symbol)
protoSymbolArray.add(symbolData)
protoSymbolArray.size - 1
// val symbolData = serializeIrSymbolData(symbol)
// val uniqId = symbolData.uniqId
// Pair(UniqId(uniqId.index, uniqId.isLocal), symbolData)
}
return proto.build()
}
@@ -304,13 +306,13 @@ open class IrFileSerializer(
return proto.build()
}
fun serializeIrTypeVariance(variance: Variance) = when (variance) {
private fun serializeIrTypeVariance(variance: Variance) = when (variance) {
Variance.IN_VARIANCE -> ProtoTypeVariance.IN
Variance.OUT_VARIANCE -> ProtoTypeVariance.OUT
Variance.INVARIANT -> ProtoTypeVariance.INV
}
fun serializeAnnotations(annotations: List<IrConstructorCall>): ProtoAnnotations {
private fun serializeAnnotations(annotations: List<IrConstructorCall>): ProtoAnnotations {
val proto = ProtoAnnotations.newBuilder()
annotations.forEach {
proto.addAnnotation(serializeConstructorCall(it))
@@ -318,7 +320,7 @@ open class IrFileSerializer(
return proto.build()
}
fun serializeFqName(fqName: FqName): ProtoFqName {
private fun serializeFqName(fqName: FqName): ProtoFqName {
val proto = ProtoFqName.newBuilder()
fqName.pathSegments().forEach {
proto.addSegment(serializeString(it.identifier))
@@ -327,12 +329,12 @@ open class IrFileSerializer(
return proto.build()
}
fun serializeIrTypeProjection(argument: IrTypeProjection) = ProtoTypeProjection.newBuilder()
private fun serializeIrTypeProjection(argument: IrTypeProjection): ProtoTypeProjection = ProtoTypeProjection.newBuilder()
.setVariance(serializeIrTypeVariance(argument.variance))
.setType(serializeIrType(argument.type))
.build()
fun serializeTypeArgument(argument: IrTypeArgument): ProtoTypeArgument {
private fun serializeTypeArgument(argument: IrTypeArgument): ProtoTypeArgument {
val proto = ProtoTypeArgument.newBuilder()
when (argument) {
is IrStarProjection ->
@@ -345,7 +347,7 @@ open class IrFileSerializer(
return proto.build()
}
fun serializeSimpleType(type: IrSimpleType): ProtoSimpleType {
private fun serializeSimpleType(type: IrSimpleType): ProtoSimpleType {
val proto = ProtoSimpleType.newBuilder()
.setAnnotations(serializeAnnotations(type.annotations))
.setClassifier(serializeIrSymbol(type.classifier))
@@ -370,11 +372,11 @@ open class IrFileSerializer(
return proto.build()
}
fun serializeDynamicType(type: IrDynamicType) = ProtoDynamicType.newBuilder()
private fun serializeDynamicType(type: IrDynamicType): ProtoDynamicType = ProtoDynamicType.newBuilder()
.setAnnotations(serializeAnnotations(type.annotations))
.build()
fun serializeErrorType(type: IrErrorType) = ProtoErrorType.newBuilder()
private fun serializeErrorType(type: IrErrorType): ProtoErrorType = ProtoErrorType.newBuilder()
.setAnnotations(serializeAnnotations(type.annotations))
.build()
@@ -419,7 +421,7 @@ open class IrFileSerializer(
val type: IrTypeKey?
)
val IrType.toIrTypeKey: IrTypeKey
private val IrType.toIrTypeKey: IrTypeKey
get() = IrTypeKey(
kind = when (this) {
is IrSimpleType -> IrTypeKind.SIMPLE
@@ -433,7 +435,7 @@ open class IrFileSerializer(
annotations = this.annotations
)
val IrTypeArgument.toIrTypeArgumentKey: IrTypeArgumentKey
private val IrTypeArgument.toIrTypeArgumentKey: IrTypeArgumentKey
get() = IrTypeArgumentKey(
kind = when (this) {
is IrStarProjection -> IrTypeArgumentKind.STAR
@@ -444,7 +446,7 @@ open class IrFileSerializer(
type = (this as? IrTypeProjection)?.type?.toIrTypeKey
)
fun serializeIrType(type: IrType): ProtoTypeIndex {
private fun serializeIrType(type: IrType): ProtoTypeIndex {
val proto = ProtoTypeIndex.newBuilder()
val key = type.toIrTypeKey
proto.index = protoTypeMap.getOrPut(key) {
@@ -522,7 +524,7 @@ open class IrFileSerializer(
}
proto.typeArguments = serializeTypeArguments(call)
for (index in 0..call.valueArgumentsCount - 1) {
for (index in 0 until call.valueArgumentsCount) {
val actual = call.getValueArgument(index)
val argOrNull = ProtoNullableIrExpression.newBuilder()
if (actual == null) {
@@ -530,7 +532,7 @@ open class IrFileSerializer(
// I see a lack of arg for an empty vararg,
// rather than an empty vararg node.
// TODO: how do we assert that without descriptora?
// TODO: how do we assert that without descriptor?
//assert(it.varargElementType != null || it.hasDefaultValue())
} else {
argOrNull.expression = serializeExpression(actual)
@@ -637,7 +639,7 @@ open class IrFileSerializer(
.setLoop(serializeLoop(expression))
.build()
fun serializeEnumConstructorCall(call: IrEnumConstructorCall): ProtoEnumConstructorCall {
private fun serializeEnumConstructorCall(call: IrEnumConstructorCall): ProtoEnumConstructorCall {
val proto = ProtoEnumConstructorCall.newBuilder()
.setSymbol(serializeIrSymbol(call.symbol))
.setMemberAccess(serializeMemberAccessCommon(call))
@@ -667,14 +669,14 @@ open class IrFileSerializer(
private fun serializeGetField(expression: IrGetField): ProtoGetField =
ProtoGetField.newBuilder()
.setFieldAccess(serializeFieldAccessCommon(expression)).apply {
expression.origin?.let { setOrigin(serializeIrStatementOrigin(it)) }
expression.origin?.let { origin = serializeIrStatementOrigin(it) }
}
.build()
private fun serializeGetValue(expression: IrGetValue): ProtoGetValue =
ProtoGetValue.newBuilder()
.setSymbol(serializeIrSymbol(expression.symbol)).apply {
expression.origin?.let { setOrigin(serializeIrStatementOrigin(it)) }
expression.origin?.let { origin = serializeIrStatementOrigin(it) }
}
.build()
@@ -703,7 +705,7 @@ open class IrFileSerializer(
ProtoSetField.newBuilder()
.setFieldAccess(serializeFieldAccessCommon(expression))
.setValue(serializeExpression(expression.value)).apply {
expression.origin?.let { setOrigin(serializeIrStatementOrigin(it)) }
expression.origin?.let { origin = serializeIrStatementOrigin(it) }
}
.build()
@@ -711,7 +713,7 @@ open class IrFileSerializer(
ProtoSetVariable.newBuilder()
.setSymbol(serializeIrSymbol(expression.symbol))
.setValue(serializeExpression(expression.value)).apply {
expression.origin?.let { setOrigin(serializeIrStatementOrigin(it)) }
expression.origin?.let { origin = serializeIrStatementOrigin(it) }
}
.build()
@@ -1288,7 +1290,7 @@ open class IrFileSerializer(
// ---------- Top level ------------------------------------------------------
fun serializeFileEntry(entry: SourceManager.FileEntry) = ProtoFileEntry.newBuilder()
private fun serializeFileEntry(entry: SourceManager.FileEntry): ProtoFileEntry = ProtoFileEntry.newBuilder()
.setName(entry.name)
.addAllLineStartOffsets(entry.lineStartOffsets.asIterable())
.build()
@@ -1299,17 +1301,6 @@ open class IrFileSerializer(
fun serializeIrFile(file: IrFile): IrIrSerializedIrFile {
val topLevelDeclarations = mutableListOf<SerializedDeclaration>()
protoSymbolMap = mutableMapOf()
protoSymbolArray = arrayListOf()
protoTypeMap = mutableMapOf()
protoTypeArray = arrayListOf()
protoStringMap = mutableMapOf()
protoStringArray = arrayListOf()
protoBodyArray = mutableListOf()
val proto = ProtoFile.newBuilder()
.setFileEntry(serializeFileEntry(file.fileEntry))
.setFqName(serializeFqName(file.fqName))
@@ -1324,7 +1315,9 @@ open class IrFileSerializer(
val byteArray = serializeDeclaration(it).toByteArray()
val uniqId = declarationTable.uniqIdByDeclaration(it)
topLevelDeclarations.add(TopLevelDeclaration(uniqId.index, uniqId.isLocal, it.descriptor.toString(), byteArray))
proto.addDeclarationId(protoUniqId(uniqId))
if (uniqId.isPublic) {
proto.addDeclarationId(protoUniqId(uniqId))
}
}
// TODO: is it Konan specific?
@@ -1333,6 +1326,7 @@ open class IrFileSerializer(
file.declarations
.filterIsInstance<IrProperty>()
.filter { it.backingField?.initializer != null && !it.isConst }
.filter { it.visibility.let { it == Visibilities.PUBLIC || it == Visibilities.INTERNAL } }
.forEach { proto.addExplicitlyExportedToCompiler(serializeIrSymbol(it.backingField!!.symbol)) }
// TODO: Konan specific
compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/IrModuleSerializer.kt
+4 -3
View File
@@ -11,9 +11,10 @@ import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
import org.jetbrains.kotlin.library.IrIrSerializedIrFile
import org.jetbrains.kotlin.library.SerializedIrModule
abstract class IrModuleSerializer<F : IrFileSerializer>(protected val logger: LoggingContext,
protected val mangler: KotlinMangler,
protected val declarationTable: DeclarationTable) {
abstract class IrModuleSerializer<F : IrFileSerializer>(
protected val logger: LoggingContext,
protected val mangler: KotlinMangler
) {
abstract fun createSerializerForFile(file: IrFile): F
compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/KlibStructure.kt
-33
View File
@@ -1,33 +0,0 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.backend.common.serialization
import org.jetbrains.kotlin.name.FqName
class Table
sealed class KlibEntry(val fqName: FqName)
class KlibFileEntry(fqName: FqName) : KlibEntry(fqName) {
lateinit var bodyData: ByteArray
lateinit var headerData: ByteArray
val stringTable = Table()
val symbolTable = Table()
val typeTable = Table()
}
class KlibPackageEntry(fqName: FqName) : KlibEntry(fqName) {
val childs = mutableListOf<KlibEntry>()
}
class KlibModuleEntry(val name: String) : KlibEntry(FqName.ROOT) {
val packages = mutableListOf<KlibEntry>()
val stringTable = Table()
val symbolTable = Table()
val typeTable = Table()
}
compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/KotlinIrLinker.kt
+153 -76
View File
@@ -33,8 +33,8 @@ import org.jetbrains.kotlin.ir.types.impl.IrErrorTypeImpl
import org.jetbrains.kotlin.ir.util.IrDeserializer
import org.jetbrains.kotlin.ir.util.NaiveSourceBasedFileEntryImpl
import org.jetbrains.kotlin.ir.util.SymbolTable
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite.newInstance
import org.jetbrains.kotlin.resolve.descriptorUtil.isPublishedApi
import org.jetbrains.kotlin.resolve.descriptorUtil.module
import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedCallableMemberDescriptor
import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedClassDescriptor
@@ -61,9 +61,56 @@ abstract class KotlinIrLinker(
private val firstKnownBuiltinsIndex: Long
) : DescriptorUniqIdAware, IrDeserializer {
protected val deserializedSymbols = mutableMapOf<UniqIdKey, IrSymbol>()
private val reachableTopLevels = mutableSetOf<UniqIdKey>()
private val deserializedTopLevels = mutableSetOf<UniqIdKey>()
sealed class DeserializationState {
val deserializedSymbols = mutableMapOf<UniqIdKey, IrSymbol>()
val reachableTopLevels = mutableSetOf<UniqIdKey>()
val deserializedTopLevels = mutableSetOf<UniqIdKey>()
operator fun contains(key: UniqIdKey) = key in deserializedSymbols
operator fun get(key: UniqIdKey): IrSymbol = deserializedSymbols[key] ?: error("jfsdhsdfhksjdhfkjshdk")
abstract fun addUniqID(key: UniqIdKey)
protected open fun peekReachableKey(): UniqIdKey? = reachableTopLevels.firstOrNull()
class ModuleDeserializationState(val module: IrModuleDeserializer): DeserializationState() {
override fun addUniqID(key: UniqIdKey) {
module.addModuleReachableTopLevel(key)
}
}
class SimpleDeserializationState: DeserializationState() {
override fun addUniqID(key: UniqIdKey) {
reachableTopLevels.add(key)
}
override fun peekReachableKey(): UniqIdKey {
return reachableTopLevels.firstOrNull() ?: error("Expecting non-empty set")
}
}
fun processPendingDeclaration(processor: (UniqIdKey) -> Unit) {
do {
val reachableKey = peekReachableKey() ?: return
if (deserializedSymbols[reachableKey]?.isBound == true) {
reachableTopLevels.remove(reachableKey)
deserializedTopLevels.add(reachableKey)
continue
}
processor(reachableKey)
reachableTopLevels.remove(reachableKey)
deserializedTopLevels.add(reachableKey)
} while (reachableTopLevels.isNotEmpty())
}
}
private val globalDeserializationState = DeserializationState.SimpleDeserializationState()
private val modulesWithReachableTopLevels = mutableSetOf<IrModuleDeserializer>()
//TODO: This is Native specific. Eliminate me.
private val forwardDeclarations = mutableSetOf<IrSymbol>()
@@ -71,32 +118,41 @@ abstract class KotlinIrLinker(
protected val deserializersForModules = mutableMapOf<ModuleDescriptor, IrModuleDeserializer>()
private fun getForwardDeclararationModuleDeserializer() = deserializersForModules.entries.single { it.key.isForwardDeclarationModule }.value
inner class IrModuleDeserializer(
private val moduleDescriptor: ModuleDescriptor,
private val deserializationStrategy: DeserializationStrategy
) {
val fileToDeserializerMap = mutableMapOf<IrFile, IrDeserializerForFile>()
protected val moduleResolvedForwardDeclarations = mutableMapOf<UniqIdKey, UniqIdKey>()
private val moduleDeserializationState = DeserializationState.ModuleDeserializationState(this)
private val moduleReversedFileIndex = mutableMapOf<UniqIdKey, IrDeserializerForFile>()
private val moduleDependencies = mutableListOf<IrModuleDeserializer>()
// This is a heavy initializer
val module = deserializeIrModuleHeader()
private var moduleLoops = mutableMapOf<Int, IrLoopBase>()
inner class IrDeserializerForFile(private var annotationsProto: ProtoAnnotations?, private val fileIndex: Int, onlyHeaders: Boolean) :
IrFileDeserializer(logger, builtIns, symbolTable) {
private var fileLoops = mutableMapOf<Int, IrLoopBase>()
private val symbolProtosCache = mutableMapOf<Int, ProtoSymbolData>()
private val typeProtosCache = mutableMapOf<Int, ProtoType>()
private val stringsCache = mutableMapOf<Int, String>()
private val declarationsBytes: ByteArray by lazy { TODO("") }
private val bodiesBytes: ByteArray by lazy { TODO("") }
private val symbolsBytes: ByteArray by lazy { TODO("") }
private val typesBytes: ByteArray by lazy { TODO("") }
private val stringsBytes: ByteArray by lazy { TODO("") }
lateinit var file: IrFile
private val deserializeBodies: Boolean = !onlyHeaders
private val fileLocalResolvedForwardDeclarations = mutableMapOf<UniqIdKey, UniqIdKey>()
val fileLocalDeserializationState = DeserializationState.SimpleDeserializationState()
fun deserializeDeclaration(key: UniqIdKey): IrDeclaration {
return deserializeDeclaration(loadTopLevelDeclarationProto(key), file)
}
@@ -215,13 +271,39 @@ abstract class KotlinIrLinker(
}
}
private fun isGlobalUniqID(uniqIdKey: UniqIdKey): Boolean {
return uniqIdKey in globalDeserializationState ||
descriptorReferenceDeserializer.checkIfSpecialDescriptorId(uniqIdKey.uniqId.index)
}
private fun getModuleForTopLevelId(key: UniqIdKey): IrModuleDeserializer {
if (key in moduleReversedFileIndex) return this@IrModuleDeserializer
return moduleDependencies.firstOrNull { key in it.moduleReversedFileIndex } ?:
error("Deserializer for declaration $key is not found")
}
private fun getStateForID(key: UniqIdKey): DeserializationState {
if (key.uniqId.isLocal) return fileLocalDeserializationState
if (isGlobalUniqID(key)) return globalDeserializationState
return getModuleForTopLevelId(key).moduleDeserializationState
}
private fun deserializeIrSymbolData(proto: ProtoSymbolData): IrSymbol {
val key = proto.uniqId.uniqIdKey(moduleDescriptor)
val topLevelKey = proto.topLevelUniqId.uniqIdKey(moduleDescriptor)
if (!deserializedTopLevels.contains(topLevelKey)) reachableTopLevels.add(topLevelKey)
val topLevelDeserializationState = getStateForID(topLevelKey)
val symbol = deserializedSymbols.getOrPut(key) {
if (topLevelKey !in topLevelDeserializationState) {
topLevelDeserializationState.addUniqID(topLevelKey)
}
// If topLevel declaration is module-public and current declaration in not (e.g. value parameter)
// they should be proccesed via different tables
val deserializationState =
if (topLevelKey.uniqId.isLocal xor key.uniqId.isLocal) getStateForID(key) else topLevelDeserializationState
val symbol = deserializationState.deserializedSymbols.getOrPut(key) {
val descriptor = if (proto.hasDescriptorReference()) {
deserializeDescriptorReference(proto.descriptorReference)
} else {
@@ -229,12 +311,8 @@ abstract class KotlinIrLinker(
}
resolvedForwardDeclarations[key]?.let {
if (!deserializedTopLevels.contains(it)) reachableTopLevels.add(it) // Assuming forward declarations are always top levels.
}
descriptor?.module?.let {
if (!deserializersForModules.containsKey(it) && !it.isForwardDeclarationModule) {
deserializeIrModuleHeader(it)!!
with (getForwardDeclararationModuleDeserializer().moduleDeserializationState) {
if (it !in this) addUniqID(it)
}
}
@@ -279,7 +357,7 @@ abstract class KotlinIrLinker(
loadStringProto(proto.index)
override fun deserializeLoopHeader(loopIndex: Int, loopBuilder: () -> IrLoopBase) =
moduleLoops.getOrPut(loopIndex, loopBuilder)
fileLoops.getOrPut(loopIndex, loopBuilder)
override fun deserializeExpressionBody(proto: ProtoBodyIndex): IrExpression {
if (deserializeBodies) {
@@ -301,16 +379,22 @@ abstract class KotlinIrLinker(
}
}
// TODO: this is JS specific. Eliminate me.
override fun getPrimitiveTypeOrNull(symbol: IrClassifierSymbol, hasQuestionMark: Boolean) =
this@KotlinIrLinker.getPrimitiveTypeOrNull(symbol, hasQuestionMark)
fun deserializeFileAnnotationsIfFirstUse() {
annotationsProto?.let {
file.annotations.addAll(deserializeAnnotations(it))
annotationsProto = null
}
}
fun deserializeFileTopLevelDeclaration(key: UniqIdKey) {
fileLocalDeserializationState.addUniqID(key)
fileLocalDeserializationState.processPendingDeclaration {
val declaration = deserializeDeclaration(it)
file.declarations.add(declaration)
}
}
}
private fun deserializeIrFile(fileProto: ProtoFile, fileIndex: Int): IrFile {
@@ -329,27 +413,36 @@ abstract class KotlinIrLinker(
val file = IrFileImpl(fileEntry, symbol, fqName)
fileDeserializer.file = file
fileToDeserializerMap[file] = fileDeserializer
fileProto.declarationIdList.forEach {
val uniqIdKey = it.uniqIdKey(moduleDescriptor)
reversedFileIndex.getOrPut(uniqIdKey) { mutableListOf() }.add(file)
fileToDeserializerMap[file] = fileDeserializer
assert(uniqIdKey.uniqId.isPublic)
moduleReversedFileIndex.getOrPut(uniqIdKey) { fileDeserializer }
}
when (deserializationStrategy) {
DeserializationStrategy.EXPLICITLY_EXPORTED -> {
fileProto.explicitlyExportedToCompilerList.forEach {
val symbolProto = fileDeserializer.loadSymbolData(it.index)
reachableTopLevels.add(symbolProto.topLevelUniqId.uniqIdKey(moduleDescriptor))
val topLevelUniqId = symbolProto.topLevelUniqId
val uniqIdKey = topLevelUniqId.uniqIdKey (moduleDescriptor)
assert(uniqIdKey.uniqId.isPublic)
// if (topLevelUniqId.isLocal) {
// fileDeserializer.fileLocalDeserializationState.addUniqID(uniqIdKey)
// } else {
moduleDeserializationState.addUniqID(uniqIdKey)
// }
}
}
DeserializationStrategy.ALL -> {
fileProto.declarationIdList.forEach {
val uniqIdKey = it.uniqIdKey(moduleDescriptor)
reachableTopLevels.add(uniqIdKey)
assert(uniqIdKey.uniqId.isPublic)
moduleDeserializationState.addUniqID(uniqIdKey)
}
}
else -> error("Unixpected deserialization strategy")
else -> error("Unexpected deserialization strategy")
}
return file
@@ -364,16 +457,27 @@ abstract class KotlinIrLinker(
files.add(deserializeIrFile(ProtoFile.parseFrom(readFile(moduleDescriptor, i), newInstance()), i))
}
// val files = readFiles(moduleDescriptor).map {
// deserializeIrFile(ProtoFile.parseFrom(it.codedInputStream, newInstance()))
// }
moduleDescriptor.allDependencyModules.filter { it != moduleDescriptor }.mapTo(moduleDependencies) { deserializersForModules[it]!! }
return IrModuleFragmentImpl(moduleDescriptor, builtIns, files)
}
}
// TODO: this is JS specific. Eliminate me.
protected open fun getPrimitiveTypeOrNull(symbol: IrClassifierSymbol, hasQuestionMark: Boolean): IrSimpleType? = null
fun deserializeAllModuleReachableTopLevels() {
moduleDeserializationState.processPendingDeclaration {
val fileDeserializer = moduleReversedFileIndex[it]
?: error("No file deserializer for key $it")
fileDeserializer.deserializeFileTopLevelDeclaration(it)
fileDeserializer.deserializeFileAnnotationsIfFirstUse()
}
}
fun addModuleReachableTopLevel(topLevelKey: UniqIdKey) {
moduleDeserializationState.reachableTopLevels.add(topLevelKey)
modulesWithReachableTopLevels.add(this)
}
}
protected abstract val descriptorReferenceDeserializer: DescriptorReferenceDeserializer
@@ -381,8 +485,9 @@ abstract class KotlinIrLinker(
private fun loadKnownBuiltinSymbols(): Long {
var currentIndex = firstKnownBuiltinsIndex
val globalDeserializedSymbols = globalDeserializationState.deserializedSymbols
builtIns.knownBuiltins.forEach {
deserializedSymbols[UniqIdKey(null, UniqId(currentIndex, isLocal = false))] = it
globalDeserializedSymbols[UniqIdKey(null, UniqId(currentIndex, isLocal = false))] = it
assert(symbolTable.referenceSimpleFunction(it.descriptor) == it)
currentIndex++
}
@@ -396,12 +501,6 @@ abstract class KotlinIrLinker(
return codedInputStream
}
private val reversedFileIndex = mutableMapOf<UniqIdKey, MutableList<IrFile>>()
private val UniqIdKey.moduleOfOrigin
get() =
this.moduleDescriptor ?: reversedFileIndex[this]?.handleClashes(this)?.packageFragmentDescriptor?.containingDeclaration
protected abstract fun reader(moduleDescriptor: ModuleDescriptor, fileIndex: Int, uniqId: UniqId): ByteArray
protected abstract fun readSymbol(moduleDescriptor: ModuleDescriptor, fileIndex: Int, symbolIndex: Int): ByteArray
protected abstract fun readType(moduleDescriptor: ModuleDescriptor, fileIndex: Int, typeIndex: Int): ByteArray
@@ -410,44 +509,21 @@ abstract class KotlinIrLinker(
protected abstract fun readFile(moduleDescriptor: ModuleDescriptor, fileIndex: Int): ByteArray
protected abstract fun readFileCount(moduleDescriptor: ModuleDescriptor): Int
protected open fun List<IrFile>.handleClashes(uniqIdKey: UniqIdKey): IrFile {
if (size == 1)
return this[0]
assert(size != 0)
error("UniqId clash: ${uniqIdKey.uniqId.index}. Found in the " +
"[${this.joinToString { it.packageFragmentDescriptor.containingDeclaration.name.asString() }}]")
}
private fun deserializeAllReachableTopLevels() {
do {
val key = reachableTopLevels.first()
val moduleOfOrigin = key.moduleOfOrigin
val moduleDeserializer = modulesWithReachableTopLevels.first()
modulesWithReachableTopLevels.remove(moduleDeserializer)
if (deserializedSymbols[key]?.isBound == true ||
// The key.moduleOrigin is null for uniqIds that we haven't seen in any of the library headers.
// Just skip it for now and handle it elsewhere.
moduleOfOrigin == null
) {
reachableTopLevels.remove(key)
deserializedTopLevels.add(key)
continue
}
val moduleDeserializer = deserializersForModules[moduleOfOrigin] ?: error("No module found")
val file = reversedFileIndex[key]!!.handleClashes(key)
val fileDeserializer: IrModuleDeserializer.IrDeserializerForFile = moduleDeserializer.fileToDeserializerMap[file] ?: error("dkjfkljfls")
val reachable = fileDeserializer.deserializeDeclaration(key)
file.declarations.add(reachable)
fileDeserializer.deserializeFileAnnotationsIfFirstUse()
reachableTopLevels.remove(key)
deserializedTopLevels.add(key)
} while (reachableTopLevels.isNotEmpty())
moduleDeserializer.deserializeAllModuleReachableTopLevels()
} while (modulesWithReachableTopLevels.isNotEmpty())
}
private fun findDeserializedDeclarationForDescriptor(descriptor: DeclarationDescriptor): DeclarationDescriptor? {
val topLevelDescriptor = descriptor.findTopLevelDescriptor()
val topLevelDescriptor = descriptor.findTopLevelDescriptor() as DeclarationDescriptorWithVisibility
require(topLevelDescriptor.isPublishedApi() || topLevelDescriptor.visibility.let { it.isPublicAPI || it == Visibilities.INTERNAL }) {
"Locally accessible declarations should not be accessed here $topLevelDescriptor"
}
// This is Native specific. Try to eliminate.
if (topLevelDescriptor.module.isForwardDeclarationModule) return null
@@ -461,10 +537,11 @@ abstract class KotlinIrLinker(
val uniqId = UniqId(descriptorUniqId, isLocal = false)
val topLevelKey = UniqIdKey(topLevelDescriptor.module, uniqId)
// This top level descriptor doesn't have a serialized IR declaration.
if (topLevelKey.moduleOfOrigin == null) return null
val moduleOfOrigin = topLevelDescriptor.module
reachableTopLevels.add(topLevelKey)
val moduleDeserializer = deserializersForModules[moduleOfOrigin] ?: error("No module deserializer found for $moduleOfOrigin")
moduleDeserializer.addModuleReachableTopLevel(topLevelKey)
deserializeAllReachableTopLevels()
return topLevelDescriptor
compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/UniqId.kt
+3 -1
View File
@@ -16,7 +16,9 @@ import org.jetbrains.kotlin.backend.common.serialization.proto.UniqId as ProtoUn
data class UniqId(
val index: Long,
val isLocal: Boolean
)
) {
val isPublic: Boolean get() = !isLocal
}
// isLocal=true in UniqId is good while we dealing with a single current module.
// To disambiguate module local declarations of different modules we use UniqIdKey.
compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/lower/serialization/ir/JsIrLinker.kt
-8
View File
@@ -50,13 +50,5 @@ class JsIrLinker(
override fun readFileCount(moduleDescriptor: ModuleDescriptor) =
moduleDescriptor.kotlinLibrary.fileCount()
override fun List<IrFile>.handleClashes(uniqIdKey: UniqIdKey): IrFile {
if (size == 1)
return this[0]
assert(size != 0)
error("UniqId clash: ${uniqIdKey.uniqId.index}. Found in the " +
"[${this.joinToString { it.packageFragmentDescriptor.containingDeclaration.userName }}]")
}
private val ModuleDescriptor.userName get() = kotlinLibrary.libraryFile.absolutePath
}