JS PIR: carrier serializer implementation

This commit is contained in:
Anton Bannykh
2021-04-02 17:49:33 +03:00
committed by TeamCityServer
parent c97066527c
commit 1e822aa28a
5 changed files with 506 additions and 1 deletions
@@ -0,0 +1,139 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.serialization
import org.jetbrains.kotlin.backend.common.serialization.IrDeclarationDeserializer
import org.jetbrains.kotlin.backend.common.serialization.codedInputStream
import org.jetbrains.kotlin.backend.common.serialization.proto.PirBodyCarrier
import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET
import org.jetbrains.kotlin.ir.declarations.IrDeclaration
import org.jetbrains.kotlin.ir.declarations.IrSymbolOwner
import org.jetbrains.kotlin.ir.declarations.persistent.*
import org.jetbrains.kotlin.ir.declarations.persistent.PersistentIrAnonymousInitializer
import org.jetbrains.kotlin.ir.declarations.persistent.PersistentIrClass
import org.jetbrains.kotlin.ir.declarations.persistent.carriers.BodyCarrier
import org.jetbrains.kotlin.ir.declarations.persistent.carriers.BodyCarrierImpl
import org.jetbrains.kotlin.ir.declarations.persistent.carriers.Carrier
import org.jetbrains.kotlin.ir.declarations.persistent.carriers.DeclarationCarrier
import org.jetbrains.kotlin.ir.expressions.*
import org.jetbrains.kotlin.ir.expressions.impl.IrSyntheticBodyImpl
import org.jetbrains.kotlin.ir.util.IdSignature
import org.jetbrains.kotlin.library.impl.IrArrayMemoryReader
import org.jetbrains.kotlin.library.impl.IrIntArrayMemoryReader
import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite
class CarrierDeserializer(
val declarationDeserializer: IrDeclarationDeserializer,
val serializedCarriers: SerializedCarriers,
) {
private val carrierDeserializerImpl =
IrCarrierDeserializerImpl(declarationDeserializer, ::deserializeBody, ::deserializeExpressionBody)
private val blockBodyCache = mutableMapOf<Int, IrBody>()
private fun deserializeBody(index: Int): IrBody = blockBodyCache.getOrPut(index) {
when (index) {
-1 -> IrSyntheticBodyImpl(UNDEFINED_OFFSET, UNDEFINED_OFFSET, IrSyntheticBodyKind.ENUM_VALUEOF)
-2 -> IrSyntheticBodyImpl(UNDEFINED_OFFSET, UNDEFINED_OFFSET, IrSyntheticBodyKind.ENUM_VALUES)
else -> (declarationDeserializer.deserializeStatementBody(index) as IrBlockBody).also {
injectCarriers(it, index)
}
}
}
private val expressionBodyCache = mutableMapOf<Int, IrExpressionBody>()
private fun deserializeExpressionBody(index: Int): IrExpressionBody = expressionBodyCache.getOrPut(index) {
declarationDeserializer.deserializeExpressionBody(index).also {
injectCarriers(it, index)
}
}
private val signatureToIndex = mutableMapOf<IdSignature, Int>().also { map ->
IrIntArrayMemoryReader(serializedCarriers.signatures).array.forEachIndexed { i, index ->
val idSig = declarationDeserializer.symbolDeserializer.deserializeIdSignature(index)
map[idSig] = i
}
}
private val declarationReader = IrArrayMemoryReader(serializedCarriers.declarationCarriers)
private val bodyReader = IrArrayMemoryReader(serializedCarriers.bodyCarriers)
private val removedOnReader = IrIntArrayMemoryReader(serializedCarriers.removedOn)
fun injectCarriers(declaration: IrDeclaration, signature: IdSignature) {
if (declaration is PersistentIrDeclarationBase<*>) {
when (declaration) {
is PersistentIrAnonymousInitializer -> declaration.load(signature, carrierDeserializerImpl::deserializeAnonymousInitializerCarrier)
is PersistentIrClass -> declaration.load(signature, carrierDeserializerImpl::deserializeClassCarrier)
is PersistentIrConstructor -> declaration.load(signature, carrierDeserializerImpl::deserializeConstructorCarrier)
is PersistentIrEnumEntry -> declaration.load(signature, carrierDeserializerImpl::deserializeEnumEntryCarrier)
is PersistentIrErrorDeclaration -> declaration.load(signature, carrierDeserializerImpl::deserializeErrorDeclarationCarrier)
is PersistentIrField -> declaration.load(signature, carrierDeserializerImpl::deserializeFieldCarrier)
is PersistentIrFunctionCommon -> declaration.load(signature, carrierDeserializerImpl::deserializeFunctionCarrier)
is PersistentIrLocalDelegatedProperty -> declaration.load(signature, carrierDeserializerImpl::deserializeLocalDelegatedPropertyCarrier)
is PersistentIrPropertyCommon -> declaration.load(signature, carrierDeserializerImpl::deserializePropertyCarrier)
is PersistentIrTypeAlias -> declaration.load(signature, carrierDeserializerImpl::deserializeTypeAliasCarrier)
is PersistentIrTypeParameter -> declaration.load(signature, carrierDeserializerImpl::deserializeTypeParameterCarrier)
is PersistentIrValueParameter -> declaration.load(signature, carrierDeserializerImpl::deserializeValueParameterCarrier)
else -> error("unknown declaration type: ${declaration::class.qualifiedName}")
}
}
}
fun injectCarriers(body: IrBody, index: Int) {
if (body is PersistentIrBodyBase<*>) {
val bodyCarriers = bodyReader.tableItemBytes(index)
val carriers = IrArrayMemoryReader(bodyCarriers).mapToArray { bodyBytes ->
val bodyProto = PirBodyCarrier.parseFrom(bodyBytes.codedInputStream, ExtensionRegistryLite.newInstance())
deserializeBodyCarrier(bodyProto)
}
body.load(carriers)
}
}
private inline fun <reified C : DeclarationCarrier> PersistentIrDeclarationBase<C>.load(signature: IdSignature, fn: (ByteArray) -> C) {
val index = signatureToIndex[signature] ?: return
// ?: error("Not found: $signature")
val bodyCarriers = declarationReader.tableItemBytes(index)
load(IrArrayMemoryReader(bodyCarriers).mapToArray(fn))
removedOn = removedOnReader.array[index]
}
private inline fun <reified C : Carrier> PersistentIrElementBase<C>.load(carriers: Array<C>) {
if (carriers.isNotEmpty()) {
@Suppress("UNCHECKED_CAST")
this.values = carriers as Array<Carrier>
// pretend to be lowered "all the way"
// TODO: make less hacky?
this.lastModified = 1000
this.loweredUpTo = 1000
}
}
private fun deserializeBodyCarrier(proto: PirBodyCarrier): BodyCarrier {
return BodyCarrierImpl(
proto.lastModified,
if (proto.hasContainerFieldSymbol()) declarationDeserializer.symbolDeserializer.deserializeIrSymbol(proto.containerFieldSymbol) else null,
)
}
}
private inline fun <reified T> IrArrayMemoryReader.mapToArray(fn: (ByteArray) -> T): Array<T> {
return Array<T>(entryCount()) { i ->
fn(tableItemBytes(i))
}
}
@@ -0,0 +1,131 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.serialization
import org.jetbrains.kotlin.backend.common.serialization.IrFileSerializer
import org.jetbrains.kotlin.backend.common.serialization.proto.PirBodyCarrier
import org.jetbrains.kotlin.ir.declarations.IrDeclaration
import org.jetbrains.kotlin.ir.declarations.persistent.PersistentIrBodyBase
import org.jetbrains.kotlin.ir.declarations.persistent.PersistentIrDeclarationBase
import org.jetbrains.kotlin.ir.declarations.persistent.PersistentIrElementBase
import org.jetbrains.kotlin.ir.declarations.persistent.carriers.*
import org.jetbrains.kotlin.ir.declarations.persistent.carriers.AnonymousInitializerCarrier
import org.jetbrains.kotlin.ir.declarations.persistent.carriers.ClassCarrier
import org.jetbrains.kotlin.ir.declarations.persistent.carriers.ConstructorCarrier
import org.jetbrains.kotlin.ir.declarations.persistent.carriers.EnumEntryCarrier
import org.jetbrains.kotlin.ir.declarations.persistent.carriers.ErrorDeclarationCarrier
import org.jetbrains.kotlin.ir.expressions.IrBody
import org.jetbrains.kotlin.ir.expressions.IrSyntheticBody
import org.jetbrains.kotlin.ir.expressions.IrSyntheticBodyKind
import org.jetbrains.kotlin.ir.util.IdSignature
import org.jetbrains.kotlin.library.impl.IrMemoryArrayWriter
import org.jetbrains.kotlin.library.impl.IrMemoryIntArrayWriter
class SerializedCarriers(
val signatures: ByteArray,
val declarationCarriers: ByteArray,
val bodyCarriers: ByteArray,
val removedOn: ByteArray,
)
// Declarations are references by signatures, bodies are referenced by index.
fun IrFileSerializer.serializeCarriers(declarations: Iterable<IrDeclaration>, bodies: List<IrBody>, signaturer: (IrDeclaration) -> IdSignature): SerializedCarriers {
val serializer = CarrierSerializer(this, bodies, signaturer)
declarations.forEach { serializer.serializeDeclarationCarrier(it) }
bodies.forEach { serializer.serializeBodyCarriers(it) }
return serializer.build()
}
private class CarrierSerializer(val fileSerializer: IrFileSerializer, bodies: List<IrBody>, val signaturer: (IrDeclaration) -> IdSignature) {
private val bodyToIndex = mutableMapOf<IrBody, Int>().also { map ->
bodies.forEachIndexed { index, irBody ->
map[irBody] = index
}
}
private val serializerImpl = IrCarrierSerializerImpl(fileSerializer) {
if (it is IrSyntheticBody) {
when (it.kind) {
IrSyntheticBodyKind.ENUM_VALUEOF -> -1
IrSyntheticBodyKind.ENUM_VALUES -> -2
}
} else bodyToIndex[it]
?: error("")
}
val signatures = mutableListOf<Int>()
val declarationCarriers = mutableListOf<ByteArray>()
val bodyCarriers = mutableListOf<ByteArray>()
val removedOn = mutableListOf<Int>()
fun build(): SerializedCarriers {
return SerializedCarriers(
IrMemoryIntArrayWriter(signatures).writeIntoMemory(),
IrMemoryArrayWriter(declarationCarriers).writeIntoMemory(),
IrMemoryArrayWriter(bodyCarriers).writeIntoMemory(),
IrMemoryIntArrayWriter(removedOn).writeIntoMemory(),
)
}
fun serializeDeclarationCarrier(declaration: IrDeclaration) {
if (declaration is PersistentIrDeclarationBase<*>) {
// TODO proper signature calculations?
val signature = signaturer(declaration)
signatures += fileSerializer.protoIdSignature(signature)
declarationCarriers += serializeCarriers(declaration)
removedOn += declaration.removedOn
} // else -> TODO?
}
fun serializeBodyCarriers(body: IrBody) {
if (body is PersistentIrBodyBase<*>) {
bodyCarriers += serializeCarriers(body)
}
}
private fun serializeCarriers(element: PersistentIrElementBase<*>): ByteArray {
val carriers = with(serializerImpl) {
// Serialize all state
// TODO maybe skip state in the initial declarations?
val values = ((element.values ?: arrayOf()) + element)
values.map {
when (it) {
is AnonymousInitializerCarrier -> serializeAnonymousInitializerCarrier(it)
is ClassCarrier -> serializeClassCarrier(it)
is ConstructorCarrier -> serializeConstructorCarrier(it)
is EnumEntryCarrier -> serializeEnumEntryCarrier(it)
is ErrorDeclarationCarrier -> serializeErrorDeclarationCarrier(it)
is FieldCarrier -> serializeFieldCarrier(it)
is FunctionCarrier -> serializeFunctionCarrier(it)
is LocalDelegatedPropertyCarrier -> serializeLocalDelegatedPropertyCarrier(it)
is PropertyCarrier -> serializePropertyCarrier(it)
is TypeAliasCarrier -> serializeTypeAliasCarrier(it)
is TypeParameterCarrier -> serializeTypeParameterCarrier(it)
is ValueParameterCarrier -> serializeValueParameterCarrier(it)
is BodyCarrier -> serializeBodyCarrier(it)
else -> error("unknown Carrier")
}
}
}
return IrMemoryArrayWriter(carriers).writeIntoMemory()
}
private fun serializeBodyCarrier(bodyCarrier: BodyCarrier): ByteArray {
val proto = PirBodyCarrier.newBuilder()
proto.lastModified = bodyCarrier.lastModified
bodyCarrier.containerFieldSymbol?.let { proto.containerFieldSymbol = serializerImpl.fileSerializer.serializeIrSymbol(it) }
return proto.build().toByteArray()
}
}
@@ -0,0 +1,120 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.serialization
import org.jetbrains.kotlin.backend.common.serialization.IrDeclarationDeserializer
import org.jetbrains.kotlin.backend.common.serialization.IrFlags
import org.jetbrains.kotlin.descriptors.DescriptorVisibility
import org.jetbrains.kotlin.descriptors.Modality
import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET
import org.jetbrains.kotlin.ir.declarations.IrDeclarationOrigin
import org.jetbrains.kotlin.ir.declarations.IrVariable
import org.jetbrains.kotlin.ir.expressions.IrBlockBody
import org.jetbrains.kotlin.ir.expressions.IrBody
import org.jetbrains.kotlin.ir.expressions.IrConstructorCall
import org.jetbrains.kotlin.ir.expressions.IrExpressionBody
import org.jetbrains.kotlin.ir.symbols.*
import org.jetbrains.kotlin.ir.types.IrType
import org.jetbrains.kotlin.serialization.deserialization.ProtoEnumFlags
import org.jetbrains.kotlin.serialization.deserialization.descriptorVisibility
import org.jetbrains.kotlin.backend.common.serialization.proto.IrConstructorCall as ProtoIrConstructorCall
import org.jetbrains.kotlin.backend.common.serialization.proto.IrVariable as ProtoIrVariable
internal class IrCarrierDeserializerImpl(
val declarationDeserializer: IrDeclarationDeserializer,
val indexToBody: (Int) -> IrBody,
val indexToExpressionBody: (Int) -> IrExpressionBody
) : IrCarrierDeserializer() {
override fun deserializeParentSymbol(proto: Long): IrSymbol {
return declarationDeserializer.symbolDeserializer.deserializeIrSymbol(proto)
}
override fun deserializeOrigin(proto: Int): IrDeclarationOrigin {
return declarationDeserializer.deserializeIrDeclarationOrigin(proto)
}
override fun deserializeAnnotation(proto: ProtoIrConstructorCall): IrConstructorCall {
return declarationDeserializer.bodyDeserializer.deserializeAnnotation(proto)
}
override fun deserializeBody(proto: Int): IrBody {
return indexToBody(proto)
}
override fun deserializeBlockBody(proto: Int): IrBlockBody {
return indexToBody(proto) as IrBlockBody
}
override fun deserializeExpressionBody(proto: Int): IrExpressionBody {
return indexToExpressionBody(proto)
}
override fun deserializeValueParameter(proto: Long): IrValueParameterSymbol {
return declarationDeserializer.symbolDeserializer.deserializeIrSymbol(proto) as IrValueParameterSymbol
}
override fun deserializeTypeParameter(proto: Long): IrTypeParameterSymbol {
return declarationDeserializer.symbolDeserializer.deserializeIrSymbol(proto) as IrTypeParameterSymbol
}
override fun deserializeSuperType(proto: Int): IrType {
return declarationDeserializer.deserializeIrType(proto)
}
override fun deserializeType(proto: Int): IrType {
return declarationDeserializer.deserializeIrType(proto)
}
override fun deserializeClass(proto: Long): IrClassSymbol {
return declarationDeserializer.symbolDeserializer.deserializeIrSymbol(proto) as IrClassSymbol
}
override fun deserializePropertySymbol(proto: Long): IrPropertySymbol {
return declarationDeserializer.symbolDeserializer.deserializeIrSymbol(proto) as IrPropertySymbol
}
override fun deserializeSimpleFunction(proto: Long): IrSimpleFunctionSymbol {
return declarationDeserializer.symbolDeserializer.deserializeIrSymbol(proto) as IrSimpleFunctionSymbol
}
override fun deserializeSimpleFunctionSymbol(proto: Long): IrSimpleFunctionSymbol {
return declarationDeserializer.symbolDeserializer.deserializeIrSymbol(proto) as IrSimpleFunctionSymbol
}
override fun deserializeField(proto: Long): IrFieldSymbol {
return declarationDeserializer.symbolDeserializer.deserializeIrSymbol(proto) as IrFieldSymbol
}
override fun deserializeVariable(proto: ProtoIrVariable): IrVariable {
return declarationDeserializer.deserializeIrVariable(proto)
}
override fun deserializeVisibility(proto: Long): DescriptorVisibility {
return ProtoEnumFlags.descriptorVisibility(IrFlags.VISIBILITY.get(proto.toInt()))
}
override fun deserializeModality(proto: Long): Modality {
return ProtoEnumFlags.modality(IrFlags.MODALITY.get(proto.toInt()))
}
override fun deserializeIsExternalClass(proto: Long): Boolean {
return IrFlags.IS_EXTERNAL_CLASS.get(proto.toInt())
}
override fun deserializeIsExternalField(proto: Long): Boolean {
return IrFlags.IS_EXTERNAL_FIELD.get(proto.toInt())
}
override fun deserializeIsExternalFunction(proto: Long): Boolean {
return IrFlags.IS_EXTERNAL_FUNCTION.get(proto.toInt())
}
override fun deserializeIsExternalProperty(proto: Long): Boolean {
return IrFlags.IS_EXTERNAL_PROPERTY.get(proto.toInt())
}
}
@@ -0,0 +1,115 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.serialization
import org.jetbrains.kotlin.backend.common.serialization.IrFileSerializer
import org.jetbrains.kotlin.backend.common.serialization.IrFlags
import org.jetbrains.kotlin.descriptors.DescriptorVisibility
import org.jetbrains.kotlin.descriptors.Modality
import org.jetbrains.kotlin.ir.declarations.IrDeclarationOrigin
import org.jetbrains.kotlin.ir.declarations.IrVariable
import org.jetbrains.kotlin.ir.expressions.IrBlockBody
import org.jetbrains.kotlin.ir.expressions.IrBody
import org.jetbrains.kotlin.ir.expressions.IrConstructorCall
import org.jetbrains.kotlin.ir.expressions.IrExpressionBody
import org.jetbrains.kotlin.ir.symbols.*
import org.jetbrains.kotlin.ir.types.IrType
import org.jetbrains.kotlin.metadata.deserialization.Flags
import org.jetbrains.kotlin.serialization.deserialization.ProtoEnumFlags
import org.jetbrains.kotlin.serialization.deserialization.descriptorVisibility
import org.jetbrains.kotlin.backend.common.serialization.proto.IrConstructorCall as ProtoIrConstructorCall
import org.jetbrains.kotlin.backend.common.serialization.proto.IrVariable as ProtoIrVariable
internal class IrCarrierSerializerImpl(val fileSerializer: IrFileSerializer, val bodyIndex: (IrBody) -> Int) : IrCarrierSerializer() {
override fun serializeParentSymbol(value: IrSymbol): Long {
return fileSerializer.serializeIrSymbol(value)
}
override fun serializeOrigin(value: IrDeclarationOrigin): Int {
return fileSerializer.serializeIrDeclarationOrigin(value)
}
override fun serializeAnnotation(value: IrConstructorCall): ProtoIrConstructorCall {
return fileSerializer.serializeConstructorCall(value)
}
override fun serializeBody(value: IrBody): Int {
return bodyIndex(value)
}
override fun serializeBlockBody(value: IrBlockBody): Int {
return serializeBody(value)
}
override fun serializeExpressionBody(value: IrExpressionBody): Int {
return serializeBody(value)
}
override fun serializeValueParameter(value: IrValueParameterSymbol): Long {
return fileSerializer.serializeIrSymbol(value)
}
override fun serializeTypeParameter(value: IrTypeParameterSymbol): Long {
return fileSerializer.serializeIrSymbol(value)
}
override fun serializeSuperType(value: IrType): Int {
return fileSerializer.serializeIrType(value)
}
override fun serializeType(value: IrType): Int {
return fileSerializer.serializeIrType(value)
}
override fun serializeClass(value: IrClassSymbol): Long {
return fileSerializer.serializeIrSymbol(value)
}
override fun serializePropertySymbol(value: IrPropertySymbol): Long {
return fileSerializer.serializeIrSymbol(value)
}
override fun serializeSimpleFunction(value: IrSimpleFunctionSymbol): Long {
return fileSerializer.serializeIrSymbol(value)
}
override fun serializeSimpleFunctionSymbol(value: IrSimpleFunctionSymbol): Long {
return fileSerializer.serializeIrSymbol(value)
}
override fun serializeField(value: IrFieldSymbol): Long {
return fileSerializer.serializeIrSymbol(value)
}
override fun serializeVariable(value: IrVariable): ProtoIrVariable {
return fileSerializer.serializeIrVariable(value)
}
override fun serializeVisibility(value: DescriptorVisibility): Long {
return Flags.VISIBILITY.toFlags(ProtoEnumFlags.descriptorVisibility(value)).toLong()
}
override fun serializeModality(value: Modality): Long {
return Flags.MODALITY.toFlags(ProtoEnumFlags.modality(value)).toLong()
}
override fun serializeIsExternalClass(value: Boolean): Long {
return IrFlags.IS_EXTERNAL_CLASS.toFlags(value).toLong()
}
override fun serializeIsExternalField(value: Boolean): Long {
return IrFlags.IS_EXTERNAL_FIELD.toFlags(value).toLong()
}
override fun serializeIsExternalFunction(value: Boolean): Long {
return IrFlags.IS_EXTERNAL_FUNCTION.toFlags(value).toLong()
}
override fun serializeIsExternalProperty(value: Boolean): Long {
return IrFlags.IS_EXTERNAL_PROPERTY.toFlags(value).toLong()
}
}
@@ -174,7 +174,7 @@ fun IrModuleDeserializer.findModuleDeserializerForTopLevelId(idSignature: IdSign
return moduleDependencies.firstOrNull { idSignature in it }
}
internal val ByteArray.codedInputStream: CodedInputStream
val ByteArray.codedInputStream: CodedInputStream
get() {
val codedInputStream = CodedInputStream.newInstance(this)
codedInputStream.setRecursionLimit(65535) // The default 64 is blatantly not enough for IR.