From ff8acd21e605a9a92682879c394752b686c9a1be Mon Sep 17 00:00:00 2001 From: Alexander Gorshenev Date: Tue, 10 Jan 2017 15:13:40 +0300 Subject: [PATCH] This is initial implementation of serializer/deserializer for inline IR bodies. The deserialization is disabled by default. Use `-enable deserializer` to enable it. Use `-verbose deserializer` to watch it crunch. --- .../kotlin/backend/common/IrValidator.kt | 12 +- .../jetbrains/kotlin/backend/konan/Context.kt | 7 + .../kotlin/backend/konan/KonanDriver.kt | 11 +- .../kotlin/backend/konan/KonanPhases.kt | 7 + .../backend/konan/descriptors/DeepVisitor.kt | 18 + .../konan/descriptors/DescriptorUtils.kt | 21 +- .../jetbrains/kotlin/backend/konan/ir/Ir.kt | 1 + .../kotlin/backend/konan/llvm/IrToBitcode.kt | 6 - .../backend/konan/llvm/MetadataGenerator.kt | 48 +- .../kotlin/backend/konan/llvm/NameUtils.kt | 0 .../backend/konan/lower/FunctionInlining.kt | 14 +- .../serialization/BackingFieldVisitor.kt | 27 + .../konan/serialization/DescriptorTable.kt | 67 ++ .../serialization/IrDescriptorDeserializer.kt | 416 ++++++++ .../serialization/IrDescriptorSerializer.kt | 139 +++ .../KonanDescriptorSerializer.kt | 666 +++++++++++++ .../backend/konan/serialization/KonanIr.proto | 271 +++++ .../konan/serialization/KonanLinkData.proto | 39 +- .../serialization/KonanPackageFragment.kt | 2 +- .../serialization/KonanSerializationUtil.kt | 84 +- .../serialization/KonanSerializerExtension.kt | 135 ++- .../konan/serialization/KonanStringTable.kt | 12 + .../LocalDeclarationDeserializer.kt | 132 +++ .../konan/serialization/SerializeIr.kt | 934 ++++++++++++++++++ .../konan/serialization/SerializerVisitor.kt | 73 ++ .../konan/serialization/StringTableUtil.kt | 30 + backend.native/tests/build.gradle | 9 + .../serialization/deserialized_inline0.kt | 37 + .../serialization/deserialized_listof0.kt | 29 + .../org/jetbrains/kotlin/KonanTest.groovy | 3 +- 30 files changed, 3180 insertions(+), 70 deletions(-) delete mode 100644 backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/NameUtils.kt create mode 100644 backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/BackingFieldVisitor.kt create mode 100644 backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/DescriptorTable.kt create mode 100644 backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/IrDescriptorDeserializer.kt create mode 100644 backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/IrDescriptorSerializer.kt create mode 100644 backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanDescriptorSerializer.kt create mode 100644 backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanIr.proto create mode 100644 backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/LocalDeclarationDeserializer.kt create mode 100644 backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/SerializeIr.kt create mode 100644 backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/SerializerVisitor.kt create mode 100644 backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/StringTableUtil.kt create mode 100644 backend.native/tests/serialization/deserialized_inline0.kt create mode 100644 backend.native/tests/serialization/deserialized_listof0.kt diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/common/IrValidator.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/common/IrValidator.kt index 44f7cf0d7ab..4f88bfb2390 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/common/IrValidator.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/common/IrValidator.kt @@ -11,6 +11,11 @@ import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid import org.jetbrains.kotlin.ir.visitors.acceptVoid import org.jetbrains.kotlin.resolve.descriptorUtil.module +fun validateIrFunction(context: BackendContext, irFunction: IrFunction) { + val visitor = IrValidator(context) + irFunction.acceptVoid(visitor) +} + fun validateIrFile(context: BackendContext, irFile: IrFile) { val visitor = IrValidator(context) irFile.acceptVoid(visitor) @@ -65,7 +70,12 @@ fun validateIrModule(context: BackendContext, irModule: IrModuleFragment) { } private fun BackendContext.reportIrValidationError(message: String, irFile: IrFile, irElement: IrElement) { - this.reportWarning("[IR VALIDATION] $message", irFile, irElement) + try { + this.reportWarning("[IR VALIDATION] $message", irFile, irElement) + } catch (e: Throwable) { + println("an error trying to print a warning message: $e") + e.printStackTrace() + } // TODO: throw an exception after fixing bugs leading to invalid IR. } diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt index 6f69f99ee98..9b0fe6a0911 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt @@ -10,6 +10,7 @@ import org.jetbrains.kotlin.backend.konan.llvm.Llvm import org.jetbrains.kotlin.backend.konan.llvm.LlvmDeclarations import org.jetbrains.kotlin.backend.konan.llvm.functionName import org.jetbrains.kotlin.backend.konan.llvm.verifyModule +import org.jetbrains.kotlin.backend.konan.llvm.* import org.jetbrains.kotlin.descriptors.* import org.jetbrains.kotlin.descriptors.annotations.Annotations import org.jetbrains.kotlin.descriptors.impl.PropertyDescriptorImpl @@ -202,6 +203,12 @@ internal class Context(config: KonanConfig) : KonanBackendContext(config) { ClassVtablesBuilder(classDescriptor, this) } + // We serialize untouched descriptor tree and inline IR bodies + // right after the frontend. + // But we have to wait until the code generation phase, + // to dump this information into generated file. + lateinit var serializedLinkData: String + // TODO: make lateinit? var irModule: IrModuleFragment? = null set(module: IrModuleFragment?) { diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt index 2e29ba55b2d..16e4a9429f9 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanDriver.kt @@ -3,9 +3,11 @@ package org.jetbrains.kotlin.backend.konan import org.jetbrains.kotlin.analyzer.AnalysisResult import org.jetbrains.kotlin.backend.common.messageCollector import org.jetbrains.kotlin.backend.common.validateIrModule +import org.jetbrains.kotlin.backend.konan.ir.DeserializerDriver import org.jetbrains.kotlin.backend.konan.ir.ModuleIndex import org.jetbrains.kotlin.backend.konan.llvm.emitLLVM -import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys +import org.jetbrains.kotlin.backend.konan.serialization.KonanSerializationUtil +import org.jetbrains.kotlin.backend.konan.serialization.markBackingFields import org.jetbrains.kotlin.cli.common.messages.AnalyzerWithCompilerReport import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment import org.jetbrains.kotlin.config.kotlinSourceRoots @@ -70,6 +72,13 @@ public fun runTopLevelPhases(konanConfig: KonanConfig, environment: KotlinCoreEn validateIrModule(context, module) } + phaser.phase(KonanPhase.SERIALIZER) { + markBackingFields(context) + val serializer = KonanSerializationUtil(context) + context.serializedLinkData = + serializer.serializeModule(context.moduleDescriptor!!) + DeserializerDriver(context).dumpAllInlineBodies() + } phaser.phase(KonanPhase.BACKEND) { phaser.phase(KonanPhase.LOWER) { KonanLower(context).lower() diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanPhases.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanPhases.kt index 5e444a6c586..757b4511394 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanPhases.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanPhases.kt @@ -8,9 +8,11 @@ enum class KonanPhase(val description: String, /* */ FRONTEND("Frontend builds AST"), /* */ PSI_TO_IR("Psi to IR conversion"), + /* */ SERIALIZER("Serialize descriptor tree and inline IR bodies"), /* */ BACKEND("All backend"), /* ... */ LOWER("IR Lowering"), /* ... ... */ LOWER_INLINE("Functions inlining"), + /* ... ... ... */ DESERIALIZER("Deserialize inline bodies"), /* ... ... */ LOWER_INTEROP("Interop lowering"), /* ... ... */ LOWER_ENUMS("Enum classes lowering"), /* ... ... */ LOWER_DELEGATION("Delegation lowering"), @@ -53,6 +55,11 @@ object KonanPhases { fun config(config: KonanConfig) { with (config.configuration) { with (KonanConfigKeys) { + + // We disable inline IR deserialization + // until we have enough test passes + KonanPhase.DESERIALIZER.enabled = false + val disabled = get(DISABLED_PHASES) disabled?.forEach { phases[known(it)]!!.enabled = false } diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/descriptors/DeepVisitor.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/descriptors/DeepVisitor.kt index e5c66d8d0f4..32a7b674066 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/descriptors/DeepVisitor.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/descriptors/DeepVisitor.kt @@ -98,3 +98,21 @@ open class DeepVisitor(val worker: DeclarationDescriptorVisitor) } } +open public class EmptyDescriptorVisitorVoid: DeclarationDescriptorVisitor { + override fun visitPackageFragmentDescriptor(descriptor: PackageFragmentDescriptor, data: Unit) = true + override fun visitPackageViewDescriptor(descriptor: PackageViewDescriptor, data: Unit) = true + override fun visitVariableDescriptor(descriptor: VariableDescriptor, data: Unit) = true + override fun visitFunctionDescriptor(descriptor: FunctionDescriptor, data: Unit) = true + override fun visitTypeParameterDescriptor(descriptor: TypeParameterDescriptor, data: Unit) = true + override fun visitClassDescriptor(descriptor: ClassDescriptor, data: Unit) = true + override fun visitTypeAliasDescriptor(descriptor: TypeAliasDescriptor, data: Unit) = true + override fun visitModuleDeclaration(descriptor: ModuleDescriptor, data: Unit) = true + override fun visitConstructorDescriptor(descriptor: ConstructorDescriptor, data: Unit) = true + override fun visitScriptDescriptor(descriptor: ScriptDescriptor, data: Unit) = true + override fun visitPropertyDescriptor(descriptor: PropertyDescriptor, data: Unit) = true + override fun visitValueParameterDescriptor(descriptor: ValueParameterDescriptor, data: Unit) = true + override fun visitPropertyGetterDescriptor(descriptor: PropertyGetterDescriptor, data: Unit) = true + override fun visitPropertySetterDescriptor(descriptor: PropertySetterDescriptor, data: Unit) = true + override fun visitReceiverParameterDescriptor(descriptor: ReceiverParameterDescriptor, data: Unit) = true +} + diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/descriptors/DescriptorUtils.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/descriptors/DescriptorUtils.kt index b8da57fad12..1f157209079 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/descriptors/DescriptorUtils.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/descriptors/DescriptorUtils.kt @@ -157,9 +157,11 @@ internal val T.allOverriddenDescriptors: List } internal val ClassDescriptor.contributedMethods: List + get () = unsubstitutedMemberScope.contributedMethods + +internal val MemberScope.contributedMethods: List get () { - val contributedDescriptors = unsubstitutedMemberScope.getContributedDescriptors() - // (includes declarations from supers) + val contributedDescriptors = this.getContributedDescriptors() val functions = contributedDescriptors.filterIsInstance() @@ -174,6 +176,7 @@ internal val ClassDescriptor.contributedMethods: List return allMethods } + fun ClassDescriptor.isAbstract() = this.modality == Modality.SEALED || this.modality == Modality.ABSTRACT || this.kind == ClassKind.ENUM_CLASS @@ -264,4 +267,16 @@ internal fun FunctionDescriptor.bridgeDirectionsTo(overriddenDescriptor: Functio } return ourDirections -} \ No newline at end of file +} + + +internal fun DeclarationDescriptor.getMemberScope(): MemberScope { + val containingScope = when (this) { + is ClassDescriptor -> this.unsubstitutedMemberScope + is PackageViewDescriptor -> this.memberScope + else -> error("Unexpected member scope: $containingDeclaration") + } + return containingScope +} + + diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt index 9384b1f2de0..3a9b1909932 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt @@ -3,6 +3,7 @@ package org.jetbrains.kotlin.backend.konan.ir import org.jetbrains.kotlin.backend.konan.Context import org.jetbrains.kotlin.descriptors.FunctionDescriptor import org.jetbrains.kotlin.ir.declarations.IrModuleFragment +import org.jetbrains.kotlin.ir.declarations.IrFunction import org.jetbrains.kotlin.descriptors.PropertyDescriptor // This is what Context collects about IR. diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt index 5a6efcc35c5..7e44fa0bd11 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt @@ -108,12 +108,6 @@ internal class MetadatorVisitor(val context: Context) : IrElementVisitorVoid { element.acceptChildrenVoid(this) } - override fun visitProperty(declaration: IrProperty) { - declaration.acceptChildrenVoid(this) - metadator.property(declaration) - - } - override fun visitModuleFragment(module: IrModuleFragment) { module.acceptChildrenVoid(this) metadator.endModule(module) diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/MetadataGenerator.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/MetadataGenerator.kt index fd97be999d6..a23bccad76f 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/MetadataGenerator.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/MetadataGenerator.kt @@ -2,16 +2,14 @@ package org.jetbrains.kotlin.backend.konan.llvm import kotlinx.cinterop.* import llvm.* -import org.jetbrains.kotlin.backend.konan.ir.Ir import org.jetbrains.kotlin.backend.konan.Context -import org.jetbrains.kotlin.backend.konan.serialization.* -import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.backend.konan.KonanConfigKeys -import org.jetbrains.kotlin.ir.declarations.IrModuleFragment -import org.jetbrains.kotlin.ir.declarations.IrProperty -import org.jetbrains.kotlin.descriptors.PropertyDescriptor +import org.jetbrains.kotlin.backend.konan.serialization.deserializeModule +import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl -import java.io.* +import org.jetbrains.kotlin.ir.declarations.IrModuleFragment +import java.io.Closeable +import java.io.File fun loadMetadata(configuration: CompilerConfiguration, file: File): ModuleDescriptorImpl { @@ -86,12 +84,28 @@ class MetadataReader(file: File) : Closeable { } } + fun namedMetadataNodes(name: String): Array { + val result = mutableListOf() + memScoped { + val nodeCount = LLVMGetNamedMetadataNumOperands(llvmModule, name) + val nodeArray = allocArray(nodeCount) + + LLVMGetNamedMetadataOperands(llvmModule, name, nodeArray[0].ptr) + + //return Pair(nodeCount, nodeArray[0].value!!) + for (index in 0..nodeCount-1) { + result.add(nodeArray[index].value!!) + } + } + return result.toTypedArray() + } + fun namedMetadataNode(name: String): Pair { memScoped { val nodeCount = LLVMGetNamedMetadataNumOperands(llvmModule, name) val nodeArray = allocArray(nodeCount) - LLVMGetNamedMetadataOperands(llvmModule, "kmetadata", nodeArray[0].ptr) + LLVMGetNamedMetadataOperands(llvmModule, name, nodeArray[0].ptr) return Pair(nodeCount, nodeArray[0].value!!) } @@ -126,13 +140,6 @@ internal class MetadataGenerator(override val context: Context): ContextUtils { return md } - internal fun property(declaration: IrProperty) { - if (declaration.backingField == null) return - assert(declaration.backingField!!.descriptor == declaration.descriptor || declaration.isDelegated) - - context.ir.propertiesWithBackingFields.add(declaration.descriptor) - } - private fun emitModuleMetadata(name: String, md: LLVMValueRef?) { LLVMAddNamedMetadataOperand(context.llvmModule, name, md) } @@ -141,18 +148,19 @@ internal class MetadataGenerator(override val context: Context): ContextUtils { return LLVMMDString(str, str.length)!! } - internal fun endModule(module: IrModuleFragment) { + private fun addLinkData(module: IrModuleFragment) { val abiVersion = context.config.configuration.get(KonanConfigKeys.ABI_VERSION) val abiNode = metadataString("$abiVersion") - val moduleName = metadataString(module.descriptor.name.asString()) - - val serializer = KonanSerializationUtil(context) - val moduleAsString = serializer.serializeModule(module.descriptor) + val moduleAsString = context.serializedLinkData val dataNode = metadataString(moduleAsString) val kmetadataArg = metadataNode(listOf(abiNode, moduleName, dataNode)) emitModuleMetadata("kmetadata", kmetadataArg) } + + internal fun endModule(module: IrModuleFragment) { + addLinkData(module) + } } diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/NameUtils.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/NameUtils.kt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/FunctionInlining.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/FunctionInlining.kt index 90c7f38d3e9..7ad35123ccb 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/FunctionInlining.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/FunctionInlining.kt @@ -2,6 +2,7 @@ package org.jetbrains.kotlin.backend.konan.lower import org.jetbrains.kotlin.backend.konan.Context import org.jetbrains.kotlin.backend.konan.descriptors.isFunctionInvoke +import org.jetbrains.kotlin.backend.konan.ir.DeserializerDriver import org.jetbrains.kotlin.backend.konan.ir.IrInlineFunctionBody import org.jetbrains.kotlin.descriptors.* import org.jetbrains.kotlin.ir.IrElement @@ -29,6 +30,7 @@ internal class FunctionInlining(val context: Context): IrElementTransformerVoid( var currentScope : Scope? = null //-------------------------------------------------------------------------// + val deserializer = DeserializerDriver(context) fun inline(irFile: IrFile) = irFile.accept(this, null) @@ -143,15 +145,13 @@ internal class FunctionInlining(val context: Context): IrElementTransformerVoid( } //-------------------------------------------------------------------------// - fun inlineFunction(irCall: IrCall): IrExpression { - val functionDescriptor = irCall.descriptor as FunctionDescriptor - val functionDeclaration = context.ir.originalModuleIndex - .functions[functionDescriptor.original] // Get FunctionDeclaration by FunctionDescriptor. - - if (functionDeclaration == null) return super.visitCall(irCall) // Function is declared in another module. - + val originalDescriptor = functionDescriptor.original + val functionDeclaration = + context.ir.originalModuleIndex.functions[originalDescriptor] ?: // Function is declared in the current module. + deserializer.deserializeInlineBody(originalDescriptor) // Function is declared in another module. + if (functionDeclaration == null) return super.visitCall(irCall) val copyFuncDeclaration = functionDeclaration.accept(InlineCopyIr(), // Create copy of the function. null) as IrFunction diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/BackingFieldVisitor.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/BackingFieldVisitor.kt new file mode 100644 index 00000000000..04d45e7abc9 --- /dev/null +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/BackingFieldVisitor.kt @@ -0,0 +1,27 @@ +package org.jetbrains.kotlin.backend.konan.serialization + +import org.jetbrains.kotlin.backend.konan.Context +import org.jetbrains.kotlin.ir.IrElement +import org.jetbrains.kotlin.ir.declarations.IrProperty +import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid +import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid + +internal class BackingFieldVisitor(val context: Context) : IrElementVisitorVoid { + + override fun visitElement(element: IrElement) { + element.acceptChildrenVoid(this) + } + + override fun visitProperty(declaration: IrProperty) { + if (declaration.backingField == null) return + assert(declaration.backingField!!.descriptor + == declaration.descriptor || declaration.isDelegated) + + context.ir.propertiesWithBackingFields.add(declaration.descriptor) + } +} + +internal fun markBackingFields(context: Context) { + context.irModule!!.accept(BackingFieldVisitor(context), null) +} + diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/DescriptorTable.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/DescriptorTable.kt new file mode 100644 index 00000000000..a9de8fa5615 --- /dev/null +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/DescriptorTable.kt @@ -0,0 +1,67 @@ +package org.jetbrains.kotlin.backend.konan.serialization + +import kotlin.properties.Delegates +import org.jetbrains.kotlin.descriptors.* +import org.jetbrains.kotlin.ir.descriptors.IrBuiltIns +import org.jetbrains.kotlin.ir.declarations.IrFunction +import org.jetbrains.kotlin.types.KotlinType +import org.jetbrains.kotlin.serialization.deserialization.NameResolver +import org.jetbrains.kotlin.serialization.ProtoBuf +import org.jetbrains.kotlin.serialization.ProtoBuf.QualifiedNameTable.QualifiedName + + +// TODO: we currently just assign each encountered +// descriptor a new int id. +// That;s okay until we have more than one external module. +// In that case the descriptors will get different ids. +// Need to come up with a stable scheme of ids +// dependant on something like FunctionDescriptor.functionName +class DescriptorTable(val builtIns: IrBuiltIns) { + + val table = mutableMapOf() + var currentIndex = 0 + + init { + builtIns.irBuiltInDescriptors.forEachIndexed { + index, descriptor -> + + table.put(descriptor, index) + currentIndex = index + 1 + } + } + + fun indexByValue(value: DeclarationDescriptor): Int { + val index = table.getOrPut(value) { + currentIndex ++ + } + return index + } +} + +class IrDeserializationDescriptorIndex(irBuiltIns: IrBuiltIns) { + + val map = mutableMapOf() + + init { + irBuiltIns.irBuiltInDescriptors.forEachIndexed { + index, descriptor -> + map.put(index, descriptor) + } + } + +} + +val IrBuiltIns.irBuiltInDescriptors + get() = listOf( + this.eqeqeq, + this.eqeq, + this.lt0, + this.lteq0, + this.gt0, + this.gteq0, + this.throwNpe, + this.booleanNot, + this.noWhenBranchMatchedException, + this.enumValueOf) + + diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/IrDescriptorDeserializer.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/IrDescriptorDeserializer.kt new file mode 100644 index 00000000000..5c3fbaa9eeb --- /dev/null +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/IrDescriptorDeserializer.kt @@ -0,0 +1,416 @@ +package org.jetbrains.kotlin.backend.konan.serialization + +import org.jetbrains.kotlin.backend.konan.Context +import org.jetbrains.kotlin.backend.konan.descriptors.contributedMethods +import org.jetbrains.kotlin.backend.konan.llvm.base64Decode +import org.jetbrains.kotlin.descriptors.* +import org.jetbrains.kotlin.ir.expressions.IrLoop +import org.jetbrains.kotlin.name.Name +import org.jetbrains.kotlin.resolve.scopes.MemberScope +import org.jetbrains.kotlin.serialization.KonanIr +import org.jetbrains.kotlin.serialization.KonanIr.KotlinDescriptor +import org.jetbrains.kotlin.serialization.KonanLinkData +import org.jetbrains.kotlin.serialization.ProtoBuf +import org.jetbrains.kotlin.serialization.deserialization.NameResolver +import org.jetbrains.kotlin.serialization.deserialization.NameResolverImpl +import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedClassConstructorDescriptor +import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedPropertyDescriptor +import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedSimpleFunctionDescriptor +import org.jetbrains.kotlin.types.KotlinType + + +internal fun printType(proto: ProtoBuf.Type) { + println("debug text: " + proto.getExtension(KonanLinkData.typeText)) +} + +internal fun printTypeTable(proto: ProtoBuf.TypeTable) { + proto.getTypeList().forEach { + printType(it) + } +} + +internal fun DeclarationDescriptor.findPackage(): PackageFragmentDescriptor { + return if (this is PackageFragmentDescriptor) this + else this.containingDeclaration!!.findPackage() +} + +internal fun DeserializedSimpleFunctionDescriptor.nameTable(): ProtoBuf.QualifiedNameTable { + val pkg = this.findPackage() + assert(pkg is KonanPackageFragment) + return (pkg as KonanPackageFragment).proto.getNameTable() +} + + +internal fun DeserializedSimpleFunctionDescriptor.nameResolver(): NameResolver { + val pkg = this.findPackage() as KonanPackageFragment + return NameResolverImpl(pkg.proto.getStringTable(), pkg.proto.getNameTable()) +} + + +// This is the class that knowns how to deserialize +// Kotlin descriptors and types for IR. + +internal class IrDescriptorDeserializer(val context: Context, + val deserializationDescriptorIndex: IrDeserializationDescriptorIndex, + val rootFunction: DeserializedSimpleFunctionDescriptor, + val localDeserializer: LocalDeclarationDeserializer) { + + val loopIndex = mutableMapOf() + val nameResolver = rootFunction.nameResolver() as NameResolverImpl + val nameTable = rootFunction.nameTable() + val descriptorIndex = deserializationDescriptorIndex.map + + fun deserializeKotlinType(proto: KonanIr.KotlinType): KotlinType { + + val index = proto.getIndex() + val text = proto.getDebugText() + val typeProto = localDeserializer.typeTable!![index] + val type = localDeserializer.deserializeInlineType(typeProto) + context.log("### deserialized Kotlin Type index=$index, text=$text:\t$type") + + return type + } + + fun deserializeLocalDeclaration(proto: KonanLinkData.Descriptor): DeclarationDescriptor { + when { + proto.hasFunction() -> { + val functionProto = proto.function + val index = functionProto.getExtension(KonanLinkData.functionIndex) + val descriptor = localDeserializer.deserializeFunction(functionProto) + descriptorIndex.put(index, descriptor) + return descriptor + } + + proto.hasProperty() -> { + val propertyProto = proto.property + val index = propertyProto.getExtension(KonanLinkData.propertyIndex) + val descriptor = localDeserializer.deserializeProperty(propertyProto) + descriptorIndex.put(index, descriptor) + return descriptor + } + // TODO + // proto.hasClazz() -> + else -> error("Unexpected descriptor kind") + } + } + + // Either (a substitution of) a public descriptor + // or an already seen local declaration. + fun deserializeKnownDescriptor(proto: KonanIr.KotlinDescriptor): DeclarationDescriptor { + + val index = proto.index + val kind = proto.kind + + val descriptor = when (kind) { + KonanIr.KotlinDescriptor.Kind.VALUE_PARAMETER, + KonanIr.KotlinDescriptor.Kind.TYPE_PARAMETER, + KonanIr.KotlinDescriptor.Kind.RECEIVER, + KonanIr.KotlinDescriptor.Kind.VARIABLE -> + descriptorIndex[index]!! + + KonanIr.KotlinDescriptor.Kind.CONSTRUCTOR, + KonanIr.KotlinDescriptor.Kind.FUNCTION, + KonanIr.KotlinDescriptor.Kind.ACCESSOR -> { + descriptorIndex[index] ?: + findInTheDescriptorTree(proto)!! + } + else -> TODO("Unexpected descriptor kind: $kind") + } + + return descriptor + } + + + fun deserializeDescriptor(proto: KonanIr.KotlinDescriptor): DeclarationDescriptor { + + context.log("### deserializeDescriptor ${proto.kind} ${proto.index}") + + val descriptor = if (proto.hasIrLocalDeclaration()) { + val realDescriptor = proto.irLocalDeclaration.descriptor + deserializeLocalDeclaration(realDescriptor) + } else + deserializeKnownDescriptor(proto) + + descriptorIndex.put(proto.index, descriptor) + + context.log("### descriptor ${proto.kind} ${proto.index} -> $descriptor") + + // Now there are several descriptors that automatically + // recreated in addition to this one. Register them + // all too. + + if (descriptor is FunctionDescriptor) { + registerParameterDescriptors(proto, descriptor) + } + return descriptor + } + + + // -------------------------------------------------------- + + // + // The section below is a poor man's resolve. + // Having a 'kind' and a 'name' of a descriptor, + // we need to find it's original unsubstituted version + // in the descriptor tree. And then apply the substitutions + // to get the same substituted descriptor we've obtained + // in the original ir. + // + + // We don't deserialize value, type and receiver parameters, rather + // just obtain them from their function descriptor and match them to the indices + // stored in the IR serialization. + fun registerParameterDescriptors ( + proto: KonanIr.KotlinDescriptor, + functionDescriptor: FunctionDescriptor) { + + //val functionProto = functionDescriptor.proto + functionDescriptor.valueParameters.forEachIndexed { i, parameter -> + //val parameterProto = functionProto.getValueParameter(i) + // Assume they are in the same order. + val index = proto.getValueParameter(i).index + descriptorIndex.put(index, parameter) + } + + functionDescriptor.typeParameters.forEachIndexed {i, parameter -> + //val parameterProto = functionProto.getTypeParameter(i) + // Assume they are in the same order too. + val index = proto.getTypeParameter(i).index + descriptorIndex.put(index, parameter) + } + + val dispatchReceiver = functionDescriptor.dispatchReceiverParameter + if (dispatchReceiver != null) { + assert(proto.hasDispatchReceiverIndex()) + val dispatchReceiverIndex = proto.dispatchReceiverIndex + descriptorIndex.put(dispatchReceiverIndex, dispatchReceiver) + } + + + val extensionReceiver = functionDescriptor.extensionReceiverParameter + if (extensionReceiver != null) { + assert(proto.hasExtensionReceiverIndex()) + val extensionReceiverIndex = proto.extensionReceiverIndex + descriptorIndex.put(extensionReceiverIndex, extensionReceiver) + } + } + + + fun substituteFunction(proto: KonanIr.KotlinDescriptor, + originalDescriptor: FunctionDescriptor): FunctionDescriptor { + + val newDescriptor = + originalDescriptor.newCopyBuilder().apply() { + setOriginal(originalDescriptor) + setReturnType(deserializeKotlinType(proto.type)) + if (proto.hasExtensionReceiverType()) { + setExtensionReceiverType(deserializeKotlinType(proto.extensionReceiverType)) + } + }.build()!! + + descriptorIndex.put(proto.index, newDescriptor) + + return newDescriptor + } + + fun substituteConstructor(proto: KonanIr.KotlinDescriptor, + originalDescriptor: ClassConstructorDescriptor): ClassConstructorDescriptor { + + val newDescriptor = originalDescriptor.newCopyBuilder().apply() { + setOriginal(originalDescriptor) + setReturnType(deserializeKotlinType(proto.type)) + if (proto.hasExtensionReceiverType()) { + setExtensionReceiverType(deserializeKotlinType(proto.extensionReceiverType)) + } + }.build()!! + + descriptorIndex.put(proto.index, newDescriptor) + + // TODO: why? + return newDescriptor as ClassConstructorDescriptor + } + + // Property accessors are different because the accessors don't have + // ProtoBuf serializations. Only their properties do. + // And also an accessor can not be copied, only properties can. + fun substituteAccessor(proto: KonanIr.KotlinDescriptor, + originalDescriptor: PropertyAccessorDescriptor): PropertyAccessorDescriptor { + + val originalPropertyDescriptor = originalDescriptor.correspondingProperty as DeserializedPropertyDescriptor + + + // TODO: property copy building is a mess. + // Plus it is changing in the big Kotlin under my fingers. + // What should I do here??? + // how do I set type and extensionreceiver for a property??? + val newPropertyDescriptor = originalPropertyDescriptor + + val newDescriptor = when (originalDescriptor) { + is PropertyGetterDescriptor -> + newPropertyDescriptor.getter + is PropertySetterDescriptor -> + newPropertyDescriptor.setter + else -> error("Unexpected accessor kind") + } + + descriptorIndex.put(proto.index, newDescriptor!!) + + newDescriptor.valueParameters.forEachIndexed { i, parameter -> + // Assume they are in the same order. + val index = proto.getValueParameter(i).index + descriptorIndex.put(index, parameter) + } + + newPropertyDescriptor.typeParameters.forEachIndexed { i, parameter -> + // Assume they are in the same order too. + val index = proto.getTypeParameter(i).index + descriptorIndex.put(index, parameter) + } + + val dispatchReceiver = newPropertyDescriptor.dispatchReceiverParameter + if (dispatchReceiver != null) { + assert(proto.hasDispatchReceiverIndex()) + val dispatchReceiverIndex = proto.dispatchReceiverIndex + descriptorIndex.put(dispatchReceiverIndex, dispatchReceiver) + } + + val extensionReceiver = newPropertyDescriptor.extensionReceiverParameter + if (extensionReceiver != null) { + assert(proto.hasExtensionReceiverIndex()) + val extensionReceiverIndex = proto.extensionReceiverIndex + descriptorIndex.put(extensionReceiverIndex, extensionReceiver) + } + + return newDescriptor + } + + fun parentMemberScopeByFqNameIndex(index: Int): MemberScope { + val parent = parentByFqNameIndex(index) + return when (parent) { + is ClassDescriptor -> parent.getUnsubstitutedMemberScope() + is PackageFragmentDescriptor -> parent.getMemberScope() + is PackageViewDescriptor -> parent.memberScope + else -> error("could not get a member scope") + } + } + + fun parentByFqNameIndex(index: Int): DeclarationDescriptor { + val module = context.moduleDescriptor!! + val parent = nameResolver.getDescriptorByFqNameIndex(module, nameTable, index) + return parent + + } + + fun matchNameInParentScope(proto: KonanIr.KotlinDescriptor): Collection { + val classOrPackage = proto.classOrPackage + val name = proto.name + + when (proto.kind) { + KotlinDescriptor.Kind.CONSTRUCTOR -> { + val parent = parentByFqNameIndex(classOrPackage) + assert(parent is ClassDescriptor) + return (parent as ClassDescriptor).constructors + } + KotlinDescriptor.Kind.ACCESSOR, + KotlinDescriptor.Kind.FUNCTION -> { + val parentScope = + parentMemberScopeByFqNameIndex(classOrPackage) + return parentScope.contributedMethods.filter{ + it.name == Name.guessByFirstCharacter(name) + } + } + else -> TODO("Can't find matching names for ${proto.kind}") + } + } + + fun selectFunction( + functions: Collection, + proto: KonanIr.KotlinDescriptor ): + DeserializedSimpleFunctionDescriptor { + + val originalIndex = proto.originalIndex + return functions.single() { + val proto = (it as DeserializedSimpleFunctionDescriptor).proto + proto.getExtension(KonanLinkData.functionIndex) == originalIndex + } as DeserializedSimpleFunctionDescriptor + } + + fun selectConstructor( + constructors: Collection, + proto: KonanIr.KotlinDescriptor): + DeserializedClassConstructorDescriptor { + + val originalIndex = proto.originalIndex + return constructors.single { + val proto = (it as DeserializedClassConstructorDescriptor).proto + proto.getExtension(KonanLinkData.constructorIndex) == originalIndex + } as DeserializedClassConstructorDescriptor + } + + + fun selectAccessor( + functions: Collection, + proto: KonanIr.KotlinDescriptor): + PropertyAccessorDescriptor { + + // TODO: property accessors are not serialized by + // descriptor serialization mechanisms, we can't match + // the "original" field of the deserialized descriptors. + // So KonanIr.KonanDescriptor.original contains + // original index of the property rather than accessor's. + val originalIndex = proto.originalIndex + return functions.single { + val property = (it as PropertyAccessorDescriptor).correspondingProperty as DeserializedPropertyDescriptor + property.proto.getExtension(KonanLinkData.propertyIndex) == originalIndex + } as PropertyAccessorDescriptor + } + + fun selectAmongMatchingNames( + matching: Collection, + proto: KonanIr.KotlinDescriptor): + DeclarationDescriptor { + + return when(proto.kind) { + KotlinDescriptor.Kind.FUNCTION -> + selectFunction(matching, proto) + KotlinDescriptor.Kind.ACCESSOR -> + selectAccessor(matching, proto) + KotlinDescriptor.Kind.CONSTRUCTOR -> + selectConstructor(matching, proto) + else -> TODO("don't know how to select ${proto.kind}") + } + } + + fun performSubstitutions(proto: KonanIr.KotlinDescriptor, + originalDescriptor: DeclarationDescriptor): + DeclarationDescriptor { + + //TODO: We need to properly skip creating substituted + // copies for public descriptors if we know the substituted + // descriptor is going to be the same. + // But that's not a trivial thing to find out. + if (originalDescriptor == rootFunction) + return rootFunction + + return when (originalDescriptor) { + is SimpleFunctionDescriptor -> + substituteFunction(proto, originalDescriptor) + is PropertyAccessorDescriptor -> + substituteAccessor(proto, originalDescriptor) + is ClassConstructorDescriptor -> + substituteConstructor(proto, originalDescriptor) + else -> error("unexpected type of public function") + } + } + + fun findInTheDescriptorTree(proto: KonanIr.KotlinDescriptor): DeclarationDescriptor? { + + val matchingNames = matchNameInParentScope(proto) + if (matchingNames.size == 0) return null + + val originalDescriptor = selectAmongMatchingNames(matchingNames, proto) + + return performSubstitutions(proto, originalDescriptor) + } +} + diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/IrDescriptorSerializer.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/IrDescriptorSerializer.kt new file mode 100644 index 00000000000..6fbcb1e4f25 --- /dev/null +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/IrDescriptorSerializer.kt @@ -0,0 +1,139 @@ +package org.jetbrains.kotlin.backend.konan.serialization + +import org.jetbrains.kotlin.backend.konan.Context +import org.jetbrains.kotlin.descriptors.* +import org.jetbrains.kotlin.descriptors.impl.LocalVariableDescriptor +import org.jetbrains.kotlin.ir.descriptors.IrTemporaryVariableDescriptor +import org.jetbrains.kotlin.resolve.DescriptorUtils +import org.jetbrains.kotlin.resolve.descriptorUtil.fqNameSafe +import org.jetbrains.kotlin.serialization.KonanIr +import org.jetbrains.kotlin.types.KotlinType + +val DeclarationDescriptor.classOrPackage: DeclarationDescriptor? + get() { + return if (this.containingDeclaration!! is ClassOrPackageFragmentDescriptor) + containingDeclaration + else null + } + +internal class IrDescriptorSerializer( + val context: Context, + val descriptorTable: DescriptorTable, + val stringTable: KonanStringTable, + var typeSerializer: ((KotlinType)->Int), + var rootFunction: FunctionDescriptor) { + + fun serializeKotlinType(type: KotlinType): KonanIr.KotlinType { + val index = this.typeSerializer!!(type) + val proto = KonanIr.KotlinType.newBuilder() + .setIndex(index) + .setDebugText(type.toString()) + .build() + return proto + } + + fun kotlinDescriptorKind(descriptor: DeclarationDescriptor) = + when (descriptor) { + is ConstructorDescriptor + -> KonanIr.KotlinDescriptor.Kind.CONSTRUCTOR + is PropertyAccessorDescriptor + -> KonanIr.KotlinDescriptor.Kind.ACCESSOR + is FunctionDescriptor + -> KonanIr.KotlinDescriptor.Kind.FUNCTION + is ClassDescriptor + -> KonanIr.KotlinDescriptor.Kind.CLASS + is ValueParameterDescriptor + -> KonanIr.KotlinDescriptor.Kind.VALUE_PARAMETER + is LocalVariableDescriptor, + is IrTemporaryVariableDescriptor + -> KonanIr.KotlinDescriptor.Kind.VARIABLE + is TypeParameterDescriptor + -> KonanIr.KotlinDescriptor.Kind.TYPE_PARAMETER + is ReceiverParameterDescriptor + -> KonanIr.KotlinDescriptor.Kind.RECEIVER + else -> TODO("Unexpected local descriptor.") + } + + fun functionDescriptorSpecifics(descriptor: FunctionDescriptor, proto: KonanIr.KotlinDescriptor.Builder) { + + descriptor.valueParameters.forEach { + proto.addValueParameter(serializeDescriptor(it)) + } + + descriptor.typeParameters.forEach { + proto.addTypeParameter(serializeDescriptor(it)) + } + + // Allocate two indicies for the receivers. + // They are not deserialized from protobuf, + // just recreated together with their function. + + val dispatchReceiver = descriptor.dispatchReceiverParameter + if (dispatchReceiver != null) + proto.setDispatchReceiverIndex( + descriptorTable.indexByValue(dispatchReceiver)) + + val extensionReceiver = descriptor.extensionReceiverParameter + if (extensionReceiver != null) { + proto.setExtensionReceiverIndex( + descriptorTable.indexByValue(extensionReceiver)) + proto.setExtensionReceiverType( + serializeKotlinType(extensionReceiver.type!!)) + } + + proto.setType(serializeKotlinType( + descriptor.returnType!!)) + + } + + fun variableDescriptorSpecifics(descriptor: VariableDescriptor, proto: KonanIr.KotlinDescriptor.Builder) { + proto.setType(serializeKotlinType(descriptor.type)) + } + + fun serializeDescriptor(descriptor: DeclarationDescriptor): KonanIr.KotlinDescriptor { + + if (descriptor is CallableMemberDescriptor && + descriptor.kind == CallableMemberDescriptor.Kind.FAKE_OVERRIDE) { + // TODO: It seems rather braindead. + // Do we need to do anything more than that? + return serializeDescriptor(DescriptorUtils.unwrapFakeOverride(descriptor)) + } + + val classOrPackage = descriptor.classOrPackage + val parentFqNameIndex = if (classOrPackage is ClassOrPackageFragmentDescriptor) { + stringTable.getClassOrPackageFqNameIndex(classOrPackage) + } else { -1 } + + val index = descriptorTable.indexByValue(descriptor) + // For getters and setters we use + // the *protperty* original index. + val originalIndex = if (descriptor is PropertyAccessorDescriptor) { + descriptorTable.indexByValue(descriptor.correspondingProperty.original) + } else { + descriptorTable.indexByValue(descriptor.original) + + } + + context.log("index = $index") + context.log("originalIndex = $originalIndex") + context.log("") + + val proto = KonanIr.KotlinDescriptor.newBuilder() + .setName(descriptor.name.asString()) + .setKind(kotlinDescriptorKind(descriptor)) + .setIndex(index) + .setOriginalIndex(originalIndex) + .setClassOrPackage(parentFqNameIndex) + + when (descriptor) { + is FunctionDescriptor -> + functionDescriptorSpecifics(descriptor, proto) + is VariableDescriptor -> + variableDescriptorSpecifics(descriptor, proto) + } + + return proto.build() + } + +} + diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanDescriptorSerializer.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanDescriptorSerializer.kt new file mode 100644 index 00000000000..c2d0727a631 --- /dev/null +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanDescriptorSerializer.kt @@ -0,0 +1,666 @@ +/* + * Copyright 2010-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.serialization + +import org.jetbrains.kotlin.builtins.KotlinBuiltIns +import org.jetbrains.kotlin.builtins.isSuspendFunctionType +import org.jetbrains.kotlin.builtins.transformSuspendFunctionToRuntimeFunctionType +import org.jetbrains.kotlin.config.LanguageFeature +import org.jetbrains.kotlin.descriptors.* +import org.jetbrains.kotlin.descriptors.annotations.Annotated +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.name.Name +import org.jetbrains.kotlin.protobuf.MessageLite +import org.jetbrains.kotlin.resolve.DescriptorUtils +import org.jetbrains.kotlin.resolve.DescriptorUtils.isEnumEntry +import org.jetbrains.kotlin.resolve.MemberComparator +import org.jetbrains.kotlin.resolve.constants.NullValue +import org.jetbrains.kotlin.serialization.deserialization.descriptors.SinceKotlinInfo +import org.jetbrains.kotlin.types.* +import org.jetbrains.kotlin.types.typeUtil.contains +import org.jetbrains.kotlin.utils.Interner +import java.io.ByteArrayOutputStream +import java.util.* + +import org.jetbrains.kotlin.backend.konan.serialization.KonanSerializerExtension + +// This is an almost exact copy of DescriptorSerializer +// from the big Kotlin. +// The only difference for now is typeId is public now. +// Consider bringing the new functionaity upstream. + +class KonanDescriptorSerializer private constructor( + private val containingDeclaration: DeclarationDescriptor?, + private val typeParameters: Interner, + private val extension: SerializerExtension, + private val typeTable: MutableTypeTable, + private val sinceKotlinInfoTable: MutableSinceKotlinInfoTable, + private val serializeTypeTableToFunction: Boolean +) { + fun serialize(message: MessageLite): ByteArray { + return ByteArrayOutputStream().apply { + stringTable.serializeTo(this) + message.writeTo(this) + }.toByteArray() + } + + private fun createChildSerializer(descriptor: DeclarationDescriptor): KonanDescriptorSerializer = + KonanDescriptorSerializer(descriptor, Interner(typeParameters), extension, typeTable, sinceKotlinInfoTable, + serializeTypeTableToFunction = false) + + val stringTable: StringTable + get() = extension.stringTable + + private fun useTypeTable(): Boolean = extension.shouldUseTypeTable() + + fun classProto(classDescriptor: ClassDescriptor): ProtoBuf.Class.Builder { + val builder = ProtoBuf.Class.newBuilder() + + val flags = Flags.getClassFlags( + hasAnnotations(classDescriptor), classDescriptor.visibility, classDescriptor.modality, classDescriptor.kind, + classDescriptor.isInner, classDescriptor.isCompanionObject, classDescriptor.isData, classDescriptor.isExternal + ) + if (flags != builder.flags) { + builder.flags = flags + } + + builder.fqName = getClassifierId(classDescriptor) + + for (typeParameterDescriptor in classDescriptor.declaredTypeParameters) { + builder.addTypeParameter(typeParameter(typeParameterDescriptor)) + } + + if (!KotlinBuiltIns.isSpecialClassWithNoSupertypes(classDescriptor)) { + // Special classes (Any, Nothing) have no supertypes + for (supertype in classDescriptor.typeConstructor.supertypes) { + if (useTypeTable()) { + builder.addSupertypeId(typeId(supertype)) + } + else { + builder.addSupertype(type(supertype)) + } + } + } + + for (descriptor in classDescriptor.constructors) { + builder.addConstructor(constructorProto(descriptor)) + } + + for (descriptor in sort(DescriptorUtils.getAllDescriptors(classDescriptor.defaultType.memberScope))) { + if (descriptor is CallableMemberDescriptor) { + if (descriptor.kind == CallableMemberDescriptor.Kind.FAKE_OVERRIDE) continue + + when (descriptor) { + is PropertyDescriptor -> builder.addProperty(propertyProto(descriptor)) + is FunctionDescriptor -> builder.addFunction(functionProto(descriptor)) + } + } + } + + for (descriptor in sort(DescriptorUtils.getAllDescriptors(classDescriptor.unsubstitutedInnerClassesScope))) { + if (descriptor is TypeAliasDescriptor) { + builder.addTypeAlias(typeAliasProto(descriptor)) + } + else { + val name = getSimpleNameIndex(descriptor.name) + if (isEnumEntry(descriptor)) { + builder.addEnumEntry(enumEntryProto(descriptor as ClassDescriptor)) + } + else { + builder.addNestedClassName(name) + } + } + } + + for (sealedSubclass in classDescriptor.sealedSubclasses) { + builder.addSealedSubclassFqName(getClassifierId(sealedSubclass)) + } + + val companionObjectDescriptor = classDescriptor.companionObjectDescriptor + if (companionObjectDescriptor != null) { + builder.companionObjectName = getSimpleNameIndex(companionObjectDescriptor.name) + } + + val typeTableProto = typeTable.serialize() + if (typeTableProto != null) { + builder.typeTable = typeTableProto + } + + val sinceKotlinInfoProto = sinceKotlinInfoTable.serialize() + if (sinceKotlinInfoProto != null) { + builder.sinceKotlinInfoTable = sinceKotlinInfoProto + } + + extension.serializeClass(classDescriptor, builder) + + return builder + } + + fun propertyProto(descriptor: PropertyDescriptor): ProtoBuf.Property.Builder { + val builder = ProtoBuf.Property.newBuilder() + + val local = createChildSerializer(descriptor) + + var hasGetter = false + var hasSetter = false + val lateInit = descriptor.isLateInit + val isConst = descriptor.isConst + + val compileTimeConstant = descriptor.compileTimeInitializer + val hasConstant = compileTimeConstant != null && compileTimeConstant !is NullValue + + val hasAnnotations = descriptor.annotations.getAllAnnotations().isNotEmpty() + + val propertyFlags = Flags.getAccessorFlags(hasAnnotations, descriptor.visibility, descriptor.modality, false, false, false) + + val getter = descriptor.getter + if (getter != null) { + hasGetter = true + val accessorFlags = getAccessorFlags(getter) + if (accessorFlags != propertyFlags) { + builder.getterFlags = accessorFlags + } + } + + val setter = descriptor.setter + if (setter != null) { + hasSetter = true + val accessorFlags = getAccessorFlags(setter) + if (accessorFlags != propertyFlags) { + builder.setterFlags = accessorFlags + } + + if (!setter.isDefault) { + val setterLocal = local.createChildSerializer(setter) + for (valueParameterDescriptor in setter.valueParameters) { + builder.setSetterValueParameter(setterLocal.valueParameter(valueParameterDescriptor)) + } + } + } + + val flags = Flags.getPropertyFlags( + hasAnnotations, descriptor.visibility, descriptor.modality, descriptor.kind, descriptor.isVar, + hasGetter, hasSetter, hasConstant, isConst, lateInit, descriptor.isExternal, + @Suppress("DEPRECATION") descriptor.isDelegated + ) + if (flags != builder.flags) { + builder.flags = flags + } + + builder.name = getSimpleNameIndex(descriptor.name) + + if (useTypeTable()) { + builder.returnTypeId = local.typeId(descriptor.type) + } + else { + builder.setReturnType(local.type(descriptor.type)) + } + + for (typeParameterDescriptor in descriptor.typeParameters) { + builder.addTypeParameter(local.typeParameter(typeParameterDescriptor)) + } + + val receiverParameter = descriptor.extensionReceiverParameter + if (receiverParameter != null) { + if (useTypeTable()) { + builder.receiverTypeId = local.typeId(receiverParameter.type) + } + else { + builder.setReceiverType(local.type(receiverParameter.type)) + } + } + + if (descriptor.isSuspendOrHasSuspendTypesInSignature()) { + builder.sinceKotlinInfo = writeSinceKotlinInfo(LanguageFeature.Coroutines) + } + + extension.serializeProperty(descriptor, builder) + + return builder + } + + fun functionProto(descriptor: FunctionDescriptor): ProtoBuf.Function.Builder { + val builder = ProtoBuf.Function.newBuilder() + + val local = createChildSerializer(descriptor) + + val flags = Flags.getFunctionFlags( + hasAnnotations(descriptor), descriptor.visibility, descriptor.modality, descriptor.kind, descriptor.isOperator, + descriptor.isInfix, descriptor.isInline, descriptor.isTailrec, descriptor.isExternal, descriptor.isSuspend + ) + if (flags != builder.flags) { + builder.flags = flags + } + + builder.name = getSimpleNameIndex(descriptor.name) + + if (useTypeTable()) { + builder.returnTypeId = local.typeId(descriptor.returnType!!) + } + else { + builder.setReturnType(local.type(descriptor.returnType!!)) + } + + for (typeParameterDescriptor in descriptor.typeParameters) { + builder.addTypeParameter(local.typeParameter(typeParameterDescriptor)) + } + + val receiverParameter = descriptor.extensionReceiverParameter + if (receiverParameter != null) { + if (useTypeTable()) { + builder.receiverTypeId = local.typeId(receiverParameter.type) + } + else { + builder.setReceiverType(local.type(receiverParameter.type)) + } + } + + for (valueParameterDescriptor in descriptor.valueParameters) { + builder.addValueParameter(local.valueParameter(valueParameterDescriptor)) + } + + if (serializeTypeTableToFunction) { + val typeTableProto = typeTable.serialize() + if (typeTableProto != null) { + builder.typeTable = typeTableProto + } + } + + if (descriptor.isSuspendOrHasSuspendTypesInSignature()) { + builder.sinceKotlinInfo = writeSinceKotlinInfo(LanguageFeature.Coroutines) + } + + extension.serializeFunction(descriptor, builder) + + return builder + } + + fun constructorProto(descriptor: ConstructorDescriptor): ProtoBuf.Constructor.Builder { + val builder = ProtoBuf.Constructor.newBuilder() + + val local = createChildSerializer(descriptor) + + val flags = Flags.getConstructorFlags(hasAnnotations(descriptor), descriptor.visibility, !descriptor.isPrimary) + if (flags != builder.flags) { + builder.flags = flags + } + + for (valueParameterDescriptor in descriptor.valueParameters) { + builder.addValueParameter(local.valueParameter(valueParameterDescriptor)) + } + + if (descriptor.isSuspendOrHasSuspendTypesInSignature()) { + builder.sinceKotlinInfo = writeSinceKotlinInfo(LanguageFeature.Coroutines) + } + + extension.serializeConstructor(descriptor, builder) + + return builder + } + + private fun CallableMemberDescriptor.isSuspendOrHasSuspendTypesInSignature(): Boolean { + if (this is FunctionDescriptor && isSuspend) return true + + return listOfNotNull( + extensionReceiverParameter?.type, + returnType, + *valueParameters.map(ValueParameterDescriptor::getType).toTypedArray() + ).any { type -> type.contains(UnwrappedType::isSuspendFunctionType) } + } + + fun typeAliasProto(descriptor: TypeAliasDescriptor): ProtoBuf.TypeAlias.Builder { + val builder = ProtoBuf.TypeAlias.newBuilder() + val local = createChildSerializer(descriptor) + + val flags = Flags.getTypeAliasFlags(hasAnnotations(descriptor), descriptor.visibility) + if (flags != builder.flags) { + builder.flags = flags + } + + builder.name = getSimpleNameIndex(descriptor.name) + + for (typeParameterDescriptor in descriptor.declaredTypeParameters) { + builder.addTypeParameter(local.typeParameter(typeParameterDescriptor)) + } + + val underlyingType = descriptor.underlyingType + if (useTypeTable()) { + builder.underlyingTypeId = local.typeId(underlyingType) + } + else { + builder.setUnderlyingType(local.type(underlyingType)) + } + + val expandedType = descriptor.expandedType + if (useTypeTable()) { + builder.expandedTypeId = local.typeId(expandedType) + } + else { + builder.setExpandedType(local.type(expandedType)) + } + + builder.addAllAnnotation(descriptor.annotations.map { extension.annotationSerializer.serializeAnnotation(it) }) + + return builder + } + + fun enumEntryProto(descriptor: ClassDescriptor): ProtoBuf.EnumEntry.Builder { + val builder = ProtoBuf.EnumEntry.newBuilder() + builder.name = getSimpleNameIndex(descriptor.name) + extension.serializeEnumEntry(descriptor, builder) + return builder + } + + private fun valueParameter(descriptor: ValueParameterDescriptor): ProtoBuf.ValueParameter.Builder { + val builder = ProtoBuf.ValueParameter.newBuilder() + + val flags = Flags.getValueParameterFlags( + hasAnnotations(descriptor), descriptor.declaresDefaultValue(), + descriptor.isCrossinline, descriptor.isNoinline + ) + if (flags != builder.flags) { + builder.flags = flags + } + + builder.name = getSimpleNameIndex(descriptor.name) + + if (useTypeTable()) { + builder.typeId = typeId(descriptor.type) + } + else { + builder.setType(type(descriptor.type)) + } + + val varargElementType = descriptor.varargElementType + if (varargElementType != null) { + if (useTypeTable()) { + builder.varargElementTypeId = typeId(varargElementType) + } + else { + builder.setVarargElementType(type(varargElementType)) + } + } + + extension.serializeValueParameter(descriptor, builder) + + return builder + } + + private fun typeParameter(typeParameter: TypeParameterDescriptor): ProtoBuf.TypeParameter.Builder { + val builder = ProtoBuf.TypeParameter.newBuilder() + + builder.id = getTypeParameterId(typeParameter) + + builder.name = getSimpleNameIndex(typeParameter.name) + + if (typeParameter.isReified != builder.reified) { + builder.reified = typeParameter.isReified + } + + val variance = variance(typeParameter.variance) + if (variance != builder.variance) { + builder.variance = variance + } + extension.serializeTypeParameter(typeParameter, builder) + + val upperBounds = typeParameter.upperBounds + if (upperBounds.size == 1 && KotlinBuiltIns.isDefaultBound(upperBounds.single())) return builder + + for (upperBound in upperBounds) { + if (useTypeTable()) { + builder.addUpperBoundId(typeId(upperBound)) + } + else { + builder.addUpperBound(type(upperBound)) + } + } + + return builder + } + + public fun typeId(type: KotlinType): Int = typeTable[type(type)] + + private fun type(type: KotlinType): ProtoBuf.Type.Builder { + val builder = ProtoBuf.Type.newBuilder() + + if (type.isError) { + extension.serializeErrorType(type, builder) + return builder + } + + if (type.isFlexible()) { + val flexibleType = type.asFlexibleType() + + val lowerBound = type(flexibleType.lowerBound) + val upperBound = type(flexibleType.upperBound) + extension.serializeFlexibleType(flexibleType, lowerBound, upperBound) + if (useTypeTable()) { + lowerBound.flexibleUpperBoundId = typeTable[upperBound] + } + else { + lowerBound.setFlexibleUpperBound(upperBound) + } + return lowerBound + } + + if (type.isSuspendFunctionType) { + val functionType = type(transformSuspendFunctionToRuntimeFunctionType(type)) + functionType.flags = Flags.getTypeFlags(true) + return functionType + } + + val descriptor = type.constructor.declarationDescriptor + when (descriptor) { + is ClassDescriptor, is TypeAliasDescriptor -> { + val possiblyInnerType = type.buildPossiblyInnerType() ?: error("possiblyInnerType should not be null: $type") + fillFromPossiblyInnerType(builder, possiblyInnerType) + } + is TypeParameterDescriptor -> { + if (descriptor.containingDeclaration === containingDeclaration) { + builder.typeParameterName = getSimpleNameIndex(descriptor.name) + } + else { + builder.typeParameter = getTypeParameterId(descriptor) + } + + assert(type.arguments.isEmpty()) { "Found arguments for type constructor build on type parameter: $descriptor" } + } + } + + if (type.isMarkedNullable != builder.nullable) { + builder.nullable = type.isMarkedNullable + } + + val abbreviation = type.getAbbreviatedType()?.abbreviation + if (abbreviation != null) { + if (useTypeTable()) { + builder.abbreviatedTypeId = typeId(abbreviation) + } + else { + builder.setAbbreviatedType(type(abbreviation)) + } + } + + extension.serializeType(type, builder) + + return builder + } + + private fun fillFromPossiblyInnerType(builder: ProtoBuf.Type.Builder, type: PossiblyInnerType) { + val classifierDescriptor = type.classifierDescriptor + val classifierId = getClassifierId(classifierDescriptor) + when (classifierDescriptor) { + is ClassDescriptor -> builder.className = classifierId + is TypeAliasDescriptor -> builder.typeAliasName = classifierId + } + + for (projection in type.arguments) { + builder.addArgument(typeArgument(projection)) + } + + if (type.outerType != null) { + val outerBuilder = ProtoBuf.Type.newBuilder() + fillFromPossiblyInnerType(outerBuilder, type.outerType!!) + if (useTypeTable()) { + builder.outerTypeId = typeTable[outerBuilder] + } + else { + builder.setOuterType(outerBuilder) + } + } + } + + private fun typeArgument(typeProjection: TypeProjection): ProtoBuf.Type.Argument.Builder { + val builder = ProtoBuf.Type.Argument.newBuilder() + + if (typeProjection.isStarProjection) { + builder.projection = ProtoBuf.Type.Argument.Projection.STAR + } + else { + val projection = projection(typeProjection.projectionKind) + + if (projection != builder.projection) { + builder.projection = projection + } + + if (useTypeTable()) { + builder.typeId = typeId(typeProjection.type) + } + else { + builder.setType(type(typeProjection.type)) + } + } + + return builder + } + + fun packagePartProto(packageFqName: FqName, members: Collection): ProtoBuf.Package.Builder { + val builder = ProtoBuf.Package.newBuilder() + + for (declaration in sort(members)) { + when (declaration) { + is PropertyDescriptor -> builder.addProperty(propertyProto(declaration)) + is FunctionDescriptor -> builder.addFunction(functionProto(declaration)) + is TypeAliasDescriptor -> builder.addTypeAlias(typeAliasProto(declaration)) + } + } + + val typeTableProto = typeTable.serialize() + if (typeTableProto != null) { + builder.typeTable = typeTableProto + } + + val sinceKotlinInfoProto = sinceKotlinInfoTable.serialize() + if (sinceKotlinInfoProto != null) { + builder.sinceKotlinInfoTable = sinceKotlinInfoProto + } + + extension.serializePackage(packageFqName, builder) + + return builder + } + + private fun writeSinceKotlinInfo(languageFeature: LanguageFeature): Int { + val languageVersion = languageFeature.sinceVersion!! + val sinceKotlinInfo = ProtoBuf.SinceKotlinInfo.newBuilder().apply { + SinceKotlinInfo.Version(languageVersion.major, languageVersion.minor).encode( + writeVersion = { version = it }, + writeVersionFull = { versionFull = it } + ) + } + return sinceKotlinInfoTable[sinceKotlinInfo] + } + + private fun getClassifierId(descriptor: ClassifierDescriptorWithTypeParameters): Int = + stringTable.getFqNameIndex(descriptor) + + private fun getSimpleNameIndex(name: Name): Int = + stringTable.getStringIndex(name.asString()) + + private fun getTypeParameterId(descriptor: TypeParameterDescriptor): Int = + typeParameters.intern(descriptor) + + companion object { + @JvmStatic + fun createTopLevel(extension: SerializerExtension): KonanDescriptorSerializer { + return KonanDescriptorSerializer(null, Interner(), extension, MutableTypeTable(), MutableSinceKotlinInfoTable(), + serializeTypeTableToFunction = false) + } + + @JvmStatic + fun createForLambda(extension: SerializerExtension): KonanDescriptorSerializer { + return KonanDescriptorSerializer(null, Interner(), extension, MutableTypeTable(), MutableSinceKotlinInfoTable(), + serializeTypeTableToFunction = true) + } + + @JvmStatic + fun create(descriptor: ClassDescriptor, extension: SerializerExtension): KonanDescriptorSerializer { + val container = descriptor.containingDeclaration + val parentSerializer = if (container is ClassDescriptor) + create(container, extension) + else + createTopLevel(extension) + + // Calculate type parameter ids for the outer class beforehand, as it would've had happened if we were always + // serializing outer classes before nested classes. + // Otherwise our interner can get wrong ids because we may serialize classes in any order. + val serializer = KonanDescriptorSerializer( + descriptor, + Interner(parentSerializer.typeParameters), + parentSerializer.extension, + MutableTypeTable(), + MutableSinceKotlinInfoTable(), + serializeTypeTableToFunction = false + ) + for (typeParameter in descriptor.declaredTypeParameters) { + serializer.typeParameters.intern(typeParameter) + } + return serializer + } + + private fun getAccessorFlags(accessor: PropertyAccessorDescriptor): Int { + return Flags.getAccessorFlags( + hasAnnotations(accessor), + accessor.visibility, + accessor.modality, + !accessor.isDefault, + accessor.isExternal, + accessor.isInline + ) + } + + private fun variance(variance: Variance): ProtoBuf.TypeParameter.Variance = when (variance) { + Variance.INVARIANT -> ProtoBuf.TypeParameter.Variance.INV + Variance.IN_VARIANCE -> ProtoBuf.TypeParameter.Variance.IN + Variance.OUT_VARIANCE -> ProtoBuf.TypeParameter.Variance.OUT + } + + private fun projection(projectionKind: Variance): ProtoBuf.Type.Argument.Projection = when (projectionKind) { + Variance.INVARIANT -> ProtoBuf.Type.Argument.Projection.INV + Variance.IN_VARIANCE -> ProtoBuf.Type.Argument.Projection.IN + Variance.OUT_VARIANCE -> ProtoBuf.Type.Argument.Projection.OUT + } + + private fun hasAnnotations(descriptor: Annotated): Boolean = !descriptor.annotations.isEmpty() + + fun sort(descriptors: Collection): List = + ArrayList(descriptors).apply { + //NOTE: the exact comparator does matter here + Collections.sort(this, MemberComparator.INSTANCE) + } + } +} diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanIr.proto b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanIr.proto new file mode 100644 index 00000000000..4fde0fcf4e5 --- /dev/null +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanIr.proto @@ -0,0 +1,271 @@ + +syntax = "proto2"; +package org.jetbrains.kotlin.serialization; + +import "org/jetbrains/kotlin/backend/konan/serialization/KonanLinkData.proto"; +option java_outer_classname = "KonanIr"; +option optimize_for = LITE_RUNTIME; + +// This is subject to change. +// One can think of it as a 'descriptor reference'. +// The real descriptor data is encoded in KonanLinkdData. +message KotlinDescriptor { + enum Kind { + FUNCTION = 1; + VARIABLE = 2; + CLASS = 3; + VALUE_PARAMETER = 4; + CONSTRUCTOR = 5; + TYPE_PARAMETER = 6; + RECEIVER = 7; + ACCESSOR = 8; + } + // TODO: Use the string table? + optional string name = 1; + required Kind kind = 2; + required int32 index = 3; + required int32 original_index = 4; + // index in fq name table + required int32 class_or_package = 5; + // Descriptor kind specific fields. + optional KotlinType type = 6; + repeated KotlinDescriptor value_parameter = 7; + repeated KotlinDescriptor type_parameter = 8; + optional int32 dispatch_receiver_index = 9; + optional int32 extension_receiver_index = 10; + optional KotlinType extension_receiver_type = 11; + optional LocalDeclaration ir_local_declaration = 12; +} + +message LocalDeclaration { + // This is Descriptor message from KonanLinkData + required Descriptor descriptor = 1; +} + +message KotlinType { + required int32 index = 1; + optional string debug_text = 2; // TODO: remove me +} + +message Coordinates { + required int32 start_offset = 1; + required int32 end_offset = 2; +} + +message TypeMap { + message Pair { + required KotlinDescriptor descriptor = 1; + required KotlinType type = 2; + } + repeated Pair pair = 1; +} + +/* ------ IrExpressions --------------------------------------------- */ + +message IrBreak { + required int32 loop_id = 1; + optional string label = 2; +} + +message IrBlock { + required bool is_transparent_scope = 1; + repeated IrStatement statement = 2; +} + +message IrCall { + enum Primitive { + NOT_PRIMITIVE = 1; + NULLARY = 2; + UNARY = 3; + BINARY = 4; + } + required Primitive kind = 1; + required KotlinDescriptor descriptor = 2; + required TypeMap type_map = 3; + optional KotlinDescriptor super = 4; + optional IrExpression dispatch_receiver = 5; + optional IrExpression extension_receiver = 6; + repeated IrExpression value_argument = 7; +} + +message IrCallableReference { + required KotlinDescriptor descriptor = 1; + required TypeMap type_map = 2; + //required int32 value_parameters_size = 3; +} + +message IrConst { + oneof value { + bool null = 1; + bool boolean = 2; + int32 char = 3; + int32 byte = 4; + int32 short = 5; + int32 int = 6; + int64 long = 7; + float float = 8; + double double = 9; + string string = 10; + } +} + +message IrDelegatingConstructorCall { + required KotlinDescriptor descriptor = 1; + required TypeMap type_map = 2; + repeated IrExpression value_argument = 3; +} + +message IrGetValue { + required KotlinDescriptor descriptor = 1; +} + +message IrGetObject { + required KotlinDescriptor descriptor = 1; +} + +message IrInstanceInitializerCall { + required KotlinDescriptor descriptor = 1; +} + +message IrReturn { + required KotlinDescriptor return_target = 1; + required IrExpression value = 2; +} + +message IrSetVariable { + required KotlinDescriptor descriptor = 1; + required IrExpression value = 2; +} + +message IrStringConcat { + repeated IrExpression argument = 1; +} + +message IrThrow { + required IrExpression value = 1; +} + +message IrTry { + required IrExpression result = 1; + repeated IrStatement catch = 2; + optional IrExpression finally = 3; +} + +message IrTypeOp { + required IrTypeOperator operator = 1; + required KotlinType operand = 2; + required IrExpression argument = 3; +} + +message IrVararg { + required KotlinType element_type = 1; +} + +message IrWhen { + repeated IrStatement branch = 1; +} + +message IrWhile { + required int32 loop_id = 1; + required IrExpression condition = 2; + optional string label = 3; + optional IrExpression body = 4; +} + +message IrOperation { + oneof operation { + IrBlock block = 1; + IrBreak break = 2; + IrCall call = 3; + IrCallableReference callable_reference = 4; + IrConst const = 5; + IrDelegatingConstructorCall delegating_constructor_call = 6; + IrGetValue get_value = 7; + IrGetObject get_object = 8; + IrInstanceInitializerCall instance_initializer_call = 9; + IrReturn return = 10; + IrSetVariable set_variable = 11; + IrStringConcat string_concat = 12; + IrThrow throw = 13; + IrTry try = 14; + IrTypeOp type_op = 15; + IrVararg vararg = 16; + IrWhen when = 17; + IrWhile while = 18; + } +} + +enum IrTypeOperator { + CAST = 1; + IMPLICIT_CAST = 2; + IMPLICIT_NOTNULL = 3; + IMPLICIT_COERCION_TO_UNIT = 4; + SAFE_CAST = 5; + INSTANCEOF = 6; + NOT_INSTANCEOF = 7; +} + +message IrExpression { + required IrOperation operation = 1; + required KotlinType type = 2; + required Coordinates coordinates = 3; +} + + +/* ------ Declarations --------------------------------------------- */ + +message IrFunc { + optional IrStatement body = 1; +} + +message IrVar { + optional IrExpression initializer = 1; +} + +message IrClass { + repeated IrDeclaration member = 1; +} + +message IrDeclarator { + oneof symbol { + IrVar variable = 1; + IrFunc function = 2; + IrClass ir_class = 3; + } +} + +message IrDeclaration { + required KotlinDescriptor descriptor = 1; + required Coordinates coordinates = 2; + required IrDeclarator declarator = 3; + repeated IrDeclaration nested = 4; +} + +/* ------- IrStatements --------------------------------------------- */ + +message IrBranch { + required IrExpression condition = 1; + required IrExpression result = 2; +} + +message IrBlockBody { + repeated IrStatement statement = 1; +} + +message IrCatch { + required KotlinDescriptor parameter = 1; + required IrExpression result = 2; +} + +// Let's try to map IrElement as well as IrStatement to IrStatement. +message IrStatement { + required Coordinates coordinates = 1; + oneof statement { + IrDeclaration declaration = 2; + IrExpression expression = 3; + IrBlockBody block_body = 4; + IrBranch branch = 5; + IrCatch catch = 6; + } +} + diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanLinkData.proto b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanLinkData.proto index 2bb717d9e89..81119d6a465 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanLinkData.proto +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanLinkData.proto @@ -2,7 +2,7 @@ syntax = "proto2"; package org.jetbrains.kotlin.serialization; -// This, an all the rest of the included proto files have ".proto1" extension. +// This, and all the rest of the included proto files have ".proto1" extension. // That allows us to construct the current file, // but protoc will skip .class files for them. // The issue here is that we need to co-exist with the big Kotlin, @@ -21,35 +21,70 @@ extend Package { extend Class { repeated Annotation class_annotation = 170; + optional int32 class_index = 171; } extend Constructor { repeated Annotation constructor_annotation = 170; + optional int32 constructor_index = 171; + optional int32 constructor_parent = 172; } extend Function { repeated Annotation function_annotation = 170; + optional int32 function_index = 171; + optional int32 function_parent = 172; + optional InlineIrBody inline_ir_body = 174; } extend Property { repeated Annotation property_annotation = 170; - optional Annotation.Argument.Value compile_time_value = 171; + optional int32 property_index = 171; + optional int32 property_parent = 172; + optional bool used_as_variable = 173; + optional Annotation.Argument.Value compile_time_value = 174; } extend EnumEntry { repeated Annotation enum_entry_annotation = 170; + optional int32 enum_entry_index = 171; } extend ValueParameter { repeated Annotation parameter_annotation = 170; + optional int32 value_parameter_index = 171; } extend Type { repeated Annotation type_annotation = 170; + optional int32 type_index = 171; + optional string type_text = 172; // TODO: remove me } extend TypeParameter { repeated Annotation type_parameter_annotation = 170; + //optional int32 type_parameter_index = 171; + //optional int32 type_parameter_parent = 172; +} + +message InlineIrBody { + // We need to refer from descriptors to ir inline body. + // And in ir we need to refer local declaration descriptors + // That requires mutual import of KonanIr and KonanLinkData. + // I break the circle here by storing encoded IR. + // May be we need to merge KonanIr into KonanLinkData. + // That'd allow mutually recursive messages. + required string encoded_ir = 11; +} + +// Inner descriptors used for inlining. +message Descriptor { + oneof descriptor { + Function function = 1; + Class clazz = 2; + Property property = 3; + Constructor constructor = 4; + } } // Konan Binary Linkdata structures. diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanPackageFragment.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanPackageFragment.kt index 0451545e0e4..cd6aaf0cf3c 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanPackageFragment.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanPackageFragment.kt @@ -9,7 +9,7 @@ import org.jetbrains.kotlin.serialization.deserialization.NameResolverImpl import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedPackageMemberScope import org.jetbrains.kotlin.storage.StorageManager -class KonanPackageFragment( private val proto: KonanLinkData.PackageFragment, +class KonanPackageFragment( val proto: KonanLinkData.PackageFragment, storageManager: StorageManager, module: ModuleDescriptor) : DeserializedPackageFragment(FqName(proto.getFqName()), storageManager, module) { diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanSerializationUtil.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanSerializationUtil.kt index e38fbc8c25d..94111f71af6 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanSerializationUtil.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanSerializationUtil.kt @@ -6,15 +6,23 @@ import org.jetbrains.kotlin.backend.konan.llvm.base64Decode import org.jetbrains.kotlin.backend.konan.llvm.base64Encode import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.descriptors.* +import org.jetbrains.kotlin.descriptors.CallableMemberDescriptor.Kind.DECLARATION +import org.jetbrains.kotlin.descriptors.Modality.FINAL +import org.jetbrains.kotlin.descriptors.SourceElement.NO_SOURCE +import org.jetbrains.kotlin.descriptors.Visibilities.INTERNAL +import org.jetbrains.kotlin.descriptors.annotations.Annotations.Companion.EMPTY +import org.jetbrains.kotlin.descriptors.impl.LocalVariableDescriptor import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl +import org.jetbrains.kotlin.descriptors.impl.PropertyDescriptorImpl import org.jetbrains.kotlin.incremental.components.LookupTracker +import org.jetbrains.kotlin.ir.descriptors.IrTemporaryVariableDescriptor import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.name.Name import org.jetbrains.kotlin.resolve.CompilerDeserializationConfiguration import org.jetbrains.kotlin.resolve.DescriptorUtils import org.jetbrains.kotlin.resolve.scopes.DescriptorKindFilter import org.jetbrains.kotlin.resolve.scopes.MemberScope -import org.jetbrains.kotlin.serialization.DescriptorSerializer +import org.jetbrains.kotlin.serialization.KonanDescriptorSerializer import org.jetbrains.kotlin.serialization.KonanLinkData import org.jetbrains.kotlin.serialization.ProtoBuf import org.jetbrains.kotlin.serialization.deserialization.* @@ -73,7 +81,6 @@ internal fun deserializeModule(configuration: CompilerConfiguration, builtIns.builtInsModule = moduleDescriptor val deserializationConfiguration = CompilerDeserializationConfiguration(configuration) - val libraryAsByteArray = base64Decode(base64) val libraryProto = KonanLinkData.Library.parseFrom( libraryAsByteArray, KonanSerializerProtocol.extensionRegistry) @@ -92,8 +99,9 @@ internal fun deserializeModule(configuration: CompilerConfiguration, internal class KonanSerializationUtil(val context: Context) { - val serializerExtension = KonanSerializerExtension(context) - val serializer = DescriptorSerializer.createTopLevel(serializerExtension) + val serializerExtension = KonanSerializerExtension(context, this) + val serializer = KonanDescriptorSerializer.createTopLevel(serializerExtension) + val typeSerializer: (KotlinType)->Int = { it -> serializer.typeId(it) } fun serializeClass(packageName: FqName, builder: KonanLinkData.Classes.Builder, @@ -102,7 +110,7 @@ internal class KonanSerializationUtil(val context: Context) { if (skip(classDescriptor)) return - val classProto = serializer.classProto(classDescriptor).build() + val classProto = serializer.classProto(classDescriptor).build() ?: error("Class not serialized: $classDescriptor") builder.addClasses(classProto) @@ -136,7 +144,7 @@ internal class KonanSerializationUtil(val context: Context) { val skip: (DeclarationDescriptor) -> Boolean = { DescriptorUtils.getContainingModule(it) != module } - val classifierDescriptors = DescriptorSerializer.sort(packageView.memberScope.getContributedDescriptors( + val classifierDescriptors = KonanDescriptorSerializer.sort(packageView.memberScope.getContributedDescriptors( DescriptorKindFilter.CLASSIFIERS)) val classesBuilder = KonanLinkData.Classes.newBuilder() @@ -190,6 +198,7 @@ internal class KonanSerializationUtil(val context: Context) { internal fun serializeModule(moduleDescriptor: ModuleDescriptor): String { val library = KonanLinkData.Library.newBuilder() + getPackagesFqNames(moduleDescriptor).forEach { val packageProto = serializePackage(it, moduleDescriptor) library.addPackageFragment(packageProto) @@ -199,5 +208,68 @@ internal class KonanSerializationUtil(val context: Context) { val base64 = base64Encode(libraryAsByteArray) return base64 } + + /* The section below is specific for IR serialization */ + + // TODO: We utilize DescriptorSerializer's property + // serialization to serialize variables for now. + // Need to negotiate the ability to deserialize + // variables with the big kotlin somehow. + fun variableAsProperty(variable: VariableDescriptor): PropertyDescriptor { + + val isDelegated = when (variable) { + is LocalVariableDescriptor -> variable.isDelegated + is IrTemporaryVariableDescriptor -> false + else -> error("Unexpected variable descriptor.") + } + + val property = PropertyDescriptorImpl.create( + variable.containingDeclaration, + EMPTY, + FINAL, + INTERNAL, + variable.isVar(), + variable.name, + DECLARATION, + NO_SOURCE, + false, false, false, false, false, + isDelegated) + + property.setType(variable.type, listOf(), null, null as KotlinType?) + + // TODO: transform the getter and the setter too. + property.initialize(null, null) + return property + } + + fun serializeLocalDeclaration(descriptor: DeclarationDescriptor): KonanLinkData.Descriptor { + + val proto = KonanLinkData.Descriptor.newBuilder() + + context.log("### serializeLocalDeclaration: $descriptor") + + when (descriptor) { + is FunctionDescriptor -> + proto.setFunction(serializer.functionProto(descriptor)) + is PropertyDescriptor -> + proto.setProperty(serializer.propertyProto(descriptor)) + + is ClassDescriptor -> + proto.setClazz(serializer.classProto(descriptor)) + + is VariableDescriptor -> { + val property = variableAsProperty(descriptor) + serializerExtension.originalVariables.put(property, descriptor) + proto.setProperty(serializer.propertyProto(property)) + } + + else -> error("Unexpected descriptor kind: $descriptor") + } + + return proto.build() + } + + + } diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanSerializerExtension.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanSerializerExtension.kt index eddf457278e..188e3ff2747 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanSerializerExtension.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanSerializerExtension.kt @@ -1,19 +1,3 @@ -/* - * Copyright 2010-2016 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package org.jetbrains.kotlin.backend.konan.serialization import org.jetbrains.kotlin.backend.konan.Context @@ -23,23 +7,130 @@ import org.jetbrains.kotlin.descriptors.SourceElement import org.jetbrains.kotlin.descriptors.annotations.AnnotationDescriptorImpl import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite import org.jetbrains.kotlin.serialization.* +import org.jetbrains.kotlin.serialization.Flags +import org.jetbrains.kotlin.serialization.KonanLinkData +import org.jetbrains.kotlin.serialization.KonanLinkData.InlineIrBody +import org.jetbrains.kotlin.descriptors.* +import org.jetbrains.kotlin.descriptors.annotations.* +import org.jetbrains.kotlin.backend.konan.descriptors.* +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.psi.KtFile +import org.jetbrains.kotlin.resolve.DescriptorUtils +import org.jetbrains.kotlin.resolve.source.PsiSourceFile +import org.jetbrains.kotlin.serialization.KotlinSerializerExtensionBase +import org.jetbrains.kotlin.serialization.ProtoBuf +import org.jetbrains.kotlin.serialization.StringTableImpl +import org.jetbrains.kotlin.serialization.SerializerExtensionProtocol +import org.jetbrains.kotlin.types.FlexibleType +import org.jetbrains.kotlin.types.KotlinType -internal class KonanSerializerExtension(val context: Context) : + +internal class KonanSerializerExtension(val context: Context, val util: KonanSerializationUtil) : KotlinSerializerExtensionBase(KonanSerializerProtocol) { + val inlineDescriptorTable = DescriptorTable(context.irBuiltIns) + val originalVariables = mutableMapOf() override val stringTable = KonanStringTable() + override fun shouldUseTypeTable(): Boolean = true - private val backingFieldClass = + override fun serializeType(descriptor: KotlinType, proto: ProtoBuf.Type.Builder) { + + proto.setExtension(KonanLinkData.typeText, descriptor.toString()) + + super.serializeType(descriptor, proto) + } + + override fun serializeTypeParameter(descriptor: TypeParameterDescriptor, proto: ProtoBuf.TypeParameter.Builder) { + super.serializeTypeParameter(descriptor, proto) + } + + override fun serializeValueParameter(descriptor: ValueParameterDescriptor, proto: ProtoBuf.ValueParameter.Builder) { + + val index = inlineDescriptorTable.indexByValue(descriptor) + proto.setExtension(KonanLinkData.valueParameterIndex, index) + + super.serializeValueParameter(descriptor, proto) + } + + override fun serializeEnumEntry(descriptor: ClassDescriptor, proto: ProtoBuf.EnumEntry.Builder) { + + super.serializeEnumEntry(descriptor, proto) + } + + fun DeclarationDescriptor.parentFqNameIndex(): Int { + + if (this.containingDeclaration is ClassOrPackageFragmentDescriptor) { + val parentIndex = stringTable.getClassOrPackageFqNameIndex( + this.containingDeclaration as ClassOrPackageFragmentDescriptor) + return parentIndex + } else { + return -1 + } + } + + override fun serializeConstructor(descriptor: ConstructorDescriptor, proto: ProtoBuf.Constructor.Builder) { + + proto.setExtension(KonanLinkData.constructorIndex, + inlineDescriptorTable.indexByValue(descriptor)) + val parentIndex = descriptor.parentFqNameIndex() + proto.setExtension(KonanLinkData.constructorParent, parentIndex) + + super.serializeConstructor(descriptor, proto) + } + + override fun serializeClass(descriptor: ClassDescriptor, proto: ProtoBuf.Class.Builder) { + + proto.setExtension(KonanLinkData.classIndex, inlineDescriptorTable.indexByValue(descriptor)) + super.serializeClass(descriptor, proto) + } + + override fun serializeFunction(descriptor: FunctionDescriptor, proto: ProtoBuf.Function.Builder) { + + proto.setExtension(KonanLinkData.functionIndex, + inlineDescriptorTable.indexByValue(descriptor)) + val parentIndex = descriptor.parentFqNameIndex() + proto.setExtension(KonanLinkData.functionParent, parentIndex) + + if (descriptor.isInline()) { + + val encodedIR: String = IrSerializer( context, + inlineDescriptorTable, stringTable, util, descriptor) + .serializeInlineBody() + + val inlineIrBody = KonanLinkData.InlineIrBody + .newBuilder() + .setEncodedIr(encodedIR) + .build() + + proto.setExtension(KonanLinkData.inlineIrBody, inlineIrBody) + + } + super.serializeFunction(descriptor, proto) + } + + private val backingFieldClass = context.builtIns.getKonanInternalClass("HasBackingField").getDefaultType() private val backingFieldAnnotation = AnnotationDescriptorImpl( backingFieldClass, emptyMap(), SourceElement.NO_SOURCE) - override fun serializeProperty(descriptor: PropertyDescriptor, proto: ProtoBuf.Property.Builder) { - super.serializeProperty(descriptor, proto) - if (context.ir.propertiesWithBackingFields.contains(descriptor)) { - proto.addExtension(KonanLinkData.propertyAnnotation, + override fun serializeProperty(property: PropertyDescriptor, proto: ProtoBuf.Property.Builder) { + val parentIndex = property.parentFqNameIndex() + proto.setExtension(KonanLinkData.propertyParent, parentIndex) + val variable = originalVariables[property] + if (variable != null) { + proto.setExtension(KonanLinkData.usedAsVariable, true) + proto.setExtension(KonanLinkData.propertyIndex, inlineDescriptorTable.indexByValue(variable)) + + } else { + proto.setExtension(KonanLinkData.propertyIndex, inlineDescriptorTable.indexByValue(property)) + } + + super.serializeProperty(property, proto) + + if (context.ir.propertiesWithBackingFields.contains(property)) { + proto.addExtension(KonanLinkData.propertyAnnotation, annotationSerializer.serializeAnnotation(backingFieldAnnotation)) proto.flags = proto.flags or Flags.HAS_ANNOTATIONS.toFlags(true) diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanStringTable.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanStringTable.kt index b69f256d742..f9bcf4dcbd4 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanStringTable.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanStringTable.kt @@ -7,6 +7,18 @@ import org.jetbrains.kotlin.resolve.descriptorUtil.getAllSuperClassifiers import org.jetbrains.kotlin.resolve.descriptorUtil.module class KonanStringTable : StringTableImpl() { + + fun getClassOrPackageFqNameIndex(descriptor: ClassOrPackageFragmentDescriptor): Int { + when (descriptor) { + is PackageFragmentDescriptor -> + return getPackageFqNameIndex(descriptor.fqName) + + is ClassDescriptor -> + return getFqNameIndex(descriptor as ClassifierDescriptorWithTypeParameters) + else -> error("Can not get fqNameIndex for $descriptor") + } + } + override fun getFqNameIndexOfLocalAnonymousClass(descriptor: ClassifierDescriptorWithTypeParameters): Int { return if (descriptor.containingDeclaration is CallableMemberDescriptor) { val superClassifiers = descriptor.getAllSuperClassifiers() diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/LocalDeclarationDeserializer.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/LocalDeclarationDeserializer.kt new file mode 100644 index 00000000000..7620cc29453 --- /dev/null +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/LocalDeclarationDeserializer.kt @@ -0,0 +1,132 @@ +package org.jetbrains.kotlin.backend.konan.serialization + +import org.jetbrains.kotlin.descriptors.* +import org.jetbrains.kotlin.descriptors.impl.LocalVariableDescriptor +import org.jetbrains.kotlin.serialization.Flags +import org.jetbrains.kotlin.serialization.KonanLinkData +import org.jetbrains.kotlin.serialization.ProtoBuf +import org.jetbrains.kotlin.serialization.ProtoBuf.QualifiedNameTable.QualifiedName +import org.jetbrains.kotlin.serialization.deserialization.MemberDeserializer +import org.jetbrains.kotlin.serialization.deserialization.NameResolverImpl +import org.jetbrains.kotlin.serialization.deserialization.TypeTable +import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedClassDescriptor +import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedSimpleFunctionDescriptor +import org.jetbrains.kotlin.serialization.deserialization.descriptors.SinceKotlinInfoTable +import org.jetbrains.kotlin.serialization.deserialization.findClassAcrossModuleDependencies +import org.jetbrains.kotlin.types.KotlinType + +// This class knows how to construct contexts for +// MemberDeserializer to deserialize descriptors declared in IR. +// Consider merging it all to the very MemberDesereializer itself eventually. + +class LocalDeclarationDeserializer(val rootFunction: FunctionDescriptor, val module: ModuleDescriptor) { + + val pkg = rootFunction.getContainingDeclaration() as KonanPackageFragment + init { + assert(pkg is KonanPackageFragment) + } + + val components = pkg.components + val nameResolver = NameResolverImpl(pkg.proto.getStringTable(), pkg.proto.getNameTable()) + val nameTable = pkg.proto.getNameTable() + val typeTable = TypeTable(pkg.proto.getPackage().getTypeTable()) + val context = components.createContext( + pkg, nameResolver, typeTable, SinceKotlinInfoTable.EMPTY, null) + + val typeParameters = (rootFunction as DeserializedSimpleFunctionDescriptor).proto.typeParameterList + + val childContext = context.childContext(rootFunction, typeParameters, nameResolver, typeTable) + val typeDeserializer = childContext.typeDeserializer + val memberDeserializer = MemberDeserializer(childContext) + + fun deserializeInlineType(type: ProtoBuf.Type): KotlinType { + val result = typeDeserializer.type(type) + + return result + } + + fun getDescriptorByFqNameIndex(module: ModuleDescriptor, fqNameIndex: Int): DeclarationDescriptor { + + val packageName = nameResolver.getPackageFqName(fqNameIndex) + val shortName = nameResolver.getString(fqNameIndex) + // Here we using internals of NameresolverImpl. + val proto = nameTable.getQualifiedName(fqNameIndex) + when (proto.kind) { + QualifiedName.Kind.CLASS, + QualifiedName.Kind.LOCAL -> + return module.findClassAcrossModuleDependencies(nameResolver.getClassId(fqNameIndex))!! + QualifiedName.Kind.PACKAGE -> + return module.getPackage(packageName) + } + } + + fun memberDeserializerByParentFqNameIndex(fqName: Int): MemberDeserializer { + + val parent = getDescriptorByFqNameIndex(module, fqName) + val typeParameters = if (parent is DeserializedClassDescriptor) { + parent.classProto.typeParameterList + } else listOf() + + val childContext = context.childContext(parent, typeParameters, nameResolver, typeTable) + return MemberDeserializer(childContext) + + } + + fun deserializeFunction(proto: ProtoBuf.Function): FunctionDescriptor { + val containingFqName = proto.getExtension(KonanLinkData.functionParent) + // TODO: learn to take the containing IR declaration + val memberDeserializer = if (containingFqName == -1) + this.memberDeserializer + else + memberDeserializerByParentFqNameIndex(containingFqName) + + val function = memberDeserializer.loadFunction(proto) + + return function + } + + fun deserializeConstructor(proto: ProtoBuf.Constructor): ConstructorDescriptor { + + val containingFqName = proto.getExtension(KonanLinkData.constructorParent) + val memberDeserializer = memberDeserializerByParentFqNameIndex(containingFqName) + val isPrimary = !Flags.IS_SECONDARY.get(proto.flags) + val constructor = memberDeserializer.loadConstructor(proto, isPrimary) + + return constructor + } + + + fun deserializeProperty(proto: ProtoBuf.Property): VariableDescriptor { + val containingFqName = proto.getExtension(KonanLinkData.propertyParent) + + // TODO: learn to take the containing IR declaration + val memberDeserializer = if (containingFqName == -1) + this.memberDeserializer + else + memberDeserializerByParentFqNameIndex(containingFqName) + + val property = memberDeserializer.loadProperty(proto) + + return if (proto.getExtension(KonanLinkData.usedAsVariable)) { + propertyToVariable(property) + } else { + property + } + } + + fun propertyToVariable(property: PropertyDescriptor): LocalVariableDescriptor { + val variable = LocalVariableDescriptor( + property.containingDeclaration, + property.annotations, + property.name, + property.type, + property.isVar, + property.isDelegated, + SourceElement.NO_SOURCE) + + // TODO: Should we transform the getter and the setter too? + return variable + } +} + + diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/SerializeIr.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/SerializeIr.kt new file mode 100644 index 00000000000..15a63418a1c --- /dev/null +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/SerializeIr.kt @@ -0,0 +1,934 @@ +package org.jetbrains.kotlin.backend.konan.serialization + +import org.jetbrains.kotlin.backend.konan.Context +import org.jetbrains.kotlin.backend.konan.descriptors.getMemberScope +import org.jetbrains.kotlin.backend.konan.ir.ir2string +import org.jetbrains.kotlin.backend.konan.ir.ir2stringWhole +import org.jetbrains.kotlin.backend.konan.llvm.base64Decode +import org.jetbrains.kotlin.backend.konan.llvm.base64Encode +import org.jetbrains.kotlin.descriptors.* +import org.jetbrains.kotlin.incremental.components.NoLookupLocation +import org.jetbrains.kotlin.ir.IrElement +import org.jetbrains.kotlin.ir.IrStatement +import org.jetbrains.kotlin.ir.declarations.* +import org.jetbrains.kotlin.ir.declarations.IrDeclarationOrigin.DEFINED +import org.jetbrains.kotlin.ir.declarations.impl.IrClassImpl +import org.jetbrains.kotlin.ir.declarations.impl.IrFunctionImpl +import org.jetbrains.kotlin.ir.declarations.impl.IrVariableImpl +import org.jetbrains.kotlin.ir.expressions.* +import org.jetbrains.kotlin.ir.expressions.impl.* +import org.jetbrains.kotlin.serialization.KonanIr +import org.jetbrains.kotlin.serialization.KonanLinkData +import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedSimpleFunctionDescriptor +import org.jetbrains.kotlin.types.KotlinType + +internal class IrSerializer(val context: Context, + val descriptorTable: DescriptorTable, + val stringTable: KonanStringTable, + val util: KonanSerializationUtil, + var rootFunction: FunctionDescriptor) { + + val loopIndex = mutableMapOf() + var currentLoopIndex = 0 + val irDescriptorSerializer = IrDescriptorSerializer(context, + descriptorTable, stringTable, util.typeSerializer, rootFunction) + + fun serializeInlineBody(): String { + val declaration = context.ir.originalModuleIndex.functions[rootFunction]!! + context.log("INLINE: ${ir2stringWhole(declaration)}") + return encodeDeclaration(declaration as IrFunction) + } + + fun serializeKotlinType(type: KotlinType): KonanIr.KotlinType { + context.log("### serializing KotlinType: " + type) + return irDescriptorSerializer.serializeKotlinType(type) + } + + fun serializeDescriptor(descriptor: DeclarationDescriptor): KonanIr.KotlinDescriptor { + context.log("### serializeDescriptor $descriptor") + + // Behind this call starts a large world of + // descriptor serialization for IR. + return irDescriptorSerializer.serializeDescriptor(descriptor) + } + + fun serializeCoordinates(start: Int, end: Int): KonanIr.Coordinates { + val proto = KonanIr.Coordinates.newBuilder() + .setStartOffset(start) + .setEndOffset(end) + .build() + return proto + } + + fun serializeTypeMap(typeArguments: Map): KonanIr.TypeMap { + val proto = KonanIr.TypeMap.newBuilder() + typeArguments.forEach { key, value -> + val pair = KonanIr.TypeMap.Pair.newBuilder() + .setDescriptor(serializeDescriptor(key)) + .setType(serializeKotlinType(value)) + .build() + proto.addPair(pair) + } + return proto.build() + } + + /* -------------------------------------------------------------------------- */ + + fun serializeBlockBody(expression: IrBlockBody): KonanIr.IrBlockBody { + val proto = KonanIr.IrBlockBody.newBuilder() + expression.statements.forEach { + proto.addStatement(serializeStatement(it)) + } + return proto.build() + } + + fun serializeBranch(branch: IrBranch): KonanIr.IrBranch { + val proto = KonanIr.IrBranch.newBuilder() + + proto.setCondition(serializeExpression(branch.condition)) + proto.setResult(serializeExpression(branch.result)) + + return proto.build() + } + + fun serializeBlock(block: IrBlock): KonanIr.IrBlock { + val proto = KonanIr.IrBlock.newBuilder() + .setIsTransparentScope(block.isTransparentScope) + block.statements.forEach { + proto.addStatement(serializeStatement(it)) + } + return proto.build() + } + + fun serializeCatch(catch: IrCatch): KonanIr.IrCatch { + val proto = KonanIr.IrCatch.newBuilder() + .setParameter(serializeDescriptor(catch.parameter)) + .setResult(serializeExpression(catch.result)) + return proto.build() + } + + fun serializeStringConcat(expression: IrStringConcatenation): KonanIr.IrStringConcat { + val proto = KonanIr.IrStringConcat.newBuilder() + expression.arguments.forEach { + proto.addArgument(serializeExpression(it)) + } + return proto.build() + } + + fun irCallToPrimitiveKind(call: IrCall): KonanIr.IrCall.Primitive { + return when (call) { + is IrNullaryPrimitiveImpl + -> return KonanIr.IrCall.Primitive.NULLARY + is IrUnaryPrimitiveImpl + -> return KonanIr.IrCall.Primitive.UNARY + is IrBinaryPrimitiveImpl + -> return KonanIr.IrCall.Primitive.BINARY + else + -> return KonanIr.IrCall.Primitive.NOT_PRIMITIVE + } + } + + fun serializeCall(call: IrCall): KonanIr.IrCall { + val proto = KonanIr.IrCall.newBuilder() + + proto.setKind(irCallToPrimitiveKind(call)) + proto.setDescriptor(serializeDescriptor(call.descriptor)) + + if (call.extensionReceiver != null) { + proto.setExtensionReceiver(serializeExpression(call.extensionReceiver!!)) + } + + if (call.dispatchReceiver != null) { + proto.setDispatchReceiver(serializeExpression(call.dispatchReceiver!!)) + } + + if (call.superQualifier != null) { + proto.setSuper(serializeDescriptor(call.superQualifier!!)) + } + + val typeMap = mutableMapOf() + call.descriptor.original.typeParameters.forEach { + val type = call.getTypeArgument(it) + if (type != null) typeMap.put(it, type) + } + proto.setTypeMap(serializeTypeMap(typeMap)) + + call.descriptor.valueParameters.forEach { + val actual = call.getValueArgument(it.index) + if (actual == null) { + // Am I observing an IR generation regtession? + // I see a lack of arg for an empty vararg, + // rather than an empty vararg node. + assert(it.varargElementType != null) + } else { + val arg = actual!! + val argProto = serializeExpression(arg) + proto.addValueArgument(argProto) + } + } + + return proto.build() + } + + fun serializeCallableReference(callable: IrCallableReference): KonanIr.IrCallableReference { + val proto = KonanIr.IrCallableReference.newBuilder() + .setDescriptor(serializeDescriptor(callable.descriptor)) + + val typeMap = mutableMapOf() + callable.descriptor.original.typeParameters.forEach { + val type = callable.getTypeArgument(it) + if (type != null) { + typeMap.put(it, type) + } + } + proto.setTypeMap(serializeTypeMap(typeMap)) + return proto.build() + } + + fun serializeConst(value: IrConst<*>): KonanIr.IrConst { + val proto = KonanIr.IrConst.newBuilder() + when (value.kind) { + IrConstKind.Null -> proto.setNull(true) + IrConstKind.Boolean -> proto.setBoolean(value.value as Boolean) + IrConstKind.Byte -> proto.setByte((value.value as Byte).toInt()) + IrConstKind.Int -> proto.setInt(value.value as Int) + IrConstKind.Long -> proto.setLong(value.value as Long) + IrConstKind.String -> proto.setString(value.value as String) + IrConstKind.Float -> proto.setFloat(value.value as Float) + IrConstKind.Double -> proto.setDouble(value.value as Double) + else -> { + TODO("Const type serialization not implemented yet: ${ir2string(value)}") + } + } + return proto.build() + } + + fun serializeDelegatingConstructorCall(call: IrDelegatingConstructorCall): KonanIr.IrDelegatingConstructorCall { + val proto = KonanIr.IrDelegatingConstructorCall.newBuilder() + + proto.setDescriptor(serializeDescriptor(call.descriptor)) + + val typeMap = mutableMapOf() + call.descriptor.original.typeParameters.forEach { + val type = call.getTypeArgument(it) + if (type != null) typeMap.put(it, type) + } + proto.setTypeMap(serializeTypeMap(typeMap)) + + return proto.build() + } + + fun serializeGetValue(expression: IrGetValue): KonanIr.IrGetValue { + val proto = KonanIr.IrGetValue.newBuilder() + .setDescriptor(serializeDescriptor(expression.descriptor)) + return proto.build() + } + + fun serializeGetObject(expression: IrGetObjectValue): KonanIr.IrGetObject { + val proto = KonanIr.IrGetObject.newBuilder() + .setDescriptor(serializeDescriptor(expression.descriptor)) + return proto.build() + } + + fun serializeInstanceInitializerCall(call: IrInstanceInitializerCall): KonanIr.IrInstanceInitializerCall { + val proto = KonanIr.IrInstanceInitializerCall.newBuilder() + + proto.setDescriptor(serializeDescriptor(call.classDescriptor)) + + return proto.build() + } + + fun serializeReturn(expression: IrReturn): KonanIr.IrReturn { + val proto = KonanIr.IrReturn.newBuilder() + .setReturnTarget(serializeDescriptor(expression.returnTarget)) + .setValue(serializeExpression(expression.value)) + return proto.build() + } + + fun serializeSetVariable(expression: IrSetVariable): KonanIr.IrSetVariable { + val proto = KonanIr.IrSetVariable.newBuilder() + .setDescriptor(serializeDescriptor(expression.descriptor)) + .setValue(serializeExpression(expression.value)) + return proto.build() + } + + fun serializeThrow(expression: IrThrow): KonanIr.IrThrow { + val proto = KonanIr.IrThrow.newBuilder() + .setValue(serializeExpression(expression.value)) + return proto.build() + } + + fun serializeTry(expression: IrTry): KonanIr.IrTry { + val proto = KonanIr.IrTry.newBuilder() + .setResult(serializeExpression(expression.tryResult)) + val catchList = expression.catches + catchList.forEach { + proto.addCatch(serializeStatement(it)) + } + val finallyExpression = expression.finallyExpression + if (finallyExpression != null) { + proto.setFinally(serializeExpression(finallyExpression)) + } + return proto.build() + } + + fun serializeTypeOperator(operator: IrTypeOperator): KonanIr.IrTypeOperator { + when (operator) { + IrTypeOperator.CAST + -> return KonanIr.IrTypeOperator.CAST + IrTypeOperator.IMPLICIT_CAST + -> return KonanIr.IrTypeOperator.IMPLICIT_CAST + IrTypeOperator.IMPLICIT_NOTNULL + -> return KonanIr.IrTypeOperator.IMPLICIT_NOTNULL + IrTypeOperator.IMPLICIT_COERCION_TO_UNIT + -> return KonanIr.IrTypeOperator.IMPLICIT_COERCION_TO_UNIT + IrTypeOperator.SAFE_CAST + -> return KonanIr.IrTypeOperator.SAFE_CAST + IrTypeOperator.INSTANCEOF + -> return KonanIr.IrTypeOperator.INSTANCEOF + IrTypeOperator.NOT_INSTANCEOF + -> return KonanIr.IrTypeOperator.NOT_INSTANCEOF + else -> error("Unknown type operator") + } + } + + fun serializeTypeOp(expression: IrTypeOperatorCall): KonanIr.IrTypeOp { + val proto = KonanIr.IrTypeOp.newBuilder() + .setOperator(serializeTypeOperator(expression.operator)) + .setOperand(serializeKotlinType(expression.typeOperand)) + .setArgument(serializeExpression(expression.argument)) + return proto.build() + + } + + fun serializeVararg(expression: IrVararg): KonanIr.IrVararg { + val proto = KonanIr.IrVararg.newBuilder() + .setElementType(serializeKotlinType(expression.varargElementType)) + return proto.build() + } + + fun serializeWhen(expression: IrWhen): KonanIr.IrWhen { + val proto = KonanIr.IrWhen.newBuilder() + + val branches = expression.branches + branches.forEach { + proto.addBranch(serializeStatement(it)) + } + + return proto.build() + } + + fun serializeWhile(expression: IrWhileLoop): KonanIr.IrWhile { + val proto = KonanIr.IrWhile.newBuilder() + .setCondition(serializeExpression(expression.condition)) + val label = expression.label + if (label != null) { + proto.setLabel(label) + } + + proto.setLoopId(currentLoopIndex) + loopIndex.put(expression, currentLoopIndex++) + + val body = expression.body + if (body != null) { + proto.setBody(serializeExpression(body)) + } + return proto.build() + + } + + fun serializeBreak(expression: IrBreak): KonanIr.IrBreak { + val proto = KonanIr.IrBreak.newBuilder() + val label = expression.label + if (label != null) { + proto.setLabel(label) + } + val loopId = loopIndex[expression.loop!!]!! + proto.setLoopId(loopId) + + return proto.build() + } + + fun serializeExpression(expression: IrExpression): KonanIr.IrExpression { + context.log("### serializing Expression: ${ir2string(expression)}") + + val coordinates = serializeCoordinates(expression.startOffset, expression.endOffset) + val proto = KonanIr.IrExpression.newBuilder() + .setType(serializeKotlinType(expression.type)) + .setCoordinates(coordinates) + + val operationProto = KonanIr.IrOperation.newBuilder() + + when (expression) { + is IrBlock -> operationProto.setBlock(serializeBlock(expression)) + is IrBreak -> operationProto.setBreak(serializeBreak(expression)) + is IrCall -> operationProto.setCall(serializeCall(expression)) + is IrCallableReference + -> operationProto.setCallableReference(serializeCallableReference(expression)) + is IrConst<*> -> operationProto.setConst(serializeConst(expression)) + is IrDelegatingConstructorCall + -> operationProto.setDelegatingConstructorCall(serializeDelegatingConstructorCall(expression)) + is IrGetValue -> operationProto.setGetValue(serializeGetValue(expression)) + is IrGetObjectValue + -> operationProto.setGetObject(serializeGetObject(expression)) + is IrInstanceInitializerCall + -> operationProto.setInstanceInitializerCall(serializeInstanceInitializerCall(expression)) + is IrReturn -> operationProto.setReturn(serializeReturn(expression)) + is IrSetVariable -> operationProto.setSetVariable(serializeSetVariable(expression)) + is IrStringConcatenation + -> operationProto.setStringConcat(serializeStringConcat(expression)) + is IrThrow -> operationProto.setThrow(serializeThrow(expression)) + is IrTry -> operationProto.setTry(serializeTry(expression)) + is IrTypeOperatorCall + -> operationProto.setTypeOp(serializeTypeOp(expression)) + is IrVararg -> operationProto.setVararg(serializeVararg(expression)) + is IrWhen -> operationProto.setWhen(serializeWhen(expression)) + is IrWhileLoop -> operationProto.setWhile(serializeWhile(expression)) + else -> { + TODO("Expression serialization not implemented yet: ${ir2string(expression)}.") + } + } + proto.setOperation(operationProto) + + return proto.build() + } + + fun serializeStatement(statement: IrElement): KonanIr.IrStatement { + context.log("### serializing Statement: ${ir2string(statement)}") + + val coordinates = serializeCoordinates(statement.startOffset, statement.endOffset) + val proto = KonanIr.IrStatement.newBuilder() + .setCoordinates(coordinates) + + when (statement) { + is IrDeclaration -> proto.setDeclaration(serializeDeclaration(statement)) + is IrExpression -> proto.setExpression(serializeExpression(statement)) + is IrBlockBody -> proto.setBlockBody(serializeBlockBody(statement)) + is IrBranch -> proto.setBranch(serializeBranch(statement)) + is IrCatch -> proto.setCatch(serializeCatch(statement)) + else -> { + TODO("Statement not implemented yet: ${ir2string(statement)}") + } + } + return proto.build() + } + + fun serializeFunction(function: IrFunction): KonanIr.IrFunc { + val proto = KonanIr.IrFunc.newBuilder() + val body = function.body + if (body != null) proto.setBody(serializeStatement(body)) + return proto.build() + } + + fun serializeVariable(variable: IrVariable): KonanIr.IrVar { + val proto = KonanIr.IrVar.newBuilder() + val initializer = variable.initializer + if (initializer != null) { + proto.setInitializer(serializeExpression(initializer)) + } + return proto.build() + } + + fun serializeIrClass(clazz: IrClass): KonanIr.IrClass { + val proto = KonanIr.IrClass.newBuilder() + val declarations = clazz.declarations + declarations.forEach { + proto.addMember(serializeDeclaration(it)) + } + return proto.build() + } + + fun serializeDeclaration(declaration: IrDeclaration): KonanIr.IrDeclaration { + context.log("### serializing Declaration: ${ir2string(declaration)}") + + val descriptor = declaration.descriptor + var kotlinDescriptor = serializeDescriptor(descriptor) + if (descriptor != rootFunction) { + val realDescriptor = util.serializeLocalDeclaration(descriptor) + val localDeclaration = KonanIr.LocalDeclaration + .newBuilder() + .setDescriptor(realDescriptor) + .build() + kotlinDescriptor = kotlinDescriptor + .toBuilder() + .setIrLocalDeclaration(localDeclaration) + .build() + } else if (descriptor is ClassDescriptor) { + // TODO + context.log("Can't serialize local class declarations in inline functions") + } + + val coordinates = serializeCoordinates(declaration.startOffset, declaration.endOffset) + val proto = KonanIr.IrDeclaration.newBuilder() + //.setKind(declaration.irKind()) + .setDescriptor(kotlinDescriptor) + .setCoordinates(coordinates) + + val declarator = KonanIr.IrDeclarator.newBuilder() + + when (declaration) { + is IrFunction + -> declarator.setFunction(serializeFunction(declaration)) + is IrVariable + -> declarator.setVariable(serializeVariable(declaration)) + is IrClass + -> declarator.setIrClass(serializeIrClass(declaration)) + else + -> { + TODO("Declaration serialization not supported yet.") + } + } + + proto.setDeclarator(declarator) + + return proto.build() + } + + fun encodeDeclaration(declaration: IrDeclaration): String { + val proto = serializeDeclaration(declaration) + val byteArray = proto.toByteArray() + val base64 = base64Encode(byteArray) + return base64 + } + + +} + +// --------- Deserializer part ----------------------------- + +internal class IrDeserializer(val context: Context, + val descriptorIndex: IrDeserializationDescriptorIndex, + val rootFunction: DeserializedSimpleFunctionDescriptor) { + + val loopIndex = mutableMapOf() + + val localDeserializer = LocalDeclarationDeserializer(rootFunction, context.moduleDescriptor!!) + + val descriptorDeserializer = IrDescriptorDeserializer( + context, descriptorIndex, rootFunction, localDeserializer) + + + fun deserializeKotlinType(proto: KonanIr.KotlinType) + = descriptorDeserializer.deserializeKotlinType(proto) + + fun deserializeDescriptor(proto: KonanIr.KotlinDescriptor) + = descriptorDeserializer.deserializeDescriptor(proto) + + fun deserializeTypeMap(descriptor: CallableDescriptor, proto: KonanIr.TypeMap): + Map { + val typeMap = mutableMapOf() + val pairProtos = proto.getPairList() + pairProtos.forEachIndexed { index, pair -> + val typeParameter = descriptor.original.typeParameters[index] + + typeMap.put(typeParameter, + deserializeKotlinType(pair.getType())) + } + context.log("### deserialized typeMap = $typeMap") + return typeMap + } + + fun deserializeBlockBody(proto: KonanIr.IrBlockBody, + start: Int, end: Int): IrBlockBody { + + val statements = mutableListOf() + + val statementProtos = proto.getStatementList() + statementProtos.forEach { + statements.add(deserializeStatement(it) as IrStatement) + } + + return IrBlockBodyImpl(start, end, statements) + } + + fun deserializeBranch(proto: KonanIr.IrBranch, start: Int, end: Int): IrBranch { + + val condition = deserializeExpression(proto.getCondition()) + val result = deserializeExpression(proto.getResult()) + + return IrBranchImpl(start, end, condition, result) + } + + fun deserializeCatch(proto: KonanIr.IrCatch, start: Int, end: Int): IrCatch { + val parameter = deserializeDescriptor(proto.getParameter()) as VariableDescriptor + val result = deserializeExpression(proto.getResult()) + + return IrCatchImpl(start, end, parameter, result) + } + + fun deserializeStatement(proto: KonanIr.IrStatement): IrElement { + val start = proto.getCoordinates().getStartOffset() + val end = proto.getCoordinates().getEndOffset() + //val statement = proto.getStatement() + val element = when { + proto.hasBlockBody() + -> deserializeBlockBody(proto.getBlockBody(), start, end) + proto.hasBranch() + -> deserializeBranch(proto.getBranch(), start, end) + proto.hasCatch() + -> deserializeCatch(proto.getCatch(), start, end) + proto.hasDeclaration() + -> deserializeDeclaration(proto.getDeclaration()) + proto.hasExpression() + -> deserializeExpression(proto.getExpression()) + else -> { + TODO("Statement deserialization not implemented}") + } + } + + context.log("Deserialized statement: ${ir2string(element)}") + + return element + } + + fun deserializeBlock(proto: KonanIr.IrBlock, start: Int, end: Int, type: KotlinType): IrBlock { + val statements = mutableListOf() + val statementProtos = proto.getStatementList() + statementProtos.forEach { + statements.add(deserializeStatement(it) as IrStatement) + } + + val block = IrBlockImpl(start, end, type, null, statements) + + // TODO: Need to set isTransparentScope somehow + return block + } + + fun deserializeCall(proto: KonanIr.IrCall, start: Int, end: Int, type: KotlinType): IrCall { + val descriptor = deserializeDescriptor(proto.getDescriptor()) as CallableDescriptor + + val typeArgs = deserializeTypeMap(descriptor, proto.getTypeMap()) + val superDescriptor = if (proto.hasSuper()) { + deserializeDescriptor(proto.getSuper()) as ClassDescriptor + } else null + + val call: IrCall = when (proto.kind) { + KonanIr.IrCall.Primitive.NOT_PRIMITIVE -> + // TODO: implement the last three args here. + IrCallImpl(start, end, type, descriptor, typeArgs , null, superDescriptor) + KonanIr.IrCall.Primitive.NULLARY -> + IrNullaryPrimitiveImpl(start, end, null, descriptor) + KonanIr.IrCall.Primitive.UNARY -> + IrUnaryPrimitiveImpl(start, end, null, descriptor) + KonanIr.IrCall.Primitive.BINARY -> + IrBinaryPrimitiveImpl(start, end, null, descriptor) + } + + descriptor.valueParameters.mapIndexed { i, valueParameterDescriptor -> + call.putValueArgument(i, deserializeExpression(proto.getValueArgument(i))) + } + + if (proto.hasDispatchReceiver()) { + call.dispatchReceiver = deserializeExpression(proto.dispatchReceiver) + } + if (proto.hasExtensionReceiver()) { + call.extensionReceiver = deserializeExpression(proto.extensionReceiver) + } + + return call + } + + fun deserializeCallableReference(proto: KonanIr.IrCallableReference, + start: Int, end: Int, type: KotlinType): IrCallableReference { + + val descriptor = deserializeDescriptor(proto.getDescriptor()) as CallableDescriptor + if (descriptor.typeParameters.isNotEmpty()) error("We can't deserialize typa arguments") + val typeMap = deserializeTypeMap(descriptor, proto.getTypeMap()) + + val callable = IrCallableReferenceImpl(start, end, type, descriptor, typeMap, null) + return callable + } + + fun deserializeDelegatingConstructorCall(proto: KonanIr.IrDelegatingConstructorCall, start: Int, end: Int, type: KotlinType): IrDelegatingConstructorCall { + val descriptor = deserializeDescriptor(proto.getDescriptor()) as ClassConstructorDescriptor + val typeArgs = deserializeTypeMap(descriptor, proto.getTypeMap()) + + // TODO: implement the last three args here. + val call = IrDelegatingConstructorCallImpl(start, end, descriptor, typeArgs) + + descriptor.valueParameters.mapIndexed { i, valueParameterDescriptor -> + call.putValueArgument(i, deserializeExpression(proto.getValueArgument(i))) + } + + return call + } + + fun deserializeGetValue(proto: KonanIr.IrGetValue, start: Int, end: Int, type: KotlinType): IrGetValue { + val descriptor = deserializeDescriptor(proto.descriptor) as ValueDescriptor + + // TODO: origin! + return IrGetValueImpl(start, end, descriptor, null) + } + + fun deserializeGetObject(proto: KonanIr.IrGetObject, start: Int, end: Int, type: KotlinType): IrGetObjectValue { + val descriptor = deserializeDescriptor(proto.descriptor) as ClassDescriptor + return IrGetObjectValueImpl(start, end, type, descriptor) + } + + fun deserializeInstanceInitializerCall(proto: KonanIr.IrInstanceInitializerCall, start: Int, end: Int, type: KotlinType): IrInstanceInitializerCall { + val descriptor = deserializeDescriptor(proto.getDescriptor()) as ClassDescriptor + + // TODO: implement the last three args here. + return IrInstanceInitializerCallImpl(start, end, descriptor) + } + + + fun deserializeReturn(proto: KonanIr.IrReturn, start: Int, end: Int, type: KotlinType): IrReturn { + val descriptor = + deserializeDescriptor(proto.getReturnTarget()) as CallableDescriptor + val value = deserializeExpression(proto.getValue()) + return IrReturnImpl(start, end, type, descriptor, value) + } + + fun deserializeSetVariable(proto: KonanIr.IrSetVariable, start: Int, end: Int, type: KotlinType): IrSetVariable { + val descriptor = deserializeDescriptor(proto.getDescriptor()) as VariableDescriptor + val value = deserializeExpression(proto.getValue()) + val setVar = IrSetVariableImpl(start, end, descriptor, null) + setVar.value = value + return setVar + } + + fun deserializeStringConcat(proto: KonanIr.IrStringConcat, start: Int, end: Int, type: KotlinType): IrStringConcatenation { + val argumentProtos = proto.getArgumentList() + val arguments = mutableListOf() + + argumentProtos.forEach { + arguments.add(deserializeExpression(it) as IrExpression) + } + return IrStringConcatenationImpl(start, end, type, arguments) + } + + fun deserializeThrow(proto: KonanIr.IrThrow, start: Int, end: Int, type: KotlinType): IrThrowImpl { + return IrThrowImpl(start, end, type, deserializeExpression(proto.getValue())) + } + + fun deserializeTry(proto: KonanIr.IrTry, start: Int, end: Int, type: KotlinType): IrTryImpl { + val result = deserializeExpression(proto.getResult()) + val catches = mutableListOf() + proto.getCatchList().forEach { + catches.add(deserializeStatement(it) as IrCatch) + } + val finallyExpression = + if (proto.hasFinally()) deserializeExpression(proto.getFinally()) else null + return IrTryImpl(start, end, type, result, catches, finallyExpression) + } + + fun deserializeTypeOperator(operator: KonanIr.IrTypeOperator): IrTypeOperator { + when (operator) { + KonanIr.IrTypeOperator.CAST + -> return IrTypeOperator.CAST + KonanIr.IrTypeOperator.IMPLICIT_CAST + -> return IrTypeOperator.IMPLICIT_CAST + KonanIr.IrTypeOperator.IMPLICIT_NOTNULL + -> return IrTypeOperator.IMPLICIT_NOTNULL + KonanIr.IrTypeOperator.IMPLICIT_COERCION_TO_UNIT + -> return IrTypeOperator.IMPLICIT_COERCION_TO_UNIT + KonanIr.IrTypeOperator.SAFE_CAST + -> return IrTypeOperator.SAFE_CAST + KonanIr.IrTypeOperator.INSTANCEOF + -> return IrTypeOperator.INSTANCEOF + KonanIr.IrTypeOperator.NOT_INSTANCEOF + -> return IrTypeOperator.NOT_INSTANCEOF + else -> error("Unknown type operator") + } + } + + fun deserializeTypeOp(proto: KonanIr.IrTypeOp, start: Int, end: Int, type: KotlinType) : IrTypeOperatorCall { + val operator = deserializeTypeOperator(proto.getOperator()) + val operand = deserializeKotlinType(proto.getOperand()) + val argument = deserializeExpression(proto.getArgument()) + return IrTypeOperatorCallImpl(start, end, type, operator, operand, argument) + } + + fun deserializeVararg(proto: KonanIr.IrVararg, start: Int, end: Int, type: KotlinType): IrVararg { + val elementType = deserializeKotlinType(proto.getElementType()) + return IrVarargImpl(start, end, type, elementType) + } + + fun deserializeWhen(proto: KonanIr.IrWhen, start: Int, end: Int, type: KotlinType): IrWhen { + val branches = mutableListOf() + + proto.getBranchList().forEach { + branches.add(deserializeStatement(it) as IrBranch) + } + + // TODO: provide some origin! + return IrWhenImpl(start, end, type, null, branches) + } + + fun deserializeWhile(proto: KonanIr.IrWhile, start: Int, end: Int, type: KotlinType): IrWhileLoop { + // we create the IrLoop before deserializing the body, so that + // IrBreak statements have something to put into 'loop' field. + val loop = IrWhileLoopImpl(start, end, type, null) + + val loopId = proto.getLoopId() + loopIndex.getOrPut(loopId){loop} + + val condition = deserializeExpression(proto.getCondition()) + val label = if (proto.hasLabel()) proto.getLabel() else null + val body = if (proto.hasBody()) deserializeExpression(proto.getBody()) else null + + loop.label = label + loop.condition = condition + loop.body = body + + return loop + } + + fun deserializeBreak(proto: KonanIr.IrBreak, start: Int, end: Int, type: KotlinType): IrBreak { + val label = if(proto.hasLabel()) proto.getLabel() else null + val loopId = proto.getLoopId() + val loop = loopIndex[loopId]!! + val irBreak = IrBreakImpl(start, end, type, loop) + irBreak.label = label + + return irBreak + } + + fun deserializeConst(proto: KonanIr.IrConst, start: Int, end: Int, type: KotlinType): IrExpression { + + when { + proto.hasNull() + -> return IrConstImpl.constNull(start, end, type) + proto.hasBoolean() + -> return IrConstImpl.boolean(start, end, type, proto.getBoolean()) + proto.hasInt() + -> return IrConstImpl.int(start, end, type, proto.getInt()) + proto.hasString() + -> return IrConstImpl.string(start, end, type, proto.getString()) + proto.hasDouble() + -> return IrConstImpl.double(start, end, type, proto.getDouble()) + else -> { + TODO("Not all const types have been implemented") + } + } + } + + fun deserializeOperation(proto: KonanIr.IrOperation, start: Int, end: Int, type: KotlinType): IrExpression { + when { + proto.hasBlock() + -> return deserializeBlock(proto.getBlock(), start, end, type) + proto.hasBreak() + -> return deserializeBreak(proto.getBreak(), start, end, type) + proto.hasCall() + -> return deserializeCall(proto.getCall(), start, end, type) + proto.hasCallableReference() + -> return deserializeCallableReference(proto.getCallableReference(), start, end, type) + proto.hasConst() + -> return deserializeConst(proto.getConst(), start, end, type) + proto.hasDelegatingConstructorCall() + -> return deserializeDelegatingConstructorCall(proto.getDelegatingConstructorCall(), start, end, type) + proto.hasGetValue() + -> return deserializeGetValue(proto.getGetValue(), start, end, type) + proto.hasGetObject() + -> return deserializeGetObject(proto.getGetObject(), start, end, type) + proto.hasInstanceInitializerCall() + -> return deserializeInstanceInitializerCall(proto.getInstanceInitializerCall(), start, end, type) + proto.hasReturn() + -> return deserializeReturn(proto.getReturn(), start, end, type) + proto.hasSetVariable() + -> return deserializeSetVariable(proto.getSetVariable(), start, end, type) + proto.hasStringConcat() + -> return deserializeStringConcat(proto.getStringConcat(), start, end, type) + proto.hasThrow() + -> return deserializeThrow(proto.getThrow(), start, end, type) + proto.hasTry() + -> return deserializeTry(proto.getTry(), start, end, type) + proto.hasTypeOp() + -> return deserializeTypeOp(proto.getTypeOp(), start, end, type) + proto.hasVararg() + -> return deserializeVararg(proto.getVararg(), start, end, type) + proto.hasWhen() + -> return deserializeWhen(proto.getWhen(), start, end, type) + proto.hasWhile() + -> return deserializeWhile(proto.getWhile(), start, end, type) + else -> { + TODO("Expression deserialization not implemented}") + } + } + } + + fun deserializeExpression(proto: KonanIr.IrExpression): IrExpression { + val start = proto.getCoordinates().getStartOffset() + val end = proto.getCoordinates().getEndOffset() + val type = deserializeKotlinType(proto.getType()) + val operation = proto.getOperation() + val expression = deserializeOperation(operation, start, end, type) + + context.log("Deserialized expression: ${ir2string(expression)}") + return expression + } + + fun deserializeIrClass(proto: KonanIr.IrClass, descriptor: ClassDescriptor, start: Int, end: Int, origin: IrDeclarationOrigin): IrClass { + val members = mutableListOf() + + proto.getMemberList().forEach { + members.add(deserializeDeclaration(it)) + } + + val clazz = IrClassImpl(start, end, origin, descriptor, members) + return clazz + + } + + fun deserializeIrFunction(proto: KonanIr.IrFunc, descriptor: FunctionDescriptor, + start: Int, end: Int, origin: IrDeclarationOrigin): IrFunction { + + val body = deserializeStatement(proto.getBody()) + val function = IrFunctionImpl(start, end, origin, + descriptor as FunctionDescriptor, body as IrBody) + + return function + } + + fun deserializeIrVariable(proto: KonanIr.IrVar, descriptor: VariableDescriptor, + start: Int, end: Int, origin: IrDeclarationOrigin): IrVariable { + + val initializer = deserializeExpression(proto.getInitializer()) + val variable = IrVariableImpl(start, end, origin, descriptor, initializer) + + return variable + } + + fun deserializeDeclaration(proto: KonanIr.IrDeclaration): IrDeclaration { + + val descriptor = deserializeDescriptor(proto.descriptor) + + val start = proto.getCoordinates().getStartOffset() + val end = proto.getCoordinates().getEndOffset() + val origin = DEFINED // TODO: retore the real origins + val declarator = proto.getDeclarator() + + val declaration = when { + declarator.hasIrClass() + -> deserializeIrClass(declarator.getIrClass(), + descriptor as ClassDescriptor, start, end, origin) + declarator.hasFunction() + -> deserializeIrFunction(declarator.getFunction(), + descriptor as FunctionDescriptor, start, end, origin) + declarator.hasVariable() + -> deserializeIrVariable(declarator.getVariable(), + descriptor as VariableDescriptor, start, end, origin) + else -> { + TODO("Declaration deserialization not implemented") + } + } + context.log("Deserialized declaration: ${ir2string(declaration)}") + return declaration + } + + fun decodeDeclaration(): IrDeclaration { + val proto = (rootFunction as DeserializedSimpleFunctionDescriptor).proto + + if (!proto.hasExtension(KonanLinkData.inlineIrBody)) { + error("$rootFunction doesn't have ir serialized.") + } + + val inlineProto = proto.getExtension(KonanLinkData.inlineIrBody) + val base64 = inlineProto.encodedIr + val byteArray = base64Decode(base64) + val irProto = KonanIr.IrDeclaration.parseFrom(byteArray) + return deserializeDeclaration(irProto) + } +} + diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/SerializerVisitor.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/SerializerVisitor.kt new file mode 100644 index 00000000000..d7f4137887d --- /dev/null +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/SerializerVisitor.kt @@ -0,0 +1,73 @@ +package org.jetbrains.kotlin.backend.konan.ir + +import org.jetbrains.kotlin.backend.konan.Context +import org.jetbrains.kotlin.descriptors.FunctionDescriptor +import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedSimpleFunctionDescriptor +import org.jetbrains.kotlin.descriptors.ModuleDescriptor +import org.jetbrains.kotlin.descriptors.DeclarationDescriptorVisitor +import org.jetbrains.kotlin.ir.IrElement +import org.jetbrains.kotlin.ir.declarations.* +import org.jetbrains.kotlin.ir.descriptors.IrBuiltIns +import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid +import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid +import org.jetbrains.kotlin.ir.visitors.acceptVoid +import org.jetbrains.kotlin.backend.konan.descriptors.EmptyDescriptorVisitorVoid +import org.jetbrains.kotlin.backend.konan.descriptors.DeepVisitor +import org.jetbrains.kotlin.backend.konan.serialization.* +import org.jetbrains.kotlin.backend.konan.llvm.base64Encode +import org.jetbrains.kotlin.backend.konan.llvm.base64Decode +import org.jetbrains.kotlin.backend.konan.PhaseManager +import org.jetbrains.kotlin.backend.konan.KonanPhase +import org.jetbrains.kotlin.serialization.MutableTypeTable +import org.jetbrains.kotlin.serialization.KonanLinkData +import org.jetbrains.kotlin.backend.common.validateIrFunction + +internal class DeserializerDriver(val context: Context) { + + val descriptorIndex = IrDeserializationDescriptorIndex(context.irBuiltIns) + + internal fun deserializeInlineBody(descriptor: FunctionDescriptor): IrDeclaration? { + if (!descriptor.isInline()) return null + + if (descriptor !is DeserializedSimpleFunctionDescriptor) { + return null + } + + var deserializedIr: IrDeclaration? = null + PhaseManager(context).phase(KonanPhase.DESERIALIZER) { + context.log("### IR deserialization attempt:\t$descriptor") + try { + deserializedIr = IrDeserializer(context, descriptorIndex, descriptor).decodeDeclaration() + context.log("${deserializedIr!!.descriptor}") + context.log(ir2stringWhole(deserializedIr!!)) + context.log("IR deserialization SUCCESS:\t$descriptor") + } catch(e: Throwable) { + context.log("IR deserialization FAILURE:\t$descriptor") + e.printStackTrace() + } + } + return deserializedIr + } + + internal fun dumpAllInlineBodies() { + if (! context.phase!!.verbose) return + context.log("Now deserializing all inlines for debugging purpose.") + context.moduleDescriptor!!.accept( + InlineBodiesPrinterVisitor(InlineBodyPrinter()), Unit) + } + + + inner class InlineBodiesPrinterVisitor(worker: EmptyDescriptorVisitorVoid): DeepVisitor(worker) { } + + inner class InlineBodyPrinter: EmptyDescriptorVisitorVoid() { + + override fun visitFunctionDescriptor(descriptor: FunctionDescriptor, data: Unit): Boolean { + if (descriptor is DeserializedSimpleFunctionDescriptor) { + this@DeserializerDriver.deserializeInlineBody(descriptor) + } + + return true + + } + } +} diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/StringTableUtil.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/StringTableUtil.kt new file mode 100644 index 00000000000..c82b07ecc64 --- /dev/null +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/StringTableUtil.kt @@ -0,0 +1,30 @@ +package org.jetbrains.kotlin.backend.konan.serialization + +import org.jetbrains.kotlin.descriptors.* +import org.jetbrains.kotlin.serialization.deserialization.findClassAcrossModuleDependencies +import org.jetbrains.kotlin.serialization.ProtoBuf +import org.jetbrains.kotlin.serialization.ProtoBuf.QualifiedNameTable.QualifiedName +import org.jetbrains.kotlin.serialization.deserialization.DeserializedPackageFragment +import org.jetbrains.kotlin.serialization.deserialization.NameResolver +import org.jetbrains.kotlin.serialization.deserialization.NameResolverImpl + +// TODO Come up with a better file name. + +internal fun NameResolverImpl.getDescriptorByFqNameIndex( + module: ModuleDescriptor, + nameTable: ProtoBuf.QualifiedNameTable, + fqNameIndex: Int): DeclarationDescriptor { + + val packageName = this.getPackageFqName(fqNameIndex) + // TODO: Here we are using internals of NameresolverImpl. + // Consider extending NameResolver. + val proto = nameTable.getQualifiedName(fqNameIndex) + when (proto.kind) { + QualifiedName.Kind.CLASS, + QualifiedName.Kind.LOCAL -> + return module.findClassAcrossModuleDependencies(this.getClassId(fqNameIndex))!! + QualifiedName.Kind.PACKAGE -> + return module.getPackage(packageName) + } +} + diff --git a/backend.native/tests/build.gradle b/backend.native/tests/build.gradle index f632d383737..c16e3142fd6 100644 --- a/backend.native/tests/build.gradle +++ b/backend.native/tests/build.gradle @@ -1532,6 +1532,15 @@ task inline23(type: RunKonanTest) { source = "codegen/inline/inline23.kt" } +task deserialized_inline0(type: RunKonanTest) { + source = "serialization/deserialized_inline0.kt" + flags = ["--enable", "deserializer"] +} +task deserialized_listof0(type: RunKonanTest) { + source = "serialization/deserialized_listof0.kt" + flags = ["--enable", "deserializer"] +} + kotlinNativeInterop { sysstat { pkg 'sysstat' diff --git a/backend.native/tests/serialization/deserialized_inline0.kt b/backend.native/tests/serialization/deserialized_inline0.kt new file mode 100644 index 00000000000..bb787e1c1c8 --- /dev/null +++ b/backend.native/tests/serialization/deserialized_inline0.kt @@ -0,0 +1,37 @@ + +fun inline_todo() { + try { + TODO("OK") + } catch (e: Throwable) { + println(e.message) + } +} + +fun inline_maxof() { + println(maxOf(10, 17)) + println(maxOf(17, 13)) + println(maxOf(17, 17)) +} + +fun inline_assert() { + //assert(true) +} + +fun inline_areEqual() { + val a = 17 + val b = "some string" + println(konan.internal.areEqual(a, 17)) + println(konan.internal.areEqual(a, a)) + println(konan.internal.areEqual(17, 17)) + println(konan.internal.areEqual(b, "some string")) + println(konan.internal.areEqual("some string", b)) + println(konan.internal.areEqual(b, b)) +} + +fun main(args: Array) { + inline_todo() + inline_assert() + inline_maxof() + inline_areEqual() +} + diff --git a/backend.native/tests/serialization/deserialized_listof0.kt b/backend.native/tests/serialization/deserialized_listof0.kt new file mode 100644 index 00000000000..ce5e429fb61 --- /dev/null +++ b/backend.native/tests/serialization/deserialized_listof0.kt @@ -0,0 +1,29 @@ + +fun test_arrayList() { + val l = listOf(1, 2, 3) + val m = listOf() + val n = l + m + println(n) +} + +fun test_arrayList2(x: T, y: T, z: T) { + val l = listOf(x, y, z) + val m = listOf() + val n = m + l + println(l) +} + +fun test_arrayList3() { + val l = listOf() + val m = listOf("a", "b", "c") + val n = l + m + println(n) +} + +fun main(args: Array) { + test_arrayList() + test_arrayList2(5, 6, 7) + test_arrayList2("a", "b", "c") + test_arrayList3() +} + diff --git a/buildSrc/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy b/buildSrc/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy index 125fa3358be..6f53e936787 100644 --- a/buildSrc/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy +++ b/buildSrc/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy @@ -23,6 +23,7 @@ abstract class KonanTest extends JavaExec { String testData = null int expectedExitStatus = 0 List arguments = null + List flags = null boolean enabled = true boolean run = true @@ -193,7 +194,7 @@ class TestFailedException extends RuntimeException { @ParallelizableTask class RunKonanTest extends KonanTest { void compileTest(List filesToCompile, String exe) { - runCompiler(filesToCompile, exe, []) + runCompiler(filesToCompile, exe, flags?:[]) } }