[IR SERIALIZATION] Support new deserializer scheme

(cherry picked from commit a4f7260ce49ed102be6006cfaf574e2a154875d9)
2019-08-15 17:21:04 +03:00
parent a345b3ce1c
commit d928eabeb1
11 changed files with 108 additions and 66 deletions
@@ -53,7 +53,7 @@ import org.jetbrains.kotlin.backend.konan.llvm.coverage.CoverageManager
 import org.jetbrains.kotlin.ir.symbols.impl.IrTypeParameterSymbolImpl
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.konan.library.KonanLibraryLayout
-import org.jetbrains.kotlin.library.SerializedIr
+import org.jetbrains.kotlin.library.SerializedIrModule

 /**
 * Offset for synthetic elements created by lowerings and not attributable to other places in the source code.
@@ -270,7 +270,7 @@ internal class Context(config: KonanConfig) : KonanBackendContext(config) {
    // But we have to wait until the code generation phase,
    // to dump this information into generated file.
    var serializedMetadata: SerializedMetadata? = null
-    var serializedIr: SerializedIr? = null
+    var serializedIr: SerializedIrModule? = null
    var dataFlowGraph: ByteArray? = null

    val librariesWithDependencies by lazy {
@@ -14,15 +14,23 @@ import org.jetbrains.kotlin.backend.konan.serialization.*
 import org.jetbrains.kotlin.cli.common.messages.AnalyzerWithCompilerReport
 import org.jetbrains.kotlin.config.CommonConfigurationKeys
 import org.jetbrains.kotlin.config.languageVersionSettings
+import org.jetbrains.kotlin.descriptors.ModuleDescriptor
 import org.jetbrains.kotlin.descriptors.konan.isKonanStdlib
+import org.jetbrains.kotlin.ir.IrElement
 import org.jetbrains.kotlin.ir.declarations.IrFile
 import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
+import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction
 import org.jetbrains.kotlin.ir.util.SymbolTable
+import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid
+import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid
+import org.jetbrains.kotlin.ir.visitors.acceptVoid
 import org.jetbrains.kotlin.ir.util.addChild
 import org.jetbrains.kotlin.ir.util.file
 import org.jetbrains.kotlin.konan.target.CompilerOutputKind
+import org.jetbrains.kotlin.library.KotlinLibrary
 import org.jetbrains.kotlin.psi2ir.Psi2IrConfiguration
 import org.jetbrains.kotlin.psi2ir.Psi2IrTranslator
+import org.jetbrains.kotlin.utils.DFS
 import java.util.Collections.emptySet

 internal fun moduleValidationCallback(state: ActionState, module: IrModuleFragment, context: Context) {
@@ -152,7 +160,14 @@ internal val psiToIrPhase = konanUnitPhase(
                val dependencies = moduleDescriptor.allDependencyModules.filter {
                    config.librariesWithDependencies(moduleDescriptor).contains(it.konanLibrary)
                }
-                for (dependency in dependencies) {
+
+                fun sortDependencies(dependencies: List<ModuleDescriptor>): Collection<ModuleDescriptor> {
+                    return DFS.topologicalOrder(dependencies) {
+                        it.allDependencyModules
+                    }.reversed()
+                }
+
+                for (dependency in sortDependencies(dependencies)) {
                    deserializer.deserializeIrModuleHeader(dependency)
                }
                if (dependencies.size == dependenciesCount) break
@@ -214,9 +229,10 @@ internal val copyDefaultValuesToActualPhase = konanUnitPhase(

 internal val serializerPhase = konanUnitPhase(
        op = {
-            val declarationTable = KonanDeclarationTable(irModule!!.irBuiltins, DescriptorTable())
-            serializedIr = KonanIrModuleSerializer(this, declarationTable).serializedIrModule(irModule!!)
-            val serializer = KonanSerializationUtil(this, config.configuration.get(CommonConfigurationKeys.METADATA_VERSION)!!, declarationTable)
+            val descriptorTable = DescriptorTable()
+//            val declarationTable = KonanDeclarationTable(irModule!!.irBuiltins)
+            serializedIr = KonanIrModuleSerializer(this, irModule!!.irBuiltins, descriptorTable).serializedIrModule(irModule!!)
+            val serializer = KonanSerializationUtil(this, config.configuration.get(CommonConfigurationKeys.METADATA_VERSION)!!, descriptorTable)
            serializedMetadata = serializer.serializeModule(moduleDescriptor)
        },
        name = "Serializer",
@@ -6,7 +6,6 @@
 package org.jetbrains.kotlin.backend.konan.library.impl

 import llvm.LLVMModuleRef
-import org.jetbrains.kotlin.library.SerializedIr
 import org.jetbrains.kotlin.backend.konan.library.BitcodeWriter
 import org.jetbrains.kotlin.backend.konan.library.KonanLibraryWriter
 import org.jetbrains.kotlin.konan.file.File
@@ -15,10 +14,7 @@ import org.jetbrains.kotlin.konan.library.KonanLibraryLayout
 import org.jetbrains.kotlin.konan.properties.Properties
 import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.library.*
-import org.jetbrains.kotlin.library.impl.BaseWriterImpl
-import org.jetbrains.kotlin.library.impl.IrWriterImpl
-import org.jetbrains.kotlin.library.impl.KotlinLibraryLayoutForWriter
-import org.jetbrains.kotlin.library.impl.MetadataWriterImpl
+import org.jetbrains.kotlin.library.impl.*

 class KonanLibraryLayoutForWriter(
    override val libDir: File,
@@ -41,7 +37,7 @@ class KonanLibraryWriterImpl(
    base: BaseWriter = BaseWriterImpl(layout, moduleName, versions, nopack),
    bitcode: BitcodeWriter = BitcodeWriterImpl(layout),
    metadata: MetadataWriter = MetadataWriterImpl(layout),
-    ir: IrWriter = IrWriterImpl(layout)
+    ir: IrWriter = IrMonoliticWriterImpl(layout)

 ) : BaseWriter by base, BitcodeWriter by bitcode, MetadataWriter by metadata, IrWriter by ir, KonanLibraryWriter

@@ -50,7 +46,7 @@ internal fun buildLibrary(
    included: List<String>,
    linkDependencies: List<KonanLibrary>,
    metadata: SerializedMetadata,
-    ir: SerializedIr,
+    ir: SerializedIrModule,
    versions: KonanLibraryVersioning,
    target: KonanTarget,
    output: String,
@@ -1,17 +1,13 @@
 package org.jetbrains.kotlin.backend.konan.serialization

 import org.jetbrains.kotlin.backend.konan.llvm.*
-import org.jetbrains.kotlin.backend.common.serialization.DeclarationTable
-import org.jetbrains.kotlin.backend.common.serialization.DescriptorTable
+
+import org.jetbrains.kotlin.backend.common.serialization.GlobalDeclarationTable
+
 import org.jetbrains.kotlin.ir.descriptors.IrBuiltIns

-
-// TODO: We don't manage id clashes anyhow now.
-class KonanDeclarationTable(builtIns: IrBuiltIns, descriptorTable: DescriptorTable):
-    DeclarationTable(builtIns, descriptorTable, KonanMangler) {
-    override var currentIndex = 0L
-
+class KonanGlobalDeclarationTable(builtIns: IrBuiltIns) : GlobalDeclarationTable(KonanMangler) {
    init {
-        loadKnownBuiltins()
+        loadKnownBuiltins(builtIns, 0)
    }
-}
+}
@@ -8,6 +8,6 @@ class KonanDescriptorReferenceDeserializer(
    currentModule: ModuleDescriptor,
    mangler: KotlinMangler,
    builtIns: IrBuiltIns,
-    resolvedForwardDeclarations: MutableMap<UniqIdKey, UniqIdKey>
+    resolvedForwardDeclarations: MutableMap<UniqId, UniqId>
 ): DescriptorReferenceDeserializer(currentModule, mangler, builtIns, resolvedForwardDeclarations),
   DescriptorUniqIdAware by KonanDescriptorUniqIdAware
@@ -0,0 +1,22 @@
+package org.jetbrains.kotlin.backend.konan.serialization
+
+import org.jetbrains.kotlin.backend.common.LoggingContext
+import org.jetbrains.kotlin.backend.common.serialization.DeclarationTable
+import org.jetbrains.kotlin.backend.common.serialization.IrFileSerializer
+import org.jetbrains.kotlin.backend.konan.RuntimeNames
+import org.jetbrains.kotlin.ir.declarations.IrClass
+import org.jetbrains.kotlin.ir.declarations.IrFunction
+import org.jetbrains.kotlin.ir.util.hasAnnotation
+
+class KonanIrFileSerializer(
+        logger: LoggingContext,
+        declarationTable: DeclarationTable,
+        bodiesOnlyForInlines: Boolean = false
+): IrFileSerializer(logger, declarationTable, bodiesOnlyForInlines) {
+
+    override fun backendSpecificExplicitRoot(declaration: IrFunction) =
+            declaration.annotations.hasAnnotation(RuntimeNames.exportForCppRuntime)
+
+    override fun backendSpecificExplicitRoot(declaration: IrClass) =
+            declaration.annotations.hasAnnotation(RuntimeNames.exportTypeInfoAnnotation)
+}
@@ -5,20 +5,25 @@ import org.jetbrains.kotlin.backend.common.serialization.IrModuleSerializer
 import org.jetbrains.kotlin.backend.konan.RuntimeNames
 import org.jetbrains.kotlin.backend.konan.llvm.KonanMangler
 import org.jetbrains.kotlin.backend.common.serialization.DeclarationTable
+import org.jetbrains.kotlin.backend.common.serialization.DescriptorTable
+import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsGlobalDeclarationTable
+import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrFileSerializer
 import org.jetbrains.kotlin.ir.declarations.IrClass
+import org.jetbrains.kotlin.ir.declarations.IrFile
 import org.jetbrains.kotlin.ir.declarations.IrFunction
+import org.jetbrains.kotlin.ir.descriptors.IrBuiltIns
 import org.jetbrains.kotlin.ir.util.hasAnnotation

 class KonanIrModuleSerializer(
    logger: LoggingContext,
-    declarationTable: DeclarationTable,
-    bodiesOnlyForInlines: Boolean = false
-) : IrModuleSerializer(logger, declarationTable, KonanMangler, bodiesOnlyForInlines) {
+    irBuiltIns: IrBuiltIns,
+    private val descriptorTable: DescriptorTable
+) : IrModuleSerializer<KonanIrFileSerializer>(logger) {

-    override fun backendSpecificExplicitRoot(declaration: IrFunction) =
-        declaration.annotations.hasAnnotation(RuntimeNames.exportForCppRuntime)

-    override fun backendSpecificExplicitRoot(declaration: IrClass) =
-        declaration.annotations.hasAnnotation(RuntimeNames.exportTypeInfoAnnotation)
+    private val globalDeclarationTable = KonanGlobalDeclarationTable(irBuiltIns)
+
+    override fun createSerializerForFile(file: IrFile): KonanIrFileSerializer =
+            KonanIrFileSerializer(logger, DeclarationTable(descriptorTable, globalDeclarationTable, 0))

 }
@@ -19,52 +19,58 @@ package org.jetbrains.kotlin.backend.konan.serialization
 import org.jetbrains.kotlin.backend.common.LoggingContext
 import org.jetbrains.kotlin.backend.common.serialization.*
 import org.jetbrains.kotlin.backend.konan.descriptors.konanLibrary
+import org.jetbrains.kotlin.backend.konan.llvm.KonanMangler
+import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
-import org.jetbrains.kotlin.ir.declarations.IrFile
 import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
 import org.jetbrains.kotlin.ir.descriptors.IrBuiltIns
 import org.jetbrains.kotlin.ir.util.SymbolTable

 class KonanIrLinker(
-    currentModule: ModuleDescriptor,
-    logger: LoggingContext,
-    builtIns: IrBuiltIns,
-    symbolTable: SymbolTable,
-    forwardModuleDescriptor: ModuleDescriptor?,
-    exportedDependencies: List<ModuleDescriptor>
+        currentModule: ModuleDescriptor,
+        logger: LoggingContext,
+        builtIns: IrBuiltIns,
+        symbolTable: SymbolTable,
+        forwardModuleDescriptor: ModuleDescriptor?,
+        exportedDependencies: List<ModuleDescriptor>
 ) : KotlinIrLinker(logger, builtIns, symbolTable, exportedDependencies, forwardModuleDescriptor, 0L),
-    DescriptorUniqIdAware by KonanDescriptorUniqIdAware {
+        DescriptorUniqIdAware by KonanDescriptorUniqIdAware {

    override val descriptorReferenceDeserializer =
-        KonanDescriptorReferenceDeserializer(currentModule, KonanDeclarationTable(builtIns, DescriptorTable()), builtIns, resolvedForwardDeclarations)
+            KonanDescriptorReferenceDeserializer(currentModule, KonanMangler, builtIns, resolvedForwardDeclarations)

-    override fun reader(moduleDescriptor: ModuleDescriptor, uniqId: UniqId) =
-        moduleDescriptor.konanLibrary!!.irDeclaration(uniqId.index, uniqId.isLocal)
+    override fun reader(moduleDescriptor: ModuleDescriptor, fileIndex: Int, uniqId: UniqId) =
+            moduleDescriptor.konanLibrary!!.irDeclaration(uniqId.index, uniqId.isLocal, fileIndex)

-    override fun readSymbol(moduleDescriptor: ModuleDescriptor, symbolIndex: Int) =
-            moduleDescriptor.konanLibrary!!.symbol(symbolIndex)
+    override fun readSymbol(moduleDescriptor: ModuleDescriptor, fileIndex: Int, symbolIndex: Int) =
+            moduleDescriptor.konanLibrary!!.symbol(symbolIndex, fileIndex)

-    override fun readType(moduleDescriptor: ModuleDescriptor, typeIndex: Int) =
-            moduleDescriptor.konanLibrary!!.type(typeIndex)
+    override fun readType(moduleDescriptor: ModuleDescriptor, fileIndex: Int, typeIndex: Int) =
+            moduleDescriptor.konanLibrary!!.type(typeIndex, fileIndex)

-    override fun readString(moduleDescriptor: ModuleDescriptor, stringIndex: Int) =
-            moduleDescriptor.konanLibrary!!.string(stringIndex)
+    override fun readString(moduleDescriptor: ModuleDescriptor, fileIndex: Int, stringIndex: Int) =
+            moduleDescriptor.konanLibrary!!.string(stringIndex, fileIndex)

-    override val ModuleDescriptor.irHeader get() = this.konanLibrary!!.irHeader
+    override fun readBody(moduleDescriptor: ModuleDescriptor, fileIndex: Int, bodyIndex: Int) =
+            moduleDescriptor.konanLibrary!!.body(bodyIndex, fileIndex)

-    override fun List<IrFile>.handleClashes(uniqIdKey: UniqIdKey): IrFile {
-        if (size == 1)
-            return this[0]
-        assert(size != 0)
-        error("UniqId clash: ${uniqIdKey.uniqId.index}. Found in the " +
-                "[${this.joinToString { it.packageFragmentDescriptor.containingDeclaration.userName }}]")
+    override fun readFile(moduleDescriptor: ModuleDescriptor, fileIndex: Int) =
+            moduleDescriptor.konanLibrary!!.file(fileIndex)
+
+    override fun readFileCount(moduleDescriptor: ModuleDescriptor) =
+            moduleDescriptor.run { if (isForwardDeclarationModule) 0 else konanLibrary!!.fileCount() }
+
+    private val ModuleDescriptor.userName get() = konanLibrary!!.libraryFile.absolutePath
+
+    override fun checkAccessibility(declarationDescriptor: DeclarationDescriptor) = true
+
+    override fun handleNoModuleDeserializerFound(key: UniqId): DeserializationState {
+        return globalDeserializationState
    }

-    private val ModuleDescriptor.userName get() = konanLibrary?.libraryFile?.absolutePath ?: name.asString()
-
    val modules: Map<String, IrModuleFragment> get() = mutableMapOf<String, IrModuleFragment>().apply {
-        deserializersForModules.forEach {
+        deserializersForModules.filter { !it.key.isForwardDeclarationModule }.forEach {
            this.put(it.key.konanLibrary!!.libraryName, it.value.module)
        }
    }
-}
+}
@@ -9,7 +9,7 @@ import org.jetbrains.kotlin.backend.konan.Context
 import org.jetbrains.kotlin.backend.konan.descriptors.isExpectMember
 import org.jetbrains.kotlin.backend.konan.descriptors.isSerializableExpectClass
 import org.jetbrains.kotlin.descriptors.*
-import org.jetbrains.kotlin.backend.common.serialization.DeclarationTable
+import org.jetbrains.kotlin.backend.common.serialization.DescriptorTable
 import org.jetbrains.kotlin.library.SerializedMetadata
 import org.jetbrains.kotlin.metadata.ProtoBuf
 import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion
@@ -34,7 +34,7 @@ import org.jetbrains.kotlin.serialization.konan.SourceFileMap
 * with MemberDeserializer class.
 */

-internal class KonanSerializationUtil(val context: Context, val metadataVersion: BinaryVersion, val declarationTable: DeclarationTable) {
+internal class KonanSerializationUtil(val context: Context, val metadataVersion: BinaryVersion, val descriptorTable: DescriptorTable) {

    lateinit var serializerContext: SerializerContext

@@ -46,7 +46,7 @@ internal class KonanSerializationUtil(val context: Context, val metadataVersion:
            var classSerializer: DescriptorSerializer = topSerializer
    )
    private fun createNewContext(): SerializerContext {
-        val extension = KonanSerializerExtension(context, metadataVersion, sourceFileMap, declarationTable)
+        val extension = KonanSerializerExtension(context, metadataVersion, sourceFileMap, descriptorTable)
        return SerializerContext(
                extension,
                DescriptorSerializer.createTopLevel(extension)
@@ -237,4 +237,4 @@ internal class KonanSerializationUtil(val context: Context, val metadataVersion:
 }

 private const val TOP_LEVEL_DECLARATION_COUNT_PER_FILE = 128
-private const val TOP_LEVEL_CLASS_DECLARATION_COUNT_PER_FILE = 64
+private const val TOP_LEVEL_CLASS_DECLARATION_COUNT_PER_FILE = 64
@@ -10,6 +10,7 @@ import org.jetbrains.kotlin.config.LanguageFeature
 import org.jetbrains.kotlin.config.languageVersionSettings
 import org.jetbrains.kotlin.descriptors.*
 import org.jetbrains.kotlin.backend.common.serialization.DeclarationTable
+import org.jetbrains.kotlin.backend.common.serialization.DescriptorTable
 import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion
 import org.jetbrains.kotlin.metadata.ProtoBuf
 import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf
@@ -21,7 +22,7 @@ import org.jetbrains.kotlin.serialization.konan.SourceFileMap
 import org.jetbrains.kotlin.types.KotlinType

 internal class KonanSerializerExtension(val context: Context, override val metadataVersion: BinaryVersion,
-                                        val sourceFileMap: SourceFileMap, val declarationTable: DeclarationTable
+                                        val sourceFileMap: SourceFileMap, val descriptorTable: DescriptorTable
 ) :
        KotlinSerializerExtensionBase(KonanSerializerProtocol) {

@@ -29,7 +30,7 @@ internal class KonanSerializerExtension(val context: Context, override val metad
    override fun shouldUseTypeTable(): Boolean = true

    fun uniqId(descriptor: DeclarationDescriptor): KonanProtoBuf.DescriptorUniqId? {
-        val index = declarationTable.descriptorTable.get(descriptor)
+        val index = descriptorTable.get(descriptor)
        return index?.let { newKonanDescriptorUniqId(it) }
    }

@@ -96,7 +96,7 @@ fun createKonanLibrary(
    val base = BaseKotlinLibraryImpl(baseAccess, isDefault)
    val targeted = TargetedLibraryImpl(targetedAccess, base)
    val metadata = MetadataLibraryImpl(metadataAccess)
-    val ir = IrLibraryImpl(irAccess)
+    val ir = IrMonoliticLibraryImpl(irAccess)
    val bitcode = BitcodeLibraryImpl(bitcodeAccess, targeted)

    return KonanLibraryImpl(targeted, metadata, ir, bitcode)