[IR SERIALIZATION] Support new deserializer scheme

(cherry picked from commit a4f7260ce49ed102be6006cfaf574e2a154875d9)

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt

@@ -53,7 +53,7 @@ import org.jetbrains.kotlin.backend.konan.llvm.coverage.CoverageManager
 import org.jetbrains.kotlin.ir.symbols.impl.IrTypeParameterSymbolImpl
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.konan.library.KonanLibraryLayout
-import org.jetbrains.kotlin.library.SerializedIr
+import org.jetbrains.kotlin.library.SerializedIrModule
 
 /**
  * Offset for synthetic elements created by lowerings and not attributable to other places in the source code.
@@ -270,7 +270,7 @@ internal class Context(config: KonanConfig) : KonanBackendContext(config) {
     // But we have to wait until the code generation phase,
     // to dump this information into generated file.
     var serializedMetadata: SerializedMetadata? = null
-    var serializedIr: SerializedIr? = null
+    var serializedIr: SerializedIrModule? = null
     var dataFlowGraph: ByteArray? = null
 
     val librariesWithDependencies by lazy {

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ToplevelPhases.kt

@@ -14,15 +14,23 @@ import org.jetbrains.kotlin.backend.konan.serialization.*
 import org.jetbrains.kotlin.cli.common.messages.AnalyzerWithCompilerReport
 import org.jetbrains.kotlin.config.CommonConfigurationKeys
 import org.jetbrains.kotlin.config.languageVersionSettings
+import org.jetbrains.kotlin.descriptors.ModuleDescriptor
 import org.jetbrains.kotlin.descriptors.konan.isKonanStdlib
+import org.jetbrains.kotlin.ir.IrElement
 import org.jetbrains.kotlin.ir.declarations.IrFile
 import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
+import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction
 import org.jetbrains.kotlin.ir.util.SymbolTable
+import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid
+import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid
+import org.jetbrains.kotlin.ir.visitors.acceptVoid
 import org.jetbrains.kotlin.ir.util.addChild
 import org.jetbrains.kotlin.ir.util.file
 import org.jetbrains.kotlin.konan.target.CompilerOutputKind
+import org.jetbrains.kotlin.library.KotlinLibrary
 import org.jetbrains.kotlin.psi2ir.Psi2IrConfiguration
 import org.jetbrains.kotlin.psi2ir.Psi2IrTranslator
+import org.jetbrains.kotlin.utils.DFS
 import java.util.Collections.emptySet
 
 internal fun moduleValidationCallback(state: ActionState, module: IrModuleFragment, context: Context) {
@@ -152,7 +160,14 @@ internal val psiToIrPhase = konanUnitPhase(
                 val dependencies = moduleDescriptor.allDependencyModules.filter {
                     config.librariesWithDependencies(moduleDescriptor).contains(it.konanLibrary)
                 }
-                for (dependency in dependencies) {
+
+                fun sortDependencies(dependencies: List<ModuleDescriptor>): Collection<ModuleDescriptor> {
+                    return DFS.topologicalOrder(dependencies) {
+                        it.allDependencyModules
+                    }.reversed()
+                }
+
+                for (dependency in sortDependencies(dependencies)) {
                     deserializer.deserializeIrModuleHeader(dependency)
                 }
                 if (dependencies.size == dependenciesCount) break
@@ -214,9 +229,10 @@ internal val copyDefaultValuesToActualPhase = konanUnitPhase(
 
 internal val serializerPhase = konanUnitPhase(
         op = {
-            val declarationTable = KonanDeclarationTable(irModule!!.irBuiltins, DescriptorTable())
+            val descriptorTable = DescriptorTable()
-            serializedIr = KonanIrModuleSerializer(this, declarationTable).serializedIrModule(irModule!!)
+//            val declarationTable = KonanDeclarationTable(irModule!!.irBuiltins)
-            val serializer = KonanSerializationUtil(this, config.configuration.get(CommonConfigurationKeys.METADATA_VERSION)!!, declarationTable)
+            serializedIr = KonanIrModuleSerializer(this, irModule!!.irBuiltins, descriptorTable).serializedIrModule(irModule!!)
+            val serializer = KonanSerializationUtil(this, config.configuration.get(CommonConfigurationKeys.METADATA_VERSION)!!, descriptorTable)
             serializedMetadata = serializer.serializeModule(moduleDescriptor)
         },
         name = "Serializer",

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/library/impl/KonanLibraryWriterImpl.kt

@@ -6,7 +6,6 @@
 package org.jetbrains.kotlin.backend.konan.library.impl
 
 import llvm.LLVMModuleRef
-import org.jetbrains.kotlin.library.SerializedIr
 import org.jetbrains.kotlin.backend.konan.library.BitcodeWriter
 import org.jetbrains.kotlin.backend.konan.library.KonanLibraryWriter
 import org.jetbrains.kotlin.konan.file.File
@@ -15,10 +14,7 @@ import org.jetbrains.kotlin.konan.library.KonanLibraryLayout
 import org.jetbrains.kotlin.konan.properties.Properties
 import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.library.*
-import org.jetbrains.kotlin.library.impl.BaseWriterImpl
+import org.jetbrains.kotlin.library.impl.*
-import org.jetbrains.kotlin.library.impl.IrWriterImpl
-import org.jetbrains.kotlin.library.impl.KotlinLibraryLayoutForWriter
-import org.jetbrains.kotlin.library.impl.MetadataWriterImpl
 
 class KonanLibraryLayoutForWriter(
     override val libDir: File,
@@ -41,7 +37,7 @@ class KonanLibraryWriterImpl(
     base: BaseWriter = BaseWriterImpl(layout, moduleName, versions, nopack),
     bitcode: BitcodeWriter = BitcodeWriterImpl(layout),
     metadata: MetadataWriter = MetadataWriterImpl(layout),
-    ir: IrWriter = IrWriterImpl(layout)
+    ir: IrWriter = IrMonoliticWriterImpl(layout)
 
 ) : BaseWriter by base, BitcodeWriter by bitcode, MetadataWriter by metadata, IrWriter by ir, KonanLibraryWriter
 
@@ -50,7 +46,7 @@ internal fun buildLibrary(
     included: List<String>,
     linkDependencies: List<KonanLibrary>,
     metadata: SerializedMetadata,
-    ir: SerializedIr,
+    ir: SerializedIrModule,
     versions: KonanLibraryVersioning,
     target: KonanTarget,
     output: String,

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanDeclarationTable.kt

@@ -1,17 +1,13 @@
 package org.jetbrains.kotlin.backend.konan.serialization
 
 import org.jetbrains.kotlin.backend.konan.llvm.*
-import org.jetbrains.kotlin.backend.common.serialization.DeclarationTable
+
-import org.jetbrains.kotlin.backend.common.serialization.DescriptorTable
+import org.jetbrains.kotlin.backend.common.serialization.GlobalDeclarationTable
+
 import org.jetbrains.kotlin.ir.descriptors.IrBuiltIns
 
-
+class KonanGlobalDeclarationTable(builtIns: IrBuiltIns) : GlobalDeclarationTable(KonanMangler) {
-// TODO: We don't manage id clashes anyhow now.
-class KonanDeclarationTable(builtIns: IrBuiltIns, descriptorTable: DescriptorTable):
-    DeclarationTable(builtIns, descriptorTable, KonanMangler) {
-    override var currentIndex = 0L
-
     init {
-        loadKnownBuiltins()
+        loadKnownBuiltins(builtIns, 0)
     }
-}
+}

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanDeserializeDescriptorReference.kt

@@ -8,6 +8,6 @@ class KonanDescriptorReferenceDeserializer(
     currentModule: ModuleDescriptor,
     mangler: KotlinMangler,
     builtIns: IrBuiltIns,
-    resolvedForwardDeclarations: MutableMap<UniqIdKey, UniqIdKey>
+    resolvedForwardDeclarations: MutableMap<UniqId, UniqId>
 ): DescriptorReferenceDeserializer(currentModule, mangler, builtIns, resolvedForwardDeclarations),
    DescriptorUniqIdAware by KonanDescriptorUniqIdAware

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanIrFileSerializer.kt

@@ -0,0 +1,22 @@
+package org.jetbrains.kotlin.backend.konan.serialization
+
+import org.jetbrains.kotlin.backend.common.LoggingContext
+import org.jetbrains.kotlin.backend.common.serialization.DeclarationTable
+import org.jetbrains.kotlin.backend.common.serialization.IrFileSerializer
+import org.jetbrains.kotlin.backend.konan.RuntimeNames
+import org.jetbrains.kotlin.ir.declarations.IrClass
+import org.jetbrains.kotlin.ir.declarations.IrFunction
+import org.jetbrains.kotlin.ir.util.hasAnnotation
+
+class KonanIrFileSerializer(
+        logger: LoggingContext,
+        declarationTable: DeclarationTable,
+        bodiesOnlyForInlines: Boolean = false
+): IrFileSerializer(logger, declarationTable, bodiesOnlyForInlines) {
+
+    override fun backendSpecificExplicitRoot(declaration: IrFunction) =
+            declaration.annotations.hasAnnotation(RuntimeNames.exportForCppRuntime)
+
+    override fun backendSpecificExplicitRoot(declaration: IrClass) =
+            declaration.annotations.hasAnnotation(RuntimeNames.exportTypeInfoAnnotation)
+}

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanIrModuleSerializer.kt

@@ -5,20 +5,25 @@ import org.jetbrains.kotlin.backend.common.serialization.IrModuleSerializer
 import org.jetbrains.kotlin.backend.konan.RuntimeNames
 import org.jetbrains.kotlin.backend.konan.llvm.KonanMangler
 import org.jetbrains.kotlin.backend.common.serialization.DeclarationTable
+import org.jetbrains.kotlin.backend.common.serialization.DescriptorTable
+import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsGlobalDeclarationTable
+import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrFileSerializer
 import org.jetbrains.kotlin.ir.declarations.IrClass
+import org.jetbrains.kotlin.ir.declarations.IrFile
 import org.jetbrains.kotlin.ir.declarations.IrFunction
+import org.jetbrains.kotlin.ir.descriptors.IrBuiltIns
 import org.jetbrains.kotlin.ir.util.hasAnnotation
 
 class KonanIrModuleSerializer(
     logger: LoggingContext,
-    declarationTable: DeclarationTable,
+    irBuiltIns: IrBuiltIns,
-    bodiesOnlyForInlines: Boolean = false
+    private val descriptorTable: DescriptorTable
-) : IrModuleSerializer(logger, declarationTable, KonanMangler, bodiesOnlyForInlines) {
+) : IrModuleSerializer<KonanIrFileSerializer>(logger) {
 
-    override fun backendSpecificExplicitRoot(declaration: IrFunction) =
-        declaration.annotations.hasAnnotation(RuntimeNames.exportForCppRuntime)
 
-    override fun backendSpecificExplicitRoot(declaration: IrClass) =
+    private val globalDeclarationTable = KonanGlobalDeclarationTable(irBuiltIns)
-        declaration.annotations.hasAnnotation(RuntimeNames.exportTypeInfoAnnotation)
+
+    override fun createSerializerForFile(file: IrFile): KonanIrFileSerializer =
+            KonanIrFileSerializer(logger, DeclarationTable(descriptorTable, globalDeclarationTable, 0))
 
 }

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanIrlinker.kt

@@ -19,52 +19,58 @@ package org.jetbrains.kotlin.backend.konan.serialization
 import org.jetbrains.kotlin.backend.common.LoggingContext
 import org.jetbrains.kotlin.backend.common.serialization.*
 import org.jetbrains.kotlin.backend.konan.descriptors.konanLibrary
+import org.jetbrains.kotlin.backend.konan.llvm.KonanMangler
+import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
-import org.jetbrains.kotlin.ir.declarations.IrFile
 import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
 import org.jetbrains.kotlin.ir.descriptors.IrBuiltIns
 import org.jetbrains.kotlin.ir.util.SymbolTable
 
 class KonanIrLinker(
-    currentModule: ModuleDescriptor,
+        currentModule: ModuleDescriptor,
-    logger: LoggingContext,
+        logger: LoggingContext,
-    builtIns: IrBuiltIns,
+        builtIns: IrBuiltIns,
-    symbolTable: SymbolTable,
+        symbolTable: SymbolTable,
-    forwardModuleDescriptor: ModuleDescriptor?,
+        forwardModuleDescriptor: ModuleDescriptor?,
-    exportedDependencies: List<ModuleDescriptor>
+        exportedDependencies: List<ModuleDescriptor>
 ) : KotlinIrLinker(logger, builtIns, symbolTable, exportedDependencies, forwardModuleDescriptor, 0L),
-    DescriptorUniqIdAware by KonanDescriptorUniqIdAware {
+        DescriptorUniqIdAware by KonanDescriptorUniqIdAware {
 
     override val descriptorReferenceDeserializer =
-        KonanDescriptorReferenceDeserializer(currentModule, KonanDeclarationTable(builtIns, DescriptorTable()), builtIns, resolvedForwardDeclarations)
+            KonanDescriptorReferenceDeserializer(currentModule, KonanMangler, builtIns, resolvedForwardDeclarations)
 
-    override fun reader(moduleDescriptor: ModuleDescriptor, uniqId: UniqId) =
+    override fun reader(moduleDescriptor: ModuleDescriptor, fileIndex: Int, uniqId: UniqId) =
-        moduleDescriptor.konanLibrary!!.irDeclaration(uniqId.index, uniqId.isLocal)
+            moduleDescriptor.konanLibrary!!.irDeclaration(uniqId.index, uniqId.isLocal, fileIndex)
 
-    override fun readSymbol(moduleDescriptor: ModuleDescriptor, symbolIndex: Int) =
+    override fun readSymbol(moduleDescriptor: ModuleDescriptor, fileIndex: Int, symbolIndex: Int) =
-            moduleDescriptor.konanLibrary!!.symbol(symbolIndex)
+            moduleDescriptor.konanLibrary!!.symbol(symbolIndex, fileIndex)
 
-    override fun readType(moduleDescriptor: ModuleDescriptor, typeIndex: Int) =
+    override fun readType(moduleDescriptor: ModuleDescriptor, fileIndex: Int, typeIndex: Int) =
-            moduleDescriptor.konanLibrary!!.type(typeIndex)
+            moduleDescriptor.konanLibrary!!.type(typeIndex, fileIndex)
 
-    override fun readString(moduleDescriptor: ModuleDescriptor, stringIndex: Int) =
+    override fun readString(moduleDescriptor: ModuleDescriptor, fileIndex: Int, stringIndex: Int) =
-            moduleDescriptor.konanLibrary!!.string(stringIndex)
+            moduleDescriptor.konanLibrary!!.string(stringIndex, fileIndex)
 
-    override val ModuleDescriptor.irHeader get() = this.konanLibrary!!.irHeader
+    override fun readBody(moduleDescriptor: ModuleDescriptor, fileIndex: Int, bodyIndex: Int) =
+            moduleDescriptor.konanLibrary!!.body(bodyIndex, fileIndex)
 
-    override fun List<IrFile>.handleClashes(uniqIdKey: UniqIdKey): IrFile {
+    override fun readFile(moduleDescriptor: ModuleDescriptor, fileIndex: Int) =
-        if (size == 1)
+            moduleDescriptor.konanLibrary!!.file(fileIndex)
-            return this[0]
+
-        assert(size != 0)
+    override fun readFileCount(moduleDescriptor: ModuleDescriptor) =
-        error("UniqId clash: ${uniqIdKey.uniqId.index}. Found in the " +
+            moduleDescriptor.run { if (isForwardDeclarationModule) 0 else konanLibrary!!.fileCount() }
-                "[${this.joinToString { it.packageFragmentDescriptor.containingDeclaration.userName }}]")
+
+    private val ModuleDescriptor.userName get() = konanLibrary!!.libraryFile.absolutePath
+
+    override fun checkAccessibility(declarationDescriptor: DeclarationDescriptor) = true
+
+    override fun handleNoModuleDeserializerFound(key: UniqId): DeserializationState {
+        return globalDeserializationState
     }
 
-    private val ModuleDescriptor.userName get() = konanLibrary?.libraryFile?.absolutePath ?: name.asString()
-
     val modules: Map<String, IrModuleFragment> get() = mutableMapOf<String, IrModuleFragment>().apply {
-        deserializersForModules.forEach {
+        deserializersForModules.filter { !it.key.isForwardDeclarationModule }.forEach {
             this.put(it.key.konanLibrary!!.libraryName, it.value.module)
         }
     }
-}
+}

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanSerializationUtil.kt

@@ -9,7 +9,7 @@ import org.jetbrains.kotlin.backend.konan.Context
 import org.jetbrains.kotlin.backend.konan.descriptors.isExpectMember
 import org.jetbrains.kotlin.backend.konan.descriptors.isSerializableExpectClass
 import org.jetbrains.kotlin.descriptors.*
-import org.jetbrains.kotlin.backend.common.serialization.DeclarationTable
+import org.jetbrains.kotlin.backend.common.serialization.DescriptorTable
 import org.jetbrains.kotlin.library.SerializedMetadata
 import org.jetbrains.kotlin.metadata.ProtoBuf
 import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion
@@ -34,7 +34,7 @@ import org.jetbrains.kotlin.serialization.konan.SourceFileMap
  * with MemberDeserializer class.
  */
 
-internal class KonanSerializationUtil(val context: Context, val metadataVersion: BinaryVersion, val declarationTable: DeclarationTable) {
+internal class KonanSerializationUtil(val context: Context, val metadataVersion: BinaryVersion, val descriptorTable: DescriptorTable) {
 
     lateinit var serializerContext: SerializerContext
 
@@ -46,7 +46,7 @@ internal class KonanSerializationUtil(val context: Context, val metadataVersion:
             var classSerializer: DescriptorSerializer = topSerializer
     )
     private fun createNewContext(): SerializerContext {
-        val extension = KonanSerializerExtension(context, metadataVersion, sourceFileMap, declarationTable)
+        val extension = KonanSerializerExtension(context, metadataVersion, sourceFileMap, descriptorTable)
         return SerializerContext(
                 extension,
                 DescriptorSerializer.createTopLevel(extension)
@@ -237,4 +237,4 @@ internal class KonanSerializationUtil(val context: Context, val metadataVersion:
 }
 
 private const val TOP_LEVEL_DECLARATION_COUNT_PER_FILE = 128
-private const val TOP_LEVEL_CLASS_DECLARATION_COUNT_PER_FILE = 64
+private const val TOP_LEVEL_CLASS_DECLARATION_COUNT_PER_FILE = 64

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanSerializerExtension.kt

@@ -10,6 +10,7 @@ import org.jetbrains.kotlin.config.LanguageFeature
 import org.jetbrains.kotlin.config.languageVersionSettings
 import org.jetbrains.kotlin.descriptors.*
 import org.jetbrains.kotlin.backend.common.serialization.DeclarationTable
+import org.jetbrains.kotlin.backend.common.serialization.DescriptorTable
 import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion
 import org.jetbrains.kotlin.metadata.ProtoBuf
 import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf
@@ -21,7 +22,7 @@ import org.jetbrains.kotlin.serialization.konan.SourceFileMap
 import org.jetbrains.kotlin.types.KotlinType
 
 internal class KonanSerializerExtension(val context: Context, override val metadataVersion: BinaryVersion,
-                                        val sourceFileMap: SourceFileMap, val declarationTable: DeclarationTable
+                                        val sourceFileMap: SourceFileMap, val descriptorTable: DescriptorTable
 ) :
         KotlinSerializerExtensionBase(KonanSerializerProtocol) {
 
@@ -29,7 +30,7 @@ internal class KonanSerializerExtension(val context: Context, override val metad
     override fun shouldUseTypeTable(): Boolean = true
 
     fun uniqId(descriptor: DeclarationDescriptor): KonanProtoBuf.DescriptorUniqId? {
-        val index = declarationTable.descriptorTable.get(descriptor)
+        val index = descriptorTable.get(descriptor)
         return index?.let { newKonanDescriptorUniqId(it) }
     }
 

shared/src/library/kotlin/org/jetbrains/kotlin/konan/library/impl/KonanLibraryImpl.kt

@@ -96,7 +96,7 @@ fun createKonanLibrary(
     val base = BaseKotlinLibraryImpl(baseAccess, isDefault)
     val targeted = TargetedLibraryImpl(targetedAccess, base)
     val metadata = MetadataLibraryImpl(metadataAccess)
-    val ir = IrLibraryImpl(irAccess)
+    val ir = IrMonoliticLibraryImpl(irAccess)
     val bitcode = BitcodeLibraryImpl(bitcodeAccess, targeted)
 
     return KonanLibraryImpl(targeted, metadata, ir, bitcode)