[IR SERIALIZATION] Make [de]serialization processes incremental

- Use local uniqID local indexation file-local
 - Rethink the way declaration is looked for
 - Store in public file index only publicly accessible symbols
 - refactor API
 - code clean up

compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/DeserializeDescriptorReference.kt

@@ -30,6 +30,9 @@ abstract class DescriptorReferenceDeserializer(
 
     protected open fun resolveSpecialDescriptor(fqn: FqName) = builtIns.builtIns.getBuiltInClassByFqName(fqn)
 
+    protected abstract fun resolveSpecialDescriptor(fqn: FqName): DeclarationDescriptor
+    abstract fun checkIfSpecialDescriptorId(id: Long): Boolean
+    protected abstract fun getDescriptorIdOrNull(descriptor: DeclarationDescriptor): Long?
     protected open fun checkIfSpecialDescriptorId(id: Long) = with(mangler) { id.isSpecial }
 
     protected open fun getDescriptorIdOrNull(descriptor: DeclarationDescriptor) =

compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/IrFileSerializer.kt

@@ -5,6 +5,7 @@
 
 package org.jetbrains.kotlin.backend.common.serialization
 
+//import org.jetbrains.kotlin.library.SerializedIrFile
 import org.jetbrains.kotlin.backend.common.LoggingContext
 import org.jetbrains.kotlin.backend.common.ir.ir2string
 import org.jetbrains.kotlin.descriptors.*
@@ -12,7 +13,6 @@ import org.jetbrains.kotlin.builtins.FunctionInterfacePackageFragment
 import org.jetbrains.kotlin.descriptors.ClassKind
 import org.jetbrains.kotlin.descriptors.ClassKind.*
 import org.jetbrains.kotlin.ir.IrElement
-import org.jetbrains.kotlin.ir.IrStatement
 import org.jetbrains.kotlin.ir.SourceManager
 import org.jetbrains.kotlin.ir.declarations.*
 import org.jetbrains.kotlin.ir.expressions.*
@@ -24,9 +24,9 @@ import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid
 import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid
 import org.jetbrains.kotlin.ir.visitors.acceptVoid
 import org.jetbrains.kotlin.library.IrIrSerializedIrFile
-import org.jetbrains.kotlin.library.SerializedIrModule
+import org.jetbrains.kotlin.library.impl.SerializedDeclaration
-//import org.jetbrains.kotlin.library.SerializedIrFile
+import org.jetbrains.kotlin.library.impl.SkippedDeclaration
-import org.jetbrains.kotlin.library.impl.*
+import org.jetbrains.kotlin.library.impl.TopLevelDeclaration
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.types.Variance
@@ -35,9 +35,9 @@ import org.jetbrains.kotlin.backend.common.serialization.proto.ClassKind as Prot
 import org.jetbrains.kotlin.backend.common.serialization.proto.Coordinates as ProtoCoordinates
 import org.jetbrains.kotlin.backend.common.serialization.proto.FieldAccessCommon as ProtoFieldAccessCommon
 import org.jetbrains.kotlin.backend.common.serialization.proto.FileEntry as ProtoFileEntry
+import org.jetbrains.kotlin.backend.common.serialization.proto.FqName as ProtoFqName
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrAnonymousInit as ProtoAnonymousInit
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrBlock as ProtoBlock
-import org.jetbrains.kotlin.backend.common.serialization.proto.IrDataIndex as ProtoBodyIndex
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrBlockBody as ProtoBlockBody
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrBranch as ProtoBranch
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrBreak as ProtoBreak
@@ -50,6 +50,10 @@ import org.jetbrains.kotlin.backend.common.serialization.proto.IrConst as ProtoC
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrConstructor as ProtoConstructor
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrConstructorCall as ProtoConstructorCall
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrContinue as ProtoContinue
+import org.jetbrains.kotlin.backend.common.serialization.proto.IrDataIndex as ProtoBodyIndex
+import org.jetbrains.kotlin.backend.common.serialization.proto.IrDataIndex as ProtoStringIndex
+import org.jetbrains.kotlin.backend.common.serialization.proto.IrDataIndex as ProtoSymbolIndex
+import org.jetbrains.kotlin.backend.common.serialization.proto.IrDataIndex as ProtoTypeIndex
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrDeclaration as ProtoDeclaration
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrDeclarationBase as ProtoDeclarationBase
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrDeclarationContainer as ProtoDeclarationContainer
@@ -77,7 +81,6 @@ import org.jetbrains.kotlin.backend.common.serialization.proto.IrGetValue as Pro
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrInstanceInitializerCall as ProtoInstanceInitializerCall
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrLocalDelegatedProperty as ProtoLocalDelegatedProperty
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrLocalDelegatedPropertyReference as ProtoLocalDelegatedPropertyReference
-import org.jetbrains.kotlin.backend.common.serialization.proto.IrModule as ProtoModule
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrOperation as ProtoOperation
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrProperty as ProtoProperty
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrPropertyReference as ProtoPropertyReference
@@ -90,7 +93,6 @@ import org.jetbrains.kotlin.backend.common.serialization.proto.IrStarProjection
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrStatement as ProtoStatement
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrStatementOrigin as ProtoStatementOrigin
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrStringConcat as ProtoStringConcat
-import org.jetbrains.kotlin.backend.common.serialization.proto.IrDataIndex as ProtoSymbolIndex
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrSymbolData as ProtoSymbolData
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrSymbolKind as ProtoSymbolKind
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrSyntheticBody as ProtoSyntheticBody
@@ -101,7 +103,6 @@ import org.jetbrains.kotlin.backend.common.serialization.proto.IrType as ProtoTy
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrTypeAbbreviation as ProtoTypeAbbreviation
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrTypeAlias as ProtoTypeAlias
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrTypeArgument as ProtoTypeArgument
-import org.jetbrains.kotlin.backend.common.serialization.proto.IrDataIndex as ProtoTypeIndex
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrTypeOp as ProtoTypeOp
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrTypeOperator as ProtoTypeOperator
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrTypeParameter as ProtoTypeParameter
@@ -118,50 +119,55 @@ import org.jetbrains.kotlin.backend.common.serialization.proto.Loop as ProtoLoop
 import org.jetbrains.kotlin.backend.common.serialization.proto.MemberAccessCommon as ProtoMemberAccessCommon
 import org.jetbrains.kotlin.backend.common.serialization.proto.ModalityKind as ProtoModalityKind
 import org.jetbrains.kotlin.backend.common.serialization.proto.NullableIrExpression as ProtoNullableIrExpression
-import org.jetbrains.kotlin.backend.common.serialization.proto.IrDataIndex as ProtoStringIndex
 import org.jetbrains.kotlin.backend.common.serialization.proto.TypeArguments as ProtoTypeArguments
 import org.jetbrains.kotlin.backend.common.serialization.proto.Visibility as ProtoVisibility
-import org.jetbrains.kotlin.backend.common.serialization.proto.FqName as ProtoFqName
 
 open class IrFileSerializer(
     val logger: LoggingContext,
-    val declarationTable: DeclarationTable,
+    private val declarationTable: DeclarationTable,
-    val mangler: KotlinMangler,
+    mangler: KotlinMangler,
-    val bodiesOnlyForInlines: Boolean = false
+    private val bodiesOnlyForInlines: Boolean = false
 ) {
 
     private val loopIndex = mutableMapOf<IrLoop, Int>()
     private var currentLoopIndex = 0
-    val descriptorReferenceSerializer = DescriptorReferenceSerializer(declarationTable, { serializeString(it) }, { serializeFqName(it) }, mangler)
 
-    // The same symbol can be used multiple times in a module
+    // The same symbol can be used multiple times in a file
     // so use this index to store symbol data only once.
-    var protoSymbolMap = mutableMapOf<IrSymbol, Int>()
+    private val protoSymbolMap = mutableMapOf<IrSymbol, Int>()
-    var protoSymbolArray = arrayListOf<ProtoSymbolData>()
+    private val protoSymbolArray = arrayListOf<ProtoSymbolData>()
 
-    // The same type can be used multiple times in a module
+    // The same type can be used multiple times in a file
     // so use this index to store type data only once.
-    var protoTypeMap = mutableMapOf<IrTypeKey, Int>()
+    private val protoTypeMap = mutableMapOf<IrTypeKey, Int>()
-    var protoTypeArray = arrayListOf<ProtoType>()
+    private val protoTypeArray = arrayListOf<ProtoType>()
 
-    var protoStringMap = mutableMapOf<String, Int>()
+    private val protoStringMap = mutableMapOf<String, Int>()
-    var protoStringArray = arrayListOf<String>()
+    private val protoStringArray = arrayListOf<String>()
 
-    var protoBodyArray = mutableListOf<XStatementOrExpression>()
+    private val protoBodyArray = mutableListOf<XStatementOrExpression>()
 
-    sealed class XStatementOrExpression() {
+    private val descriptorReferenceSerializer =
+        DescriptorReferenceSerializer(declarationTable, { serializeString(it) }, { serializeFqName(it) }, mangler)
+
+    sealed class XStatementOrExpression {
         abstract fun toByteArray(): ByteArray
 
-        open fun toProtoStatement(): ProtoStatement { error("It is not a ProtoStatement") }
+        open fun toProtoStatement(): ProtoStatement {
-        open fun toProtoExpression(): ProtoExpression { error("It is not a ProtoExpression") }
+            error("It is not a ProtoStatement")
+        }
 
-        class XStatement(private val proto: ProtoStatement): XStatementOrExpression() {
+        open fun toProtoExpression(): ProtoExpression {
-            override fun toByteArray() = proto.toByteArray()
+            error("It is not a ProtoExpression")
+        }
+
+        class XStatement(private val proto: ProtoStatement) : XStatementOrExpression() {
+            override fun toByteArray(): ByteArray = proto.toByteArray()
             override fun toProtoStatement() = proto
         }
 
-        class XExpression(private val proto: ProtoExpression): XStatementOrExpression() {
+        class XExpression(private val proto: ProtoExpression) : XStatementOrExpression() {
-            override fun toByteArray() = proto.toByteArray()
+            override fun toByteArray(): ByteArray = proto.toByteArray()
             override fun toProtoExpression() = proto
         }
     }
@@ -215,11 +221,11 @@ open class IrFileSerializer(
         return proto.build()
     }
 
-    fun serializeName(name: Name): ProtoStringIndex = serializeString(name.toString())
+    private fun serializeName(name: Name): ProtoStringIndex = serializeString(name.toString())
 
     /* ------- IrSymbols -------------------------------------------------------- */
 
-    fun protoSymbolKind(symbol: IrSymbol): ProtoSymbolKind = when (symbol) {
+    private fun protoSymbolKind(symbol: IrSymbol): ProtoSymbolKind = when (symbol) {
         is IrAnonymousInitializerSymbol ->
             ProtoSymbolKind.ANONYMOUS_INIT_SYMBOL
         is IrClassSymbol ->
@@ -256,7 +262,7 @@ open class IrFileSerializer(
             TODO("Unexpected symbol kind: $symbol")
     }
 
-    fun serializeIrSymbolData(symbol: IrSymbol): ProtoSymbolData {
+    private fun serializeIrSymbolData(symbol: IrSymbol): ProtoSymbolData {
 
         val declaration = symbol.owner as? IrDeclaration ?: error("Expected IrDeclaration: ${symbol.owner}")
 
@@ -284,10 +290,6 @@ open class IrFileSerializer(
             val symbolData = serializeIrSymbolData(symbol)
             protoSymbolArray.add(symbolData)
             protoSymbolArray.size - 1
-
-//            val symbolData = serializeIrSymbolData(symbol)
-//            val uniqId = symbolData.uniqId
-//            Pair(UniqId(uniqId.index, uniqId.isLocal), symbolData)
         }
         return proto.build()
     }
@@ -304,13 +306,13 @@ open class IrFileSerializer(
         return proto.build()
     }
 
-    fun serializeIrTypeVariance(variance: Variance) = when (variance) {
+    private fun serializeIrTypeVariance(variance: Variance) = when (variance) {
         Variance.IN_VARIANCE -> ProtoTypeVariance.IN
         Variance.OUT_VARIANCE -> ProtoTypeVariance.OUT
         Variance.INVARIANT -> ProtoTypeVariance.INV
     }
 
-    fun serializeAnnotations(annotations: List<IrConstructorCall>): ProtoAnnotations {
+    private fun serializeAnnotations(annotations: List<IrConstructorCall>): ProtoAnnotations {
         val proto = ProtoAnnotations.newBuilder()
         annotations.forEach {
             proto.addAnnotation(serializeConstructorCall(it))
@@ -318,7 +320,7 @@ open class IrFileSerializer(
         return proto.build()
     }
 
-    fun serializeFqName(fqName: FqName): ProtoFqName {
+    private fun serializeFqName(fqName: FqName): ProtoFqName {
         val proto = ProtoFqName.newBuilder()
         fqName.pathSegments().forEach {
             proto.addSegment(serializeString(it.identifier))
@@ -327,12 +329,12 @@ open class IrFileSerializer(
         return proto.build()
     }
 
-    fun serializeIrTypeProjection(argument: IrTypeProjection) = ProtoTypeProjection.newBuilder()
+    private fun serializeIrTypeProjection(argument: IrTypeProjection): ProtoTypeProjection = ProtoTypeProjection.newBuilder()
         .setVariance(serializeIrTypeVariance(argument.variance))
         .setType(serializeIrType(argument.type))
         .build()
 
-    fun serializeTypeArgument(argument: IrTypeArgument): ProtoTypeArgument {
+    private fun serializeTypeArgument(argument: IrTypeArgument): ProtoTypeArgument {
         val proto = ProtoTypeArgument.newBuilder()
         when (argument) {
             is IrStarProjection ->
@@ -345,7 +347,7 @@ open class IrFileSerializer(
         return proto.build()
     }
 
-    fun serializeSimpleType(type: IrSimpleType): ProtoSimpleType {
+    private fun serializeSimpleType(type: IrSimpleType): ProtoSimpleType {
         val proto = ProtoSimpleType.newBuilder()
             .setAnnotations(serializeAnnotations(type.annotations))
             .setClassifier(serializeIrSymbol(type.classifier))
@@ -370,11 +372,11 @@ open class IrFileSerializer(
         return proto.build()
     }
 
-    fun serializeDynamicType(type: IrDynamicType) = ProtoDynamicType.newBuilder()
+    private fun serializeDynamicType(type: IrDynamicType): ProtoDynamicType = ProtoDynamicType.newBuilder()
         .setAnnotations(serializeAnnotations(type.annotations))
         .build()
 
-    fun serializeErrorType(type: IrErrorType) = ProtoErrorType.newBuilder()
+    private fun serializeErrorType(type: IrErrorType): ProtoErrorType = ProtoErrorType.newBuilder()
         .setAnnotations(serializeAnnotations(type.annotations))
         .build()
 
@@ -419,7 +421,7 @@ open class IrFileSerializer(
         val type: IrTypeKey?
     )
 
-    val IrType.toIrTypeKey: IrTypeKey
+    private val IrType.toIrTypeKey: IrTypeKey
         get() = IrTypeKey(
             kind = when (this) {
                 is IrSimpleType -> IrTypeKind.SIMPLE
@@ -433,7 +435,7 @@ open class IrFileSerializer(
             annotations = this.annotations
         )
 
-    val IrTypeArgument.toIrTypeArgumentKey: IrTypeArgumentKey
+    private val IrTypeArgument.toIrTypeArgumentKey: IrTypeArgumentKey
         get() = IrTypeArgumentKey(
             kind = when (this) {
                 is IrStarProjection -> IrTypeArgumentKind.STAR
@@ -444,7 +446,7 @@ open class IrFileSerializer(
             type = (this as? IrTypeProjection)?.type?.toIrTypeKey
         )
 
-    fun serializeIrType(type: IrType): ProtoTypeIndex {
+    private fun serializeIrType(type: IrType): ProtoTypeIndex {
         val proto = ProtoTypeIndex.newBuilder()
         val key = type.toIrTypeKey
         proto.index = protoTypeMap.getOrPut(key) {
@@ -522,7 +524,7 @@ open class IrFileSerializer(
         }
         proto.typeArguments = serializeTypeArguments(call)
 
-        for (index in 0..call.valueArgumentsCount - 1) {
+        for (index in 0 until call.valueArgumentsCount) {
             val actual = call.getValueArgument(index)
             val argOrNull = ProtoNullableIrExpression.newBuilder()
             if (actual == null) {
@@ -530,7 +532,7 @@ open class IrFileSerializer(
                 // I see a lack of arg for an empty vararg,
                 // rather than an empty vararg node.
 
-                // TODO: how do we assert that without descriptora?
+                // TODO: how do we assert that without descriptor?
                 //assert(it.varargElementType != null || it.hasDefaultValue())
             } else {
                 argOrNull.expression = serializeExpression(actual)
@@ -637,7 +639,7 @@ open class IrFileSerializer(
             .setLoop(serializeLoop(expression))
             .build()
 
-    fun serializeEnumConstructorCall(call: IrEnumConstructorCall): ProtoEnumConstructorCall {
+    private fun serializeEnumConstructorCall(call: IrEnumConstructorCall): ProtoEnumConstructorCall {
         val proto = ProtoEnumConstructorCall.newBuilder()
             .setSymbol(serializeIrSymbol(call.symbol))
             .setMemberAccess(serializeMemberAccessCommon(call))
@@ -667,14 +669,14 @@ open class IrFileSerializer(
     private fun serializeGetField(expression: IrGetField): ProtoGetField =
         ProtoGetField.newBuilder()
             .setFieldAccess(serializeFieldAccessCommon(expression)).apply {
-                expression.origin?.let { setOrigin(serializeIrStatementOrigin(it)) }
+                expression.origin?.let { origin = serializeIrStatementOrigin(it) }
             }
             .build()
 
     private fun serializeGetValue(expression: IrGetValue): ProtoGetValue =
         ProtoGetValue.newBuilder()
             .setSymbol(serializeIrSymbol(expression.symbol)).apply {
-                expression.origin?.let { setOrigin(serializeIrStatementOrigin(it)) }
+                expression.origin?.let { origin = serializeIrStatementOrigin(it) }
             }
             .build()
 
@@ -703,7 +705,7 @@ open class IrFileSerializer(
         ProtoSetField.newBuilder()
             .setFieldAccess(serializeFieldAccessCommon(expression))
             .setValue(serializeExpression(expression.value)).apply {
-                expression.origin?.let { setOrigin(serializeIrStatementOrigin(it)) }
+                expression.origin?.let { origin = serializeIrStatementOrigin(it) }
             }
             .build()
 
@@ -711,7 +713,7 @@ open class IrFileSerializer(
         ProtoSetVariable.newBuilder()
             .setSymbol(serializeIrSymbol(expression.symbol))
             .setValue(serializeExpression(expression.value)).apply {
-                expression.origin?.let { setOrigin(serializeIrStatementOrigin(it)) }
+                expression.origin?.let { origin = serializeIrStatementOrigin(it) }
             }
             .build()
 
@@ -1288,7 +1290,7 @@ open class IrFileSerializer(
 
 // ---------- Top level ------------------------------------------------------
 
-    fun serializeFileEntry(entry: SourceManager.FileEntry) = ProtoFileEntry.newBuilder()
+    private fun serializeFileEntry(entry: SourceManager.FileEntry): ProtoFileEntry = ProtoFileEntry.newBuilder()
         .setName(entry.name)
         .addAllLineStartOffsets(entry.lineStartOffsets.asIterable())
         .build()
@@ -1299,17 +1301,6 @@ open class IrFileSerializer(
     fun serializeIrFile(file: IrFile): IrIrSerializedIrFile {
         val topLevelDeclarations = mutableListOf<SerializedDeclaration>()
 
-        protoSymbolMap = mutableMapOf()
-        protoSymbolArray = arrayListOf()
-
-        protoTypeMap = mutableMapOf()
-        protoTypeArray = arrayListOf()
-
-        protoStringMap = mutableMapOf()
-        protoStringArray = arrayListOf()
-
-        protoBodyArray = mutableListOf()
-
         val proto = ProtoFile.newBuilder()
             .setFileEntry(serializeFileEntry(file.fileEntry))
             .setFqName(serializeFqName(file.fqName))
@@ -1324,7 +1315,9 @@ open class IrFileSerializer(
             val byteArray = serializeDeclaration(it).toByteArray()
             val uniqId = declarationTable.uniqIdByDeclaration(it)
             topLevelDeclarations.add(TopLevelDeclaration(uniqId.index, uniqId.isLocal, it.descriptor.toString(), byteArray))
-            proto.addDeclarationId(protoUniqId(uniqId))
+            if (uniqId.isPublic) {
+                proto.addDeclarationId(protoUniqId(uniqId))
+            }
         }
 
         // TODO: is it Konan specific?
@@ -1333,6 +1326,7 @@ open class IrFileSerializer(
         file.declarations
             .filterIsInstance<IrProperty>()
             .filter { it.backingField?.initializer != null && !it.isConst }
+            .filter { it.visibility.let { it == Visibilities.PUBLIC || it == Visibilities.INTERNAL } }
             .forEach { proto.addExplicitlyExportedToCompiler(serializeIrSymbol(it.backingField!!.symbol)) }
 
         // TODO: Konan specific

compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/IrModuleSerializer.kt

@@ -11,9 +11,10 @@ import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
 import org.jetbrains.kotlin.library.IrIrSerializedIrFile
 import org.jetbrains.kotlin.library.SerializedIrModule
 
-abstract class IrModuleSerializer<F : IrFileSerializer>(protected val logger: LoggingContext,
+abstract class IrModuleSerializer<F : IrFileSerializer>(
-                                                        protected val mangler: KotlinMangler,
+    protected val logger: LoggingContext,
-                                                        protected val declarationTable: DeclarationTable) {
+    protected val mangler: KotlinMangler
+) {
 
 
     abstract fun createSerializerForFile(file: IrFile): F

compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/KlibStructure.kt

@@ -1,33 +0,0 @@
-/*
- * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.backend.common.serialization
-
-import org.jetbrains.kotlin.name.FqName
-
-class Table
-
-sealed class KlibEntry(val fqName: FqName)
-
-class KlibFileEntry(fqName: FqName) : KlibEntry(fqName) {
-    lateinit var bodyData: ByteArray
-    lateinit var headerData: ByteArray
-
-    val stringTable = Table()
-    val symbolTable = Table()
-    val typeTable = Table()
-}
-
-class KlibPackageEntry(fqName: FqName) : KlibEntry(fqName) {
-    val childs = mutableListOf<KlibEntry>()
-}
-
-class KlibModuleEntry(val name: String) : KlibEntry(FqName.ROOT) {
-    val packages = mutableListOf<KlibEntry>()
-
-    val stringTable = Table()
-    val symbolTable = Table()
-    val typeTable = Table()
-}

compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/KotlinIrLinker.kt

@@ -33,8 +33,8 @@ import org.jetbrains.kotlin.ir.types.impl.IrErrorTypeImpl
 import org.jetbrains.kotlin.ir.util.IrDeserializer
 import org.jetbrains.kotlin.ir.util.NaiveSourceBasedFileEntryImpl
 import org.jetbrains.kotlin.ir.util.SymbolTable
-import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite.newInstance
+import org.jetbrains.kotlin.resolve.descriptorUtil.isPublishedApi
 import org.jetbrains.kotlin.resolve.descriptorUtil.module
 import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedCallableMemberDescriptor
 import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedClassDescriptor
@@ -61,9 +61,56 @@ abstract class KotlinIrLinker(
     private val firstKnownBuiltinsIndex: Long
 ) : DescriptorUniqIdAware, IrDeserializer {
 
-    protected val deserializedSymbols = mutableMapOf<UniqIdKey, IrSymbol>()
+
-    private val reachableTopLevels = mutableSetOf<UniqIdKey>()
+    sealed class DeserializationState {
-    private val deserializedTopLevels = mutableSetOf<UniqIdKey>()
+        val deserializedSymbols = mutableMapOf<UniqIdKey, IrSymbol>()
+        val reachableTopLevels = mutableSetOf<UniqIdKey>()
+        val deserializedTopLevels = mutableSetOf<UniqIdKey>()
+
+        operator fun contains(key: UniqIdKey) = key in deserializedSymbols
+        operator fun get(key: UniqIdKey): IrSymbol = deserializedSymbols[key] ?: error("jfsdhsdfhksjdhfkjshdk")
+
+        abstract fun addUniqID(key: UniqIdKey)
+        protected open fun peekReachableKey(): UniqIdKey? = reachableTopLevels.firstOrNull()
+
+        class ModuleDeserializationState(val module: IrModuleDeserializer): DeserializationState() {
+            override fun addUniqID(key: UniqIdKey) {
+                module.addModuleReachableTopLevel(key)
+            }
+        }
+
+        class SimpleDeserializationState: DeserializationState() {
+            override fun addUniqID(key: UniqIdKey) {
+                reachableTopLevels.add(key)
+            }
+
+            override fun peekReachableKey(): UniqIdKey {
+                return reachableTopLevels.firstOrNull() ?: error("Expecting non-empty set")
+            }
+        }
+
+        fun processPendingDeclaration(processor: (UniqIdKey) -> Unit) {
+            do {
+
+                val reachableKey = peekReachableKey() ?: return
+
+                if (deserializedSymbols[reachableKey]?.isBound == true) {
+                    reachableTopLevels.remove(reachableKey)
+                    deserializedTopLevels.add(reachableKey)
+                    continue
+                }
+
+                processor(reachableKey)
+
+                reachableTopLevels.remove(reachableKey)
+                deserializedTopLevels.add(reachableKey)
+
+            } while (reachableTopLevels.isNotEmpty())
+        }
+    }
+
+    private val globalDeserializationState = DeserializationState.SimpleDeserializationState()
+    private val modulesWithReachableTopLevels = mutableSetOf<IrModuleDeserializer>()
 
     //TODO: This is Native specific. Eliminate me.
     private val forwardDeclarations = mutableSetOf<IrSymbol>()
@@ -71,32 +118,41 @@ abstract class KotlinIrLinker(
 
     protected val deserializersForModules = mutableMapOf<ModuleDescriptor, IrModuleDeserializer>()
 
+    private fun getForwardDeclararationModuleDeserializer() = deserializersForModules.entries.single { it.key.isForwardDeclarationModule }.value
+
     inner class IrModuleDeserializer(
         private val moduleDescriptor: ModuleDescriptor,
         private val deserializationStrategy: DeserializationStrategy
     ) {
 
         val fileToDeserializerMap = mutableMapOf<IrFile, IrDeserializerForFile>()
+
+        protected val moduleResolvedForwardDeclarations = mutableMapOf<UniqIdKey, UniqIdKey>()
+
+        private val moduleDeserializationState = DeserializationState.ModuleDeserializationState(this)
+        private val moduleReversedFileIndex = mutableMapOf<UniqIdKey, IrDeserializerForFile>()
+        private val moduleDependencies = mutableListOf<IrModuleDeserializer>()
+
         // This is a heavy initializer
         val module = deserializeIrModuleHeader()
-        private var moduleLoops = mutableMapOf<Int, IrLoopBase>()
 
         inner class IrDeserializerForFile(private var annotationsProto: ProtoAnnotations?, private val fileIndex: Int, onlyHeaders: Boolean) :
             IrFileDeserializer(logger, builtIns, symbolTable) {
 
+            private var fileLoops = mutableMapOf<Int, IrLoopBase>()
+
             private val symbolProtosCache = mutableMapOf<Int, ProtoSymbolData>()
             private val typeProtosCache = mutableMapOf<Int, ProtoType>()
             private val stringsCache = mutableMapOf<Int, String>()
 
-            private val declarationsBytes: ByteArray by lazy { TODO("") }
-            private val bodiesBytes: ByteArray by lazy { TODO("") }
-            private val symbolsBytes: ByteArray by lazy { TODO("") }
-            private val typesBytes: ByteArray by lazy { TODO("") }
-            private val stringsBytes: ByteArray by lazy { TODO("") }
             lateinit var file: IrFile
 
             private val deserializeBodies: Boolean = !onlyHeaders
 
+            private val fileLocalResolvedForwardDeclarations = mutableMapOf<UniqIdKey, UniqIdKey>()
+
+            val fileLocalDeserializationState = DeserializationState.SimpleDeserializationState()
+
             fun deserializeDeclaration(key: UniqIdKey): IrDeclaration {
                 return deserializeDeclaration(loadTopLevelDeclarationProto(key), file)
             }
@@ -215,13 +271,39 @@ abstract class KotlinIrLinker(
                 }
             }
 
+            private fun isGlobalUniqID(uniqIdKey: UniqIdKey): Boolean {
+                return uniqIdKey in globalDeserializationState ||
+                        descriptorReferenceDeserializer.checkIfSpecialDescriptorId(uniqIdKey.uniqId.index)
+            }
+
+            private fun getModuleForTopLevelId(key: UniqIdKey): IrModuleDeserializer {
+                if (key in moduleReversedFileIndex) return this@IrModuleDeserializer
+                return moduleDependencies.firstOrNull { key in it.moduleReversedFileIndex } ?:
+                error("Deserializer for declaration $key is not found")
+            }
+
+            private fun getStateForID(key: UniqIdKey): DeserializationState {
+                if (key.uniqId.isLocal) return fileLocalDeserializationState
+                if (isGlobalUniqID(key)) return globalDeserializationState
+                return getModuleForTopLevelId(key).moduleDeserializationState
+            }
+
             private fun deserializeIrSymbolData(proto: ProtoSymbolData): IrSymbol {
                 val key = proto.uniqId.uniqIdKey(moduleDescriptor)
                 val topLevelKey = proto.topLevelUniqId.uniqIdKey(moduleDescriptor)
 
-                if (!deserializedTopLevels.contains(topLevelKey)) reachableTopLevels.add(topLevelKey)
+                val topLevelDeserializationState = getStateForID(topLevelKey)
 
-                val symbol = deserializedSymbols.getOrPut(key) {
+                if (topLevelKey !in topLevelDeserializationState) {
+                    topLevelDeserializationState.addUniqID(topLevelKey)
+                }
+
+                // If topLevel declaration is module-public and current declaration in not (e.g. value parameter)
+                // they should be proccesed via different tables
+                val deserializationState =
+                    if (topLevelKey.uniqId.isLocal xor key.uniqId.isLocal) getStateForID(key) else topLevelDeserializationState
+
+                val symbol = deserializationState.deserializedSymbols.getOrPut(key) {
                     val descriptor = if (proto.hasDescriptorReference()) {
                         deserializeDescriptorReference(proto.descriptorReference)
                     } else {
@@ -229,12 +311,8 @@ abstract class KotlinIrLinker(
                     }
 
                     resolvedForwardDeclarations[key]?.let {
-                        if (!deserializedTopLevels.contains(it)) reachableTopLevels.add(it) // Assuming forward declarations are always top levels.
+                        with (getForwardDeclararationModuleDeserializer().moduleDeserializationState) {
-                    }
+                            if (it !in this) addUniqID(it)
-
-                    descriptor?.module?.let {
-                        if (!deserializersForModules.containsKey(it) && !it.isForwardDeclarationModule) {
-                            deserializeIrModuleHeader(it)!!
                         }
                     }
 
@@ -279,7 +357,7 @@ abstract class KotlinIrLinker(
                 loadStringProto(proto.index)
 
             override fun deserializeLoopHeader(loopIndex: Int, loopBuilder: () -> IrLoopBase) =
-                moduleLoops.getOrPut(loopIndex, loopBuilder)
+                fileLoops.getOrPut(loopIndex, loopBuilder)
 
             override fun deserializeExpressionBody(proto: ProtoBodyIndex): IrExpression {
                 if (deserializeBodies) {
@@ -301,16 +379,22 @@ abstract class KotlinIrLinker(
                 }
             }
 
-            // TODO: this is JS specific. Eliminate me.
-            override fun getPrimitiveTypeOrNull(symbol: IrClassifierSymbol, hasQuestionMark: Boolean) =
-                this@KotlinIrLinker.getPrimitiveTypeOrNull(symbol, hasQuestionMark)
-
             fun deserializeFileAnnotationsIfFirstUse() {
                 annotationsProto?.let {
                     file.annotations.addAll(deserializeAnnotations(it))
                     annotationsProto = null
                 }
             }
+
+            fun deserializeFileTopLevelDeclaration(key: UniqIdKey) {
+
+                fileLocalDeserializationState.addUniqID(key)
+
+                fileLocalDeserializationState.processPendingDeclaration {
+                    val declaration = deserializeDeclaration(it)
+                    file.declarations.add(declaration)
+                }
+            }
         }
 
         private fun deserializeIrFile(fileProto: ProtoFile, fileIndex: Int): IrFile {
@@ -329,27 +413,36 @@ abstract class KotlinIrLinker(
             val file = IrFileImpl(fileEntry, symbol, fqName)
 
             fileDeserializer.file = file
+            fileToDeserializerMap[file] = fileDeserializer
 
             fileProto.declarationIdList.forEach {
                 val uniqIdKey = it.uniqIdKey(moduleDescriptor)
-                reversedFileIndex.getOrPut(uniqIdKey) { mutableListOf() }.add(file)
+                assert(uniqIdKey.uniqId.isPublic)
-                fileToDeserializerMap[file] = fileDeserializer
+                moduleReversedFileIndex.getOrPut(uniqIdKey) { fileDeserializer }
             }
 
             when (deserializationStrategy) {
                 DeserializationStrategy.EXPLICITLY_EXPORTED -> {
                     fileProto.explicitlyExportedToCompilerList.forEach {
                         val symbolProto = fileDeserializer.loadSymbolData(it.index)
-                        reachableTopLevels.add(symbolProto.topLevelUniqId.uniqIdKey(moduleDescriptor))
+                        val topLevelUniqId = symbolProto.topLevelUniqId
+                        val uniqIdKey = topLevelUniqId.uniqIdKey (moduleDescriptor)
+                        assert(uniqIdKey.uniqId.isPublic)
+//                        if (topLevelUniqId.isLocal) {
+//                            fileDeserializer.fileLocalDeserializationState.addUniqID(uniqIdKey)
+//                        } else {
+                            moduleDeserializationState.addUniqID(uniqIdKey)
+//                        }
                     }
                 }
                 DeserializationStrategy.ALL -> {
                     fileProto.declarationIdList.forEach {
                         val uniqIdKey = it.uniqIdKey(moduleDescriptor)
-                        reachableTopLevels.add(uniqIdKey)
+                        assert(uniqIdKey.uniqId.isPublic)
+                        moduleDeserializationState.addUniqID(uniqIdKey)
                     }
                 }
-                else -> error("Unixpected deserialization strategy")
+                else -> error("Unexpected deserialization strategy")
             }
 
             return file
@@ -364,16 +457,27 @@ abstract class KotlinIrLinker(
                 files.add(deserializeIrFile(ProtoFile.parseFrom(readFile(moduleDescriptor, i), newInstance()), i))
             }
 
-//            val files = readFiles(moduleDescriptor).map {
+            moduleDescriptor.allDependencyModules.filter { it != moduleDescriptor }.mapTo(moduleDependencies) { deserializersForModules[it]!! }
-//                deserializeIrFile(ProtoFile.parseFrom(it.codedInputStream, newInstance()))
-//            }
 
             return IrModuleFragmentImpl(moduleDescriptor, builtIns, files)
         }
-    }
 
-    // TODO: this is JS specific. Eliminate me.
+        fun deserializeAllModuleReachableTopLevels() {
-    protected open fun getPrimitiveTypeOrNull(symbol: IrClassifierSymbol, hasQuestionMark: Boolean): IrSimpleType? = null
+
+            moduleDeserializationState.processPendingDeclaration {
+                val fileDeserializer = moduleReversedFileIndex[it]
+                    ?: error("No file deserializer for key $it")
+
+                fileDeserializer.deserializeFileTopLevelDeclaration(it)
+                fileDeserializer.deserializeFileAnnotationsIfFirstUse()
+            }
+        }
+
+        fun addModuleReachableTopLevel(topLevelKey: UniqIdKey) {
+            moduleDeserializationState.reachableTopLevels.add(topLevelKey)
+            modulesWithReachableTopLevels.add(this)
+        }
+    }
 
     protected abstract val descriptorReferenceDeserializer: DescriptorReferenceDeserializer
 
@@ -381,8 +485,9 @@ abstract class KotlinIrLinker(
 
     private fun loadKnownBuiltinSymbols(): Long {
         var currentIndex = firstKnownBuiltinsIndex
+        val globalDeserializedSymbols = globalDeserializationState.deserializedSymbols
         builtIns.knownBuiltins.forEach {
-            deserializedSymbols[UniqIdKey(null, UniqId(currentIndex, isLocal = false))] = it
+            globalDeserializedSymbols[UniqIdKey(null, UniqId(currentIndex, isLocal = false))] = it
             assert(symbolTable.referenceSimpleFunction(it.descriptor) == it)
             currentIndex++
         }
@@ -396,12 +501,6 @@ abstract class KotlinIrLinker(
             return codedInputStream
         }
 
-    private val reversedFileIndex = mutableMapOf<UniqIdKey, MutableList<IrFile>>()
-
-    private val UniqIdKey.moduleOfOrigin
-        get() =
-            this.moduleDescriptor ?: reversedFileIndex[this]?.handleClashes(this)?.packageFragmentDescriptor?.containingDeclaration
-
     protected abstract fun reader(moduleDescriptor: ModuleDescriptor, fileIndex: Int, uniqId: UniqId): ByteArray
     protected abstract fun readSymbol(moduleDescriptor: ModuleDescriptor, fileIndex: Int, symbolIndex: Int): ByteArray
     protected abstract fun readType(moduleDescriptor: ModuleDescriptor, fileIndex: Int, typeIndex: Int): ByteArray
@@ -410,44 +509,21 @@ abstract class KotlinIrLinker(
     protected abstract fun readFile(moduleDescriptor: ModuleDescriptor, fileIndex: Int): ByteArray
     protected abstract fun readFileCount(moduleDescriptor: ModuleDescriptor): Int
 
-    protected open fun List<IrFile>.handleClashes(uniqIdKey: UniqIdKey): IrFile {
-        if (size == 1)
-            return this[0]
-        assert(size != 0)
-        error("UniqId clash: ${uniqIdKey.uniqId.index}. Found in the " +
-                      "[${this.joinToString { it.packageFragmentDescriptor.containingDeclaration.name.asString() }}]")
-    }
-
     private fun deserializeAllReachableTopLevels() {
         do {
-            val key = reachableTopLevels.first()
+            val moduleDeserializer = modulesWithReachableTopLevels.first()
-            val moduleOfOrigin = key.moduleOfOrigin
+            modulesWithReachableTopLevels.remove(moduleDeserializer)
 
-            if (deserializedSymbols[key]?.isBound == true ||
+            moduleDeserializer.deserializeAllModuleReachableTopLevels()
-                // The key.moduleOrigin is null for uniqIds that we haven't seen in any of the library headers.
+        } while (modulesWithReachableTopLevels.isNotEmpty())
-                // Just skip it for now and handle it elsewhere.
-                moduleOfOrigin == null
-            ) {
-
-                reachableTopLevels.remove(key)
-                deserializedTopLevels.add(key)
-                continue
-            }
-
-            val moduleDeserializer = deserializersForModules[moduleOfOrigin] ?: error("No module found")
-            val file = reversedFileIndex[key]!!.handleClashes(key)
-            val fileDeserializer: IrModuleDeserializer.IrDeserializerForFile = moduleDeserializer.fileToDeserializerMap[file] ?: error("dkjfkljfls")
-            val reachable = fileDeserializer.deserializeDeclaration(key)
-            file.declarations.add(reachable)
-            fileDeserializer.deserializeFileAnnotationsIfFirstUse()
-
-            reachableTopLevels.remove(key)
-            deserializedTopLevels.add(key)
-        } while (reachableTopLevels.isNotEmpty())
     }
 
     private fun findDeserializedDeclarationForDescriptor(descriptor: DeclarationDescriptor): DeclarationDescriptor? {
-        val topLevelDescriptor = descriptor.findTopLevelDescriptor()
+        val topLevelDescriptor = descriptor.findTopLevelDescriptor() as DeclarationDescriptorWithVisibility
+
+        require(topLevelDescriptor.isPublishedApi() || topLevelDescriptor.visibility.let { it.isPublicAPI || it == Visibilities.INTERNAL }) {
+            "Locally accessible declarations should not be accessed here $topLevelDescriptor"
+        }
 
         // This is Native specific. Try to eliminate.
         if (topLevelDescriptor.module.isForwardDeclarationModule) return null
@@ -461,10 +537,11 @@ abstract class KotlinIrLinker(
         val uniqId = UniqId(descriptorUniqId, isLocal = false)
         val topLevelKey = UniqIdKey(topLevelDescriptor.module, uniqId)
 
-        // This top level descriptor doesn't have a serialized IR declaration.
+        val moduleOfOrigin = topLevelDescriptor.module
-        if (topLevelKey.moduleOfOrigin == null) return null
 
-        reachableTopLevels.add(topLevelKey)
+        val moduleDeserializer = deserializersForModules[moduleOfOrigin] ?: error("No module deserializer found for $moduleOfOrigin")
+
+        moduleDeserializer.addModuleReachableTopLevel(topLevelKey)
 
         deserializeAllReachableTopLevels()
         return topLevelDescriptor

compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/UniqId.kt

@@ -16,7 +16,9 @@ import org.jetbrains.kotlin.backend.common.serialization.proto.UniqId as ProtoUn
 data class UniqId(
     val index: Long,
     val isLocal: Boolean
-)
+) {
+    val isPublic: Boolean get() = !isLocal
+}
 
 // isLocal=true in UniqId is good while we dealing with a single current module.
 // To disambiguate module local declarations of different modules we use UniqIdKey.

compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/lower/serialization/ir/JsIrLinker.kt

@@ -50,13 +50,5 @@ class JsIrLinker(
     override fun readFileCount(moduleDescriptor: ModuleDescriptor) =
         moduleDescriptor.kotlinLibrary.fileCount()
 
-    override fun List<IrFile>.handleClashes(uniqIdKey: UniqIdKey): IrFile {
-        if (size == 1)
-            return this[0]
-        assert(size != 0)
-        error("UniqId clash: ${uniqIdKey.uniqId.index}. Found in the " +
-                      "[${this.joinToString { it.packageFragmentDescriptor.containingDeclaration.userName }}]")
-    }
-
     private val ModuleDescriptor.userName get() = kotlinLibrary.libraryFile.absolutePath
 }