[KLIB] Implement new linker based on IdSignature

- Remove klib dependency on metadata and uniqID
 - Refactored proto format to make it more effective and compact
  -- Use special encoding for some types of data (coordinates, flags, types)
  -- Remove symbols table
  -- Use packed proto list if it is possible
 - Remove extension from metadata
 - Remove special ids for function interfaces
 - Fix klib IO
 - Fix incremental cache
 - General code clean up

compiler/ir/serialization.jvm/src/org/jetbrains/kotlin/backend/jvm/serialization/JvmIdSignatureDescriptor.kt

@@ -9,6 +9,7 @@ import org.jetbrains.kotlin.backend.common.serialization.signature.IdSignatureDe
 import org.jetbrains.kotlin.descriptors.PropertyDescriptor
 import org.jetbrains.kotlin.ir.util.KotlinMangler
 import org.jetbrains.kotlin.load.java.descriptors.JavaForKotlinOverridePropertyDescriptor
+import org.jetbrains.kotlin.load.java.descriptors.JavaPropertyDescriptor
 
 class JvmIdSignatureDescriptor(private val mangler: KotlinMangler.DescriptorMangler) : IdSignatureDescriptor(mangler) {
 

compiler/ir/serialization.jvm/src/org/jetbrains/kotlin/ir/backend/jvm/serialization/JvmDescriptorReferenceDeserializer.kt

@@ -1,26 +0,0 @@
-/*
- * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.ir.backend.jvm.serialization
-
-import org.jetbrains.kotlin.backend.common.serialization.DescriptorReferenceDeserializer
-import org.jetbrains.kotlin.backend.common.serialization.DescriptorUniqIdAware
-import org.jetbrains.kotlin.descriptors.ClassDescriptor
-import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
-import org.jetbrains.kotlin.descriptors.ModuleDescriptor
-import org.jetbrains.kotlin.name.FqName
-
-class JvmDescriptorReferenceDeserializer(currentModule: ModuleDescriptor, private val uniqIdAware: DescriptorUniqIdAware) :
-    DescriptorReferenceDeserializer(currentModule, JvmMangler, builtIns = null, resolvedForwardDeclarations = mutableMapOf()),
-    DescriptorUniqIdAware by uniqIdAware {
-
-    override fun resolveSpecialDescriptor(fqn: FqName): ClassDescriptor {
-        error("Should never be called")
-    }
-
-    override fun checkIfSpecialDescriptorId(id: Long) = false
-
-    override fun getDescriptorIdOrNull(descriptor: DeclarationDescriptor): Long? = null
-}

compiler/ir/serialization.jvm/src/org/jetbrains/kotlin/ir/backend/jvm/serialization/JvmDescriptorUniqIdAware.kt

@@ -1,86 +0,0 @@
-/*
- * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.ir.backend.jvm.serialization
-
-import org.jetbrains.kotlin.backend.common.serialization.DescriptorUniqIdAware
-import org.jetbrains.kotlin.backend.common.serialization.tryGetExtension
-import org.jetbrains.kotlin.descriptors.*
-import org.jetbrains.kotlin.ir.declarations.IrDeclaration
-import org.jetbrains.kotlin.ir.declarations.IrDeclarationParent
-import org.jetbrains.kotlin.ir.declarations.IrEnumEntry
-import org.jetbrains.kotlin.ir.util.*
-import org.jetbrains.kotlin.library.metadata.KlibMetadataProtoBuf
-import org.jetbrains.kotlin.serialization.deserialization.descriptors.*
-
-class JvmDescriptorUniqIdAware(val symbolTable: SymbolTable, val stubGenerator: DeclarationStubGenerator) : DescriptorUniqIdAware {
-    override fun DeclarationDescriptor.getUniqId(): Long? =
-        when (this) {
-            is DeserializedClassDescriptor -> this.classProto.tryGetExtension(KlibMetadataProtoBuf.classUniqId)?.index
-                ?: referenceAndHash(this)
-            is DeserializedSimpleFunctionDescriptor -> this.proto.tryGetExtension(KlibMetadataProtoBuf.functionUniqId)?.index
-                ?: referenceAndHash(this)
-            is DeserializedPropertyDescriptor -> this.proto.tryGetExtension(KlibMetadataProtoBuf.propertyUniqId)?.index
-                ?: referenceAndHash(this)
-            is DeserializedClassConstructorDescriptor -> this.proto.tryGetExtension(KlibMetadataProtoBuf.constructorUniqId)?.index
-                ?: referenceAndHash(this)
-            is DeserializedTypeParameterDescriptor -> this.proto.tryGetExtension(KlibMetadataProtoBuf.typeParamUniqId)?.index
-                ?: referenceAndHash(this)
-            is DeserializedTypeAliasDescriptor -> this.proto.tryGetExtension(KlibMetadataProtoBuf.typeAliasUniqId)?.index
-                ?: referenceAndHash(this)
-            else -> referenceAndHash(this)
-        }
-
-    private fun referenceAndHash(descriptor: DeclarationDescriptor): Long? =
-        if (descriptor is CallableMemberDescriptor && descriptor.kind === CallableMemberDescriptor.Kind.FAKE_OVERRIDE)
-            null
-        else with(JvmMangler) {
-            referenceWithParents(descriptor).hashedMangle
-        }
-
-
-    private fun referenceWithParents(descriptor: DeclarationDescriptor): IrDeclaration {
-        val original = descriptor.original
-        val result = referenceOrDeclare(original)
-        var currentDescriptor = original
-        var current = result
-        // If current is a lazy declaration, the parent may already be set.
-        while (current.parent == null) {
-            val nextDescriptor = when {
-                currentDescriptor is TypeParameterDescriptor && currentDescriptor.containingDeclaration is PropertyDescriptor -> {
-                    val property = currentDescriptor.containingDeclaration as PropertyDescriptor
-                    // No way to choose between getter and setter by descriptor alone :(
-                    property.getter ?: property.setter!!
-                }
-                else ->
-                    currentDescriptor.containingDeclaration!!
-            }
-            if (nextDescriptor is PackageFragmentDescriptor) {
-                current.parent = symbolTable.findOrDeclareExternalPackageFragment(nextDescriptor)
-                break
-            } else {
-                val next = referenceOrDeclare(nextDescriptor, correspondingClassForEnum = true)
-                current.parent = next as IrDeclarationParent
-                currentDescriptor = nextDescriptor
-                current = next
-            }
-        }
-        return result
-    }
-
-    private fun referenceOrDeclare(descriptor: DeclarationDescriptor, correspondingClassForEnum: Boolean = false): IrDeclaration =
-        symbolTable.referenceMember(descriptor, correspondingClassForEnum).also {
-            if (!it.isBound) {
-                stubGenerator.getDeclaration(it)
-            }
-        }.owner as IrDeclaration
-}
-
-// May be needed in the future
-//
-//fun DeclarationDescriptor.willBeEliminatedInLowerings(): Boolean =
-//        isAnnotationConstructor() ||
-//                (this is PropertyAccessorDescriptor &&
-//                        correspondingProperty.hasJvmFieldAnnotation())

compiler/ir/serialization.jvm/src/org/jetbrains/kotlin/ir/backend/jvm/serialization/JvmIrLinker.kt

@@ -6,50 +6,31 @@
 package org.jetbrains.kotlin.ir.backend.jvm.serialization
 
 import org.jetbrains.kotlin.backend.common.LoggingContext
-import org.jetbrains.kotlin.backend.common.serialization.*
-import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
-import org.jetbrains.kotlin.descriptors.DeclarationDescriptorWithVisibility
+import org.jetbrains.kotlin.backend.common.serialization.DeserializationStrategy
+import org.jetbrains.kotlin.backend.common.serialization.KotlinIrLinker
 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
-import org.jetbrains.kotlin.descriptors.Visibilities
 import org.jetbrains.kotlin.descriptors.konan.kotlinLibrary
 import org.jetbrains.kotlin.ir.descriptors.IrBuiltIns
-import org.jetbrains.kotlin.ir.util.DeclarationStubGenerator
+import org.jetbrains.kotlin.ir.util.IdSignature
 import org.jetbrains.kotlin.ir.util.SymbolTable
-import org.jetbrains.kotlin.ir.util.UniqId
-import org.jetbrains.kotlin.load.java.JavaVisibilities
-import org.jetbrains.kotlin.load.java.descriptors.JavaClassDescriptor
-import org.jetbrains.kotlin.resolve.descriptorUtil.isPublishedApi
 
-class JvmIrLinker(
-    currentModule: ModuleDescriptor,
-    logger: LoggingContext,
-    builtIns: IrBuiltIns,
-    symbolTable: SymbolTable,
-    stubGenerator: DeclarationStubGenerator
-) : KotlinIrLinker(logger, builtIns, symbolTable, emptyList(), null, JvmMangler,
-                   // TODO: Use protected shouldBeDeserialized() ?
-                   tolerateNonKlibDescriptors = true),
-    DescriptorUniqIdAware {
+class JvmIrLinker(logger: LoggingContext, builtIns: IrBuiltIns, symbolTable: SymbolTable) :
+    KotlinIrLinker(logger, builtIns, symbolTable, emptyList(), null) {
 
-    override val descriptorReferenceDeserializer =
-        JvmDescriptorReferenceDeserializer(currentModule, JvmDescriptorUniqIdAware(symbolTable, stubGenerator))
-    override fun DeclarationDescriptor.getUniqId() = with(descriptorReferenceDeserializer) {
-        getUniqId()
-    }
-
-    override fun handleNoModuleDeserializerFound(key: UniqId): DeserializationState<*> {
+    override fun handleNoModuleDeserializerFound(idSignature: IdSignature): DeserializationState<*> {
+        // TODO: Implement special java-module deserializer instead of this hack
         return globalDeserializationState // !!!!!! Wrong, as external references will all have UniqId.NONE
     }
 
-    override fun reader(moduleDescriptor: ModuleDescriptor, fileIndex: Int, uniqId: UniqId) =
-        moduleDescriptor.kotlinLibrary.irDeclaration(uniqId.index, fileIndex)
-
-    override fun readSymbol(moduleDescriptor: ModuleDescriptor, fileIndex: Int, symbolIndex: Int) =
-        moduleDescriptor.kotlinLibrary.symbol(symbolIndex, fileIndex)
+    override fun reader(moduleDescriptor: ModuleDescriptor, fileIndex: Int, idSigIndex: Int) =
+        moduleDescriptor.kotlinLibrary.irDeclaration(idSigIndex, fileIndex)
 
     override fun readType(moduleDescriptor: ModuleDescriptor, fileIndex: Int, typeIndex: Int) =
         moduleDescriptor.kotlinLibrary.type(typeIndex, fileIndex)
 
+    override fun readSignature(moduleDescriptor: ModuleDescriptor, fileIndex: Int, signatureIndex: Int) =
+        moduleDescriptor.kotlinLibrary.signature(signatureIndex, fileIndex)
+
     override fun readString(moduleDescriptor: ModuleDescriptor, fileIndex: Int, stringIndex: Int) =
         moduleDescriptor.kotlinLibrary.string(stringIndex, fileIndex)
 
@@ -62,5 +43,14 @@ class JvmIrLinker(
     override fun readFileCount(moduleDescriptor: ModuleDescriptor) =
         moduleDescriptor.kotlinLibrary.fileCount()
 
-    private val ModuleDescriptor.userName get() = kotlinLibrary.libraryFile.absolutePath
+    override fun resolveModuleDeserializer(moduleDescriptor: ModuleDescriptor): IrModuleDeserializer? {
+        return deserializersForModules[moduleDescriptor]
+    }
+
+    // TODO: implement special Java deserializer
+    override fun createModuleDeserializer(moduleDescriptor: ModuleDescriptor, strategy: DeserializationStrategy): IrModuleDeserializer =
+        JvmModuleDeserializer(moduleDescriptor, strategy)
+
+    private inner class JvmModuleDeserializer(moduleDescriptor: ModuleDescriptor, strategy: DeserializationStrategy) :
+        KotlinIrLinker.IrModuleDeserializer(moduleDescriptor, strategy)
 }

compiler/ir/serialization.jvm/src/org/jetbrains/kotlin/ir/backend/jvm/serialization/JvmMangler.kt

@@ -7,9 +7,8 @@ package org.jetbrains.kotlin.ir.backend.jvm.serialization
 
 import org.jetbrains.kotlin.backend.common.serialization.mangle.KotlinExportChecker
 import org.jetbrains.kotlin.backend.common.serialization.mangle.KotlinMangleComputer
-import org.jetbrains.kotlin.backend.common.serialization.mangle.classic.ClassicExportChecker
-import org.jetbrains.kotlin.backend.common.serialization.mangle.classic.ClassicKotlinManglerImpl
-import org.jetbrains.kotlin.backend.common.serialization.mangle.classic.ClassicMangleComputer
+import org.jetbrains.kotlin.backend.common.serialization.mangle.MangleConstant
+import org.jetbrains.kotlin.backend.common.serialization.mangle.MangleMode
 import org.jetbrains.kotlin.backend.common.serialization.mangle.descriptor.DescriptorBasedKotlinManglerImpl
 import org.jetbrains.kotlin.backend.common.serialization.mangle.descriptor.DescriptorExportCheckerVisitor
 import org.jetbrains.kotlin.backend.common.serialization.mangle.descriptor.DescriptorMangleComputer
@@ -18,6 +17,7 @@ import org.jetbrains.kotlin.backend.common.serialization.mangle.ir.IrExportCheck
 import org.jetbrains.kotlin.backend.common.serialization.mangle.ir.IrMangleComputer
 import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
 import org.jetbrains.kotlin.descriptors.FunctionDescriptor
+import org.jetbrains.kotlin.descriptors.PropertyDescriptor
 import org.jetbrains.kotlin.idea.MainFunctionDetector
 import org.jetbrains.kotlin.ir.declarations.IrDeclaration
 import org.jetbrains.kotlin.load.java.lazy.descriptors.isJavaField
@@ -32,14 +32,14 @@ abstract class AbstractJvmManglerIr : IrBasedKotlinManglerImpl() {
         override fun IrDeclaration.isPlatformSpecificExported() = false
     }
 
-    private class JvmIrManglerComputer(builder: StringBuilder) : IrMangleComputer(builder) {
-        override fun copy(): IrMangleComputer = JvmIrManglerComputer(builder)
+    private class JvmIrManglerComputer(builder: StringBuilder, mode: MangleMode) : IrMangleComputer(builder, mode) {
+        override fun copy(newMode: MangleMode): IrMangleComputer = JvmIrManglerComputer(builder, newMode)
     }
 
     override fun getExportChecker(): KotlinExportChecker<IrDeclaration> = exportChecker
 
-    override fun getMangleComputer(prefix: String): KotlinMangleComputer<IrDeclaration> {
-        return JvmIrManglerComputer(StringBuilder(256))
+    override fun getMangleComputer(mode: MangleMode): KotlinMangleComputer<IrDeclaration> {
+        return JvmIrManglerComputer(StringBuilder(256), mode)
     }
 }
 
@@ -55,9 +55,10 @@ abstract class AbstractJvmDescriptorMangler(private val mainDetector: MainFuncti
         override fun DeclarationDescriptor.isPlatformSpecificExported() = false
     }
 
-    private class JvmDescriptorManglerComputer(builder: StringBuilder, prefix: String, private val mainDetector: MainFunctionDetector?) :
-        DescriptorMangleComputer(builder, prefix) {
-        override fun copy(): DescriptorMangleComputer = JvmDescriptorManglerComputer(builder, specialPrefix, mainDetector)
+    private class JvmDescriptorManglerComputer(builder: StringBuilder, private val mainDetector: MainFunctionDetector?, mode: MangleMode) :
+        DescriptorMangleComputer(builder, mode) {
+        override fun copy(newMode: MangleMode): DescriptorMangleComputer =
+            JvmDescriptorManglerComputer(builder, mainDetector, newMode)
 
         private fun isMainFunction(descriptor: FunctionDescriptor): Boolean = mainDetector?.isMain(descriptor) ?: false
 
@@ -66,12 +67,19 @@ abstract class AbstractJvmDescriptorMangler(private val mainDetector: MainFuncti
                 return source.containingFile.name
             } else null
         }
+
+        override fun PropertyDescriptor.platformSpecificSuffix(): String? {
+            // Since LV 1.4 there is a feature PreferJavaFieldOverload which allows to have java and kotlin
+            // properties with the same signature on the same level.
+            // For more details see JvmPlatformOverloadsSpecificityComparator.kt
+            return if (isJavaField) MangleConstant.JAVA_FIELD_SUFFIX else null
+        }
     }
 
     override fun getExportChecker(): KotlinExportChecker<DeclarationDescriptor> = exportChecker
 
-    override fun getMangleComputer(prefix: String): KotlinMangleComputer<DeclarationDescriptor> {
-        return JvmDescriptorManglerComputer(StringBuilder(256), prefix, mainDetector)
+    override fun getMangleComputer(mode: MangleMode): KotlinMangleComputer<DeclarationDescriptor> {
+        return JvmDescriptorManglerComputer(StringBuilder(256), mainDetector, mode)
     }
 }