Commonized FileRegistry with SourceFileMap for klib metadata

compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/LegacyDescriptorUtils.kt

@@ -12,8 +12,13 @@ import org.jetbrains.kotlin.descriptors.MemberDescriptor
 import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor
 import org.jetbrains.kotlin.resolve.checkers.ExpectedActualDeclarationChecker
 import org.jetbrains.kotlin.descriptors.*
+import org.jetbrains.kotlin.library.KotlinLibrary
+import org.jetbrains.kotlin.library.metadata.KlibMetadataPackageFragment
+import org.jetbrains.kotlin.library.metadata.KlibMetadataProtoBuf
 import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.resolve.OverridingUtil
+import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedPropertyDescriptor
+import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedSimpleFunctionDescriptor
 
 internal val DeclarationDescriptor.isExpectMember: Boolean
     get() = this is MemberDescriptor && this.isExpect
@@ -49,3 +54,23 @@ internal fun <T : CallableMemberDescriptor> T.resolveFakeOverrideMaybeAbstract()
 // This is Native specific. Try to eliminate.
 val ModuleDescriptor.isForwardDeclarationModule get() =
     name == Name.special("<forward declarations>")
+
+private fun sourceByIndex(descriptor: CallableMemberDescriptor, index: Int): SourceFile {
+    val fragment = descriptor.findPackage() as KlibMetadataPackageFragment
+    return fragment.fileRegistry.sourceFile(index)
+}
+
+fun CallableMemberDescriptor.findSourceFile(): SourceFile {
+    val source = this.source.containingFile
+    if (source != SourceFile.NO_SOURCE_FILE)
+        return source
+    return when {
+        this is DeserializedSimpleFunctionDescriptor && proto.hasExtension(KlibMetadataProtoBuf.functionFile) ->
+            sourceByIndex(
+                this, proto.getExtension(KlibMetadataProtoBuf.functionFile))
+        this is DeserializedPropertyDescriptor && proto.hasExtension(KlibMetadataProtoBuf.propertyFile) ->
+            sourceByIndex(
+                this, proto.getExtension(KlibMetadataProtoBuf.propertyFile))
+        else -> TODO()
+    }
+}

compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/metadata/KlibFileMetadata.kt

@@ -3,7 +3,7 @@
  * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
  */
 
-package org.jetbrains.kotlin.ir.backend.js.lower.serialization.metadata
+package org.jetbrains.kotlin.backend.common.serialization.metadata
 
 import org.jetbrains.kotlin.library.metadata.KlibMetadataPackageFragment
 import org.jetbrains.kotlin.psi.KtFile

compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/metadata/KlibMetadataFileRegistry.kt

@@ -1,41 +0,0 @@
-/*
- * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.ir.backend.js.lower.serialization.metadata
-
-import org.jetbrains.kotlin.backend.common.serialization.metadata.extractFileId
-import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
-import org.jetbrains.kotlin.descriptors.DeclarationDescriptorWithSource
-import org.jetbrains.kotlin.library.metadata.KlibMetadataPackageFragment
-import org.jetbrains.kotlin.psi.KtFile
-import org.jetbrains.kotlin.resolve.DescriptorUtils
-import org.jetbrains.kotlin.resolve.source.PsiSourceFile
-
-
-class KlibMetadataFileRegistry {
-    private val fileIdsImpl = mutableMapOf<KlibFileMetadata, Int>()
-
-    fun lookup(file: KlibFileMetadata) = fileIdsImpl.getOrPut(file) { fileIdsImpl.size }
-
-    val fileIds: Map<KlibFileMetadata, Int>
-        get() = fileIdsImpl
-
-    fun getFileId(descriptor: DeclarationDescriptor): Int? {
-        if (!DescriptorUtils.isTopLevelDeclaration(descriptor) || descriptor !is DeclarationDescriptorWithSource) return null
-
-        val fileId = descriptor.extractFileId()
-        if (fileId != null) {
-            (descriptor.containingDeclaration as? KlibMetadataPackageFragment)?.let { packageFragment ->
-                return this.lookup(KotlinDeserializedFileMetadata(packageFragment, fileId))
-            }
-        }
-
-        val file = descriptor.source.containingFile as? PsiSourceFile ?: return null
-
-        val psiFile = file.psiFile
-        return (psiFile as? KtFile)?.let { this.lookup(KotlinPsiFileMetadata(it)) }
-    }
-
-}

compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/metadata/KlibMetadataSerializer.kt

@@ -11,8 +11,7 @@ import org.jetbrains.kotlin.config.AnalysisFlags
 import org.jetbrains.kotlin.config.LanguageVersionSettings
 import org.jetbrains.kotlin.descriptors.*
 import org.jetbrains.kotlin.incremental.components.NoLookupLocation
-import org.jetbrains.kotlin.ir.backend.js.lower.serialization.metadata.KlibMetadataFileRegistry
+import org.jetbrains.kotlin.library.metadata.KlibMetadataFileRegistry
-import org.jetbrains.kotlin.ir.backend.js.lower.serialization.metadata.KotlinPsiFileMetadata
 import org.jetbrains.kotlin.library.metadata.KlibMetadataProtoBuf
 import org.jetbrains.kotlin.metadata.ProtoBuf
 import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion
@@ -35,7 +34,6 @@ abstract class KlibMetadataSerializer(
     val metadataVersion: BinaryVersion,
     val descriptorTable: DescriptorTable
 ) {
-
     val fileRegistry = KlibMetadataFileRegistry()
 
     lateinit var serializerContext: SerializerContext
@@ -57,7 +55,7 @@ abstract class KlibMetadataSerializer(
             languageVersionSettings,
             metadataVersion,
             ::declarationTableHandler,
-            {descriptor -> fileRegistry.getFileId(descriptor) } ,
+            {descriptor: DeclarationDescriptorWithSource -> fileRegistry.assign(descriptor.source.containingFile) } ,
             KlibMetadataStringTable()
         )
         return SerializerContext(
@@ -82,10 +80,6 @@ abstract class KlibMetadataSerializer(
 
         val (stringTableProto, nameTableProto) = serializerExtension.stringTable.buildProto()
 
-        // TODO: we place files table to each and every fragment.
-        // Need to refactor it out sonehow.
-        val files = serializeFiles(fileRegistry, bindingContext, AnnotationSerializer(serializerExtension.stringTable))
-
         return ProtoBuf.PackageFragment.newBuilder()
             .setPackage(packageProto)
             .addAllClass_(classesProto.map { it.first })
@@ -95,7 +89,6 @@ abstract class KlibMetadataSerializer(
                 classesProto.forEach {
                     packageFragment.addExtension(KlibMetadataProtoBuf.className, it.second )
                 }
-                packageFragment.setExtension(KlibMetadataProtoBuf.packageFragmentFiles, files)
                 packageFragment.setExtension(KlibMetadataProtoBuf.isEmpty, isEmpty)
                 packageFragment.setExtension(KlibMetadataProtoBuf.fqName, fqName.asString())
             }
@@ -224,32 +217,17 @@ abstract class KlibMetadataSerializer(
     }
 
     private fun serializeFiles(
-        fileRegistry: KlibMetadataFileRegistry,
+        header: KlibMetadataProtoBuf.Header.Builder,
-        bindingContext: BindingContext,
+        fileRegistry: KlibMetadataFileRegistry
-        serializer: AnnotationSerializer
+    ) {
-    ): KlibMetadataProtoBuf.Files {
+
-        val filesProto = KlibMetadataProtoBuf.Files.newBuilder()
+        fileRegistry.filesAndClear().map { it.name ?: "" }.forEach {
-        for ((file, id) in fileRegistry.fileIds.entries.sortedBy { it.value }) {
             val fileProto = KlibMetadataProtoBuf.File.newBuilder()
-            if (id != filesProto.fileCount) {
+                .setName(it)
-                fileProto.id = id
+                .build()
-            }
+
-            val annotations = when (file) {
+            header.addFile(fileProto)
-                is KotlinPsiFileMetadata -> file.ktFile.annotationEntries.map { bindingContext[BindingContext.ANNOTATION, it]!! }
-                //is KotlinDeserializedFileMetadata -> file.packageFragment.fileMap[file.fileId]!!.annotations
-                else -> TODO("support other file types")
-            }
-            for (annotation in annotations.filterOutSourceAnnotations()) {
-                fileProto.addAnnotation(serializer.serializeAnnotation(annotation))
-            }
-            val name = when (file) {
-                is KotlinPsiFileMetadata -> file.ktFile.getName()
-                else -> TODO("support other file types")
-            }
-            fileProto.name = name
-            filesProto.addFile(fileProto)
         }
-        return filesProto.build()
     }
 
     protected fun getPackagesFqNames(module: ModuleDescriptor): Set<FqName> {
@@ -296,6 +274,9 @@ abstract class KlibMetadataSerializer(
         emptyPackages.forEach {
             header.addEmptyPackage(it)
         }
+
+        serializeFiles(header, fileRegistry)
+
         return header.build()
     }
 

compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/metadata/KlibMetadataSerializerExtension.kt

@@ -22,7 +22,7 @@ class KlibMetadataSerializerExtension(
     private val languageVersionSettings: LanguageVersionSettings,
     override val metadataVersion: BinaryVersion,
     val declarationTableHandler: (DeclarationDescriptor) -> KlibMetadataProtoBuf.DescriptorUniqId?,
-    val descriptorFileId: (DeclarationDescriptor) -> Int?,
+    val descriptorFileId: (DeclarationDescriptorWithSource) -> Int?,
     override val stringTable: StringTableImpl
 ) : KotlinSerializerExtensionBase(KlibMetadataSerializerProtocol) {
     override fun shouldUseTypeTable(): Boolean = true

compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/lower/serialization/metadata/JsKlibMetadataPackageFragment.kt

@@ -1,76 +0,0 @@
-/*
- * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.ir.backend.js.lower.serialization.metadata
-
-import org.jetbrains.kotlin.descriptors.ModuleDescriptor
-import org.jetbrains.kotlin.descriptors.SourceFile
-import org.jetbrains.kotlin.descriptors.annotations.AnnotationDescriptor
-import org.jetbrains.kotlin.library.metadata.KlibMetadataProtoBuf
-import org.jetbrains.kotlin.metadata.ProtoBuf
-import org.jetbrains.kotlin.metadata.deserialization.NameResolverImpl
-import org.jetbrains.kotlin.name.ClassId
-import org.jetbrains.kotlin.name.FqName
-import org.jetbrains.kotlin.serialization.deserialization.*
-import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedContainerSource
-import org.jetbrains.kotlin.storage.StorageManager
-import org.jetbrains.kotlin.storage.getValue
-
-class JsKlibMetadataPackageFragment(
-    fqName: FqName,
-    storageManager: StorageManager,
-    module: ModuleDescriptor,
-    proto: ProtoBuf.PackageFragment,
-    header: KlibMetadataProtoBuf.Header,
-    metadataVersion: JsKlibMetadataVersion,
-    configuration: DeserializationConfiguration
-) : DeserializedPackageFragmentImpl(
-    fqName, storageManager, module, proto, metadataVersion, JsContainerSource(fqName, header, configuration)
-) {
-    val fileMap: Map<Int, FileHolder> =
-        proto.getExtension(KlibMetadataProtoBuf.packageFragmentFiles).fileList.withIndex().associate { (index, file) ->
-            (if (file.hasId()) file.id else index) to FileHolder(file.annotationList)
-        }
-
-    private lateinit var annotationDeserializer: AnnotationDeserializer
-
-    override fun initialize(components: DeserializationComponents) {
-        super.initialize(components)
-        this.annotationDeserializer = AnnotationDeserializer(components.moduleDescriptor, components.notFoundClasses)
-    }
-
-    inner class FileHolder(private val annotationsProto: List<ProtoBuf.Annotation>) {
-        val annotations: List<AnnotationDescriptor> by storageManager.createLazyValue {
-            annotationsProto.map { annotationDeserializer.deserializeAnnotation(it, nameResolver) }
-        }
-    }
-
-    class JsContainerSource(
-        private val fqName: FqName,
-        header: KlibMetadataProtoBuf.Header,
-        configuration: DeserializationConfiguration
-    ) : DeserializedContainerSource {
-        val annotations: List<ClassId> =
-            if (header.annotationCount == 0) emptyList()
-            else NameResolverImpl(header.strings, header.qualifiedNames).let { nameResolver ->
-                // TODO: read arguments of module annotations
-                header.annotationList.map { annotation -> nameResolver.getClassId(annotation.id) }
-            }
-
-        // TODO
-        override fun getContainingFile(): SourceFile = SourceFile.NO_SOURCE_FILE
-
-        // This is null because we look for incompatible libraries in dependencies in the beginning of the compilation anyway,
-        // and refuse to compile against them completely
-        override val incompatibility: IncompatibleVersionErrorData<*>?
-            get() = null
-
-        override val isPreReleaseInvisible: Boolean =
-            configuration.reportErrorsOnPreReleaseDependencies && (header.flags and 1) != 0
-
-        override val presentableString: String
-            get() = "Package '$fqName'"
-    }
-}

compiler/util-klib-metadata/src/KlibMetadataProtoBuf.proto

@@ -33,6 +33,7 @@ message Header {
   repeated org.jetbrains.kotlin.metadata.Annotation annotation = 6;
   repeated string package_fragment_name = 7;
   repeated string empty_package = 8;
+  repeated File file = 9;
 
   // TODO: Unused.
   // TODO: We need to have backend specific extension for versionings.
@@ -42,14 +43,7 @@ message Header {
 }
 
 message File {
-  // If absent, id is the index of the file in the Files.file list
+  required string name = 1;
-  optional int32 id = 1;
-  required string name = 2;
-  repeated org.jetbrains.kotlin.metadata.Annotation annotation = 3;
-}
-
-message Files {
-  repeated File file = 1;
 }
 
 message DescriptorUniqId {
@@ -92,6 +86,10 @@ extend org.jetbrains.kotlin.metadata.EnumEntry {
   optional DescriptorUniqId enum_entry_uniq_id = 172;
 }
 
+extend org.jetbrains.kotlin.metadata.ValueParameter {
+   repeated org.jetbrains.kotlin.metadata.Annotation parameter_annotation = 170;
+}
+
 extend org.jetbrains.kotlin.metadata.Type {
   repeated org.jetbrains.kotlin.metadata.Annotation type_annotation = 170;
 }
@@ -102,7 +100,7 @@ extend org.jetbrains.kotlin.metadata.TypeParameter {
 }
 
 extend org.jetbrains.kotlin.metadata.PackageFragment {
-  optional Files package_fragment_files = 170;
+  repeated int32 package_fragment_files = 170;
   optional bool is_empty = 172;
   optional string fq_name = 173;
   // id in StringTable

compiler/util-klib-metadata/src/org/jetbrains/kotlin/library/metadata/KlibMetadataFileRegistry.kt

@@ -1,26 +1,13 @@
 /*
- * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * that can be found in the LICENSE file.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
  */
-package org.jetbrains.kotlin.backend.common.serialization.metadata
+package org.jetbrains.kotlin.library.metadata
 
 import org.jetbrains.kotlin.descriptors.SourceFile
 import org.jetbrains.kotlin.library.KotlinLibrary
 
-private class DeserializedSourceFile(
+class KlibMetadataFileRegistry {
-        val name_: String, val index: Int, val library: KotlinLibrary) : SourceFile {
-    override fun getName(): String? = name_
-
-    override fun equals(other: Any?): Boolean {
-        return other is DeserializedSourceFile && library == other.library && index == other.index
-    }
-
-    override fun hashCode(): Int {
-        return library.hashCode() xor index
-    }
-}
-
-class SourceFileMap {
     private val sourceToIndex = mutableMapOf<SourceFile, Int>()
     private val indexToSource = mutableMapOf<Int, SourceFile>()
 
@@ -37,7 +24,7 @@ class SourceFileMap {
     }
 
     fun sourceFile(index: Int): SourceFile =
-            indexToSource[index] ?: throw Error("Unknown file for $index")
+        indexToSource[index] ?: throw Error("Unknown file for $index")
 
     fun filesAndClear() =
         sourceToIndex.keys.sortedBy {
@@ -51,4 +38,18 @@ class SourceFileMap {
         sourceToIndex.clear()
         indexToSource.clear()
     }
+}
+
+class DeserializedSourceFile(
+    val name_: String, val index: Int, val library: KotlinLibrary
+) : SourceFile {
+    override fun getName(): String? = name_
+
+    override fun equals(other: Any?): Boolean {
+        return other is DeserializedSourceFile && library == other.library && index == other.index
+    }
+
+    override fun hashCode(): Int {
+        return library.hashCode() xor index
+    }
 }

compiler/util-klib-metadata/src/org/jetbrains/kotlin/library/metadata/KlibMetadataPackageFragment.kt

@@ -19,6 +19,16 @@ import org.jetbrains.kotlin.serialization.deserialization.getClassId
 import org.jetbrains.kotlin.serialization.deserialization.getName
 import org.jetbrains.kotlin.storage.StorageManager
 
+private val KotlinLibrary.fileSources: KlibMetadataFileRegistry get() {
+        val result = KlibMetadataFileRegistry()
+        val proto = parseModuleHeader(moduleHeaderData)
+        proto.fileList.forEachIndexed { index, it ->
+                result.provide(it.name, index, this)
+            }
+        return result
+}
+
+
 class KlibMetadataPackageFragment(
     fqName: FqName,
     private val library: KotlinLibrary,
@@ -28,6 +38,10 @@ class KlibMetadataPackageFragment(
     private val partName: String
 ) : DeserializedPackageFragment(fqName, storageManager, module) {
 
+    val fileRegistry: KlibMetadataFileRegistry by lazy {
+            library.fileSources
+    }
+
     lateinit var components: DeserializationComponents
 
     override fun initialize(components: DeserializationComponents) {

compiler/util-klib/src/org/jetbrains/kotlin/library/KotlinAbiVersion.kt

@@ -22,7 +22,7 @@ fun String.parseKonanAbiVersion(): KotlinAbiVersion {
 
 data class KotlinAbiVersion(val version: Int) {
     companion object {
-        val CURRENT = KotlinAbiVersion(16)
+        val CURRENT = KotlinAbiVersion(17)
     }
     override fun toString() = "$version"
 }