Commonized FileRegistry with SourceFileMap for klib metadata

2019-10-05 01:23:02 +03:00
parent b7a0546634
commit e5dbec4523
11 changed files with 336 additions and 1054 deletions
@@ -12,8 +12,13 @@ import org.jetbrains.kotlin.descriptors.MemberDescriptor
 import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor
 import org.jetbrains.kotlin.resolve.checkers.ExpectedActualDeclarationChecker
 import org.jetbrains.kotlin.descriptors.*
+import org.jetbrains.kotlin.library.KotlinLibrary
+import org.jetbrains.kotlin.library.metadata.KlibMetadataPackageFragment
+import org.jetbrains.kotlin.library.metadata.KlibMetadataProtoBuf
 import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.resolve.OverridingUtil
+import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedPropertyDescriptor
+import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedSimpleFunctionDescriptor

 internal val DeclarationDescriptor.isExpectMember: Boolean
    get() = this is MemberDescriptor && this.isExpect
@@ -49,3 +54,23 @@ internal fun <T : CallableMemberDescriptor> T.resolveFakeOverrideMaybeAbstract()
 // This is Native specific. Try to eliminate.
 val ModuleDescriptor.isForwardDeclarationModule get() =
    name == Name.special("<forward declarations>")
+
+private fun sourceByIndex(descriptor: CallableMemberDescriptor, index: Int): SourceFile {
+    val fragment = descriptor.findPackage() as KlibMetadataPackageFragment
+    return fragment.fileRegistry.sourceFile(index)
+}
+
+fun CallableMemberDescriptor.findSourceFile(): SourceFile {
+    val source = this.source.containingFile
+    if (source != SourceFile.NO_SOURCE_FILE)
+        return source
+    return when {
+        this is DeserializedSimpleFunctionDescriptor && proto.hasExtension(KlibMetadataProtoBuf.functionFile) ->
+            sourceByIndex(
+                this, proto.getExtension(KlibMetadataProtoBuf.functionFile))
+        this is DeserializedPropertyDescriptor && proto.hasExtension(KlibMetadataProtoBuf.propertyFile) ->
+            sourceByIndex(
+                this, proto.getExtension(KlibMetadataProtoBuf.propertyFile))
+        else -> TODO()
+    }
+}
@@ -3,7 +3,7 @@
 * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
 */

-package org.jetbrains.kotlin.ir.backend.js.lower.serialization.metadata
+package org.jetbrains.kotlin.backend.common.serialization.metadata

 import org.jetbrains.kotlin.library.metadata.KlibMetadataPackageFragment
 import org.jetbrains.kotlin.psi.KtFile
@@ -1,41 +0,0 @@
-/*
- * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.ir.backend.js.lower.serialization.metadata
-
-import org.jetbrains.kotlin.backend.common.serialization.metadata.extractFileId
-import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
-import org.jetbrains.kotlin.descriptors.DeclarationDescriptorWithSource
-import org.jetbrains.kotlin.library.metadata.KlibMetadataPackageFragment
-import org.jetbrains.kotlin.psi.KtFile
-import org.jetbrains.kotlin.resolve.DescriptorUtils
-import org.jetbrains.kotlin.resolve.source.PsiSourceFile
-
-
-class KlibMetadataFileRegistry {
-    private val fileIdsImpl = mutableMapOf<KlibFileMetadata, Int>()
-
-    fun lookup(file: KlibFileMetadata) = fileIdsImpl.getOrPut(file) { fileIdsImpl.size }
-
-    val fileIds: Map<KlibFileMetadata, Int>
-        get() = fileIdsImpl
-
-    fun getFileId(descriptor: DeclarationDescriptor): Int? {
-        if (!DescriptorUtils.isTopLevelDeclaration(descriptor) || descriptor !is DeclarationDescriptorWithSource) return null
-
-        val fileId = descriptor.extractFileId()
-        if (fileId != null) {
-            (descriptor.containingDeclaration as? KlibMetadataPackageFragment)?.let { packageFragment ->
-                return this.lookup(KotlinDeserializedFileMetadata(packageFragment, fileId))
-            }
-        }
-
-        val file = descriptor.source.containingFile as? PsiSourceFile ?: return null
-
-        val psiFile = file.psiFile
-        return (psiFile as? KtFile)?.let { this.lookup(KotlinPsiFileMetadata(it)) }
-    }
-
-}
@@ -11,8 +11,7 @@ import org.jetbrains.kotlin.config.AnalysisFlags
 import org.jetbrains.kotlin.config.LanguageVersionSettings
 import org.jetbrains.kotlin.descriptors.*
 import org.jetbrains.kotlin.incremental.components.NoLookupLocation
-import org.jetbrains.kotlin.ir.backend.js.lower.serialization.metadata.KlibMetadataFileRegistry
-import org.jetbrains.kotlin.ir.backend.js.lower.serialization.metadata.KotlinPsiFileMetadata
+import org.jetbrains.kotlin.library.metadata.KlibMetadataFileRegistry
 import org.jetbrains.kotlin.library.metadata.KlibMetadataProtoBuf
 import org.jetbrains.kotlin.metadata.ProtoBuf
 import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion
@@ -35,7 +34,6 @@ abstract class KlibMetadataSerializer(
    val metadataVersion: BinaryVersion,
    val descriptorTable: DescriptorTable
 ) {
-
    val fileRegistry = KlibMetadataFileRegistry()

    lateinit var serializerContext: SerializerContext
@@ -57,7 +55,7 @@ abstract class KlibMetadataSerializer(
            languageVersionSettings,
            metadataVersion,
            ::declarationTableHandler,
-            {descriptor -> fileRegistry.getFileId(descriptor) } ,
+            {descriptor: DeclarationDescriptorWithSource -> fileRegistry.assign(descriptor.source.containingFile) } ,
            KlibMetadataStringTable()
        )
        return SerializerContext(
@@ -82,10 +80,6 @@ abstract class KlibMetadataSerializer(

        val (stringTableProto, nameTableProto) = serializerExtension.stringTable.buildProto()

-        // TODO: we place files table to each and every fragment.
-        // Need to refactor it out sonehow.
-        val files = serializeFiles(fileRegistry, bindingContext, AnnotationSerializer(serializerExtension.stringTable))
-
        return ProtoBuf.PackageFragment.newBuilder()
            .setPackage(packageProto)
            .addAllClass_(classesProto.map { it.first })
@@ -95,7 +89,6 @@ abstract class KlibMetadataSerializer(
                classesProto.forEach {
                    packageFragment.addExtension(KlibMetadataProtoBuf.className, it.second )
                }
-                packageFragment.setExtension(KlibMetadataProtoBuf.packageFragmentFiles, files)
                packageFragment.setExtension(KlibMetadataProtoBuf.isEmpty, isEmpty)
                packageFragment.setExtension(KlibMetadataProtoBuf.fqName, fqName.asString())
            }
@@ -224,32 +217,17 @@ abstract class KlibMetadataSerializer(
    }

    private fun serializeFiles(
-        fileRegistry: KlibMetadataFileRegistry,
-        bindingContext: BindingContext,
-        serializer: AnnotationSerializer
-    ): KlibMetadataProtoBuf.Files {
-        val filesProto = KlibMetadataProtoBuf.Files.newBuilder()
-        for ((file, id) in fileRegistry.fileIds.entries.sortedBy { it.value }) {
+        header: KlibMetadataProtoBuf.Header.Builder,
+        fileRegistry: KlibMetadataFileRegistry
+    ) {
+
+        fileRegistry.filesAndClear().map { it.name ?: "" }.forEach {
            val fileProto = KlibMetadataProtoBuf.File.newBuilder()
-            if (id != filesProto.fileCount) {
-                fileProto.id = id
-            }
-            val annotations = when (file) {
-                is KotlinPsiFileMetadata -> file.ktFile.annotationEntries.map { bindingContext[BindingContext.ANNOTATION, it]!! }
-                //is KotlinDeserializedFileMetadata -> file.packageFragment.fileMap[file.fileId]!!.annotations
-                else -> TODO("support other file types")
-            }
-            for (annotation in annotations.filterOutSourceAnnotations()) {
-                fileProto.addAnnotation(serializer.serializeAnnotation(annotation))
-            }
-            val name = when (file) {
-                is KotlinPsiFileMetadata -> file.ktFile.getName()
-                else -> TODO("support other file types")
-            }
-            fileProto.name = name
-            filesProto.addFile(fileProto)
+                .setName(it)
+                .build()
+
+            header.addFile(fileProto)
        }
-        return filesProto.build()
    }

    protected fun getPackagesFqNames(module: ModuleDescriptor): Set<FqName> {
@@ -296,6 +274,9 @@ abstract class KlibMetadataSerializer(
        emptyPackages.forEach {
            header.addEmptyPackage(it)
        }
+
+        serializeFiles(header, fileRegistry)
+
        return header.build()
    }

@@ -22,7 +22,7 @@ class KlibMetadataSerializerExtension(
    private val languageVersionSettings: LanguageVersionSettings,
    override val metadataVersion: BinaryVersion,
    val declarationTableHandler: (DeclarationDescriptor) -> KlibMetadataProtoBuf.DescriptorUniqId?,
-    val descriptorFileId: (DeclarationDescriptor) -> Int?,
+    val descriptorFileId: (DeclarationDescriptorWithSource) -> Int?,
    override val stringTable: StringTableImpl
 ) : KotlinSerializerExtensionBase(KlibMetadataSerializerProtocol) {
    override fun shouldUseTypeTable(): Boolean = true
@@ -1,76 +0,0 @@
-/*
- * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.ir.backend.js.lower.serialization.metadata
-
-import org.jetbrains.kotlin.descriptors.ModuleDescriptor
-import org.jetbrains.kotlin.descriptors.SourceFile
-import org.jetbrains.kotlin.descriptors.annotations.AnnotationDescriptor
-import org.jetbrains.kotlin.library.metadata.KlibMetadataProtoBuf
-import org.jetbrains.kotlin.metadata.ProtoBuf
-import org.jetbrains.kotlin.metadata.deserialization.NameResolverImpl
-import org.jetbrains.kotlin.name.ClassId
-import org.jetbrains.kotlin.name.FqName
-import org.jetbrains.kotlin.serialization.deserialization.*
-import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedContainerSource
-import org.jetbrains.kotlin.storage.StorageManager
-import org.jetbrains.kotlin.storage.getValue
-
-class JsKlibMetadataPackageFragment(
-    fqName: FqName,
-    storageManager: StorageManager,
-    module: ModuleDescriptor,
-    proto: ProtoBuf.PackageFragment,
-    header: KlibMetadataProtoBuf.Header,
-    metadataVersion: JsKlibMetadataVersion,
-    configuration: DeserializationConfiguration
-) : DeserializedPackageFragmentImpl(
-    fqName, storageManager, module, proto, metadataVersion, JsContainerSource(fqName, header, configuration)
-) {
-    val fileMap: Map<Int, FileHolder> =
-        proto.getExtension(KlibMetadataProtoBuf.packageFragmentFiles).fileList.withIndex().associate { (index, file) ->
-            (if (file.hasId()) file.id else index) to FileHolder(file.annotationList)
-        }
-
-    private lateinit var annotationDeserializer: AnnotationDeserializer
-
-    override fun initialize(components: DeserializationComponents) {
-        super.initialize(components)
-        this.annotationDeserializer = AnnotationDeserializer(components.moduleDescriptor, components.notFoundClasses)
-    }
-
-    inner class FileHolder(private val annotationsProto: List<ProtoBuf.Annotation>) {
-        val annotations: List<AnnotationDescriptor> by storageManager.createLazyValue {
-            annotationsProto.map { annotationDeserializer.deserializeAnnotation(it, nameResolver) }
-        }
-    }
-
-    class JsContainerSource(
-        private val fqName: FqName,
-        header: KlibMetadataProtoBuf.Header,
-        configuration: DeserializationConfiguration
-    ) : DeserializedContainerSource {
-        val annotations: List<ClassId> =
-            if (header.annotationCount == 0) emptyList()
-            else NameResolverImpl(header.strings, header.qualifiedNames).let { nameResolver ->
-                // TODO: read arguments of module annotations
-                header.annotationList.map { annotation -> nameResolver.getClassId(annotation.id) }
-            }
-
-        // TODO
-        override fun getContainingFile(): SourceFile = SourceFile.NO_SOURCE_FILE
-
-        // This is null because we look for incompatible libraries in dependencies in the beginning of the compilation anyway,
-        // and refuse to compile against them completely
-        override val incompatibility: IncompatibleVersionErrorData<*>?
-            get() = null
-
-        override val isPreReleaseInvisible: Boolean =
-            configuration.reportErrorsOnPreReleaseDependencies && (header.flags and 1) != 0
-
-        override val presentableString: String
-            get() = "Package '$fqName'"
-    }
-}
@@ -33,6 +33,7 @@ message Header {
  repeated org.jetbrains.kotlin.metadata.Annotation annotation = 6;
  repeated string package_fragment_name = 7;
  repeated string empty_package = 8;
+  repeated File file = 9;

  // TODO: Unused.
  // TODO: We need to have backend specific extension for versionings.
@@ -42,14 +43,7 @@ message Header {
 }

 message File {
-  // If absent, id is the index of the file in the Files.file list
-  optional int32 id = 1;
-  required string name = 2;
-  repeated org.jetbrains.kotlin.metadata.Annotation annotation = 3;
-}
-
-message Files {
-  repeated File file = 1;
+  required string name = 1;
 }

 message DescriptorUniqId {
@@ -92,6 +86,10 @@ extend org.jetbrains.kotlin.metadata.EnumEntry {
  optional DescriptorUniqId enum_entry_uniq_id = 172;
 }

+extend org.jetbrains.kotlin.metadata.ValueParameter {
+   repeated org.jetbrains.kotlin.metadata.Annotation parameter_annotation = 170;
+}
+
 extend org.jetbrains.kotlin.metadata.Type {
  repeated org.jetbrains.kotlin.metadata.Annotation type_annotation = 170;
 }
@@ -102,7 +100,7 @@ extend org.jetbrains.kotlin.metadata.TypeParameter {
 }

 extend org.jetbrains.kotlin.metadata.PackageFragment {
-  optional Files package_fragment_files = 170;
+  repeated int32 package_fragment_files = 170;
  optional bool is_empty = 172;
  optional string fq_name = 173;
  // id in StringTable
@@ -1,26 +1,13 @@
 /*
- * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
- * that can be found in the LICENSE file.
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
 */
-package org.jetbrains.kotlin.backend.common.serialization.metadata
+package org.jetbrains.kotlin.library.metadata

 import org.jetbrains.kotlin.descriptors.SourceFile
 import org.jetbrains.kotlin.library.KotlinLibrary

-private class DeserializedSourceFile(
-        val name_: String, val index: Int, val library: KotlinLibrary) : SourceFile {
-    override fun getName(): String? = name_
-
-    override fun equals(other: Any?): Boolean {
-        return other is DeserializedSourceFile && library == other.library && index == other.index
-    }
-
-    override fun hashCode(): Int {
-        return library.hashCode() xor index
-    }
-}
-
-class SourceFileMap {
+class KlibMetadataFileRegistry {
    private val sourceToIndex = mutableMapOf<SourceFile, Int>()
    private val indexToSource = mutableMapOf<Int, SourceFile>()

@@ -37,7 +24,7 @@ class SourceFileMap {
    }

    fun sourceFile(index: Int): SourceFile =
-            indexToSource[index] ?: throw Error("Unknown file for $index")
+        indexToSource[index] ?: throw Error("Unknown file for $index")

    fun filesAndClear() =
        sourceToIndex.keys.sortedBy {
@@ -51,4 +38,18 @@ class SourceFileMap {
        sourceToIndex.clear()
        indexToSource.clear()
    }
+}
+
+class DeserializedSourceFile(
+    val name_: String, val index: Int, val library: KotlinLibrary
+) : SourceFile {
+    override fun getName(): String? = name_
+
+    override fun equals(other: Any?): Boolean {
+        return other is DeserializedSourceFile && library == other.library && index == other.index
+    }
+
+    override fun hashCode(): Int {
+        return library.hashCode() xor index
+    }
 }
@@ -19,6 +19,16 @@ import org.jetbrains.kotlin.serialization.deserialization.getClassId
 import org.jetbrains.kotlin.serialization.deserialization.getName
 import org.jetbrains.kotlin.storage.StorageManager

+private val KotlinLibrary.fileSources: KlibMetadataFileRegistry get() {
+        val result = KlibMetadataFileRegistry()
+        val proto = parseModuleHeader(moduleHeaderData)
+        proto.fileList.forEachIndexed { index, it ->
+                result.provide(it.name, index, this)
+            }
+        return result
+}
+
+
 class KlibMetadataPackageFragment(
    fqName: FqName,
    private val library: KotlinLibrary,
@@ -28,6 +38,10 @@ class KlibMetadataPackageFragment(
    private val partName: String
 ) : DeserializedPackageFragment(fqName, storageManager, module) {

+    val fileRegistry: KlibMetadataFileRegistry by lazy {
+            library.fileSources
+    }
+
    lateinit var components: DeserializationComponents

    override fun initialize(components: DeserializationComponents) {
@@ -22,7 +22,7 @@ fun String.parseKonanAbiVersion(): KotlinAbiVersion {

 data class KotlinAbiVersion(val version: Int) {
    companion object {
-        val CURRENT = KotlinAbiVersion(16)
+        val CURRENT = KotlinAbiVersion(17)
    }
    override fun toString() = "$version"
 }