Commonizing klib metadata between native and js

2019-07-26 14:45:18 +03:00
parent f38123e78c
commit c227c13799
98 changed files with 3265 additions and 7848 deletions
@@ -14,28 +14,18 @@ import com.intellij.psi.SingleRootFileViewProvider
 import com.intellij.psi.impl.PsiFileFactoryImpl
 import com.intellij.psi.stubs.PsiFileStub
 import com.intellij.testFramework.LightVirtualFile
-import org.jetbrains.kotlin.config.LanguageVersionSettings
-import org.jetbrains.kotlin.descriptors.ModuleDescriptor
-import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
 import org.jetbrains.kotlin.ide.konan.decompiler.KotlinNativeLoadingMetadataCache
 import org.jetbrains.kotlin.idea.KotlinFileType
 import org.jetbrains.kotlin.idea.KotlinLanguage
-import org.jetbrains.kotlin.idea.caches.project.LibraryInfo
 import org.jetbrains.kotlin.idea.decompiler.textBuilder.LoggingErrorReporter
-import org.jetbrains.kotlin.konan.file.File
-import org.jetbrains.kotlin.konan.util.KonanFactories.DefaultPackageFragmentsFactory
-import org.jetbrains.kotlin.konan.library.KonanLibrary
-import org.jetbrains.kotlin.konan.library.KonanLibraryLayout
-import org.jetbrains.kotlin.konan.library.KonanLibrarySource.KonanLibraryDir
-import org.jetbrains.kotlin.konan.library.KonanLibrarySource.KonanLibraryFile
-import org.jetbrains.kotlin.konan.library.MetadataReader
-import org.jetbrains.kotlin.konan.library.createKonanLibrary
-import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf
+import org.jetbrains.kotlin.library.KotlinLibrary
+import org.jetbrains.kotlin.library.impl.BaseKotlinLibraryImpl
+import org.jetbrains.kotlin.library.impl.MetadataLibraryImpl
+import org.jetbrains.kotlin.library.metadata.KlibMetadataProtoBuf
+import org.jetbrains.kotlin.library.metadata.PackageAccessHandler
+import org.jetbrains.kotlin.metadata.ProtoBuf
 import org.jetbrains.kotlin.psi.stubs.elements.KtStubElementTypes
-import org.jetbrains.kotlin.resolve.CompilerDeserializationConfiguration
-import org.jetbrains.kotlin.storage.StorageManager
-
-const val KOTLIN_NATIVE_CURRENT_ABI_VERSION = 1
+import org.jetbrains.kotlin.konan.file.File as KFile

 fun createFileStub(project: Project, text: String): PsiFileStub<*> {
    val virtualFile = LightVirtualFile("dummy.kt", KotlinFileType.INSTANCE, text)
@@ -49,62 +39,48 @@ fun createFileStub(project: Project, text: String): PsiFileStub<*> {

 fun createLoggingErrorReporter(log: Logger) = LoggingErrorReporter(log)

-fun KonanLibrary.createPackageFragmentProvider(
-    storageManager: StorageManager,
-    languageVersionSettings: LanguageVersionSettings,
-    moduleDescriptor: ModuleDescriptor
-): PackageFragmentProvider {
-
-    val library = this
-
-    val libraryProto = library.moduleHeaderData
-
-    //TODO: Is it required somehow?
-    //val moduleName = Name.special(libraryProto.moduleName)
-    //val moduleOrigin = DeserializedKonanModuleOrigin(library)
-
-    val deserializationConfiguration = CompilerDeserializationConfiguration(languageVersionSettings)
-
-    return DefaultPackageFragmentsFactory.createPackageFragmentProvider(
-        library,
-        null,
-        libraryProto.packageFragmentNameList,
-        storageManager,
-        moduleDescriptor,
-        deserializationConfiguration
-    )
-}
-
-internal object CachingIdeMetadataReaderImpl : MetadataReader {
-
-    override fun loadSerializedModule(libraryLayout: KonanLibraryLayout): KonanProtoBuf.LinkDataLibrary =
-        cache.getCachedModuleHeader(libraryLayout.getVirtualFile(libraryLayout.moduleHeaderFile))
-
-    override fun loadSerializedPackageFragment(
-        libraryLayout: KonanLibraryLayout,
-        packageFqName: String,
-        partName: String
-    ): KonanProtoBuf.LinkDataPackageFragment =
-        cache.getCachedPackageFragment(libraryLayout.getVirtualFile(libraryLayout.packageFragmentFile(packageFqName, partName)))
-
-    private fun KonanLibraryLayout.getVirtualFile(file: File): VirtualFile {
-        val source = this.source
-        return when (source) {
-            is KonanLibraryFile -> asJarFileSystemFile(source.klibFile, file)
-            is KonanLibraryDir -> asLocalFile(file)
-        }
+internal object CachingIdeKonanLibraryMetadataLoader : PackageAccessHandler {
+    override fun loadModuleHeader(
+        library: KotlinLibrary
+    ): KlibMetadataProtoBuf.Header {
+        val virtualFile = getVirtualFile(library, library.moduleHeaderFile)
+        return cache.getCachedModuleHeader(virtualFile)
    }

-    private fun asJarFileSystemFile(jarFile: File, localFile: File): VirtualFile {
+    override fun loadPackageFragment(
+        library: KotlinLibrary,
+        packageFqName: String,
+        partName: String
+    ): ProtoBuf.PackageFragment {
+        val virtualFile = getVirtualFile(library, library.packageFragmentFile(packageFqName, partName))
+        return cache.getCachedPackageFragment(virtualFile)
+    }
+
+    private fun getVirtualFile(library: KotlinLibrary, file: KFile): VirtualFile =
+        if (library.isZipped) asJarFileSystemFile(library.libraryFile, file) else asLocalFile(file)
+
+    private fun asJarFileSystemFile(jarFile: KFile, localFile: KFile): VirtualFile {
        val fullPath = jarFile.absolutePath + "!" + localFile.absolutePath
        return StandardFileSystems.jar().findFileByPath(fullPath) ?: error("File not found: $fullPath")
    }

-    private fun asLocalFile(localFile: File): VirtualFile {
+    private fun asLocalFile(localFile: KFile): VirtualFile {
        val fullPath = localFile.absolutePath
        return StandardFileSystems.local().findFileByPath(fullPath) ?: error("File not found: $fullPath")
    }

    private val cache
        get() = KotlinNativeLoadingMetadataCache.getInstance()
+
+
+    private val KotlinLibrary.moduleHeaderFile get() =
+        (this as MetadataLibraryImpl).access.layout.moduleHeaderFile
+
+    private fun KotlinLibrary.packageFragmentFile(packageFqName: String, partName: String) =
+        (this as MetadataLibraryImpl).access.layout.packageFragmentFile(packageFqName, partName)
+
+    private val KotlinLibrary.isZipped get() =
+        (this as BaseKotlinLibraryImpl).access.layout.isZipped
+
 }
+
@@ -24,7 +24,8 @@ import org.jetbrains.kotlin.context.ProjectContext
 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
 import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
 import org.jetbrains.kotlin.descriptors.impl.CompositePackageFragmentProvider
-import org.jetbrains.kotlin.descriptors.konan.DeserializedKonanModuleOrigin
+import org.jetbrains.kotlin.descriptors.konan.DeserializedKlibModuleOrigin
+import org.jetbrains.kotlin.descriptors.konan.KlibModuleOrigin
 import org.jetbrains.kotlin.ide.konan.analyzer.NativeResolverForModuleFactory
 import org.jetbrains.kotlin.idea.caches.project.LibraryInfo
 import org.jetbrains.kotlin.idea.caches.project.lazyClosure
@@ -32,15 +33,47 @@ import org.jetbrains.kotlin.idea.caches.resolve.BuiltInsCacheKey
 import org.jetbrains.kotlin.idea.compiler.IDELanguageSettingsProvider
 import org.jetbrains.kotlin.konan.file.File
 import org.jetbrains.kotlin.konan.library.*
-import org.jetbrains.kotlin.konan.util.KonanFactories
+import org.jetbrains.kotlin.library.KotlinLibrary
+import org.jetbrains.kotlin.library.impl.createKotlinLibrary
+import org.jetbrains.kotlin.library.isInterop
+import org.jetbrains.kotlin.library.metadata.parseModuleHeader
 import org.jetbrains.kotlin.platform.TargetPlatform
 import org.jetbrains.kotlin.platform.impl.NativeIdePlatformKind
 import org.jetbrains.kotlin.platform.konan.KonanPlatforms
 import org.jetbrains.kotlin.resolve.CompilerDeserializationConfiguration
 import org.jetbrains.kotlin.resolve.ImplicitIntegerCoercion
 import org.jetbrains.kotlin.resolve.TargetEnvironment
+import org.jetbrains.kotlin.serialization.deserialization.DeserializationConfiguration
+import org.jetbrains.kotlin.serialization.konan.impl.KlibMetadataModuleDescriptorFactoryImpl
 import org.jetbrains.kotlin.storage.StorageManager

+fun KotlinLibrary.createPackageFragmentProvider(
+        storageManager: StorageManager,
+        languageVersionSettings: LanguageVersionSettings,
+        moduleDescriptor: ModuleDescriptor
+    ): PackageFragmentProvider {
+
+    val library = this
+
+    val libraryProto = parseModuleHeader(library.moduleHeaderData)
+
+    //TODO: Is it required somehow?
+    //val moduleName = Name.special(libraryProto.moduleName)
+    //val moduleOrigin = DeserializedKonanModuleOrigin(library)
+
+    val deserializationConfiguration = CompilerDeserializationConfiguration(languageVersionSettings)
+
+    return KonanFactories.DefaultDeserializedDescriptorFactory.createPackageFragmentProvider(
+        library,
+        null,
+        libraryProto.packageFragmentNameList,
+        storageManager,
+        moduleDescriptor,
+        deserializationConfiguration,
+        null
+    )
+}
+
 class NativePlatformKindResolution : IdePlatformKindResolution {

    override fun createLibraryInfo(project: Project, library: Library): List<LibraryInfo> {
@@ -116,32 +149,34 @@ private fun createKotlinNativeBuiltIns(moduleInfo: ModuleInfo, projectContext: P
    val builtInsModule = KonanFactories.DefaultDescriptorFactory.createDescriptorAndNewBuiltIns(
        KotlinBuiltIns.BUILTINS_MODULE_NAME,
        storageManager,
-        DeserializedKonanModuleOrigin(konanLibrary),
+        DeserializedKlibModuleOrigin(konanLibrary),
        stdlibInfo.capabilities
    )

    val languageSettings = IDELanguageSettingsProvider.getLanguageVersionSettings(stdlibInfo, project, isReleaseCoroutines = false)
    val deserializationConfiguration = CompilerDeserializationConfiguration(languageSettings)

-    val libraryProto = konanLibrary.moduleHeaderData
+    val libraryProto = parseModuleHeader(konanLibrary.moduleHeaderData)

-    val stdlibFragmentProvider = KonanFactories.DefaultPackageFragmentsFactory.createPackageFragmentProvider(
+    val stdlibFragmentProvider = KonanFactories.DefaultDeserializedDescriptorFactory.createPackageFragmentProvider(
        konanLibrary,
-        null,
+        CachingIdeKonanLibraryMetadataLoader,
        libraryProto.packageFragmentNameList,
        storageManager,
        builtInsModule,
-        deserializationConfiguration
+        deserializationConfiguration,
+        null
    )

    builtInsModule.initialize(
        CompositePackageFragmentProvider(
            listOf(
                stdlibFragmentProvider,
-                KonanFactories.DefaultPackageFragmentsFactory.createForwardDeclarationHackPackagePartProvider(
-                    storageManager,
-                    builtInsModule
-                )
+                (KonanFactories.DefaultDeserializedDescriptorFactory as KlibMetadataModuleDescriptorFactoryImpl)
+                    .createForwardDeclarationHackPackagePartProvider(
+                        storageManager,
+                        builtInsModule
+                    )
            )
        )
    )
@@ -158,11 +193,7 @@ private fun ModuleInfo.findNativeStdlib(project: Project): NativeLibraryInfo? =

 class NativeLibraryInfo(project: Project, library: Library, root: File) : LibraryInfo(project, library) {

-    private val nativeLibrary = createKonanLibrary(
-        root,
-        KOTLIN_NATIVE_CURRENT_ABI_VERSION,
-        metadataReader = CachingIdeMetadataReaderImpl
-    )
+    private val nativeLibrary = createKotlinLibrary(root)

    private val roots = listOf(root.absolutePath)

@@ -173,8 +204,8 @@ class NativeLibraryInfo(project: Project, library: Library, root: File) : Librar
    override val capabilities: Map<ModuleDescriptor.Capability<*>, Any?>
        get() = super.capabilities +
                mapOf(
-                    ImplicitIntegerCoercion.MODULE_CAPABILITY to nativeLibrary.isInterop,
-                    NATIVE_LIBRARY_CAPABILITY to nativeLibrary
+                    KlibModuleOrigin.CAPABILITY to this,
+                    ImplicitIntegerCoercion.MODULE_CAPABILITY to nativeLibrary.isInterop
                )

    override val platform: TargetPlatform
@@ -183,6 +214,6 @@ class NativeLibraryInfo(project: Project, library: Library, root: File) : Librar
    override fun toString() = "Native" + super.toString()

    companion object {
-        val NATIVE_LIBRARY_CAPABILITY = ModuleDescriptor.Capability<KonanLibrary>("KonanLibrary")
+        val NATIVE_LIBRARY_CAPABILITY = ModuleDescriptor.Capability<KotlinLibrary>("KotlinLibrary")
    }
 }
@@ -11,9 +11,10 @@ import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.util.containers.ContainerUtil.createConcurrentWeakValueMap
 import org.jetbrains.kotlin.konan.library.KLIB_METADATA_FILE_EXTENSION
 import org.jetbrains.kotlin.konan.library.KLIB_MODULE_METADATA_FILE_NAME
-import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf
-import org.jetbrains.kotlin.serialization.konan.parseModuleHeader
-import org.jetbrains.kotlin.serialization.konan.parsePackageFragment
+import org.jetbrains.kotlin.library.metadata.KlibMetadataProtoBuf
+import org.jetbrains.kotlin.library.metadata.parseModuleHeader
+import org.jetbrains.kotlin.library.metadata.parsePackageFragment
+import org.jetbrains.kotlin.metadata.ProtoBuf

 class KotlinNativeLoadingMetadataCache : BaseComponent {

@@ -32,15 +33,15 @@ class KotlinNativeLoadingMetadataCache : BaseComponent {
        constructor(virtualFile: VirtualFile) : this(virtualFile.url, virtualFile.modificationStamp)
    }

-    private val packageFragmentCache = createConcurrentWeakValueMap<CacheKey, KonanProtoBuf.LinkDataPackageFragment>()
-    private val moduleHeaderCache = createConcurrentWeakValueMap<CacheKey, KonanProtoBuf.LinkDataLibrary>()
+    private val packageFragmentCache = createConcurrentWeakValueMap<CacheKey, ProtoBuf.PackageFragment>()
+    private val moduleHeaderCache = createConcurrentWeakValueMap<CacheKey, KlibMetadataProtoBuf.Header>()

-    fun getCachedPackageFragment(virtualFile: VirtualFile): KonanProtoBuf.LinkDataPackageFragment =
+    fun getCachedPackageFragment(virtualFile: VirtualFile): ProtoBuf.PackageFragment =
        packageFragmentCache.computeIfAbsent(CacheKey(virtualFile.ensurePackageMetadataFile)) {
            virtualFile.createPackageFragmentCacheEntry
        }

-    fun getCachedModuleHeader(virtualFile: VirtualFile): KonanProtoBuf.LinkDataLibrary =
+    fun getCachedModuleHeader(virtualFile: VirtualFile): KlibMetadataProtoBuf.Header =
        moduleHeaderCache.computeIfAbsent(CacheKey(virtualFile.ensureModuleHeaderFile)) {
            virtualFile.createModuleHeaderCacheEntry
        }
@@ -10,12 +10,12 @@ import com.intellij.openapi.fileTypes.FileTypeConsumer
 import com.intellij.openapi.fileTypes.FileTypeFactory
 import com.intellij.openapi.vfs.VirtualFile
 import org.jetbrains.kotlin.konan.library.KLIB_METADATA_FILE_EXTENSION
+import org.jetbrains.kotlin.library.metadata.KlibMetadataSerializerProtocol
 import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion
-import org.jetbrains.kotlin.serialization.konan.KonanSerializerProtocol
 import org.jetbrains.kotlin.serialization.konan.NullFlexibleTypeDeserializer

 class KotlinNativeMetadataDecompiler : KotlinNativeMetadataDecompilerBase<KotlinNativeMetadataVersion>(
-    KotlinNativeMetaFileType, { KonanSerializerProtocol }, NullFlexibleTypeDeserializer,
+    KotlinNativeMetaFileType, { KlibMetadataSerializerProtocol }, NullFlexibleTypeDeserializer,
    { KotlinNativeMetadataVersion.DEFAULT_INSTANCE },
    { KotlinNativeMetadataVersion.INVALID_VERSION },
    KotlinNativeMetaFileType.STUB_VERSION
@@ -23,7 +23,7 @@ class KotlinNativeMetadataDecompiler : KotlinNativeMetadataDecompilerBase<Kotlin

    override fun doReadFile(file: VirtualFile): FileWithMetadata? {
        val proto = KotlinNativeLoadingMetadataCache.getInstance().getCachedPackageFragment(file)
-        return FileWithMetadata.Compatible(proto, KonanSerializerProtocol) //todo: check version compatibility
+        return FileWithMetadata.Compatible(proto, KlibMetadataSerializerProtocol) //todo: check version compatibility
    }
 }

@@ -15,10 +15,10 @@ import org.jetbrains.kotlin.idea.decompiler.common.createIncompatibleAbiVersionD
 import org.jetbrains.kotlin.idea.decompiler.textBuilder.DecompiledText
 import org.jetbrains.kotlin.idea.decompiler.textBuilder.buildDecompiledText
 import org.jetbrains.kotlin.idea.decompiler.textBuilder.defaultDecompilerRendererOptions
+import org.jetbrains.kotlin.library.metadata.KlibMetadataProtoBuf
 import org.jetbrains.kotlin.metadata.ProtoBuf
 import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion
 import org.jetbrains.kotlin.metadata.deserialization.NameResolverImpl
-import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.renderer.DescriptorRenderer
 import org.jetbrains.kotlin.serialization.SerializerExtensionProtocol
@@ -99,14 +99,14 @@ sealed class FileWithMetadata {
    class Incompatible(val version: BinaryVersion) : FileWithMetadata()

    open class Compatible(
-        val proto: KonanProtoBuf.LinkDataPackageFragment,
+        val proto: ProtoBuf.PackageFragment,
        serializerProtocol: SerializerExtensionProtocol // TODO: Is it required?
    ) : FileWithMetadata() {
-        val nameResolver = NameResolverImpl(proto.stringTable, proto.nameTable)
-        val packageFqName = FqName(proto.fqName)
+        val nameResolver = NameResolverImpl(proto.strings, proto.qualifiedNames)
+        val packageFqName = FqName(proto.getExtension(KlibMetadataProtoBuf.fqName))

        open val classesToDecompile: List<ProtoBuf.Class> =
-            proto.classes.classesList.filter { proto ->
+            proto.class_List.filter { proto ->
                val classId = nameResolver.getClassId(proto.fqName)
                !classId.isNestedClass && classId !in ClassDeserializer.BLACK_LIST
            }
@@ -15,8 +15,8 @@ import org.jetbrains.kotlin.ide.konan.createLoggingErrorReporter
 import org.jetbrains.kotlin.idea.decompiler.textBuilder.DeserializerForDecompilerBase
 import org.jetbrains.kotlin.idea.decompiler.textBuilder.ResolveEverythingToKotlinAnyLocalClassifierResolver
 import org.jetbrains.kotlin.incremental.components.LookupTracker
+import org.jetbrains.kotlin.metadata.ProtoBuf
 import org.jetbrains.kotlin.metadata.deserialization.NameResolver
-import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.serialization.SerializerExtensionProtocol
 import org.jetbrains.kotlin.serialization.deserialization.*
@@ -24,7 +24,7 @@ import org.jetbrains.kotlin.serialization.deserialization.descriptors.Deserializ

 class KotlinNativeMetadataDeserializerForDecompiler(
    packageFqName: FqName,
-    private val proto: KonanProtoBuf.LinkDataPackageFragment,
+    private val proto: ProtoBuf.PackageFragment,
    private val nameResolver: NameResolver,
    serializerProtocol: SerializerExtensionProtocol,
    flexibleTypeDeserializer: FlexibleTypeDeserializer
@@ -6,20 +6,20 @@
 package org.jetbrains.kotlin.ide.konan.decompiler

 import org.jetbrains.kotlin.descriptors.SourceElement
+import org.jetbrains.kotlin.metadata.ProtoBuf
 import org.jetbrains.kotlin.metadata.deserialization.NameResolver
-import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf
 import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.serialization.deserialization.ClassData
 import org.jetbrains.kotlin.serialization.deserialization.ClassDataFinder
 import org.jetbrains.kotlin.serialization.deserialization.getClassId

 class KotlinNativeProtoBasedClassDataFinder(
-    proto: KonanProtoBuf.LinkDataPackageFragment,
+    proto: ProtoBuf.PackageFragment,
    private val nameResolver: NameResolver,
    private val classSource: (ClassId) -> SourceElement = { SourceElement.NO_SOURCE }
 ) : ClassDataFinder {
    private val classIdToProto =
-        proto.classes.classesList.associateBy { klass ->
+        proto.class_List.associateBy { klass ->
            nameResolver.getClassId(klass.fqName)
        }

@@ -9,6 +9,7 @@ import com.intellij.util.indexing.FileBasedIndex
 import org.jetbrains.kotlin.ide.konan.decompiler.KotlinNativeLoadingMetadataCache
 import org.jetbrains.kotlin.ide.konan.decompiler.KotlinNativeMetaFileType
 import org.jetbrains.kotlin.idea.vfilefinder.KotlinFileIndexBase
+import org.jetbrains.kotlin.library.metadata.KlibMetadataProtoBuf
 import org.jetbrains.kotlin.name.FqName

 object KotlinNativeMetaFileIndex : KotlinFileIndexBase<KotlinNativeMetaFileIndex>(KotlinNativeMetaFileIndex::class.java) {
@@ -27,6 +28,6 @@ object KotlinNativeMetaFileIndex : KotlinFileIndexBase<KotlinNativeMetaFileIndex
    /*todo: check version?!*/
    private val INDEXER = indexer { fileContent ->
        val fragment = KotlinNativeLoadingMetadataCache.getInstance().getCachedPackageFragment(fileContent.file)
-        FqName(fragment.fqName)
+        FqName(fragment.getExtension(KlibMetadataProtoBuf.fqName))
    }
 }