K/N: Cache library metadata in IDEA plugin

2018-09-10 14:45:24 +03:00
parent 1df91ff7ec
commit 71be94b3e4
17 changed files with 90 additions and 61 deletions
@@ -7,33 +7,32 @@ package org.jetbrains.kotlin.ide.konan

 import com.intellij.openapi.diagnostic.Logger
 import com.intellij.openapi.project.Project
+import com.intellij.openapi.vfs.LocalFileSystem
+import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.psi.PsiFileFactory
 import com.intellij.psi.SingleRootFileViewProvider
 import com.intellij.psi.impl.PsiFileFactoryImpl
-import com.intellij.psi.search.GlobalSearchScope
 import com.intellij.psi.stubs.PsiFileStub
 import com.intellij.testFramework.LightVirtualFile
-
-import org.jetbrains.kotlin.analyzer.ModuleInfo
 import org.jetbrains.kotlin.config.LanguageVersionSettings
-import org.jetbrains.kotlin.context.ModuleContext
 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
 import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
+import org.jetbrains.kotlin.ide.konan.decompiler.KotlinNativeLoadingMetadataCache
 import org.jetbrains.kotlin.idea.KotlinFileType
 import org.jetbrains.kotlin.idea.KotlinLanguage
 import org.jetbrains.kotlin.idea.caches.project.LibraryInfo
 import org.jetbrains.kotlin.idea.decompiler.textBuilder.LoggingErrorReporter
 import org.jetbrains.kotlin.konan.file.File
 import org.jetbrains.kotlin.konan.library.KonanLibrary
+import org.jetbrains.kotlin.konan.library.KonanLibraryLayout
+import org.jetbrains.kotlin.konan.library.MetadataReader
 import org.jetbrains.kotlin.konan.library.createKonanLibrary
 import org.jetbrains.kotlin.konan.target.HostManager
 import org.jetbrains.kotlin.konan.util.KonanFactories.DefaultPackageFragmentsFactory
-import org.jetbrains.kotlin.psi.KtFile
+import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf
 import org.jetbrains.kotlin.psi.stubs.elements.KtStubElementTypes
 import org.jetbrains.kotlin.resolve.CompilerDeserializationConfiguration
 import org.jetbrains.kotlin.resolve.konan.platform.KonanPlatform
-import org.jetbrains.kotlin.resolve.lazy.declarations.DeclarationProviderFactoryService
-import org.jetbrains.kotlin.serialization.konan.parseModuleHeader
 import org.jetbrains.kotlin.storage.StorageManager

 const val KOTLIN_NATIVE_CURRENT_ABI_VERSION = 1
@@ -51,25 +50,22 @@ fun createFileStub(project: Project, text: String): PsiFileStub<*> {
 fun createLoggingErrorReporter(log: Logger) = LoggingErrorReporter(log)

 fun LibraryInfo.createPackageFragmentProviderForLibraryModule(
-    project: Project,
    storageManager: StorageManager,
    languageVersionSettings: LanguageVersionSettings,
    moduleDescriptor: ModuleDescriptor
 ): List<PackageFragmentProvider> {

-    // This is to preserve "capabilities" from the original IntelliJ LibraryInfo:
    if (this.platform != KonanPlatform) return emptyList()

    val libraryRoots = this.getLibraryRoots()

-
-    return libraryRoots.map { root ->
+    return libraryRoots.mapNotNull { libraryRoot -> File(libraryRoot).takeIf { it.exists } }.map { libraryRoot ->
        val konanLibrary =
            createKonanLibrary(
-                File(root),
+                libraryRoot,
                KOTLIN_NATIVE_CURRENT_ABI_VERSION,
-                HostManager.host, /* TODO: Inlined from KonanPaths, and looks like a bit incorrect */
-                isDefault = false /* TODO: Is it ok? */
+                HostManager.host /* TODO: Inlined from KonanPaths, and looks like a bit incorrect */,
+                metadataReader = CachingIdeMetadataReaderImpl
            )
        konanLibrary.createPackageFragmentProvider(storageManager, languageVersionSettings, moduleDescriptor)
    }
@@ -80,8 +76,10 @@ private fun KonanLibrary.createPackageFragmentProvider(
    languageVersionSettings: LanguageVersionSettings,
    moduleDescriptor: ModuleDescriptor
 ): PackageFragmentProvider {
+
    val library = this
-    val libraryProto = parseModuleHeader(library.moduleHeaderData)
+
+    val libraryProto = library.moduleHeaderData

    //TODO: Is it required somehow?
    //val moduleName = Name.special(libraryProto.moduleName)
@@ -98,3 +96,21 @@ private fun KonanLibrary.createPackageFragmentProvider(
        deserializationConfiguration
    )
 }
+
+internal object CachingIdeMetadataReaderImpl : MetadataReader {
+
+    override fun loadSerializedModule(libraryLayout: KonanLibraryLayout): KonanProtoBuf.LinkDataLibrary =
+        cache.getCachedModuleHeader(libraryLayout.moduleHeaderFile.virtualFile)
+
+    override fun loadSerializedPackageFragment(
+        libraryLayout: KonanLibraryLayout,
+        packageFqName: String
+    ): KonanProtoBuf.LinkDataPackageFragment =
+        cache.getCachedPackageFragment(libraryLayout.packageFragmentFile(packageFqName).virtualFile)
+
+    private val File.virtualFile: VirtualFile
+        get() = LocalFileSystem.getInstance().refreshAndFindFileByPath(path) ?: error("File not found: $path")
+
+    private val cache
+        get() = KotlinNativeLoadingMetadataCache.getInstance()
+}
@@ -67,7 +67,11 @@ private fun createKotlinNativeBuiltIns(projectContext: ProjectContext): KotlinBu
    if (stdlib != null) {

        val (path, libraryInfo) = stdlib
-        val library = createKonanLibrary(File(path), KOTLIN_NATIVE_CURRENT_ABI_VERSION)
+        val library = createKonanLibrary(
+            File(path),
+            KOTLIN_NATIVE_CURRENT_ABI_VERSION,
+            metadataReader = CachingIdeMetadataReaderImpl
+        )

        val builtInsModule = DefaultDeserializedDescriptorFactory.createDescriptorAndNewBuiltIns(
            library,
@@ -62,7 +62,6 @@ object NativeAnalyzerFacade : ResolverForModuleFactory() {

        if (moduleInfo is LibraryInfo) {
            val libPackageFragmentProviders = moduleInfo.createPackageFragmentProviderForLibraryModule(
-                moduleContext.project,
                moduleContext.storageManager,
                languageVersionSettings,
                moduleDescriptor
@@ -8,26 +8,31 @@ package org.jetbrains.kotlin.ide.konan.decompiler
 import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.components.ApplicationComponent
 import com.intellij.openapi.vfs.*
-import com.intellij.util.containers.ContainerUtil.createConcurrentWeakValueMap
+import com.intellij.util.containers.ContainerUtil.createConcurrentSoftValueMap
 import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf
+import org.jetbrains.kotlin.serialization.konan.parseModuleHeader
 import org.jetbrains.kotlin.serialization.konan.parsePackageFragment

-class KotlinNativeDescriptorManager : ApplicationComponent {
+class KotlinNativeLoadingMetadataCache : ApplicationComponent {

    companion object {
        @JvmStatic
-        fun getInstance(): KotlinNativeDescriptorManager =
-            ApplicationManager.getApplication().getComponent(KotlinNativeDescriptorManager::class.java)
+        fun getInstance(): KotlinNativeLoadingMetadataCache =
+            ApplicationManager.getApplication().getComponent(KotlinNativeLoadingMetadataCache::class.java)
    }

-    private val protoCache = createConcurrentWeakValueMap<VirtualFile, KonanProtoBuf.LinkDataPackageFragment>()
+    private val packageFragmentCache = createConcurrentSoftValueMap<VirtualFile, KonanProtoBuf.LinkDataPackageFragment>()
+    private val moduleHeaderCache = createConcurrentSoftValueMap<VirtualFile, KonanProtoBuf.LinkDataLibrary>()

-    fun getCachedPackageFragment(virtualFile: VirtualFile): KonanProtoBuf.LinkDataPackageFragment {
-        return protoCache.computeIfAbsent(virtualFile) {
-            val bytes = virtualFile.contentsToByteArray(false)
-            parsePackageFragment(bytes)
+    fun getCachedPackageFragment(virtualFile: VirtualFile): KonanProtoBuf.LinkDataPackageFragment =
+        packageFragmentCache.computeIfAbsent(virtualFile) {
+            parsePackageFragment(virtualFile.contentsToByteArray(false))
+        }
+
+    fun getCachedModuleHeader(virtualFile: VirtualFile): KonanProtoBuf.LinkDataLibrary =
+        moduleHeaderCache.computeIfAbsent(virtualFile) {
+            parseModuleHeader(virtualFile.contentsToByteArray(false))
        }
-    }

    override fun initComponent() {
        VirtualFileManager.getInstance().addVirtualFileListener(object : VirtualFileListener {
@@ -40,6 +45,7 @@ class KotlinNativeDescriptorManager : ApplicationComponent {
    }

    private fun invalidateCaches(virtualFile: VirtualFile) {
-        protoCache.remove(virtualFile)
+        packageFragmentCache.remove(virtualFile)
+        moduleHeaderCache.remove(virtualFile)
    }
 }
@@ -23,7 +23,7 @@ class KotlinNativeMetadataDecompiler : KotlinNativeMetadataDecompilerBase<Kotlin
 ) {

    override fun doReadFile(file: VirtualFile): FileWithMetadata? {
-        val proto = KotlinNativeDescriptorManager.getInstance().getCachedPackageFragment(file)
+        val proto = KotlinNativeLoadingMetadataCache.getInstance().getCachedPackageFragment(file)
        return FileWithMetadata.Compatible(proto, KonanSerializerProtocol) //todo: check version compatibility
    }
 }
@@ -6,7 +6,7 @@
 package org.jetbrains.kotlin.ide.konan.index

 import com.intellij.util.indexing.FileBasedIndex
-import org.jetbrains.kotlin.ide.konan.decompiler.KotlinNativeDescriptorManager
+import org.jetbrains.kotlin.ide.konan.decompiler.KotlinNativeLoadingMetadataCache
 import org.jetbrains.kotlin.ide.konan.decompiler.KotlinNativeMetaFileType
 import org.jetbrains.kotlin.idea.vfilefinder.KotlinFileIndexBase
 import org.jetbrains.kotlin.name.FqName
@@ -26,7 +26,7 @@ object KotlinNativeMetaFileIndex : KotlinFileIndexBase<KotlinNativeMetaFileIndex

    /*todo: check version?!*/
    private val INDEXER = indexer { fileContent ->
-        val fragment = KotlinNativeDescriptorManager.getInstance().getCachedPackageFragment(fileContent.file)
+        val fragment = KotlinNativeLoadingMetadataCache.getInstance().getCachedPackageFragment(fileContent.file)
        FqName(fragment.fqName)
    }
 }
@@ -4,7 +4,7 @@
            <implementation-class>org.jetbrains.kotlin.ide.konan.KotlinNativeApplicationComponent</implementation-class>
        </component>
        <component>
-            <implementation-class>org.jetbrains.kotlin.ide.konan.decompiler.KotlinNativeDescriptorManager</implementation-class>
+            <implementation-class>org.jetbrains.kotlin.ide.konan.decompiler.KotlinNativeLoadingMetadataCache</implementation-class>
        </component>
    </application-components>

@@ -8,6 +8,7 @@ package org.jetbrains.kotlin.konan.library
 import org.jetbrains.kotlin.konan.file.File
 import org.jetbrains.kotlin.konan.properties.Properties
 import org.jetbrains.kotlin.konan.properties.propertyList
+import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf
 import org.jetbrains.kotlin.name.FqName

 const val KLIB_PROPERTY_ABI_VERSION = "abi_version"
@@ -42,8 +43,8 @@ interface KonanLibrary {
    val targetList: List<String>

    val dataFlowGraph: ByteArray?
-    val moduleHeaderData: ByteArray
-    fun packageMetadata(fqName: String): ByteArray
+    val moduleHeaderData: KonanProtoBuf.LinkDataLibrary
+    fun packageMetadata(packageFqName: String): KonanProtoBuf.LinkDataPackageFragment
 }

 val KonanLibrary.uniqueName
@@ -44,5 +44,6 @@ interface KonanLibraryLayout {
    val dataFlowGraphFile
        get() = File(linkdataDir, "module_data_flow_graph")

-    fun packageFile(packageName: String) = File(linkdataDir, if (packageName == "") "root_package.knm" else "package_$packageName.knm")
+    fun packageFragmentFile(packageFqName: String) =
+        File(linkdataDir, if (packageFqName == "") "root_package.knm" else "package_$packageFqName.knm")
 }
@@ -5,7 +5,9 @@

 package org.jetbrains.kotlin.konan.library

+import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf
+
 interface MetadataReader {
-    fun loadSerializedModule(libraryLayout: KonanLibraryLayout): ByteArray
-    fun loadSerializedPackageFragment(libraryLayout: KonanLibraryLayout, fqName: String): ByteArray
+    fun loadSerializedModule(libraryLayout: KonanLibraryLayout): KonanProtoBuf.LinkDataLibrary
+    fun loadSerializedPackageFragment(libraryLayout: KonanLibraryLayout, packageFqName: String): KonanProtoBuf.LinkDataPackageFragment
 }
@@ -7,12 +7,18 @@ package org.jetbrains.kotlin.konan.library.impl

 import org.jetbrains.kotlin.konan.library.KonanLibraryLayout
 import org.jetbrains.kotlin.konan.library.MetadataReader
+import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf
+import org.jetbrains.kotlin.serialization.konan.parseModuleHeader
+import org.jetbrains.kotlin.serialization.konan.parsePackageFragment

-internal object DefaultMetadataReaderImpl : MetadataReader {
+object DefaultMetadataReaderImpl : MetadataReader {

-    override fun loadSerializedModule(libraryLayout: KonanLibraryLayout): ByteArray =
-        libraryLayout.moduleHeaderFile.readBytes()
+    override fun loadSerializedModule(libraryLayout: KonanLibraryLayout): KonanProtoBuf.LinkDataLibrary =
+        parseModuleHeader(libraryLayout.moduleHeaderFile.readBytes())

-    override fun loadSerializedPackageFragment(libraryLayout: KonanLibraryLayout, fqName: String): ByteArray =
-        libraryLayout.packageFile(fqName).readBytes()
+    override fun loadSerializedPackageFragment(
+        libraryLayout: KonanLibraryLayout,
+        packageFqName: String
+    ): KonanProtoBuf.LinkDataPackageFragment =
+        parsePackageFragment(libraryLayout.packageFragmentFile(packageFqName).readBytes())
 }
@@ -6,13 +6,17 @@
 package org.jetbrains.kotlin.konan.library.impl

 import org.jetbrains.kotlin.konan.file.File
-import org.jetbrains.kotlin.konan.library.*
+import org.jetbrains.kotlin.konan.library.KLIB_PROPERTY_ABI_VERSION
+import org.jetbrains.kotlin.konan.library.KLIB_PROPERTY_LINKED_OPTS
+import org.jetbrains.kotlin.konan.library.KonanLibrary
+import org.jetbrains.kotlin.konan.library.MetadataReader
 import org.jetbrains.kotlin.konan.properties.Properties
 import org.jetbrains.kotlin.konan.properties.loadProperties
 import org.jetbrains.kotlin.konan.properties.propertyList
 import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.konan.util.defaultTargetSubstitutions
 import org.jetbrains.kotlin.konan.util.substitute
+import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf

 internal class KonanLibraryImpl(
    override val libraryFile: File,
@@ -59,9 +63,9 @@ internal class KonanLibraryImpl(

    override val dataFlowGraph by lazy { layout.inPlace { it.dataFlowGraphFile.let { if (it.exists) it.readBytes() else null } } }

-    override val moduleHeaderData: ByteArray by lazy { layout.inPlace { metadataReader.loadSerializedModule(it) } }
+    override val moduleHeaderData: KonanProtoBuf.LinkDataLibrary by lazy { layout.inPlace { metadataReader.loadSerializedModule(it) } }

-    override fun packageMetadata(fqName: String) = layout.inPlace { metadataReader.loadSerializedPackageFragment(it, fqName) }
+    override fun packageMetadata(packageFqName: String) = layout.inPlace { metadataReader.loadSerializedPackageFragment(it, packageFqName) }

    override fun toString() = "$libraryName[default=$isDefault]"
-}
+}
@@ -8,14 +8,13 @@ package org.jetbrains.kotlin.konan.library.resolver.impl
 import org.jetbrains.kotlin.konan.library.KonanLibrary
 import org.jetbrains.kotlin.konan.library.resolver.KonanResolvedLibrary
 import org.jetbrains.kotlin.name.FqName
-import org.jetbrains.kotlin.serialization.konan.parseModuleHeader

 internal class KonanResolvedLibraryImpl(
    override val library: KonanLibrary
 ) : KonanResolvedLibrary {

    private val _resolvedDependencies = mutableListOf<KonanResolvedLibrary>()
-    private val _emptyPackages by lazy { parseModuleHeader(library.moduleHeaderData).emptyPackageList }
+    private val _emptyPackages by lazy { library.moduleHeaderData.emptyPackageList }

    override val resolvedDependencies: List<KonanResolvedLibrary>
        get() = _resolvedDependencies
@@ -29,7 +29,6 @@ interface KonanDeserializedPackageFragmentsFactory {
        moduleDescriptor: ModuleDescriptor
    ): List<PackageFragmentDescriptor>

-
    fun createPackageFragmentProvider(
        library: KonanLibrary,
        packageAccessedHandler: PackageAccessedHandler?,
@@ -35,9 +35,7 @@ class KonanPackageFragment(

    // The proto field is lazy so that we can load only needed
    // packages from the library.
-    private val protoForNames: KonanProtoBuf.LinkDataPackageFragment by lazy {
-        parsePackageFragment(library.packageMetadata(fqName.asString()))
-    }
+    private val protoForNames: KonanProtoBuf.LinkDataPackageFragment by lazy { library.packageMetadata(fqName.asString()) }

    val proto: KonanProtoBuf.LinkDataPackageFragment
        get() = protoForNames.also { packageAccessedHandler?.markPackageAccessed(fqName) }
@@ -8,19 +8,15 @@ package org.jetbrains.kotlin.serialization.konan.impl
 import org.jetbrains.kotlin.builtins.KotlinBuiltIns
 import org.jetbrains.kotlin.config.LanguageVersionSettings
 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
-import org.jetbrains.kotlin.descriptors.NotFoundClasses
-import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
-import org.jetbrains.kotlin.descriptors.PackageFragmentProviderImpl
 import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
 import org.jetbrains.kotlin.descriptors.konan.DeserializedKonanModuleOrigin
 import org.jetbrains.kotlin.descriptors.konan.KonanModuleDescriptorFactory
-import org.jetbrains.kotlin.incremental.components.LookupTracker
 import org.jetbrains.kotlin.konan.library.KonanLibrary
 import org.jetbrains.kotlin.konan.library.resolver.PackageAccessedHandler
 import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.resolve.CompilerDeserializationConfiguration
-import org.jetbrains.kotlin.serialization.deserialization.*
-import org.jetbrains.kotlin.serialization.konan.*
+import org.jetbrains.kotlin.serialization.konan.KonanDeserializedModuleDescriptorFactory
+import org.jetbrains.kotlin.serialization.konan.KonanDeserializedPackageFragmentsFactory
 import org.jetbrains.kotlin.storage.StorageManager

 internal class KonanDeserializedModuleDescriptorFactoryImpl(
@@ -61,7 +57,7 @@ internal class KonanDeserializedModuleDescriptorFactoryImpl(
        customCapabilities: Map<ModuleDescriptor.Capability<*>, Any?>
    ): ModuleDescriptorImpl {

-        val libraryProto = parseModuleHeader(library.moduleHeaderData)
+        val libraryProto = library.moduleHeaderData

        val moduleName = Name.special(libraryProto.moduleName)
        val moduleOrigin = DeserializedKonanModuleOrigin(library)
@@ -86,5 +82,4 @@ internal class KonanDeserializedModuleDescriptorFactoryImpl(

        return moduleDescriptor
    }
-
 }
@@ -68,7 +68,6 @@ internal object KonanDeserializedPackageFragmentsFactoryImpl : KonanDeserialized
        return result
    }

-
    override fun createPackageFragmentProvider(
        library: KonanLibrary,
        packageAccessedHandler: PackageAccessedHandler?,