K/N: Cache library metadata in IDEA plugin

idea/idea-native/src/org/jetbrains/kotlin/ide/konan/KotlinNativePluginUtil.kt

@@ -7,33 +7,32 @@ package org.jetbrains.kotlin.ide.konan
 
 import com.intellij.openapi.diagnostic.Logger
 import com.intellij.openapi.project.Project
+import com.intellij.openapi.vfs.LocalFileSystem
+import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.psi.PsiFileFactory
 import com.intellij.psi.SingleRootFileViewProvider
 import com.intellij.psi.impl.PsiFileFactoryImpl
-import com.intellij.psi.search.GlobalSearchScope
 import com.intellij.psi.stubs.PsiFileStub
 import com.intellij.testFramework.LightVirtualFile
-
-import org.jetbrains.kotlin.analyzer.ModuleInfo
 import org.jetbrains.kotlin.config.LanguageVersionSettings
-import org.jetbrains.kotlin.context.ModuleContext
 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
 import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
+import org.jetbrains.kotlin.ide.konan.decompiler.KotlinNativeLoadingMetadataCache
 import org.jetbrains.kotlin.idea.KotlinFileType
 import org.jetbrains.kotlin.idea.KotlinLanguage
 import org.jetbrains.kotlin.idea.caches.project.LibraryInfo
 import org.jetbrains.kotlin.idea.decompiler.textBuilder.LoggingErrorReporter
 import org.jetbrains.kotlin.konan.file.File
 import org.jetbrains.kotlin.konan.library.KonanLibrary
+import org.jetbrains.kotlin.konan.library.KonanLibraryLayout
+import org.jetbrains.kotlin.konan.library.MetadataReader
 import org.jetbrains.kotlin.konan.library.createKonanLibrary
 import org.jetbrains.kotlin.konan.target.HostManager
 import org.jetbrains.kotlin.konan.util.KonanFactories.DefaultPackageFragmentsFactory
-import org.jetbrains.kotlin.psi.KtFile
+import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf
 import org.jetbrains.kotlin.psi.stubs.elements.KtStubElementTypes
 import org.jetbrains.kotlin.resolve.CompilerDeserializationConfiguration
 import org.jetbrains.kotlin.resolve.konan.platform.KonanPlatform
-import org.jetbrains.kotlin.resolve.lazy.declarations.DeclarationProviderFactoryService
-import org.jetbrains.kotlin.serialization.konan.parseModuleHeader
 import org.jetbrains.kotlin.storage.StorageManager
 
 const val KOTLIN_NATIVE_CURRENT_ABI_VERSION = 1
@@ -51,25 +50,22 @@ fun createFileStub(project: Project, text: String): PsiFileStub<*> {
 fun createLoggingErrorReporter(log: Logger) = LoggingErrorReporter(log)
 
 fun LibraryInfo.createPackageFragmentProviderForLibraryModule(
-    project: Project,
     storageManager: StorageManager,
     languageVersionSettings: LanguageVersionSettings,
     moduleDescriptor: ModuleDescriptor
 ): List<PackageFragmentProvider> {
 
-    // This is to preserve "capabilities" from the original IntelliJ LibraryInfo:
     if (this.platform != KonanPlatform) return emptyList()
 
     val libraryRoots = this.getLibraryRoots()
 
-
+    return libraryRoots.mapNotNull { libraryRoot -> File(libraryRoot).takeIf { it.exists } }.map { libraryRoot ->
-    return libraryRoots.map { root ->
         val konanLibrary =
             createKonanLibrary(
-                File(root),
+                libraryRoot,
                 KOTLIN_NATIVE_CURRENT_ABI_VERSION,
-                HostManager.host, /* TODO: Inlined from KonanPaths, and looks like a bit incorrect */
+                HostManager.host /* TODO: Inlined from KonanPaths, and looks like a bit incorrect */,
-                isDefault = false /* TODO: Is it ok? */
+                metadataReader = CachingIdeMetadataReaderImpl
             )
         konanLibrary.createPackageFragmentProvider(storageManager, languageVersionSettings, moduleDescriptor)
     }
@@ -80,8 +76,10 @@ private fun KonanLibrary.createPackageFragmentProvider(
     languageVersionSettings: LanguageVersionSettings,
     moduleDescriptor: ModuleDescriptor
 ): PackageFragmentProvider {
+
     val library = this
-    val libraryProto = parseModuleHeader(library.moduleHeaderData)
+
+    val libraryProto = library.moduleHeaderData
 
     //TODO: Is it required somehow?
     //val moduleName = Name.special(libraryProto.moduleName)
@@ -98,3 +96,21 @@ private fun KonanLibrary.createPackageFragmentProvider(
         deserializationConfiguration
     )
 }
+
+internal object CachingIdeMetadataReaderImpl : MetadataReader {
+
+    override fun loadSerializedModule(libraryLayout: KonanLibraryLayout): KonanProtoBuf.LinkDataLibrary =
+        cache.getCachedModuleHeader(libraryLayout.moduleHeaderFile.virtualFile)
+
+    override fun loadSerializedPackageFragment(
+        libraryLayout: KonanLibraryLayout,
+        packageFqName: String
+    ): KonanProtoBuf.LinkDataPackageFragment =
+        cache.getCachedPackageFragment(libraryLayout.packageFragmentFile(packageFqName).virtualFile)
+
+    private val File.virtualFile: VirtualFile
+        get() = LocalFileSystem.getInstance().refreshAndFindFileByPath(path) ?: error("File not found: $path")
+
+    private val cache
+        get() = KotlinNativeLoadingMetadataCache.getInstance()
+}

idea/idea-native/src/org/jetbrains/kotlin/ide/konan/NativePlatformKindResolution.kt

@@ -67,7 +67,11 @@ private fun createKotlinNativeBuiltIns(projectContext: ProjectContext): KotlinBu
     if (stdlib != null) {
 
         val (path, libraryInfo) = stdlib
-        val library = createKonanLibrary(File(path), KOTLIN_NATIVE_CURRENT_ABI_VERSION)
+        val library = createKonanLibrary(
+            File(path),
+            KOTLIN_NATIVE_CURRENT_ABI_VERSION,
+            metadataReader = CachingIdeMetadataReaderImpl
+        )
 
         val builtInsModule = DefaultDeserializedDescriptorFactory.createDescriptorAndNewBuiltIns(
             library,

idea/idea-native/src/org/jetbrains/kotlin/ide/konan/analyzer/NativeAnalyzerFacade.kt

@@ -62,7 +62,6 @@ object NativeAnalyzerFacade : ResolverForModuleFactory() {
 
         if (moduleInfo is LibraryInfo) {
             val libPackageFragmentProviders = moduleInfo.createPackageFragmentProviderForLibraryModule(
-                moduleContext.project,
                 moduleContext.storageManager,
                 languageVersionSettings,
                 moduleDescriptor

idea/idea-native/src/org/jetbrains/kotlin/ide/konan/decompiler/KotlinNativeLoadingMetadataCache.kt

@@ -8,26 +8,31 @@ package org.jetbrains.kotlin.ide.konan.decompiler
 import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.components.ApplicationComponent
 import com.intellij.openapi.vfs.*
-import com.intellij.util.containers.ContainerUtil.createConcurrentWeakValueMap
+import com.intellij.util.containers.ContainerUtil.createConcurrentSoftValueMap
 import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf
+import org.jetbrains.kotlin.serialization.konan.parseModuleHeader
 import org.jetbrains.kotlin.serialization.konan.parsePackageFragment
 
-class KotlinNativeDescriptorManager : ApplicationComponent {
+class KotlinNativeLoadingMetadataCache : ApplicationComponent {
 
     companion object {
         @JvmStatic
-        fun getInstance(): KotlinNativeDescriptorManager =
+        fun getInstance(): KotlinNativeLoadingMetadataCache =
-            ApplicationManager.getApplication().getComponent(KotlinNativeDescriptorManager::class.java)
+            ApplicationManager.getApplication().getComponent(KotlinNativeLoadingMetadataCache::class.java)
     }
 
-    private val protoCache = createConcurrentWeakValueMap<VirtualFile, KonanProtoBuf.LinkDataPackageFragment>()
+    private val packageFragmentCache = createConcurrentSoftValueMap<VirtualFile, KonanProtoBuf.LinkDataPackageFragment>()
+    private val moduleHeaderCache = createConcurrentSoftValueMap<VirtualFile, KonanProtoBuf.LinkDataLibrary>()
 
-    fun getCachedPackageFragment(virtualFile: VirtualFile): KonanProtoBuf.LinkDataPackageFragment {
+    fun getCachedPackageFragment(virtualFile: VirtualFile): KonanProtoBuf.LinkDataPackageFragment =
-        return protoCache.computeIfAbsent(virtualFile) {
+        packageFragmentCache.computeIfAbsent(virtualFile) {
-            val bytes = virtualFile.contentsToByteArray(false)
+            parsePackageFragment(virtualFile.contentsToByteArray(false))
-            parsePackageFragment(bytes)
+        }
+
+    fun getCachedModuleHeader(virtualFile: VirtualFile): KonanProtoBuf.LinkDataLibrary =
+        moduleHeaderCache.computeIfAbsent(virtualFile) {
+            parseModuleHeader(virtualFile.contentsToByteArray(false))
         }
-    }
 
     override fun initComponent() {
         VirtualFileManager.getInstance().addVirtualFileListener(object : VirtualFileListener {
@@ -40,6 +45,7 @@ class KotlinNativeDescriptorManager : ApplicationComponent {
     }
 
     private fun invalidateCaches(virtualFile: VirtualFile) {
-        protoCache.remove(virtualFile)
+        packageFragmentCache.remove(virtualFile)
+        moduleHeaderCache.remove(virtualFile)
     }
 }

idea/idea-native/src/org/jetbrains/kotlin/ide/konan/decompiler/KotlinNativeMetadataDecompiler.kt

@@ -23,7 +23,7 @@ class KotlinNativeMetadataDecompiler : KotlinNativeMetadataDecompilerBase<Kotlin
 ) {
 
     override fun doReadFile(file: VirtualFile): FileWithMetadata? {
-        val proto = KotlinNativeDescriptorManager.getInstance().getCachedPackageFragment(file)
+        val proto = KotlinNativeLoadingMetadataCache.getInstance().getCachedPackageFragment(file)
         return FileWithMetadata.Compatible(proto, KonanSerializerProtocol) //todo: check version compatibility
     }
 }

idea/idea-native/src/org/jetbrains/kotlin/ide/konan/index/KotlinNativeMetaFileIndex.kt

@@ -6,7 +6,7 @@
 package org.jetbrains.kotlin.ide.konan.index
 
 import com.intellij.util.indexing.FileBasedIndex
-import org.jetbrains.kotlin.ide.konan.decompiler.KotlinNativeDescriptorManager
+import org.jetbrains.kotlin.ide.konan.decompiler.KotlinNativeLoadingMetadataCache
 import org.jetbrains.kotlin.ide.konan.decompiler.KotlinNativeMetaFileType
 import org.jetbrains.kotlin.idea.vfilefinder.KotlinFileIndexBase
 import org.jetbrains.kotlin.name.FqName
@@ -26,7 +26,7 @@ object KotlinNativeMetaFileIndex : KotlinFileIndexBase<KotlinNativeMetaFileIndex
 
     /*todo: check version?!*/
     private val INDEXER = indexer { fileContent ->
-        val fragment = KotlinNativeDescriptorManager.getInstance().getCachedPackageFragment(fileContent.file)
+        val fragment = KotlinNativeLoadingMetadataCache.getInstance().getCachedPackageFragment(fileContent.file)
         FqName(fragment.fqName)
     }
 }

idea/src/META-INF/native.xml

@@ -4,7 +4,7 @@
             <implementation-class>org.jetbrains.kotlin.ide.konan.KotlinNativeApplicationComponent</implementation-class>
         </component>
         <component>
-            <implementation-class>org.jetbrains.kotlin.ide.konan.decompiler.KotlinNativeDescriptorManager</implementation-class>
+            <implementation-class>org.jetbrains.kotlin.ide.konan.decompiler.KotlinNativeLoadingMetadataCache</implementation-class>
         </component>
     </application-components>
 

konan/library-reader/src/org/jetbrains/kotlin/konan/library/KonanLibrary.kt

@@ -8,6 +8,7 @@ package org.jetbrains.kotlin.konan.library
 import org.jetbrains.kotlin.konan.file.File
 import org.jetbrains.kotlin.konan.properties.Properties
 import org.jetbrains.kotlin.konan.properties.propertyList
+import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf
 import org.jetbrains.kotlin.name.FqName
 
 const val KLIB_PROPERTY_ABI_VERSION = "abi_version"
@@ -42,8 +43,8 @@ interface KonanLibrary {
     val targetList: List<String>
 
     val dataFlowGraph: ByteArray?
-    val moduleHeaderData: ByteArray
+    val moduleHeaderData: KonanProtoBuf.LinkDataLibrary
-    fun packageMetadata(fqName: String): ByteArray
+    fun packageMetadata(packageFqName: String): KonanProtoBuf.LinkDataPackageFragment
 }
 
 val KonanLibrary.uniqueName

konan/library-reader/src/org/jetbrains/kotlin/konan/library/KonanLibraryLayout.kt

@@ -44,5 +44,6 @@ interface KonanLibraryLayout {
     val dataFlowGraphFile
         get() = File(linkdataDir, "module_data_flow_graph")
 
-    fun packageFile(packageName: String) = File(linkdataDir, if (packageName == "") "root_package.knm" else "package_$packageName.knm")
+    fun packageFragmentFile(packageFqName: String) =
+        File(linkdataDir, if (packageFqName == "") "root_package.knm" else "package_$packageFqName.knm")
 }

konan/library-reader/src/org/jetbrains/kotlin/konan/library/KonanMetadataReader.kt

@@ -5,7 +5,9 @@
 
 package org.jetbrains.kotlin.konan.library
 
+import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf
+
 interface MetadataReader {
-    fun loadSerializedModule(libraryLayout: KonanLibraryLayout): ByteArray
+    fun loadSerializedModule(libraryLayout: KonanLibraryLayout): KonanProtoBuf.LinkDataLibrary
-    fun loadSerializedPackageFragment(libraryLayout: KonanLibraryLayout, fqName: String): ByteArray
+    fun loadSerializedPackageFragment(libraryLayout: KonanLibraryLayout, packageFqName: String): KonanProtoBuf.LinkDataPackageFragment
 }

konan/library-reader/src/org/jetbrains/kotlin/konan/library/impl/DefaultMetadataReaderImpl.kt

@@ -7,12 +7,18 @@ package org.jetbrains.kotlin.konan.library.impl
 
 import org.jetbrains.kotlin.konan.library.KonanLibraryLayout
 import org.jetbrains.kotlin.konan.library.MetadataReader
+import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf
+import org.jetbrains.kotlin.serialization.konan.parseModuleHeader
+import org.jetbrains.kotlin.serialization.konan.parsePackageFragment
 
-internal object DefaultMetadataReaderImpl : MetadataReader {
+object DefaultMetadataReaderImpl : MetadataReader {
 
-    override fun loadSerializedModule(libraryLayout: KonanLibraryLayout): ByteArray =
+    override fun loadSerializedModule(libraryLayout: KonanLibraryLayout): KonanProtoBuf.LinkDataLibrary =
-        libraryLayout.moduleHeaderFile.readBytes()
+        parseModuleHeader(libraryLayout.moduleHeaderFile.readBytes())
 
-    override fun loadSerializedPackageFragment(libraryLayout: KonanLibraryLayout, fqName: String): ByteArray =
+    override fun loadSerializedPackageFragment(
-        libraryLayout.packageFile(fqName).readBytes()
+        libraryLayout: KonanLibraryLayout,
+        packageFqName: String
+    ): KonanProtoBuf.LinkDataPackageFragment =
+        parsePackageFragment(libraryLayout.packageFragmentFile(packageFqName).readBytes())
 }

konan/library-reader/src/org/jetbrains/kotlin/konan/library/impl/KonanLibraryImpl.kt

@@ -6,13 +6,17 @@
 package org.jetbrains.kotlin.konan.library.impl
 
 import org.jetbrains.kotlin.konan.file.File
-import org.jetbrains.kotlin.konan.library.*
+import org.jetbrains.kotlin.konan.library.KLIB_PROPERTY_ABI_VERSION
+import org.jetbrains.kotlin.konan.library.KLIB_PROPERTY_LINKED_OPTS
+import org.jetbrains.kotlin.konan.library.KonanLibrary
+import org.jetbrains.kotlin.konan.library.MetadataReader
 import org.jetbrains.kotlin.konan.properties.Properties
 import org.jetbrains.kotlin.konan.properties.loadProperties
 import org.jetbrains.kotlin.konan.properties.propertyList
 import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.konan.util.defaultTargetSubstitutions
 import org.jetbrains.kotlin.konan.util.substitute
+import org.jetbrains.kotlin.metadata.konan.KonanProtoBuf
 
 internal class KonanLibraryImpl(
     override val libraryFile: File,
@@ -59,9 +63,9 @@ internal class KonanLibraryImpl(
 
     override val dataFlowGraph by lazy { layout.inPlace { it.dataFlowGraphFile.let { if (it.exists) it.readBytes() else null } } }
 
-    override val moduleHeaderData: ByteArray by lazy { layout.inPlace { metadataReader.loadSerializedModule(it) } }
+    override val moduleHeaderData: KonanProtoBuf.LinkDataLibrary by lazy { layout.inPlace { metadataReader.loadSerializedModule(it) } }
 
-    override fun packageMetadata(fqName: String) = layout.inPlace { metadataReader.loadSerializedPackageFragment(it, fqName) }
+    override fun packageMetadata(packageFqName: String) = layout.inPlace { metadataReader.loadSerializedPackageFragment(it, packageFqName) }
 
     override fun toString() = "$libraryName[default=$isDefault]"
 }

konan/library-reader/src/org/jetbrains/kotlin/konan/library/resolver/impl/KonanResolvedLibraryImpl.kt

@@ -8,14 +8,13 @@ package org.jetbrains.kotlin.konan.library.resolver.impl
 import org.jetbrains.kotlin.konan.library.KonanLibrary
 import org.jetbrains.kotlin.konan.library.resolver.KonanResolvedLibrary
 import org.jetbrains.kotlin.name.FqName
-import org.jetbrains.kotlin.serialization.konan.parseModuleHeader
 
 internal class KonanResolvedLibraryImpl(
     override val library: KonanLibrary
 ) : KonanResolvedLibrary {
 
     private val _resolvedDependencies = mutableListOf<KonanResolvedLibrary>()
-    private val _emptyPackages by lazy { parseModuleHeader(library.moduleHeaderData).emptyPackageList }
+    private val _emptyPackages by lazy { library.moduleHeaderData.emptyPackageList }
 
     override val resolvedDependencies: List<KonanResolvedLibrary>
         get() = _resolvedDependencies

konan/library-reader/src/org/jetbrains/kotlin/serialization/konan/KonanDeserializedPackageFragmentsFactory.kt

@@ -29,7 +29,6 @@ interface KonanDeserializedPackageFragmentsFactory {
         moduleDescriptor: ModuleDescriptor
     ): List<PackageFragmentDescriptor>
 
-
     fun createPackageFragmentProvider(
         library: KonanLibrary,
         packageAccessedHandler: PackageAccessedHandler?,

konan/library-reader/src/org/jetbrains/kotlin/serialization/konan/KonanPackageFragment.kt

@@ -35,9 +35,7 @@ class KonanPackageFragment(
 
     // The proto field is lazy so that we can load only needed
     // packages from the library.
-    private val protoForNames: KonanProtoBuf.LinkDataPackageFragment by lazy {
+    private val protoForNames: KonanProtoBuf.LinkDataPackageFragment by lazy { library.packageMetadata(fqName.asString()) }
-        parsePackageFragment(library.packageMetadata(fqName.asString()))
-    }
 
     val proto: KonanProtoBuf.LinkDataPackageFragment
         get() = protoForNames.also { packageAccessedHandler?.markPackageAccessed(fqName) }

konan/library-reader/src/org/jetbrains/kotlin/serialization/konan/impl/KonanDeserializedModuleDescriptorFactoryImpl.kt

@@ -8,19 +8,15 @@ package org.jetbrains.kotlin.serialization.konan.impl
 import org.jetbrains.kotlin.builtins.KotlinBuiltIns
 import org.jetbrains.kotlin.config.LanguageVersionSettings
 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
-import org.jetbrains.kotlin.descriptors.NotFoundClasses
-import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
-import org.jetbrains.kotlin.descriptors.PackageFragmentProviderImpl
 import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
 import org.jetbrains.kotlin.descriptors.konan.DeserializedKonanModuleOrigin
 import org.jetbrains.kotlin.descriptors.konan.KonanModuleDescriptorFactory
-import org.jetbrains.kotlin.incremental.components.LookupTracker
 import org.jetbrains.kotlin.konan.library.KonanLibrary
 import org.jetbrains.kotlin.konan.library.resolver.PackageAccessedHandler
 import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.resolve.CompilerDeserializationConfiguration
-import org.jetbrains.kotlin.serialization.deserialization.*
+import org.jetbrains.kotlin.serialization.konan.KonanDeserializedModuleDescriptorFactory
-import org.jetbrains.kotlin.serialization.konan.*
+import org.jetbrains.kotlin.serialization.konan.KonanDeserializedPackageFragmentsFactory
 import org.jetbrains.kotlin.storage.StorageManager
 
 internal class KonanDeserializedModuleDescriptorFactoryImpl(
@@ -61,7 +57,7 @@ internal class KonanDeserializedModuleDescriptorFactoryImpl(
         customCapabilities: Map<ModuleDescriptor.Capability<*>, Any?>
     ): ModuleDescriptorImpl {
 
-        val libraryProto = parseModuleHeader(library.moduleHeaderData)
+        val libraryProto = library.moduleHeaderData
 
         val moduleName = Name.special(libraryProto.moduleName)
         val moduleOrigin = DeserializedKonanModuleOrigin(library)
@@ -86,5 +82,4 @@ internal class KonanDeserializedModuleDescriptorFactoryImpl(
 
         return moduleDescriptor
     }
-
 }

konan/library-reader/src/org/jetbrains/kotlin/serialization/konan/impl/KonanDeserializedPackageFragmentsFactoryImpl.kt

@@ -68,7 +68,6 @@ internal object KonanDeserializedPackageFragmentsFactoryImpl : KonanDeserialized
         return result
     }
 
-
     override fun createPackageFragmentProvider(
         library: KonanLibrary,
         packageAccessedHandler: PackageAccessedHandler?,