Refactor NativeAnalyzerFacade to reuse existing library dependency model

Fix memory blowing up in quadratic native stdlib dependencies
Reuse ModuleInfo & LibraryInfo for Native properly

idea/idea-native/src/org/jetbrains/kotlin/ide/konan/KotlinNativePluginUtil.kt

@@ -10,20 +10,30 @@ import com.intellij.openapi.project.Project
 import com.intellij.psi.PsiFileFactory
 import com.intellij.psi.SingleRootFileViewProvider
 import com.intellij.psi.impl.PsiFileFactoryImpl
+import com.intellij.psi.search.GlobalSearchScope
 import com.intellij.psi.stubs.PsiFileStub
 import com.intellij.testFramework.LightVirtualFile
+
 import org.jetbrains.kotlin.analyzer.ModuleInfo
-import org.jetbrains.kotlin.builtins.KotlinBuiltIns
 import org.jetbrains.kotlin.config.LanguageVersionSettings
+import org.jetbrains.kotlin.context.ModuleContext
+import org.jetbrains.kotlin.descriptors.ModuleDescriptor
+import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
 import org.jetbrains.kotlin.idea.KotlinFileType
 import org.jetbrains.kotlin.idea.KotlinLanguage
 import org.jetbrains.kotlin.idea.caches.project.LibraryInfo
 import org.jetbrains.kotlin.idea.decompiler.textBuilder.LoggingErrorReporter
-import org.jetbrains.kotlin.konan.library.libraryResolver
+import org.jetbrains.kotlin.konan.file.File
-import org.jetbrains.kotlin.konan.util.KonanFactories.DefaultResolvedDescriptorsFactory
+import org.jetbrains.kotlin.konan.library.KonanLibrary
+import org.jetbrains.kotlin.konan.library.createKonanLibrary
+import org.jetbrains.kotlin.konan.target.HostManager
+import org.jetbrains.kotlin.konan.util.KonanFactories.DefaultPackageFragmentsFactory
+import org.jetbrains.kotlin.psi.KtFile
 import org.jetbrains.kotlin.psi.stubs.elements.KtStubElementTypes
+import org.jetbrains.kotlin.resolve.CompilerDeserializationConfiguration
 import org.jetbrains.kotlin.resolve.konan.platform.KonanPlatform
-import org.jetbrains.kotlin.serialization.konan.KonanResolvedModuleDescriptors
+import org.jetbrains.kotlin.resolve.lazy.declarations.DeclarationProviderFactoryService
+import org.jetbrains.kotlin.serialization.konan.parseModuleHeader
 import org.jetbrains.kotlin.storage.StorageManager
 
 const val KOTLIN_NATIVE_CURRENT_ABI_VERSION = 1
@@ -40,36 +50,51 @@ fun createFileStub(project: Project, text: String): PsiFileStub<*> {
 
 fun createLoggingErrorReporter(log: Logger) = LoggingErrorReporter(log)
 
-fun ModuleInfo.createResolvedModuleDescriptors(
+fun LibraryInfo.createPackageFragmentProviderForLibraryModule(
+    project: Project,
     storageManager: StorageManager,
-    builtIns: KotlinBuiltIns,
+    languageVersionSettings: LanguageVersionSettings,
-    languageVersionSettings: LanguageVersionSettings
+    moduleDescriptor: ModuleDescriptor
-): KonanResolvedModuleDescriptors {
+): List<PackageFragmentProvider> {
 
-    // This is to preserve "capabilities" from the original IDEA LibraryInfo:
+    // This is to preserve "capabilities" from the original IntelliJ LibraryInfo:
-    val libraryMap =
+    if (this.platform != KonanPlatform) return emptyList()
-        this.dependencies().filterIsInstance<LibraryInfo>().flatMap { dependency ->
-            if (dependency.platform == KonanPlatform) {
-                dependency.getLibraryRoots().map { libraryRoot ->
-                    libraryRoot to dependency
-                }
-            } else {
-                emptyList()
-            }
-        }.toMap()
 
-    val libraryPaths = libraryMap.keys.toList()
+    val libraryRoots = this.getLibraryRoots()
 
-    val resolvedLibraries = KotlinNativePluginSearchPathResolver(libraryPaths)
-        .libraryResolver(KOTLIN_NATIVE_CURRENT_ABI_VERSION)
-        .resolveWithDependencies(libraryPaths)
 
-    return DefaultResolvedDescriptorsFactory.createResolved(
+    return libraryRoots.map { root ->
-        resolvedLibraries,
+        val konanLibrary =
-        storageManager,
+            createKonanLibrary(
-        builtIns,
+                File(root),
-        languageVersionSettings,
+                KOTLIN_NATIVE_CURRENT_ABI_VERSION,
-        null
+                HostManager.host, /* TODO: Inlined from KonanPaths, and looks like a bit incorrect */
-        // This is to preserve "capabilities" from the original IDEA LibraryInfo:
+                isDefault = false /* TODO: Is it ok? */
-    ) { library -> libraryMap[library.libraryFile.path]?.capabilities ?: emptyMap() }
+            )
+        konanLibrary.createPackageFragmentProvider(storageManager, languageVersionSettings, moduleDescriptor)
+    }
+}
+
+private fun KonanLibrary.createPackageFragmentProvider(
+    storageManager: StorageManager,
+    languageVersionSettings: LanguageVersionSettings,
+    moduleDescriptor: ModuleDescriptor
+): PackageFragmentProvider {
+    val library = this
+    val libraryProto = parseModuleHeader(library.moduleHeaderData)
+
+    //TODO: Is it required somehow?
+    //val moduleName = Name.special(libraryProto.moduleName)
+    //val moduleOrigin = DeserializedKonanModuleOrigin(library)
+
+    val deserializationConfiguration = CompilerDeserializationConfiguration(languageVersionSettings)
+
+    return DefaultPackageFragmentsFactory.createPackageFragmentProvider(
+        library,
+        null,
+        libraryProto.packageFragmentNameList,
+        storageManager,
+        moduleDescriptor,
+        deserializationConfiguration
+    )
 }

idea/idea-native/src/org/jetbrains/kotlin/ide/konan/analyzer/NativeAnalyzerFacade.kt

@@ -13,7 +13,8 @@ import org.jetbrains.kotlin.context.ModuleContext
 import org.jetbrains.kotlin.descriptors.impl.CompositePackageFragmentProvider
 import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
 import org.jetbrains.kotlin.frontend.di.createContainerForLazyResolve
-import org.jetbrains.kotlin.ide.konan.createResolvedModuleDescriptors
+import org.jetbrains.kotlin.ide.konan.createPackageFragmentProviderForLibraryModule
+import org.jetbrains.kotlin.idea.caches.project.LibraryInfo
 import org.jetbrains.kotlin.resolve.BindingTraceContext
 import org.jetbrains.kotlin.resolve.TargetEnvironment
 import org.jetbrains.kotlin.resolve.TargetPlatform
@@ -40,7 +41,7 @@ object NativeAnalyzerFacade : ResolverForModuleFactory() {
             moduleContext.project,
             moduleContext.storageManager,
             moduleContent.syntheticFiles,
-            moduleContent.moduleContentScope!!,
+            moduleContent.moduleContentScope,
             moduleContent.moduleInfo
         )
 
@@ -54,16 +55,21 @@ object NativeAnalyzerFacade : ResolverForModuleFactory() {
             languageVersionSettings
         )
 
-        val moduleDescriptors = moduleContent.moduleInfo.createResolvedModuleDescriptors(
-            moduleContext.storageManager,
-            moduleContext.module.builtIns,
-            languageVersionSettings
-        )
-
         val packageFragmentProvider = container.get<ResolveSession>().packageFragmentProvider
         val fragmentProviders = mutableListOf(packageFragmentProvider)
-        moduleDescriptors.resolvedDescriptors.mapTo(fragmentProviders) { it.packageFragmentProvider }
+
-        fragmentProviders.add(moduleDescriptors.forwardDeclarationsModule.packageFragmentProvider)
+        val moduleInfo = moduleContent.moduleInfo
+
+        if (moduleInfo is LibraryInfo) {
+            val libPackageFragmentProviders = moduleInfo.createPackageFragmentProviderForLibraryModule(
+                moduleContext.project,
+                moduleContext.storageManager,
+                languageVersionSettings,
+                moduleDescriptor
+            )
+            fragmentProviders.addAll(libPackageFragmentProviders)
+            //TODO: Forward declarations?
+        }
 
         return ResolverForModule(CompositePackageFragmentProvider(fragmentProviders), container)
     }

konan/library-reader/src/org/jetbrains/kotlin/serialization/konan/KonanDeserializedPackageFragmentsFactory.kt

@@ -7,8 +7,10 @@ package org.jetbrains.kotlin.serialization.konan
 
 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
 import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor
+import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
 import org.jetbrains.kotlin.konan.library.KonanLibrary
 import org.jetbrains.kotlin.konan.library.resolver.PackageAccessedHandler
+import org.jetbrains.kotlin.serialization.deserialization.DeserializationConfiguration
 import org.jetbrains.kotlin.storage.StorageManager
 
 interface KonanDeserializedPackageFragmentsFactory {
@@ -26,4 +28,14 @@ interface KonanDeserializedPackageFragmentsFactory {
         deserializedPackageFragments: List<KonanPackageFragment>,
         moduleDescriptor: ModuleDescriptor
     ): List<PackageFragmentDescriptor>
+
+
+    fun createPackageFragmentProvider(
+        library: KonanLibrary,
+        packageAccessedHandler: PackageAccessedHandler?,
+        packageFragmentNames: List<String>,
+        storageManager: StorageManager,
+        moduleDescriptor: ModuleDescriptor,
+        configuration: DeserializationConfiguration
+    ): PackageFragmentProvider
 }

konan/library-reader/src/org/jetbrains/kotlin/serialization/konan/impl/KonanDeserializedModuleDescriptorFactoryImpl.kt

@@ -73,7 +73,7 @@ internal class KonanDeserializedModuleDescriptorFactoryImpl(
 
         val deserializationConfiguration = CompilerDeserializationConfiguration(languageVersionSettings)
 
-        val provider = createPackageFragmentProvider(
+        val provider = packageFragmentsFactory.createPackageFragmentProvider(
             library,
             packageAccessedHandler,
             libraryProto.packageFragmentNameList,
@@ -87,54 +87,4 @@ internal class KonanDeserializedModuleDescriptorFactoryImpl(
         return moduleDescriptor
     }
 
-    private fun createPackageFragmentProvider(
-        library: KonanLibrary,
-        packageAccessedHandler: PackageAccessedHandler?,
-        packageFragmentNames: List<String>,
-        storageManager: StorageManager,
-        moduleDescriptor: ModuleDescriptor,
-        configuration: DeserializationConfiguration
-    ): PackageFragmentProvider {
-
-        val deserializedPackageFragments = packageFragmentsFactory.createDeserializedPackageFragments(
-            library, packageFragmentNames, moduleDescriptor, packageAccessedHandler, storageManager
-        )
-
-        val syntheticPackageFragments = packageFragmentsFactory.createSyntheticPackageFragments(
-            library, deserializedPackageFragments, moduleDescriptor
-        )
-
-        val provider = PackageFragmentProviderImpl(deserializedPackageFragments + syntheticPackageFragments)
-
-        val notFoundClasses = NotFoundClasses(storageManager, moduleDescriptor)
-
-        val annotationAndConstantLoader = AnnotationAndConstantLoaderImpl(
-            moduleDescriptor,
-            notFoundClasses,
-            KonanSerializerProtocol
-        )
-
-        val components = DeserializationComponents(
-            storageManager,
-            moduleDescriptor,
-            configuration,
-            DeserializedClassDataFinder(provider),
-            annotationAndConstantLoader,
-            provider,
-            LocalClassifierTypeSettings.Default,
-            ErrorReporter.DO_NOTHING,
-            LookupTracker.DO_NOTHING,
-            NullFlexibleTypeDeserializer,
-            emptyList(),
-            notFoundClasses,
-            ContractDeserializer.DEFAULT,
-            extensionRegistryLite = KonanSerializerProtocol.extensionRegistry
-        )
-
-        for (packageFragment in deserializedPackageFragments) {
-            packageFragment.initialize(components)
-        }
-
-        return provider
-    }
 }

konan/library-reader/src/org/jetbrains/kotlin/serialization/konan/impl/KonanDeserializedPackageFragmentsFactoryImpl.kt

@@ -5,11 +5,10 @@
 
 package org.jetbrains.kotlin.serialization.konan.impl
 
-import org.jetbrains.kotlin.descriptors.ClassifierDescriptor
+import org.jetbrains.kotlin.descriptors.*
-import org.jetbrains.kotlin.descriptors.ModuleDescriptor
-import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor
 import org.jetbrains.kotlin.descriptors.impl.PackageFragmentDescriptorImpl
 import org.jetbrains.kotlin.incremental.components.LookupLocation
+import org.jetbrains.kotlin.incremental.components.LookupTracker
 import org.jetbrains.kotlin.konan.library.KonanLibrary
 import org.jetbrains.kotlin.konan.library.exportForwardDeclarations
 import org.jetbrains.kotlin.konan.library.isInterop
@@ -19,8 +18,11 @@ import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.resolve.scopes.MemberScope
 import org.jetbrains.kotlin.resolve.scopes.MemberScopeImpl
+import org.jetbrains.kotlin.serialization.deserialization.*
 import org.jetbrains.kotlin.serialization.konan.KonanDeserializedPackageFragmentsFactory
 import org.jetbrains.kotlin.serialization.konan.KonanPackageFragment
+import org.jetbrains.kotlin.serialization.konan.KonanSerializerProtocol
+import org.jetbrains.kotlin.serialization.konan.NullFlexibleTypeDeserializer
 import org.jetbrains.kotlin.storage.StorageManager
 import org.jetbrains.kotlin.utils.Printer
 import org.jetbrains.kotlin.utils.addToStdlib.firstNotNullResult
@@ -65,6 +67,58 @@ internal object KonanDeserializedPackageFragmentsFactoryImpl : KonanDeserialized
 
         return result
     }
+
+
+    override fun createPackageFragmentProvider(
+        library: KonanLibrary,
+        packageAccessedHandler: PackageAccessedHandler?,
+        packageFragmentNames: List<String>,
+        storageManager: StorageManager,
+        moduleDescriptor: ModuleDescriptor,
+        configuration: DeserializationConfiguration
+    ): PackageFragmentProvider {
+
+        val deserializedPackageFragments = createDeserializedPackageFragments(
+            library, packageFragmentNames, moduleDescriptor, packageAccessedHandler, storageManager
+        )
+
+        val syntheticPackageFragments = createSyntheticPackageFragments(
+            library, deserializedPackageFragments, moduleDescriptor
+        )
+
+        val provider = PackageFragmentProviderImpl(deserializedPackageFragments + syntheticPackageFragments)
+
+        val notFoundClasses = NotFoundClasses(storageManager, moduleDescriptor)
+
+        val annotationAndConstantLoader = AnnotationAndConstantLoaderImpl(
+            moduleDescriptor,
+            notFoundClasses,
+            KonanSerializerProtocol
+        )
+
+        val components = DeserializationComponents(
+            storageManager,
+            moduleDescriptor,
+            configuration,
+            DeserializedClassDataFinder(provider),
+            annotationAndConstantLoader,
+            provider,
+            LocalClassifierTypeSettings.Default,
+            ErrorReporter.DO_NOTHING,
+            LookupTracker.DO_NOTHING,
+            NullFlexibleTypeDeserializer,
+            emptyList(),
+            notFoundClasses,
+            ContractDeserializer.DEFAULT,
+            extensionRegistryLite = KonanSerializerProtocol.extensionRegistry
+        )
+
+        for (packageFragment in deserializedPackageFragments) {
+            packageFragment.initialize(components)
+        }
+
+        return provider
+    }
 }
 
 /**