[AA] IDE mode tests: Build and index stubs for binary libraries

- Instead of indexing binary library declarations from decompiled PSI,
  the static declaration provider now builds and indexes stubs. As noted
  in KT-65960, this brings IDE mode tests much more in line with
  decompiled stubs indexing in the IDE. It should allow us to catch
  issues with the stub-based deserialized symbol provider outside of IDE
  tests.
- In the Standalone mode, we can skip stub-indexing completely, as we
  provide FIR symbols via class-based deserialization. This also extends
  to shared binary roots.

^KT-65960 fixed

analysis/analysis-api-impl-base/tests/org/jetbrains/kotlin/analysis/api/impl/base/test/configurators/AnalysisApiBaseTestServiceRegistrar.kt

@@ -23,8 +23,11 @@ import org.jetbrains.kotlin.analysis.project.structure.KtBinaryModule
 import org.jetbrains.kotlin.analysis.providers.*
 import org.jetbrains.kotlin.analysis.providers.impl.*
 import org.jetbrains.kotlin.analysis.test.framework.project.structure.ktModuleProvider
+import org.jetbrains.kotlin.analysis.test.framework.services.configuration.AnalysisApiBinaryLibraryIndexingMode
+import org.jetbrains.kotlin.analysis.test.framework.services.configuration.libraryIndexingConfiguration
 import org.jetbrains.kotlin.analysis.test.framework.services.environmentManager
 import org.jetbrains.kotlin.analysis.test.framework.test.configurators.AnalysisApiTestServiceRegistrar
+import org.jetbrains.kotlin.analysis.test.framework.test.configurators.TestModuleKind
 import org.jetbrains.kotlin.psi.KtClassOrObject
 import org.jetbrains.kotlin.psi.KtFile
 import org.jetbrains.kotlin.psi.KtFileClassProvider
@@ -66,32 +69,51 @@ object AnalysisApiBaseTestServiceRegistrar : AnalysisApiTestServiceRegistrar() {
 
     override fun registerProjectModelServices(project: MockProject, testServices: TestServices) {
         val moduleStructure = testServices.ktModuleProvider.getModuleStructure()
-        val allSourceKtFiles = moduleStructure.mainModules.flatMap { it.files.filterIsInstance<KtFile>() }
+        val testKtFiles = moduleStructure.mainModules.flatMap { it.files.filterIsInstance<KtFile>() }
-        val binaryDependencies = moduleStructure.binaryModules.toMutableSet()
+
+        // We explicitly exclude decompiled libraries. Their decompiled PSI files are indexed by the declaration provider, so it shouldn't
+        // additionally build and index stubs for the library.
+        val mainBinaryModules = moduleStructure.mainModules
+            .filter { it.moduleKind == TestModuleKind.LibraryBinary }
+            .map { it.ktModule as KtBinaryModule }
+
+        val sharedBinaryDependencies = moduleStructure.binaryModules.toMutableSet()
         for (mainModule in moduleStructure.mainModules) {
             val ktModule = mainModule.ktModule
             if (ktModule !is KtBinaryModule) continue
-            binaryDependencies -= ktModule
+            sharedBinaryDependencies -= ktModule
         }
 
-        val roots = StandaloneProjectFactory.getVirtualFilesForLibraryRoots(
+        val mainBinaryRoots = StandaloneProjectFactory.getVirtualFilesForLibraryRoots(
-            binaryDependencies.flatMap { binary -> binary.getBinaryRoots() },
+            mainBinaryModules.flatMap { it.getBinaryRoots() },
+            testServices.environmentManager.getProjectEnvironment(),
+        ).distinct()
+
+        val sharedBinaryRoots = StandaloneProjectFactory.getVirtualFilesForLibraryRoots(
+            sharedBinaryDependencies.flatMap { binary -> binary.getBinaryRoots() },
             testServices.environmentManager.getProjectEnvironment()
         ).distinct()
 
         project.apply {
-            registerService(KotlinAnnotationsResolverFactory::class.java, KotlinStaticAnnotationsResolverFactory(project, allSourceKtFiles))
+            registerService(KotlinAnnotationsResolverFactory::class.java, KotlinStaticAnnotationsResolverFactory(project, testKtFiles))
 
             val filter = BuiltInDefinitionFile.FILTER_OUT_CLASSES_EXISTING_AS_JVM_CLASS_FILES
             val ktFilesForBinaries: List<KtFile>
             try {
                 BuiltInDefinitionFile.FILTER_OUT_CLASSES_EXISTING_AS_JVM_CLASS_FILES = false
+
+                val shouldBuildStubsForBinaryLibraries =
+                    testServices.libraryIndexingConfiguration.binaryLibraryIndexingMode == AnalysisApiBinaryLibraryIndexingMode.INDEX_STUBS
+
                 val declarationProviderFactory = KotlinStaticDeclarationProviderFactory(
                     project,
-                    allSourceKtFiles,
+                    testKtFiles,
-                    additionalRoots = roots,
+                    binaryRoots = mainBinaryRoots,
+                    sharedBinaryRoots = sharedBinaryRoots,
                     skipBuiltins = testServices.moduleStructure.allDirectives.contains(NO_RUNTIME),
+                    shouldBuildStubsForBinaryLibraries = shouldBuildStubsForBinaryLibraries,
                 )
+
                 ktFilesForBinaries = declarationProviderFactory.getAdditionalCreatedKtFiles()
                 registerService(
                     KotlinDeclarationProviderFactory::class.java, declarationProviderFactory
@@ -102,7 +124,7 @@ object AnalysisApiBaseTestServiceRegistrar : AnalysisApiTestServiceRegistrar() {
             registerService(KotlinDeclarationProviderMerger::class.java, KotlinStaticDeclarationProviderMerger(project))
             registerService(
                 KotlinPackageProviderFactory::class.java,
-                KotlinStaticPackageProviderFactory(project, allSourceKtFiles + ktFilesForBinaries)
+                KotlinStaticPackageProviderFactory(project, testKtFiles + ktFilesForBinaries)
             )
             registerService(KotlinPackageProviderMerger::class.java, KotlinStaticPackageProviderMerger(project))
             registerService(KotlinResolutionScopeProvider::class.java, KotlinByModulesResolutionScopeProvider::class.java)

analysis/analysis-api-providers/src/org/jetbrains/kotlin/analysis/providers/impl/KotlinStaticDeclarationProvider.kt

@@ -11,7 +11,6 @@ import com.intellij.openapi.project.Project
 import com.intellij.openapi.vfs.VfsUtilCore
 import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.openapi.vfs.VirtualFileVisitor
-import com.intellij.openapi.vfs.impl.jar.CoreJarFileSystem
 import com.intellij.psi.PsiElement
 import com.intellij.psi.PsiManager
 import com.intellij.psi.SingleRootFileViewProvider
@@ -138,11 +137,22 @@ public class KotlinStaticDeclarationProvider internal constructor(
     }
 }
 
+/**
+ * [binaryRoots] and [sharedBinaryRoots] are used to build stubs for symbols from binary libraries. They only need to be specified if
+ * [shouldBuildStubsForBinaryLibraries] is true. In Standalone mode, binary roots don't need to be specified because library symbols are
+ * provided via class-based deserialization, not stub-based deserialization.
+ *
+ * @param binaryRoots Binary roots of the binary libraries that are specific to [project].
+ * @param sharedBinaryRoots Binary roots that are shared between multiple different projects. This allows Kotlin tests to cache stubs for
+ * shared libraries like the Kotlin stdlib.
+ */
 public class KotlinStaticDeclarationProviderFactory(
     private val project: Project,
-    files: Collection<KtFile>,
+    testKtFiles: Collection<KtFile>,
-    additionalRoots: List<VirtualFile> = emptyList(),
+    binaryRoots: List<VirtualFile> = emptyList(),
+    sharedBinaryRoots: List<VirtualFile> = emptyList(),
     skipBuiltins: Boolean = false,
+    shouldBuildStubsForBinaryLibraries: Boolean = false,
 ) : KotlinDeclarationProviderFactory() {
 
     private val index = KotlinStaticDeclarationIndex()
@@ -260,46 +270,6 @@ public class KotlinStaticDeclarationProviderFactory(
     init {
         val recorder = KtDeclarationRecorder()
 
-        fun indexStub(stub: StubElement<*>) {
-            when (stub) {
-                is KotlinClassStubImpl -> {
-                    addToClassMap(stub.psi)
-                    // member functions and properties
-                    stub.childrenStubs.forEach(::indexStub)
-                }
-                is KotlinObjectStubImpl -> {
-                    addToClassMap(stub.psi)
-                    // member functions and properties
-                    stub.childrenStubs.forEach(::indexStub)
-                }
-                is KotlinTypeAliasStubImpl -> addToTypeAliasMap(stub.psi)
-                is KotlinFunctionStubImpl -> addToFunctionMap(stub.psi)
-                is KotlinPropertyStubImpl -> addToPropertyMap(stub.psi)
-                is KotlinPlaceHolderStubImpl -> {
-                    if (stub.stubType == KtStubElementTypes.CLASS_BODY) {
-                        stub.getChildrenStubs().filterIsInstance<KotlinClassOrObjectStub<*>>().forEach(::indexStub)
-                    }
-                }
-            }
-        }
-
-        fun processStub(ktFileStub: KotlinFileStubImpl) {
-            val ktFile: KtFile = ktFileStub.psi
-            addToFacadeFileMap(ktFile)
-
-            val partNames = ktFileStub.facadePartSimpleNames
-            if (partNames != null) {
-                val packageFqName = ktFileStub.getPackageFqName()
-                for (partName in partNames) {
-                    val multiFileClassPartFqName: FqName = packageFqName.child(Name.identifier(partName))
-                    index.multiFileClassPartMap.computeIfAbsent(multiFileClassPartFqName) { mutableSetOf() }.add(ktFile)
-                }
-            }
-
-            // top-level functions and properties, built-in classes
-            ktFileStub.childrenStubs.forEach(::indexStub)
-        }
-
         // Indexing built-ins
         val builtins = mutableSetOf<String>()
         if (!skipBuiltins) {
@@ -309,38 +279,86 @@ public class KotlinStaticDeclarationProviderFactory(
             }
         }
 
-        val binaryClassCache = ClsKotlinBinaryClassCache.getInstance()
+        // We only need to index binary roots if we deserialize compiled symbols from stubs. When deserializing from class files, we don't
-        for (root in additionalRoots) {
+        // need these symbols in the declaration provider.
-            KotlinFakeClsStubsCache.processAdditionalRoot(root) { additionalRoot ->
+        if (shouldBuildStubsForBinaryLibraries) {
-                val stubs = mutableMapOf<VirtualFile, KotlinFileStubImpl>()
+            val binaryClassCache = ClsKotlinBinaryClassCache.getInstance()
-                VfsUtilCore.visitChildrenRecursively(additionalRoot, object : VirtualFileVisitor<Void>() {
+
-                    override fun visitFile(file: VirtualFile): Boolean {
+            for (root in sharedBinaryRoots) {
-                        if (!file.isDirectory) {
+                KotlinFakeClsStubsCache.processAdditionalRoot(root) { additionalRoot ->
-                            val stub = buildStubByVirtualFile(file, binaryClassCache) ?: return true
+                    collectStubsFromBinaryRoot(additionalRoot, binaryClassCache)
-                            stubs.put(file, stub)
+                }?.let(::processCollectedStubs)
-                        }
+            }
-                        return true
+
-                    }
+            // In contrast to `sharedBinaryRoots`, which are shared between many different projects (e.g. the Kotlin stdlib), `binaryRoots`
-                })
+            // come from binary libraries which are specific to the project. Caching them in `KotlinFakeClsStubsCache` won't have any
-                stubs
+            // performance advantage.
-            }?.forEach { entry ->
+            binaryRoots
-                val stub = entry.value
+                .map { collectStubsFromBinaryRoot(it, binaryClassCache) }
-                val fakeFile = object : KtFile(KtClassFileViewProvider(psiManager, entry.key), isCompiled = true) {
+                .forEach { processCollectedStubs(it) }
-                    override fun getStub() = stub
+        }
-                    override fun isPhysical() = false
+
+        testKtFiles.forEach { file ->
+            file.accept(recorder)
+        }
+    }
+
+    private fun indexStub(stub: StubElement<*>) {
+        when (stub) {
+            is KotlinClassStubImpl -> {
+                addToClassMap(stub.psi)
+                // member functions and properties
+                stub.childrenStubs.forEach(::indexStub)
+            }
+            is KotlinObjectStubImpl -> {
+                addToClassMap(stub.psi)
+                // member functions and properties
+                stub.childrenStubs.forEach(::indexStub)
+            }
+            is KotlinTypeAliasStubImpl -> addToTypeAliasMap(stub.psi)
+            is KotlinFunctionStubImpl -> addToFunctionMap(stub.psi)
+            is KotlinPropertyStubImpl -> addToPropertyMap(stub.psi)
+            is KotlinPlaceHolderStubImpl -> {
+                if (stub.stubType == KtStubElementTypes.CLASS_BODY) {
+                    stub.getChildrenStubs().filterIsInstance<KotlinClassOrObjectStub<*>>().forEach(::indexStub)
                 }
-                stub.psi = fakeFile
+            }
-                createdFakeKtFiles.add(fakeFile)
+        }
-                processStub(stub)
+    }
+
+    private fun processStub(ktFileStub: KotlinFileStubImpl) {
+        val ktFile: KtFile = ktFileStub.psi
+        addToFacadeFileMap(ktFile)
+
+        val partNames = ktFileStub.facadePartSimpleNames
+        if (partNames != null) {
+            val packageFqName = ktFileStub.getPackageFqName()
+            for (partName in partNames) {
+                val multiFileClassPartFqName: FqName = packageFqName.child(Name.identifier(partName))
+                index.multiFileClassPartMap.computeIfAbsent(multiFileClassPartFqName) { mutableSetOf() }.add(ktFile)
             }
         }
 
-        // Indexing user source files
+        // top-level functions and properties, built-in classes
-        files.forEach {
+        ktFileStub.childrenStubs.forEach(::indexStub)
-            it.accept(recorder)
-        }
     }
 
+    private fun collectStubsFromBinaryRoot(
+        binaryRoot: VirtualFile,
+        binaryClassCache: ClsKotlinBinaryClassCache,
+    ): Map<VirtualFile, KotlinFileStubImpl> =
+        buildMap {
+            VfsUtilCore.visitChildrenRecursively(binaryRoot, object : VirtualFileVisitor<Void>() {
+                override fun visitFile(file: VirtualFile): Boolean {
+                    if (!file.isDirectory) {
+                        val stub = buildStubByVirtualFile(file, binaryClassCache) ?: return true
+                        put(file, stub)
+                    }
+                    return true
+                }
+            })
+        }
+
     private fun buildStubByVirtualFile(file: VirtualFile, binaryClassCache: ClsKotlinBinaryClassCache): KotlinFileStubImpl? {
         val fileContent = FileContentImpl.createByFile(file)
         val fileType = file.fileType
@@ -358,6 +376,19 @@ public class KotlinStaticDeclarationProviderFactory(
         return stubBuilder.buildFileStub(fileContent) as? KotlinFileStubImpl
     }
 
+    private fun processCollectedStubs(stubs: Map<VirtualFile, KotlinFileStubImpl>) {
+        stubs.forEach { entry ->
+            val stub = entry.value
+            val fakeFile = object : KtFile(KtClassFileViewProvider(psiManager, entry.key), isCompiled = true) {
+                override fun getStub() = stub
+                override fun isPhysical() = false
+            }
+            stub.psi = fakeFile
+            createdFakeKtFiles.add(fakeFile)
+            processStub(stub)
+        }
+    }
+
     override fun createDeclarationProvider(scope: GlobalSearchScope, contextualModule: KtModule?): KotlinDeclarationProvider {
         return KotlinStaticDeclarationProvider(index, scope)
     }