[kotlin] fix "IllegalStateException: Could not find stdlib"

The commit removes all the KLib resolution logic,
now Analysis API Standalone clients need to provide all the KLib
list directly.

The resolution logic was removed as too error-prone and requiring compiler configurations.

Kotlin Gradle plugin can provide a full set of required KLibs,
so if a client is a Gradle plugin, this should not be an issue.

Probably, some fancy API which will explicitly perform all KLib dependency searches
should be introduced in the future (KT-63395)

^KT-63126 fixed

analysis/analysis-api-standalone/analysis-api-fir-standalone-base/src/org/jetbrains/kotlin/analysis/api/standalone/base/project/structure/LLFirStandaloneLibrarySymbolProviderFactory.kt

@@ -11,7 +11,6 @@ import com.intellij.psi.search.GlobalSearchScope
 import org.jetbrains.kotlin.analysis.low.level.api.fir.project.structure.LLFirLibrarySymbolProviderFactory
 import org.jetbrains.kotlin.analysis.low.level.api.fir.project.structure.LLFirModuleData
 import org.jetbrains.kotlin.analysis.project.structure.KtLibraryModule
-import org.jetbrains.kotlin.backend.common.CommonKLibResolver
 import org.jetbrains.kotlin.fir.BinaryModuleData
 import org.jetbrains.kotlin.fir.FirSession
 import org.jetbrains.kotlin.fir.deserialization.SingleModuleDataProvider
@@ -23,12 +22,18 @@ import org.jetbrains.kotlin.fir.scopes.FirKotlinScopeProvider
 import org.jetbrains.kotlin.fir.session.KlibBasedSymbolProvider
 import org.jetbrains.kotlin.fir.session.MetadataSymbolProvider
 import org.jetbrains.kotlin.fir.session.NativeForwardDeclarationsSymbolProvider
+import org.jetbrains.kotlin.library.KLIB_FILE_EXTENSION
 import org.jetbrains.kotlin.library.KotlinLibrary
+import org.jetbrains.kotlin.library.ToolingSingleFileKlibResolveStrategy
 import org.jetbrains.kotlin.library.metadata.impl.KlibResolvedModuleDescriptorsFactoryImpl.Companion.FORWARD_DECLARATIONS_MODULE_NAME
 import org.jetbrains.kotlin.load.kotlin.PackageAndMetadataPartProvider
 import org.jetbrains.kotlin.load.kotlin.PackagePartProvider
 import org.jetbrains.kotlin.load.kotlin.VirtualFileFinderFactory
 import java.lang.IllegalStateException
+import java.nio.file.Path
+import kotlin.io.path.absolutePathString
+import kotlin.io.path.extension
+import kotlin.io.path.isDirectory
 
 class LLFirStandaloneLibrarySymbolProviderFactory(private val project: Project) : LLFirLibrarySymbolProviderFactory() {
     override fun createJvmLibrarySymbolProvider(
@@ -115,7 +120,7 @@ class LLFirStandaloneLibrarySymbolProviderFactory(private val project: Project)
     override fun createBuiltinsSymbolProvider(
         session: FirSession,
         moduleData: LLFirModuleData,
-        kotlinScopeProvider: FirKotlinScopeProvider
+        kotlinScopeProvider: FirKotlinScopeProvider,
     ): List<FirSymbolProvider> {
         return listOf(
             FirBuiltinSymbolProvider(session, moduleData, kotlinScopeProvider)
@@ -126,12 +131,19 @@ class LLFirStandaloneLibrarySymbolProviderFactory(private val project: Project)
     private fun LLFirModuleData.getLibraryKLibs(): List<KotlinLibrary> {
         val ktLibraryModule = ktModule as? KtLibraryModule ?: return emptyList()
 
-        val resolveResult = CommonKLibResolver.resolve(
-            ktLibraryModule.getBinaryRoots().map { it.toString() },
-            IntellijLogBasedLogger,
-            lenient = true,
-        )
-        return resolveResult.getFullResolvedList().map { it.library }
+        return ktLibraryModule.getBinaryRoots()
+            .filter { it.isDirectory() || it.extension == KLIB_FILE_EXTENSION }
+            .mapNotNull { it.tryResolveAsKLib() }
+    }
+
+    private fun Path.tryResolveAsKLib(): KotlinLibrary? {
+        return try {
+            val konanFile = org.jetbrains.kotlin.konan.file.File(absolutePathString())
+            ToolingSingleFileKlibResolveStrategy.tryResolve(konanFile, IntellijLogBasedLogger)
+        } catch (e: Exception) {
+            LOG.warn("Cannot resolve a KLib $this", e)
+            null
+        }
     }
 
     companion object {

analysis/analysis-api-standalone/analysis-api-standalone-native/build.gradle.kts

@@ -0,0 +1,37 @@
+import org.jetbrains.kotlin.kotlinNativeDist
+
+plugins {
+    kotlin("jvm")
+    id("jps-compatible")
+}
+
+dependencies {
+    testImplementation(projectTests(":compiler:tests-common"))
+    testImplementation(project(":analysis:analysis-api-standalone"))
+    testImplementation(projectTests(":analysis:analysis-api-standalone"))
+    testImplementation(projectTests(":native:native.tests"))
+
+    testImplementation(platform(libs.junit.bom))
+    testImplementation(libs.junit.jupiter.api)
+    testRuntimeOnly(libs.junit.jupiter.engine)
+}
+
+sourceSets {
+    "main" { none() }
+    "test" {
+        projectDefault()
+    }
+}
+
+
+projectTest(jUnitMode = JUnitMode.JUnit5) {
+    dependsOn(":dist")
+    workingDir = rootDir
+    useJUnitPlatform()
+}
+
+val test by nativeTest("test", null) {
+    systemProperty("kotlin.native.home", kotlinNativeDist.absolutePath)
+}
+
+testsJar()

analysis/analysis-api-standalone/analysis-api-standalone-native/testData/nativeSessionBuilder/resolveAgainstNativeKLib/klibSrc/nativeKLibFunction.kt

@@ -0,0 +1,5 @@
+package nativeKLib
+
+fun nativeKLibFunction(arg: String): Int {
+    return 1
+}

analysis/analysis-api-standalone/analysis-api-standalone-native/testData/nativeSessionBuilder/resolveAgainstNativeKLib/src/main.kt

@@ -0,0 +1,5 @@
+import nativeKLib.nativeKLibFunction
+
+fun main() {
+    nativeKLibFunction("aaa")
+}

analysis/analysis-api-standalone/analysis-api-standalone-native/tests/org/jetbrains/kotlin/analysis/api/standalone/konan/fir/test/cases/session/builder/NativeStandaloneSessionBuilderTest.kt

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.analysis.api.standalone.konan.fir.test.cases.session.builder
+
+import org.jetbrains.kotlin.analysis.api.KtAnalysisApiInternals
+import org.jetbrains.kotlin.analysis.api.lifetime.KtLifetimeTokenProvider
+import org.jetbrains.kotlin.analysis.api.standalone.KtAlwaysAccessibleLifetimeTokenProvider
+import org.jetbrains.kotlin.analysis.api.standalone.buildStandaloneAnalysisAPISession
+import org.jetbrains.kotlin.analysis.api.standalone.fir.test.cases.session.builder.assertIsCallOf
+import org.jetbrains.kotlin.analysis.project.structure.KtSourceModule
+import org.jetbrains.kotlin.analysis.project.structure.builder.buildKtLibraryModule
+import org.jetbrains.kotlin.analysis.project.structure.builder.buildKtSourceModule
+import org.jetbrains.kotlin.konan.target.HostManager
+import org.jetbrains.kotlin.name.CallableId
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.name.Name
+import org.jetbrains.kotlin.platform.konan.NativePlatforms
+import org.jetbrains.kotlin.psi.KtCallExpression
+import org.jetbrains.kotlin.psi.KtFile
+import org.jetbrains.kotlin.psi.psiUtil.findDescendantOfType
+import org.junit.jupiter.api.Test
+
+@OptIn(KtAnalysisApiInternals::class)
+class NativeStandaloneSessionBuilderTest {
+    @Test
+    fun testResolveAgainstCommonKlib() {
+        lateinit var sourceModule: KtSourceModule
+        val currentArchitectureTarget = HostManager.host
+        val nativePlatform = NativePlatforms.nativePlatformByTargets(listOf(currentArchitectureTarget))
+        val session = buildStandaloneAnalysisAPISession {
+            registerProjectService(KtLifetimeTokenProvider::class.java, KtAlwaysAccessibleLifetimeTokenProvider())
+
+            buildKtModuleProvider {
+                platform = nativePlatform
+                val kLib = addModule(
+                    buildKtLibraryModule {
+                        val compiledKLibRoot = compileToNativeKLib(testDataPath("resolveAgainstNativeKLib/klibSrc"))
+                        addBinaryRoot(compiledKLibRoot)
+                        platform = nativePlatform
+                        libraryName = "klib"
+                    }
+                )
+                sourceModule = addModule(
+                    buildKtSourceModule {
+                        addSourceRoot(testDataPath("resolveAgainstNativeKLib/src"))
+                        addRegularDependency(kLib)
+                        platform = nativePlatform
+                        moduleName = "source"
+                    }
+                )
+            }
+        }
+        val ktFile = session.modulesWithFiles.getValue(sourceModule).single() as KtFile
+
+        val ktCallExpression = ktFile.findDescendantOfType<KtCallExpression>()!!
+        ktCallExpression.assertIsCallOf(CallableId(FqName("nativeKLib"), Name.identifier("nativeKLibFunction")))
+    }
+}

analysis/analysis-api-standalone/analysis-api-standalone-native/tests/org/jetbrains/kotlin/analysis/api/standalone/konan/fir/test/cases/session/builder/testUtils.kt

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.analysis.api.standalone.konan.fir.test.cases.session.builder
+
+import org.jetbrains.kotlin.cli.common.ExitCode
+import org.jetbrains.kotlin.konan.test.blackbox.support.compilation.callCompilerWithoutOutputInterceptor
+import org.jetbrains.kotlin.test.util.KtTestUtil
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.Paths
+import kotlin.io.path.absolutePathString
+import kotlin.io.path.extension
+import kotlin.streams.asSequence
+
+internal fun testDataPath(path: String): Path {
+    return Paths.get("analysis/analysis-api-standalone/analysis-api-standalone-native/testData/nativeSessionBuilder").resolve(path)
+}
+
+internal fun compileToNativeKLib(kLibSourcesRoot: Path): Path {
+    val ktFiles = Files.walk(kLibSourcesRoot).asSequence().filter { it.extension == "kt" }.toList()
+    val testKlib = KtTestUtil.tmpDir("testLibrary").resolve("library.klib").toPath()
+
+    val arguments = buildList {
+        ktFiles.mapTo(this) { it.absolutePathString() }
+        addAll(listOf("-produce", "library"))
+        addAll(listOf("-output", testKlib.absolutePathString()))
+    }
+
+    val compileResult = callCompilerWithoutOutputInterceptor(arguments.toTypedArray())
+
+    check(compileResult.exitCode == ExitCode.OK) {
+        "Compilation error: $compileResult"
+    }
+
+    return testKlib
+}

analysis/analysis-api-standalone/src/org/jetbrains/kotlin/analysis/project/structure/builder/KtBinaryModuleBuilder.kt

@@ -11,10 +11,26 @@ import java.nio.file.Path
 public abstract class KtBinaryModuleBuilder : KtModuleBuilder() {
     private val binaryRoots: MutableList<Path> = mutableListOf()
 
+    /**
+     * Adds a [root] to the current library.
+     *
+     * The [root] can be:
+     * * A .jar file for JVM libraries or common metadata KLibs
+     * * A directory with a set of .classfiles for JVM Libraries
+     * * A Kotlin/Native, Kotlin/Common, Kotlin/JS KLib.
+     * In this case, all KLib dependencies should be provided together with the KLib itself.
+     */
     public fun addBinaryRoot(root: Path) {
         binaryRoots.add(root)
     }
 
+    /**
+     * Adds a collection of [roots] to the current library.
+     *
+     * See [addBinaryRoot] for details
+     *
+     * @see addBinaryRoot for details
+     */
     public fun addBinaryRoots(roots: Collection<Path>) {
         binaryRoots.addAll(roots)
     }

analysis/build.gradle.kts

@@ -14,4 +14,8 @@ tasks.register("analysisAllTests") {
         ":analysis:low-level-api-fir:test",
         ":analysis:symbol-light-classes:test"
     )
+
+    if (kotlinBuildProperties.isKotlinNativeEnabled) {
+        dependsOn(":analysis:analysis-api-standalone:analysis-api-standalone-native:test")
+    }
 }

compiler/util-klib/src/org/jetbrains/kotlin/library/ToolingResolve.kt

@@ -27,6 +27,10 @@ import java.io.IOException
  */
 object ToolingSingleFileKlibResolveStrategy : SingleFileKlibResolveStrategy {
     override fun resolve(libraryFile: File, logger: Logger): KotlinLibrary =
+        tryResolve(libraryFile, logger)
+            ?: fakeLibrary(libraryFile)
+
+    fun tryResolve(libraryFile: File, logger: Logger): KotlinLibrary? =
         withSafeAccess(libraryFile) { localRoot ->
             if (localRoot.looksLikeKlibComponent) {
                 // old style library
@@ -49,7 +53,7 @@ object ToolingSingleFileKlibResolveStrategy : SingleFileKlibResolveStrategy {
                     }
                 }
             }
-        } ?: fakeLibrary(libraryFile)
+        }
 
     private const val NONEXISTENT_COMPONENT_NAME = "__nonexistent_component_name__"
 

native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/NativeTestSupport.kt

@@ -69,7 +69,7 @@ internal object CastCompatibleKotlinNativeClassLoader {
     val kotlinNativeClassLoader = NativeTestSupport.computeNativeClassLoader(this::class.java.classLoader)
 }
 
-private object NativeTestSupport {
+internal object NativeTestSupport {
     private val NAMESPACE = ExtensionContext.Namespace.create(NativeTestSupport::class.java.simpleName)
 
     /*************** Test process settings ***************/

native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/compilation/CompilationToolCall.kt

@@ -10,6 +10,7 @@ import org.jetbrains.kotlin.cli.common.messages.*
 import org.jetbrains.kotlin.compilerRunner.OutputItemsCollectorImpl
 import org.jetbrains.kotlin.compilerRunner.processCompilerOutput
 import org.jetbrains.kotlin.config.Services
+import org.jetbrains.kotlin.konan.test.blackbox.support.NativeTestSupport
 import org.jetbrains.kotlin.konan.test.blackbox.support.settings.KotlinNativeTargets
 import java.io.ByteArrayOutputStream
 import java.io.File
@@ -20,7 +21,7 @@ import kotlin.time.measureTime
 import kotlin.time.measureTimedValue
 import org.jetbrains.kotlin.konan.file.File as KonanFile
 
-internal data class CompilationToolCallResult(
+data class CompilationToolCallResult(
     val exitCode: ExitCode,
     val toolOutput: String,
     val toolOutputHasErrors: Boolean,
@@ -74,6 +75,12 @@ internal fun callCompiler(compilerArgs: Array<String>, kotlinNativeClassLoader:
     return CompilationToolCallResult(exitCode, compilerOutput, messageCollector.hasErrors(), duration)
 }
 
+
+fun callCompilerWithoutOutputInterceptor(compilerArgs: Array<String>): CompilationToolCallResult {
+    val compilerClassLoader = NativeTestSupport.computeNativeClassLoader(parent = null).classLoader
+    return callCompilerWithoutOutputInterceptor(compilerArgs, compilerClassLoader)
+}
+
 internal fun callCompilerWithoutOutputInterceptor(
     compilerArgs: Array<String>,
     kotlinNativeClassLoader: ClassLoader

repo/gradle-build-conventions/buildsrc-compat/src/main/kotlin/nativeTest.kt

@@ -94,7 +94,8 @@ fun Project.nativeTest(
     requirePlatformLibs: Boolean = false,
     customCompilerDependencies: List<Configuration> = emptyList(),
     customTestDependencies: List<Configuration> = emptyList(),
-    compilerPluginDependencies: List<Configuration> = emptyList()
+    compilerPluginDependencies: List<Configuration> = emptyList(),
+    body: Test.() -> Unit = {},
 ) = projectTest(
     taskName,
     jUnitMode = JUnitMode.JUnit5,
@@ -227,4 +228,5 @@ fun Project.nativeTest(
                 """.trimIndent()
             )
         }
+    body()
 }

settings.gradle

@@ -579,7 +579,8 @@ include ":generators:analysis-api-generator",
 
 if (buildProperties.isKotlinNativeEnabled) {
     include ":generators:analysis-api-generator:generator-kotlin-native",
-            ":analysis:low-level-api-fir:low-level-api-fir-native"
+            ":analysis:low-level-api-fir:low-level-api-fir-native",
+            ":analysis:analysis-api-standalone:analysis-api-standalone-native"
 }