From 6aeabc83ea8d10e053dd23fdb67ec374290ddfb4 Mon Sep 17 00:00:00 2001 From: Ilya Kirillov Date: Mon, 6 Nov 2023 11:49:46 +0100 Subject: [PATCH] [kotlin] fix "IllegalStateException: Could not find stdlib" The commit removes all the KLib resolution logic, now Analysis API Standalone clients need to provide all the KLib list directly. The resolution logic was removed as too error-prone and requiring compiler configurations. Kotlin Gradle plugin can provide a full set of required KLibs, so if a client is a Gradle plugin, this should not be an issue. Probably, some fancy API which will explicitly perform all KLib dependency searches should be introduced in the future (KT-63395) ^KT-63126 fixed --- ...rStandaloneLibrarySymbolProviderFactory.kt | 28 ++++++--- .../build.gradle.kts | 37 +++++++++++ .../klibSrc/nativeKLibFunction.kt | 5 ++ .../resolveAgainstNativeKLib/src/main.kt | 5 ++ .../NativeStandaloneSessionBuilderTest.kt | 61 +++++++++++++++++++ .../test/cases/session/builder/testUtils.kt | 39 ++++++++++++ .../builder/KtBinaryModuleBuilder.kt | 16 +++++ analysis/build.gradle.kts | 4 ++ .../kotlin/library/ToolingResolve.kt | 6 +- .../blackbox/support/NativeTestSupport.kt | 2 +- .../compilation/CompilationToolCall.kt | 9 ++- .../src/main/kotlin/nativeTest.kt | 4 +- settings.gradle | 3 +- 13 files changed, 206 insertions(+), 13 deletions(-) create mode 100644 analysis/analysis-api-standalone/analysis-api-standalone-native/build.gradle.kts create mode 100644 analysis/analysis-api-standalone/analysis-api-standalone-native/testData/nativeSessionBuilder/resolveAgainstNativeKLib/klibSrc/nativeKLibFunction.kt create mode 100644 analysis/analysis-api-standalone/analysis-api-standalone-native/testData/nativeSessionBuilder/resolveAgainstNativeKLib/src/main.kt create mode 100644 analysis/analysis-api-standalone/analysis-api-standalone-native/tests/org/jetbrains/kotlin/analysis/api/standalone/konan/fir/test/cases/session/builder/NativeStandaloneSessionBuilderTest.kt create mode 100644 analysis/analysis-api-standalone/analysis-api-standalone-native/tests/org/jetbrains/kotlin/analysis/api/standalone/konan/fir/test/cases/session/builder/testUtils.kt diff --git a/analysis/analysis-api-standalone/analysis-api-fir-standalone-base/src/org/jetbrains/kotlin/analysis/api/standalone/base/project/structure/LLFirStandaloneLibrarySymbolProviderFactory.kt b/analysis/analysis-api-standalone/analysis-api-fir-standalone-base/src/org/jetbrains/kotlin/analysis/api/standalone/base/project/structure/LLFirStandaloneLibrarySymbolProviderFactory.kt index acd18090dc9..803ef6dfe0a 100644 --- a/analysis/analysis-api-standalone/analysis-api-fir-standalone-base/src/org/jetbrains/kotlin/analysis/api/standalone/base/project/structure/LLFirStandaloneLibrarySymbolProviderFactory.kt +++ b/analysis/analysis-api-standalone/analysis-api-fir-standalone-base/src/org/jetbrains/kotlin/analysis/api/standalone/base/project/structure/LLFirStandaloneLibrarySymbolProviderFactory.kt @@ -11,7 +11,6 @@ import com.intellij.psi.search.GlobalSearchScope import org.jetbrains.kotlin.analysis.low.level.api.fir.project.structure.LLFirLibrarySymbolProviderFactory import org.jetbrains.kotlin.analysis.low.level.api.fir.project.structure.LLFirModuleData import org.jetbrains.kotlin.analysis.project.structure.KtLibraryModule -import org.jetbrains.kotlin.backend.common.CommonKLibResolver import org.jetbrains.kotlin.fir.BinaryModuleData import org.jetbrains.kotlin.fir.FirSession import org.jetbrains.kotlin.fir.deserialization.SingleModuleDataProvider @@ -23,12 +22,18 @@ import org.jetbrains.kotlin.fir.scopes.FirKotlinScopeProvider import org.jetbrains.kotlin.fir.session.KlibBasedSymbolProvider import org.jetbrains.kotlin.fir.session.MetadataSymbolProvider import org.jetbrains.kotlin.fir.session.NativeForwardDeclarationsSymbolProvider +import org.jetbrains.kotlin.library.KLIB_FILE_EXTENSION import org.jetbrains.kotlin.library.KotlinLibrary +import org.jetbrains.kotlin.library.ToolingSingleFileKlibResolveStrategy import org.jetbrains.kotlin.library.metadata.impl.KlibResolvedModuleDescriptorsFactoryImpl.Companion.FORWARD_DECLARATIONS_MODULE_NAME import org.jetbrains.kotlin.load.kotlin.PackageAndMetadataPartProvider import org.jetbrains.kotlin.load.kotlin.PackagePartProvider import org.jetbrains.kotlin.load.kotlin.VirtualFileFinderFactory import java.lang.IllegalStateException +import java.nio.file.Path +import kotlin.io.path.absolutePathString +import kotlin.io.path.extension +import kotlin.io.path.isDirectory class LLFirStandaloneLibrarySymbolProviderFactory(private val project: Project) : LLFirLibrarySymbolProviderFactory() { override fun createJvmLibrarySymbolProvider( @@ -115,7 +120,7 @@ class LLFirStandaloneLibrarySymbolProviderFactory(private val project: Project) override fun createBuiltinsSymbolProvider( session: FirSession, moduleData: LLFirModuleData, - kotlinScopeProvider: FirKotlinScopeProvider + kotlinScopeProvider: FirKotlinScopeProvider, ): List { return listOf( FirBuiltinSymbolProvider(session, moduleData, kotlinScopeProvider) @@ -126,12 +131,19 @@ class LLFirStandaloneLibrarySymbolProviderFactory(private val project: Project) private fun LLFirModuleData.getLibraryKLibs(): List { val ktLibraryModule = ktModule as? KtLibraryModule ?: return emptyList() - val resolveResult = CommonKLibResolver.resolve( - ktLibraryModule.getBinaryRoots().map { it.toString() }, - IntellijLogBasedLogger, - lenient = true, - ) - return resolveResult.getFullResolvedList().map { it.library } + return ktLibraryModule.getBinaryRoots() + .filter { it.isDirectory() || it.extension == KLIB_FILE_EXTENSION } + .mapNotNull { it.tryResolveAsKLib() } + } + + private fun Path.tryResolveAsKLib(): KotlinLibrary? { + return try { + val konanFile = org.jetbrains.kotlin.konan.file.File(absolutePathString()) + ToolingSingleFileKlibResolveStrategy.tryResolve(konanFile, IntellijLogBasedLogger) + } catch (e: Exception) { + LOG.warn("Cannot resolve a KLib $this", e) + null + } } companion object { diff --git a/analysis/analysis-api-standalone/analysis-api-standalone-native/build.gradle.kts b/analysis/analysis-api-standalone/analysis-api-standalone-native/build.gradle.kts new file mode 100644 index 00000000000..637e01316a6 --- /dev/null +++ b/analysis/analysis-api-standalone/analysis-api-standalone-native/build.gradle.kts @@ -0,0 +1,37 @@ +import org.jetbrains.kotlin.kotlinNativeDist + +plugins { + kotlin("jvm") + id("jps-compatible") +} + +dependencies { + testImplementation(projectTests(":compiler:tests-common")) + testImplementation(project(":analysis:analysis-api-standalone")) + testImplementation(projectTests(":analysis:analysis-api-standalone")) + testImplementation(projectTests(":native:native.tests")) + + testImplementation(platform(libs.junit.bom)) + testImplementation(libs.junit.jupiter.api) + testRuntimeOnly(libs.junit.jupiter.engine) +} + +sourceSets { + "main" { none() } + "test" { + projectDefault() + } +} + + +projectTest(jUnitMode = JUnitMode.JUnit5) { + dependsOn(":dist") + workingDir = rootDir + useJUnitPlatform() +} + +val test by nativeTest("test", null) { + systemProperty("kotlin.native.home", kotlinNativeDist.absolutePath) +} + +testsJar() diff --git a/analysis/analysis-api-standalone/analysis-api-standalone-native/testData/nativeSessionBuilder/resolveAgainstNativeKLib/klibSrc/nativeKLibFunction.kt b/analysis/analysis-api-standalone/analysis-api-standalone-native/testData/nativeSessionBuilder/resolveAgainstNativeKLib/klibSrc/nativeKLibFunction.kt new file mode 100644 index 00000000000..76056b42f66 --- /dev/null +++ b/analysis/analysis-api-standalone/analysis-api-standalone-native/testData/nativeSessionBuilder/resolveAgainstNativeKLib/klibSrc/nativeKLibFunction.kt @@ -0,0 +1,5 @@ +package nativeKLib + +fun nativeKLibFunction(arg: String): Int { + return 1 +} \ No newline at end of file diff --git a/analysis/analysis-api-standalone/analysis-api-standalone-native/testData/nativeSessionBuilder/resolveAgainstNativeKLib/src/main.kt b/analysis/analysis-api-standalone/analysis-api-standalone-native/testData/nativeSessionBuilder/resolveAgainstNativeKLib/src/main.kt new file mode 100644 index 00000000000..591ea0f820e --- /dev/null +++ b/analysis/analysis-api-standalone/analysis-api-standalone-native/testData/nativeSessionBuilder/resolveAgainstNativeKLib/src/main.kt @@ -0,0 +1,5 @@ +import nativeKLib.nativeKLibFunction + +fun main() { + nativeKLibFunction("aaa") +} \ No newline at end of file diff --git a/analysis/analysis-api-standalone/analysis-api-standalone-native/tests/org/jetbrains/kotlin/analysis/api/standalone/konan/fir/test/cases/session/builder/NativeStandaloneSessionBuilderTest.kt b/analysis/analysis-api-standalone/analysis-api-standalone-native/tests/org/jetbrains/kotlin/analysis/api/standalone/konan/fir/test/cases/session/builder/NativeStandaloneSessionBuilderTest.kt new file mode 100644 index 00000000000..a11eb4cd61d --- /dev/null +++ b/analysis/analysis-api-standalone/analysis-api-standalone-native/tests/org/jetbrains/kotlin/analysis/api/standalone/konan/fir/test/cases/session/builder/NativeStandaloneSessionBuilderTest.kt @@ -0,0 +1,61 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.analysis.api.standalone.konan.fir.test.cases.session.builder + +import org.jetbrains.kotlin.analysis.api.KtAnalysisApiInternals +import org.jetbrains.kotlin.analysis.api.lifetime.KtLifetimeTokenProvider +import org.jetbrains.kotlin.analysis.api.standalone.KtAlwaysAccessibleLifetimeTokenProvider +import org.jetbrains.kotlin.analysis.api.standalone.buildStandaloneAnalysisAPISession +import org.jetbrains.kotlin.analysis.api.standalone.fir.test.cases.session.builder.assertIsCallOf +import org.jetbrains.kotlin.analysis.project.structure.KtSourceModule +import org.jetbrains.kotlin.analysis.project.structure.builder.buildKtLibraryModule +import org.jetbrains.kotlin.analysis.project.structure.builder.buildKtSourceModule +import org.jetbrains.kotlin.konan.target.HostManager +import org.jetbrains.kotlin.name.CallableId +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.name.Name +import org.jetbrains.kotlin.platform.konan.NativePlatforms +import org.jetbrains.kotlin.psi.KtCallExpression +import org.jetbrains.kotlin.psi.KtFile +import org.jetbrains.kotlin.psi.psiUtil.findDescendantOfType +import org.junit.jupiter.api.Test + +@OptIn(KtAnalysisApiInternals::class) +class NativeStandaloneSessionBuilderTest { + @Test + fun testResolveAgainstCommonKlib() { + lateinit var sourceModule: KtSourceModule + val currentArchitectureTarget = HostManager.host + val nativePlatform = NativePlatforms.nativePlatformByTargets(listOf(currentArchitectureTarget)) + val session = buildStandaloneAnalysisAPISession { + registerProjectService(KtLifetimeTokenProvider::class.java, KtAlwaysAccessibleLifetimeTokenProvider()) + + buildKtModuleProvider { + platform = nativePlatform + val kLib = addModule( + buildKtLibraryModule { + val compiledKLibRoot = compileToNativeKLib(testDataPath("resolveAgainstNativeKLib/klibSrc")) + addBinaryRoot(compiledKLibRoot) + platform = nativePlatform + libraryName = "klib" + } + ) + sourceModule = addModule( + buildKtSourceModule { + addSourceRoot(testDataPath("resolveAgainstNativeKLib/src")) + addRegularDependency(kLib) + platform = nativePlatform + moduleName = "source" + } + ) + } + } + val ktFile = session.modulesWithFiles.getValue(sourceModule).single() as KtFile + + val ktCallExpression = ktFile.findDescendantOfType()!! + ktCallExpression.assertIsCallOf(CallableId(FqName("nativeKLib"), Name.identifier("nativeKLibFunction"))) + } +} \ No newline at end of file diff --git a/analysis/analysis-api-standalone/analysis-api-standalone-native/tests/org/jetbrains/kotlin/analysis/api/standalone/konan/fir/test/cases/session/builder/testUtils.kt b/analysis/analysis-api-standalone/analysis-api-standalone-native/tests/org/jetbrains/kotlin/analysis/api/standalone/konan/fir/test/cases/session/builder/testUtils.kt new file mode 100644 index 00000000000..45fe9e7c604 --- /dev/null +++ b/analysis/analysis-api-standalone/analysis-api-standalone-native/tests/org/jetbrains/kotlin/analysis/api/standalone/konan/fir/test/cases/session/builder/testUtils.kt @@ -0,0 +1,39 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.analysis.api.standalone.konan.fir.test.cases.session.builder + +import org.jetbrains.kotlin.cli.common.ExitCode +import org.jetbrains.kotlin.konan.test.blackbox.support.compilation.callCompilerWithoutOutputInterceptor +import org.jetbrains.kotlin.test.util.KtTestUtil +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import kotlin.io.path.absolutePathString +import kotlin.io.path.extension +import kotlin.streams.asSequence + +internal fun testDataPath(path: String): Path { + return Paths.get("analysis/analysis-api-standalone/analysis-api-standalone-native/testData/nativeSessionBuilder").resolve(path) +} + +internal fun compileToNativeKLib(kLibSourcesRoot: Path): Path { + val ktFiles = Files.walk(kLibSourcesRoot).asSequence().filter { it.extension == "kt" }.toList() + val testKlib = KtTestUtil.tmpDir("testLibrary").resolve("library.klib").toPath() + + val arguments = buildList { + ktFiles.mapTo(this) { it.absolutePathString() } + addAll(listOf("-produce", "library")) + addAll(listOf("-output", testKlib.absolutePathString())) + } + + val compileResult = callCompilerWithoutOutputInterceptor(arguments.toTypedArray()) + + check(compileResult.exitCode == ExitCode.OK) { + "Compilation error: $compileResult" + } + + return testKlib +} \ No newline at end of file diff --git a/analysis/analysis-api-standalone/src/org/jetbrains/kotlin/analysis/project/structure/builder/KtBinaryModuleBuilder.kt b/analysis/analysis-api-standalone/src/org/jetbrains/kotlin/analysis/project/structure/builder/KtBinaryModuleBuilder.kt index 7d3ac5fd2d7..d53fa589605 100644 --- a/analysis/analysis-api-standalone/src/org/jetbrains/kotlin/analysis/project/structure/builder/KtBinaryModuleBuilder.kt +++ b/analysis/analysis-api-standalone/src/org/jetbrains/kotlin/analysis/project/structure/builder/KtBinaryModuleBuilder.kt @@ -11,10 +11,26 @@ import java.nio.file.Path public abstract class KtBinaryModuleBuilder : KtModuleBuilder() { private val binaryRoots: MutableList = mutableListOf() + /** + * Adds a [root] to the current library. + * + * The [root] can be: + * * A .jar file for JVM libraries or common metadata KLibs + * * A directory with a set of .classfiles for JVM Libraries + * * A Kotlin/Native, Kotlin/Common, Kotlin/JS KLib. + * In this case, all KLib dependencies should be provided together with the KLib itself. + */ public fun addBinaryRoot(root: Path) { binaryRoots.add(root) } + /** + * Adds a collection of [roots] to the current library. + * + * See [addBinaryRoot] for details + * + * @see addBinaryRoot for details + */ public fun addBinaryRoots(roots: Collection) { binaryRoots.addAll(roots) } diff --git a/analysis/build.gradle.kts b/analysis/build.gradle.kts index 61d23c3ec43..242b4bab799 100644 --- a/analysis/build.gradle.kts +++ b/analysis/build.gradle.kts @@ -14,4 +14,8 @@ tasks.register("analysisAllTests") { ":analysis:low-level-api-fir:test", ":analysis:symbol-light-classes:test" ) + + if (kotlinBuildProperties.isKotlinNativeEnabled) { + dependsOn(":analysis:analysis-api-standalone:analysis-api-standalone-native:test") + } } diff --git a/compiler/util-klib/src/org/jetbrains/kotlin/library/ToolingResolve.kt b/compiler/util-klib/src/org/jetbrains/kotlin/library/ToolingResolve.kt index b5d4b8fe594..67ae0e9fc80 100644 --- a/compiler/util-klib/src/org/jetbrains/kotlin/library/ToolingResolve.kt +++ b/compiler/util-klib/src/org/jetbrains/kotlin/library/ToolingResolve.kt @@ -27,6 +27,10 @@ import java.io.IOException */ object ToolingSingleFileKlibResolveStrategy : SingleFileKlibResolveStrategy { override fun resolve(libraryFile: File, logger: Logger): KotlinLibrary = + tryResolve(libraryFile, logger) + ?: fakeLibrary(libraryFile) + + fun tryResolve(libraryFile: File, logger: Logger): KotlinLibrary? = withSafeAccess(libraryFile) { localRoot -> if (localRoot.looksLikeKlibComponent) { // old style library @@ -49,7 +53,7 @@ object ToolingSingleFileKlibResolveStrategy : SingleFileKlibResolveStrategy { } } } - } ?: fakeLibrary(libraryFile) + } private const val NONEXISTENT_COMPONENT_NAME = "__nonexistent_component_name__" diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/NativeTestSupport.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/NativeTestSupport.kt index a57d6c3ffb0..938a93aa855 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/NativeTestSupport.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/NativeTestSupport.kt @@ -69,7 +69,7 @@ internal object CastCompatibleKotlinNativeClassLoader { val kotlinNativeClassLoader = NativeTestSupport.computeNativeClassLoader(this::class.java.classLoader) } -private object NativeTestSupport { +internal object NativeTestSupport { private val NAMESPACE = ExtensionContext.Namespace.create(NativeTestSupport::class.java.simpleName) /*************** Test process settings ***************/ diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/compilation/CompilationToolCall.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/compilation/CompilationToolCall.kt index c255000d878..2b5438c8cef 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/compilation/CompilationToolCall.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/compilation/CompilationToolCall.kt @@ -10,6 +10,7 @@ import org.jetbrains.kotlin.cli.common.messages.* import org.jetbrains.kotlin.compilerRunner.OutputItemsCollectorImpl import org.jetbrains.kotlin.compilerRunner.processCompilerOutput import org.jetbrains.kotlin.config.Services +import org.jetbrains.kotlin.konan.test.blackbox.support.NativeTestSupport import org.jetbrains.kotlin.konan.test.blackbox.support.settings.KotlinNativeTargets import java.io.ByteArrayOutputStream import java.io.File @@ -20,7 +21,7 @@ import kotlin.time.measureTime import kotlin.time.measureTimedValue import org.jetbrains.kotlin.konan.file.File as KonanFile -internal data class CompilationToolCallResult( +data class CompilationToolCallResult( val exitCode: ExitCode, val toolOutput: String, val toolOutputHasErrors: Boolean, @@ -74,6 +75,12 @@ internal fun callCompiler(compilerArgs: Array, kotlinNativeClassLoader: return CompilationToolCallResult(exitCode, compilerOutput, messageCollector.hasErrors(), duration) } + +fun callCompilerWithoutOutputInterceptor(compilerArgs: Array): CompilationToolCallResult { + val compilerClassLoader = NativeTestSupport.computeNativeClassLoader(parent = null).classLoader + return callCompilerWithoutOutputInterceptor(compilerArgs, compilerClassLoader) +} + internal fun callCompilerWithoutOutputInterceptor( compilerArgs: Array, kotlinNativeClassLoader: ClassLoader diff --git a/repo/gradle-build-conventions/buildsrc-compat/src/main/kotlin/nativeTest.kt b/repo/gradle-build-conventions/buildsrc-compat/src/main/kotlin/nativeTest.kt index a74a8406e21..fc24ac7dae0 100644 --- a/repo/gradle-build-conventions/buildsrc-compat/src/main/kotlin/nativeTest.kt +++ b/repo/gradle-build-conventions/buildsrc-compat/src/main/kotlin/nativeTest.kt @@ -94,7 +94,8 @@ fun Project.nativeTest( requirePlatformLibs: Boolean = false, customCompilerDependencies: List = emptyList(), customTestDependencies: List = emptyList(), - compilerPluginDependencies: List = emptyList() + compilerPluginDependencies: List = emptyList(), + body: Test.() -> Unit = {}, ) = projectTest( taskName, jUnitMode = JUnitMode.JUnit5, @@ -227,4 +228,5 @@ fun Project.nativeTest( """.trimIndent() ) } + body() } diff --git a/settings.gradle b/settings.gradle index 5a97fab3973..3c21c8f0234 100644 --- a/settings.gradle +++ b/settings.gradle @@ -579,7 +579,8 @@ include ":generators:analysis-api-generator", if (buildProperties.isKotlinNativeEnabled) { include ":generators:analysis-api-generator:generator-kotlin-native", - ":analysis:low-level-api-fir:low-level-api-fir-native" + ":analysis:low-level-api-fir:low-level-api-fir-native", + ":analysis:analysis-api-standalone:analysis-api-standalone-native" }