[AA] Add symbol tests for symbols from JS klibs

Use test compiler runner to compile JS libraries. Determine compiler
by the specified TARGET_PLATFORM. Add tests for dynamic type and
JS-specific .proto extensions.

Refactor the code for mock library compilation to make the switch
between platforms more straightforward.

KTIJ-27566
KT-63217
This commit is contained in:
Pavel Kirpichenkov
2023-11-17 20:11:08 +02:00
committed by Space Team
parent 0736cb3fac
commit 0eb1a63a2f
35 changed files with 543 additions and 116 deletions
@@ -18,7 +18,7 @@ import org.jetbrains.kotlin.analysis.test.framework.project.structure.ktModulePr
import org.jetbrains.kotlin.analysis.test.framework.services.ExpressionMarkerProvider
import org.jetbrains.kotlin.analysis.test.framework.services.ExpressionMarkersSourceFilePreprocessor
import org.jetbrains.kotlin.analysis.test.framework.services.expressionMarkerProvider
import org.jetbrains.kotlin.analysis.test.framework.services.libraries.CompilerExecutor
import org.jetbrains.kotlin.analysis.test.framework.services.libraries.TestModuleCompiler
import org.jetbrains.kotlin.analysis.test.framework.test.configurators.AnalysisApiTestConfigurator
import org.jetbrains.kotlin.analysis.test.framework.test.configurators.FrontendKind
import org.jetbrains.kotlin.analysis.test.framework.utils.SkipTestException
@@ -117,7 +117,7 @@ abstract class AbstractAnalysisApiBasedTest : TestWithDisposable() {
useDirectives(*AbstractKotlinCompilerTest.defaultDirectiveContainers.toTypedArray())
useDirectives(JvmEnvironmentConfigurationDirectives)
useDirectives(CompilerExecutor.Directives)
useDirectives(TestModuleCompiler.Directives)
useSourcePreprocessor(::ExpressionMarkersSourceFilePreprocessor)
@@ -10,6 +10,7 @@ import com.intellij.psi.PsiFile
import com.intellij.psi.search.GlobalSearchScope
import org.jetbrains.kotlin.analysis.api.standalone.base.project.structure.StandaloneProjectFactory
import org.jetbrains.kotlin.analysis.project.structure.*
import org.jetbrains.kotlin.analysis.test.framework.services.environmentManager
import org.jetbrains.kotlin.cli.jvm.compiler.TopDownAnalyzerFacadeForJVM
import org.jetbrains.kotlin.cli.jvm.config.jvmClasspathRoots
import org.jetbrains.kotlin.cli.jvm.config.jvmModularRoots
@@ -17,7 +18,6 @@ import org.jetbrains.kotlin.config.JVMConfigurationKeys
import org.jetbrains.kotlin.config.LanguageVersionSettings
import org.jetbrains.kotlin.platform.TargetPlatform
import org.jetbrains.kotlin.platform.jvm.JvmPlatforms
import org.jetbrains.kotlin.platform.jvm.isJvm
import org.jetbrains.kotlin.platform.konan.isNative
import org.jetbrains.kotlin.psi.KtFile
import org.jetbrains.kotlin.resolve.PlatformDependentAnalyzerServices
@@ -71,7 +71,10 @@ abstract class KtModuleByCompilerConfiguration(
private fun createJdkFromConfiguration(): KtSdkModule? = configuration.get(JVMConfigurationKeys.JDK_HOME)?.let { jdkHome ->
val jdkHomePaths = StandaloneProjectFactory.getDefaultJdkModulePaths(project, jdkHome.toPath())
val scope = TestModuleStructureFactory.getScopeForLibraryByRoots(jdkHomePaths, testServices)
val scope = StandaloneProjectFactory.createSearchScopeByLibraryRoots(
jdkHomePaths,
testServices.environmentManager.getProjectEnvironment()
)
KtJdkModuleImpl(
"jdk",
@@ -171,7 +174,10 @@ private class LibraryByRoots(
override val project: Project,
testServices: TestServices,
) : KtLibraryModule {
override val contentScope: GlobalSearchScope = TestModuleStructureFactory.getScopeForLibraryByRoots(roots, testServices)
override val contentScope: GlobalSearchScope = StandaloneProjectFactory.createSearchScopeByLibraryRoots(
roots,
testServices.environmentManager.getProjectEnvironment(),
)
override val libraryName: String get() = "Test Library $roots"
override val directRegularDependencies: List<KtModule> get() = emptyList()
override val directDependsOnDependencies: List<KtModule> get() = emptyList()
@@ -8,7 +8,6 @@ package org.jetbrains.kotlin.analysis.test.framework.project.structure
import com.intellij.openapi.project.Project
import com.intellij.psi.PsiFile
import com.intellij.psi.PsiManager
import com.intellij.psi.search.GlobalSearchScope
import org.jetbrains.kotlin.analysis.api.standalone.base.project.structure.KtModuleProjectStructure
import org.jetbrains.kotlin.analysis.api.standalone.base.project.structure.KtModuleWithFiles
import org.jetbrains.kotlin.analysis.api.standalone.base.project.structure.StandaloneProjectFactory
@@ -20,7 +19,10 @@ import org.jetbrains.kotlin.analysis.test.framework.services.environmentManager
import org.jetbrains.kotlin.analysis.utils.errors.requireIsInstance
import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
import org.jetbrains.kotlin.cli.jvm.config.JvmClasspathRoot
import org.jetbrains.kotlin.platform.isCommon
import org.jetbrains.kotlin.platform.isJs
import org.jetbrains.kotlin.platform.jvm.JvmPlatforms
import org.jetbrains.kotlin.platform.jvm.isJvm
import org.jetbrains.kotlin.test.TestInfrastructureInternals
import org.jetbrains.kotlin.test.model.DependencyRelation
import org.jetbrains.kotlin.test.model.TestModule
@@ -56,13 +58,7 @@ object TestModuleStructureFactory {
}
is KtModuleWithModifiableDependencies -> {
addModuleDependencies(testModule, moduleEntriesByName, ktModule)
buildList {
addIfNotNull(getJdkModule(testModule, project, testServices))
addAll(getStdlibModules(testModule, project, testServices))
addAll(getLibraryModules(testServices, testModule, project))
addAll(createLibrariesByCompilerConfigurators(testModule, testServices, project))
}.forEach { library ->
stdlibAndSdkDependencies(ktModule, testModule, project, testServices).forEach { library ->
val cachedLibrary = binaryModulesBySourceRoots.getOrPut(library.getBinaryRoots().toSet()) { library }
ktModule.directRegularDependencies.add(cachedLibrary)
}
@@ -74,6 +70,28 @@ object TestModuleStructureFactory {
return KtModuleProjectStructure(moduleEntries, binaryModulesBySourceRoots.values)
}
private fun stdlibAndSdkDependencies(
module: KtModule,
testModule: TestModule,
project: Project,
testServices: TestServices,
): List<KtBinaryModule> {
return when {
module.platform.isJvm() -> jvmStdlibAndJdkDependencies(testModule, project, testServices)
module.platform.isJs() -> jsStdlibDependencies(project, testServices)
else -> error("Unsupported platform ${module.platform} in this test")
}
}
private fun jvmStdlibAndJdkDependencies(testModule: TestModule, project: Project, testServices: TestServices): List<KtBinaryModule> {
return buildList {
addIfNotNull(getJdkModule(testModule, project, testServices))
addAll(getStdlibModules(testModule, project, testServices))
addAll(getLibraryModules(testServices, testModule, project))
addAll(createLibrariesByCompilerConfigurators(testModule, testServices, project))
}
}
@OptIn(TestInfrastructureInternals::class)
private fun createLibrariesByCompilerConfigurators(
testModule: TestModule,
@@ -130,7 +148,10 @@ object TestModuleStructureFactory {
return KtLibraryModuleImpl(
libraryName,
JvmPlatforms.defaultJvmPlatform,
getScopeForLibraryByRoots(listOf(jar), testServices),
StandaloneProjectFactory.createSearchScopeByLibraryRoots(
listOf(jar),
testServices.environmentManager.getProjectEnvironment()
),
project,
listOf(jar),
librarySources = null,
@@ -167,16 +188,33 @@ object TestModuleStructureFactory {
return KtJdkModuleImpl(
"jdk",
JvmPlatforms.defaultJvmPlatform,
getScopeForLibraryByRoots(jdkSourceRoots, testServices),
StandaloneProjectFactory.createSearchScopeByLibraryRoots(
jdkSourceRoots,
testServices.environmentManager.getProjectEnvironment()
),
project,
jdkSourceRoots
)
}
fun getScopeForLibraryByRoots(roots: Collection<Path>, testServices: TestServices): GlobalSearchScope {
return StandaloneProjectFactory.createSearchScopeByLibraryRoots(
roots,
testServices.environmentManager.getProjectEnvironment()
private fun jsStdlibDependencies(project: Project, testServices: TestServices): List<KtBinaryModule> {
return listOf(
jsStdlib(project, testServices),
)
}
private fun jsStdlib(project: Project, testServices: TestServices): KtBinaryModule {
val jar = testServices.standardLibrariesPathProvider.fullJsStdlib().toPath()
return KtLibraryModuleImpl(
PathUtil.JS_LIB_NAME,
JvmPlatforms.defaultJvmPlatform,
StandaloneProjectFactory.createSearchScopeByLibraryRoots(
listOf(jar),
testServices.environmentManager.getProjectEnvironment()
),
project,
listOf(jar),
librarySources = null,
)
}
@@ -0,0 +1,143 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.analysis.test.framework.services.libraries
import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments
import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments
import org.jetbrains.kotlin.cli.common.arguments.cliArgument
import org.jetbrains.kotlin.config.JvmTarget
import org.jetbrains.kotlin.platform.isJs
import org.jetbrains.kotlin.platform.jvm.isJvm
import org.jetbrains.kotlin.test.directives.JvmEnvironmentConfigurationDirectives
import org.jetbrains.kotlin.test.directives.LanguageSettingsDirectives
import org.jetbrains.kotlin.test.model.TestModule
import org.jetbrains.kotlin.test.services.TestServices
import org.jetbrains.kotlin.test.services.sourceFileProvider
import org.jetbrains.kotlin.test.services.standardLibrariesPathProvider
import org.jetbrains.kotlin.test.util.KtTestUtil
import java.io.ByteArrayInputStream
import java.nio.file.Path
import java.util.jar.Attributes
import java.util.jar.JarEntry
import java.util.jar.JarOutputStream
import java.util.jar.Manifest
import kotlin.io.path.div
import kotlin.io.path.outputStream
abstract class CliTestModuleCompiler : TestModuleCompiler() {
internal abstract val compilerKind: CompilerExecutor.CompilerKind
protected abstract fun buildPlatformCompilerOptions(module: TestModule, testServices: TestServices): List<String>
override fun compile(tmpDir: Path, module: TestModule, testServices: TestServices): Path = CompilerExecutor.compileLibrary(
compilerKind,
tmpDir,
buildCompilerOptions(module, testServices),
compilationErrorExpected = Directives.COMPILATION_ERRORS in module.directives
)
override fun compileTestModuleToLibrarySources(module: TestModule, testServices: TestServices): Path {
val tmpDir = KtTestUtil.tmpDir("testSourcesToCompile").toPath()
val librarySourcesPath = tmpDir / "library-sources.jar"
val manifest = Manifest().apply { mainAttributes[Attributes.Name.MANIFEST_VERSION] = "1.0" }
JarOutputStream(librarySourcesPath.outputStream(), manifest).use { jarOutputStream ->
for (testFile in module.files) {
val text = testServices.sourceFileProvider.getContentOfSourceFile(testFile)
addFileToJar(testFile.relativePath, text, jarOutputStream)
}
}
return librarySourcesPath
}
private fun buildCompilerOptions(module: TestModule, testServices: TestServices): List<String> = buildList {
addAll(buildCommonCompilerOptions(module))
addAll(buildPlatformCompilerOptions(module, testServices))
}
private fun buildCommonCompilerOptions(module: TestModule): List<String> = buildList {
module.directives[LanguageSettingsDirectives.API_VERSION].firstOrNull()?.let { apiVersion ->
addAll(listOf(CommonCompilerArguments::apiVersion.cliArgument, apiVersion.versionString))
}
module.directives[LanguageSettingsDirectives.LANGUAGE].firstOrNull()?.let {
add("-XXLanguage:$it")
}
if (LanguageSettingsDirectives.ALLOW_KOTLIN_PACKAGE in module.directives) {
add(CommonCompilerArguments::allowKotlinPackage.cliArgument)
}
addAll(module.directives[Directives.COMPILER_ARGUMENTS])
}
private fun addFileToJar(path: String, text: String, jarOutputStream: JarOutputStream) {
jarOutputStream.putNextEntry(JarEntry(path))
ByteArrayInputStream(text.toByteArray()).copyTo(jarOutputStream)
jarOutputStream.closeEntry()
}
}
class JvmJarTestModuleCompiler : CliTestModuleCompiler() {
override val compilerKind = CompilerExecutor.CompilerKind.JVM
override fun buildPlatformCompilerOptions(module: TestModule, testServices: TestServices): List<String> = buildList {
module.directives[JvmEnvironmentConfigurationDirectives.JVM_TARGET].firstOrNull()?.let { jvmTarget ->
addAll(listOf(K2JVMCompilerArguments::jvmTarget.cliArgument, jvmTarget.description))
val jdkHome = when {
jvmTarget <= JvmTarget.JVM_1_8 -> KtTestUtil.getJdk8Home()
jvmTarget <= JvmTarget.JVM_11 -> KtTestUtil.getJdk11Home()
jvmTarget <= JvmTarget.JVM_17 -> KtTestUtil.getJdk17Home()
jvmTarget <= JvmTarget.JVM_21 -> KtTestUtil.getJdk21Home()
else -> error("JDK for $jvmTarget is not found")
}
addAll(listOf(K2JVMCompilerArguments::jdkHome.cliArgument, jdkHome.toString()))
}
}
}
class JsKlibTestModuleCompiler : CliTestModuleCompiler() {
override val compilerKind = CompilerExecutor.CompilerKind.JS
override fun buildPlatformCompilerOptions(module: TestModule, testServices: TestServices): List<String> {
return listOf(
K2JSCompilerArguments::libraries.cliArgument, testServices.standardLibrariesPathProvider.fullJsStdlib().absolutePath,
)
}
}
/**
* [DispatchingTestModuleCompiler] chooses the appropriate compiler for a module based on its platform.
* In case all tests in a suite should compile libraries for a single platform, one of the underlying [TestModuleCompiler]s
* can be registered directly. Once new test compilers are introduced, they should be added to [DispatchingTestModuleCompiler].
*/
class DispatchingTestModuleCompiler : TestModuleCompiler() {
private val compilersByKind = mapOf(
CompilerExecutor.CompilerKind.JVM to JvmJarTestModuleCompiler(),
CompilerExecutor.CompilerKind.JS to JsKlibTestModuleCompiler(),
)
override fun compile(tmpDir: Path, module: TestModule, testServices: TestServices): Path {
return getCompiler(module).compileTestModuleToLibrary(module, testServices)
}
override fun compileTestModuleToLibrarySources(module: TestModule, testServices: TestServices): Path {
return getCompiler(module).compileTestModuleToLibrarySources(module, testServices)
}
private fun getCompiler(module: TestModule): CliTestModuleCompiler {
val compilerKindForModule = when {
module.targetPlatform.isJvm() -> CompilerExecutor.CompilerKind.JVM
module.targetPlatform.isJs() -> CompilerExecutor.CompilerKind.JS
else -> error("DispatchingTestModuleCompiler doesn't support the platform: ${module.targetPlatform}")
}
return compilersByKind[compilerKindForModule]
?: error("TestModuleCompiler is not available for ${compilerKindForModule.name}")
}
}
@@ -6,83 +6,68 @@
package org.jetbrains.kotlin.analysis.test.framework.services.libraries
import org.jetbrains.kotlin.analysis.test.framework.utils.SkipTestException
import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
import org.jetbrains.kotlin.cli.common.arguments.cliArgument
import org.jetbrains.kotlin.cli.common.arguments.K2JSCompilerArguments
import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments
import org.jetbrains.kotlin.config.JvmTarget
import org.jetbrains.kotlin.cli.common.arguments.cliArgument
import org.jetbrains.kotlin.config.LanguageFeature
import org.jetbrains.kotlin.test.MockLibraryUtil
import org.jetbrains.kotlin.test.directives.JvmEnvironmentConfigurationDirectives
import org.jetbrains.kotlin.test.directives.LanguageSettingsDirectives
import org.jetbrains.kotlin.test.directives.model.SimpleDirectivesContainer
import org.jetbrains.kotlin.test.model.TestModule
import org.jetbrains.kotlin.test.util.KtTestUtil
import java.nio.file.Path
import kotlin.io.path.absolutePathString
import kotlin.io.path.div
import kotlin.io.path.exists
import kotlin.io.path.notExists
import kotlin.io.path.*
object CompilerExecutor {
fun compileLibrary(sourcesPath: Path, options: List<String>, compilationErrorExpected: Boolean): Path {
val library = sourcesPath / "library.jar"
val sourceFiles = sourcesPath.toFile().walkBottomUp()
val commands = buildList {
sourceFiles.mapTo(this) { it.absolutePath }
addAll(options)
add("-d")
add(library.absolutePathString())
add("-XXLanguage:-SkipStandaloneScriptsInSourceRoots")
}
try {
MockLibraryUtil.runJvmCompiler(commands)
internal object CompilerExecutor {
fun compileLibrary(compilerKind: CompilerKind, sourcesPath: Path, options: List<String>, compilationErrorExpected: Boolean): Path {
val library = try {
compile(compilerKind, sourcesPath, options)
} catch (e: Throwable) {
if (!compilationErrorExpected) {
throw IllegalStateException("Unexpected compilation error while compiling library", e)
}
null
}
if (library.exists() && compilationErrorExpected) {
if (library?.exists() == true && compilationErrorExpected) {
error("Compilation error expected but, code was compiled successfully")
}
if (library.notExists()) {
if (library == null || library.notExists()) {
throw LibraryWasNotCompiledDueToExpectedCompilationError()
}
return library
}
fun parseCompilerOptionsFromTestdata(module: TestModule): List<String> = buildList {
module.directives[LanguageSettingsDirectives.API_VERSION].firstOrNull()?.let { apiVersion ->
addAll(listOf(CommonCompilerArguments::apiVersion.cliArgument, apiVersion.versionString))
private fun compile(compilerKind: CompilerKind, sourcesPath: Path, options: List<String>): Path {
val sourceFiles = sourcesPath.toFile().walkBottomUp()
val library = when (compilerKind) {
CompilerKind.JVM -> sourcesPath / "library.jar"
CompilerKind.JS -> sourcesPath / "library.klib"
}
module.directives[LanguageSettingsDirectives.LANGUAGE].firstOrNull()?.let {
add("-XXLanguage:$it")
}
if (LanguageSettingsDirectives.ALLOW_KOTLIN_PACKAGE in module.directives) {
add(CommonCompilerArguments::allowKotlinPackage.cliArgument)
}
module.directives[JvmEnvironmentConfigurationDirectives.JVM_TARGET].firstOrNull()?.let { jvmTarget ->
addAll(listOf(K2JVMCompilerArguments::jvmTarget.cliArgument, jvmTarget.description))
val jdkHome = when {
jvmTarget <= JvmTarget.JVM_1_8 -> KtTestUtil.getJdk8Home()
jvmTarget <= JvmTarget.JVM_11 -> KtTestUtil.getJdk11Home()
jvmTarget <= JvmTarget.JVM_17 -> KtTestUtil.getJdk17Home()
jvmTarget <= JvmTarget.JVM_21 -> KtTestUtil.getJdk21Home()
else -> error("JDK for $jvmTarget is not found")
when (compilerKind) {
CompilerKind.JVM -> {
val commands = buildList {
sourceFiles.mapTo(this) { it.absolutePath }
addAll(options)
add(K2JVMCompilerArguments::destination.cliArgument); add(library.absolutePathString())
add("-XXLanguage:-${LanguageFeature.SkipStandaloneScriptsInSourceRoots.name}")
}
MockLibraryUtil.runJvmCompiler(commands)
}
CompilerKind.JS -> {
val commands = buildList {
add(K2JSCompilerArguments::metaInfo.cliArgument)
add(K2JSCompilerArguments::moduleName.cliArgument); add("library")
add(K2JSCompilerArguments::outputDir.cliArgument); add(library.parent.absolutePathString())
add(K2JSCompilerArguments::irProduceKlibFile.cliArgument)
sourceFiles.mapTo(this) { it.absolutePath }
addAll(options)
}
MockLibraryUtil.runJsCompiler(commands)
}
addAll(listOf(K2JVMCompilerArguments::jdkHome.cliArgument, jdkHome.toString()))
}
addAll(module.directives[Directives.COMPILER_ARGUMENTS])
return library
}
object Directives : SimpleDirectivesContainer() {
val COMPILER_ARGUMENTS by stringDirective("List of additional compiler arguments")
val COMPILATION_ERRORS by directive("Is compilation errors expected in the file")
enum class CompilerKind {
JVM, JS
}
}
@@ -5,20 +5,15 @@
package org.jetbrains.kotlin.analysis.test.framework.services.libraries
import org.jetbrains.kotlin.test.directives.model.SimpleDirectivesContainer
import org.jetbrains.kotlin.test.model.TestModule
import org.jetbrains.kotlin.test.services.TestService
import org.jetbrains.kotlin.test.services.TestServices
import org.jetbrains.kotlin.test.services.sourceFileProvider
import org.jetbrains.kotlin.test.util.KtTestUtil
import java.io.ByteArrayInputStream
import java.nio.file.Path
import java.util.jar.Attributes
import java.util.jar.JarEntry
import java.util.jar.JarOutputStream
import java.util.jar.Manifest
import kotlin.io.path.createFile
import kotlin.io.path.div
import kotlin.io.path.outputStream
import kotlin.io.path.writeText
abstract class TestModuleCompiler : TestService {
@@ -29,37 +24,15 @@ abstract class TestModuleCompiler : TestService {
val tmpSourceFile = (tmpDir / testFile.name).createFile()
tmpSourceFile.writeText(text)
}
return compile(tmpDir, module)
return compile(tmpDir, module, testServices)
}
abstract fun compile(tmpDir: Path, module: TestModule): Path
abstract fun compile(tmpDir: Path, module: TestModule, testServices: TestServices): Path
abstract fun compileTestModuleToLibrarySources(module: TestModule, testServices: TestServices): Path?
}
class TestModuleCompilerJar : TestModuleCompiler() {
override fun compile(tmpDir: Path, module: TestModule): Path = CompilerExecutor.compileLibrary(
tmpDir,
CompilerExecutor.parseCompilerOptionsFromTestdata(module),
compilationErrorExpected = CompilerExecutor.Directives.COMPILATION_ERRORS in module.directives
)
override fun compileTestModuleToLibrarySources(module: TestModule, testServices: TestServices): Path {
fun addFileToJar(path: String, text: String, jarOutputStream: JarOutputStream) {
jarOutputStream.putNextEntry(JarEntry(path))
ByteArrayInputStream(text.toByteArray()).copyTo(jarOutputStream)
jarOutputStream.closeEntry()
}
val tmpDir = KtTestUtil.tmpDir("testSourcesToCompile").toPath()
val librarySourcesPath = tmpDir / "library-sources.jar"
val manifest = Manifest().apply { mainAttributes[Attributes.Name.MANIFEST_VERSION] = "1.0" }
JarOutputStream(librarySourcesPath.outputStream(), manifest).use { jarOutputStream ->
for (testFile in module.files) {
val text = testServices.sourceFileProvider.getContentOfSourceFile(testFile)
addFileToJar(testFile.relativePath, text, jarOutputStream)
}
}
return librarySourcesPath
object Directives : SimpleDirectivesContainer() {
val COMPILER_ARGUMENTS by stringDirective("List of additional compiler arguments")
val COMPILATION_ERRORS by directive("Is compilation errors expected in the file")
}
}