From 59e69481a3af738f8bd096a172aee34a001cd218 Mon Sep 17 00:00:00 2001 From: Simon Ogorodnik Date: Wed, 1 Feb 2023 20:32:58 +0100 Subject: [PATCH] Modularized tests. Support parsing of original compiler arguments Now testData contains info about original compiler arguments, which can be reused to support plugins and various other arguments Support original arguments in modularized tests - Properly support JDK9 modules - Opt-ins from original arguments - jdkHome from module data or arguments, as well as noJdk option - Fix friendPaths not configured - jvmDefault now handled properly ^KT-56075 ^KT-55879 --- .../fir/AbstractFrontendModularizedTest.kt | 84 ++++++++++++++++--- .../AbstractFullPipelineModularizedTest.kt | 23 ++++- .../kotlin/fir/AbstractModularizedTest.kt | 29 ++++++- .../fir/FE1FullPipelineModularizedTest.kt | 31 ++++--- .../FirResolveModularizedTotalKotlinTest.kt | 2 + .../kotlin/fir/FullPipelineModularizedTest.kt | 29 ++++--- ...NonFirResolveModularizedTotalKotlinTest.kt | 54 +++++++----- 7 files changed, 193 insertions(+), 59 deletions(-) diff --git a/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/AbstractFrontendModularizedTest.kt b/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/AbstractFrontendModularizedTest.kt index f5e9369b89e..8644a068b43 100644 --- a/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/AbstractFrontendModularizedTest.kt +++ b/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/AbstractFrontendModularizedTest.kt @@ -5,34 +5,96 @@ package org.jetbrains.kotlin.fir +import com.intellij.psi.PsiJavaModule import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys +import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments import org.jetbrains.kotlin.cli.common.config.KotlinSourceRoot +import org.jetbrains.kotlin.cli.jvm.config.JvmClasspathRoot +import org.jetbrains.kotlin.cli.jvm.config.JvmModulePathRoot import org.jetbrains.kotlin.cli.jvm.config.addJavaSourceRoot -import org.jetbrains.kotlin.cli.jvm.config.addJvmClasspathRoots import org.jetbrains.kotlin.cli.jvm.config.configureJdkClasspathRoots import org.jetbrains.kotlin.config.* import org.jetbrains.kotlin.test.KotlinTestUtils +import java.io.File abstract class AbstractFrontendModularizedTest : AbstractModularizedTest() { - fun createDefaultConfiguration(moduleData: ModuleData): CompilerConfiguration { + + fun configureLanguageVersionSettings( + configuration: CompilerConfiguration, + moduleData: ModuleData, + languageVersion: LanguageVersion, + configureFlags: MutableMap, Any?>.() -> Unit = {}, + configureFeatures: MutableMap.() -> Unit = {} + ) { + val originalArguments = moduleData.arguments as? K2JVMCompilerArguments + + configuration.languageVersionSettings = LanguageVersionSettingsImpl( + languageVersion, + originalArguments?.apiVersion?.let { ApiVersion.parse(it) } ?: ApiVersion.LATEST_STABLE, + analysisFlags = buildMap { + put(AnalysisFlags.optIn, moduleData.optInAnnotations + originalArguments?.optIn.orEmpty()) + if (originalArguments != null) { + put(AnalysisFlags.skipPrereleaseCheck, originalArguments.skipPrereleaseCheck) + put(JvmAnalysisFlags.jvmDefaultMode, JvmDefaultMode.fromStringOrNull(originalArguments.jvmDefault)) + } + + configureFlags() + }, + specificFeatures = buildMap { + configureFeatures() + } + ) + } + + fun createDefaultConfiguration( + moduleData: ModuleData + ): CompilerConfiguration { val configuration = KotlinTestUtils.newConfiguration() + val originalArguments = moduleData.arguments as? K2JVMCompilerArguments + moduleData.javaSourceRoots.forEach { configuration.addJavaSourceRoot(it.path, it.packagePrefix) } - configuration.addJvmClasspathRoots(moduleData.classpath) - configuration.languageVersionSettings = LanguageVersionSettingsImpl( - LanguageVersion.LATEST_STABLE, - ApiVersion.LATEST_STABLE, - analysisFlags = mutableMapOf(AnalysisFlags.optIn to moduleData.optInAnnotations) - ) - configuration.configureJdkClasspathRoots() - // in case of modular jdk only - configuration.putIfNotNull(JVMConfigurationKeys.JDK_HOME, moduleData.modularJdkRoot) + val isJava9Module = moduleData.javaSourceRoots.any { (file, packagePrefix) -> + packagePrefix == null && + (file.name == PsiJavaModule.MODULE_INFO_FILE || + (file.isDirectory && file.listFiles()!!.any { it.name == PsiJavaModule.MODULE_INFO_FILE })) + } + + for (rootPath in moduleData.classpath) { + if (isJava9Module) { + configuration.add(CLIConfigurationKeys.CONTENT_ROOTS, JvmModulePathRoot(rootPath)) + } + configuration.add(CLIConfigurationKeys.CONTENT_ROOTS, JvmClasspathRoot(rootPath)) + } + + val jdkHome = + moduleData.modularJdkRoot + ?: moduleData.jdkHome?.absoluteFile + ?: originalArguments?.jdkHome?.fixPath()?.absoluteFile + + if (originalArguments != null) { + configuration.put(JVMConfigurationKeys.NO_JDK, originalArguments.noJdk) + + for (modularRoot in originalArguments.javaModulePath?.split(File.pathSeparatorChar).orEmpty()) { + configuration.add(CLIConfigurationKeys.CONTENT_ROOTS, JvmModulePathRoot(modularRoot.fixPath())) + } + + configuration.put( + JVMConfigurationKeys.JVM_TARGET, + originalArguments.jvmTargetIfSupported() ?: JvmTarget.DEFAULT + ) + } + configuration.putIfNotNull(JVMConfigurationKeys.JDK_HOME, jdkHome) + configuration.configureJdkClasspathRoots() configuration.addAll( CLIConfigurationKeys.CONTENT_ROOTS, moduleData.sources.filter { it.extension == "kt" || it.isDirectory }.map { KotlinSourceRoot(it.absolutePath, false) }) + + configuration.addAll(JVMConfigurationKeys.FRIEND_PATHS, moduleData.friendDirs.map { it.absolutePath }) + return configuration } } \ No newline at end of file diff --git a/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/AbstractFullPipelineModularizedTest.kt b/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/AbstractFullPipelineModularizedTest.kt index 5b96d07d19e..8f73805ded8 100644 --- a/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/AbstractFullPipelineModularizedTest.kt +++ b/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/AbstractFullPipelineModularizedTest.kt @@ -128,10 +128,27 @@ abstract class AbstractFullPipelineModularizedTest : AbstractModularizedTest() { } private fun configureBaseArguments(args: K2JVMCompilerArguments, moduleData: ModuleData, tmp: Path) { + val originalArguments = moduleData.arguments as? K2JVMCompilerArguments + if (originalArguments != null) { + args.apiVersion = originalArguments.apiVersion + args.noJdk = originalArguments.noJdk + args.noStdlib = originalArguments.noStdlib + args.noReflect = originalArguments.noReflect + args.jvmTarget = originalArguments.jvmTargetIfSupported()?.description + args.jsr305 = originalArguments.jsr305 + args.nullabilityAnnotations = originalArguments.nullabilityAnnotations + args.jspecifyAnnotations = originalArguments.jspecifyAnnotations + args.jvmDefault = originalArguments.jvmDefault + args.jdkRelease = originalArguments.jdkRelease + args.progressiveMode = originalArguments.progressiveMode + args.optIn = (moduleData.optInAnnotations + (originalArguments.optIn ?: emptyArray())).toTypedArray() + args.allowKotlinPackage = originalArguments.allowKotlinPackage + } else { + args.jvmTarget = JVM_TARGET + args.allowKotlinPackage = true + } args.reportPerf = true - args.jvmTarget = JVM_TARGET - args.allowKotlinPackage = true - args.jdkHome = moduleData.jdkHome?.absolutePath + args.jdkHome = moduleData.jdkHome?.absolutePath ?: originalArguments?.jdkHome?.fixPath()?.absolutePath args.renderInternalDiagnosticNames = true configureArgsUsingBuildFile(args, moduleData, tmp) } diff --git a/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/AbstractModularizedTest.kt b/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/AbstractModularizedTest.kt index 8adc07d458c..e87d83e1996 100644 --- a/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/AbstractModularizedTest.kt +++ b/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/AbstractModularizedTest.kt @@ -6,7 +6,11 @@ package org.jetbrains.kotlin.fir import com.intellij.openapi.util.JDOMUtil +import com.intellij.util.xmlb.XmlSerializer import org.jdom.Element +import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments +import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments +import org.jetbrains.kotlin.config.JvmTarget import org.jetbrains.kotlin.fir.scopes.ProcessorAction import org.jetbrains.kotlin.test.testFramework.KtUsefulTestCase import org.jetbrains.kotlin.types.AbstractTypeChecker @@ -37,11 +41,16 @@ data class ModuleData( val friendDirs = rawFriendDirs.map { it.fixPath() } val jdkHome = rawJdkHome?.fixPath() val modularJdkRoot = rawModularJdkRoot?.fixPath() + + /** + * Raw compiler arguments, as it was passed to original module build + */ + var arguments: CommonCompilerArguments? = null } data class JavaSourceRootData(val path: Path, val packagePrefix: String?) -private fun String.fixPath(): File = File(ROOT_PATH_PREFIX, this.removePrefix("/")) +internal fun String.fixPath(): File = File(ROOT_PATH_PREFIX, this.removePrefix("/")) private val ROOT_PATH_PREFIX:String = System.getProperty("fir.bench.prefix", "/") private val OUTPUT_DIR_REGEX_FILTER:String = System.getProperty("fir.bench.filter", ".*") @@ -136,7 +145,16 @@ abstract class AbstractModularizedTest : KtUsefulTestCase() { private fun loadModuleDumpFile(file: File): List { val rootElement = JDOMUtil.load(file) val modules = rootElement.getChildren("module") - return modules.map { node -> loadModule(node) } + val arguments = rootElement.getChild("compilerArguments")?.let { loadCompilerArguments(it) } + return modules.map { node -> loadModule(node).also { it.arguments = arguments } } + } + + private fun loadCompilerArguments(argumentsRoot: Element): CommonCompilerArguments? { + val element = argumentsRoot.children.singleOrNull() ?: return null + return when (element.name) { + "K2JVMCompilerArguments" -> K2JVMCompilerArguments().also { XmlSerializer.deserializeInto(it, element) } + else -> null + } } protected abstract fun beforePass(pass: Int) @@ -172,3 +190,10 @@ abstract class AbstractModularizedTest : KtUsefulTestCase() { afterPass(pass) } } + + +internal fun K2JVMCompilerArguments.jvmTargetIfSupported(): JvmTarget? { + val specified = jvmTarget?.let { JvmTarget.fromString(it) } ?: return null + if (specified != JvmTarget.JVM_1_6) return specified + return null +} \ No newline at end of file diff --git a/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/FE1FullPipelineModularizedTest.kt b/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/FE1FullPipelineModularizedTest.kt index bf27aad4549..1a598f6d9f1 100644 --- a/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/FE1FullPipelineModularizedTest.kt +++ b/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/FE1FullPipelineModularizedTest.kt @@ -7,21 +7,30 @@ package org.jetbrains.kotlin.fir import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments +/** + * Language version to be used for K1 FP tests + */ +internal val LANGUAGE_VERSION_K1: String = System.getProperty("fir.bench.language.version.k1", "1.8") + class FE1FullPipelineModularizedTest : AbstractFullPipelineModularizedTest() { override fun configureArguments(args: K2JVMCompilerArguments, moduleData: ModuleData) { args.useK2 = false args.useIR = true - args.jvmDefault = "compatibility" - args.apiVersion = API_VERSION - args.optIn = arrayOf( - "kotlin.RequiresOptIn", - "kotlin.contracts.ExperimentalContracts", - "kotlin.io.path.ExperimentalPathApi", - "org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI" - ) - args.multiPlatform = true - args.noStdlib = true - args.noReflect = true + args.languageVersion = LANGUAGE_VERSION_K1 + // TODO: Remove when support for old modularized tests is removed + if (moduleData.arguments == null) { + args.jvmDefault = "compatibility" + args.apiVersion = API_VERSION + args.optIn = arrayOf( + "kotlin.RequiresOptIn", + "kotlin.contracts.ExperimentalContracts", + "kotlin.io.path.ExperimentalPathApi", + "org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI" + ) + args.multiPlatform = true + args.noStdlib = true + args.noReflect = true + } } fun testTotalKotlin() { diff --git a/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/FirResolveModularizedTotalKotlinTest.kt b/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/FirResolveModularizedTotalKotlinTest.kt index 89420c30120..df11f5d5f7e 100644 --- a/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/FirResolveModularizedTotalKotlinTest.kt +++ b/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/FirResolveModularizedTotalKotlinTest.kt @@ -19,6 +19,7 @@ import org.jetbrains.kotlin.asJava.finder.JavaElementFinder import org.jetbrains.kotlin.cli.common.toBooleanLenient import org.jetbrains.kotlin.cli.jvm.compiler.* import org.jetbrains.kotlin.cli.jvm.compiler.pipeline.collectSources +import org.jetbrains.kotlin.config.LanguageVersion import org.jetbrains.kotlin.config.languageVersionSettings import org.jetbrains.kotlin.diagnostics.DiagnosticReporterFactory import org.jetbrains.kotlin.fir.analysis.collectors.AbstractDiagnosticCollector @@ -213,6 +214,7 @@ class FirResolveModularizedTotalKotlinTest : AbstractFrontendModularizedTest() { override fun processModule(moduleData: ModuleData): ProcessorAction { val disposable = Disposer.newDisposable() val configuration = createDefaultConfiguration(moduleData) + configureLanguageVersionSettings(configuration, moduleData, LanguageVersion.fromVersionString(LANGUAGE_VERSION_K2)!!) val environment = KotlinCoreEnvironment.createForTests(disposable, configuration, EnvironmentConfigFiles.JVM_CONFIG_FILES) PsiElementFinder.EP.getPoint(environment.project) diff --git a/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/FullPipelineModularizedTest.kt b/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/FullPipelineModularizedTest.kt index 71737029bca..39f1874c807 100644 --- a/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/FullPipelineModularizedTest.kt +++ b/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/FullPipelineModularizedTest.kt @@ -8,11 +8,11 @@ package org.jetbrains.kotlin.fir import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments // This is used for API version configuration for both frontends -// TODO: rename system property +// TODO: Deprecated and only used in old FP tests internal val API_VERSION: String = System.getProperty("fir.bench.language.version", "1.4") -// This is used for language version configuration for K2 only. K1 always uses default version -private val LANGUAGE_VERSION_K2: String = System.getProperty("fir.bench.language.version.k2", "2.0") +// This is used for language version configuration for K2 only. K1 uses LANGUAGE_VERSION_K1 +internal val LANGUAGE_VERSION_K2: String = System.getProperty("fir.bench.language.version.k2", "2.0") class FullPipelineModularizedTest : AbstractFullPipelineModularizedTest() { @@ -20,16 +20,19 @@ class FullPipelineModularizedTest : AbstractFullPipelineModularizedTest() { args.useK2 = true args.useIR = true args.languageVersion = LANGUAGE_VERSION_K2 - args.apiVersion = API_VERSION - args.jvmDefault = "compatibility" - args.optIn = moduleData.optInAnnotations.toTypedArray() + arrayOf( - "kotlin.RequiresOptIn", - "kotlin.contracts.ExperimentalContracts", - "kotlin.io.path.ExperimentalPathApi", - "org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI" - ) - args.noStdlib = true - args.noReflect = true + // TODO: Remove when support for old modularized tests is removed + if (moduleData.arguments == null) { + args.apiVersion = API_VERSION + args.jvmDefault = "compatibility" + args.optIn = moduleData.optInAnnotations.toTypedArray() + arrayOf( + "kotlin.RequiresOptIn", + "kotlin.contracts.ExperimentalContracts", + "kotlin.io.path.ExperimentalPathApi", + "org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI" + ) + args.noStdlib = true + args.noReflect = true + } } fun testTotalKotlin() { diff --git a/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/NonFirResolveModularizedTotalKotlinTest.kt b/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/NonFirResolveModularizedTotalKotlinTest.kt index 966cca37f2b..647ac658089 100644 --- a/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/NonFirResolveModularizedTotalKotlinTest.kt +++ b/compiler/fir/modularized-tests/tests/org/jetbrains/kotlin/fir/NonFirResolveModularizedTotalKotlinTest.kt @@ -5,6 +5,7 @@ package org.jetbrains.kotlin.fir +import com.intellij.openapi.Disposable import com.intellij.openapi.util.Disposer import com.intellij.openapi.util.text.StringUtil import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY @@ -16,6 +17,9 @@ import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment import org.jetbrains.kotlin.cli.jvm.compiler.KotlinToJVMBytecodeCompiler import org.jetbrains.kotlin.config.* import org.jetbrains.kotlin.fir.scopes.ProcessorAction +import org.jetbrains.kotlin.load.kotlin.ModuleVisibilityManager +import org.jetbrains.kotlin.types.AbstractTypeChecker +import org.jetbrains.kotlin.types.FlexibleTypeImpl import java.io.FileOutputStream import java.io.PrintStream import kotlin.system.measureNanoTime @@ -67,28 +71,28 @@ class NonFirResolveModularizedTotalKotlinTest : AbstractFrontendModularizedTest( writeMessageToLog("$message: ${time * 1e-6} ms") } - override fun processModule(moduleData: ModuleData): ProcessorAction { - val disposable = Disposer.newDisposable() - + private fun configureAndSetupEnvironment(moduleData: ModuleData, disposable: Disposable): KotlinCoreEnvironment { val configuration = createDefaultConfiguration(moduleData) - - configuration.languageVersionSettings = - LanguageVersionSettingsImpl( - configuration.languageVersionSettings.languageVersion, - configuration.languageVersionSettings.apiVersion, - specificFeatures = mapOf( - LanguageFeature.NewInference to if (USE_NI) LanguageFeature.State.ENABLED else LanguageFeature.State.DISABLED - ), - analysisFlags = mapOf( - AnalysisFlags.skipPrereleaseCheck to true, - AnalysisFlags.optIn to moduleData.optInAnnotations - ) - ) - - System.getProperty("fir.bench.oldfe.jvm_target")?.let { - configuration.put(JVMConfigurationKeys.JVM_TARGET, JvmTarget.fromString(it) ?: error("Unknown JvmTarget")) + configureLanguageVersionSettings( + configuration, moduleData, + LanguageVersion.fromVersionString(LANGUAGE_VERSION_K1)!!, + configureFeatures = { + put(LanguageFeature.NewInference, if (USE_NI) LanguageFeature.State.ENABLED else LanguageFeature.State.DISABLED) + }, + configureFlags = { + // TODO: Remove when old tests are no longer supported + if (moduleData.arguments == null) { + put(AnalysisFlags.skipPrereleaseCheck, true) + } + } + ) + // TODO: Remove when old tests are no longer supported + if (moduleData.arguments == null) { + System.getProperty("fir.bench.oldfe.jvm_target")?.let { + configuration.put(JVMConfigurationKeys.JVM_TARGET, JvmTarget.fromString(it) ?: error("Unknown JvmTarget")) + } } configuration.put(MESSAGE_COLLECTOR_KEY, object : MessageCollector { override fun clear() { @@ -111,6 +115,18 @@ class NonFirResolveModularizedTotalKotlinTest : AbstractFrontendModularizedTest( }) val environment = KotlinCoreEnvironment.createForTests(disposable, configuration, EnvironmentConfigFiles.JVM_CONFIG_FILES) + val visibilityManager = ModuleVisibilityManager.SERVICE.getInstance(environment.project) + for (friendDir in configuration.getList(JVMConfigurationKeys.FRIEND_PATHS)) { + visibilityManager.addFriendPath(friendDir) + } + + return environment + } + + override fun processModule(moduleData: ModuleData): ProcessorAction { + val disposable = Disposer.newDisposable() + val environment = configureAndSetupEnvironment(moduleData, disposable) + runAnalysis(environment) Disposer.dispose(disposable)