Modularized tests. Support parsing of original compiler arguments

Now testData contains info about original compiler arguments, which can be reused to support plugins and various other arguments Support original arguments in modularized tests - Properly support JDK9 modules - Opt-ins from original arguments - jdkHome from module data or arguments, as well as noJdk option - Fix friendPaths not configured - jvmDefault now handled properly ^KT-56075 ^KT-55879
2023-02-01 20:32:58 +01:00
parent 36c3d62ba1
commit 59e69481a3
7 changed files with 193 additions and 59 deletions
@@ -5,34 +5,96 @@

 package org.jetbrains.kotlin.fir

+import com.intellij.psi.PsiJavaModule
 import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
+import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments
 import org.jetbrains.kotlin.cli.common.config.KotlinSourceRoot
+import org.jetbrains.kotlin.cli.jvm.config.JvmClasspathRoot
+import org.jetbrains.kotlin.cli.jvm.config.JvmModulePathRoot
 import org.jetbrains.kotlin.cli.jvm.config.addJavaSourceRoot
-import org.jetbrains.kotlin.cli.jvm.config.addJvmClasspathRoots
 import org.jetbrains.kotlin.cli.jvm.config.configureJdkClasspathRoots
 import org.jetbrains.kotlin.config.*
 import org.jetbrains.kotlin.test.KotlinTestUtils
+import java.io.File

 abstract class AbstractFrontendModularizedTest : AbstractModularizedTest() {
-    fun createDefaultConfiguration(moduleData: ModuleData): CompilerConfiguration {
+
+    fun configureLanguageVersionSettings(
+        configuration: CompilerConfiguration,
+        moduleData: ModuleData,
+        languageVersion: LanguageVersion,
+        configureFlags: MutableMap<AnalysisFlag<*>, Any?>.() -> Unit = {},
+        configureFeatures: MutableMap<LanguageFeature, LanguageFeature.State>.() -> Unit = {}
+    ) {
+        val originalArguments = moduleData.arguments as? K2JVMCompilerArguments
+
+        configuration.languageVersionSettings = LanguageVersionSettingsImpl(
+            languageVersion,
+            originalArguments?.apiVersion?.let { ApiVersion.parse(it) } ?: ApiVersion.LATEST_STABLE,
+            analysisFlags = buildMap {
+                put(AnalysisFlags.optIn, moduleData.optInAnnotations + originalArguments?.optIn.orEmpty())
+                if (originalArguments != null) {
+                    put(AnalysisFlags.skipPrereleaseCheck, originalArguments.skipPrereleaseCheck)
+                    put(JvmAnalysisFlags.jvmDefaultMode, JvmDefaultMode.fromStringOrNull(originalArguments.jvmDefault))
+                }
+
+                configureFlags()
+            },
+            specificFeatures = buildMap {
+                configureFeatures()
+            }
+        )
+    }
+
+    fun createDefaultConfiguration(
+        moduleData: ModuleData
+    ): CompilerConfiguration {
        val configuration = KotlinTestUtils.newConfiguration()
+        val originalArguments = moduleData.arguments as? K2JVMCompilerArguments
+
        moduleData.javaSourceRoots.forEach {
            configuration.addJavaSourceRoot(it.path, it.packagePrefix)
        }
-        configuration.addJvmClasspathRoots(moduleData.classpath)
-        configuration.languageVersionSettings = LanguageVersionSettingsImpl(
-            LanguageVersion.LATEST_STABLE,
-            ApiVersion.LATEST_STABLE,
-            analysisFlags = mutableMapOf(AnalysisFlags.optIn to moduleData.optInAnnotations)
-        )
-        configuration.configureJdkClasspathRoots()

-        // in case of modular jdk only
-        configuration.putIfNotNull(JVMConfigurationKeys.JDK_HOME, moduleData.modularJdkRoot)
+        val isJava9Module = moduleData.javaSourceRoots.any { (file, packagePrefix) ->
+            packagePrefix == null &&
+                    (file.name == PsiJavaModule.MODULE_INFO_FILE ||
+                            (file.isDirectory && file.listFiles()!!.any { it.name == PsiJavaModule.MODULE_INFO_FILE }))
+        }
+
+        for (rootPath in moduleData.classpath) {
+            if (isJava9Module) {
+                configuration.add(CLIConfigurationKeys.CONTENT_ROOTS, JvmModulePathRoot(rootPath))
+            }
+            configuration.add(CLIConfigurationKeys.CONTENT_ROOTS, JvmClasspathRoot(rootPath))
+        }
+
+        val jdkHome =
+            moduleData.modularJdkRoot
+                ?: moduleData.jdkHome?.absoluteFile
+                ?: originalArguments?.jdkHome?.fixPath()?.absoluteFile
+
+        if (originalArguments != null) {
+            configuration.put(JVMConfigurationKeys.NO_JDK, originalArguments.noJdk)
+
+            for (modularRoot in originalArguments.javaModulePath?.split(File.pathSeparatorChar).orEmpty()) {
+                configuration.add(CLIConfigurationKeys.CONTENT_ROOTS, JvmModulePathRoot(modularRoot.fixPath()))
+            }
+
+            configuration.put(
+                JVMConfigurationKeys.JVM_TARGET,
+                originalArguments.jvmTargetIfSupported() ?: JvmTarget.DEFAULT
+            )
+        }
+        configuration.putIfNotNull(JVMConfigurationKeys.JDK_HOME, jdkHome)
+        configuration.configureJdkClasspathRoots()

        configuration.addAll(
            CLIConfigurationKeys.CONTENT_ROOTS,
            moduleData.sources.filter { it.extension == "kt" || it.isDirectory }.map { KotlinSourceRoot(it.absolutePath, false) })
+
+        configuration.addAll(JVMConfigurationKeys.FRIEND_PATHS, moduleData.friendDirs.map { it.absolutePath })
+
        return configuration
    }
 }
@@ -128,10 +128,27 @@ abstract class AbstractFullPipelineModularizedTest : AbstractModularizedTest() {
    }

    private fun configureBaseArguments(args: K2JVMCompilerArguments, moduleData: ModuleData, tmp: Path) {
+        val originalArguments = moduleData.arguments as? K2JVMCompilerArguments
+        if (originalArguments != null) {
+            args.apiVersion = originalArguments.apiVersion
+            args.noJdk = originalArguments.noJdk
+            args.noStdlib = originalArguments.noStdlib
+            args.noReflect = originalArguments.noReflect
+            args.jvmTarget = originalArguments.jvmTargetIfSupported()?.description
+            args.jsr305 = originalArguments.jsr305
+            args.nullabilityAnnotations = originalArguments.nullabilityAnnotations
+            args.jspecifyAnnotations = originalArguments.jspecifyAnnotations
+            args.jvmDefault = originalArguments.jvmDefault
+            args.jdkRelease = originalArguments.jdkRelease
+            args.progressiveMode = originalArguments.progressiveMode
+            args.optIn = (moduleData.optInAnnotations + (originalArguments.optIn ?: emptyArray())).toTypedArray()
+            args.allowKotlinPackage = originalArguments.allowKotlinPackage
+        } else {
+            args.jvmTarget = JVM_TARGET
+            args.allowKotlinPackage = true
+        }
        args.reportPerf = true
-        args.jvmTarget = JVM_TARGET
-        args.allowKotlinPackage = true
-        args.jdkHome = moduleData.jdkHome?.absolutePath
+        args.jdkHome = moduleData.jdkHome?.absolutePath ?: originalArguments?.jdkHome?.fixPath()?.absolutePath
        args.renderInternalDiagnosticNames = true
        configureArgsUsingBuildFile(args, moduleData, tmp)
    }
@@ -6,7 +6,11 @@
 package org.jetbrains.kotlin.fir

 import com.intellij.openapi.util.JDOMUtil
+import com.intellij.util.xmlb.XmlSerializer
 import org.jdom.Element
+import org.jetbrains.kotlin.cli.common.arguments.CommonCompilerArguments
+import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments
+import org.jetbrains.kotlin.config.JvmTarget
 import org.jetbrains.kotlin.fir.scopes.ProcessorAction
 import org.jetbrains.kotlin.test.testFramework.KtUsefulTestCase
 import org.jetbrains.kotlin.types.AbstractTypeChecker
@@ -37,11 +41,16 @@ data class ModuleData(
    val friendDirs = rawFriendDirs.map { it.fixPath() }
    val jdkHome = rawJdkHome?.fixPath()
    val modularJdkRoot = rawModularJdkRoot?.fixPath()
+
+    /**
+     * Raw compiler arguments, as it was passed to original module build
+     */
+    var arguments: CommonCompilerArguments? = null
 }

 data class JavaSourceRootData<Path : Any>(val path: Path, val packagePrefix: String?)

-private fun String.fixPath(): File = File(ROOT_PATH_PREFIX, this.removePrefix("/"))
+internal fun String.fixPath(): File = File(ROOT_PATH_PREFIX, this.removePrefix("/"))

 private val ROOT_PATH_PREFIX:String = System.getProperty("fir.bench.prefix", "/")
 private val OUTPUT_DIR_REGEX_FILTER:String = System.getProperty("fir.bench.filter", ".*")
@@ -136,7 +145,16 @@ abstract class AbstractModularizedTest : KtUsefulTestCase() {
    private fun loadModuleDumpFile(file: File): List<ModuleData> {
        val rootElement = JDOMUtil.load(file)
        val modules = rootElement.getChildren("module")
-        return modules.map { node -> loadModule(node) }
+        val arguments = rootElement.getChild("compilerArguments")?.let { loadCompilerArguments(it) }
+        return modules.map { node -> loadModule(node).also { it.arguments = arguments } }
+    }
+
+    private fun loadCompilerArguments(argumentsRoot: Element): CommonCompilerArguments? {
+        val element = argumentsRoot.children.singleOrNull() ?: return null
+        return when (element.name) {
+            "K2JVMCompilerArguments" -> K2JVMCompilerArguments().also { XmlSerializer.deserializeInto(it, element) }
+            else -> null
+        }
    }

    protected abstract fun beforePass(pass: Int)
@@ -172,3 +190,10 @@ abstract class AbstractModularizedTest : KtUsefulTestCase() {
        afterPass(pass)
    }
 }
+
+
+internal fun K2JVMCompilerArguments.jvmTargetIfSupported(): JvmTarget? {
+    val specified = jvmTarget?.let { JvmTarget.fromString(it) } ?: return null
+    if (specified != JvmTarget.JVM_1_6) return specified
+    return null
+}
@@ -7,21 +7,30 @@ package org.jetbrains.kotlin.fir

 import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments

+/**
+ * Language version to be used for K1 FP tests
+ */
+internal val LANGUAGE_VERSION_K1: String = System.getProperty("fir.bench.language.version.k1", "1.8")
+
 class FE1FullPipelineModularizedTest : AbstractFullPipelineModularizedTest() {
    override fun configureArguments(args: K2JVMCompilerArguments, moduleData: ModuleData) {
        args.useK2 = false
        args.useIR = true
-        args.jvmDefault = "compatibility"
-        args.apiVersion = API_VERSION
-        args.optIn = arrayOf(
-            "kotlin.RequiresOptIn",
-            "kotlin.contracts.ExperimentalContracts",
-            "kotlin.io.path.ExperimentalPathApi",
-            "org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI"
-        )
-        args.multiPlatform = true
-        args.noStdlib = true
-        args.noReflect = true
+        args.languageVersion = LANGUAGE_VERSION_K1
+        // TODO: Remove when support for old modularized tests is removed
+        if (moduleData.arguments == null) {
+            args.jvmDefault = "compatibility"
+            args.apiVersion = API_VERSION
+            args.optIn = arrayOf(
+                "kotlin.RequiresOptIn",
+                "kotlin.contracts.ExperimentalContracts",
+                "kotlin.io.path.ExperimentalPathApi",
+                "org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI"
+            )
+            args.multiPlatform = true
+            args.noStdlib = true
+            args.noReflect = true
+        }
    }

    fun testTotalKotlin() {
@@ -19,6 +19,7 @@ import org.jetbrains.kotlin.asJava.finder.JavaElementFinder
 import org.jetbrains.kotlin.cli.common.toBooleanLenient
 import org.jetbrains.kotlin.cli.jvm.compiler.*
 import org.jetbrains.kotlin.cli.jvm.compiler.pipeline.collectSources
+import org.jetbrains.kotlin.config.LanguageVersion
 import org.jetbrains.kotlin.config.languageVersionSettings
 import org.jetbrains.kotlin.diagnostics.DiagnosticReporterFactory
 import org.jetbrains.kotlin.fir.analysis.collectors.AbstractDiagnosticCollector
@@ -213,6 +214,7 @@ class FirResolveModularizedTotalKotlinTest : AbstractFrontendModularizedTest() {
    override fun processModule(moduleData: ModuleData): ProcessorAction {
        val disposable = Disposer.newDisposable()
        val configuration = createDefaultConfiguration(moduleData)
+        configureLanguageVersionSettings(configuration, moduleData, LanguageVersion.fromVersionString(LANGUAGE_VERSION_K2)!!)
        val environment = KotlinCoreEnvironment.createForTests(disposable, configuration, EnvironmentConfigFiles.JVM_CONFIG_FILES)

        PsiElementFinder.EP.getPoint(environment.project)
@@ -8,11 +8,11 @@ package org.jetbrains.kotlin.fir
 import org.jetbrains.kotlin.cli.common.arguments.K2JVMCompilerArguments

 // This is used for API version configuration for both frontends
-// TODO: rename system property
+// TODO: Deprecated and only used in old FP tests
 internal val API_VERSION: String = System.getProperty("fir.bench.language.version", "1.4")

-// This is used for language version configuration for K2 only. K1 always uses default version
-private val LANGUAGE_VERSION_K2: String = System.getProperty("fir.bench.language.version.k2", "2.0")
+// This is used for language version configuration for K2 only. K1 uses LANGUAGE_VERSION_K1
+internal val LANGUAGE_VERSION_K2: String = System.getProperty("fir.bench.language.version.k2", "2.0")

 class FullPipelineModularizedTest : AbstractFullPipelineModularizedTest() {

@@ -20,16 +20,19 @@ class FullPipelineModularizedTest : AbstractFullPipelineModularizedTest() {
        args.useK2 = true
        args.useIR = true
        args.languageVersion = LANGUAGE_VERSION_K2
-        args.apiVersion = API_VERSION
-        args.jvmDefault = "compatibility"
-        args.optIn = moduleData.optInAnnotations.toTypedArray() + arrayOf(
-            "kotlin.RequiresOptIn",
-            "kotlin.contracts.ExperimentalContracts",
-            "kotlin.io.path.ExperimentalPathApi",
-            "org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI"
-        )
-        args.noStdlib = true
-        args.noReflect = true
+        // TODO: Remove when support for old modularized tests is removed
+        if (moduleData.arguments == null) {
+            args.apiVersion = API_VERSION
+            args.jvmDefault = "compatibility"
+            args.optIn = moduleData.optInAnnotations.toTypedArray() + arrayOf(
+                "kotlin.RequiresOptIn",
+                "kotlin.contracts.ExperimentalContracts",
+                "kotlin.io.path.ExperimentalPathApi",
+                "org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI"
+            )
+            args.noStdlib = true
+            args.noReflect = true
+        }
    }

    fun testTotalKotlin() {
@@ -5,6 +5,7 @@

 package org.jetbrains.kotlin.fir

+import com.intellij.openapi.Disposable
 import com.intellij.openapi.util.Disposer
 import com.intellij.openapi.util.text.StringUtil
 import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY
@@ -16,6 +17,9 @@ import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
 import org.jetbrains.kotlin.cli.jvm.compiler.KotlinToJVMBytecodeCompiler
 import org.jetbrains.kotlin.config.*
 import org.jetbrains.kotlin.fir.scopes.ProcessorAction
+import org.jetbrains.kotlin.load.kotlin.ModuleVisibilityManager
+import org.jetbrains.kotlin.types.AbstractTypeChecker
+import org.jetbrains.kotlin.types.FlexibleTypeImpl
 import java.io.FileOutputStream
 import java.io.PrintStream
 import kotlin.system.measureNanoTime
@@ -67,28 +71,28 @@ class NonFirResolveModularizedTotalKotlinTest : AbstractFrontendModularizedTest(
        writeMessageToLog("$message: ${time * 1e-6} ms")
    }

-    override fun processModule(moduleData: ModuleData): ProcessorAction {
-        val disposable = Disposer.newDisposable()
-
+    private fun configureAndSetupEnvironment(moduleData: ModuleData, disposable: Disposable): KotlinCoreEnvironment {

        val configuration = createDefaultConfiguration(moduleData)

-
-        configuration.languageVersionSettings =
-            LanguageVersionSettingsImpl(
-                configuration.languageVersionSettings.languageVersion,
-                configuration.languageVersionSettings.apiVersion,
-                specificFeatures = mapOf(
-                    LanguageFeature.NewInference to if (USE_NI) LanguageFeature.State.ENABLED else LanguageFeature.State.DISABLED
-                ),
-                analysisFlags = mapOf(
-                    AnalysisFlags.skipPrereleaseCheck to true,
-                    AnalysisFlags.optIn to moduleData.optInAnnotations
-                )
-            )
-
-        System.getProperty("fir.bench.oldfe.jvm_target")?.let {
-            configuration.put(JVMConfigurationKeys.JVM_TARGET, JvmTarget.fromString(it) ?: error("Unknown JvmTarget"))
+        configureLanguageVersionSettings(
+            configuration, moduleData,
+            LanguageVersion.fromVersionString(LANGUAGE_VERSION_K1)!!,
+            configureFeatures = {
+                put(LanguageFeature.NewInference, if (USE_NI) LanguageFeature.State.ENABLED else LanguageFeature.State.DISABLED)
+            },
+            configureFlags = {
+                // TODO: Remove when old tests are no longer supported
+                if (moduleData.arguments == null) {
+                    put(AnalysisFlags.skipPrereleaseCheck, true)
+                }
+            }
+        )
+        // TODO: Remove when old tests are no longer supported
+        if (moduleData.arguments == null) {
+            System.getProperty("fir.bench.oldfe.jvm_target")?.let {
+                configuration.put(JVMConfigurationKeys.JVM_TARGET, JvmTarget.fromString(it) ?: error("Unknown JvmTarget"))
+            }
        }
        configuration.put(MESSAGE_COLLECTOR_KEY, object : MessageCollector {
            override fun clear() {
@@ -111,6 +115,18 @@ class NonFirResolveModularizedTotalKotlinTest : AbstractFrontendModularizedTest(
        })
        val environment = KotlinCoreEnvironment.createForTests(disposable, configuration, EnvironmentConfigFiles.JVM_CONFIG_FILES)

+        val visibilityManager = ModuleVisibilityManager.SERVICE.getInstance(environment.project)
+        for (friendDir in configuration.getList(JVMConfigurationKeys.FRIEND_PATHS)) {
+            visibilityManager.addFriendPath(friendDir)
+        }
+
+        return environment
+    }
+
+    override fun processModule(moduleData: ModuleData): ProcessorAction {
+        val disposable = Disposer.newDisposable()
+        val environment = configureAndSetupEnvironment(moduleData, disposable)
+
        runAnalysis(environment)

        Disposer.dispose(disposable)