FIR IDE: introduce diagnostics tests bases on FIR diagnostics tests

compiler/android-tests/tests/org/jetbrains/kotlin/android/tests/CodegenTestsOnAndroidGenerator.kt

@@ -354,8 +354,8 @@ class CodegenTestsOnAndroidGenerator private constructor(private val pathManager
                         extractor.provideConfigurationKeys()
                         extractor.configure(keyConfiguration, module.directives)
                     }
-                    val kind = configuratorForFlags.extractConfigurationKind(module.directives)
+                    val kind = JvmEnvironmentConfigurator.extractConfigurationKind(module.directives)
-                    val jdkKind = configuratorForFlags.extractJdkKind(module.directives)
+                    val jdkKind = JvmEnvironmentConfigurator.extractJdkKind(module.directives)
 
                     keyConfiguration.languageVersionSettings = module.languageVersionSettings
 

compiler/fir/entrypoint/src/org/jetbrains/kotlin/fir/analysis/FirAnalyzerFacade.kt

@@ -29,16 +29,25 @@ import org.jetbrains.kotlin.psi.KtFile
 import org.jetbrains.kotlin.psi2ir.generators.GeneratorExtensions
 import java.io.File
 
+abstract class AbstractFirAnalyzerFacade {
+    abstract val scopeSession: ScopeSession
+    abstract fun runCheckers(): Map<FirFile, List<FirDiagnostic<*>>>
+
+    abstract fun runResolution(): List<FirFile>
+
+    abstract fun convertToIr(extensions: GeneratorExtensions): Fir2IrResult
+}
+
 class FirAnalyzerFacade(
     val session: FirSession,
     val languageVersionSettings: LanguageVersionSettings,
     val ktFiles: Collection<KtFile> = emptyList(), // may be empty if light tree mode enabled
     val originalFiles: Collection<File> = emptyList(), // may be empty if light tree mode disabled
     val useLightTree: Boolean = false
-) {
+) : AbstractFirAnalyzerFacade() {
     private var firFiles: List<FirFile>? = null
     private var _scopeSession: ScopeSession? = null
-    val scopeSession: ScopeSession
+    override val scopeSession: ScopeSession
         get() = _scopeSession!!
 
     private var collectedDiagnostics: Map<FirFile, List<FirDiagnostic<*>>>? = null
@@ -63,7 +72,7 @@ class FirAnalyzerFacade(
         }
     }
 
-    fun runResolution(): List<FirFile> {
+    override fun runResolution(): List<FirFile> {
         if (firFiles == null) buildRawFir()
         if (_scopeSession != null) return firFiles!!
         val resolveProcessor = FirTotalResolveProcessor(session)
@@ -73,7 +82,7 @@ class FirAnalyzerFacade(
     }
 
     @OptIn(ExperimentalStdlibApi::class)
-    fun runCheckers(): Map<FirFile, List<FirDiagnostic<*>>> {
+    override fun runCheckers(): Map<FirFile, List<FirDiagnostic<*>>> {
         if (_scopeSession == null) runResolution()
         if (collectedDiagnostics != null) return collectedDiagnostics!!
         val collector = FirDiagnosticsCollector.create(session, scopeSession)
@@ -87,7 +96,7 @@ class FirAnalyzerFacade(
         return collectedDiagnostics!!
     }
 
-    fun convertToIr(extensions: GeneratorExtensions): Fir2IrResult {
+    override fun convertToIr(extensions: GeneratorExtensions): Fir2IrResult {
         if (_scopeSession == null) runResolution()
         val signaturer = JvmIdSignatureDescriptor(JvmManglerDesc())
 

compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/FirFrontendFacade.kt

@@ -78,7 +78,7 @@ class FirFrontendFacade(
             testFile to firFile
         }.toMap()
 
-        return FirOutputArtifact(session, filesMap, firAnalyzerFacade)
+        return FirOutputArtifactImpl(session, filesMap, firAnalyzerFacade)
     }
 
     private fun createSessionForBuiltins(

compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/FirOutputArtifact.kt

@@ -6,19 +6,27 @@
 package org.jetbrains.kotlin.test.frontend.fir
 
 import org.jetbrains.kotlin.fir.FirSession
+import org.jetbrains.kotlin.fir.analysis.AbstractFirAnalyzerFacade
 import org.jetbrains.kotlin.fir.analysis.FirAnalyzerFacade
 import org.jetbrains.kotlin.fir.declarations.FirFile
 import org.jetbrains.kotlin.test.model.FrontendKinds
 import org.jetbrains.kotlin.test.model.ResultingArtifact
 import org.jetbrains.kotlin.test.model.TestFile
 
-data class FirOutputArtifact(
+abstract class FirOutputArtifact : ResultingArtifact.FrontendOutput<FirOutputArtifact>() {
-    val session: FirSession,
+    abstract val session: FirSession
-    val allFirFiles: Map<TestFile, FirFile>,
+    abstract val firAnalyzerFacade: AbstractFirAnalyzerFacade
-    val firAnalyzerFacade: FirAnalyzerFacade
+    abstract val allFirFiles: Map<TestFile, FirFile>
-) : ResultingArtifact.FrontendOutput<FirOutputArtifact>() {
+
     override val kind: FrontendKinds.FIR
         get() = FrontendKinds.FIR
 
-    val firFiles: Map<TestFile, FirFile> = allFirFiles.filterKeys { !it.isAdditional }
+
+    val firFiles: Map<TestFile, FirFile> by lazy { allFirFiles.filterKeys { !it.isAdditional } }
 }
+
+data class FirOutputArtifactImpl(
+    override val session: FirSession,
+    override val allFirFiles: Map<TestFile, FirFile>,
+    override val firAnalyzerFacade: FirAnalyzerFacade
+) : FirOutputArtifact()

compiler/tests-common-new/tests/org/jetbrains/kotlin/test/runners/AbstractFirDiagnosticTest.kt

@@ -28,75 +28,7 @@ import org.jetbrains.kotlin.test.services.fir.FirOldFrontendMetaConfigurator
 
 abstract class AbstractFirDiagnosticTest : AbstractKotlinCompilerTest() {
     override fun TestConfigurationBuilder.configuration() {
-        globalDefaults {
+        baseFirDiagnosticTestConfiguration()
-            frontend = FrontendKinds.FIR
-            targetPlatform = JvmPlatforms.defaultJvmPlatform
-            dependencyKind = DependencyKind.Source
-        }
-
-        enableMetaInfoHandler()
-
-        useConfigurators(
-            ::CommonEnvironmentConfigurator,
-            ::JvmEnvironmentConfigurator,
-        )
-
-        useAdditionalSourceProviders(
-            ::AdditionalDiagnosticsSourceFilesProvider,
-            ::CoroutineHelpersSourceFilesProvider,
-        )
-        useFrontendFacades(::FirFrontendFacade)
-        useFrontendHandlers(
-            ::FirDiagnosticsHandler,
-            ::FirDumpHandler,
-            ::FirCfgDumpHandler,
-            ::FirCfgConsistencyHandler,
-            ::FirNoImplicitTypesHandler,
-        )
-
-        useMetaInfoProcessors(::PsiLightTreeMetaInfoProcessor)
-
-        defaultDirectives {
-            +COMPARE_WITH_LIGHT_TREE
-        }
-
-        forTestsMatching("compiler/testData/diagnostics/*") {
-            useAfterAnalysisCheckers(
-                ::FirIdenticalChecker,
-                ::FirFailingTestSuppressor,
-            )
-            useMetaTestConfigurators(::FirOldFrontendMetaConfigurator)
-        }
-
-        forTestsMatching("compiler/fir/analysis-tests/testData/*") {
-            defaultDirectives {
-                +FIR_DUMP
-            }
-        }
-
-        forTestsMatching(
-            "compiler/testData/diagnostics/testsWithStdLib/*" or
-                    "compiler/fir/analysis-tests/testData/resolveWithStdlib/*" or
-                    "compiler/testData/diagnostics/tests/unsignedTypes/*"
-        ) {
-            defaultDirectives {
-                +WITH_STDLIB
-            }
-        }
-
-        forTestsMatching("compiler/fir/analysis-tests/testData/resolve/extendedCheckers/*") {
-            defaultDirectives {
-                +WITH_EXTENDED_CHECKERS
-            }
-        }
-
-        forTestsMatching("compiler/testData/diagnostics/tests/testsWithJava15/*") {
-            defaultDirectives {
-                JDK_KIND with TestJdkKind.FULL_JDK_15
-                +WITH_STDLIB
-                +WITH_REFLECT
-            }
-        }
     }
 }
 
@@ -110,3 +42,76 @@ abstract class AbstractFirDiagnosticsWithLightTreeTest : AbstractFirDiagnosticTe
         }
     }
 }
+
+
+fun TestConfigurationBuilder.baseFirDiagnosticTestConfiguration() {
+    globalDefaults {
+        frontend = FrontendKinds.FIR
+        targetPlatform = JvmPlatforms.defaultJvmPlatform
+        dependencyKind = DependencyKind.Source
+    }
+
+    enableMetaInfoHandler()
+
+    useConfigurators(
+        ::CommonEnvironmentConfigurator,
+        ::JvmEnvironmentConfigurator,
+    )
+
+    useAdditionalSourceProviders(
+        ::AdditionalDiagnosticsSourceFilesProvider,
+        ::CoroutineHelpersSourceFilesProvider,
+    )
+    useFrontendFacades(::FirFrontendFacade)
+    useFrontendHandlers(
+        ::FirDiagnosticsHandler,
+        ::FirDumpHandler,
+        ::FirCfgDumpHandler,
+        ::FirCfgConsistencyHandler,
+        ::FirNoImplicitTypesHandler,
+    )
+
+    useMetaInfoProcessors(::PsiLightTreeMetaInfoProcessor)
+
+    defaultDirectives {
+        +COMPARE_WITH_LIGHT_TREE
+    }
+
+    forTestsMatching("compiler/testData/diagnostics/*") {
+        useAfterAnalysisCheckers(
+            ::FirIdenticalChecker,
+            ::FirFailingTestSuppressor,
+        )
+        useMetaTestConfigurators(::FirOldFrontendMetaConfigurator)
+    }
+
+    forTestsMatching("compiler/fir/analysis-tests/testData/*") {
+        defaultDirectives {
+            +FIR_DUMP
+        }
+    }
+
+    forTestsMatching(
+        "compiler/testData/diagnostics/testsWithStdLib/*" or
+                "compiler/fir/analysis-tests/testData/resolveWithStdlib/*" or
+                "compiler/testData/diagnostics/tests/unsignedTypes/*"
+    ) {
+        defaultDirectives {
+            +WITH_STDLIB
+        }
+    }
+
+    forTestsMatching("compiler/fir/analysis-tests/testData/resolve/extendedCheckers/*") {
+        defaultDirectives {
+            +WITH_EXTENDED_CHECKERS
+        }
+    }
+
+    forTestsMatching("compiler/testData/diagnostics/tests/testsWithJava15/*") {
+        defaultDirectives {
+            JDK_KIND with TestJdkKind.FULL_JDK_15
+            +WITH_STDLIB
+            +WITH_REFLECT
+        }
+    }
+}

compiler/tests-common-new/tests/org/jetbrains/kotlin/test/runners/AbstractKotlinCompilerTest.kt

@@ -34,23 +34,27 @@ abstract class AbstractKotlinCompilerTest {
         val defaultPreprocessors: List<Constructor<SourceFilePreprocessor>> = listOf(
             ::MetaInfosCleanupPreprocessor
         )
+
+        private fun configureDebugFlags() {
+            AbstractTypeChecker.RUN_SLOW_ASSERTIONS = true
+            FlexibleTypeImpl.RUN_SLOW_ASSERTIONS = true
+        }
+
+        val defaultConfiguration: TestConfigurationBuilder.() -> Unit = {
+            useAdditionalService<TemporaryDirectoryManager>(::TemporaryDirectoryManagerImpl)
+            useAdditionalService<BackendKindExtractor>(::BackendKindExtractorImpl)
+            useSourcePreprocessor(*defaultPreprocessors.toTypedArray())
+            useDirectives(*defaultDirectiveContainers.toTypedArray())
+            configureDebugFlags()
+        }
     }
 
     private val configuration: TestConfigurationBuilder.() -> Unit = {
         assertions = JUnit5Assertions
-        useAdditionalService<TemporaryDirectoryManager>(::TemporaryDirectoryManagerImpl)
+        defaultConfiguration()
-        useAdditionalService<BackendKindExtractor>(::BackendKindExtractorImpl)
-        useSourcePreprocessor(*defaultPreprocessors.toTypedArray())
-        useDirectives(*defaultDirectiveContainers.toTypedArray())
-        configureDebugFlags()
         configure(this)
     }
 
-    private fun configureDebugFlags() {
-        AbstractTypeChecker.RUN_SLOW_ASSERTIONS = true
-        FlexibleTypeImpl.RUN_SLOW_ASSERTIONS = true
-    }
-
     abstract fun TestConfigurationBuilder.configuration()
     private lateinit var testInfo: KotlinTestInfo
 

compiler/tests-common-new/tests/org/jetbrains/kotlin/test/services/configuration/JvmEnvironmentConfigurator.kt

@@ -65,6 +65,61 @@ class JvmEnvironmentConfigurator(testServices: TestServices) : EnvironmentConfig
         private val DEFAULT_JVM_TARGET_FROM_PROPERTY: String? = System.getProperty("kotlin.test.default.jvm.target")
 
         private const val JAVA_BINARIES_JAR_NAME = "java-binaries"
+
+        fun extractConfigurationKind(registeredDirectives: RegisteredDirectives): ConfigurationKind {
+            val withRuntime = JvmEnvironmentConfigurationDirectives.WITH_RUNTIME in registeredDirectives ||
+                    JvmEnvironmentConfigurationDirectives.WITH_STDLIB in registeredDirectives
+            val withReflect = JvmEnvironmentConfigurationDirectives.WITH_REFLECT in registeredDirectives
+            val noRuntime = JvmEnvironmentConfigurationDirectives.NO_RUNTIME in registeredDirectives
+            if (noRuntime && withRuntime) {
+                error("NO_RUNTIME and WITH_RUNTIME can not be used together")
+            }
+            return when {
+                withRuntime && !withReflect -> ConfigurationKind.NO_KOTLIN_REFLECT
+                withRuntime || withReflect -> ConfigurationKind.ALL
+                noRuntime -> ConfigurationKind.JDK_NO_RUNTIME
+                else -> ConfigurationKind.JDK_ONLY
+            }
+        }
+
+        fun extractJdkKind(registeredDirectives: RegisteredDirectives): TestJdkKind {
+            val fullJdkEnabled = JvmEnvironmentConfigurationDirectives.FULL_JDK in registeredDirectives
+            val jdkKinds = registeredDirectives[JvmEnvironmentConfigurationDirectives.JDK_KIND]
+
+            if (fullJdkEnabled) {
+                if (jdkKinds.isNotEmpty()) {
+                    error("FULL_JDK and JDK_KIND can not be used together")
+                }
+                return TestJdkKind.FULL_JDK
+            }
+
+            return when (jdkKinds.size) {
+                0 -> TestJdkKind.MOCK_JDK
+                1 -> jdkKinds.single()
+                else -> error("Too many jdk kinds passed: ${jdkKinds.joinToArrayString()}")
+            }
+        }
+
+        fun getLibraryFiles(configurationKind: ConfigurationKind, directives: RegisteredDirectives): List<File> {
+            val files = mutableListOf<File>()
+            if (configurationKind.withRuntime) {
+                files.add(ForTestCompileRuntime.runtimeJarForTests())
+                files.add(ForTestCompileRuntime.scriptRuntimeJarForTests())
+                files.add(ForTestCompileRuntime.kotlinTestJarForTests())
+            } else if (configurationKind.withMockRuntime) {
+                files.add(ForTestCompileRuntime.minimalRuntimeJarForTests())
+                files.add(ForTestCompileRuntime.scriptRuntimeJarForTests())
+            }
+            if (configurationKind.withReflection) {
+                files.add(ForTestCompileRuntime.reflectJarForTests())
+            }
+            files.add(KtTestUtil.getAnnotationsJar())
+
+            if (JvmEnvironmentConfigurationDirectives.STDLIB_JDK8 in directives) {
+                files.add(ForTestCompileRuntime.runtimeJarForTestsWithJdk8())
+            }
+            return files
+        }
     }
 
     override val directivesContainers: List<DirectivesContainer>
@@ -130,27 +185,8 @@ class JvmEnvironmentConfigurator(testServices: TestServices) : EnvironmentConfig
             configuration.put(TEST_CONFIGURATION_KIND_KEY, it)
         }
 
-        val javaVersionToCompile = registeredDirectives[COMPILE_JAVA_USING].singleOrNull()
+        configuration.addJvmClasspathRoots(getLibraryFiles(configurationKind, module.directives))
-        val javaBinaryFiles = if (ALL_JAVA_AS_BINARY !in registeredDirectives) {
-            module.javaFiles.filter { INCLUDE_JAVA_AS_BINARY in it.directives }
-        } else module.javaFiles
 
-        val useJava9ToCompileIncludedJavaFiles = javaVersionToCompile == TestJavacVersion.JAVAC_9
-
-        if (configurationKind.withRuntime) {
-            configuration.configureStandardLibs(PathUtil.kotlinPathsForDistDirectory, K2JVMCompilerArguments().also { it.noReflect = true })
-            configuration.addJvmClasspathRoot(ForTestCompileRuntime.kotlinTestJarForTests())
-        } else if (configurationKind.withMockRuntime) {
-            configuration.addJvmClasspathRoot(ForTestCompileRuntime.minimalRuntimeJarForTests())
-            configuration.addJvmClasspathRoot(ForTestCompileRuntime.scriptRuntimeJarForTests())
-        }
-        if (configurationKind.withReflection) {
-            configuration.addJvmClasspathRoot(ForTestCompileRuntime.reflectJarForTests())
-        }
-
-        if (JvmEnvironmentConfigurationDirectives.STDLIB_JDK8 in module.directives) {
-            configuration.addJvmClasspathRoot(ForTestCompileRuntime.runtimeJarForTestsWithJdk8())
-        }
 
         configuration.addJvmClasspathRoot(KtTestUtil.getAnnotationsJar())
 
@@ -309,39 +345,6 @@ class JvmEnvironmentConfigurator(testServices: TestServices) : EnvironmentConfig
         }
     }
 
-    fun extractJdkKind(registeredDirectives: RegisteredDirectives): TestJdkKind {
-        val fullJdkEnabled = JvmEnvironmentConfigurationDirectives.FULL_JDK in registeredDirectives
-        val jdkKinds = registeredDirectives[JvmEnvironmentConfigurationDirectives.JDK_KIND]
-
-        if (fullJdkEnabled) {
-            if (jdkKinds.isNotEmpty()) {
-                error("FULL_JDK and JDK_KIND can not be used together")
-            }
-            return TestJdkKind.FULL_JDK
-        }
-
-        return when (jdkKinds.size) {
-            0 -> TestJdkKind.MOCK_JDK
-            1 -> jdkKinds.single()
-            else -> error("Too many jdk kinds passed: ${jdkKinds.joinToArrayString()}")
-        }
-    }
-
-    fun extractConfigurationKind(registeredDirectives: RegisteredDirectives): ConfigurationKind {
-        val withRuntime = JvmEnvironmentConfigurationDirectives.WITH_RUNTIME in registeredDirectives ||
-                JvmEnvironmentConfigurationDirectives.WITH_STDLIB in registeredDirectives
-        val withReflect = JvmEnvironmentConfigurationDirectives.WITH_REFLECT in registeredDirectives
-        val noRuntime = JvmEnvironmentConfigurationDirectives.NO_RUNTIME in registeredDirectives
-        if (noRuntime && withRuntime) {
-            error("NO_RUNTIME and WITH_RUNTIME can not be used together")
-        }
-        return when {
-            withRuntime && !withReflect -> ConfigurationKind.NO_KOTLIN_REFLECT
-            withRuntime || withReflect -> ConfigurationKind.ALL
-            noRuntime -> ConfigurationKind.JDK_NO_RUNTIME
-            else -> ConfigurationKind.JDK_ONLY
-        }
-    }
 
     private fun CompilerConfiguration.registerModuleDependencies(module: TestModule) {
         val dependencyProvider = testServices.dependencyProvider

generators/tests/org/jetbrains/kotlin/generators/tests/GenerateTests.kt

@@ -27,6 +27,7 @@ import org.jetbrains.kotlin.formatter.AbstractFormatterTest
 import org.jetbrains.kotlin.formatter.AbstractTypingIndentationTestBase
 import org.jetbrains.kotlin.generators.TestGroup
 import org.jetbrains.kotlin.generators.impl.generateTestGroupSuite
+import org.jetbrains.kotlin.generators.util.TestGeneratorUtil
 import org.jetbrains.kotlin.generators.util.TestGeneratorUtil.KT_OR_KTS
 import org.jetbrains.kotlin.generators.util.TestGeneratorUtil.KT_OR_KTS_WITHOUT_DOTS_IN_NAME
 import org.jetbrains.kotlin.generators.util.TestGeneratorUtil.KT_WITHOUT_DOTS_IN_NAME
@@ -86,6 +87,7 @@ import org.jetbrains.kotlin.idea.fir.low.level.api.AbstractFirMultiModuleLazyRes
 import org.jetbrains.kotlin.idea.fir.low.level.api.AbstractFirMultiModuleResolveTest
 import org.jetbrains.kotlin.idea.fir.low.level.api.diagnostic.AbstractDiagnosticTraversalCounterTest
 import org.jetbrains.kotlin.idea.fir.low.level.api.diagnostic.AbstractFirContextCollectionTest
+import org.jetbrains.kotlin.idea.fir.low.level.api.diagnostic.compiler.based.AbstractDiagnosisCompilerTestDataTest
 import org.jetbrains.kotlin.idea.fir.low.level.api.file.structure.AbstractFileStructureAndOutOfBlockModificationTrackerConsistencyTest
 import org.jetbrains.kotlin.idea.fir.low.level.api.file.structure.AbstractFileStructureTest
 import org.jetbrains.kotlin.idea.fir.low.level.api.resolve.AbstractInnerDeclarationsResolvePhaseTest
@@ -1100,6 +1102,34 @@ fun main(args: Array<String>) {
             }
         }
 
+        testGroup(
+            "idea/idea-frontend-fir/idea-fir-low-level-api/tests",
+            "compiler/fir/analysis-tests/testData",
+        ) {
+            testClass<AbstractDiagnosisCompilerTestDataTest>(suiteTestClassName = "DiagnosisCompilerFirTestdataTestGenerated") {
+                model("resolve", pattern = TestGeneratorUtil.KT_WITHOUT_DOTS_IN_NAME)
+                model("resolveWithStdlib", pattern = TestGeneratorUtil.KT_WITHOUT_DOTS_IN_NAME, )
+            }
+        }
+
+        testGroup(
+            "idea/idea-frontend-fir/idea-fir-low-level-api/tests",
+            "compiler/testData",
+        ) {
+            testClass<AbstractDiagnosisCompilerTestDataTest>(suiteTestClassName = "DiagnosisCompilerTestFE10TestdataTestGenerated") {
+                model(
+                    "diagnostics/tests",
+                    excludedPattern = excludedFirTestdataPattern,
+                )
+                model(
+                    "diagnostics/testsWithStdLib",
+                    excludedPattern = excludedFirTestdataPattern,
+                    excludeDirs = listOf("native")
+                )
+            }
+        }
+
+
         testGroup("idea/idea-fir/tests", "idea") {
             testClass<AbstractFirHighlightingTest> {
                 model("testData/highlighter")

idea/idea-frontend-fir/idea-fir-low-level-api/build.gradle.kts

@@ -27,7 +27,11 @@ dependencies {
 // <neededFor>`AbstractFirLazyResolveTest` uses fir implementation of references which are not in classpath otherwise
     testRuntimeOnly(project(":idea:idea-frontend-fir"))
 // </neededFor>
+    testCompile(projectTests(":compiler:test-infrastructure-utils"))
+    testCompile(projectTests(":compiler:test-infrastructure"))
+    testCompile(projectTests(":compiler:tests-common-new"))
 
+    testImplementation("org.opentest4j:opentest4j:1.2.0")
     testCompile(toolsJar())
     testCompile(projectTests(":idea"))
     testCompile(project(":idea:idea-fir"))

idea/idea-frontend-fir/idea-fir-low-level-api/tests/org/jetbrains/kotlin/idea/fir/low/level/api/compiler/based/AbstractCompilerBasedTest.kt

@@ -0,0 +1,190 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.idea.fir.low.level.api.compiler.based
+
+import com.intellij.codeInsight.ExternalAnnotationsManager
+import com.intellij.openapi.components.serviceIfCreated
+import com.intellij.openapi.project.impl.ProjectExImpl
+import com.intellij.openapi.projectRoots.Sdk
+import com.intellij.openapi.roots.LanguageLevelProjectExtension
+import com.intellij.serviceContainer.ComponentManagerImpl
+import com.intellij.testFramework.LightProjectDescriptor
+import org.jetbrains.kotlin.cli.jvm.compiler.MockExternalAnnotationsManager
+import org.jetbrains.kotlin.config.LanguageVersionSettings
+import org.jetbrains.kotlin.idea.caches.project.IdeaModuleInfo
+import org.jetbrains.kotlin.idea.caches.project.ModuleSourceInfo
+import org.jetbrains.kotlin.idea.caches.project.getModuleInfo
+import org.jetbrains.kotlin.idea.fir.low.level.api.api.DiagnosticCheckerFilter
+import org.jetbrains.kotlin.idea.fir.low.level.api.api.FirModuleResolveState
+import org.jetbrains.kotlin.idea.fir.low.level.api.api.getFirFile
+import org.jetbrains.kotlin.idea.fir.low.level.api.createResolveStateForNoCaching
+import org.jetbrains.kotlin.idea.project.withLanguageVersionSettings
+import org.jetbrains.kotlin.idea.test.*
+import org.jetbrains.kotlin.idea.test.PluginTestCaseBase.getLanguageLevel
+import org.jetbrains.kotlin.psi.KtFile
+import org.jetbrains.kotlin.test.TestConfiguration
+import org.jetbrains.kotlin.test.TestRunner
+import org.jetbrains.kotlin.test.builders.TestConfigurationBuilder
+import org.jetbrains.kotlin.test.builders.testConfiguration
+import org.jetbrains.kotlin.test.directives.FirDiagnosticsDirectives
+import org.jetbrains.kotlin.test.directives.model.DirectivesContainer
+import org.jetbrains.kotlin.test.directives.model.RegisteredDirectives
+import org.jetbrains.kotlin.test.frontend.fir.FirOutputArtifact
+import org.jetbrains.kotlin.test.model.*
+import org.jetbrains.kotlin.test.runners.*
+import org.jetbrains.kotlin.test.services.*
+import org.jetbrains.kotlin.test.services.configuration.JvmEnvironmentConfigurator
+import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstance
+
+abstract class AbstractCompilerBasedTest : KotlinLightCodeInsightFixtureTestCase() {
+    private var _configuration: TestConfiguration? = null
+    private var _modules: TestModuleStructure? = null
+
+    protected val modules: TestModuleStructure get() = _modules!!
+    protected val configuration: TestConfiguration get() = _configuration!!
+
+    private var oldAnnoatationManager: ExternalAnnotationsManager? = null
+
+    private fun <R> withLanguageLevel(directives: RegisteredDirectives, action: () -> R): R {
+        val oldJavaLanguageLevel = LanguageLevelProjectExtension.getInstance(project).languageLevel
+        val requiredLanguageLevel = getLanguageLevel(JvmEnvironmentConfigurator.extractJdkKind(directives))
+        LanguageLevelProjectExtension.getInstance(project).languageLevel = requiredLanguageLevel
+        return try {
+            action()
+        } finally {
+            LanguageLevelProjectExtension.getInstance(project).languageLevel = oldJavaLanguageLevel
+        }
+    }
+
+    inner class LowLevelFirFrontendFacade(
+        testServices: TestServices
+    ) : FrontendFacade<FirOutputArtifact>(testServices, FrontendKinds.FIR) {
+        override val additionalDirectives: List<DirectivesContainer>
+            get() = listOf(FirDiagnosticsDirectives)
+
+        override fun analyze(module: TestModule): LowLevelFirOutputArtifact {
+            val files = module.files.associateWith { file ->
+                val text = testServices.sourceFileProvider.getContentOfSourceFile(file)
+                myFixture.addFileToProject(file.relativePath, text)
+            }
+            val ktFile = files.values.firstIsInstance<KtFile>()
+            val moduleInfo = ktFile.getModuleInfo()
+            val resolveState = createResolveState(moduleInfo, module.languageVersionSettings)
+            val allFirFiles = files.mapNotNull { (testFile, psiFile) ->
+                if (psiFile !is KtFile) return@mapNotNull null
+                testFile to psiFile.getFirFile(resolveState)
+            }.toMap()
+
+            val diagnosticCheckerFilter = if (FirDiagnosticsDirectives.WITH_EXTENDED_CHECKERS in module.directives) {
+                DiagnosticCheckerFilter.EXTENDED_AND_COMMON_CHECKERS
+            } else DiagnosticCheckerFilter.ONLY_COMMON_CHECKERS
+
+            val analyzerFacade = LowLevelFirAnalyzerFacade(resolveState, allFirFiles, diagnosticCheckerFilter)
+            return LowLevelFirOutputArtifact(resolveState.rootModuleSession, analyzerFacade)
+        }
+    }
+
+    private fun createResolveState(moduleInfo: IdeaModuleInfo, languageVersionSettings: LanguageVersionSettings): FirModuleResolveState {
+        require(moduleInfo is ModuleSourceInfo)
+        var resolveState: FirModuleResolveState? = null
+        moduleInfo.module.withLanguageVersionSettings(languageVersionSettings) {
+            resolveState = createResolveStateForNoCaching(moduleInfo)
+        }
+        return resolveState!!
+    }
+
+    override fun isFirPlugin(): Boolean = true
+
+
+    override fun setUp() {
+        if (!isAllFilesPresentInTest()) {
+            setupConfiguration()
+        }
+        super.setUp()
+        registerMockExternalAnnotationManager()
+    }
+
+    private fun registerMockExternalAnnotationManager() {
+        oldAnnoatationManager = project.serviceIfCreated()
+        registerAnnotationManager(MockExternalAnnotationsManager())
+    }
+
+    private fun unregisterMockExternalAnnotationManager() {
+        oldAnnoatationManager?.let { old ->
+            registerAnnotationManager(old)
+        }
+    }
+
+    private fun registerAnnotationManager(manager: ExternalAnnotationsManager) {
+        @Suppress("UnstableApiUsage") val projectExImpl = project as ProjectExImpl
+        projectExImpl.registerServiceInstance(
+            ExternalAnnotationsManager::class.java,
+            manager,
+            ComponentManagerImpl.fakeCorePluginDescriptor
+        )
+    }
+
+    private fun setupConfiguration() {
+        _configuration = testConfiguration(testPath()) {
+            testInfo = KotlinTestInfo(
+                className = "_undefined_",
+                methodName = "_testUndefined_",
+                tags = emptySet()
+            )
+            configureTest()
+            AbstractKotlinCompilerTest.defaultConfiguration(this)
+            unregisterAllFacades()
+            useFrontendFacades(::LowLevelFirFrontendFacade)
+            assertions = JUnit4AssertionsService
+        }
+        _modules = configuration.moduleStructureExtractor.splitTestDataByModules(testPath(), configuration.directives)
+    }
+
+    override fun tearDown() {
+        _configuration = null
+        _modules = null
+        unregisterMockExternalAnnotationManager()
+        super.tearDown()
+    }
+
+    abstract fun TestConfigurationBuilder.configureTest()
+
+
+    fun doTest(path: String) {
+        if (ignoreTest()) return
+        withLanguageLevel(modules.allDirectives) {
+            TestRunner(configuration).runTest(path)
+        }
+    }
+
+    private fun ignoreTest(): Boolean {
+        if (modules.modules.size > 1) {
+            return true // multimodule tests are not supported
+        }
+
+        val singleModule = modules.modules.single()
+        if (singleModule.files.none { it.isKtFile }) {
+            return true // nothing to highlight
+        }
+
+        return false
+    }
+
+    override fun getProjectDescriptor(): LightProjectDescriptor {
+        if (isAllFilesPresentInTest()) return KotlinLightProjectDescriptor.INSTANCE
+
+        val allDirectives = modules.allDirectives
+
+        val configurationKind = JvmEnvironmentConfigurator.extractConfigurationKind(allDirectives)
+        val jdkKind = JvmEnvironmentConfigurator.extractJdkKind(allDirectives)
+
+        val libraryFiles = JvmEnvironmentConfigurator.getLibraryFiles(configurationKind, allDirectives)
+
+        return object : KotlinJdkAndLibraryProjectDescriptor(libraryFiles) {
+            override fun getSdk(): Sdk = PluginTestCaseBase.jdk(jdkKind)
+        }
+    }
+}

idea/idea-frontend-fir/idea-fir-low-level-api/tests/org/jetbrains/kotlin/idea/fir/low/level/api/compiler/based/JUnit4AssertionsService.kt

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.idea.fir.low.level.api.compiler.based
+
+import com.intellij.rt.execution.junit.FileComparisonFailure
+import org.jetbrains.kotlin.test.services.AssertionsService
+import org.jetbrains.kotlin.test.util.JUnit4Assertions
+import org.opentest4j.MultipleFailuresError
+import java.io.File
+import java.util.function.Consumer
+
+internal object JUnit4AssertionsService : AssertionsService() {
+    override fun assertEqualsToFile(expectedFile: File, actual: String, sanitizer: (String) -> String, message: () -> String) {
+        JUnit4Assertions.assertEqualsToFile(expectedFile, actual, sanitizer, message)
+    }
+
+    override fun assertEquals(expected: Any?, actual: Any?, message: (() -> String)?) {
+        JUnit4Assertions.assertEquals(expected, actual, message)
+    }
+
+    override fun assertNotEquals(expected: Any?, actual: Any?, message: (() -> String)?) {
+        JUnit4Assertions.assertNotEquals(expected, actual, message)
+    }
+
+    override fun assertTrue(value: Boolean, message: (() -> String)?) {
+        JUnit4Assertions.assertTrue(value, message)
+    }
+
+    override fun assertFalse(value: Boolean, message: (() -> String)?) {
+        JUnit4Assertions.assertFalse(value, message)
+    }
+
+    override fun assertNotNull(value: Any?, message: (() -> String)?) {
+        JUnit4Assertions.assertNotNull(value, message)
+    }
+
+    override fun <T> assertSameElements(expected: Collection<T>, actual: Collection<T>, message: (() -> String)?) {
+        JUnit4Assertions.assertSameElements(expected, actual, message)
+    }
+
+    override fun assertAll(exceptions: List<Throwable>) {
+        when (exceptions.size) {
+            0 -> Unit
+            1 -> throw exceptions.single()
+            else -> {
+                throwMultipleExceptions(exceptions)
+            }
+        }
+
+    }
+
+    private fun throwMultipleExceptions(exceptions: List<Throwable>) {
+        val multipleFailuresError = MultipleFailuresError("", exceptions)
+        exceptions.forEach(multipleFailuresError::addSuppressed)
+        throw multipleFailuresError
+    }
+
+    override fun fail(message: () -> String): Nothing {
+        JUnit4Assertions.fail(message)
+    }
+}

idea/idea-frontend-fir/idea-fir-low-level-api/tests/org/jetbrains/kotlin/idea/fir/low/level/api/compiler/based/LowLevelFirAnalyzerFacade.kt

@@ -0,0 +1,41 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.idea.fir.low.level.api.compiler.based
+
+import org.jetbrains.kotlin.fir.analysis.AbstractFirAnalyzerFacade
+import org.jetbrains.kotlin.fir.analysis.diagnostics.FirDiagnostic
+import org.jetbrains.kotlin.fir.backend.Fir2IrResult
+import org.jetbrains.kotlin.fir.declarations.FirFile
+import org.jetbrains.kotlin.fir.psi
+import org.jetbrains.kotlin.fir.resolve.ScopeSession
+import org.jetbrains.kotlin.idea.fir.low.level.api.api.DiagnosticCheckerFilter
+import org.jetbrains.kotlin.idea.fir.low.level.api.api.FirModuleResolveState
+import org.jetbrains.kotlin.idea.fir.low.level.api.api.collectDiagnosticsForFile
+import org.jetbrains.kotlin.psi.KtFile
+import org.jetbrains.kotlin.psi2ir.generators.GeneratorExtensions
+import org.jetbrains.kotlin.test.model.TestFile
+
+class LowLevelFirAnalyzerFacade(
+    val resolveState: FirModuleResolveState,
+    val allFirFiles: Map<TestFile, FirFile>,
+    private val diagnosticCheckerFilter: DiagnosticCheckerFilter,
+) : AbstractFirAnalyzerFacade() {
+    override val scopeSession: ScopeSession get() = shouldNotBeCalled()
+
+    override fun runCheckers(): Map<FirFile, List<FirDiagnostic<*>>> {
+        return allFirFiles.values.associateWith { firFile ->
+            val ktFile = firFile.psi as KtFile
+            val diagnostics = ktFile.collectDiagnosticsForFile(resolveState, diagnosticCheckerFilter)
+            @Suppress("UNCHECKED_CAST")
+            diagnostics.toList() as List<FirDiagnostic<*>>
+        }
+    }
+
+    override fun runResolution(): List<FirFile> = shouldNotBeCalled()
+    override fun convertToIr(extensions: GeneratorExtensions): Fir2IrResult = shouldNotBeCalled()
+}
+
+private fun shouldNotBeCalled(): Nothing = error("Should not be called for LL test")

idea/idea-frontend-fir/idea-fir-low-level-api/tests/org/jetbrains/kotlin/idea/fir/low/level/api/compiler/based/LowLevelFirOutputArtifact.kt

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.idea.fir.low.level.api.compiler.based
+
+import org.jetbrains.kotlin.fir.FirSession
+import org.jetbrains.kotlin.fir.declarations.FirFile
+import org.jetbrains.kotlin.test.frontend.fir.FirOutputArtifact
+import org.jetbrains.kotlin.test.model.TestFile
+
+class LowLevelFirOutputArtifact(
+    override val session: FirSession,
+    override val firAnalyzerFacade: LowLevelFirAnalyzerFacade,
+) : FirOutputArtifact() {
+    override val allFirFiles: Map<TestFile, FirFile>
+        get() = firAnalyzerFacade.allFirFiles
+}

idea/idea-frontend-fir/idea-fir-low-level-api/tests/org/jetbrains/kotlin/idea/fir/low/level/api/diagnostic/compiler/based/AbstractDiagnosisCompilerTestDataTest.kt

@@ -0,0 +1,16 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.idea.fir.low.level.api.diagnostic.compiler.based
+
+import org.jetbrains.kotlin.idea.fir.low.level.api.compiler.based.AbstractCompilerBasedTest
+import org.jetbrains.kotlin.test.builders.TestConfigurationBuilder
+import org.jetbrains.kotlin.test.runners.baseFirDiagnosticTestConfiguration
+
+abstract class AbstractDiagnosisCompilerTestDataTest : AbstractCompilerBasedTest() {
+    override fun TestConfigurationBuilder.configureTest() {
+        baseFirDiagnosticTestConfiguration()
+    }
+}

idea/idea-test-framework/test/org/jetbrains/kotlin/idea/test/PluginTestCaseBase.java

@@ -25,6 +25,7 @@ import com.intellij.openapi.projectRoots.impl.JavaSdkImpl;
 import com.intellij.openapi.util.Disposer;
 import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.openapi.vfs.newvfs.impl.VfsRootAccess;
+import com.intellij.pom.java.LanguageLevel;
 import com.intellij.testFramework.IdeaTestUtil;
 import kotlin.jvm.functions.Function0;
 import org.jetbrains.annotations.NotNull;
@@ -110,17 +111,47 @@ public class PluginTestCaseBase {
         switch (kind) {
             case MOCK_JDK:
                 return mockJdk();
+            case MODIFIED_MOCK_JDK:
+                return jdkByVersion("MODIFIED MOCK", KtTestUtil.findMockJdkRtModified());
+            case FULL_JDK_6:
+                return jdkByVersion("6", KtTestUtil.getJdk6Home());
             case FULL_JDK_9:
-                String jre9 = KtTestUtil.getJdk9Home().getPath();
+                return jdkByVersion("9", KtTestUtil.getJdk9Home());
-                VfsRootAccess.allowRootAccess(jre9);
-                return getSdk(jre9, "Full JDK 9");
             case FULL_JDK:
                 return fullJdk();
+            case FULL_JDK_15:
+                return jdkByVersion("15", KtTestUtil.getJdk15Home());
             default:
                 throw new UnsupportedOperationException(kind.toString());
         }
     }
 
+    @NotNull
+    public static LanguageLevel getLanguageLevel(@NotNull TestJdkKind kind) {
+        switch (kind) {
+            case MOCK_JDK:
+                return LanguageLevel.JDK_1_8;
+            case MODIFIED_MOCK_JDK:
+                return LanguageLevel.JDK_11;
+            case FULL_JDK_6:
+                return LanguageLevel.JDK_1_6;
+            case FULL_JDK_9:
+                return LanguageLevel.JDK_1_9;
+            case FULL_JDK:
+                return LanguageLevel.JDK_1_8;
+            case FULL_JDK_15:
+                return LanguageLevel.JDK_15_PREVIEW;
+            default:
+                throw new UnsupportedOperationException(kind.toString());
+        }
+    }
+
+    private static Sdk jdkByVersion(String version, File jdkDir) {
+        String path = jdkDir.getPath();
+        VfsRootAccess.allowRootAccess(path);
+        return getSdk(path, "Full JDK " + version);
+    }
+
     public static boolean isAllFilesPresentTest(@NotNull String testName) {
         return StringUtil.startsWithIgnoreCase(testName, "allFilesPresentIn");
     }