[LL FIR] implement diagnostic tests on custom script definitions

it is impossible to declare test data with another output yet
as `myScriptFile.test.ll.kts` won't be treated as custom definition as
it requires `test.kts` extension.

^KT-66232 Fixed
^KT-66276

analysis/analysis-api-impl-base/tests/org/jetbrains/kotlin/analysis/api/impl/base/test/configurators/AnalysisApiScriptTestServiceRegistrar.kt

@@ -12,7 +12,7 @@ import org.jetbrains.kotlin.scripting.definitions.ScriptDefinitionProvider
 import org.jetbrains.kotlin.test.services.TestServices
 
 object AnalysisApiScriptTestServiceRegistrar : AnalysisApiTestServiceRegistrar() {
-    override fun registerProjectModelServices(project: MockProject, testServices: TestServices) {
+    override fun registerProjectServices(project: MockProject, testServices: TestServices) {
         project.registerService(ScriptDefinitionProvider::class.java, CliScriptDefinitionProvider())
     }
 }

analysis/low-level-api-fir/build.gradle.kts

@@ -5,6 +5,8 @@ plugins {
     id("jps-compatible")
 }
 
+val scriptingTestDefinition by configurations.creating
+
 dependencies {
     api(project(":compiler:psi"))
     implementation(project(":analysis:project-structure"))
@@ -57,6 +59,8 @@ dependencies {
     testImplementation(libs.junit.jupiter.api)
     testRuntimeOnly(libs.junit.jupiter.engine)
     testImplementation(project(":analysis:symbol-light-classes"))
+    testImplementation(projectTests(":plugins:scripting:scripting-tests"))
+    testImplementation(project(":kotlin-scripting-common"))
 
     testRuntimeOnly(project(":core:descriptors.runtime"))
 
@@ -64,6 +68,8 @@ dependencies {
     // We use 'api' instead of 'implementation' because other modules might be using these jars indirectly
     testApi(project(":plugins:fir-plugin-prototype"))
     testApi(projectTests(":plugins:fir-plugin-prototype"))
+
+    scriptingTestDefinition(projectTests(":plugins:scripting:test-script-definition"))
 }
 
 sourceSets {
@@ -78,9 +84,14 @@ kotlin {
 }
 
 projectTest(jUnitMode = JUnitMode.JUnit5) {
-    dependsOn(":dist")
+    dependsOn(":dist", ":plugins:scripting:test-script-definition:testJar")
     workingDir = rootDir
     useJUnitPlatform()
+
+    val scriptingTestDefinitionClasspath = scriptingTestDefinition.asPath
+    doFirst {
+        systemProperty("kotlin.script.test.script.definition.classpath", scriptingTestDefinitionClasspath)
+    }
 }
 
 allprojects {

analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/diagnostic/compiler/based/AbstractLLFirPreresolvedReversedScriptDiagnosticCompilerTestDataTest.kt

@@ -0,0 +1,18 @@
+/*
+ * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.analysis.low.level.api.fir.diagnostic.compiler.based
+
+import org.jetbrains.kotlin.scripting.test.configureWithCustomScriptDef
+import org.jetbrains.kotlin.test.builders.TestConfigurationBuilder
+
+abstract class AbstractLLFirPreresolvedReversedScriptDiagnosticCompilerTestDataTest :
+    AbstractLLFirPreresolvedReversedDiagnosticCompilerTestDataTest() {
+    override fun configureTest(builder: TestConfigurationBuilder) {
+        super.configureTest(builder)
+        builder.configureWithCustomScriptDef()
+        builder.useConfigurators(::CustomScriptDefinitionEnvironmentConfigurator)
+    }
+}

analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/diagnostic/compiler/based/AbstractLLFirScriptDiagnosticCompilerTestDataTest.kt

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.analysis.low.level.api.fir.diagnostic.compiler.based
+
+import com.intellij.mock.MockProject
+import com.intellij.openapi.project.Project
+import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.scripting.compiler.plugin.ScriptingCompilerConfigurationExtension
+import org.jetbrains.kotlin.scripting.test.configureWithCustomScriptDef
+import org.jetbrains.kotlin.test.builders.TestConfigurationBuilder
+import org.jetbrains.kotlin.test.model.TestModule
+import org.jetbrains.kotlin.test.services.EnvironmentConfigurator
+import org.jetbrains.kotlin.test.services.TestServices
+import kotlin.script.experimental.host.ScriptingHostConfiguration
+import kotlin.script.experimental.jvm.defaultJvmScriptingHostConfiguration
+
+abstract class AbstractLLFirScriptDiagnosticCompilerTestDataTest : AbstractDiagnosticCompilerTestDataTest() {
+    override fun configureTest(builder: TestConfigurationBuilder) {
+        super.configureTest(builder)
+        builder.configureWithCustomScriptDef()
+        builder.useConfigurators(::CustomScriptDefinitionEnvironmentConfigurator)
+    }
+}
+
+internal class CustomScriptDefinitionEnvironmentConfigurator(testServices: TestServices) : EnvironmentConfigurator(testServices) {
+    override fun legacyRegisterCompilerExtensions(project: Project, module: TestModule, configuration: CompilerConfiguration) {
+        val hostConfiguration = ScriptingHostConfiguration(defaultJvmScriptingHostConfiguration) {}
+
+        ScriptingCompilerConfigurationExtension(project as MockProject, hostConfiguration).updateConfiguration(configuration)
+    }
+}

analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/diagnostic/compiler/based/LLFirPreresolvedReversedScriptDiagnosticCompilerTestDataTestGenerated.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.analysis.low.level.api.fir.diagnostic.compiler.based;
+
+import com.intellij.testFramework.TestDataPath;
+import org.jetbrains.kotlin.test.util.KtTestUtil;
+import org.jetbrains.kotlin.test.TestMetadata;
+import org.junit.jupiter.api.Test;
+
+import java.io.File;
+import java.util.regex.Pattern;
+
+/** This class is generated by {@link org.jetbrains.kotlin.generators.tests.analysis.api.GenerateAnalysisApiTestsKt}. DO NOT MODIFY MANUALLY */
+@SuppressWarnings("all")
+@TestMetadata("plugins/scripting/scripting-tests/testData/diagnostics/testScripts")
+@TestDataPath("$PROJECT_ROOT")
+public class LLFirPreresolvedReversedScriptDiagnosticCompilerTestDataTestGenerated extends AbstractLLFirPreresolvedReversedScriptDiagnosticCompilerTestDataTest {
+  @Test
+  public void testAllFilesPresentInTestScripts() {
+    KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("plugins/scripting/scripting-tests/testData/diagnostics/testScripts"), Pattern.compile("^(.+)\\.(kts)$"), null, true);
+  }
+
+  @Test
+  @TestMetadata("conflictingImportsWithJava.test.kts")
+  public void testConflictingImportsWithJava_test() {
+    runTest("plugins/scripting/scripting-tests/testData/diagnostics/testScripts/conflictingImportsWithJava.test.kts");
+  }
+
+  @Test
+  @TestMetadata("providedPropertiesShadowing.test.kts")
+  public void testProvidedPropertiesShadowing_test() {
+    runTest("plugins/scripting/scripting-tests/testData/diagnostics/testScripts/providedPropertiesShadowing.test.kts");
+  }
+
+  @Test
+  @TestMetadata("providedProperties.test.kts")
+  public void testProvidedProperties_test() {
+    runTest("plugins/scripting/scripting-tests/testData/diagnostics/testScripts/providedProperties.test.kts");
+  }
+
+  @Test
+  @TestMetadata("simple.test.kts")
+  public void testSimple_test() {
+    runTest("plugins/scripting/scripting-tests/testData/diagnostics/testScripts/simple.test.kts");
+  }
+}

analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/diagnostic/compiler/based/LLFirScriptDiagnosticCompilerTestDataTestGenerated.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.analysis.low.level.api.fir.diagnostic.compiler.based;
+
+import com.intellij.testFramework.TestDataPath;
+import org.jetbrains.kotlin.test.util.KtTestUtil;
+import org.jetbrains.kotlin.test.TestMetadata;
+import org.junit.jupiter.api.Test;
+
+import java.io.File;
+import java.util.regex.Pattern;
+
+/** This class is generated by {@link org.jetbrains.kotlin.generators.tests.analysis.api.GenerateAnalysisApiTestsKt}. DO NOT MODIFY MANUALLY */
+@SuppressWarnings("all")
+@TestMetadata("plugins/scripting/scripting-tests/testData/diagnostics/testScripts")
+@TestDataPath("$PROJECT_ROOT")
+public class LLFirScriptDiagnosticCompilerTestDataTestGenerated extends AbstractLLFirScriptDiagnosticCompilerTestDataTest {
+  @Test
+  public void testAllFilesPresentInTestScripts() {
+    KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("plugins/scripting/scripting-tests/testData/diagnostics/testScripts"), Pattern.compile("^(.+)\\.(kts)$"), null, true);
+  }
+
+  @Test
+  @TestMetadata("conflictingImportsWithJava.test.kts")
+  public void testConflictingImportsWithJava_test() {
+    runTest("plugins/scripting/scripting-tests/testData/diagnostics/testScripts/conflictingImportsWithJava.test.kts");
+  }
+
+  @Test
+  @TestMetadata("providedPropertiesShadowing.test.kts")
+  public void testProvidedPropertiesShadowing_test() {
+    runTest("plugins/scripting/scripting-tests/testData/diagnostics/testScripts/providedPropertiesShadowing.test.kts");
+  }
+
+  @Test
+  @TestMetadata("providedProperties.test.kts")
+  public void testProvidedProperties_test() {
+    runTest("plugins/scripting/scripting-tests/testData/diagnostics/testScripts/providedProperties.test.kts");
+  }
+
+  @Test
+  @TestMetadata("simple.test.kts")
+  public void testSimple_test() {
+    runTest("plugins/scripting/scripting-tests/testData/diagnostics/testScripts/simple.test.kts");
+  }
+}

generators/analysis-api-generator/tests/org/jetbrains/kotlin/generators/tests/analysis/api/firLowLevel.kt

@@ -17,6 +17,8 @@ import org.jetbrains.kotlin.analysis.low.level.api.fir.diagnostic.compiler.based
 import org.jetbrains.kotlin.analysis.low.level.api.fir.diagnostic.compiler.based.AbstractLLFirPreresolvedReversedDiagnosticCompilerTestDataSpecTest
 import org.jetbrains.kotlin.analysis.low.level.api.fir.diagnostic.compiler.based.AbstractLLFirPreresolvedReversedDiagnosticCompilerTestDataTest
 import org.jetbrains.kotlin.analysis.low.level.api.fir.diagnostic.compiler.based.AbstractLLFirReversedBlackBoxCodegenBasedTest
+import org.jetbrains.kotlin.analysis.low.level.api.fir.diagnostic.compiler.based.AbstractLLFirScriptDiagnosticCompilerTestDataTest
+import org.jetbrains.kotlin.analysis.low.level.api.fir.diagnostic.compiler.based.AbstractLLFirPreresolvedReversedScriptDiagnosticCompilerTestDataTest
 import org.jetbrains.kotlin.analysis.low.level.api.fir.file.structure.*
 import org.jetbrains.kotlin.analysis.low.level.api.fir.resolve.AbstractErrorResistanceTest
 import org.jetbrains.kotlin.analysis.low.level.api.fir.resolve.AbstractOutOfContentRootLazyDeclarationResolveScopeBasedTest
@@ -337,6 +339,19 @@ internal fun TestGroupSuite.generateFirLowLevelApiTests() {
         }
     }
 
+    testGroup(
+        "analysis/low-level-api-fir/tests",
+        "plugins/scripting/scripting-tests/testData",
+    ) {
+        testClass<AbstractLLFirScriptDiagnosticCompilerTestDataTest> {
+            model("diagnostics/testScripts", pattern = TestGeneratorUtil.KTS)
+        }
+
+        testClass<AbstractLLFirPreresolvedReversedScriptDiagnosticCompilerTestDataTest>() {
+            model("diagnostics/testScripts", pattern = TestGeneratorUtil.KTS)
+        }
+    }
+
     testGroup(
         "analysis/low-level-api-fir/tests",
         "compiler/testData",

plugins/scripting/scripting-tests/test/org/jetbrains/kotlin/scripting/test/AbstractScriptTestBaseClasses.kt

@@ -30,7 +30,7 @@ open class AbstractScriptWithCustomDefBlackBoxCodegenTest : AbstractFirScriptCod
     }
 }
 
-private fun TestConfigurationBuilder.configureWithCustomScriptDef() {
+fun TestConfigurationBuilder.configureWithCustomScriptDef() {
     useConfigurators(
         ::ScriptWithCustomDefEnvironmentConfigurator
     )

plugins/scripting/scripting-tests/testData/diagnostics/testScripts/providedProperties.test.kts

@@ -1,3 +1,4 @@
 // SCRIPT_PROVIDED_PROPERTIES: prop1: kotlin.String, prop2: java.io.File
+// MUTE_LL_FIR: KT-66276
 
 val rv = args[1] + prop1 + prop2.path

plugins/scripting/scripting-tests/testData/diagnostics/testScripts/providedPropertiesShadowing.test.kts

@@ -1,4 +1,5 @@
 // SCRIPT_PROVIDED_PROPERTIES: prop1: kotlin.String
+// MUTE_LL_FIR: KT-66276
 
 val prop1 = 42 // TODO: Error should be reported on this shadowing, see KT-65809