Test, scripting: add infra for script tests with customizable def

based on the regular compiler tests infrastructure, but adding
directives that can customize the definition from testdata.
So far only default imports and provided properties are supported, but
the infrastructure is easily extendable to other customizations.
Another limitations that provided properties are not supported for
the black box tests - the constructor parameters computing code
should be adapted to support it.
Note: in order to pick up the customized definition, the script files
should have an extension .test.kts

build.gradle.kts

@@ -832,6 +832,7 @@ tasks {
         dependsOn(":kotlin-scripting-common:test")
         dependsOn(":kotlin-scripting-jvm:test")
         dependsOn(":kotlin-scripting-jvm-host-test:test")
+        dependsOn(":plugins:scripting:scripting-tests:test")
         dependsOn(":kotlin-scripting-dependencies:test")
         dependsOn(":kotlin-scripting-dependencies-maven:test")
         dependsOn(":kotlin-scripting-dependencies-maven-all:test")

compiler/tests-common-new/tests/org/jetbrains/kotlin/test/runners/codegen/AbstractFirScriptCodegenTest.kt

@@ -131,10 +131,7 @@ class FirJvmScriptRunChecker(testServices: TestServices) : JvmBinaryArtifactHand
             Regex("param: (\\S.*)").find(ktFile.text)?.let { it.groups[1]?.value?.split(" ") }
                 .orEmpty().toTypedArray()
         val scriptInstance = ctor.newInstance(args)
-        var anyExpectationFound = false
         for ((fieldName, expectedValue) in expected) {
-            anyExpectationFound = true
-
             if (expectedValue == "<nofield>") {
                 try {
                     scriptClass.getDeclaredField(fieldName)
@@ -150,7 +147,6 @@ class FirJvmScriptRunChecker(testServices: TestServices) : JvmBinaryArtifactHand
             val resultString = result?.toString() ?: "null"
             assertions.assertEquals(expectedValue, resultString) { "comparing field $fieldName" }
         }
-        assertions.assertTrue(anyExpectationFound) { "expecting at least one expectation" }
     }
 
     override fun processAfterAllModules(someAssertionWasFailed: Boolean) {

generators/build.gradle.kts

@@ -73,6 +73,7 @@ dependencies {
     testApi(projectTests(":plugins:fir-plugin-prototype"))
     testApi(projectTests(":plugins:fir-plugin-prototype:fir-plugin-ic-test"))
     testApi(projectTests(":generators:test-generator"))
+    testApi(projectTests(":plugins:scripting:scripting-tests"))
     testImplementation(commonDependency("org.jetbrains.kotlin:kotlin-reflect")) { isTransitive = false }
     testImplementation(projectTests(":compiler:test-infrastructure-utils"))
     testImplementation(projectTests(":compiler:test-infrastructure"))

generators/tests/org/jetbrains/kotlin/generators/tests/GenerateTests.kt

@@ -44,6 +44,8 @@ import org.jetbrains.kotlin.parcelize.test.runners.*
 import org.jetbrains.kotlin.powerassert.AbstractFirLightTreeBlackBoxCodegenTestForPowerAssert
 import org.jetbrains.kotlin.powerassert.AbstractIrBlackBoxCodegenTestForPowerAssert
 import org.jetbrains.kotlin.samWithReceiver.*
+import org.jetbrains.kotlin.scripting.test.AbstractScriptWithCustomDefBlackBoxCodegenTest
+import org.jetbrains.kotlin.scripting.test.AbstractScriptWithCustomDefDiagnosticsTestBase
 import org.jetbrains.kotlin.test.TargetBackend
 import org.jetbrains.kotlinx.atomicfu.AbstractAtomicfuJsIrTest
 import org.jetbrains.kotlinx.atomicfu.AbstractAtomicfuJvmIrTest
@@ -408,6 +410,18 @@ fun main(args: Array<String>) {
             }
         }
 
+        testGroup("plugins/scripting/scripting-tests/tests-gen", "plugins/scripting/scripting-tests") {
+            testClass<AbstractScriptWithCustomDefDiagnosticsTestBase> {
+                model("testData/diagnostics/testScripts", extension = "kts")
+            }
+        }
+
+        testGroup("plugins/scripting/scripting-tests/tests-gen", "plugins/scripting/scripting-tests") {
+            testClass<AbstractScriptWithCustomDefBlackBoxCodegenTest> {
+                model("testData/codegen/testScripts", extension = "kts")
+            }
+        }
+
         testGroup("plugins/assign-plugin/tests-gen", "plugins/assign-plugin/testData") {
             testClass<AbstractAssignmentPluginDiagnosticTest> {
                 model("diagnostics", excludedPattern = excludedFirTestdataPattern)

plugins/scripting/scripting-tests/build.gradle.kts

@@ -0,0 +1,43 @@
+
+plugins {
+    kotlin("jvm")
+}
+
+val scriptingTestDefinition by configurations.creating
+
+dependencies {
+    testApi(project(":kotlin-scripting-jvm"))
+    testApi(project(":kotlin-scripting-compiler-impl"))
+    testApi(projectTests(":compiler:test-infrastructure"))
+    testApi(projectTests(":compiler:test-infrastructure-utils"))
+    testApi(projectTests(":compiler:tests-compiler-utils"))
+    testApi(projectTests(":compiler:tests-common-new"))
+
+    testApi(platform(libs.junit.bom))
+    testImplementation(libs.junit.jupiter.api)
+    testRuntimeOnly(libs.junit.jupiter.engine)
+    testRuntimeOnly(commonDependency("org.codehaus.woodstox:stax2-api"))
+    testRuntimeOnly(commonDependency("com.fasterxml:aalto-xml"))
+
+    scriptingTestDefinition(projectTests(":plugins:scripting:test-script-definition"))
+}
+
+sourceSets {
+    "main" {}
+    "test" {
+        projectDefault()
+        generatedTestDir()
+    }
+}
+
+projectTest(parallel = true, jUnitMode = JUnitMode.JUnit5) {
+    dependsOn(":dist", ":plugins:scripting:test-script-definition:testJar")
+    workingDir = rootDir
+    useJUnitPlatform()
+    val scriptingTestDefinitionClasspath = scriptingTestDefinition.asPath
+    doFirst {
+        systemProperty("kotlin.script.test.script.definition.classpath", scriptingTestDefinitionClasspath)
+    }
+}
+
+testsJar()

plugins/scripting/scripting-tests/test/org/jetbrains/kotlin/scripting/test/AbstractScriptTestBaseClasses.kt

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.scripting.test
+
+import org.jetbrains.kotlin.test.FirParser
+import org.jetbrains.kotlin.test.builders.TestConfigurationBuilder
+import org.jetbrains.kotlin.test.directives.ConfigurationDirectives.WITH_STDLIB
+import org.jetbrains.kotlin.test.runners.AbstractFirDiagnosticTestBase
+import org.jetbrains.kotlin.test.runners.codegen.AbstractFirScriptCodegenTest
+
+open class AbstractScriptWithCustomDefDiagnosticsTestBase : AbstractFirDiagnosticTestBase(FirParser.Psi) {
+    override fun configure(builder: TestConfigurationBuilder) {
+        super.configure(builder)
+        with(builder) {
+            configureWithCustomScriptDef()
+        }
+    }
+}
+
+open class AbstractScriptWithCustomDefBlackBoxCodegenTest : AbstractFirScriptCodegenTest() {
+    override fun configure(builder: TestConfigurationBuilder) {
+        super.configure(builder)
+        with(builder) {
+            configureWithCustomScriptDef()
+            useCustomRuntimeClasspathProviders(::ScriptWithCustomDefRuntimeClassPathProvider)
+        }
+    }
+}
+
+private fun TestConfigurationBuilder.configureWithCustomScriptDef() {
+    useConfigurators(
+        ::ScriptWithCustomDefEnvironmentConfigurator
+    )
+    defaultDirectives {
+        +WITH_STDLIB
+    }
+}

plugins/scripting/scripting-tests/test/org/jetbrains/kotlin/scripting/test/ScriptWithCustomDefConfigurators.kt

@@ -0,0 +1,46 @@
+/*
+ * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.scripting.test
+
+import org.jetbrains.kotlin.cli.jvm.config.addJvmClasspathRoots
+import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.scripting.configuration.ScriptingConfigurationKeys
+import org.jetbrains.kotlin.test.directives.model.DirectivesContainer
+import org.jetbrains.kotlin.test.model.TestModule
+import org.jetbrains.kotlin.test.services.EnvironmentConfigurator
+import org.jetbrains.kotlin.test.services.RuntimeClasspathProvider
+import org.jetbrains.kotlin.test.services.TestServices
+import java.io.File
+
+private val testScriptDefinitionClasspath by lazy {
+    System.getProperty("kotlin.script.test.script.definition.classpath")!!.split(File.pathSeparator).map {
+        File(it).also {
+            require(it.exists()) {
+                "The file required for custom test script definition not found: $it"
+            }
+        }
+    }
+}
+
+class ScriptWithCustomDefEnvironmentConfigurator(testServices: TestServices) : EnvironmentConfigurator(testServices) {
+    override fun configureCompilerConfiguration(configuration: CompilerConfiguration, module: TestModule) {
+        configuration.addJvmClasspathRoots(testScriptDefinitionClasspath)
+        val dirSplitRegex = Regex(" *, *")
+        ScriptingTestDirectives.directivesToPassViaEnvironment.forEach { (directive, envName) ->
+            module.directives[directive].flatMap { it.split(dirSplitRegex).filter { it.isNotEmpty() } }.let {
+                if (it.isNotEmpty()) {
+                    configuration.put(ScriptingConfigurationKeys.LEGACY_SCRIPT_RESOLVER_ENVIRONMENT_OPTION, envName, it)
+                }
+            }
+        }
+    }
+
+    override val directiveContainers: List<DirectivesContainer> = listOf(ScriptingTestDirectives)
+}
+
+class ScriptWithCustomDefRuntimeClassPathProvider(testServices: TestServices) : RuntimeClasspathProvider(testServices) {
+    override fun runtimeClassPaths(module: TestModule): List<File> = testScriptDefinitionClasspath
+}

plugins/scripting/scripting-tests/test/org/jetbrains/kotlin/scripting/test/ScriptingTestDirectives.kt

@@ -0,0 +1,19 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.scripting.test
+
+import org.jetbrains.kotlin.test.directives.model.SimpleDirectivesContainer
+
+object ScriptingTestDirectives : SimpleDirectivesContainer() {
+    val SCRIPT_DEFAULT_IMPORTS by stringDirective("Default imports", multiLine = true)
+    val SCRIPT_PROVIDED_PROPERTIES by stringDirective("Provided properties", multiLine = true)
+
+    val directivesToPassViaEnvironment =
+        listOf(
+            SCRIPT_DEFAULT_IMPORTS to "defaultImports",
+            SCRIPT_PROVIDED_PROPERTIES to "providedProperties",
+        )
+}

plugins/scripting/scripting-tests/testData/codegen/testScripts/simple.test.kts

@@ -0,0 +1,4 @@
+
+val rv = "O" + "K"
+
+// expected: rv: OK

plugins/scripting/scripting-tests/testData/diagnostics/testScripts/providedProperties.test.kts

@@ -0,0 +1,3 @@
+// SCRIPT_PROVIDED_PROPERTIES: prop1: kotlin.String, prop2: java.io.File
+
+val rv = args[1] + prop1 + prop2.path

plugins/scripting/scripting-tests/testData/diagnostics/testScripts/simple.test.kts

@@ -0,0 +1,4 @@
+// SCRIPT_DEFAULT_IMPORTS: java.io.File, java.net.URI
+
+val f = File("")
+val u = URI("")

plugins/scripting/scripting-tests/tests-gen/org/jetbrains/kotlin/scripting/test/ScriptWithCustomDefBlackBoxCodegenTestGenerated.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.scripting.test;
+
+import com.intellij.testFramework.TestDataPath;
+import org.jetbrains.kotlin.test.util.KtTestUtil;
+import org.jetbrains.kotlin.test.TargetBackend;
+import org.jetbrains.kotlin.test.TestMetadata;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+
+import java.io.File;
+import java.util.regex.Pattern;
+
+/** This class is generated by {@link org.jetbrains.kotlin.generators.tests.GenerateTestsKt}. DO NOT MODIFY MANUALLY */
+@SuppressWarnings("all")
+@TestMetadata("plugins/scripting/scripting-tests/testData/codegen/testScripts")
+@TestDataPath("$PROJECT_ROOT")
+public class ScriptWithCustomDefBlackBoxCodegenTestGenerated extends AbstractScriptWithCustomDefBlackBoxCodegenTest {
+  @Test
+  public void testAllFilesPresentInTestScripts() {
+    KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("plugins/scripting/scripting-tests/testData/codegen/testScripts"), Pattern.compile("^(.+)\\.kts$"), null, TargetBackend.JVM_IR, true);
+  }
+
+  @Test
+  @TestMetadata("simple.test.kts")
+  public void testSimple_test() {
+    runTest("plugins/scripting/scripting-tests/testData/codegen/testScripts/simple.test.kts");
+  }
+}

plugins/scripting/scripting-tests/tests-gen/org/jetbrains/kotlin/scripting/test/ScriptWithCustomDefDiagnosticsTestBaseGenerated.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.scripting.test;
+
+import com.intellij.testFramework.TestDataPath;
+import org.jetbrains.kotlin.test.util.KtTestUtil;
+import org.jetbrains.kotlin.test.TestMetadata;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+
+import java.io.File;
+import java.util.regex.Pattern;
+
+/** This class is generated by {@link org.jetbrains.kotlin.generators.tests.GenerateTestsKt}. DO NOT MODIFY MANUALLY */
+@SuppressWarnings("all")
+@TestMetadata("plugins/scripting/scripting-tests/testData/diagnostics/testScripts")
+@TestDataPath("$PROJECT_ROOT")
+public class ScriptWithCustomDefDiagnosticsTestBaseGenerated extends AbstractScriptWithCustomDefDiagnosticsTestBase {
+  @Test
+  public void testAllFilesPresentInTestScripts() {
+    KtTestUtil.assertAllTestsPresentByMetadataWithExcluded(this.getClass(), new File("plugins/scripting/scripting-tests/testData/diagnostics/testScripts"), Pattern.compile("^(.+)\\.kts$"), null, true);
+  }
+
+  @Test
+  @TestMetadata("providedProperties.test.kts")
+  public void testProvidedProperties_test() {
+    runTest("plugins/scripting/scripting-tests/testData/diagnostics/testScripts/providedProperties.test.kts");
+  }
+
+  @Test
+  @TestMetadata("simple.test.kts")
+  public void testSimple_test() {
+    runTest("plugins/scripting/scripting-tests/testData/diagnostics/testScripts/simple.test.kts");
+  }
+}

plugins/scripting/test-script-definition/build.gradle.kts

@@ -0,0 +1,16 @@
+
+plugins {
+    kotlin("jvm")
+}
+
+dependencies {
+    testApi(project(":kotlin-scripting-jvm"))
+    testApi(project(":kotlin-scripting-compiler-impl"))
+}
+
+sourceSets {
+    "main" {}
+    "test" { projectDefault() }
+}
+
+testsJar()

plugins/scripting/test-script-definition/test/org/jetbrains/kotlin/scripting/test/definition/customTestScriptDefinition.kt

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.scripting.test.definition
+
+import kotlin.script.experimental.annotations.KotlinScript
+import org.jetbrains.kotlin.scripting.definitions.getEnvironment
+import kotlin.script.experimental.api.*
+import kotlin.script.experimental.host.ScriptingHostConfiguration
+
+@Suppress("unused", "UNUSED_PARAMETER")
+@KotlinScript(fileExtension = "test.kts", compilationConfiguration = ConfigurableTestScriptConfiguration::class)
+abstract class ConfigurableTestScript(vararg val args: String)
+
+class ConfigurableTestScriptConfiguration : ScriptCompilationConfiguration(
+    {
+        refineConfiguration {
+            beforeCompiling { ctx ->
+                val hostConfiguration = ctx.compilationConfiguration[ScriptCompilationConfiguration.hostConfiguration]!!
+                val env = hostConfiguration[ScriptingHostConfiguration.getEnvironment]?.invoke()
+                if (env == null) makeFailureResult("Unable to retrieve environment for the custom test script")
+                else
+                    ScriptCompilationConfiguration(ctx.compilationConfiguration) {
+                        env["defaultImports"]?.let {
+                            @Suppress("UNCHECKED_CAST")
+                            defaultImports.append(it as List<String>)
+                        }
+                        env["providedProperties"]?.let {
+                            @Suppress("UNCHECKED_CAST")
+                            (it as List<String>).forEach {
+                                it.split(Regex(" *: *")).let {
+                                    providedProperties.append(listOf(it.first() to KotlinType(it.last())))
+                                }
+                            }
+                        }
+                    }.asSuccess()
+            }
+        }
+    }
+)

settings.gradle

@@ -260,6 +260,8 @@ include ":kotlin-imports-dumper-compiler-plugin",
         ":kotlin-scripting-compiler-embeddable",
         ":kotlin-scripting-compiler-impl",
         ":kotlin-scripting-compiler-impl-embeddable",
+        ":plugins:scripting:test-script-definition",
+        ":plugins:scripting:scripting-tests",
         ":kotlin-scripting-dependencies",
         ":kotlin-scripting-dependencies-maven",
         ":kotlin-scripting-dependencies-maven-all",