Scripting: update scripts in source roots handling

#KT-52525 fixed
related to #KT-52735

compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/CommonCompilerArguments.kt

@@ -415,6 +415,9 @@ abstract class CommonCompilerArguments : CommonToolArguments() {
     @Argument(value = "-Xrender-internal-diagnostic-names", description = "Render internal names of warnings and errors")
     var renderInternalDiagnosticNames: Boolean by FreezableVar(false)
 
+    @Argument(value = "-Xallow-any-scripts-in-source-roots", description = "Allow to compile any scripts along with regular Kotlin sources")
+    var allowAnyScriptsInSourceRoots: Boolean by FreezableVar(false)
+
     @OptIn(IDEAPluginsCompatibilityAPI::class)
     open fun configureAnalysisFlags(collector: MessageCollector, languageVersion: LanguageVersion): MutableMap<AnalysisFlag<*>, Any> {
         return HashMap<AnalysisFlag<*>, Any>().apply {
@@ -508,6 +511,10 @@ abstract class CommonCompilerArguments : CommonToolArguments() {
                 }
             }
 
+            if (allowAnyScriptsInSourceRoots) {
+                put(LanguageFeature.SkipStandaloneScriptsInSourceRoots, LanguageFeature.State.DISABLED)
+            }
+
             // Internal arguments should go last, because it may be useful to override
             // some feature state via -XX (even if some -X flags were passed)
             if (internalArguments.isNotEmpty()) {

compiler/cli/src/org/jetbrains/kotlin/cli/common/arguments.kt

@@ -28,6 +28,7 @@ fun <A : CommonCompilerArguments> CompilerConfiguration.setupCommonArguments(
     putIfNotNull(CLIConfigurationKeys.INTELLIJ_PLUGIN_ROOT, arguments.intellijPluginRoot)
     put(CommonConfigurationKeys.REPORT_OUTPUT_FILES, arguments.reportOutputFiles)
     put(CommonConfigurationKeys.INCREMENTAL_COMPILATION, incrementalCompilationIsEnabled(arguments))
+    put(CommonConfigurationKeys.ALLOW_ANY_SCRIPTS_IN_SOURCE_ROOTS, arguments.allowAnyScriptsInSourceRoots)
 
     val metadataVersionString = arguments.metadataVersion
     if (metadataVersionString != null) {

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCoreEnvironment.kt

@@ -637,6 +637,7 @@ class KotlinCoreEnvironment private constructor(
             JsSyntheticTranslateExtension.registerExtensionPoint(project)
             CompilerConfigurationExtension.registerExtensionPoint(project)
             CollectAdditionalSourcesExtension.registerExtensionPoint(project)
+            ProcessSourcesBeforeCompilingExtension.registerExtensionPoint(project)
             ExtraImportsProviderExtension.registerExtensionPoint(project)
             IrGenerationExtension.registerExtensionPoint(project)
             ScriptEvaluationExtension.registerExtensionPoint(project)

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinToJVMBytecodeCompiler.kt

@@ -37,12 +37,10 @@ import org.jetbrains.kotlin.codegen.ClassBuilderFactories
 import org.jetbrains.kotlin.codegen.CodegenFactory
 import org.jetbrains.kotlin.codegen.DefaultCodegenFactory
 import org.jetbrains.kotlin.codegen.state.GenerationState
-import org.jetbrains.kotlin.config.CommonConfigurationKeys
+import org.jetbrains.kotlin.config.*
-import org.jetbrains.kotlin.config.CompilerConfiguration
-import org.jetbrains.kotlin.config.JVMConfigurationKeys
-import org.jetbrains.kotlin.config.languageVersionSettings
 import org.jetbrains.kotlin.diagnostics.DiagnosticReporterFactory
 import org.jetbrains.kotlin.diagnostics.impl.BaseDiagnosticsCollector
+import org.jetbrains.kotlin.extensions.ProcessSourcesBeforeCompilingExtension
 import org.jetbrains.kotlin.ir.backend.jvm.jvmResolveLibraries
 import org.jetbrains.kotlin.load.kotlin.ModuleVisibilityManager
 import org.jetbrains.kotlin.modules.Module
@@ -236,8 +234,11 @@ object KotlinToJVMBytecodeCompiler {
     }
 
     fun analyze(environment: KotlinCoreEnvironment): AnalysisResult? {
-        val sourceFiles = environment.getSourceFiles()
         val collector = environment.messageCollector
+        val sourceFiles = ProcessSourcesBeforeCompilingExtension.getInstances(environment.project)
+            .fold(environment.getSourceFiles() as Collection<KtFile>) { files, extension ->
+                extension.processSources(files, environment.configuration)
+            }
 
         // Can be null for Scripts/REPL
         val performanceManager = environment.configuration.get(CLIConfigurationKeys.PERF_MANAGER)

compiler/config/src/org/jetbrains/kotlin/config/CommonConfigurationKeys.kt

@@ -74,6 +74,10 @@ object CommonConfigurationKeys {
     @JvmField
     val INCREMENTAL_COMPILATION =
         CompilerConfigurationKey.create<Boolean>("Enable incremental compilation")
+
+    @JvmField
+    val ALLOW_ANY_SCRIPTS_IN_SOURCE_ROOTS =
+        CompilerConfigurationKey.create<Boolean>("Allow to compile any scripts along with regular Kotlin sources")
 }
 
 var CompilerConfiguration.languageVersionSettings: LanguageVersionSettings

compiler/frontend/src/org/jetbrains/kotlin/extensions/ProcessSourcesBeforeCompilingExtension.kt

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.extensions
+
+import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.psi.KtFile
+
+interface ProcessSourcesBeforeCompilingExtension {
+
+    companion object : ProjectExtensionDescriptor<ProcessSourcesBeforeCompilingExtension>(
+        "org.jetbrains.kotlin.processSourcesBeforeCompilingExtension",
+        ProcessSourcesBeforeCompilingExtension::class.java
+    )
+
+    fun processSources(
+        sources: Collection<KtFile>,
+        configuration: CompilerConfiguration
+    ): Collection<KtFile>
+}

compiler/testData/cli/js/jsExtraHelp.out

@@ -45,6 +45,8 @@ where advanced options include:
                              Turn on range checks for the array access functions
   -Xwasm-enable-asserts      Turn on asserts
   -Xwasm-kclass-fqn          Enable support for FQ names in KClass
+  -Xallow-any-scripts-in-source-roots
+                             Allow to compile any scripts along with regular Kotlin sources
   -Xallow-kotlin-package     Allow compiling code in package 'kotlin' and allow not requiring kotlin.stdlib in module-info
   -Xallow-result-return-type Allow compiling code when `kotlin.Result` is used as a return type
   -Xbuiltins-from-sources    Compile builtIns from sources

compiler/testData/cli/jvm/extraHelp.out

@@ -150,6 +150,8 @@ where advanced options include:
   -Xuse-type-table           Use type table in metadata serialization
   -Xvalidate-bytecode        Validate generated JVM bytecode before and after optimizations
   -Xvalidate-ir              Validate IR before and after lowering
+  -Xallow-any-scripts-in-source-roots
+                             Allow to compile any scripts along with regular Kotlin sources
   -Xallow-kotlin-package     Allow compiling code in package 'kotlin' and allow not requiring kotlin.stdlib in module-info
   -Xallow-result-return-type Allow compiling code when `kotlin.Result` is used as a return type
   -Xbuiltins-from-sources    Compile builtIns from sources

compiler/tests/org/jetbrains/kotlin/cli/LauncherScriptTest.kt

@@ -299,7 +299,7 @@ println(42)
             expectedStderr = "error: unrecognized script type: noInline.myscript; Specify path to the script file as the first argument\n"
         )
         runProcess(
-            "kotlin", "-howtorun", ".kts", "$testDataDirectory/noInline.myscript",
+            "kotlin", "-Xallow-any-scripts-in-source-roots", "-howtorun", ".kts", "$testDataDirectory/noInline.myscript",
             expectedExitCode = 1, expectedStderr = """error: unresolved reference: CompilerOptions (noInline.myscript:1:7)
 compiler/testData/launcher/noInline.myscript:1:7: error: unresolved reference: CompilerOptions
 @file:CompilerOptions("-Xno-inline")

compiler/tests/org/jetbrains/kotlin/integration/CompilerSmokeTest.java

@@ -172,7 +172,7 @@ public class CompilerSmokeTest extends CompilerSmokeTestBase {
     public void testCompileScript() throws Exception {
         String jar = tmpdir.getAbsolutePath() + File.separator + "script.jar";
 
-        runCompiler("script", "script.kts", "-d", jar);
+        runCompiler("script", "-Xallow-any-scripts-in-source-roots", "script.kts", "-d", jar);
     }
 
     public void testInlineOnly() throws Exception {

compiler/util/src/org/jetbrains/kotlin/config/LanguageVersionSettings.kt

@@ -278,6 +278,7 @@ enum class LanguageFeature(
     ForbidInferringTypeVariablesIntoEmptyIntersection(KOTLIN_1_9, kind = BUG_FIX), // KT-51221
     ForbidExtensionCallsOnInlineFunctionalParameters(KOTLIN_1_9, kind = BUG_FIX), // KT-52502
     ForbidInferringPostponedTypeVariableIntoDeclaredUpperBound(KOTLIN_1_9, kind = BUG_FIX), // KT-47986
+    SkipStandaloneScriptsInSourceRoots(KOTLIN_1_9, kind = OTHER), // KT-52525
 
     // Disabled for indefinite time. See KT-48535 and related discussion
     ApproximateIntegerLiteralTypesInReceiverPosition(sinceVersion = null),

libraries/scripting/common/src/kotlin/script/experimental/api/scriptCompilation.kt

@@ -170,6 +170,13 @@ val ScriptCompilationConfigurationKeys.sourceFragments by PropertiesCollection.k
  */
 val ScriptCompilationConfigurationKeys.hostConfiguration by PropertiesCollection.key<ScriptingHostConfiguration>(isTransient = true)
 
+/**
+ * Should the script be always considered standalone
+ * If true, it is ignored when compiled along with other sources (starting from 1.9, according to SkipStandaloneScriptsInSourceRoots language feature)
+ * true by default
+ */
+val ScriptCompilationConfigurationKeys.isStandalone by PropertiesCollection.key<Boolean>(true)
+
 /**
  * The sub-builder DSL for configuring refinement callbacks
  */

libraries/scripting/common/src/kotlin/script/experimental/api/scriptIdeConfiguration.kt

@@ -20,8 +20,8 @@ enum class ScriptAcceptedLocation {
     Sources,     // Under sources roots
     Tests,       // Under test sources roots
     Libraries,   // Under libraries classes or sources
-    Project,     // Under project folder, including sources and test sources roots
+    Project,     // Project infrastructure: project files excluding source roots
-    Everywhere;
+    Everywhere;  // All places in the project
 }
 
 val ScriptCompilationConfigurationKeys.ide
@@ -30,9 +30,4 @@ val ScriptCompilationConfigurationKeys.ide
 val IdeScriptCompilationConfigurationKeys.dependenciesSources by PropertiesCollection.key<List<ScriptDependency>>()
 
 val IdeScriptCompilationConfigurationKeys.acceptedLocations
-        by PropertiesCollection.key(
+        by PropertiesCollection.key(listOf(ScriptAcceptedLocation.Everywhere))
-            listOf(
-                ScriptAcceptedLocation.Sources,
-                ScriptAcceptedLocation.Tests
-            )
-        )

libraries/scripting/common/src/kotlin/script/experimental/host/configurationFromTemplate.kt

@@ -156,6 +156,10 @@ private fun ScriptCompilationConfiguration.Builder.propertiesFromTemplate(
 ) {
     baseClass.replaceOnlyDefault(if (templateClass == baseClassType.fromClass) baseClassType else KotlinType(templateClass))
     fileExtension.replaceOnlyDefault(mainAnnotation.fileExtension)
+    // TODO: remove this exception when gradle switches to the new definitions and sets the property accordingly
+    if (get(fileExtension) == "gradle.kts") {
+        isStandalone(false)
+    }
     filePathPattern.replaceOnlyDefault(mainAnnotation.filePathPattern)
     displayName.replaceOnlyDefault(mainAnnotation.displayName)
 }

plugins/scripting/scripting-compiler-impl/src/org/jetbrains/kotlin/scripting/definitions/ScriptCompilationConfigurationFromDefinition.kt

@@ -36,6 +36,10 @@ class ScriptCompilationConfigurationFromDefinition(
         platform(scriptDefinition.platform)
         @Suppress("DEPRECATION")
         compilerOptions.putIfAny(scriptDefinition.additionalCompilerArguments)
+        // TODO: remove this exception when gradle switches to the new definitions and sets the property accordingly
+        if (scriptDefinition.fileExtension == "gradle.kts") {
+            isStandalone(false)
+        }
         ide {
             acceptedLocations.put(scriptDefinition.scriptExpectedLocations.mapLegacyExpectedLocations())
         }

plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/compiler/plugin/extensions/ScriptingProcessSourcesBeforeCompilingExtension.kt

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.scripting.compiler.plugin.extensions
+
+import com.intellij.openapi.project.Project
+import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY
+import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
+import org.jetbrains.kotlin.config.CommonConfigurationKeys
+import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.config.LanguageFeature
+import org.jetbrains.kotlin.config.languageVersionSettings
+import org.jetbrains.kotlin.extensions.ProcessSourcesBeforeCompilingExtension
+import org.jetbrains.kotlin.psi.KtFile
+import org.jetbrains.kotlin.scripting.definitions.ScriptDefinitionProvider
+import org.jetbrains.kotlin.scripting.resolve.KtFileScriptSource
+import kotlin.script.experimental.api.*
+
+class ScriptingProcessSourcesBeforeCompilingExtension(val project: Project) : ProcessSourcesBeforeCompilingExtension {
+
+    override fun processSources(sources: Collection<KtFile>, configuration: CompilerConfiguration): Collection<KtFile> {
+        val versionSettings = configuration.languageVersionSettings
+        val shouldSkipStandaloneScripts = versionSettings.supportsFeature(LanguageFeature.SkipStandaloneScriptsInSourceRoots)
+        val definitionProvider by lazy(LazyThreadSafetyMode.NONE) { ScriptDefinitionProvider.getInstance(project) }
+        val messageCollector = configuration.getNotNull(MESSAGE_COLLECTOR_KEY)
+
+        fun KtFile.isStandaloneScript(): Boolean {
+            val scriptDefinition = definitionProvider?.findDefinition(KtFileScriptSource(this))
+            return scriptDefinition?.compilationConfiguration?.get(ScriptCompilationConfiguration.isStandalone) ?: true
+        }
+
+        // filter out scripts that are not suitable for source roots, according to the compiler configuration and script definitions
+        return sources.filter { ktFile ->
+            when {
+                !ktFile.isScript() -> true
+                configuration.getBoolean(CommonConfigurationKeys.ALLOW_ANY_SCRIPTS_IN_SOURCE_ROOTS) -> true
+                !ktFile.isStandaloneScript() -> true
+                else -> {
+                    if (!shouldSkipStandaloneScripts) {
+                        messageCollector.report(
+                            CompilerMessageSeverity.WARNING,
+                            "Script '${ktFile.name}' is not supposed to be used along with regular Kotlin sources, and will be ignored in the future versions by default. (Use -Xallow-any-scripts-in-source-roots command line option to opt-in for the old behavior.)"
+                        )
+                    }
+                    !shouldSkipStandaloneScripts
+                }
+            }
+        }
+    }
+}

plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/compiler/plugin/pluginRegisrar.kt

@@ -16,6 +16,7 @@ import org.jetbrains.kotlin.compiler.plugin.ComponentRegistrar
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.extensions.CollectAdditionalSourcesExtension
 import org.jetbrains.kotlin.extensions.CompilerConfigurationExtension
+import org.jetbrains.kotlin.extensions.ProcessSourcesBeforeCompilingExtension
 import org.jetbrains.kotlin.extensions.ProjectExtensionDescriptor
 import org.jetbrains.kotlin.resolve.extensions.ExtraImportsProviderExtension
 import org.jetbrains.kotlin.resolve.extensions.SyntheticResolveExtension
@@ -24,6 +25,7 @@ import org.jetbrains.kotlin.scripting.compiler.plugin.definitions.CliScriptDepen
 import org.jetbrains.kotlin.scripting.compiler.plugin.definitions.CliScriptReportSink
 import org.jetbrains.kotlin.scripting.compiler.plugin.extensions.JvmStandardReplFactoryExtension
 import org.jetbrains.kotlin.scripting.compiler.plugin.extensions.ScriptingCollectAdditionalSourcesExtension
+import org.jetbrains.kotlin.scripting.compiler.plugin.extensions.ScriptingProcessSourcesBeforeCompilingExtension
 import org.jetbrains.kotlin.scripting.definitions.ScriptDefinitionProvider
 import org.jetbrains.kotlin.scripting.definitions.ScriptDependenciesProvider
 import org.jetbrains.kotlin.scripting.extensions.ScriptExtraImportsProviderExtension
@@ -56,6 +58,7 @@ class ScriptingCompilerConfigurationComponentRegistrar : ComponentRegistrar {
         withClassloadingProblemsReporting(messageCollector) {
             CompilerConfigurationExtension.registerExtension(project, ScriptingCompilerConfigurationExtension(project, hostConfiguration))
             CollectAdditionalSourcesExtension.registerExtension(project, ScriptingCollectAdditionalSourcesExtension(project))
+            ProcessSourcesBeforeCompilingExtension.registerExtension(project, ScriptingProcessSourcesBeforeCompilingExtension(project))
             ScriptEvaluationExtension.registerExtensionIfRequired(project, JvmCliScriptEvaluationExtension())
             ShellExtension.registerExtensionIfRequired(project, JvmCliReplShellExtension())
             ReplFactoryExtension.registerExtensionIfRequired(project, JvmStandardReplFactoryExtension())

plugins/scripting/scripting-compiler/testData/compiler/mixedCompilation/simpleScript.main.kts

@@ -0,0 +1,2 @@
+
+val ok = "OK"

plugins/scripting/scripting-compiler/testData/compiler/mixedCompilation/simpleScriptInstance.kt

@@ -0,0 +1,4 @@
+
+fun main() {
+    println(SimpleScript_main(emptyArray<String>(), java.io.File("")).ok)
+}

plugins/scripting/scripting-compiler/tests/org/jetbrains/kotlin/scripting/compiler/plugin/ScriptingWithCliCompilerTest.kt

@@ -6,6 +6,7 @@
 package org.jetbrains.kotlin.scripting.compiler.plugin
 
 import org.jetbrains.kotlin.cli.common.CLITool
+import org.jetbrains.kotlin.cli.common.ExitCode
 import org.jetbrains.kotlin.cli.common.environment.setIdeaIoUseFallback
 import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler
 import org.jetbrains.kotlin.scripting.compiler.test.linesSplitTrim
@@ -187,6 +188,58 @@ class ScriptingWithCliCompilerTest {
         )
     }
 
+    @Test
+    fun testCompileScriptWithRegularKotlin() {
+
+        fun compileVariant(vararg flags: String): Pair<List<String>, ExitCode> {
+            return withTempDir { tmpdir ->
+                val (_, err, exitCode) = captureOutErrRet {
+                    CLITool.doMainNoExit(
+                        K2JVMCompiler(),
+                        arrayOf(
+                            "-d", tmpdir.path,
+                            "-cp", getMainKtsClassPath().joinToString(File.pathSeparator),
+                            *flags,
+                            "$TEST_DATA_DIR/compiler/mixedCompilation/simpleScriptInstance.kt",
+                            "$TEST_DATA_DIR/compiler/mixedCompilation/simpleScript.main.kts"
+                        )
+                    )
+                }
+                err.linesSplitTrim() to exitCode
+            }
+        }
+
+        val scriptInSourceRootWarning =
+            "warning: script 'simpleScript.main.kts' is not supposed to be used along with regular Kotlin sources, and will be ignored in the future versions"
+
+        val unresolvedScriptError =
+            "simpleScriptInstance.kt:3:13: error: unresolved reference: SimpleScript_main"
+
+        compileVariant("-language-version", "1.7").let { (errLines, exitCode) ->
+            Assert.assertTrue(errLines.any { it.startsWith(scriptInSourceRootWarning) })
+            Assert.assertEquals(ExitCode.OK, exitCode)
+        }
+
+        compileVariant("-language-version", "1.7", "-Xallow-any-scripts-in-source-roots").let { (errLines, exitCode) ->
+            Assert.assertTrue(errLines.none { it.startsWith(scriptInSourceRootWarning) })
+            Assert.assertEquals(ExitCode.OK, exitCode)
+        }
+
+        compileVariant("-language-version", "1.9").let { (errLines, exitCode) ->
+            if (errLines.none { it.endsWith(unresolvedScriptError) }) {
+                Assert.fail("Expecting unresolved reference: SimpleScript_main error, got:\n${errLines.joinToString("\n")}")
+            }
+            Assert.assertEquals(ExitCode.COMPILATION_ERROR, exitCode)
+        }
+
+        compileVariant("-language-version", "1.9", "-Xallow-any-scripts-in-source-roots").let { (errLines, exitCode) ->
+            Assert.assertTrue(errLines.none {
+                it.endsWith(unresolvedScriptError) || it.startsWith(scriptInSourceRootWarning)
+            })
+            Assert.assertEquals(ExitCode.OK, exitCode)
+        }
+    }
+
     private fun getMainKtsClassPath(): List<File> {
         return listOf(
             File("dist/kotlinc/lib/kotlin-main-kts.jar").also {

plugins/scripting/scripting-compiler/tests/org/jetbrains/kotlin/scripting/compiler/plugin/testUtil.kt

@@ -224,10 +224,10 @@ internal fun <T> captureOutErrRet(body: () -> T): Triple<String, String, T> {
     return Triple(outStream.toString().trim(), errStream.toString().trim(), ret)
 }
 
-internal fun <R> withTempDir(keyName: String = "tmp", body: (File) -> R) {
+internal fun <R> withTempDir(keyName: String = "tmp", body: (File) -> R): R {
     val tempDir = Files.createTempDirectory(keyName).toFile()
     try {
-        body(tempDir)
+        return body(tempDir)
     } finally {
         tempDir.deleteRecursively()
     }