Scripting: refactor test infrastructure - add missing stderr handling

libraries/tools/kotlin-main-kts-test/test/org/jetbrains/kotlin/mainKts/test/mainKtsIT.kt

@@ -163,12 +163,13 @@ class MainKtsIT {
 fun runWithKotlincAndMainKts(
     scriptPath: String,
     expectedOutPatterns: List<String> = emptyList(),
+    expectedErrPatterns: List<String> = emptyList(),
     expectedExitCode: Int = 0,
     cacheDir: Path? = null
 ) {
     val paths = PathUtil.kotlinPathsForDistDirectory
     runWithKotlinc(
-        scriptPath, expectedOutPatterns, expectedExitCode,
+        scriptPath, expectedOutPatterns, expectedErrPatterns, expectedExitCode,
         classpath = listOf(
             paths.jar(KotlinPaths.Jar.MainKts).also {
                 Assert.assertTrue("kotlin-main-kts.jar not found, run dist task: ${it.absolutePath}", it.exists())
@@ -181,12 +182,13 @@ fun runWithKotlincAndMainKts(
 fun runWithKotlinRunner(
     scriptPath: String,
     expectedOutPatterns: List<String> = emptyList(),
+    expectedErrPatterns: List<String> = emptyList(),
     expectedExitCode: Int = 0,
     cacheDir: Path? = null,
     expectErrorOnK2: Boolean = false
 ) {
     runWithKotlinLauncherScript(
-        "kotlin", listOf(scriptPath), expectedOutPatterns, expectedExitCode,
+        "kotlin", listOf(scriptPath), expectedOutPatterns, expectedErrPatterns, expectedExitCode,
         additionalEnvVars = listOf(COMPILED_SCRIPTS_CACHE_DIR_ENV_VAR to (cacheDir?.toAbsolutePath()?.toString() ?: "")),
         expectErrorOnK2 = expectErrorOnK2
     )

libraries/tools/kotlin-main-kts-test/test/org/jetbrains/kotlin/mainKts/test/mainKtsTest.kt

@@ -403,17 +403,24 @@ class CacheDirectoryDetectorTest {
     private val directories = Directories(systemProperties, environment)
 }
 
-internal fun captureOut(body: () -> Unit): String {
+internal fun captureOut(body: () -> Unit): String = captureOutAndErr(body).first
+
+internal fun captureOutAndErr(body: () -> Unit): Pair<String, String> {
     val outStream = ByteArrayOutputStream()
+    val errStream = ByteArrayOutputStream()
     val prevOut = System.out
+    val prevErr = System.err
     System.setOut(PrintStream(outStream))
+    System.setErr(PrintStream(errStream))
     try {
         body()
     } finally {
         System.out.flush()
+        System.err.flush()
         System.setOut(prevOut)
+        System.setErr(prevErr)
     }
-    return outStream.toString().trim()
+    return outStream.toString().trim() to errStream.toString().trim()
 }
 
 internal fun <T> withProperty(name: String, value: String?, body: () -> T): T {

plugins/scripting/scripting-compiler/tests/org/jetbrains/kotlin/scripting/compiler/plugin/testUtil.kt

@@ -30,6 +30,7 @@ internal fun getBaseCompilerArgumentsFromProperty(): List<String>? =
 fun runWithKotlinc(
     scriptPath: String,
     expectedOutPatterns: List<String> = emptyList(),
+    expectedErrPatterns: List<String> = emptyList(),
     expectedExitCode: Int = 0,
     workDirectory: File? = null,
     classpath: List<File> = emptyList(),
@@ -38,7 +39,7 @@ fun runWithKotlinc(
 ) {
     runWithKotlinc(
         arrayOf("-script", scriptPath),
-        expectedOutPatterns, expectedExitCode, workDirectory, classpath, additionalEnvVars, expectErrorOnK2
+        expectedOutPatterns, expectedErrPatterns, expectedExitCode, workDirectory, classpath, additionalEnvVars, expectErrorOnK2
     )
 }
 
@@ -46,6 +47,7 @@ fun runWithKotlinLauncherScript(
     launcherScriptName: String,
     compilerArgs: Iterable<String>,
     expectedOutPatterns: List<String> = emptyList(),
+    expectedErrPatterns: List<String> = emptyList(),
     expectedExitCode: Int = 0,
     workDirectory: File? = null,
     classpath: List<File> = emptyList(),
@@ -66,12 +68,16 @@ fun runWithKotlinLauncherScript(
         addAll(compilerArgs)
     }
 
-    runAndCheckResults(args, expectedOutPatterns, expectedExitCode, workDirectory, additionalEnvVars, expectErrorOnK2 = expectErrorOnK2)
+    runAndCheckResults(
+        args, expectedOutPatterns, expectedErrPatterns, expectedExitCode, workDirectory, additionalEnvVars,
+        expectErrorOnK2 = expectErrorOnK2
+    )
 }
 
 fun runWithKotlinc(
     compilerArgs: Array<String>,
     expectedOutPatterns: List<String> = emptyList(),
+    expectedErrPatterns: List<String> = emptyList(),
     expectedExitCode: Int = 0,
     workDirectory: File? = null,
     classpath: List<File> = emptyList(),
@@ -79,14 +85,15 @@ fun runWithKotlinc(
     expectErrorOnK2: Boolean = false
 ) {
     runWithKotlinLauncherScript(
-        "kotlinc", compilerArgs.asIterable(), expectedOutPatterns, expectedExitCode, workDirectory, classpath,
+        "kotlinc", compilerArgs.asIterable(), expectedOutPatterns, expectedErrPatterns,
-        additionalEnvVars, expectErrorOnK2
+        expectedExitCode, workDirectory, classpath, additionalEnvVars, expectErrorOnK2
     )
 }
 
 fun runAndCheckResults(
     args: List<String>,
     expectedOutPatterns: List<String> = emptyList(),
+    expectedErrPatterns: List<String> = emptyList(),
     expectedExitCode: Int = 0,
     workDirectory: File? = null,
     additionalEnvVars: Iterable<Pair<String, String>>? = null,
@@ -143,12 +150,18 @@ fun runAndCheckResults(
                 process.exitValue() != 0 || processOut.contains("error: ")
             )
         } else {
-            Assert.assertEquals(expectedOutPatterns.size, processOut.size)
+            fun checkExpectedOutputPatterns(expectedPatterns: List<String>, actualOut: List<String>) {
-            for ((expectedPattern, actualLine) in expectedOutPatterns.zip(processOut)) {
+                Assert.assertEquals(expectedPatterns.size, actualOut.size)
-                Assert.assertTrue(
+                for ((expectedPattern, actualLine) in expectedPatterns.zip(actualOut)) {
-                    "line \"$actualLine\" do not match with expected pattern \"$expectedPattern\"",
+                    Assert.assertTrue(
-                    Regex(expectedPattern).matches(actualLine)
+                        "line \"$actualLine\" do not match with expected pattern \"$expectedPattern\"",
-                )
+                        Regex(expectedPattern).matches(actualLine)
+                    )
+                }
+            }
+            checkExpectedOutputPatterns(expectedOutPatterns, processOut)
+            if (expectedErrPatterns.isNotEmpty()) {
+                checkExpectedOutputPatterns(expectedErrPatterns, processErr)
             }
             Assert.assertEquals(expectedExitCode, process.exitValue())
         }