[K/N][tests] Support OUTPUT_REGEX in new testing infra ^KT-61259

native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/TestDirectives.kt

@@ -14,6 +14,7 @@ import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.INPUT_DAT
 import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.KIND
 import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.LLDB_TRACE
 import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.OUTPUT_DATA_FILE
+import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.OUTPUT_REGEX
 import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.PROGRAM_ARGS
 import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.TEST_RUNNER
 import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck
@@ -22,6 +23,7 @@ import org.jetbrains.kotlin.konan.test.blackbox.support.util.LLDBSessionSpec
 import org.jetbrains.kotlin.test.directives.model.RegisteredDirectives
 import org.jetbrains.kotlin.test.directives.model.SimpleDirectivesContainer
 import org.jetbrains.kotlin.test.directives.model.StringDirective
+import org.jetbrains.kotlin.test.directives.model.singleValue
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
 import org.jetbrains.kotlin.test.services.impl.RegisteredDirectivesParser
@@ -83,13 +85,13 @@ internal object TestDirectives : SimpleDirectivesContainer() {
         """.trimIndent()
     )
 
-    // TODO: to be supported later
+    val OUTPUT_REGEX by stringDirective(
-//    val OUTPUT_REGEX by stringDirective(
+        description = """
-//        description = """
+            The regex that the expected program output should match to. When program finishes its execution, the actual output (stdout)
-//            The regex that the expected program output should match to. When program finishes its execution, the actual output (stdout)
+            will be checked against this regex.
-//            will be checked against this regex.
+        """.trimIndent(),
-//        """.trimIndent()
+        multiLine = true,
-//    )
+    )
 
     // TODO: to be supported later
 //    val OUTPUT_INCLUDES by stringDirective(
@@ -394,7 +396,7 @@ internal fun parseFreeCompilerArgs(registeredDirectives: RegisteredDirectives, l
 }
 
 internal fun parseOutputDataFile(baseDir: File, registeredDirectives: RegisteredDirectives, location: Location): OutputDataFile? =
-    parseFileBasedDirective(baseDir, OUTPUT_DATA_FILE, registeredDirectives, location)?.let(TestRunCheck::OutputDataFile)
+    parseFileBasedDirective(baseDir, OUTPUT_DATA_FILE, registeredDirectives, location)?.let { OutputDataFile(file = it) }
 
 internal fun parseInputDataFile(baseDir: File, registeredDirectives: RegisteredDirectives, location: Location): File? =
     parseFileBasedDirective(baseDir, INPUT_DATA_FILE, registeredDirectives, location)
@@ -418,6 +420,19 @@ private fun parseFileBasedDirective(
 
 internal fun parseProgramArguments(registeredDirectives: RegisteredDirectives): List<String> = registeredDirectives[PROGRAM_ARGS]
 
+internal fun parseOutputRegex(registeredDirectives: RegisteredDirectives): TestRunCheck.OutputMatcher? {
+    if (OUTPUT_REGEX !in registeredDirectives)
+        return null
+    val regexStr = registeredDirectives.singleValue(OUTPUT_REGEX)
+    val regex = regexStr.toRegex(RegexOption.DOT_MATCHES_ALL)
+    return TestRunCheck.OutputMatcher {
+        assertTrue(regex.matches(it)) {
+            "Regex `$regex` failed to match `$it`"
+        }
+        true
+    }
+}
+
 internal class Location(private val testDataFile: File, val lineNumber: Int? = null) {
     override fun toString() = buildString {
         append(testDataFile.path)

native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/StandardTestCaseGroupProvider.kt

@@ -212,6 +212,10 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider {
             )
         else null
 
+        val outputMatcher = lldbSpec?.let {
+            OutputMatcher(Output.STDOUT) { output -> lldbSpec.checkLLDBOutput(output, settings.get()) }
+        } ?: parseOutputRegex(registeredDirectives)
+
         val testCase = TestCase(
             id = TestCaseId.TestDataFile(testDataFile),
             kind = testKind,
@@ -222,7 +226,7 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider {
                 computeExecutionTimeoutCheck(settings, expectedTimeoutFailure),
                 computeExitCodeCheck(testKind, registeredDirectives, location),
                 computeOutputDataFileCheck(testDataFile, registeredDirectives, location),
-                lldbSpec?.let { OutputMatcher { output -> lldbSpec.checkLLDBOutput(output, settings.get()) } },
+                outputMatcher,
                 fileCheckMatcher = null,
             ),
             extras = when (testKind) {

native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/AbstractLocalProcessRunner.kt

@@ -93,6 +93,12 @@ internal abstract class AbstractLocalProcessRunner<R>(protected val checks: Test
     abstract override fun buildResultHandler(runResult: RunResult): LocalResultHandler<R> // ?? Narrow returned type.
 }
 
+private fun RunResult.processOutputAsString(output: TestRunCheck.Output) = when (output) {
+    TestRunCheck.Output.STDOUT -> processOutput.stdOut.filteredOutput
+    TestRunCheck.Output.STDERR -> processOutput.stdErr
+    TestRunCheck.Output.ALL -> processOutput.stdOut.filteredOutput + processOutput.stdErr
+}
+
 internal abstract class LocalResultHandler<R>(
     runResult: RunResult,
     private val visibleProcessName: String,
@@ -124,7 +130,7 @@ internal abstract class LocalResultHandler<R>(
                     }
                     is TestRunCheck.OutputDataFile -> {
                         val expectedOutput = check.file.readText()
-                        val actualFilteredOutput = runResult.processOutput.stdOut.filteredOutput + runResult.processOutput.stdErr
+                        val actualFilteredOutput = runResult.processOutputAsString(check.output)
 
                         // Don't use verifyExpectation(expected, actual) to avoid exposing potentially large test output in exception message
                         // and blowing up test logs.
@@ -134,7 +140,7 @@ internal abstract class LocalResultHandler<R>(
                     }
                     is TestRunCheck.OutputMatcher -> {
                         try {
-                            if(!check.match(runResult.processOutput.stdOut.filteredOutput)) add(
+                            if(!check.match(runResult.processOutputAsString(check.output))) add(
                                 "Tested process output has not passed validation."
                             )
                         } catch (t: Throwable) {

native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunChecks.kt

@@ -22,9 +22,19 @@ internal sealed interface TestRunCheck {
         class Expected(val expectedExitCode: Int) : ExitCode()
     }
 
-    class OutputDataFile(val file: File) : TestRunCheck
+    enum class Output {
+        STDOUT,
+        STDERR,
 
-    class OutputMatcher(val match: (String) -> Boolean): TestRunCheck
+        /**
+         * [STDOUT] followed by [STDERR]
+         */
+        ALL,
+    }
+
+    class OutputDataFile(val output: Output = Output.ALL, val file: File) : TestRunCheck
+
+    class OutputMatcher(val output: Output = Output.ALL, val match: (String) -> Boolean): TestRunCheck
 
     class FileCheckMatcher(val settings: Settings, val testDataFile: File): TestRunCheck
 }