From 226ee3c3671af4653c76563c32fddb0b3128e646 Mon Sep 17 00:00:00 2001 From: Alexander Shabalin Date: Wed, 8 Nov 2023 13:15:46 +0100 Subject: [PATCH] [K/N][tests] Support OUTPUT_REGEX in new testing infra ^KT-61259 --- .../test/blackbox/support/TestDirectives.kt | 31 ++++++++++++++----- .../group/StandardTestCaseGroupProvider.kt | 6 +++- .../runner/AbstractLocalProcessRunner.kt | 10 ++++-- .../blackbox/support/runner/TestRunChecks.kt | 14 +++++++-- 4 files changed, 48 insertions(+), 13 deletions(-) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/TestDirectives.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/TestDirectives.kt index 6424f2c0ce6..a076afdff2e 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/TestDirectives.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/TestDirectives.kt @@ -14,6 +14,7 @@ import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.INPUT_DAT import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.KIND import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.LLDB_TRACE import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.OUTPUT_DATA_FILE +import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.OUTPUT_REGEX import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.PROGRAM_ARGS import org.jetbrains.kotlin.konan.test.blackbox.support.TestDirectives.TEST_RUNNER import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck @@ -22,6 +23,7 @@ import org.jetbrains.kotlin.konan.test.blackbox.support.util.LLDBSessionSpec import org.jetbrains.kotlin.test.directives.model.RegisteredDirectives import org.jetbrains.kotlin.test.directives.model.SimpleDirectivesContainer import org.jetbrains.kotlin.test.directives.model.StringDirective +import org.jetbrains.kotlin.test.directives.model.singleValue import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail import org.jetbrains.kotlin.test.services.impl.RegisteredDirectivesParser @@ -83,13 +85,13 @@ internal object TestDirectives : SimpleDirectivesContainer() { """.trimIndent() ) - // TODO: to be supported later -// val OUTPUT_REGEX by stringDirective( -// description = """ -// The regex that the expected program output should match to. When program finishes its execution, the actual output (stdout) -// will be checked against this regex. -// """.trimIndent() -// ) + val OUTPUT_REGEX by stringDirective( + description = """ + The regex that the expected program output should match to. When program finishes its execution, the actual output (stdout) + will be checked against this regex. + """.trimIndent(), + multiLine = true, + ) // TODO: to be supported later // val OUTPUT_INCLUDES by stringDirective( @@ -394,7 +396,7 @@ internal fun parseFreeCompilerArgs(registeredDirectives: RegisteredDirectives, l } internal fun parseOutputDataFile(baseDir: File, registeredDirectives: RegisteredDirectives, location: Location): OutputDataFile? = - parseFileBasedDirective(baseDir, OUTPUT_DATA_FILE, registeredDirectives, location)?.let(TestRunCheck::OutputDataFile) + parseFileBasedDirective(baseDir, OUTPUT_DATA_FILE, registeredDirectives, location)?.let { OutputDataFile(file = it) } internal fun parseInputDataFile(baseDir: File, registeredDirectives: RegisteredDirectives, location: Location): File? = parseFileBasedDirective(baseDir, INPUT_DATA_FILE, registeredDirectives, location) @@ -418,6 +420,19 @@ private fun parseFileBasedDirective( internal fun parseProgramArguments(registeredDirectives: RegisteredDirectives): List = registeredDirectives[PROGRAM_ARGS] +internal fun parseOutputRegex(registeredDirectives: RegisteredDirectives): TestRunCheck.OutputMatcher? { + if (OUTPUT_REGEX !in registeredDirectives) + return null + val regexStr = registeredDirectives.singleValue(OUTPUT_REGEX) + val regex = regexStr.toRegex(RegexOption.DOT_MATCHES_ALL) + return TestRunCheck.OutputMatcher { + assertTrue(regex.matches(it)) { + "Regex `$regex` failed to match `$it`" + } + true + } +} + internal class Location(private val testDataFile: File, val lineNumber: Int? = null) { override fun toString() = buildString { append(testDataFile.path) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/StandardTestCaseGroupProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/StandardTestCaseGroupProvider.kt index b38f64d955f..19ac6c12b09 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/StandardTestCaseGroupProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/StandardTestCaseGroupProvider.kt @@ -212,6 +212,10 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider { ) else null + val outputMatcher = lldbSpec?.let { + OutputMatcher(Output.STDOUT) { output -> lldbSpec.checkLLDBOutput(output, settings.get()) } + } ?: parseOutputRegex(registeredDirectives) + val testCase = TestCase( id = TestCaseId.TestDataFile(testDataFile), kind = testKind, @@ -222,7 +226,7 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider { computeExecutionTimeoutCheck(settings, expectedTimeoutFailure), computeExitCodeCheck(testKind, registeredDirectives, location), computeOutputDataFileCheck(testDataFile, registeredDirectives, location), - lldbSpec?.let { OutputMatcher { output -> lldbSpec.checkLLDBOutput(output, settings.get()) } }, + outputMatcher, fileCheckMatcher = null, ), extras = when (testKind) { diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/AbstractLocalProcessRunner.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/AbstractLocalProcessRunner.kt index 5c83f57621c..9eff608dc63 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/AbstractLocalProcessRunner.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/AbstractLocalProcessRunner.kt @@ -93,6 +93,12 @@ internal abstract class AbstractLocalProcessRunner(protected val checks: Test abstract override fun buildResultHandler(runResult: RunResult): LocalResultHandler // ?? Narrow returned type. } +private fun RunResult.processOutputAsString(output: TestRunCheck.Output) = when (output) { + TestRunCheck.Output.STDOUT -> processOutput.stdOut.filteredOutput + TestRunCheck.Output.STDERR -> processOutput.stdErr + TestRunCheck.Output.ALL -> processOutput.stdOut.filteredOutput + processOutput.stdErr +} + internal abstract class LocalResultHandler( runResult: RunResult, private val visibleProcessName: String, @@ -124,7 +130,7 @@ internal abstract class LocalResultHandler( } is TestRunCheck.OutputDataFile -> { val expectedOutput = check.file.readText() - val actualFilteredOutput = runResult.processOutput.stdOut.filteredOutput + runResult.processOutput.stdErr + val actualFilteredOutput = runResult.processOutputAsString(check.output) // Don't use verifyExpectation(expected, actual) to avoid exposing potentially large test output in exception message // and blowing up test logs. @@ -134,7 +140,7 @@ internal abstract class LocalResultHandler( } is TestRunCheck.OutputMatcher -> { try { - if(!check.match(runResult.processOutput.stdOut.filteredOutput)) add( + if(!check.match(runResult.processOutputAsString(check.output))) add( "Tested process output has not passed validation." ) } catch (t: Throwable) { diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunChecks.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunChecks.kt index 78c1128550f..3cd12a62494 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunChecks.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunChecks.kt @@ -22,9 +22,19 @@ internal sealed interface TestRunCheck { class Expected(val expectedExitCode: Int) : ExitCode() } - class OutputDataFile(val file: File) : TestRunCheck + enum class Output { + STDOUT, + STDERR, - class OutputMatcher(val match: (String) -> Boolean): TestRunCheck + /** + * [STDOUT] followed by [STDERR] + */ + ALL, + } + + class OutputDataFile(val output: Output = Output.ALL, val file: File) : TestRunCheck + + class OutputMatcher(val output: Output = Output.ALL, val match: (String) -> Boolean): TestRunCheck class FileCheckMatcher(val settings: Settings, val testDataFile: File): TestRunCheck }