diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/AbstractNativeKlibABITest.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/AbstractNativeKlibABITest.kt index 42816553aa7..6f8e365dc99 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/AbstractNativeKlibABITest.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/AbstractNativeKlibABITest.kt @@ -116,7 +116,6 @@ abstract class AbstractNativeKlibABITest : AbstractNativeSimpleTest() { modules = setOf(module), freeCompilerArgs = compilerArgs, nominalPackageName = PackageName.EMPTY, - expectedOutputDataFile = null, checks = TestRunChecks.Default(testRunSettings.get().executionTimeout), extras = DEFAULT_EXTRAS ).apply { diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/InfrastructureDumpedTestListingTest.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/InfrastructureDumpedTestListingTest.kt index fc2486dc03e..9229a46b15c 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/InfrastructureDumpedTestListingTest.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/InfrastructureDumpedTestListingTest.kt @@ -112,7 +112,6 @@ class InfrastructureDumpedTestListingTest : AbstractNativeSimpleTest() { modules = setOf(module), freeCompilerArgs = TestCompilerArgs.EMPTY, nominalPackageName = PackageName.EMPTY, - expectedOutputDataFile = null, checks = TestRunChecks.Default(testRunSettings.get().executionTimeout), extras = DEFAULT_EXTRAS ).apply { diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCase.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCase.kt index c4a68f6e213..269cb03d25a 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCase.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCase.kt @@ -170,7 +170,6 @@ internal class TestCase( val modules: Set, val freeCompilerArgs: TestCompilerArgs, val nominalPackageName: PackageName, - val expectedOutputDataFile: File?, // TODO: refactor to TestRunCheck val checks: TestRunChecks, val extras: Extras ) { diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestDirectives.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestDirectives.kt index 4d701580753..2e259ca8bde 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestDirectives.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestDirectives.kt @@ -14,6 +14,7 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.OUTPUT_DAT import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.EXPECTED_TIMEOUT_FAILURE import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.TEST_RUNNER import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck +import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.OutputDataFile import org.jetbrains.kotlin.test.directives.model.RegisteredDirectives import org.jetbrains.kotlin.test.directives.model.SimpleDirectivesContainer import org.jetbrains.kotlin.test.directives.model.StringDirective @@ -276,8 +277,8 @@ internal fun parseFreeCompilerArgs(registeredDirectives: RegisteredDirectives, l return TestCompilerArgs(freeCompilerArgs) } -internal fun parseOutputDataFile(baseDir: File, registeredDirectives: RegisteredDirectives, location: Location): File? = - parseFileBasedDirective(baseDir, OUTPUT_DATA_FILE, registeredDirectives, location) +internal fun parseOutputDataFile(baseDir: File, registeredDirectives: RegisteredDirectives, location: Location): OutputDataFile? = + parseFileBasedDirective(baseDir, OUTPUT_DATA_FILE, registeredDirectives, location)?.let(TestRunCheck::OutputDataFile) internal fun parseInputDataFile(baseDir: File, registeredDirectives: RegisteredDirectives, location: Location): File? = parseFileBasedDirective(baseDir, INPUT_DATA_FILE, registeredDirectives, location) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/ExtTestCaseGroupProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/ExtTestCaseGroupProvider.kt index 2535b72cea2..8e85c4d7666 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/ExtTestCaseGroupProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/ExtTestCaseGroupProvider.kt @@ -553,7 +553,6 @@ private class ExtTestDataFile( modules = modules, freeCompilerArgs = assembleFreeCompilerArgs(), nominalPackageName = testDataFileSettings.nominalPackageName, - expectedOutputDataFile = null, checks = TestRunChecks.Default(timeouts.executionTimeout), extras = WithTestRunnerExtras(runnerType = TestRunnerType.DEFAULT) ) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/PredefinedTestCaseGroupProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/PredefinedTestCaseGroupProvider.kt index fbf60c53231..31cfc6ace33 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/PredefinedTestCaseGroupProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/PredefinedTestCaseGroupProvider.kt @@ -63,7 +63,6 @@ internal class PredefinedTestCaseGroupProvider(annotation: PredefinedTestCases) freeCompilerArgs = predefinedTestCase.freeCompilerArgs .parseCompilerArgs(settings) { "Failed to parse free compiler arguments for test case $testCaseId" }, nominalPackageName = PackageName(testCaseId.uniqueName), - expectedOutputDataFile = null, checks = TestRunChecks.Default(settings.get().executionTimeout), extras = WithTestRunnerExtras( runnerType = predefinedTestCase.runnerType, diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt index e07f48b5e02..9ac0b777e3b 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt @@ -8,8 +8,7 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.group import org.jetbrains.kotlin.konan.blackboxtest.support.* import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.NoTestRunnerExtras import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.WithTestRunnerExtras -import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.ExecutionTimeout -import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.ExitCode +import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.* import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunChecks import org.jetbrains.kotlin.konan.blackboxtest.support.settings.GeneratedSources import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Settings @@ -183,7 +182,6 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider { val registeredDirectives = directivesParser.build() val freeCompilerArgs = parseFreeCompilerArgs(registeredDirectives, location) - val expectedOutputDataFile = parseOutputDataFile(baseDir = testDataFile.parentFile, registeredDirectives, location) val testKind = parseTestKind(registeredDirectives, location) val expectedTimeoutFailure = parseExpectedTimeoutFailure(registeredDirectives) @@ -198,10 +196,10 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider { modules = testModules.values.toSet(), freeCompilerArgs = freeCompilerArgs, nominalPackageName = nominalPackageName, - expectedOutputDataFile = expectedOutputDataFile, checks = TestRunChecks( computeExecutionTimeoutCheck(settings, expectedTimeoutFailure), - computeExitCodeCheck(testKind, registeredDirectives, location) + computeExitCodeCheck(testKind, registeredDirectives, location), + computeOutputDataFileCheck(testDataFile, registeredDirectives, location) ), extras = if (testKind == TestKind.STANDALONE_NO_TR) NoTestRunnerExtras( @@ -254,5 +252,11 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider { parseExpectedExitCode(registeredDirectives, location) else ExitCode.Expected(0) + + private fun computeOutputDataFileCheck( + testDataFile: File, + registeredDirectives: RegisteredDirectives, + location: Location + ): OutputDataFile? = parseOutputDataFile(baseDir = testDataFile.parentFile, registeredDirectives, location) } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt index f3e3a5a0082..a611041c88b 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt @@ -5,6 +5,7 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.runner +import com.intellij.openapi.util.text.StringUtilRt.convertLineSeparators import kotlinx.coroutines.* import org.jetbrains.kotlin.konan.blackboxtest.support.runner.AbstractRunner.AbstractRun import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.ExecutionTimeout @@ -94,6 +95,16 @@ internal abstract class AbstractLocalProcessRunner(private val checks: TestRu } } } + is TestRunCheck.OutputDataFile -> { + val expectedOutput = check.file.readText() + val actualFilteredOutput = runResult.processOutput.stdOut.filteredOutput + runResult.processOutput.stdErr + + // Don't use verifyExpectation(expected, actual) to avoid exposing potentially large test output in exception message + // and blowing up test logs. + verifyExpectation(convertLineSeparators(expectedOutput) == convertLineSeparators(actualFilteredOutput)) { + "Tested process output mismatch. See \"TEST STDOUT\" and \"EXPECTED OUTPUT DATA FILE\" below." + } + } } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/BaseTestRunProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/BaseTestRunProvider.kt index dec510829d5..dcffc199e97 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/BaseTestRunProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/BaseTestRunProvider.kt @@ -26,7 +26,7 @@ internal open class BaseTestRunProvider { ) private fun getTestRunParameters(testCase: TestCase, testName: TestName?): List = buildList { - addIfNotNull(testCase.expectedOutputDataFile?.let(TestRunParameter::WithExpectedOutputData)) + addIfNotNull(testCase.checks.outputDataFile?.file?.let(TestRunParameter::WithExpectedOutputData)) when (testCase.kind) { TestKind.STANDALONE_NO_TR -> { diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt index 5c01a4689ca..a3edfc4d9e8 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt @@ -5,7 +5,6 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.runner -import com.intellij.openapi.util.text.StringUtilRt.convertLineSeparators import org.jetbrains.kotlin.konan.blackboxtest.support.* import org.jetbrains.kotlin.konan.blackboxtest.support.util.TCTestOutputFilter import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestOutputFilter @@ -51,9 +50,6 @@ internal class LocalTestRunner(private val testRun: TestRun) : AbstractLocalProc override fun doHandle() { verifyTestReport(runResult.processOutput.stdOut.testReport) - - val mergedOutput = runResult.processOutput.stdOut.filteredOutput + runResult.processOutput.stdErr - verifyNonTestOutput(mergedOutput) } private fun verifyTestReport(testReport: TestReport?) { @@ -84,15 +80,5 @@ internal class LocalTestRunner(private val testRun: TestRun) : AbstractLocalProc tests.forEach { appendLine().append(" - ").append(it) } } } - - private fun verifyNonTestOutput(nonTestOutput: String) { - testRun.runParameters.get { - // Don't use verifyExpectation(expected, actual) to avoid exposing potentially large test output in exception message - // and blowing up test logs. - verifyExpectation(convertLineSeparators(expectedOutputDataFile.readText()) == convertLineSeparators(nonTestOutput)) { - "Tested process output mismatch. See \"TEST STDOUT\" and \"EXPECTED OUTPUT DATA FILE\" below." - } - } - } } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRun.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRun.kt index 93f5f516fe2..02f6c14ed09 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRun.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRun.kt @@ -96,6 +96,7 @@ internal sealed interface TestRunParameter { override fun applyTo(programArgs: MutableList) = Unit } + // Currently, used only for logging the data. class WithExpectedOutputData(val expectedOutputDataFile: File) : TestRunParameter { override fun applyTo(programArgs: MutableList) = Unit } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunChecks.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunChecks.kt index 9e15e23a02c..324d6886e2e 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunChecks.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunChecks.kt @@ -5,8 +5,9 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.runner -import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.ExecutionTimeout -import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.ExitCode +import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.* +import org.jetbrains.kotlin.utils.yieldIfNotNull +import java.io.File import kotlin.time.Duration internal sealed interface TestRunCheck { @@ -19,24 +20,29 @@ internal sealed interface TestRunCheck { object AnyNonZero : ExitCode() class Expected(val expectedExitCode: Int) : ExitCode() } + + class OutputDataFile(val file: File) : TestRunCheck } internal class TestRunChecks( val executionTimeoutCheck: ExecutionTimeout, - private val exitCodeCheck: ExitCode + private val exitCodeCheck: ExitCode, + val outputDataFile: OutputDataFile? ) : Iterable { override fun iterator() = iterator { yield(executionTimeoutCheck) yield(exitCodeCheck) + yieldIfNotNull(outputDataFile) } companion object { // The most frequently used case: @Suppress("TestFunctionName") fun Default(timeout: Duration) = TestRunChecks( - ExecutionTimeout.ShouldNotExceed(timeout), - ExitCode.Expected(0) + executionTimeoutCheck = ExecutionTimeout.ShouldNotExceed(timeout), + exitCodeCheck = ExitCode.Expected(0), + outputDataFile = null ) } }