From 85882413dbe1479e3d694d4b4f5f4f5a255c7aef Mon Sep 17 00:00:00 2001 From: Vladimir Sukharev Date: Wed, 29 Nov 2023 12:24:21 +0100 Subject: [PATCH] [K/N] Fix approach of expected failures caused by outputCheck fail. `Expected failure flag` was wrongly included in list of test checks. However, the nature of checks and this flag is different: - any test check can pronounce the test as failed. - `Expected failure flag` must invert the accumulated effect of all checks. Having flag as an additional check cannot catch usecase, when test is failed only by producing wrong stdout. This commit extracts this flag out of check list. ^KT-59288 --- .../ObjCToKotlinSteppingInLLDBTest.kt | 1 - .../konan/test/blackbox/support/TestCase.kt | 1 + .../support/group/ExtTestCaseGroupProvider.kt | 9 +- .../group/StandardTestCaseGroupProvider.kt | 1 - .../runner/AbstractLocalProcessRunner.kt | 216 +++++++++--------- .../support/runner/BaseTestRunProvider.kt | 2 +- .../support/runner/LocalTestNameExtractor.kt | 2 +- .../support/runner/LocalTestRunner.kt | 13 +- .../test/blackbox/support/runner/TestRun.kt | 3 +- .../blackbox/support/runner/TestRunChecks.kt | 5 - 10 files changed, 123 insertions(+), 130 deletions(-) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/ObjCToKotlinSteppingInLLDBTest.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/ObjCToKotlinSteppingInLLDBTest.kt index 50307c8119c..be7f8d03e49 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/ObjCToKotlinSteppingInLLDBTest.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/ObjCToKotlinSteppingInLLDBTest.kt @@ -256,7 +256,6 @@ class ObjCToKotlinSteppingInLLDBTest : AbstractNativeSimpleTest() { checks = TestRunChecks( executionTimeoutCheck = ExecutionTimeout.ShouldNotExceed(testRunSettings.get().executionTimeout), exitCodeCheck = TestRunCheck.ExitCode.Expected(0), - expectedFailureCheck = null, outputDataFile = null, outputMatcher = spec.let { TestRunCheck.OutputMatcher { output -> spec.checkLLDBOutput(output, testRunSettings.get()) } }, fileCheckMatcher = null, diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/TestCase.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/TestCase.kt index 5224519f00b..b4925efc863 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/TestCase.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/TestCase.kt @@ -190,6 +190,7 @@ internal class TestCase( val checks: TestRunChecks, val extras: Extras, val fileCheckStage: String? = null, // KT-62157: TODO move it to extras + val expectedFailure: Boolean = false, ) { sealed interface Extras class NoTestRunnerExtras(val entryPoint: String, val inputDataFile: File? = null, val arguments: List = emptyList()) : Extras diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/ExtTestCaseGroupProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/ExtTestCaseGroupProvider.kt index efd381e521b..3c6635afa65 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/ExtTestCaseGroupProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/ExtTestCaseGroupProvider.kt @@ -124,7 +124,7 @@ private class ExtTestDataFile( structureFactory.ExtTestDataFileStructure(testDataFile, allSourceTransformers) } - private val isIgnoredTarget: Boolean = settings.isIgnoredTarget(structure.directives) + private val isExpectedFailure: Boolean = settings.isIgnoredTarget(structure.directives) private val testDataFileSettings by lazy { val optIns = structure.directives.multiValues(OPT_IN_DIRECTIVE) @@ -197,7 +197,7 @@ private class ExtTestDataFile( private fun determineIfStandaloneTest(): Boolean = with(structure) { if (directives.contains(NATIVE_STANDALONE_DIRECTIVE)) return true if (directives.contains(FILECHECK_STAGE.name)) return true - if (isIgnoredTarget) return true + if (isExpectedFailure) return true // To make the debug of possible failed testruns easier, it makes sense to run dodgy tests alone if (directives.contains(IGNORE_NATIVE.name) || directives.contains(IGNORE_NATIVE_K1.name) || @@ -524,10 +524,11 @@ private class ExtTestDataFile( modules = modules, freeCompilerArgs = assembleFreeCompilerArgs(), nominalPackageName = testDataFileSettings.nominalPackageName, + expectedFailure = isExpectedFailure, checks = TestRunChecks.Default(timeouts.executionTimeout).copy( fileCheckMatcher = fileCheckStage?.let { TestRunCheck.FileCheckMatcher(settings, testDataFile) }, - exitCodeCheck = TestRunCheck.ExitCode.Expected(0).takeUnless { isIgnoredTarget }, - expectedFailureCheck = TestRunCheck.ExpectedFailure.takeIf { isIgnoredTarget }, + // for expected failures, it does not matter, which exit code would the process have, since test might fail with other reasons + exitCodeCheck = TestRunCheck.ExitCode.Expected(0).takeUnless { isExpectedFailure }, ), fileCheckStage = fileCheckStage, extras = WithTestRunnerExtras(runnerType = TestRunnerType.DEFAULT) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/StandardTestCaseGroupProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/StandardTestCaseGroupProvider.kt index 8c7394c7e13..b38f64d955f 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/StandardTestCaseGroupProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/group/StandardTestCaseGroupProvider.kt @@ -221,7 +221,6 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider { checks = TestRunChecks( computeExecutionTimeoutCheck(settings, expectedTimeoutFailure), computeExitCodeCheck(testKind, registeredDirectives, location), - expectedFailureCheck = null, computeOutputDataFileCheck(testDataFile, registeredDirectives, location), lldbSpec?.let { OutputMatcher { output -> lldbSpec.checkLLDBOutput(output, settings.get()) } }, fileCheckMatcher = null, diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/AbstractLocalProcessRunner.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/AbstractLocalProcessRunner.kt index 080ce935dac..5c83f57621c 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/AbstractLocalProcessRunner.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/AbstractLocalProcessRunner.kt @@ -10,6 +10,7 @@ import com.intellij.openapi.util.text.StringUtilRt.convertLineSeparators import kotlinx.coroutines.* import org.jetbrains.kotlin.konan.target.Architecture import org.jetbrains.kotlin.konan.target.KonanTarget +import org.jetbrains.kotlin.konan.test.blackbox.support.TestCaseId import org.jetbrains.kotlin.konan.test.blackbox.support.runner.AbstractRunner.AbstractRun import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck.ExecutionTimeout import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck.ExitCode @@ -19,7 +20,6 @@ import org.jetbrains.kotlin.konan.test.blackbox.support.settings.OptimizationMod import org.jetbrains.kotlin.konan.test.blackbox.support.settings.configurables import org.jetbrains.kotlin.konan.test.blackbox.support.util.TestOutputFilter import org.jetbrains.kotlin.util.capitalizeDecapitalize.toUpperCaseAsciiOnly -import org.junit.jupiter.api.Assertions.fail import java.io.* import java.util.concurrent.atomic.AtomicBoolean import kotlin.time.* @@ -96,117 +96,123 @@ internal abstract class AbstractLocalProcessRunner(protected val checks: Test internal abstract class LocalResultHandler( runResult: RunResult, private val visibleProcessName: String, - protected val checks: TestRunChecks + protected val checks: TestRunChecks, + private val testCaseId: TestCaseId?, + private val expectedFailure: Boolean, ) : AbstractResultHandler(runResult) { override fun handle(): R { - checks.forEach { check -> - when (check) { - is ExecutionTimeout.ShouldNotExceed -> verifyExpectation(runResult.hasFinishedOnTime) { - "Timeout exceeded during test execution." - } - is ExecutionTimeout.ShouldExceed -> verifyExpectation(!runResult.hasFinishedOnTime) { - "Test is expected to fail with exceeded timeout, which hasn't happened." - } - is TestRunCheck.ExpectedFailure -> { - val testReport = runResult.processOutput.stdOut.testReport - verifyExpectation(testReport != null) { - "testReport is expected to be non-null" - } - verifyExpectation(!testReport!!.isEmpty()) { - "testReport is expected to be non-empty" - } - verifyExpectation(testReport.failedTests.isNotEmpty()) { - "Test did not fail as expected" - } - verifyExpectation(testReport.passedTests.isEmpty()) { - "Test unexpectedly passed" - } - } - is ExitCode -> { - // Don't check exit code if it is unknown. - val knownExitCode: Int = runResult.exitCode ?: return@forEach - when (check) { - is ExitCode.Expected -> verifyExpectation(knownExitCode == check.expectedExitCode) { - "$visibleProcessName exit code is $knownExitCode while ${check.expectedExitCode} was expected." - } - is ExitCode.AnyNonZero -> verifyExpectation(knownExitCode != 0) { - "$visibleProcessName exited with zero code, which wasn't expected." - } - } - } - is TestRunCheck.OutputDataFile -> { - val expectedOutput = check.file.readText() - val actualFilteredOutput = runResult.processOutput.stdOut.filteredOutput + runResult.processOutput.stdErr - - // Don't use verifyExpectation(expected, actual) to avoid exposing potentially large test output in exception message - // and blowing up test logs. - verifyExpectation(convertLineSeparators(expectedOutput) == convertLineSeparators(actualFilteredOutput)) { - "Tested process output mismatch. See \"TEST STDOUT\" and \"EXPECTED OUTPUT DATA FILE\" below." - } - } - is TestRunCheck.OutputMatcher -> { - try { - verifyExpectation(check.match(runResult.processOutput.stdOut.filteredOutput)) { - "Tested process output has not passed validation." - } - } catch (t: Throwable) { - if (t is Exception || t is AssertionError) { - fail( - getLoggedRun().withErrorMessage("Tested process output has not passed validation: " + t.message), - t + val diagnostics = buildList { + checks.forEach { check -> + when (check) { + is ExecutionTimeout.ShouldNotExceed -> if(!runResult.hasFinishedOnTime) add( + "Timeout exceeded during test execution." + ) + is ExecutionTimeout.ShouldExceed -> if(runResult.hasFinishedOnTime) add( + "Test is expected to fail with exceeded timeout, which hasn't happened." + ) + is ExitCode -> { + // Don't check exit code if it is unknown. + val knownExitCode: Int = runResult.exitCode ?: return@forEach + when (check) { + is ExitCode.Expected -> if (knownExitCode != check.expectedExitCode) add( + "$visibleProcessName exit code is $knownExitCode while ${check.expectedExitCode} was expected." + ) + is ExitCode.AnyNonZero -> if (knownExitCode == 0) add( + "$visibleProcessName exited with zero code, which wasn't expected." + ) + } + } + is TestRunCheck.OutputDataFile -> { + val expectedOutput = check.file.readText() + val actualFilteredOutput = runResult.processOutput.stdOut.filteredOutput + runResult.processOutput.stdErr + + // Don't use verifyExpectation(expected, actual) to avoid exposing potentially large test output in exception message + // and blowing up test logs. + if(convertLineSeparators(expectedOutput) != convertLineSeparators(actualFilteredOutput)) add( + "Tested process output mismatch. See \"TEST STDOUT\" and \"EXPECTED OUTPUT DATA FILE\" below." + ) + } + is TestRunCheck.OutputMatcher -> { + try { + if(!check.match(runResult.processOutput.stdOut.filteredOutput)) add( + "Tested process output has not passed validation." + ) + } catch (t: Throwable) { + if (t is Exception || t is AssertionError) { + add("Tested process output has not passed validation: " + t.message) + } else { + throw t + } + } + } + is TestRunCheck.FileCheckMatcher -> { + val fileCheckExecutable = check.settings.configurables.absoluteLlvmHome + File.separator + "bin" + File.separator + + if (SystemInfo.isWindows) "FileCheck.exe" else "FileCheck" + require(File(fileCheckExecutable).exists()) { + "$fileCheckExecutable does not exist. Make sure Distribution for `settings.configurables` " + + "was created using `propertyOverrides` to specify development variant of LLVM instead of user variant." + } + val fileCheckDump = runResult.testExecutable.executable.fileCheckDump!! + val fileCheckOut = File(fileCheckDump.absolutePath + ".out") + val fileCheckErr = File(fileCheckDump.absolutePath + ".err") + + val testTarget = check.settings.get().testTarget + val checkPrefixes = buildList { + add("CHECK") + add("CHECK-${testTarget.abiInfoString}") + add("CHECK-${testTarget.name.toUpperCaseAsciiOnly()}") + if (testTarget.family.isAppleFamily) { + add("CHECK-APPLE") + } + } + val optimizationMode = check.settings.get().name + val checkPrefixesWithOptMode = checkPrefixes.map { "$it-$optimizationMode" } + val commaSeparatedCheckPrefixes = (checkPrefixes + checkPrefixesWithOptMode).joinToString(",") + + val result = ProcessBuilder( + fileCheckExecutable, + check.testDataFile.absolutePath, + "--input-file", + fileCheckDump.absolutePath, + "--check-prefixes", commaSeparatedCheckPrefixes, + "--allow-deprecated-dag-overlap" // TODO specify it via new test directive for `function_attributes_at_callsite.kt` + ).redirectOutput(fileCheckOut) + .redirectError(fileCheckErr) + .start() + .waitFor() + val errText = fileCheckErr.readText() + val outText = fileCheckOut.readText() + if(!(result == 0 && errText.isEmpty() && outText.isEmpty())) { + val shortOutText = outText.lines().take(100) + val shortErrText = errText.lines().take(100) + add("FileCheck matching of ${fileCheckDump.absolutePath}\n" + + "with '--check-prefixes $commaSeparatedCheckPrefixes'\n" + + "failed with result=$result:\n" + + shortOutText.joinToString("\n") + "\n" + + shortErrText.joinToString("\n") ) - } else { - throw t } } } - is TestRunCheck.FileCheckMatcher -> { - val fileCheckExecutable = check.settings.configurables.absoluteLlvmHome + File.separator + "bin" + File.separator + - if (SystemInfo.isWindows) "FileCheck.exe" else "FileCheck" - require(File(fileCheckExecutable).exists()) { - "$fileCheckExecutable does not exist. Make sure Distribution for `settings.configurables` " + - "was created using `propertyOverrides` to specify development variant of LLVM instead of user variant." - } - val fileCheckDump = runResult.testExecutable.executable.fileCheckDump!! - val fileCheckOut = File(fileCheckDump.absolutePath + ".out") - val fileCheckErr = File(fileCheckDump.absolutePath + ".err") - - val testTarget = check.settings.get().testTarget - val checkPrefixes = buildList { - add("CHECK") - add("CHECK-${testTarget.abiInfoString}") - add("CHECK-${testTarget.name.toUpperCaseAsciiOnly()}") - if (testTarget.family.isAppleFamily) { - add("CHECK-APPLE") - } - } - val optimizationMode = check.settings.get().name - val checkPrefixesWithOptMode = checkPrefixes.map { "$it-$optimizationMode" } - val commaSeparatedCheckPrefixes = (checkPrefixes + checkPrefixesWithOptMode).joinToString(",") - - val result = ProcessBuilder( - fileCheckExecutable, - check.testDataFile.absolutePath, - "--input-file", - fileCheckDump.absolutePath, - "--check-prefixes", commaSeparatedCheckPrefixes, - "--allow-deprecated-dag-overlap" // TODO specify it via new test directive for `function_attributes_at_callsite.kt` - ).redirectOutput(fileCheckOut) - .redirectError(fileCheckErr) - .start() - .waitFor() - val errText = fileCheckErr.readText() - val outText = fileCheckOut.readText() - verifyExpectation(result == 0 && errText.isEmpty() && outText.isEmpty()) { - val shortOutText = outText.lines().take(100) - val shortErrText = errText.lines().take(100) - "FileCheck matching of ${fileCheckDump.absolutePath}\n" + - "with '--check-prefixes $commaSeparatedCheckPrefixes'\n" + - "failed with result=$result:\n" + - shortOutText.joinToString("\n") + "\n" + - shortErrText.joinToString("\n") - } + } + } + if (!expectedFailure) { + verifyExpectation(diagnostics.isEmpty()) { + diagnostics.joinToString("\n") + } + } else { + val runResultInfo = "TestCaseId: $testCaseId\nExit code: ${runResult.exitCode}\nFiltered test output is${ + runResult.processOutput.stdOut.filteredOutput.let { + if (it.isNotEmpty()) ":\n$it" else " empty." } + }" + verifyExpectation(diagnostics.isNotEmpty() || runResult.processOutput.stdOut.testReport?.failedTests?.isNotEmpty() == true) { + "Test did not fail as expected: $runResultInfo" + } + println("Test failed as expected.\n$runResultInfo") + if (diagnostics.isNotEmpty()) { + println("Diagnostics are:") + diagnostics.forEach(::println) } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/BaseTestRunProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/BaseTestRunProvider.kt index 8340ffd9282..5e5e203dfe3 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/BaseTestRunProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/BaseTestRunProvider.kt @@ -15,7 +15,7 @@ import org.jetbrains.kotlin.utils.addIfNotNull internal open class BaseTestRunProvider { protected fun createTestRun(testCase: TestCase, executable: TestExecutable, testRunName: String, testName: TestName?): TestRun { val runParameters = getTestRunParameters(testCase, testName) - return TestRun(displayName = testRunName, executable, runParameters, testCase.id, testCase.checks) + return TestRun(displayName = testRunName, executable, runParameters, testCase.id, testCase.checks, testCase.expectedFailure) } protected fun createSingleTestRun(testCase: TestCase, executable: TestExecutable): TestRun = createTestRun( diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/LocalTestNameExtractor.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/LocalTestNameExtractor.kt index 0e424b4f8be..6eafaec5d38 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/LocalTestNameExtractor.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/LocalTestNameExtractor.kt @@ -40,7 +40,7 @@ internal class TestNameResultHandler( visibleProcessName: String, checks: TestRunChecks, private val loggedParameters: LoggedData.TestRunParameters -) : LocalResultHandler>(runResult, visibleProcessName, checks) { +) : LocalResultHandler>(runResult, visibleProcessName, checks, null, false) { override fun getLoggedRun() = LoggedData.TestRun(loggedParameters, runResult) override fun doHandle() = GTestListing.parse(runResult.processOutput.stdOut.filteredOutput) } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/LocalTestRunner.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/LocalTestRunner.kt index f790d920a2f..762066886eb 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/LocalTestRunner.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/LocalTestRunner.kt @@ -59,7 +59,7 @@ internal class ResultHandler( checks: TestRunChecks, private val testRun: TestRun, private val loggedParameters: LoggedData.TestRunParameters -) : LocalResultHandler(runResult, visibleProcessName, checks) { +) : LocalResultHandler(runResult, visibleProcessName, checks, testRun.testCaseId, testRun.expectedFailure) { override fun getLoggedRun() = LoggedData.TestRun(loggedParameters, runResult) override fun doHandle() { @@ -83,17 +83,8 @@ internal class ResultHandler( ) } - if (checks.expectedFailureCheck !is TestRunCheck.ExpectedFailure) + if (!testRun.expectedFailure) verifyNoSuchTests(testReport.failedTests, "There are failed tests") - else { - runResult.processOutput.stdOut.filteredOutput.let { - if (it.isNotEmpty()) - println("Failure is expected. Exit code=${runResult.exitCode}, filtered test output is below:\n$it") - else - println("Failure is expected. Exit code=${runResult.exitCode}, filtered test output is empty.") - } - verifyNoSuchTests(testReport.passedTests, "There are unexpectedly passed tests") - } assumeFalse(testReport.ignoredTests.isNotEmpty() && testReport.passedTests.isEmpty(), "Test case is disabled") } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRun.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRun.kt index 868845dcbf3..b4696d9adf4 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRun.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRun.kt @@ -55,7 +55,8 @@ internal class TestRun( val executable: TestExecutable, val runParameters: List, val testCaseId: TestCaseId, - val checks: TestRunChecks + val checks: TestRunChecks, + val expectedFailure: Boolean, ) internal sealed interface TestRunParameter { diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunChecks.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunChecks.kt index a84e5e5e6e7..78c1128550f 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunChecks.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunChecks.kt @@ -22,8 +22,6 @@ internal sealed interface TestRunCheck { class Expected(val expectedExitCode: Int) : ExitCode() } - object ExpectedFailure : TestRunCheck - class OutputDataFile(val file: File) : TestRunCheck class OutputMatcher(val match: (String) -> Boolean): TestRunCheck @@ -34,7 +32,6 @@ internal sealed interface TestRunCheck { internal data class TestRunChecks( val executionTimeoutCheck: ExecutionTimeout, private val exitCodeCheck: ExitCode?, - val expectedFailureCheck: ExpectedFailure?, val outputDataFile: OutputDataFile?, val outputMatcher: OutputMatcher?, val fileCheckMatcher: FileCheckMatcher?, @@ -43,7 +40,6 @@ internal data class TestRunChecks( override fun iterator() = iterator { yield(executionTimeoutCheck) yieldIfNotNull(exitCodeCheck) - yieldIfNotNull(expectedFailureCheck) yieldIfNotNull(outputDataFile) yieldIfNotNull(outputMatcher) yieldIfNotNull(fileCheckMatcher) @@ -55,7 +51,6 @@ internal data class TestRunChecks( fun Default(timeout: Duration) = TestRunChecks( executionTimeoutCheck = ExecutionTimeout.ShouldNotExceed(timeout), exitCodeCheck = ExitCode.Expected(0), - expectedFailureCheck = null, outputDataFile = null, outputMatcher = null, fileCheckMatcher = null,