[Native][tests] Introduce EXPECTED_TIMEOUT_FAILURE directive

It allows to correctly handle tests that were failed by timeout where such failure is the expected behavior.
2022-04-05 23:16:14 +03:00
parent 7cf3ca29af
commit 261e177b39
21 changed files with 216 additions and 120 deletions
@@ -1,4 +1,5 @@
 // KIND: STANDALONE_NO_TR
+// EXPECTED_TIMEOUT_FAILURE

 import kotlin.math.E
 import kotlin.math.sqrt
@@ -1,4 +1,5 @@
 // KIND: STANDALONE_NO_TR
+// EXPECTED_TIMEOUT_FAILURE

 import kotlin.math.E
 import kotlin.math.sqrt
@@ -14,10 +14,8 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilati
 import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationDependencyType.Library
 import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationResult.Companion.assertSuccess
 import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestExecutable
-import org.jetbrains.kotlin.konan.blackboxtest.support.settings.CacheMode
-import org.jetbrains.kotlin.konan.blackboxtest.support.settings.KotlinNativeHome
-import org.jetbrains.kotlin.konan.blackboxtest.support.settings.KotlinNativeTargets
-import org.jetbrains.kotlin.konan.blackboxtest.support.settings.SimpleTestDirectories
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunChecks
+import org.jetbrains.kotlin.konan.blackboxtest.support.settings.*
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.*
 import org.junit.jupiter.api.Tag
 import java.io.File
@@ -119,6 +117,7 @@ abstract class AbstractNativeKlibABITest : AbstractNativeSimpleTest() {
        freeCompilerArgs = compilerArgs,
        nominalPackageName = PackageName.EMPTY,
        expectedOutputDataFile = null,
+        checks = TestRunChecks.Default(testRunSettings.get<Timeouts>().executionTimeout),
        extras = DEFAULT_EXTRAS
    ).apply {
        initialize(null)
@@ -21,9 +21,11 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilati
 import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationResult.Companion.assertSuccess
 import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationResult.Success
 import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestExecutable
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunChecks
 import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunners
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.KotlinNativeTargets
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.SimpleTestDirectories
+import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Timeouts
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.DumpedTestListing
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.LAUNCHER_MODULE_NAME
 import org.jetbrains.kotlin.test.TestMetadata
@@ -111,6 +113,7 @@ class InfrastructureDumpedTestListingTest : AbstractNativeSimpleTest() {
            freeCompilerArgs = TestCompilerArgs.EMPTY,
            nominalPackageName = PackageName.EMPTY,
            expectedOutputDataFile = null,
+            checks = TestRunChecks.Default(testRunSettings.get<Timeouts>().executionTimeout),
            extras = DEFAULT_EXTRAS
        ).apply {
            initialize(null)
@@ -152,11 +152,16 @@ internal abstract class LoggedData {

    class TestRun(
        private val parameters: TestRunParameters,
-        private val runResult: RunResult.Completed
+        private val runResult: RunResult
    ) : LoggedData() {
        override fun computeText() = buildString {
            appendLine("TEST RUN:")
-            appendCommonRunResult(runResult)
+            appendLine("- Exit code: ${runResult.exitCode ?: "<unknown>"}")
+            appendDuration(runResult.timeout, runResult.duration, runResult.hasFinishedOnTime)
+            appendLine()
+            appendPotentiallyLargeOutput(runResult.processOutput.stdOut.filteredOutput, "STDOUT", truncateLargeOutput = true)
+            appendLine()
+            appendPotentiallyLargeOutput(runResult.processOutput.stdErr, "STDERR", truncateLargeOutput = true)
            appendLine()
            appendLine(parameters)
        }
@@ -179,19 +184,6 @@ internal abstract class LoggedData {
        }
    }

-    class TestRunTimeoutExceeded(
-        private val parameters: TestRunParameters,
-        private val runResult: RunResult.TimeoutExceeded
-    ) : LoggedData() {
-        override fun computeText() = buildString {
-            appendLine("TIMED OUT:")
-            appendLine("- Max permitted duration: ${runResult.timeout}")
-            appendCommonRunResult(runResult)
-            appendLine()
-            appendLine(parameters)
-        }
-    }
-
    companion object {
        protected fun StringBuilder.appendList(header: String, list: Iterable<Any?>): StringBuilder {
            appendLine(header)
@@ -222,17 +214,14 @@ internal abstract class LoggedData {
            return this
        }

-        protected fun StringBuilder.appendDuration(duration: Duration): StringBuilder =
-            append("- Duration: ").appendLine(duration.toString(DurationUnit.SECONDS, 2))
+        protected fun StringBuilder.appendDuration(timeout: Duration, duration: Duration, hasFinishedOnTime: Boolean) {
+            append("- Max permitted duration: ").appendLine(timeout.toString(DurationUnit.SECONDS, 2))
+            appendDuration(duration)
+            appendLine("- Finished on time: $hasFinishedOnTime")
+        }

-        protected fun StringBuilder.appendCommonRunResult(runResult: RunResult): StringBuilder {
-            appendLine("- Exit code: ${runResult.exitCode ?: "<unknown>"}")
-            appendDuration(runResult.duration)
-            appendLine()
-            appendPotentiallyLargeOutput(runResult.processOutput.stdOut.filteredOutput, "STDOUT", truncateLargeOutput = true)
-            appendLine()
-            appendPotentiallyLargeOutput(runResult.processOutput.stdErr, "STDERR", truncateLargeOutput = true)
-            return this
+        protected fun StringBuilder.appendDuration(duration: Duration) {
+            append("- Duration: ").appendLine(duration.toString(DurationUnit.SECONDS, 2))
        }

        private fun StringBuilder.appendPotentiallyLargeOutput(output: String, subject: String, truncateLargeOutput: Boolean) {
@@ -32,7 +32,6 @@ import kotlin.reflect.KClass
 import kotlin.reflect.KParameter
 import kotlin.reflect.full.findAnnotation
 import kotlin.time.Duration.Companion.milliseconds
-import kotlin.time.Duration.Companion.seconds

 class NativeBlackBoxTestSupport : BeforeEachCallback {
    /**
@@ -218,7 +217,7 @@ private object NativeTestSupport {
        val executionTimeout = ClassLevelProperty.EXECUTION_TIMEOUT.readValue(
            enforcedProperties,
            { it.toLongOrNull()?.milliseconds },
-            default = 10.seconds
+            default = Timeouts.DEFAULT_EXECUTION_TIMEOUT
        )
        return Timeouts(executionTimeout)
    }
@@ -9,6 +9,7 @@ package org.jetbrains.kotlin.konan.blackboxtest.support

 import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.WithTestRunnerExtras
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestModule.Companion.allDependencies
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunChecks
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.*
 import org.jetbrains.kotlin.storage.LockBasedStorageManager
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue
@@ -169,7 +170,8 @@ internal class TestCase(
    val modules: Set<TestModule.Exclusive>,
    val freeCompilerArgs: TestCompilerArgs,
    val nominalPackageName: PackageName,
-    val expectedOutputDataFile: File?,
+    val expectedOutputDataFile: File?, // TODO: refactor to TestRunCheck
+    val checks: TestRunChecks,
    val extras: Extras
 ) {
    sealed interface Extras
@@ -10,6 +10,7 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.FREE_COMPI
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.INPUT_DATA_FILE
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.KIND
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.OUTPUT_DATA_FILE
+import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.EXPECTED_TIMEOUT_FAILURE
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.TEST_RUNNER
 import org.jetbrains.kotlin.test.directives.model.RegisteredDirectives
 import org.jetbrains.kotlin.test.directives.model.SimpleDirectivesContainer
@@ -67,11 +68,35 @@ internal object TestDirectives : SimpleDirectivesContainer() {

    val OUTPUT_DATA_FILE by stringDirective(
        description = """
-            Specify the file which contains the expected program output. When program finishes its execution the actual output (stdout)
+            Specify the file which contains the expected program output. When program finishes its execution, the actual output (stdout)
            will be compared to the contents of this file.
        """.trimIndent()
    )

+    // TODO: to be supported later
+//    val OUTPUT_REGEX by stringDirective(
+//        description = """
+//            The regex that the expected program output should match to. When program finishes its execution, the actual output (stdout)
+//            will be checked against this regex.
+//        """.trimIndent()
+//    )
+
+    // TODO: to be supported later
+//    val OUTPUT_INCLUDES by stringDirective(
+//        description = """
+//            The text that the expected program output should contain. When program finishes its execution, it will be checked if
+//            the actual output (stdout) contains this text.
+//        """.trimIndent()
+//    )
+
+    // TODO: to be supported later
+//    val OUTPUT_NOT_INCLUDES by stringDirective(
+//        description = """
+//            The text that the expected program output should NOT contain. When program finishes its execution, it will be checked if
+//            the actual output (stdout) does nto contain this text.
+//        """.trimIndent()
+//    )
+
    val INPUT_DATA_FILE by stringDirective(
        description = """
            Specify the file which contains the text to be passed to process' input (stdin).
@@ -79,6 +104,19 @@ internal object TestDirectives : SimpleDirectivesContainer() {
        """.trimIndent()
    )

+    // TODO: to be supported later
+//    val EXIT_CODE by stringDirective(
+//        description = """
+//            Specify the exit code that the test should finish with. Example: // EXIT_CODE: 42
+//            To indicate any non-zero exit code use // EXIT_CODE: !0
+//            Note that this directive makes sense only in combination with // KIND: STANDALONE_NO_TR
+//        """.trimIndent()
+//    )
+
+    val EXPECTED_TIMEOUT_FAILURE by directive(
+        description = "Whether the test is expected to fail on timeout"
+    )
+
    val FREE_COMPILER_ARGS by stringDirective(
        description = "Specify free compiler arguments for Kotlin/Native compiler"
    )
@@ -203,6 +241,9 @@ internal fun parseFileName(parsedDirective: RegisteredDirectivesParser.ParsedDir
    return fileName
 }

+internal fun parseExpectedTimeoutFailure(registeredDirectives: RegisteredDirectives): Boolean =
+    EXPECTED_TIMEOUT_FAILURE in registeredDirectives
+
 internal fun parseFreeCompilerArgs(registeredDirectives: RegisteredDirectives, location: Location): TestCompilerArgs {
    if (FREE_COMPILER_ARGS !in registeredDirectives)
        return TestCompilerArgs.EMPTY
@@ -19,10 +19,8 @@ import org.jetbrains.kotlin.config.CommonConfigurationKeys
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.konan.blackboxtest.support.*
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.WithTestRunnerExtras
-import org.jetbrains.kotlin.konan.blackboxtest.support.settings.ExternalSourceTransformersProvider
-import org.jetbrains.kotlin.konan.blackboxtest.support.settings.GeneratedSources
-import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Settings
-import org.jetbrains.kotlin.konan.blackboxtest.support.settings.TestRoots
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunChecks
+import org.jetbrains.kotlin.konan.blackboxtest.support.settings.*
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.*
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.Name
@@ -69,7 +67,8 @@ internal class ExtTestCaseGroupProvider : TestCaseGroupProvider, TestDisposable(
                    structureFactory = structureFactory,
                    customSourceTransformers = settings.get<ExternalSourceTransformersProvider>().getSourceTransformers(testDataFile),
                    testRoots = settings.get(),
-                    generatedSources = settings.get()
+                    generatedSources = settings.get(),
+                    timeouts = settings.get()
                )

                if (extTestDataFile.isRelevant)
@@ -102,7 +101,8 @@ private class ExtTestDataFile(
    structureFactory: ExtTestDataFileStructureFactory,
    customSourceTransformers: ExternalSourceTransformers?,
    testRoots: TestRoots,
-    private val generatedSources: GeneratedSources
+    private val generatedSources: GeneratedSources,
+    private val timeouts: Timeouts
 ) {
    private val structure by lazy {
        val allSourceTransformers: ExternalSourceTransformers = if (customSourceTransformers.isNullOrEmpty())
@@ -554,6 +554,7 @@ private class ExtTestDataFile(
            freeCompilerArgs = assembleFreeCompilerArgs(),
            nominalPackageName = testDataFileSettings.nominalPackageName,
            expectedOutputDataFile = null,
+            checks = TestRunChecks.Default(timeouts.executionTimeout),
            extras = WithTestRunnerExtras(runnerType = TestRunnerType.DEFAULT)
        )
        testCase.initialize(sharedModules::get)
@@ -7,8 +7,10 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.group

 import org.jetbrains.kotlin.konan.blackboxtest.support.*
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.WithTestRunnerExtras
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunChecks
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.KotlinNativeHome
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Settings
+import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Timeouts
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.ThreadSafeCache
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.expandGlobTo
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.getAbsoluteFile
@@ -62,6 +64,7 @@ internal class PredefinedTestCaseGroupProvider(annotation: PredefinedTestCases)
                    .parseCompilerArgs(settings) { "Failed to parse free compiler arguments for test case $testCaseId" },
                nominalPackageName = PackageName(testCaseId.uniqueName),
                expectedOutputDataFile = null,
+                checks = TestRunChecks.Default(settings.get<Timeouts>().executionTimeout),
                extras = WithTestRunnerExtras(
                    runnerType = predefinedTestCase.runnerType,
                    ignoredTests = predefinedTestCase.ignoredTests.toSet()
@@ -8,9 +8,12 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.group
 import org.jetbrains.kotlin.konan.blackboxtest.support.*
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.NoTestRunnerExtras
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.WithTestRunnerExtras
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunChecks
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.GeneratedSources
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Settings
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.TestRoots
+import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Timeouts
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.*
 import org.jetbrains.kotlin.test.directives.model.Directive
 import org.jetbrains.kotlin.test.services.JUnit5Assertions
@@ -180,6 +183,14 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider {
        val freeCompilerArgs = parseFreeCompilerArgs(registeredDirectives, location)
        val expectedOutputDataFile = parseOutputDataFile(baseDir = testDataFile.parentFile, registeredDirectives, location)
        val testKind = parseTestKind(registeredDirectives, location)
+        val expectedTimeoutFailure = parseExpectedTimeoutFailure(registeredDirectives)
+
+        val checks = TestRunChecks {
+            this += if (expectedTimeoutFailure)
+                TestRunCheck.ExecutionTimeout.ShouldExceed(settings.get<Timeouts>().executionTimeout)
+            else
+                TestRunCheck.ExecutionTimeout.ShouldNotExceed(settings.get<Timeouts>().executionTimeout)
+        }

        if (testKind == TestKind.REGULAR) {
            // Fix package declarations to avoid unintended conflicts between symbols with the same name in different test cases.
@@ -193,6 +204,7 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider {
            freeCompilerArgs = freeCompilerArgs,
            nominalPackageName = nominalPackageName,
            expectedOutputDataFile = expectedOutputDataFile,
+            checks = checks,
            extras = if (testKind == TestKind.STANDALONE_NO_TR)
                NoTestRunnerExtras(
                    entryPoint = parseEntryPoint(registeredDirectives, location),
@@ -12,7 +12,7 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestOutputFilter
 import java.io.ByteArrayOutputStream
 import kotlin.time.*

-internal abstract class AbstractLocalProcessRunner<R>(private val executionTimeout: Duration) : AbstractRunner<R>() {
+internal abstract class AbstractLocalProcessRunner<R>(private val checks: TestRunChecks) : AbstractRunner<R>() {
    protected abstract val visibleProcessName: String
    protected abstract val executable: TestExecutable
    protected abstract val programArgs: List<String>
@@ -26,6 +26,8 @@ internal abstract class AbstractLocalProcessRunner<R>(private val executionTimeo
            val unfilteredOutput = UnfilteredProcessOutput()
            val unfilteredOutputReader: Job

+            val executionTimeout: Duration = checks.executionTimeoutCheck.timeout
+
            val process: Process
            val hasFinishedOnTime: Boolean

@@ -41,16 +43,11 @@ internal abstract class AbstractLocalProcessRunner<R>(private val executionTimeo
                )
            }

-            if (hasFinishedOnTime) {
+            val exitCode: Int? = if (hasFinishedOnTime) {
                unfilteredOutputReader.join() // Wait until all output streams are drained.
-
-                RunResult.Completed(
-                    exitCode = process.exitValue(),
-                    duration = duration,
-                    processOutput = unfilteredOutput.toProcessOutput(outputFilter)
-                )
+                process.exitValue()
            } else {
-                val exitCode: Int? = try { // It could happen just by an accident that the process has exited by itself.
+                try { // It could happen just by an accident that the process has exited by itself.
                    val exitCode = process.exitValue() // Fetch exit code.
                    unfilteredOutputReader.join() // Wait until all streams are drained.
                    exitCode
@@ -59,22 +56,37 @@ internal abstract class AbstractLocalProcessRunner<R>(private val executionTimeo
                    process.destroyForcibly() // kill -9
                    null
                }
-
-                RunResult.TimeoutExceeded(
-                    timeout = executionTimeout,
-                    exitCode = exitCode,
-                    duration = duration,
-                    processOutput = unfilteredOutput.toProcessOutput(outputFilter)
-                )
            }
+
+            RunResult(
+                exitCode = exitCode,
+                timeout = executionTimeout,
+                duration = duration,
+                hasFinishedOnTime = hasFinishedOnTime,
+                processOutput = unfilteredOutput.toProcessOutput(outputFilter)
+            )
        }
    }

-    abstract override fun buildResultHandler(runResult: RunResult.Completed): ResultHandler // Narrow returned type.
+    abstract override fun buildResultHandler(runResult: RunResult): ResultHandler // Narrow returned type.

-    abstract inner class ResultHandler(runResult: RunResult.Completed) : AbstractRunner<R>.ResultHandler(runResult) {
+    abstract inner class ResultHandler(runResult: RunResult) : AbstractRunner<R>.ResultHandler(runResult) {
        override fun handle(): R {
-            verifyExpectation(runResult.exitCode == 0) { "$visibleProcessName exited with non-zero code." }
+            checks.forEach { check ->
+                when (check) {
+                    is TestRunCheck.ExecutionTimeout.ShouldNotExceed -> verifyExpectation(runResult.hasFinishedOnTime) {
+                        "Timeout exceeded during test execution."
+                    }
+                    is TestRunCheck.ExecutionTimeout.ShouldExceed -> verifyExpectation(!runResult.hasFinishedOnTime) {
+                        "Test is expected to fail with exceeded timeout, which hasn't happened."
+                    }
+                }
+            }
+
+            // Don't check exit code if it is unknown.
+            runResult.exitCode?.let { exitCode ->
+                verifyExpectation(exitCode == 0) { "$visibleProcessName exited with non-zero code." }
+            }

            return doHandle()
        }
@@ -7,26 +7,18 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.runner

 import org.jetbrains.kotlin.konan.blackboxtest.support.LoggedData
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue
-import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
 import org.opentest4j.TestAbortedException

 internal abstract class AbstractRunner<R> {
    protected abstract fun buildRun(): AbstractRun
-    protected abstract fun buildResultHandler(runResult: RunResult.Completed): ResultHandler
+    protected abstract fun buildResultHandler(runResult: RunResult): ResultHandler
    protected abstract fun getLoggedParameters(): LoggedData.TestRunParameters
    protected abstract fun handleUnexpectedFailure(t: Throwable): Nothing

    fun run(): R = try {
        val run = buildRun()
-
-        val resultHandler = when (val runResult = run.run()) {
-            is RunResult.TimeoutExceeded -> fail {
-                LoggedData.TestRunTimeoutExceeded(getLoggedParameters(), runResult)
-                    .withErrorMessage("Timeout exceeded during test execution.")
-            }
-            is RunResult.Completed -> buildResultHandler(runResult)
-        }
-
+        val runResult = run.run()
+        val resultHandler = buildResultHandler(runResult)
        resultHandler.handle()
    } catch (t: Throwable) {
        when (t) {
@@ -42,7 +34,7 @@ internal abstract class AbstractRunner<R> {
        fun run(): RunResult
    }

-    abstract inner class ResultHandler(protected val runResult: RunResult.Completed) {
+    abstract inner class ResultHandler(protected val runResult: RunResult) {
        abstract fun getLoggedRun(): LoggedData
        abstract fun handle(): R

@@ -9,12 +9,13 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.NoTestRunnerExtras
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestKind
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestName
-import org.jetbrains.kotlin.test.services.JUnit5Assertions
+import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue
+import org.jetbrains.kotlin.utils.addIfNotNull

 internal open class BaseTestRunProvider {
    protected fun createTestRun(testCase: TestCase, executable: TestExecutable, testRunName: String, testName: TestName?): TestRun {
        val runParameters = getTestRunParameters(testCase, testName)
-        return TestRun(displayName = testRunName, executable, runParameters, testCase.id)
+        return TestRun(displayName = testRunName, executable, runParameters, testCase.id, testCase.checks)
    }

    protected fun createSingleTestRun(testCase: TestCase, executable: TestExecutable): TestRun = createTestRun(
@@ -24,25 +25,24 @@ internal open class BaseTestRunProvider {
        testName = null
    )

-    private fun getTestRunParameters(testCase: TestCase, testName: TestName?): List<TestRunParameter> = with(testCase) {
-        when (kind) {
+    private fun getTestRunParameters(testCase: TestCase, testName: TestName?): List<TestRunParameter> = buildList {
+        addIfNotNull(testCase.expectedOutputDataFile?.let(TestRunParameter::WithExpectedOutputData))
+
+        when (testCase.kind) {
            TestKind.STANDALONE_NO_TR -> {
-                JUnit5Assertions.assertTrue(testName == null)
-                listOfNotNull(
-                    extras<NoTestRunnerExtras>().inputDataFile?.let(TestRunParameter::WithInputData),
-                    expectedOutputDataFile?.let(TestRunParameter::WithExpectedOutputData)
+                assertTrue(testName == null)
+                addIfNotNull(testCase.extras<NoTestRunnerExtras>().inputDataFile?.let(TestRunParameter::WithInputData))
+            }
+            TestKind.STANDALONE -> {
+                add(TestRunParameter.WithTCTestLogger)
+                addIfNotNull(testName?.let(TestRunParameter::WithTestFilter))
+            }
+            TestKind.REGULAR -> {
+                add(TestRunParameter.WithTCTestLogger)
+                addIfNotNull(
+                    testName?.let(TestRunParameter::WithTestFilter) ?: TestRunParameter.WithPackageFilter(testCase.nominalPackageName)
                )
            }
-            TestKind.STANDALONE -> listOfNotNull(
-                TestRunParameter.WithTCTestLogger,
-                testName?.let(TestRunParameter::WithTestFilter),
-                expectedOutputDataFile?.let(TestRunParameter::WithExpectedOutputData)
-            )
-            TestKind.REGULAR -> listOfNotNull(
-                TestRunParameter.WithTCTestLogger,
-                testName?.let(TestRunParameter::WithTestFilter) ?: TestRunParameter.WithPackageFilter(nominalPackageName),
-                expectedOutputDataFile?.let(TestRunParameter::WithExpectedOutputData)
-            )
        }
    }
 }
@@ -10,12 +10,11 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.TestName
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.GTestListing
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestOutputFilter
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
-import kotlin.time.Duration

 internal class LocalTestNameExtractor(
    override val executable: TestExecutable,
-    executionTimeout: Duration
-) : AbstractLocalProcessRunner<Collection<TestName>>(executionTimeout) {
+    checks: TestRunChecks
+) : AbstractLocalProcessRunner<Collection<TestName>>(checks) {
    override val visibleProcessName get() = "Test name extractor"
    override val programArgs = listOf(executable.executableFile.path, "--ktest_list_tests")
    override val outputFilter get() = TestOutputFilter.NO_FILTERING
@@ -27,7 +26,7 @@ internal class LocalTestNameExtractor(
        runParameters = null
    )

-    override fun buildResultHandler(runResult: RunResult.Completed) = ResultHandler(runResult)
+    override fun buildResultHandler(runResult: RunResult) = ResultHandler(runResult)

    override fun handleUnexpectedFailure(t: Throwable) = fail {
        LoggedData.TestRunUnexpectedFailure(getLoggedParameters(), t)
@@ -35,7 +34,7 @@ internal class LocalTestNameExtractor(
    }

    inner class ResultHandler(
-        runResult: RunResult.Completed
+        runResult: RunResult
    ) : AbstractLocalProcessRunner<Collection<TestName>>.ResultHandler(runResult) {
        override fun getLoggedRun() = LoggedData.TestRun(getLoggedParameters(), runResult)
        override fun doHandle() = GTestListing.parse(runResult.processOutput.stdOut.filteredOutput)
@@ -12,12 +12,8 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestOutputFilter
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestReport
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
 import org.junit.jupiter.api.Assumptions.assumeFalse
-import kotlin.time.Duration

-internal class LocalTestRunner(
-    private val testRun: TestRun,
-    executionTimeout: Duration
-) : AbstractLocalProcessRunner<Unit>(executionTimeout) {
+internal class LocalTestRunner(private val testRun: TestRun) : AbstractLocalProcessRunner<Unit>(testRun.checks) {
    override val visibleProcessName get() = "Tested process"
    override val executable get() = testRun.executable

@@ -43,14 +39,14 @@ internal class LocalTestRunner(
        }
    }

-    override fun buildResultHandler(runResult: RunResult.Completed) = ResultHandler(runResult)
+    override fun buildResultHandler(runResult: RunResult) = ResultHandler(runResult)

    override fun handleUnexpectedFailure(t: Throwable) = fail {
        LoggedData.TestRunUnexpectedFailure(getLoggedParameters(), t)
            .withErrorMessage("Test execution failed with unexpected exception.")
    }

-    inner class ResultHandler(runResult: RunResult.Completed) : AbstractLocalProcessRunner<Unit>.ResultHandler(runResult) {
+    inner class ResultHandler(runResult: RunResult) : AbstractLocalProcessRunner<Unit>.ResultHandler(runResult) {
        override fun getLoggedRun() = LoggedData.TestRun(getLoggedParameters(), runResult)

        override fun doHandle() {
@@ -8,23 +8,17 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.runner
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestOutputFilter
 import kotlin.time.Duration

-internal sealed interface RunResult {
-    val exitCode: Int?
-    val duration: Duration
+internal data class RunResult(
+    val exitCode: Int?,
+    val timeout: Duration,
+    val duration: Duration,
+    val hasFinishedOnTime: Boolean,
    val processOutput: ProcessOutput
-
-    data class Completed(
-        override val exitCode: Int,
-        override val duration: Duration,
-        override val processOutput: ProcessOutput
-    ) : RunResult
-
-    data class TimeoutExceeded(
-        val timeout: Duration,
-        override val exitCode: Int?,
-        override val duration: Duration,
-        override val processOutput: ProcessOutput
-    ) : RunResult
+) {
+    init {
+        // null exit code is possible only when test run hasn't finished on time.
+        check(exitCode != null || !hasFinishedOnTime)
+    }
 }

 internal class ProcessOutput(val stdOut: TestOutputFilter.FilteredOutput, val stdErr: String)
@@ -54,7 +54,8 @@ internal class TestRun(
    val displayName: String,
    val executable: TestExecutable,
    val runParameters: List<TestRunParameter>,
-    val testCaseId: TestCaseId
+    val testCaseId: TestCaseId,
+    val checks: TestRunChecks
 )

 internal sealed interface TestRunParameter {
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.konan.blackboxtest.support.runner
+
+import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Timeouts
+import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstance
+import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull
+import kotlin.time.Duration
+
+internal sealed interface TestRunCheck {
+    sealed class ExecutionTimeout(val timeout: Duration) : TestRunCheck {
+        class ShouldNotExceed(timeout: Duration) : ExecutionTimeout(timeout)
+        class ShouldExceed(timeout: Duration) : ExecutionTimeout(timeout)
+    }
+}
+
+internal class TestRunChecks(builderAction: MutableCollection<TestRunCheck>.() -> Unit) : Iterable<TestRunCheck> {
+    constructor(vararg checks: TestRunCheck) : this({ addAll(checks) })
+
+    private val allChecks = buildList {
+        builderAction()
+        addDefaults()
+    }
+
+    val executionTimeoutCheck: TestRunCheck.ExecutionTimeout get() = allChecks.firstIsInstance()
+
+    override fun iterator() = allChecks.iterator()
+
+    companion object {
+        // The most frequently used case:
+        @Suppress("TestFunctionName")
+        fun Default(timeout: Duration) = TestRunChecks(TestRunCheck.ExecutionTimeout.ShouldNotExceed(timeout))
+
+        private fun MutableCollection<TestRunCheck>.addDefaults() {
+            if (isMissing<TestRunCheck.ExecutionTimeout>()) {
+                add(TestRunCheck.ExecutionTimeout.ShouldNotExceed(Timeouts.DEFAULT_EXECUTION_TIMEOUT))
+            }
+        }
+
+        private inline fun <reified T : TestRunCheck> Collection<TestRunCheck>.isMissing(): Boolean =
+            firstIsInstanceOrNull<T>() == null
+    }
+}
@@ -16,7 +16,7 @@ internal object TestRunners {
    fun createProperTestRunner(testRun: TestRun, settings: Settings): AbstractRunner<*> = with(settings) {
        with(get<KotlinNativeTargets>()) {
            if (testTarget == hostTarget)
-                LocalTestRunner(testRun, get<Timeouts>().executionTimeout)
+                LocalTestRunner(testRun)
            else
                runningAtNonHostTarget()
        }
@@ -26,7 +26,7 @@ internal object TestRunners {
    fun extractTestNames(executable: TestExecutable, settings: Settings): Collection<TestName> = with(settings) {
        with(get<KotlinNativeTargets>()) {
            if (testTarget == hostTarget)
-                LocalTestNameExtractor(executable, get<Timeouts>().executionTimeout).run()
+                LocalTestNameExtractor(executable, TestRunChecks.Default(get<Timeouts>().executionTimeout)).run()
            else
                runningAtNonHostTarget()
        }
@@ -9,6 +9,7 @@ import org.jetbrains.kotlin.konan.target.KonanTarget
 import java.io.File
 import java.util.*
 import kotlin.time.Duration
+import kotlin.time.Duration.Companion.seconds

 /**
 * The tested and the host Kotlin/Native targets.
@@ -113,7 +114,11 @@ internal class BaseDirs(val testBuildDir: File)
 /**
 * Timeouts.
 */
-internal class Timeouts(val executionTimeout: Duration)
+internal class Timeouts(val executionTimeout: Duration) {
+    companion object {
+        val DEFAULT_EXECUTION_TIMEOUT: Duration get() = 15.seconds
+    }
+}

 /**
 * Used cache mode.