[Native][tests] Introduce EXPECTED_TIMEOUT_FAILURE directive

It allows to correctly handle tests that were failed by timeout where such failure is the expected behavior.

native/native.tests/testData/samples/standalone_notr_long_running.kt

@@ -1,4 +1,5 @@
 // KIND: STANDALONE_NO_TR
+// EXPECTED_TIMEOUT_FAILURE
 
 import kotlin.math.E
 import kotlin.math.sqrt

native/native.tests/testData/samples/standalone_notr_long_running_and_verbose.kt

@@ -1,4 +1,5 @@
 // KIND: STANDALONE_NO_TR
+// EXPECTED_TIMEOUT_FAILURE
 
 import kotlin.math.E
 import kotlin.math.sqrt

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/AbstractNativeKlibABITest.kt

@@ -14,10 +14,8 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilati
 import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationDependencyType.Library
 import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationResult.Companion.assertSuccess
 import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestExecutable
-import org.jetbrains.kotlin.konan.blackboxtest.support.settings.CacheMode
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunChecks
-import org.jetbrains.kotlin.konan.blackboxtest.support.settings.KotlinNativeHome
+import org.jetbrains.kotlin.konan.blackboxtest.support.settings.*
-import org.jetbrains.kotlin.konan.blackboxtest.support.settings.KotlinNativeTargets
-import org.jetbrains.kotlin.konan.blackboxtest.support.settings.SimpleTestDirectories
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.*
 import org.junit.jupiter.api.Tag
 import java.io.File
@@ -119,6 +117,7 @@ abstract class AbstractNativeKlibABITest : AbstractNativeSimpleTest() {
         freeCompilerArgs = compilerArgs,
         nominalPackageName = PackageName.EMPTY,
         expectedOutputDataFile = null,
+        checks = TestRunChecks.Default(testRunSettings.get<Timeouts>().executionTimeout),
         extras = DEFAULT_EXTRAS
     ).apply {
         initialize(null)

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/InfrastructureDumpedTestListingTest.kt

@@ -21,9 +21,11 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilati
 import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationResult.Companion.assertSuccess
 import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationResult.Success
 import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestExecutable
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunChecks
 import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunners
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.KotlinNativeTargets
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.SimpleTestDirectories
+import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Timeouts
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.DumpedTestListing
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.LAUNCHER_MODULE_NAME
 import org.jetbrains.kotlin.test.TestMetadata
@@ -111,6 +113,7 @@ class InfrastructureDumpedTestListingTest : AbstractNativeSimpleTest() {
             freeCompilerArgs = TestCompilerArgs.EMPTY,
             nominalPackageName = PackageName.EMPTY,
             expectedOutputDataFile = null,
+            checks = TestRunChecks.Default(testRunSettings.get<Timeouts>().executionTimeout),
             extras = DEFAULT_EXTRAS
         ).apply {
             initialize(null)

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/LoggedData.kt

@@ -152,11 +152,16 @@ internal abstract class LoggedData {
 
     class TestRun(
         private val parameters: TestRunParameters,
-        private val runResult: RunResult.Completed
+        private val runResult: RunResult
     ) : LoggedData() {
         override fun computeText() = buildString {
             appendLine("TEST RUN:")
-            appendCommonRunResult(runResult)
+            appendLine("- Exit code: ${runResult.exitCode ?: "<unknown>"}")
+            appendDuration(runResult.timeout, runResult.duration, runResult.hasFinishedOnTime)
+            appendLine()
+            appendPotentiallyLargeOutput(runResult.processOutput.stdOut.filteredOutput, "STDOUT", truncateLargeOutput = true)
+            appendLine()
+            appendPotentiallyLargeOutput(runResult.processOutput.stdErr, "STDERR", truncateLargeOutput = true)
             appendLine()
             appendLine(parameters)
         }
@@ -179,19 +184,6 @@ internal abstract class LoggedData {
         }
     }
 
-    class TestRunTimeoutExceeded(
-        private val parameters: TestRunParameters,
-        private val runResult: RunResult.TimeoutExceeded
-    ) : LoggedData() {
-        override fun computeText() = buildString {
-            appendLine("TIMED OUT:")
-            appendLine("- Max permitted duration: ${runResult.timeout}")
-            appendCommonRunResult(runResult)
-            appendLine()
-            appendLine(parameters)
-        }
-    }
-
     companion object {
         protected fun StringBuilder.appendList(header: String, list: Iterable<Any?>): StringBuilder {
             appendLine(header)
@@ -222,17 +214,14 @@ internal abstract class LoggedData {
             return this
         }
 
-        protected fun StringBuilder.appendDuration(duration: Duration): StringBuilder =
+        protected fun StringBuilder.appendDuration(timeout: Duration, duration: Duration, hasFinishedOnTime: Boolean) {
-            append("- Duration: ").appendLine(duration.toString(DurationUnit.SECONDS, 2))
+            append("- Max permitted duration: ").appendLine(timeout.toString(DurationUnit.SECONDS, 2))
+            appendDuration(duration)
+            appendLine("- Finished on time: $hasFinishedOnTime")
+        }
 
-        protected fun StringBuilder.appendCommonRunResult(runResult: RunResult): StringBuilder {
+        protected fun StringBuilder.appendDuration(duration: Duration) {
-            appendLine("- Exit code: ${runResult.exitCode ?: "<unknown>"}")
+            append("- Duration: ").appendLine(duration.toString(DurationUnit.SECONDS, 2))
-            appendDuration(runResult.duration)
-            appendLine()
-            appendPotentiallyLargeOutput(runResult.processOutput.stdOut.filteredOutput, "STDOUT", truncateLargeOutput = true)
-            appendLine()
-            appendPotentiallyLargeOutput(runResult.processOutput.stdErr, "STDERR", truncateLargeOutput = true)
-            return this
         }
 
         private fun StringBuilder.appendPotentiallyLargeOutput(output: String, subject: String, truncateLargeOutput: Boolean) {

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/NativeTestSupport.kt

@@ -32,7 +32,6 @@ import kotlin.reflect.KClass
 import kotlin.reflect.KParameter
 import kotlin.reflect.full.findAnnotation
 import kotlin.time.Duration.Companion.milliseconds
-import kotlin.time.Duration.Companion.seconds
 
 class NativeBlackBoxTestSupport : BeforeEachCallback {
     /**
@@ -218,7 +217,7 @@ private object NativeTestSupport {
         val executionTimeout = ClassLevelProperty.EXECUTION_TIMEOUT.readValue(
             enforcedProperties,
             { it.toLongOrNull()?.milliseconds },
-            default = 10.seconds
+            default = Timeouts.DEFAULT_EXECUTION_TIMEOUT
         )
         return Timeouts(executionTimeout)
     }

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCase.kt

@@ -9,6 +9,7 @@ package org.jetbrains.kotlin.konan.blackboxtest.support
 
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.WithTestRunnerExtras
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestModule.Companion.allDependencies
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunChecks
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.*
 import org.jetbrains.kotlin.storage.LockBasedStorageManager
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue
@@ -169,7 +170,8 @@ internal class TestCase(
     val modules: Set<TestModule.Exclusive>,
     val freeCompilerArgs: TestCompilerArgs,
     val nominalPackageName: PackageName,
-    val expectedOutputDataFile: File?,
+    val expectedOutputDataFile: File?, // TODO: refactor to TestRunCheck
+    val checks: TestRunChecks,
     val extras: Extras
 ) {
     sealed interface Extras

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestDirectives.kt

@@ -10,6 +10,7 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.FREE_COMPI
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.INPUT_DATA_FILE
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.KIND
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.OUTPUT_DATA_FILE
+import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.EXPECTED_TIMEOUT_FAILURE
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.TEST_RUNNER
 import org.jetbrains.kotlin.test.directives.model.RegisteredDirectives
 import org.jetbrains.kotlin.test.directives.model.SimpleDirectivesContainer
@@ -67,11 +68,35 @@ internal object TestDirectives : SimpleDirectivesContainer() {
 
     val OUTPUT_DATA_FILE by stringDirective(
         description = """
-            Specify the file which contains the expected program output. When program finishes its execution the actual output (stdout)
+            Specify the file which contains the expected program output. When program finishes its execution, the actual output (stdout)
             will be compared to the contents of this file.
         """.trimIndent()
     )
 
+    // TODO: to be supported later
+//    val OUTPUT_REGEX by stringDirective(
+//        description = """
+//            The regex that the expected program output should match to. When program finishes its execution, the actual output (stdout)
+//            will be checked against this regex.
+//        """.trimIndent()
+//    )
+
+    // TODO: to be supported later
+//    val OUTPUT_INCLUDES by stringDirective(
+//        description = """
+//            The text that the expected program output should contain. When program finishes its execution, it will be checked if
+//            the actual output (stdout) contains this text.
+//        """.trimIndent()
+//    )
+
+    // TODO: to be supported later
+//    val OUTPUT_NOT_INCLUDES by stringDirective(
+//        description = """
+//            The text that the expected program output should NOT contain. When program finishes its execution, it will be checked if
+//            the actual output (stdout) does nto contain this text.
+//        """.trimIndent()
+//    )
+
     val INPUT_DATA_FILE by stringDirective(
         description = """
             Specify the file which contains the text to be passed to process' input (stdin).
@@ -79,6 +104,19 @@ internal object TestDirectives : SimpleDirectivesContainer() {
         """.trimIndent()
     )
 
+    // TODO: to be supported later
+//    val EXIT_CODE by stringDirective(
+//        description = """
+//            Specify the exit code that the test should finish with. Example: // EXIT_CODE: 42
+//            To indicate any non-zero exit code use // EXIT_CODE: !0
+//            Note that this directive makes sense only in combination with // KIND: STANDALONE_NO_TR
+//        """.trimIndent()
+//    )
+
+    val EXPECTED_TIMEOUT_FAILURE by directive(
+        description = "Whether the test is expected to fail on timeout"
+    )
+
     val FREE_COMPILER_ARGS by stringDirective(
         description = "Specify free compiler arguments for Kotlin/Native compiler"
     )
@@ -203,6 +241,9 @@ internal fun parseFileName(parsedDirective: RegisteredDirectivesParser.ParsedDir
     return fileName
 }
 
+internal fun parseExpectedTimeoutFailure(registeredDirectives: RegisteredDirectives): Boolean =
+    EXPECTED_TIMEOUT_FAILURE in registeredDirectives
+
 internal fun parseFreeCompilerArgs(registeredDirectives: RegisteredDirectives, location: Location): TestCompilerArgs {
     if (FREE_COMPILER_ARGS !in registeredDirectives)
         return TestCompilerArgs.EMPTY

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/ExtTestCaseGroupProvider.kt

@@ -19,10 +19,8 @@ import org.jetbrains.kotlin.config.CommonConfigurationKeys
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.konan.blackboxtest.support.*
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.WithTestRunnerExtras
-import org.jetbrains.kotlin.konan.blackboxtest.support.settings.ExternalSourceTransformersProvider
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunChecks
-import org.jetbrains.kotlin.konan.blackboxtest.support.settings.GeneratedSources
+import org.jetbrains.kotlin.konan.blackboxtest.support.settings.*
-import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Settings
-import org.jetbrains.kotlin.konan.blackboxtest.support.settings.TestRoots
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.*
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.Name
@@ -69,7 +67,8 @@ internal class ExtTestCaseGroupProvider : TestCaseGroupProvider, TestDisposable(
                     structureFactory = structureFactory,
                     customSourceTransformers = settings.get<ExternalSourceTransformersProvider>().getSourceTransformers(testDataFile),
                     testRoots = settings.get(),
-                    generatedSources = settings.get()
+                    generatedSources = settings.get(),
+                    timeouts = settings.get()
                 )
 
                 if (extTestDataFile.isRelevant)
@@ -102,7 +101,8 @@ private class ExtTestDataFile(
     structureFactory: ExtTestDataFileStructureFactory,
     customSourceTransformers: ExternalSourceTransformers?,
     testRoots: TestRoots,
-    private val generatedSources: GeneratedSources
+    private val generatedSources: GeneratedSources,
+    private val timeouts: Timeouts
 ) {
     private val structure by lazy {
         val allSourceTransformers: ExternalSourceTransformers = if (customSourceTransformers.isNullOrEmpty())
@@ -554,6 +554,7 @@ private class ExtTestDataFile(
             freeCompilerArgs = assembleFreeCompilerArgs(),
             nominalPackageName = testDataFileSettings.nominalPackageName,
             expectedOutputDataFile = null,
+            checks = TestRunChecks.Default(timeouts.executionTimeout),
             extras = WithTestRunnerExtras(runnerType = TestRunnerType.DEFAULT)
         )
         testCase.initialize(sharedModules::get)

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/PredefinedTestCaseGroupProvider.kt

@@ -7,8 +7,10 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.group
 
 import org.jetbrains.kotlin.konan.blackboxtest.support.*
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.WithTestRunnerExtras
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunChecks
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.KotlinNativeHome
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Settings
+import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Timeouts
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.ThreadSafeCache
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.expandGlobTo
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.getAbsoluteFile
@@ -62,6 +64,7 @@ internal class PredefinedTestCaseGroupProvider(annotation: PredefinedTestCases)
                     .parseCompilerArgs(settings) { "Failed to parse free compiler arguments for test case $testCaseId" },
                 nominalPackageName = PackageName(testCaseId.uniqueName),
                 expectedOutputDataFile = null,
+                checks = TestRunChecks.Default(settings.get<Timeouts>().executionTimeout),
                 extras = WithTestRunnerExtras(
                     runnerType = predefinedTestCase.runnerType,
                     ignoredTests = predefinedTestCase.ignoredTests.toSet()

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt

@@ -8,9 +8,12 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.group
 import org.jetbrains.kotlin.konan.blackboxtest.support.*
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.NoTestRunnerExtras
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.WithTestRunnerExtras
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunChecks
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.GeneratedSources
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Settings
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.TestRoots
+import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Timeouts
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.*
 import org.jetbrains.kotlin.test.directives.model.Directive
 import org.jetbrains.kotlin.test.services.JUnit5Assertions
@@ -180,6 +183,14 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider {
         val freeCompilerArgs = parseFreeCompilerArgs(registeredDirectives, location)
         val expectedOutputDataFile = parseOutputDataFile(baseDir = testDataFile.parentFile, registeredDirectives, location)
         val testKind = parseTestKind(registeredDirectives, location)
+        val expectedTimeoutFailure = parseExpectedTimeoutFailure(registeredDirectives)
+
+        val checks = TestRunChecks {
+            this += if (expectedTimeoutFailure)
+                TestRunCheck.ExecutionTimeout.ShouldExceed(settings.get<Timeouts>().executionTimeout)
+            else
+                TestRunCheck.ExecutionTimeout.ShouldNotExceed(settings.get<Timeouts>().executionTimeout)
+        }
 
         if (testKind == TestKind.REGULAR) {
             // Fix package declarations to avoid unintended conflicts between symbols with the same name in different test cases.
@@ -193,6 +204,7 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider {
             freeCompilerArgs = freeCompilerArgs,
             nominalPackageName = nominalPackageName,
             expectedOutputDataFile = expectedOutputDataFile,
+            checks = checks,
             extras = if (testKind == TestKind.STANDALONE_NO_TR)
                 NoTestRunnerExtras(
                     entryPoint = parseEntryPoint(registeredDirectives, location),

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt

@@ -12,7 +12,7 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestOutputFilter
 import java.io.ByteArrayOutputStream
 import kotlin.time.*
 
-internal abstract class AbstractLocalProcessRunner<R>(private val executionTimeout: Duration) : AbstractRunner<R>() {
+internal abstract class AbstractLocalProcessRunner<R>(private val checks: TestRunChecks) : AbstractRunner<R>() {
     protected abstract val visibleProcessName: String
     protected abstract val executable: TestExecutable
     protected abstract val programArgs: List<String>
@@ -26,6 +26,8 @@ internal abstract class AbstractLocalProcessRunner<R>(private val executionTimeo
             val unfilteredOutput = UnfilteredProcessOutput()
             val unfilteredOutputReader: Job
 
+            val executionTimeout: Duration = checks.executionTimeoutCheck.timeout
+
             val process: Process
             val hasFinishedOnTime: Boolean
 
@@ -41,16 +43,11 @@ internal abstract class AbstractLocalProcessRunner<R>(private val executionTimeo
                 )
             }
 
-            if (hasFinishedOnTime) {
+            val exitCode: Int? = if (hasFinishedOnTime) {
                 unfilteredOutputReader.join() // Wait until all output streams are drained.
-
+                process.exitValue()
-                RunResult.Completed(
-                    exitCode = process.exitValue(),
-                    duration = duration,
-                    processOutput = unfilteredOutput.toProcessOutput(outputFilter)
-                )
             } else {
-                val exitCode: Int? = try { // It could happen just by an accident that the process has exited by itself.
+                try { // It could happen just by an accident that the process has exited by itself.
                     val exitCode = process.exitValue() // Fetch exit code.
                     unfilteredOutputReader.join() // Wait until all streams are drained.
                     exitCode
@@ -59,22 +56,37 @@ internal abstract class AbstractLocalProcessRunner<R>(private val executionTimeo
                     process.destroyForcibly() // kill -9
                     null
                 }
-
-                RunResult.TimeoutExceeded(
-                    timeout = executionTimeout,
-                    exitCode = exitCode,
-                    duration = duration,
-                    processOutput = unfilteredOutput.toProcessOutput(outputFilter)
-                )
             }
+
+            RunResult(
+                exitCode = exitCode,
+                timeout = executionTimeout,
+                duration = duration,
+                hasFinishedOnTime = hasFinishedOnTime,
+                processOutput = unfilteredOutput.toProcessOutput(outputFilter)
+            )
         }
     }
 
-    abstract override fun buildResultHandler(runResult: RunResult.Completed): ResultHandler // Narrow returned type.
+    abstract override fun buildResultHandler(runResult: RunResult): ResultHandler // Narrow returned type.
 
-    abstract inner class ResultHandler(runResult: RunResult.Completed) : AbstractRunner<R>.ResultHandler(runResult) {
+    abstract inner class ResultHandler(runResult: RunResult) : AbstractRunner<R>.ResultHandler(runResult) {
         override fun handle(): R {
-            verifyExpectation(runResult.exitCode == 0) { "$visibleProcessName exited with non-zero code." }
+            checks.forEach { check ->
+                when (check) {
+                    is TestRunCheck.ExecutionTimeout.ShouldNotExceed -> verifyExpectation(runResult.hasFinishedOnTime) {
+                        "Timeout exceeded during test execution."
+                    }
+                    is TestRunCheck.ExecutionTimeout.ShouldExceed -> verifyExpectation(!runResult.hasFinishedOnTime) {
+                        "Test is expected to fail with exceeded timeout, which hasn't happened."
+                    }
+                }
+            }
+
+            // Don't check exit code if it is unknown.
+            runResult.exitCode?.let { exitCode ->
+                verifyExpectation(exitCode == 0) { "$visibleProcessName exited with non-zero code." }
+            }
 
             return doHandle()
         }

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractRunner.kt

@@ -7,26 +7,18 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.runner
 
 import org.jetbrains.kotlin.konan.blackboxtest.support.LoggedData
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue
-import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
 import org.opentest4j.TestAbortedException
 
 internal abstract class AbstractRunner<R> {
     protected abstract fun buildRun(): AbstractRun
-    protected abstract fun buildResultHandler(runResult: RunResult.Completed): ResultHandler
+    protected abstract fun buildResultHandler(runResult: RunResult): ResultHandler
     protected abstract fun getLoggedParameters(): LoggedData.TestRunParameters
     protected abstract fun handleUnexpectedFailure(t: Throwable): Nothing
 
     fun run(): R = try {
         val run = buildRun()
-
+        val runResult = run.run()
-        val resultHandler = when (val runResult = run.run()) {
+        val resultHandler = buildResultHandler(runResult)
-            is RunResult.TimeoutExceeded -> fail {
-                LoggedData.TestRunTimeoutExceeded(getLoggedParameters(), runResult)
-                    .withErrorMessage("Timeout exceeded during test execution.")
-            }
-            is RunResult.Completed -> buildResultHandler(runResult)
-        }
-
         resultHandler.handle()
     } catch (t: Throwable) {
         when (t) {
@@ -42,7 +34,7 @@ internal abstract class AbstractRunner<R> {
         fun run(): RunResult
     }
 
-    abstract inner class ResultHandler(protected val runResult: RunResult.Completed) {
+    abstract inner class ResultHandler(protected val runResult: RunResult) {
         abstract fun getLoggedRun(): LoggedData
         abstract fun handle(): R
 

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/BaseTestRunProvider.kt

@@ -9,12 +9,13 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.NoTestRunnerExtras
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestKind
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestName
-import org.jetbrains.kotlin.test.services.JUnit5Assertions
+import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue
+import org.jetbrains.kotlin.utils.addIfNotNull
 
 internal open class BaseTestRunProvider {
     protected fun createTestRun(testCase: TestCase, executable: TestExecutable, testRunName: String, testName: TestName?): TestRun {
         val runParameters = getTestRunParameters(testCase, testName)
-        return TestRun(displayName = testRunName, executable, runParameters, testCase.id)
+        return TestRun(displayName = testRunName, executable, runParameters, testCase.id, testCase.checks)
     }
 
     protected fun createSingleTestRun(testCase: TestCase, executable: TestExecutable): TestRun = createTestRun(
@@ -24,25 +25,24 @@ internal open class BaseTestRunProvider {
         testName = null
     )
 
-    private fun getTestRunParameters(testCase: TestCase, testName: TestName?): List<TestRunParameter> = with(testCase) {
+    private fun getTestRunParameters(testCase: TestCase, testName: TestName?): List<TestRunParameter> = buildList {
-        when (kind) {
+        addIfNotNull(testCase.expectedOutputDataFile?.let(TestRunParameter::WithExpectedOutputData))
+
+        when (testCase.kind) {
             TestKind.STANDALONE_NO_TR -> {
-                JUnit5Assertions.assertTrue(testName == null)
+                assertTrue(testName == null)
-                listOfNotNull(
+                addIfNotNull(testCase.extras<NoTestRunnerExtras>().inputDataFile?.let(TestRunParameter::WithInputData))
-                    extras<NoTestRunnerExtras>().inputDataFile?.let(TestRunParameter::WithInputData),
+            }
-                    expectedOutputDataFile?.let(TestRunParameter::WithExpectedOutputData)
+            TestKind.STANDALONE -> {
+                add(TestRunParameter.WithTCTestLogger)
+                addIfNotNull(testName?.let(TestRunParameter::WithTestFilter))
+            }
+            TestKind.REGULAR -> {
+                add(TestRunParameter.WithTCTestLogger)
+                addIfNotNull(
+                    testName?.let(TestRunParameter::WithTestFilter) ?: TestRunParameter.WithPackageFilter(testCase.nominalPackageName)
                 )
             }
-            TestKind.STANDALONE -> listOfNotNull(
-                TestRunParameter.WithTCTestLogger,
-                testName?.let(TestRunParameter::WithTestFilter),
-                expectedOutputDataFile?.let(TestRunParameter::WithExpectedOutputData)
-            )
-            TestKind.REGULAR -> listOfNotNull(
-                TestRunParameter.WithTCTestLogger,
-                testName?.let(TestRunParameter::WithTestFilter) ?: TestRunParameter.WithPackageFilter(nominalPackageName),
-                expectedOutputDataFile?.let(TestRunParameter::WithExpectedOutputData)
-            )
         }
     }
 }

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestNameExtractor.kt

@@ -10,12 +10,11 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.TestName
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.GTestListing
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestOutputFilter
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
-import kotlin.time.Duration
 
 internal class LocalTestNameExtractor(
     override val executable: TestExecutable,
-    executionTimeout: Duration
+    checks: TestRunChecks
-) : AbstractLocalProcessRunner<Collection<TestName>>(executionTimeout) {
+) : AbstractLocalProcessRunner<Collection<TestName>>(checks) {
     override val visibleProcessName get() = "Test name extractor"
     override val programArgs = listOf(executable.executableFile.path, "--ktest_list_tests")
     override val outputFilter get() = TestOutputFilter.NO_FILTERING
@@ -27,7 +26,7 @@ internal class LocalTestNameExtractor(
         runParameters = null
     )
 
-    override fun buildResultHandler(runResult: RunResult.Completed) = ResultHandler(runResult)
+    override fun buildResultHandler(runResult: RunResult) = ResultHandler(runResult)
 
     override fun handleUnexpectedFailure(t: Throwable) = fail {
         LoggedData.TestRunUnexpectedFailure(getLoggedParameters(), t)
@@ -35,7 +34,7 @@ internal class LocalTestNameExtractor(
     }
 
     inner class ResultHandler(
-        runResult: RunResult.Completed
+        runResult: RunResult
     ) : AbstractLocalProcessRunner<Collection<TestName>>.ResultHandler(runResult) {
         override fun getLoggedRun() = LoggedData.TestRun(getLoggedParameters(), runResult)
         override fun doHandle() = GTestListing.parse(runResult.processOutput.stdOut.filteredOutput)

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt

@@ -12,12 +12,8 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestOutputFilter
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestReport
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
 import org.junit.jupiter.api.Assumptions.assumeFalse
-import kotlin.time.Duration
 
-internal class LocalTestRunner(
+internal class LocalTestRunner(private val testRun: TestRun) : AbstractLocalProcessRunner<Unit>(testRun.checks) {
-    private val testRun: TestRun,
-    executionTimeout: Duration
-) : AbstractLocalProcessRunner<Unit>(executionTimeout) {
     override val visibleProcessName get() = "Tested process"
     override val executable get() = testRun.executable
 
@@ -43,14 +39,14 @@ internal class LocalTestRunner(
         }
     }
 
-    override fun buildResultHandler(runResult: RunResult.Completed) = ResultHandler(runResult)
+    override fun buildResultHandler(runResult: RunResult) = ResultHandler(runResult)
 
     override fun handleUnexpectedFailure(t: Throwable) = fail {
         LoggedData.TestRunUnexpectedFailure(getLoggedParameters(), t)
             .withErrorMessage("Test execution failed with unexpected exception.")
     }
 
-    inner class ResultHandler(runResult: RunResult.Completed) : AbstractLocalProcessRunner<Unit>.ResultHandler(runResult) {
+    inner class ResultHandler(runResult: RunResult) : AbstractLocalProcessRunner<Unit>.ResultHandler(runResult) {
         override fun getLoggedRun() = LoggedData.TestRun(getLoggedParameters(), runResult)
 
         override fun doHandle() {

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/RunResult.kt

@@ -8,23 +8,17 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.runner
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestOutputFilter
 import kotlin.time.Duration
 
-internal sealed interface RunResult {
+internal data class RunResult(
-    val exitCode: Int?
+    val exitCode: Int?,
-    val duration: Duration
+    val timeout: Duration,
+    val duration: Duration,
+    val hasFinishedOnTime: Boolean,
     val processOutput: ProcessOutput
-
+) {
-    data class Completed(
+    init {
-        override val exitCode: Int,
+        // null exit code is possible only when test run hasn't finished on time.
-        override val duration: Duration,
+        check(exitCode != null || !hasFinishedOnTime)
-        override val processOutput: ProcessOutput
+    }
-    ) : RunResult
-
-    data class TimeoutExceeded(
-        val timeout: Duration,
-        override val exitCode: Int?,
-        override val duration: Duration,
-        override val processOutput: ProcessOutput
-    ) : RunResult
 }
 
 internal class ProcessOutput(val stdOut: TestOutputFilter.FilteredOutput, val stdErr: String)

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRun.kt

@@ -54,7 +54,8 @@ internal class TestRun(
     val displayName: String,
     val executable: TestExecutable,
     val runParameters: List<TestRunParameter>,
-    val testCaseId: TestCaseId
+    val testCaseId: TestCaseId,
+    val checks: TestRunChecks
 )
 
 internal sealed interface TestRunParameter {

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunChecks.kt

@@ -0,0 +1,46 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.konan.blackboxtest.support.runner
+
+import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Timeouts
+import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstance
+import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull
+import kotlin.time.Duration
+
+internal sealed interface TestRunCheck {
+    sealed class ExecutionTimeout(val timeout: Duration) : TestRunCheck {
+        class ShouldNotExceed(timeout: Duration) : ExecutionTimeout(timeout)
+        class ShouldExceed(timeout: Duration) : ExecutionTimeout(timeout)
+    }
+}
+
+internal class TestRunChecks(builderAction: MutableCollection<TestRunCheck>.() -> Unit) : Iterable<TestRunCheck> {
+    constructor(vararg checks: TestRunCheck) : this({ addAll(checks) })
+
+    private val allChecks = buildList {
+        builderAction()
+        addDefaults()
+    }
+
+    val executionTimeoutCheck: TestRunCheck.ExecutionTimeout get() = allChecks.firstIsInstance()
+
+    override fun iterator() = allChecks.iterator()
+
+    companion object {
+        // The most frequently used case:
+        @Suppress("TestFunctionName")
+        fun Default(timeout: Duration) = TestRunChecks(TestRunCheck.ExecutionTimeout.ShouldNotExceed(timeout))
+
+        private fun MutableCollection<TestRunCheck>.addDefaults() {
+            if (isMissing<TestRunCheck.ExecutionTimeout>()) {
+                add(TestRunCheck.ExecutionTimeout.ShouldNotExceed(Timeouts.DEFAULT_EXECUTION_TIMEOUT))
+            }
+        }
+
+        private inline fun <reified T : TestRunCheck> Collection<TestRunCheck>.isMissing(): Boolean =
+            firstIsInstanceOrNull<T>() == null
+    }
+}

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunners.kt

@@ -16,7 +16,7 @@ internal object TestRunners {
     fun createProperTestRunner(testRun: TestRun, settings: Settings): AbstractRunner<*> = with(settings) {
         with(get<KotlinNativeTargets>()) {
             if (testTarget == hostTarget)
-                LocalTestRunner(testRun, get<Timeouts>().executionTimeout)
+                LocalTestRunner(testRun)
             else
                 runningAtNonHostTarget()
         }
@@ -26,7 +26,7 @@ internal object TestRunners {
     fun extractTestNames(executable: TestExecutable, settings: Settings): Collection<TestName> = with(settings) {
         with(get<KotlinNativeTargets>()) {
             if (testTarget == hostTarget)
-                LocalTestNameExtractor(executable, get<Timeouts>().executionTimeout).run()
+                LocalTestNameExtractor(executable, TestRunChecks.Default(get<Timeouts>().executionTimeout)).run()
             else
                 runningAtNonHostTarget()
         }

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/settings/TestProcessSettings.kt

@@ -9,6 +9,7 @@ import org.jetbrains.kotlin.konan.target.KonanTarget
 import java.io.File
 import java.util.*
 import kotlin.time.Duration
+import kotlin.time.Duration.Companion.seconds
 
 /**
  * The tested and the host Kotlin/Native targets.
@@ -113,7 +114,11 @@ internal class BaseDirs(val testBuildDir: File)
 /**
  * Timeouts.
  */
-internal class Timeouts(val executionTimeout: Duration)
+internal class Timeouts(val executionTimeout: Duration) {
+    companion object {
+        val DEFAULT_EXECUTION_TIMEOUT: Duration get() = 15.seconds
+    }
+}
 
 /**
  * Used cache mode.