[Native][tests] Introduce EXIT_CODE directive

It allows to specify the expected exit code for STANDALONE_NO_TR tests.

native/native.tests/testData/samples/standalone_notr_nonzero_exit_code.kt

@@ -0,0 +1,8 @@
+// KIND: STANDALONE_NO_TR
+// EXIT_CODE: !0
+
+import kotlin.system.exitProcess
+
+fun main() {
+    exitProcess(42)
+}

native/native.tests/testData/samples/standalone_notr_nonzero_exit_code2.kt

@@ -0,0 +1,8 @@
+// KIND: STANDALONE_NO_TR
+// EXIT_CODE: 42
+
+import kotlin.system.exitProcess
+
+fun main() {
+    exitProcess(42)
+}

native/native.tests/testData/samples/standalone_notr_zero_exit_code.kt

@@ -0,0 +1,8 @@
+// KIND: STANDALONE_NO_TR
+// EXIT_CODE: 0
+
+import kotlin.system.exitProcess
+
+fun main() {
+    exitProcess(0)
+}

native/native.tests/tests-gen/org/jetbrains/kotlin/konan/blackboxtest/InfrastructureTestGenerated.java

@@ -144,6 +144,18 @@ public class InfrastructureTestGenerated extends AbstractNativeBlackBoxTest {
             runTest("native/native.tests/testData/samples/standalone_notr_multifile_entry_point.kt");
         }
 
+        @Test
+        @TestMetadata("standalone_notr_nonzero_exit_code.kt")
+        public void testStandalone_notr_nonzero_exit_code() throws Exception {
+            runTest("native/native.tests/testData/samples/standalone_notr_nonzero_exit_code.kt");
+        }
+
+        @Test
+        @TestMetadata("standalone_notr_nonzero_exit_code2.kt")
+        public void testStandalone_notr_nonzero_exit_code2() throws Exception {
+            runTest("native/native.tests/testData/samples/standalone_notr_nonzero_exit_code2.kt");
+        }
+
         @Test
         @TestMetadata("standalone_notr_simple.kt")
         public void testStandalone_notr_simple() throws Exception {
@@ -186,6 +198,12 @@ public class InfrastructureTestGenerated extends AbstractNativeBlackBoxTest {
             runTest("native/native.tests/testData/samples/standalone_notr_too_verbose.kt");
         }
 
+        @Test
+        @TestMetadata("standalone_notr_zero_exit_code.kt")
+        public void testStandalone_notr_zero_exit_code() throws Exception {
+            runTest("native/native.tests/testData/samples/standalone_notr_zero_exit_code.kt");
+        }
+
         @Test
         @TestMetadata("standalone_simple.kt")
         public void testStandalone_simple() throws Exception {

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestDirectives.kt

@@ -6,12 +6,14 @@
 package org.jetbrains.kotlin.konan.blackboxtest.support
 
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.ENTRY_POINT
+import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.EXIT_CODE
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.FREE_COMPILER_ARGS
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.INPUT_DATA_FILE
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.KIND
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.OUTPUT_DATA_FILE
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.EXPECTED_TIMEOUT_FAILURE
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.TEST_RUNNER
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck
 import org.jetbrains.kotlin.test.directives.model.RegisteredDirectives
 import org.jetbrains.kotlin.test.directives.model.SimpleDirectivesContainer
 import org.jetbrains.kotlin.test.directives.model.StringDirective
@@ -104,14 +106,13 @@ internal object TestDirectives : SimpleDirectivesContainer() {
         """.trimIndent()
     )
 
-    // TODO: to be supported later
+    val EXIT_CODE by stringDirective(
-//    val EXIT_CODE by stringDirective(
+        description = """
-//        description = """
+            Specify the exit code that the test should finish with. Example: // EXIT_CODE: 42
-//            Specify the exit code that the test should finish with. Example: // EXIT_CODE: 42
+            To indicate any non-zero exit code use // EXIT_CODE: !0
-//            To indicate any non-zero exit code use // EXIT_CODE: !0
+            Note that this directive makes sense only in combination with // KIND: STANDALONE_NO_TR
-//            Note that this directive makes sense only in combination with // KIND: STANDALONE_NO_TR
+        """.trimIndent()
-//        """.trimIndent()
+    )
-//    )
 
     val EXPECTED_TIMEOUT_FAILURE by directive(
         description = "Whether the test is expected to fail on timeout"
@@ -244,6 +245,21 @@ internal fun parseFileName(parsedDirective: RegisteredDirectivesParser.ParsedDir
 internal fun parseExpectedTimeoutFailure(registeredDirectives: RegisteredDirectives): Boolean =
     EXPECTED_TIMEOUT_FAILURE in registeredDirectives
 
+internal fun parseExpectedExitCode(registeredDirectives: RegisteredDirectives, location: Location): TestRunCheck.ExitCode {
+    if (EXIT_CODE !in registeredDirectives)
+        return TestRunCheck.ExitCode.Expected(0)
+
+    val values = registeredDirectives[EXIT_CODE]
+    val exitCode = values.singleOrNull()
+        ?: fail { "$location: Exactly one exit code expected in $EXIT_CODE directive: $values" }
+
+    return when (exitCode) {
+        "!0" -> TestRunCheck.ExitCode.AnyNonZero
+        else -> exitCode.toIntOrNull()?.let(TestRunCheck.ExitCode::Expected)
+            ?: fail { "$location: Invalid exit code specified in $EXIT_CODE directive: $exitCode" }
+    }
+}
+
 internal fun parseFreeCompilerArgs(registeredDirectives: RegisteredDirectives, location: Location): TestCompilerArgs {
     if (FREE_COMPILER_ARGS !in registeredDirectives)
         return TestCompilerArgs.EMPTY

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt

@@ -8,7 +8,8 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.group
 import org.jetbrains.kotlin.konan.blackboxtest.support.*
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.NoTestRunnerExtras
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.WithTestRunnerExtras
-import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.ExecutionTimeout
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.ExitCode
 import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunChecks
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.GeneratedSources
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Settings
@@ -16,6 +17,7 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.settings.TestRoots
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Timeouts
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.*
 import org.jetbrains.kotlin.test.directives.model.Directive
+import org.jetbrains.kotlin.test.directives.model.RegisteredDirectives
 import org.jetbrains.kotlin.test.services.JUnit5Assertions
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertEquals
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertFalse
@@ -185,13 +187,6 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider {
         val testKind = parseTestKind(registeredDirectives, location)
         val expectedTimeoutFailure = parseExpectedTimeoutFailure(registeredDirectives)
 
-        val checks = TestRunChecks {
-            this += if (expectedTimeoutFailure)
-                TestRunCheck.ExecutionTimeout.ShouldExceed(settings.get<Timeouts>().executionTimeout)
-            else
-                TestRunCheck.ExecutionTimeout.ShouldNotExceed(settings.get<Timeouts>().executionTimeout)
-        }
-
         if (testKind == TestKind.REGULAR) {
             // Fix package declarations to avoid unintended conflicts between symbols with the same name in different test cases.
             fixPackageNames(testModules.values, nominalPackageName, testDataFile)
@@ -204,7 +199,10 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider {
             freeCompilerArgs = freeCompilerArgs,
             nominalPackageName = nominalPackageName,
             expectedOutputDataFile = expectedOutputDataFile,
-            checks = checks,
+            checks = TestRunChecks(
+                computeExecutionTimeoutCheck(settings, expectedTimeoutFailure),
+                computeExitCodeCheck(testKind, registeredDirectives, location)
+            ),
             extras = if (testKind == TestKind.STANDALONE_NO_TR)
                 NoTestRunnerExtras(
                     entryPoint = parseEntryPoint(registeredDirectives, location),
@@ -242,5 +240,19 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider {
                 }
             }
         }
+
+        private fun computeExecutionTimeoutCheck(settings: Settings, expectedTimeoutFailure: Boolean): ExecutionTimeout {
+            val executionTimeout = settings.get<Timeouts>().executionTimeout
+            return if (expectedTimeoutFailure)
+                ExecutionTimeout.ShouldExceed(executionTimeout)
+            else
+                ExecutionTimeout.ShouldNotExceed(executionTimeout)
+        }
+
+        private fun computeExitCodeCheck(testKind: TestKind, registeredDirectives: RegisteredDirectives, location: Location): ExitCode =
+            if (testKind == TestKind.STANDALONE_NO_TR)
+                parseExpectedExitCode(registeredDirectives, location)
+            else
+                ExitCode.Expected(0)
     }
 }

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt

@@ -7,6 +7,8 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.runner
 
 import kotlinx.coroutines.*
 import org.jetbrains.kotlin.konan.blackboxtest.support.runner.AbstractRunner.AbstractRun
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.ExecutionTimeout
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.ExitCode
 import org.jetbrains.kotlin.konan.blackboxtest.support.runner.UnfilteredProcessOutput.Companion.launchReader
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestOutputFilter
 import java.io.ByteArrayOutputStream
@@ -74,20 +76,27 @@ internal abstract class AbstractLocalProcessRunner<R>(private val checks: TestRu
         override fun handle(): R {
             checks.forEach { check ->
                 when (check) {
-                    is TestRunCheck.ExecutionTimeout.ShouldNotExceed -> verifyExpectation(runResult.hasFinishedOnTime) {
+                    is ExecutionTimeout.ShouldNotExceed -> verifyExpectation(runResult.hasFinishedOnTime) {
                         "Timeout exceeded during test execution."
                     }
-                    is TestRunCheck.ExecutionTimeout.ShouldExceed -> verifyExpectation(!runResult.hasFinishedOnTime) {
+                    is ExecutionTimeout.ShouldExceed -> verifyExpectation(!runResult.hasFinishedOnTime) {
                         "Test is expected to fail with exceeded timeout, which hasn't happened."
                     }
+                    is ExitCode -> {
+                        // Don't check exit code if it is unknown.
+                        val knownExitCode: Int = runResult.exitCode ?: return@forEach
+                        when (check) {
+                            is ExitCode.Expected -> verifyExpectation(knownExitCode == check.expectedExitCode) {
+                                "$visibleProcessName exit code is $knownExitCode while ${check.expectedExitCode} was expected."
+                            }
+                            is ExitCode.AnyNonZero -> verifyExpectation(knownExitCode != 0) {
+                                "$visibleProcessName exited with zero code, which wasn't expected."
+                            }
+                        }
+                    }
                 }
             }
 
-            // Don't check exit code if it is unknown.
-            runResult.exitCode?.let { exitCode ->
-                verifyExpectation(exitCode == 0) { "$visibleProcessName exited with non-zero code." }
-            }
-
             return doHandle()
         }
 

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunChecks.kt

@@ -5,9 +5,8 @@
 
 package org.jetbrains.kotlin.konan.blackboxtest.support.runner
 
-import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Timeouts
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.ExecutionTimeout
-import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstance
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.ExitCode
-import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull
 import kotlin.time.Duration
 
 internal sealed interface TestRunCheck {
@@ -15,32 +14,29 @@ internal sealed interface TestRunCheck {
         class ShouldNotExceed(timeout: Duration) : ExecutionTimeout(timeout)
         class ShouldExceed(timeout: Duration) : ExecutionTimeout(timeout)
     }
+
+    sealed class ExitCode : TestRunCheck {
+        object AnyNonZero : ExitCode()
+        class Expected(val expectedExitCode: Int) : ExitCode()
+    }
 }
 
-internal class TestRunChecks(builderAction: MutableCollection<TestRunCheck>.() -> Unit) : Iterable<TestRunCheck> {
+internal class TestRunChecks(
-    constructor(vararg checks: TestRunCheck) : this({ addAll(checks) })
+    val executionTimeoutCheck: ExecutionTimeout,
+    private val exitCodeCheck: ExitCode
+) : Iterable<TestRunCheck> {
 
-    private val allChecks = buildList {
+    override fun iterator() = iterator {
-        builderAction()
+        yield(executionTimeoutCheck)
-        addDefaults()
+        yield(exitCodeCheck)
     }
 
-    val executionTimeoutCheck: TestRunCheck.ExecutionTimeout get() = allChecks.firstIsInstance()
-
-    override fun iterator() = allChecks.iterator()
-
     companion object {
         // The most frequently used case:
         @Suppress("TestFunctionName")
-        fun Default(timeout: Duration) = TestRunChecks(TestRunCheck.ExecutionTimeout.ShouldNotExceed(timeout))
+        fun Default(timeout: Duration) = TestRunChecks(
-
+            ExecutionTimeout.ShouldNotExceed(timeout),
-        private fun MutableCollection<TestRunCheck>.addDefaults() {
+            ExitCode.Expected(0)
-            if (isMissing<TestRunCheck.ExecutionTimeout>()) {
+        )
-                add(TestRunCheck.ExecutionTimeout.ShouldNotExceed(Timeouts.DEFAULT_EXECUTION_TIMEOUT))
-            }
-        }
-
-        private inline fun <reified T : TestRunCheck> Collection<TestRunCheck>.isMissing(): Boolean =
-            firstIsInstanceOrNull<T>() == null
     }
 }