[Native][tests] Introduce EXIT_CODE directive

It allows to specify the expected exit code for STANDALONE_NO_TR tests.
2022-04-07 15:05:03 +03:00
parent 261e177b39
commit 996ff5085d
8 changed files with 121 additions and 46 deletions
@@ -0,0 +1,8 @@
+// KIND: STANDALONE_NO_TR
+// EXIT_CODE: !0
+
+import kotlin.system.exitProcess
+
+fun main() {
+    exitProcess(42)
+}
@@ -0,0 +1,8 @@
+// KIND: STANDALONE_NO_TR
+// EXIT_CODE: 42
+
+import kotlin.system.exitProcess
+
+fun main() {
+    exitProcess(42)
+}
@@ -0,0 +1,8 @@
+// KIND: STANDALONE_NO_TR
+// EXIT_CODE: 0
+
+import kotlin.system.exitProcess
+
+fun main() {
+    exitProcess(0)
+}
@@ -144,6 +144,18 @@ public class InfrastructureTestGenerated extends AbstractNativeBlackBoxTest {
            runTest("native/native.tests/testData/samples/standalone_notr_multifile_entry_point.kt");
        }

+        @Test
+        @TestMetadata("standalone_notr_nonzero_exit_code.kt")
+        public void testStandalone_notr_nonzero_exit_code() throws Exception {
+            runTest("native/native.tests/testData/samples/standalone_notr_nonzero_exit_code.kt");
+        }
+
+        @Test
+        @TestMetadata("standalone_notr_nonzero_exit_code2.kt")
+        public void testStandalone_notr_nonzero_exit_code2() throws Exception {
+            runTest("native/native.tests/testData/samples/standalone_notr_nonzero_exit_code2.kt");
+        }
+
        @Test
        @TestMetadata("standalone_notr_simple.kt")
        public void testStandalone_notr_simple() throws Exception {
@@ -186,6 +198,12 @@ public class InfrastructureTestGenerated extends AbstractNativeBlackBoxTest {
            runTest("native/native.tests/testData/samples/standalone_notr_too_verbose.kt");
        }

+        @Test
+        @TestMetadata("standalone_notr_zero_exit_code.kt")
+        public void testStandalone_notr_zero_exit_code() throws Exception {
+            runTest("native/native.tests/testData/samples/standalone_notr_zero_exit_code.kt");
+        }
+
        @Test
        @TestMetadata("standalone_simple.kt")
        public void testStandalone_simple() throws Exception {
@@ -6,12 +6,14 @@
 package org.jetbrains.kotlin.konan.blackboxtest.support

 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.ENTRY_POINT
+import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.EXIT_CODE
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.FREE_COMPILER_ARGS
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.INPUT_DATA_FILE
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.KIND
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.OUTPUT_DATA_FILE
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.EXPECTED_TIMEOUT_FAILURE
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.TEST_RUNNER
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck
 import org.jetbrains.kotlin.test.directives.model.RegisteredDirectives
 import org.jetbrains.kotlin.test.directives.model.SimpleDirectivesContainer
 import org.jetbrains.kotlin.test.directives.model.StringDirective
@@ -104,14 +106,13 @@ internal object TestDirectives : SimpleDirectivesContainer() {
        """.trimIndent()
    )

-    // TODO: to be supported later
-//    val EXIT_CODE by stringDirective(
-//        description = """
-//            Specify the exit code that the test should finish with. Example: // EXIT_CODE: 42
-//            To indicate any non-zero exit code use // EXIT_CODE: !0
-//            Note that this directive makes sense only in combination with // KIND: STANDALONE_NO_TR
-//        """.trimIndent()
-//    )
+    val EXIT_CODE by stringDirective(
+        description = """
+            Specify the exit code that the test should finish with. Example: // EXIT_CODE: 42
+            To indicate any non-zero exit code use // EXIT_CODE: !0
+            Note that this directive makes sense only in combination with // KIND: STANDALONE_NO_TR
+        """.trimIndent()
+    )

    val EXPECTED_TIMEOUT_FAILURE by directive(
        description = "Whether the test is expected to fail on timeout"
@@ -244,6 +245,21 @@ internal fun parseFileName(parsedDirective: RegisteredDirectivesParser.ParsedDir
 internal fun parseExpectedTimeoutFailure(registeredDirectives: RegisteredDirectives): Boolean =
    EXPECTED_TIMEOUT_FAILURE in registeredDirectives

+internal fun parseExpectedExitCode(registeredDirectives: RegisteredDirectives, location: Location): TestRunCheck.ExitCode {
+    if (EXIT_CODE !in registeredDirectives)
+        return TestRunCheck.ExitCode.Expected(0)
+
+    val values = registeredDirectives[EXIT_CODE]
+    val exitCode = values.singleOrNull()
+        ?: fail { "$location: Exactly one exit code expected in $EXIT_CODE directive: $values" }
+
+    return when (exitCode) {
+        "!0" -> TestRunCheck.ExitCode.AnyNonZero
+        else -> exitCode.toIntOrNull()?.let(TestRunCheck.ExitCode::Expected)
+            ?: fail { "$location: Invalid exit code specified in $EXIT_CODE directive: $exitCode" }
+    }
+}
+
 internal fun parseFreeCompilerArgs(registeredDirectives: RegisteredDirectives, location: Location): TestCompilerArgs {
    if (FREE_COMPILER_ARGS !in registeredDirectives)
        return TestCompilerArgs.EMPTY
@@ -8,7 +8,8 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.group
 import org.jetbrains.kotlin.konan.blackboxtest.support.*
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.NoTestRunnerExtras
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.WithTestRunnerExtras
-import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.ExecutionTimeout
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.ExitCode
 import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunChecks
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.GeneratedSources
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Settings
@@ -16,6 +17,7 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.settings.TestRoots
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Timeouts
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.*
 import org.jetbrains.kotlin.test.directives.model.Directive
+import org.jetbrains.kotlin.test.directives.model.RegisteredDirectives
 import org.jetbrains.kotlin.test.services.JUnit5Assertions
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertEquals
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertFalse
@@ -185,13 +187,6 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider {
        val testKind = parseTestKind(registeredDirectives, location)
        val expectedTimeoutFailure = parseExpectedTimeoutFailure(registeredDirectives)

-        val checks = TestRunChecks {
-            this += if (expectedTimeoutFailure)
-                TestRunCheck.ExecutionTimeout.ShouldExceed(settings.get<Timeouts>().executionTimeout)
-            else
-                TestRunCheck.ExecutionTimeout.ShouldNotExceed(settings.get<Timeouts>().executionTimeout)
-        }
-
        if (testKind == TestKind.REGULAR) {
            // Fix package declarations to avoid unintended conflicts between symbols with the same name in different test cases.
            fixPackageNames(testModules.values, nominalPackageName, testDataFile)
@@ -204,7 +199,10 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider {
            freeCompilerArgs = freeCompilerArgs,
            nominalPackageName = nominalPackageName,
            expectedOutputDataFile = expectedOutputDataFile,
-            checks = checks,
+            checks = TestRunChecks(
+                computeExecutionTimeoutCheck(settings, expectedTimeoutFailure),
+                computeExitCodeCheck(testKind, registeredDirectives, location)
+            ),
            extras = if (testKind == TestKind.STANDALONE_NO_TR)
                NoTestRunnerExtras(
                    entryPoint = parseEntryPoint(registeredDirectives, location),
@@ -242,5 +240,19 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider {
                }
            }
        }
+
+        private fun computeExecutionTimeoutCheck(settings: Settings, expectedTimeoutFailure: Boolean): ExecutionTimeout {
+            val executionTimeout = settings.get<Timeouts>().executionTimeout
+            return if (expectedTimeoutFailure)
+                ExecutionTimeout.ShouldExceed(executionTimeout)
+            else
+                ExecutionTimeout.ShouldNotExceed(executionTimeout)
+        }
+
+        private fun computeExitCodeCheck(testKind: TestKind, registeredDirectives: RegisteredDirectives, location: Location): ExitCode =
+            if (testKind == TestKind.STANDALONE_NO_TR)
+                parseExpectedExitCode(registeredDirectives, location)
+            else
+                ExitCode.Expected(0)
    }
 }
@@ -7,6 +7,8 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.runner

 import kotlinx.coroutines.*
 import org.jetbrains.kotlin.konan.blackboxtest.support.runner.AbstractRunner.AbstractRun
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.ExecutionTimeout
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.ExitCode
 import org.jetbrains.kotlin.konan.blackboxtest.support.runner.UnfilteredProcessOutput.Companion.launchReader
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestOutputFilter
 import java.io.ByteArrayOutputStream
@@ -74,20 +76,27 @@ internal abstract class AbstractLocalProcessRunner<R>(private val checks: TestRu
        override fun handle(): R {
            checks.forEach { check ->
                when (check) {
-                    is TestRunCheck.ExecutionTimeout.ShouldNotExceed -> verifyExpectation(runResult.hasFinishedOnTime) {
+                    is ExecutionTimeout.ShouldNotExceed -> verifyExpectation(runResult.hasFinishedOnTime) {
                        "Timeout exceeded during test execution."
                    }
-                    is TestRunCheck.ExecutionTimeout.ShouldExceed -> verifyExpectation(!runResult.hasFinishedOnTime) {
+                    is ExecutionTimeout.ShouldExceed -> verifyExpectation(!runResult.hasFinishedOnTime) {
                        "Test is expected to fail with exceeded timeout, which hasn't happened."
                    }
+                    is ExitCode -> {
+                        // Don't check exit code if it is unknown.
+                        val knownExitCode: Int = runResult.exitCode ?: return@forEach
+                        when (check) {
+                            is ExitCode.Expected -> verifyExpectation(knownExitCode == check.expectedExitCode) {
+                                "$visibleProcessName exit code is $knownExitCode while ${check.expectedExitCode} was expected."
+                            }
+                            is ExitCode.AnyNonZero -> verifyExpectation(knownExitCode != 0) {
+                                "$visibleProcessName exited with zero code, which wasn't expected."
+                            }
+                        }
+                    }
                }
            }

-            // Don't check exit code if it is unknown.
-            runResult.exitCode?.let { exitCode ->
-                verifyExpectation(exitCode == 0) { "$visibleProcessName exited with non-zero code." }
-            }
-
            return doHandle()
        }

@@ -5,9 +5,8 @@

 package org.jetbrains.kotlin.konan.blackboxtest.support.runner

-import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Timeouts
-import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstance
-import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.ExecutionTimeout
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.ExitCode
 import kotlin.time.Duration

 internal sealed interface TestRunCheck {
@@ -15,32 +14,29 @@ internal sealed interface TestRunCheck {
        class ShouldNotExceed(timeout: Duration) : ExecutionTimeout(timeout)
        class ShouldExceed(timeout: Duration) : ExecutionTimeout(timeout)
    }
+
+    sealed class ExitCode : TestRunCheck {
+        object AnyNonZero : ExitCode()
+        class Expected(val expectedExitCode: Int) : ExitCode()
+    }
 }

-internal class TestRunChecks(builderAction: MutableCollection<TestRunCheck>.() -> Unit) : Iterable<TestRunCheck> {
-    constructor(vararg checks: TestRunCheck) : this({ addAll(checks) })
+internal class TestRunChecks(
+    val executionTimeoutCheck: ExecutionTimeout,
+    private val exitCodeCheck: ExitCode
+) : Iterable<TestRunCheck> {

-    private val allChecks = buildList {
-        builderAction()
-        addDefaults()
+    override fun iterator() = iterator {
+        yield(executionTimeoutCheck)
+        yield(exitCodeCheck)
    }

-    val executionTimeoutCheck: TestRunCheck.ExecutionTimeout get() = allChecks.firstIsInstance()
-
-    override fun iterator() = allChecks.iterator()
-
    companion object {
        // The most frequently used case:
        @Suppress("TestFunctionName")
-        fun Default(timeout: Duration) = TestRunChecks(TestRunCheck.ExecutionTimeout.ShouldNotExceed(timeout))
-
-        private fun MutableCollection<TestRunCheck>.addDefaults() {
-            if (isMissing<TestRunCheck.ExecutionTimeout>()) {
-                add(TestRunCheck.ExecutionTimeout.ShouldNotExceed(Timeouts.DEFAULT_EXECUTION_TIMEOUT))
-            }
-        }
-
-        private inline fun <reified T : TestRunCheck> Collection<TestRunCheck>.isMissing(): Boolean =
-            firstIsInstanceOrNull<T>() == null
+        fun Default(timeout: Duration) = TestRunChecks(
+            ExecutionTimeout.ShouldNotExceed(timeout),
+            ExitCode.Expected(0)
+        )
    }
 }