diff --git a/native/native.tests/testData/samples/standalone_notr_nonzero_exit_code.kt b/native/native.tests/testData/samples/standalone_notr_nonzero_exit_code.kt new file mode 100644 index 00000000000..9a679c67a4e --- /dev/null +++ b/native/native.tests/testData/samples/standalone_notr_nonzero_exit_code.kt @@ -0,0 +1,8 @@ +// KIND: STANDALONE_NO_TR +// EXIT_CODE: !0 + +import kotlin.system.exitProcess + +fun main() { + exitProcess(42) +} diff --git a/native/native.tests/testData/samples/standalone_notr_nonzero_exit_code2.kt b/native/native.tests/testData/samples/standalone_notr_nonzero_exit_code2.kt new file mode 100644 index 00000000000..ac66fea6ac9 --- /dev/null +++ b/native/native.tests/testData/samples/standalone_notr_nonzero_exit_code2.kt @@ -0,0 +1,8 @@ +// KIND: STANDALONE_NO_TR +// EXIT_CODE: 42 + +import kotlin.system.exitProcess + +fun main() { + exitProcess(42) +} diff --git a/native/native.tests/testData/samples/standalone_notr_zero_exit_code.kt b/native/native.tests/testData/samples/standalone_notr_zero_exit_code.kt new file mode 100644 index 00000000000..35293c92ccf --- /dev/null +++ b/native/native.tests/testData/samples/standalone_notr_zero_exit_code.kt @@ -0,0 +1,8 @@ +// KIND: STANDALONE_NO_TR +// EXIT_CODE: 0 + +import kotlin.system.exitProcess + +fun main() { + exitProcess(0) +} diff --git a/native/native.tests/tests-gen/org/jetbrains/kotlin/konan/blackboxtest/InfrastructureTestGenerated.java b/native/native.tests/tests-gen/org/jetbrains/kotlin/konan/blackboxtest/InfrastructureTestGenerated.java index 466a5c7a10b..9b2f2e44a70 100644 --- a/native/native.tests/tests-gen/org/jetbrains/kotlin/konan/blackboxtest/InfrastructureTestGenerated.java +++ b/native/native.tests/tests-gen/org/jetbrains/kotlin/konan/blackboxtest/InfrastructureTestGenerated.java @@ -144,6 +144,18 @@ public class InfrastructureTestGenerated extends AbstractNativeBlackBoxTest { runTest("native/native.tests/testData/samples/standalone_notr_multifile_entry_point.kt"); } + @Test + @TestMetadata("standalone_notr_nonzero_exit_code.kt") + public void testStandalone_notr_nonzero_exit_code() throws Exception { + runTest("native/native.tests/testData/samples/standalone_notr_nonzero_exit_code.kt"); + } + + @Test + @TestMetadata("standalone_notr_nonzero_exit_code2.kt") + public void testStandalone_notr_nonzero_exit_code2() throws Exception { + runTest("native/native.tests/testData/samples/standalone_notr_nonzero_exit_code2.kt"); + } + @Test @TestMetadata("standalone_notr_simple.kt") public void testStandalone_notr_simple() throws Exception { @@ -186,6 +198,12 @@ public class InfrastructureTestGenerated extends AbstractNativeBlackBoxTest { runTest("native/native.tests/testData/samples/standalone_notr_too_verbose.kt"); } + @Test + @TestMetadata("standalone_notr_zero_exit_code.kt") + public void testStandalone_notr_zero_exit_code() throws Exception { + runTest("native/native.tests/testData/samples/standalone_notr_zero_exit_code.kt"); + } + @Test @TestMetadata("standalone_simple.kt") public void testStandalone_simple() throws Exception { diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestDirectives.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestDirectives.kt index fd8255e77a8..4d701580753 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestDirectives.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestDirectives.kt @@ -6,12 +6,14 @@ package org.jetbrains.kotlin.konan.blackboxtest.support import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.ENTRY_POINT +import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.EXIT_CODE import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.FREE_COMPILER_ARGS import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.INPUT_DATA_FILE import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.KIND import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.OUTPUT_DATA_FILE import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.EXPECTED_TIMEOUT_FAILURE import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.TEST_RUNNER +import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck import org.jetbrains.kotlin.test.directives.model.RegisteredDirectives import org.jetbrains.kotlin.test.directives.model.SimpleDirectivesContainer import org.jetbrains.kotlin.test.directives.model.StringDirective @@ -104,14 +106,13 @@ internal object TestDirectives : SimpleDirectivesContainer() { """.trimIndent() ) - // TODO: to be supported later -// val EXIT_CODE by stringDirective( -// description = """ -// Specify the exit code that the test should finish with. Example: // EXIT_CODE: 42 -// To indicate any non-zero exit code use // EXIT_CODE: !0 -// Note that this directive makes sense only in combination with // KIND: STANDALONE_NO_TR -// """.trimIndent() -// ) + val EXIT_CODE by stringDirective( + description = """ + Specify the exit code that the test should finish with. Example: // EXIT_CODE: 42 + To indicate any non-zero exit code use // EXIT_CODE: !0 + Note that this directive makes sense only in combination with // KIND: STANDALONE_NO_TR + """.trimIndent() + ) val EXPECTED_TIMEOUT_FAILURE by directive( description = "Whether the test is expected to fail on timeout" @@ -244,6 +245,21 @@ internal fun parseFileName(parsedDirective: RegisteredDirectivesParser.ParsedDir internal fun parseExpectedTimeoutFailure(registeredDirectives: RegisteredDirectives): Boolean = EXPECTED_TIMEOUT_FAILURE in registeredDirectives +internal fun parseExpectedExitCode(registeredDirectives: RegisteredDirectives, location: Location): TestRunCheck.ExitCode { + if (EXIT_CODE !in registeredDirectives) + return TestRunCheck.ExitCode.Expected(0) + + val values = registeredDirectives[EXIT_CODE] + val exitCode = values.singleOrNull() + ?: fail { "$location: Exactly one exit code expected in $EXIT_CODE directive: $values" } + + return when (exitCode) { + "!0" -> TestRunCheck.ExitCode.AnyNonZero + else -> exitCode.toIntOrNull()?.let(TestRunCheck.ExitCode::Expected) + ?: fail { "$location: Invalid exit code specified in $EXIT_CODE directive: $exitCode" } + } +} + internal fun parseFreeCompilerArgs(registeredDirectives: RegisteredDirectives, location: Location): TestCompilerArgs { if (FREE_COMPILER_ARGS !in registeredDirectives) return TestCompilerArgs.EMPTY diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt index 51525bd2f5d..e07f48b5e02 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt @@ -8,7 +8,8 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.group import org.jetbrains.kotlin.konan.blackboxtest.support.* import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.NoTestRunnerExtras import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.WithTestRunnerExtras -import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck +import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.ExecutionTimeout +import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.ExitCode import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunChecks import org.jetbrains.kotlin.konan.blackboxtest.support.settings.GeneratedSources import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Settings @@ -16,6 +17,7 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.settings.TestRoots import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Timeouts import org.jetbrains.kotlin.konan.blackboxtest.support.util.* import org.jetbrains.kotlin.test.directives.model.Directive +import org.jetbrains.kotlin.test.directives.model.RegisteredDirectives import org.jetbrains.kotlin.test.services.JUnit5Assertions import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertEquals import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertFalse @@ -185,13 +187,6 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider { val testKind = parseTestKind(registeredDirectives, location) val expectedTimeoutFailure = parseExpectedTimeoutFailure(registeredDirectives) - val checks = TestRunChecks { - this += if (expectedTimeoutFailure) - TestRunCheck.ExecutionTimeout.ShouldExceed(settings.get().executionTimeout) - else - TestRunCheck.ExecutionTimeout.ShouldNotExceed(settings.get().executionTimeout) - } - if (testKind == TestKind.REGULAR) { // Fix package declarations to avoid unintended conflicts between symbols with the same name in different test cases. fixPackageNames(testModules.values, nominalPackageName, testDataFile) @@ -204,7 +199,10 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider { freeCompilerArgs = freeCompilerArgs, nominalPackageName = nominalPackageName, expectedOutputDataFile = expectedOutputDataFile, - checks = checks, + checks = TestRunChecks( + computeExecutionTimeoutCheck(settings, expectedTimeoutFailure), + computeExitCodeCheck(testKind, registeredDirectives, location) + ), extras = if (testKind == TestKind.STANDALONE_NO_TR) NoTestRunnerExtras( entryPoint = parseEntryPoint(registeredDirectives, location), @@ -242,5 +240,19 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider { } } } + + private fun computeExecutionTimeoutCheck(settings: Settings, expectedTimeoutFailure: Boolean): ExecutionTimeout { + val executionTimeout = settings.get().executionTimeout + return if (expectedTimeoutFailure) + ExecutionTimeout.ShouldExceed(executionTimeout) + else + ExecutionTimeout.ShouldNotExceed(executionTimeout) + } + + private fun computeExitCodeCheck(testKind: TestKind, registeredDirectives: RegisteredDirectives, location: Location): ExitCode = + if (testKind == TestKind.STANDALONE_NO_TR) + parseExpectedExitCode(registeredDirectives, location) + else + ExitCode.Expected(0) } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt index 2cbde88a5aa..f3e3a5a0082 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt @@ -7,6 +7,8 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.runner import kotlinx.coroutines.* import org.jetbrains.kotlin.konan.blackboxtest.support.runner.AbstractRunner.AbstractRun +import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.ExecutionTimeout +import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.ExitCode import org.jetbrains.kotlin.konan.blackboxtest.support.runner.UnfilteredProcessOutput.Companion.launchReader import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestOutputFilter import java.io.ByteArrayOutputStream @@ -74,20 +76,27 @@ internal abstract class AbstractLocalProcessRunner(private val checks: TestRu override fun handle(): R { checks.forEach { check -> when (check) { - is TestRunCheck.ExecutionTimeout.ShouldNotExceed -> verifyExpectation(runResult.hasFinishedOnTime) { + is ExecutionTimeout.ShouldNotExceed -> verifyExpectation(runResult.hasFinishedOnTime) { "Timeout exceeded during test execution." } - is TestRunCheck.ExecutionTimeout.ShouldExceed -> verifyExpectation(!runResult.hasFinishedOnTime) { + is ExecutionTimeout.ShouldExceed -> verifyExpectation(!runResult.hasFinishedOnTime) { "Test is expected to fail with exceeded timeout, which hasn't happened." } + is ExitCode -> { + // Don't check exit code if it is unknown. + val knownExitCode: Int = runResult.exitCode ?: return@forEach + when (check) { + is ExitCode.Expected -> verifyExpectation(knownExitCode == check.expectedExitCode) { + "$visibleProcessName exit code is $knownExitCode while ${check.expectedExitCode} was expected." + } + is ExitCode.AnyNonZero -> verifyExpectation(knownExitCode != 0) { + "$visibleProcessName exited with zero code, which wasn't expected." + } + } + } } } - // Don't check exit code if it is unknown. - runResult.exitCode?.let { exitCode -> - verifyExpectation(exitCode == 0) { "$visibleProcessName exited with non-zero code." } - } - return doHandle() } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunChecks.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunChecks.kt index d0289c855f9..9e15e23a02c 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunChecks.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunChecks.kt @@ -5,9 +5,8 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.runner -import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Timeouts -import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstance -import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull +import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.ExecutionTimeout +import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck.ExitCode import kotlin.time.Duration internal sealed interface TestRunCheck { @@ -15,32 +14,29 @@ internal sealed interface TestRunCheck { class ShouldNotExceed(timeout: Duration) : ExecutionTimeout(timeout) class ShouldExceed(timeout: Duration) : ExecutionTimeout(timeout) } + + sealed class ExitCode : TestRunCheck { + object AnyNonZero : ExitCode() + class Expected(val expectedExitCode: Int) : ExitCode() + } } -internal class TestRunChecks(builderAction: MutableCollection.() -> Unit) : Iterable { - constructor(vararg checks: TestRunCheck) : this({ addAll(checks) }) +internal class TestRunChecks( + val executionTimeoutCheck: ExecutionTimeout, + private val exitCodeCheck: ExitCode +) : Iterable { - private val allChecks = buildList { - builderAction() - addDefaults() + override fun iterator() = iterator { + yield(executionTimeoutCheck) + yield(exitCodeCheck) } - val executionTimeoutCheck: TestRunCheck.ExecutionTimeout get() = allChecks.firstIsInstance() - - override fun iterator() = allChecks.iterator() - companion object { // The most frequently used case: @Suppress("TestFunctionName") - fun Default(timeout: Duration) = TestRunChecks(TestRunCheck.ExecutionTimeout.ShouldNotExceed(timeout)) - - private fun MutableCollection.addDefaults() { - if (isMissing()) { - add(TestRunCheck.ExecutionTimeout.ShouldNotExceed(Timeouts.DEFAULT_EXECUTION_TIMEOUT)) - } - } - - private inline fun Collection.isMissing(): Boolean = - firstIsInstanceOrNull() == null + fun Default(timeout: Duration) = TestRunChecks( + ExecutionTimeout.ShouldNotExceed(timeout), + ExitCode.Expected(0) + ) } }