diff --git a/native/native.tests/testData/samples/standalone_notr_long_running.kt b/native/native.tests/testData/samples/standalone_notr_long_running.kt index 13b02ddad20..8a23826c272 100644 --- a/native/native.tests/testData/samples/standalone_notr_long_running.kt +++ b/native/native.tests/testData/samples/standalone_notr_long_running.kt @@ -1,4 +1,5 @@ // KIND: STANDALONE_NO_TR +// EXPECTED_TIMEOUT_FAILURE import kotlin.math.E import kotlin.math.sqrt diff --git a/native/native.tests/testData/samples/standalone_notr_long_running_and_verbose.kt b/native/native.tests/testData/samples/standalone_notr_long_running_and_verbose.kt index b550b0d2810..d9e0781cb25 100644 --- a/native/native.tests/testData/samples/standalone_notr_long_running_and_verbose.kt +++ b/native/native.tests/testData/samples/standalone_notr_long_running_and_verbose.kt @@ -1,4 +1,5 @@ // KIND: STANDALONE_NO_TR +// EXPECTED_TIMEOUT_FAILURE import kotlin.math.E import kotlin.math.sqrt diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/AbstractNativeKlibABITest.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/AbstractNativeKlibABITest.kt index e3a611036b2..42816553aa7 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/AbstractNativeKlibABITest.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/AbstractNativeKlibABITest.kt @@ -14,10 +14,8 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilati import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationDependencyType.Library import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationResult.Companion.assertSuccess import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestExecutable -import org.jetbrains.kotlin.konan.blackboxtest.support.settings.CacheMode -import org.jetbrains.kotlin.konan.blackboxtest.support.settings.KotlinNativeHome -import org.jetbrains.kotlin.konan.blackboxtest.support.settings.KotlinNativeTargets -import org.jetbrains.kotlin.konan.blackboxtest.support.settings.SimpleTestDirectories +import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunChecks +import org.jetbrains.kotlin.konan.blackboxtest.support.settings.* import org.jetbrains.kotlin.konan.blackboxtest.support.util.* import org.junit.jupiter.api.Tag import java.io.File @@ -119,6 +117,7 @@ abstract class AbstractNativeKlibABITest : AbstractNativeSimpleTest() { freeCompilerArgs = compilerArgs, nominalPackageName = PackageName.EMPTY, expectedOutputDataFile = null, + checks = TestRunChecks.Default(testRunSettings.get().executionTimeout), extras = DEFAULT_EXTRAS ).apply { initialize(null) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/InfrastructureDumpedTestListingTest.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/InfrastructureDumpedTestListingTest.kt index 7f08faa5a6a..fc2486dc03e 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/InfrastructureDumpedTestListingTest.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/InfrastructureDumpedTestListingTest.kt @@ -21,9 +21,11 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilati import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationResult.Companion.assertSuccess import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationResult.Success import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestExecutable +import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunChecks import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunners import org.jetbrains.kotlin.konan.blackboxtest.support.settings.KotlinNativeTargets import org.jetbrains.kotlin.konan.blackboxtest.support.settings.SimpleTestDirectories +import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Timeouts import org.jetbrains.kotlin.konan.blackboxtest.support.util.DumpedTestListing import org.jetbrains.kotlin.konan.blackboxtest.support.util.LAUNCHER_MODULE_NAME import org.jetbrains.kotlin.test.TestMetadata @@ -111,6 +113,7 @@ class InfrastructureDumpedTestListingTest : AbstractNativeSimpleTest() { freeCompilerArgs = TestCompilerArgs.EMPTY, nominalPackageName = PackageName.EMPTY, expectedOutputDataFile = null, + checks = TestRunChecks.Default(testRunSettings.get().executionTimeout), extras = DEFAULT_EXTRAS ).apply { initialize(null) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/LoggedData.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/LoggedData.kt index 6fa9b70e174..d324f67bc03 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/LoggedData.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/LoggedData.kt @@ -152,11 +152,16 @@ internal abstract class LoggedData { class TestRun( private val parameters: TestRunParameters, - private val runResult: RunResult.Completed + private val runResult: RunResult ) : LoggedData() { override fun computeText() = buildString { appendLine("TEST RUN:") - appendCommonRunResult(runResult) + appendLine("- Exit code: ${runResult.exitCode ?: ""}") + appendDuration(runResult.timeout, runResult.duration, runResult.hasFinishedOnTime) + appendLine() + appendPotentiallyLargeOutput(runResult.processOutput.stdOut.filteredOutput, "STDOUT", truncateLargeOutput = true) + appendLine() + appendPotentiallyLargeOutput(runResult.processOutput.stdErr, "STDERR", truncateLargeOutput = true) appendLine() appendLine(parameters) } @@ -179,19 +184,6 @@ internal abstract class LoggedData { } } - class TestRunTimeoutExceeded( - private val parameters: TestRunParameters, - private val runResult: RunResult.TimeoutExceeded - ) : LoggedData() { - override fun computeText() = buildString { - appendLine("TIMED OUT:") - appendLine("- Max permitted duration: ${runResult.timeout}") - appendCommonRunResult(runResult) - appendLine() - appendLine(parameters) - } - } - companion object { protected fun StringBuilder.appendList(header: String, list: Iterable): StringBuilder { appendLine(header) @@ -222,17 +214,14 @@ internal abstract class LoggedData { return this } - protected fun StringBuilder.appendDuration(duration: Duration): StringBuilder = - append("- Duration: ").appendLine(duration.toString(DurationUnit.SECONDS, 2)) + protected fun StringBuilder.appendDuration(timeout: Duration, duration: Duration, hasFinishedOnTime: Boolean) { + append("- Max permitted duration: ").appendLine(timeout.toString(DurationUnit.SECONDS, 2)) + appendDuration(duration) + appendLine("- Finished on time: $hasFinishedOnTime") + } - protected fun StringBuilder.appendCommonRunResult(runResult: RunResult): StringBuilder { - appendLine("- Exit code: ${runResult.exitCode ?: ""}") - appendDuration(runResult.duration) - appendLine() - appendPotentiallyLargeOutput(runResult.processOutput.stdOut.filteredOutput, "STDOUT", truncateLargeOutput = true) - appendLine() - appendPotentiallyLargeOutput(runResult.processOutput.stdErr, "STDERR", truncateLargeOutput = true) - return this + protected fun StringBuilder.appendDuration(duration: Duration) { + append("- Duration: ").appendLine(duration.toString(DurationUnit.SECONDS, 2)) } private fun StringBuilder.appendPotentiallyLargeOutput(output: String, subject: String, truncateLargeOutput: Boolean) { diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/NativeTestSupport.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/NativeTestSupport.kt index 7ecf63e499b..43b6dbbe372 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/NativeTestSupport.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/NativeTestSupport.kt @@ -32,7 +32,6 @@ import kotlin.reflect.KClass import kotlin.reflect.KParameter import kotlin.reflect.full.findAnnotation import kotlin.time.Duration.Companion.milliseconds -import kotlin.time.Duration.Companion.seconds class NativeBlackBoxTestSupport : BeforeEachCallback { /** @@ -218,7 +217,7 @@ private object NativeTestSupport { val executionTimeout = ClassLevelProperty.EXECUTION_TIMEOUT.readValue( enforcedProperties, { it.toLongOrNull()?.milliseconds }, - default = 10.seconds + default = Timeouts.DEFAULT_EXECUTION_TIMEOUT ) return Timeouts(executionTimeout) } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCase.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCase.kt index 82cc1fd7511..c4a68f6e213 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCase.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCase.kt @@ -9,6 +9,7 @@ package org.jetbrains.kotlin.konan.blackboxtest.support import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.WithTestRunnerExtras import org.jetbrains.kotlin.konan.blackboxtest.support.TestModule.Companion.allDependencies +import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunChecks import org.jetbrains.kotlin.konan.blackboxtest.support.util.* import org.jetbrains.kotlin.storage.LockBasedStorageManager import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue @@ -169,7 +170,8 @@ internal class TestCase( val modules: Set, val freeCompilerArgs: TestCompilerArgs, val nominalPackageName: PackageName, - val expectedOutputDataFile: File?, + val expectedOutputDataFile: File?, // TODO: refactor to TestRunCheck + val checks: TestRunChecks, val extras: Extras ) { sealed interface Extras diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestDirectives.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestDirectives.kt index 38009194d63..fd8255e77a8 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestDirectives.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestDirectives.kt @@ -10,6 +10,7 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.FREE_COMPI import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.INPUT_DATA_FILE import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.KIND import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.OUTPUT_DATA_FILE +import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.EXPECTED_TIMEOUT_FAILURE import org.jetbrains.kotlin.konan.blackboxtest.support.TestDirectives.TEST_RUNNER import org.jetbrains.kotlin.test.directives.model.RegisteredDirectives import org.jetbrains.kotlin.test.directives.model.SimpleDirectivesContainer @@ -67,11 +68,35 @@ internal object TestDirectives : SimpleDirectivesContainer() { val OUTPUT_DATA_FILE by stringDirective( description = """ - Specify the file which contains the expected program output. When program finishes its execution the actual output (stdout) + Specify the file which contains the expected program output. When program finishes its execution, the actual output (stdout) will be compared to the contents of this file. """.trimIndent() ) + // TODO: to be supported later +// val OUTPUT_REGEX by stringDirective( +// description = """ +// The regex that the expected program output should match to. When program finishes its execution, the actual output (stdout) +// will be checked against this regex. +// """.trimIndent() +// ) + + // TODO: to be supported later +// val OUTPUT_INCLUDES by stringDirective( +// description = """ +// The text that the expected program output should contain. When program finishes its execution, it will be checked if +// the actual output (stdout) contains this text. +// """.trimIndent() +// ) + + // TODO: to be supported later +// val OUTPUT_NOT_INCLUDES by stringDirective( +// description = """ +// The text that the expected program output should NOT contain. When program finishes its execution, it will be checked if +// the actual output (stdout) does nto contain this text. +// """.trimIndent() +// ) + val INPUT_DATA_FILE by stringDirective( description = """ Specify the file which contains the text to be passed to process' input (stdin). @@ -79,6 +104,19 @@ internal object TestDirectives : SimpleDirectivesContainer() { """.trimIndent() ) + // TODO: to be supported later +// val EXIT_CODE by stringDirective( +// description = """ +// Specify the exit code that the test should finish with. Example: // EXIT_CODE: 42 +// To indicate any non-zero exit code use // EXIT_CODE: !0 +// Note that this directive makes sense only in combination with // KIND: STANDALONE_NO_TR +// """.trimIndent() +// ) + + val EXPECTED_TIMEOUT_FAILURE by directive( + description = "Whether the test is expected to fail on timeout" + ) + val FREE_COMPILER_ARGS by stringDirective( description = "Specify free compiler arguments for Kotlin/Native compiler" ) @@ -203,6 +241,9 @@ internal fun parseFileName(parsedDirective: RegisteredDirectivesParser.ParsedDir return fileName } +internal fun parseExpectedTimeoutFailure(registeredDirectives: RegisteredDirectives): Boolean = + EXPECTED_TIMEOUT_FAILURE in registeredDirectives + internal fun parseFreeCompilerArgs(registeredDirectives: RegisteredDirectives, location: Location): TestCompilerArgs { if (FREE_COMPILER_ARGS !in registeredDirectives) return TestCompilerArgs.EMPTY diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/ExtTestCaseGroupProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/ExtTestCaseGroupProvider.kt index 624406a6444..2535b72cea2 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/ExtTestCaseGroupProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/ExtTestCaseGroupProvider.kt @@ -19,10 +19,8 @@ import org.jetbrains.kotlin.config.CommonConfigurationKeys import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.konan.blackboxtest.support.* import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.WithTestRunnerExtras -import org.jetbrains.kotlin.konan.blackboxtest.support.settings.ExternalSourceTransformersProvider -import org.jetbrains.kotlin.konan.blackboxtest.support.settings.GeneratedSources -import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Settings -import org.jetbrains.kotlin.konan.blackboxtest.support.settings.TestRoots +import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunChecks +import org.jetbrains.kotlin.konan.blackboxtest.support.settings.* import org.jetbrains.kotlin.konan.blackboxtest.support.util.* import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.name.Name @@ -69,7 +67,8 @@ internal class ExtTestCaseGroupProvider : TestCaseGroupProvider, TestDisposable( structureFactory = structureFactory, customSourceTransformers = settings.get().getSourceTransformers(testDataFile), testRoots = settings.get(), - generatedSources = settings.get() + generatedSources = settings.get(), + timeouts = settings.get() ) if (extTestDataFile.isRelevant) @@ -102,7 +101,8 @@ private class ExtTestDataFile( structureFactory: ExtTestDataFileStructureFactory, customSourceTransformers: ExternalSourceTransformers?, testRoots: TestRoots, - private val generatedSources: GeneratedSources + private val generatedSources: GeneratedSources, + private val timeouts: Timeouts ) { private val structure by lazy { val allSourceTransformers: ExternalSourceTransformers = if (customSourceTransformers.isNullOrEmpty()) @@ -554,6 +554,7 @@ private class ExtTestDataFile( freeCompilerArgs = assembleFreeCompilerArgs(), nominalPackageName = testDataFileSettings.nominalPackageName, expectedOutputDataFile = null, + checks = TestRunChecks.Default(timeouts.executionTimeout), extras = WithTestRunnerExtras(runnerType = TestRunnerType.DEFAULT) ) testCase.initialize(sharedModules::get) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/PredefinedTestCaseGroupProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/PredefinedTestCaseGroupProvider.kt index 80db45acf80..fbf60c53231 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/PredefinedTestCaseGroupProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/PredefinedTestCaseGroupProvider.kt @@ -7,8 +7,10 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.group import org.jetbrains.kotlin.konan.blackboxtest.support.* import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.WithTestRunnerExtras +import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunChecks import org.jetbrains.kotlin.konan.blackboxtest.support.settings.KotlinNativeHome import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Settings +import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Timeouts import org.jetbrains.kotlin.konan.blackboxtest.support.util.ThreadSafeCache import org.jetbrains.kotlin.konan.blackboxtest.support.util.expandGlobTo import org.jetbrains.kotlin.konan.blackboxtest.support.util.getAbsoluteFile @@ -62,6 +64,7 @@ internal class PredefinedTestCaseGroupProvider(annotation: PredefinedTestCases) .parseCompilerArgs(settings) { "Failed to parse free compiler arguments for test case $testCaseId" }, nominalPackageName = PackageName(testCaseId.uniqueName), expectedOutputDataFile = null, + checks = TestRunChecks.Default(settings.get().executionTimeout), extras = WithTestRunnerExtras( runnerType = predefinedTestCase.runnerType, ignoredTests = predefinedTestCase.ignoredTests.toSet() diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt index 06710b707f6..51525bd2f5d 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt @@ -8,9 +8,12 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.group import org.jetbrains.kotlin.konan.blackboxtest.support.* import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.NoTestRunnerExtras import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.WithTestRunnerExtras +import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunCheck +import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunChecks import org.jetbrains.kotlin.konan.blackboxtest.support.settings.GeneratedSources import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Settings import org.jetbrains.kotlin.konan.blackboxtest.support.settings.TestRoots +import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Timeouts import org.jetbrains.kotlin.konan.blackboxtest.support.util.* import org.jetbrains.kotlin.test.directives.model.Directive import org.jetbrains.kotlin.test.services.JUnit5Assertions @@ -180,6 +183,14 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider { val freeCompilerArgs = parseFreeCompilerArgs(registeredDirectives, location) val expectedOutputDataFile = parseOutputDataFile(baseDir = testDataFile.parentFile, registeredDirectives, location) val testKind = parseTestKind(registeredDirectives, location) + val expectedTimeoutFailure = parseExpectedTimeoutFailure(registeredDirectives) + + val checks = TestRunChecks { + this += if (expectedTimeoutFailure) + TestRunCheck.ExecutionTimeout.ShouldExceed(settings.get().executionTimeout) + else + TestRunCheck.ExecutionTimeout.ShouldNotExceed(settings.get().executionTimeout) + } if (testKind == TestKind.REGULAR) { // Fix package declarations to avoid unintended conflicts between symbols with the same name in different test cases. @@ -193,6 +204,7 @@ internal class StandardTestCaseGroupProvider : TestCaseGroupProvider { freeCompilerArgs = freeCompilerArgs, nominalPackageName = nominalPackageName, expectedOutputDataFile = expectedOutputDataFile, + checks = checks, extras = if (testKind == TestKind.STANDALONE_NO_TR) NoTestRunnerExtras( entryPoint = parseEntryPoint(registeredDirectives, location), diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt index f9e146cac6a..2cbde88a5aa 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt @@ -12,7 +12,7 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestOutputFilter import java.io.ByteArrayOutputStream import kotlin.time.* -internal abstract class AbstractLocalProcessRunner(private val executionTimeout: Duration) : AbstractRunner() { +internal abstract class AbstractLocalProcessRunner(private val checks: TestRunChecks) : AbstractRunner() { protected abstract val visibleProcessName: String protected abstract val executable: TestExecutable protected abstract val programArgs: List @@ -26,6 +26,8 @@ internal abstract class AbstractLocalProcessRunner(private val executionTimeo val unfilteredOutput = UnfilteredProcessOutput() val unfilteredOutputReader: Job + val executionTimeout: Duration = checks.executionTimeoutCheck.timeout + val process: Process val hasFinishedOnTime: Boolean @@ -41,16 +43,11 @@ internal abstract class AbstractLocalProcessRunner(private val executionTimeo ) } - if (hasFinishedOnTime) { + val exitCode: Int? = if (hasFinishedOnTime) { unfilteredOutputReader.join() // Wait until all output streams are drained. - - RunResult.Completed( - exitCode = process.exitValue(), - duration = duration, - processOutput = unfilteredOutput.toProcessOutput(outputFilter) - ) + process.exitValue() } else { - val exitCode: Int? = try { // It could happen just by an accident that the process has exited by itself. + try { // It could happen just by an accident that the process has exited by itself. val exitCode = process.exitValue() // Fetch exit code. unfilteredOutputReader.join() // Wait until all streams are drained. exitCode @@ -59,22 +56,37 @@ internal abstract class AbstractLocalProcessRunner(private val executionTimeo process.destroyForcibly() // kill -9 null } - - RunResult.TimeoutExceeded( - timeout = executionTimeout, - exitCode = exitCode, - duration = duration, - processOutput = unfilteredOutput.toProcessOutput(outputFilter) - ) } + + RunResult( + exitCode = exitCode, + timeout = executionTimeout, + duration = duration, + hasFinishedOnTime = hasFinishedOnTime, + processOutput = unfilteredOutput.toProcessOutput(outputFilter) + ) } } - abstract override fun buildResultHandler(runResult: RunResult.Completed): ResultHandler // Narrow returned type. + abstract override fun buildResultHandler(runResult: RunResult): ResultHandler // Narrow returned type. - abstract inner class ResultHandler(runResult: RunResult.Completed) : AbstractRunner.ResultHandler(runResult) { + abstract inner class ResultHandler(runResult: RunResult) : AbstractRunner.ResultHandler(runResult) { override fun handle(): R { - verifyExpectation(runResult.exitCode == 0) { "$visibleProcessName exited with non-zero code." } + checks.forEach { check -> + when (check) { + is TestRunCheck.ExecutionTimeout.ShouldNotExceed -> verifyExpectation(runResult.hasFinishedOnTime) { + "Timeout exceeded during test execution." + } + is TestRunCheck.ExecutionTimeout.ShouldExceed -> verifyExpectation(!runResult.hasFinishedOnTime) { + "Test is expected to fail with exceeded timeout, which hasn't happened." + } + } + } + + // Don't check exit code if it is unknown. + runResult.exitCode?.let { exitCode -> + verifyExpectation(exitCode == 0) { "$visibleProcessName exited with non-zero code." } + } return doHandle() } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractRunner.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractRunner.kt index f424512d3e9..efdaec4d2e4 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractRunner.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractRunner.kt @@ -7,26 +7,18 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.runner import org.jetbrains.kotlin.konan.blackboxtest.support.LoggedData import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue -import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail import org.opentest4j.TestAbortedException internal abstract class AbstractRunner { protected abstract fun buildRun(): AbstractRun - protected abstract fun buildResultHandler(runResult: RunResult.Completed): ResultHandler + protected abstract fun buildResultHandler(runResult: RunResult): ResultHandler protected abstract fun getLoggedParameters(): LoggedData.TestRunParameters protected abstract fun handleUnexpectedFailure(t: Throwable): Nothing fun run(): R = try { val run = buildRun() - - val resultHandler = when (val runResult = run.run()) { - is RunResult.TimeoutExceeded -> fail { - LoggedData.TestRunTimeoutExceeded(getLoggedParameters(), runResult) - .withErrorMessage("Timeout exceeded during test execution.") - } - is RunResult.Completed -> buildResultHandler(runResult) - } - + val runResult = run.run() + val resultHandler = buildResultHandler(runResult) resultHandler.handle() } catch (t: Throwable) { when (t) { @@ -42,7 +34,7 @@ internal abstract class AbstractRunner { fun run(): RunResult } - abstract inner class ResultHandler(protected val runResult: RunResult.Completed) { + abstract inner class ResultHandler(protected val runResult: RunResult) { abstract fun getLoggedRun(): LoggedData abstract fun handle(): R diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/BaseTestRunProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/BaseTestRunProvider.kt index 8969e6b72a8..dec510829d5 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/BaseTestRunProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/BaseTestRunProvider.kt @@ -9,12 +9,13 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.NoTestRunnerExtras import org.jetbrains.kotlin.konan.blackboxtest.support.TestKind import org.jetbrains.kotlin.konan.blackboxtest.support.TestName -import org.jetbrains.kotlin.test.services.JUnit5Assertions +import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue +import org.jetbrains.kotlin.utils.addIfNotNull internal open class BaseTestRunProvider { protected fun createTestRun(testCase: TestCase, executable: TestExecutable, testRunName: String, testName: TestName?): TestRun { val runParameters = getTestRunParameters(testCase, testName) - return TestRun(displayName = testRunName, executable, runParameters, testCase.id) + return TestRun(displayName = testRunName, executable, runParameters, testCase.id, testCase.checks) } protected fun createSingleTestRun(testCase: TestCase, executable: TestExecutable): TestRun = createTestRun( @@ -24,25 +25,24 @@ internal open class BaseTestRunProvider { testName = null ) - private fun getTestRunParameters(testCase: TestCase, testName: TestName?): List = with(testCase) { - when (kind) { + private fun getTestRunParameters(testCase: TestCase, testName: TestName?): List = buildList { + addIfNotNull(testCase.expectedOutputDataFile?.let(TestRunParameter::WithExpectedOutputData)) + + when (testCase.kind) { TestKind.STANDALONE_NO_TR -> { - JUnit5Assertions.assertTrue(testName == null) - listOfNotNull( - extras().inputDataFile?.let(TestRunParameter::WithInputData), - expectedOutputDataFile?.let(TestRunParameter::WithExpectedOutputData) + assertTrue(testName == null) + addIfNotNull(testCase.extras().inputDataFile?.let(TestRunParameter::WithInputData)) + } + TestKind.STANDALONE -> { + add(TestRunParameter.WithTCTestLogger) + addIfNotNull(testName?.let(TestRunParameter::WithTestFilter)) + } + TestKind.REGULAR -> { + add(TestRunParameter.WithTCTestLogger) + addIfNotNull( + testName?.let(TestRunParameter::WithTestFilter) ?: TestRunParameter.WithPackageFilter(testCase.nominalPackageName) ) } - TestKind.STANDALONE -> listOfNotNull( - TestRunParameter.WithTCTestLogger, - testName?.let(TestRunParameter::WithTestFilter), - expectedOutputDataFile?.let(TestRunParameter::WithExpectedOutputData) - ) - TestKind.REGULAR -> listOfNotNull( - TestRunParameter.WithTCTestLogger, - testName?.let(TestRunParameter::WithTestFilter) ?: TestRunParameter.WithPackageFilter(nominalPackageName), - expectedOutputDataFile?.let(TestRunParameter::WithExpectedOutputData) - ) } } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestNameExtractor.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestNameExtractor.kt index 122b70c6e74..7838d77902c 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestNameExtractor.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestNameExtractor.kt @@ -10,12 +10,11 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.TestName import org.jetbrains.kotlin.konan.blackboxtest.support.util.GTestListing import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestOutputFilter import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail -import kotlin.time.Duration internal class LocalTestNameExtractor( override val executable: TestExecutable, - executionTimeout: Duration -) : AbstractLocalProcessRunner>(executionTimeout) { + checks: TestRunChecks +) : AbstractLocalProcessRunner>(checks) { override val visibleProcessName get() = "Test name extractor" override val programArgs = listOf(executable.executableFile.path, "--ktest_list_tests") override val outputFilter get() = TestOutputFilter.NO_FILTERING @@ -27,7 +26,7 @@ internal class LocalTestNameExtractor( runParameters = null ) - override fun buildResultHandler(runResult: RunResult.Completed) = ResultHandler(runResult) + override fun buildResultHandler(runResult: RunResult) = ResultHandler(runResult) override fun handleUnexpectedFailure(t: Throwable) = fail { LoggedData.TestRunUnexpectedFailure(getLoggedParameters(), t) @@ -35,7 +34,7 @@ internal class LocalTestNameExtractor( } inner class ResultHandler( - runResult: RunResult.Completed + runResult: RunResult ) : AbstractLocalProcessRunner>.ResultHandler(runResult) { override fun getLoggedRun() = LoggedData.TestRun(getLoggedParameters(), runResult) override fun doHandle() = GTestListing.parse(runResult.processOutput.stdOut.filteredOutput) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt index 906a5c92abd..5c01a4689ca 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt @@ -12,12 +12,8 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestOutputFilter import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestReport import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail import org.junit.jupiter.api.Assumptions.assumeFalse -import kotlin.time.Duration -internal class LocalTestRunner( - private val testRun: TestRun, - executionTimeout: Duration -) : AbstractLocalProcessRunner(executionTimeout) { +internal class LocalTestRunner(private val testRun: TestRun) : AbstractLocalProcessRunner(testRun.checks) { override val visibleProcessName get() = "Tested process" override val executable get() = testRun.executable @@ -43,14 +39,14 @@ internal class LocalTestRunner( } } - override fun buildResultHandler(runResult: RunResult.Completed) = ResultHandler(runResult) + override fun buildResultHandler(runResult: RunResult) = ResultHandler(runResult) override fun handleUnexpectedFailure(t: Throwable) = fail { LoggedData.TestRunUnexpectedFailure(getLoggedParameters(), t) .withErrorMessage("Test execution failed with unexpected exception.") } - inner class ResultHandler(runResult: RunResult.Completed) : AbstractLocalProcessRunner.ResultHandler(runResult) { + inner class ResultHandler(runResult: RunResult) : AbstractLocalProcessRunner.ResultHandler(runResult) { override fun getLoggedRun() = LoggedData.TestRun(getLoggedParameters(), runResult) override fun doHandle() { diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/RunResult.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/RunResult.kt index b7f012f1429..e5a5f08fe2f 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/RunResult.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/RunResult.kt @@ -8,23 +8,17 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.runner import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestOutputFilter import kotlin.time.Duration -internal sealed interface RunResult { - val exitCode: Int? - val duration: Duration +internal data class RunResult( + val exitCode: Int?, + val timeout: Duration, + val duration: Duration, + val hasFinishedOnTime: Boolean, val processOutput: ProcessOutput - - data class Completed( - override val exitCode: Int, - override val duration: Duration, - override val processOutput: ProcessOutput - ) : RunResult - - data class TimeoutExceeded( - val timeout: Duration, - override val exitCode: Int?, - override val duration: Duration, - override val processOutput: ProcessOutput - ) : RunResult +) { + init { + // null exit code is possible only when test run hasn't finished on time. + check(exitCode != null || !hasFinishedOnTime) + } } internal class ProcessOutput(val stdOut: TestOutputFilter.FilteredOutput, val stdErr: String) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRun.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRun.kt index 848145937ad..93f5f516fe2 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRun.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRun.kt @@ -54,7 +54,8 @@ internal class TestRun( val displayName: String, val executable: TestExecutable, val runParameters: List, - val testCaseId: TestCaseId + val testCaseId: TestCaseId, + val checks: TestRunChecks ) internal sealed interface TestRunParameter { diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunChecks.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunChecks.kt new file mode 100644 index 00000000000..d0289c855f9 --- /dev/null +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunChecks.kt @@ -0,0 +1,46 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.konan.blackboxtest.support.runner + +import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Timeouts +import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstance +import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull +import kotlin.time.Duration + +internal sealed interface TestRunCheck { + sealed class ExecutionTimeout(val timeout: Duration) : TestRunCheck { + class ShouldNotExceed(timeout: Duration) : ExecutionTimeout(timeout) + class ShouldExceed(timeout: Duration) : ExecutionTimeout(timeout) + } +} + +internal class TestRunChecks(builderAction: MutableCollection.() -> Unit) : Iterable { + constructor(vararg checks: TestRunCheck) : this({ addAll(checks) }) + + private val allChecks = buildList { + builderAction() + addDefaults() + } + + val executionTimeoutCheck: TestRunCheck.ExecutionTimeout get() = allChecks.firstIsInstance() + + override fun iterator() = allChecks.iterator() + + companion object { + // The most frequently used case: + @Suppress("TestFunctionName") + fun Default(timeout: Duration) = TestRunChecks(TestRunCheck.ExecutionTimeout.ShouldNotExceed(timeout)) + + private fun MutableCollection.addDefaults() { + if (isMissing()) { + add(TestRunCheck.ExecutionTimeout.ShouldNotExceed(Timeouts.DEFAULT_EXECUTION_TIMEOUT)) + } + } + + private inline fun Collection.isMissing(): Boolean = + firstIsInstanceOrNull() == null + } +} diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunners.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunners.kt index c3409902f6c..2ffa3d398f8 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunners.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunners.kt @@ -16,7 +16,7 @@ internal object TestRunners { fun createProperTestRunner(testRun: TestRun, settings: Settings): AbstractRunner<*> = with(settings) { with(get()) { if (testTarget == hostTarget) - LocalTestRunner(testRun, get().executionTimeout) + LocalTestRunner(testRun) else runningAtNonHostTarget() } @@ -26,7 +26,7 @@ internal object TestRunners { fun extractTestNames(executable: TestExecutable, settings: Settings): Collection = with(settings) { with(get()) { if (testTarget == hostTarget) - LocalTestNameExtractor(executable, get().executionTimeout).run() + LocalTestNameExtractor(executable, TestRunChecks.Default(get().executionTimeout)).run() else runningAtNonHostTarget() } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/settings/TestProcessSettings.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/settings/TestProcessSettings.kt index 2eef713eb8d..64f15b8d69b 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/settings/TestProcessSettings.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/settings/TestProcessSettings.kt @@ -9,6 +9,7 @@ import org.jetbrains.kotlin.konan.target.KonanTarget import java.io.File import java.util.* import kotlin.time.Duration +import kotlin.time.Duration.Companion.seconds /** * The tested and the host Kotlin/Native targets. @@ -113,7 +114,11 @@ internal class BaseDirs(val testBuildDir: File) /** * Timeouts. */ -internal class Timeouts(val executionTimeout: Duration) +internal class Timeouts(val executionTimeout: Duration) { + companion object { + val DEFAULT_EXECUTION_TIMEOUT: Duration get() = 15.seconds + } +} /** * Used cache mode.