diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/ResultHandler.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/ResultHandler.kt index 9cfbf5583a8..823c258e931 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/ResultHandler.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/ResultHandler.kt @@ -8,11 +8,13 @@ package org.jetbrains.kotlin.konan.test.blackbox.support.runner import com.intellij.openapi.util.SystemInfo import com.intellij.openapi.util.text.StringUtilRt.convertLineSeparators import org.jetbrains.kotlin.konan.test.blackbox.support.LoggedData +import org.jetbrains.kotlin.konan.test.blackbox.support.TestKind import org.jetbrains.kotlin.konan.test.blackbox.support.TestName import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck.ExecutionTimeout import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck.ExitCode import org.jetbrains.kotlin.konan.test.blackbox.support.settings.configurables import org.jetbrains.kotlin.konan.test.blackbox.support.util.TestOutputFilter +import org.jetbrains.kotlin.konan.test.blackbox.support.util.TestReport import org.jetbrains.kotlin.native.executors.RunProcessResult import org.jetbrains.kotlin.native.executors.runProcess import org.junit.jupiter.api.Assumptions @@ -100,10 +102,7 @@ internal class ResultHandler( if (check.testOutputFilter != TestOutputFilter.NO_FILTERING) { val testReport = runResult.processOutput.stdOut.testReport - if (testReport == null) { - add("TestRun has TestFiltering enabled, but test report is null") - } - checkNotNull(testReport) + checkNotNull(testReport) { "TestRun has TestFiltering enabled, but test report is null" } if (testReport.isEmpty()) add("No tests have been found. Test report is empty") @@ -117,14 +116,48 @@ internal class ResultHandler( testReport.ignoredTests.filter { testName -> !testMatches(testName) }, "Excessive tests have been ignored" ) + + verifyNoSuchTests(testReport.failedTests, "Failed tests found in the test report") + + testReport.checkDisabled() } - verifyNoSuchTests(testReport.failedTests, "Failed tests found in the test report") + testRun.runParameters.getAll { + verifyNoSuchTests( + testReport.passedTests.filter { testName -> !testMatches(testName) }, + "Ignored tests have been executed" + ) + verifyNoSuchTests( + testReport.failedTests.filter { testName -> + testName.packageName == testRun.testCase.nominalPackageName + }, + "Test failure found in the test report" + ) - Assumptions.assumeFalse( - testReport.ignoredTests.isNotEmpty() && testReport.passedTests.isEmpty(), - "Test case is disabled" - ) + Assumptions.assumeFalse( + testReport.ignoredTests.any { testName -> + testName.packageName == testRun.testCase.nominalPackageName + }, + "Test case is disabled" + ) + } + + if (!testRun.runParameters.has()) { + verifyNoSuchTests( + testReport.failedTests.filter { testName -> + testName.packageName == testRun.testCase.nominalPackageName + }, + "Test ${testRun.testCase.id} failure found in the test report" + ) + + testReport.checkDisabled() + } + + if (testRun.testCase.kind == TestKind.STANDALONE) { + verifyNoSuchTests(testReport.failedTests, "Failed tests found in the test report") + + testReport.checkDisabled() + } } } } @@ -136,6 +169,7 @@ internal class ResultHandler( } } else { val runResultInfo = buildString { + appendLine("TestCase Kind: ${testRun.testCase.kind}") appendLine("TestCaseId: ${testRun.testCase.id}") appendLine("Exit code: ${runResult.exitCode}") appendLine("Filtered test output is") @@ -165,6 +199,13 @@ internal class ResultHandler( ) } } + + private fun TestReport.checkDisabled() { + Assumptions.assumeFalse( + ignoredTests.isNotEmpty() && passedTests.isEmpty(), + "Test case is disabled" + ) + } } internal fun doFileCheck(check: TestRunCheck.FileCheckMatcher, fileCheckDump: File): RunProcessResult { diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt index 88612e4d48b..aa645c76dd5 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt @@ -24,7 +24,7 @@ import java.util.concurrent.ConcurrentHashMap * @see SharedExecutionTestRunner */ internal object SharedExecutionBuilder { - private val executionResults: ConcurrentHashMap> = ConcurrentHashMap() + private val executableToSharedRun: ConcurrentHashMap = ConcurrentHashMap() private val testCasesToExecuteSeparately: ConcurrentHashMap> = ConcurrentHashMap() fun buildRunner(settings: Settings, executor: Executor, testRun: TestRun): AbstractRunner { @@ -38,7 +38,7 @@ internal object SharedExecutionBuilder { return RunnerWithExecutor(executor, testRun) } - return executionResults.computeIfAbsent(testRun.executable) { + val sharedTestRun = executableToSharedRun.computeIfAbsent(testRun.executable) { // Get ignored tests to exclude them from run by adding the test filtering option val ignoredTests = if (testRun.testCase.extras is TestCase.WithTestRunnerExtras) { testRun.testCase.extras.ignoredTests @@ -60,7 +60,7 @@ internal object SharedExecutionBuilder { executionTimeoutCheck = TestRunCheck.ExecutionTimeout.ShouldNotExceed(timeout) ) - val sharedTestRun = TestRun( + TestRun( displayName = "Shared TestRun for ${testRun.executable.executable.path} made from ${testRun.displayName}", executable = testRun.executable, runParameters = runParameters, @@ -68,8 +68,9 @@ internal object SharedExecutionBuilder { checks = checks, expectedFailure = false ) - CachedRunResultRunner(executor, sharedTestRun) } + + return CachedRunResultRunner(executor, testRun, sharedTestRun) } private fun Settings.computeSeparateTestCases(testRun: TestRun): MutableList = @@ -100,11 +101,23 @@ internal object SharedExecutionBuilder { }.toMutableList() } - private class CachedRunResultRunner(executor: Executor, testRun: TestRun) : RunnerWithExecutor(executor, testRun) { - private val cachedRunResult by lazy(LazyThreadSafetyMode.SYNCHRONIZED) { - super.buildRun().run() + private val cachedRunResult = ConcurrentHashMap() + + private class CachedRunResultRunner(executor: Executor, private val testRun: TestRun, private val sharedTestRun: TestRun) : + RunnerWithExecutor(executor, sharedTestRun) { + + override fun buildRun() = AbstractRun { + cachedRunResult.computeIfAbsent(sharedTestRun) { super.buildRun().run() } } - override fun buildRun() = AbstractRun { cachedRunResult } + override fun buildResultHandler(runResult: RunResult): ResultHandler = ResultHandler( + runResult = runResult, + visibleProcessName = "Test process under ${this::class.simpleName}", + checks = sharedTestRun.checks, + testRun = testRun.copy( + runParameters = sharedTestRun.runParameters + ), + loggedParameters = getLoggedParameters() + ) } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRun.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRun.kt index cfd2f884282..9d0e526d3d0 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRun.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRun.kt @@ -146,6 +146,9 @@ internal inline fun List.get(on firstIsInstanceOrNull()?.let(onFound) } +internal inline fun List.getAll(onFound: T.() -> Unit) { + filterIsInstance().forEach(onFound) +} // must be in sync with `fromGTestPattern(String)` in kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestRunner.kt internal fun fromGTestPattern(pattern: String): Regex {