From 5ddfd4fd9197773de7441769e4bec257d92db484 Mon Sep 17 00:00:00 2001 From: Pavel Punegov Date: Tue, 20 Feb 2024 11:10:57 +0100 Subject: [PATCH] [K/N][test] Check test filtering with different run parameters Adds additional checks when different test filtering is used, like ignored tests filter. This makes SharedExecutionBuilder be able to handle tests separately but run together in the executable. --- .../blackbox/support/runner/ResultHandler.kt | 59 ++++++++++++++++--- .../support/runner/SharedExecutionBuilder.kt | 29 ++++++--- .../test/blackbox/support/runner/TestRun.kt | 3 + 3 files changed, 74 insertions(+), 17 deletions(-) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/ResultHandler.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/ResultHandler.kt index 9cfbf5583a8..823c258e931 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/ResultHandler.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/ResultHandler.kt @@ -8,11 +8,13 @@ package org.jetbrains.kotlin.konan.test.blackbox.support.runner import com.intellij.openapi.util.SystemInfo import com.intellij.openapi.util.text.StringUtilRt.convertLineSeparators import org.jetbrains.kotlin.konan.test.blackbox.support.LoggedData +import org.jetbrains.kotlin.konan.test.blackbox.support.TestKind import org.jetbrains.kotlin.konan.test.blackbox.support.TestName import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck.ExecutionTimeout import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck.ExitCode import org.jetbrains.kotlin.konan.test.blackbox.support.settings.configurables import org.jetbrains.kotlin.konan.test.blackbox.support.util.TestOutputFilter +import org.jetbrains.kotlin.konan.test.blackbox.support.util.TestReport import org.jetbrains.kotlin.native.executors.RunProcessResult import org.jetbrains.kotlin.native.executors.runProcess import org.junit.jupiter.api.Assumptions @@ -100,10 +102,7 @@ internal class ResultHandler( if (check.testOutputFilter != TestOutputFilter.NO_FILTERING) { val testReport = runResult.processOutput.stdOut.testReport - if (testReport == null) { - add("TestRun has TestFiltering enabled, but test report is null") - } - checkNotNull(testReport) + checkNotNull(testReport) { "TestRun has TestFiltering enabled, but test report is null" } if (testReport.isEmpty()) add("No tests have been found. Test report is empty") @@ -117,14 +116,48 @@ internal class ResultHandler( testReport.ignoredTests.filter { testName -> !testMatches(testName) }, "Excessive tests have been ignored" ) + + verifyNoSuchTests(testReport.failedTests, "Failed tests found in the test report") + + testReport.checkDisabled() } - verifyNoSuchTests(testReport.failedTests, "Failed tests found in the test report") + testRun.runParameters.getAll { + verifyNoSuchTests( + testReport.passedTests.filter { testName -> !testMatches(testName) }, + "Ignored tests have been executed" + ) + verifyNoSuchTests( + testReport.failedTests.filter { testName -> + testName.packageName == testRun.testCase.nominalPackageName + }, + "Test failure found in the test report" + ) - Assumptions.assumeFalse( - testReport.ignoredTests.isNotEmpty() && testReport.passedTests.isEmpty(), - "Test case is disabled" - ) + Assumptions.assumeFalse( + testReport.ignoredTests.any { testName -> + testName.packageName == testRun.testCase.nominalPackageName + }, + "Test case is disabled" + ) + } + + if (!testRun.runParameters.has()) { + verifyNoSuchTests( + testReport.failedTests.filter { testName -> + testName.packageName == testRun.testCase.nominalPackageName + }, + "Test ${testRun.testCase.id} failure found in the test report" + ) + + testReport.checkDisabled() + } + + if (testRun.testCase.kind == TestKind.STANDALONE) { + verifyNoSuchTests(testReport.failedTests, "Failed tests found in the test report") + + testReport.checkDisabled() + } } } } @@ -136,6 +169,7 @@ internal class ResultHandler( } } else { val runResultInfo = buildString { + appendLine("TestCase Kind: ${testRun.testCase.kind}") appendLine("TestCaseId: ${testRun.testCase.id}") appendLine("Exit code: ${runResult.exitCode}") appendLine("Filtered test output is") @@ -165,6 +199,13 @@ internal class ResultHandler( ) } } + + private fun TestReport.checkDisabled() { + Assumptions.assumeFalse( + ignoredTests.isNotEmpty() && passedTests.isEmpty(), + "Test case is disabled" + ) + } } internal fun doFileCheck(check: TestRunCheck.FileCheckMatcher, fileCheckDump: File): RunProcessResult { diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt index 88612e4d48b..aa645c76dd5 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt @@ -24,7 +24,7 @@ import java.util.concurrent.ConcurrentHashMap * @see SharedExecutionTestRunner */ internal object SharedExecutionBuilder { - private val executionResults: ConcurrentHashMap> = ConcurrentHashMap() + private val executableToSharedRun: ConcurrentHashMap = ConcurrentHashMap() private val testCasesToExecuteSeparately: ConcurrentHashMap> = ConcurrentHashMap() fun buildRunner(settings: Settings, executor: Executor, testRun: TestRun): AbstractRunner { @@ -38,7 +38,7 @@ internal object SharedExecutionBuilder { return RunnerWithExecutor(executor, testRun) } - return executionResults.computeIfAbsent(testRun.executable) { + val sharedTestRun = executableToSharedRun.computeIfAbsent(testRun.executable) { // Get ignored tests to exclude them from run by adding the test filtering option val ignoredTests = if (testRun.testCase.extras is TestCase.WithTestRunnerExtras) { testRun.testCase.extras.ignoredTests @@ -60,7 +60,7 @@ internal object SharedExecutionBuilder { executionTimeoutCheck = TestRunCheck.ExecutionTimeout.ShouldNotExceed(timeout) ) - val sharedTestRun = TestRun( + TestRun( displayName = "Shared TestRun for ${testRun.executable.executable.path} made from ${testRun.displayName}", executable = testRun.executable, runParameters = runParameters, @@ -68,8 +68,9 @@ internal object SharedExecutionBuilder { checks = checks, expectedFailure = false ) - CachedRunResultRunner(executor, sharedTestRun) } + + return CachedRunResultRunner(executor, testRun, sharedTestRun) } private fun Settings.computeSeparateTestCases(testRun: TestRun): MutableList = @@ -100,11 +101,23 @@ internal object SharedExecutionBuilder { }.toMutableList() } - private class CachedRunResultRunner(executor: Executor, testRun: TestRun) : RunnerWithExecutor(executor, testRun) { - private val cachedRunResult by lazy(LazyThreadSafetyMode.SYNCHRONIZED) { - super.buildRun().run() + private val cachedRunResult = ConcurrentHashMap() + + private class CachedRunResultRunner(executor: Executor, private val testRun: TestRun, private val sharedTestRun: TestRun) : + RunnerWithExecutor(executor, sharedTestRun) { + + override fun buildRun() = AbstractRun { + cachedRunResult.computeIfAbsent(sharedTestRun) { super.buildRun().run() } } - override fun buildRun() = AbstractRun { cachedRunResult } + override fun buildResultHandler(runResult: RunResult): ResultHandler = ResultHandler( + runResult = runResult, + visibleProcessName = "Test process under ${this::class.simpleName}", + checks = sharedTestRun.checks, + testRun = testRun.copy( + runParameters = sharedTestRun.runParameters + ), + loggedParameters = getLoggedParameters() + ) } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRun.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRun.kt index cfd2f884282..9d0e526d3d0 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRun.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRun.kt @@ -146,6 +146,9 @@ internal inline fun List.get(on firstIsInstanceOrNull()?.let(onFound) } +internal inline fun List.getAll(onFound: T.() -> Unit) { + filterIsInstance().forEach(onFound) +} // must be in sync with `fromGTestPattern(String)` in kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestRunner.kt internal fun fromGTestPattern(pattern: String): Regex {