From 42cba927e65eaaddd1f0565c16b8fec96701d38e Mon Sep 17 00:00:00 2001 From: Pavel Punegov Date: Tue, 13 Feb 2024 19:41:27 +0100 Subject: [PATCH] [K/N][test] Infra: implement race-free way to separate test cases With shared test running test cases that may fail or timeout should be excluded from shared run and executed separately. This is done by getting all test cases for the current executable and putting them to the map that is checked for all TestRuns --- .../support/runner/SharedExecutionBuilder.kt | 50 ++++++++++++++----- .../support/runner/TestRunProvider.kt | 2 +- .../blackbox/support/runner/TestRunners.kt | 2 +- 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt index ed2a783a36c..88612e4d48b 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt @@ -8,6 +8,8 @@ package org.jetbrains.kotlin.konan.test.blackbox.support.runner import org.jetbrains.kotlin.konan.test.blackbox.support.TestCase import org.jetbrains.kotlin.konan.test.blackbox.support.TestKind import org.jetbrains.kotlin.konan.test.blackbox.support.TestName +import org.jetbrains.kotlin.konan.test.blackbox.support.settings.BlackBoxTestInstances +import org.jetbrains.kotlin.konan.test.blackbox.support.settings.Settings import org.jetbrains.kotlin.konan.test.blackbox.support.settings.SharedExecutionTestRunner import org.jetbrains.kotlin.native.executors.Executor import java.util.concurrent.ConcurrentHashMap @@ -23,18 +25,16 @@ import java.util.concurrent.ConcurrentHashMap */ internal object SharedExecutionBuilder { private val executionResults: ConcurrentHashMap> = ConcurrentHashMap() - private val testRunsToExecuteSeparately: ConcurrentHashMap> = ConcurrentHashMap() - - fun buildRunner(executor: Executor, testRun: TestRun): AbstractRunner { - if (testRun.expectedFailure || testRun.checks.executionTimeoutCheck is TestRunCheck.ExecutionTimeout.ShouldExceed) { - // If the test run is expected to fail or timeout it should not be executed with others. - // Add it to the map of ignored test cases for the executable - testRunsToExecuteSeparately.computeIfAbsent(testRun.executable) { mutableListOf() } += testRun.testCase + private val testCasesToExecuteSeparately: ConcurrentHashMap> = ConcurrentHashMap() + fun buildRunner(settings: Settings, executor: Executor, testRun: TestRun): AbstractRunner { + if (testRun.testCase.kind != TestKind.REGULAR) { return RunnerWithExecutor(executor, testRun) } - if (testRun.testCase.kind != TestKind.REGULAR) { + val separateTestCases = settings.computeSeparateTestCases(testRun) + + if (testRun.testCase in separateTestCases) { return RunnerWithExecutor(executor, testRun) } @@ -46,9 +46,7 @@ internal object SharedExecutionBuilder { emptySet() // Get tests that are not compatible with others - val testsThatMayFail = testRunsToExecuteSeparately[testRun.executable] - ?.map { it.nominalPackageName.toString() } - ?: emptyList() + val testsThatMayFail = separateTestCases.map { it.nominalPackageName.toString() } val ignoredParameters = (ignoredTests + testsThatMayFail).map { TestRunParameter.WithIgnoredTestFilter(TestName(it)) @@ -56,7 +54,7 @@ internal object SharedExecutionBuilder { val runParameters = testRun.runParameters.filterNot { it is TestRunParameter.WithFilter } + ignoredParameters // Increase timeout for the run, as there are multiple tests to be run. - // At this point there is only amount of tests available, but not each TestRun instance with exact timeout value. + // At this point, there is only a number of tests available, but not each TestRun instance with exact timeout value. val timeout = testRun.checks.executionTimeoutCheck.timeout * testRun.executable.testNames.count() val checks = testRun.checks.copy( executionTimeoutCheck = TestRunCheck.ExecutionTimeout.ShouldNotExceed(timeout) @@ -74,6 +72,34 @@ internal object SharedExecutionBuilder { } } + private fun Settings.computeSeparateTestCases(testRun: TestRun): MutableList = + testCasesToExecuteSeparately.computeIfAbsent(testRun.executable) { + val testRunProvider = get().enclosingTestInstance.testRunProvider + val testCaseGroupId = testRun.testCase.id.testCaseGroupId + + // First, try to get all TestCases for the current executable from the same TestCaseGroup + val testCases = testRunProvider.testCaseGroupProvider.getTestCaseGroup(testCaseGroupId, this) + ?.getRegularOnly( + testRun.testCase.freeCompilerArgs, + testRun.testCase.sharedModules, + testRun.testCase.extras().runnerType + ) + + check(testCases != null && testRun.testCase in testCases) { + "Should be not empty and contain at least $testRun" + } + + check(testCases.all { it.id.testCaseGroupId == testCaseGroupId }) { + "Got TestCases with different group ids. TestCases: $testCases" + } + + // If the test run is expected to fail or timeout, it should not be executed with others. + // Add it to the map of ignored test cases for the executable + testCases.filter { + it.expectedFailure || it.checks.executionTimeoutCheck is TestRunCheck.ExecutionTimeout.ShouldExceed + }.toMutableList() + } + private class CachedRunResultRunner(executor: Executor, testRun: TestRun) : RunnerWithExecutor(executor, testRun) { private val cachedRunResult by lazy(LazyThreadSafetyMode.SYNCHRONIZED) { super.buildRun().run() diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunProvider.kt index 7fee9c7ebf2..21d9cbf20d0 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunProvider.kt @@ -29,7 +29,7 @@ import org.junit.jupiter.api.extension.ExtensionContext * [TestRun] provider that is used in Kotlin/Native black box tests together with the corresponding [TestCaseGroupProvider]. */ internal class TestRunProvider( - private val testCaseGroupProvider: TestCaseGroupProvider + internal val testCaseGroupProvider: TestCaseGroupProvider ) : BaseTestRunProvider(), ExtensionContext.Store.CloseableResource { private val compilationFactory = TestCompilationFactory() private val cachedCompilations = ThreadSafeCache>() diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunners.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunners.kt index 38c125a37fa..c5f047c767e 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunners.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunners.kt @@ -22,7 +22,7 @@ internal object TestRunners { private fun Executor.toRunner(settings: Settings, testRun: TestRun): AbstractRunner = if (settings.get().value) { - SharedExecutionBuilder.buildRunner(this, testRun) + SharedExecutionBuilder.buildRunner(settings, this, testRun) } else { RunnerWithExecutor(this, testRun) }