diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt index ed2a783a36c..88612e4d48b 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt @@ -8,6 +8,8 @@ package org.jetbrains.kotlin.konan.test.blackbox.support.runner import org.jetbrains.kotlin.konan.test.blackbox.support.TestCase import org.jetbrains.kotlin.konan.test.blackbox.support.TestKind import org.jetbrains.kotlin.konan.test.blackbox.support.TestName +import org.jetbrains.kotlin.konan.test.blackbox.support.settings.BlackBoxTestInstances +import org.jetbrains.kotlin.konan.test.blackbox.support.settings.Settings import org.jetbrains.kotlin.konan.test.blackbox.support.settings.SharedExecutionTestRunner import org.jetbrains.kotlin.native.executors.Executor import java.util.concurrent.ConcurrentHashMap @@ -23,18 +25,16 @@ import java.util.concurrent.ConcurrentHashMap */ internal object SharedExecutionBuilder { private val executionResults: ConcurrentHashMap> = ConcurrentHashMap() - private val testRunsToExecuteSeparately: ConcurrentHashMap> = ConcurrentHashMap() - - fun buildRunner(executor: Executor, testRun: TestRun): AbstractRunner { - if (testRun.expectedFailure || testRun.checks.executionTimeoutCheck is TestRunCheck.ExecutionTimeout.ShouldExceed) { - // If the test run is expected to fail or timeout it should not be executed with others. - // Add it to the map of ignored test cases for the executable - testRunsToExecuteSeparately.computeIfAbsent(testRun.executable) { mutableListOf() } += testRun.testCase + private val testCasesToExecuteSeparately: ConcurrentHashMap> = ConcurrentHashMap() + fun buildRunner(settings: Settings, executor: Executor, testRun: TestRun): AbstractRunner { + if (testRun.testCase.kind != TestKind.REGULAR) { return RunnerWithExecutor(executor, testRun) } - if (testRun.testCase.kind != TestKind.REGULAR) { + val separateTestCases = settings.computeSeparateTestCases(testRun) + + if (testRun.testCase in separateTestCases) { return RunnerWithExecutor(executor, testRun) } @@ -46,9 +46,7 @@ internal object SharedExecutionBuilder { emptySet() // Get tests that are not compatible with others - val testsThatMayFail = testRunsToExecuteSeparately[testRun.executable] - ?.map { it.nominalPackageName.toString() } - ?: emptyList() + val testsThatMayFail = separateTestCases.map { it.nominalPackageName.toString() } val ignoredParameters = (ignoredTests + testsThatMayFail).map { TestRunParameter.WithIgnoredTestFilter(TestName(it)) @@ -56,7 +54,7 @@ internal object SharedExecutionBuilder { val runParameters = testRun.runParameters.filterNot { it is TestRunParameter.WithFilter } + ignoredParameters // Increase timeout for the run, as there are multiple tests to be run. - // At this point there is only amount of tests available, but not each TestRun instance with exact timeout value. + // At this point, there is only a number of tests available, but not each TestRun instance with exact timeout value. val timeout = testRun.checks.executionTimeoutCheck.timeout * testRun.executable.testNames.count() val checks = testRun.checks.copy( executionTimeoutCheck = TestRunCheck.ExecutionTimeout.ShouldNotExceed(timeout) @@ -74,6 +72,34 @@ internal object SharedExecutionBuilder { } } + private fun Settings.computeSeparateTestCases(testRun: TestRun): MutableList = + testCasesToExecuteSeparately.computeIfAbsent(testRun.executable) { + val testRunProvider = get().enclosingTestInstance.testRunProvider + val testCaseGroupId = testRun.testCase.id.testCaseGroupId + + // First, try to get all TestCases for the current executable from the same TestCaseGroup + val testCases = testRunProvider.testCaseGroupProvider.getTestCaseGroup(testCaseGroupId, this) + ?.getRegularOnly( + testRun.testCase.freeCompilerArgs, + testRun.testCase.sharedModules, + testRun.testCase.extras().runnerType + ) + + check(testCases != null && testRun.testCase in testCases) { + "Should be not empty and contain at least $testRun" + } + + check(testCases.all { it.id.testCaseGroupId == testCaseGroupId }) { + "Got TestCases with different group ids. TestCases: $testCases" + } + + // If the test run is expected to fail or timeout, it should not be executed with others. + // Add it to the map of ignored test cases for the executable + testCases.filter { + it.expectedFailure || it.checks.executionTimeoutCheck is TestRunCheck.ExecutionTimeout.ShouldExceed + }.toMutableList() + } + private class CachedRunResultRunner(executor: Executor, testRun: TestRun) : RunnerWithExecutor(executor, testRun) { private val cachedRunResult by lazy(LazyThreadSafetyMode.SYNCHRONIZED) { super.buildRun().run() diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunProvider.kt index 7fee9c7ebf2..21d9cbf20d0 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunProvider.kt @@ -29,7 +29,7 @@ import org.junit.jupiter.api.extension.ExtensionContext * [TestRun] provider that is used in Kotlin/Native black box tests together with the corresponding [TestCaseGroupProvider]. */ internal class TestRunProvider( - private val testCaseGroupProvider: TestCaseGroupProvider + internal val testCaseGroupProvider: TestCaseGroupProvider ) : BaseTestRunProvider(), ExtensionContext.Store.CloseableResource { private val compilationFactory = TestCompilationFactory() private val cachedCompilations = ThreadSafeCache>() diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunners.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunners.kt index 38c125a37fa..c5f047c767e 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunners.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunners.kt @@ -22,7 +22,7 @@ internal object TestRunners { private fun Executor.toRunner(settings: Settings, testRun: TestRun): AbstractRunner = if (settings.get().value) { - SharedExecutionBuilder.buildRunner(this, testRun) + SharedExecutionBuilder.buildRunner(settings, this, testRun) } else { RunnerWithExecutor(this, testRun) }