diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/ConfigurationProperties.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/ConfigurationProperties.kt index 7977621fd15..1f5b5dcc14d 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/ConfigurationProperties.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/ConfigurationProperties.kt @@ -71,6 +71,7 @@ internal enum class ClassLevelProperty(val shortName: String) { SANITIZER("sanitizer"), COMPILER_OUTPUT_INTERCEPTOR("compilerOutputInterceptor"), PIPELINE_TYPE("pipelineType"), + SHARED_TEST_EXECUTION("sharedTestExecution"), ; internal val propertyName = fullPropertyName(shortName) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/NativeTestSupport.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/NativeTestSupport.kt index 418f8daab7a..de9cffed51d 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/NativeTestSupport.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/NativeTestSupport.kt @@ -203,6 +203,7 @@ internal object NativeTestSupport { output += computeCustomKlibs(enforcedProperties) output += computeTestKind(enforcedProperties) output += computeForcedNoopTestRunner(enforcedProperties) + output += computeSharedExecutionTestRunner(enforcedProperties) output += computeTimeouts(enforcedProperties) // Parse annotations of current class, since there's no way to put annotations to upper-level enclosing class output += computePipelineType(enforcedProperties, testClass.get()) @@ -325,6 +326,15 @@ internal object NativeTestSupport { ) ) + private fun computeSharedExecutionTestRunner(enforcedProperties: EnforcedProperties): SharedExecutionTestRunner = + SharedExecutionTestRunner( + ClassLevelProperty.SHARED_TEST_EXECUTION.readValue( + enforcedProperties, + String::toBooleanStrictOrNull, + default = false + ) + ) + private fun computeTimeouts(enforcedProperties: EnforcedProperties): Timeouts { val executionTimeout = ClassLevelProperty.EXECUTION_TIMEOUT.readValue( enforcedProperties, diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/BaseTestRunProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/BaseTestRunProvider.kt index c4b58a5d16e..34c393a313f 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/BaseTestRunProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/BaseTestRunProvider.kt @@ -15,7 +15,7 @@ import org.jetbrains.kotlin.utils.addIfNotNull internal open class BaseTestRunProvider { protected fun createTestRun(testCase: TestCase, executable: TestExecutable, testRunName: String, testName: TestName?): TestRun { val runParameters = getTestRunParameters(testCase, testName) - return TestRun(displayName = testRunName, executable, runParameters, testCase.id, testCase.checks, testCase.expectedFailure) + return TestRun(displayName = testRunName, executable, runParameters, testCase, testCase.checks, testCase.expectedFailure) } protected fun createSingleTestRun(testCase: TestCase, executable: TestExecutable): TestRun = createTestRun( diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/ResultHandler.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/ResultHandler.kt index c39ea1b88de..2ced210f3bf 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/ResultHandler.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/ResultHandler.kt @@ -103,7 +103,7 @@ internal class ResultHandler( diagnostics.joinToString("\n") } } else { - val runResultInfo = "TestCaseId: ${testRun.testCaseId}\nExit code: ${runResult.exitCode}\nFiltered test output is${ + val runResultInfo = "TestCaseId: ${testRun.testCase.id}\nExit code: ${runResult.exitCode}\nFiltered test output is${ runResult.processOutput.stdOut.filteredOutput.let { if (it.isNotEmpty()) ":\n$it" else " empty." } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/RunnerWithExecutor.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/RunnerWithExecutor.kt index b3807360e70..bc986fe7ae0 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/RunnerWithExecutor.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/RunnerWithExecutor.kt @@ -17,7 +17,7 @@ import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream import java.io.InputStream -internal class RunnerWithExecutor( +internal open class RunnerWithExecutor( private val executor: Executor, private val testRun: TestRun ) : AbstractRunner() { @@ -74,7 +74,7 @@ internal class RunnerWithExecutor( override fun getLoggedParameters() = LoggedData.TestRunParameters( compilationToolCall = executable.loggedCompilationToolCall, - testCaseId = testRun.testCaseId, + testCaseId = testRun.testCase.id, runArgs = programArgs, runParameters = testRun.runParameters ) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt new file mode 100644 index 00000000000..ed2a783a36c --- /dev/null +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt @@ -0,0 +1,84 @@ +/* + * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.konan.test.blackbox.support.runner + +import org.jetbrains.kotlin.konan.test.blackbox.support.TestCase +import org.jetbrains.kotlin.konan.test.blackbox.support.TestKind +import org.jetbrains.kotlin.konan.test.blackbox.support.TestName +import org.jetbrains.kotlin.konan.test.blackbox.support.settings.SharedExecutionTestRunner +import org.jetbrains.kotlin.native.executors.Executor +import java.util.concurrent.ConcurrentHashMap + +/** + * This is a caching shared TestRun execution builder. + * + * Builds a [Runner] that is able to execute tests, caching results for all tests in the executable, + * compiled with [TestCase.WithTestRunnerExtras]. The idea is to run [TestExecutable] that contains multiple [TestCase]s only once + * and pass this shared result to all tests. + * + * @see SharedExecutionTestRunner + */ +internal object SharedExecutionBuilder { + private val executionResults: ConcurrentHashMap> = ConcurrentHashMap() + private val testRunsToExecuteSeparately: ConcurrentHashMap> = ConcurrentHashMap() + + fun buildRunner(executor: Executor, testRun: TestRun): AbstractRunner { + if (testRun.expectedFailure || testRun.checks.executionTimeoutCheck is TestRunCheck.ExecutionTimeout.ShouldExceed) { + // If the test run is expected to fail or timeout it should not be executed with others. + // Add it to the map of ignored test cases for the executable + testRunsToExecuteSeparately.computeIfAbsent(testRun.executable) { mutableListOf() } += testRun.testCase + + return RunnerWithExecutor(executor, testRun) + } + + if (testRun.testCase.kind != TestKind.REGULAR) { + return RunnerWithExecutor(executor, testRun) + } + + return executionResults.computeIfAbsent(testRun.executable) { + // Get ignored tests to exclude them from run by adding the test filtering option + val ignoredTests = if (testRun.testCase.extras is TestCase.WithTestRunnerExtras) { + testRun.testCase.extras.ignoredTests + } else + emptySet() + + // Get tests that are not compatible with others + val testsThatMayFail = testRunsToExecuteSeparately[testRun.executable] + ?.map { it.nominalPackageName.toString() } + ?: emptyList() + + val ignoredParameters = (ignoredTests + testsThatMayFail).map { + TestRunParameter.WithIgnoredTestFilter(TestName(it)) + } + val runParameters = testRun.runParameters.filterNot { it is TestRunParameter.WithFilter } + ignoredParameters + + // Increase timeout for the run, as there are multiple tests to be run. + // At this point there is only amount of tests available, but not each TestRun instance with exact timeout value. + val timeout = testRun.checks.executionTimeoutCheck.timeout * testRun.executable.testNames.count() + val checks = testRun.checks.copy( + executionTimeoutCheck = TestRunCheck.ExecutionTimeout.ShouldNotExceed(timeout) + ) + + val sharedTestRun = TestRun( + displayName = "Shared TestRun for ${testRun.executable.executable.path} made from ${testRun.displayName}", + executable = testRun.executable, + runParameters = runParameters, + testCase = testRun.testCase, + checks = checks, + expectedFailure = false + ) + CachedRunResultRunner(executor, sharedTestRun) + } + } + + private class CachedRunResultRunner(executor: Executor, testRun: TestRun) : RunnerWithExecutor(executor, testRun) { + private val cachedRunResult by lazy(LazyThreadSafetyMode.SYNCHRONIZED) { + super.buildRun().run() + } + + override fun buildRun() = AbstractRun { cachedRunResult } + } +} diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRun.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRun.kt index f3ca992cdfa..cfd2f884282 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRun.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRun.kt @@ -50,11 +50,11 @@ internal class TestExecutable( } } -internal class TestRun( +internal data class TestRun( val displayName: String, val executable: TestExecutable, val runParameters: List, - val testCaseId: TestCaseId, + val testCase: TestCase, val checks: TestRunChecks, val expectedFailure: Boolean, ) @@ -86,6 +86,14 @@ internal sealed interface TestRunParameter { override fun testMatches(testName: TestName) = this.testName == testName } + class WithIgnoredTestFilter(val testName: TestName) : WithFilter() { + override fun applyTo(programArgs: MutableList) { + programArgs += "--ktest_filter=*-$testName" + } + + override fun testMatches(testName: TestName) = this.testName != testName + } + class WithGTestPatterns(val positivePatterns: Set = setOf("*"), val negativePatterns: Set) : WithFilter() { val positiveRegexes = positivePatterns.map(::fromGTestPattern) val negativeRegexes = negativePatterns.map(::fromGTestPattern) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunners.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunners.kt index 9a54a754c14..38c125a37fa 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunners.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunners.kt @@ -7,14 +7,23 @@ package org.jetbrains.kotlin.konan.test.blackbox.support.runner import org.jetbrains.kotlin.konan.test.blackbox.support.settings.ForcedNoopTestRunner import org.jetbrains.kotlin.konan.test.blackbox.support.settings.Settings +import org.jetbrains.kotlin.konan.test.blackbox.support.settings.SharedExecutionTestRunner import org.jetbrains.kotlin.konan.test.blackbox.support.settings.executor +import org.jetbrains.kotlin.native.executors.Executor internal object TestRunners { fun createProperTestRunner(testRun: TestRun, settings: Settings): Runner = with(settings) { if (get().value) { NoopTestRunner } else { - RunnerWithExecutor(executor, testRun) + executor.toRunner(settings, testRun) } } + + private fun Executor.toRunner(settings: Settings, testRun: TestRun): AbstractRunner = + if (settings.get().value) { + SharedExecutionBuilder.buildRunner(this, testRun) + } else { + RunnerWithExecutor(this, testRun) + } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/settings/TestProcessSettings.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/settings/TestProcessSettings.kt index eb50100a460..a4c0dedf472 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/settings/TestProcessSettings.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/settings/TestProcessSettings.kt @@ -10,7 +10,6 @@ import org.jetbrains.kotlin.konan.properties.resolvablePropertyList import org.jetbrains.kotlin.konan.target.Distribution import org.jetbrains.kotlin.konan.target.KonanTarget import org.jetbrains.kotlin.konan.test.blackbox.support.MutedOption -import org.jetbrains.kotlin.konan.test.blackbox.support.TestKind import org.jetbrains.kotlin.konan.test.blackbox.support.runner.RunnerWithExecutor import org.jetbrains.kotlin.konan.test.blackbox.support.runner.NoopTestRunner import org.jetbrains.kotlin.konan.test.blackbox.support.runner.Runner @@ -111,6 +110,13 @@ internal value class CustomKlibs(val klibs: Set) { @JvmInline internal value class ForcedNoopTestRunner(val value: Boolean) +/** + * Controls whether tests that support TestRunner should be executed once in the binary. + * Their execution result is shared between tests from the same test executable. + */ +@JvmInline +internal value class SharedExecutionTestRunner(val value: Boolean) + /** * Optimization mode to be applied. */ diff --git a/repo/gradle-build-conventions/buildsrc-compat/src/main/kotlin/nativeTest.kt b/repo/gradle-build-conventions/buildsrc-compat/src/main/kotlin/nativeTest.kt index 205e2ddc890..454071115dd 100644 --- a/repo/gradle-build-conventions/buildsrc-compat/src/main/kotlin/nativeTest.kt +++ b/repo/gradle-build-conventions/buildsrc-compat/src/main/kotlin/nativeTest.kt @@ -27,6 +27,7 @@ private enum class TestProperty(shortName: String) { CACHE_MODE("cacheMode"), EXECUTION_TIMEOUT("executionTimeout"), SANITIZER("sanitizer"), + SHARED_TEST_EXECUTION("sharedTestExecution"), TEAMCITY("teamcity"); val fullName = "kotlin.internal.native.test.$shortName" @@ -195,6 +196,7 @@ fun Project.nativeTest( compute(CACHE_MODE) compute(EXECUTION_TIMEOUT) compute(SANITIZER) + compute(SHARED_TEST_EXECUTION) // Pass whether tests are running at TeamCity. computePrivate(TEAMCITY) { kotlinBuildProperties.isTeamcityBuild.toString() }