From 07422d4feb0694dd738eaac0516898de8e18a3b4 Mon Sep 17 00:00:00 2001 From: Pavel Punegov Date: Thu, 8 Feb 2024 15:32:57 +0100 Subject: [PATCH] [K/N][test] Implement shared test execution of TestCases Added the ability to execute multiple tests once and share the results. With Test runner turned on tests are being compiled into the single executable. Then they executed each one separately using --ktest_* options. This commit makes test run executable once and the result goes to the result handler. --- .../support/ConfigurationProperties.kt | 1 + .../blackbox/support/NativeTestSupport.kt | 10 +++ .../support/runner/BaseTestRunProvider.kt | 2 +- .../blackbox/support/runner/ResultHandler.kt | 2 +- .../support/runner/RunnerWithExecutor.kt | 4 +- .../support/runner/SharedExecutionBuilder.kt | 84 +++++++++++++++++++ .../test/blackbox/support/runner/TestRun.kt | 12 ++- .../blackbox/support/runner/TestRunners.kt | 11 ++- .../support/settings/TestProcessSettings.kt | 8 +- .../src/main/kotlin/nativeTest.kt | 2 + 10 files changed, 128 insertions(+), 8 deletions(-) create mode 100644 native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/ConfigurationProperties.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/ConfigurationProperties.kt index 7977621fd15..1f5b5dcc14d 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/ConfigurationProperties.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/ConfigurationProperties.kt @@ -71,6 +71,7 @@ internal enum class ClassLevelProperty(val shortName: String) { SANITIZER("sanitizer"), COMPILER_OUTPUT_INTERCEPTOR("compilerOutputInterceptor"), PIPELINE_TYPE("pipelineType"), + SHARED_TEST_EXECUTION("sharedTestExecution"), ; internal val propertyName = fullPropertyName(shortName) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/NativeTestSupport.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/NativeTestSupport.kt index 418f8daab7a..de9cffed51d 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/NativeTestSupport.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/NativeTestSupport.kt @@ -203,6 +203,7 @@ internal object NativeTestSupport { output += computeCustomKlibs(enforcedProperties) output += computeTestKind(enforcedProperties) output += computeForcedNoopTestRunner(enforcedProperties) + output += computeSharedExecutionTestRunner(enforcedProperties) output += computeTimeouts(enforcedProperties) // Parse annotations of current class, since there's no way to put annotations to upper-level enclosing class output += computePipelineType(enforcedProperties, testClass.get()) @@ -325,6 +326,15 @@ internal object NativeTestSupport { ) ) + private fun computeSharedExecutionTestRunner(enforcedProperties: EnforcedProperties): SharedExecutionTestRunner = + SharedExecutionTestRunner( + ClassLevelProperty.SHARED_TEST_EXECUTION.readValue( + enforcedProperties, + String::toBooleanStrictOrNull, + default = false + ) + ) + private fun computeTimeouts(enforcedProperties: EnforcedProperties): Timeouts { val executionTimeout = ClassLevelProperty.EXECUTION_TIMEOUT.readValue( enforcedProperties, diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/BaseTestRunProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/BaseTestRunProvider.kt index c4b58a5d16e..34c393a313f 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/BaseTestRunProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/BaseTestRunProvider.kt @@ -15,7 +15,7 @@ import org.jetbrains.kotlin.utils.addIfNotNull internal open class BaseTestRunProvider { protected fun createTestRun(testCase: TestCase, executable: TestExecutable, testRunName: String, testName: TestName?): TestRun { val runParameters = getTestRunParameters(testCase, testName) - return TestRun(displayName = testRunName, executable, runParameters, testCase.id, testCase.checks, testCase.expectedFailure) + return TestRun(displayName = testRunName, executable, runParameters, testCase, testCase.checks, testCase.expectedFailure) } protected fun createSingleTestRun(testCase: TestCase, executable: TestExecutable): TestRun = createTestRun( diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/ResultHandler.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/ResultHandler.kt index c39ea1b88de..2ced210f3bf 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/ResultHandler.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/ResultHandler.kt @@ -103,7 +103,7 @@ internal class ResultHandler( diagnostics.joinToString("\n") } } else { - val runResultInfo = "TestCaseId: ${testRun.testCaseId}\nExit code: ${runResult.exitCode}\nFiltered test output is${ + val runResultInfo = "TestCaseId: ${testRun.testCase.id}\nExit code: ${runResult.exitCode}\nFiltered test output is${ runResult.processOutput.stdOut.filteredOutput.let { if (it.isNotEmpty()) ":\n$it" else " empty." } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/RunnerWithExecutor.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/RunnerWithExecutor.kt index b3807360e70..bc986fe7ae0 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/RunnerWithExecutor.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/RunnerWithExecutor.kt @@ -17,7 +17,7 @@ import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream import java.io.InputStream -internal class RunnerWithExecutor( +internal open class RunnerWithExecutor( private val executor: Executor, private val testRun: TestRun ) : AbstractRunner() { @@ -74,7 +74,7 @@ internal class RunnerWithExecutor( override fun getLoggedParameters() = LoggedData.TestRunParameters( compilationToolCall = executable.loggedCompilationToolCall, - testCaseId = testRun.testCaseId, + testCaseId = testRun.testCase.id, runArgs = programArgs, runParameters = testRun.runParameters ) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt new file mode 100644 index 00000000000..ed2a783a36c --- /dev/null +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt @@ -0,0 +1,84 @@ +/* + * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.konan.test.blackbox.support.runner + +import org.jetbrains.kotlin.konan.test.blackbox.support.TestCase +import org.jetbrains.kotlin.konan.test.blackbox.support.TestKind +import org.jetbrains.kotlin.konan.test.blackbox.support.TestName +import org.jetbrains.kotlin.konan.test.blackbox.support.settings.SharedExecutionTestRunner +import org.jetbrains.kotlin.native.executors.Executor +import java.util.concurrent.ConcurrentHashMap + +/** + * This is a caching shared TestRun execution builder. + * + * Builds a [Runner] that is able to execute tests, caching results for all tests in the executable, + * compiled with [TestCase.WithTestRunnerExtras]. The idea is to run [TestExecutable] that contains multiple [TestCase]s only once + * and pass this shared result to all tests. + * + * @see SharedExecutionTestRunner + */ +internal object SharedExecutionBuilder { + private val executionResults: ConcurrentHashMap> = ConcurrentHashMap() + private val testRunsToExecuteSeparately: ConcurrentHashMap> = ConcurrentHashMap() + + fun buildRunner(executor: Executor, testRun: TestRun): AbstractRunner { + if (testRun.expectedFailure || testRun.checks.executionTimeoutCheck is TestRunCheck.ExecutionTimeout.ShouldExceed) { + // If the test run is expected to fail or timeout it should not be executed with others. + // Add it to the map of ignored test cases for the executable + testRunsToExecuteSeparately.computeIfAbsent(testRun.executable) { mutableListOf() } += testRun.testCase + + return RunnerWithExecutor(executor, testRun) + } + + if (testRun.testCase.kind != TestKind.REGULAR) { + return RunnerWithExecutor(executor, testRun) + } + + return executionResults.computeIfAbsent(testRun.executable) { + // Get ignored tests to exclude them from run by adding the test filtering option + val ignoredTests = if (testRun.testCase.extras is TestCase.WithTestRunnerExtras) { + testRun.testCase.extras.ignoredTests + } else + emptySet() + + // Get tests that are not compatible with others + val testsThatMayFail = testRunsToExecuteSeparately[testRun.executable] + ?.map { it.nominalPackageName.toString() } + ?: emptyList() + + val ignoredParameters = (ignoredTests + testsThatMayFail).map { + TestRunParameter.WithIgnoredTestFilter(TestName(it)) + } + val runParameters = testRun.runParameters.filterNot { it is TestRunParameter.WithFilter } + ignoredParameters + + // Increase timeout for the run, as there are multiple tests to be run. + // At this point there is only amount of tests available, but not each TestRun instance with exact timeout value. + val timeout = testRun.checks.executionTimeoutCheck.timeout * testRun.executable.testNames.count() + val checks = testRun.checks.copy( + executionTimeoutCheck = TestRunCheck.ExecutionTimeout.ShouldNotExceed(timeout) + ) + + val sharedTestRun = TestRun( + displayName = "Shared TestRun for ${testRun.executable.executable.path} made from ${testRun.displayName}", + executable = testRun.executable, + runParameters = runParameters, + testCase = testRun.testCase, + checks = checks, + expectedFailure = false + ) + CachedRunResultRunner(executor, sharedTestRun) + } + } + + private class CachedRunResultRunner(executor: Executor, testRun: TestRun) : RunnerWithExecutor(executor, testRun) { + private val cachedRunResult by lazy(LazyThreadSafetyMode.SYNCHRONIZED) { + super.buildRun().run() + } + + override fun buildRun() = AbstractRun { cachedRunResult } + } +} diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRun.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRun.kt index f3ca992cdfa..cfd2f884282 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRun.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRun.kt @@ -50,11 +50,11 @@ internal class TestExecutable( } } -internal class TestRun( +internal data class TestRun( val displayName: String, val executable: TestExecutable, val runParameters: List, - val testCaseId: TestCaseId, + val testCase: TestCase, val checks: TestRunChecks, val expectedFailure: Boolean, ) @@ -86,6 +86,14 @@ internal sealed interface TestRunParameter { override fun testMatches(testName: TestName) = this.testName == testName } + class WithIgnoredTestFilter(val testName: TestName) : WithFilter() { + override fun applyTo(programArgs: MutableList) { + programArgs += "--ktest_filter=*-$testName" + } + + override fun testMatches(testName: TestName) = this.testName != testName + } + class WithGTestPatterns(val positivePatterns: Set = setOf("*"), val negativePatterns: Set) : WithFilter() { val positiveRegexes = positivePatterns.map(::fromGTestPattern) val negativeRegexes = negativePatterns.map(::fromGTestPattern) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunners.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunners.kt index 9a54a754c14..38c125a37fa 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunners.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunners.kt @@ -7,14 +7,23 @@ package org.jetbrains.kotlin.konan.test.blackbox.support.runner import org.jetbrains.kotlin.konan.test.blackbox.support.settings.ForcedNoopTestRunner import org.jetbrains.kotlin.konan.test.blackbox.support.settings.Settings +import org.jetbrains.kotlin.konan.test.blackbox.support.settings.SharedExecutionTestRunner import org.jetbrains.kotlin.konan.test.blackbox.support.settings.executor +import org.jetbrains.kotlin.native.executors.Executor internal object TestRunners { fun createProperTestRunner(testRun: TestRun, settings: Settings): Runner = with(settings) { if (get().value) { NoopTestRunner } else { - RunnerWithExecutor(executor, testRun) + executor.toRunner(settings, testRun) } } + + private fun Executor.toRunner(settings: Settings, testRun: TestRun): AbstractRunner = + if (settings.get().value) { + SharedExecutionBuilder.buildRunner(this, testRun) + } else { + RunnerWithExecutor(this, testRun) + } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/settings/TestProcessSettings.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/settings/TestProcessSettings.kt index eb50100a460..a4c0dedf472 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/settings/TestProcessSettings.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/settings/TestProcessSettings.kt @@ -10,7 +10,6 @@ import org.jetbrains.kotlin.konan.properties.resolvablePropertyList import org.jetbrains.kotlin.konan.target.Distribution import org.jetbrains.kotlin.konan.target.KonanTarget import org.jetbrains.kotlin.konan.test.blackbox.support.MutedOption -import org.jetbrains.kotlin.konan.test.blackbox.support.TestKind import org.jetbrains.kotlin.konan.test.blackbox.support.runner.RunnerWithExecutor import org.jetbrains.kotlin.konan.test.blackbox.support.runner.NoopTestRunner import org.jetbrains.kotlin.konan.test.blackbox.support.runner.Runner @@ -111,6 +110,13 @@ internal value class CustomKlibs(val klibs: Set) { @JvmInline internal value class ForcedNoopTestRunner(val value: Boolean) +/** + * Controls whether tests that support TestRunner should be executed once in the binary. + * Their execution result is shared between tests from the same test executable. + */ +@JvmInline +internal value class SharedExecutionTestRunner(val value: Boolean) + /** * Optimization mode to be applied. */ diff --git a/repo/gradle-build-conventions/buildsrc-compat/src/main/kotlin/nativeTest.kt b/repo/gradle-build-conventions/buildsrc-compat/src/main/kotlin/nativeTest.kt index 205e2ddc890..454071115dd 100644 --- a/repo/gradle-build-conventions/buildsrc-compat/src/main/kotlin/nativeTest.kt +++ b/repo/gradle-build-conventions/buildsrc-compat/src/main/kotlin/nativeTest.kt @@ -27,6 +27,7 @@ private enum class TestProperty(shortName: String) { CACHE_MODE("cacheMode"), EXECUTION_TIMEOUT("executionTimeout"), SANITIZER("sanitizer"), + SHARED_TEST_EXECUTION("sharedTestExecution"), TEAMCITY("teamcity"); val fullName = "kotlin.internal.native.test.$shortName" @@ -195,6 +196,7 @@ fun Project.nativeTest( compute(CACHE_MODE) compute(EXECUTION_TIMEOUT) compute(SANITIZER) + compute(SHARED_TEST_EXECUTION) // Pass whether tests are running at TeamCity. computePrivate(TEAMCITY) { kotlinBuildProperties.isTeamcityBuild.toString() }