[K/N][test] Implement shared test execution of TestCases

Added the ability to execute multiple tests once and share the results.
With Test runner turned on tests are being compiled into the single
executable. Then they executed each one separately using --ktest_*
options. This commit makes test run executable once and the result goes
to the result handler.

native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/ConfigurationProperties.kt

@@ -71,6 +71,7 @@ internal enum class ClassLevelProperty(val shortName: String) {
     SANITIZER("sanitizer"),
     COMPILER_OUTPUT_INTERCEPTOR("compilerOutputInterceptor"),
     PIPELINE_TYPE("pipelineType"),
+    SHARED_TEST_EXECUTION("sharedTestExecution"),
     ;
 
     internal val propertyName = fullPropertyName(shortName)

native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/NativeTestSupport.kt

@@ -203,6 +203,7 @@ internal object NativeTestSupport {
         output += computeCustomKlibs(enforcedProperties)
         output += computeTestKind(enforcedProperties)
         output += computeForcedNoopTestRunner(enforcedProperties)
+        output += computeSharedExecutionTestRunner(enforcedProperties)
         output += computeTimeouts(enforcedProperties)
         // Parse annotations of current class, since there's no way to put annotations to upper-level enclosing class
         output += computePipelineType(enforcedProperties, testClass.get())
@@ -325,6 +326,15 @@ internal object NativeTestSupport {
             )
         )
 
+    private fun computeSharedExecutionTestRunner(enforcedProperties: EnforcedProperties): SharedExecutionTestRunner =
+        SharedExecutionTestRunner(
+            ClassLevelProperty.SHARED_TEST_EXECUTION.readValue(
+                enforcedProperties,
+                String::toBooleanStrictOrNull,
+                default = false
+            )
+        )
+
     private fun computeTimeouts(enforcedProperties: EnforcedProperties): Timeouts {
         val executionTimeout = ClassLevelProperty.EXECUTION_TIMEOUT.readValue(
             enforcedProperties,

native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/BaseTestRunProvider.kt

@@ -15,7 +15,7 @@ import org.jetbrains.kotlin.utils.addIfNotNull
 internal open class BaseTestRunProvider {
     protected fun createTestRun(testCase: TestCase, executable: TestExecutable, testRunName: String, testName: TestName?): TestRun {
         val runParameters = getTestRunParameters(testCase, testName)
-        return TestRun(displayName = testRunName, executable, runParameters, testCase.id, testCase.checks, testCase.expectedFailure)
+        return TestRun(displayName = testRunName, executable, runParameters, testCase, testCase.checks, testCase.expectedFailure)
     }
 
     protected fun createSingleTestRun(testCase: TestCase, executable: TestExecutable): TestRun = createTestRun(

native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/ResultHandler.kt

@@ -103,7 +103,7 @@ internal class ResultHandler(
                 diagnostics.joinToString("\n")
             }
         } else {
-            val runResultInfo = "TestCaseId: ${testRun.testCaseId}\nExit code: ${runResult.exitCode}\nFiltered test output is${
+            val runResultInfo = "TestCaseId: ${testRun.testCase.id}\nExit code: ${runResult.exitCode}\nFiltered test output is${
                 runResult.processOutput.stdOut.filteredOutput.let {
                     if (it.isNotEmpty()) ":\n$it" else " empty."
                 }

native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/RunnerWithExecutor.kt

@@ -17,7 +17,7 @@ import java.io.ByteArrayInputStream
 import java.io.ByteArrayOutputStream
 import java.io.InputStream
 
-internal class RunnerWithExecutor(
+internal open class RunnerWithExecutor(
     private val executor: Executor,
     private val testRun: TestRun
 ) : AbstractRunner<Unit>() {
@@ -74,7 +74,7 @@ internal class RunnerWithExecutor(
 
     override fun getLoggedParameters() = LoggedData.TestRunParameters(
         compilationToolCall = executable.loggedCompilationToolCall,
-        testCaseId = testRun.testCaseId,
+        testCaseId = testRun.testCase.id,
         runArgs = programArgs,
         runParameters = testRun.runParameters
     )

native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/SharedExecutionBuilder.kt

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.konan.test.blackbox.support.runner
+
+import org.jetbrains.kotlin.konan.test.blackbox.support.TestCase
+import org.jetbrains.kotlin.konan.test.blackbox.support.TestKind
+import org.jetbrains.kotlin.konan.test.blackbox.support.TestName
+import org.jetbrains.kotlin.konan.test.blackbox.support.settings.SharedExecutionTestRunner
+import org.jetbrains.kotlin.native.executors.Executor
+import java.util.concurrent.ConcurrentHashMap
+
+/**
+ * This is a caching shared TestRun execution builder.
+ *
+ * Builds a [Runner] that is able to execute tests, caching results for all tests in the executable,
+ * compiled with [TestCase.WithTestRunnerExtras]. The idea is to run [TestExecutable] that contains multiple [TestCase]s only once
+ * and pass this shared result to all tests.
+ *
+ * @see SharedExecutionTestRunner
+ */
+internal object SharedExecutionBuilder {
+    private val executionResults: ConcurrentHashMap<TestExecutable, AbstractRunner<Unit>> = ConcurrentHashMap()
+    private val testRunsToExecuteSeparately: ConcurrentHashMap<TestExecutable, MutableList<TestCase>> = ConcurrentHashMap()
+
+    fun buildRunner(executor: Executor, testRun: TestRun): AbstractRunner<Unit> {
+        if (testRun.expectedFailure || testRun.checks.executionTimeoutCheck is TestRunCheck.ExecutionTimeout.ShouldExceed) {
+            // If the test run is expected to fail or timeout it should not be executed with others.
+            // Add it to the map of ignored test cases for the executable
+            testRunsToExecuteSeparately.computeIfAbsent(testRun.executable) { mutableListOf() } += testRun.testCase
+
+            return RunnerWithExecutor(executor, testRun)
+        }
+
+        if (testRun.testCase.kind != TestKind.REGULAR) {
+            return RunnerWithExecutor(executor, testRun)
+        }
+
+        return executionResults.computeIfAbsent(testRun.executable) {
+            // Get ignored tests to exclude them from run by adding the test filtering option
+            val ignoredTests = if (testRun.testCase.extras is TestCase.WithTestRunnerExtras) {
+                testRun.testCase.extras.ignoredTests
+            } else
+                emptySet()
+
+            // Get tests that are not compatible with others
+            val testsThatMayFail = testRunsToExecuteSeparately[testRun.executable]
+                ?.map { it.nominalPackageName.toString() }
+                ?: emptyList()
+
+            val ignoredParameters = (ignoredTests + testsThatMayFail).map {
+                TestRunParameter.WithIgnoredTestFilter(TestName(it))
+            }
+            val runParameters = testRun.runParameters.filterNot { it is TestRunParameter.WithFilter } + ignoredParameters
+
+            // Increase timeout for the run, as there are multiple tests to be run.
+            // At this point there is only amount of tests available, but not each TestRun instance with exact timeout value.
+            val timeout = testRun.checks.executionTimeoutCheck.timeout * testRun.executable.testNames.count()
+            val checks = testRun.checks.copy(
+                executionTimeoutCheck = TestRunCheck.ExecutionTimeout.ShouldNotExceed(timeout)
+            )
+
+            val sharedTestRun = TestRun(
+                displayName = "Shared TestRun for ${testRun.executable.executable.path} made from ${testRun.displayName}",
+                executable = testRun.executable,
+                runParameters = runParameters,
+                testCase = testRun.testCase,
+                checks = checks,
+                expectedFailure = false
+            )
+            CachedRunResultRunner(executor, sharedTestRun)
+        }
+    }
+
+    private class CachedRunResultRunner(executor: Executor, testRun: TestRun) : RunnerWithExecutor(executor, testRun) {
+        private val cachedRunResult by lazy(LazyThreadSafetyMode.SYNCHRONIZED) {
+            super.buildRun().run()
+        }
+
+        override fun buildRun() = AbstractRun { cachedRunResult }
+    }
+}

native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRun.kt

@@ -50,11 +50,11 @@ internal class TestExecutable(
     }
 }
 
-internal class TestRun(
+internal data class TestRun(
     val displayName: String,
     val executable: TestExecutable,
     val runParameters: List<TestRunParameter>,
-    val testCaseId: TestCaseId,
+    val testCase: TestCase,
     val checks: TestRunChecks,
     val expectedFailure: Boolean,
 )
@@ -86,6 +86,14 @@ internal sealed interface TestRunParameter {
         override fun testMatches(testName: TestName) = this.testName == testName
     }
 
+    class WithIgnoredTestFilter(val testName: TestName) : WithFilter() {
+        override fun applyTo(programArgs: MutableList<String>) {
+            programArgs += "--ktest_filter=*-$testName"
+        }
+
+        override fun testMatches(testName: TestName) = this.testName != testName
+    }
+
     class WithGTestPatterns(val positivePatterns: Set<String> = setOf("*"), val negativePatterns: Set<String>) : WithFilter() {
         val positiveRegexes = positivePatterns.map(::fromGTestPattern)
         val negativeRegexes = negativePatterns.map(::fromGTestPattern)

native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/runner/TestRunners.kt

@@ -7,14 +7,23 @@ package org.jetbrains.kotlin.konan.test.blackbox.support.runner
 
 import org.jetbrains.kotlin.konan.test.blackbox.support.settings.ForcedNoopTestRunner
 import org.jetbrains.kotlin.konan.test.blackbox.support.settings.Settings
+import org.jetbrains.kotlin.konan.test.blackbox.support.settings.SharedExecutionTestRunner
 import org.jetbrains.kotlin.konan.test.blackbox.support.settings.executor
+import org.jetbrains.kotlin.native.executors.Executor
 
 internal object TestRunners {
     fun createProperTestRunner(testRun: TestRun, settings: Settings): Runner<Unit> = with(settings) {
         if (get<ForcedNoopTestRunner>().value) {
             NoopTestRunner
         } else {
-            RunnerWithExecutor(executor, testRun)
+            executor.toRunner(settings, testRun)
         }
     }
+
+    private fun Executor.toRunner(settings: Settings, testRun: TestRun): AbstractRunner<Unit> =
+        if (settings.get<SharedExecutionTestRunner>().value) {
+            SharedExecutionBuilder.buildRunner(this, testRun)
+        } else {
+            RunnerWithExecutor(this, testRun)
+        }
 }

native/native.tests/tests/org/jetbrains/kotlin/konan/test/blackbox/support/settings/TestProcessSettings.kt

@@ -10,7 +10,6 @@ import org.jetbrains.kotlin.konan.properties.resolvablePropertyList
 import org.jetbrains.kotlin.konan.target.Distribution
 import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.konan.test.blackbox.support.MutedOption
-import org.jetbrains.kotlin.konan.test.blackbox.support.TestKind
 import org.jetbrains.kotlin.konan.test.blackbox.support.runner.RunnerWithExecutor
 import org.jetbrains.kotlin.konan.test.blackbox.support.runner.NoopTestRunner
 import org.jetbrains.kotlin.konan.test.blackbox.support.runner.Runner
@@ -111,6 +110,13 @@ internal value class CustomKlibs(val klibs: Set<File>) {
 @JvmInline
 internal value class ForcedNoopTestRunner(val value: Boolean)
 
+/**
+ * Controls whether tests that support TestRunner should be executed once in the binary.
+ * Their execution result is shared between tests from the same test executable.
+ */
+@JvmInline
+internal value class SharedExecutionTestRunner(val value: Boolean)
+
 /**
  * Optimization mode to be applied.
  */

repo/gradle-build-conventions/buildsrc-compat/src/main/kotlin/nativeTest.kt

@@ -27,6 +27,7 @@ private enum class TestProperty(shortName: String) {
     CACHE_MODE("cacheMode"),
     EXECUTION_TIMEOUT("executionTimeout"),
     SANITIZER("sanitizer"),
+    SHARED_TEST_EXECUTION("sharedTestExecution"),
     TEAMCITY("teamcity");
 
     val fullName = "kotlin.internal.native.test.$shortName"
@@ -195,6 +196,7 @@ fun Project.nativeTest(
             compute(CACHE_MODE)
             compute(EXECUTION_TIMEOUT)
             compute(SANITIZER)
+            compute(SHARED_TEST_EXECUTION)
 
             // Pass whether tests are running at TeamCity.
             computePrivate(TEAMCITY) { kotlinBuildProperties.isTeamcityBuild.toString() }