[K/N][test] Use Executors in the New test infra

Add RunnerWithExecutor class that uses Executors
Re-use checkers in the new executor

Merge-request: KT-MR-8964

native/executors/src/main/kotlin/org/jetbrains/kotlin/native/executors/XcRunRuntimeUtils.kt

@@ -108,10 +108,10 @@ data class SimulatorRuntimeDescriptor(
 }
 
 data class DeviceType(
-    @Expose val bundlePath: String,
+        @Expose val bundlePath: String,
-    @Expose val name: String,
+        @Expose val name: String,
-    @Expose val identifier: String,
+        @Expose val identifier: String,
-    @Expose val productFamily: String
+        @Expose val productFamily: String
 )
 
 /**

native/native.tests/build.gradle.kts

@@ -18,6 +18,8 @@ dependencies {
     testImplementation(projectTests(":compiler:tests-common-new"))
     testImplementation(projectTests(":compiler:test-infrastructure"))
     testImplementation(projectTests(":generators:test-generator"))
+    testImplementation(project(":native:kotlin-native-utils"))
+    testImplementation(project(":native:executors"))
     testApiJUnit5()
     testImplementation(commonDependency("org.jetbrains.kotlinx", "kotlinx-coroutines-core")) { isTransitive = false }
 

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt

@@ -16,7 +16,7 @@ import org.junit.jupiter.api.Assertions.fail
 import java.io.ByteArrayOutputStream
 import kotlin.time.*
 
-internal abstract class AbstractLocalProcessRunner<R>(private val checks: TestRunChecks) : AbstractRunner<R>() {
+internal abstract class AbstractLocalProcessRunner<R>(protected val checks: TestRunChecks) : AbstractRunner<R>() {
     protected abstract val visibleProcessName: String
     protected abstract val executable: TestExecutable
     protected abstract val programArgs: List<String>
@@ -72,64 +72,68 @@ internal abstract class AbstractLocalProcessRunner<R>(private val checks: TestRu
         }
     }
 
-    abstract override fun buildResultHandler(runResult: RunResult): ResultHandler // Narrow returned type.
+    abstract override fun buildResultHandler(runResult: RunResult): LocalResultHandler<R> // ?? Narrow returned type.
+}
 
-    abstract inner class ResultHandler(runResult: RunResult) : AbstractRunner<R>.ResultHandler(runResult) {
+internal abstract class LocalResultHandler<R>(
-        override fun handle(): R {
+    runResult: RunResult,
-            checks.forEach { check ->
+    private val visibleProcessName: String,
-                when (check) {
+    private val checks: TestRunChecks
-                    is ExecutionTimeout.ShouldNotExceed -> verifyExpectation(runResult.hasFinishedOnTime) {
+) : AbstractResultHandler<R>(runResult) {
-                        "Timeout exceeded during test execution."
+    override fun handle(): R {
-                    }
+        checks.forEach { check ->
-                    is ExecutionTimeout.ShouldExceed -> verifyExpectation(!runResult.hasFinishedOnTime) {
+            when (check) {
-                        "Test is expected to fail with exceeded timeout, which hasn't happened."
+                is ExecutionTimeout.ShouldNotExceed -> verifyExpectation(runResult.hasFinishedOnTime) {
-                    }
+                    "Timeout exceeded during test execution."
-                    is ExitCode -> {
+                }
-                        // Don't check exit code if it is unknown.
+                is ExecutionTimeout.ShouldExceed -> verifyExpectation(!runResult.hasFinishedOnTime) {
-                        val knownExitCode: Int = runResult.exitCode ?: return@forEach
+                    "Test is expected to fail with exceeded timeout, which hasn't happened."
-                        when (check) {
+                }
-                            is ExitCode.Expected -> verifyExpectation(knownExitCode == check.expectedExitCode) {
+                is ExitCode -> {
-                                "$visibleProcessName exit code is $knownExitCode while ${check.expectedExitCode} was expected."
+                    // Don't check exit code if it is unknown.
-                            }
+                    val knownExitCode: Int = runResult.exitCode ?: return@forEach
-                            is ExitCode.AnyNonZero -> verifyExpectation(knownExitCode != 0) {
+                    when (check) {
-                                "$visibleProcessName exited with zero code, which wasn't expected."
+                        is ExitCode.Expected -> verifyExpectation(knownExitCode == check.expectedExitCode) {
-                            }
+                            "$visibleProcessName exit code is $knownExitCode while ${check.expectedExitCode} was expected."
+                        }
+                        is ExitCode.AnyNonZero -> verifyExpectation(knownExitCode != 0) {
+                            "$visibleProcessName exited with zero code, which wasn't expected."
                         }
                     }
-                    is TestRunCheck.OutputDataFile -> {
+                }
-                        val expectedOutput = check.file.readText()
+                is TestRunCheck.OutputDataFile -> {
-                        val actualFilteredOutput = runResult.processOutput.stdOut.filteredOutput + runResult.processOutput.stdErr
+                    val expectedOutput = check.file.readText()
+                    val actualFilteredOutput = runResult.processOutput.stdOut.filteredOutput + runResult.processOutput.stdErr
 
-                        // Don't use verifyExpectation(expected, actual) to avoid exposing potentially large test output in exception message
+                    // Don't use verifyExpectation(expected, actual) to avoid exposing potentially large test output in exception message
-                        // and blowing up test logs.
+                    // and blowing up test logs.
-                        verifyExpectation(convertLineSeparators(expectedOutput) == convertLineSeparators(actualFilteredOutput)) {
+                    verifyExpectation(convertLineSeparators(expectedOutput) == convertLineSeparators(actualFilteredOutput)) {
-                            "Tested process output mismatch. See \"TEST STDOUT\" and \"EXPECTED OUTPUT DATA FILE\" below."
+                        "Tested process output mismatch. See \"TEST STDOUT\" and \"EXPECTED OUTPUT DATA FILE\" below."
-                        }
                     }
-                    is TestRunCheck.OutputMatcher -> {
+                }
-                        try {
+                is TestRunCheck.OutputMatcher -> {
-                            verifyExpectation(check.match(runResult.processOutput.stdOut.filteredOutput)) {
+                    try {
-                                "Tested process output has not passed validation."
+                        verifyExpectation(check.match(runResult.processOutput.stdOut.filteredOutput)) {
-                            }
+                            "Tested process output has not passed validation."
-                        } catch (t: Throwable) {
+                        }
-                            if (t is Exception || t is AssertionError) {
+                    } catch (t: Throwable) {
-                                fail<Nothing>(
+                        if (t is Exception || t is AssertionError) {
-                                    getLoggedRun().withErrorMessage("Tested process output has not passed validation: " + t.message),
+                            fail<Nothing>(
-                                    t
+                                getLoggedRun().withErrorMessage("Tested process output has not passed validation: " + t.message),
-                                )
+                                t
-                            } else {
+                            )
-                                throw t
+                        } else {
-                            }
+                            throw t
                         }
                     }
                 }
             }
-
-            return doHandle()
         }
 
-        protected abstract fun doHandle(): R
+        return doHandle()
     }
+
+    protected abstract fun doHandle(): R
 }
 
 private class UnfilteredProcessOutput {

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractRunner.kt

@@ -9,9 +9,9 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.LoggedData
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue
 import org.opentest4j.TestAbortedException
 
-internal abstract class AbstractRunner<R>: Runner<R> {
+internal abstract class AbstractRunner<R> : Runner<R> {
     protected abstract fun buildRun(): AbstractRun
-    protected abstract fun buildResultHandler(runResult: RunResult): ResultHandler
+    protected abstract fun buildResultHandler(runResult: RunResult): AbstractResultHandler<R>
     protected abstract fun getLoggedParameters(): LoggedData.TestRunParameters
     protected abstract fun handleUnexpectedFailure(t: Throwable): Nothing
 
@@ -33,13 +33,13 @@ internal abstract class AbstractRunner<R>: Runner<R> {
     fun interface AbstractRun {
         fun run(): RunResult
     }
+}
 
-    abstract inner class ResultHandler(protected val runResult: RunResult) {
+internal abstract class AbstractResultHandler<R>(protected val runResult: RunResult) {
-        abstract fun getLoggedRun(): LoggedData
+    abstract fun getLoggedRun(): LoggedData
-        abstract fun handle(): R
+    abstract fun handle(): R
 
-        protected inline fun verifyExpectation(shouldBeTrue: Boolean, crossinline errorMessage: () -> String) {
+    protected inline fun verifyExpectation(shouldBeTrue: Boolean, crossinline errorMessage: () -> String) {
-            assertTrue(shouldBeTrue) { getLoggedRun().withErrorMessage(errorMessage()) }
+        assertTrue(shouldBeTrue) { getLoggedRun().withErrorMessage(errorMessage()) }
-        }
     }
 }

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestNameExtractor.kt

@@ -26,17 +26,21 @@ internal class LocalTestNameExtractor(
         runParameters = null
     )
 
-    override fun buildResultHandler(runResult: RunResult) = ResultHandler(runResult)
+    override fun buildResultHandler(runResult: RunResult) =
+        TestNameResultHandler(runResult, visibleProcessName, checks, getLoggedParameters())
 
     override fun handleUnexpectedFailure(t: Throwable) = fail {
         LoggedData.TestRunUnexpectedFailure(getLoggedParameters(), t)
             .withErrorMessage("Test name extraction failed with unexpected exception.")
     }
-
+}
-    inner class ResultHandler(
+
-        runResult: RunResult
+internal class TestNameResultHandler(
-    ) : AbstractLocalProcessRunner<Collection<TestName>>.ResultHandler(runResult) {
+    runResult: RunResult,
-        override fun getLoggedRun() = LoggedData.TestRun(getLoggedParameters(), runResult)
+    visibleProcessName: String,
-        override fun doHandle() = GTestListing.parse(runResult.processOutput.stdOut.filteredOutput)
+    checks: TestRunChecks,
-    }
+    private val loggedParameters: LoggedData.TestRunParameters
+) : LocalResultHandler<Collection<TestName>>(runResult, visibleProcessName, checks) {
+    override fun getLoggedRun() = LoggedData.TestRun(loggedParameters, runResult)
+    override fun doHandle() = GTestListing.parse(runResult.processOutput.stdOut.filteredOutput)
 }

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt

@@ -38,47 +38,59 @@ internal class LocalTestRunner(private val testRun: TestRun) : AbstractLocalProc
         }
     }
 
-    override fun buildResultHandler(runResult: RunResult) = ResultHandler(runResult)
+    override fun buildResultHandler(runResult: RunResult) = ResultHandler(
+        runResult = runResult,
+        visibleProcessName = visibleProcessName,
+        checks = testRun.checks,
+        testRun = testRun,
+        loggedParameters = getLoggedParameters()
+    )
 
     override fun handleUnexpectedFailure(t: Throwable) = fail {
         LoggedData.TestRunUnexpectedFailure(getLoggedParameters(), t)
             .withErrorMessage("Test execution failed with unexpected exception.")
     }
+}
 
-    inner class ResultHandler(runResult: RunResult) : AbstractLocalProcessRunner<Unit>.ResultHandler(runResult) {
+internal class ResultHandler(
-        override fun getLoggedRun() = LoggedData.TestRun(getLoggedParameters(), runResult)
+    runResult: RunResult,
+    visibleProcessName: String,
+    checks: TestRunChecks,
+    private val testRun: TestRun,
+    private val loggedParameters: LoggedData.TestRunParameters
+) : LocalResultHandler<Unit>(runResult, visibleProcessName, checks) {
+    override fun getLoggedRun() = LoggedData.TestRun(loggedParameters, runResult)
 
-        override fun doHandle() {
+    override fun doHandle() {
-            verifyTestReport(runResult.processOutput.stdOut.testReport)
+        verifyTestReport(runResult.processOutput.stdOut.testReport)
+    }
+
+    private fun verifyTestReport(testReport: TestReport?) {
+        if (testReport == null) return
+
+        verifyExpectation(!testReport.isEmpty()) { "No tests have been found." }
+
+        testRun.runParameters.get<TestRunParameter.WithFilter> {
+            verifyNoSuchTests(
+                testReport.passedTests.filter { testName -> !testMatches(testName) },
+                "Excessive tests have been executed"
+            )
+
+            verifyNoSuchTests(
+                testReport.ignoredTests.filter { testName -> !testMatches(testName) },
+                "Excessive tests have been ignored"
+            )
         }
 
-        private fun verifyTestReport(testReport: TestReport?) {
+        verifyNoSuchTests(testReport.failedTests, "There are failed tests")
-            if (testReport == null) return
 
-            verifyExpectation(!testReport.isEmpty()) { "No tests have been found." }
+        assumeFalse(testReport.ignoredTests.isNotEmpty() && testReport.passedTests.isEmpty(), "Test case is disabled")
+    }
 
-            testRun.runParameters.get<TestRunParameter.WithFilter> {
+    private fun verifyNoSuchTests(tests: Collection<TestName>, subject: String) = verifyExpectation(tests.isEmpty()) {
-                verifyNoSuchTests(
+        buildString {
-                    testReport.passedTests.filter { testName -> !testMatches(testName) },
+            append(subject).append(':')
-                    "Excessive tests have been executed"
+            tests.forEach { appendLine().append(" - ").append(it) }
-                )
-
-                verifyNoSuchTests(
-                    testReport.ignoredTests.filter { testName -> !testMatches(testName) },
-                    "Excessive tests have been ignored"
-                )
-            }
-
-            verifyNoSuchTests(testReport.failedTests, "There are failed tests")
-
-            assumeFalse(testReport.ignoredTests.isNotEmpty() && testReport.passedTests.isEmpty(), "Test case is disabled")
-        }
-
-        private fun verifyNoSuchTests(tests: Collection<TestName>, subject: String) = verifyExpectation(tests.isEmpty()) {
-            buildString {
-                append(subject).append(':')
-                tests.forEach { appendLine().append(" - ").append(it) }
-            }
         }
     }
 }

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/RunnerWithExecutor.kt

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.konan.blackboxtest.support.runner
+
+import org.jetbrains.kotlin.native.executors.ExecuteRequest
+import org.jetbrains.kotlin.native.executors.Executor
+import org.jetbrains.kotlin.konan.blackboxtest.support.LoggedData
+import org.jetbrains.kotlin.konan.blackboxtest.support.util.TCTestOutputFilter
+import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestOutputFilter
+import org.jetbrains.kotlin.test.services.JUnit5Assertions
+import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull
+import java.io.ByteArrayInputStream
+import java.io.ByteArrayOutputStream
+import java.io.InputStream
+
+internal class RunnerWithExecutor(
+    private val executor: Executor,
+    private val testRun: TestRun
+) : AbstractRunner<Unit>() {
+    private val executable get() = testRun.executable
+
+    private val outputFilter: TestOutputFilter
+        get() = if (testRun.runParameters.has<TestRunParameter.WithTCTestLogger>()) TCTestOutputFilter else TestOutputFilter.NO_FILTERING
+
+    private val programArgs = mutableListOf<String>().apply {
+        testRun.runParameters.forEach { it.applyTo(this) }
+    }
+
+    private fun inputStreamFromTestParameter(): InputStream =
+        testRun.runParameters.firstIsInstanceOrNull<TestRunParameter.WithInputData>()
+            ?.let {
+                ByteArrayInputStream(it.inputDataFile.readBytes())
+            } ?: ByteArrayInputStream(byteArrayOf())
+
+    override fun buildRun() = AbstractRun {
+        val stdin = inputStreamFromTestParameter()
+        val stdout = ByteArrayOutputStream()
+        val stderr = ByteArrayOutputStream()
+        val request = ExecuteRequest(
+            executableAbsolutePath = executable.executableFile.absolutePath,
+            args = programArgs,
+            stdin = stdin,
+            stdout = stdout,
+            stderr = stderr,
+            timeout = testRun.checks.executionTimeoutCheck.timeout
+        )
+        val response = executor.execute(request)
+        RunResult(
+            exitCode = response.exitCode,
+            timeout = request.timeout,
+            duration = response.executionTime,
+            hasFinishedOnTime = request.timeout >= response.executionTime,
+            processOutput = ProcessOutput(
+                stdOut = outputFilter.filter(stdout.toString("UTF-8")),
+                stdErr = stderr.toString("UTF-8")
+            )
+        )
+    }
+
+    override fun buildResultHandler(runResult: RunResult): ResultHandler = ResultHandler(
+        runResult = runResult,
+        visibleProcessName = "Test process under Executor ${executor::class.simpleName}",
+        checks = testRun.checks,
+        testRun = testRun,
+        loggedParameters = getLoggedParameters()
+    )
+
+    override fun getLoggedParameters() = LoggedData.TestRunParameters(
+        compilationToolCall = executable.loggedCompilationToolCall,
+        testCaseId = testRun.testCaseId,
+        runArgs = programArgs,
+        runParameters = testRun.runParameters
+    )
+
+    override fun handleUnexpectedFailure(t: Throwable) = JUnit5Assertions.fail {
+        LoggedData.TestRunUnexpectedFailure(getLoggedParameters(), t)
+            .withErrorMessage("Test execution failed with unexpected exception.")
+    }
+}

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunners.kt

@@ -7,39 +7,58 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.runner
 
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestName
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.ForcedNoopTestRunner
+import org.jetbrains.kotlin.konan.blackboxtest.support.settings.KotlinNativeHome
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.KotlinNativeTargets
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Settings
 import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Timeouts
+import org.jetbrains.kotlin.native.executors.Executor
+import org.jetbrains.kotlin.native.executors.EmulatorExecutor
+import org.jetbrains.kotlin.native.executors.XcodeSimulatorExecutor
+import org.jetbrains.kotlin.konan.target.*
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
+import java.util.concurrent.ConcurrentHashMap
 
 internal object TestRunners {
-    // Currently, only local and noop test runners are supported.
     fun createProperTestRunner(testRun: TestRun, settings: Settings): Runner<Unit> = with(settings) {
         if (get<ForcedNoopTestRunner>().value) {
             NoopTestRunner
         } else with(get<KotlinNativeTargets>()) {
-            if (testTarget == hostTarget)
+            if (testTarget == hostTarget) {
                 LocalTestRunner(testRun)
-            else
+            } else {
-                runningAtNonHostTarget()
+                val nativeHome = get<KotlinNativeHome>()
+                val distribution = Distribution(nativeHome.dir.path)
+                val configurables = PlatformManager(distribution, true).platform(testTarget).configurables
+
+                val executor = cached(
+                    when {
+                        configurables is ConfigurablesWithEmulator -> EmulatorExecutor(configurables)
+                        configurables is AppleConfigurables && configurables.targetTriple.isSimulator ->
+                            XcodeSimulatorExecutor(configurables)
+                        else -> runningOnUnsupportedTarget()
+                    }
+                )
+                RunnerWithExecutor(executor, testRun)
+            }
         }
     }
 
+    private val runnersCache: ConcurrentHashMap<String, Executor> = ConcurrentHashMap()
+
+    private inline fun <reified T : Executor> cached(executor: T): Executor =
+        runnersCache.computeIfAbsent(T::class.java.simpleName) { executor }
+
     // Currently, only local test name extractor is supported.
     fun extractTestNames(executable: TestExecutable, settings: Settings): Collection<TestName> = with(settings) {
         with(get<KotlinNativeTargets>()) {
             if (testTarget == hostTarget)
                 LocalTestNameExtractor(executable, TestRunChecks.Default(get<Timeouts>().executionTimeout)).run()
             else
-                runningAtNonHostTarget()
+                runningOnUnsupportedTarget()
         }
     }
 
-    private fun KotlinNativeTargets.runningAtNonHostTarget(): Nothing = fail {
+    private fun KotlinNativeTargets.runningOnUnsupportedTarget(): Nothing = fail {
-        """
+        "Running tests for $testTarget on $hostTarget is not supported yet."
-            Running at non-host target is not supported yet.
-            Compilation target: $testTarget
-            Host target: $hostTarget
-        """.trimIndent()
     }
 }

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/settings/TestProcessSettings.kt

@@ -23,7 +23,9 @@ import kotlin.time.Duration.Companion.seconds
 /**
  * The tested and the host Kotlin/Native targets.
  */
-internal class KotlinNativeTargets(val testTarget: KonanTarget, val hostTarget: KonanTarget)
+internal class KotlinNativeTargets(val testTarget: KonanTarget, val hostTarget: KonanTarget) {
+    fun areDifferentTargets() = testTarget != hostTarget
+}
 
 /**
  * The Kotlin/Native home.