diff --git a/native/executors/src/main/kotlin/org/jetbrains/kotlin/native/executors/XcRunRuntimeUtils.kt b/native/executors/src/main/kotlin/org/jetbrains/kotlin/native/executors/XcRunRuntimeUtils.kt index d9fec58eb69..f8856f92a13 100644 --- a/native/executors/src/main/kotlin/org/jetbrains/kotlin/native/executors/XcRunRuntimeUtils.kt +++ b/native/executors/src/main/kotlin/org/jetbrains/kotlin/native/executors/XcRunRuntimeUtils.kt @@ -108,10 +108,10 @@ data class SimulatorRuntimeDescriptor( } data class DeviceType( - @Expose val bundlePath: String, - @Expose val name: String, - @Expose val identifier: String, - @Expose val productFamily: String + @Expose val bundlePath: String, + @Expose val name: String, + @Expose val identifier: String, + @Expose val productFamily: String ) /** diff --git a/native/native.tests/build.gradle.kts b/native/native.tests/build.gradle.kts index ff9a067699c..ce76d96bf60 100644 --- a/native/native.tests/build.gradle.kts +++ b/native/native.tests/build.gradle.kts @@ -18,6 +18,8 @@ dependencies { testImplementation(projectTests(":compiler:tests-common-new")) testImplementation(projectTests(":compiler:test-infrastructure")) testImplementation(projectTests(":generators:test-generator")) + testImplementation(project(":native:kotlin-native-utils")) + testImplementation(project(":native:executors")) testApiJUnit5() testImplementation(commonDependency("org.jetbrains.kotlinx", "kotlinx-coroutines-core")) { isTransitive = false } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt index 30b29098364..ba7399f2393 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt @@ -16,7 +16,7 @@ import org.junit.jupiter.api.Assertions.fail import java.io.ByteArrayOutputStream import kotlin.time.* -internal abstract class AbstractLocalProcessRunner(private val checks: TestRunChecks) : AbstractRunner() { +internal abstract class AbstractLocalProcessRunner(protected val checks: TestRunChecks) : AbstractRunner() { protected abstract val visibleProcessName: String protected abstract val executable: TestExecutable protected abstract val programArgs: List @@ -72,64 +72,68 @@ internal abstract class AbstractLocalProcessRunner(private val checks: TestRu } } - abstract override fun buildResultHandler(runResult: RunResult): ResultHandler // Narrow returned type. + abstract override fun buildResultHandler(runResult: RunResult): LocalResultHandler // ?? Narrow returned type. +} - abstract inner class ResultHandler(runResult: RunResult) : AbstractRunner.ResultHandler(runResult) { - override fun handle(): R { - checks.forEach { check -> - when (check) { - is ExecutionTimeout.ShouldNotExceed -> verifyExpectation(runResult.hasFinishedOnTime) { - "Timeout exceeded during test execution." - } - is ExecutionTimeout.ShouldExceed -> verifyExpectation(!runResult.hasFinishedOnTime) { - "Test is expected to fail with exceeded timeout, which hasn't happened." - } - is ExitCode -> { - // Don't check exit code if it is unknown. - val knownExitCode: Int = runResult.exitCode ?: return@forEach - when (check) { - is ExitCode.Expected -> verifyExpectation(knownExitCode == check.expectedExitCode) { - "$visibleProcessName exit code is $knownExitCode while ${check.expectedExitCode} was expected." - } - is ExitCode.AnyNonZero -> verifyExpectation(knownExitCode != 0) { - "$visibleProcessName exited with zero code, which wasn't expected." - } +internal abstract class LocalResultHandler( + runResult: RunResult, + private val visibleProcessName: String, + private val checks: TestRunChecks +) : AbstractResultHandler(runResult) { + override fun handle(): R { + checks.forEach { check -> + when (check) { + is ExecutionTimeout.ShouldNotExceed -> verifyExpectation(runResult.hasFinishedOnTime) { + "Timeout exceeded during test execution." + } + is ExecutionTimeout.ShouldExceed -> verifyExpectation(!runResult.hasFinishedOnTime) { + "Test is expected to fail with exceeded timeout, which hasn't happened." + } + is ExitCode -> { + // Don't check exit code if it is unknown. + val knownExitCode: Int = runResult.exitCode ?: return@forEach + when (check) { + is ExitCode.Expected -> verifyExpectation(knownExitCode == check.expectedExitCode) { + "$visibleProcessName exit code is $knownExitCode while ${check.expectedExitCode} was expected." + } + is ExitCode.AnyNonZero -> verifyExpectation(knownExitCode != 0) { + "$visibleProcessName exited with zero code, which wasn't expected." } } - is TestRunCheck.OutputDataFile -> { - val expectedOutput = check.file.readText() - val actualFilteredOutput = runResult.processOutput.stdOut.filteredOutput + runResult.processOutput.stdErr + } + is TestRunCheck.OutputDataFile -> { + val expectedOutput = check.file.readText() + val actualFilteredOutput = runResult.processOutput.stdOut.filteredOutput + runResult.processOutput.stdErr - // Don't use verifyExpectation(expected, actual) to avoid exposing potentially large test output in exception message - // and blowing up test logs. - verifyExpectation(convertLineSeparators(expectedOutput) == convertLineSeparators(actualFilteredOutput)) { - "Tested process output mismatch. See \"TEST STDOUT\" and \"EXPECTED OUTPUT DATA FILE\" below." - } + // Don't use verifyExpectation(expected, actual) to avoid exposing potentially large test output in exception message + // and blowing up test logs. + verifyExpectation(convertLineSeparators(expectedOutput) == convertLineSeparators(actualFilteredOutput)) { + "Tested process output mismatch. See \"TEST STDOUT\" and \"EXPECTED OUTPUT DATA FILE\" below." } - is TestRunCheck.OutputMatcher -> { - try { - verifyExpectation(check.match(runResult.processOutput.stdOut.filteredOutput)) { - "Tested process output has not passed validation." - } - } catch (t: Throwable) { - if (t is Exception || t is AssertionError) { - fail( - getLoggedRun().withErrorMessage("Tested process output has not passed validation: " + t.message), - t - ) - } else { - throw t - } + } + is TestRunCheck.OutputMatcher -> { + try { + verifyExpectation(check.match(runResult.processOutput.stdOut.filteredOutput)) { + "Tested process output has not passed validation." + } + } catch (t: Throwable) { + if (t is Exception || t is AssertionError) { + fail( + getLoggedRun().withErrorMessage("Tested process output has not passed validation: " + t.message), + t + ) + } else { + throw t } } } } - - return doHandle() } - protected abstract fun doHandle(): R + return doHandle() } + + protected abstract fun doHandle(): R } private class UnfilteredProcessOutput { diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractRunner.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractRunner.kt index d76974633b9..2b703508991 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractRunner.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractRunner.kt @@ -9,9 +9,9 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.LoggedData import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue import org.opentest4j.TestAbortedException -internal abstract class AbstractRunner: Runner { +internal abstract class AbstractRunner : Runner { protected abstract fun buildRun(): AbstractRun - protected abstract fun buildResultHandler(runResult: RunResult): ResultHandler + protected abstract fun buildResultHandler(runResult: RunResult): AbstractResultHandler protected abstract fun getLoggedParameters(): LoggedData.TestRunParameters protected abstract fun handleUnexpectedFailure(t: Throwable): Nothing @@ -33,13 +33,13 @@ internal abstract class AbstractRunner: Runner { fun interface AbstractRun { fun run(): RunResult } +} - abstract inner class ResultHandler(protected val runResult: RunResult) { - abstract fun getLoggedRun(): LoggedData - abstract fun handle(): R +internal abstract class AbstractResultHandler(protected val runResult: RunResult) { + abstract fun getLoggedRun(): LoggedData + abstract fun handle(): R - protected inline fun verifyExpectation(shouldBeTrue: Boolean, crossinline errorMessage: () -> String) { - assertTrue(shouldBeTrue) { getLoggedRun().withErrorMessage(errorMessage()) } - } + protected inline fun verifyExpectation(shouldBeTrue: Boolean, crossinline errorMessage: () -> String) { + assertTrue(shouldBeTrue) { getLoggedRun().withErrorMessage(errorMessage()) } } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestNameExtractor.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestNameExtractor.kt index ddad4c9229a..ab7b063bfaf 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestNameExtractor.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestNameExtractor.kt @@ -26,17 +26,21 @@ internal class LocalTestNameExtractor( runParameters = null ) - override fun buildResultHandler(runResult: RunResult) = ResultHandler(runResult) + override fun buildResultHandler(runResult: RunResult) = + TestNameResultHandler(runResult, visibleProcessName, checks, getLoggedParameters()) override fun handleUnexpectedFailure(t: Throwable) = fail { LoggedData.TestRunUnexpectedFailure(getLoggedParameters(), t) .withErrorMessage("Test name extraction failed with unexpected exception.") } - - inner class ResultHandler( - runResult: RunResult - ) : AbstractLocalProcessRunner>.ResultHandler(runResult) { - override fun getLoggedRun() = LoggedData.TestRun(getLoggedParameters(), runResult) - override fun doHandle() = GTestListing.parse(runResult.processOutput.stdOut.filteredOutput) - } +} + +internal class TestNameResultHandler( + runResult: RunResult, + visibleProcessName: String, + checks: TestRunChecks, + private val loggedParameters: LoggedData.TestRunParameters +) : LocalResultHandler>(runResult, visibleProcessName, checks) { + override fun getLoggedRun() = LoggedData.TestRun(loggedParameters, runResult) + override fun doHandle() = GTestListing.parse(runResult.processOutput.stdOut.filteredOutput) } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt index c22775b879e..c0f3f25ca01 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt @@ -38,47 +38,59 @@ internal class LocalTestRunner(private val testRun: TestRun) : AbstractLocalProc } } - override fun buildResultHandler(runResult: RunResult) = ResultHandler(runResult) + override fun buildResultHandler(runResult: RunResult) = ResultHandler( + runResult = runResult, + visibleProcessName = visibleProcessName, + checks = testRun.checks, + testRun = testRun, + loggedParameters = getLoggedParameters() + ) override fun handleUnexpectedFailure(t: Throwable) = fail { LoggedData.TestRunUnexpectedFailure(getLoggedParameters(), t) .withErrorMessage("Test execution failed with unexpected exception.") } +} - inner class ResultHandler(runResult: RunResult) : AbstractLocalProcessRunner.ResultHandler(runResult) { - override fun getLoggedRun() = LoggedData.TestRun(getLoggedParameters(), runResult) +internal class ResultHandler( + runResult: RunResult, + visibleProcessName: String, + checks: TestRunChecks, + private val testRun: TestRun, + private val loggedParameters: LoggedData.TestRunParameters +) : LocalResultHandler(runResult, visibleProcessName, checks) { + override fun getLoggedRun() = LoggedData.TestRun(loggedParameters, runResult) - override fun doHandle() { - verifyTestReport(runResult.processOutput.stdOut.testReport) + override fun doHandle() { + verifyTestReport(runResult.processOutput.stdOut.testReport) + } + + private fun verifyTestReport(testReport: TestReport?) { + if (testReport == null) return + + verifyExpectation(!testReport.isEmpty()) { "No tests have been found." } + + testRun.runParameters.get { + verifyNoSuchTests( + testReport.passedTests.filter { testName -> !testMatches(testName) }, + "Excessive tests have been executed" + ) + + verifyNoSuchTests( + testReport.ignoredTests.filter { testName -> !testMatches(testName) }, + "Excessive tests have been ignored" + ) } - private fun verifyTestReport(testReport: TestReport?) { - if (testReport == null) return + verifyNoSuchTests(testReport.failedTests, "There are failed tests") - verifyExpectation(!testReport.isEmpty()) { "No tests have been found." } + assumeFalse(testReport.ignoredTests.isNotEmpty() && testReport.passedTests.isEmpty(), "Test case is disabled") + } - testRun.runParameters.get { - verifyNoSuchTests( - testReport.passedTests.filter { testName -> !testMatches(testName) }, - "Excessive tests have been executed" - ) - - verifyNoSuchTests( - testReport.ignoredTests.filter { testName -> !testMatches(testName) }, - "Excessive tests have been ignored" - ) - } - - verifyNoSuchTests(testReport.failedTests, "There are failed tests") - - assumeFalse(testReport.ignoredTests.isNotEmpty() && testReport.passedTests.isEmpty(), "Test case is disabled") - } - - private fun verifyNoSuchTests(tests: Collection, subject: String) = verifyExpectation(tests.isEmpty()) { - buildString { - append(subject).append(':') - tests.forEach { appendLine().append(" - ").append(it) } - } + private fun verifyNoSuchTests(tests: Collection, subject: String) = verifyExpectation(tests.isEmpty()) { + buildString { + append(subject).append(':') + tests.forEach { appendLine().append(" - ").append(it) } } } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/RunnerWithExecutor.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/RunnerWithExecutor.kt new file mode 100644 index 00000000000..7ee2f9f542d --- /dev/null +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/RunnerWithExecutor.kt @@ -0,0 +1,82 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.konan.blackboxtest.support.runner + +import org.jetbrains.kotlin.native.executors.ExecuteRequest +import org.jetbrains.kotlin.native.executors.Executor +import org.jetbrains.kotlin.konan.blackboxtest.support.LoggedData +import org.jetbrains.kotlin.konan.blackboxtest.support.util.TCTestOutputFilter +import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestOutputFilter +import org.jetbrains.kotlin.test.services.JUnit5Assertions +import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.io.InputStream + +internal class RunnerWithExecutor( + private val executor: Executor, + private val testRun: TestRun +) : AbstractRunner() { + private val executable get() = testRun.executable + + private val outputFilter: TestOutputFilter + get() = if (testRun.runParameters.has()) TCTestOutputFilter else TestOutputFilter.NO_FILTERING + + private val programArgs = mutableListOf().apply { + testRun.runParameters.forEach { it.applyTo(this) } + } + + private fun inputStreamFromTestParameter(): InputStream = + testRun.runParameters.firstIsInstanceOrNull() + ?.let { + ByteArrayInputStream(it.inputDataFile.readBytes()) + } ?: ByteArrayInputStream(byteArrayOf()) + + override fun buildRun() = AbstractRun { + val stdin = inputStreamFromTestParameter() + val stdout = ByteArrayOutputStream() + val stderr = ByteArrayOutputStream() + val request = ExecuteRequest( + executableAbsolutePath = executable.executableFile.absolutePath, + args = programArgs, + stdin = stdin, + stdout = stdout, + stderr = stderr, + timeout = testRun.checks.executionTimeoutCheck.timeout + ) + val response = executor.execute(request) + RunResult( + exitCode = response.exitCode, + timeout = request.timeout, + duration = response.executionTime, + hasFinishedOnTime = request.timeout >= response.executionTime, + processOutput = ProcessOutput( + stdOut = outputFilter.filter(stdout.toString("UTF-8")), + stdErr = stderr.toString("UTF-8") + ) + ) + } + + override fun buildResultHandler(runResult: RunResult): ResultHandler = ResultHandler( + runResult = runResult, + visibleProcessName = "Test process under Executor ${executor::class.simpleName}", + checks = testRun.checks, + testRun = testRun, + loggedParameters = getLoggedParameters() + ) + + override fun getLoggedParameters() = LoggedData.TestRunParameters( + compilationToolCall = executable.loggedCompilationToolCall, + testCaseId = testRun.testCaseId, + runArgs = programArgs, + runParameters = testRun.runParameters + ) + + override fun handleUnexpectedFailure(t: Throwable) = JUnit5Assertions.fail { + LoggedData.TestRunUnexpectedFailure(getLoggedParameters(), t) + .withErrorMessage("Test execution failed with unexpected exception.") + } +} \ No newline at end of file diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunners.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunners.kt index 2e13c74b11a..a95472377ee 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunners.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunners.kt @@ -7,39 +7,58 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.runner import org.jetbrains.kotlin.konan.blackboxtest.support.TestName import org.jetbrains.kotlin.konan.blackboxtest.support.settings.ForcedNoopTestRunner +import org.jetbrains.kotlin.konan.blackboxtest.support.settings.KotlinNativeHome import org.jetbrains.kotlin.konan.blackboxtest.support.settings.KotlinNativeTargets import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Settings import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Timeouts +import org.jetbrains.kotlin.native.executors.Executor +import org.jetbrains.kotlin.native.executors.EmulatorExecutor +import org.jetbrains.kotlin.native.executors.XcodeSimulatorExecutor +import org.jetbrains.kotlin.konan.target.* import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail +import java.util.concurrent.ConcurrentHashMap internal object TestRunners { - // Currently, only local and noop test runners are supported. fun createProperTestRunner(testRun: TestRun, settings: Settings): Runner = with(settings) { if (get().value) { NoopTestRunner } else with(get()) { - if (testTarget == hostTarget) + if (testTarget == hostTarget) { LocalTestRunner(testRun) - else - runningAtNonHostTarget() + } else { + val nativeHome = get() + val distribution = Distribution(nativeHome.dir.path) + val configurables = PlatformManager(distribution, true).platform(testTarget).configurables + + val executor = cached( + when { + configurables is ConfigurablesWithEmulator -> EmulatorExecutor(configurables) + configurables is AppleConfigurables && configurables.targetTriple.isSimulator -> + XcodeSimulatorExecutor(configurables) + else -> runningOnUnsupportedTarget() + } + ) + RunnerWithExecutor(executor, testRun) + } } } + private val runnersCache: ConcurrentHashMap = ConcurrentHashMap() + + private inline fun cached(executor: T): Executor = + runnersCache.computeIfAbsent(T::class.java.simpleName) { executor } + // Currently, only local test name extractor is supported. fun extractTestNames(executable: TestExecutable, settings: Settings): Collection = with(settings) { with(get()) { if (testTarget == hostTarget) LocalTestNameExtractor(executable, TestRunChecks.Default(get().executionTimeout)).run() else - runningAtNonHostTarget() + runningOnUnsupportedTarget() } } - private fun KotlinNativeTargets.runningAtNonHostTarget(): Nothing = fail { - """ - Running at non-host target is not supported yet. - Compilation target: $testTarget - Host target: $hostTarget - """.trimIndent() + private fun KotlinNativeTargets.runningOnUnsupportedTarget(): Nothing = fail { + "Running tests for $testTarget on $hostTarget is not supported yet." } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/settings/TestProcessSettings.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/settings/TestProcessSettings.kt index a45ed2aa557..8eac2e57d63 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/settings/TestProcessSettings.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/settings/TestProcessSettings.kt @@ -23,7 +23,9 @@ import kotlin.time.Duration.Companion.seconds /** * The tested and the host Kotlin/Native targets. */ -internal class KotlinNativeTargets(val testTarget: KonanTarget, val hostTarget: KonanTarget) +internal class KotlinNativeTargets(val testTarget: KonanTarget, val hostTarget: KonanTarget) { + fun areDifferentTargets() = testTarget != hostTarget +} /** * The Kotlin/Native home.