azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[K/N][test] Use Executors in the New test infra

Browse Source 
Add RunnerWithExecutor class that uses Executors
Re-use checkers in the new executor

Merge-request: KT-MR-8964
This commit is contained in:
Pavel Punegov
2023-02-22 15:11:21 +01:00
committed by Space Team
parent 76ab130011
commit 60f43d6d4f
9 changed files with 234 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
native/executors/src/main/kotlin/org/jetbrains/kotlin/native/executors/XcRunRuntimeUtils.kt
+4 -4
View File
@@ -108,10 +108,10 @@ data class SimulatorRuntimeDescriptor(
}
data class DeviceType(
@Expose val bundlePath: String,
@Expose val name: String,
@Expose val identifier: String,
@Expose val productFamily: String
@Expose val bundlePath: String,
@Expose val name: String,
@Expose val identifier: String,
@Expose val productFamily: String
)
/**
native/native.tests/build.gradle.kts
+2
View File
@@ -18,6 +18,8 @@ dependencies {
testImplementation(projectTests(":compiler:tests-common-new"))
testImplementation(projectTests(":compiler:test-infrastructure"))
testImplementation(projectTests(":generators:test-generator"))
testImplementation(project(":native:kotlin-native-utils"))
testImplementation(project(":native:executors"))
testApiJUnit5()
testImplementation(commonDependency("org.jetbrains.kotlinx", "kotlinx-coroutines-core")) { isTransitive = false }
native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt
+51 -47
View File
@@ -16,7 +16,7 @@ import org.junit.jupiter.api.Assertions.fail
import java.io.ByteArrayOutputStream
import kotlin.time.*
internal abstract class AbstractLocalProcessRunner<R>(private val checks: TestRunChecks) : AbstractRunner<R>() {
internal abstract class AbstractLocalProcessRunner<R>(protected val checks: TestRunChecks) : AbstractRunner<R>() {
protected abstract val visibleProcessName: String
protected abstract val executable: TestExecutable
protected abstract val programArgs: List<String>
@@ -72,64 +72,68 @@ internal abstract class AbstractLocalProcessRunner<R>(private val checks: TestRu
}
}
abstract override fun buildResultHandler(runResult: RunResult): ResultHandler // Narrow returned type.
abstract override fun buildResultHandler(runResult: RunResult): LocalResultHandler<R> // ?? Narrow returned type.
}
abstract inner class ResultHandler(runResult: RunResult) : AbstractRunner<R>.ResultHandler(runResult) {
override fun handle(): R {
checks.forEach { check ->
when (check) {
is ExecutionTimeout.ShouldNotExceed -> verifyExpectation(runResult.hasFinishedOnTime) {
"Timeout exceeded during test execution."
}
is ExecutionTimeout.ShouldExceed -> verifyExpectation(!runResult.hasFinishedOnTime) {
"Test is expected to fail with exceeded timeout, which hasn't happened."
}
is ExitCode -> {
// Don't check exit code if it is unknown.
val knownExitCode: Int = runResult.exitCode ?: return@forEach
when (check) {
is ExitCode.Expected -> verifyExpectation(knownExitCode == check.expectedExitCode) {
"$visibleProcessName exit code is $knownExitCode while ${check.expectedExitCode} was expected."
}
is ExitCode.AnyNonZero -> verifyExpectation(knownExitCode != 0) {
"$visibleProcessName exited with zero code, which wasn't expected."
}
internal abstract class LocalResultHandler<R>(
runResult: RunResult,
private val visibleProcessName: String,
private val checks: TestRunChecks
) : AbstractResultHandler<R>(runResult) {
override fun handle(): R {
checks.forEach { check ->
when (check) {
is ExecutionTimeout.ShouldNotExceed -> verifyExpectation(runResult.hasFinishedOnTime) {
"Timeout exceeded during test execution."
}
is ExecutionTimeout.ShouldExceed -> verifyExpectation(!runResult.hasFinishedOnTime) {
"Test is expected to fail with exceeded timeout, which hasn't happened."
}
is ExitCode -> {
// Don't check exit code if it is unknown.
val knownExitCode: Int = runResult.exitCode ?: return@forEach
when (check) {
is ExitCode.Expected -> verifyExpectation(knownExitCode == check.expectedExitCode) {
"$visibleProcessName exit code is $knownExitCode while ${check.expectedExitCode} was expected."
}
is ExitCode.AnyNonZero -> verifyExpectation(knownExitCode != 0) {
"$visibleProcessName exited with zero code, which wasn't expected."
}
}
is TestRunCheck.OutputDataFile -> {
val expectedOutput = check.file.readText()
val actualFilteredOutput = runResult.processOutput.stdOut.filteredOutput + runResult.processOutput.stdErr
}
is TestRunCheck.OutputDataFile -> {
val expectedOutput = check.file.readText()
val actualFilteredOutput = runResult.processOutput.stdOut.filteredOutput + runResult.processOutput.stdErr
// Don't use verifyExpectation(expected, actual) to avoid exposing potentially large test output in exception message
// and blowing up test logs.
verifyExpectation(convertLineSeparators(expectedOutput) == convertLineSeparators(actualFilteredOutput)) {
"Tested process output mismatch. See \"TEST STDOUT\" and \"EXPECTED OUTPUT DATA FILE\" below."
}
// Don't use verifyExpectation(expected, actual) to avoid exposing potentially large test output in exception message
// and blowing up test logs.
verifyExpectation(convertLineSeparators(expectedOutput) == convertLineSeparators(actualFilteredOutput)) {
"Tested process output mismatch. See \"TEST STDOUT\" and \"EXPECTED OUTPUT DATA FILE\" below."
}
is TestRunCheck.OutputMatcher -> {
try {
verifyExpectation(check.match(runResult.processOutput.stdOut.filteredOutput)) {
"Tested process output has not passed validation."
}
} catch (t: Throwable) {
if (t is Exception || t is AssertionError) {
fail<Nothing>(
getLoggedRun().withErrorMessage("Tested process output has not passed validation: " + t.message),
t
)
} else {
throw t
}
}
is TestRunCheck.OutputMatcher -> {
try {
verifyExpectation(check.match(runResult.processOutput.stdOut.filteredOutput)) {
"Tested process output has not passed validation."
}
} catch (t: Throwable) {
if (t is Exception || t is AssertionError) {
fail<Nothing>(
getLoggedRun().withErrorMessage("Tested process output has not passed validation: " + t.message),
t
)
} else {
throw t
}
}
}
}
return doHandle()
}
protected abstract fun doHandle(): R
return doHandle()
}
protected abstract fun doHandle(): R
}
private class UnfilteredProcessOutput {
native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractRunner.kt
+8 -8
View File
@@ -9,9 +9,9 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.LoggedData
import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue
import org.opentest4j.TestAbortedException
internal abstract class AbstractRunner<R>: Runner<R> {
internal abstract class AbstractRunner<R> : Runner<R> {
protected abstract fun buildRun(): AbstractRun
protected abstract fun buildResultHandler(runResult: RunResult): ResultHandler
protected abstract fun buildResultHandler(runResult: RunResult): AbstractResultHandler<R>
protected abstract fun getLoggedParameters(): LoggedData.TestRunParameters
protected abstract fun handleUnexpectedFailure(t: Throwable): Nothing
@@ -33,13 +33,13 @@ internal abstract class AbstractRunner<R>: Runner<R> {
fun interface AbstractRun {
fun run(): RunResult
}
}
abstract inner class ResultHandler(protected val runResult: RunResult) {
abstract fun getLoggedRun(): LoggedData
abstract fun handle(): R
internal abstract class AbstractResultHandler<R>(protected val runResult: RunResult) {
abstract fun getLoggedRun(): LoggedData
abstract fun handle(): R
protected inline fun verifyExpectation(shouldBeTrue: Boolean, crossinline errorMessage: () -> String) {
assertTrue(shouldBeTrue) { getLoggedRun().withErrorMessage(errorMessage()) }
}
protected inline fun verifyExpectation(shouldBeTrue: Boolean, crossinline errorMessage: () -> String) {
assertTrue(shouldBeTrue) { getLoggedRun().withErrorMessage(errorMessage()) }
}
}
native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestNameExtractor.kt
+12 -8
View File
@@ -26,17 +26,21 @@ internal class LocalTestNameExtractor(
runParameters = null
)
override fun buildResultHandler(runResult: RunResult) = ResultHandler(runResult)
override fun buildResultHandler(runResult: RunResult) =
TestNameResultHandler(runResult, visibleProcessName, checks, getLoggedParameters())
override fun handleUnexpectedFailure(t: Throwable) = fail {
LoggedData.TestRunUnexpectedFailure(getLoggedParameters(), t)
.withErrorMessage("Test name extraction failed with unexpected exception.")
}
inner class ResultHandler(
runResult: RunResult
) : AbstractLocalProcessRunner<Collection<TestName>>.ResultHandler(runResult) {
override fun getLoggedRun() = LoggedData.TestRun(getLoggedParameters(), runResult)
override fun doHandle() = GTestListing.parse(runResult.processOutput.stdOut.filteredOutput)
}
}
internal class TestNameResultHandler(
runResult: RunResult,
visibleProcessName: String,
checks: TestRunChecks,
private val loggedParameters: LoggedData.TestRunParameters
) : LocalResultHandler<Collection<TestName>>(runResult, visibleProcessName, checks) {
override fun getLoggedRun() = LoggedData.TestRun(loggedParameters, runResult)
override fun doHandle() = GTestListing.parse(runResult.processOutput.stdOut.filteredOutput)
}
native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt
+42 -30
View File
@@ -38,47 +38,59 @@ internal class LocalTestRunner(private val testRun: TestRun) : AbstractLocalProc
}
}
override fun buildResultHandler(runResult: RunResult) = ResultHandler(runResult)
override fun buildResultHandler(runResult: RunResult) = ResultHandler(
runResult = runResult,
visibleProcessName = visibleProcessName,
checks = testRun.checks,
testRun = testRun,
loggedParameters = getLoggedParameters()
)
override fun handleUnexpectedFailure(t: Throwable) = fail {
LoggedData.TestRunUnexpectedFailure(getLoggedParameters(), t)
.withErrorMessage("Test execution failed with unexpected exception.")
}
}
inner class ResultHandler(runResult: RunResult) : AbstractLocalProcessRunner<Unit>.ResultHandler(runResult) {
override fun getLoggedRun() = LoggedData.TestRun(getLoggedParameters(), runResult)
internal class ResultHandler(
runResult: RunResult,
visibleProcessName: String,
checks: TestRunChecks,
private val testRun: TestRun,
private val loggedParameters: LoggedData.TestRunParameters
) : LocalResultHandler<Unit>(runResult, visibleProcessName, checks) {
override fun getLoggedRun() = LoggedData.TestRun(loggedParameters, runResult)
override fun doHandle() {
verifyTestReport(runResult.processOutput.stdOut.testReport)
override fun doHandle() {
verifyTestReport(runResult.processOutput.stdOut.testReport)
}
private fun verifyTestReport(testReport: TestReport?) {
if (testReport == null) return
verifyExpectation(!testReport.isEmpty()) { "No tests have been found." }
testRun.runParameters.get<TestRunParameter.WithFilter> {
verifyNoSuchTests(
testReport.passedTests.filter { testName -> !testMatches(testName) },
"Excessive tests have been executed"
)
verifyNoSuchTests(
testReport.ignoredTests.filter { testName -> !testMatches(testName) },
"Excessive tests have been ignored"
)
}
private fun verifyTestReport(testReport: TestReport?) {
if (testReport == null) return
verifyNoSuchTests(testReport.failedTests, "There are failed tests")
verifyExpectation(!testReport.isEmpty()) { "No tests have been found." }
assumeFalse(testReport.ignoredTests.isNotEmpty() && testReport.passedTests.isEmpty(), "Test case is disabled")
}
testRun.runParameters.get<TestRunParameter.WithFilter> {
verifyNoSuchTests(
testReport.passedTests.filter { testName -> !testMatches(testName) },
"Excessive tests have been executed"
)
verifyNoSuchTests(
testReport.ignoredTests.filter { testName -> !testMatches(testName) },
"Excessive tests have been ignored"
)
}
verifyNoSuchTests(testReport.failedTests, "There are failed tests")
assumeFalse(testReport.ignoredTests.isNotEmpty() && testReport.passedTests.isEmpty(), "Test case is disabled")
}
private fun verifyNoSuchTests(tests: Collection<TestName>, subject: String) = verifyExpectation(tests.isEmpty()) {
buildString {
append(subject).append(':')
tests.forEach { appendLine().append(" - ").append(it) }
}
private fun verifyNoSuchTests(tests: Collection<TestName>, subject: String) = verifyExpectation(tests.isEmpty()) {
buildString {
append(subject).append(':')
tests.forEach { appendLine().append(" - ").append(it) }
}
}
}
native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/RunnerWithExecutor.kt
+82
View File
@@ -0,0 +1,82 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.konan.blackboxtest.support.runner
import org.jetbrains.kotlin.native.executors.ExecuteRequest
import org.jetbrains.kotlin.native.executors.Executor
import org.jetbrains.kotlin.konan.blackboxtest.support.LoggedData
import org.jetbrains.kotlin.konan.blackboxtest.support.util.TCTestOutputFilter
import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestOutputFilter
import org.jetbrains.kotlin.test.services.JUnit5Assertions
import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull
import java.io.ByteArrayInputStream
import java.io.ByteArrayOutputStream
import java.io.InputStream
internal class RunnerWithExecutor(
private val executor: Executor,
private val testRun: TestRun
) : AbstractRunner<Unit>() {
private val executable get() = testRun.executable
private val outputFilter: TestOutputFilter
get() = if (testRun.runParameters.has<TestRunParameter.WithTCTestLogger>()) TCTestOutputFilter else TestOutputFilter.NO_FILTERING
private val programArgs = mutableListOf<String>().apply {
testRun.runParameters.forEach { it.applyTo(this) }
}
private fun inputStreamFromTestParameter(): InputStream =
testRun.runParameters.firstIsInstanceOrNull<TestRunParameter.WithInputData>()
?.let {
ByteArrayInputStream(it.inputDataFile.readBytes())
} ?: ByteArrayInputStream(byteArrayOf())
override fun buildRun() = AbstractRun {
val stdin = inputStreamFromTestParameter()
val stdout = ByteArrayOutputStream()
val stderr = ByteArrayOutputStream()
val request = ExecuteRequest(
executableAbsolutePath = executable.executableFile.absolutePath,
args = programArgs,
stdin = stdin,
stdout = stdout,
stderr = stderr,
timeout = testRun.checks.executionTimeoutCheck.timeout
)
val response = executor.execute(request)
RunResult(
exitCode = response.exitCode,
timeout = request.timeout,
duration = response.executionTime,
hasFinishedOnTime = request.timeout >= response.executionTime,
processOutput = ProcessOutput(
stdOut = outputFilter.filter(stdout.toString("UTF-8")),
stdErr = stderr.toString("UTF-8")
)
)
}
override fun buildResultHandler(runResult: RunResult): ResultHandler = ResultHandler(
runResult = runResult,
visibleProcessName = "Test process under Executor ${executor::class.simpleName}",
checks = testRun.checks,
testRun = testRun,
loggedParameters = getLoggedParameters()
)
override fun getLoggedParameters() = LoggedData.TestRunParameters(
compilationToolCall = executable.loggedCompilationToolCall,
testCaseId = testRun.testCaseId,
runArgs = programArgs,
runParameters = testRun.runParameters
)
override fun handleUnexpectedFailure(t: Throwable) = JUnit5Assertions.fail {
LoggedData.TestRunUnexpectedFailure(getLoggedParameters(), t)
.withErrorMessage("Test execution failed with unexpected exception.")
}
}
native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/TestRunners.kt
+30 -11
View File
@@ -7,39 +7,58 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.runner
import org.jetbrains.kotlin.konan.blackboxtest.support.TestName
import org.jetbrains.kotlin.konan.blackboxtest.support.settings.ForcedNoopTestRunner
import org.jetbrains.kotlin.konan.blackboxtest.support.settings.KotlinNativeHome
import org.jetbrains.kotlin.konan.blackboxtest.support.settings.KotlinNativeTargets
import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Settings
import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Timeouts
import org.jetbrains.kotlin.native.executors.Executor
import org.jetbrains.kotlin.native.executors.EmulatorExecutor
import org.jetbrains.kotlin.native.executors.XcodeSimulatorExecutor
import org.jetbrains.kotlin.konan.target.*
import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
import java.util.concurrent.ConcurrentHashMap
internal object TestRunners {
// Currently, only local and noop test runners are supported.
fun createProperTestRunner(testRun: TestRun, settings: Settings): Runner<Unit> = with(settings) {
if (get<ForcedNoopTestRunner>().value) {
NoopTestRunner
} else with(get<KotlinNativeTargets>()) {
if (testTarget == hostTarget)
if (testTarget == hostTarget) {
LocalTestRunner(testRun)
else
runningAtNonHostTarget()
} else {
val nativeHome = get<KotlinNativeHome>()
val distribution = Distribution(nativeHome.dir.path)
val configurables = PlatformManager(distribution, true).platform(testTarget).configurables
val executor = cached(
when {
configurables is ConfigurablesWithEmulator -> EmulatorExecutor(configurables)
configurables is AppleConfigurables && configurables.targetTriple.isSimulator ->
XcodeSimulatorExecutor(configurables)
else -> runningOnUnsupportedTarget()
}
)
RunnerWithExecutor(executor, testRun)
}
}
}
private val runnersCache: ConcurrentHashMap<String, Executor> = ConcurrentHashMap()
private inline fun <reified T : Executor> cached(executor: T): Executor =
runnersCache.computeIfAbsent(T::class.java.simpleName) { executor }
// Currently, only local test name extractor is supported.
fun extractTestNames(executable: TestExecutable, settings: Settings): Collection<TestName> = with(settings) {
with(get<KotlinNativeTargets>()) {
if (testTarget == hostTarget)
LocalTestNameExtractor(executable, TestRunChecks.Default(get<Timeouts>().executionTimeout)).run()
else
runningAtNonHostTarget()
runningOnUnsupportedTarget()
}
}
private fun KotlinNativeTargets.runningAtNonHostTarget(): Nothing = fail {
"""
Running at non-host target is not supported yet.
Compilation target: $testTarget
Host target: $hostTarget
""".trimIndent()
private fun KotlinNativeTargets.runningOnUnsupportedTarget(): Nothing = fail {
"Running tests for $testTarget on $hostTarget is not supported yet."
}
}
native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/settings/TestProcessSettings.kt
+3 -1
View File
@@ -23,7 +23,9 @@ import kotlin.time.Duration.Companion.seconds
/**
* The tested and the host Kotlin/Native targets.
*/
internal class KotlinNativeTargets(val testTarget: KonanTarget, val hostTarget: KonanTarget)
internal class KotlinNativeTargets(val testTarget: KonanTarget, val hostTarget: KonanTarget) {
fun areDifferentTargets() = testTarget != hostTarget
}
/**
* The Kotlin/Native home.