azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Native][tests] Introduce TestRunner that knows how exactly to run an arbitrary test

Browse Source
This commit is contained in:
Dmitriy Dolovov
2021-11-16 00:24:52 +03:00
parent aedcfe78e1
commit 0f77c078e8
8 changed files with 250 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/AbstractNativeBlackBoxTest.kt
+5 -2
View File
@@ -13,7 +13,10 @@ import org.junit.jupiter.api.extension.ExtendWith
abstract class AbstractNativeBlackBoxTest {
internal lateinit var testRunProvider: TestRunProvider
fun runTest(@TestDataFile testDataFilePath: String) {
testRunProvider.getSingleTestRun(getAbsoluteFile(testDataFilePath)).runAndVerify()
fun runTest(@TestDataFile testDataFilePath: String): Unit = with(testRunProvider) {
val testDataFile = getAbsoluteFile(testDataFilePath)
val testRun = getSingleTestRun(testDataFile)
val testRunner = createRunner(testRun)
testRunner.run()
}
}
native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/TestEnvironment.kt
+2
View File
@@ -29,6 +29,8 @@ internal class GlobalTestEnvironment(
val cacheSettings: TestCacheSettings = defaultCacheSettings,
val baseBuildDir: File = projectBuildDir
) {
val hostTarget: KonanTarget = HostManager.host
fun getRootCacheDirectory(debuggable: Boolean): File? =
(cacheSettings as? TestCacheSettings.WithCache)?.getRootCacheDirectory(this, debuggable)
native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/TestExecutable.kt
+18 -3
View File
@@ -22,12 +22,27 @@ internal class TestRun(
internal sealed interface TestRunParameter {
fun applyTo(programArgs: MutableList<String>)
class WithPackageName(val packageName: String) : TestRunParameter {
override fun applyTo(programArgs: MutableList<String>) {
programArgs += "--ktest_filter=$packageName.*"
sealed class WithFilter : TestRunParameter {
protected abstract val wildcard: String
abstract fun testMatches(testName: String): Boolean
final override fun applyTo(programArgs: MutableList<String>) {
programArgs += "--ktest_filter=$wildcard"
}
}
class WithPackageFilter(private val packageFQN: PackageName) : WithFilter() {
override val wildcard get() = "$packageFQN.*"
override fun testMatches(testName: String) = testName.startsWith("$packageFQN.")
}
/*
class WithFunctionFilter(private val functionFQN: String) : WithFilter() {
override val wildcard get() = functionFQN
override fun testMatches(testName: String) = testName == functionFQN
}
*/
object WithGTestLogger : TestRunParameter {
override fun applyTo(programArgs: MutableList<String>) {
programArgs += "--ktest_logger=GTEST"
native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/TestRunProvider.kt
+15 -1
View File
@@ -8,6 +8,8 @@ package org.jetbrains.kotlin.konan.blackboxtest
import com.intellij.openapi.util.Disposer
import org.jetbrains.kotlin.konan.blackboxtest.TestCompilationResult.Companion.assertSuccess
import org.jetbrains.kotlin.konan.blackboxtest.group.TestCaseGroupProvider
import org.jetbrains.kotlin.konan.blackboxtest.runner.AbstractRunner
import org.jetbrains.kotlin.konan.blackboxtest.runner.LocalTestRunner
import org.jetbrains.kotlin.konan.blackboxtest.util.ThreadSafeCache
import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue
import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
@@ -64,7 +66,7 @@ internal class TestRunProvider(
)
TestKind.REGULAR -> listOfNotNull(
TestRunParameter.WithGTestLogger,
TestRunParameter.WithPackageName(packageName = testCase.nominalPackageName),
TestRunParameter.WithPackageFilter(testCase.nominalPackageName),
testCase.expectedOutputDataFile?.let(TestRunParameter::WithExpectedOutputData)
)
}
@@ -72,6 +74,18 @@ internal class TestRunProvider(
return TestRun(executable, runParameters, testCase.origin)
}
// Currently, only local test runner is supported.
fun createRunner(testRun: TestRun): AbstractRunner<*> = when (val target = environment.globalEnvironment.target) {
environment.globalEnvironment.hostTarget -> LocalTestRunner(testRun)
else -> fail {
"""
Running at non-host target is not supported yet.
Compilation target: $target
Host target: ${environment.globalEnvironment.hostTarget}
""".trimIndent()
}
}
override fun close() {
Disposer.dispose(environment)
}
native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/TestRunner.kt
-128
View File
@@ -1,128 +0,0 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.konan.blackboxtest
import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertEquals
import kotlin.properties.Delegates
internal fun TestRun.runAndVerify() {
val programArgs = mutableListOf<String>(executable.executableFile.path)
runParameters.forEach { it.applyTo(programArgs) }
val loggedParameters = LoggedData.TestRunParameters(executable.loggedCompilerCall, origin, programArgs, runParameters)
val startTimeMillis = System.currentTimeMillis()
val process = ProcessBuilder(programArgs).directory(executable.executableFile.parentFile).start()
runParameters.get<TestRunParameter.WithInputData> {
process.outputStream.write(inputDataFile.readBytes())
process.outputStream.flush()
}
TestOutput(runParameters, process, startTimeMillis, loggedParameters).verify()
}
private class TestOutput(
private val runParameters: List<TestRunParameter>,
private val process: Process,
private val startTimeMillis: Long,
private val loggedParameters: LoggedData.TestRunParameters
) {
private var exitCode: Int by Delegates.notNull()
private lateinit var stdOut: String
private lateinit var stdErr: String
private var finishTimeMillis by Delegates.notNull<Long>()
fun verify() {
waitUntilExecutionFinished()
verifyExpectation(0, exitCode) { "Tested process exited with non-zero code." }
if (runParameters.has<TestRunParameter.WithGTestLogger>()) {
verifyTestWithGTestRunner()
} else {
verifyPlainTest()
}
}
private fun verifyTestWithGTestRunner() {
val testStatuses = hashMapOf<TestStatus, MutableSet<TestName>>()
val cleanStdOut = StringBuilder()
var expectStatusLine = false
stdOut.lines().forEach { line ->
when {
expectStatusLine -> {
val matcher = GTEST_STATUS_LINE_REGEX.matchEntire(line)
if (matcher != null) {
// Read the line with test status.
val testStatus = matcher.groupValues[1]
val testName = matcher.groupValues[2]
testStatuses.getOrPut(testStatus) { hashSetOf() } += testName
expectStatusLine = false
} else {
// If current line is not a status line then it could be only the line with the process' output.
cleanStdOut.appendLine(line)
}
}
line.startsWith(GTEST_RUN_LINE_PREFIX) -> {
expectStatusLine = true // Next line contains either test status.
}
else -> Unit
}
}
verifyExpectation(true, testStatuses.isNotEmpty()) { "No tests have been executed." }
val passedTests = testStatuses[GTEST_STATUS_OK]?.size ?: 0
verifyExpectation(true, passedTests > 0) { "No passed tests." }
runParameters.get<TestRunParameter.WithPackageName> {
val excessiveTests = testStatuses.getValue(GTEST_STATUS_OK).filter { testName -> !testName.startsWith(packageName) }
verifyExpectation(true, excessiveTests.isEmpty()) { "Excessive tests have been executed: $excessiveTests." }
}
val failedTests = (testStatuses - GTEST_STATUS_OK).values.sumOf { it.size }
verifyExpectation(0, failedTests) { "There are failed tests." }
runParameters.get<TestRunParameter.WithExpectedOutputData> {
val mergedOutput = cleanStdOut.toString() + stdErr
verifyExpectation(expectedOutputDataFile.readText(), mergedOutput) { "Tested process output mismatch." }
}
}
private fun verifyPlainTest() {
runParameters.get<TestRunParameter.WithExpectedOutputData> {
val mergedOutput = stdOut + stdErr
verifyExpectation(expectedOutputDataFile.readText(), mergedOutput) { "Tested process output mismatch." }
}
}
private fun waitUntilExecutionFinished() {
exitCode = process.waitFor()
finishTimeMillis = System.currentTimeMillis()
stdOut = process.inputStream.bufferedReader().readText()
stdErr = process.errorStream.bufferedReader().readText()
}
private inline fun <T> verifyExpectation(expected: T, actual: T, crossinline errorMessageHeader: () -> String) {
assertEquals(expected, actual) {
buildString {
appendLine(errorMessageHeader())
appendLine()
appendLine(LoggedData.TestRun(loggedParameters, exitCode, stdOut, stdErr, finishTimeMillis - startTimeMillis))
}
}
}
companion object {
private const val GTEST_RUN_LINE_PREFIX = "[ RUN ]"
private val GTEST_STATUS_LINE_REGEX = Regex("^\\[\\s+([A-Z]+)\\s+]\\s+(\\S+)\\s+.*")
private const val GTEST_STATUS_OK = "OK"
}
}
private typealias TestStatus = String
private typealias TestName = String
native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/runner/AbstractLocalProcessRunner.kt
+45
View File
@@ -0,0 +1,45 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.konan.blackboxtest.runner
import org.jetbrains.kotlin.konan.blackboxtest.TestExecutable
internal abstract class AbstractLocalProcessRunner<R> : AbstractRunner<R>() {
protected abstract val visibleProcessName: String
protected abstract val executable: TestExecutable
protected abstract val programArgs: List<String>
protected open fun customizeProcess(process: Process) = Unit
final override fun buildRun() = object : AbstractRun {
override fun run(): RunResult {
val startTimeMillis = System.currentTimeMillis()
val process = ProcessBuilder(programArgs).directory(executable.executableFile.parentFile).start()
customizeProcess(process)
val exitCode = process.waitFor()
val finishTimeMillis = System.currentTimeMillis()
val stdOut = process.inputStream.bufferedReader().readText()
val stdErr = process.errorStream.bufferedReader().readText()
return RunResult(exitCode, finishTimeMillis - startTimeMillis, stdOut, stdErr)
}
}
abstract override fun buildResultHandler(runResult: RunResult): ResultHandler // Narrow returned type.
abstract inner class ResultHandler(runResult: RunResult) : AbstractRunner<R>.ResultHandler(runResult) {
override fun handle(): R {
verifyExpectation(0, runResult.exitCode) { "$visibleProcessName exited with non-zero code." }
return doHandle()
}
protected abstract fun doHandle(): R
}
}
native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/runner/AbstractRunner.kt
+53
View File
@@ -0,0 +1,53 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.konan.blackboxtest.runner
import org.jetbrains.kotlin.konan.blackboxtest.LoggedData
import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertEquals
import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue
internal abstract class AbstractRunner<R> {
protected abstract fun buildRun(): AbstractRun
protected abstract fun buildResultHandler(runResult: RunResult): ResultHandler
fun run(): R {
val run = buildRun()
val runResult = run.run()
val resultHandler = buildResultHandler(runResult)
return resultHandler.handle()
}
interface AbstractRun {
fun run(): RunResult
}
data class RunResult(val exitCode: Int, val durationMillis: Long, val stdOut: String, val stdErr: String)
abstract inner class ResultHandler(protected val runResult: RunResult) {
abstract fun getLoggedRun(): LoggedData
abstract fun handle(): R
val exitCode get() = runResult.exitCode
val durationMillis get() = runResult.durationMillis
val stdOut get() = runResult.stdOut
val stdErr get() = runResult.stdErr
protected inline fun <T> verifyExpectation(expected: T, actual: T, crossinline errorMessageHeader: () -> String) {
assertEquals(expected, actual) { formatErrorMessage(errorMessageHeader) }
}
protected inline fun verifyExpectation(shouldBeTrue: Boolean, crossinline errorMessageHeader: () -> String) {
assertTrue(shouldBeTrue) { formatErrorMessage(errorMessageHeader) }
}
private inline fun formatErrorMessage(errorMessageHeader: () -> String) = buildString {
appendLine(errorMessageHeader())
appendLine()
appendLine(getLoggedRun())
}
}
}
native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/runner/LocalTestRunner.kt
+112
View File
@@ -0,0 +1,112 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.konan.blackboxtest.runner
import org.jetbrains.kotlin.konan.blackboxtest.*
import org.jetbrains.kotlin.konan.blackboxtest.LoggedData
import org.jetbrains.kotlin.konan.blackboxtest.TestRun
import org.jetbrains.kotlin.konan.blackboxtest.TestRunParameter
import org.jetbrains.kotlin.konan.blackboxtest.get
internal class LocalTestRunner(private val testRun: TestRun) : AbstractLocalProcessRunner<Unit>() {
override val visibleProcessName get() = "Tested process"
override val executable get() = testRun.executable
override val programArgs = buildList<String> {
add(executable.executableFile.path)
testRun.runParameters.forEach { it.applyTo(this) }
}
private fun getLoggedParameters() = LoggedData.TestRunParameters(
compilerCall = executable.loggedCompilerCall,
origin = testRun.origin,
runArgs = programArgs,
runParameters = testRun.runParameters
)
override fun customizeProcess(process: Process) {
testRun.runParameters.get<TestRunParameter.WithInputData> {
process.outputStream.write(inputDataFile.readBytes())
process.outputStream.flush()
}
}
override fun buildResultHandler(runResult: RunResult) = ResultHandler(runResult)
inner class ResultHandler(runResult: RunResult) : AbstractLocalProcessRunner<Unit>.ResultHandler(runResult) {
override fun getLoggedRun() = LoggedData.TestRun(getLoggedParameters(), exitCode, stdOut, stdErr, durationMillis)
override fun doHandle() {
if (testRun.runParameters.has<TestRunParameter.WithGTestLogger>()) {
verifyTestWithGTestRunner()
} else {
verifyPlainTest()
}
}
private fun verifyTestWithGTestRunner() {
val testStatuses = hashMapOf<TestStatus, MutableSet<TestName>>()
val cleanStdOut = StringBuilder()
var expectStatusLine = false
stdOut.lines().forEach { line ->
when {
expectStatusLine -> {
val matcher = GTEST_STATUS_LINE_REGEX.matchEntire(line)
if (matcher != null) {
// Read the line with test status.
val testStatus = matcher.groupValues[1]
val testName = matcher.groupValues[2]
testStatuses.getOrPut(testStatus) { hashSetOf() } += testName
expectStatusLine = false
} else {
// If current line is not a status line then it could be only the line with the process' output.
cleanStdOut.appendLine(line)
}
}
line.startsWith(GTEST_RUN_LINE_PREFIX) -> {
expectStatusLine = true // Next line contains either test status.
}
else -> Unit
}
}
verifyExpectation(true, testStatuses.isNotEmpty()) { "No tests have been executed." }
val passedTests = testStatuses[GTEST_STATUS_OK]?.size ?: 0
verifyExpectation(true, passedTests > 0) { "No passed tests." }
testRun.runParameters.get<TestRunParameter.WithFilter> {
val excessiveTests = testStatuses.getValue(GTEST_STATUS_OK).filter { testName -> !testMatches(testName) }
verifyExpectation(true, excessiveTests.isEmpty()) { "Excessive tests have been executed: $excessiveTests." }
}
val failedTests = (testStatuses - GTEST_STATUS_OK).values.sumOf { it.size }
verifyExpectation(0, failedTests) { "There are failed tests." }
testRun.runParameters.get<TestRunParameter.WithExpectedOutputData> {
val mergedOutput = cleanStdOut.toString() + stdErr
verifyExpectation(expectedOutputDataFile.readText(), mergedOutput) { "Tested process output mismatch." }
}
}
private fun verifyPlainTest() {
testRun.runParameters.get<TestRunParameter.WithExpectedOutputData> {
val mergedOutput = stdOut + stdErr
verifyExpectation(expectedOutputDataFile.readText(), mergedOutput) { "Tested process output mismatch." }
}
}
}
companion object {
private const val GTEST_RUN_LINE_PREFIX = "[ RUN ]"
private val GTEST_STATUS_LINE_REGEX = Regex("^\\[\\s+([A-Z]+)\\s+]\\s+(\\S+)\\s+.*")
private const val GTEST_STATUS_OK = "OK"
}
}
private typealias TestStatus = String
private typealias TestName = String