[K/N][test] Refactor test results handling

Move handling to appropriate TestRunCheck instances to make
them separate from each other. Also this makes it possible to
use separately from the ResultHandler on specific scenarios like
in FrameworkTest.
This commit is contained in:
Pavel Punegov
2024-03-05 14:23:22 +01:00
committed by Space Team
parent 5ddfd4fd91
commit 8ef2d49ab0
5 changed files with 224 additions and 212 deletions
@@ -14,18 +14,15 @@ import org.jetbrains.kotlin.konan.test.blackbox.support.group.FirPipeline
import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestExecutable
import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck
import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunChecks
import org.jetbrains.kotlin.konan.test.blackbox.support.runner.doFileCheck
import org.jetbrains.kotlin.konan.test.blackbox.support.settings.*
import org.jetbrains.kotlin.konan.test.blackbox.support.util.createTestProvider
import org.jetbrains.kotlin.native.executors.runProcess
import org.jetbrains.kotlin.test.KtAssert.fail
import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue
import org.junit.jupiter.api.Assumptions
import org.junit.jupiter.api.Disabled
import org.junit.jupiter.api.Tag
import org.junit.jupiter.api.Test
import java.io.File
import java.io.FileWriter
import kotlin.time.Duration
@TestDataPath("\$PROJECT_ROOT")
@@ -83,7 +80,7 @@ abstract class FrameworkTestBase : AbstractNativeSimpleTest() {
objCFrameworkCompilation.result.assertSuccess()
val fileCheckDump = buildDir.resolve("out.$fileCheckStage.ll").also { assert(it.exists()) }
val result = doFileCheck(testCase.checks.fileCheckMatcher!!, fileCheckDump)
val result = testCase.checks.fileCheckMatcher!!.doFileCheck(fileCheckDump)
if (!(result.stdout.isEmpty() && result.stderr.isEmpty())) {
val shortOutText = result.stdout.lines().take(100)
val shortErrText = result.stderr.lines().take(100)
@@ -5,31 +5,10 @@
package org.jetbrains.kotlin.konan.test.blackbox.support.runner
import com.intellij.openapi.util.SystemInfo
import com.intellij.openapi.util.text.StringUtilRt.convertLineSeparators
import org.jetbrains.kotlin.konan.test.blackbox.support.LoggedData
import org.jetbrains.kotlin.konan.test.blackbox.support.TestKind
import org.jetbrains.kotlin.konan.test.blackbox.support.TestName
import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck.ExecutionTimeout
import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck.ExitCode
import org.jetbrains.kotlin.konan.test.blackbox.support.settings.configurables
import org.jetbrains.kotlin.konan.test.blackbox.support.util.TestOutputFilter
import org.jetbrains.kotlin.konan.test.blackbox.support.util.TestReport
import org.jetbrains.kotlin.native.executors.RunProcessResult
import org.jetbrains.kotlin.native.executors.runProcess
import org.junit.jupiter.api.Assumptions
import java.io.File
import kotlin.time.Duration
private fun RunResult.processOutputAsString(output: TestRunCheck.Output) = when (output) {
TestRunCheck.Output.STDOUT -> processOutput.stdOut.filteredOutput
TestRunCheck.Output.STDERR -> processOutput.stdErr
TestRunCheck.Output.ALL -> processOutput.stdOut.filteredOutput + processOutput.stdErr
}
internal class ResultHandler(
runResult: RunResult,
private val visibleProcessName: String,
private val checks: TestRunChecks,
private val testRun: TestRun,
private val loggedParameters: LoggedData.TestRunParameters,
@@ -38,134 +17,12 @@ internal class ResultHandler(
override fun getLoggedRun() = LoggedData.TestRun(loggedParameters, runResult)
override fun handle() {
val diagnostics = buildList<String> {
checks.forEach { check ->
when (check) {
is ExecutionTimeout.ShouldNotExceed -> if (!runResult.hasFinishedOnTime) add(
"Timeout exceeded during test execution."
)
is ExecutionTimeout.ShouldExceed -> if (runResult.hasFinishedOnTime) add(
"Test is expected to fail with exceeded timeout, which hasn't happened."
)
is ExitCode -> {
// Don't check exit code if it is unknown.
val knownExitCode: Int = runResult.exitCode ?: return@forEach
when (check) {
is ExitCode.Expected -> if (knownExitCode != check.expectedExitCode) add(
"$visibleProcessName exit code is $knownExitCode while ${check.expectedExitCode} was expected."
)
is ExitCode.AnyNonZero -> if (knownExitCode == 0) add(
"$visibleProcessName exited with zero code, which wasn't expected."
)
}
}
is TestRunCheck.OutputDataFile -> {
val expectedOutput = check.file.readText()
val actualFilteredOutput = runResult.processOutputAsString(check.output)
// Don't use verifyExpectation(expected, actual) to avoid exposing potentially large test output in exception message
// and blowing up test logs.
if (convertLineSeparators(expectedOutput) != convertLineSeparators(actualFilteredOutput)) add(
"Tested process output mismatch. See \"TEST STDOUT\" and \"EXPECTED OUTPUT DATA FILE\" below."
)
}
is TestRunCheck.OutputMatcher -> {
try {
if (!check.match(runResult.processOutputAsString(check.output))) add(
"Tested process output has not passed validation."
)
} catch (t: Throwable) {
if (t is Exception || t is AssertionError) {
add("Tested process output has not passed validation: " + t.message)
} else {
throw t
}
}
}
is TestRunCheck.FileCheckMatcher -> {
val fileCheckDump = runResult.testExecutable.executable.fileCheckDump!!
val result = doFileCheck(check, fileCheckDump)
if (!(result.stdout.isEmpty() && result.stderr.isEmpty())) {
val shortOutText = result.stdout.lines().take(100)
val shortErrText = result.stderr.lines().take(100)
add(
"FileCheck matching of ${fileCheckDump.absolutePath}\n" +
"with '--check-prefixes ${check.prefixes}'\n" +
"failed with result=$result:\n" +
shortOutText.joinToString("\n") + "\n" +
shortErrText.joinToString("\n")
)
}
}
is TestRunCheck.TestFiltering -> {
if (check.testOutputFilter != TestOutputFilter.NO_FILTERING) {
val testReport = runResult.processOutput.stdOut.testReport
checkNotNull(testReport) { "TestRun has TestFiltering enabled, but test report is null" }
if (testReport.isEmpty()) add("No tests have been found. Test report is empty")
testRun.runParameters.get<TestRunParameter.WithFilter> {
verifyNoSuchTests(
testReport.passedTests.filter { testName -> !testMatches(testName) },
"Excessive tests have been executed"
)
verifyNoSuchTests(
testReport.ignoredTests.filter { testName -> !testMatches(testName) },
"Excessive tests have been ignored"
)
verifyNoSuchTests(testReport.failedTests, "Failed tests found in the test report")
testReport.checkDisabled()
}
testRun.runParameters.getAll<TestRunParameter.WithIgnoredTestFilter> {
verifyNoSuchTests(
testReport.passedTests.filter { testName -> !testMatches(testName) },
"Ignored tests have been executed"
)
verifyNoSuchTests(
testReport.failedTests.filter { testName ->
testName.packageName == testRun.testCase.nominalPackageName
},
"Test failure found in the test report"
)
Assumptions.assumeFalse(
testReport.ignoredTests.any { testName ->
testName.packageName == testRun.testCase.nominalPackageName
},
"Test case is disabled"
)
}
if (!testRun.runParameters.has<TestRunParameter.WithFilter>()) {
verifyNoSuchTests(
testReport.failedTests.filter { testName ->
testName.packageName == testRun.testCase.nominalPackageName
},
"Test ${testRun.testCase.id} failure found in the test report"
)
testReport.checkDisabled()
}
if (testRun.testCase.kind == TestKind.STANDALONE) {
verifyNoSuchTests(testReport.failedTests, "Failed tests found in the test report")
testReport.checkDisabled()
}
}
}
}
}
}
val failedResults = checks.map { check ->
check.apply(testRun, runResult)
}.filterIsInstance<TestRunCheck.Result.Failed>()
if (!testRun.expectedFailure) {
verifyExpectation(diagnostics.isEmpty()) {
diagnostics.joinToString("\n")
verifyExpectation(failedResults.isEmpty()) {
failedResults.joinToString("\n")
}
} else {
val runResultInfo = buildString {
@@ -178,53 +35,14 @@ internal class ResultHandler(
})
appendLine(runResult.processOutput.stdOut.testReport)
}
verifyExpectation(diagnostics.isNotEmpty()) {
verifyExpectation(failedResults.isNotEmpty()) {
"Test did not fail as expected: $runResultInfo"
}
println("Test failed as expected.\n$runResultInfo")
if (diagnostics.isNotEmpty()) {
if (failedResults.isNotEmpty()) {
println("Diagnostics are:")
diagnostics.forEach(::println)
failedResults.forEach(::println)
}
}
}
private fun MutableList<String>.verifyNoSuchTests(tests: Collection<TestName>, subject: String) {
if (tests.isNotEmpty()) {
add(
buildString {
append(subject).append(':')
tests.forEach { appendLine().append(" - ").append(it) }
}
)
}
}
private fun TestReport.checkDisabled() {
Assumptions.assumeFalse(
ignoredTests.isNotEmpty() && passedTests.isEmpty(),
"Test case is disabled"
)
}
}
internal fun doFileCheck(check: TestRunCheck.FileCheckMatcher, fileCheckDump: File): RunProcessResult {
val fileCheckExecutable = check.settings.configurables.absoluteLlvmHome + File.separator + "bin" + File.separator +
if (SystemInfo.isWindows) "FileCheck.exe" else "FileCheck"
require(File(fileCheckExecutable).exists()) {
"$fileCheckExecutable does not exist. Make sure Distribution for `settings.configurables` " +
"was created using `propertyOverrides` to specify development variant of LLVM instead of user variant."
}
return try {
runProcess(
fileCheckExecutable,
check.testDataFile.absolutePath,
"--input-file",
fileCheckDump.absolutePath,
"--check-prefixes", check.prefixes,
"--allow-deprecated-dag-overlap" // TODO specify it via new test directive for `function_attributes_at_callsite.kt`
)
} catch (t: Throwable) {
RunProcessResult(Duration.ZERO, "FileCheck utility failed:", t.toString())
}
}
@@ -7,7 +7,6 @@ package org.jetbrains.kotlin.konan.test.blackbox.support.runner
import org.jetbrains.kotlin.konan.test.blackbox.support.LoggedData
import org.jetbrains.kotlin.konan.test.blackbox.support.runner.AbstractRunner.AbstractRun
import org.jetbrains.kotlin.konan.test.blackbox.support.util.TCTestOutputFilter
import org.jetbrains.kotlin.konan.test.blackbox.support.util.TestOutputFilter
import org.jetbrains.kotlin.native.executors.ExecuteRequest
import org.jetbrains.kotlin.native.executors.Executor
@@ -66,7 +65,6 @@ internal open class RunnerWithExecutor(
override fun buildResultHandler(runResult: RunResult): ResultHandler = ResultHandler(
runResult = runResult,
visibleProcessName = "Test process under Executor ${executor::class.simpleName}",
checks = testRun.checks,
testRun = testRun,
loggedParameters = getLoggedParameters()
@@ -112,7 +112,6 @@ internal object SharedExecutionBuilder {
override fun buildResultHandler(runResult: RunResult): ResultHandler = ResultHandler(
runResult = runResult,
visibleProcessName = "Test process under ${this::class.simpleName}",
checks = sharedTestRun.checks,
testRun = testRun.copy(
runParameters = sharedTestRun.runParameters
@@ -5,29 +5,76 @@
package org.jetbrains.kotlin.konan.test.blackbox.support.runner
import com.intellij.openapi.util.SystemInfo
import com.intellij.openapi.util.text.StringUtilRt
import org.jetbrains.kotlin.konan.target.Architecture
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.target.needSmallBinary
import org.jetbrains.kotlin.konan.test.blackbox.support.TestKind
import org.jetbrains.kotlin.konan.test.blackbox.support.TestName
import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck.*
import org.jetbrains.kotlin.konan.test.blackbox.support.settings.*
import org.jetbrains.kotlin.konan.test.blackbox.support.settings.CacheMode
import org.jetbrains.kotlin.konan.test.blackbox.support.settings.KotlinNativeTargets
import org.jetbrains.kotlin.konan.test.blackbox.support.settings.OptimizationMode
import org.jetbrains.kotlin.konan.test.blackbox.support.settings.Settings
import org.jetbrains.kotlin.konan.test.blackbox.support.util.TestOutputFilter
import org.jetbrains.kotlin.konan.test.blackbox.support.util.TestReport
import org.jetbrains.kotlin.native.executors.RunProcessResult
import org.jetbrains.kotlin.native.executors.runProcess
import org.jetbrains.kotlin.util.capitalizeDecapitalize.toUpperCaseAsciiOnly
import org.jetbrains.kotlin.utils.yieldIfNotNull
import org.junit.jupiter.api.Assumptions
import java.io.File
import kotlin.time.Duration
internal sealed interface TestRunCheck {
fun apply(testRun: TestRun, runResult: RunResult): Result
sealed interface Result {
data object Passed : Result
data class Failed(val reason: String) : Result
}
sealed class ExecutionTimeout(val timeout: Duration) : TestRunCheck {
class ShouldNotExceed(timeout: Duration) : ExecutionTimeout(timeout)
class ShouldExceed(timeout: Duration) : ExecutionTimeout(timeout)
class ShouldNotExceed(timeout: Duration) : ExecutionTimeout(timeout) {
override fun apply(testRun: TestRun, runResult: RunResult): Result =
if (!runResult.hasFinishedOnTime)
Result.Failed("Timeout exceeded during test execution.")
else Result.Passed
}
class ShouldExceed(timeout: Duration) : ExecutionTimeout(timeout) {
override fun apply(testRun: TestRun, runResult: RunResult): Result =
if (runResult.hasFinishedOnTime)
Result.Failed("Test is expected to fail with exceeded timeout, which hasn't happened.")
else Result.Passed
}
}
sealed class ExitCode : TestRunCheck {
object AnyNonZero : ExitCode()
class Expected(val expectedExitCode: Int) : ExitCode()
data object AnyNonZero : ExitCode()
data class Expected(val expectedExitCode: Int) : ExitCode()
override fun apply(testRun: TestRun, runResult: RunResult): Result {
// Don't check the exit code if it is unknown.
val knownExitCode: Int = runResult.exitCode ?: return Result.Passed
return when (this) {
AnyNonZero -> {
if (knownExitCode == 0)
Result.Failed("Test exited with zero code, which wasn't expected.")
else Result.Passed
}
is Expected -> {
if (knownExitCode != expectedExitCode)
Result.Failed("Exit code is $knownExitCode while $expectedExitCode was expected.")
else Result.Passed
}
}
}
}
enum class Output {
@@ -40,11 +87,123 @@ internal sealed interface TestRunCheck {
ALL,
}
class OutputDataFile(val output: Output = Output.ALL, val file: File) : TestRunCheck
fun RunResult.processOutputAsString(output: Output) = when (output) {
Output.STDOUT -> processOutput.stdOut.filteredOutput
Output.STDERR -> processOutput.stdErr
Output.ALL -> processOutput.stdOut.filteredOutput + processOutput.stdErr
}
class OutputMatcher(val output: Output = Output.ALL, val match: (String) -> Boolean) : TestRunCheck
class OutputDataFile(val output: Output = Output.ALL, val file: File) : TestRunCheck {
override fun apply(testRun: TestRun, runResult: RunResult): Result {
val expectedOutput = file.readText()
val actualFilteredOutput = runResult.processOutputAsString(output)
class TestFiltering(val testOutputFilter: TestOutputFilter) : TestRunCheck
return if (StringUtilRt.convertLineSeparators(expectedOutput) != StringUtilRt.convertLineSeparators(actualFilteredOutput))
Result.Failed("Tested process output mismatch. See \"TEST STDOUT\" and \"EXPECTED OUTPUT DATA FILE\" below.")
else Result.Passed
}
}
class OutputMatcher(val output: Output = Output.ALL, val match: (String) -> Boolean) : TestRunCheck {
override fun apply(testRun: TestRun, runResult: RunResult): Result =
try {
if (!match(runResult.processOutputAsString(output))) {
Result.Failed("Tested process output has not passed validation.")
} else Result.Passed
} catch (t: Throwable) {
if (t is Exception || t is AssertionError) {
Result.Failed("Tested process output has not passed validation: ${t.message}")
} else {
throw t
}
}
}
class TestFiltering(val testOutputFilter: TestOutputFilter) : TestRunCheck {
override fun apply(testRun: TestRun, runResult: RunResult): Result {
if (testOutputFilter != TestOutputFilter.NO_FILTERING) {
val testReport = runResult.processOutput.stdOut.testReport
checkNotNull(testReport) { "TestRun has TestFiltering enabled, but test report is null" }
if (testReport.isEmpty()) Result.Failed("No tests have been found. Test report is empty")
testRun.runParameters.get<TestRunParameter.WithFilter> {
testReport.checkDisabled()
return listOf(
verifyNoSuchTests(
testReport.passedTests.filter { testName -> !testMatches(testName) },
"Excessive tests have been executed"
),
verifyNoSuchTests(
testReport.ignoredTests.filter { testName -> !testMatches(testName) },
"Excessive tests have been ignored"
),
verifyNoSuchTests(testReport.failedTests, "Failed tests found in the test report")
).filterIsInstance<Result.Failed>().firstOrNull() ?: Result.Passed
}
testRun.runParameters.getAll<TestRunParameter.WithIgnoredTestFilter> {
Assumptions.assumeFalse(
testReport.ignoredTests.any { testName ->
testName.packageName == testRun.testCase.nominalPackageName
},
"Test case is disabled"
)
return listOf(
verifyNoSuchTests(
testReport.passedTests.filter { testName -> !testMatches(testName) },
"Ignored tests have been executed"
),
verifyNoSuchTests(
testReport.failedTests.filter { testName ->
testName.packageName == testRun.testCase.nominalPackageName
},
"Test failure found in the test report"
)
).filterIsInstance<Result.Failed>().firstOrNull() ?: Result.Passed
}
if (testRun.testCase.kind == TestKind.STANDALONE) {
testReport.checkDisabled()
return verifyNoSuchTests(testReport.failedTests, "Failed tests found in the test report")
}
if (!testRun.runParameters.has<TestRunParameter.WithFilter>()) {
testReport.checkDisabled()
return verifyNoSuchTests(
testReport.failedTests.filter { testName ->
testName.packageName == testRun.testCase.nominalPackageName
},
"Test ${testRun.testCase.id} failure found in the test report"
)
}
}
return Result.Passed
}
private fun verifyNoSuchTests(tests: Collection<TestName>, subject: String): Result {
return if (tests.isNotEmpty()) {
Result.Failed(
buildString {
append(subject).append(':')
tests.forEach { appendLine().append(" - ").append(it) }
}
)
} else Result.Passed
}
private fun TestReport.checkDisabled() {
Assumptions.assumeFalse(
ignoredTests.isNotEmpty() && passedTests.isEmpty(),
"Test case is disabled"
)
}
}
class FileCheckMatcher(val settings: Settings, val testDataFile: File) : TestRunCheck {
val prefixes: String
@@ -69,6 +228,55 @@ internal sealed interface TestRunCheck {
val checkPrefixesWithCacheMode = checkPrefixes.map { "$it-CACHE_$cacheMode" }
return (checkPrefixes + checkPrefixesWithOptMode + checkPrefixesWithCacheMode).joinToString(",")
}
// Shameless borrowing `val KonanTarget.abiInfo` from module `:kotlin-native:backend.native`, which cannot be imported here for now.
private val KonanTarget.abiInfoString: String
get() = when {
this == KonanTarget.MINGW_X64 -> "WINDOWSX64"
!family.isAppleFamily && architecture == Architecture.ARM64 -> "AAPCS"
else -> "DEFAULTABI"
}
override fun apply(testRun: TestRun, runResult: RunResult): Result {
val fileCheckDump = runResult.testExecutable.executable.fileCheckDump!!
val result = doFileCheck(fileCheckDump)
return if (!(result.stdout.isEmpty() && result.stderr.isEmpty())) {
val shortOutText = result.stdout.lines().take(100)
val shortErrText = result.stderr.lines().take(100)
Result.Failed(
"""
FileCheck matching of ${fileCheckDump.absolutePath}
with '--check-prefixes $prefixes'
failed with result=$result:
${shortOutText.joinToString("\n")}
${shortErrText.joinToString("\n")}
""".trimIndent()
)
} else Result.Passed
}
internal fun doFileCheck(fileCheckDump: File): RunProcessResult {
val fileCheckExecutable = settings.configurables.absoluteLlvmHome + File.separator + "bin" + File.separator +
if (SystemInfo.isWindows) "FileCheck.exe" else "FileCheck"
require(File(fileCheckExecutable).exists()) {
"$fileCheckExecutable does not exist. Make sure Distribution for `settings.configurables` " +
"was created using `propertyOverrides` to specify development variant of LLVM instead of user variant."
}
return try {
runProcess(
fileCheckExecutable,
testDataFile.absolutePath,
"--input-file",
fileCheckDump.absolutePath,
"--check-prefixes", prefixes,
"--allow-deprecated-dag-overlap" // TODO specify it via new test directive for `function_attributes_at_callsite.kt`
)
} catch (t: Throwable) {
RunProcessResult(Duration.ZERO, "FileCheck utility failed:", t.toString())
}
}
}
}
@@ -103,11 +311,3 @@ internal data class TestRunChecks(
)
}
}
// Shameless borrowing `val KonanTarget.abiInfo` from module `:kotlin-native:backend.native`, which cannot be imported here for now.
private val KonanTarget.abiInfoString: String
get() = when {
this == KonanTarget.MINGW_X64 -> "WINDOWSX64"
!family.isAppleFamily && architecture == Architecture.ARM64 -> "AAPCS"
else -> "DEFAULTABI"
}