[K/N][test] Refactor test results handling
Move handling to appropriate TestRunCheck instances to make them separate from each other. Also this makes it possible to use separately from the ResultHandler on specific scenarios like in FrameworkTest.
This commit is contained in:
committed by
Space Team
parent
5ddfd4fd91
commit
8ef2d49ab0
@@ -14,18 +14,15 @@ import org.jetbrains.kotlin.konan.test.blackbox.support.group.FirPipeline
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestExecutable
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunChecks
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.runner.doFileCheck
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.settings.*
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.util.createTestProvider
|
||||
import org.jetbrains.kotlin.native.executors.runProcess
|
||||
import org.jetbrains.kotlin.test.KtAssert.fail
|
||||
import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue
|
||||
import org.junit.jupiter.api.Assumptions
|
||||
import org.junit.jupiter.api.Disabled
|
||||
import org.junit.jupiter.api.Tag
|
||||
import org.junit.jupiter.api.Test
|
||||
import java.io.File
|
||||
import java.io.FileWriter
|
||||
import kotlin.time.Duration
|
||||
|
||||
@TestDataPath("\$PROJECT_ROOT")
|
||||
@@ -83,7 +80,7 @@ abstract class FrameworkTestBase : AbstractNativeSimpleTest() {
|
||||
objCFrameworkCompilation.result.assertSuccess()
|
||||
|
||||
val fileCheckDump = buildDir.resolve("out.$fileCheckStage.ll").also { assert(it.exists()) }
|
||||
val result = doFileCheck(testCase.checks.fileCheckMatcher!!, fileCheckDump)
|
||||
val result = testCase.checks.fileCheckMatcher!!.doFileCheck(fileCheckDump)
|
||||
if (!(result.stdout.isEmpty() && result.stderr.isEmpty())) {
|
||||
val shortOutText = result.stdout.lines().take(100)
|
||||
val shortErrText = result.stderr.lines().take(100)
|
||||
|
||||
+8
-190
@@ -5,31 +5,10 @@
|
||||
|
||||
package org.jetbrains.kotlin.konan.test.blackbox.support.runner
|
||||
|
||||
import com.intellij.openapi.util.SystemInfo
|
||||
import com.intellij.openapi.util.text.StringUtilRt.convertLineSeparators
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.LoggedData
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.TestKind
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.TestName
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck.ExecutionTimeout
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck.ExitCode
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.settings.configurables
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.util.TestOutputFilter
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.util.TestReport
|
||||
import org.jetbrains.kotlin.native.executors.RunProcessResult
|
||||
import org.jetbrains.kotlin.native.executors.runProcess
|
||||
import org.junit.jupiter.api.Assumptions
|
||||
import java.io.File
|
||||
import kotlin.time.Duration
|
||||
|
||||
private fun RunResult.processOutputAsString(output: TestRunCheck.Output) = when (output) {
|
||||
TestRunCheck.Output.STDOUT -> processOutput.stdOut.filteredOutput
|
||||
TestRunCheck.Output.STDERR -> processOutput.stdErr
|
||||
TestRunCheck.Output.ALL -> processOutput.stdOut.filteredOutput + processOutput.stdErr
|
||||
}
|
||||
|
||||
internal class ResultHandler(
|
||||
runResult: RunResult,
|
||||
private val visibleProcessName: String,
|
||||
private val checks: TestRunChecks,
|
||||
private val testRun: TestRun,
|
||||
private val loggedParameters: LoggedData.TestRunParameters,
|
||||
@@ -38,134 +17,12 @@ internal class ResultHandler(
|
||||
override fun getLoggedRun() = LoggedData.TestRun(loggedParameters, runResult)
|
||||
|
||||
override fun handle() {
|
||||
val diagnostics = buildList<String> {
|
||||
checks.forEach { check ->
|
||||
when (check) {
|
||||
is ExecutionTimeout.ShouldNotExceed -> if (!runResult.hasFinishedOnTime) add(
|
||||
"Timeout exceeded during test execution."
|
||||
)
|
||||
is ExecutionTimeout.ShouldExceed -> if (runResult.hasFinishedOnTime) add(
|
||||
"Test is expected to fail with exceeded timeout, which hasn't happened."
|
||||
)
|
||||
is ExitCode -> {
|
||||
// Don't check exit code if it is unknown.
|
||||
val knownExitCode: Int = runResult.exitCode ?: return@forEach
|
||||
when (check) {
|
||||
is ExitCode.Expected -> if (knownExitCode != check.expectedExitCode) add(
|
||||
"$visibleProcessName exit code is $knownExitCode while ${check.expectedExitCode} was expected."
|
||||
)
|
||||
is ExitCode.AnyNonZero -> if (knownExitCode == 0) add(
|
||||
"$visibleProcessName exited with zero code, which wasn't expected."
|
||||
)
|
||||
}
|
||||
}
|
||||
is TestRunCheck.OutputDataFile -> {
|
||||
val expectedOutput = check.file.readText()
|
||||
val actualFilteredOutput = runResult.processOutputAsString(check.output)
|
||||
|
||||
// Don't use verifyExpectation(expected, actual) to avoid exposing potentially large test output in exception message
|
||||
// and blowing up test logs.
|
||||
if (convertLineSeparators(expectedOutput) != convertLineSeparators(actualFilteredOutput)) add(
|
||||
"Tested process output mismatch. See \"TEST STDOUT\" and \"EXPECTED OUTPUT DATA FILE\" below."
|
||||
)
|
||||
}
|
||||
is TestRunCheck.OutputMatcher -> {
|
||||
try {
|
||||
if (!check.match(runResult.processOutputAsString(check.output))) add(
|
||||
"Tested process output has not passed validation."
|
||||
)
|
||||
} catch (t: Throwable) {
|
||||
if (t is Exception || t is AssertionError) {
|
||||
add("Tested process output has not passed validation: " + t.message)
|
||||
} else {
|
||||
throw t
|
||||
}
|
||||
}
|
||||
}
|
||||
is TestRunCheck.FileCheckMatcher -> {
|
||||
val fileCheckDump = runResult.testExecutable.executable.fileCheckDump!!
|
||||
val result = doFileCheck(check, fileCheckDump)
|
||||
|
||||
if (!(result.stdout.isEmpty() && result.stderr.isEmpty())) {
|
||||
val shortOutText = result.stdout.lines().take(100)
|
||||
val shortErrText = result.stderr.lines().take(100)
|
||||
add(
|
||||
"FileCheck matching of ${fileCheckDump.absolutePath}\n" +
|
||||
"with '--check-prefixes ${check.prefixes}'\n" +
|
||||
"failed with result=$result:\n" +
|
||||
shortOutText.joinToString("\n") + "\n" +
|
||||
shortErrText.joinToString("\n")
|
||||
)
|
||||
}
|
||||
}
|
||||
is TestRunCheck.TestFiltering -> {
|
||||
if (check.testOutputFilter != TestOutputFilter.NO_FILTERING) {
|
||||
val testReport = runResult.processOutput.stdOut.testReport
|
||||
|
||||
checkNotNull(testReport) { "TestRun has TestFiltering enabled, but test report is null" }
|
||||
|
||||
if (testReport.isEmpty()) add("No tests have been found. Test report is empty")
|
||||
|
||||
testRun.runParameters.get<TestRunParameter.WithFilter> {
|
||||
verifyNoSuchTests(
|
||||
testReport.passedTests.filter { testName -> !testMatches(testName) },
|
||||
"Excessive tests have been executed"
|
||||
)
|
||||
|
||||
verifyNoSuchTests(
|
||||
testReport.ignoredTests.filter { testName -> !testMatches(testName) },
|
||||
"Excessive tests have been ignored"
|
||||
)
|
||||
|
||||
verifyNoSuchTests(testReport.failedTests, "Failed tests found in the test report")
|
||||
|
||||
testReport.checkDisabled()
|
||||
}
|
||||
|
||||
testRun.runParameters.getAll<TestRunParameter.WithIgnoredTestFilter> {
|
||||
verifyNoSuchTests(
|
||||
testReport.passedTests.filter { testName -> !testMatches(testName) },
|
||||
"Ignored tests have been executed"
|
||||
)
|
||||
verifyNoSuchTests(
|
||||
testReport.failedTests.filter { testName ->
|
||||
testName.packageName == testRun.testCase.nominalPackageName
|
||||
},
|
||||
"Test failure found in the test report"
|
||||
)
|
||||
|
||||
Assumptions.assumeFalse(
|
||||
testReport.ignoredTests.any { testName ->
|
||||
testName.packageName == testRun.testCase.nominalPackageName
|
||||
},
|
||||
"Test case is disabled"
|
||||
)
|
||||
}
|
||||
|
||||
if (!testRun.runParameters.has<TestRunParameter.WithFilter>()) {
|
||||
verifyNoSuchTests(
|
||||
testReport.failedTests.filter { testName ->
|
||||
testName.packageName == testRun.testCase.nominalPackageName
|
||||
},
|
||||
"Test ${testRun.testCase.id} failure found in the test report"
|
||||
)
|
||||
|
||||
testReport.checkDisabled()
|
||||
}
|
||||
|
||||
if (testRun.testCase.kind == TestKind.STANDALONE) {
|
||||
verifyNoSuchTests(testReport.failedTests, "Failed tests found in the test report")
|
||||
|
||||
testReport.checkDisabled()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
val failedResults = checks.map { check ->
|
||||
check.apply(testRun, runResult)
|
||||
}.filterIsInstance<TestRunCheck.Result.Failed>()
|
||||
if (!testRun.expectedFailure) {
|
||||
verifyExpectation(diagnostics.isEmpty()) {
|
||||
diagnostics.joinToString("\n")
|
||||
verifyExpectation(failedResults.isEmpty()) {
|
||||
failedResults.joinToString("\n")
|
||||
}
|
||||
} else {
|
||||
val runResultInfo = buildString {
|
||||
@@ -178,53 +35,14 @@ internal class ResultHandler(
|
||||
})
|
||||
appendLine(runResult.processOutput.stdOut.testReport)
|
||||
}
|
||||
verifyExpectation(diagnostics.isNotEmpty()) {
|
||||
verifyExpectation(failedResults.isNotEmpty()) {
|
||||
"Test did not fail as expected: $runResultInfo"
|
||||
}
|
||||
println("Test failed as expected.\n$runResultInfo")
|
||||
if (diagnostics.isNotEmpty()) {
|
||||
if (failedResults.isNotEmpty()) {
|
||||
println("Diagnostics are:")
|
||||
diagnostics.forEach(::println)
|
||||
failedResults.forEach(::println)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun MutableList<String>.verifyNoSuchTests(tests: Collection<TestName>, subject: String) {
|
||||
if (tests.isNotEmpty()) {
|
||||
add(
|
||||
buildString {
|
||||
append(subject).append(':')
|
||||
tests.forEach { appendLine().append(" - ").append(it) }
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private fun TestReport.checkDisabled() {
|
||||
Assumptions.assumeFalse(
|
||||
ignoredTests.isNotEmpty() && passedTests.isEmpty(),
|
||||
"Test case is disabled"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
internal fun doFileCheck(check: TestRunCheck.FileCheckMatcher, fileCheckDump: File): RunProcessResult {
|
||||
val fileCheckExecutable = check.settings.configurables.absoluteLlvmHome + File.separator + "bin" + File.separator +
|
||||
if (SystemInfo.isWindows) "FileCheck.exe" else "FileCheck"
|
||||
require(File(fileCheckExecutable).exists()) {
|
||||
"$fileCheckExecutable does not exist. Make sure Distribution for `settings.configurables` " +
|
||||
"was created using `propertyOverrides` to specify development variant of LLVM instead of user variant."
|
||||
}
|
||||
return try {
|
||||
runProcess(
|
||||
fileCheckExecutable,
|
||||
check.testDataFile.absolutePath,
|
||||
"--input-file",
|
||||
fileCheckDump.absolutePath,
|
||||
"--check-prefixes", check.prefixes,
|
||||
"--allow-deprecated-dag-overlap" // TODO specify it via new test directive for `function_attributes_at_callsite.kt`
|
||||
)
|
||||
} catch (t: Throwable) {
|
||||
RunProcessResult(Duration.ZERO, "FileCheck utility failed:", t.toString())
|
||||
}
|
||||
}
|
||||
|
||||
-2
@@ -7,7 +7,6 @@ package org.jetbrains.kotlin.konan.test.blackbox.support.runner
|
||||
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.LoggedData
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.runner.AbstractRunner.AbstractRun
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.util.TCTestOutputFilter
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.util.TestOutputFilter
|
||||
import org.jetbrains.kotlin.native.executors.ExecuteRequest
|
||||
import org.jetbrains.kotlin.native.executors.Executor
|
||||
@@ -66,7 +65,6 @@ internal open class RunnerWithExecutor(
|
||||
|
||||
override fun buildResultHandler(runResult: RunResult): ResultHandler = ResultHandler(
|
||||
runResult = runResult,
|
||||
visibleProcessName = "Test process under Executor ${executor::class.simpleName}",
|
||||
checks = testRun.checks,
|
||||
testRun = testRun,
|
||||
loggedParameters = getLoggedParameters()
|
||||
|
||||
-1
@@ -112,7 +112,6 @@ internal object SharedExecutionBuilder {
|
||||
|
||||
override fun buildResultHandler(runResult: RunResult): ResultHandler = ResultHandler(
|
||||
runResult = runResult,
|
||||
visibleProcessName = "Test process under ${this::class.simpleName}",
|
||||
checks = sharedTestRun.checks,
|
||||
testRun = testRun.copy(
|
||||
runParameters = sharedTestRun.runParameters
|
||||
|
||||
+215
-15
@@ -5,29 +5,76 @@
|
||||
|
||||
package org.jetbrains.kotlin.konan.test.blackbox.support.runner
|
||||
|
||||
import com.intellij.openapi.util.SystemInfo
|
||||
import com.intellij.openapi.util.text.StringUtilRt
|
||||
import org.jetbrains.kotlin.konan.target.Architecture
|
||||
import org.jetbrains.kotlin.konan.target.KonanTarget
|
||||
import org.jetbrains.kotlin.konan.target.needSmallBinary
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.TestKind
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.TestName
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck.*
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.settings.*
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.settings.CacheMode
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.settings.KotlinNativeTargets
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.settings.OptimizationMode
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.settings.Settings
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.util.TestOutputFilter
|
||||
import org.jetbrains.kotlin.konan.test.blackbox.support.util.TestReport
|
||||
import org.jetbrains.kotlin.native.executors.RunProcessResult
|
||||
import org.jetbrains.kotlin.native.executors.runProcess
|
||||
import org.jetbrains.kotlin.util.capitalizeDecapitalize.toUpperCaseAsciiOnly
|
||||
import org.jetbrains.kotlin.utils.yieldIfNotNull
|
||||
import org.junit.jupiter.api.Assumptions
|
||||
import java.io.File
|
||||
import kotlin.time.Duration
|
||||
|
||||
internal sealed interface TestRunCheck {
|
||||
|
||||
fun apply(testRun: TestRun, runResult: RunResult): Result
|
||||
|
||||
sealed interface Result {
|
||||
data object Passed : Result
|
||||
data class Failed(val reason: String) : Result
|
||||
}
|
||||
|
||||
sealed class ExecutionTimeout(val timeout: Duration) : TestRunCheck {
|
||||
class ShouldNotExceed(timeout: Duration) : ExecutionTimeout(timeout)
|
||||
class ShouldExceed(timeout: Duration) : ExecutionTimeout(timeout)
|
||||
class ShouldNotExceed(timeout: Duration) : ExecutionTimeout(timeout) {
|
||||
override fun apply(testRun: TestRun, runResult: RunResult): Result =
|
||||
if (!runResult.hasFinishedOnTime)
|
||||
Result.Failed("Timeout exceeded during test execution.")
|
||||
else Result.Passed
|
||||
}
|
||||
|
||||
class ShouldExceed(timeout: Duration) : ExecutionTimeout(timeout) {
|
||||
override fun apply(testRun: TestRun, runResult: RunResult): Result =
|
||||
if (runResult.hasFinishedOnTime)
|
||||
Result.Failed("Test is expected to fail with exceeded timeout, which hasn't happened.")
|
||||
else Result.Passed
|
||||
}
|
||||
}
|
||||
|
||||
sealed class ExitCode : TestRunCheck {
|
||||
object AnyNonZero : ExitCode()
|
||||
class Expected(val expectedExitCode: Int) : ExitCode()
|
||||
data object AnyNonZero : ExitCode()
|
||||
|
||||
data class Expected(val expectedExitCode: Int) : ExitCode()
|
||||
|
||||
override fun apply(testRun: TestRun, runResult: RunResult): Result {
|
||||
// Don't check the exit code if it is unknown.
|
||||
val knownExitCode: Int = runResult.exitCode ?: return Result.Passed
|
||||
|
||||
return when (this) {
|
||||
AnyNonZero -> {
|
||||
if (knownExitCode == 0)
|
||||
Result.Failed("Test exited with zero code, which wasn't expected.")
|
||||
else Result.Passed
|
||||
}
|
||||
is Expected -> {
|
||||
if (knownExitCode != expectedExitCode)
|
||||
Result.Failed("Exit code is $knownExitCode while $expectedExitCode was expected.")
|
||||
else Result.Passed
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum class Output {
|
||||
@@ -40,11 +87,123 @@ internal sealed interface TestRunCheck {
|
||||
ALL,
|
||||
}
|
||||
|
||||
class OutputDataFile(val output: Output = Output.ALL, val file: File) : TestRunCheck
|
||||
fun RunResult.processOutputAsString(output: Output) = when (output) {
|
||||
Output.STDOUT -> processOutput.stdOut.filteredOutput
|
||||
Output.STDERR -> processOutput.stdErr
|
||||
Output.ALL -> processOutput.stdOut.filteredOutput + processOutput.stdErr
|
||||
}
|
||||
|
||||
class OutputMatcher(val output: Output = Output.ALL, val match: (String) -> Boolean) : TestRunCheck
|
||||
class OutputDataFile(val output: Output = Output.ALL, val file: File) : TestRunCheck {
|
||||
override fun apply(testRun: TestRun, runResult: RunResult): Result {
|
||||
val expectedOutput = file.readText()
|
||||
val actualFilteredOutput = runResult.processOutputAsString(output)
|
||||
|
||||
class TestFiltering(val testOutputFilter: TestOutputFilter) : TestRunCheck
|
||||
return if (StringUtilRt.convertLineSeparators(expectedOutput) != StringUtilRt.convertLineSeparators(actualFilteredOutput))
|
||||
Result.Failed("Tested process output mismatch. See \"TEST STDOUT\" and \"EXPECTED OUTPUT DATA FILE\" below.")
|
||||
else Result.Passed
|
||||
}
|
||||
}
|
||||
|
||||
class OutputMatcher(val output: Output = Output.ALL, val match: (String) -> Boolean) : TestRunCheck {
|
||||
override fun apply(testRun: TestRun, runResult: RunResult): Result =
|
||||
try {
|
||||
if (!match(runResult.processOutputAsString(output))) {
|
||||
Result.Failed("Tested process output has not passed validation.")
|
||||
} else Result.Passed
|
||||
} catch (t: Throwable) {
|
||||
if (t is Exception || t is AssertionError) {
|
||||
Result.Failed("Tested process output has not passed validation: ${t.message}")
|
||||
} else {
|
||||
throw t
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class TestFiltering(val testOutputFilter: TestOutputFilter) : TestRunCheck {
|
||||
override fun apply(testRun: TestRun, runResult: RunResult): Result {
|
||||
if (testOutputFilter != TestOutputFilter.NO_FILTERING) {
|
||||
val testReport = runResult.processOutput.stdOut.testReport
|
||||
|
||||
checkNotNull(testReport) { "TestRun has TestFiltering enabled, but test report is null" }
|
||||
|
||||
if (testReport.isEmpty()) Result.Failed("No tests have been found. Test report is empty")
|
||||
|
||||
testRun.runParameters.get<TestRunParameter.WithFilter> {
|
||||
testReport.checkDisabled()
|
||||
|
||||
return listOf(
|
||||
verifyNoSuchTests(
|
||||
testReport.passedTests.filter { testName -> !testMatches(testName) },
|
||||
"Excessive tests have been executed"
|
||||
),
|
||||
verifyNoSuchTests(
|
||||
testReport.ignoredTests.filter { testName -> !testMatches(testName) },
|
||||
"Excessive tests have been ignored"
|
||||
),
|
||||
verifyNoSuchTests(testReport.failedTests, "Failed tests found in the test report")
|
||||
).filterIsInstance<Result.Failed>().firstOrNull() ?: Result.Passed
|
||||
}
|
||||
|
||||
testRun.runParameters.getAll<TestRunParameter.WithIgnoredTestFilter> {
|
||||
Assumptions.assumeFalse(
|
||||
testReport.ignoredTests.any { testName ->
|
||||
testName.packageName == testRun.testCase.nominalPackageName
|
||||
},
|
||||
"Test case is disabled"
|
||||
)
|
||||
|
||||
return listOf(
|
||||
verifyNoSuchTests(
|
||||
testReport.passedTests.filter { testName -> !testMatches(testName) },
|
||||
"Ignored tests have been executed"
|
||||
),
|
||||
verifyNoSuchTests(
|
||||
testReport.failedTests.filter { testName ->
|
||||
testName.packageName == testRun.testCase.nominalPackageName
|
||||
},
|
||||
"Test failure found in the test report"
|
||||
)
|
||||
).filterIsInstance<Result.Failed>().firstOrNull() ?: Result.Passed
|
||||
}
|
||||
|
||||
if (testRun.testCase.kind == TestKind.STANDALONE) {
|
||||
testReport.checkDisabled()
|
||||
|
||||
return verifyNoSuchTests(testReport.failedTests, "Failed tests found in the test report")
|
||||
}
|
||||
|
||||
if (!testRun.runParameters.has<TestRunParameter.WithFilter>()) {
|
||||
testReport.checkDisabled()
|
||||
|
||||
return verifyNoSuchTests(
|
||||
testReport.failedTests.filter { testName ->
|
||||
testName.packageName == testRun.testCase.nominalPackageName
|
||||
},
|
||||
"Test ${testRun.testCase.id} failure found in the test report"
|
||||
)
|
||||
}
|
||||
}
|
||||
return Result.Passed
|
||||
}
|
||||
|
||||
private fun verifyNoSuchTests(tests: Collection<TestName>, subject: String): Result {
|
||||
return if (tests.isNotEmpty()) {
|
||||
Result.Failed(
|
||||
buildString {
|
||||
append(subject).append(':')
|
||||
tests.forEach { appendLine().append(" - ").append(it) }
|
||||
}
|
||||
)
|
||||
} else Result.Passed
|
||||
}
|
||||
|
||||
private fun TestReport.checkDisabled() {
|
||||
Assumptions.assumeFalse(
|
||||
ignoredTests.isNotEmpty() && passedTests.isEmpty(),
|
||||
"Test case is disabled"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
class FileCheckMatcher(val settings: Settings, val testDataFile: File) : TestRunCheck {
|
||||
val prefixes: String
|
||||
@@ -69,6 +228,55 @@ internal sealed interface TestRunCheck {
|
||||
val checkPrefixesWithCacheMode = checkPrefixes.map { "$it-CACHE_$cacheMode" }
|
||||
return (checkPrefixes + checkPrefixesWithOptMode + checkPrefixesWithCacheMode).joinToString(",")
|
||||
}
|
||||
|
||||
// Shameless borrowing `val KonanTarget.abiInfo` from module `:kotlin-native:backend.native`, which cannot be imported here for now.
|
||||
private val KonanTarget.abiInfoString: String
|
||||
get() = when {
|
||||
this == KonanTarget.MINGW_X64 -> "WINDOWSX64"
|
||||
!family.isAppleFamily && architecture == Architecture.ARM64 -> "AAPCS"
|
||||
else -> "DEFAULTABI"
|
||||
}
|
||||
|
||||
override fun apply(testRun: TestRun, runResult: RunResult): Result {
|
||||
val fileCheckDump = runResult.testExecutable.executable.fileCheckDump!!
|
||||
val result = doFileCheck(fileCheckDump)
|
||||
|
||||
return if (!(result.stdout.isEmpty() && result.stderr.isEmpty())) {
|
||||
val shortOutText = result.stdout.lines().take(100)
|
||||
val shortErrText = result.stderr.lines().take(100)
|
||||
|
||||
Result.Failed(
|
||||
"""
|
||||
FileCheck matching of ${fileCheckDump.absolutePath}
|
||||
with '--check-prefixes $prefixes'
|
||||
failed with result=$result:
|
||||
${shortOutText.joinToString("\n")}
|
||||
${shortErrText.joinToString("\n")}
|
||||
""".trimIndent()
|
||||
)
|
||||
} else Result.Passed
|
||||
}
|
||||
|
||||
internal fun doFileCheck(fileCheckDump: File): RunProcessResult {
|
||||
val fileCheckExecutable = settings.configurables.absoluteLlvmHome + File.separator + "bin" + File.separator +
|
||||
if (SystemInfo.isWindows) "FileCheck.exe" else "FileCheck"
|
||||
require(File(fileCheckExecutable).exists()) {
|
||||
"$fileCheckExecutable does not exist. Make sure Distribution for `settings.configurables` " +
|
||||
"was created using `propertyOverrides` to specify development variant of LLVM instead of user variant."
|
||||
}
|
||||
return try {
|
||||
runProcess(
|
||||
fileCheckExecutable,
|
||||
testDataFile.absolutePath,
|
||||
"--input-file",
|
||||
fileCheckDump.absolutePath,
|
||||
"--check-prefixes", prefixes,
|
||||
"--allow-deprecated-dag-overlap" // TODO specify it via new test directive for `function_attributes_at_callsite.kt`
|
||||
)
|
||||
} catch (t: Throwable) {
|
||||
RunProcessResult(Duration.ZERO, "FileCheck utility failed:", t.toString())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -103,11 +311,3 @@ internal data class TestRunChecks(
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Shameless borrowing `val KonanTarget.abiInfo` from module `:kotlin-native:backend.native`, which cannot be imported here for now.
|
||||
private val KonanTarget.abiInfoString: String
|
||||
get() = when {
|
||||
this == KonanTarget.MINGW_X64 -> "WINDOWSX64"
|
||||
!family.isAppleFamily && architecture == Architecture.ARM64 -> "AAPCS"
|
||||
else -> "DEFAULTABI"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user