[K/N][tests] Merge LocalResultHandler and ResultHandler
This commit is contained in:
Alexander Shabalin
committed by
Space Team
Space Team
parent
8d696dae03
commit
5bd78e1bf2
-168
@@ -1,168 +0,0 @@
|
|||||||
/*
|
|
||||||
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
|
||||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package org.jetbrains.kotlin.konan.test.blackbox.support.runner
|
|
||||||
|
|
||||||
import com.intellij.openapi.util.SystemInfo
|
|
||||||
import com.intellij.openapi.util.text.StringUtilRt.convertLineSeparators
|
|
||||||
import org.jetbrains.kotlin.konan.target.Architecture
|
|
||||||
import org.jetbrains.kotlin.konan.target.KonanTarget
|
|
||||||
import org.jetbrains.kotlin.konan.target.needSmallBinary
|
|
||||||
import org.jetbrains.kotlin.konan.test.blackbox.support.TestCaseId
|
|
||||||
import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck.ExecutionTimeout
|
|
||||||
import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck.ExitCode
|
|
||||||
import org.jetbrains.kotlin.konan.test.blackbox.support.settings.KotlinNativeTargets
|
|
||||||
import org.jetbrains.kotlin.konan.test.blackbox.support.settings.OptimizationMode
|
|
||||||
import org.jetbrains.kotlin.konan.test.blackbox.support.settings.configurables
|
|
||||||
import org.jetbrains.kotlin.util.capitalizeDecapitalize.toUpperCaseAsciiOnly
|
|
||||||
import java.io.File
|
|
||||||
|
|
||||||
private fun RunResult.processOutputAsString(output: TestRunCheck.Output) = when (output) {
|
|
||||||
TestRunCheck.Output.STDOUT -> processOutput.stdOut.filteredOutput
|
|
||||||
TestRunCheck.Output.STDERR -> processOutput.stdErr
|
|
||||||
TestRunCheck.Output.ALL -> processOutput.stdOut.filteredOutput + processOutput.stdErr
|
|
||||||
}
|
|
||||||
|
|
||||||
internal abstract class LocalResultHandler<R>(
|
|
||||||
runResult: RunResult,
|
|
||||||
private val visibleProcessName: String,
|
|
||||||
protected val checks: TestRunChecks,
|
|
||||||
private val testCaseId: TestCaseId?,
|
|
||||||
private val expectedFailure: Boolean,
|
|
||||||
) : AbstractResultHandler<R>(runResult) {
|
|
||||||
override fun handle(): R {
|
|
||||||
val diagnostics = buildList<String> {
|
|
||||||
checks.forEach { check ->
|
|
||||||
when (check) {
|
|
||||||
is ExecutionTimeout.ShouldNotExceed -> if(!runResult.hasFinishedOnTime) add(
|
|
||||||
"Timeout exceeded during test execution."
|
|
||||||
)
|
|
||||||
is ExecutionTimeout.ShouldExceed -> if(runResult.hasFinishedOnTime) add(
|
|
||||||
"Test is expected to fail with exceeded timeout, which hasn't happened."
|
|
||||||
)
|
|
||||||
is ExitCode -> {
|
|
||||||
// Don't check exit code if it is unknown.
|
|
||||||
val knownExitCode: Int = runResult.exitCode ?: return@forEach
|
|
||||||
when (check) {
|
|
||||||
is ExitCode.Expected -> if (knownExitCode != check.expectedExitCode) add(
|
|
||||||
"$visibleProcessName exit code is $knownExitCode while ${check.expectedExitCode} was expected."
|
|
||||||
)
|
|
||||||
is ExitCode.AnyNonZero -> if (knownExitCode == 0) add(
|
|
||||||
"$visibleProcessName exited with zero code, which wasn't expected."
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
is TestRunCheck.OutputDataFile -> {
|
|
||||||
val expectedOutput = check.file.readText()
|
|
||||||
val actualFilteredOutput = runResult.processOutputAsString(check.output)
|
|
||||||
|
|
||||||
// Don't use verifyExpectation(expected, actual) to avoid exposing potentially large test output in exception message
|
|
||||||
// and blowing up test logs.
|
|
||||||
if(convertLineSeparators(expectedOutput) != convertLineSeparators(actualFilteredOutput)) add(
|
|
||||||
"Tested process output mismatch. See \"TEST STDOUT\" and \"EXPECTED OUTPUT DATA FILE\" below."
|
|
||||||
)
|
|
||||||
}
|
|
||||||
is TestRunCheck.OutputMatcher -> {
|
|
||||||
try {
|
|
||||||
if(!check.match(runResult.processOutputAsString(check.output))) add(
|
|
||||||
"Tested process output has not passed validation."
|
|
||||||
)
|
|
||||||
} catch (t: Throwable) {
|
|
||||||
if (t is Exception || t is AssertionError) {
|
|
||||||
add("Tested process output has not passed validation: " + t.message)
|
|
||||||
} else {
|
|
||||||
throw t
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
is TestRunCheck.FileCheckMatcher -> {
|
|
||||||
val fileCheckExecutable = check.settings.configurables.absoluteLlvmHome + File.separator + "bin" + File.separator +
|
|
||||||
if (SystemInfo.isWindows) "FileCheck.exe" else "FileCheck"
|
|
||||||
require(File(fileCheckExecutable).exists()) {
|
|
||||||
"$fileCheckExecutable does not exist. Make sure Distribution for `settings.configurables` " +
|
|
||||||
"was created using `propertyOverrides` to specify development variant of LLVM instead of user variant."
|
|
||||||
}
|
|
||||||
val fileCheckDump = runResult.testExecutable.executable.fileCheckDump!!
|
|
||||||
val fileCheckOut = File(fileCheckDump.absolutePath + ".out")
|
|
||||||
val fileCheckErr = File(fileCheckDump.absolutePath + ".err")
|
|
||||||
|
|
||||||
val testTarget = check.settings.get<KotlinNativeTargets>().testTarget
|
|
||||||
val checkPrefixes = buildList {
|
|
||||||
add("CHECK")
|
|
||||||
add("CHECK-${testTarget.abiInfoString}")
|
|
||||||
add("CHECK-${testTarget.name.toUpperCaseAsciiOnly()}")
|
|
||||||
if (testTarget.family.isAppleFamily) {
|
|
||||||
add("CHECK-APPLE")
|
|
||||||
}
|
|
||||||
if (testTarget.needSmallBinary()) {
|
|
||||||
add("CHECK-SMALLBINARY")
|
|
||||||
} else {
|
|
||||||
add("CHECK-BIGBINARY")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
val optimizationMode = check.settings.get<OptimizationMode>().name
|
|
||||||
val checkPrefixesWithOptMode = checkPrefixes.map { "$it-$optimizationMode" }
|
|
||||||
val commaSeparatedCheckPrefixes = (checkPrefixes + checkPrefixesWithOptMode).joinToString(",")
|
|
||||||
|
|
||||||
val result = ProcessBuilder(
|
|
||||||
fileCheckExecutable,
|
|
||||||
check.testDataFile.absolutePath,
|
|
||||||
"--input-file",
|
|
||||||
fileCheckDump.absolutePath,
|
|
||||||
"--check-prefixes", commaSeparatedCheckPrefixes,
|
|
||||||
"--allow-deprecated-dag-overlap" // TODO specify it via new test directive for `function_attributes_at_callsite.kt`
|
|
||||||
).redirectOutput(fileCheckOut)
|
|
||||||
.redirectError(fileCheckErr)
|
|
||||||
.start()
|
|
||||||
.waitFor()
|
|
||||||
val errText = fileCheckErr.readText()
|
|
||||||
val outText = fileCheckOut.readText()
|
|
||||||
if(!(result == 0 && errText.isEmpty() && outText.isEmpty())) {
|
|
||||||
val shortOutText = outText.lines().take(100)
|
|
||||||
val shortErrText = errText.lines().take(100)
|
|
||||||
add("FileCheck matching of ${fileCheckDump.absolutePath}\n" +
|
|
||||||
"with '--check-prefixes $commaSeparatedCheckPrefixes'\n" +
|
|
||||||
"failed with result=$result:\n" +
|
|
||||||
shortOutText.joinToString("\n") + "\n" +
|
|
||||||
shortErrText.joinToString("\n")
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (!expectedFailure) {
|
|
||||||
verifyExpectation(diagnostics.isEmpty()) {
|
|
||||||
diagnostics.joinToString("\n")
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
val runResultInfo = "TestCaseId: $testCaseId\nExit code: ${runResult.exitCode}\nFiltered test output is${
|
|
||||||
runResult.processOutput.stdOut.filteredOutput.let {
|
|
||||||
if (it.isNotEmpty()) ":\n$it" else " empty."
|
|
||||||
}
|
|
||||||
}"
|
|
||||||
verifyExpectation(diagnostics.isNotEmpty() || runResult.processOutput.stdOut.testReport?.failedTests?.isNotEmpty() == true) {
|
|
||||||
"Test did not fail as expected: $runResultInfo"
|
|
||||||
}
|
|
||||||
println("Test failed as expected.\n$runResultInfo")
|
|
||||||
if (diagnostics.isNotEmpty()) {
|
|
||||||
println("Diagnostics are:")
|
|
||||||
diagnostics.forEach(::println)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return doHandle()
|
|
||||||
}
|
|
||||||
|
|
||||||
protected abstract fun doHandle(): R
|
|
||||||
}
|
|
||||||
|
|
||||||
// Shameless borrowing `val KonanTarget.abiInfo` from module `:kotlin-native:backend.native`, which cannot be imported here for now.
|
|
||||||
private val KonanTarget.abiInfoString: String
|
|
||||||
get() = when {
|
|
||||||
this == KonanTarget.MINGW_X64 -> "WINDOWSX64"
|
|
||||||
!family.isAppleFamily && architecture == Architecture.ARM64 -> "AAPCS"
|
|
||||||
else -> "DEFAULTABI"
|
|
||||||
}
|
|
||||||
+154
-7
@@ -5,21 +5,160 @@
|
|||||||
|
|
||||||
package org.jetbrains.kotlin.konan.test.blackbox.support.runner
|
package org.jetbrains.kotlin.konan.test.blackbox.support.runner
|
||||||
|
|
||||||
|
import com.intellij.openapi.util.SystemInfo
|
||||||
|
import com.intellij.openapi.util.text.StringUtilRt.convertLineSeparators
|
||||||
|
import org.jetbrains.kotlin.konan.target.Architecture
|
||||||
|
import org.jetbrains.kotlin.konan.target.KonanTarget
|
||||||
|
import org.jetbrains.kotlin.konan.target.needSmallBinary
|
||||||
import org.jetbrains.kotlin.konan.test.blackbox.support.LoggedData
|
import org.jetbrains.kotlin.konan.test.blackbox.support.LoggedData
|
||||||
import org.jetbrains.kotlin.konan.test.blackbox.support.TestName
|
import org.jetbrains.kotlin.konan.test.blackbox.support.TestName
|
||||||
|
import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck.ExecutionTimeout
|
||||||
|
import org.jetbrains.kotlin.konan.test.blackbox.support.runner.TestRunCheck.ExitCode
|
||||||
|
import org.jetbrains.kotlin.konan.test.blackbox.support.settings.KotlinNativeTargets
|
||||||
|
import org.jetbrains.kotlin.konan.test.blackbox.support.settings.OptimizationMode
|
||||||
|
import org.jetbrains.kotlin.konan.test.blackbox.support.settings.configurables
|
||||||
import org.jetbrains.kotlin.konan.test.blackbox.support.util.TestReport
|
import org.jetbrains.kotlin.konan.test.blackbox.support.util.TestReport
|
||||||
import org.junit.jupiter.api.Assumptions.assumeFalse
|
import org.jetbrains.kotlin.util.capitalizeDecapitalize.toUpperCaseAsciiOnly
|
||||||
|
import org.junit.jupiter.api.Assumptions
|
||||||
|
import java.io.File
|
||||||
|
|
||||||
|
private fun RunResult.processOutputAsString(output: TestRunCheck.Output) = when (output) {
|
||||||
|
TestRunCheck.Output.STDOUT -> processOutput.stdOut.filteredOutput
|
||||||
|
TestRunCheck.Output.STDERR -> processOutput.stdErr
|
||||||
|
TestRunCheck.Output.ALL -> processOutput.stdOut.filteredOutput + processOutput.stdErr
|
||||||
|
}
|
||||||
|
|
||||||
internal class ResultHandler(
|
internal class ResultHandler(
|
||||||
runResult: RunResult,
|
runResult: RunResult,
|
||||||
visibleProcessName: String,
|
private val visibleProcessName: String,
|
||||||
checks: TestRunChecks,
|
private val checks: TestRunChecks,
|
||||||
private val testRun: TestRun,
|
private val testRun: TestRun,
|
||||||
private val loggedParameters: LoggedData.TestRunParameters
|
private val loggedParameters: LoggedData.TestRunParameters,
|
||||||
) : LocalResultHandler<Unit>(runResult, visibleProcessName, checks, testRun.testCaseId, testRun.expectedFailure) {
|
) : AbstractResultHandler<Unit>(runResult) {
|
||||||
|
|
||||||
override fun getLoggedRun() = LoggedData.TestRun(loggedParameters, runResult)
|
override fun getLoggedRun() = LoggedData.TestRun(loggedParameters, runResult)
|
||||||
|
|
||||||
override fun doHandle() {
|
override fun handle() {
|
||||||
|
val diagnostics = buildList<String> {
|
||||||
|
checks.forEach { check ->
|
||||||
|
when (check) {
|
||||||
|
is ExecutionTimeout.ShouldNotExceed -> if(!runResult.hasFinishedOnTime) add(
|
||||||
|
"Timeout exceeded during test execution."
|
||||||
|
)
|
||||||
|
is ExecutionTimeout.ShouldExceed -> if(runResult.hasFinishedOnTime) add(
|
||||||
|
"Test is expected to fail with exceeded timeout, which hasn't happened."
|
||||||
|
)
|
||||||
|
is ExitCode -> {
|
||||||
|
// Don't check exit code if it is unknown.
|
||||||
|
val knownExitCode: Int = runResult.exitCode ?: return@forEach
|
||||||
|
when (check) {
|
||||||
|
is ExitCode.Expected -> if (knownExitCode != check.expectedExitCode) add(
|
||||||
|
"$visibleProcessName exit code is $knownExitCode while ${check.expectedExitCode} was expected."
|
||||||
|
)
|
||||||
|
is ExitCode.AnyNonZero -> if (knownExitCode == 0) add(
|
||||||
|
"$visibleProcessName exited with zero code, which wasn't expected."
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
is TestRunCheck.OutputDataFile -> {
|
||||||
|
val expectedOutput = check.file.readText()
|
||||||
|
val actualFilteredOutput = runResult.processOutputAsString(check.output)
|
||||||
|
|
||||||
|
// Don't use verifyExpectation(expected, actual) to avoid exposing potentially large test output in exception message
|
||||||
|
// and blowing up test logs.
|
||||||
|
if(convertLineSeparators(expectedOutput) != convertLineSeparators(actualFilteredOutput)) add(
|
||||||
|
"Tested process output mismatch. See \"TEST STDOUT\" and \"EXPECTED OUTPUT DATA FILE\" below."
|
||||||
|
)
|
||||||
|
}
|
||||||
|
is TestRunCheck.OutputMatcher -> {
|
||||||
|
try {
|
||||||
|
if(!check.match(runResult.processOutputAsString(check.output))) add(
|
||||||
|
"Tested process output has not passed validation."
|
||||||
|
)
|
||||||
|
} catch (t: Throwable) {
|
||||||
|
if (t is Exception || t is AssertionError) {
|
||||||
|
add("Tested process output has not passed validation: " + t.message)
|
||||||
|
} else {
|
||||||
|
throw t
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
is TestRunCheck.FileCheckMatcher -> {
|
||||||
|
val fileCheckExecutable = check.settings.configurables.absoluteLlvmHome + File.separator + "bin" + File.separator +
|
||||||
|
if (SystemInfo.isWindows) "FileCheck.exe" else "FileCheck"
|
||||||
|
require(File(fileCheckExecutable).exists()) {
|
||||||
|
"$fileCheckExecutable does not exist. Make sure Distribution for `settings.configurables` " +
|
||||||
|
"was created using `propertyOverrides` to specify development variant of LLVM instead of user variant."
|
||||||
|
}
|
||||||
|
val fileCheckDump = runResult.testExecutable.executable.fileCheckDump!!
|
||||||
|
val fileCheckOut = File(fileCheckDump.absolutePath + ".out")
|
||||||
|
val fileCheckErr = File(fileCheckDump.absolutePath + ".err")
|
||||||
|
|
||||||
|
val testTarget = check.settings.get<KotlinNativeTargets>().testTarget
|
||||||
|
val checkPrefixes = buildList {
|
||||||
|
add("CHECK")
|
||||||
|
add("CHECK-${testTarget.abiInfoString}")
|
||||||
|
add("CHECK-${testTarget.name.toUpperCaseAsciiOnly()}")
|
||||||
|
if (testTarget.family.isAppleFamily) {
|
||||||
|
add("CHECK-APPLE")
|
||||||
|
}
|
||||||
|
if (testTarget.needSmallBinary()) {
|
||||||
|
add("CHECK-SMALLBINARY")
|
||||||
|
} else {
|
||||||
|
add("CHECK-BIGBINARY")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
val optimizationMode = check.settings.get<OptimizationMode>().name
|
||||||
|
val checkPrefixesWithOptMode = checkPrefixes.map { "$it-$optimizationMode" }
|
||||||
|
val commaSeparatedCheckPrefixes = (checkPrefixes + checkPrefixesWithOptMode).joinToString(",")
|
||||||
|
|
||||||
|
val result = ProcessBuilder(
|
||||||
|
fileCheckExecutable,
|
||||||
|
check.testDataFile.absolutePath,
|
||||||
|
"--input-file",
|
||||||
|
fileCheckDump.absolutePath,
|
||||||
|
"--check-prefixes", commaSeparatedCheckPrefixes,
|
||||||
|
"--allow-deprecated-dag-overlap" // TODO specify it via new test directive for `function_attributes_at_callsite.kt`
|
||||||
|
).redirectOutput(fileCheckOut)
|
||||||
|
.redirectError(fileCheckErr)
|
||||||
|
.start()
|
||||||
|
.waitFor()
|
||||||
|
val errText = fileCheckErr.readText()
|
||||||
|
val outText = fileCheckOut.readText()
|
||||||
|
if(!(result == 0 && errText.isEmpty() && outText.isEmpty())) {
|
||||||
|
val shortOutText = outText.lines().take(100)
|
||||||
|
val shortErrText = errText.lines().take(100)
|
||||||
|
add("FileCheck matching of ${fileCheckDump.absolutePath}\n" +
|
||||||
|
"with '--check-prefixes $commaSeparatedCheckPrefixes'\n" +
|
||||||
|
"failed with result=$result:\n" +
|
||||||
|
shortOutText.joinToString("\n") + "\n" +
|
||||||
|
shortErrText.joinToString("\n")
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!testRun.expectedFailure) {
|
||||||
|
verifyExpectation(diagnostics.isEmpty()) {
|
||||||
|
diagnostics.joinToString("\n")
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
val runResultInfo = "TestCaseId: ${testRun.testCaseId}\nExit code: ${runResult.exitCode}\nFiltered test output is${
|
||||||
|
runResult.processOutput.stdOut.filteredOutput.let {
|
||||||
|
if (it.isNotEmpty()) ":\n$it" else " empty."
|
||||||
|
}
|
||||||
|
}"
|
||||||
|
verifyExpectation(diagnostics.isNotEmpty() || runResult.processOutput.stdOut.testReport?.failedTests?.isNotEmpty() == true) {
|
||||||
|
"Test did not fail as expected: $runResultInfo"
|
||||||
|
}
|
||||||
|
println("Test failed as expected.\n$runResultInfo")
|
||||||
|
if (diagnostics.isNotEmpty()) {
|
||||||
|
println("Diagnostics are:")
|
||||||
|
diagnostics.forEach(::println)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
verifyTestReport(runResult.processOutput.stdOut.testReport)
|
verifyTestReport(runResult.processOutput.stdOut.testReport)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -42,7 +181,7 @@ internal class ResultHandler(
|
|||||||
|
|
||||||
if (!testRun.expectedFailure)
|
if (!testRun.expectedFailure)
|
||||||
verifyNoSuchTests(testReport.failedTests, "There are failed tests")
|
verifyNoSuchTests(testReport.failedTests, "There are failed tests")
|
||||||
assumeFalse(testReport.ignoredTests.isNotEmpty() && testReport.passedTests.isEmpty(), "Test case is disabled")
|
Assumptions.assumeFalse(testReport.ignoredTests.isNotEmpty() && testReport.passedTests.isEmpty(), "Test case is disabled")
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun verifyNoSuchTests(tests: Collection<TestName>, subject: String) = verifyExpectation(tests.isEmpty()) {
|
private fun verifyNoSuchTests(tests: Collection<TestName>, subject: String) = verifyExpectation(tests.isEmpty()) {
|
||||||
@@ -52,3 +191,11 @@ internal class ResultHandler(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Shameless borrowing `val KonanTarget.abiInfo` from module `:kotlin-native:backend.native`, which cannot be imported here for now.
|
||||||
|
private val KonanTarget.abiInfoString: String
|
||||||
|
get() = when {
|
||||||
|
this == KonanTarget.MINGW_X64 -> "WINDOWSX64"
|
||||||
|
!family.isAppleFamily && architecture == Architecture.ARM64 -> "AAPCS"
|
||||||
|
else -> "DEFAULTABI"
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user