Add test for multirun and check gold value on the whole execution output
This commit is contained in:
committed by
Pavel Punegov
parent
83b1db86f9
commit
ce9f2b7c0b
@@ -2233,6 +2233,16 @@ task args0(type: RunStandaloneKonanTest) {
|
|||||||
source = "runtime/basic/args0.kt"
|
source = "runtime/basic/args0.kt"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
task multiargs(type: RunStandaloneKonanTest) {
|
||||||
|
arguments = ["AAA", "BB", "C"]
|
||||||
|
multiRuns = true
|
||||||
|
multiArguments = [["1", "2", "3"], ["---"], ["Hello", "world"]]
|
||||||
|
goldValue = "AAA\nBB\nC\n1\n2\n3" +
|
||||||
|
"AAA\nBB\nC\n---" +
|
||||||
|
"AAA\nBB\nC\nHello\nworld\n"
|
||||||
|
source = "runtime/basic/args0.kt"
|
||||||
|
}
|
||||||
|
|
||||||
task spread_operator_0(type: RunKonanTest) {
|
task spread_operator_0(type: RunKonanTest) {
|
||||||
goldValue = "[K, o, t, l, i, n, , i, s, , c, o, o, l, , l, a, n, g, u, a, g, e]\n"
|
goldValue = "[K, o, t, l, i, n, , i, s, , c, o, o, l, , l, a, n, g, u, a, g, e]\n"
|
||||||
source = "codegen/basics/spread_operator_0.kt"
|
source = "codegen/basics/spread_operator_0.kt"
|
||||||
|
|||||||
@@ -287,6 +287,8 @@ abstract class KonanTest extends JavaExec {
|
|||||||
|
|
||||||
def times = multiRuns ? multiArguments.size() : 1
|
def times = multiRuns ? multiArguments.size() : 1
|
||||||
|
|
||||||
|
def result = compilerMessagesText
|
||||||
|
def exitCodeMismatch = false
|
||||||
for (int i = 0; i < times; i++) {
|
for (int i = 0; i < times; i++) {
|
||||||
ExecResult execResult = project.execute {
|
ExecResult execResult = project.execute {
|
||||||
|
|
||||||
@@ -305,11 +307,11 @@ abstract class KonanTest extends JavaExec {
|
|||||||
|
|
||||||
ignoreExitValue = true
|
ignoreExitValue = true
|
||||||
}
|
}
|
||||||
def result = compilerMessagesText + out.toString("UTF-8")
|
def currentResult = out.toString("UTF-8")
|
||||||
|
println(currentResult)
|
||||||
|
result += currentResult
|
||||||
|
|
||||||
println(result)
|
exitCodeMismatch |= execResult.exitValue != expectedExitStatus
|
||||||
|
|
||||||
def exitCodeMismatch = execResult.exitValue != expectedExitStatus
|
|
||||||
if (exitCodeMismatch) {
|
if (exitCodeMismatch) {
|
||||||
def message = "Expected exit status: $expectedExitStatus, actual: ${execResult.exitValue}"
|
def message = "Expected exit status: $expectedExitStatus, actual: ${execResult.exitValue}"
|
||||||
if (this.expectedFail) {
|
if (this.expectedFail) {
|
||||||
@@ -318,25 +320,25 @@ abstract class KonanTest extends JavaExec {
|
|||||||
throw new TestFailedException("Test failed on iteration $i. $message")
|
throw new TestFailedException("Test failed on iteration $i. $message")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
result = result.replace(System.lineSeparator(), "\n")
|
|
||||||
def goldValueMismatch = !outputChecker.apply(result)
|
|
||||||
if (goldValueMismatch) {
|
|
||||||
def message
|
|
||||||
if (goldValue != null) {
|
|
||||||
message = "Iteration $i. Expected output: $goldValue, actual output: $result"
|
|
||||||
} else {
|
|
||||||
message = "Iteration $i. Actual output doesn't match output checker: $result"
|
|
||||||
}
|
|
||||||
if (this.expectedFail) {
|
|
||||||
println("Expected failure. $message")
|
|
||||||
} else {
|
|
||||||
throw new TestFailedException("Test failed on iteration $i. $message")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!exitCodeMismatch && !goldValueMismatch && this.expectedFail) println("Unexpected pass")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
result = result.replace(System.lineSeparator(), "\n")
|
||||||
|
def goldValueMismatch = !outputChecker.apply(result)
|
||||||
|
if (goldValueMismatch) {
|
||||||
|
def message
|
||||||
|
if (goldValue != null) {
|
||||||
|
message = "Expected output: $goldValue, actual output: $result"
|
||||||
|
} else {
|
||||||
|
message = "Actual output doesn't match output checker: $result"
|
||||||
|
}
|
||||||
|
if (this.expectedFail) {
|
||||||
|
println("Expected failure. $message")
|
||||||
|
} else {
|
||||||
|
throw new TestFailedException("Test failed. $message")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!exitCodeMismatch && !goldValueMismatch && this.expectedFail) println("Unexpected pass")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user