Add test for multirun and check gold value on the whole execution output

This commit is contained in:
Pavel Punegov
2019-03-27 12:31:09 +03:00
committed by Pavel Punegov
parent 83b1db86f9
commit ce9f2b7c0b
2 changed files with 34 additions and 22 deletions
+10
View File
@@ -2233,6 +2233,16 @@ task args0(type: RunStandaloneKonanTest) {
source = "runtime/basic/args0.kt"
}
task multiargs(type: RunStandaloneKonanTest) {
arguments = ["AAA", "BB", "C"]
multiRuns = true
multiArguments = [["1", "2", "3"], ["---"], ["Hello", "world"]]
goldValue = "AAA\nBB\nC\n1\n2\n3" +
"AAA\nBB\nC\n---" +
"AAA\nBB\nC\nHello\nworld\n"
source = "runtime/basic/args0.kt"
}
task spread_operator_0(type: RunKonanTest) {
goldValue = "[K, o, t, l, i, n, , i, s, , c, o, o, l, , l, a, n, g, u, a, g, e]\n"
source = "codegen/basics/spread_operator_0.kt"
@@ -287,6 +287,8 @@ abstract class KonanTest extends JavaExec {
def times = multiRuns ? multiArguments.size() : 1
def result = compilerMessagesText
def exitCodeMismatch = false
for (int i = 0; i < times; i++) {
ExecResult execResult = project.execute {
@@ -305,11 +307,11 @@ abstract class KonanTest extends JavaExec {
ignoreExitValue = true
}
def result = compilerMessagesText + out.toString("UTF-8")
def currentResult = out.toString("UTF-8")
println(currentResult)
result += currentResult
println(result)
def exitCodeMismatch = execResult.exitValue != expectedExitStatus
exitCodeMismatch |= execResult.exitValue != expectedExitStatus
if (exitCodeMismatch) {
def message = "Expected exit status: $expectedExitStatus, actual: ${execResult.exitValue}"
if (this.expectedFail) {
@@ -318,25 +320,25 @@ abstract class KonanTest extends JavaExec {
throw new TestFailedException("Test failed on iteration $i. $message")
}
}
result = result.replace(System.lineSeparator(), "\n")
def goldValueMismatch = !outputChecker.apply(result)
if (goldValueMismatch) {
def message
if (goldValue != null) {
message = "Iteration $i. Expected output: $goldValue, actual output: $result"
} else {
message = "Iteration $i. Actual output doesn't match output checker: $result"
}
if (this.expectedFail) {
println("Expected failure. $message")
} else {
throw new TestFailedException("Test failed on iteration $i. $message")
}
}
if (!exitCodeMismatch && !goldValueMismatch && this.expectedFail) println("Unexpected pass")
}
result = result.replace(System.lineSeparator(), "\n")
def goldValueMismatch = !outputChecker.apply(result)
if (goldValueMismatch) {
def message
if (goldValue != null) {
message = "Expected output: $goldValue, actual output: $result"
} else {
message = "Actual output doesn't match output checker: $result"
}
if (this.expectedFail) {
println("Expected failure. $message")
} else {
throw new TestFailedException("Test failed. $message")
}
}
if (!exitCodeMismatch && !goldValueMismatch && this.expectedFail) println("Unexpected pass")
}
}