From f096e272b355af4128d49234b78beed10ea5bc7d Mon Sep 17 00:00:00 2001 From: Ilya Matveev Date: Tue, 24 Jan 2017 19:26:43 +0300 Subject: [PATCH] backend.native/tests: Add JSON output to run-external task --- .gitignore | 2 +- backend.native/tests/build.gradle | 40 +++++---- .../org/jetbrains/kotlin/KonanTest.groovy | 83 ++++++++++--------- 3 files changed, 69 insertions(+), 56 deletions(-) diff --git a/.gitignore b/.gitignore index 6c68619c0a0..f87a4bceab3 100644 --- a/.gitignore +++ b/.gitignore @@ -21,7 +21,7 @@ kotstd/kotstd.iml *.kt.S *.kt.exe *.log -testOutput +test.output # Ignore Gradle GUI config gradle-app.setting diff --git a/backend.native/tests/build.gradle b/backend.native/tests/build.gradle index d01c3e9808b..ec765f98bd0 100644 --- a/backend.native/tests/build.gradle +++ b/backend.native/tests/build.gradle @@ -1,3 +1,4 @@ +import groovy.json.JsonOutput import org.jetbrains.kotlin.* configurations { @@ -8,7 +9,7 @@ dependencies { cli_bc project(path: ':backend.native', configuration: 'cli_bc') } -def testOutputRoot = rootProject.file("testOutput").absolutePath +def testOutputRoot = rootProject.file("test.output").absolutePath allprojects { // Root directories for test output (logs, compiled files, statistics etc). Only single path must be in each set. @@ -31,7 +32,6 @@ task clean { } } - task regenerate_external_tests() { doLast { def externalTestsProject = childProjects["external"] @@ -86,24 +86,28 @@ task run() { } task run_external () { - project.subprojects { - dependsOn(it.tasks.withType(RunExternalTestGroup).matching { it.enabled }) - } + // TODO Consider test output directory cleaning before execution. + // TODO Consider using some logger instead of println + def testTasks = childProjects["external"].tasks.withType(RunExternalTestGroup).matching { it.enabled } + dependsOn(testTasks) + doLast { - def logFile = childProjects["external"].file(logFileName) - def logText = logFile.text + Map> results = [:] + def statistics = new RunExternalTestGroup.Statistics() + testTasks.matching { it.state.executed }.each { + results.put(it.name, it.results) + statistics.add(it.statistics) + } - def passed = logText.count("|PASSED|") - def failed = logText.count("|FAILED|") - def skipped = logText.count("|SKIPPED|") - - def report = "\nDONE.\n\n" + - "PASSED: $passed\n" + - "FAILED: $failed\n" + - "TOTAL (FAILED + PASSED): ${passed + failed}\n" + - "SKIPPED : $skipped" - logFile.append(report) - println(report) + def output = ["statistics" : statistics, "tests" : results] + def json = JsonOutput.toJson(output) + def reportFile = new File(sourceSets.testOutputExternal.output.getDirs().getSingleFile(), "results.json") + reportFile.write(JsonOutput.prettyPrint(json)) + println("DONE.\n\n" + + "TOTAL: $statistics.total\n" + + "PASSED: $statistics.passed\n" + + "FAILED: $statistics.failed\n" + + "SKIPPED: $statistics.skipped") } } diff --git a/buildSrc/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy b/buildSrc/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy index 7c0b65c59f7..713c99edbc0 100644 --- a/buildSrc/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy +++ b/buildSrc/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy @@ -1,6 +1,6 @@ package org.jetbrains.kotlin -import groovy.json.JsonBuilder +import groovy.json.JsonOutput import org.gradle.api.DefaultTask import org.gradle.api.tasks.ParallelizableTask import org.gradle.api.tasks.TaskAction @@ -141,19 +141,48 @@ class RunExternalTestGroup extends RunKonanTest { def outputSourceSetName = "testOutputExternal" String filter = project.findProperty("filter") String goldValue = "OK" + Map results = [:] + Statistics statistics = new Statistics() - class TestResult { - String name = null + static class TestResult { String status = null String comment = null - TestResult(String name, String status, String comment){ - this.name = name; + TestResult(String status, String comment = ""){ this.status = status; this.comment = comment; } } + static class Statistics { + int total = 0 + int passed = 0 + int failed = 0 + int skipped = 0 + + void pass(int count = 1) { + passed += count + total += count + } + + void skip(int count = 1) { + skipped += count + total += count + } + + void fail(int count = 1) { + failed += count + total += count + } + + void add(Statistics other) { + total += other.total + passed += other.passed + failed += other.failed + skipped += other.skipped + } + } + // TODO refactor List buildCompileList() { def result = [] @@ -259,53 +288,33 @@ class RunExternalTestGroup extends RunKonanTest { } // Run the tests. - def current = 0 - def passed = 0 - def skipped = 0 - def failed = 0 - def total = ktFiles.size() - def results = [] def currentResult = null + statistics = new Statistics() ktFiles.each { - current++ source = project.relativePath(it) - println("TEST: $it.name ($current/$total, passed: $passed, skipped: $skipped)") + println("TEST: $it.name ($statistics.total/$ktFiles.size, passed: $statistics.passed, skipped: $statistics.skipped)") if (isEnabledForNativeBackend(source)) { try { super.executeTest() - currentResult = new TestResult(it.name, "PASSED", "") - passed++ + currentResult = new TestResult("PASSED") + statistics.pass() } catch (Exception ex) { - currentResult = new TestResult(it.name, "FAILED", + currentResult = new TestResult("FAILED", "Exception: ${ex.getMessage()}. Cause: ${ex.getCause()?.getMessage()}") - failed++ + statistics.fail() } } else { - currentResult = new TestResult(it.name, "SKIPPED", "") - skipped++ + currentResult = new TestResult("SKIPPED") + statistics.skip() } println("TEST $currentResult.status\n") - results.add(currentResult) + results.put(it.name, currentResult) } // Save the report. def reportFile = project.file("${outputDirectory}/results.json") - def json = new JsonBuilder() - json { - "statistics" "total" : total, "passed" : passed, "failed" : failed, "skipped" : skipped - - "tests" { - results.each { result -> - "$result.name" { - "status" result.status - "comment" result.comment - } - } - } - - - } - reportFile.write(json.toPrettyString()) - println("TOTAL PASSED: $passed/$total (SKIPPED: $skipped)") + def json = JsonOutput.toJson(["statistics" : statistics, "tests" : results]) + reportFile.write(JsonOutput.prettyPrint(json)) + println("TOTAL PASSED: $statistics.passed/$statistics.total (SKIPPED: $statistics.skipped)") } }