backend.native/tests: Add JSON output to run-external task

.gitignore

@@ -21,7 +21,7 @@ kotstd/kotstd.iml
 *.kt.S
 *.kt.exe
 *.log
-testOutput
+test.output
 
 # Ignore Gradle GUI config
 gradle-app.setting

backend.native/tests/build.gradle

@@ -1,3 +1,4 @@
+import groovy.json.JsonOutput
 import org.jetbrains.kotlin.*
 
 configurations {
@@ -8,7 +9,7 @@ dependencies {
     cli_bc project(path: ':backend.native', configuration: 'cli_bc')
 }
 
-def testOutputRoot = rootProject.file("testOutput").absolutePath
+def testOutputRoot = rootProject.file("test.output").absolutePath
 
 allprojects {
     // Root directories for test output (logs, compiled files, statistics etc). Only single path must be in each set.
@@ -31,7 +32,6 @@ task clean {
     }
 }
 
-
 task regenerate_external_tests() {
     doLast {
         def externalTestsProject = childProjects["external"]
@@ -86,24 +86,28 @@ task run() {
 }
 
 task run_external () {
-    project.subprojects {
-        dependsOn(it.tasks.withType(RunExternalTestGroup).matching { it.enabled })
-    }
+    // TODO Consider test output directory cleaning before execution.
+    // TODO Consider using some logger instead of println
+    def testTasks = childProjects["external"].tasks.withType(RunExternalTestGroup).matching { it.enabled }
+    dependsOn(testTasks)
+
     doLast {
-        def logFile = childProjects["external"].file(logFileName)
-        def logText = logFile.text
+        Map<String, Map<String, RunExternalTestGroup.TestResult>> results = [:]
+        def statistics = new RunExternalTestGroup.Statistics()
+        testTasks.matching { it.state.executed }.each {
+            results.put(it.name, it.results)
+            statistics.add(it.statistics)
+        }
 
-        def passed = logText.count("|PASSED|")
-        def failed = logText.count("|FAILED|")
-        def skipped = logText.count("|SKIPPED|")
-
-        def report = "\nDONE.\n\n" +
-                     "PASSED: $passed\n" +
-                     "FAILED: $failed\n" +
-                     "TOTAL (FAILED + PASSED): ${passed + failed}\n" +
-                     "SKIPPED : $skipped"
-        logFile.append(report)
-        println(report)
+        def output = ["statistics" : statistics, "tests" : results]
+        def json = JsonOutput.toJson(output)
+        def reportFile = new File(sourceSets.testOutputExternal.output.getDirs().getSingleFile(), "results.json")
+        reportFile.write(JsonOutput.prettyPrint(json))
+        println("DONE.\n\n" +
+                "TOTAL: $statistics.total\n" +
+                "PASSED: $statistics.passed\n" +
+                "FAILED: $statistics.failed\n" +
+                "SKIPPED: $statistics.skipped")
     }
 }
 

buildSrc/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy

@@ -1,6 +1,6 @@
 package org.jetbrains.kotlin
 
-import groovy.json.JsonBuilder
+import groovy.json.JsonOutput
 import org.gradle.api.DefaultTask
 import org.gradle.api.tasks.ParallelizableTask
 import org.gradle.api.tasks.TaskAction
@@ -141,19 +141,48 @@ class RunExternalTestGroup extends RunKonanTest {
     def outputSourceSetName = "testOutputExternal"
     String filter = project.findProperty("filter")
     String goldValue = "OK"
+    Map<String, TestResult> results = [:]
+    Statistics statistics = new Statistics()
 
-    class TestResult {
-        String name = null
+    static class TestResult {
         String status = null
         String comment = null
 
-        TestResult(String name, String status, String comment){
-            this.name = name;
+        TestResult(String status, String comment = ""){
             this.status = status;
             this.comment = comment;
         }
     }
 
+    static class Statistics {
+        int total = 0
+        int passed = 0
+        int failed = 0
+        int skipped = 0
+
+        void pass(int count = 1) {
+            passed += count
+            total += count
+        }
+
+        void skip(int count = 1) {
+            skipped += count
+            total += count
+        }
+
+        void fail(int count = 1) {
+            failed += count
+            total += count
+        }
+
+        void add(Statistics other) {
+            total   += other.total
+            passed  += other.passed
+            failed  += other.failed
+            skipped += other.skipped
+        }
+    }
+
     // TODO refactor
     List<String> buildCompileList() {
         def result = []
@@ -259,53 +288,33 @@ class RunExternalTestGroup extends RunKonanTest {
         }
 
         // Run the tests.
-        def current = 0
-        def passed = 0
-        def skipped = 0
-        def failed = 0
-        def total = ktFiles.size()
-        def results = []
         def currentResult = null
+        statistics = new Statistics()
         ktFiles.each {
-            current++
             source = project.relativePath(it)
-            println("TEST: $it.name ($current/$total, passed: $passed, skipped: $skipped)")
+            println("TEST: $it.name ($statistics.total/$ktFiles.size, passed: $statistics.passed, skipped: $statistics.skipped)")
             if (isEnabledForNativeBackend(source)) {
                 try {
                     super.executeTest()
-                    currentResult = new TestResult(it.name, "PASSED", "")
-                    passed++
+                    currentResult = new TestResult("PASSED")
+                    statistics.pass()
                 } catch (Exception ex) {
-                    currentResult = new TestResult(it.name, "FAILED",
+                    currentResult = new TestResult("FAILED",
                             "Exception: ${ex.getMessage()}. Cause: ${ex.getCause()?.getMessage()}")
-                    failed++
+                    statistics.fail()
                 }
             } else {
-                currentResult = new TestResult(it.name, "SKIPPED", "")
-                skipped++
+                currentResult = new TestResult("SKIPPED")
+                statistics.skip()
             }
             println("TEST $currentResult.status\n")
-            results.add(currentResult)
+            results.put(it.name, currentResult)
         }
 
         // Save the report.
         def reportFile = project.file("${outputDirectory}/results.json")
-        def json = new JsonBuilder()
-        json {
-            "statistics" "total" : total, "passed" : passed, "failed" : failed, "skipped" : skipped
-
-            "tests" {
-                results.each { result ->
-                    "$result.name" {
-                        "status" result.status
-                        "comment" result.comment
-                    }
-                }
-            }
-
-
-        }
-        reportFile.write(json.toPrettyString())
-        println("TOTAL PASSED: $passed/$total (SKIPPED: $skipped)")
+        def json = JsonOutput.toJson(["statistics" : statistics, "tests" : results])
+        reportFile.write(JsonOutput.prettyPrint(json))
+        println("TOTAL PASSED: $statistics.passed/$statistics.total (SKIPPED: $statistics.skipped)")
     }
 }