Gradle, Tests runner: handle case when test runtime exits abnormally

In this case we should close all test nodes with failures and gracefully
close all other enclosing nodes.

#KT-32136 Fixed

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/testing/KotlinTestRunnerListener.kt

@@ -0,0 +1,10 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.internal.testing
+
+interface KotlinTestRunnerListener {
+    fun runningFailure(failure: Error)
+}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/testing/TCServiceMessagesClient.kt

@@ -39,9 +39,11 @@ internal open class TCServiceMessagesClient(
     inline fun root(operation: OperationIdentifier, actions: () -> Unit) {
         rootOperationId = operation
 
-        RootNode(operation).open {
+        val tsStart = System.currentTimeMillis()
-            actions()
+        val root = RootNode(operation)
-        }
+        open(tsStart, root)
+        actions()
+        ensureNodesClosed(root)
     }
 
     override fun parseException(e: ParseException, text: String) {
@@ -136,8 +138,8 @@ internal open class TCServiceMessagesClient(
             }
 
             if (settings.treatFailedTestOutputAsStacktrace) {
-                append(output)
+                append(stackTraceOutput)
-                output.setLength(0)
+                stackTraceOutput.setLength(0)
             }
         }
 
@@ -179,8 +181,9 @@ internal open class TCServiceMessagesClient(
         destination: TestOutputEvent.Destination,
         text: String
     ) {
+        allOutput.append(text)
         if (settings.treatFailedTestOutputAsStacktrace) {
-            output.append(text)
+            stackTraceOutput.append(text)
         } else {
             results.output(descriptor.id, DefaultTestOutputEvent(destination, text))
         }
@@ -445,7 +448,8 @@ internal open class TCServiceMessagesClient(
         localId: String,
         ignored: Boolean = false
     ) : Node(parent, localId) {
-        val output by lazy { StringBuilder() }
+        val stackTraceOutput by lazy { StringBuilder() }
+        val allOutput by lazy { StringBuilder() }
 
         private val parentDescriptor = (this@TestNode.parent as GroupNode).requireReportingNode()
 
@@ -460,6 +464,8 @@ internal open class TCServiceMessagesClient(
         }
 
         override fun markCompleted(ts: Long) {
+            stackTraceOutput.setLength(0)
+            allOutput.setLength(0)
             reportCompleted(ts)
         }
 
@@ -476,12 +482,51 @@ internal open class TCServiceMessagesClient(
     private fun push(node: Node) = node.also { leaf = node }
     private fun pop() = leaf!!.also { leaf = it.parent }
 
-    fun closeAll() {
+    fun ensureNodesClosed(root: RootNode? = null, cause: Throwable? = null, throwError: Boolean = true): Error? {
         val ts = System.currentTimeMillis()
 
-        while (leaf != null) {
+        when (leaf) {
-            close(ts, leaf!!.localId)
+            null -> return null
+            root -> close(ts, leaf!!.localId)
+            else -> {
+                val output = StringBuilder()
+                var currentTest: TestNode? = null
+
+                while (leaf != null) {
+                    val currentLeaf = leaf!!
+
+                    if (currentLeaf is TestNode) {
+                        currentTest = currentLeaf
+                        output.append(currentLeaf.allOutput)
+                        currentLeaf.failure(TestFailed(currentLeaf.cleanName, null))
+                    }
+
+                    close(ts, currentLeaf.localId)
+                }
+
+                @Suppress("ThrowableNotThrown")
+                val error = Error(
+                    buildString {
+                        append("Test running process exited unexpectedly.\n")
+                        if (currentTest != null) {
+                            append("Current test: ${currentTest.cleanName}\n")
+                        }
+                        if (output.toString().isNotBlank()) {
+                            append("Process output:\n $output")
+                        }
+                    },
+                    cause
+                )
+
+                if (throwError) {
+                    throw error
+                } else {
+                    return error
+                }
+            }
         }
+
+        return null
     }
 
     private fun requireLeaf() = leaf ?: error("test out of group")

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/testing/TCServiceMessagesTestExecutor.kt

@@ -28,7 +28,9 @@ private val log = LoggerFactory.getLogger("org.jetbrains.kotlin.gradle.tasks.tes
 
 class TCServiceMessagesTestExecutor(
     val execHandleFactory: ExecHandleFactory,
-    val buildOperationExecutor: BuildOperationExecutor
+    val buildOperationExecutor: BuildOperationExecutor,
+    val runListeners: MutableList<KotlinTestRunnerListener>,
+    val ignoreRunFailures: Boolean
 ) : TestExecuter<TCServiceMessagesTestExecutionSpec> {
     var execHandle: ExecHandle? = null
     var outputReaderThread: Thread? = null
@@ -54,13 +56,22 @@ class TCServiceMessagesTestExecutor(
                 }
 
                 if (spec.checkExitCode && result.exitValue != 0) {
-                    spec.showSuppressedOutput()
                     error("$execHandle exited with errors (exit code: ${result.exitValue})")
                 }
             } catch (e: Throwable) {
-                client.closeAll()
                 spec.showSuppressedOutput()
-                throw e
+
+                val wrappedError = client.ensureNodesClosed(null, e, false) ?: if (e is Error) e else Error(e)
+
+                runListeners.forEach {
+                    it.runningFailure(wrappedError)
+                }
+
+                if (ignoreRunFailures) {
+                    log.error(wrappedError.message)
+                } else {
+                    throw e
+                }
             }
         }
     }

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/KotlinTest.kt

@@ -12,6 +12,7 @@ import org.gradle.api.tasks.Internal
 import org.gradle.api.tasks.Optional
 import org.gradle.api.tasks.testing.AbstractTestTask
 import org.gradle.process.internal.ExecHandleFactory
+import org.jetbrains.kotlin.gradle.internal.testing.KotlinTestRunnerListener
 import org.jetbrains.kotlin.gradle.internal.testing.TCServiceMessagesTestExecutor
 import org.jetbrains.kotlin.gradle.utils.injected
 import javax.inject.Inject
@@ -46,8 +47,19 @@ abstract class KotlinTest : AbstractTestTask() {
     open val execHandleFactory: ExecHandleFactory
         get() = injected
 
+    private val runListeners = mutableListOf<KotlinTestRunnerListener>()
+
+    @Input
+    var ignoreRunFailures: Boolean = false
+
+    fun addRunListener(listener: KotlinTestRunnerListener) {
+        runListeners.add(listener)
+    }
+
     override fun createTestExecuter() = TCServiceMessagesTestExecutor(
         execHandleFactory,
-        buildOperationExecutor
+        buildOperationExecutor,
+        runListeners,
+        ignoreRunFailures
     )
 }

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/testing/internal/DefaultMultiCauseException.kt

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.testing.internal
+
+import java.io.PrintStream
+import java.io.PrintWriter
+
+class MultiCauseException(message: String, val causes: List<Error>) : Error(message, causes.firstOrNull()) {
+    override fun printStackTrace(printStream: PrintStream) {
+        val writer = PrintWriter(printStream)
+        this.printStackTrace(writer)
+        writer.flush()
+    }
+
+    override fun printStackTrace(printWriter: PrintWriter) {
+        if (this.causes.size <= 1) {
+            super.printStackTrace(printWriter)
+        } else {
+            super.printStackTrace(printWriter)
+
+            causes.forEachIndexed { i, it ->
+                printWriter.format("Cause %s: ", i + 1)
+                it.printStackTrace(printWriter)
+            }
+        }
+    }
+}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/testing/internal/KotlinTestReport.kt

@@ -11,7 +11,9 @@ import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.Internal
 import org.gradle.api.tasks.TaskAction
 import org.gradle.api.tasks.testing.*
+import org.jetbrains.kotlin.gradle.internal.testing.KotlinTestRunnerListener
 import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider
+import org.jetbrains.kotlin.gradle.tasks.KotlinTest
 import java.io.File
 import java.net.URI
 
@@ -61,6 +63,8 @@ open class KotlinTestReport : TestReport() {
     private val hasFailedTests: Boolean
         get() = hasOwnFailedTests || children.any { it.hasFailedTests }
 
+    private val ownSuppressedRunningFailures = mutableListOf<Pair<KotlinTest, Error>>()
+
     private val failedTestsListener = object : TestListener {
         override fun beforeTest(testDescriptor: TestDescriptor) {
         }
@@ -98,6 +102,11 @@ open class KotlinTestReport : TestReport() {
     fun registerTestTask(task: AbstractTestTask) {
         testTasks.add(task)
         task.addTestListener(failedTestsListener)
+        if (task is KotlinTest) task.addRunListener(object : KotlinTestRunnerListener {
+            override fun runningFailure(failure: Error) {
+                ownSuppressedRunningFailures.add(task to failure)
+            }
+        })
         reportOn(task)
 
         addToParents(task)
@@ -125,20 +134,72 @@ open class KotlinTestReport : TestReport() {
 
     @TaskAction
     fun checkFailedTests() {
-        if (checkFailedTests && hasFailedTests) {
+        if (checkFailedTests) {
-            val message = StringBuilder("There were failing tests.")
+            checkSuppressedRunningFailures()
 
-            val reportUrl = htmlReportUrl
+            if (hasFailedTests) {
-            if (reportUrl != null) {
+                if (ignoreFailures) {
-                message.append(" See the report at: $reportUrl")
+                    logger.warn(getFailingTestsMessage())
+                } else {
+                    throw GradleException(getFailingTestsMessage())
+                }
+            }
+        }
+    }
+
+    private fun getFailingTestsMessage(): String {
+        val message = StringBuilder("There were failing tests.")
+
+        val reportUrl = htmlReportUrl
+        if (reportUrl != null) {
+            message.append(" See the report at: $reportUrl")
+        }
+        return message.toString()
+    }
+
+    private fun checkSuppressedRunningFailures() {
+        val allSuppressedRunningFailures = mutableListOf<Pair<KotlinTest, Error>>()
+
+        fun visitSuppressedRunningFailures(report: KotlinTestReport) {
+            report.ownSuppressedRunningFailures.forEach {
+                allSuppressedRunningFailures.add(it)
             }
 
-            if (ignoreFailures) {
+            report.children.forEach {
-                logger.warn(message.toString())
+                visitSuppressedRunningFailures(it)
-            } else {
-                throw GradleException(message.toString())
             }
         }
+
+        visitSuppressedRunningFailures(this)
+
+        if (allSuppressedRunningFailures.isNotEmpty()) {
+            val allErrors = mutableListOf<Error>()
+            val msg = buildString {
+                appendln("Failed to execute all tests:")
+                allSuppressedRunningFailures.groupBy { it.first }.forEach { test, errors ->
+                    append(test.path)
+                    append(": ")
+                    var first = true
+                    errors.forEach { (_, error) ->
+                        allErrors.add(error)
+                        append(error.message)
+                        if (first) first = false else appendln()
+                    }
+                }
+
+                if (hasFailedTests) {
+                    val failedTestsMessage = getFailingTestsMessage()
+                    if (ignoreFailures) {
+                        logger.warn(getFailingTestsMessage())
+                    } else {
+                        allErrors.add(Error(failedTestsMessage))
+                        appendln("Also: $failedTestsMessage")
+                    }
+                }
+            }
+
+            throw MultiCauseException(msg, allErrors)
+        }
     }
 
     fun maybeOverrideReporting(graph: TaskExecutionGraph) {
@@ -175,6 +236,9 @@ open class KotlinTestReport : TestReport() {
 
     private fun disableTestReporting(task: AbstractTestTask) {
         task.ignoreFailures = true
+        if (task is KotlinTest) {
+            task.ignoreRunFailures = true
+        }
 
         @Suppress("UnstableApiUsage")
         task.reports.html.isEnabled = false

libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/internal/testing/tcsmc/TestCrash.kt

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.internal.testing.tcsmc
+
+import jetbrains.buildServer.messages.serviceMessages.TestStarted
+import jetbrains.buildServer.messages.serviceMessages.TestSuiteStarted
+import org.junit.Test
+import kotlin.test.assertEquals
+
+class TestCrash : TCServiceMessagesClientTest() {
+    @Test
+    fun testNativeCrash() {
+        treatFailedTestOutputAsStacktrace = true
+
+        var e: Throwable? = null
+        try {
+            assertEvents(
+                """
+STARTED SUITE root // root
+  STARTED SUITE  // root/
+    STARTED TEST displayName: Test, classDisplayName: , className: , name: Test // root//Test
+      StdOut[Crash info] // root//Test
+      FAILURE  // root//Test
+    COMPLETED FAILURE // root//Test
+  COMPLETED FAILURE // root/
+COMPLETED FAILURE // root                
+            """.trimIndent()
+            ) {
+                serviceMessage(TestSuiteStarted(""))
+                serviceMessage(TestStarted("Test", false, null))
+                regularText(
+                    """Crash info"""
+                )
+            }
+        } catch (t: Throwable) {
+            e = t
+        }
+
+        assertEquals(
+            "Test running process exits unexpectedly.\n" +
+                    "Current test: Test\n" +
+                    "Process output:\n" +
+                    " Crash info",
+            e?.message
+        )
+    }
+}