Gradle, Tests runner: handle case when test runtime exits abnormally

In this case we should close all test nodes with failures and gracefully close all other enclosing nodes. #KT-32136 Fixed
2019-06-28 11:05:52 +03:00
parent 96dcafdf35
commit 38ff228bc8
7 changed files with 246 additions and 24 deletions
@@ -0,0 +1,10 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.internal.testing
+
+interface KotlinTestRunnerListener {
+    fun runningFailure(failure: Error)
+}
@@ -39,9 +39,11 @@ internal open class TCServiceMessagesClient(
    inline fun root(operation: OperationIdentifier, actions: () -> Unit) {
        rootOperationId = operation

-        RootNode(operation).open {
-            actions()
-        }
+        val tsStart = System.currentTimeMillis()
+        val root = RootNode(operation)
+        open(tsStart, root)
+        actions()
+        ensureNodesClosed(root)
    }

    override fun parseException(e: ParseException, text: String) {
@@ -136,8 +138,8 @@ internal open class TCServiceMessagesClient(
            }

            if (settings.treatFailedTestOutputAsStacktrace) {
-                append(output)
-                output.setLength(0)
+                append(stackTraceOutput)
+                stackTraceOutput.setLength(0)
            }
        }

@@ -179,8 +181,9 @@ internal open class TCServiceMessagesClient(
        destination: TestOutputEvent.Destination,
        text: String
    ) {
+        allOutput.append(text)
        if (settings.treatFailedTestOutputAsStacktrace) {
-            output.append(text)
+            stackTraceOutput.append(text)
        } else {
            results.output(descriptor.id, DefaultTestOutputEvent(destination, text))
        }
@@ -445,7 +448,8 @@ internal open class TCServiceMessagesClient(
        localId: String,
        ignored: Boolean = false
    ) : Node(parent, localId) {
-        val output by lazy { StringBuilder() }
+        val stackTraceOutput by lazy { StringBuilder() }
+        val allOutput by lazy { StringBuilder() }

        private val parentDescriptor = (this@TestNode.parent as GroupNode).requireReportingNode()

@@ -460,6 +464,8 @@ internal open class TCServiceMessagesClient(
        }

        override fun markCompleted(ts: Long) {
+            stackTraceOutput.setLength(0)
+            allOutput.setLength(0)
            reportCompleted(ts)
        }

@@ -476,12 +482,51 @@ internal open class TCServiceMessagesClient(
    private fun push(node: Node) = node.also { leaf = node }
    private fun pop() = leaf!!.also { leaf = it.parent }

-    fun closeAll() {
+    fun ensureNodesClosed(root: RootNode? = null, cause: Throwable? = null, throwError: Boolean = true): Error? {
        val ts = System.currentTimeMillis()

-        while (leaf != null) {
-            close(ts, leaf!!.localId)
+        when (leaf) {
+            null -> return null
+            root -> close(ts, leaf!!.localId)
+            else -> {
+                val output = StringBuilder()
+                var currentTest: TestNode? = null
+
+                while (leaf != null) {
+                    val currentLeaf = leaf!!
+
+                    if (currentLeaf is TestNode) {
+                        currentTest = currentLeaf
+                        output.append(currentLeaf.allOutput)
+                        currentLeaf.failure(TestFailed(currentLeaf.cleanName, null))
+                    }
+
+                    close(ts, currentLeaf.localId)
+                }
+
+                @Suppress("ThrowableNotThrown")
+                val error = Error(
+                    buildString {
+                        append("Test running process exited unexpectedly.\n")
+                        if (currentTest != null) {
+                            append("Current test: ${currentTest.cleanName}\n")
+                        }
+                        if (output.toString().isNotBlank()) {
+                            append("Process output:\n $output")
+                        }
+                    },
+                    cause
+                )
+
+                if (throwError) {
+                    throw error
+                } else {
+                    return error
+                }
+            }
        }
+
+        return null
    }

    private fun requireLeaf() = leaf ?: error("test out of group")
@@ -28,7 +28,9 @@ private val log = LoggerFactory.getLogger("org.jetbrains.kotlin.gradle.tasks.tes

 class TCServiceMessagesTestExecutor(
    val execHandleFactory: ExecHandleFactory,
-    val buildOperationExecutor: BuildOperationExecutor
+    val buildOperationExecutor: BuildOperationExecutor,
+    val runListeners: MutableList<KotlinTestRunnerListener>,
+    val ignoreRunFailures: Boolean
 ) : TestExecuter<TCServiceMessagesTestExecutionSpec> {
    var execHandle: ExecHandle? = null
    var outputReaderThread: Thread? = null
@@ -54,13 +56,22 @@ class TCServiceMessagesTestExecutor(
                }

                if (spec.checkExitCode && result.exitValue != 0) {
-                    spec.showSuppressedOutput()
                    error("$execHandle exited with errors (exit code: ${result.exitValue})")
                }
            } catch (e: Throwable) {
-                client.closeAll()
                spec.showSuppressedOutput()
-                throw e
+
+                val wrappedError = client.ensureNodesClosed(null, e, false) ?: if (e is Error) e else Error(e)
+
+                runListeners.forEach {
+                    it.runningFailure(wrappedError)
+                }
+
+                if (ignoreRunFailures) {
+                    log.error(wrappedError.message)
+                } else {
+                    throw e
+                }
            }
        }
    }
@@ -12,6 +12,7 @@ import org.gradle.api.tasks.Internal
 import org.gradle.api.tasks.Optional
 import org.gradle.api.tasks.testing.AbstractTestTask
 import org.gradle.process.internal.ExecHandleFactory
+import org.jetbrains.kotlin.gradle.internal.testing.KotlinTestRunnerListener
 import org.jetbrains.kotlin.gradle.internal.testing.TCServiceMessagesTestExecutor
 import org.jetbrains.kotlin.gradle.utils.injected
 import javax.inject.Inject
@@ -46,8 +47,19 @@ abstract class KotlinTest : AbstractTestTask() {
    open val execHandleFactory: ExecHandleFactory
        get() = injected

+    private val runListeners = mutableListOf<KotlinTestRunnerListener>()
+
+    @Input
+    var ignoreRunFailures: Boolean = false
+
+    fun addRunListener(listener: KotlinTestRunnerListener) {
+        runListeners.add(listener)
+    }
+
    override fun createTestExecuter() = TCServiceMessagesTestExecutor(
        execHandleFactory,
-        buildOperationExecutor
+        buildOperationExecutor,
+        runListeners,
+        ignoreRunFailures
    )
 }
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.testing.internal
+
+import java.io.PrintStream
+import java.io.PrintWriter
+
+class MultiCauseException(message: String, val causes: List<Error>) : Error(message, causes.firstOrNull()) {
+    override fun printStackTrace(printStream: PrintStream) {
+        val writer = PrintWriter(printStream)
+        this.printStackTrace(writer)
+        writer.flush()
+    }
+
+    override fun printStackTrace(printWriter: PrintWriter) {
+        if (this.causes.size <= 1) {
+            super.printStackTrace(printWriter)
+        } else {
+            super.printStackTrace(printWriter)
+
+            causes.forEachIndexed { i, it ->
+                printWriter.format("Cause %s: ", i + 1)
+                it.printStackTrace(printWriter)
+            }
+        }
+    }
+}
@@ -11,7 +11,9 @@ import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.Internal
 import org.gradle.api.tasks.TaskAction
 import org.gradle.api.tasks.testing.*
+import org.jetbrains.kotlin.gradle.internal.testing.KotlinTestRunnerListener
 import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider
+import org.jetbrains.kotlin.gradle.tasks.KotlinTest
 import java.io.File
 import java.net.URI

@@ -61,6 +63,8 @@ open class KotlinTestReport : TestReport() {
    private val hasFailedTests: Boolean
        get() = hasOwnFailedTests || children.any { it.hasFailedTests }

+    private val ownSuppressedRunningFailures = mutableListOf<Pair<KotlinTest, Error>>()
+
    private val failedTestsListener = object : TestListener {
        override fun beforeTest(testDescriptor: TestDescriptor) {
        }
@@ -98,6 +102,11 @@ open class KotlinTestReport : TestReport() {
    fun registerTestTask(task: AbstractTestTask) {
        testTasks.add(task)
        task.addTestListener(failedTestsListener)
+        if (task is KotlinTest) task.addRunListener(object : KotlinTestRunnerListener {
+            override fun runningFailure(failure: Error) {
+                ownSuppressedRunningFailures.add(task to failure)
+            }
+        })
        reportOn(task)

        addToParents(task)
@@ -125,20 +134,72 @@ open class KotlinTestReport : TestReport() {

    @TaskAction
    fun checkFailedTests() {
-        if (checkFailedTests && hasFailedTests) {
-            val message = StringBuilder("There were failing tests.")
+        if (checkFailedTests) {
+            checkSuppressedRunningFailures()

-            val reportUrl = htmlReportUrl
-            if (reportUrl != null) {
-                message.append(" See the report at: $reportUrl")
+            if (hasFailedTests) {
+                if (ignoreFailures) {
+                    logger.warn(getFailingTestsMessage())
+                } else {
+                    throw GradleException(getFailingTestsMessage())
+                }
+            }
+        }
+    }
+
+    private fun getFailingTestsMessage(): String {
+        val message = StringBuilder("There were failing tests.")
+
+        val reportUrl = htmlReportUrl
+        if (reportUrl != null) {
+            message.append(" See the report at: $reportUrl")
+        }
+        return message.toString()
+    }
+
+    private fun checkSuppressedRunningFailures() {
+        val allSuppressedRunningFailures = mutableListOf<Pair<KotlinTest, Error>>()
+
+        fun visitSuppressedRunningFailures(report: KotlinTestReport) {
+            report.ownSuppressedRunningFailures.forEach {
+                allSuppressedRunningFailures.add(it)
            }

-            if (ignoreFailures) {
-                logger.warn(message.toString())
-            } else {
-                throw GradleException(message.toString())
+            report.children.forEach {
+                visitSuppressedRunningFailures(it)
            }
        }
+
+        visitSuppressedRunningFailures(this)
+
+        if (allSuppressedRunningFailures.isNotEmpty()) {
+            val allErrors = mutableListOf<Error>()
+            val msg = buildString {
+                appendln("Failed to execute all tests:")
+                allSuppressedRunningFailures.groupBy { it.first }.forEach { test, errors ->
+                    append(test.path)
+                    append(": ")
+                    var first = true
+                    errors.forEach { (_, error) ->
+                        allErrors.add(error)
+                        append(error.message)
+                        if (first) first = false else appendln()
+                    }
+                }
+
+                if (hasFailedTests) {
+                    val failedTestsMessage = getFailingTestsMessage()
+                    if (ignoreFailures) {
+                        logger.warn(getFailingTestsMessage())
+                    } else {
+                        allErrors.add(Error(failedTestsMessage))
+                        appendln("Also: $failedTestsMessage")
+                    }
+                }
+            }
+
+            throw MultiCauseException(msg, allErrors)
+        }
    }

    fun maybeOverrideReporting(graph: TaskExecutionGraph) {
@@ -175,6 +236,9 @@ open class KotlinTestReport : TestReport() {

    private fun disableTestReporting(task: AbstractTestTask) {
        task.ignoreFailures = true
+        if (task is KotlinTest) {
+            task.ignoreRunFailures = true
+        }

        @Suppress("UnstableApiUsage")
        task.reports.html.isEnabled = false
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.internal.testing.tcsmc
+
+import jetbrains.buildServer.messages.serviceMessages.TestStarted
+import jetbrains.buildServer.messages.serviceMessages.TestSuiteStarted
+import org.junit.Test
+import kotlin.test.assertEquals
+
+class TestCrash : TCServiceMessagesClientTest() {
+    @Test
+    fun testNativeCrash() {
+        treatFailedTestOutputAsStacktrace = true
+
+        var e: Throwable? = null
+        try {
+            assertEvents(
+                """
+STARTED SUITE root // root
+  STARTED SUITE  // root/
+    STARTED TEST displayName: Test, classDisplayName: , className: , name: Test // root//Test
+      StdOut[Crash info] // root//Test
+      FAILURE  // root//Test
+    COMPLETED FAILURE // root//Test
+  COMPLETED FAILURE // root/
+COMPLETED FAILURE // root                
+            """.trimIndent()
+            ) {
+                serviceMessage(TestSuiteStarted(""))
+                serviceMessage(TestStarted("Test", false, null))
+                regularText(
+                    """Crash info"""
+                )
+            }
+        } catch (t: Throwable) {
+            e = t
+        }
+
+        assertEquals(
+            "Test running process exits unexpectedly.\n" +
+                    "Current test: Test\n" +
+                    "Process output:\n" +
+                    " Crash info",
+            e?.message
+        )
+    }
+}