diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/testing/KotlinTestRunnerListener.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/testing/KotlinTestRunnerListener.kt new file mode 100644 index 00000000000..a7af0c9092f --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/testing/KotlinTestRunnerListener.kt @@ -0,0 +1,10 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.internal.testing + +interface KotlinTestRunnerListener { + fun runningFailure(failure: Error) +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/testing/TCServiceMessagesClient.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/testing/TCServiceMessagesClient.kt index d950a9e6fe0..a41fb566656 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/testing/TCServiceMessagesClient.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/testing/TCServiceMessagesClient.kt @@ -39,9 +39,11 @@ internal open class TCServiceMessagesClient( inline fun root(operation: OperationIdentifier, actions: () -> Unit) { rootOperationId = operation - RootNode(operation).open { - actions() - } + val tsStart = System.currentTimeMillis() + val root = RootNode(operation) + open(tsStart, root) + actions() + ensureNodesClosed(root) } override fun parseException(e: ParseException, text: String) { @@ -136,8 +138,8 @@ internal open class TCServiceMessagesClient( } if (settings.treatFailedTestOutputAsStacktrace) { - append(output) - output.setLength(0) + append(stackTraceOutput) + stackTraceOutput.setLength(0) } } @@ -179,8 +181,9 @@ internal open class TCServiceMessagesClient( destination: TestOutputEvent.Destination, text: String ) { + allOutput.append(text) if (settings.treatFailedTestOutputAsStacktrace) { - output.append(text) + stackTraceOutput.append(text) } else { results.output(descriptor.id, DefaultTestOutputEvent(destination, text)) } @@ -445,7 +448,8 @@ internal open class TCServiceMessagesClient( localId: String, ignored: Boolean = false ) : Node(parent, localId) { - val output by lazy { StringBuilder() } + val stackTraceOutput by lazy { StringBuilder() } + val allOutput by lazy { StringBuilder() } private val parentDescriptor = (this@TestNode.parent as GroupNode).requireReportingNode() @@ -460,6 +464,8 @@ internal open class TCServiceMessagesClient( } override fun markCompleted(ts: Long) { + stackTraceOutput.setLength(0) + allOutput.setLength(0) reportCompleted(ts) } @@ -476,12 +482,51 @@ internal open class TCServiceMessagesClient( private fun push(node: Node) = node.also { leaf = node } private fun pop() = leaf!!.also { leaf = it.parent } - fun closeAll() { + fun ensureNodesClosed(root: RootNode? = null, cause: Throwable? = null, throwError: Boolean = true): Error? { val ts = System.currentTimeMillis() - while (leaf != null) { - close(ts, leaf!!.localId) + when (leaf) { + null -> return null + root -> close(ts, leaf!!.localId) + else -> { + val output = StringBuilder() + var currentTest: TestNode? = null + + while (leaf != null) { + val currentLeaf = leaf!! + + if (currentLeaf is TestNode) { + currentTest = currentLeaf + output.append(currentLeaf.allOutput) + currentLeaf.failure(TestFailed(currentLeaf.cleanName, null)) + } + + close(ts, currentLeaf.localId) + } + + @Suppress("ThrowableNotThrown") + val error = Error( + buildString { + append("Test running process exited unexpectedly.\n") + if (currentTest != null) { + append("Current test: ${currentTest.cleanName}\n") + } + if (output.toString().isNotBlank()) { + append("Process output:\n $output") + } + }, + cause + ) + + if (throwError) { + throw error + } else { + return error + } + } } + + return null } private fun requireLeaf() = leaf ?: error("test out of group") diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/testing/TCServiceMessagesTestExecutor.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/testing/TCServiceMessagesTestExecutor.kt index d859446ce7e..1f34cd969b7 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/testing/TCServiceMessagesTestExecutor.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/testing/TCServiceMessagesTestExecutor.kt @@ -28,7 +28,9 @@ private val log = LoggerFactory.getLogger("org.jetbrains.kotlin.gradle.tasks.tes class TCServiceMessagesTestExecutor( val execHandleFactory: ExecHandleFactory, - val buildOperationExecutor: BuildOperationExecutor + val buildOperationExecutor: BuildOperationExecutor, + val runListeners: MutableList, + val ignoreRunFailures: Boolean ) : TestExecuter { var execHandle: ExecHandle? = null var outputReaderThread: Thread? = null @@ -54,13 +56,22 @@ class TCServiceMessagesTestExecutor( } if (spec.checkExitCode && result.exitValue != 0) { - spec.showSuppressedOutput() error("$execHandle exited with errors (exit code: ${result.exitValue})") } } catch (e: Throwable) { - client.closeAll() spec.showSuppressedOutput() - throw e + + val wrappedError = client.ensureNodesClosed(null, e, false) ?: if (e is Error) e else Error(e) + + runListeners.forEach { + it.runningFailure(wrappedError) + } + + if (ignoreRunFailures) { + log.error(wrappedError.message) + } else { + throw e + } } } } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/KotlinTest.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/KotlinTest.kt index 80ee7079fbc..70c784b2ca9 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/KotlinTest.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/tasks/KotlinTest.kt @@ -12,6 +12,7 @@ import org.gradle.api.tasks.Internal import org.gradle.api.tasks.Optional import org.gradle.api.tasks.testing.AbstractTestTask import org.gradle.process.internal.ExecHandleFactory +import org.jetbrains.kotlin.gradle.internal.testing.KotlinTestRunnerListener import org.jetbrains.kotlin.gradle.internal.testing.TCServiceMessagesTestExecutor import org.jetbrains.kotlin.gradle.utils.injected import javax.inject.Inject @@ -46,8 +47,19 @@ abstract class KotlinTest : AbstractTestTask() { open val execHandleFactory: ExecHandleFactory get() = injected + private val runListeners = mutableListOf() + + @Input + var ignoreRunFailures: Boolean = false + + fun addRunListener(listener: KotlinTestRunnerListener) { + runListeners.add(listener) + } + override fun createTestExecuter() = TCServiceMessagesTestExecutor( execHandleFactory, - buildOperationExecutor + buildOperationExecutor, + runListeners, + ignoreRunFailures ) } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/testing/internal/DefaultMultiCauseException.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/testing/internal/DefaultMultiCauseException.kt new file mode 100644 index 00000000000..8125a4f9569 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/testing/internal/DefaultMultiCauseException.kt @@ -0,0 +1,30 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.testing.internal + +import java.io.PrintStream +import java.io.PrintWriter + +class MultiCauseException(message: String, val causes: List) : Error(message, causes.firstOrNull()) { + override fun printStackTrace(printStream: PrintStream) { + val writer = PrintWriter(printStream) + this.printStackTrace(writer) + writer.flush() + } + + override fun printStackTrace(printWriter: PrintWriter) { + if (this.causes.size <= 1) { + super.printStackTrace(printWriter) + } else { + super.printStackTrace(printWriter) + + causes.forEachIndexed { i, it -> + printWriter.format("Cause %s: ", i + 1) + it.printStackTrace(printWriter) + } + } + } +} diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/testing/internal/KotlinTestReport.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/testing/internal/KotlinTestReport.kt index de20c12a1f2..89405ad86fa 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/testing/internal/KotlinTestReport.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/testing/internal/KotlinTestReport.kt @@ -11,7 +11,9 @@ import org.gradle.api.tasks.Input import org.gradle.api.tasks.Internal import org.gradle.api.tasks.TaskAction import org.gradle.api.tasks.testing.* +import org.jetbrains.kotlin.gradle.internal.testing.KotlinTestRunnerListener import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider +import org.jetbrains.kotlin.gradle.tasks.KotlinTest import java.io.File import java.net.URI @@ -61,6 +63,8 @@ open class KotlinTestReport : TestReport() { private val hasFailedTests: Boolean get() = hasOwnFailedTests || children.any { it.hasFailedTests } + private val ownSuppressedRunningFailures = mutableListOf>() + private val failedTestsListener = object : TestListener { override fun beforeTest(testDescriptor: TestDescriptor) { } @@ -98,6 +102,11 @@ open class KotlinTestReport : TestReport() { fun registerTestTask(task: AbstractTestTask) { testTasks.add(task) task.addTestListener(failedTestsListener) + if (task is KotlinTest) task.addRunListener(object : KotlinTestRunnerListener { + override fun runningFailure(failure: Error) { + ownSuppressedRunningFailures.add(task to failure) + } + }) reportOn(task) addToParents(task) @@ -125,20 +134,72 @@ open class KotlinTestReport : TestReport() { @TaskAction fun checkFailedTests() { - if (checkFailedTests && hasFailedTests) { - val message = StringBuilder("There were failing tests.") + if (checkFailedTests) { + checkSuppressedRunningFailures() - val reportUrl = htmlReportUrl - if (reportUrl != null) { - message.append(" See the report at: $reportUrl") + if (hasFailedTests) { + if (ignoreFailures) { + logger.warn(getFailingTestsMessage()) + } else { + throw GradleException(getFailingTestsMessage()) + } + } + } + } + + private fun getFailingTestsMessage(): String { + val message = StringBuilder("There were failing tests.") + + val reportUrl = htmlReportUrl + if (reportUrl != null) { + message.append(" See the report at: $reportUrl") + } + return message.toString() + } + + private fun checkSuppressedRunningFailures() { + val allSuppressedRunningFailures = mutableListOf>() + + fun visitSuppressedRunningFailures(report: KotlinTestReport) { + report.ownSuppressedRunningFailures.forEach { + allSuppressedRunningFailures.add(it) } - if (ignoreFailures) { - logger.warn(message.toString()) - } else { - throw GradleException(message.toString()) + report.children.forEach { + visitSuppressedRunningFailures(it) } } + + visitSuppressedRunningFailures(this) + + if (allSuppressedRunningFailures.isNotEmpty()) { + val allErrors = mutableListOf() + val msg = buildString { + appendln("Failed to execute all tests:") + allSuppressedRunningFailures.groupBy { it.first }.forEach { test, errors -> + append(test.path) + append(": ") + var first = true + errors.forEach { (_, error) -> + allErrors.add(error) + append(error.message) + if (first) first = false else appendln() + } + } + + if (hasFailedTests) { + val failedTestsMessage = getFailingTestsMessage() + if (ignoreFailures) { + logger.warn(getFailingTestsMessage()) + } else { + allErrors.add(Error(failedTestsMessage)) + appendln("Also: $failedTestsMessage") + } + } + } + + throw MultiCauseException(msg, allErrors) + } } fun maybeOverrideReporting(graph: TaskExecutionGraph) { @@ -175,6 +236,9 @@ open class KotlinTestReport : TestReport() { private fun disableTestReporting(task: AbstractTestTask) { task.ignoreFailures = true + if (task is KotlinTest) { + task.ignoreRunFailures = true + } @Suppress("UnstableApiUsage") task.reports.html.isEnabled = false diff --git a/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/internal/testing/tcsmc/TestCrash.kt b/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/internal/testing/tcsmc/TestCrash.kt new file mode 100644 index 00000000000..37994b0f8a0 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/test/kotlin/org/jetbrains/kotlin/gradle/internal/testing/tcsmc/TestCrash.kt @@ -0,0 +1,50 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.internal.testing.tcsmc + +import jetbrains.buildServer.messages.serviceMessages.TestStarted +import jetbrains.buildServer.messages.serviceMessages.TestSuiteStarted +import org.junit.Test +import kotlin.test.assertEquals + +class TestCrash : TCServiceMessagesClientTest() { + @Test + fun testNativeCrash() { + treatFailedTestOutputAsStacktrace = true + + var e: Throwable? = null + try { + assertEvents( + """ +STARTED SUITE root // root + STARTED SUITE // root/ + STARTED TEST displayName: Test, classDisplayName: , className: , name: Test // root//Test + StdOut[Crash info] // root//Test + FAILURE // root//Test + COMPLETED FAILURE // root//Test + COMPLETED FAILURE // root/ +COMPLETED FAILURE // root + """.trimIndent() + ) { + serviceMessage(TestSuiteStarted("")) + serviceMessage(TestStarted("Test", false, null)) + regularText( + """Crash info""" + ) + } + } catch (t: Throwable) { + e = t + } + + assertEquals( + "Test running process exits unexpectedly.\n" + + "Current test: Test\n" + + "Process output:\n" + + " Crash info", + e?.message + ) + } +} \ No newline at end of file