From bb01c4cbbb881d3fd7ad021cadcd3abc111b7550 Mon Sep 17 00:00:00 2001 From: Dmitriy Dolovov Date: Fri, 3 Dec 2021 00:06:10 +0300 Subject: [PATCH] [Native][tests] Filter test output to suppress TEAMCITY-related messages --- .../konan/blackboxtest/support/LoggedData.kt | 6 ++- .../runner/AbstractLocalProcessRunner.kt | 4 +- .../runner/LocalTestFunctionExtractor.kt | 4 +- .../support/runner/LocalTestRunner.kt | 32 +++++------ .../blackboxtest/support/runner/RunResult.kt | 9 ++-- .../konan/blackboxtest/support/util/IO.kt | 4 +- .../blackboxtest/support/util/TestOutput.kt | 53 ++++++++++++------- 7 files changed, 67 insertions(+), 45 deletions(-) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/LoggedData.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/LoggedData.kt index bd948f85f73..5fb62039f6d 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/LoggedData.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/LoggedData.kt @@ -203,11 +203,13 @@ internal abstract class LoggedData { appendDuration(runResult.duration) appendLine() appendLine("========== BEGIN: STDOUT ==========") - if (runResult.output.stdOut.isNotEmpty()) appendLine(runResult.output.stdOut.trimEnd()) + val stdOut = runResult.processOutput.stdOut.filteredOutput + if (stdOut.isNotEmpty()) appendLine(stdOut.trimEnd()) appendLine("========== END: STDOUT ==========") appendLine() appendLine("========== BEGIN: STDERR ==========") - if (runResult.output.stdErr.isNotEmpty()) appendLine(runResult.output.stdErr.trimEnd()) + val stdErr = runResult.processOutput.stdErr + if (stdErr.isNotEmpty()) appendLine(stdErr.trimEnd()) appendLine("========== END: STDERR ==========") return this } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt index 14c7ad4c5a3..84735bfb4a5 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt @@ -7,6 +7,7 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.runner import org.jetbrains.kotlin.konan.blackboxtest.support.TestExecutable import org.jetbrains.kotlin.konan.blackboxtest.support.runner.AbstractRunner.AbstractRun +import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestOutputFilter import org.jetbrains.kotlin.konan.blackboxtest.support.util.readOutput import kotlin.time.* @@ -14,6 +15,7 @@ internal abstract class AbstractLocalProcessRunner(private val executionTimeo protected abstract val visibleProcessName: String protected abstract val executable: TestExecutable protected abstract val programArgs: List + protected abstract val outputFilter: TestOutputFilter protected open fun customizeProcess(process: Process) = Unit @@ -33,7 +35,7 @@ internal abstract class AbstractLocalProcessRunner(private val executionTimeo val (process, hasFinishedOnTime) = result // Don't use blocking read from stdout/stderr on non-finished process. If the process is hanging this would result in hanging test. - val output = process.readOutput(nonBlocking = !hasFinishedOnTime) + val output = process.readOutput(outputFilter, nonBlocking = !hasFinishedOnTime) if (hasFinishedOnTime) { val exitCode: Int = process.exitValue() diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestFunctionExtractor.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestFunctionExtractor.kt index 3bf18877b77..508d6c4c277 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestFunctionExtractor.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestFunctionExtractor.kt @@ -8,6 +8,7 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.runner import org.jetbrains.kotlin.konan.blackboxtest.support.LoggedData import org.jetbrains.kotlin.konan.blackboxtest.support.TestExecutable import org.jetbrains.kotlin.konan.blackboxtest.support.TestFunction +import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestOutputFilter import org.jetbrains.kotlin.konan.blackboxtest.support.util.parseGTestListing import org.jetbrains.kotlin.test.services.JUnit5Assertions import kotlin.time.Duration @@ -18,6 +19,7 @@ internal class LocalTestFunctionExtractor( ) : AbstractLocalProcessRunner>(executionTimeout) { override val visibleProcessName get() = "Test function extractor" override val programArgs = listOf(executable.executableFile.path, "--ktest_list_tests") + override val outputFilter get() = TestOutputFilter.NO_FILTERING override fun getLoggedParameters() = LoggedData.TestRunParameters( compilerCall = executable.loggedCompilerCall, @@ -37,6 +39,6 @@ internal class LocalTestFunctionExtractor( runResult: RunResult.Completed ) : AbstractLocalProcessRunner>.ResultHandler(runResult) { override fun getLoggedRun() = LoggedData.TestRun(getLoggedParameters(), runResult) - override fun doHandle() = parseGTestListing(runResult.output.stdOut) + override fun doHandle() = parseGTestListing(runResult.processOutput.stdOut.filteredOutput) } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt index 5460f2cd409..ba04fc0ffb6 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt @@ -7,8 +7,10 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.runner import com.intellij.openapi.util.text.StringUtilRt.convertLineSeparators import org.jetbrains.kotlin.konan.blackboxtest.support.* +import org.jetbrains.kotlin.konan.blackboxtest.support.util.TCTestOutputFilter import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestName -import org.jetbrains.kotlin.konan.blackboxtest.support.util.parseTCTestReport +import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestOutputFilter +import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestReport import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail import org.junit.jupiter.api.Assumptions.assumeFalse import kotlin.time.Duration @@ -25,6 +27,9 @@ internal class LocalTestRunner( testRun.runParameters.forEach { it.applyTo(this) } } + override val outputFilter: TestOutputFilter + get() = if (testRun.runParameters.has()) TCTestOutputFilter else TestOutputFilter.NO_FILTERING + override fun getLoggedParameters() = LoggedData.TestRunParameters( compilerCall = executable.loggedCompilerCall, testCaseId = testRun.testCaseId, @@ -50,15 +55,14 @@ internal class LocalTestRunner( override fun getLoggedRun() = LoggedData.TestRun(getLoggedParameters(), runResult) override fun doHandle() { - if (testRun.runParameters.has()) { - verifyTCTestLoggerOutput() - } else { - verifyPlainOutput() - } + verifyTestReport(runResult.processOutput.stdOut.testReport) + + val mergedOutput = runResult.processOutput.stdOut.filteredOutput + runResult.processOutput.stdErr + verifyNonTestOutput(mergedOutput) } - private fun verifyTCTestLoggerOutput() { - val testReport = parseTCTestReport(runResult.output.stdOut) + private fun verifyTestReport(testReport: TestReport?) { + if (testReport == null) return verifyExpectation(!testReport.isEmpty()) { "No tests have been found." } @@ -77,22 +81,18 @@ internal class LocalTestRunner( verifyNoSuchTests(testReport.failedTests, "There are failed tests") assumeFalse(testReport.ignoredTests.isNotEmpty() && testReport.passedTests.isEmpty(), "Test case is disabled") - - verifyNonTestOutput { testReport.nonTestOutput + runResult.output.stdErr } } private fun verifyNoSuchTests(tests: Collection, subject: String) = verifyExpectation(tests.isEmpty()) { buildString { - append(subject).appendLine(":") - tests.forEach { append(" - ").appendLine(it) } + append(subject).append(':') + tests.forEach { appendLine().append(" - ").append(it) } } } - private fun verifyPlainOutput() = verifyNonTestOutput { runResult.output.stdOut + runResult.output.stdErr } - - private fun verifyNonTestOutput(nonTestOutput: () -> String) { + private fun verifyNonTestOutput(nonTestOutput: String) { testRun.runParameters.get { - verifyExpectation(convertLineSeparators(expectedOutputDataFile.readText()), convertLineSeparators(nonTestOutput())) { + verifyExpectation(convertLineSeparators(expectedOutputDataFile.readText()), convertLineSeparators(nonTestOutput)) { "Tested process output mismatch. See \"TEST STDOUT\" and \"EXPECTED OUTPUT DATA FILE\" below." } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/RunResult.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/RunResult.kt index cc978509d01..b7f012f1429 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/RunResult.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/RunResult.kt @@ -5,25 +5,26 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.runner +import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestOutputFilter import kotlin.time.Duration internal sealed interface RunResult { val exitCode: Int? val duration: Duration - val output: ProcessOutput + val processOutput: ProcessOutput data class Completed( override val exitCode: Int, override val duration: Duration, - override val output: ProcessOutput + override val processOutput: ProcessOutput ) : RunResult data class TimeoutExceeded( val timeout: Duration, override val exitCode: Int?, override val duration: Duration, - override val output: ProcessOutput + override val processOutput: ProcessOutput ) : RunResult } -internal class ProcessOutput(val stdOut: String, val stdErr: String) +internal class ProcessOutput(val stdOut: TestOutputFilter.FilteredOutput, val stdErr: String) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/IO.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/IO.kt index 89f28ae88c6..be1896c9123 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/IO.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/IO.kt @@ -32,12 +32,12 @@ internal fun InputStream.readBytesNonBlocking(): ByteArray { return result.toByteArray() } -internal fun Process.readOutput(nonBlocking: Boolean): ProcessOutput { +internal fun Process.readOutput(outputFilter: TestOutputFilter, nonBlocking: Boolean): ProcessOutput { val stdOut = if (nonBlocking) inputStream.readBytesNonBlocking() else inputStream.readBytes() val stdErr = if (nonBlocking) errorStream.readBytesNonBlocking() else errorStream.readBytes() return ProcessOutput( - stdOut = stdOut.toString(Charsets.UTF_8), + stdOut = outputFilter.filter(stdOut.toString(Charsets.UTF_8)), stdErr = stdErr.toString(Charsets.UTF_8) ) } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/TestOutput.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/TestOutput.kt index 75cb2d7c259..29aa1d7b5d3 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/TestOutput.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/TestOutput.kt @@ -18,14 +18,25 @@ internal typealias TestName = String internal class TestReport( val passedTests: Collection, val failedTests: Collection, - val ignoredTests: Collection, - val nonTestOutput: String + val ignoredTests: Collection ) { fun isEmpty(): Boolean = passedTests.isEmpty() && failedTests.isEmpty() && ignoredTests.isEmpty() } +internal interface TestOutputFilter { + fun filter(testOutput: String): FilteredOutput + + class FilteredOutput(val filteredOutput: String, val testReport: TestReport?) + + companion object { + val NO_FILTERING = object : TestOutputFilter { + override fun filter(testOutput: String) = FilteredOutput(testOutput, null) + } + } +} + /** - * Parses TC test reports like this: + * Processed TC test reports like this: * * ##teamcity[testSuiteStarted name='sample.test.SampleTestKt' locationHint='ktest:suite://sample.test.SampleTestKt'] * ##teamcity[testStarted name='one' locationHint='ktest:test://sample.test.SampleTestKt.one'] @@ -33,24 +44,30 @@ internal class TestReport( * ##teamcity[testIgnored name='two'] * ##teamcity[testSuiteFinished name='sample.test.SampleTestKt'] */ -internal fun parseTCTestReport(rawTCTestOutput: String): TestReport { - val callback = TCTestMessageParserCallback() - ServiceMessagesParser().parse(rawTCTestOutput, callback) +internal object TCTestOutputFilter : TestOutputFilter { + override fun filter(testOutput: String): TestOutputFilter.FilteredOutput { + val callback = TCTestMessageParserCallback() + ServiceMessagesParser().parse(testOutput, callback) + callback.finish() - assertTrue(callback.errors.isEmpty()) { - buildString { - appendLine("Failed to parse TC test output.") - callback.errors.forEach { error -> + assertTrue(callback.errors.isEmpty()) { + buildString { + appendLine("Failed to parse TC test output.") + callback.errors.forEach { error -> + appendLine() + append("Error: ").appendLine(error) + } appendLine() - append("Error: ").appendLine(error) + appendLine("Full test output:") + append(testOutput) } - appendLine() - appendLine("Test output:") - append(rawTCTestOutput) } - } - return callback.getReport() + return TestOutputFilter.FilteredOutput( + filteredOutput = callback.nonTestOutput.toString(), + testReport = TestReport(callback.passedTests, callback.failedTests, callback.ignoredTests) + ) + } } private class TCTestMessageParserCallback : ServiceMessageParserCallback { @@ -143,11 +160,9 @@ private class TCTestMessageParserCallback : ServiceMessageParserCallback { errors += error() } - fun getReport(): TestReport { + fun finish() { // The last test state is "TestStarted" this likely means that the test process terminated during test execution (SIGSEGV, etc). state.safeAs()?.let { failedTests += it.testName } - - return TestReport(passedTests, failedTests, ignoredTests, nonTestOutput.toString()) } }