[Native][tests] Use TEAMCITY test logger instead of GTEST one

That's because currently K/N tests do not report ignored tests with GTEST logger, but do so with TEAMCITY logger.
2021-12-02 12:41:42 +03:00
parent 553ea22504
commit 09dbd43b97
8 changed files with 296 additions and 173 deletions
@@ -12,6 +12,7 @@ dependencies {
    testImplementation(project(":kotlin-reflect"))
    testImplementation(intellijCore())
    testImplementation(commonDependency("commons-lang:commons-lang"))
+    testImplementation(commonDependency("org.jetbrains.teamcity:serviceMessages"))
    testImplementation(project(":kotlin-compiler-runner-unshaded"))
    testImplementation(projectTests(":compiler:tests-common"))
    testImplementation(projectTests(":compiler:tests-common-new"))
@@ -66,9 +66,10 @@ internal sealed interface TestRunParameter {
        }
    }

-    object WithGTestLogger : TestRunParameter {
+    object WithTCTestLogger : TestRunParameter {
        override fun applyTo(programArgs: MutableList<String>) {
-            programArgs += "--ktest_logger=GTEST"
+            programArgs += "--ktest_logger=TEAMCITY"
+            programArgs += "--ktest_no_exit_code"
        }
    }

@@ -169,12 +169,12 @@ internal class TestRunProvider(
                )
            }
            TestKind.STANDALONE -> listOfNotNull(
-                TestRunParameter.WithGTestLogger,
+                TestRunParameter.WithTCTestLogger,
                testFunction?.let(TestRunParameter::WithFunctionFilter),
                expectedOutputDataFile?.let(TestRunParameter::WithExpectedOutputData)
            )
            TestKind.REGULAR -> listOfNotNull(
-                TestRunParameter.WithGTestLogger,
+                TestRunParameter.WithTCTestLogger,
                testFunction?.let(TestRunParameter::WithFunctionFilter) ?: TestRunParameter.WithPackageFilter(nominalPackageName),
                expectedOutputDataFile?.let(TestRunParameter::WithExpectedOutputData)
            )
@@ -9,6 +9,7 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.LoggedData
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertEquals
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
+import org.opentest4j.TestAbortedException

 internal abstract class AbstractRunner<R> {
    protected abstract fun buildRun(): AbstractRun
@@ -29,11 +30,12 @@ internal abstract class AbstractRunner<R> {

        resultHandler.handle()
    } catch (t: Throwable) {
-        if (t is AssertionError)
-            throw t
-        else {
-            // An unexpected failure.
-            handleUnexpectedFailure(t)
+        when (t) {
+            is AssertionError, is TestAbortedException -> throw t
+            else -> {
+                // An unexpected failure.
+                handleUnexpectedFailure(t)
+            }
        }
    }

@@ -7,8 +7,10 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.runner

 import com.intellij.openapi.util.text.StringUtilRt.convertLineSeparators
 import org.jetbrains.kotlin.konan.blackboxtest.support.*
-import org.jetbrains.kotlin.konan.blackboxtest.support.util.parseGTestReport
+import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestName
+import org.jetbrains.kotlin.konan.blackboxtest.support.util.parseTCTestReport
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
+import org.junit.jupiter.api.Assumptions.assumeFalse
 import kotlin.time.Duration

 internal class LocalTestRunner(
@@ -48,37 +50,49 @@ internal class LocalTestRunner(
        override fun getLoggedRun() = LoggedData.TestRun(getLoggedParameters(), runResult)

        override fun doHandle() {
-            if (testRun.runParameters.has<TestRunParameter.WithGTestLogger>()) {
-                verifyTestWithGTestRunner()
+            if (testRun.runParameters.has<TestRunParameter.WithTCTestLogger>()) {
+                verifyTCTestLoggerOutput()
            } else {
-                verifyPlainTest()
+                verifyPlainOutput()
            }
        }

-        private fun verifyTestWithGTestRunner() {
-            val testReport = parseGTestReport(runResult.output.stdOut)
+        private fun verifyTCTestLoggerOutput() {
+            val testReport = parseTCTestReport(runResult.output.stdOut)

-            verifyExpectation(!testReport.isEmpty()) { "No tests have been executed." }
-
-            val passedTests = testReport.getPassedTests()
-            verifyExpectation(passedTests.isNotEmpty()) { "No passed tests." }
+            verifyExpectation(!testReport.isEmpty()) { "No tests have been found." }

            testRun.runParameters.get<TestRunParameter.WithFilter> {
-                val excessiveTests = passedTests.filter { testName -> !testMatches(testName) }
-                verifyExpectation(excessiveTests.isEmpty()) { "Excessive tests have been executed: $excessiveTests." }
+                verifyNoSuchTests(
+                    testReport.passedTests.filter { testName -> !testMatches(testName) },
+                    "Excessive tests have been executed"
+                )
+
+                verifyNoSuchTests(
+                    testReport.ignoredTests.filter { testName -> !testMatches(testName) },
+                    "Excessive tests have been ignored"
+                )
            }

-            val failedTests = testReport.getFailedTests().size
-            verifyExpectation(0, failedTests) { "There are $failedTests failed tests." }
+            verifyNoSuchTests(testReport.failedTests, "There are failed tests")

-            verifyOutputData(mergedOutput = testReport.cleanStdOut + runResult.output.stdErr)
+            assumeFalse(testReport.ignoredTests.isNotEmpty() && testReport.passedTests.isEmpty(), "Test case is disabled")
+
+            verifyNonTestOutput { testReport.nonTestOutput + runResult.output.stdErr }
        }

-        private fun verifyPlainTest() = verifyOutputData(mergedOutput = runResult.output.stdOut + runResult.output.stdErr)
+        private fun verifyNoSuchTests(tests: Collection<TestName>, subject: String) = verifyExpectation(tests.isEmpty()) {
+            buildString {
+                append(subject).appendLine(":")
+                tests.forEach { append(" - ").appendLine(it) }
+            }
+        }

-        private fun verifyOutputData(mergedOutput: String) {
+        private fun verifyPlainOutput() = verifyNonTestOutput { runResult.output.stdOut + runResult.output.stdErr }
+
+        private fun verifyNonTestOutput(nonTestOutput: () -> String) {
            testRun.runParameters.get<TestRunParameter.WithExpectedOutputData> {
-                verifyExpectation(convertLineSeparators(expectedOutputDataFile.readText()), convertLineSeparators(mergedOutput)) {
+                verifyExpectation(convertLineSeparators(expectedOutputDataFile.readText()), convertLineSeparators(nonTestOutput())) {
                    "Tested process output mismatch. See \"TEST STDOUT\" and \"EXPECTED OUTPUT DATA FILE\" below."
                }
            }
@@ -1,146 +0,0 @@
-/*
- * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.konan.blackboxtest.support.util
-
-import org.jetbrains.kotlin.konan.blackboxtest.support.PackageFQN
-import org.jetbrains.kotlin.konan.blackboxtest.support.TestFunction
-import org.jetbrains.kotlin.konan.blackboxtest.support.util.GTestListingParseState.*
-import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
-
-internal typealias TestName = String
-internal typealias TestStatus = String
-
-internal class GTestReport(
-    private val testStatuses: Map<TestStatus, Collection<TestName>>,
-    val cleanStdOut: String
-) {
-    fun getPassedTests(): Collection<TestName> = testStatuses[STATUS_OK].orEmpty()
-    fun getFailedTests(): Collection<TestName> = (testStatuses - STATUS_OK).flatMap { it.value }
-
-    fun isEmpty(): Boolean = testStatuses.isEmpty()
-
-    companion object {
-        private const val STATUS_OK = "OK"
-    }
-}
-
-/**
- * Parses GTest reports like this:
- *
- *   [==========] Running 2 tests from 1 test cases.
- *   [----------] Global test environment set-up.
- *   [----------] 2 tests from sample.test.SampleTestKt
- *   [ RUN      ] sample.test.SampleTestKt.one
- *   [       OK ] sample.test.SampleTestKt.one (0 ms)
- *   [ RUN      ] sample.test.SampleTestKt.two
- *   [       OK ] sample.test.SampleTestKt.two (0 ms)
- *   [----------] 2 tests from sample.test.SampleTestKt (0 ms total)
- *
- *   [----------] Global test environment tear-down
- *   [==========] 2 tests from 1 test cases ran. (0 ms total)
- *   [  PASSED  ] 2 tests.
- */
-internal fun parseGTestReport(stdOut: String): GTestReport {
-    val testStatuses = hashMapOf<TestStatus, MutableSet<TestName>>()
-    val cleanStdOut = StringBuilder()
-
-    var expectStatusLine = false
-    stdOut.lineSequence().forEachIndexed { index, line ->
-        when {
-            index == 0 && line.startsWith(STDLIB_TESTS_IGNORED_LINE_PREFIX) -> Unit
-            expectStatusLine -> {
-                val matcher = STATUS_LINE_REGEX.matchEntire(line)
-                if (matcher != null) {
-                    // Read the line with test status.
-                    val testStatus = matcher.groupValues[1]
-                    val testName = matcher.groupValues[2]
-                    testStatuses.getOrPut(testStatus) { hashSetOf() } += testName
-                    expectStatusLine = false
-                } else {
-                    // If current line is not a status line then it could be only the line with the process' output.
-                    cleanStdOut.appendLine(line)
-                }
-            }
-            line.matches(RUN_LINE_REGEX) -> {
-                expectStatusLine = true // Next line contains either  test status.
-            }
-            else -> Unit
-        }
-    }
-
-    return GTestReport(testStatuses, cleanStdOut.toString())
-}
-
-/**
- * Extracts [TestFunction]s from GTest listing.
- *
- * Example:
- *   sample.test.SampleTestKt.
- *     one
- *     two
- *
- * yields TestFunction(sample.test, one) and TestFunction(sample.test, two).
- */
-internal fun parseGTestListing(rawGTestListing: String): Collection<TestFunction> = buildList {
-    var state: GTestListingParseState = Begin
-
-    rawGTestListing.lineSequence().forEachIndexed { index, line ->
-        fun parseError(message: String): Nothing = fail {
-            buildString {
-                appendLine("$message at line #$index: \"$line\"")
-                appendLine()
-                appendLine("Full listing:")
-                appendLine(rawGTestListing)
-            }
-        }
-
-        state = when {
-            index == 0 && line.startsWith(STDLIB_TESTS_IGNORED_LINE_PREFIX) -> state
-            line.isBlank() -> when (state) {
-                is NewTest, is End -> End
-                else -> parseError("Unexpected empty line")
-            }
-            line[0].isWhitespace() -> when (val s = state) {
-                is HasPackageName -> {
-                    this += TestFunction(s.packageName, line.trim())
-                    NewTest(s.packageName)
-                }
-                else -> parseError("Test name encountered before test suite name")
-            }
-            else -> when (state) {
-                is Begin, is NewTest -> {
-                    val packageParts = line.trimEnd().removeSuffix(".").split('.')
-                    if (packageParts.isEmpty()) parseError("Malformed test suite name")
-
-                    // Drop the last part because it is related to class name (or file-class name).
-                    // TODO: How to handle nested classes?
-                    val packageName = packageParts.dropLast(1).joinToString(".")
-
-                    NewTestSuite(packageName)
-                }
-                else -> parseError("Unexpected test suite name")
-            }
-        }
-    }
-}
-
-private val RUN_LINE_REGEX = Regex("""^\[\s+RUN\s+]\s+.*""")
-private val STATUS_LINE_REGEX = Regex("""^\[\s+([A-Z]+)\s+]\s+(\S+)\s+.*""")
-
-// The very first line of stdlib test output may contain seed of Random. Such line should be ignored.
-private const val STDLIB_TESTS_IGNORED_LINE_PREFIX = "Seed: "
-
-private sealed interface GTestListingParseState {
-    object Begin : GTestListingParseState
-    object End : GTestListingParseState
-
-    interface HasPackageName : GTestListingParseState {
-        val packageName: PackageFQN
-    }
-
-    class NewTestSuite(override val packageName: PackageFQN) : HasPackageName
-    class NewTest(override val packageName: PackageFQN) : HasPackageName
-}
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.konan.blackboxtest.support.util
+
+import org.jetbrains.kotlin.konan.blackboxtest.support.PackageFQN
+import org.jetbrains.kotlin.konan.blackboxtest.support.TestFunction
+import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
+import org.jetbrains.kotlin.konan.blackboxtest.support.util.GTestListingParseState as State
+
+/**
+ * Extracts [TestFunction]s from GTest listing.
+ *
+ * Example:
+ *   sample.test.SampleTestKt.
+ *     one
+ *     two
+ *
+ * yields TestFunction(sample.test, one) and TestFunction(sample.test, two).
+ */
+internal fun parseGTestListing(rawGTestListing: String): Collection<TestFunction> = buildList {
+    var state: State = State.Begin
+
+    rawGTestListing.lineSequence().forEachIndexed { index, line ->
+        fun parseError(message: String): Nothing = fail {
+            buildString {
+                appendLine("$message at line #$index: \"$line\"")
+                appendLine()
+                appendLine("Full listing:")
+                appendLine(rawGTestListing)
+            }
+        }
+
+        state = when {
+            index == 0 && line.startsWith(STDLIB_TESTS_IGNORED_LINE_PREFIX) -> state
+            line.isBlank() -> when (state) {
+                is State.NewTest, is State.End -> State.End
+                else -> parseError("Unexpected empty line")
+            }
+            line[0].isWhitespace() -> when (val s = state) {
+                is State.HasPackageName -> {
+                    this += TestFunction(s.packageName, line.trim())
+                    State.NewTest(s.packageName)
+                }
+                else -> parseError("Test name encountered before test suite name")
+            }
+            else -> when (state) {
+                is State.Begin, is State.NewTest -> {
+                    val packageParts = line.trimEnd().removeSuffix(".").split('.')
+                    if (packageParts.isEmpty()) parseError("Malformed test suite name")
+
+                    // Drop the last part because it is related to class name (or file-class name).
+                    // TODO: How to handle nested classes?
+                    val packageName = packageParts.dropLast(1).joinToString(".")
+
+                    State.NewTestSuite(packageName)
+                }
+                else -> parseError("Unexpected test suite name")
+            }
+        }
+    }
+}
+
+private sealed interface GTestListingParseState {
+    object Begin : State
+    object End : State
+    class NewTestSuite(override val packageName: PackageFQN) : State, HasPackageName
+    class NewTest(override val packageName: PackageFQN) : State, HasPackageName
+
+    interface HasPackageName {
+        val packageName: PackageFQN
+    }
+}
+
+// The very first line of stdlib test output may contain seed of Random. Such line should be ignored.
+private const val STDLIB_TESTS_IGNORED_LINE_PREFIX = "Seed: "
@@ -0,0 +1,173 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.konan.blackboxtest.support.util
+
+import jetbrains.buildServer.messages.serviceMessages.*
+import org.jetbrains.kotlin.konan.blackboxtest.support.FunctionName
+import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue
+import org.jetbrains.kotlin.utils.addToStdlib.cast
+import org.jetbrains.kotlin.utils.addToStdlib.safeAs
+import org.jetbrains.kotlin.konan.blackboxtest.support.util.TCTestReportParseState as State
+import java.text.ParseException
+
+internal typealias TestName = String
+
+internal class TestReport(
+    val passedTests: Collection<TestName>,
+    val failedTests: Collection<TestName>,
+    val ignoredTests: Collection<TestName>,
+    val nonTestOutput: String
+) {
+    fun isEmpty(): Boolean = passedTests.isEmpty() && failedTests.isEmpty() && ignoredTests.isEmpty()
+}
+
+/**
+ * Parses TC test reports like this:
+ *
+ *   ##teamcity[testSuiteStarted name='sample.test.SampleTestKt' locationHint='ktest:suite://sample.test.SampleTestKt']
+ *   ##teamcity[testStarted name='one' locationHint='ktest:test://sample.test.SampleTestKt.one']
+ *   ##teamcity[testFinished name='one' duration='0']
+ *   ##teamcity[testIgnored name='two']
+ *   ##teamcity[testSuiteFinished name='sample.test.SampleTestKt']
+ */
+internal fun parseTCTestReport(rawTCTestOutput: String): TestReport {
+    val callback = TCTestMessageParserCallback()
+    ServiceMessagesParser().parse(rawTCTestOutput, callback)
+
+    assertTrue(callback.errors.isEmpty()) {
+        buildString {
+            appendLine("Failed to parse TC test output.")
+            callback.errors.forEach { error ->
+                appendLine()
+                append("Error: ").appendLine(error)
+            }
+            appendLine()
+            appendLine("Test output:")
+            append(rawTCTestOutput)
+        }
+    }
+
+    return callback.getReport()
+}
+
+private class TCTestMessageParserCallback : ServiceMessageParserCallback {
+    private var afterMessage = false
+    private var state: State = State.Begin
+
+    val passedTests = mutableListOf<TestName>()
+    val failedTests = mutableListOf<TestName>()
+    val ignoredTests = mutableListOf<TestName>()
+
+    val nonTestOutput = StringBuilder()
+    val errors = mutableListOf<String>()
+
+    override fun regularText(text: String) {
+        val actualText = if (afterMessage) {
+            if (text.startsWith("\r\n"))
+                text.removePrefix("\r\n")
+            else
+                text.removePrefix("\n")
+        } else
+            text
+
+        nonTestOutput.append(actualText)
+        afterMessage = false
+    }
+
+    override fun serviceMessage(message: ServiceMessage) {
+        fun unexpectedMessage(): State {
+            addError {
+                """
+                    Unexpected TC test message: "$message"
+                    State: $state
+                """.trimIndent()
+            }
+            return state
+        }
+
+        state = when (message) {
+            is TestSuiteStarted -> when (state) {
+                is State.Begin,
+                is State.TestSuiteFinished -> State.TestSuiteStarted(message.suiteName)
+                else -> unexpectedMessage()
+            }
+            is TestSuiteFinished -> when (state) {
+                is State.TestSuiteStarted,
+                is State.TestIgnored,
+                is State.TestFinished -> State.TestSuiteFinished
+                else -> unexpectedMessage()
+            }
+            is TestIgnored -> when (state) {
+                is State.TestSuiteStarted,
+                is State.TestIgnored,
+                is State.TestFinished -> State.TestIgnored(state.testSuite, message.functionName).also { ignoredTests += it.testName }
+                else -> unexpectedMessage()
+            }
+            is TestStarted -> when (state) {
+                is State.TestSuiteStarted,
+                is State.TestIgnored,
+                is State.TestFinished -> State.TestStarted(state.testSuite, message.testName)
+                else -> unexpectedMessage()
+            }
+            is TestFailed -> when (val s = state) {
+                is State.TestStarted -> {
+                    nonTestOutput.append(message.stacktrace)
+                    State.TestFailed(s.testSuite, message.testName).also { failedTests += it.testName }
+                }
+                else -> unexpectedMessage()
+            }
+            is TestFinished -> when (state) {
+                is State.TestStarted -> State.TestFinished(state.testSuite, message.testName).also { passedTests += it.testName }
+                is State.TestFailed -> State.TestFinished(state.testSuite, message.testName)
+                else -> unexpectedMessage()
+            }
+            else -> {
+                addError { "Unsupported TC test message: $message" }
+                state
+            }
+        }
+        afterMessage = true
+    }
+
+    override fun parseException(e: ParseException, text: String) {
+        errors += buildString {
+            append("Failed to parse TC test message: \"").append(text).appendLine("\"")
+            appendLine(e.toString())
+        }
+    }
+
+    private fun addError(error: () -> String) {
+        errors += error()
+    }
+
+    fun getReport(): TestReport {
+        // The last test state is "TestStarted" this likely means that the test process terminated during test execution (SIGSEGV, etc).
+        state.safeAs<State.TestStarted>()?.let { failedTests += it.testName }
+
+        return TestReport(passedTests, failedTests, ignoredTests, nonTestOutput.toString())
+    }
+}
+
+private sealed interface TCTestReportParseState {
+    object Begin : State
+
+    class TestSuiteStarted(val testSuiteName: String) : State
+    object TestSuiteFinished : State
+
+    sealed class TestState(val testSuite: TestSuiteStarted, val functionName: FunctionName) : State {
+        val testName: TestName get() = "${testSuite.testSuiteName}.$functionName"
+    }
+
+    class TestIgnored(testSuite: TestSuiteStarted, functionName: FunctionName) : TestState(testSuite, functionName)
+    class TestStarted(testSuite: TestSuiteStarted, functionName: FunctionName) : TestState(testSuite, functionName)
+    class TestFailed(testSuite: TestSuiteStarted, functionName: FunctionName) : TestState(testSuite, functionName)
+    class TestFinished(testSuite: TestSuiteStarted, functionName: FunctionName) : TestState(testSuite, functionName)
+}
+
+private val State.testSuite: State.TestSuiteStarted
+    get() = if (this is State.TestSuiteStarted) this else cast<State.TestState>().testSuite
+
+private val BaseTestMessage.functionName: FunctionName get() = testName