diff --git a/native/native.tests/build.gradle.kts b/native/native.tests/build.gradle.kts index 5f4fb28ed61..5e268537351 100644 --- a/native/native.tests/build.gradle.kts +++ b/native/native.tests/build.gradle.kts @@ -12,6 +12,7 @@ dependencies { testImplementation(project(":kotlin-reflect")) testImplementation(intellijCore()) testImplementation(commonDependency("commons-lang:commons-lang")) + testImplementation(commonDependency("org.jetbrains.teamcity:serviceMessages")) testImplementation(project(":kotlin-compiler-runner-unshaded")) testImplementation(projectTests(":compiler:tests-common")) testImplementation(projectTests(":compiler:tests-common-new")) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestExecutable.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestExecutable.kt index c57728c657a..90bcab6048c 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestExecutable.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestExecutable.kt @@ -66,9 +66,10 @@ internal sealed interface TestRunParameter { } } - object WithGTestLogger : TestRunParameter { + object WithTCTestLogger : TestRunParameter { override fun applyTo(programArgs: MutableList) { - programArgs += "--ktest_logger=GTEST" + programArgs += "--ktest_logger=TEAMCITY" + programArgs += "--ktest_no_exit_code" } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestRunProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestRunProvider.kt index ad2ceb1b819..4c75af45dbd 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestRunProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestRunProvider.kt @@ -169,12 +169,12 @@ internal class TestRunProvider( ) } TestKind.STANDALONE -> listOfNotNull( - TestRunParameter.WithGTestLogger, + TestRunParameter.WithTCTestLogger, testFunction?.let(TestRunParameter::WithFunctionFilter), expectedOutputDataFile?.let(TestRunParameter::WithExpectedOutputData) ) TestKind.REGULAR -> listOfNotNull( - TestRunParameter.WithGTestLogger, + TestRunParameter.WithTCTestLogger, testFunction?.let(TestRunParameter::WithFunctionFilter) ?: TestRunParameter.WithPackageFilter(nominalPackageName), expectedOutputDataFile?.let(TestRunParameter::WithExpectedOutputData) ) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractRunner.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractRunner.kt index 17be1b3fd44..52dfdc5bdd6 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractRunner.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractRunner.kt @@ -9,6 +9,7 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.LoggedData import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertEquals import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail +import org.opentest4j.TestAbortedException internal abstract class AbstractRunner { protected abstract fun buildRun(): AbstractRun @@ -29,11 +30,12 @@ internal abstract class AbstractRunner { resultHandler.handle() } catch (t: Throwable) { - if (t is AssertionError) - throw t - else { - // An unexpected failure. - handleUnexpectedFailure(t) + when (t) { + is AssertionError, is TestAbortedException -> throw t + else -> { + // An unexpected failure. + handleUnexpectedFailure(t) + } } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt index f9acb4abc54..5460f2cd409 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt @@ -7,8 +7,10 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.runner import com.intellij.openapi.util.text.StringUtilRt.convertLineSeparators import org.jetbrains.kotlin.konan.blackboxtest.support.* -import org.jetbrains.kotlin.konan.blackboxtest.support.util.parseGTestReport +import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestName +import org.jetbrains.kotlin.konan.blackboxtest.support.util.parseTCTestReport import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail +import org.junit.jupiter.api.Assumptions.assumeFalse import kotlin.time.Duration internal class LocalTestRunner( @@ -48,37 +50,49 @@ internal class LocalTestRunner( override fun getLoggedRun() = LoggedData.TestRun(getLoggedParameters(), runResult) override fun doHandle() { - if (testRun.runParameters.has()) { - verifyTestWithGTestRunner() + if (testRun.runParameters.has()) { + verifyTCTestLoggerOutput() } else { - verifyPlainTest() + verifyPlainOutput() } } - private fun verifyTestWithGTestRunner() { - val testReport = parseGTestReport(runResult.output.stdOut) + private fun verifyTCTestLoggerOutput() { + val testReport = parseTCTestReport(runResult.output.stdOut) - verifyExpectation(!testReport.isEmpty()) { "No tests have been executed." } - - val passedTests = testReport.getPassedTests() - verifyExpectation(passedTests.isNotEmpty()) { "No passed tests." } + verifyExpectation(!testReport.isEmpty()) { "No tests have been found." } testRun.runParameters.get { - val excessiveTests = passedTests.filter { testName -> !testMatches(testName) } - verifyExpectation(excessiveTests.isEmpty()) { "Excessive tests have been executed: $excessiveTests." } + verifyNoSuchTests( + testReport.passedTests.filter { testName -> !testMatches(testName) }, + "Excessive tests have been executed" + ) + + verifyNoSuchTests( + testReport.ignoredTests.filter { testName -> !testMatches(testName) }, + "Excessive tests have been ignored" + ) } - val failedTests = testReport.getFailedTests().size - verifyExpectation(0, failedTests) { "There are $failedTests failed tests." } + verifyNoSuchTests(testReport.failedTests, "There are failed tests") - verifyOutputData(mergedOutput = testReport.cleanStdOut + runResult.output.stdErr) + assumeFalse(testReport.ignoredTests.isNotEmpty() && testReport.passedTests.isEmpty(), "Test case is disabled") + + verifyNonTestOutput { testReport.nonTestOutput + runResult.output.stdErr } } - private fun verifyPlainTest() = verifyOutputData(mergedOutput = runResult.output.stdOut + runResult.output.stdErr) + private fun verifyNoSuchTests(tests: Collection, subject: String) = verifyExpectation(tests.isEmpty()) { + buildString { + append(subject).appendLine(":") + tests.forEach { append(" - ").appendLine(it) } + } + } - private fun verifyOutputData(mergedOutput: String) { + private fun verifyPlainOutput() = verifyNonTestOutput { runResult.output.stdOut + runResult.output.stdErr } + + private fun verifyNonTestOutput(nonTestOutput: () -> String) { testRun.runParameters.get { - verifyExpectation(convertLineSeparators(expectedOutputDataFile.readText()), convertLineSeparators(mergedOutput)) { + verifyExpectation(convertLineSeparators(expectedOutputDataFile.readText()), convertLineSeparators(nonTestOutput())) { "Tested process output mismatch. See \"TEST STDOUT\" and \"EXPECTED OUTPUT DATA FILE\" below." } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/GTest.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/GTest.kt deleted file mode 100644 index f394d0efc9c..00000000000 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/GTest.kt +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.konan.blackboxtest.support.util - -import org.jetbrains.kotlin.konan.blackboxtest.support.PackageFQN -import org.jetbrains.kotlin.konan.blackboxtest.support.TestFunction -import org.jetbrains.kotlin.konan.blackboxtest.support.util.GTestListingParseState.* -import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail - -internal typealias TestName = String -internal typealias TestStatus = String - -internal class GTestReport( - private val testStatuses: Map>, - val cleanStdOut: String -) { - fun getPassedTests(): Collection = testStatuses[STATUS_OK].orEmpty() - fun getFailedTests(): Collection = (testStatuses - STATUS_OK).flatMap { it.value } - - fun isEmpty(): Boolean = testStatuses.isEmpty() - - companion object { - private const val STATUS_OK = "OK" - } -} - -/** - * Parses GTest reports like this: - * - * [==========] Running 2 tests from 1 test cases. - * [----------] Global test environment set-up. - * [----------] 2 tests from sample.test.SampleTestKt - * [ RUN ] sample.test.SampleTestKt.one - * [ OK ] sample.test.SampleTestKt.one (0 ms) - * [ RUN ] sample.test.SampleTestKt.two - * [ OK ] sample.test.SampleTestKt.two (0 ms) - * [----------] 2 tests from sample.test.SampleTestKt (0 ms total) - * - * [----------] Global test environment tear-down - * [==========] 2 tests from 1 test cases ran. (0 ms total) - * [ PASSED ] 2 tests. - */ -internal fun parseGTestReport(stdOut: String): GTestReport { - val testStatuses = hashMapOf>() - val cleanStdOut = StringBuilder() - - var expectStatusLine = false - stdOut.lineSequence().forEachIndexed { index, line -> - when { - index == 0 && line.startsWith(STDLIB_TESTS_IGNORED_LINE_PREFIX) -> Unit - expectStatusLine -> { - val matcher = STATUS_LINE_REGEX.matchEntire(line) - if (matcher != null) { - // Read the line with test status. - val testStatus = matcher.groupValues[1] - val testName = matcher.groupValues[2] - testStatuses.getOrPut(testStatus) { hashSetOf() } += testName - expectStatusLine = false - } else { - // If current line is not a status line then it could be only the line with the process' output. - cleanStdOut.appendLine(line) - } - } - line.matches(RUN_LINE_REGEX) -> { - expectStatusLine = true // Next line contains either test status. - } - else -> Unit - } - } - - return GTestReport(testStatuses, cleanStdOut.toString()) -} - -/** - * Extracts [TestFunction]s from GTest listing. - * - * Example: - * sample.test.SampleTestKt. - * one - * two - * - * yields TestFunction(sample.test, one) and TestFunction(sample.test, two). - */ -internal fun parseGTestListing(rawGTestListing: String): Collection = buildList { - var state: GTestListingParseState = Begin - - rawGTestListing.lineSequence().forEachIndexed { index, line -> - fun parseError(message: String): Nothing = fail { - buildString { - appendLine("$message at line #$index: \"$line\"") - appendLine() - appendLine("Full listing:") - appendLine(rawGTestListing) - } - } - - state = when { - index == 0 && line.startsWith(STDLIB_TESTS_IGNORED_LINE_PREFIX) -> state - line.isBlank() -> when (state) { - is NewTest, is End -> End - else -> parseError("Unexpected empty line") - } - line[0].isWhitespace() -> when (val s = state) { - is HasPackageName -> { - this += TestFunction(s.packageName, line.trim()) - NewTest(s.packageName) - } - else -> parseError("Test name encountered before test suite name") - } - else -> when (state) { - is Begin, is NewTest -> { - val packageParts = line.trimEnd().removeSuffix(".").split('.') - if (packageParts.isEmpty()) parseError("Malformed test suite name") - - // Drop the last part because it is related to class name (or file-class name). - // TODO: How to handle nested classes? - val packageName = packageParts.dropLast(1).joinToString(".") - - NewTestSuite(packageName) - } - else -> parseError("Unexpected test suite name") - } - } - } -} - -private val RUN_LINE_REGEX = Regex("""^\[\s+RUN\s+]\s+.*""") -private val STATUS_LINE_REGEX = Regex("""^\[\s+([A-Z]+)\s+]\s+(\S+)\s+.*""") - -// The very first line of stdlib test output may contain seed of Random. Such line should be ignored. -private const val STDLIB_TESTS_IGNORED_LINE_PREFIX = "Seed: " - -private sealed interface GTestListingParseState { - object Begin : GTestListingParseState - object End : GTestListingParseState - - interface HasPackageName : GTestListingParseState { - val packageName: PackageFQN - } - - class NewTestSuite(override val packageName: PackageFQN) : HasPackageName - class NewTest(override val packageName: PackageFQN) : HasPackageName -} diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/TestListing.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/TestListing.kt new file mode 100644 index 00000000000..2f753141b27 --- /dev/null +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/TestListing.kt @@ -0,0 +1,78 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.konan.blackboxtest.support.util + +import org.jetbrains.kotlin.konan.blackboxtest.support.PackageFQN +import org.jetbrains.kotlin.konan.blackboxtest.support.TestFunction +import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail +import org.jetbrains.kotlin.konan.blackboxtest.support.util.GTestListingParseState as State + +/** + * Extracts [TestFunction]s from GTest listing. + * + * Example: + * sample.test.SampleTestKt. + * one + * two + * + * yields TestFunction(sample.test, one) and TestFunction(sample.test, two). + */ +internal fun parseGTestListing(rawGTestListing: String): Collection = buildList { + var state: State = State.Begin + + rawGTestListing.lineSequence().forEachIndexed { index, line -> + fun parseError(message: String): Nothing = fail { + buildString { + appendLine("$message at line #$index: \"$line\"") + appendLine() + appendLine("Full listing:") + appendLine(rawGTestListing) + } + } + + state = when { + index == 0 && line.startsWith(STDLIB_TESTS_IGNORED_LINE_PREFIX) -> state + line.isBlank() -> when (state) { + is State.NewTest, is State.End -> State.End + else -> parseError("Unexpected empty line") + } + line[0].isWhitespace() -> when (val s = state) { + is State.HasPackageName -> { + this += TestFunction(s.packageName, line.trim()) + State.NewTest(s.packageName) + } + else -> parseError("Test name encountered before test suite name") + } + else -> when (state) { + is State.Begin, is State.NewTest -> { + val packageParts = line.trimEnd().removeSuffix(".").split('.') + if (packageParts.isEmpty()) parseError("Malformed test suite name") + + // Drop the last part because it is related to class name (or file-class name). + // TODO: How to handle nested classes? + val packageName = packageParts.dropLast(1).joinToString(".") + + State.NewTestSuite(packageName) + } + else -> parseError("Unexpected test suite name") + } + } + } +} + +private sealed interface GTestListingParseState { + object Begin : State + object End : State + class NewTestSuite(override val packageName: PackageFQN) : State, HasPackageName + class NewTest(override val packageName: PackageFQN) : State, HasPackageName + + interface HasPackageName { + val packageName: PackageFQN + } +} + +// The very first line of stdlib test output may contain seed of Random. Such line should be ignored. +private const val STDLIB_TESTS_IGNORED_LINE_PREFIX = "Seed: " diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/TestOutput.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/TestOutput.kt new file mode 100644 index 00000000000..75cb2d7c259 --- /dev/null +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/TestOutput.kt @@ -0,0 +1,173 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.konan.blackboxtest.support.util + +import jetbrains.buildServer.messages.serviceMessages.* +import org.jetbrains.kotlin.konan.blackboxtest.support.FunctionName +import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue +import org.jetbrains.kotlin.utils.addToStdlib.cast +import org.jetbrains.kotlin.utils.addToStdlib.safeAs +import org.jetbrains.kotlin.konan.blackboxtest.support.util.TCTestReportParseState as State +import java.text.ParseException + +internal typealias TestName = String + +internal class TestReport( + val passedTests: Collection, + val failedTests: Collection, + val ignoredTests: Collection, + val nonTestOutput: String +) { + fun isEmpty(): Boolean = passedTests.isEmpty() && failedTests.isEmpty() && ignoredTests.isEmpty() +} + +/** + * Parses TC test reports like this: + * + * ##teamcity[testSuiteStarted name='sample.test.SampleTestKt' locationHint='ktest:suite://sample.test.SampleTestKt'] + * ##teamcity[testStarted name='one' locationHint='ktest:test://sample.test.SampleTestKt.one'] + * ##teamcity[testFinished name='one' duration='0'] + * ##teamcity[testIgnored name='two'] + * ##teamcity[testSuiteFinished name='sample.test.SampleTestKt'] + */ +internal fun parseTCTestReport(rawTCTestOutput: String): TestReport { + val callback = TCTestMessageParserCallback() + ServiceMessagesParser().parse(rawTCTestOutput, callback) + + assertTrue(callback.errors.isEmpty()) { + buildString { + appendLine("Failed to parse TC test output.") + callback.errors.forEach { error -> + appendLine() + append("Error: ").appendLine(error) + } + appendLine() + appendLine("Test output:") + append(rawTCTestOutput) + } + } + + return callback.getReport() +} + +private class TCTestMessageParserCallback : ServiceMessageParserCallback { + private var afterMessage = false + private var state: State = State.Begin + + val passedTests = mutableListOf() + val failedTests = mutableListOf() + val ignoredTests = mutableListOf() + + val nonTestOutput = StringBuilder() + val errors = mutableListOf() + + override fun regularText(text: String) { + val actualText = if (afterMessage) { + if (text.startsWith("\r\n")) + text.removePrefix("\r\n") + else + text.removePrefix("\n") + } else + text + + nonTestOutput.append(actualText) + afterMessage = false + } + + override fun serviceMessage(message: ServiceMessage) { + fun unexpectedMessage(): State { + addError { + """ + Unexpected TC test message: "$message" + State: $state + """.trimIndent() + } + return state + } + + state = when (message) { + is TestSuiteStarted -> when (state) { + is State.Begin, + is State.TestSuiteFinished -> State.TestSuiteStarted(message.suiteName) + else -> unexpectedMessage() + } + is TestSuiteFinished -> when (state) { + is State.TestSuiteStarted, + is State.TestIgnored, + is State.TestFinished -> State.TestSuiteFinished + else -> unexpectedMessage() + } + is TestIgnored -> when (state) { + is State.TestSuiteStarted, + is State.TestIgnored, + is State.TestFinished -> State.TestIgnored(state.testSuite, message.functionName).also { ignoredTests += it.testName } + else -> unexpectedMessage() + } + is TestStarted -> when (state) { + is State.TestSuiteStarted, + is State.TestIgnored, + is State.TestFinished -> State.TestStarted(state.testSuite, message.testName) + else -> unexpectedMessage() + } + is TestFailed -> when (val s = state) { + is State.TestStarted -> { + nonTestOutput.append(message.stacktrace) + State.TestFailed(s.testSuite, message.testName).also { failedTests += it.testName } + } + else -> unexpectedMessage() + } + is TestFinished -> when (state) { + is State.TestStarted -> State.TestFinished(state.testSuite, message.testName).also { passedTests += it.testName } + is State.TestFailed -> State.TestFinished(state.testSuite, message.testName) + else -> unexpectedMessage() + } + else -> { + addError { "Unsupported TC test message: $message" } + state + } + } + afterMessage = true + } + + override fun parseException(e: ParseException, text: String) { + errors += buildString { + append("Failed to parse TC test message: \"").append(text).appendLine("\"") + appendLine(e.toString()) + } + } + + private fun addError(error: () -> String) { + errors += error() + } + + fun getReport(): TestReport { + // The last test state is "TestStarted" this likely means that the test process terminated during test execution (SIGSEGV, etc). + state.safeAs()?.let { failedTests += it.testName } + + return TestReport(passedTests, failedTests, ignoredTests, nonTestOutput.toString()) + } +} + +private sealed interface TCTestReportParseState { + object Begin : State + + class TestSuiteStarted(val testSuiteName: String) : State + object TestSuiteFinished : State + + sealed class TestState(val testSuite: TestSuiteStarted, val functionName: FunctionName) : State { + val testName: TestName get() = "${testSuite.testSuiteName}.$functionName" + } + + class TestIgnored(testSuite: TestSuiteStarted, functionName: FunctionName) : TestState(testSuite, functionName) + class TestStarted(testSuite: TestSuiteStarted, functionName: FunctionName) : TestState(testSuite, functionName) + class TestFailed(testSuite: TestSuiteStarted, functionName: FunctionName) : TestState(testSuite, functionName) + class TestFinished(testSuite: TestSuiteStarted, functionName: FunctionName) : TestState(testSuite, functionName) +} + +private val State.testSuite: State.TestSuiteStarted + get() = if (this is State.TestSuiteStarted) this else cast().testSuite + +private val BaseTestMessage.functionName: FunctionName get() = testName