From 4f037b8b811d4d7ea8e5f3a3e962521410475c27 Mon Sep 17 00:00:00 2001 From: Ilya Matveev Date: Wed, 3 Feb 2021 12:32:50 +0700 Subject: [PATCH] [kotlin.test] Don't run filtered out test suites by default Popular frameworks such as JUnit, TestNG and Google Test do not run test suites whose tests were fully filtered out. Thus @BeforeClass/@AfterClass of such suites are not executed and the suites are not reported in a test report. This patch introduces the same behaviour to the Kotlin/Native's kotlin.test runner. See also: #4522, #4615. --- .../backend.native/tests/build.gradle | 60 +++++++++++++++++++ .../tests/testing/filtered_suites.kt | 57 ++++++++++++++++++ .../kotlin/native/internal/test/TestRunner.kt | 8 +-- 3 files changed, 119 insertions(+), 6 deletions(-) create mode 100644 kotlin-native/backend.native/tests/testing/filtered_suites.kt diff --git a/kotlin-native/backend.native/tests/build.gradle b/kotlin-native/backend.native/tests/build.gradle index e15d7ed564c..f14ae027cb1 100644 --- a/kotlin-native/backend.native/tests/build.gradle +++ b/kotlin-native/backend.native/tests/build.gradle @@ -3521,6 +3521,66 @@ standaloneTest("testing_filters") { } } +standaloneTest("testing_filtered_suites") { + source = "testing/filtered_suites.kt" + flags = ["-tr", "-ea"] + + def filters = [ + ["Filtered_suitesKt.*"], // filter out a class. + ["A.*"], // filter out a top-level suite. + ["*.common"], // run a test from all suites -> all hooks executed. + ["Ignored.*"], // an ignored suite -> no hooks executed. + ["A.ignored"] // a suite with only ignored tests. + // TODO: Currently we execute hooks while other frameworks don't. Fix it. + ] + def expectedHooks = [ + ["Filtered_suitesKt.before", "Filtered_suitesKt.after"], + ["A.before", "A.after"], + ["A.before", "A.after", "Filtered_suitesKt.before", "Filtered_suitesKt.after"], + [], + ["A.before", "A.after"], // TODO: fix it. + ] + + multiRuns = true + multiArguments = filters.collect { + def filter = it.collect { "kotlin.test.tests.$it" }.join(",") + ["--ktest_gradle_filter=$filter", "--ktest_logger=SIMPLE"] + } + outputChecker = { String output -> + // The first chunk is empty - drop it. + def outputs = output.split("Starting testing\n").drop(1) + if (outputs.size() != expectedHooks.size()) { + println("Incorrect number of test runs. Expected: ${expectedHooks.size()}, actual: ${outputs.size()}") + return false + } + + // Check the correct set of hooks was executed on each run. + for (int i = 0; i < outputs.size(); i++) { + def actual = outputs[i].split('\n') + .findAll { it.startsWith("Hook:") } + .collect { it.replace("Hook: ", "") } + def expected = expectedHooks[i] + + if (actual.size() != expected.size()) { + println("Incorrect number of executed hooks for run #$i. Expected: ${expected.size()}. Actual: ${actual.size()}") + println("Expected hooks: $expected") + println("Actual hooks: $actual") + return false + } + + for (expectedHook in expected) { + if (!actual.contains(expectedHook)) { + println("Expected hook wasn't executed for run #$i: $expectedHook") + println("Expected hooks: $expected") + println("Actual hooks: $actual") + return false + } + } + } + return true + } +} + // Check that stacktraces and ignored suite are correctly reported in the TC logger. standaloneTest("testing_stacktrace") { source = "testing/stacktrace.kt" diff --git a/kotlin-native/backend.native/tests/testing/filtered_suites.kt b/kotlin-native/backend.native/tests/testing/filtered_suites.kt new file mode 100644 index 00000000000..3b8a0fd66f5 --- /dev/null +++ b/kotlin-native/backend.native/tests/testing/filtered_suites.kt @@ -0,0 +1,57 @@ +package kotlin.test.tests + +import kotlin.test.* + +private fun hook(message: String) { + print("Hook: ") + println(message) +} + +class A { + @Test + fun foo() {} + + @Test + fun common() {} + + @Ignore + @Test + fun ignored() {} + + companion object { + @BeforeClass + fun before() = hook("A.before") + + @AfterClass + fun after() = hook("A.after") + } +} + +@Ignore +class Ignored { + @Test + fun bar() {} + + @Test + fun common() {} + + companion object { + @BeforeClass + fun before() = hook("Ignored.before") + + @AfterClass + fun after() = hook("Ignored.after") + } +} + +@BeforeClass +fun before() = hook("Filtered_suitesKt.before") + +@AfterClass +fun after() = hook("Filtered_suitesKt.after") + +@Test +fun baz() {} + +@Test +fun common() {} diff --git a/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestRunner.kt b/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestRunner.kt index 8d580aa5a86..d254458da16 100644 --- a/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestRunner.kt +++ b/kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestRunner.kt @@ -16,7 +16,6 @@ internal class TestRunner(val suites: List, args: Array) { private var logger: TestLogger = GTestLogger() private var runTests = true private var useExitCode = true - private var reportExcludedTestSuites = true var iterations = 1 private set var exitCode = 0 @@ -38,7 +37,6 @@ internal class TestRunner(val suites: List, args: Array) { logger.log(help); runTests = false } "--ktest_no_exit_code" -> useExitCode = false - "--ktest_no_excluded_test_suites" -> reportExcludedTestSuites = false else -> throw IllegalArgumentException("Unknown option: $it\n$help") } 2 -> { @@ -219,9 +217,6 @@ internal class TestRunner(val suites: List, args: Array) { |--ktest_logger=GTEST|TEAMCITY|SIMPLE|SILENT - Use the specified output format. The default one is GTEST. | |--ktest_no_exit_code - Don't return a non-zero exit code if there are failing tests. - | - |--ktest_no_excluded_test_suites - Don't report test suites that don't match the filter. - | Has no effect when filter is not specified. """.trimMargin() private inline fun sendToListeners(event: TestListener.() -> Unit) { @@ -259,7 +254,8 @@ internal class TestRunner(val suites: List, args: Array) { if (it.ignored) { sendToListeners { ignoreSuite(it) } } else { - if (!reportExcludedTestSuites && it.size == 0) { + // Do not run filtered out suites. + if (it.size == 0) { return@forEach } sendToListeners { startSuite(it) }