[kotlin.test] Don't run filtered out test suites by default

Popular frameworks such as JUnit, TestNG and Google Test do not
run test suites whose tests were fully filtered out. Thus
@BeforeClass/@AfterClass of such suites are not executed and the
suites are not reported in a test report.

This patch introduces the same behaviour to the Kotlin/Native's
kotlin.test runner.

See also: #4522, #4615.

kotlin-native/backend.native/tests/build.gradle

@@ -3521,6 +3521,66 @@ standaloneTest("testing_filters") {
     }
 }
 
+standaloneTest("testing_filtered_suites") {
+    source = "testing/filtered_suites.kt"
+    flags = ["-tr", "-ea"]
+
+    def filters = [
+            ["Filtered_suitesKt.*"],    // filter out a class.
+            ["A.*"],                    // filter out a top-level suite.
+            ["*.common"],               // run a test from all suites -> all hooks executed.
+            ["Ignored.*"],              // an ignored suite -> no hooks executed.
+            ["A.ignored"]               // a suite with only ignored tests.
+                                        //      TODO: Currently we execute hooks while other frameworks don't. Fix it.
+    ]
+    def expectedHooks = [
+            ["Filtered_suitesKt.before", "Filtered_suitesKt.after"],
+            ["A.before", "A.after"],
+            ["A.before", "A.after", "Filtered_suitesKt.before", "Filtered_suitesKt.after"],
+            [],
+            ["A.before", "A.after"],  // TODO: fix it.
+    ]
+
+    multiRuns = true
+    multiArguments = filters.collect {
+        def filter = it.collect { "kotlin.test.tests.$it" }.join(",")
+        ["--ktest_gradle_filter=$filter", "--ktest_logger=SIMPLE"]
+    }
+    outputChecker = { String output ->
+        // The first chunk is empty - drop it.
+        def outputs = output.split("Starting testing\n").drop(1)
+        if (outputs.size() != expectedHooks.size()) {
+            println("Incorrect number of test runs. Expected: ${expectedHooks.size()}, actual: ${outputs.size()}")
+            return false
+        }
+
+        // Check the correct set of hooks was executed on each run.
+        for (int i = 0; i < outputs.size(); i++) {
+            def actual = outputs[i].split('\n')
+                    .findAll { it.startsWith("Hook:") }
+                    .collect { it.replace("Hook: ", "") }
+            def expected = expectedHooks[i]
+
+            if (actual.size() != expected.size()) {
+                println("Incorrect number of executed hooks for run #$i. Expected: ${expected.size()}. Actual: ${actual.size()}")
+                println("Expected hooks: $expected")
+                println("Actual hooks: $actual")
+                return false
+            }
+
+            for (expectedHook in expected) {
+                if (!actual.contains(expectedHook)) {
+                    println("Expected hook wasn't executed for run #$i: $expectedHook")
+                    println("Expected hooks: $expected")
+                    println("Actual hooks: $actual")
+                    return false
+                }
+            }
+        }
+        return true
+    }
+}
+
 // Check that stacktraces and ignored suite are correctly reported in the TC logger.
 standaloneTest("testing_stacktrace") {
     source = "testing/stacktrace.kt"

kotlin-native/backend.native/tests/testing/filtered_suites.kt

@@ -0,0 +1,57 @@
+package kotlin.test.tests
+
+import kotlin.test.*
+
+private fun hook(message: String) {
+    print("Hook: ")
+    println(message)
+}
+
+class A {
+    @Test
+    fun foo() {}
+
+    @Test
+    fun common() {}
+
+    @Ignore
+    @Test
+    fun ignored() {}
+
+    companion object {
+        @BeforeClass
+        fun before() = hook("A.before")
+
+        @AfterClass
+        fun after() = hook("A.after")
+    }
+}
+
+@Ignore
+class Ignored {
+    @Test
+    fun bar() {}
+
+    @Test
+    fun common() {}
+
+    companion object {
+        @BeforeClass
+        fun before() = hook("Ignored.before")
+
+        @AfterClass
+        fun after() = hook("Ignored.after")
+    }
+}
+
+@BeforeClass
+fun before() = hook("Filtered_suitesKt.before")
+
+@AfterClass
+fun after() = hook("Filtered_suitesKt.after")
+
+@Test
+fun baz() {}
+
+@Test
+fun common() {}

kotlin-native/runtime/src/main/kotlin/kotlin/native/internal/test/TestRunner.kt

@@ -16,7 +16,6 @@ internal class TestRunner(val suites: List<TestSuite>, args: Array<String>) {
     private var logger: TestLogger = GTestLogger()
     private var runTests = true
     private var useExitCode = true
-    private var reportExcludedTestSuites = true
     var iterations = 1
         private set
     var exitCode = 0
@@ -38,7 +37,6 @@ internal class TestRunner(val suites: List<TestSuite>, args: Array<String>) {
                         logger.log(help); runTests = false
                     }
                     "--ktest_no_exit_code" -> useExitCode = false
-                    "--ktest_no_excluded_test_suites" -> reportExcludedTestSuites = false
                     else -> throw IllegalArgumentException("Unknown option: $it\n$help")
                 }
                 2 -> {
@@ -219,9 +217,6 @@ internal class TestRunner(val suites: List<TestSuite>, args: Array<String>) {
             |--ktest_logger=GTEST|TEAMCITY|SIMPLE|SILENT         - Use the specified output format. The default one is GTEST.
             |
             |--ktest_no_exit_code                                - Don't return a non-zero exit code if there are failing tests.
-            |
-            |--ktest_no_excluded_test_suites                     - Don't report test suites that don't match the filter.
-            |                                                      Has no effect when filter is not specified.
         """.trimMargin()
 
     private inline fun sendToListeners(event: TestListener.() -> Unit) {
@@ -259,7 +254,8 @@ internal class TestRunner(val suites: List<TestSuite>, args: Array<String>) {
                 if (it.ignored) {
                     sendToListeners { ignoreSuite(it) }
                 } else {
-                    if (!reportExcludedTestSuites && it.size == 0) {
+                    // Do not run filtered out suites.
+                    if (it.size == 0) {
                         return@forEach
                     }
                     sendToListeners { startSuite(it) }