Test runner: Add tests for Gradle test filter

backend.native/tests/build.gradle

@@ -2880,6 +2880,106 @@ task testing_library_usage(type: RunStandaloneKonanTest) {
     arguments = ['--ktest_logger=SILENT']
 }
 
+class Filter {
+    List<String> positive
+    List<String> negative
+    List<String> expectedTests
+
+    Filter(List<String> positive, List<String> negative, List<String> expectedTests) {
+        this.positive = positive
+        this.negative = negative
+        this.expectedTests = expectedTests
+    }
+
+    List<String> args() {
+        List<String> result = []
+        if (positive != null && !positive.isEmpty()) {
+            result += "--ktest_gradle_filter=${asListOfPatterns(positive)}"
+        }
+        if (negative != null && !negative.isEmpty()) {
+            result += "--ktest_negative_gradle_filter=${asListOfPatterns(negative)}"
+        }
+        return result
+    }
+
+    static private String asListOfPatterns(List<String> input) {
+        return input.collect { "kotlin.test.tests.$it" }.join(",")
+    }
+}
+
+task testing_filters(type: RunStandaloneKonanTest) {
+    source = "testing/filters.kt"
+    flags = ['-tr', '-ea']
+    def filters = [
+        new Filter(["A.foo1", "B", "FiltersKt.foo1"], [], ["A.foo1", "B.foo1", "B.foo2", "B.bar", "FiltersKt.foo1"]),
+        new Filter([], ["A.foo1", "B", "FiltersKt.foo1"], ["A.foo2", "A.bar", "FiltersKt.foo2", "FiltersKt.bar"]),
+        new Filter(["A", "FiltersKt"], ["A.foo1", "FiltersKt.foo1"], ["A.foo2", "A.bar", "FiltersKt.foo2", "FiltersKt.bar"]),
+        new Filter(["A.foo*", "B.*"], [], ["A.foo1", "A.foo2", "B.foo1", "B.foo2", "B.bar"]),
+        new Filter([], ["A.foo*", "B.*"], ["A.bar", "FiltersKt.foo1", "FiltersKt.foo2", "FiltersKt.bar"]),
+        new Filter(["*.foo*"], ["B"], ["A.foo1", "A.foo2", "FiltersKt.foo1", "FiltersKt.foo2"]),
+        new Filter(["A"], ["*.foo*"], ["A.bar"])
+    ]
+    multiRuns = true
+    multiArguments = filters.collect { it.args() + '--ktest_logger=SIMPLE' }
+    outputChecker = { String output ->
+        // The first chunk is empty - drop it.
+        def outputs = output.split("Starting testing\n").drop(1)
+        if (outputs.size() != filters.size()) {
+            println("Incorrect number of test runs. Expected: ${filters.size()}, actual: ${outputs.size()}")
+            return false
+        }
+
+        // Check the correct set of tests was executed in each run.
+        for (int i = 0; i < outputs.size(); i++) {
+            def actualMessages = outputs[i].split('\n').findAll { it.startsWith("Passed: ") }
+            def expectedMessages = filters[i].expectedTests.collect { String test ->
+                def method = test.substring(test.lastIndexOf('.') + 1)
+                def suite = test.substring(0, test.lastIndexOf('.'))
+                "Passed: $method (kotlin.test.tests.$suite)".toString()
+            }
+
+            if (actualMessages.size() != expectedMessages.size()) {
+                println("Incorrect number of tests executed for filters[$i]. Expected: ${expectedMessages.size()}. Actual: ${actualMessages.size()}")
+                return false
+            }
+
+            for (message in expectedMessages) {
+                if (!actualMessages.contains(message)) {
+                    println("Test run output for filters[$i] doesn't contain the expected message '$message'")
+                    return false
+                }
+            }
+        }
+        return true
+    }
+}
+
+// Check that stacktraces and ignored suite are correctly reported in the TC logger.
+task testing_stacktrace(type: RunStandaloneKonanTest) {
+    source = "testing/stacktrace.kt"
+    flags = ['-tr', '-ea']
+    arguments = ["--ktest_logger=TEAMCITY", "--ktest_no_exit_code"]
+    // This test prints TeamCity service messages about failed test causing false negative failure at CI.
+    // So we need to suppress printing these messages to stdout.
+    printOutput = false
+    outputChecker = { String output ->
+        def ignoredOutput = """\
+            |##teamcity[testSuiteStarted name='kotlin.test.tests.Ignored' locationHint='ktest:suite://kotlin.test.tests.Ignored']
+            |##teamcity[testIgnored name='foo']
+            |##teamcity[testSuiteFinished name='kotlin.test.tests.Ignored']""".stripMargin()
+        def failedOutput = """\
+            |##teamcity[testSuiteStarted name='kotlin.test.tests.Failed' locationHint='ktest:suite://kotlin.test.tests.Failed']
+            |##teamcity[testStarted name='bar' locationHint='ktest:test://kotlin.test.tests.Failed.bar']
+            |##teamcity[testFailed name='bar' message='Bar' details='kotlin.Exception: Bar|n""".stripMargin()
+
+        def shownInnerException = output.readLines()
+                .find { it.startsWith("##teamcity[testFailed name='bar'") }
+                ?.contains("Caused by: kotlin.Exception: Baz") ?: false
+
+        return output.contains(ignoredOutput) && output.contains(failedOutput) && shownInnerException
+    }
+}
+
 // Just check that the driver is able to produce runnable binaries.
 task driver0(type: RunDriverKonanTest) {
     goldValue = "Hello, world!\n"

backend.native/tests/testing/filters.kt

@@ -0,0 +1,34 @@
+package kotlin.test.tests
+
+import kotlin.test.*
+
+class A {
+    @Test
+    fun foo1() {}
+
+    @Test
+    fun foo2() {}
+
+    @Test
+    fun bar() {}
+}
+
+class B {
+    @Test
+    fun foo1() {}
+
+    @Test
+    fun foo2() {}
+
+    @Test
+    fun bar() {}
+}
+
+@Test
+fun foo1() {}
+
+@Test
+fun foo2() {}
+
+@Test
+fun bar() {}

backend.native/tests/testing/stacktrace.kt

@@ -0,0 +1,24 @@
+package kotlin.test.tests
+
+import kotlin.test.*
+
+@Ignore
+class Ignored {
+    @Test
+    fun foo() {}
+}
+
+class Failed {
+    @Test
+    fun bar() {
+        try {
+            baz()
+        } catch(e: Exception) {
+            throw Exception("Bar", e)
+        }
+    }
+
+    fun baz() {
+        throw Exception("Baz")
+    }
+}

buildSrc/plugins/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy

@@ -42,6 +42,7 @@ abstract class KonanTest extends JavaExec {
     String goldValue = null
     // Checks test's output against gold value and returns true if the output matches the expectation
     Function<String, Boolean> outputChecker = { str -> (goldValue == null || goldValue == str) }
+    boolean printOutput = true
     String testData = null
     int expectedExitStatus = 0
     List<String> arguments = null
@@ -365,7 +366,9 @@ abstract class KonanTest extends JavaExec {
             }
         }
         def result = compilerMessagesText + out.toString("UTF-8")
-        println(result)
+        if (printOutput) {
+            println(result)
+        }
         result = result.replace(System.lineSeparator(), "\n")
         def goldValueMismatch = !outputChecker.apply(result)
         if (goldValueMismatch) {