diff --git a/backend.native/tests/build.gradle b/backend.native/tests/build.gradle index e61e371bd9f..90b2a2faf37 100644 --- a/backend.native/tests/build.gradle +++ b/backend.native/tests/build.gradle @@ -2880,6 +2880,106 @@ task testing_library_usage(type: RunStandaloneKonanTest) { arguments = ['--ktest_logger=SILENT'] } +class Filter { + List positive + List negative + List expectedTests + + Filter(List positive, List negative, List expectedTests) { + this.positive = positive + this.negative = negative + this.expectedTests = expectedTests + } + + List args() { + List result = [] + if (positive != null && !positive.isEmpty()) { + result += "--ktest_gradle_filter=${asListOfPatterns(positive)}" + } + if (negative != null && !negative.isEmpty()) { + result += "--ktest_negative_gradle_filter=${asListOfPatterns(negative)}" + } + return result + } + + static private String asListOfPatterns(List input) { + return input.collect { "kotlin.test.tests.$it" }.join(",") + } +} + +task testing_filters(type: RunStandaloneKonanTest) { + source = "testing/filters.kt" + flags = ['-tr', '-ea'] + def filters = [ + new Filter(["A.foo1", "B", "FiltersKt.foo1"], [], ["A.foo1", "B.foo1", "B.foo2", "B.bar", "FiltersKt.foo1"]), + new Filter([], ["A.foo1", "B", "FiltersKt.foo1"], ["A.foo2", "A.bar", "FiltersKt.foo2", "FiltersKt.bar"]), + new Filter(["A", "FiltersKt"], ["A.foo1", "FiltersKt.foo1"], ["A.foo2", "A.bar", "FiltersKt.foo2", "FiltersKt.bar"]), + new Filter(["A.foo*", "B.*"], [], ["A.foo1", "A.foo2", "B.foo1", "B.foo2", "B.bar"]), + new Filter([], ["A.foo*", "B.*"], ["A.bar", "FiltersKt.foo1", "FiltersKt.foo2", "FiltersKt.bar"]), + new Filter(["*.foo*"], ["B"], ["A.foo1", "A.foo2", "FiltersKt.foo1", "FiltersKt.foo2"]), + new Filter(["A"], ["*.foo*"], ["A.bar"]) + ] + multiRuns = true + multiArguments = filters.collect { it.args() + '--ktest_logger=SIMPLE' } + outputChecker = { String output -> + // The first chunk is empty - drop it. + def outputs = output.split("Starting testing\n").drop(1) + if (outputs.size() != filters.size()) { + println("Incorrect number of test runs. Expected: ${filters.size()}, actual: ${outputs.size()}") + return false + } + + // Check the correct set of tests was executed in each run. + for (int i = 0; i < outputs.size(); i++) { + def actualMessages = outputs[i].split('\n').findAll { it.startsWith("Passed: ") } + def expectedMessages = filters[i].expectedTests.collect { String test -> + def method = test.substring(test.lastIndexOf('.') + 1) + def suite = test.substring(0, test.lastIndexOf('.')) + "Passed: $method (kotlin.test.tests.$suite)".toString() + } + + if (actualMessages.size() != expectedMessages.size()) { + println("Incorrect number of tests executed for filters[$i]. Expected: ${expectedMessages.size()}. Actual: ${actualMessages.size()}") + return false + } + + for (message in expectedMessages) { + if (!actualMessages.contains(message)) { + println("Test run output for filters[$i] doesn't contain the expected message '$message'") + return false + } + } + } + return true + } +} + +// Check that stacktraces and ignored suite are correctly reported in the TC logger. +task testing_stacktrace(type: RunStandaloneKonanTest) { + source = "testing/stacktrace.kt" + flags = ['-tr', '-ea'] + arguments = ["--ktest_logger=TEAMCITY", "--ktest_no_exit_code"] + // This test prints TeamCity service messages about failed test causing false negative failure at CI. + // So we need to suppress printing these messages to stdout. + printOutput = false + outputChecker = { String output -> + def ignoredOutput = """\ + |##teamcity[testSuiteStarted name='kotlin.test.tests.Ignored' locationHint='ktest:suite://kotlin.test.tests.Ignored'] + |##teamcity[testIgnored name='foo'] + |##teamcity[testSuiteFinished name='kotlin.test.tests.Ignored']""".stripMargin() + def failedOutput = """\ + |##teamcity[testSuiteStarted name='kotlin.test.tests.Failed' locationHint='ktest:suite://kotlin.test.tests.Failed'] + |##teamcity[testStarted name='bar' locationHint='ktest:test://kotlin.test.tests.Failed.bar'] + |##teamcity[testFailed name='bar' message='Bar' details='kotlin.Exception: Bar|n""".stripMargin() + + def shownInnerException = output.readLines() + .find { it.startsWith("##teamcity[testFailed name='bar'") } + ?.contains("Caused by: kotlin.Exception: Baz") ?: false + + return output.contains(ignoredOutput) && output.contains(failedOutput) && shownInnerException + } +} + // Just check that the driver is able to produce runnable binaries. task driver0(type: RunDriverKonanTest) { goldValue = "Hello, world!\n" diff --git a/backend.native/tests/testing/filters.kt b/backend.native/tests/testing/filters.kt new file mode 100644 index 00000000000..c63b196c564 --- /dev/null +++ b/backend.native/tests/testing/filters.kt @@ -0,0 +1,34 @@ +package kotlin.test.tests + +import kotlin.test.* + +class A { + @Test + fun foo1() {} + + @Test + fun foo2() {} + + @Test + fun bar() {} +} + +class B { + @Test + fun foo1() {} + + @Test + fun foo2() {} + + @Test + fun bar() {} +} + +@Test +fun foo1() {} + +@Test +fun foo2() {} + +@Test +fun bar() {} \ No newline at end of file diff --git a/backend.native/tests/testing/stacktrace.kt b/backend.native/tests/testing/stacktrace.kt new file mode 100644 index 00000000000..deb859a1398 --- /dev/null +++ b/backend.native/tests/testing/stacktrace.kt @@ -0,0 +1,24 @@ +package kotlin.test.tests + +import kotlin.test.* + +@Ignore +class Ignored { + @Test + fun foo() {} +} + +class Failed { + @Test + fun bar() { + try { + baz() + } catch(e: Exception) { + throw Exception("Bar", e) + } + } + + fun baz() { + throw Exception("Baz") + } +} diff --git a/buildSrc/plugins/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy b/buildSrc/plugins/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy index ffd6177ba3e..bfd431714d7 100644 --- a/buildSrc/plugins/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy +++ b/buildSrc/plugins/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy @@ -42,6 +42,7 @@ abstract class KonanTest extends JavaExec { String goldValue = null // Checks test's output against gold value and returns true if the output matches the expectation Function outputChecker = { str -> (goldValue == null || goldValue == str) } + boolean printOutput = true String testData = null int expectedExitStatus = 0 List arguments = null @@ -365,7 +366,9 @@ abstract class KonanTest extends JavaExec { } } def result = compilerMessagesText + out.toString("UTF-8") - println(result) + if (printOutput) { + println(result) + } result = result.replace(System.lineSeparator(), "\n") def goldValueMismatch = !outputChecker.apply(result) if (goldValueMismatch) {