From 33df86338b2748aca8505703a6ac12f9c4f20255 Mon Sep 17 00:00:00 2001 From: Ilya Matveev Date: Fri, 6 Dec 2019 19:40:35 +0700 Subject: [PATCH] Gradle, tests: Improve tests for running simulator unit-tests --- .../kotlin/gradle/NewMultiplatformIT.kt | 95 +++++++++++++++++-- .../new-mpp-native-tests/TEST-TestKt.xml | 5 +- .../new-mpp-native-tests/build.gradle | 7 ++ .../new-mpp-native-tests/gradle.properties | 1 - 4 files changed, 99 insertions(+), 9 deletions(-) diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/NewMultiplatformIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/NewMultiplatformIT.kt index 2583034266d..8e8b272091a 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/NewMultiplatformIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/NewMultiplatformIT.kt @@ -4,6 +4,8 @@ */ package org.jetbrains.kotlin.gradle +import org.gradle.api.logging.LogLevel +import org.jdom.input.SAXBuilder import org.jetbrains.kotlin.gradle.internals.* import org.jetbrains.kotlin.gradle.plugin.ProjectLocalConfigurations import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinJvmWithJavaTargetPreset @@ -20,6 +22,7 @@ import org.jetbrains.kotlin.konan.target.HostManager import org.junit.Assert import org.junit.Ignore import org.junit.Test +import java.io.File import java.util.jar.JarFile import java.util.zip.ZipFile import kotlin.test.assertEquals @@ -1277,9 +1280,20 @@ class NewMultiplatformIT : BaseGradleIT() { } } } + + private fun getBootedSimulators(workingDirectory: File): Set? = + if (HostManager.hostIsMac) { + val simulators = runProcess(listOf("xcrun", "simctl", "list"), workingDirectory, System.getenv()).also { + assertTrue(it.isSuccessful, "xcrun exection failed") + }.output + + simulators.split('\n').filter { it.contains("(Booted)") }.map { it.trim() }.toSet() + } else { + null + } @Test - fun testNativeTests() = with(Project("new-mpp-native-tests", gradleVersion)) { + fun testNativeTests() = with(Project("new-mpp-native-tests", gradleVersion, minLogLevel = LogLevel.INFO)) { val testTasks = listOf("macos64Test", "linux64Test", "mingw64Test", "iosTest") val hostTestTask = "${nativeHostTargetName}Test" @@ -1288,6 +1302,8 @@ class NewMultiplatformIT : BaseGradleIT() { val defaultOutputFile = "build/bin/$nativeHostTargetName/debugTest/test.$suffix" val anotherOutputFile = "build/bin/$nativeHostTargetName/anotherDebugTest/another.$suffix" + val hostIsMac = HostManager.hostIsMac + build("tasks") { assertSuccessful() testTasks.forEach { @@ -1296,6 +1312,7 @@ class NewMultiplatformIT : BaseGradleIT() { } } + // Perform all following checks in a single test to avoid running the K/N compiler several times. // Check that tests are not built during the ":assemble" execution build("assemble") { assertSuccessful() @@ -1303,27 +1320,91 @@ class NewMultiplatformIT : BaseGradleIT() { assertNoSuchFile(anotherOutputFile) } + val testsToExecute = mutableListOf(":$hostTestTask") + if (hostIsMac) { + testsToExecute.add(":iosTest") + } + val testsToSkip = testTasks.map { ":$it" } - testsToExecute + + // Store currently booted simulators to check that they don't leak (MacOS only). + val bootedSimulatorsBefore = getBootedSimulators(projectDir) + + // Check the case when all tests pass. build("check") { assertSuccessful() - val testsToExecute = mutableListOf(":$hostTestTask") - if (HostManager.hostIsMac) { - testsToExecute.add(":iosTest") - } - val testsToSkip = testTasks.map { ":$it" } - testsToExecute - assertTasksExecuted(*testsToExecute.toTypedArray()) assertTasksSkipped(*testsToSkip.toTypedArray()) + assertContainsRegex("org\\.foo\\.test\\.TestKt\\.fooTest\\s+PASSED".toRegex()) + assertContainsRegex("org\\.foo\\.test\\.TestKt\\.barTest\\s+PASSED".toRegex()) + assertFileExists(defaultOutputFile) + } + + // Check simulator process leaking. + val bootedSimulatorsAfter = getBootedSimulators(projectDir) + assertEquals(bootedSimulatorsBefore, bootedSimulatorsAfter) + + // Check the case with failed tests. + projectDir.resolve("src/commonTest/kotlin/test.kt").appendText( + """ + @Test + fun fail() { + error("FAILURE!") + } + """.trimIndent() + ) + build("check") { + assertFailed() + + assertTasksFailed(":allTests") + // In the aggregation report mode platform-specific tasks + // are executed successfully even if there are failing tests. + assertTasksExecuted(*testsToExecute.toTypedArray()) + assertTasksSkipped(*testsToSkip.toTypedArray()) + + assertContainsRegex("org\\.foo\\.test\\.TestKt\\.fail\\s+FAILED".toRegex()) + } + + // Check that individual test reports are created correctly. + build("check", "-Pkotlin.tests.individualTaskReports=true", "--continue") { + assertFailed() + + // In the individual report mode platform-specific tasks + // fail if there are failing tests. + assertTasksFailed(*testsToExecute.toTypedArray()) + assertTasksSkipped(*testsToSkip.toTypedArray()) + + + fun assertStacktrace(taskName: String) { + val testReport = projectDir.resolve("build/test-results/$taskName/TEST-org.foo.test.TestKt.xml") + val stacktrace = SAXBuilder().build(testReport).rootElement + .getChildren("testcase") + .single { it.getAttribute("name").value == "fail" } + .getChild("failure") + .text + assertTrue(stacktrace.contains("""at org\.foo\.test\.fail\(.*test\.kt:24\)""".toRegex())) + } + assertTestResults("testProject/new-mpp-native-tests/TEST-TestKt.xml", hostTestTask) + // K/N doesn't report line numbers correctly on Linux (see KT-35408). + // TODO: Uncomment when this is fixed. + //assertStacktrace(hostTestTask) + if (hostIsMac) { + assertTestResults("testProject/new-mpp-native-tests/TEST-TestKt.xml", "iosTest") + assertStacktrace("iosTest") + } } build("linkAnotherDebugTest${nativeHostTargetName}") { assertSuccessful() assertFileExists(anotherOutputFile) } + } + @Test + fun testNativeTestGetters() = with(Project("new-mpp-native-tests", gradleVersion)) { // Check that test binaries can be accessed in a buildscript. build("checkNewGetters") { assertSuccessful() diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/new-mpp-native-tests/TEST-TestKt.xml b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/new-mpp-native-tests/TEST-TestKt.xml index c3ed56e91d0..41626ade0ba 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/new-mpp-native-tests/TEST-TestKt.xml +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/new-mpp-native-tests/TEST-TestKt.xml @@ -1,9 +1,12 @@ - + + + ... + diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/new-mpp-native-tests/build.gradle b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/new-mpp-native-tests/build.gradle index 01c12c80b58..833f3c70dd9 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/new-mpp-native-tests/build.gradle +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/new-mpp-native-tests/build.gradle @@ -48,6 +48,13 @@ kotlin { } } +// Check that test events are correctly reported in CLI. +configure([macos64Test, linux64Test, mingw64Test, iosTest]){ + testLogging { + events "passed", "skipped", "failed" + } +} + // Check that getting a test binary in an old way fails showing the corresponding warning task checkOldGet { doLast { diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/new-mpp-native-tests/gradle.properties b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/new-mpp-native-tests/gradle.properties index 682e6a5662e..1c0ad1195e2 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/new-mpp-native-tests/gradle.properties +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/new-mpp-native-tests/gradle.properties @@ -1,2 +1 @@ -kotlin.tests.individualTaskReports=true kotlin.native.disableCompilerDaemon=true \ No newline at end of file