Gradle, tests: Improve tests for running simulator unit-tests

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/NewMultiplatformIT.kt

@@ -4,6 +4,8 @@
  */
 package org.jetbrains.kotlin.gradle
 
+import org.gradle.api.logging.LogLevel
+import org.jdom.input.SAXBuilder
 import org.jetbrains.kotlin.gradle.internals.*
 import org.jetbrains.kotlin.gradle.plugin.ProjectLocalConfigurations
 import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinJvmWithJavaTargetPreset
@@ -20,6 +22,7 @@ import org.jetbrains.kotlin.konan.target.HostManager
 import org.junit.Assert
 import org.junit.Ignore
 import org.junit.Test
+import java.io.File
 import java.util.jar.JarFile
 import java.util.zip.ZipFile
 import kotlin.test.assertEquals
@@ -1277,9 +1280,20 @@ class NewMultiplatformIT : BaseGradleIT() {
             }
         }
     }
+    
+    private fun getBootedSimulators(workingDirectory: File): Set<String>? =
+        if (HostManager.hostIsMac) {
+            val simulators = runProcess(listOf("xcrun", "simctl", "list"), workingDirectory, System.getenv()).also {
+                assertTrue(it.isSuccessful, "xcrun exection failed")
+            }.output
+
+            simulators.split('\n').filter { it.contains("(Booted)") }.map { it.trim() }.toSet()
+        } else {
+            null
+        }
 
     @Test
-    fun testNativeTests() = with(Project("new-mpp-native-tests", gradleVersion)) {
+    fun testNativeTests() = with(Project("new-mpp-native-tests", gradleVersion, minLogLevel = LogLevel.INFO)) {
         val testTasks = listOf("macos64Test", "linux64Test", "mingw64Test", "iosTest")
         val hostTestTask = "${nativeHostTargetName}Test"
 
@@ -1288,6 +1302,8 @@ class NewMultiplatformIT : BaseGradleIT() {
         val defaultOutputFile = "build/bin/$nativeHostTargetName/debugTest/test.$suffix"
         val anotherOutputFile = "build/bin/$nativeHostTargetName/anotherDebugTest/another.$suffix"
 
+        val hostIsMac = HostManager.hostIsMac
+
         build("tasks") {
             assertSuccessful()
             testTasks.forEach {
@@ -1296,6 +1312,7 @@ class NewMultiplatformIT : BaseGradleIT() {
             }
         }
 
+        // Perform all following checks in a single test to avoid running the K/N compiler several times.
         // Check that tests are not built during the ":assemble" execution
         build("assemble") {
             assertSuccessful()
@@ -1303,27 +1320,91 @@ class NewMultiplatformIT : BaseGradleIT() {
             assertNoSuchFile(anotherOutputFile)
         }
 
+        val testsToExecute = mutableListOf(":$hostTestTask")
+        if (hostIsMac) {
+            testsToExecute.add(":iosTest")
+        }
+        val testsToSkip = testTasks.map { ":$it" } - testsToExecute
+
+        // Store currently booted simulators to check that they don't leak (MacOS only).
+        val bootedSimulatorsBefore = getBootedSimulators(projectDir)
+
+        // Check the case when all tests pass.
         build("check") {
             assertSuccessful()
 
-            val testsToExecute = mutableListOf(":$hostTestTask")
-            if (HostManager.hostIsMac) {
-                testsToExecute.add(":iosTest")
-            }
-            val testsToSkip = testTasks.map { ":$it" } - testsToExecute
-
             assertTasksExecuted(*testsToExecute.toTypedArray())
             assertTasksSkipped(*testsToSkip.toTypedArray())
 
+            assertContainsRegex("org\\.foo\\.test\\.TestKt\\.fooTest\\s+PASSED".toRegex())
+            assertContainsRegex("org\\.foo\\.test\\.TestKt\\.barTest\\s+PASSED".toRegex())
+
             assertFileExists(defaultOutputFile)
+        }
+
+        // Check simulator process leaking.
+        val bootedSimulatorsAfter = getBootedSimulators(projectDir)
+        assertEquals(bootedSimulatorsBefore, bootedSimulatorsAfter)
+
+        // Check the case with failed tests.
+        projectDir.resolve("src/commonTest/kotlin/test.kt").appendText(
+            """
+                @Test
+                fun fail() {
+                    error("FAILURE!")
+                }
+            """.trimIndent()
+        )
+        build("check") {
+            assertFailed()
+
+            assertTasksFailed(":allTests")
+            // In the aggregation report mode platform-specific tasks
+            // are executed successfully even if there are failing tests.
+            assertTasksExecuted(*testsToExecute.toTypedArray())
+            assertTasksSkipped(*testsToSkip.toTypedArray())
+
+            assertContainsRegex("org\\.foo\\.test\\.TestKt\\.fail\\s+FAILED".toRegex())
+        }
+
+        // Check that individual test reports are created correctly.
+        build("check", "-Pkotlin.tests.individualTaskReports=true", "--continue") {
+            assertFailed()
+
+            // In the individual report mode platform-specific tasks
+            // fail if there are failing tests.
+            assertTasksFailed(*testsToExecute.toTypedArray())
+            assertTasksSkipped(*testsToSkip.toTypedArray())
+
+
+            fun assertStacktrace(taskName: String) {
+                val testReport = projectDir.resolve("build/test-results/$taskName/TEST-org.foo.test.TestKt.xml")
+                val stacktrace = SAXBuilder().build(testReport).rootElement
+                    .getChildren("testcase")
+                    .single { it.getAttribute("name").value == "fail" }
+                    .getChild("failure")
+                    .text
+                assertTrue(stacktrace.contains("""at org\.foo\.test\.fail\(.*test\.kt:24\)""".toRegex()))
+            }
+
             assertTestResults("testProject/new-mpp-native-tests/TEST-TestKt.xml", hostTestTask)
+            // K/N doesn't report line numbers correctly on Linux (see KT-35408).
+            // TODO: Uncomment when this is fixed.
+            //assertStacktrace(hostTestTask)
+            if (hostIsMac) {
+                assertTestResults("testProject/new-mpp-native-tests/TEST-TestKt.xml", "iosTest")
+                assertStacktrace("iosTest")
+            }
         }
 
         build("linkAnotherDebugTest${nativeHostTargetName}") {
             assertSuccessful()
             assertFileExists(anotherOutputFile)
         }
+    }
 
+    @Test
+    fun testNativeTestGetters() = with(Project("new-mpp-native-tests", gradleVersion)) {
         // Check that test binaries can be accessed in a buildscript.
         build("checkNewGetters") {
             assertSuccessful()

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/new-mpp-native-tests/TEST-TestKt.xml

@@ -1,9 +1,12 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <results>
-  <testsuite name="org.foo.test.TestKt" tests="2" skipped="0" failures="0" errors="0" timestamp="..." hostname="..." time="...">
+  <testsuite name="org.foo.test.TestKt" tests="3" skipped="0" failures="1" errors="0" timestamp="..." hostname="..." time="...">
     <properties/>
     <testcase name="fooTest" classname="org.foo.test.TestKt" time="..."/>
     <testcase name="barTest" classname="org.foo.test.TestKt" time="..."/>
+    <testcase name="fail" classname="org.foo.test.TestKt" time="...">
+      <failure message="..." type="kotlin.IllegalStateException">...</failure>
+    </testcase>
     <system-out><![CDATA[Foo call
 Bar call]]></system-out>
     <system-err/>

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/new-mpp-native-tests/build.gradle

@@ -48,6 +48,13 @@ kotlin {
     }
 }
 
+// Check that test events are correctly reported in CLI.
+configure([macos64Test, linux64Test, mingw64Test, iosTest]){
+    testLogging {
+        events "passed", "skipped", "failed"
+    }
+}
+
 // Check that getting a test binary in an old way fails showing the corresponding warning
 task checkOldGet {
     doLast {

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/new-mpp-native-tests/gradle.properties

@@ -1,2 +1 @@
-kotlin.tests.individualTaskReports=true
 kotlin.native.disableCompilerDaemon=true