[K/N][test] Improve simulator executor

Makes executor be able to find and download necessary runtimes and
create device with appropriate runtime available. Replaces device names
with identifiers


Co-authored-by: Alexander Shabalin <Alexander.Shabalin@jetbrains.com>

Merge-request: KT-MR-7919
Merged-by: Pavel Punegov <Pavel.Punegov@jetbrains.com>

kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/ExecutorService.kt

@@ -99,8 +99,9 @@ fun create(project: Project): ExecutorService {
         configurables is WasmConfigurables -> WasmExecutor(configurables).service(project)
         configurables is ConfigurablesWithEmulator && testTarget != HostManager.host -> EmulatorExecutor(configurables).service(project)
         configurables is AppleConfigurables && configurables.targetTriple.isSimulator -> XcodeSimulatorExecutor(configurables).apply {
+            // Property can specify device identifier to be run on. For example, `com.apple.CoreSimulator.SimDeviceType.iPhone-11`
             project.findProperty("iosDevice")?.toString()?.let {
-                deviceName = it
+                deviceId = it
             }
         }.service(project)
         supportsRunningTestsOnDevice(testTarget) -> deviceLauncher(project)

kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/XcRunRuntimeUtils.kt

@@ -20,26 +20,57 @@ private fun compareStringsAsVersions(version1: String, version2: String): Int {
     return splitVersion1.size.compareTo(splitVersion2.size)
 }
 
-/**
+internal fun simulatorOsName(family: Family): String {
- * Returns parsed output of `xcrun simctl list runtimes -j`.
+    return when (family) {
- */
-private fun Xcode.getSimulatorRuntimeDescriptors(): List<SimulatorRuntimeDescriptor> = gson.fromJson(simulatorRuntimes, ListRuntimesReport::class.java).runtimes
-
-/**
- * Returns first available simulator runtime for [target] with at least [osMinVersion] OS version.
- * */
-fun Xcode.getLatestSimulatorRuntimeFor(family: Family, osMinVersion: String): SimulatorRuntimeDescriptor? {
-    val osName = when (family) {
         Family.IOS -> "iOS"
         Family.WATCHOS -> "watchOS"
         Family.TVOS -> "tvOS"
         else -> error("Unexpected simulator OS: $family")
     }
-    return getSimulatorRuntimeDescriptors().firstOrNull {
+}
+
+/**
+ * Returns parsed output of `xcrun simctl list runtimes -j`.
+ */
+private fun Xcode.getSimulatorRuntimeDescriptors(): List<SimulatorRuntimeDescriptor> =
+        gson.fromJson(simulatorRuntimes, ListRuntimesReport::class.java).runtimes
+private fun getSimulatorRuntimeDescriptors(json: String): List<SimulatorRuntimeDescriptor> =
+        gson.fromJson(json, ListRuntimesReport::class.java).runtimes
+
+private fun getLatestSimulatorRuntimeFor(
+        descriptors: List<SimulatorRuntimeDescriptor>,
+        family: Family,
+        osMinVersion: String
+): SimulatorRuntimeDescriptor? {
+    val osName = simulatorOsName(family)
+    return descriptors.firstOrNull {
         it.checkAvailability() && it.name.startsWith(osName) && compareStringsAsVersions(it.version, osMinVersion) >= 0
     }
 }
 
+private fun getSimulatorRuntimesFor(
+        descriptors: List<SimulatorRuntimeDescriptor>,
+        family: Family,
+        osMinVersion: String
+): List<SimulatorRuntimeDescriptor> {
+    val osName = simulatorOsName(family)
+    return descriptors.filter {
+        it.checkAvailability() && it.name.startsWith(osName) && compareStringsAsVersions(it.version, osMinVersion) >= 0
+    }
+}
+
+/**
+ * Returns first available simulator runtime for [target] with at least [osMinVersion] OS version.
+ * */
+fun Xcode.getLatestSimulatorRuntimeFor(family: Family, osMinVersion: String): SimulatorRuntimeDescriptor? =
+        getLatestSimulatorRuntimeFor(getSimulatorRuntimeDescriptors(), family, osMinVersion)
+
+fun getLatestSimulatorRuntimeFor(json: String, family: Family, osMinVersion: String): SimulatorRuntimeDescriptor? =
+        getLatestSimulatorRuntimeFor(getSimulatorRuntimeDescriptors(json), family, osMinVersion)
+
+fun getSimulatorRuntimesFor(json: String, family: Family, osMinVersion: String): List<SimulatorRuntimeDescriptor> =
+        getSimulatorRuntimesFor(getSimulatorRuntimeDescriptors(json), family, osMinVersion)
+
 // Result of `xcrun simctl list runtimes -j`.
 data class ListRuntimesReport(
         @Expose val runtimes: List<SimulatorRuntimeDescriptor>
@@ -53,7 +84,8 @@ data class SimulatorRuntimeDescriptor(
         @Expose val availability: String? = null,
         @Expose val name: String,
         @Expose val identifier: String,
-        @Expose val buildversion: String
+        @Expose val buildversion: String,
+        @Expose val supportedDeviceTypes: List<DeviceType>
 ) {
     /**
      * Different Xcode/macOS combinations give different fields that checks
@@ -65,3 +97,33 @@ data class SimulatorRuntimeDescriptor(
         return false
     }
 }
+
+data class DeviceType(
+    @Expose val bundlePath: String,
+    @Expose val name: String,
+    @Expose val identifier: String,
+    @Expose val productFamily: String
+)
+
+/**
+ * Returns map of simulator devices from the json input
+ */
+fun getSimulatorDevices(json: String): Map<String, List<SimulatorDeviceDescriptor>> =
+        gson.fromJson(json, ListDevicesReport::class.java).devices
+
+// Result of `xcrun simctl list devices -j`
+data class ListDevicesReport(
+        @Expose val devices: Map<String, List<SimulatorDeviceDescriptor>>
+)
+
+data class SimulatorDeviceDescriptor(
+        @Expose val lastBootedAt: String?,
+        @Expose val dataPath: String,
+        @Expose val dataPathSize: Long,
+        @Expose val logPath: String,
+        @Expose val udid: String,
+        @Expose val isAvailable: Boolean?,
+        @Expose val deviceTypeIdentifier: String,
+        @Expose val state: String,
+        @Expose val name: String
+)

kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/executors/XcodeSimulatorExecutor.kt

@@ -6,13 +6,15 @@
 package org.jetbrains.kotlin.executors
 
 import org.jetbrains.kotlin.konan.target.*
+import org.jetbrains.kotlin.*
 import java.io.ByteArrayOutputStream
 import java.io.File
+import java.util.logging.Logger
 
-private fun defaultDeviceName(target: KonanTarget) = when (target.family) {
+private fun defaultDeviceId(target: KonanTarget) = when (target.family) {
-    Family.TVOS -> "Apple TV 4K"
+    Family.TVOS -> "com.apple.CoreSimulator.SimDeviceType.Apple-TV-4K-4K"
-    Family.IOS -> "iPhone 11"
+    Family.IOS -> "com.apple.CoreSimulator.SimDeviceType.iPhone-14"
-    Family.WATCHOS -> "Apple Watch Series 6 " + (if (Xcode.findCurrent().version.startsWith("14")) "(40mm)" else "- 40mm")
+    Family.WATCHOS -> "com.apple.CoreSimulator.SimDeviceType.Apple-Watch-Series-6-40mm"
     else -> error("Unexpected simulation target: $target")
 }
 
@@ -33,10 +35,12 @@ private fun Executor.run(executableAbsolutePath: String, vararg args: String) =
  */
 class XcodeSimulatorExecutor(
         private val configurables: AppleConfigurables,
-        var deviceName: String = defaultDeviceName(configurables.target),
+        var deviceId: String = defaultDeviceId(configurables.target),
 ) : Executor {
     private val hostExecutor: Executor = HostExecutor()
 
+    private val logger = Logger.getLogger(this::class.java.name)
+
     private val target by configurables::target
 
     init {
@@ -62,20 +66,91 @@ class XcodeSimulatorExecutor(
         else -> error("${target.architecture} can't be used in simulator.")
     }.toTypedArray()
 
-    private var _deviceNameChecked: String? = null
+    private fun simctl(vararg args: String): String {
+        val out = hostExecutor.run("/usr/bin/xcrun", *arrayOf("simctl", *args))
+        return out.toString("UTF-8").trim()
+    }
+
+    private var deviceChecked: SimulatorDeviceDescriptor? = null
+
     private fun ensureSimulatorExists() {
-        // Already ensured that simulator for `deviceName` exists.
+        // Already ensured that simulator for `deviceId` exists.
-        if (deviceName == _deviceNameChecked)
+        if (deviceId == deviceChecked?.deviceTypeIdentifier) {
+            logger.info("Device already exists: ${deviceChecked?.deviceTypeIdentifier} with name ${deviceChecked?.name}")
             return
-        // Find out if the default device is available
-        val out = hostExecutor.run("/usr/bin/xcrun", "simctl", "list", "devices", "available")
-        val deviceNameExists = out.toString("UTF-8").trim().contains(deviceName)
-        // Create if it's not available
-        if (!deviceNameExists) {
-            hostExecutor.run("/usr/bin/xcrun", "simctl", "create", deviceName, deviceName)
         }
-        // If successfully created, remember that.
+        logger.info("Device was not cheked before. Find the device with id: $deviceId")
-        _deviceNameChecked = deviceName
+        val simulatorRuntime = getSimulatorRuntime()
+        logger.info("Runtime used for the $deviceId is $simulatorRuntime")
+
+        val device = getDeviceFor(simulatorRuntime)
+        logger.info("Found device: $device")
+        // If successfully found, remember that.
+        deviceChecked = device
+    }
+
+    private fun getDeviceFor(simulatorRuntime: SimulatorRuntimeDescriptor): SimulatorDeviceDescriptor {
+        val runtimeIdentifier = simulatorRuntime.identifier
+        val deviceOrNull = {
+            getSimulatorDevices(simctl("list", "devices", "--json"))[runtimeIdentifier]
+                    ?.find { it.deviceTypeIdentifier == deviceId && it.isAvailable == true }
+        }
+        // If the device already exists, nothing to do.
+        deviceOrNull()?.let { return it }
+
+        // Create the device
+        logger.info("Current runtime doesn't have $deviceId available")
+        val deviceType = simulatorRuntime.supportedDeviceTypes.find { it.identifier == deviceId }
+        checkNotNull(deviceType) {
+            """
+            Default device $deviceId is not available for the runtime ${simulatorRuntime.name}
+            Supported devices: ${simulatorRuntime.supportedDeviceTypes.map { it.identifier }.joinToString(", ")}
+            """.trimIndent()
+        }
+        val out = simctl("create", deviceType.name, deviceType.identifier, runtimeIdentifier)
+        logger.info("Create device $deviceId: simctl output is: $out")
+
+        return checkNotNull(deviceOrNull()) { "Unable to get or create simulator device $deviceId for $target with runtime ${simulatorRuntime.name}" }
+    }
+
+    private fun getSimulatorRuntime(): SimulatorRuntimeDescriptor {
+        val simulatorRuntimeOrNull = {
+            getSimulatorRuntimesFor(
+                    json = simctl("list", "runtimes", "--json"),
+                    family = target.family,
+                    osMinVersion = configurables.osVersionMin
+            ).firstOrNull {
+                it.supportedDeviceTypes.any { deviceType -> deviceType.identifier == deviceId }
+            }
+        }
+        // If the simulator runtime already exists, nothing to do.
+        simulatorRuntimeOrNull()?.let { return it }
+
+        // Download this platform if not available
+        logger.info("Runtime for the $deviceId is not available. Downloading runtimes for $target with Xcode")
+        downloadRuntimeFor(simulatorOsName(target.family))
+
+        return checkNotNull(simulatorRuntimeOrNull()) { "Runtime is not available for the selected $deviceId. Check Xcode installation" }
+    }
+
+    private fun downloadRuntimeFor(osName: String) {
+        val version = Xcode.findCurrent().version
+        val versionSplit = version.split(".")
+        check(versionSplit.size >= 2) {
+            "Unrecognised version of Xcode $version was split to $versionSplit"
+        }
+        val major = versionSplit[0].toInt()
+        val minor = versionSplit[1].toInt()
+        check(major >= 14) {
+            "Was unable to get the required runtimes running on Xcode $version. Check the Xcode installation"
+        }
+        if (minor >= 1) {
+            // Option -downloadPlatform NAME available only since 14.1
+            hostExecutor.run("/usr/bin/xcrun", "xcodebuild", "-downloadPlatform", osName)
+        } else {
+            // Have to download all platforms :(
+            hostExecutor.run("/usr/bin/xcrun", "xcodebuild", "-downloadAllPlatforms")
+        }
     }
 
     override fun execute(request: ExecuteRequest): ExecuteResponse {
@@ -85,11 +160,12 @@ class XcodeSimulatorExecutor(
             "SIMCTL_CHILD_" + it.key
         }
         val workingDirectory = request.workingDirectory ?: File(request.executableAbsolutePath).parentFile
+        val name = deviceChecked?.name ?: error("No device available for $deviceId")
         // Starting Xcode 11 `simctl spawn` requires explicit `--standalone` flag.
         return hostExecutor.execute(request.copying {
             this.executableAbsolutePath = "/usr/bin/xcrun"
             this.workingDirectory = workingDirectory
-            this.args.addAll(0, listOf("simctl", "spawn", "--standalone", *archSpecification, deviceName, executable))
+            this.args.addAll(0, listOf("simctl", "spawn", "--standalone", *archSpecification, name, executable))
             this.environment.clear()
             this.environment.putAll(env)
         })