From 6a19c4ce96fb935be90286b7ca8de03fa939446d Mon Sep 17 00:00:00 2001 From: Alexander Shabalin Date: Sun, 3 Sep 2023 19:57:29 +0200 Subject: [PATCH] [K/N] Remove legacy executors. --- .../backend.native/tests/build.gradle | 19 - .../org/jetbrains/kotlin/ExecutorService.kt | 375 +----------------- .../main/kotlin/org/jetbrains/kotlin/Utils.kt | 6 - .../kotlin/native/executors/WasmExecutor.kt | 36 -- 4 files changed, 9 insertions(+), 427 deletions(-) delete mode 100644 native/executors/src/main/kotlin/org/jetbrains/kotlin/native/executors/WasmExecutor.kt diff --git a/kotlin-native/backend.native/tests/build.gradle b/kotlin-native/backend.native/tests/build.gradle index dca5a4e6e19..92baa794a72 100644 --- a/kotlin-native/backend.native/tests/build.gradle +++ b/kotlin-native/backend.native/tests/build.gradle @@ -6285,25 +6285,6 @@ dependencies { api RepoDependencies.commonDependency(project, "junit") } -// Configure build for iOS device targets. -if (UtilsKt.supportsRunningTestsOnDevice(target) && !UtilsKt.getCompileOnlyTests(project)) { - project.tasks - .withType(KonanTestExecutable.class) - .configureEach { - ExecutorServiceKt.configureXcodeBuild((KonanTestExecutable) it) - } - // Exclude tasks that cannot be run on device - project.tasks - .withType(KonanTest.class) - .matching { (it instanceof KonanLocalTest && ((KonanLocalTest) it).useTestData) || - // Filter out tests that depend on libs. TODO: copy them too - it instanceof KonanInteropTest || - it instanceof KonanDynamicTest || - it instanceof KonanLinkTest - } - .configureEach { it.enabled = false } -} - project.afterEvaluate { // Don't treat any task as up-to-date, no matter what. // Note: this project should contain only test tasks, including ones that build binaries, diff --git a/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/ExecutorService.kt b/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/ExecutorService.kt index e4fc4274c05..b7abc29aeec 100644 --- a/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/ExecutorService.kt +++ b/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/ExecutorService.kt @@ -17,7 +17,6 @@ @file:OptIn(kotlin.time.ExperimentalTime::class) package org.jetbrains.kotlin -import com.google.gson.annotations.Expose import groovy.lang.Closure import org.gradle.api.Action import org.gradle.api.Project @@ -31,12 +30,7 @@ import org.jetbrains.kotlin.native.executors.* import java.io.* import java.nio.file.Path -import java.nio.file.Paths -import java.time.LocalDateTime -import java.time.format.DateTimeFormatter -import java.util.concurrent.TimeUnit -import kotlin.time.Duration import kotlin.time.DurationUnit import kotlin.time.toDuration @@ -99,21 +93,21 @@ fun create(project: Project): ExecutorService { val testTarget = project.testTarget val configurables = project.testTargetConfigurables - return when { - project.compileOnlyTests -> NoOpExecutor(explanation = "compile-only tests").service(project) - project.hasProperty("remote") -> sshExecutor(project, testTarget) - configurables is WasmConfigurables -> WasmExecutor(configurables).service(project) - configurables is ConfigurablesWithEmulator && testTarget != HostManager.host -> EmulatorExecutor(configurables).service(project) + val executor = when { + project.compileOnlyTests -> NoOpExecutor(explanation = "compile-only tests") + testTarget == HostManager.host -> HostExecutor() + configurables is ConfigurablesWithEmulator && testTarget != HostManager.host -> EmulatorExecutor(configurables) configurables is AppleConfigurables && configurables.targetTriple.isSimulator -> XcodeSimulatorExecutor(configurables).apply { // Property can specify device identifier to be run on. For example, `com.apple.CoreSimulator.SimDeviceType.iPhone-11` project.findProperty("iosDevice")?.toString()?.let { deviceId = it } - }.service(project) - supportsRunningTestsOnDevice(testTarget) -> deviceLauncher(project) - configurables is AppleConfigurables && RosettaExecutor.availableFor(configurables) -> RosettaExecutor(configurables).service(project) - else -> HostExecutor().service(project) + } + configurables is AppleConfigurables && RosettaExecutor.availableFor(configurables) -> RosettaExecutor(configurables) + else -> error("Cannot run for target $testTarget") } + + return executor.service(project) } data class ProcessOutput(var stdOut: String, var stdErr: String, var exitCode: Int) @@ -230,354 +224,3 @@ fun localExecutorService(project: Project): ExecutorService = object : ExecutorS override val project: Project get() = project override fun execute(action: Action): ExecResult? = project.exec(action) } - -/** - * Remote process executor. - * - * @param remote makes binaries be executed on a remote host - * Specify it as -Premote=user@host - */ -@Suppress("KDocUnresolvedReference") -private fun sshExecutor(project: Project, testTarget: KonanTarget): ExecutorService = object : ExecutorService { - - private val remote: String = project.property("remote").toString() - private val sshArgs: List = System.getenv("SSH_ARGS")?.split(" ") ?: emptyList() - private val sshHome = System.getenv("SSH_HOME") ?: "/usr/bin" - - // Unique remote dir name to be used in the target host - private val remoteDir = run { - val date = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMddHHmmss")) - Paths.get(project.findProperty("remoteRoot").toString(), "tmp", - System.getProperty("user.name") + "_" + date).toString() - } - - private val environmentArgs: List = when { - testTarget.family.isAppleFamily -> - listOf("export DYLD_LIBRARY_PATH=$remoteDir:\$DYLD_LIBRARY_PATH;") - testTarget.family in listOf(Family.LINUX, Family.ANDROID) -> - listOf("export LD_LIBRARY_PATH=$remoteDir:\$LD_LIBRARY_PATH;") - else -> emptyList() - } - - override val project: Project get() = project - - override fun execute(action: Action): ExecResult { - lateinit var executableWithLibs: ExecutableWithDynamicLibs - - createRemoteDir() - val execResult = project.exec { - action.execute(this) - - executableWithLibs = ExecutableWithDynamicLibs(executable) - executableWithLibs.upload() - - this.executable = executableWithLibs.remoteExecutable - commandLine = arrayListOf("$sshHome/ssh") + sshArgs + remote + environmentArgs + commandLine - } - executableWithLibs.cleanup() - return execResult - } - - private fun createRemoteDir() { - project.exec { - commandLine = arrayListOf("$sshHome/ssh") + sshArgs + remote + "mkdir" + "-p" + remoteDir - } - } - - private fun upload(fileName: String) { - project.exec { - commandLine = arrayListOf("$sshHome/scp") + sshArgs + fileName + "$remote:$remoteDir" - } - } - - private fun cleanup(fileName: String) { - project.exec { - commandLine = arrayListOf("$sshHome/ssh") + sshArgs + remote + "rm" + fileName - } - } - - inner class ExecutableWithDynamicLibs(val localExecutable: String) { - - val remoteExecutable: String = "$remoteDir/${File(localExecutable).name}" - - val localDynamicLibs: List = collectLocalDynamicLibs() - val remoteDynamicLibs: List = localDynamicLibs.map { "$remoteDir/${File(it).name}" } - - fun upload() { - upload(localExecutable) - localDynamicLibs.forEach(::upload) - } - - fun cleanup() { - cleanup(remoteExecutable) - remoteDynamicLibs.forEach(::cleanup) - } - - private val String.remotePath - get() = "$remoteDir/${File(this).name}" - - private fun collectLocalDynamicLibs(): List { - val dynamicSuffix = CompilerOutputKind.DYNAMIC.suffix(testTarget) - val directory = File(localExecutable).parentFile?.takeIf { it.isDirectory } - ?: return emptyList() - return directory.listFiles() - .filter { it.isFile && it.name.endsWith(dynamicSuffix) } - .map { it.path } - } - } -} - -internal data class DeviceTarget( - @Expose val name: String, - @Expose val udid: String, - @Expose val state: String, - @Expose val type: String -) - -private fun deviceLauncher(project: Project) = object : ExecutorService { - private val xcProject = Paths.get(project.testOutputRoot, "launcher") - - private val idb = project.findProperty("idb_path") as? String ?: "idb" - - private val deviceName = project.findProperty("device_name") as? String - - private val bundleID = "org.jetbrains.kotlin.KonanTestLauncher" - - override val project: Project get() = project - - override fun execute(action: Action): ExecResult? { - val result: ExecResult? - try { - val udid = targetUDID() - println("Found device UDID: $udid") - install(udid, xcProject.resolve("build/KonanTestLauncher.ipa").toString()) - val commands = startDebugServer(udid) - .split("\n") - .filter { it.isNotBlank() } - .flatMap { listOf("-o", it) } - - var savedOut: OutputStream? = null - val out = ByteArrayOutputStream() - result = project.exec { - action.execute(this) - executable = "lldb" - args = commands + "-b" + "-o" + "command script import ${pythonScript()}" + - "-o" + ("process launch" + - (args.takeUnless { it.isEmpty() } - ?.let { " -- ${it.joinToString(" ")}" } - ?: "")) + - "-o" + "get_exit_code" + - "-k" + "get_exit_code" + - "-k" + "exit -1" - // A test task that uses project.exec { } sets the stdOut to parse the result, - // but the test executable is being run under debugger that has its own output mixed with the - // output from the test. Save the stdOut from the test to write the parsed output to it. - savedOut = this.standardOutput - standardOutput = out - } - out.toString() - .also { if (project.verboseTest) println(it) } - .split("\n") - .dropWhile { s -> !s.startsWith("(lldb) process launch") } - .drop(1) // drop 'process launch' also - .dropLastWhile { !it.matches(".*Process [0-9]* exited with status .*".toRegex()) } - .joinToString("\n") { - it.replace("Process [0-9]* exited with status .*".toRegex(), "") - .replace("\r", "") // TODO: investigate: where does the \r comes from - } - .also { - savedOut?.write(it.toByteArray()) - } - - uninstall(udid) - } catch (exc: Exception) { - throw RuntimeException("iOS-device execution failed", exc) - } finally { - kill() - } - return result - } - - /* - * This script kills the target process in case it has been stopped, - * and exists lldb with the same exit code as a target process. - */ - private fun pythonScript(): String = xcProject.resolve("lldb_cmd.py").toFile().run { - writeText( // language=Python - """ - import lldb - - def exit_code(debugger, command, exe_ctx, result, internal_dict): - process = exe_ctx.GetProcess() - state = process.GetState() - if state == lldb.eStateStopped: - # Call flush method, otherwise some output isn't shown in the debugger - debugger.HandleCommand("expression -- (int) fflush(NULL)") - debugger.HandleCommand("bt all") - process.Kill() - code = process.GetExitStatus() - debugger.HandleCommand("exit %d" % code) - - def __lldb_init_module(debugger, _): - debugger.HandleCommand('command script add -f lldb_cmd.exit_code get_exit_code') - """.trimIndent()) - absolutePath - } - - private fun kill() = project.exec { commandLine(idb, "kill") } - - private inline fun tryUntilTrue(times: Int = 3, f: () -> Boolean) { - for (i in 1..times) { - if (f()) break - else TimeUnit.SECONDS.sleep(i.toLong()) - } - } - - private fun targetUDID(): String { - val out = ByteArrayOutputStream() - // idb launches idb_companion but doesn't wait for it and just exits. - // So relaunch `list-targets` again. - tryUntilTrue { - project.exec { - commandLine(idb, "list-targets", "--json") - standardOutput = out - }.assertNormalExitValue() - out.toString().trim().isNotEmpty() - } - return out.toString().run { - check(isNotEmpty()) - split("\n") - .filter { it.isNotEmpty() } - .map { - gson.fromJson(it, DeviceTarget::class.java) - } - .first { - it.type == "device" && deviceName?.run { this == it.name } ?: true - } - .udid - } - } - - private fun install(udid: String, bundlePath: String) { - val out = ByteArrayOutputStream() - lateinit var result: ExecResult - tryUntilTrue { - result = project.exec { - workingDir = xcProject.toFile() - commandLine = listOf(idb, "install", "--udid", udid, bundlePath) - standardOutput = out - errorOutput = out - isIgnoreExitValue = true - } - println(out.toString()) - result.exitValue == 0 - } - check(result.exitValue == 0) { "Installation of $bundlePath failed: $out" } - } - - private fun uninstall(udid: String) { - val out = ByteArrayOutputStream() - - project.exec { - workingDir = xcProject.toFile() - commandLine = listOf(idb, "uninstall", "--udid", udid, bundleID) - standardOutput = out - errorOutput = out - isIgnoreExitValue = true - } - println(out.toString()) - } - - private fun startDebugServer(udid: String): String { - val out = ByteArrayOutputStream() - - val result = project.exec { - workingDir = xcProject.toFile() - commandLine = listOf(idb, "debugserver", "start", "--udid", udid, bundleID) - standardOutput = out - errorOutput = out - isIgnoreExitValue = true - } - check(result.exitValue == 0) { "Failed to start debug server: $out" } - return out.toString() - } -} - -fun KonanTestExecutable.configureXcodeBuild() { - this.doBeforeRun = Action { - val signIdentity = project.findProperty("sign_identity") as? String ?: "iPhone Developer" - val developmentTeam = project.findProperty("development_team") as? String - requireNotNull(developmentTeam) { "Specify '-Pdevelopment_team=' with the your team id" } - val xcProject = Paths.get(project.testOutputRoot, "launcher") - - val shellScript: String = // language=Bash - mutableListOf(""" - set -x - # Copy executable to the build dir. - COPY_TO="${"$"}TARGET_BUILD_DIR/${"$"}EXECUTABLE_PATH" - cp "${project.file(executable).absolutePath}" "${"$"}COPY_TO" - # copy dSYM if it exists - DSYM_DIR="${project.file("$executable.dSYM").absolutePath}" - if [ -d "${"$"}DSYM_DIR" ]; then - cp -r "${"$"}DSYM_DIR" "${"$"}TARGET_BUILD_DIR/${"$"}EXECUTABLE_FOLDER_PATH/" - fi - """.trimIndent()).also { - if (this is FrameworkTest) { - // Create a Frameworks folder inside the build dir. - it += "mkdir -p \"\$TARGET_BUILD_DIR/\$FRAMEWORKS_FOLDER_PATH\"" - // Copy each framework to the Frameworks dir. - it += frameworks.filter { framework -> !framework.isStatic } - .map { framework -> - val artifact = framework.artifact - "cp -r \"$testOutput/${this.name}/${project.testTarget.name}/$artifact.framework\" " + - "\"\$TARGET_BUILD_DIR/\$FRAMEWORKS_FOLDER_PATH/$artifact.framework\"" - } - } - }.joinToString(separator = "\\n") { it.replace("\"", "\\\"") } - - // Copy template xcode project. - project.file("iosLauncher").copyRecursively(xcProject.toFile(), overwrite = true) - - xcProject.resolve("KonanTestLauncher.xcodeproj/project.pbxproj") - .toFile() - .apply { - val text = readLines().joinToString("\n") { - when { - it.contains("CODE_SIGN_IDENTITY") -> - it.replaceAfter("= ", "\"$signIdentity\";") - it.contains("DEVELOPMENT_TEAM") || it.contains("DevelopmentTeam") -> - it.replaceAfter("= ", "$developmentTeam;") - it.contains("shellScript = ") -> - it.replaceAfter("= ", "\"$shellScript\";") - else -> it - } - } - writeText(text) - } - - val sdk = when (project.testTarget) { - KonanTarget.IOS_ARM32, KonanTarget.IOS_ARM64 -> Xcode.findCurrent().iphoneosSdk - else -> error("Unsupported target: ${project.testTarget}") - } - - fun xcodebuild(vararg elements: String) { - val xcode = listOf("/usr/bin/xcrun", "-sdk", sdk, "xcodebuild") - val out = ByteArrayOutputStream() - val result = project.exec { - workingDir = xcProject.toFile() - commandLine = xcode + elements.toList() - standardOutput = out - } - println(out.toString("UTF-8")) - result.assertNormalExitValue() - } - xcodebuild("-workspace", "KonanTestLauncher.xcodeproj/project.xcworkspace", - "-scheme", "KonanTestLauncher", "-allowProvisioningUpdates", "-destination", - "generic/platform=iOS", "build") - val archive = xcProject.resolve("build/KonanTestLauncher.xcarchive").toString() - xcodebuild("-workspace", "KonanTestLauncher.xcodeproj/project.xcworkspace", - "-scheme", "KonanTestLauncher", "archive", "-archivePath", archive) - xcodebuild("-exportArchive", "-archivePath", archive, "-exportOptionsPlist", "KonanTestLauncher/Info.plist", - "-exportPath", xcProject.resolve("build").toString()) - } -} diff --git a/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/Utils.kt b/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/Utils.kt index a2b3958a30e..ec9d274f3a7 100644 --- a/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/Utils.kt +++ b/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/Utils.kt @@ -156,12 +156,6 @@ fun codesign(project: Project, path: String) { fun isSimulatorTarget(project: Project, target: KonanTarget): Boolean = project.platformManager.platform(target).targetTriple.isSimulator -/** - * Check that [target] is an Apple device. - */ -fun supportsRunningTestsOnDevice(target: KonanTarget): Boolean = - target == KonanTarget.IOS_ARM32 || target == KonanTarget.IOS_ARM64 - /** * Creates a list of file paths to be compiled from the given [compile] list with regard to [exclude] list. */ diff --git a/native/executors/src/main/kotlin/org/jetbrains/kotlin/native/executors/WasmExecutor.kt b/native/executors/src/main/kotlin/org/jetbrains/kotlin/native/executors/WasmExecutor.kt deleted file mode 100644 index b8148319b15..00000000000 --- a/native/executors/src/main/kotlin/org/jetbrains/kotlin/native/executors/WasmExecutor.kt +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.native.executors - -import org.jetbrains.kotlin.konan.target.Configurables -import org.jetbrains.kotlin.konan.target.WasmConfigurables -import java.io.File - -/** - * [Executor] that runs the wasm process via `d8`. - * - * NOTE: Apart from [ExecuteRequest.executableAbsolutePath] a JS launcher is required, - * it must be located in the same folder with the same name and `.js` suffix appended. - * - * @param configurables [Configurables] for the wasm target - */ -class WasmExecutor( - private val configurables: WasmConfigurables, -) : Executor { - private val hostExecutor: Executor = HostExecutor() - - override fun execute(request: ExecuteRequest): ExecuteResponse { - val absoluteTargetToolchain = configurables.absoluteTargetToolchain - val workingDirectory = request.workingDirectory ?: File(request.executableAbsolutePath).parentFile - val executable = request.executableAbsolutePath - val launcherJs = "$executable.js" - return hostExecutor.execute(request.copying { - this.executableAbsolutePath = "$absoluteTargetToolchain/bin/d8" - this.workingDirectory = workingDirectory - this.args.addAll(0, listOf("--expose-wasm", launcherJs, "--", executable)) - }) - } -} \ No newline at end of file