[K/N] Compile runtime tests in parallel.

Merge-request: KT-MR-6164
Merged-by: Alexander Shabalin <Alexander.Shabalin@jetbrains.com>

kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/testing/native/NativeTest.kt

@@ -11,23 +11,148 @@ import javax.inject.Inject
 import org.gradle.api.*
 import org.gradle.api.file.ConfigurableFileCollection
 import org.gradle.api.tasks.*
+import org.gradle.api.model.ObjectFactory
 import org.gradle.kotlin.dsl.getByType
+import org.gradle.process.ExecOperations
+import org.gradle.workers.WorkAction
+import org.gradle.workers.WorkParameters
+import org.gradle.workers.WorkerExecutor
 import org.jetbrains.kotlin.ExecClang
 import org.jetbrains.kotlin.bitcode.CompileToBitcode
 import org.jetbrains.kotlin.bitcode.CompileToBitcodeExtension
 import org.jetbrains.kotlin.konan.target.*
 import java.io.OutputStream
 
-open class CompileNativeTest @Inject constructor(
-        @InputFile val inputFile: File,
-        @Input val target: KonanTarget
+interface CompileNativeTestParameters : WorkParameters {
+    var mainFile: File
+    var inputFiles: List<File>
+    var llvmLinkOutputFile: File
+    var compilerOutputFile: File
+    var targetName: String
+    var compilerArgs: List<String>
+    var linkCommands: List<List<String>>
+
+    var konanHome: File
+    var llvmDir: File
+    var experimentalDistribution: Boolean
+    var isInfoEnabled: Boolean
+}
+
+abstract class CompileNativeTestJob : WorkAction<CompileNativeTestParameters> {
+    @get:Inject
+    abstract val execOperations: ExecOperations
+
+    @get:Inject
+    abstract val objects: ObjectFactory
+
+    private fun llvmLink() {
+        with(parameters) {
+            val tmpOutput = File.createTempFile("runtimeTests", ".bc").apply {
+                deleteOnExit()
+            }
+
+            // The runtime provides our implementations for some standard functions (see StdCppStubs.cpp).
+            // We need to internalize these symbols to avoid clashes with symbols provided by the C++ stdlib.
+            // But llvm-link -internalize is kinda broken: it links modules one by one and can't see usages
+            // of a symbol in subsequent modules. So it will mangle such symbols causing "unresolved symbol"
+            // errors at the link stage. So we have to run llvm-link twice: the first one links all modules
+            // except the one containing the entry point to a single *.bc without internalization. The second
+            // run internalizes this big module and links it with a module containing the entry point.
+            execOperations.exec {
+                executable = "$llvmDir/bin/llvm-link"
+                args = listOf("-o", tmpOutput.absolutePath) + inputFiles.map { it.absolutePath }
+            }
+
+            execOperations.exec {
+                executable = "$llvmDir/bin/llvm-link"
+                args = listOf(
+                        "-o", llvmLinkOutputFile.absolutePath,
+                        mainFile.absolutePath,
+                        tmpOutput.absolutePath,
+                        "-internalize"
+                )
+            }
+        }
+    }
+
+    private fun compile() {
+        with(parameters) {
+            val platformManager = PlatformManager(buildDistribution(konanHome.absolutePath), experimentalDistribution)
+            val execClang = ExecClang.create(objects, platformManager, llvmDir)
+            val target = platformManager.targetByName(targetName)
+
+            if (target.family.isAppleFamily) {
+                execClang.execToolchainClang(target) {
+                    executable = "clang++"
+                    this.args = compilerArgs + listOf(llvmLinkOutputFile.absolutePath, "-o", compilerOutputFile.absolutePath)
+                }
+            } else {
+                execClang.execBareClang {
+                    executable = "clang++"
+                    this.args = compilerArgs + listOf(llvmLinkOutputFile.absolutePath, "-o", compilerOutputFile.absolutePath)
+                }
+            }
+        }
+    }
+
+    private fun link() {
+        with(parameters) {
+            for (command in linkCommands) {
+                execOperations.exec {
+                    commandLine(command)
+                    if (!isInfoEnabled && command[0].endsWith("dsymutil")) {
+                        // Suppress dsymutl's warnings.
+                        // See: https://bugs.swift.org/browse/SR-11539.
+                        val nullOutputStream = object: OutputStream() {
+                            override fun write(b: Int) {}
+                        }
+                        errorOutput = nullOutputStream
+                    }
+                }
+            }
+        }
+    }
+
+    override fun execute() {
+        llvmLink()
+        compile()
+        link()
+    }
+}
+
+abstract class CompileNativeTest @Inject constructor(
+        baseName: String,
+        @Input val target: KonanTarget,
+        @InputFile val mainFile: File,
+        private val platformManager: PlatformManager,
+        private val mimallocEnabled: Boolean,
 ) : DefaultTask() {
+
+    @SkipWhenEmpty
+    @InputFiles
+    val inputFiles: ConfigurableFileCollection = project.files()
+
     @OutputFile
-    var outputFile = project.buildDir.resolve("bin/test/${target.name}/${inputFile.nameWithoutExtension}.o")
+    var llvmLinkOutputFile: File = project.buildDir.resolve("bitcode/test/${target.name}/$baseName.bc")
+
+    @OutputFile
+    var compilerOutputFile: File = project.buildDir.resolve("bin/test/${target.name}/$baseName.o")
+
+    private val executableExtension: String = when (target) {
+        is KonanTarget.MINGW_X64 -> ".exe"
+        is KonanTarget.MINGW_X86 -> ".exe"
+        else -> ""
+    }
+
+    @OutputFile
+    var outputFile: File = project.buildDir.resolve("bin/test/${target.name}/$baseName$executableExtension")
 
     @Input
     val clangArgs = mutableListOf<String>()
 
+    @Input
+    val linkerArgs = mutableListOf<String>()
+
     @Input @Optional
     var sanitizer: SanitizerKind? = null
 
@@ -37,126 +162,14 @@ open class CompileNativeTest @Inject constructor(
         SanitizerKind.THREAD -> listOf("-fsanitize=thread")
     }
 
-    @TaskAction
-    fun compile() {
-        val plugin = project.extensions.getByType<ExecClang>()
-        val args = clangArgs + sanitizerFlags + listOf(inputFile.absolutePath, "-o", outputFile.absolutePath)
-        if (target.family.isAppleFamily) {
-            plugin.execToolchainClang(target) {
-                executable = "clang++"
-                this.args = args
-            }
-        } else {
-            plugin.execBareClang {
-                executable = "clang++"
-                this.args = args
-            }
-        }
-    }
-}
-
-open class LlvmLinkNativeTest @Inject constructor(
-        baseName: String,
-        @Input val target: String,
-        @InputFile val mainFile: File
-) : DefaultTask() {
-
-    @SkipWhenEmpty
-    @InputFiles
-    val inputFiles: ConfigurableFileCollection = project.files()
-
-    @OutputFile
-    var outputFile: File = project.buildDir.resolve("bitcode/test/$target/$baseName.bc")
-
-    @TaskAction
-    fun llvmLink() {
-        val llvmDir = project.property("llvmDir")
-        val tmpOutput = File.createTempFile("runtimeTests", ".bc").apply {
-            deleteOnExit()
-        }
-
-        // The runtime provides our implementations for some standard functions (see StdCppStubs.cpp).
-        // We need to internalize these symbols to avoid clashes with symbols provided by the C++ stdlib.
-        // But llvm-link -internalize is kinda broken: it links modules one by one and can't see usages
-        // of a symbol in subsequent modules. So it will mangle such symbols causing "unresolved symbol"
-        // errors at the link stage. So we have to run llvm-link twice: the first one links all modules
-        // except the one containing the entry point to a single *.bc without internalization. The second
-        // run internalizes this big module and links it with a module containing the entry point.
-        project.exec {
-            executable = "$llvmDir/bin/llvm-link"
-            args = listOf("-o", tmpOutput.absolutePath) + inputFiles.map { it.absolutePath }
-        }
-
-        project.exec {
-            executable = "$llvmDir/bin/llvm-link"
-            args = listOf(
-                    "-o", outputFile.absolutePath,
-                    mainFile.absolutePath,
-                    tmpOutput.absolutePath,
-                    "-internalize"
-            )
-        }
-    }
-}
-
-open class LinkNativeTest @Inject constructor(
-        @InputFiles val inputFiles: List<File>,
-        @OutputFile val outputFile: File,
-        @Internal val target: String,
-        @Internal val linkerArgs: List<String>,
-        private val  platformManager: PlatformManager,
-        private val mimallocEnabled: Boolean
-) : DefaultTask () {
-    companion object {
-        fun create(
-                project: Project,
-                platformManager: PlatformManager,
-                taskName: String,
-                inputFiles: List<File>,
-                target: String,
-                outputFile: File,
-                linkerArgs: List<String>,
-                mimallocEnabled: Boolean
-        ): LinkNativeTest = project.tasks.create(
-                taskName,
-                LinkNativeTest::class.java,
-                inputFiles,
-                outputFile,
-                target,
-                linkerArgs,
-                platformManager,
-                mimallocEnabled)
-
-        fun create(
-                project: Project,
-                platformManager: PlatformManager,
-                taskName: String,
-                inputFiles: List<File>,
-                target: String,
-                executableName: String,
-                mimallocEnabled: Boolean,
-                linkerArgs: List<String> = listOf()
-        ): LinkNativeTest = create(
-                project,
-                platformManager,
-                taskName,
-                inputFiles,
-                target,
-                project.buildDir.resolve("bin/test/$target/$executableName"),
-                linkerArgs, mimallocEnabled)
-    }
-
-    @Input @Optional
-    var sanitizer: SanitizerKind? = null
-
     @get:Input
-    val commands: List<List<String>>
+    val linkCommands: List<List<String>>
         get() {
             // Getting link commands requires presence of a target toolchain.
             // Thus we cannot get them at the configuration stage because the toolchain may be not downloaded yet.
-            val linker = platformManager.platform(platformManager.targetByName(target)).linker
+            val linker = platformManager.platform(target).linker
             return linker.finalLinkCommands(
-                    inputFiles.map { it.absolutePath },
+                    listOf(compilerOutputFile.absolutePath),
                     outputFile.absolutePath,
                     listOf(),
                     linkerArgs,
@@ -170,36 +183,79 @@ open class LinkNativeTest @Inject constructor(
             ).map { it.argsWithExecutable }
         }
 
+    @get:Inject
+    abstract val workerExecutor: WorkerExecutor
+
     @TaskAction
-    fun link() {
-        for (command in commands) {
-            project.exec {
-                commandLine(command)
-                if (!logger.isInfoEnabled() && command[0].endsWith("dsymutil")) {
-                    // Suppress dsymutl's warnings.
-                    // See: https://bugs.swift.org/browse/SR-11539.
-                    val nullOutputStream = object: OutputStream() {
-                        override fun write(b: Int) {}
-                    }
-                    errorOutput = nullOutputStream
-                }
-            }
+    fun compile() {
+        val workQueue = workerExecutor.noIsolation()
+
+        val parameters = { it: CompileNativeTestParameters ->
+            it.mainFile = mainFile
+            it.inputFiles = inputFiles.map { it }
+            it.llvmLinkOutputFile = llvmLinkOutputFile
+            it.compilerOutputFile = compilerOutputFile
+            it.targetName = target.name
+            it.compilerArgs = clangArgs + sanitizerFlags
+            it.linkCommands = linkCommands
+
+            it.konanHome = project.project(":kotlin-native").projectDir
+            it.llvmDir = project.file(project.findProperty("llvmDir")!!)
+            it.isInfoEnabled = logger.isInfoEnabled()
         }
+
+        workQueue.submit(CompileNativeTestJob::class.java, parameters)
     }
 }
 
-open class ReportWithPrefixes @Inject constructor(
-        @Input val testName: String,
-        @InputFile val inputReport: File,
-        @OutputFile val outputReport: File
+open class RunNativeTest @Inject constructor(
+    @Input val testName: String,
+    @InputFile val inputFile: File,
 ) : DefaultTask() {
+
+    @Internal
+    var workingDir: File = project.buildDir.resolve("testReports/$testName")
+
+    @OutputFile
+    var outputFile: File = workingDir.resolve("report.xml")
+
+    @OutputFile
+    var outputFileWithPrefixes: File = workingDir.resolve("report-with-prefixes.xml")
+
+    @Input @Optional
+    var filter: String? = project.findProperty("gtest_filter") as? String
+
+    @Input @Optional
+    var sanitizer: SanitizerKind? = null
+
+    @InputFile
+    var tsanSuppressionsFile = project.file("tsan_suppressions.txt")
+
     @TaskAction
-    fun process() {
+    fun run() {
+        workingDir.mkdirs()
+
+        // Do not run this in workers, because we don't want this task to run in parallel.
+        project.exec {
+            executable = inputFile.absolutePath
+
+            if (filter != null) {
+                args("--gtest_filter=${filter}")
+            }
+            args("--gtest_output=xml:${outputFile.absolutePath}")
+            when (sanitizer) {
+                SanitizerKind.THREAD -> {
+                    environment("TSAN_OPTIONS", "suppressions=${tsanSuppressionsFile.absolutePath}")
+                }
+                else -> {} // no action required
+            }
+        }
+
         // TODO: Better to use proper XML parsing.
-        var contents = inputReport.readText()
+        var contents = outputFile.readText()
         contents = contents.replace("<testsuite name=\"", "<testsuite name=\"${testName}.")
         contents = contents.replace("classname=\"", "classname=\"${testName}.")
-        outputReport.writeText(contents)
+        outputFileWithPrefixes.writeText(contents)
     }
 }
 
@@ -268,71 +324,35 @@ private fun createTestTask(
 
     val testSupportTask = project.tasks.getByName("${target}TestSupport${CompileToBitcodeExtension.suffixForSanitizer(sanitizer)}") as CompileToBitcode
 
-    // TODO: It may make sense to merge llvm-link, compile and link to a single task.
-    val llvmLinkTask = project.tasks.create(
-            "${testName}LlvmLink",
-            LlvmLinkNativeTest::class.java,
-            testName, target, testSupportTask.outFile
+    val mimallocEnabled = testedTaskNames.any { it.contains("mimalloc", ignoreCase = true) }
+    val compileTask = project.tasks.create(
+            "${testName}Compile",
+            CompileNativeTest::class.java,
+            testName,
+            konanTarget,
+            testSupportTask.outFile,
+            platformManager,
+            mimallocEnabled,
     ).apply {
         val tasksToLink = (compileToBitcodeTasks + testedTasks + testFrameworkTasks)
-        inputFiles.setFrom(tasksToLink.map { it.outFile })
+        this.sanitizer = sanitizer
+        this.inputFiles.setFrom(tasksToLink.map { it.outFile })
+        this.clangArgs.addAll(buildClangFlags(platformManager.platform(konanTarget).configurables))
         dependsOn(testSupportTask)
         dependsOn(tasksToLink)
     }
 
-    val compileTask = project.tasks.create(
-            "${testName}Compile",
-            CompileNativeTest::class.java,
-            llvmLinkTask.outputFile,
-            konanTarget
-    ).apply {
-        this.sanitizer = sanitizer
-        dependsOn(llvmLinkTask)
-        clangArgs.addAll(buildClangFlags(platformManager.platform(konanTarget).configurables))
-    }
-
-    val mimallocEnabled = testedTaskNames.any { it.contains("mimalloc", ignoreCase = true) }
-    val linkTask = LinkNativeTest.create(
-            project,
-            platformManager,
-            "${testName}Link${CompileToBitcodeExtension.suffixForSanitizer(sanitizer)}",
-            listOf(compileTask.outputFile),
-            target,
-            testName,
-            mimallocEnabled
+    val runTask = project.tasks.create(
+        testName,
+        RunNativeTest::class.java,
+        testName,
+        compileTask.outputFile,
     ).apply {
         this.sanitizer = sanitizer
         dependsOn(compileTask)
     }
 
-    return project.tasks.create(testName, Exec::class.java).apply {
-        dependsOn(linkTask)
-
-        workingDir = project.buildDir.resolve("testReports/$testName")
-        val xmlReport = workingDir.resolve("report.xml")
-        executable(linkTask.outputFile)
-        val filter = project.findProperty("gtest_filter");
-        if (filter != null) {
-            args("--gtest_filter=${filter}")
-        }
-        args("--gtest_output=xml:${xmlReport.absoluteFile}")
-        when (sanitizer) {
-            SanitizerKind.THREAD -> {
-                val file = project.file("tsan_suppressions.txt")
-                inputs.file(file)
-                environment("TSAN_OPTIONS", "suppressions=${file.absolutePath}")
-            }
-            else -> {} // no action required
-        }
-
-        doFirst {
-            workingDir.mkdirs()
-        }
-
-        val reportWithPrefixes = workingDir.resolve("report-with-prefixes.xml")
-        val reportTask = project.tasks.register("${testName}Report", ReportWithPrefixes::class.java, testName, xmlReport, reportWithPrefixes)
-        finalizedBy(reportTask)
-    }
+    return runTask
 }
 
 // TODO: These tests should be created by `CompileToBitcodeExtension`

kotlin-native/runtime/src/mm/cpp/ThreadSuspensionTest.cpp

@@ -25,11 +25,11 @@ using namespace kotlin;
 namespace {
 
 #ifdef KONAN_WINDOWS
-constexpr size_t kDefaultIterations = 1000;
-constexpr size_t kDefaultReportingStep = 100;
+constexpr size_t kDefaultIterations = 20;
+constexpr size_t kDefaultReportingStep = 2;
 #else
-constexpr size_t kDefaultIterations = 10000;
-constexpr size_t kDefaultReportingStep = 1000;
+constexpr size_t kDefaultIterations = 200;
+constexpr size_t kDefaultReportingStep = 20;
 #endif // #ifdef KONAN_WINDOWS
 
 std_support::vector<mm::ThreadData*> collectThreadData() {