[K/N][build] Parallelize runtime building

kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/ExecClang.kt

@@ -17,17 +17,29 @@
 package org.jetbrains.kotlin
 
 import org.gradle.api.Action
-import groovy.lang.Closure
 import org.gradle.api.GradleException
 import org.gradle.api.Project
+import org.gradle.api.internal.file.FileOperations
+import org.gradle.api.internal.project.ProjectInternal
+import org.gradle.api.model.ObjectFactory
+import org.gradle.process.ExecOperations
 import org.gradle.process.ExecResult
 import org.gradle.process.ExecSpec
 import org.jetbrains.kotlin.konan.target.*
 import org.jetbrains.kotlin.konan.file.*
+import org.jetbrains.kotlin.konan.file.File as KFile
+import java.io.File
+import javax.inject.Inject
 
-class ExecClang(private val project: Project) {
+abstract class ExecClang @Inject constructor(
+        private val platformManager: PlatformManager,
+        private val llvmDir: File
+    ) {
 
-    private val platformManager = project.project(":kotlin-native").findProperty("platformManager") as PlatformManager
+    @get:Inject
+    protected abstract val fileOperations: FileOperations
+    @get:Inject
+    protected abstract val execOperations: ExecOperations
 
     private fun clangArgsForCppRuntime(target: KonanTarget): List<String> {
         return platformManager.platform(target).clang.clangArgsForKonanSources.asList()
@@ -42,8 +54,7 @@ class ExecClang(private val project: Project) {
         val executable = executableOrNull ?: "clang"
 
         if (listOf("clang", "clang++").contains(executable)) {
-            val llvmDir = project.findProperty("llvmDir")
+            return "${llvmDir.absolutePath}/bin/$executable"
-            return "${llvmDir}/bin/$executable"
         } else {
             throw GradleException("unsupported clang executable: $executable")
         }
@@ -71,10 +82,6 @@ class ExecClang(private val project: Project) {
         return this.execClang(emptyList(), action)
     }
 
-    fun execBareClang(closure: Closure<in ExecSpec>): ExecResult {
-        return this.execClang(emptyList(), closure)
-    }
-
     // The konan ones invoke clang with konan provided sysroots.
     // So they require a target or assume it to be the host.
     // The target can be specified as KonanTarget or as a
@@ -88,14 +95,6 @@ class ExecClang(private val project: Project) {
         return this.execClang(clangArgsForCppRuntime(target), action)
     }
 
-    fun execKonanClang(target: String?, closure: Closure<in ExecSpec>): ExecResult {
-        return this.execClang(clangArgsForCppRuntime(target), closure)
-    }
-
-    fun execKonanClang(target: KonanTarget, closure: Closure<in ExecSpec>): ExecResult {
-        return this.execClang(clangArgsForCppRuntime(target), closure)
-    }
-
     /**
      * Execute Clang the way that produced object file is compatible with
      * the one that produced by Kotlin/Native for given [target]. It means:
@@ -104,7 +103,7 @@ class ExecClang(private val project: Project) {
      */
     fun execClangForCompilerTests(target: KonanTarget, action: Action<in ExecSpec>): ExecResult {
         val defaultArgs = platformManager.platform(target).clang.clangArgs.toList()
-        return project.exec {
+        return execOperations.exec {
             action.execute(this)
             executable = if (target.family.isAppleFamily) {
                 resolveToolchainExecutable(target, executable)
@@ -115,36 +114,18 @@ class ExecClang(private val project: Project) {
         }
     }
 
-    /**
-     * @see execClangForCompilerTests
-     */
-    fun execClangForCompilerTests(target: KonanTarget, closure: Closure<in ExecSpec>): ExecResult =
-            execClangForCompilerTests(target) { project.configure(this, closure) }
-
     // The toolchain ones execute clang from the toolchain.
 
     fun execToolchainClang(target: String?, action: Action<in ExecSpec>): ExecResult {
         return this.execToolchainClang(platformManager.targetManager(target).target, action)
     }
 
-    fun execToolchainClang(target: String?, closure: Closure<in ExecSpec>): ExecResult {
-        return this.execToolchainClang(platformManager.targetManager(target).target) { project.configure(this, closure) }
-    }
-
     fun execToolchainClang(target: KonanTarget, action: Action<in ExecSpec>): ExecResult {
         val extendedAction = Action<ExecSpec> {
             action.execute(this)
             executable = resolveToolchainExecutable(target, executable)
         }
-        return project.exec(extendedAction)
+        return execOperations.exec(extendedAction)
-    }
-
-    fun execToolchainClang(target: KonanTarget, closure: Closure<in ExecSpec>): ExecResult {
-        return this.execToolchainClang(target) { project.configure(this, closure) }
-    }
-
-    private fun execClang(defaultArgs: List<String>, closure: Closure<in ExecSpec>): ExecResult {
-        return this.execClang(defaultArgs) { project.configure(this, closure) }
     }
 
     private fun execClang(defaultArgs: List<String>, action: Action<in ExecSpec>): ExecResult {
@@ -153,10 +134,23 @@ class ExecClang(private val project: Project) {
             executable = resolveExecutable(executable)
 
             val hostPlatform = platformManager.hostPlatform
-            environment["PATH"] = project.files(hostPlatform.clang.clangPaths).asPath +
+            environment["PATH"] = fileOperations.configurableFiles(hostPlatform.clang.clangPaths).asPath +
-                    java.io.File.pathSeparator + environment["PATH"]
+                    File.pathSeparator + environment["PATH"]
             args = args + defaultArgs
         }
-        return project.exec(extendedAction)
+        return execOperations.exec(extendedAction)
+    }
+
+    companion object {
+        @JvmStatic
+        fun create(project: Project): ExecClang = create(
+                project.objects,
+                project.project(":kotlin-native").findProperty("platformManager") as PlatformManager,
+                project.file(project.findProperty("llvmDir")!!)
+        )
+
+        @JvmStatic
+        fun create(objects: ObjectFactory, platformManager: PlatformManager, llvmDir: File) =
+                objects.newInstance(ExecClang::class.java, platformManager, llvmDir)
     }
 }

kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/Utils.kt

@@ -410,7 +410,7 @@ fun Project.buildStaticLibrary(cSources: Collection<File>, output: File, objDir:
     val platform = platformManager.platform(testTarget)
 
     objDir.mkdirs()
-    ExecClang(project).execClangForCompilerTests(testTarget) {
+    ExecClang.create(project).execClangForCompilerTests(testTarget) {
         args = listOf("-c", *cSources.map { it.absolutePath }.toTypedArray())
         workingDir(objDir)
     }

kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/bitcode/CompileToBitcode.kt

@@ -10,16 +10,59 @@ import org.gradle.api.file.FileCollection
 import org.gradle.api.tasks.*
 import org.gradle.kotlin.dsl.getByType
 import org.jetbrains.kotlin.ExecClang
-import org.jetbrains.kotlin.konan.target.Family
-import org.jetbrains.kotlin.konan.target.HostManager
-import org.jetbrains.kotlin.konan.target.KonanTarget
-import org.jetbrains.kotlin.konan.target.SanitizerKind
 import java.io.File
 import javax.inject.Inject
 import kotlinBuildProperties
 import isNativeRuntimeDebugInfoEnabled
+import org.gradle.api.Project
+import org.gradle.api.model.ObjectFactory
+import org.gradle.process.ExecOperations
+import org.gradle.workers.WorkAction
+import org.gradle.workers.WorkParameters
+import org.gradle.workers.WorkerExecutor
+import org.jetbrains.kotlin.konan.target.*
 
-open class CompileToBitcode @Inject constructor(
+interface CompileToBitcodeParameters : WorkParameters {
+    var objDir: File
+    var target: String
+    var compilerExecutable: String
+    var compilerArgs: List<String>
+    var llvmLinkArgs: List<String>
+
+    var konanHome: File
+    var llvmDir: File
+    var experimentalDistribution: Boolean
+}
+
+abstract class CompileToBitcodeJob : WorkAction<CompileToBitcodeParameters> {
+    @get:Inject
+    abstract val objects: ObjectFactory
+
+    @get:Inject
+    abstract val execOperations: ExecOperations
+
+    override fun execute() {
+        with(parameters) {
+            objDir.mkdirs()
+
+            val platformManager = PlatformManager(buildDistribution(konanHome.absolutePath), experimentalDistribution)
+            val execClang = ExecClang.create(objects, platformManager, llvmDir)
+
+            execClang.execKonanClang(target) {
+                workingDir = objDir
+                executable = compilerExecutable
+                args = compilerArgs
+            }
+
+            execOperations.exec {
+                executable = "${llvmDir.absolutePath}/bin/llvm-link"
+                args = llvmLinkArgs
+            }
+        }
+    }
+}
+
+abstract class CompileToBitcode @Inject constructor(
         @Internal val srcRoot: File,
         @Input val folderName: String,
         @Input val target: String,
@@ -164,24 +207,27 @@ open class CompileToBitcode @Inject constructor(
     val outFile: File
         get() = File(targetDir, "${folderName}.bc")
 
+    @get:Inject
+    abstract val workerExecutor: WorkerExecutor
+
     @TaskAction
     fun compile() {
-        objDir.mkdirs()
+        val workQueue = workerExecutor.noIsolation()
-        val plugin = project.extensions.getByType<ExecClang>()
 
-        plugin.execKonanClang(target) {
+        val parameters = { it: CompileToBitcodeParameters ->
-            workingDir = objDir
+            it.objDir = objDir
-            executable = executable
+            it.target = target
-            args = compilerFlags + inputFiles.map { it.absolutePath }
+            it.compilerExecutable = executable
-        }
+            it.compilerArgs = compilerFlags + inputFiles.map { it.absolutePath }
-
+            it.llvmLinkArgs = listOf("-o", outFile.absolutePath) + linkerArgs +
-        project.exec {
-            val llvmDir = project.findProperty("llvmDir")
-            executable = "$llvmDir/bin/llvm-link"
-            args = listOf("-o", outFile.absolutePath) + linkerArgs +
                     inputFiles.map {
                         bitcodeFileForInputFile(it).absolutePath
                     }
+
+            it.konanHome = project.project(":kotlin-native").projectDir
+            it.llvmDir = project.file(project.findProperty("llvmDir")!!)
         }
+
+        workQueue.submit(CompileToBitcodeJob::class.java, parameters)
     }
 }

kotlin-native/build.gradle

@@ -133,9 +133,8 @@ void setupHostAndTarget() {
 }
 
 void setupClang(Project project) {
-
     project.extensions.platformManager = project.project(":kotlin-native").ext.platformManager
-    project.extensions.execClang = new org.jetbrains.kotlin.ExecClang(project)
+    project.extensions.execClang = org.jetbrains.kotlin.ExecClang.create(project)
 }
 
 void loadLocalProperties() {