[K/N][build] Split stdlib and endorsed libraries build and caching

Due to the usage of dist as an input and output at the same time by different tasks Gradle issued warning about Execution optimizations turning off. The fix is to split inputs and outputs in the build and cache tasks of stdlib and endorsed libs.
2021-10-11 19:04:27 +03:00
parent ae2d447ef4
commit 634812f96f
6 changed files with 131 additions and 51 deletions
@@ -341,7 +341,7 @@ fun parseBitcodeFile(path: String): LLVMModuleRef = memScoped {

    val res = LLVMCreateMemoryBufferWithContentsOfFile(path, bufRef.ptr, errorRef.ptr)
    if (res != 0) {
-        throw Error(errorRef.value?.toKString())
+        throw Error("Error parsing file $path : ${errorRef.value?.toKString()}")
    }

    val memoryBuffer = bufRef.value
@@ -324,12 +324,11 @@ task distCompiler(type: Copy) {
 }

 task distDef(type: Copy) {
-
-    destinationDir distDir
+    destinationDir project.file("$distDir/konan/platformDef/")

    platformManager.targetValues.each { target ->
        from(project(":kotlin-native:platformLibs").file("src/platform/${target.family.name().toLowerCase()}")) {
-            into("konan/platformDef/${target.visibleName}")
+            into target.visibleName
            include '**/*.def'
            if (target in targetsWithoutZlib) {
                exclude '**/zlib.def'
@@ -367,11 +366,11 @@ def stdlibDefaultComponent = "$stdlib/default"
 def endorsedLibs = 'klib/common/endorsedLibraries'
 def endorsedLibsBase = 'klib/common'

-task crossDistRuntime(type: Copy) {
+task crossDistRuntime {
    dependsOn.addAll(targetList.collect { "${it}CrossDistRuntime" })
 }

-task crossDistEndorsedLibraries(type: Copy) {
+task crossDistEndorsedLibraries {
    dependsOn.addAll(targetList.collect { "${it}CrossDistEndorsedLibraries" })
 }

@@ -379,6 +378,10 @@ task crossDistPlatformLibs {
    dependsOn.addAll(targetList.collect { "${it}PlatformLibs" })
 }

+task crossDistStdlib {
+    dependsOn.addAll(targetList.collect { "${it}CrossDistStdlib" })
+}
+
 task crossDistStdlibCache {
    dependsOn.addAll(targetList.findAll { it in cacheableTargetNames }.collect { "${it}StdlibCache" })
 }
@@ -387,16 +390,35 @@ task crossDistEndorsedCache {
    dependsOn.addAll(targetList.findAll { it in cacheableTargetNames }.collect { "${it}EndorsedCache" })
 }

+def endorsedLibsCopyTask = task("crossDistEndorsedLibrariesCopy", type: Copy) {
+    destinationDir project.file("$distDir/$endorsedLibsBase")
+
+    from(project(':kotlin-native:endorsedLibraries').file("build")) {
+        include('**')
+        exclude('cache')
+    }
+}
+
 targetList.each { target ->
-    task("${target}CrossDistRuntime", type: Copy) {
-        dependsOn ":kotlin-native:runtime:${target}Runtime"
+    task("${target}CopyStdlibRuntimeBitcode", type: Copy) {
        dependsOn ":kotlin-native:backend.native:${target}Stdlib"

-        destinationDir distDir
+        destinationDir project(':kotlin-native:runtime')
+                .file("build/${target}Stdlib/default")
+
+        from(project(':kotlin-native:runtime').file("build/bitcode/main/$target")) {
+            include("runtime.bc")
+            into("targets/$target/native")
+        }
+    }
+
+    task ("${target}CrossDistStdlib", type: Copy) {
+        destinationDir project.file("$distDir/$stdlib")
+
+        dependsOn ":kotlin-native:${target}CopyStdlibRuntimeBitcode"

        from(project(':kotlin-native:runtime').file("build/${target}Stdlib")) {
            include('**')
-            into(stdlib)
            eachFile {
                if (name == 'manifest') {
                    def existingManifest = file("$destinationDir/$path")
@@ -407,17 +429,31 @@ targetList.each { target ->
                }
            }
        }
-        from(project(':kotlin-native:runtime').file("build/bitcode/main/$target")) {
-            include("runtime.bc")
-            into("$stdlibDefaultComponent/targets/$target/native")
-        }
+
+        duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+    }
+
+    task("${target}CrossDistBitcodeCopy", type: Copy) {
+        dependsOn ":kotlin-native:runtime:${target}Runtime"
+
+        destinationDir project.file("$distDir/konan/targets/")
+
        from(project(':kotlin-native:runtime').file("build/bitcode/main/$target")) {
            include("*.bc")
            exclude("runtime.bc")
-            into("konan/targets/$target/native")
+            into("$target/native")
        }
+    }
+
+    task("${target}CrossDistRuntime", type: Copy) {
+        dependsOn ":kotlin-native:runtime:${target}Runtime"
+        dependsOn ":kotlin-native:${target}CrossDistStdlib"
+        dependsOn "${target}CrossDistBitcodeCopy"
+
+        destinationDir project.file("$distDir/$stdlibDefaultComponent")
+
        if (target == 'wasm32') {
-            into("$stdlibDefaultComponent/targets/wasm32/included") {
+            into("targets/wasm32/included") {
                from(project(':kotlin-native:runtime').file('src/main/js'))
                from(project(':kotlin-native:runtime').file('src/launcher/js'))
                from(project(':kotlin-native:Interop:JsRuntime').file('src/main/js'))
@@ -433,30 +469,42 @@ targetList.each { target ->
    }

    if (target in cacheableTargetNames) {
-        task "${target}StdlibCache" {
+        task ("${target}StdlibCache", type: Copy) {
+            dependsOn "${target}CrossDistStdlib"
            dependsOn ":kotlin-native:platformLibs:${target}StdlibCache"
+
+            destinationDir project.file("$distDir/klib/cache")
+
+            from("${project(":kotlin-native:runtime").buildDir}/cache/$target") {
+                include('**')
+            }
        }

-        task "${target}EndorsedCache" {
+        task ("${target}EndorsedCache", type: Copy) {
+            dependsOn "${target}CrossDistStdlib"
            dependsOn ":kotlin-native:endorsedLibraries:${target}Cache"
+
+            destinationDir project.file("$distDir/klib/cache")
+
+            from("${project(':kotlin-native:endorsedLibraries').buildDir}/cache/$target") {
+                include('**')
+            }
        }
    }

    task("${target}CrossDist") {
-        dependsOn "${target}CrossDistRuntime", "distCompiler", "${target}CrossDistEndorsedLibraries"
+        dependsOn "${target}CrossDistRuntime", "distCompiler"
+        dependsOn "${target}CrossDistEndorsedLibraries"
        if (target in cacheableTargetNames) {
            dependsOn "${target}StdlibCache", "${target}EndorsedCache"
        }
    }

-    task("${target}CrossDistEndorsedLibraries", type: Copy) {
-        dependsOn ":kotlin-native:endorsedLibraries:${target}EndorsedLibraries"
-
-        destinationDir distDir
-        from(project(':kotlin-native:endorsedLibraries').file("build")) {
-            include('**')
-            into("$endorsedLibsBase")
-        }
+    task("${target}CrossDistEndorsedLibraries") {
+        def endorsedLibsBuildTask = ":kotlin-native:endorsedLibraries:${target}EndorsedLibraries"
+        dependsOn endorsedLibsBuildTask
+        dependsOn endorsedLibsCopyTask
+        endorsedLibsCopyTask.mustRunAfter(endorsedLibsBuildTask)
    }
 }

@@ -479,6 +527,7 @@ task crossDist {
            "crossDistRuntime",
            "crossDistEndorsedLibraries",
            "distDef",
+            "crossDistStdlib",
            "crossDistStdlibCache",
            "crossDistEndorsedCache"
 }
@@ -36,23 +36,23 @@ task jvmJar {

 // Build all default libraries.
 targetList.each { target ->
-    task("${target}EndorsedLibraries", type: Copy) {
+    task("${target}EndorsedLibraries") {
        endorsedLibrariesList.each { library ->
            dependsOn "$library.projectName:${target}${library.taskName}"
-        }

-        destinationDir project.buildDir
+            dependsOn task("${target}${library.name}EndorsedLibraries", type: Copy) {

-        endorsedLibrariesList.each { library ->
-            from(library.project.file("build/${target}${library.taskName}")) {
-                include('**')
-                into(library.name)
-                eachFile {
-                    if (name == 'manifest') {
-                        def existingManifest = file("$destinationDir/$path")
-                        if (existingManifest.exists()) {
-                            UtilsKt.mergeManifestsByTargets(project, file, existingManifest)
-                            exclude()
+                destinationDir project.file("${project.buildDir}/${library.name}")
+
+                from(library.project.file("build/${target}${library.taskName}")) {
+                    include('**')
+                    eachFile {
+                        if (name == 'manifest') {
+                            def existingManifest = file("$destinationDir/$path")
+                            if (existingManifest.exists()) {
+                                UtilsKt.mergeManifestsByTargets(project, file, existingManifest)
+                                exclude()
+                            }
                        }
                    }
                }
@@ -61,17 +61,26 @@ targetList.each { target ->
    }

    if (target in cacheableTargetNames) {
-        def cacheTask = task("${target}Cache")
+        def cacheTask = task("${target}Cache", type: Copy) {
+            destinationDir project.file("${project.buildDir}/cache/$target")
+
+            endorsedLibrariesList.each { library ->
+                from(library.project.file("build/cache/$target")) {
+                    include('**')
+                }
+            }
+        }

        endorsedLibrariesList.each { library ->
            def dist = UtilsKt.getKotlinNativeDist(project)
            task("${target}${library.taskName}Cache", type: KonanCacheTask) {
                it.target = target
-                originalKlib = file("${project.buildDir}/${library.name}")
-                cacheRoot = file("$dist/klib/cache")
+                originalKlib = library.project.file("build/${target}${library.taskName}")
+                cacheRoot = library.project.file("build/cache/$target").path
                compilerDistributionPath.set(distDir)
+                cachedLibrary = "${dist}/klib/common/stdlib,${dist}/klib/cache/stdlib-cache"

-                dependsOn "${target}EndorsedLibraries"
+                dependsOn "$library.projectName:${target}${library.taskName}"
                dependsOn ":kotlin-native:${target}CrossDistRuntime"
                dependsOn ":kotlin-native:${target}StdlibCache"

@@ -58,8 +58,9 @@ rootProject.project("kotlin-native").ext.platformManager.enabled.each { target -
    if (target in cacheableTargets) {
        tasks.register("${targetName}StdlibCache", KonanCacheTask) {
            it.target = targetName
-            it.originalKlib = file("$konanHome/klib/common/stdlib")
-            it.cacheRoot = file("$konanHome/klib/cache")
+            def runtimeBuild = project(":kotlin-native:runtime").buildDir
+            it.originalKlib = project.file("$runtimeBuild/${target}Stdlib")
+            it.cacheRoot = project.file("$runtimeBuild/cache/$target").path

            it.dependsOn ":kotlin-native:${targetName}CrossDistRuntime"
        }
@@ -36,6 +36,7 @@ import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.konan.util.DependencyProcessor
 import java.nio.file.Files
 import org.jetbrains.kotlin.*
+import java.io.ByteArrayOutputStream

 internal interface KonanToolRunner : Named {
    val mainClass: String
@@ -109,7 +110,10 @@ internal abstract class KonanCliRunner(
        val launcher = project.getProperty(KonanPlugin.ProjectProperty.KONAN_JVM_LAUNCHER) as? Provider<JavaLauncher>
                ?: throw IllegalStateException("Missing property: ${KonanPlugin.ProjectProperty.KONAN_JVM_LAUNCHER}")

-        project.exec(object : Action<ExecSpec> {
+        val out = ByteArrayOutputStream()
+        val err = ByteArrayOutputStream()
+
+        val execResult = project.exec(object : Action<ExecSpec> {
            override fun execute(exec: ExecSpec) {
                exec.executable = launcher.get().executablePath.toString()
                val properties = System.getProperties().asSequence()
@@ -131,8 +135,18 @@ internal abstract class KonanCliRunner(
                })
                blacklistEnvironment.forEach { environment.remove(it) }
                exec.environment(environment)
+                exec.errorOutput = err
+                exec.standardOutput = out
+                exec.isIgnoreExitValue = true
            }
        })
+
+        check(execResult.exitValue == 0) {
+            """
+                stdout:$out
+                stderr:$err
+            """.trimIndent()
+        }
    }
 }

@@ -20,8 +20,8 @@ open class KonanCacheTask: DefaultTask() {
    lateinit var originalKlib: File

    // Taken into account by the [cacheFile] property.
-    @Internal
-    lateinit var cacheRoot: File
+    @Input
+    lateinit var cacheRoot: String

    @get:Input
    lateinit var target: String
@@ -29,7 +29,10 @@ open class KonanCacheTask: DefaultTask() {
    @get:Internal
    // TODO: Reuse NativeCacheKind from Big Kotlin plugin when it is available.
    val cacheDirectory: File
-        get() = cacheRoot.resolve("$target-g$cacheKind")
+        get() = File(cacheRoot).apply {
+            if (!exists()) mkdir()
+            resolve("$target-g$cacheKind")
+        }

    @get:OutputDirectory
    protected val cacheFile: File
@@ -49,6 +52,10 @@ open class KonanCacheTask: DefaultTask() {
        set(project.provider { project.kotlinNativeDist })
    }

+    @Input
+    @Optional
+    var cachedLibrary: String? = null
+
    @TaskAction
    fun compile() {
        // Compiler doesn't create a cache if the cacheFile already exists. So we need to remove it manually.
@@ -66,7 +73,7 @@ open class KonanCacheTask: DefaultTask() {
            "-produce", cacheKind.outputKind.name.toLowerCase(),
            "-Xadd-cache=${originalKlib.absolutePath}",
            "-Xcache-directory=${cacheDirectory.absolutePath}"
-        ) + additionalCacheFlags
+        ) + additionalCacheFlags + (cachedLibrary?.let { "-Xcached-library=$it" } ?: "")
        KonanCompilerRunner(project, konanHome = konanHome).run(args)
    }
 }