[K/N][build] Split stdlib and endorsed libraries build and caching

Due to the usage of dist as an input and output at the same time by
different tasks Gradle issued warning about Execution optimizations
turning off. The fix is to split inputs and outputs in the build and
cache tasks of stdlib and endorsed libs.

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmUtils.kt

@@ -341,7 +341,7 @@ fun parseBitcodeFile(path: String): LLVMModuleRef = memScoped {
 
     val res = LLVMCreateMemoryBufferWithContentsOfFile(path, bufRef.ptr, errorRef.ptr)
     if (res != 0) {
-        throw Error(errorRef.value?.toKString())
+        throw Error("Error parsing file $path : ${errorRef.value?.toKString()}")
     }
 
     val memoryBuffer = bufRef.value

kotlin-native/build.gradle

@@ -324,12 +324,11 @@ task distCompiler(type: Copy) {
 }
 
 task distDef(type: Copy) {
-
+    destinationDir project.file("$distDir/konan/platformDef/")
-    destinationDir distDir
 
     platformManager.targetValues.each { target ->
         from(project(":kotlin-native:platformLibs").file("src/platform/${target.family.name().toLowerCase()}")) {
-            into("konan/platformDef/${target.visibleName}")
+            into target.visibleName
             include '**/*.def'
             if (target in targetsWithoutZlib) {
                 exclude '**/zlib.def'
@@ -367,11 +366,11 @@ def stdlibDefaultComponent = "$stdlib/default"
 def endorsedLibs = 'klib/common/endorsedLibraries'
 def endorsedLibsBase = 'klib/common'
 
-task crossDistRuntime(type: Copy) {
+task crossDistRuntime {
     dependsOn.addAll(targetList.collect { "${it}CrossDistRuntime" })
 }
 
-task crossDistEndorsedLibraries(type: Copy) {
+task crossDistEndorsedLibraries {
     dependsOn.addAll(targetList.collect { "${it}CrossDistEndorsedLibraries" })
 }
 
@@ -379,6 +378,10 @@ task crossDistPlatformLibs {
     dependsOn.addAll(targetList.collect { "${it}PlatformLibs" })
 }
 
+task crossDistStdlib {
+    dependsOn.addAll(targetList.collect { "${it}CrossDistStdlib" })
+}
+
 task crossDistStdlibCache {
     dependsOn.addAll(targetList.findAll { it in cacheableTargetNames }.collect { "${it}StdlibCache" })
 }
@@ -387,16 +390,35 @@ task crossDistEndorsedCache {
     dependsOn.addAll(targetList.findAll { it in cacheableTargetNames }.collect { "${it}EndorsedCache" })
 }
 
+def endorsedLibsCopyTask = task("crossDistEndorsedLibrariesCopy", type: Copy) {
+    destinationDir project.file("$distDir/$endorsedLibsBase")
+
+    from(project(':kotlin-native:endorsedLibraries').file("build")) {
+        include('**')
+        exclude('cache')
+    }
+}
+
 targetList.each { target ->
-    task("${target}CrossDistRuntime", type: Copy) {
+    task("${target}CopyStdlibRuntimeBitcode", type: Copy) {
-        dependsOn ":kotlin-native:runtime:${target}Runtime"
         dependsOn ":kotlin-native:backend.native:${target}Stdlib"
 
-        destinationDir distDir
+        destinationDir project(':kotlin-native:runtime')
+                .file("build/${target}Stdlib/default")
+
+        from(project(':kotlin-native:runtime').file("build/bitcode/main/$target")) {
+            include("runtime.bc")
+            into("targets/$target/native")
+        }
+    }
+
+    task ("${target}CrossDistStdlib", type: Copy) {
+        destinationDir project.file("$distDir/$stdlib")
+
+        dependsOn ":kotlin-native:${target}CopyStdlibRuntimeBitcode"
 
         from(project(':kotlin-native:runtime').file("build/${target}Stdlib")) {
             include('**')
-            into(stdlib)
             eachFile {
                 if (name == 'manifest') {
                     def existingManifest = file("$destinationDir/$path")
@@ -407,17 +429,31 @@ targetList.each { target ->
                 }
             }
         }
-        from(project(':kotlin-native:runtime').file("build/bitcode/main/$target")) {
+
-            include("runtime.bc")
+        duplicatesStrategy = DuplicatesStrategy.EXCLUDE
-            into("$stdlibDefaultComponent/targets/$target/native")
+    }
-        }
+
+    task("${target}CrossDistBitcodeCopy", type: Copy) {
+        dependsOn ":kotlin-native:runtime:${target}Runtime"
+
+        destinationDir project.file("$distDir/konan/targets/")
+
         from(project(':kotlin-native:runtime').file("build/bitcode/main/$target")) {
             include("*.bc")
             exclude("runtime.bc")
-            into("konan/targets/$target/native")
+            into("$target/native")
         }
+    }
+
+    task("${target}CrossDistRuntime", type: Copy) {
+        dependsOn ":kotlin-native:runtime:${target}Runtime"
+        dependsOn ":kotlin-native:${target}CrossDistStdlib"
+        dependsOn "${target}CrossDistBitcodeCopy"
+
+        destinationDir project.file("$distDir/$stdlibDefaultComponent")
+
         if (target == 'wasm32') {
-            into("$stdlibDefaultComponent/targets/wasm32/included") {
+            into("targets/wasm32/included") {
                 from(project(':kotlin-native:runtime').file('src/main/js'))
                 from(project(':kotlin-native:runtime').file('src/launcher/js'))
                 from(project(':kotlin-native:Interop:JsRuntime').file('src/main/js'))
@@ -433,30 +469,42 @@ targetList.each { target ->
     }
 
     if (target in cacheableTargetNames) {
-        task "${target}StdlibCache" {
+        task ("${target}StdlibCache", type: Copy) {
+            dependsOn "${target}CrossDistStdlib"
             dependsOn ":kotlin-native:platformLibs:${target}StdlibCache"
+
+            destinationDir project.file("$distDir/klib/cache")
+
+            from("${project(":kotlin-native:runtime").buildDir}/cache/$target") {
+                include('**')
+            }
         }
 
-        task "${target}EndorsedCache" {
+        task ("${target}EndorsedCache", type: Copy) {
+            dependsOn "${target}CrossDistStdlib"
             dependsOn ":kotlin-native:endorsedLibraries:${target}Cache"
+
+            destinationDir project.file("$distDir/klib/cache")
+
+            from("${project(':kotlin-native:endorsedLibraries').buildDir}/cache/$target") {
+                include('**')
+            }
         }
     }
 
     task("${target}CrossDist") {
-        dependsOn "${target}CrossDistRuntime", "distCompiler", "${target}CrossDistEndorsedLibraries"
+        dependsOn "${target}CrossDistRuntime", "distCompiler"
+        dependsOn "${target}CrossDistEndorsedLibraries"
         if (target in cacheableTargetNames) {
             dependsOn "${target}StdlibCache", "${target}EndorsedCache"
         }
     }
 
-    task("${target}CrossDistEndorsedLibraries", type: Copy) {
+    task("${target}CrossDistEndorsedLibraries") {
-        dependsOn ":kotlin-native:endorsedLibraries:${target}EndorsedLibraries"
+        def endorsedLibsBuildTask = ":kotlin-native:endorsedLibraries:${target}EndorsedLibraries"
-
+        dependsOn endorsedLibsBuildTask
-        destinationDir distDir
+        dependsOn endorsedLibsCopyTask
-        from(project(':kotlin-native:endorsedLibraries').file("build")) {
+        endorsedLibsCopyTask.mustRunAfter(endorsedLibsBuildTask)
-            include('**')
-            into("$endorsedLibsBase")
-        }
     }
 }
 
@@ -479,6 +527,7 @@ task crossDist {
             "crossDistRuntime",
             "crossDistEndorsedLibraries",
             "distDef",
+            "crossDistStdlib",
             "crossDistStdlibCache",
             "crossDistEndorsedCache"
 }

kotlin-native/endorsedLibraries/build.gradle

@@ -36,23 +36,23 @@ task jvmJar {
 
 // Build all default libraries.
 targetList.each { target ->
-    task("${target}EndorsedLibraries", type: Copy) {
+    task("${target}EndorsedLibraries") {
         endorsedLibrariesList.each { library ->
             dependsOn "$library.projectName:${target}${library.taskName}"
-        }
 
-        destinationDir project.buildDir
+            dependsOn task("${target}${library.name}EndorsedLibraries", type: Copy) {
 
-        endorsedLibrariesList.each { library ->
+                destinationDir project.file("${project.buildDir}/${library.name}")
-            from(library.project.file("build/${target}${library.taskName}")) {
+
-                include('**')
+                from(library.project.file("build/${target}${library.taskName}")) {
-                into(library.name)
+                    include('**')
-                eachFile {
+                    eachFile {
-                    if (name == 'manifest') {
+                        if (name == 'manifest') {
-                        def existingManifest = file("$destinationDir/$path")
+                            def existingManifest = file("$destinationDir/$path")
-                        if (existingManifest.exists()) {
+                            if (existingManifest.exists()) {
-                            UtilsKt.mergeManifestsByTargets(project, file, existingManifest)
+                                UtilsKt.mergeManifestsByTargets(project, file, existingManifest)
-                            exclude()
+                                exclude()
+                            }
                         }
                     }
                 }
@@ -61,17 +61,26 @@ targetList.each { target ->
     }
 
     if (target in cacheableTargetNames) {
-        def cacheTask = task("${target}Cache")
+        def cacheTask = task("${target}Cache", type: Copy) {
+            destinationDir project.file("${project.buildDir}/cache/$target")
+
+            endorsedLibrariesList.each { library ->
+                from(library.project.file("build/cache/$target")) {
+                    include('**')
+                }
+            }
+        }
 
         endorsedLibrariesList.each { library ->
             def dist = UtilsKt.getKotlinNativeDist(project)
             task("${target}${library.taskName}Cache", type: KonanCacheTask) {
                 it.target = target
-                originalKlib = file("${project.buildDir}/${library.name}")
+                originalKlib = library.project.file("build/${target}${library.taskName}")
-                cacheRoot = file("$dist/klib/cache")
+                cacheRoot = library.project.file("build/cache/$target").path
                 compilerDistributionPath.set(distDir)
+                cachedLibrary = "${dist}/klib/common/stdlib,${dist}/klib/cache/stdlib-cache"
 
-                dependsOn "${target}EndorsedLibraries"
+                dependsOn "$library.projectName:${target}${library.taskName}"
                 dependsOn ":kotlin-native:${target}CrossDistRuntime"
                 dependsOn ":kotlin-native:${target}StdlibCache"
 

kotlin-native/platformLibs/build.gradle

@@ -58,8 +58,9 @@ rootProject.project("kotlin-native").ext.platformManager.enabled.each { target -
     if (target in cacheableTargets) {
         tasks.register("${targetName}StdlibCache", KonanCacheTask) {
             it.target = targetName
-            it.originalKlib = file("$konanHome/klib/common/stdlib")
+            def runtimeBuild = project(":kotlin-native:runtime").buildDir
-            it.cacheRoot = file("$konanHome/klib/cache")
+            it.originalKlib = project.file("$runtimeBuild/${target}Stdlib")
+            it.cacheRoot = project.file("$runtimeBuild/cache/$target").path
 
             it.dependsOn ":kotlin-native:${targetName}CrossDistRuntime"
         }

kotlin-native/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/KonanToolRunner.kt

@@ -36,6 +36,7 @@ import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.konan.util.DependencyProcessor
 import java.nio.file.Files
 import org.jetbrains.kotlin.*
+import java.io.ByteArrayOutputStream
 
 internal interface KonanToolRunner : Named {
     val mainClass: String
@@ -109,7 +110,10 @@ internal abstract class KonanCliRunner(
         val launcher = project.getProperty(KonanPlugin.ProjectProperty.KONAN_JVM_LAUNCHER) as? Provider<JavaLauncher>
                 ?: throw IllegalStateException("Missing property: ${KonanPlugin.ProjectProperty.KONAN_JVM_LAUNCHER}")
 
-        project.exec(object : Action<ExecSpec> {
+        val out = ByteArrayOutputStream()
+        val err = ByteArrayOutputStream()
+
+        val execResult = project.exec(object : Action<ExecSpec> {
             override fun execute(exec: ExecSpec) {
                 exec.executable = launcher.get().executablePath.toString()
                 val properties = System.getProperties().asSequence()
@@ -131,8 +135,18 @@ internal abstract class KonanCliRunner(
                 })
                 blacklistEnvironment.forEach { environment.remove(it) }
                 exec.environment(environment)
+                exec.errorOutput = err
+                exec.standardOutput = out
+                exec.isIgnoreExitValue = true
             }
         })
+
+        check(execResult.exitValue == 0) {
+            """
+                stdout:$out
+                stderr:$err
+            """.trimIndent()
+        }
     }
 }
 

kotlin-native/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanCacheTask.kt

@@ -20,8 +20,8 @@ open class KonanCacheTask: DefaultTask() {
     lateinit var originalKlib: File
 
     // Taken into account by the [cacheFile] property.
-    @Internal
+    @Input
-    lateinit var cacheRoot: File
+    lateinit var cacheRoot: String
 
     @get:Input
     lateinit var target: String
@@ -29,7 +29,10 @@ open class KonanCacheTask: DefaultTask() {
     @get:Internal
     // TODO: Reuse NativeCacheKind from Big Kotlin plugin when it is available.
     val cacheDirectory: File
-        get() = cacheRoot.resolve("$target-g$cacheKind")
+        get() = File(cacheRoot).apply {
+            if (!exists()) mkdir()
+            resolve("$target-g$cacheKind")
+        }
 
     @get:OutputDirectory
     protected val cacheFile: File
@@ -49,6 +52,10 @@ open class KonanCacheTask: DefaultTask() {
         set(project.provider { project.kotlinNativeDist })
     }
 
+    @Input
+    @Optional
+    var cachedLibrary: String? = null
+
     @TaskAction
     fun compile() {
         // Compiler doesn't create a cache if the cacheFile already exists. So we need to remove it manually.
@@ -66,7 +73,7 @@ open class KonanCacheTask: DefaultTask() {
             "-produce", cacheKind.outputKind.name.toLowerCase(),
             "-Xadd-cache=${originalKlib.absolutePath}",
             "-Xcache-directory=${cacheDirectory.absolutePath}"
-        ) + additionalCacheFlags
+        ) + additionalCacheFlags + (cachedLibrary?.let { "-Xcached-library=$it" } ?: "")
         KonanCompilerRunner(project, konanHome = konanHome).run(args)
     }
 }