From 634812f96f7dc4694f0cecffa69385f8bf4a6491 Mon Sep 17 00:00:00 2001 From: Pavel Punegov Date: Mon, 11 Oct 2021 19:04:27 +0300 Subject: [PATCH] [K/N][build] Split stdlib and endorsed libraries build and caching Due to the usage of dist as an input and output at the same time by different tasks Gradle issued warning about Execution optimizations turning off. The fix is to split inputs and outputs in the build and cache tasks of stdlib and endorsed libs. --- .../kotlin/backend/konan/llvm/LlvmUtils.kt | 2 +- kotlin-native/build.gradle | 101 +++++++++++++----- kotlin-native/endorsedLibraries/build.gradle | 43 +++++--- kotlin-native/platformLibs/build.gradle | 5 +- .../gradle/plugin/konan/KonanToolRunner.kt | 16 ++- .../plugin/konan/tasks/KonanCacheTask.kt | 15 ++- 6 files changed, 131 insertions(+), 51 deletions(-) diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmUtils.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmUtils.kt index 0ba54a6d4e4..65414651aa0 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmUtils.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmUtils.kt @@ -341,7 +341,7 @@ fun parseBitcodeFile(path: String): LLVMModuleRef = memScoped { val res = LLVMCreateMemoryBufferWithContentsOfFile(path, bufRef.ptr, errorRef.ptr) if (res != 0) { - throw Error(errorRef.value?.toKString()) + throw Error("Error parsing file $path : ${errorRef.value?.toKString()}") } val memoryBuffer = bufRef.value diff --git a/kotlin-native/build.gradle b/kotlin-native/build.gradle index 5368a6e5a30..c55684c336d 100644 --- a/kotlin-native/build.gradle +++ b/kotlin-native/build.gradle @@ -324,12 +324,11 @@ task distCompiler(type: Copy) { } task distDef(type: Copy) { - - destinationDir distDir + destinationDir project.file("$distDir/konan/platformDef/") platformManager.targetValues.each { target -> from(project(":kotlin-native:platformLibs").file("src/platform/${target.family.name().toLowerCase()}")) { - into("konan/platformDef/${target.visibleName}") + into target.visibleName include '**/*.def' if (target in targetsWithoutZlib) { exclude '**/zlib.def' @@ -367,11 +366,11 @@ def stdlibDefaultComponent = "$stdlib/default" def endorsedLibs = 'klib/common/endorsedLibraries' def endorsedLibsBase = 'klib/common' -task crossDistRuntime(type: Copy) { +task crossDistRuntime { dependsOn.addAll(targetList.collect { "${it}CrossDistRuntime" }) } -task crossDistEndorsedLibraries(type: Copy) { +task crossDistEndorsedLibraries { dependsOn.addAll(targetList.collect { "${it}CrossDistEndorsedLibraries" }) } @@ -379,6 +378,10 @@ task crossDistPlatformLibs { dependsOn.addAll(targetList.collect { "${it}PlatformLibs" }) } +task crossDistStdlib { + dependsOn.addAll(targetList.collect { "${it}CrossDistStdlib" }) +} + task crossDistStdlibCache { dependsOn.addAll(targetList.findAll { it in cacheableTargetNames }.collect { "${it}StdlibCache" }) } @@ -387,16 +390,35 @@ task crossDistEndorsedCache { dependsOn.addAll(targetList.findAll { it in cacheableTargetNames }.collect { "${it}EndorsedCache" }) } +def endorsedLibsCopyTask = task("crossDistEndorsedLibrariesCopy", type: Copy) { + destinationDir project.file("$distDir/$endorsedLibsBase") + + from(project(':kotlin-native:endorsedLibraries').file("build")) { + include('**') + exclude('cache') + } +} + targetList.each { target -> - task("${target}CrossDistRuntime", type: Copy) { - dependsOn ":kotlin-native:runtime:${target}Runtime" + task("${target}CopyStdlibRuntimeBitcode", type: Copy) { dependsOn ":kotlin-native:backend.native:${target}Stdlib" - destinationDir distDir + destinationDir project(':kotlin-native:runtime') + .file("build/${target}Stdlib/default") + + from(project(':kotlin-native:runtime').file("build/bitcode/main/$target")) { + include("runtime.bc") + into("targets/$target/native") + } + } + + task ("${target}CrossDistStdlib", type: Copy) { + destinationDir project.file("$distDir/$stdlib") + + dependsOn ":kotlin-native:${target}CopyStdlibRuntimeBitcode" from(project(':kotlin-native:runtime').file("build/${target}Stdlib")) { include('**') - into(stdlib) eachFile { if (name == 'manifest') { def existingManifest = file("$destinationDir/$path") @@ -407,17 +429,31 @@ targetList.each { target -> } } } - from(project(':kotlin-native:runtime').file("build/bitcode/main/$target")) { - include("runtime.bc") - into("$stdlibDefaultComponent/targets/$target/native") - } + + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + + task("${target}CrossDistBitcodeCopy", type: Copy) { + dependsOn ":kotlin-native:runtime:${target}Runtime" + + destinationDir project.file("$distDir/konan/targets/") + from(project(':kotlin-native:runtime').file("build/bitcode/main/$target")) { include("*.bc") exclude("runtime.bc") - into("konan/targets/$target/native") + into("$target/native") } + } + + task("${target}CrossDistRuntime", type: Copy) { + dependsOn ":kotlin-native:runtime:${target}Runtime" + dependsOn ":kotlin-native:${target}CrossDistStdlib" + dependsOn "${target}CrossDistBitcodeCopy" + + destinationDir project.file("$distDir/$stdlibDefaultComponent") + if (target == 'wasm32') { - into("$stdlibDefaultComponent/targets/wasm32/included") { + into("targets/wasm32/included") { from(project(':kotlin-native:runtime').file('src/main/js')) from(project(':kotlin-native:runtime').file('src/launcher/js')) from(project(':kotlin-native:Interop:JsRuntime').file('src/main/js')) @@ -433,30 +469,42 @@ targetList.each { target -> } if (target in cacheableTargetNames) { - task "${target}StdlibCache" { + task ("${target}StdlibCache", type: Copy) { + dependsOn "${target}CrossDistStdlib" dependsOn ":kotlin-native:platformLibs:${target}StdlibCache" + + destinationDir project.file("$distDir/klib/cache") + + from("${project(":kotlin-native:runtime").buildDir}/cache/$target") { + include('**') + } } - task "${target}EndorsedCache" { + task ("${target}EndorsedCache", type: Copy) { + dependsOn "${target}CrossDistStdlib" dependsOn ":kotlin-native:endorsedLibraries:${target}Cache" + + destinationDir project.file("$distDir/klib/cache") + + from("${project(':kotlin-native:endorsedLibraries').buildDir}/cache/$target") { + include('**') + } } } task("${target}CrossDist") { - dependsOn "${target}CrossDistRuntime", "distCompiler", "${target}CrossDistEndorsedLibraries" + dependsOn "${target}CrossDistRuntime", "distCompiler" + dependsOn "${target}CrossDistEndorsedLibraries" if (target in cacheableTargetNames) { dependsOn "${target}StdlibCache", "${target}EndorsedCache" } } - task("${target}CrossDistEndorsedLibraries", type: Copy) { - dependsOn ":kotlin-native:endorsedLibraries:${target}EndorsedLibraries" - - destinationDir distDir - from(project(':kotlin-native:endorsedLibraries').file("build")) { - include('**') - into("$endorsedLibsBase") - } + task("${target}CrossDistEndorsedLibraries") { + def endorsedLibsBuildTask = ":kotlin-native:endorsedLibraries:${target}EndorsedLibraries" + dependsOn endorsedLibsBuildTask + dependsOn endorsedLibsCopyTask + endorsedLibsCopyTask.mustRunAfter(endorsedLibsBuildTask) } } @@ -479,6 +527,7 @@ task crossDist { "crossDistRuntime", "crossDistEndorsedLibraries", "distDef", + "crossDistStdlib", "crossDistStdlibCache", "crossDistEndorsedCache" } diff --git a/kotlin-native/endorsedLibraries/build.gradle b/kotlin-native/endorsedLibraries/build.gradle index cebac150af2..1783cd27ce5 100644 --- a/kotlin-native/endorsedLibraries/build.gradle +++ b/kotlin-native/endorsedLibraries/build.gradle @@ -36,23 +36,23 @@ task jvmJar { // Build all default libraries. targetList.each { target -> - task("${target}EndorsedLibraries", type: Copy) { + task("${target}EndorsedLibraries") { endorsedLibrariesList.each { library -> dependsOn "$library.projectName:${target}${library.taskName}" - } - destinationDir project.buildDir + dependsOn task("${target}${library.name}EndorsedLibraries", type: Copy) { - endorsedLibrariesList.each { library -> - from(library.project.file("build/${target}${library.taskName}")) { - include('**') - into(library.name) - eachFile { - if (name == 'manifest') { - def existingManifest = file("$destinationDir/$path") - if (existingManifest.exists()) { - UtilsKt.mergeManifestsByTargets(project, file, existingManifest) - exclude() + destinationDir project.file("${project.buildDir}/${library.name}") + + from(library.project.file("build/${target}${library.taskName}")) { + include('**') + eachFile { + if (name == 'manifest') { + def existingManifest = file("$destinationDir/$path") + if (existingManifest.exists()) { + UtilsKt.mergeManifestsByTargets(project, file, existingManifest) + exclude() + } } } } @@ -61,17 +61,26 @@ targetList.each { target -> } if (target in cacheableTargetNames) { - def cacheTask = task("${target}Cache") + def cacheTask = task("${target}Cache", type: Copy) { + destinationDir project.file("${project.buildDir}/cache/$target") + + endorsedLibrariesList.each { library -> + from(library.project.file("build/cache/$target")) { + include('**') + } + } + } endorsedLibrariesList.each { library -> def dist = UtilsKt.getKotlinNativeDist(project) task("${target}${library.taskName}Cache", type: KonanCacheTask) { it.target = target - originalKlib = file("${project.buildDir}/${library.name}") - cacheRoot = file("$dist/klib/cache") + originalKlib = library.project.file("build/${target}${library.taskName}") + cacheRoot = library.project.file("build/cache/$target").path compilerDistributionPath.set(distDir) + cachedLibrary = "${dist}/klib/common/stdlib,${dist}/klib/cache/stdlib-cache" - dependsOn "${target}EndorsedLibraries" + dependsOn "$library.projectName:${target}${library.taskName}" dependsOn ":kotlin-native:${target}CrossDistRuntime" dependsOn ":kotlin-native:${target}StdlibCache" diff --git a/kotlin-native/platformLibs/build.gradle b/kotlin-native/platformLibs/build.gradle index aa48e95b576..2b86dd15345 100644 --- a/kotlin-native/platformLibs/build.gradle +++ b/kotlin-native/platformLibs/build.gradle @@ -58,8 +58,9 @@ rootProject.project("kotlin-native").ext.platformManager.enabled.each { target - if (target in cacheableTargets) { tasks.register("${targetName}StdlibCache", KonanCacheTask) { it.target = targetName - it.originalKlib = file("$konanHome/klib/common/stdlib") - it.cacheRoot = file("$konanHome/klib/cache") + def runtimeBuild = project(":kotlin-native:runtime").buildDir + it.originalKlib = project.file("$runtimeBuild/${target}Stdlib") + it.cacheRoot = project.file("$runtimeBuild/cache/$target").path it.dependsOn ":kotlin-native:${targetName}CrossDistRuntime" } diff --git a/kotlin-native/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/KonanToolRunner.kt b/kotlin-native/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/KonanToolRunner.kt index 0c6e43557b8..065125554d9 100644 --- a/kotlin-native/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/KonanToolRunner.kt +++ b/kotlin-native/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/KonanToolRunner.kt @@ -36,6 +36,7 @@ import org.jetbrains.kotlin.konan.target.KonanTarget import org.jetbrains.kotlin.konan.util.DependencyProcessor import java.nio.file.Files import org.jetbrains.kotlin.* +import java.io.ByteArrayOutputStream internal interface KonanToolRunner : Named { val mainClass: String @@ -109,7 +110,10 @@ internal abstract class KonanCliRunner( val launcher = project.getProperty(KonanPlugin.ProjectProperty.KONAN_JVM_LAUNCHER) as? Provider ?: throw IllegalStateException("Missing property: ${KonanPlugin.ProjectProperty.KONAN_JVM_LAUNCHER}") - project.exec(object : Action { + val out = ByteArrayOutputStream() + val err = ByteArrayOutputStream() + + val execResult = project.exec(object : Action { override fun execute(exec: ExecSpec) { exec.executable = launcher.get().executablePath.toString() val properties = System.getProperties().asSequence() @@ -131,8 +135,18 @@ internal abstract class KonanCliRunner( }) blacklistEnvironment.forEach { environment.remove(it) } exec.environment(environment) + exec.errorOutput = err + exec.standardOutput = out + exec.isIgnoreExitValue = true } }) + + check(execResult.exitValue == 0) { + """ + stdout:$out + stderr:$err + """.trimIndent() + } } } diff --git a/kotlin-native/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanCacheTask.kt b/kotlin-native/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanCacheTask.kt index 474e262b4b1..fdc28bf8752 100644 --- a/kotlin-native/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanCacheTask.kt +++ b/kotlin-native/tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanCacheTask.kt @@ -20,8 +20,8 @@ open class KonanCacheTask: DefaultTask() { lateinit var originalKlib: File // Taken into account by the [cacheFile] property. - @Internal - lateinit var cacheRoot: File + @Input + lateinit var cacheRoot: String @get:Input lateinit var target: String @@ -29,7 +29,10 @@ open class KonanCacheTask: DefaultTask() { @get:Internal // TODO: Reuse NativeCacheKind from Big Kotlin plugin when it is available. val cacheDirectory: File - get() = cacheRoot.resolve("$target-g$cacheKind") + get() = File(cacheRoot).apply { + if (!exists()) mkdir() + resolve("$target-g$cacheKind") + } @get:OutputDirectory protected val cacheFile: File @@ -49,6 +52,10 @@ open class KonanCacheTask: DefaultTask() { set(project.provider { project.kotlinNativeDist }) } + @Input + @Optional + var cachedLibrary: String? = null + @TaskAction fun compile() { // Compiler doesn't create a cache if the cacheFile already exists. So we need to remove it manually. @@ -66,7 +73,7 @@ open class KonanCacheTask: DefaultTask() { "-produce", cacheKind.outputKind.name.toLowerCase(), "-Xadd-cache=${originalKlib.absolutePath}", "-Xcache-directory=${cacheDirectory.absolutePath}" - ) + additionalCacheFlags + ) + additionalCacheFlags + (cachedLibrary?.let { "-Xcached-library=$it" } ?: "") KonanCompilerRunner(project, konanHome = konanHome).run(args) } } \ No newline at end of file