From fc1d7df0bc39417a2fe47428cef9c56589e9f647 Mon Sep 17 00:00:00 2001 From: Pavel Punegov Date: Wed, 13 Dec 2023 15:58:43 +0100 Subject: [PATCH] [K/N][build] Simplify stdlib build Build stdlib with a single task that also sets all supported targets to the manifest file. Remove manifest merging as it is not needed anymore. --- .../main/kotlin/org/jetbrains/kotlin/Utils.kt | 68 +------------------ kotlin-native/build.gradle | 56 ++++----------- kotlin-native/runtime/build.gradle.kts | 62 ++++++++--------- 3 files changed, 45 insertions(+), 141 deletions(-) diff --git a/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/Utils.kt b/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/Utils.kt index 8e089b4f913..1c048d6510c 100644 --- a/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/Utils.kt +++ b/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/Utils.kt @@ -2,7 +2,6 @@ * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license * that can be found in the LICENSE file. */ -@file:OptIn(ExperimentalStdlibApi::class) package org.jetbrains.kotlin @@ -12,15 +11,10 @@ import org.gradle.api.Task import org.gradle.api.file.FileCollection import org.gradle.api.plugins.ExtraPropertiesExtension import org.gradle.api.provider.Provider -import org.jetbrains.kotlin.konan.properties.loadProperties -import org.jetbrains.kotlin.konan.properties.propertyList -import org.jetbrains.kotlin.konan.properties.saveProperties +import org.gradle.api.tasks.TaskProvider import org.jetbrains.kotlin.konan.target.* import java.io.File import java.nio.file.Path -import org.jetbrains.kotlin.konan.file.File as KFile -import org.gradle.api.tasks.TaskProvider -import kotlin.collections.HashSet /** * Copy-pasted from [org.jetbrains.kotlin.library.KLIB_PROPERTY_NATIVE_TARGETS] @@ -329,66 +323,6 @@ fun targetSupportsCoreSymbolication(targetName: String) = fun targetSupportsThreads(targetName: String) = HostManager().targetByName(targetName).supportsThreads() -fun Project.mergeManifestsByTargets(source: File, destination: File) { - logger.info("Merging manifests: $source -> $destination") - - val sourceFile = KFile(source.absolutePath) - val sourceProperties = sourceFile.loadProperties() - - val destinationFile = KFile(destination.absolutePath) - val destinationProperties = destinationFile.loadProperties() - - // check that all properties except for KLIB_PROPERTY_NATIVE_TARGETS are equivalent - val mismatchedProperties = (sourceProperties.keys + destinationProperties.keys) - .asSequence() - .map { it.toString() } - .filterNot { it == KLIB_PROPERTY_NATIVE_TARGETS || it == KLIB_PROPERTY_COMPILER_VERSION } - .sorted() - .mapNotNull { propertyKey: String -> - val sourceProperty: String? = sourceProperties.getProperty(propertyKey) - val destinationProperty: String? = destinationProperties.getProperty(propertyKey) - when { - sourceProperty == null -> "\"$propertyKey\" is absent in $sourceFile" - destinationProperty == null -> "\"$propertyKey\" is absent in $destinationFile" - sourceProperty == destinationProperty -> { - // properties match, OK - null - } - sourceProperties.propertyList(propertyKey, escapeInQuotes = true).toSet() == - destinationProperties.propertyList(propertyKey, escapeInQuotes = true).toSet() -> { - // properties match, OK - null - } - else -> "\"$propertyKey\" differ: [$sourceProperty] vs [$destinationProperty]" - } - } - .toList() - - check(mismatchedProperties.isEmpty()) { - buildString { - appendln("Found mismatched properties while merging manifest files: $source -> $destination") - mismatchedProperties.joinTo(this, "\n") - } - } - - // merge KLIB_PROPERTY_NATIVE_TARGETS property - val sourceNativeTargets = sourceProperties.propertyList(KLIB_PROPERTY_NATIVE_TARGETS) - val destinationNativeTargets = destinationProperties.propertyList(KLIB_PROPERTY_NATIVE_TARGETS) - - val mergedNativeTargets = HashSet().apply { - addAll(sourceNativeTargets) - addAll(destinationNativeTargets) - } - - destinationProperties[KLIB_PROPERTY_NATIVE_TARGETS] = mergedNativeTargets.joinToString(" ") - - // Get KLIB_PROPERTY_COMPILER_VERSION source property and override the version - destinationProperties[KLIB_PROPERTY_COMPILER_VERSION] = sourceProperties.getProperty(KLIB_PROPERTY_COMPILER_VERSION) - - // Now save the properties to the destination file - destinationFile.saveProperties(destinationProperties) -} - fun Project.buildStaticLibrary(cSources: Collection, output: File, objDir: File) { delete(objDir) delete(output) diff --git a/kotlin-native/build.gradle b/kotlin-native/build.gradle index 712b4d8f96c..44a10599c91 100644 --- a/kotlin-native/build.gradle +++ b/kotlin-native/build.gradle @@ -365,6 +365,15 @@ tasks.register("distStdlibCache") { def stdlib = 'klib/common/stdlib' def stdlibDefaultComponent = "$stdlib/default" +tasks.register("distStdlib", Copy) { + from(project(":kotlin-native:runtime") + .tasks + .named("nativeStdlib") + .map { it.outputs } + ) + destinationDir project.file("$distDir/$stdlib") +} + tasks.register("crossDistRuntime") { dependsOn.addAll(targetList.collect { "${it}CrossDistRuntime" }) } @@ -373,51 +382,11 @@ tasks.register("crossDistPlatformLibs") { dependsOn.addAll(targetList.collect { "${it}PlatformLibs" }) } -tasks.register("crossDistStdlib") { - dependsOn.addAll(targetList.collect { "${it}CrossDistStdlib" }) -} - tasks.register("crossDistStdlibCache") { dependsOn.addAll(targetList.findAll { it in cacheableTargetNames }.collect { "${it}StdlibCache" }) } targetList.each { target -> - tasks.register("${target}CrossDistStdlib", Copy) { - dependsOn ":kotlin-native:runtime:${target}Stdlib" - // TODO: add explicit dependency on host task with IR klib stdlib parts - // As for now it is possibly to build up distribution from the tc-dist to crossdist - // by request of tests that need cross-targets or platform libs. It may create undesired - // issues with overwriting the file being used by the concurrent build/test. - // This building (by request) should be turned off when this to-do is fixed. -// if (target != hostName) { -// dependsOn ":kotlin-native:${hostName}CrossDistStdlib" -// } - - destinationDir project.file("$distDir/$stdlib") - - from(project(':kotlin-native:runtime').file("build/${target}Stdlib")) { - if (target == hostName) { - include('**') - } else { - // We don't want to rewrite stdlib parts as they are the same and may be already used - // by the other build phases like caching. - include('*/targets/**') - include('*/manifest') - } - eachFile { - if (name == 'manifest') { - def existingManifest = file("$destinationDir/$path") - if (existingManifest.exists()) { - UtilsKt.mergeManifestsByTargets(project, file, existingManifest) - exclude() - } - } - } - } - - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - } - tasks.register("${target}CrossDistBitcodeCopy", Copy) { def bitcodeFiles = configurations.runtimeBitcode.incoming.artifactView { attributes { @@ -434,7 +403,7 @@ targetList.each { target -> } tasks.register("${target}CrossDistRuntime", Copy) { - dependsOn ":kotlin-native:${target}CrossDistStdlib" + dependsOn ":kotlin-native:distStdlib" dependsOn "${target}CrossDistBitcodeCopy" destinationDir project.file("$distDir/$stdlibDefaultComponent") @@ -449,7 +418,7 @@ targetList.each { target -> if (target in cacheableTargetNames) { tasks.register("${target}StdlibCache", Copy) { - dependsOn "${target}CrossDistStdlib" + dependsOn "distStdlib" dependsOn ":kotlin-native:runtime:${target}StdlibCache" destinationDir project.file("$distDir/klib/cache/") @@ -477,6 +446,7 @@ tasks.register("dist") { dependsOn "distCompiler", "distRuntime", "distDef", + "distStdlib", "distStdlibCache" } @@ -484,7 +454,7 @@ tasks.register("crossDist") { dependsOn "distCompiler", "crossDistRuntime", "distDef", - "crossDistStdlib", + "distStdlib", "crossDistStdlibCache" } diff --git a/kotlin-native/runtime/build.gradle.kts b/kotlin-native/runtime/build.gradle.kts index 438f93628b7..8fe46e056b6 100644 --- a/kotlin-native/runtime/build.gradle.kts +++ b/kotlin-native/runtime/build.gradle.kts @@ -570,44 +570,44 @@ konanArtifacts { } } -targetList.forEach { targetName -> - val stdlibTask = tasks.register("${targetName}Stdlib", Copy::class.java) { - require(::stdlibBuildTask.isInitialized) - dependsOn(stdlibBuildTask) - dependsOn("${targetName}Runtime") +val stdlibTask = tasks.register("nativeStdlib") { + require(::stdlibBuildTask.isInitialized) - into(project.layout.buildDirectory.dir("${targetName}Stdlib")) + from(stdlibBuildTask.map { it.outputs }) + into(project.layout.buildDirectory.dir("nativeStdlib")) - from(project.layout.buildDirectory.dir("stdlib/${hostName}/stdlib")) + eachFile { + if (name == "manifest") { + // Stdlib is a common library that doesn't depend on anything target-specific. + // The current compiler can't create a library with manifest file that lists all supported targets. + // So, add all supported targets to the manifest file. + KFile(file.absolutePath).run { + val props = loadProperties() + props[KLIB_PROPERTY_NATIVE_TARGETS] = targetList.joinToString(separator = " ") - if (targetName != hostName) { - doLast { - // Change target in manifest file - with(KFile(destinationDir.resolve("default/manifest").absolutePath)) { - val props = loadProperties() - props[KLIB_PROPERTY_NATIVE_TARGETS] = targetName - val version = props[KLIB_PROPERTY_COMPILER_VERSION] - check(version == kotlinVersion) { - "Manifest file ($this) processing: $version was found while $kotlinVersion was expected" - } - saveProperties(props) + // Check that we didn't get other than the requested version from cache, previous build or due to some other build issue + val versionFromManifest = props[KLIB_PROPERTY_COMPILER_VERSION] + check(versionFromManifest == kotlinVersion) { + "Manifest file ($this) processing: $versionFromManifest was found while $kotlinVersion was expected" } + + saveProperties(props) } } } - - val cacheableTargetNames: List by project - - if (targetName in cacheableTargetNames) { - tasks.register("${targetName}StdlibCache", KonanCacheTask::class.java) { - target = targetName - originalKlib.fileProvider(stdlibTask.map { it.destinationDir }) - klibUniqName = "stdlib" - cacheRoot = project.layout.buildDirectory.dir("cache/$targetName").get().asFile.absolutePath - - dependsOn(":kotlin-native:${targetName}CrossDistRuntime") - } - } +} + +val cacheableTargetNames: List by project + +cacheableTargetNames.forEach { targetName -> + tasks.register("${targetName}StdlibCache", KonanCacheTask::class.java) { + target = targetName + originalKlib.fileProvider(stdlibTask.map { it.destinationDir }) + klibUniqName = "stdlib" + cacheRoot = project.layout.buildDirectory.dir("cache/$targetName").get().asFile.absolutePath + + dependsOn(":kotlin-native:${targetName}CrossDistRuntime") + } } // endregion