[Gradle Native] Made a k/n bundle as input for commonizeNativeDist task

Also, extracted moving konan bundle to konanHome into BuildService and wrapped moving k/n bundle to konan home with file lock.
And added an integration test for checking ^KT-65222 and ^KT-65347.

^KT-65450
^KT-65347 Fixed


Merge-request: KT-MR-14099
Merged-by: Dmitrii Krasnov <Dmitrii.Krasnov@jetbrains.com>
This commit is contained in:
Dmitrii Krasnov
2024-02-05 16:37:49 +00:00
committed by Space Team
parent f262247b2b
commit 06a3e9135c
12 changed files with 200 additions and 85 deletions
@@ -80,24 +80,22 @@ class KotlinNativeCompilerDownloadIT : KGPBaseTest() {
}
}
@DisplayName("KT-65222: Kotlin Native must be downloaded during even if klib dir exists")
@DisplayName(
"KT-65222, KT-65347: check that commonize native distribution depends on downloaded k/n bundle " +
"and downloaded bundle does not erase installed caches"
)
@GradleTest
fun shouldDownloadKotlinNativeWithExistingKlibDir(gradleVersion: GradleVersion, @TempDir konanTemp: Path) {
fun checkCommonizeNativeDistributionWithPlatform(gradleVersion: GradleVersion, @TempDir konanTemp: Path) {
nativeProject(
"native-simple-project",
"commonize-native-distribution",
gradleVersion,
buildOptions = defaultBuildOptions.copy(
konanDataDir = konanTemp,
),
) {
build(":commonizeNativeDistribution") {
assertDirectoryExists(konanTemp.resolve(STABLE_VERSION_DIR_NAME).resolve("klib"))
assertDirectoryDoesNotExist(konanTemp.resolve(STABLE_VERSION_DIR_NAME).resolve("bin"))
}
build("assemble") {
assertOutputDoesNotContain(DOWNLOAD_KONAN_FINISHED_LOG)
assertOutputContains(UNPUCK_KONAN_FINISHED_LOG)
assertOutputDoesNotContain("Please wait while Kotlin/Native")
build(":compileNativeMainKotlinMetadata") {
assertDirectoryExists(konanTemp.resolve(STABLE_VERSION_DIR_NAME).resolve("klib").resolve("platform"))
assertDirectoryExists(konanTemp.resolve(STABLE_VERSION_DIR_NAME).resolve("bin"))
}
}
}
@@ -0,0 +1,13 @@
plugins {
kotlin("multiplatform")
}
repositories {
mavenCentral()
mavenLocal()
}
kotlin {
linuxX64()
linuxArm64()
}
@@ -0,0 +1,5 @@
import platform.posix.*
fun main() {
println("Hello from simple native project")
}
@@ -45,6 +45,8 @@ import org.jetbrains.kotlin.gradle.targets.native.internal.CInteropCommonizerArt
import org.jetbrains.kotlin.gradle.targets.native.internal.CInteropKlibLibraryElements
import org.jetbrains.kotlin.gradle.targets.native.internal.CommonizerTargetAttribute
import org.jetbrains.kotlin.gradle.targets.native.toolchain.KotlinNativeBundleArtifactFormat
import org.jetbrains.kotlin.gradle.targets.native.toolchain.KotlinNativeBundleArtifactFormat.addKotlinNativeBundleConfiguration
import org.jetbrains.kotlin.gradle.targets.native.toolchain.KotlinNativeBundleBuildService
import org.jetbrains.kotlin.gradle.tasks.AbstractKotlinCompileTool
import org.jetbrains.kotlin.gradle.testing.internal.KotlinTestsRegistry
import org.jetbrains.kotlin.gradle.utils.*
@@ -85,6 +87,7 @@ abstract class DefaultKotlinBasePlugin : KotlinBasePlugin {
}
BuildMetricsService.registerIfAbsent(project)
KotlinNativeBundleBuildService.registerIfAbsent(project)
}
private fun addKotlinCompilerConfiguration(project: Project) {
@@ -120,19 +123,6 @@ abstract class DefaultKotlinBasePlugin : KotlinBasePlugin {
}
}
private fun addKotlinNativeBundleConfiguration(project: Project) {
project.configurations
.maybeCreateResolvable(KOTLIN_NATIVE_BUNDLE_CONFIGURATION_NAME).also { configuration ->
configuration.defaultDependencies {
it.add(project.dependencies.create(NativeCompilerDownloader.getCompilerDependencyNotation(project)))
}
configuration.attributes.setAttribute(
KotlinNativeBundleArtifactFormat.attribute,
KotlinNativeBundleArtifactFormat.KotlinNativeBundleArtifactsTypes.DIRECTORY
)
}
}
private fun Project.registerDefaultVariantImplementations() {
val factories = VariantImplementationFactoriesConfigurator.get(project.gradle)
factories.putIfAbsent(
@@ -14,6 +14,7 @@ import org.gradle.api.tasks.Internal
import org.gradle.api.tasks.TaskAction
import org.gradle.api.tasks.TaskProvider
import org.gradle.work.DisableCachingByDefault
import org.jetbrains.kotlin.compilerRunner.kotlinNativeToolchainEnabled
import org.jetbrains.kotlin.compilerRunner.maybeCreateCommonizerClasspathConfiguration
import org.jetbrains.kotlin.gradle.internal.isInIdeaSync
import org.jetbrains.kotlin.gradle.plugin.*
@@ -23,6 +24,9 @@ import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.Companion.kotlinPro
import org.jetbrains.kotlin.gradle.plugin.await
import org.jetbrains.kotlin.gradle.plugin.ide.Idea222Api
import org.jetbrains.kotlin.gradle.plugin.ide.ideaImportDependsOn
import org.jetbrains.kotlin.gradle.targets.native.toolchain.KotlinNativeBundleArtifactFormat
import org.jetbrains.kotlin.gradle.targets.native.toolchain.KotlinNativeBundleArtifactFormat.addKotlinNativeBundleConfiguration
import org.jetbrains.kotlin.gradle.targets.native.toolchain.KotlinNativeBundleBuildService
import org.jetbrains.kotlin.gradle.utils.whenEvaluated
import org.jetbrains.kotlin.gradle.tasks.dependsOn
import org.jetbrains.kotlin.gradle.tasks.locateOrRegisterTask
@@ -129,6 +133,12 @@ internal suspend fun Project.copyCommonizeCInteropForIdeTask(): TaskProvider<Cop
internal val Project.commonizeNativeDistributionTask: TaskProvider<NativeDistributionCommonizerTask>?
get() {
if (!isAllowCommonizer()) return null
if (project.kotlinNativeToolchainEnabled) {
KotlinNativeBundleArtifactFormat.setupAttributesMatchingStrategy(rootProject.dependencies.attributesSchema)
KotlinNativeBundleArtifactFormat.setupTransform(rootProject)
addKotlinNativeBundleConfiguration(rootProject)
KotlinNativeBundleBuildService.registerIfAbsent(rootProject)
}
return rootProject.locateOrRegisterTask(
"commonizeNativeDistribution",
invokeWhenRegistered = {
@@ -14,17 +14,14 @@ import org.gradle.api.model.ObjectFactory
import org.gradle.api.provider.ListProperty
import org.gradle.api.provider.Property
import org.gradle.api.provider.Provider
import org.gradle.api.tasks.Classpath
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.Internal
import org.gradle.api.tasks.OutputDirectory
import org.gradle.api.tasks.TaskAction
import org.gradle.api.tasks.*
import org.gradle.process.ExecOperations
import org.gradle.work.DisableCachingByDefault
import org.jetbrains.kotlin.build.report.metrics.BuildMetricsReporter
import org.jetbrains.kotlin.build.report.metrics.GradleBuildPerformanceMetric
import org.jetbrains.kotlin.build.report.metrics.GradleBuildTime
import org.jetbrains.kotlin.commonizer.SharedCommonizerTarget
import org.jetbrains.kotlin.commonizer.konanTargets
import org.jetbrains.kotlin.compilerRunner.*
import org.jetbrains.kotlin.compilerRunner.GradleCliCommonizer
import org.jetbrains.kotlin.compilerRunner.KotlinNativeCommonizerToolRunner
@@ -34,6 +31,8 @@ import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.Companion.kotlinPro
import org.jetbrains.kotlin.gradle.plugin.getKotlinPluginVersion
import org.jetbrains.kotlin.gradle.report.GradleBuildMetricsReporter
import org.jetbrains.kotlin.gradle.report.UsesBuildMetricsService
import org.jetbrains.kotlin.gradle.targets.native.toolchain.KotlinNativeProvider
import org.jetbrains.kotlin.gradle.targets.native.toolchain.UsesKotlinNativeBundleBuildService
import org.jetbrains.kotlin.gradle.utils.chainedFinalizeValueOnRead
import org.jetbrains.kotlin.gradle.utils.directoryProperty
import org.jetbrains.kotlin.gradle.utils.listProperty
@@ -50,7 +49,7 @@ internal abstract class NativeDistributionCommonizerTask
private val objectFactory: ObjectFactory,
private val execOperations: ExecOperations,
private val projectLayout: ProjectLayout,
) : DefaultTask(), UsesBuildMetricsService {
) : DefaultTask(), UsesBuildMetricsService, UsesKotlinNativeBundleBuildService {
private val konanHome = project.file(project.konanHome)
@@ -104,6 +103,11 @@ internal abstract class NativeDistributionCommonizerTask
val metrics: Property<BuildMetricsReporter<GradleBuildTime, GradleBuildPerformanceMetric>> = project.objects
.property(GradleBuildMetricsReporter())
@get:Nested
internal val kotlinNativeProvider: Provider<KotlinNativeProvider> = project.provider {
KotlinNativeProvider(project, commonizerTargets.flatMap { target -> target.konanTargets }.toSet(), kotlinNativeBundleBuildService)
}
@TaskAction
protected fun run() {
val metricsReporter = metrics.get()
@@ -133,6 +137,9 @@ internal abstract class NativeDistributionCommonizerTask
init {
outputs.upToDateWhen {
// upToDateWhen executes after configuration phase, but before inputs are calculated,
// that is why we need to get k/n bundle before commonizerCache.isUpToDate here
kotlinNativeProvider.get().kotlinNativeBundleVersion.get()
commonizerCache.isUpToDate(commonizerTargets)
}
}
@@ -33,6 +33,7 @@ import org.jetbrains.kotlin.gradle.report.UsesBuildMetricsService
import org.jetbrains.kotlin.gradle.targets.native.UsesKonanPropertiesBuildService
import org.jetbrains.kotlin.gradle.targets.native.tasks.CompilerPluginData
import org.jetbrains.kotlin.gradle.targets.native.toolchain.KotlinNativeProvider
import org.jetbrains.kotlin.gradle.targets.native.toolchain.UsesKotlinNativeBundleBuildService
import org.jetbrains.kotlin.gradle.utils.*
import org.jetbrains.kotlin.konan.target.CompilerOutputKind
import org.jetbrains.kotlin.konan.target.KonanTarget
@@ -56,7 +57,8 @@ constructor(
) : AbstractKotlinCompileTool<K2NativeCompilerArguments>(objectFactory),
UsesKonanPropertiesBuildService,
UsesBuildMetricsService,
KotlinToolTask<KotlinCommonCompilerToolOptions> {
KotlinToolTask<KotlinCommonCompilerToolOptions>,
UsesKotlinNativeBundleBuildService {
@Deprecated("Visibility will be lifted to private in the future releases")
@get:Internal
@@ -358,7 +360,7 @@ constructor(
@get:Nested
internal val kotlinNativeProvider: Provider<KotlinNativeProvider> = project.provider {
KotlinNativeProvider(project, konanTarget)
KotlinNativeProvider(project, konanTarget, kotlinNativeBundleBuildService)
}
@Deprecated(
@@ -43,6 +43,7 @@ import org.jetbrains.kotlin.gradle.report.*
import org.jetbrains.kotlin.gradle.targets.native.KonanPropertiesBuildService
import org.jetbrains.kotlin.gradle.targets.native.tasks.*
import org.jetbrains.kotlin.gradle.targets.native.toolchain.KotlinNativeProvider
import org.jetbrains.kotlin.gradle.targets.native.toolchain.UsesKotlinNativeBundleBuildService
import org.jetbrains.kotlin.gradle.utils.*
import org.jetbrains.kotlin.gradle.utils.GradleLoggerAdapter
import org.jetbrains.kotlin.gradle.utils.listFilesOrEmpty
@@ -294,7 +295,8 @@ internal constructor(
K2MultiplatformCompilationTask,
UsesBuildMetricsService,
KotlinCompilationTask<KotlinNativeCompilerOptions>,
UsesBuildFusService {
UsesBuildFusService,
UsesKotlinNativeBundleBuildService {
@get:Input
override val outputKind = LIBRARY
@@ -332,7 +334,7 @@ internal constructor(
@get:Nested
internal val kotlinNativeProvider: Provider<KotlinNativeProvider> = project.provider {
KotlinNativeProvider(project, konanTarget)
KotlinNativeProvider(project, konanTarget, kotlinNativeBundleBuildService)
}
@Deprecated(
@@ -1033,7 +1035,7 @@ internal class CacheBuilder(
@CacheableTask
abstract class CInteropProcess @Inject internal constructor(params: Params) :
DefaultTask(), UsesBuildMetricsService {
DefaultTask(), UsesBuildMetricsService, UsesKotlinNativeBundleBuildService {
internal class Params(
val settings: DefaultCInteropSettings,
@@ -1097,7 +1099,7 @@ abstract class CInteropProcess @Inject internal constructor(params: Params) :
@get:Nested
internal val kotlinNativeProvider: Provider<KotlinNativeProvider> = project.provider {
KotlinNativeProvider(project, konanTarget)
KotlinNativeProvider(project, konanTarget, kotlinNativeBundleBuildService)
}
@Deprecated(
@@ -34,6 +34,7 @@ import org.jetbrains.kotlin.gradle.report.GradleBuildMetricsReporter
import org.jetbrains.kotlin.gradle.report.UsesBuildMetricsService
import org.jetbrains.kotlin.gradle.targets.native.tasks.buildKotlinNativeBinaryLinkerArgs
import org.jetbrains.kotlin.gradle.targets.native.toolchain.KotlinNativeProvider
import org.jetbrains.kotlin.gradle.targets.native.toolchain.UsesKotlinNativeBundleBuildService
import org.jetbrains.kotlin.gradle.tasks.KotlinToolTask
import org.jetbrains.kotlin.gradle.utils.XcodeUtils
import org.jetbrains.kotlin.gradle.utils.getFile
@@ -55,7 +56,8 @@ abstract class KotlinNativeLinkArtifactTask @Inject constructor(
private val projectLayout: ProjectLayout,
) : DefaultTask(),
UsesBuildMetricsService,
KotlinToolTask<KotlinCommonCompilerToolOptions> {
KotlinToolTask<KotlinCommonCompilerToolOptions>,
UsesKotlinNativeBundleBuildService {
@get:Input
abstract val baseName: Property<String>
@@ -172,7 +174,7 @@ abstract class KotlinNativeLinkArtifactTask @Inject constructor(
@get:Nested
internal val kotlinNativeProvider: Provider<KotlinNativeProvider> = project.provider {
KotlinNativeProvider(project, konanTarget)
KotlinNativeProvider(project, konanTarget, kotlinNativeBundleBuildService)
}
@Deprecated(
@@ -8,6 +8,9 @@ package org.jetbrains.kotlin.gradle.targets.native.toolchain
import org.gradle.api.Project
import org.gradle.api.attributes.Attribute
import org.gradle.api.attributes.AttributesSchema
import org.jetbrains.kotlin.gradle.plugin.KOTLIN_NATIVE_BUNDLE_CONFIGURATION_NAME
import org.jetbrains.kotlin.gradle.utils.NativeCompilerDownloader
import org.jetbrains.kotlin.gradle.utils.maybeCreateResolvable
import org.jetbrains.kotlin.gradle.utils.setAttribute
/**
@@ -54,4 +57,16 @@ internal object KotlinNativeBundleArtifactFormat {
transform.to.setAttribute(attribute, KotlinNativeBundleArtifactsTypes.DIRECTORY)
}
}
internal fun addKotlinNativeBundleConfiguration(project: Project) {
project.configurations
.maybeCreateResolvable(KOTLIN_NATIVE_BUNDLE_CONFIGURATION_NAME).also { configuration ->
configuration.defaultDependencies {
it.add(project.dependencies.create(NativeCompilerDownloader.getCompilerDependencyNotation(project)))
}
if (!configuration.attributes.contains(attribute)) {
configuration.attributes.setAttribute(attribute, KotlinNativeBundleArtifactsTypes.DIRECTORY)
}
}
}
}
@@ -0,0 +1,104 @@
/*
* Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.gradle.targets.native.toolchain
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.file.ConfigurableFileCollection
import org.gradle.api.file.FileSystemOperations
import org.gradle.api.provider.Property
import org.gradle.api.provider.Provider
import org.gradle.api.services.BuildService
import org.gradle.api.services.BuildServiceParameters
import org.gradle.api.tasks.Internal
import org.jetbrains.kotlin.gradle.targets.native.internal.NativeDistributionCommonizerLock
import org.jetbrains.kotlin.gradle.targets.native.internal.NativeDistributionTypeProvider
import org.jetbrains.kotlin.gradle.targets.native.internal.PlatformLibrariesGenerator
import org.jetbrains.kotlin.gradle.tasks.withType
import org.jetbrains.kotlin.gradle.utils.SingleActionPerProject
import org.jetbrains.kotlin.konan.target.KonanTarget
import java.io.File
import javax.inject.Inject
private const val KONAN_DIRECTORY_NAME_TO_CHECK_EXISTENCE = "konan"
internal interface UsesKotlinNativeBundleBuildService : Task {
@get:Internal
val kotlinNativeBundleBuildService: Property<KotlinNativeBundleBuildService>
}
/**
* This service provides functionality to prepare a Kotlin/Native bundle.
*/
internal abstract class KotlinNativeBundleBuildService : BuildService<BuildServiceParameters.None> {
@get:Inject
abstract val fso: FileSystemOperations
companion object {
fun registerIfAbsent(project: Project): Provider<KotlinNativeBundleBuildService> =
project.gradle.sharedServices.registerIfAbsent(
"kotlinNativeBundleBuildService",
KotlinNativeBundleBuildService::class.java
) {}.also { serviceProvider ->
SingleActionPerProject.run(project, UsesKotlinNativeBundleBuildService::class.java.name) {
project.tasks.withType<UsesKotlinNativeBundleBuildService>().configureEach { task ->
task.kotlinNativeBundleBuildService.value(serviceProvider).disallowChanges()
task.usesService(serviceProvider)
}
}
}
}
internal fun prepareKotlinNativeBundle(
project: Project,
kotlinNativeCompilerConfiguration: ConfigurableFileCollection,
kotlinNativeVersion: String,
bundleDir: File,
reinstallFlag: Boolean,
konanTargets: Set<KonanTarget>,
) {
val lock = NativeDistributionCommonizerLock(bundleDir) { message -> project.logger.info("Kotlin Native Bundle: $message") }
if (reinstallFlag) {
lock.withLock {
bundleDir.deleteRecursively()
}
}
if (!bundleDir.resolve(KONAN_DIRECTORY_NAME_TO_CHECK_EXISTENCE).exists()) {
val gradleCachesKotlinNativeDir =
kotlinNativeCompilerConfiguration
.singleOrNull()
?.resolve(kotlinNativeVersion)
?: error(
"Kotlin Native dependency has not been properly resolved. " +
"Please, make sure that you've declared the repository, which contains $kotlinNativeVersion."
)
project.logger.info("Moving Kotlin/Native bundle from tmp directory $gradleCachesKotlinNativeDir to ${bundleDir.absolutePath}")
lock.withLock {
fso.copy {
it.from(gradleCachesKotlinNativeDir)
it.into(bundleDir)
}
}
project.logger.info("Moved Kotlin/Native bundle from $gradleCachesKotlinNativeDir to ${bundleDir.absolutePath}")
}
project.setupKotlinNativeDependencies(konanTargets)
}
private fun Project.setupKotlinNativeDependencies(konanTargets: Set<KonanTarget>) {
val distributionType = NativeDistributionTypeProvider(this).getDistributionType()
if (distributionType.mustGeneratePlatformLibs) {
konanTargets.forEach { konanTarget ->
PlatformLibrariesGenerator(project, konanTarget).generatePlatformLibsIfNeeded()
}
}
}
}
@@ -17,20 +17,24 @@ import org.jetbrains.kotlin.compilerRunner.konanHome
import org.jetbrains.kotlin.compilerRunner.kotlinNativeToolchainEnabled
import org.jetbrains.kotlin.gradle.plugin.KOTLIN_NATIVE_BUNDLE_CONFIGURATION_NAME
import org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.Companion.kotlinPropertiesProvider
import org.jetbrains.kotlin.gradle.targets.native.internal.NativeDistributionTypeProvider
import org.jetbrains.kotlin.gradle.targets.native.internal.PlatformLibrariesGenerator
import org.jetbrains.kotlin.gradle.utils.NativeCompilerDownloader
import org.jetbrains.kotlin.gradle.utils.filesProvider
import org.jetbrains.kotlin.gradle.utils.property
import org.jetbrains.kotlin.konan.target.KonanTarget
import java.io.File
private const val KONAN_DIRECTORY_NAME_TO_CHECK_EXISTENCE = "konan"
/**
* This is a nested provider for all native tasks
*/
internal class KotlinNativeProvider(project: Project, konanTarget: KonanTarget) {
internal class KotlinNativeProvider(
project: Project,
konanTargets: Set<KonanTarget>,
kotlinNativeBundleBuildService: Provider<KotlinNativeBundleBuildService>,
) {
constructor(
project: Project,
konanTarget: KonanTarget,
kotlinNativeBundleBuildService: Provider<KotlinNativeBundleBuildService>,
) : this(project, setOf(konanTarget), kotlinNativeBundleBuildService)
@get:Internal
val konanDataDir: Provider<String?> = project.provider { project.konanDataDir }
@@ -49,11 +53,13 @@ internal class KotlinNativeProvider(project: Project, konanTarget: KonanTarget)
internal val kotlinNativeBundleVersion: Provider<String> = bundleDirectory.zip(reinstallBundle) { bundleDir, reinstallFlag ->
val kotlinNativeVersion = NativeCompilerDownloader.getDependencyNameWithOsAndVersion(project)
if (project.kotlinNativeToolchainEnabled) {
project.prepareKotlinNativeBundle(
kotlinNativeBundleBuildService.get().prepareKotlinNativeBundle(
project,
kotlinNativeCompilerConfiguration,
kotlinNativeVersion,
bundleDir.asFile,
reinstallFlag,
konanTarget
konanTargets
)
}
kotlinNativeVersion
@@ -73,43 +79,4 @@ internal class KotlinNativeProvider(project: Project, konanTarget: KonanTarget)
}
}
private fun Project.prepareKotlinNativeBundle(
kotlinNativeVersion: String,
bundleDir: File,
reinstallFlag: Boolean,
konanTarget: KonanTarget,
) {
if (reinstallFlag) {
bundleDir.deleteRecursively()
}
if (!bundleDir.resolve(KONAN_DIRECTORY_NAME_TO_CHECK_EXISTENCE).exists()) {
val gradleCachesKotlinNativeDir =
kotlinNativeCompilerConfiguration
.singleOrNull()
?.resolve(kotlinNativeVersion)
?: error(
"Kotlin Native dependency has not been properly resolved. " +
"Please, make sure that you've declared the repository, which contains $kotlinNativeVersion."
)
logger.info("Moving Kotlin/Native bundle from tmp directory $gradleCachesKotlinNativeDir to ${bundleDir.absolutePath}")
copy {
it.from(gradleCachesKotlinNativeDir)
it.into(bundleDir)
}
logger.info("Moved Kotlin/Native bundle from $gradleCachesKotlinNativeDir to ${bundleDir.absolutePath}")
}
setupKotlinNativeDependencies(konanTarget)
}
private fun Project.setupKotlinNativeDependencies(konanTarget: KonanTarget) {
val distributionType = NativeDistributionTypeProvider(this).getDistributionType()
if (distributionType.mustGeneratePlatformLibs) {
PlatformLibrariesGenerator(project, konanTarget).generatePlatformLibsIfNeeded()
}
}
}