[K/N] Refactor :kotlin-native:dependencies

Move K/N dependencies handling logic into gradle plugins:

`NativeDependenciesDownloader` is instantiated in
`:kotlin-native:dependencies` and provides a single configuration that
contains all directories with the dependencies. The configuration has
a variant for each target.

`NativeDependencies` is instantiated by projects that require native
dependencies (i.e. previously used to depend on
`:kotlin-native:dependencies:update`). This plugin creates a configuration
that by default depends on `:kotlin-native:dependencies`. Its extension
provides `llvmDependency`, `libffiDependency`, `hostPlatformDependency`,
`targetDependency()` to depend on the parts needed for the project and
provides accessors `llvmPath`, `libffiPath`, `hostPlatform` that are
safe to use during configuration time, but which do not automatically
resolve those dependencies.

`llvmDir`, `*LibffiDir` properties on `:kotlin-native` are removed
completely, as their use is replaced by `NativeDependencies`. As a
consequence, `evaluationDependsOn(":kotlin-native:dependencies")` is
also gone.

All direct dependencies on `:kotlin-native:dependencies:update` are
replaced with dependencies on specific parts via `NativeDependencies`
This commit is contained in:
Alexander Shabalin
2023-08-28 11:55:32 +02:00
committed by Space Team
parent 9de791b9ac
commit 4922223bec
23 changed files with 519 additions and 186 deletions
@@ -123,5 +123,13 @@ gradlePlugin {
id = "native"
implementationClass = "org.jetbrains.kotlin.tools.NativePlugin"
}
create("nativeDependenciesDownloader") {
id = "native-dependencies-downloader"
implementationClass = "org.jetbrains.kotlin.dependencies.NativeDependenciesDownloaderPlugin"
}
create("nativeDependencies") {
id = "native-dependencies"
implementationClass = "org.jetbrains.kotlin.dependencies.NativeDependenciesPlugin"
}
}
}
@@ -27,6 +27,8 @@ import org.gradle.api.internal.CollectionCallbackActionDecorator
import org.gradle.api.tasks.JavaExec
import org.gradle.api.tasks.SourceSet
import org.gradle.internal.reflect.Instantiator
import org.jetbrains.kotlin.dependencies.NativeDependenciesExtension
import org.jetbrains.kotlin.konan.target.Platform
class NamedNativeInteropConfig implements Named {
@@ -155,6 +157,14 @@ class NamedNativeInteropConfig implements Named {
return new File(project.buildDir, "interopTemp")
}
String getLlvmDir() {
return project.extensions.nativeDependencies.llvmPath
}
Platform getHostPlatform() {
return project.extensions.nativeDependencies.hostPlatform
}
NamedNativeInteropConfig(Project project, String name, String target = null, String flavor = 'jvm') {
this.name = name
this.project = project
@@ -194,7 +204,8 @@ class NamedNativeInteropConfig implements Named {
}
genTask.configure {
dependsOn ":kotlin-native:dependencies:update"
dependsOn project.extensions.nativeDependencies.hostPlatformDependency
dependsOn project.extensions.nativeDependencies.llvmDependency
dependsOn ":kotlin-native:Interop:Indexer:nativelibs"
dependsOn ":kotlin-native:Interop:Runtime:nativelibs"
classpath = project.configurations.interopStubGenerator
@@ -293,6 +304,8 @@ class NativeInteropPlugin implements Plugin<Project> {
@Override
void apply(Project prj) {
prj.plugins.apply("native-dependencies")
prj.extensions.add("kotlinNativeInterop", new NativeInteropExtension(prj))
prj.configurations {
@@ -25,6 +25,8 @@ import org.gradle.kotlin.dsl.*
import org.gradle.language.base.plugins.LifecycleBasePlugin
import org.jetbrains.kotlin.ExecClang
import org.jetbrains.kotlin.cpp.*
import org.jetbrains.kotlin.dependencies.NativeDependenciesExtension
import org.jetbrains.kotlin.dependencies.NativeDependenciesPlugin
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.target.SanitizerKind
import org.jetbrains.kotlin.konan.target.TargetDomainObjectContainer
@@ -207,6 +209,7 @@ open class CompileToBitcodeExtension @Inject constructor(val project: Project) :
private val compilationDatabase = project.extensions.getByType<CompilationDatabaseExtension>()
private val execClang = project.extensions.getByType<ExecClang>()
private val nativeDependencies = project.extensions.getByType<NativeDependenciesExtension>()
/**
* Compiles source files into bitcode files.
@@ -228,8 +231,8 @@ open class CompileToBitcodeExtension @Inject constructor(val project: Project) :
this.inputFiles.setIncludes(this@SourceSet.inputFiles.includes)
this.inputFiles.setExcludes(this@SourceSet.inputFiles.excludes)
this.workingDirectory.set(module.compilerWorkingDirectory)
// TODO: Should depend only on the toolchain needed to build for the _target
dependsOn(":kotlin-native:dependencies:update")
dependsOn(nativeDependencies.llvmDependency)
dependsOn(nativeDependencies.targetDependency(_target))
dependsOn(this@SourceSet.dependencies)
onlyIf {
this@SourceSet.onlyIf.get().all { it.isSatisfiedBy(this@SourceSet) }
@@ -249,8 +252,8 @@ open class CompileToBitcodeExtension @Inject constructor(val project: Project) :
// Compile task depends on the toolchain (including headers) and on the source code (e.g. googletest).
// compdb task should also have these dependencies. This way the generated database will point to the
// code that actually exists.
// TODO: Should depend only on the toolchain needed to build for the _target
dependsOn(":kotlin-native:dependencies:update")
dependsOn(nativeDependencies.llvmDependency)
dependsOn(nativeDependencies.targetDependency(_target))
dependsOn(this@SourceSet.dependencies)
}
}
@@ -662,6 +665,7 @@ open class CompileToBitcodePlugin : Plugin<Project> {
project.apply<CppConsumerPlugin>()
project.apply<CompilationDatabasePlugin>()
project.apply<GitClangFormatPlugin>()
project.apply<NativeDependenciesPlugin>()
project.extensions.create<CompileToBitcodeExtension>("bitcode", project)
}
}
@@ -7,22 +7,8 @@ package org.jetbrains.kotlin.cpp
import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.api.attributes.AttributeDisambiguationRule
import org.gradle.api.attributes.MultipleCandidatesDetails
import org.jetbrains.kotlin.bitcode.CompileToBitcodePlugin
import org.jetbrains.kotlin.konan.target.TargetWithSanitizer
private class TargetDisambiguationRule : AttributeDisambiguationRule<TargetWithSanitizer> {
override fun execute(details: MultipleCandidatesDetails<TargetWithSanitizer>) = details.run {
if (consumerValue == null) {
// If the consumer didn't want a specific target, provide host target if it's available.
val default = TargetWithSanitizer.host
if (candidateValues.contains(default)) {
closestMatch(default)
}
}
}
}
import org.jetbrains.kotlin.konan.target.registerTargetWithSanitizerAttribute
/**
* Plugin for projects that depend upon C++-built projects.
@@ -36,9 +22,7 @@ private class TargetDisambiguationRule : AttributeDisambiguationRule<TargetWithS
class CppConsumerPlugin : Plugin<Project> {
override fun apply(project: Project) {
project.dependencies.attributesSchema {
attribute(TargetWithSanitizer.TARGET_ATTRIBUTE) {
disambiguationRules.add(TargetDisambiguationRule::class.java)
}
registerTargetWithSanitizerAttribute()
}
}
@@ -8,12 +8,17 @@ package org.jetbrains.kotlin.cpp
import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.kotlin.dsl.*
import org.jetbrains.kotlin.dependencies.NativeDependenciesExtension
import org.jetbrains.kotlin.dependencies.NativeDependenciesPlugin
/**
* Plugin for [GitClangFormat] that creates a task `clangFormat` that will format project sources in the current branch.
*/
open class GitClangFormatPlugin : Plugin<Project> {
override fun apply(target: Project) {
target.apply<NativeDependenciesPlugin>()
val nativeDependencies = target.extensions.getByType<NativeDependenciesExtension>()
val directory = target.layout.projectDirectory.toString()
target.tasks.register<GitClangFormat>("clangFormat") {
description = "Run clang-format in $directory"
@@ -22,8 +27,7 @@ open class GitClangFormatPlugin : Plugin<Project> {
this.directory.convention(directory)
interactive.convention(false)
// Needs LLVM toolchain.
// TODO: It does not need every dependency ever, only LLVM toolchain for the host.
dependsOn(":kotlin-native:dependencies:update")
dependsOn(nativeDependencies.llvmDependency)
}
}
@@ -0,0 +1,29 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.dependencies
import org.gradle.api.Plugin
import org.gradle.api.Project
import org.jetbrains.kotlin.konan.target.registerTargetWithSanitizerAttribute
/**
* Base plugin for projects that use native dependencies.
*
* For defining native dependencies see [NativeDependenciesDownloaderPlugin].
* For consuming native dependencies see [NativeDependenciesPlugin]
*
* @see NativeDependenciesDownloaderPlugin
* @see NativeDependenciesPlugin
*/
// TODO: Consider making it more like a standard gradle plugins that also
// creates default configurations and lifecycle tasks.
class NativeDependenciesBasePlugin : Plugin<Project> {
override fun apply(project: Project) {
project.dependencies.attributesSchema {
registerTargetWithSanitizerAttribute()
}
}
}
@@ -0,0 +1,32 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.dependencies
import org.gradle.api.DefaultTask
import org.gradle.api.provider.Property
import org.gradle.api.tasks.Internal
import org.gradle.api.tasks.TaskAction
import org.gradle.api.tasks.UntrackedTask
import org.jetbrains.kotlin.konan.util.DependencyProcessor
/**
* Downloader of native dependencies.
*
* Serves as a Gradle task wrapper around [DependencyProcessor].
*/
@UntrackedTask(because = "Output is large and work avoidance is performed in DependencyProcessor anyway")
abstract class NativeDependenciesDownloader : DefaultTask() {
/**
* [DependencyProcessor] that will perform downloading
*/
@get:Internal
abstract val dependencyProcessor: Property<DependencyProcessor>
@TaskAction
fun downloadAndExtract() {
dependencyProcessor.get().run()
}
}
@@ -0,0 +1,158 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.dependencies
import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.api.artifacts.Configuration
import org.gradle.api.attributes.Usage
import org.gradle.api.file.DirectoryProperty
import org.gradle.api.logging.LogLevel
import org.gradle.api.provider.Property
import org.gradle.kotlin.dsl.*
import org.jetbrains.kotlin.konan.properties.KonanPropertiesLoader
import org.jetbrains.kotlin.konan.target.PlatformManager
import org.jetbrains.kotlin.konan.target.TargetDomainObjectContainer
import org.jetbrains.kotlin.konan.target.TargetWithSanitizer
import org.jetbrains.kotlin.konan.util.DependencyProcessor
import org.jetbrains.kotlin.utils.capitalized
import javax.inject.Inject
/**
* Downloading native dependencies.
*
* An embedder must provide [dependenciesDirectory] and [repositoryURL] and then provide a list
* of targets.
* To configure for all targets:
* ```
* nativeDependenciesDownloader {
* dependenciesDir.set(…)
* baseUrl.set(…)
* allTargets {}
* }
* ```
* To download for host and `someTarget` only:
* ```
* nativeDependenciesDownloader {
* dependenciesDir.set(…)
* baseUrl.set(…)
* hostTarget {}
* target(someTarget) {}
* }
* ```
*
* Apply [plugin][NativeDependenciesDownloaderPlugin] to create this extension.
* The extension name is `nativeDependenciesDownloader`.
*
* @see NativeDependenciesDownloaderPlugin
*/
abstract class NativeDependenciesDownloaderExtension @Inject constructor(private val project: Project) : TargetDomainObjectContainer<NativeDependenciesDownloaderExtension.Target>(project) {
init {
this.factory = { target ->
project.objects.newInstance<Target>(this, target)
}
}
/**
* Outgoing configuration with all native dependencies.
*/
val nativeDependenciesElements: Configuration by project.configurations.creating {
description = "Native dependencies"
isCanBeConsumed = true
isCanBeResolved = false
attributes {
attribute(Usage.USAGE_ATTRIBUTE, project.objects.named(NativeDependenciesUsage.NATIVE_DEPENDENCY))
}
}
/**
* Root directory where to place all dependencies.
*/
abstract val dependenciesDirectory: DirectoryProperty
/**
* Dependency repository.
*/
abstract val repositoryURL: Property<String>
abstract class Target @Inject constructor(
private val owner: NativeDependenciesDownloaderExtension,
private val _target: TargetWithSanitizer,
) {
val target by _target::target
val sanitizer by _target::sanitizer
private val project by owner::project
private val platformManager = project.extensions.getByType<PlatformManager>()
private val loader: KonanPropertiesLoader = platformManager.loader(target).let {
require(it is KonanPropertiesLoader) {
"loader for $target must implement ${KonanPropertiesLoader::class}"
}
it
}
private val dependencyProcessor by lazy {
DependencyProcessor(
owner.dependenciesDirectory.apply { finalizeValue() }.asFile.get(),
loader,
owner.repositoryURL.apply { finalizeValue() }.get(),
keepUnstable = false) { url, currentBytes, totalBytes ->
// TODO: Consider using logger.
print("\nDownloading dependency for $_target: $url (${currentBytes}/${totalBytes}). ")
}.apply {
showInfo = project.logger.isEnabled(LogLevel.INFO)
}
}
val dependencies by loader::dependencies
val task = project.tasks.register<NativeDependenciesDownloader>("nativeDependencies${_target.name.capitalized}") {
description = "Download dependencies for $_target"
group = "native dependencies"
dependencyProcessor.set(this@Target.dependencyProcessor)
}
init {
owner.nativeDependenciesElements.outgoing {
variants {
create("$_target") {
attributes {
attribute(TargetWithSanitizer.TARGET_ATTRIBUTE, _target)
}
dependencies.forEach { dependency ->
artifact(dependencyProcessor.resolve(dependency)) {
name = dependency
type = "directory"
extension = ""
builtBy(task)
}
}
}
}
}
}
}
}
/**
* Downloading native dependencies.
*
* Provides [extension][NativeDependenciesDownloaderExtension] named `nativeDependenciesDownloader`
* that configures native dependencies.
*
* To consume native dependencies use [NativeDependenciesPlugin].
*
* @see NativeDependenciesPlugin
* @see NativeDependenciesDownloaderExtension
*/
open class NativeDependenciesDownloaderPlugin : Plugin<Project> {
override fun apply(project: Project) {
project.apply<NativeDependenciesBasePlugin>()
project.extensions.create<NativeDependenciesDownloaderExtension>("nativeDependenciesDownloader", project)
}
}
@@ -0,0 +1,131 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.dependencies
import org.gradle.api.Buildable
import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.api.artifacts.Configuration
import org.gradle.api.attributes.Usage
import org.gradle.api.file.FileCollection
import org.gradle.kotlin.dsl.*
import org.jetbrains.kotlin.konan.target.*
import javax.inject.Inject
/**
* Consuming native dependencies.
*
* Creates resolvable [nativeDependencies] configuration with default dependency
* on `:kotlin-native:dependencies` (that project must provide native dependencies by using [NativeDependenciesDownloaderPlugin])
*
* [llvmPath], [libffiPath] and [hostPlatform] are accessors that can be used during configuration
* phase and do not anyhow force the actual dependencies to be downloaded. So, anytime these
* accessors are used, make sure that execution of related tasks (or configurations resolving)
* depend on [llvmDependency], [libffiDependency] and [hostPlatformDependency] respectively.
*
* Apply [plugin][NativeDependenciesPlugin] to create this extension.
* The extension name is `nativeDependencies`.
*
* @see NativeDependenciesPlugin
*/
abstract class NativeDependenciesExtension @Inject constructor(private val project: Project) {
private val platformManager = project.extensions.getByType<PlatformManager>()
val nativeDependencies: Configuration by project.configurations.creating {
description = "Native dependencies"
isCanBeConsumed = false
isCanBeResolved = true
attributes {
attribute(Usage.USAGE_ATTRIBUTE, project.objects.named(NativeDependenciesUsage.NATIVE_DEPENDENCY))
}
defaultDependencies {
add(project.dependencies.project(":kotlin-native:dependencies"))
}
}
private val llvmFileCollection: FileCollection = nativeDependencies.incoming.artifacts.artifactFiles.filter {
it.name == platformManager.hostPlatform.llvmHome
}
private val libffiFileCollection: FileCollection = nativeDependencies.incoming.artifacts.artifactFiles.filter {
it.name == platformManager.hostPlatform.libffiDir
}
/**
* Dependency on host LLVM.
*/
val llvmDependency: Buildable by ::llvmFileCollection
/**
* Absolute path to host LLVM. Can be used during configuration.
* Note: this will not force LLVM to be downloaded. Make sure to depend
* on [llvmDependency] in tasks that need it.
*/
val llvmPath: String
get() = llvmFileCollection.singleFile.canonicalPath
/**
* Dependency on host libffi.
*/
val libffiDependency: Buildable by ::libffiFileCollection
/**
* Absolute path to host libffi. Can be used during configuration.
* Note: this will not force libffi to be downloaded. Make sure to depend
* on [libffiDependency] in tasks that need it.
*/
val libffiPath: String
get() = libffiFileCollection.singleFile.canonicalPath
/**
* Dependency on host platform.
*
* Equivalent to calling [targetDependency] with host target
*/
val hostPlatformDependency: Buildable
get() = targetDependency()
/**
* [Platform] for host. Can be used during configuration.
* Note: this will not force platform dependency to be downloaded. Make sure to depend
* on [hostPlatformDependency] in tasks that need it.
*/
val hostPlatform: Platform
get() = platformManager.hostPlatform
/**
* Dependency on [target] platform.
*/
fun targetDependency(target: TargetWithSanitizer = TargetWithSanitizer.host): Buildable =
nativeDependencies.incoming.artifactView {
attributes {
attribute(TargetWithSanitizer.TARGET_ATTRIBUTE, target)
}
}.files
/**
* Dependency on [target] platform.
*/
// TODO: Remove when all build.gradle are gone.
fun targetDependency(target: KonanTarget): Buildable = targetDependency(target.withSanitizer())
}
/**
* Consuming native dependencies.
*
* Provides [extension][NativeDependenciesExtension] named `nativeDependencies` that manages native dependencies.
*
* To provide native dependencies use [NativeDependenciesDownloaderPlugin].
*
* @see NativeDependenciesDownloaderPlugin
* @see NativeDependenciesExtension
*/
class NativeDependenciesPlugin : Plugin<Project> {
override fun apply(project: Project) {
project.apply<NativeDependenciesBasePlugin>()
project.extensions.create<NativeDependenciesExtension>("nativeDependencies", project)
}
}
@@ -0,0 +1,20 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.dependencies
import org.gradle.api.attributes.Attribute
import org.gradle.api.attributes.Usage
object NativeDependenciesUsage {
/**
* Directories with native dependencies.
*/
@JvmField
val NATIVE_DEPENDENCY = "native-dependency"
@JvmField
val USAGE_ATTRIBUTE: Attribute<Usage> = Usage.USAGE_ATTRIBUTE
}
@@ -7,6 +7,9 @@ package org.jetbrains.kotlin.konan.target
import org.gradle.api.Named
import org.gradle.api.attributes.Attribute
import org.gradle.api.attributes.AttributeDisambiguationRule
import org.gradle.api.attributes.AttributesSchema
import org.gradle.api.attributes.MultipleCandidatesDetails
import java.io.Serializable
/**
@@ -54,3 +57,24 @@ val PlatformManager.allTargetsWithSanitizers
target.withSanitizer(it)
}
}
private class TargetDisambiguationRule : AttributeDisambiguationRule<TargetWithSanitizer> {
override fun execute(details: MultipleCandidatesDetails<TargetWithSanitizer>) = details.run {
if (consumerValue == null) {
// If the consumer didn't want a specific target, provide host target if it's available.
val default = TargetWithSanitizer.host
if (candidateValues.contains(default)) {
closestMatch(default)
}
}
}
}
/**
* Register [TargetWithSanitizer] attribute with [AttributesSchema].
*/
fun AttributesSchema.registerTargetWithSanitizerAttribute() {
attribute(TargetWithSanitizer.TARGET_ATTRIBUTE) {
disambiguationRules.add(TargetDisambiguationRule::class.java)
}
}
@@ -11,9 +11,10 @@ import org.gradle.api.Project
import org.gradle.api.file.FileCollection
import org.gradle.api.plugins.BasePlugin
import org.gradle.api.tasks.*
import org.gradle.kotlin.dsl.apply
import org.gradle.kotlin.dsl.withType
import org.gradle.kotlin.dsl.*
import org.gradle.language.base.plugins.LifecycleBasePlugin
import org.jetbrains.kotlin.dependencies.NativeDependenciesExtension
import org.jetbrains.kotlin.dependencies.NativeDependenciesPlugin
import org.jetbrains.kotlin.konan.target.HostManager.Companion.hostIsMac
import org.jetbrains.kotlin.konan.target.HostManager.Companion.hostIsMingw
import java.io.File
@@ -35,6 +36,7 @@ import kotlin.collections.toTypedArray
open class NativePlugin : Plugin<Project> {
override fun apply(project: Project) {
project.apply<BasePlugin>()
project.apply<NativeDependenciesPlugin>()
project.extensions.create("native", NativeToolsExtension::class.java, project)
}
}
@@ -84,7 +86,9 @@ class ToolPatternImpl(val extension: NativeToolsExtension, val output:String, va
task.input = input.map {
extension.project.file(it)
}
task.dependsOn(":kotlin-native:dependencies:update")
val nativeDependenciesExtension = extension.project.extensions.getByType<NativeDependenciesExtension>()
task.dependsOn(nativeDependenciesExtension.hostPlatformDependency)
task.dependsOn(nativeDependenciesExtension.llvmDependency)
if (configureDepencies)
task.input.forEach { task.dependsOn(it.name) }
val file = extension.project.file(output)
@@ -183,6 +187,11 @@ class ToolConfigurationPatterns(
open class NativeToolsExtension(val project: Project) {
private val nativeDependenciesExtension = project.extensions.getByType<NativeDependenciesExtension>()
val llvmDir by nativeDependenciesExtension::llvmPath
val hostPlatform by nativeDependenciesExtension::hostPlatform
val sourceSets = SourceSets(project, this, mutableMapOf<String, SourceSet>())
val toolPatterns = ToolConfigurationPatterns(this, mutableMapOf<Pair<String, String>, ToolPatternConfiguration>())
val cleanupFiles = mutableListOf<String>()