azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[K/N] Modernize CompilationDatabase tasks

Browse Source 
Merge-request: KT-MR-6276
Merged-by: Alexander Shabalin <Alexander.Shabalin@jetbrains.com>
This commit is contained in:
Alexander Shabalin
2022-05-25 21:49:35 +00:00
committed by Space
parent c230a488e2
commit 20884d82eb
7 changed files with 392 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
buildSrc/build.gradle.kts
+4
View File
@@ -253,6 +253,10 @@ gradlePlugin {
id = "runtime-testing"
implementationClass = "org.jetbrains.kotlin.testing.native.RuntimeTestingPlugin"
}
create("compilationDatabase") {
id = "compilation-database"
implementationClass = "org.jetbrains.kotlin.cpp.CompilationDatabasePlugin"
}
create("konan") {
id = "konan"
implementationClass = "org.jetbrains.kotlin.gradle.plugin.konan.KonanPlugin"
kotlin-native/.gitignore
+1
View File
@@ -67,6 +67,7 @@ compile_commands.json
# clangd caches
.clangd/
.cache/clangd
# googletest framework used by runtime tests
runtime/googletest/
kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/CompilationDatabase.kt
-112
View File
@@ -1,112 +0,0 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
package org.jetbrains.kotlin
import java.io.File
import javax.inject.Inject
import org.gradle.api.GradleException
import com.google.gson.annotations.Expose
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.DefaultTask
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputFiles
import org.gradle.api.tasks.Internal
import org.gradle.api.tasks.OutputFile
import org.gradle.api.tasks.TaskAction
import org.gradle.kotlin.dsl.getByType
import java.io.FileReader
import java.io.FileWriter
internal data class Entry(
@Expose val directory: String,
@Expose val file: String,
@Expose val arguments: List<String>,
@Expose val output: String
) {
companion object {
fun create(
directory: File,
file: File,
args: List<String>,
outputDir: File
): Entry {
return Entry(
directory.absolutePath,
file.absolutePath,
args + listOf(file.absolutePath),
File(outputDir, file.name + ".o").absolutePath
)
}
fun writeListTo(file: File, entries: List<Entry>) = FileWriter(file).use {
gson.toJson(entries, it)
}
fun readListFrom(file: File): Array<Entry> = FileReader(file).use {
gson.fromJson(it, Array<Entry>::class.java)
}
}
}
open class GenerateCompilationDatabase @Inject constructor(@Input val target: String,
@Internal val files: Iterable<File>,
@Input val executable: String,
@Input val compilerFlags: List<String>,
@Internal val outputDir: File
) : DefaultTask() {
@OutputFile
var outputFile = File(outputDir, "compile_commands.json")
// Annotate as an input because this path affects the content of the generated file.
@get:Input
val outputDirPath: String
get() = outputDir.absolutePath
@get:Input
val pathsToFiles: Iterable<String>
get() = files.map { it.absolutePath }
@TaskAction
fun run() {
val plugin = project.extensions.getByType<ExecClang>()
val executable = plugin.resolveExecutable(executable)
val args = listOf(executable) + compilerFlags + plugin.clangArgsForCppRuntime(target)
val entries: List<Entry> = files.map { Entry.create(it.parentFile, it, args, outputDir) }
Entry.writeListTo(outputFile, entries)
}
}
open class MergeCompilationDatabases @Inject constructor() : DefaultTask() {
@InputFiles
val inputFiles = mutableListOf<File>()
@OutputFile
var outputFile = File(project.buildDir, "compile_commands.json")
@TaskAction
fun run() {
val entries = mutableListOf<Entry>()
for (file in inputFiles) {
entries.addAll(Entry.readListFrom(file))
}
Entry.writeListTo(outputFile, entries)
}
}
fun mergeCompilationDatabases(project: Project, name: String, paths: List<String>): Task {
val subtasks: List<MergeCompilationDatabases> = paths.map {
val task = project.tasks.getByPath(it)
if (task !is MergeCompilationDatabases) {
throw GradleException("Unknown task type for compdb merging: $task")
}
task
}
return project.tasks.create(name, MergeCompilationDatabases::class.java) {
dependsOn(subtasks)
inputFiles.addAll(subtasks.map { it.outputFile })
}
}
kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/bitcode/CompileToBitcodePlugin.kt
+24 -20
View File
@@ -9,8 +9,11 @@ import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.plugins.BasePlugin
import org.jetbrains.kotlin.GenerateCompilationDatabase
import org.jetbrains.kotlin.MergeCompilationDatabases
import org.gradle.kotlin.dsl.*
import org.jetbrains.kotlin.ExecClang
import org.jetbrains.kotlin.cpp.CompilationDatabaseExtension
import org.jetbrains.kotlin.cpp.CompilationDatabasePlugin
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.target.PlatformManager
import org.jetbrains.kotlin.konan.target.SanitizerKind
import org.jetbrains.kotlin.konan.target.supportedSanitizers
@@ -26,7 +29,8 @@ import javax.inject.Inject
*/
open class CompileToBitcodePlugin : Plugin<Project> {
override fun apply(target: Project) {
target.extensions.create(EXTENSION_NAME, CompileToBitcodeExtension::class.java, target)
target.apply<CompilationDatabasePlugin>()
target.extensions.create<CompileToBitcodeExtension>(EXTENSION_NAME, target)
}
companion object {
@@ -36,6 +40,9 @@ open class CompileToBitcodePlugin : Plugin<Project> {
open class CompileToBitcodeExtension @Inject constructor(val project: Project) {
private val compilationDatabase = project.extensions.getByType<CompilationDatabaseExtension>()
private val execClang = project.extensions.getByType<ExecClang>()
private val targetList = with(project) {
provider { (rootProject.project(":kotlin-native").property("targetList") as? List<*>)?.filterIsInstance<String>() ?: emptyList() } // TODO: Can we make it better?
}
@@ -54,22 +61,19 @@ open class CompileToBitcodeExtension @Inject constructor(val project: Project) {
})
}
private val compdbTasks by lazy {
targetList.get().associateBy(keySelector = { it }, valueTransform = {
val task = project.tasks.register("${it}${COMPILATION_DATABASE_TASK_NAME}", MergeCompilationDatabases::class.java)
task.configure {
outputFile = File(File(project.buildDir, it), "compile_commands.json")
private fun addToCompdb(compileTask: CompileToBitcode, konanTarget: KonanTarget) {
// No need to generate compdb entry for sanitizers.
if (compileTask.sanitizer != null) {
return
}
compilationDatabase.target(konanTarget) {
entry {
val args = listOf(execClang.resolveExecutable(compileTask.executable)) + compileTask.compilerFlags + execClang.clangArgsForCppRuntime(konanTarget.name)
directory.set(compileTask.objDir)
files.setFrom(compileTask.inputFiles)
arguments.set(args)
output.set(compileTask.outFile.absolutePath)
}
task
})
}
private fun addToCompdb(compileTask: CompileToBitcode) {
val task = project.tasks.create("${compileTask.name}_CompilationDatabase", GenerateCompilationDatabase::class.java, compileTask.target, compileTask.inputFiles, compileTask.executable, compileTask.compilerFlags, compileTask.objDir)
val compdbTask = compdbTasks[compileTask.target]!!
compdbTask.configure {
dependsOn(task)
inputFiles.add(task.outputFile)
}
}
@@ -96,7 +100,7 @@ open class CompileToBitcodeExtension @Inject constructor(val project: Project) {
dependsOn(":kotlin-native:dependencies:update")
configurationBlock()
}
addToCompdb(task)
addToCompdb(task, target)
if (outputGroup == "main" && sanitizer == null) {
allMainModulesTask.configure {
dependsOn(taskName)
@@ -135,7 +139,7 @@ open class CompileToBitcodeExtension @Inject constructor(val project: Project) {
dependsOn("downloadGoogleTest")
compilerArgs.addAll(it.compilerArgs)
addToCompdb(this)
addToCompdb(this, konanTarget)
}
if (task.inputFiles.count() == 0) null
else task
kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/cpp/CompilationDatabasePlugin.kt
+213
View File
@@ -0,0 +1,213 @@
/*
* Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.cpp
import org.gradle.api.Action
import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.api.file.ConfigurableFileCollection
import org.gradle.api.file.DirectoryProperty
import org.gradle.api.provider.ListProperty
import org.gradle.api.provider.MapProperty
import org.gradle.api.provider.Property
import org.gradle.api.provider.Provider
import org.gradle.kotlin.dsl.*
import org.jetbrains.kotlin.konan.target.HostManager
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.target.PlatformManager
import javax.inject.Inject
/**
* Generating [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html).
*
* Allows generating compilation databases for different targets. For example:
* ```
* compilationDatabase {
* target(someTarget1) {
* entry { ... }
* }
* allTargets {
* mergeFrom(provider { project("subproject") })
* }
* }
* ```
* Adds an entry for target `someTarget1`, and for each known target merges in databases generated
* by this plugin in `subproject`.
* The task that generates the database can be found via `compilationDatabase.target(someTarget1).task`.
*
* @see CompilationDatabasePlugin gradle plugin that creates this extension.
*/
abstract class CompilationDatabaseExtension @Inject constructor(private val project: Project) {
// TODO: This platformManager should acquired be from something service-ish.
// But for usefulness this service should be accessible from WorkAction.
private val platformManager = project.extensions.getByType<PlatformManager>()
/**
* Entries in the compilation database.
*
* Single [Entry] generates a number of compilation database entries: one for each file in [files].
*
* @property target target for which this [Entry] is generated.
*/
abstract class Entry @Inject constructor(val target: KonanTarget) {
/**
* **directory** from the [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html#format).
*
* The working directory of the compilation. All paths in the [arguments] must either be absolute or relative to this directory.
*/
abstract val directory: DirectoryProperty
/**
* **file** from the [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html#format).
*
* Collection of files being compiled with given [arguments]. For each file a separate
* entry will be generated in the database with the same [directory], [arguments] and [output].
*/
abstract val files: ConfigurableFileCollection
/**
* **arguments** from the [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html#format).
*
* List of arguments of the compilation commands. The first argument must be the executable.
*/
abstract val arguments: ListProperty<String>
/**
* **output** from the [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html#format).
*
* The name of the output created by the compilation step. Used as a key to distinguish
* between different modes of compilation of the same sources.
*/
abstract val output: Property<String>
}
/**
* Configure compilation database generation for [target].
*
* [entry] to add new entries.
* [mergeFrom] to merge databases from other projects with [CompilationDatabasePlugin]s.
* [task] is the gradle task for compilation database generation.
*
* @property target target for which compilation database is generated.
*/
abstract class Target @Inject constructor(
private val project: Project,
val target: KonanTarget,
) {
protected abstract val mergeFrom: ListProperty<GenerateCompilationDatabase>
/**
* Merge compilation database generated for [from] project for [target].
*
* @param from project with applied [CompilationDatabasePlugin] to merge compilation database from.
*/
fun mergeFrom(from: Provider<Project>) {
mergeFrom.add(from.flatMap { project ->
project.extensions.getByType<CompilationDatabaseExtension>().target(target).task
})
}
protected abstract val entries: ListProperty<GenerateCompilationDatabase.Entry>
/**
* Add an entry to the compilation database for [target].
*
* @param action configure [Entry]
*/
fun entry(action: Action<in Entry>) {
entries.add(project.provider {
val instance = project.objects.newInstance<Entry>(target).apply {
action.execute(this)
}
project.objects.newInstance<GenerateCompilationDatabase.Entry>().apply {
directory.set(instance.directory)
files.from(instance.files)
arguments.set(instance.arguments)
output.set(instance.output)
}
})
}
/**
* Gradle task that generates compilation database for [target].
*/
val task = project.tasks.register<GenerateCompilationDatabase>("${target}CompilationDatabase") {
description = "Generate compilation database for $target"
group = TASK_GROUP
mergeFiles.from(mergeFrom)
entries.set(this@Target.entries)
outputFile.set(project.layout.buildDirectory.file("${target}/compile_commands.json"))
}
}
protected abstract val targets: MapProperty<KonanTarget, Target>
private fun targetGetOrPut(target: KonanTarget) = targets.getting(target).orNull
?: project.objects.newInstance<Target>(project, target).apply {
targets.put(target, this)
}
/**
* Get [compilation database configuration][Target] for [target] and apply [action] to it.
*
* @param target target to configure.
* @param action action to apply to configuration.
*/
fun target(target: KonanTarget, action: Action<in Target>) = targetGetOrPut(target).apply {
action.execute(this)
}
/**
* Get [compilation database configuration][Target] for [target].
*
* @param target target to configure.
*/
fun target(target: KonanTarget) = this.target(target) {}
/**
* Get [compilation database configurations][Target] for all known targets and apply [action] to each.
*
* @param action action to apply to configurations.
*/
fun allTargets(action: Action<in Target>) = platformManager.enabled.map { target(it, action) }
/**
* Get [compilation database configurations][Target] for all known targets.
*/
val allTargets
get() = allTargets {}
/**
* Get [compilation database configuration][Target] for [host target][HostManager.host] and apply [action] to it.
*
* @param action action to apply to configuration.
*/
fun hostTarget(action: Action<in Target>) = target(HostManager.host, action)
/**
* Get [compilation database configuration][Target] for [host target][HostManager.host].
*/
val hostTarget
get() = hostTarget {}
companion object {
@JvmStatic
val TASK_GROUP = "development support"
}
}
/**
* Generating [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html).
*
* Creates [CompilationDatabaseExtension] extension named `compilationDatabase`.
*
* @see CompilationDatabaseExtension extension that this plugin creates.
*/
open class CompilationDatabasePlugin : Plugin<Project> {
override fun apply(project: Project) {
project.extensions.create<CompilationDatabaseExtension>("compilationDatabase", project)
}
}
kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/cpp/GenerateCompilationDatabase.kt
+137
View File
@@ -0,0 +1,137 @@
/*
* Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.cpp
import com.google.gson.annotations.Expose
import org.gradle.api.DefaultTask
import org.gradle.api.file.ConfigurableFileCollection
import org.gradle.api.file.DirectoryProperty
import org.gradle.api.file.RegularFileProperty
import org.gradle.api.provider.ListProperty
import org.gradle.api.provider.Property
import org.gradle.api.provider.Provider
import org.gradle.api.tasks.*
import org.jetbrains.kotlin.gson
import java.io.FileReader
import java.io.FileWriter
/**
* Generating [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html).
*
* Generate compilation database in [outputFile] from [entries] and merging in already
* generated databases in [mergeFiles].
*
* @see CompilationDatabaseExtension gradle plugin to simplify generation of these tasks.
*/
abstract class GenerateCompilationDatabase : DefaultTask() {
/**
* Entries in the compilation database.
*
* Single [Entry] generates a number of compilation database entries: one for each file in [files].
*/
abstract class Entry {
/**
* **directory** from the [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html#format).
*
* The working directory of the compilation. All paths in the [arguments] must either be absolute or relative to this directory.
*/
@get:Internal
abstract val directory: DirectoryProperty
// Only the path of the directory is an input.
@get:Input
protected val pathToDirectory: Provider<String>
get() = directory.asFile.map { it.absolutePath }
/**
* **file** from the [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html#format).
*
* Collection of files being compiled with given [arguments]. For each file a separate
* entry will be generated in the database with the same [directory], [arguments] and [output].
*/
@get:Internal
abstract val files: ConfigurableFileCollection
// Only the paths of files are an input.
@get:Input
protected val pathsToFiles: Iterable<String>
get() = files.files.map { it.absolutePath }
/**
* **arguments** from the [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html#format).
*
* List of arguments of the compilation commands. The first argument must be the executable.
*/
@get:Input
abstract val arguments: ListProperty<String>
/**
* **output** from the [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html#format).
*
* The name of the output created by the compilation step. Used as a key to distinguish
* between different modes of compilation of the same sources.
*/
@get:Input
abstract val output: Property<String>
}
// Must follow https://clang.llvm.org/docs/JSONCompilationDatabase.html#format
private data class SerializedEntry(
@Expose val directory: String,
@Expose val file: String,
@Expose val arguments: List<String>,
@Expose val output: String,
) {
companion object {
fun fromEntry(entry: Entry): List<SerializedEntry> {
val directory = entry.directory.asFile.get().absolutePath
val arguments = entry.arguments.get()
val output = entry.output.get()
return entry.files.map {
val file = it.absolutePath
SerializedEntry(
directory,
file,
arguments + listOf(file),
output,
)
}
}
}
}
/**
* List of [Entry]s to generate database from.
*/
@get:Nested
abstract val entries: ListProperty<Entry>
/**
* List of databases to merge into this database.
*/
@get:InputFiles
abstract val mergeFiles: ConfigurableFileCollection
/**
* Where to put the database.
*/
@get:OutputFile
abstract val outputFile: RegularFileProperty
@TaskAction
fun run() {
val serialized = mutableListOf<SerializedEntry>()
mergeFiles.files.forEach { file ->
FileReader(file).use {
serialized.addAll(gson.fromJson(it, Array<SerializedEntry>::class.java))
}
}
serialized.addAll(entries.get().flatMap { SerializedEntry.fromEntry(it) })
FileWriter(outputFile.asFile.get()).use {
gson.toJson(serialized, it)
}
}
}
kotlin-native/build.gradle
+13 -11
View File
@@ -23,7 +23,7 @@ import org.jetbrains.kotlin.PlatformInfo
import org.jetbrains.kotlin.KotlinBuildPusher
import org.jetbrains.kotlin.CollisionDetector
import org.jetbrains.kotlin.CollisionTransformer
import org.jetbrains.kotlin.CompilationDatabaseKt
import org.jetbrains.kotlin.cpp.CompilationDatabasePlugin
import org.jetbrains.kotlin.CompareDistributionSignatures
import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
import org.apache.tools.ant.filters.ReplaceTokens
@@ -179,6 +179,8 @@ dependencies {
commonSources project(path: ':kotlin-test:kotlin-test-annotations-common', configuration: 'sources')
}
apply plugin: CompilationDatabasePlugin
//task sharedJar {
// dependsOn gradle.includedBuild('shared').task(':jar')
//}
@@ -742,19 +744,19 @@ task pusher(type: KotlinBuildPusher){
token = project.findProperty("teamcityBearToken") ?: System.getenv("TEAMCITY_BEAR_TOKEN")
}
targetList.each { target ->
CompilationDatabaseKt.mergeCompilationDatabases(project, "${target}CompilationDatabase".toString(), [
":kotlin-native::common:${target}CompilationDatabase".toString(),
":kotlin-native:runtime:${target}CompilationDatabase".toString()
]).configure {
outputFile = file("$buildDir/${target}/compile_commands.json")
compilationDatabase {
allTargets {
mergeFrom(provider { project("common") })
mergeFrom(provider { project("runtime") })
}
}
task compdb(type: Copy) {
dependsOn "${hostName}CompilationDatabase"
from "$buildDir/${hostName}/compile_commands.json"
into "$projectDir"
tasks.register("compdb", Copy) {
from compilationDatabase.hostTarget.task
into layout.projectDirectory
group = compilationDatabase.TASK_GROUP
description = "Copy host compilation database to kotlin-native/"
}
targetList.each { targetName ->