diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 91667bb2b10..3a7616390a3 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -253,6 +253,10 @@ gradlePlugin { id = "runtime-testing" implementationClass = "org.jetbrains.kotlin.testing.native.RuntimeTestingPlugin" } + create("compilationDatabase") { + id = "compilation-database" + implementationClass = "org.jetbrains.kotlin.cpp.CompilationDatabasePlugin" + } create("konan") { id = "konan" implementationClass = "org.jetbrains.kotlin.gradle.plugin.konan.KonanPlugin" diff --git a/kotlin-native/.gitignore b/kotlin-native/.gitignore index bae104d23c4..3f2bb079a53 100644 --- a/kotlin-native/.gitignore +++ b/kotlin-native/.gitignore @@ -67,6 +67,7 @@ compile_commands.json # clangd caches .clangd/ +.cache/clangd # googletest framework used by runtime tests runtime/googletest/ diff --git a/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/CompilationDatabase.kt b/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/CompilationDatabase.kt deleted file mode 100644 index 713c4b21bdd..00000000000 --- a/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/CompilationDatabase.kt +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the LICENSE file. - */ - -package org.jetbrains.kotlin - -import java.io.File -import javax.inject.Inject -import org.gradle.api.GradleException -import com.google.gson.annotations.Expose -import org.gradle.api.Project -import org.gradle.api.Task -import org.gradle.api.DefaultTask -import org.gradle.api.tasks.Input -import org.gradle.api.tasks.InputFiles -import org.gradle.api.tasks.Internal -import org.gradle.api.tasks.OutputFile -import org.gradle.api.tasks.TaskAction -import org.gradle.kotlin.dsl.getByType -import java.io.FileReader -import java.io.FileWriter - -internal data class Entry( - @Expose val directory: String, - @Expose val file: String, - @Expose val arguments: List, - @Expose val output: String -) { - companion object { - fun create( - directory: File, - file: File, - args: List, - outputDir: File - ): Entry { - return Entry( - directory.absolutePath, - file.absolutePath, - args + listOf(file.absolutePath), - File(outputDir, file.name + ".o").absolutePath - ) - } - - fun writeListTo(file: File, entries: List) = FileWriter(file).use { - gson.toJson(entries, it) - } - - fun readListFrom(file: File): Array = FileReader(file).use { - gson.fromJson(it, Array::class.java) - } - } -} - -open class GenerateCompilationDatabase @Inject constructor(@Input val target: String, - @Internal val files: Iterable, - @Input val executable: String, - @Input val compilerFlags: List, - @Internal val outputDir: File -) : DefaultTask() { - @OutputFile - var outputFile = File(outputDir, "compile_commands.json") - - // Annotate as an input because this path affects the content of the generated file. - @get:Input - val outputDirPath: String - get() = outputDir.absolutePath - - @get:Input - val pathsToFiles: Iterable - get() = files.map { it.absolutePath } - - @TaskAction - fun run() { - val plugin = project.extensions.getByType() - val executable = plugin.resolveExecutable(executable) - val args = listOf(executable) + compilerFlags + plugin.clangArgsForCppRuntime(target) - val entries: List = files.map { Entry.create(it.parentFile, it, args, outputDir) } - Entry.writeListTo(outputFile, entries) - } -} - -open class MergeCompilationDatabases @Inject constructor() : DefaultTask() { - @InputFiles - val inputFiles = mutableListOf() - - @OutputFile - var outputFile = File(project.buildDir, "compile_commands.json") - - @TaskAction - fun run() { - val entries = mutableListOf() - for (file in inputFiles) { - entries.addAll(Entry.readListFrom(file)) - } - Entry.writeListTo(outputFile, entries) - } -} - -fun mergeCompilationDatabases(project: Project, name: String, paths: List): Task { - val subtasks: List = paths.map { - val task = project.tasks.getByPath(it) - if (task !is MergeCompilationDatabases) { - throw GradleException("Unknown task type for compdb merging: $task") - } - task - } - return project.tasks.create(name, MergeCompilationDatabases::class.java) { - dependsOn(subtasks) - inputFiles.addAll(subtasks.map { it.outputFile }) - } -} diff --git a/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/bitcode/CompileToBitcodePlugin.kt b/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/bitcode/CompileToBitcodePlugin.kt index 9eeafe21f9c..33efe43df47 100644 --- a/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/bitcode/CompileToBitcodePlugin.kt +++ b/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/bitcode/CompileToBitcodePlugin.kt @@ -9,8 +9,11 @@ import org.gradle.api.Plugin import org.gradle.api.Project import org.gradle.api.Task import org.gradle.api.plugins.BasePlugin -import org.jetbrains.kotlin.GenerateCompilationDatabase -import org.jetbrains.kotlin.MergeCompilationDatabases +import org.gradle.kotlin.dsl.* +import org.jetbrains.kotlin.ExecClang +import org.jetbrains.kotlin.cpp.CompilationDatabaseExtension +import org.jetbrains.kotlin.cpp.CompilationDatabasePlugin +import org.jetbrains.kotlin.konan.target.KonanTarget import org.jetbrains.kotlin.konan.target.PlatformManager import org.jetbrains.kotlin.konan.target.SanitizerKind import org.jetbrains.kotlin.konan.target.supportedSanitizers @@ -26,7 +29,8 @@ import javax.inject.Inject */ open class CompileToBitcodePlugin : Plugin { override fun apply(target: Project) { - target.extensions.create(EXTENSION_NAME, CompileToBitcodeExtension::class.java, target) + target.apply() + target.extensions.create(EXTENSION_NAME, target) } companion object { @@ -36,6 +40,9 @@ open class CompileToBitcodePlugin : Plugin { open class CompileToBitcodeExtension @Inject constructor(val project: Project) { + private val compilationDatabase = project.extensions.getByType() + private val execClang = project.extensions.getByType() + private val targetList = with(project) { provider { (rootProject.project(":kotlin-native").property("targetList") as? List<*>)?.filterIsInstance() ?: emptyList() } // TODO: Can we make it better? } @@ -54,22 +61,19 @@ open class CompileToBitcodeExtension @Inject constructor(val project: Project) { }) } - private val compdbTasks by lazy { - targetList.get().associateBy(keySelector = { it }, valueTransform = { - val task = project.tasks.register("${it}${COMPILATION_DATABASE_TASK_NAME}", MergeCompilationDatabases::class.java) - task.configure { - outputFile = File(File(project.buildDir, it), "compile_commands.json") + private fun addToCompdb(compileTask: CompileToBitcode, konanTarget: KonanTarget) { + // No need to generate compdb entry for sanitizers. + if (compileTask.sanitizer != null) { + return + } + compilationDatabase.target(konanTarget) { + entry { + val args = listOf(execClang.resolveExecutable(compileTask.executable)) + compileTask.compilerFlags + execClang.clangArgsForCppRuntime(konanTarget.name) + directory.set(compileTask.objDir) + files.setFrom(compileTask.inputFiles) + arguments.set(args) + output.set(compileTask.outFile.absolutePath) } - task - }) - } - - private fun addToCompdb(compileTask: CompileToBitcode) { - val task = project.tasks.create("${compileTask.name}_CompilationDatabase", GenerateCompilationDatabase::class.java, compileTask.target, compileTask.inputFiles, compileTask.executable, compileTask.compilerFlags, compileTask.objDir) - val compdbTask = compdbTasks[compileTask.target]!! - compdbTask.configure { - dependsOn(task) - inputFiles.add(task.outputFile) } } @@ -96,7 +100,7 @@ open class CompileToBitcodeExtension @Inject constructor(val project: Project) { dependsOn(":kotlin-native:dependencies:update") configurationBlock() } - addToCompdb(task) + addToCompdb(task, target) if (outputGroup == "main" && sanitizer == null) { allMainModulesTask.configure { dependsOn(taskName) @@ -135,7 +139,7 @@ open class CompileToBitcodeExtension @Inject constructor(val project: Project) { dependsOn("downloadGoogleTest") compilerArgs.addAll(it.compilerArgs) - addToCompdb(this) + addToCompdb(this, konanTarget) } if (task.inputFiles.count() == 0) null else task diff --git a/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/cpp/CompilationDatabasePlugin.kt b/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/cpp/CompilationDatabasePlugin.kt new file mode 100644 index 00000000000..d61ab14c5b1 --- /dev/null +++ b/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/cpp/CompilationDatabasePlugin.kt @@ -0,0 +1,213 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.cpp + +import org.gradle.api.Action +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.provider.ListProperty +import org.gradle.api.provider.MapProperty +import org.gradle.api.provider.Property +import org.gradle.api.provider.Provider +import org.gradle.kotlin.dsl.* +import org.jetbrains.kotlin.konan.target.HostManager +import org.jetbrains.kotlin.konan.target.KonanTarget +import org.jetbrains.kotlin.konan.target.PlatformManager +import javax.inject.Inject + +/** + * Generating [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html). + * + * Allows generating compilation databases for different targets. For example: + * ``` + * compilationDatabase { + * target(someTarget1) { + * entry { ... } + * } + * allTargets { + * mergeFrom(provider { project("subproject") }) + * } + * } + * ``` + * Adds an entry for target `someTarget1`, and for each known target merges in databases generated + * by this plugin in `subproject`. + * The task that generates the database can be found via `compilationDatabase.target(someTarget1).task`. + * + * @see CompilationDatabasePlugin gradle plugin that creates this extension. + */ +abstract class CompilationDatabaseExtension @Inject constructor(private val project: Project) { + // TODO: This platformManager should acquired be from something service-ish. + // But for usefulness this service should be accessible from WorkAction. + private val platformManager = project.extensions.getByType() + + /** + * Entries in the compilation database. + * + * Single [Entry] generates a number of compilation database entries: one for each file in [files]. + * + * @property target target for which this [Entry] is generated. + */ + abstract class Entry @Inject constructor(val target: KonanTarget) { + /** + * **directory** from the [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html#format). + * + * The working directory of the compilation. All paths in the [arguments] must either be absolute or relative to this directory. + */ + abstract val directory: DirectoryProperty + + /** + * **file** from the [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html#format). + * + * Collection of files being compiled with given [arguments]. For each file a separate + * entry will be generated in the database with the same [directory], [arguments] and [output]. + */ + abstract val files: ConfigurableFileCollection + + /** + * **arguments** from the [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html#format). + * + * List of arguments of the compilation commands. The first argument must be the executable. + */ + abstract val arguments: ListProperty + + /** + * **output** from the [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html#format). + * + * The name of the output created by the compilation step. Used as a key to distinguish + * between different modes of compilation of the same sources. + */ + abstract val output: Property + } + + /** + * Configure compilation database generation for [target]. + * + * [entry] to add new entries. + * [mergeFrom] to merge databases from other projects with [CompilationDatabasePlugin]s. + * [task] is the gradle task for compilation database generation. + * + * @property target target for which compilation database is generated. + */ + abstract class Target @Inject constructor( + private val project: Project, + val target: KonanTarget, + ) { + protected abstract val mergeFrom: ListProperty + + /** + * Merge compilation database generated for [from] project for [target]. + * + * @param from project with applied [CompilationDatabasePlugin] to merge compilation database from. + */ + fun mergeFrom(from: Provider) { + mergeFrom.add(from.flatMap { project -> + project.extensions.getByType().target(target).task + }) + } + + protected abstract val entries: ListProperty + + /** + * Add an entry to the compilation database for [target]. + * + * @param action configure [Entry] + */ + fun entry(action: Action) { + entries.add(project.provider { + val instance = project.objects.newInstance(target).apply { + action.execute(this) + } + project.objects.newInstance().apply { + directory.set(instance.directory) + files.from(instance.files) + arguments.set(instance.arguments) + output.set(instance.output) + } + }) + } + + /** + * Gradle task that generates compilation database for [target]. + */ + val task = project.tasks.register("${target}CompilationDatabase") { + description = "Generate compilation database for $target" + group = TASK_GROUP + mergeFiles.from(mergeFrom) + entries.set(this@Target.entries) + outputFile.set(project.layout.buildDirectory.file("${target}/compile_commands.json")) + } + } + + protected abstract val targets: MapProperty + + private fun targetGetOrPut(target: KonanTarget) = targets.getting(target).orNull + ?: project.objects.newInstance(project, target).apply { + targets.put(target, this) + } + + /** + * Get [compilation database configuration][Target] for [target] and apply [action] to it. + * + * @param target target to configure. + * @param action action to apply to configuration. + */ + fun target(target: KonanTarget, action: Action) = targetGetOrPut(target).apply { + action.execute(this) + } + + /** + * Get [compilation database configuration][Target] for [target]. + * + * @param target target to configure. + */ + fun target(target: KonanTarget) = this.target(target) {} + + /** + * Get [compilation database configurations][Target] for all known targets and apply [action] to each. + * + * @param action action to apply to configurations. + */ + fun allTargets(action: Action) = platformManager.enabled.map { target(it, action) } + + /** + * Get [compilation database configurations][Target] for all known targets. + */ + val allTargets + get() = allTargets {} + + /** + * Get [compilation database configuration][Target] for [host target][HostManager.host] and apply [action] to it. + * + * @param action action to apply to configuration. + */ + fun hostTarget(action: Action) = target(HostManager.host, action) + + /** + * Get [compilation database configuration][Target] for [host target][HostManager.host]. + */ + val hostTarget + get() = hostTarget {} + + companion object { + @JvmStatic + val TASK_GROUP = "development support" + } +} + +/** + * Generating [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html). + * + * Creates [CompilationDatabaseExtension] extension named `compilationDatabase`. + * + * @see CompilationDatabaseExtension extension that this plugin creates. + */ +open class CompilationDatabasePlugin : Plugin { + override fun apply(project: Project) { + project.extensions.create("compilationDatabase", project) + } +} diff --git a/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/cpp/GenerateCompilationDatabase.kt b/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/cpp/GenerateCompilationDatabase.kt new file mode 100644 index 00000000000..20409544da6 --- /dev/null +++ b/kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/cpp/GenerateCompilationDatabase.kt @@ -0,0 +1,137 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.cpp + +import com.google.gson.annotations.Expose +import org.gradle.api.DefaultTask +import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.provider.ListProperty +import org.gradle.api.provider.Property +import org.gradle.api.provider.Provider +import org.gradle.api.tasks.* +import org.jetbrains.kotlin.gson +import java.io.FileReader +import java.io.FileWriter + +/** + * Generating [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html). + * + * Generate compilation database in [outputFile] from [entries] and merging in already + * generated databases in [mergeFiles]. + * + * @see CompilationDatabaseExtension gradle plugin to simplify generation of these tasks. + */ +abstract class GenerateCompilationDatabase : DefaultTask() { + /** + * Entries in the compilation database. + * + * Single [Entry] generates a number of compilation database entries: one for each file in [files]. + */ + abstract class Entry { + /** + * **directory** from the [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html#format). + * + * The working directory of the compilation. All paths in the [arguments] must either be absolute or relative to this directory. + */ + @get:Internal + abstract val directory: DirectoryProperty + + // Only the path of the directory is an input. + @get:Input + protected val pathToDirectory: Provider + get() = directory.asFile.map { it.absolutePath } + + /** + * **file** from the [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html#format). + * + * Collection of files being compiled with given [arguments]. For each file a separate + * entry will be generated in the database with the same [directory], [arguments] and [output]. + */ + @get:Internal + abstract val files: ConfigurableFileCollection + + // Only the paths of files are an input. + @get:Input + protected val pathsToFiles: Iterable + get() = files.files.map { it.absolutePath } + + /** + * **arguments** from the [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html#format). + * + * List of arguments of the compilation commands. The first argument must be the executable. + */ + @get:Input + abstract val arguments: ListProperty + + /** + * **output** from the [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html#format). + * + * The name of the output created by the compilation step. Used as a key to distinguish + * between different modes of compilation of the same sources. + */ + @get:Input + abstract val output: Property + } + + // Must follow https://clang.llvm.org/docs/JSONCompilationDatabase.html#format + private data class SerializedEntry( + @Expose val directory: String, + @Expose val file: String, + @Expose val arguments: List, + @Expose val output: String, + ) { + companion object { + fun fromEntry(entry: Entry): List { + val directory = entry.directory.asFile.get().absolutePath + val arguments = entry.arguments.get() + val output = entry.output.get() + return entry.files.map { + val file = it.absolutePath + SerializedEntry( + directory, + file, + arguments + listOf(file), + output, + ) + } + } + } + } + + /** + * List of [Entry]s to generate database from. + */ + @get:Nested + abstract val entries: ListProperty + + /** + * List of databases to merge into this database. + */ + @get:InputFiles + abstract val mergeFiles: ConfigurableFileCollection + + /** + * Where to put the database. + */ + @get:OutputFile + abstract val outputFile: RegularFileProperty + + @TaskAction + fun run() { + val serialized = mutableListOf() + mergeFiles.files.forEach { file -> + FileReader(file).use { + serialized.addAll(gson.fromJson(it, Array::class.java)) + } + } + serialized.addAll(entries.get().flatMap { SerializedEntry.fromEntry(it) }) + FileWriter(outputFile.asFile.get()).use { + gson.toJson(serialized, it) + } + } +} diff --git a/kotlin-native/build.gradle b/kotlin-native/build.gradle index cc5ca5b0fed..f5c1c063aa8 100644 --- a/kotlin-native/build.gradle +++ b/kotlin-native/build.gradle @@ -23,7 +23,7 @@ import org.jetbrains.kotlin.PlatformInfo import org.jetbrains.kotlin.KotlinBuildPusher import org.jetbrains.kotlin.CollisionDetector import org.jetbrains.kotlin.CollisionTransformer -import org.jetbrains.kotlin.CompilationDatabaseKt +import org.jetbrains.kotlin.cpp.CompilationDatabasePlugin import org.jetbrains.kotlin.CompareDistributionSignatures import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar import org.apache.tools.ant.filters.ReplaceTokens @@ -179,6 +179,8 @@ dependencies { commonSources project(path: ':kotlin-test:kotlin-test-annotations-common', configuration: 'sources') } +apply plugin: CompilationDatabasePlugin + //task sharedJar { // dependsOn gradle.includedBuild('shared').task(':jar') //} @@ -742,19 +744,19 @@ task pusher(type: KotlinBuildPusher){ token = project.findProperty("teamcityBearToken") ?: System.getenv("TEAMCITY_BEAR_TOKEN") } -targetList.each { target -> - CompilationDatabaseKt.mergeCompilationDatabases(project, "${target}CompilationDatabase".toString(), [ - ":kotlin-native::common:${target}CompilationDatabase".toString(), - ":kotlin-native:runtime:${target}CompilationDatabase".toString() - ]).configure { - outputFile = file("$buildDir/${target}/compile_commands.json") +compilationDatabase { + allTargets { + mergeFrom(provider { project("common") }) + mergeFrom(provider { project("runtime") }) } } -task compdb(type: Copy) { - dependsOn "${hostName}CompilationDatabase" - from "$buildDir/${hostName}/compile_commands.json" - into "$projectDir" +tasks.register("compdb", Copy) { + from compilationDatabase.hostTarget.task + into layout.projectDirectory + + group = compilationDatabase.TASK_GROUP + description = "Copy host compilation database to kotlin-native/" } targetList.each { targetName ->