[K/N] Modernize CompilationDatabase tasks

Merge-request: KT-MR-6276
Merged-by: Alexander Shabalin <Alexander.Shabalin@jetbrains.com>

buildSrc/build.gradle.kts

@@ -253,6 +253,10 @@ gradlePlugin {
             id = "runtime-testing"
             implementationClass = "org.jetbrains.kotlin.testing.native.RuntimeTestingPlugin"
         }
+        create("compilationDatabase") {
+            id = "compilation-database"
+            implementationClass = "org.jetbrains.kotlin.cpp.CompilationDatabasePlugin"
+        }
         create("konan") {
             id = "konan"
             implementationClass = "org.jetbrains.kotlin.gradle.plugin.konan.KonanPlugin"

kotlin-native/.gitignore

@@ -67,6 +67,7 @@ compile_commands.json
 
 # clangd caches
 .clangd/
+.cache/clangd
 
 # googletest framework used by runtime tests
 runtime/googletest/

kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/CompilationDatabase.kt

@@ -1,112 +0,0 @@
-/*
- * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
- * that can be found in the LICENSE file.
- */
-
-package org.jetbrains.kotlin
-
-import java.io.File
-import javax.inject.Inject
-import org.gradle.api.GradleException
-import com.google.gson.annotations.Expose
-import org.gradle.api.Project
-import org.gradle.api.Task
-import org.gradle.api.DefaultTask
-import org.gradle.api.tasks.Input
-import org.gradle.api.tasks.InputFiles
-import org.gradle.api.tasks.Internal
-import org.gradle.api.tasks.OutputFile
-import org.gradle.api.tasks.TaskAction
-import org.gradle.kotlin.dsl.getByType
-import java.io.FileReader
-import java.io.FileWriter
-
-internal data class Entry(
-        @Expose val directory: String,
-        @Expose val file: String,
-        @Expose val arguments: List<String>,
-        @Expose val output: String
-) {
-    companion object {
-        fun create(
-                directory: File,
-                file: File,
-                args: List<String>,
-                outputDir: File
-        ): Entry {
-            return Entry(
-                    directory.absolutePath,
-                    file.absolutePath,
-                    args + listOf(file.absolutePath),
-                    File(outputDir, file.name + ".o").absolutePath
-            )
-        }
-
-        fun writeListTo(file: File, entries: List<Entry>) = FileWriter(file).use {
-            gson.toJson(entries, it)
-        }
-
-        fun readListFrom(file: File): Array<Entry> = FileReader(file).use {
-            gson.fromJson(it, Array<Entry>::class.java)
-        }
-    }
-}
-
-open class GenerateCompilationDatabase @Inject constructor(@Input val target: String,
-                                                           @Internal val files: Iterable<File>,
-                                                           @Input val executable: String,
-                                                           @Input val compilerFlags: List<String>,
-                                                           @Internal val outputDir: File
-) : DefaultTask() {
-    @OutputFile
-    var outputFile = File(outputDir, "compile_commands.json")
-
-    // Annotate as an input because this path affects the content of the generated file.
-    @get:Input
-    val outputDirPath: String
-        get() = outputDir.absolutePath
-
-    @get:Input
-    val pathsToFiles: Iterable<String>
-        get() = files.map { it.absolutePath }
-
-    @TaskAction
-    fun run() {
-        val plugin = project.extensions.getByType<ExecClang>()
-        val executable = plugin.resolveExecutable(executable)
-        val args = listOf(executable) + compilerFlags + plugin.clangArgsForCppRuntime(target)
-        val entries: List<Entry> = files.map { Entry.create(it.parentFile, it, args, outputDir) }
-        Entry.writeListTo(outputFile, entries)
-    }
-}
-
-open class MergeCompilationDatabases @Inject constructor() : DefaultTask() {
-    @InputFiles
-    val inputFiles = mutableListOf<File>()
-
-    @OutputFile
-    var outputFile = File(project.buildDir, "compile_commands.json")
-
-    @TaskAction
-    fun run() {
-        val entries = mutableListOf<Entry>()
-        for (file in inputFiles) {
-            entries.addAll(Entry.readListFrom(file))
-        }
-        Entry.writeListTo(outputFile, entries)
-    }
-}
-
-fun mergeCompilationDatabases(project: Project, name: String, paths: List<String>): Task {
-    val subtasks: List<MergeCompilationDatabases> = paths.map {
-        val task = project.tasks.getByPath(it)
-        if (task !is MergeCompilationDatabases) {
-            throw GradleException("Unknown task type for compdb merging: $task")
-        }
-        task
-    }
-    return project.tasks.create(name, MergeCompilationDatabases::class.java) {
-        dependsOn(subtasks)
-        inputFiles.addAll(subtasks.map { it.outputFile })
-    }
-}

kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/bitcode/CompileToBitcodePlugin.kt

@@ -9,8 +9,11 @@ import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.plugins.BasePlugin
-import org.jetbrains.kotlin.GenerateCompilationDatabase
+import org.gradle.kotlin.dsl.*
-import org.jetbrains.kotlin.MergeCompilationDatabases
+import org.jetbrains.kotlin.ExecClang
+import org.jetbrains.kotlin.cpp.CompilationDatabaseExtension
+import org.jetbrains.kotlin.cpp.CompilationDatabasePlugin
+import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.konan.target.PlatformManager
 import org.jetbrains.kotlin.konan.target.SanitizerKind
 import org.jetbrains.kotlin.konan.target.supportedSanitizers
@@ -26,7 +29,8 @@ import javax.inject.Inject
  */
 open class CompileToBitcodePlugin : Plugin<Project> {
     override fun apply(target: Project) {
-        target.extensions.create(EXTENSION_NAME, CompileToBitcodeExtension::class.java, target)
+        target.apply<CompilationDatabasePlugin>()
+        target.extensions.create<CompileToBitcodeExtension>(EXTENSION_NAME, target)
     }
 
     companion object {
@@ -36,6 +40,9 @@ open class CompileToBitcodePlugin : Plugin<Project> {
 
 open class CompileToBitcodeExtension @Inject constructor(val project: Project) {
 
+    private val compilationDatabase = project.extensions.getByType<CompilationDatabaseExtension>()
+    private val execClang = project.extensions.getByType<ExecClang>()
+
     private val targetList = with(project) {
         provider { (rootProject.project(":kotlin-native").property("targetList") as? List<*>)?.filterIsInstance<String>() ?: emptyList() } // TODO: Can we make it better?
     }
@@ -54,22 +61,19 @@ open class CompileToBitcodeExtension @Inject constructor(val project: Project) {
         })
     }
 
-    private val compdbTasks by lazy {
+    private fun addToCompdb(compileTask: CompileToBitcode, konanTarget: KonanTarget) {
-        targetList.get().associateBy(keySelector = { it }, valueTransform = {
+        // No need to generate compdb entry for sanitizers.
-            val task = project.tasks.register("${it}${COMPILATION_DATABASE_TASK_NAME}", MergeCompilationDatabases::class.java)
+        if (compileTask.sanitizer != null) {
-            task.configure {
+            return
-                outputFile = File(File(project.buildDir, it), "compile_commands.json")
+        }
+        compilationDatabase.target(konanTarget) {
+            entry {
+                val args = listOf(execClang.resolveExecutable(compileTask.executable)) + compileTask.compilerFlags + execClang.clangArgsForCppRuntime(konanTarget.name)
+                directory.set(compileTask.objDir)
+                files.setFrom(compileTask.inputFiles)
+                arguments.set(args)
+                output.set(compileTask.outFile.absolutePath)
             }
-            task
-        })
-    }
-
-    private fun addToCompdb(compileTask: CompileToBitcode) {
-        val task = project.tasks.create("${compileTask.name}_CompilationDatabase", GenerateCompilationDatabase::class.java, compileTask.target, compileTask.inputFiles, compileTask.executable, compileTask.compilerFlags, compileTask.objDir)
-        val compdbTask = compdbTasks[compileTask.target]!!
-        compdbTask.configure {
-            dependsOn(task)
-            inputFiles.add(task.outputFile)
         }
     }
 
@@ -96,7 +100,7 @@ open class CompileToBitcodeExtension @Inject constructor(val project: Project) {
                     dependsOn(":kotlin-native:dependencies:update")
                     configurationBlock()
                 }
-                addToCompdb(task)
+                addToCompdb(task, target)
                 if (outputGroup == "main" && sanitizer == null) {
                     allMainModulesTask.configure {
                         dependsOn(taskName)
@@ -135,7 +139,7 @@ open class CompileToBitcodeExtension @Inject constructor(val project: Project) {
                         dependsOn("downloadGoogleTest")
                         compilerArgs.addAll(it.compilerArgs)
 
-                        addToCompdb(this)
+                        addToCompdb(this, konanTarget)
                     }
             if (task.inputFiles.count() == 0) null
             else task

kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/cpp/CompilationDatabasePlugin.kt

@@ -0,0 +1,213 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.cpp
+
+import org.gradle.api.Action
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.api.file.ConfigurableFileCollection
+import org.gradle.api.file.DirectoryProperty
+import org.gradle.api.provider.ListProperty
+import org.gradle.api.provider.MapProperty
+import org.gradle.api.provider.Property
+import org.gradle.api.provider.Provider
+import org.gradle.kotlin.dsl.*
+import org.jetbrains.kotlin.konan.target.HostManager
+import org.jetbrains.kotlin.konan.target.KonanTarget
+import org.jetbrains.kotlin.konan.target.PlatformManager
+import javax.inject.Inject
+
+/**
+ * Generating [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html).
+ *
+ * Allows generating compilation databases for different targets. For example:
+ * ```
+ * compilationDatabase {
+ *    target(someTarget1) {
+ *        entry { ... }
+ *    }
+ *    allTargets {
+ *        mergeFrom(provider { project("subproject") })
+ *    }
+ * }
+ * ```
+ * Adds an entry for target `someTarget1`, and for each known target merges in databases generated
+ * by this plugin in `subproject`.
+ * The task that generates the database can be found via `compilationDatabase.target(someTarget1).task`.
+ *
+ * @see CompilationDatabasePlugin gradle plugin that creates this extension.
+ */
+abstract class CompilationDatabaseExtension @Inject constructor(private val project: Project) {
+    // TODO: This platformManager should acquired be from something service-ish.
+    //       But for usefulness this service should be accessible from WorkAction.
+    private val platformManager = project.extensions.getByType<PlatformManager>()
+
+    /**
+     * Entries in the compilation database.
+     *
+     * Single [Entry] generates a number of compilation database entries: one for each file in [files].
+     *
+     * @property target target for which this [Entry] is generated.
+     */
+    abstract class Entry @Inject constructor(val target: KonanTarget) {
+        /**
+         * **directory** from the [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html#format).
+         *
+         * The working directory of the compilation. All paths in the [arguments] must either be absolute or relative to this directory.
+         */
+        abstract val directory: DirectoryProperty
+
+        /**
+         * **file** from the [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html#format).
+         *
+         * Collection of files being compiled with given [arguments]. For each file a separate
+         * entry will be generated in the database with the same [directory], [arguments] and [output].
+         */
+        abstract val files: ConfigurableFileCollection
+
+        /**
+         * **arguments** from the [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html#format).
+         *
+         * List of arguments of the compilation commands. The first argument must be the executable.
+         */
+        abstract val arguments: ListProperty<String>
+
+        /**
+         * **output** from the [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html#format).
+         *
+         * The name of the output created by the compilation step. Used as a key to distinguish
+         * between different modes of compilation of the same sources.
+         */
+        abstract val output: Property<String>
+    }
+
+    /**
+     * Configure compilation database generation for [target].
+     *
+     * [entry] to add new entries.
+     * [mergeFrom] to merge databases from other projects with [CompilationDatabasePlugin]s.
+     * [task] is the gradle task for compilation database generation.
+     *
+     * @property target target for which compilation database is generated.
+     */
+    abstract class Target @Inject constructor(
+            private val project: Project,
+            val target: KonanTarget,
+    ) {
+        protected abstract val mergeFrom: ListProperty<GenerateCompilationDatabase>
+
+        /**
+         * Merge compilation database generated for [from] project for [target].
+         *
+         * @param from project with applied [CompilationDatabasePlugin] to merge compilation database from.
+         */
+        fun mergeFrom(from: Provider<Project>) {
+            mergeFrom.add(from.flatMap { project ->
+                project.extensions.getByType<CompilationDatabaseExtension>().target(target).task
+            })
+        }
+
+        protected abstract val entries: ListProperty<GenerateCompilationDatabase.Entry>
+
+        /**
+         * Add an entry to the compilation database for [target].
+         *
+         * @param action configure [Entry]
+         */
+        fun entry(action: Action<in Entry>) {
+            entries.add(project.provider {
+                val instance = project.objects.newInstance<Entry>(target).apply {
+                    action.execute(this)
+                }
+                project.objects.newInstance<GenerateCompilationDatabase.Entry>().apply {
+                    directory.set(instance.directory)
+                    files.from(instance.files)
+                    arguments.set(instance.arguments)
+                    output.set(instance.output)
+                }
+            })
+        }
+
+        /**
+         * Gradle task that generates compilation database for [target].
+         */
+        val task = project.tasks.register<GenerateCompilationDatabase>("${target}CompilationDatabase") {
+            description = "Generate compilation database for $target"
+            group = TASK_GROUP
+            mergeFiles.from(mergeFrom)
+            entries.set(this@Target.entries)
+            outputFile.set(project.layout.buildDirectory.file("${target}/compile_commands.json"))
+        }
+    }
+
+    protected abstract val targets: MapProperty<KonanTarget, Target>
+
+    private fun targetGetOrPut(target: KonanTarget) = targets.getting(target).orNull
+            ?: project.objects.newInstance<Target>(project, target).apply {
+                targets.put(target, this)
+            }
+
+    /**
+     * Get [compilation database configuration][Target] for [target] and apply [action] to it.
+     *
+     * @param target target to configure.
+     * @param action action to apply to configuration.
+     */
+    fun target(target: KonanTarget, action: Action<in Target>) = targetGetOrPut(target).apply {
+        action.execute(this)
+    }
+
+    /**
+     * Get [compilation database configuration][Target] for [target].
+     *
+     * @param target target to configure.
+     */
+    fun target(target: KonanTarget) = this.target(target) {}
+
+    /**
+     * Get [compilation database configurations][Target] for all known targets and apply [action] to each.
+     *
+     * @param action action to apply to configurations.
+     */
+    fun allTargets(action: Action<in Target>) = platformManager.enabled.map { target(it, action) }
+
+    /**
+     * Get [compilation database configurations][Target] for all known targets.
+     */
+    val allTargets
+        get() = allTargets {}
+
+    /**
+     * Get [compilation database configuration][Target] for [host target][HostManager.host] and apply [action] to it.
+     *
+     * @param action action to apply to configuration.
+     */
+    fun hostTarget(action: Action<in Target>) = target(HostManager.host, action)
+
+    /**
+     * Get [compilation database configuration][Target] for [host target][HostManager.host].
+     */
+    val hostTarget
+        get() = hostTarget {}
+
+    companion object {
+        @JvmStatic
+        val TASK_GROUP = "development support"
+    }
+}
+
+/**
+ * Generating [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html).
+ *
+ * Creates [CompilationDatabaseExtension] extension named `compilationDatabase`.
+ *
+ * @see CompilationDatabaseExtension extension that this plugin creates.
+ */
+open class CompilationDatabasePlugin : Plugin<Project> {
+    override fun apply(project: Project) {
+        project.extensions.create<CompilationDatabaseExtension>("compilationDatabase", project)
+    }
+}

kotlin-native/build-tools/src/main/kotlin/org/jetbrains/kotlin/cpp/GenerateCompilationDatabase.kt

@@ -0,0 +1,137 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.cpp
+
+import com.google.gson.annotations.Expose
+import org.gradle.api.DefaultTask
+import org.gradle.api.file.ConfigurableFileCollection
+import org.gradle.api.file.DirectoryProperty
+import org.gradle.api.file.RegularFileProperty
+import org.gradle.api.provider.ListProperty
+import org.gradle.api.provider.Property
+import org.gradle.api.provider.Provider
+import org.gradle.api.tasks.*
+import org.jetbrains.kotlin.gson
+import java.io.FileReader
+import java.io.FileWriter
+
+/**
+ * Generating [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html).
+ *
+ * Generate compilation database in [outputFile] from [entries] and merging in already
+ * generated databases in [mergeFiles].
+ *
+ * @see CompilationDatabaseExtension gradle plugin to simplify generation of these tasks.
+ */
+abstract class GenerateCompilationDatabase : DefaultTask() {
+    /**
+     * Entries in the compilation database.
+     *
+     * Single [Entry] generates a number of compilation database entries: one for each file in [files].
+     */
+    abstract class Entry {
+        /**
+         * **directory** from the [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html#format).
+         *
+         * The working directory of the compilation. All paths in the [arguments] must either be absolute or relative to this directory.
+         */
+        @get:Internal
+        abstract val directory: DirectoryProperty
+
+        // Only the path of the directory is an input.
+        @get:Input
+        protected val pathToDirectory: Provider<String>
+            get() = directory.asFile.map { it.absolutePath }
+
+        /**
+         * **file** from the [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html#format).
+         *
+         * Collection of files being compiled with given [arguments]. For each file a separate
+         * entry will be generated in the database with the same [directory], [arguments] and [output].
+         */
+        @get:Internal
+        abstract val files: ConfigurableFileCollection
+
+        // Only the paths of files are an input.
+        @get:Input
+        protected val pathsToFiles: Iterable<String>
+            get() = files.files.map { it.absolutePath }
+
+        /**
+         * **arguments** from the [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html#format).
+         *
+         * List of arguments of the compilation commands. The first argument must be the executable.
+         */
+        @get:Input
+        abstract val arguments: ListProperty<String>
+
+        /**
+         * **output** from the [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html#format).
+         *
+         * The name of the output created by the compilation step. Used as a key to distinguish
+         * between different modes of compilation of the same sources.
+         */
+        @get:Input
+        abstract val output: Property<String>
+    }
+
+    // Must follow https://clang.llvm.org/docs/JSONCompilationDatabase.html#format
+    private data class SerializedEntry(
+            @Expose val directory: String,
+            @Expose val file: String,
+            @Expose val arguments: List<String>,
+            @Expose val output: String,
+    ) {
+        companion object {
+            fun fromEntry(entry: Entry): List<SerializedEntry> {
+                val directory = entry.directory.asFile.get().absolutePath
+                val arguments = entry.arguments.get()
+                val output = entry.output.get()
+                return entry.files.map {
+                    val file = it.absolutePath
+                    SerializedEntry(
+                            directory,
+                            file,
+                            arguments + listOf(file),
+                            output,
+                    )
+                }
+            }
+        }
+    }
+
+    /**
+     * List of [Entry]s to generate database from.
+     */
+    @get:Nested
+    abstract val entries: ListProperty<Entry>
+
+    /**
+     * List of databases to merge into this database.
+     */
+    @get:InputFiles
+    abstract val mergeFiles: ConfigurableFileCollection
+
+    /**
+     * Where to put the database.
+     */
+    @get:OutputFile
+    abstract val outputFile: RegularFileProperty
+
+    @TaskAction
+    fun run() {
+        val serialized = mutableListOf<SerializedEntry>()
+        mergeFiles.files.forEach { file ->
+            FileReader(file).use {
+                serialized.addAll(gson.fromJson(it, Array<SerializedEntry>::class.java))
+            }
+        }
+        serialized.addAll(entries.get().flatMap { SerializedEntry.fromEntry(it) })
+        FileWriter(outputFile.asFile.get()).use {
+            gson.toJson(serialized, it)
+        }
+    }
+}

kotlin-native/build.gradle

@@ -23,7 +23,7 @@ import org.jetbrains.kotlin.PlatformInfo
 import org.jetbrains.kotlin.KotlinBuildPusher
 import org.jetbrains.kotlin.CollisionDetector
 import org.jetbrains.kotlin.CollisionTransformer
-import org.jetbrains.kotlin.CompilationDatabaseKt
+import org.jetbrains.kotlin.cpp.CompilationDatabasePlugin
 import org.jetbrains.kotlin.CompareDistributionSignatures
 import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
 import org.apache.tools.ant.filters.ReplaceTokens
@@ -179,6 +179,8 @@ dependencies {
     commonSources project(path: ':kotlin-test:kotlin-test-annotations-common', configuration: 'sources')
 }
 
+apply plugin: CompilationDatabasePlugin
+
 //task sharedJar {
 //    dependsOn gradle.includedBuild('shared').task(':jar')
 //}
@@ -742,19 +744,19 @@ task pusher(type: KotlinBuildPusher){
     token = project.findProperty("teamcityBearToken") ?: System.getenv("TEAMCITY_BEAR_TOKEN")
 }
 
-targetList.each { target ->
+compilationDatabase {
-    CompilationDatabaseKt.mergeCompilationDatabases(project, "${target}CompilationDatabase".toString(), [
+    allTargets {
-            ":kotlin-native::common:${target}CompilationDatabase".toString(),
+        mergeFrom(provider { project("common") })
-            ":kotlin-native:runtime:${target}CompilationDatabase".toString()
+        mergeFrom(provider { project("runtime") })
-    ]).configure {
-        outputFile = file("$buildDir/${target}/compile_commands.json")
     }
 }
 
-task compdb(type: Copy) {
+tasks.register("compdb", Copy) {
-    dependsOn "${hostName}CompilationDatabase"
+    from compilationDatabase.hostTarget.task
-    from "$buildDir/${hostName}/compile_commands.json"
+    into layout.projectDirectory
-    into "$projectDir"
+
+    group = compilationDatabase.TASK_GROUP
+    description = "Copy host compilation database to kotlin-native/"
 }
 
 targetList.each { targetName ->