Adding CompilationDatabase generation (#4091)

.gitignore

@@ -62,3 +62,9 @@ llvmCoverageMappingC/CMakeLists.txt
 tools/performance-server/node_modules
 tools/performance-server/server
 tools/performance-server/ui/js
+
+# compilation database
+compile_commands.json
+
+# clangd caches
+.clangd/

build-tools/src/main/kotlin/org/jetbrains/kotlin/CompilationDatabase.kt

@@ -0,0 +1,130 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+package org.jetbrains.kotlin
+
+import java.io.File
+import javax.inject.Inject
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.builtins.list
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonConfiguration
+import org.gradle.api.GradleException
+import org.gradle.api.Project
+import org.gradle.api.Task
+import org.gradle.api.DefaultTask
+import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.InputFiles
+import org.gradle.api.tasks.OutputFile
+import org.gradle.api.tasks.TaskAction
+import org.jetbrains.kotlin.konan.target.HostManager
+
+@Serializable
+private data class Entry(
+    val directory: String,
+    val file: String,
+    val arguments: List<String>,
+    val output: String
+) {
+    companion object {
+        fun create(
+            directory: File,
+            file: File,
+            args: List<String>,
+            outputDir: File
+        ): Entry {
+            return Entry(
+                directory.absolutePath,
+                file.absolutePath,
+                args + listOf(file.absolutePath),
+                File(outputDir, file.name + ".o").absolutePath
+            )
+        }
+
+        fun writeListTo(file: File, entries: List<Entry>) {
+            val json = Json(JsonConfiguration.Stable.copy(prettyPrint = true))
+            file.writeText(json.stringify(Entry.serializer().list, entries))
+        }
+
+        fun readListFrom(file: File): List<Entry> {
+            val json = Json(JsonConfiguration.Stable)
+            return json.parse(Entry.serializer().list, file.readText())
+        }
+    }
+}
+
+open class GenerateCompilationDatabase @Inject constructor(@Input val target: String,
+                                                           @Input val srcRoot: File,
+                                                           @Input val files: Iterable<File>,
+                                                           @Input val executable: String,
+                                                           @Input val compilerFlags: List<String>,
+                                                           @Input val outputDir: File
+): DefaultTask() {
+    @OutputFile
+    var outputFile = File(outputDir, "compile_commands.json")
+
+    @TaskAction
+    fun run() {
+        val plugin = project.convention.getPlugin(ExecClang::class.java)
+        val executable = plugin.resolveExecutable(executable)
+        val args = listOf(executable) + compilerFlags + plugin.konanArgs(target)
+        val entries: List<Entry> = files.map { Entry.create(srcRoot, it, args, outputDir) }
+        Entry.writeListTo(outputFile, entries)
+    }
+}
+
+open class MergeCompilationDatabases @Inject constructor(): DefaultTask() {
+    @InputFiles
+    val inputFiles = mutableListOf<File>()
+
+    @OutputFile
+    var outputFile = File(project.buildDir, "compile_commands.json")
+
+    @TaskAction
+    fun run() {
+        val entries = mutableListOf<Entry>()
+        for (file in inputFiles) {
+            entries.addAll(Entry.readListFrom(file))
+        }
+        Entry.writeListTo(outputFile, entries)
+    }
+}
+
+fun mergeCompilationDatabases(project: Project, name: String, paths: List<String>): Task {
+    val subtasks: List<MergeCompilationDatabases> = paths.map {
+        val task = project.tasks.getByPath(it)
+        if (task !is MergeCompilationDatabases) {
+            throw GradleException("Unknown task type for compdb merging: $task")
+        }
+        task
+    }
+    return project.tasks.create(name, MergeCompilationDatabases::class.java) { task ->
+        task.dependsOn(subtasks)
+        task.inputFiles.addAll(subtasks.map { it.outputFile })
+    }
+}
+
+fun createCompilationDatabaseFromCompileToBitcodeTasks(project: Project, name: String): Task {
+    val compileTasks = project.tasks.withType(CompileToBitcode::class.java).toList()
+    val compdbTasks = compileTasks.mapNotNull { task ->
+        // TODO: consider generating databases for more than just current host target.
+        if (task.target != HostManager.hostName) {
+            null
+        } else {
+            project.tasks.create("${task.name}_CompilationDatabase",
+                         GenerateCompilationDatabase::class.java,
+                         task.target,
+                         task.srcRoot,
+                         task.inputFiles,
+                         task.executable,
+                         task.compilerFlags,
+                         task.objDir)
+        }
+    }
+    return project.tasks.create(name, MergeCompilationDatabases::class.java) { task ->
+        task.dependsOn(compdbTasks)
+        task.inputFiles.addAll(compdbTasks.map { it.outputFile })
+    }
+}

build-tools/src/main/kotlin/org/jetbrains/kotlin/CompileToBitcode.kt

@@ -46,11 +46,45 @@ open class CompileToBitcode @Inject constructor(@InputDirectory val srcRoot: Fil
 
     private val targetDir by lazy { File(project.buildDir, target) }
 
-    private val objDir by lazy { File(targetDir, folderName) }
+    val objDir by lazy { File(targetDir, folderName) }
 
     private val KonanTarget.isMINGW
         get() = this.family == Family.MINGW
 
+    val executable
+        get() = when (language) {
+            Language.C -> "clang"
+            Language.CPP -> "clang++"
+        }
+
+    val compilerFlags: List<String>
+        get() {
+            val commonFlags = listOf("-c", "-emit-llvm", "-I$headersDir")
+            val languageFlags = when (language) {
+                Language.C ->
+                    // Used flags provided by original build of allocator C code.
+                    listOf("-std=gnu11", "-O3", "-Wall", "-Wextra", "-Wno-unknown-pragmas",
+                            "-Werror", "-ftls-model=initial-exec", "-Wno-unused-function")
+                Language.CPP ->
+                    listOfNotNull("-std=c++14", "-Werror", "-O2",
+                            "-fPIC".takeIf { !HostManager().targetByName(target).isMINGW })
+            }
+            return commonFlags + languageFlags + compilerArgs
+        }
+
+    val inputFiles: Iterable<File>
+        get() {
+            val srcFilesPatterns =
+                when (language) {
+                    Language.C -> listOf("**/*.c")
+                    Language.CPP -> listOf("**/*.cpp", "**/*.mm")
+                }
+            return project.fileTree(srcDir) {
+                it.include(srcFilesPatterns)
+                it.exclude(excludeFiles)
+            }.files
+        }
+
     @OutputFile
     val outFile = File(targetDir, "${folderName}.bc")
 
@@ -59,28 +93,11 @@ open class CompileToBitcode @Inject constructor(@InputDirectory val srcRoot: Fil
         if (target in excludedTargets) return
         objDir.mkdirs()
         val plugin = project.convention.getPlugin(ExecClang::class.java)
-        val commonFlags = listOf("-c", "-emit-llvm", "-I$headersDir")
-        val (executable, defaultFlags, srcFilesPatterns) =
-                when (language) {
-                    Language.C -> Triple("clang",
-                            // Used flags provided by original build of allocator C code.
-                            commonFlags + listOf("-std=gnu11", "-O3", "-Wall", "-Wextra", "-Wno-unknown-pragmas",
-                                    "-Werror", "-ftls-model=initial-exec", "-Wno-unused-function"),
-                            listOf("**/*.c"))
-                    Language.CPP -> Triple("clang++",
-                            commonFlags + listOfNotNull("-std=c++14", "-Werror", "-O2",
-                                    "-fPIC".takeIf { !HostManager().targetByName(target).isMINGW }),
-                            listOf("**/*.cpp", "**/*.mm"))
-                }
 
         plugin.execKonanClang(target, Action {
             it.workingDir = objDir
             it.executable = executable
-            it.args = defaultFlags + compilerArgs +
+            it.args = compilerFlags + inputFiles.map { it.absolutePath }
-                    project.fileTree(srcDir) {
-                        it.include(srcFilesPatterns)
-                        it.exclude(excludeFiles)
-                    }.files.map { it.absolutePath }
         })
 
         if (!skipLinkagePhase) {

build-tools/src/main/kotlin/org/jetbrains/kotlin/ExecClang.kt

@@ -34,11 +34,22 @@ class ExecClang(private val project: Project) {
         return platformManager.platform(target).clang.clangArgsForKonanSources.asList()
     }
 
-    private fun konanArgs(targetName: String?): List<String> {
+    fun konanArgs(targetName: String?): List<String> {
         val target = platformManager.targetManager(targetName).target
         return konanArgs(target)
     }
 
+    fun resolveExecutable(executable: String?): String {
+        val executable = executable ?: "clang"
+
+        if (listOf("clang", "clang++").contains(executable)) {
+            val llvmDir = project.findProperty("llvmDir")
+            return "${llvmDir}/bin/$executable"
+        } else {
+            throw GradleException("unsupported clang executable: $executable")
+        }
+    }
+
     // The bare ones invoke clang with system default sysroot.
 
     fun execBareClang(action: Action<in ExecSpec>): ExecResult {
@@ -82,16 +93,7 @@ class ExecClang(private val project: Project) {
                 action.execute(execSpec)
 
                 execSpec.apply {
-                    if (executable == null) {
+                    executable = resolveExecutable(executable)
-                        executable = "clang"
-                    }
-
-                    if (listOf("clang", "clang++").contains(executable)) {
-                        val llvmDir = project.findProperty("llvmDir")
-                        executable = "${llvmDir}/bin/$executable" }
-                    else {
-                        throw GradleException("unsupported clang executable: $executable")
-                    }
 
                     val hostPlatform = project.findProperty("hostPlatform") as Platform
                     environment["PATH"] = project.files(hostPlatform.clang.clangPaths).asPath +

build.gradle

@@ -23,6 +23,7 @@ import org.jetbrains.kotlin.PlatformInfo
 import org.jetbrains.kotlin.KotlinBuildPusher
 import org.jetbrains.kotlin.CollisionDetector
 import org.jetbrains.kotlin.CollisionTransformer
+import org.jetbrains.kotlin.CompilationDatabaseKt
 import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
 import org.apache.tools.ant.filters.ReplaceTokens
 import org.jetbrains.kotlin.UtilsKt
@@ -738,6 +739,7 @@ task clean {
     doLast {
         delete distDir
         delete bundle.outputs.files
+        delete tasks.findByName("compdb").outputFile
     }
 }
 
@@ -749,3 +751,10 @@ task pusher(type: KotlinBuildPusher){
     overrideConfigurationId = System.getenv("TEAMCITY_OVERRIDE_NATIVE_ID") ?: "Kotlin_KotlinDev_DeployMavenArtifacts_OverrideNative"
     token = project.findProperty("teamcityBearToken") ?: System.getenv("TEAMCITY_BEAR_TOKEN")
 }
+
+CompilationDatabaseKt.mergeCompilationDatabases(project, "compdb", [
+    ":common:compdb",
+    ":runtime:compdb"
+]).configure {
+    outputFile = file("$projectDir/compile_commands.json")
+}

common/build.gradle

@@ -3,6 +3,7 @@
  * that can be found in the LICENSE file.
  */
 import org.jetbrains.kotlin.CompileToBitcode
+import org.jetbrains.kotlin.CompilationDatabaseKt
 
 // TODO: consider using some Gradle plugins to build and test
 
@@ -14,6 +15,8 @@ targetList.each { targetName ->
     tasks.create("${targetName}Files", CompileToBitcode, file("src/files"), 'files', targetName)
 }
 
+CompilationDatabaseKt.createCompilationDatabaseFromCompileToBitcodeTasks(project, "compdb")
+
 task build {
     dependsOn "${hostName}Hash"
     dependsOn "${hostName}Files"

runtime/build.gradle

@@ -3,6 +3,7 @@
  * that can be found in the LICENSE file.
  */
 import org.jetbrains.kotlin.CompileToBitcode
+import org.jetbrains.kotlin.CompilationDatabaseKt
 import org.jetbrains.kotlin.UtilsKt
 
 // TODO: consider using some Gradle plugins to build and test
@@ -77,6 +78,8 @@ targetList.each { targetName ->
     }
 }
 
+CompilationDatabaseKt.createCompilationDatabaseFromCompileToBitcodeTasks(project, "compdb")
+
 task hostRuntime(dependsOn: "${hostName}Runtime")
 
 task clean {