[Build] Refactor compiling native code to bitcode

Also switch to kts in the corresponding buildscripts.

build-tools/build.gradle.kts

@@ -82,6 +82,10 @@ gradlePlugin {
             id = "swift-benchmarking"
             implementationClass = "org.jetbrains.kotlin.benchmark.SwiftBenchmarkingPlugin"
         }
+        create("compileToBitcode") {
+            id = "compile-to-bitcode"
+            implementationClass = "org.jetbrains.kotlin.bitcode.CompileToBitcodePlugin"
+        }
     }
 }
 

build-tools/src/main/kotlin/org/jetbrains/kotlin/CompilationDatabase.kt

@@ -16,7 +16,7 @@ import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.InputFiles
 import org.gradle.api.tasks.OutputFile
 import org.gradle.api.tasks.TaskAction
-import org.jetbrains.kotlin.konan.target.HostManager
+import org.jetbrains.kotlin.bitcode.CompileToBitcode
 import java.io.FileReader
 import java.io.FileWriter
 

build-tools/src/main/kotlin/org/jetbrains/kotlin/Internals.kt

@@ -8,11 +8,14 @@ package org.jetbrains.kotlin
 import org.gradle.api.NamedDomainObjectCollection
 import org.gradle.api.NamedDomainObjectContainer
 import org.gradle.api.Project
+import org.gradle.api.file.FileCollection
 import org.gradle.api.plugins.ExtraPropertiesExtension
+import org.gradle.api.provider.Provider
 import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension
 import org.jetbrains.kotlin.gradle.plugin.KotlinCompilation
 import org.jetbrains.kotlin.gradle.plugin.KotlinTargetPreset
 import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeCompilation
+import java.io.File
 
 /*
  * This file includes internal short-cuts visible only inside of the 'buildSrc' module.
@@ -38,3 +41,8 @@ internal val NamedDomainObjectCollection<KotlinTargetPreset<*>>.mingwX64: Kotlin
 
 internal val NamedDomainObjectContainer<out KotlinCompilation<*>>.main: KotlinNativeCompilation
     get() = getByName(::main.name) as KotlinNativeCompilation
+
+internal val FileCollection.isNotEmpty: Boolean
+    get() = !isEmpty
+
+internal fun Provider<File>.resolve(child: String): Provider<File> = map { it.resolve(child) }

build-tools/src/main/kotlin/org/jetbrains/kotlin/bitcode/CompileToBitcode.kt

@@ -1,47 +1,51 @@
 /*
- * Copyright 2010-2017 JetBrains s.r.o.
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
- *
+ * that can be found in the license/LICENSE.txt file.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
  */
 
-package org.jetbrains.kotlin
+package org.jetbrains.kotlin.bitcode
 
-import org.gradle.api.Action
 import org.gradle.api.DefaultTask
-import org.gradle.api.tasks.InputDirectory
+import org.gradle.api.file.FileCollection
-import org.gradle.api.tasks.OutputFile
+import org.gradle.api.tasks.*
-import org.gradle.api.tasks.TaskAction
+import org.jetbrains.kotlin.ExecClang
 import org.jetbrains.kotlin.konan.target.Family
 import org.jetbrains.kotlin.konan.target.HostManager
 import org.jetbrains.kotlin.konan.target.KonanTarget
-
 import java.io.File
 import javax.inject.Inject
 
-open class CompileToBitcode @Inject constructor(@InputDirectory val srcRoot: File,
+open class CompileToBitcode @Inject constructor(
-                                                val folderName: String,
+        val srcRoot: File,
-                                                val target: String) : DefaultTask() {
+        val folderName: String,
+        val target: String
+) : DefaultTask() {
+
     enum class Language {
         C, CPP
     }
 
+    // Compiler args are part of compilerFlags so we don't register them as an input.
     val compilerArgs = mutableListOf<String>()
+    @Input
     val linkerArgs = mutableListOf<String>()
-    val excludeFiles = mutableListOf<String>()
+    var excludeFiles: List<String> = listOf(
-    var srcDir = File(srcRoot, "cpp")
+            "**/*Test.cpp",
-    var headersDir = File(srcRoot, "headers")
+            "**/*Test.mm",
+    )
+    var includeFiles: List<String> = listOf(
+            "**/*.cpp",
+            "**/*.mm"
+    )
+
+    // Source files and headers are registered as inputs by the `inputFiles` and `headers` properties.
+    var srcDirs: FileCollection = project.files(srcRoot.resolve("cpp"))
+    var headersDirs: FileCollection = project.files(srcRoot.resolve("headers"))
+
+    @Input
     var skipLinkagePhase = false
-    var excludedTargets = mutableListOf<String>()
+
+    @Input
     var language = Language.CPP
 
     private val targetDir by lazy { File(project.buildDir, target) }
@@ -57,9 +61,10 @@ open class CompileToBitcode @Inject constructor(@InputDirectory val srcRoot: Fil
             Language.CPP -> "clang++"
         }
 
+    @get:Input
     val compilerFlags: List<String>
         get() {
-            val commonFlags = listOf("-c", "-emit-llvm", "-I$headersDir")
+            val commonFlags = listOf("-c", "-emit-llvm") + headersDirs.map { "-I$it" }
             val languageFlags = when (language) {
                 Language.C ->
                     // Used flags provided by original build of allocator C code.
@@ -75,17 +80,30 @@ open class CompileToBitcode @Inject constructor(@InputDirectory val srcRoot: Fil
             return commonFlags + languageFlags + compilerArgs
         }
 
+    @get:SkipWhenEmpty
+    @get:InputFiles
     val inputFiles: Iterable<File>
         get() {
-            val srcFilesPatterns =
+            return srcDirs.flatMap { srcDir ->
-                when (language) {
+                project.fileTree(srcDir) {
-                    Language.C -> listOf("**/*.c")
+                    it.include(includeFiles)
-                    Language.CPP -> listOf("**/*.cpp", "**/*.mm")
+                    it.exclude(excludeFiles)
-                }
+                }.files
-            return project.fileTree(srcDir) {
+            }
-                it.include(srcFilesPatterns)
+        }
-                it.exclude(excludeFiles)
+
-            }.files
+    @get:InputFiles
+    protected val headers: Iterable<File>
+        get() {
+            return headersDirs.files.flatMap { dir ->
+                project.fileTree(dir) {
+                    val includePatterns = when (language) {
+                        Language.C -> arrayOf("**/.h")
+                        Language.CPP -> arrayOf("**/*.h", "**/*.hpp")
+                    }
+                    it.include(*includePatterns)
+                }.files
+            }
         }
 
     @OutputFile
@@ -93,15 +111,14 @@ open class CompileToBitcode @Inject constructor(@InputDirectory val srcRoot: Fil
 
     @TaskAction
     fun compile() {
-        if (target in excludedTargets) return
         objDir.mkdirs()
         val plugin = project.convention.getPlugin(ExecClang::class.java)
 
-        plugin.execKonanClang(target, Action {
+        plugin.execKonanClang(target) {
             it.workingDir = objDir
             it.executable = executable
             it.args = compilerFlags + inputFiles.map { it.absolutePath }
-        })
+        }
 
         if (!skipLinkagePhase) {
             project.exec {
@@ -114,4 +131,4 @@ open class CompileToBitcode @Inject constructor(@InputDirectory val srcRoot: Fil
             }
         }
     }
-}
+}

build-tools/src/main/kotlin/org/jetbrains/kotlin/bitcode/CompileToBitcodePlugin.kt

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.bitcode
+
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.api.plugins.BasePlugin
+import org.jetbrains.kotlin.createCompilationDatabasesFromCompileToBitcodeTasks
+import java.io.File
+import javax.inject.Inject
+
+/**
+ * A plugin creating extensions to compile
+ */
+open class CompileToBitcodePlugin: Plugin<Project> {
+    override fun apply(target: Project) = with(target) {
+        extensions.create(EXTENSION_NAME, CompileToBitcodeExtension::class.java, target)
+
+        afterEvaluate {
+            // TODO: Support providers (https://docs.gradle.org/current/userguide/lazy_configuration.html)
+            //       in database tasks and create them along with corresponding compile tasks (not in afterEvaluate).
+            createCompilationDatabasesFromCompileToBitcodeTasks(project, COMPILATION_DATABASE_TASK_NAME)
+        }
+    }
+
+    companion object {
+        const val EXTENSION_NAME = "bitcode"
+        const val COMPILATION_DATABASE_TASK_NAME = "CompilationDatabase"
+    }
+}
+
+open class CompileToBitcodeExtension @Inject constructor(val project: Project) {
+
+    private val targetList = with(project) {
+        provider { rootProject.property("targetList") as List<String> } // TODO: Can we make it better?
+    }
+
+    fun create(name: String, srcDir: File = project.file("src/$name"), configurationBlock: CompileToBitcode.() -> Unit = {}) {
+        targetList.get().forEach { targetName ->
+            project.tasks.register(
+                    "${targetName}${name.snakeCaseToCamelCase().capitalize()}",
+                    CompileToBitcode::class.java,
+                    srcDir, name, targetName
+            ).configure {
+                it.group = BasePlugin.BUILD_GROUP
+                it.description = "Compiles '$name' to bitcode for $targetName"
+                it.configurationBlock()
+            }
+        }
+    }
+
+    companion object {
+        private fun String.snakeCaseToCamelCase() =
+                split('_').joinToString(separator = "") { it.capitalize() }
+    }
+}

common/build.gradle

@@ -1,29 +0,0 @@
-/*
- * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
- * that can be found in the LICENSE file.
- */
-import org.jetbrains.kotlin.CompileToBitcode
-import org.jetbrains.kotlin.CompilationDatabaseKt
-
-// TODO: consider using some Gradle plugins to build and test
-
-targetList.each { targetName ->
-    tasks.create("${targetName}Hash", CompileToBitcode, file("src/hash"), 'hash', targetName)
-}
-
-targetList.each { targetName ->
-    tasks.create("${targetName}Files", CompileToBitcode, file("src/files"), 'files', targetName)
-}
-
-CompilationDatabaseKt.createCompilationDatabasesFromCompileToBitcodeTasks(project, "CompilationDatabase")
-
-task build {
-    dependsOn "${hostName}Hash"
-    dependsOn "${hostName}Files"
-}
-
-task clean {
-    doLast {
-        delete buildDir
-    }
-}

common/build.gradle.kts

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+plugins {
+    id("compile-to-bitcode")
+}
+
+bitcode {
+    create("hash")
+    create("files")
+}
+
+val hostName: String by project
+
+val build by tasks.registering {
+    dependsOn("${hostName}Hash")
+    dependsOn("${hostName}Files")
+}
+
+val clean by tasks.registering {
+    doLast {
+        delete(buildDir)
+    }
+}

runtime/build.gradle

@@ -1,106 +0,0 @@
-/*
- * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
- * that can be found in the LICENSE file.
- */
-import org.jetbrains.kotlin.CompileToBitcode
-import org.jetbrains.kotlin.CompilationDatabaseKt
-import org.jetbrains.kotlin.UtilsKt
-
-// TODO: consider using some Gradle plugins to build and test
-
-void includeRuntime(CompileToBitcode task) {
-    task.compilerArgs.add('-I' + project.file('../common/src/hash/headers'))
-    task.compilerArgs.add('-I' + project.file('src/main/cpp'))
-}
-
-targetList.each { targetName ->
-    tasks.create("${targetName}Runtime", CompileToBitcode, file('src/main'), "runtime", targetName).configure {
-        dependsOn ":common:${targetName}Hash"
-        dependsOn "${targetName}StdAlloc"
-        dependsOn "${targetName}OptAlloc"
-        dependsOn "${targetName}Mimalloc"
-        dependsOn "${targetName}Launcher"
-        dependsOn "${targetName}Debug"
-        dependsOn "${targetName}Release"
-        dependsOn "${targetName}Strict"
-        dependsOn "${targetName}Relaxed"
-        dependsOn "${targetName}ProfileRuntime"
-        dependsOn "${targetName}ObjC"
-        dependsOn "${targetName}ExceptionsSupport"
-        includeRuntime(delegate)
-        linkerArgs.add(project.file("../common/build/$targetName/hash.bc").path)
-    }
-
-    tasks.create("${targetName}Mimalloc", CompileToBitcode, file('src/mimalloc'), "mimalloc", targetName).configure {
-        language = CompileToBitcode.Language.C
-        excludeFiles.addAll(["**/alloc-override*.c", "**/page-queue.c", "**/static.c"])
-        if (!UtilsKt.targetSupportsMimallocAllocator(targetName))
-            excludedTargets.add(targetName)
-        srcDir = new File(srcRoot, "c")
-        compilerArgs.add("-DKONAN_MI_MALLOC=1")
-        headersDir = new File(srcDir, "include")
-    }
-
-    tasks.create("${targetName}Launcher", CompileToBitcode, file('src/launcher'), "launcher", targetName).configure {
-        includeRuntime(delegate)
-    }
-
-    tasks.create("${targetName}Debug", CompileToBitcode, file('src/debug'), "debug", targetName).configure {
-        includeRuntime(delegate)
-    }
-
-    tasks.create("${targetName}StdAlloc", CompileToBitcode, file('src/std_alloc'), "std_alloc", targetName)
-
-    tasks.create("${targetName}OptAlloc", CompileToBitcode, file('src/opt_alloc'), "opt_alloc", targetName)
-
-    tasks.create("${targetName}ExceptionsSupport", CompileToBitcode, file('src/exceptions_support'),
-            "exceptionsSupport", targetName).configure {
-        includeRuntime(delegate)
-    }
-
-    tasks.create("${targetName}Release", CompileToBitcode, file('src/release'), "release", targetName).configure {
-        includeRuntime(delegate)
-    }
-
-    tasks.create("${targetName}Strict", CompileToBitcode, file('src/strict'), "strict", targetName).configure {
-        includeRuntime(delegate)
-    }
-
-    tasks.create("${targetName}Relaxed", CompileToBitcode, file('src/relaxed'), "relaxed", targetName).configure {
-        includeRuntime(delegate)
-    }
-
-    tasks.create("${targetName}ProfileRuntime", CompileToBitcode, file('src/profile_runtime'),
-            "profileRuntime", targetName)
-
-    tasks.create("${targetName}ObjC", CompileToBitcode, file('src/objc'), "objc", targetName).configure {
-        includeRuntime(delegate)
-    }
-}
-
-CompilationDatabaseKt.createCompilationDatabasesFromCompileToBitcodeTasks(project, "CompilationDatabase")
-
-task hostRuntime(dependsOn: "${hostName}Runtime")
-
-task clean {
-    doLast {
-        delete buildDir
-    }
-}
-
-task generateJsMath {
-    dependsOn ':distCompiler'
-    doLast {
-        def jsinteropScript = isWindows() ? "jsinterop.bat" : "jsinterop"
-        def jsinterop = "$distDir/bin/$jsinteropScript"
-        def targetDir = "$buildDir/generated"
-        "$jsinterop -pkg kotlinx.interop.wasm.math -o $targetDir/math -target wasm32".execute().waitFor()
-        def generated = file("$targetDir/math-build/natives/js_stubs.js")
-        def mathJs = file('src/main/js/math.js')
-        mathJs.write("// NOTE: THIS FILE IS AUTO-GENERATED!\n" +
-                     "// Run ':runtime:generateJsMath' to re-generate it.\n\n")
-        generated.withReader {
-            mathJs.append(it)
-        }
-    }
-}

runtime/build.gradle.kts

@@ -0,0 +1,119 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+import org.jetbrains.kotlin.*
+import org.jetbrains.kotlin.bitcode.CompileToBitcode
+
+plugins {
+    id("compile-to-bitcode")
+}
+
+fun CompileToBitcode.includeRuntime() {
+    headersDirs += files("../common/src/hash/headers", "src/main/cpp")
+}
+
+val hostName: String by project
+val targetList: List<String> by project
+
+bitcode {
+    create("runtime", file("src/main")) {
+        dependsOn(
+            ":common:${target}Hash",
+            "${target}StdAlloc",
+            "${target}OptAlloc",
+            "${target}Mimalloc",
+            "${target}Launcher",
+            "${target}Debug",
+            "${target}Release",
+            "${target}Strict",
+            "${target}Relaxed",
+            "${target}ProfileRuntime",
+            "${target}Objc",
+            "${target}ExceptionsSupport"
+        )
+        includeRuntime()
+        linkerArgs.add(project.file("../common/build/$target/hash.bc").path)
+    }
+
+    create("mimalloc") {
+        language = CompileToBitcode.Language.C
+        includeFiles = listOf("**/*.c")
+        excludeFiles += listOf("**/alloc-override*.c", "**/page-queue.c", "**/static.c")
+        srcDirs = files("$srcRoot/c")
+        compilerArgs.add("-DKONAN_MI_MALLOC=1")
+        headersDirs = files("$srcRoot/c/include")
+
+        onlyIf { targetSupportsMimallocAllocator(target) }
+    }
+
+    create("launcher") {
+        includeRuntime()
+    }
+
+    create("debug") {
+        includeRuntime()
+    }
+
+    create("std_alloc")
+    create("opt_alloc")
+
+    create("exceptionsSupport", file("src/exceptions_support")) {
+        includeRuntime()
+    }
+
+    create("release") {
+        includeRuntime()
+    }
+
+    create("strict") {
+        includeRuntime()
+    }
+
+    create("relaxed") {
+        includeRuntime()
+    }
+
+    create("profileRuntime", file("src/profile_runtime"))
+
+    create("objc") {
+        includeRuntime()
+    }
+}
+
+val hostRuntime by tasks.registering {
+    dependsOn("${hostName}Runtime")
+}
+
+val clean by tasks.registering {
+    doLast {
+        delete(buildDir)
+    }
+}
+
+val generateJsMath by tasks.registering {
+    dependsOn(":distCompiler")
+    doLast {
+        val distDir: File by project
+        val jsinteropScript = if (PlatformInfo.isWindows()) "jsinterop.bat" else "jsinterop"
+        val jsinterop = "$distDir/bin/$jsinteropScript"
+        val targetDir = "$buildDir/generated"
+
+        project.exec {
+            commandLine(
+                    jsinterop,
+                    "-pkg", "kotlinx.interop.wasm.math",
+                    "-o", "$targetDir/math",
+                    "-target", "wasm32"
+            )
+        }
+
+        val generated = file("$targetDir/math-build/natives/js_stubs.js")
+        val mathJs = file("src/main/js/math.js")
+        mathJs.writeText(
+            "// NOTE: THIS FILE IS AUTO-GENERATED!\n" +
+            "// Run ':runtime:generateJsMath' to re-generate it.\n\n"
+        )
+        mathJs.appendText(generated.readText())
+    }
+}

settings.gradle

@@ -31,7 +31,7 @@ include ':backend.native:tests'
 include ':backend.native:debugger-tests'
 include ':utilities:basic-utils'
 include ':utilities:cli-runner'
-include 'dependencyPacker'
+include ':dependencyPacker'
 
 include ':performance'
 include ':performance:ring'