User-side library generator. (#3538)

build.gradle

@@ -351,6 +351,15 @@ task distCompiler(type: Copy) {
     }
 }
 
+task distDef(type: Copy) {
+
+    destinationDir distDir
+
+    from(project("platformLibs").file("src/platform")) {
+        into('klib/platformDef')
+    }
+}
+
 task listDist(type: Exec) {
     commandLine 'find', distDir
 }
@@ -472,6 +481,7 @@ task bundle(type: (isWindows()) ? Zip : Tar) {
 }
 
 task bundleRestricted(type: Tar) {
+    dependsOn("distDef")
     def simpleOsName = HostManager.simpleOsName()
     baseName = "kotlin-native-restricted-$simpleOsName-$konanVersionFull"
     from("$project.rootDir/dist") {

cmd/generate-platform

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# Copyright 2010-2017 JetBrains s.r.o.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DIR="${BASH_SOURCE[0]%/*}"
+: ${DIR:="."}
+
+"${DIR}"/run_konan generatePlatformLibraries "$@"

cmd/generate-platform.bat

@@ -0,0 +1,17 @@
+@echo off
+
+rem Copyright 2010-2017 JetBrains s.r.o.
+rem
+rem Licensed under the Apache License, Version 2.0 (the "License");
+rem you may not use this file except in compliance with the License.
+rem You may obtain a copy of the License at
+rem
+rem http://www.apache.org/licenses/LICENSE-2.0
+rem
+rem Unless required by applicable law or agreed to in writing, software
+rem distributed under the License is distributed on an "AS IS" BASIS,
+rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+rem See the License for the specific language governing permissions and
+rem limitations under the License.
+
+call %~dps0run_konan.bat generatePlatformLibraries %*

cmd/run_konan

@@ -80,7 +80,6 @@ done < "${KONAN_HOME}/tools/env_blacklist"
 KONAN_JAR="${KONAN_HOME}/konan/lib/kotlin-native.jar"
 KONAN_CLASSPATH="$KONAN_JAR"
 TOOL_CLASS=org.jetbrains.kotlin.cli.utilities.MainKt
-
 LIBCLANG_DISABLE_CRASH_RECOVERY=1 \
 $TIMECMD "$JAVACMD" "${java_opts[@]}" "${java_args[@]}" -cp "$KONAN_CLASSPATH" "$TOOL_CLASS" "$TOOL_NAME" "${konan_args[@]}"
 

utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/GeneratePlatformLibraries.kt

@@ -0,0 +1,136 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+package org.jetbrains.kotlin.cli.utilities
+
+import org.jetbrains.kotlin.cli.bc.K2Native
+import org.jetbrains.kotlin.konan.file.File
+import java.util.concurrent.*
+import kotlinx.cli.*
+
+fun generatePlatformLibraries(args: Array<String>) {
+    val argParser = ArgParser("generate-platform", prefixStyle = ArgParser.OPTION_PREFIX_STYLE.JVM)
+    val inputDirectoryPath by argParser.option(
+            ArgType.String, "input-directory", "i", "Input directory").required()
+    val outputDirectoryPath by argParser.option(
+            ArgType.String, "output-directory", "o", "Output directory").required()
+    val target by argParser.option(
+            ArgType.String, "target", "t", "Compilation target").required()
+    val saveTemps by argParser.option(
+            ArgType.Boolean, "save-temps", "s", "Save temporary files").default(false)
+    argParser.parse(args)
+
+    val inputDirectory = File(inputDirectoryPath)
+    val outputDirectory = File(outputDirectoryPath)
+
+    if (!inputDirectory.exists) throw Error("input directory doesn't exist")
+    if (!outputDirectory.exists) {
+        outputDirectory.mkdirs()
+    }
+
+    generatePlatformLibraries(target, inputDirectory, outputDirectory, saveTemps)
+}
+
+private class DefFile(val name: String, val depends: MutableList<DefFile>) {
+
+    override fun hashCode(): Int {
+        return name.hashCode()
+    }
+
+    override fun equals(other: Any?): Boolean {
+        return other is DefFile && other.name == name
+    }
+    override fun toString(): String = "$name: [${depends.joinToString(separator = ", ") { it.name }}]"
+}
+
+private fun topoSort(defFiles: List<DefFile>): List<DefFile> {
+    // Do DFS toposort.
+    val markGray = mutableSetOf<DefFile>()
+    val markBlack = mutableSetOf<DefFile>()
+    val result = mutableListOf<DefFile>()
+
+    fun visit(def: DefFile) {
+        if (markBlack.contains(def)) return
+        if (markGray.contains(def)) throw Error("$def is part of cycle")
+        markGray += def
+        def.depends.forEach {
+            visit(it)
+        }
+        markGray -= def
+        markBlack += def
+        result += def
+    }
+
+    var index = 0
+    while (markBlack.size < defFiles.size) {
+        visit(defFiles[index++])
+    }
+    return result
+}
+
+private fun generatePlatformLibraries(target: String, inputDirectory: File, outputDirectory: File, saveTemps: Boolean) {
+    fun buildKlib(def: DefFile) {
+        val file = File("$inputDirectory/${def.name}.def")
+        File("${outputDirectory.absolutePath}/build-${def.name}").mkdirs()
+        val outKlib = "${outputDirectory.absolutePath}/build-${def.name}/${def.name}.klib"
+        val args = arrayOf("-o", outKlib,
+                "-target", target,
+                "-def", file.absolutePath,
+                "-compiler-option", "-fmodules-cache-path=$outputDirectory/clangModulesCache",
+                "-repo",  "${outputDirectory.absolutePath}",
+                "-no-default-libs", "-no-endorsed-libs", "-Xpurge-user-libs",
+                *def.depends.flatMap { listOf("-l", "$outputDirectory/${it.name}") }.toTypedArray())
+        println("Processing ${def.name}...")
+        K2Native.mainNoExit(invokeInterop("native", args))
+        org.jetbrains.kotlin.cli.klib.main(arrayOf("install", outKlib,
+                "-target", target,
+                "-repository", "${outputDirectory.absolutePath}"
+        ))
+        if (!saveTemps)  {
+            File("$outputDirectory/build-${def.name}").deleteRecursively()
+        }
+    }
+    println("generate platform libraries from $inputDirectory to $outputDirectory for $target")
+    // Build dependencies graph.
+    val defFiles = mutableMapOf<String, DefFile>()
+    val dependsRegex = Regex("^depends = (.*)")
+    inputDirectory.listFilesOrEmpty.filter { it.extension == "def" }.forEach { file ->
+        val name = file.name.split(".").also { assert(it.size == 2) }[0]
+        val def = defFiles.getOrPut(name) {
+            DefFile(name, mutableListOf())
+        }
+        file.forEachLine { line ->
+            val match = dependsRegex.matchEntire(line)
+            if (match != null) {
+                match.groupValues[1].split(" ").forEach { dependency ->
+                    def.depends.add(defFiles.getOrPut(dependency) {
+                        DefFile(dependency, mutableListOf())
+                    })
+                }
+            }
+        }
+    }
+    val sorted = topoSort(defFiles.values.toList())
+    val numCores = Runtime.getRuntime().availableProcessors()
+    val executorPool = ThreadPoolExecutor(numCores, numCores,
+            10, TimeUnit.SECONDS, ArrayBlockingQueue(1000),
+            Executors.defaultThreadFactory(), RejectedExecutionHandler { r, executor ->
+        println("Execution rejected: $r")
+        throw Error("Must not happen!")
+    })
+    val built = ConcurrentHashMap(sorted.associateWith { 0 })
+    // Now run interop tool on toposorted dependencies.
+    sorted.forEach { def ->
+        executorPool.execute {
+            // A bit ugly, we just block here until all dependencies are built.
+            while (def.depends.any { built[it] == 0 }) {
+                Thread.sleep(100)
+            }
+            buildKlib(def)
+            built[def] = 1
+        }
+    }
+    executorPool.shutdown()
+}

utilities/src/main/kotlin/org/jetbrains/kotlin/cli/utilities/main.kt

@@ -17,16 +17,18 @@ private fun mainImpl(args: Array<String>, konancMain: (Array<String>) -> Unit) {
             konancMain(utilityArgs)
         "cinterop" -> {
             val konancArgs = invokeInterop("native", utilityArgs)
-            konancArgs?.let { konancMain(it) }
+            konancMain(konancArgs)
         }
         "jsinterop" -> {
             val konancArgs = invokeInterop("wasm", utilityArgs)
-            konancArgs?.let { konancMain(it) }
+            konancMain(konancArgs)
         }
         "klib" ->
             klibMain(utilityArgs)
         "defFileDependencies" ->
             defFileDependencies(utilityArgs)
+        "generatePlatformLibraries" ->
+            generatePlatformLibraries(utilityArgs)
         else ->
             error("Unexpected utility name")
     }