tools: Add dependecy download helpers

This patch adds 2 helpers to download dependencies for compiler and
stub generator during their execution.

They take list of external dependencies as a parameter and directory
where they must be located. Helpers check if these dependencies exist
in the given directory and download not existing ones.

.gitignore

@@ -43,3 +43,9 @@ proto/compiler/google/src/google/protobuf/compiler/kotlin/protoc
 
 # translator auto generated artifacts
 kotstd/ll
+
+# directory for manual debug runs
+run-debug/**
+!run-debug/interop
+!run-debug/konanc
+

Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt

@@ -7,6 +7,7 @@ import org.jetbrains.kotlin.native.interop.indexer.buildNativeIndex
 import java.io.File
 import java.lang.IllegalArgumentException
 import java.util.*
+import kotlin.reflect.KFunction
 
 fun main(args: Array<String>) {
     val konanHome = System.getProperty("konan.home")!!
@@ -116,7 +117,20 @@ private fun runCmd(command: Array<String>, workDir: File, verbose: Boolean = fal
                 .runExpectingSuccess()
 }
 
-private fun Properties.defaultCompilerOpts(target: String, dependencies: String): List<String> {
+private fun maybeExecuteHelper(dependenciesRoot: String, propertiesFile: String, dependencies: List<String>) {
+    try {
+        val kClass = Class.forName("org.jetbrains.kotlin.konan.InteropHelper0").kotlin
+        val ctor = kClass.constructors.single() as KFunction<Runnable>
+        val result = ctor.call(dependenciesRoot, propertiesFile, dependencies)
+        result.run()
+    } catch (notFound: ClassNotFoundException) {
+        // Just ignore, no helper.
+    } catch (e: Throwable) {
+        throw IllegalStateException("Cannot download dependencies.", e)
+    }
+}
+
+private fun Properties.defaultCompilerOpts(target: String, dependencies: String, konanFileName: String): List<String> {
 
     val hostSysRootDir = this.getOsSpecific("sysRoot", target)!!
     val hostSysRoot = "$dependencies/$hostSysRootDir"
@@ -127,6 +141,12 @@ private fun Properties.defaultCompilerOpts(target: String, dependencies: String)
     val llvmHome = "$dependencies/$llvmHomeDir"
     val llvmVersion = this.getProperty("llvmVersion")!!
 
+    val dependencyList = mutableListOf<String>(sysRoot, targetSysRoot, llvmHome)
+    if (target == "linux") {
+        dependencyList.add("$dependencies/${getOsSpecific("gccToolChain", target)!!}")
+    }
+    maybeExecuteHelper(dependencies, konanFileName, dependencyList)
+
     // StubGenerator passes the arguments to libclang which 
     // works not exactly the same way as the clang binary and 
     // (in particular) uses different default header search path.
@@ -242,7 +262,7 @@ private fun processLib(konanHome: String,
     val generateShims = args["-shims"].isTrue()
     val verbose = args["-verbose"].isTrue()
 
-    val defaultOpts = konanProperties.defaultCompilerOpts(target, dependencies)
+    val defaultOpts = konanProperties.defaultCompilerOpts(target, dependencies, konanFileName)
     val headerFiles = config.getSpaceSeparated("headers") + additionalHeaders
     val compilerOpts = 
         config.getSpaceSeparated("compilerOpts") +

RELEASE_NOTES.md

@@ -48,6 +48,8 @@ between threads are allowed.
 
 	./dist/bin/kotlinc <some_file>.kt <dir_with_kt_files> -o <executable>.kexe
 
+  During the first run it will download all external dependencies such as LLVM.
+
   One may use `'-h'` flag to `kotlinc` to see available flags.
 
   For documentation on C interoperability stubs see INTEROP.md.

backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt

@@ -16,17 +16,15 @@ import kotlin.reflect.KFunction
 
 private fun maybeExecuteHelper(configuration: CompilerConfiguration) {
     try {
-        val kClass = Class.forName("org.jetbrains.kotlin.konan.Helper0").kotlin
+        val kClass = Class.forName("org.jetbrains.kotlin.konan.CompilerHelper0").kotlin
         val ctor = kClass.constructors.single() as KFunction<Runnable>
         val distribution = Distribution(configuration)
-        val target = TargetManager(configuration)
-        val result = ctor.call(
-                distribution.konanHome, TargetManager.host, target.current)
+        val result = ctor.call(distribution)
         result.run()
     } catch (notFound: ClassNotFoundException) {
         // Just ignore, no helper.
     } catch (e: Throwable) {
-        println(e.toString())
+        throw IllegalStateException("Cannot download dependencies.", e)
     }
 }
 

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Distribution.kt

@@ -35,7 +35,13 @@ class Distribution(val config: CompilerConfiguration) {
 
     val llvmHome = "$dependencies/${properties.propertyString("llvmHome.$suffix")}" 
     val sysRoot = "$dependencies/${properties.propertyString("sysRoot.$suffix")}"
-    val libGcc = "$dependencies/${properties.propertyString("libGcc.$suffix")}" 
+    val libGcc = "$dependencies/${properties.propertyString("libGcc.$suffix")}"
+
+    val targetSysRoot = if (properties.hasProperty("targetSysRoot.$suffix")) {
+            "$dependencies/${properties.propertyString("targetSysRoot.$suffix")}"
+        } else {
+            sysRoot
+        }
 
     val llvmBin = "$llvmHome/bin"
     val llvmLib = "$llvmHome/lib"

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Helper0.kt

@@ -4,12 +4,24 @@
 //
 package org.jetbrains.kotlin.konan
 
-import org.jetbrains.kotlin.backend.konan.KonanTarget
+import org.jetbrains.kotlin.backend.konan.Distribution
+import org.jetbrains.kotlin.backend.konan.TargetManager
 
-// Name it Helper0 so that it get loaded.
-class Helper0Disabled(
-        val konanHome: String, val host: KonanTarget, val target: KonanTarget) : Runnable {
+// Name it CompilerHelper0 so that it get loaded.
+class CompilerHelper0Disabled(val distribution: Distribution) : Runnable {
     override fun run() {
-        println("Running helper with $konanHome $host $target")
+        println("Running helper with ${distribution.konanHome} ${distribution.target} ${TargetManager.host}")
+    }
+}
+
+//
+// Example startup helper for Kotlin N stub generator. Compile to JAR and add it to
+// Kotlin N classpath - it will get loaded and executed automatically.
+//
+
+// Name it InteropHelper0 so that it get loaded.
+class InteropHelper0Disabled(val dependenciesRoot: String, val dependencies: List<String>) : Runnable {
+    override fun run() {
+        println("Running helper with $dependenciesRoot $dependencies")
     }
 }

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanProperties.kt

@@ -15,10 +15,13 @@ public class KonanProperties(val propertyFile: String) {
     }
 
     fun propertyString(key: String): String? = properties.getProperty(key)
+    fun propertyString(key: String, default: String) : String = properties.getProperty(key, default)
 
     fun propertyList(key: String): List<String> {
         val value =  properties.getProperty(key)
         return value?.split(' ') ?: listOf<String>()
     }
+
+    fun hasProperty(key: String): Boolean = properties.getProperty(key) != null
 }
 

backend.native/konan.properties

@@ -1,6 +1,7 @@
 
 // TODO: Do we need a $variable substitution mechanism here?
 llvmVersion = 3.9.0
+dependenciesUrl = https://jetbrains.bintray.com/kotlin-native-dependencies
 
 // macbook
 arch.osx = x86_64

backend.native/tests/build.gradle

@@ -25,8 +25,8 @@ dependencies {
     }
 }
 
-def testOutputRoot = rootProject.file("test.output").absolutePath
-def externalTestsDir = project.file("external")
+ext.testOutputRoot = rootProject.file("test.output").absolutePath
+ext.externalTestsDir = project.file("external")
 externalTestsDir.mkdirs()
 
 allprojects {

build.gradle

@@ -125,6 +125,7 @@ class PlatformInfo {
 
 task dist_compiler(type: Copy) {
     dependsOn ':backend.native:jars'
+    dependsOn ':tools:helpers:jar'
 
     destinationDir file('dist')
 
@@ -174,6 +175,11 @@ task dist_compiler(type: Copy) {
         includeEmptyDirs = false
     }
 
+    // TODO: check if we use native libraries copied to dist (see above) or not.
+    from(project(':tools:helpers').file('build/libs')) {
+        into('konan/lib')
+    }
+
     doLast {
         // MacOS System Integrity Protection strips DYLD_LIBRARY_PATH from
         // environment of all the binaries residing in /bin /usr/bin etc.

buildSrc/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy

@@ -12,7 +12,7 @@ abstract class KonanTest extends JavaExec {
     def backendNative = project.project(":backend.native")
     def runtimeProject = project.project(":runtime")
     def dist = project.rootProject.file("dist")
-    def dependencies = project.findProject(":dependencies").file("all")
+    def dependenciesDir = project.findProject(":dependencies").file("all")
     def runtimeBc = new File("${dist.canonicalPath}/lib/runtime.bc").absolutePath
     def launcherBc = new File("${dist.canonicalPath}/lib/launcher.bc").absolutePath
     def startKtBc = new File("${dist.canonicalPath}/lib/start.kt.bc").absolutePath
@@ -59,7 +59,7 @@ abstract class KonanTest extends JavaExec {
     void setJvmArgs(Iterable<?> arguments) {
         super.setJvmArgs(arguments +
                 "-Dkonan.home=${dist.canonicalPath}" +
-                "-Dkonan.dependencies=${dependencies.canonicalPat}",
+                "-Dkonan.dependencies=${dependenciesDir.canonicalPath}" +
                 "-Djava.library.path=${dist.canonicalPath}/konan/nativelib")
     }
 

cmd/interop

@@ -40,7 +40,6 @@ findHome() {
 KONAN_HOME="$(findHome)"
 
 NATIVE_LIB="${KONAN_HOME}/konan/nativelib"
-DEPENDENCIES="${KONAN_HOME}/../dependencies/all"
 JAVA_OPTS="-ea \
     -Djava.library.path=${NATIVE_LIB} \
     -Dkonan.home=${KONAN_HOME}"
@@ -49,7 +48,8 @@ STUB_GENERATOR_JAR="${KONAN_HOME}/konan/lib/StubGenerator.jar"
 KOTLIN_JAR="${KONAN_HOME}/konan/lib/kotlin-compiler.jar"
 INTEROP_INDEXER_JAR="${KONAN_HOME}/konan/lib/Indexer.jar"
 INTEROP_RUNTIME_JAR="${KONAN_HOME}/konan/lib/Runtime.jar"
-INTEROP_CLASSPATH="$STUB_GENERATOR_JAR:$KOTLIN_JAR:$INTEROP_INDEXER_JAR:$INTEROP_RUNTIME_JAR"
+HELPERS_JAR="${KONAN_HOME}/konan/lib/helpers.jar"
+INTEROP_CLASSPATH="$STUB_GENERATOR_JAR:$KOTLIN_JAR:$INTEROP_INDEXER_JAR:$INTEROP_RUNTIME_JAR:$HELPERS_JAR"
 INTEROP_TOOL=org.jetbrains.kotlin.native.interop.gen.jvm.MainKt
 
 FLAVOR_ARG=-flavor:native

cmd/konanc

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash 
+#!/usr/bin/env bash
 
 if [ -z "$JAVACMD" -a -n "$JAVA_HOME" -a -x "$JAVA_HOME/bin/java" ]; then
     JAVACMD="$JAVA_HOME/bin/java"
@@ -52,8 +52,9 @@ KONAN_HOME="$(findHome)"
 KONAN_JAR="${KONAN_HOME}/konan/lib/backend.native.jar"
 KOTLIN_JAR="${KONAN_HOME}/konan/lib/kotlin-compiler.jar"
 INTEROP_JAR="${KONAN_HOME}/konan/lib/Runtime.jar"
+HELPERS_JAR="${KONAN_HOME}/konan/lib/helpers.jar"
 NATIVE_LIB="${KONAN_HOME}/konan/nativelib"
-KONAN_CLASSPATH="$KOTLIN_JAR:$INTEROP_JAR:$KONAN_JAR"
+KONAN_CLASSPATH="$KOTLIN_JAR:$INTEROP_JAR:$KONAN_JAR:$HELPERS_JAR"
 KONAN_COMPILER=org.jetbrains.kotlin.cli.bc.K2NativeKt
 JAVA_OPTS=-ea
 

settings.gradle

@@ -7,3 +7,4 @@ include ':backend.native'
 include ':runtime'
 include ':common'
 include ':backend.native:tests'
+include ':tools:helpers'

tools/helpers/build.gradle

@@ -0,0 +1,128 @@
+import org.jetbrains.kotlin.NativeInteropPlugin
+import org.jetbrains.kotlin.RunInteropKonanTest
+import org.jetbrains.kotlin.RunKonanTest
+import org.jetbrains.kotlin.TestFailedException
+
+import java.util.concurrent.atomic.AtomicInteger
+
+buildscript {
+    repositories {
+        mavenCentral()
+    }
+
+    dependencies {
+        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
+    }
+}
+
+repositories {
+    maven {
+        url 'http://oss.sonatype.org/content/repositories/snapshots'
+    }
+}
+
+apply plugin: 'kotlin'
+apply plugin: NativeInteropPlugin
+
+configurations {
+    cli_bc.extendsFrom compile
+}
+
+dependencies {
+    compile project(path: ':backend.native', configuration: 'cli_bc')
+    cli_bc jar.outputs.files
+}
+
+sourceSets {
+    testOutputLocal {
+        output.dir(rootProject.file("${findProject(":backend.native:tests").testOutputRoot}/tools/helpers"))
+    }
+}
+
+def testDependencies = project.file("${sourceSets.testOutputLocal.output.dirs.singleFile.absolutePath}/dependencies")
+
+void prepareDependenciesDir(File testDependencies) {
+    project.delete testDependencies
+    testDependencies.mkdirs()
+    // We don't download big clang dependency in this test - only sysroots.
+    project.copy {
+        from project.findProject(":dependencies").file("all")
+        into testDependencies
+        include "clang*.tar.gz/**"
+    }
+}
+
+kotlinNativeInterop {
+    sysstat {
+        pkg 'sysstat'
+        headers 'sys/stat.h'
+        flavor 'native'
+    }
+}
+
+task testInteropHelper(type: RunInteropKonanTest) {
+    dependsOn jar
+    dependsOn ':dist'
+
+    dependenciesDir = testDependencies
+    goldValue = "0\n0\n"
+    source = "testData/interop.kt"
+    interop = 'sysstat'
+
+    doFirst {
+        prepareDependenciesDir(testDependencies)
+    }
+}
+
+task testCompilerHelper(type: RunKonanTest) {
+    dependenciesDir = testDependencies
+    source = "testData/main.kt"
+
+    doFirst {
+        prepareDependenciesDir(testDependencies)
+    }
+}
+
+// Simple test for parallel compiler execution
+task testParallel(type: DefaultTask) {
+    for (int i=0; i<4; i++) {
+        task("runParallel$i", type: RunKonanTest) {
+            dependenciesDir = testDependencies
+            source = "testData/main.kt"
+        }
+    }
+
+    doFirst {
+        prepareDependenciesDir(testDependencies)
+    }
+
+    doLast {
+        List<Thread> threads = new ArrayList<>()
+        def downloadCnt = new AtomicInteger(0)
+        tasks.withType(RunKonanTest).matching { it.name.startsWith("runParallel") }.each { task ->
+            threads.add(Thread.start {
+                task.executeTest()
+                // The helper prints messages when it is downloading.
+                // We check that there is only one downloading using compilation logs.
+                if (file("${task.buildExePath()}.compilation.log").text.contains("Download dependency")) {
+                    downloadCnt.incrementAndGet()
+                }
+            })
+        }
+        threads.each { it.join() }
+        if (downloadCnt.intValue() != 1) {
+            throw new TestFailedException("Actual downloads: ${downloadCnt.intValue()}, expected: 1")
+        }
+    }
+}
+
+task testHelpers {
+    dependsOn testInteropHelper
+    dependsOn testCompilerHelper
+    dependsOn testParallel
+
+    doLast {
+        delete testDependencies
+    }
+}
+

tools/helpers/src/main/kotlin/CompilerHelper0.kt

@@ -0,0 +1,22 @@
+package org.jetbrains.kotlin.konan
+
+import org.jetbrains.kotlin.backend.konan.Distribution
+import org.jetbrains.kotlin.backend.konan.KonanTarget
+import org.jetbrains.kotlin.backend.konan.TargetManager
+import java.io.File
+
+class CompilerHelper0(val dist: Distribution): Runnable {
+
+    override fun run() {
+        val dependencies = mutableListOf<String>()
+        dependencies.add(dist.sysRoot)
+        dependencies.add(dist.llvmHome)
+        if (dist.sysRoot != dist.targetSysRoot) {
+            dependencies.add(dist.targetSysRoot)
+        }
+        if (TargetManager.host == KonanTarget.LINUX) {
+            dependencies.add(dist.libGcc)
+        }
+        DependencyDownloader(File(dist.dependencies), dist.properties.properties, dependencies).run()
+    }
+}

tools/helpers/src/main/kotlin/DependencyDownloader.kt

@@ -0,0 +1,85 @@
+package org.jetbrains.kotlin.konan
+
+import java.io.File
+import java.io.RandomAccessFile
+import java.net.URL
+import java.nio.file.*
+import java.util.*
+
+// TODO: Try to use some dependency management system (Ivy?)
+// TODO: Show % and size during downloading
+class DependencyDownloader(dependenciesRoot: File, val properties: Properties, val dependencies: List<String>) {
+
+    val dependenciesRoot = dependenciesRoot.apply { mkdirs() }
+
+    val lockFile = File(dependenciesRoot, ".lock").apply { createNewFile() }
+    val listFile = File(dependenciesRoot, ".downloaded")
+
+    val dependenciesUrl: String =
+            properties.getProperty("dependenciesUrl", "https://jetbrains.bintray.com/kotlin-native-dependencies")
+
+    var isInfoShown = false
+
+    private fun File.containsLine(line: String): Boolean {
+        if (!exists()) {
+            return false
+        }
+        var result = false
+        listFile.forEachLine {
+            if (!result && it == line) {
+                result = true
+            }
+        }
+        return result
+    }
+
+    private fun processDependency(path: String) {
+        val depDir = File(path)
+        val depName = depDir.name
+        val inListFile = listFile.containsLine(depName)
+        if (inListFile && depDir.exists()) {
+            return
+        }
+        if (!isInfoShown) {
+            println("Downloading native dependencies (LLVM, sysroot etc). This is one-time action performing only for the first run of the compiler.")
+            isInfoShown = true
+        }
+        val downloaded = download(depName)
+        println("Extract dependency: $downloaded -> $depDir")
+        extract(downloaded)
+        if (!inListFile) {
+            listFile.appendText("$depName\n")
+        }
+    }
+
+    private fun extract(tarGz: File) {
+        val tarProcess = ProcessBuilder().apply {
+            command("tar", "-x", "-f", "${tarGz.canonicalPath}")
+            directory(tarGz.parentFile)
+        }.start()
+        tarProcess.waitFor()
+        if (tarProcess.exitValue() != 0) {
+            throw RuntimeException("Cannot extract archive with dependency: ${tarGz.canonicalPath}")
+        }
+    }
+
+    private fun download(dependencyName: String): File {
+        val to = File(dependenciesRoot.canonicalPath, "$dependencyName.tar.gz")
+        if (!to.exists()) {
+            val from = URL("$dependenciesUrl/$dependencyName.tar.gz")
+            println("Download dependency: $dependencyName from $from")
+            from.openStream().use {
+                Files.copy(it, Paths.get(to.toURI()), StandardCopyOption.REPLACE_EXISTING)
+            }
+        }
+        return to
+    }
+
+    fun run() {
+        val systemLock = RandomAccessFile(lockFile, "rw").channel.lock()
+        dependencies.forEach {
+            processDependency(it)
+        }
+        systemLock.release()
+    }
+}

tools/helpers/src/main/kotlin/InteropHelper0.kt

@@ -0,0 +1,18 @@
+package org.jetbrains.kotlin.konan
+
+import org.jetbrains.kotlin.backend.konan.KonanProperties
+import java.io.File
+import java.util.*
+
+class InteropHelper0(
+        val dependenciesRoot: String,
+        val propertiesFile: String,
+        val dependencies: List<String>): Runnable {
+
+    override fun run() =
+            DependencyDownloader(
+                    File(dependenciesRoot),
+                    Properties().apply { load(File(propertiesFile).inputStream()) },
+                    dependencies
+            ).run()
+}

tools/helpers/testData/interop.kt

@@ -0,0 +1,9 @@
+import sysstat.*
+import kotlinx.cinterop.*
+
+fun main(args: Array<String>) {
+    val statBuf = nativeHeap.alloc<statStruct>()
+    val res = stat("/", statBuf.ptr)
+    println(res)
+    println(statBuf.st_uid.value)
+}

tools/helpers/testData/main.kt

@@ -0,0 +1,3 @@
+fun main(args: Array<String>) {
+    println(42)
+}