diff --git a/.gitignore b/.gitignore index f87a4bceab3..85b92c1cb62 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,9 @@ proto/compiler/google/src/google/protobuf/compiler/kotlin/protoc # translator auto generated artifacts kotstd/ll + +# directory for manual debug runs +run-debug/** +!run-debug/interop +!run-debug/konanc + diff --git a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt index f54a20fb0a8..d861676cbd6 100644 --- a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt +++ b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt @@ -7,6 +7,7 @@ import org.jetbrains.kotlin.native.interop.indexer.buildNativeIndex import java.io.File import java.lang.IllegalArgumentException import java.util.* +import kotlin.reflect.KFunction fun main(args: Array) { val konanHome = System.getProperty("konan.home")!! @@ -116,7 +117,20 @@ private fun runCmd(command: Array, workDir: File, verbose: Boolean = fal .runExpectingSuccess() } -private fun Properties.defaultCompilerOpts(target: String, dependencies: String): List { +private fun maybeExecuteHelper(dependenciesRoot: String, propertiesFile: String, dependencies: List) { + try { + val kClass = Class.forName("org.jetbrains.kotlin.konan.InteropHelper0").kotlin + val ctor = kClass.constructors.single() as KFunction + val result = ctor.call(dependenciesRoot, propertiesFile, dependencies) + result.run() + } catch (notFound: ClassNotFoundException) { + // Just ignore, no helper. + } catch (e: Throwable) { + throw IllegalStateException("Cannot download dependencies.", e) + } +} + +private fun Properties.defaultCompilerOpts(target: String, dependencies: String, konanFileName: String): List { val hostSysRootDir = this.getOsSpecific("sysRoot", target)!! val hostSysRoot = "$dependencies/$hostSysRootDir" @@ -127,6 +141,12 @@ private fun Properties.defaultCompilerOpts(target: String, dependencies: String) val llvmHome = "$dependencies/$llvmHomeDir" val llvmVersion = this.getProperty("llvmVersion")!! + val dependencyList = mutableListOf(sysRoot, targetSysRoot, llvmHome) + if (target == "linux") { + dependencyList.add("$dependencies/${getOsSpecific("gccToolChain", target)!!}") + } + maybeExecuteHelper(dependencies, konanFileName, dependencyList) + // StubGenerator passes the arguments to libclang which // works not exactly the same way as the clang binary and // (in particular) uses different default header search path. @@ -242,7 +262,7 @@ private fun processLib(konanHome: String, val generateShims = args["-shims"].isTrue() val verbose = args["-verbose"].isTrue() - val defaultOpts = konanProperties.defaultCompilerOpts(target, dependencies) + val defaultOpts = konanProperties.defaultCompilerOpts(target, dependencies, konanFileName) val headerFiles = config.getSpaceSeparated("headers") + additionalHeaders val compilerOpts = config.getSpaceSeparated("compilerOpts") + diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index afbc4856b75..4f01c5b35c2 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -48,6 +48,8 @@ between threads are allowed. ./dist/bin/kotlinc .kt -o .kexe + During the first run it will download all external dependencies such as LLVM. + One may use `'-h'` flag to `kotlinc` to see available flags. For documentation on C interoperability stubs see INTEROP.md. diff --git a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt index bb42087b247..4267adeb67b 100644 --- a/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt +++ b/backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt @@ -16,17 +16,15 @@ import kotlin.reflect.KFunction private fun maybeExecuteHelper(configuration: CompilerConfiguration) { try { - val kClass = Class.forName("org.jetbrains.kotlin.konan.Helper0").kotlin + val kClass = Class.forName("org.jetbrains.kotlin.konan.CompilerHelper0").kotlin val ctor = kClass.constructors.single() as KFunction val distribution = Distribution(configuration) - val target = TargetManager(configuration) - val result = ctor.call( - distribution.konanHome, TargetManager.host, target.current) + val result = ctor.call(distribution) result.run() } catch (notFound: ClassNotFoundException) { // Just ignore, no helper. } catch (e: Throwable) { - println(e.toString()) + throw IllegalStateException("Cannot download dependencies.", e) } } diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Distribution.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Distribution.kt index 7646206041e..29ac5d194b7 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Distribution.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Distribution.kt @@ -35,7 +35,13 @@ class Distribution(val config: CompilerConfiguration) { val llvmHome = "$dependencies/${properties.propertyString("llvmHome.$suffix")}" val sysRoot = "$dependencies/${properties.propertyString("sysRoot.$suffix")}" - val libGcc = "$dependencies/${properties.propertyString("libGcc.$suffix")}" + val libGcc = "$dependencies/${properties.propertyString("libGcc.$suffix")}" + + val targetSysRoot = if (properties.hasProperty("targetSysRoot.$suffix")) { + "$dependencies/${properties.propertyString("targetSysRoot.$suffix")}" + } else { + sysRoot + } val llvmBin = "$llvmHome/bin" val llvmLib = "$llvmHome/lib" diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Helper0.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Helper0.kt index 02d94df46ec..122fd291ce9 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Helper0.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Helper0.kt @@ -4,12 +4,24 @@ // package org.jetbrains.kotlin.konan -import org.jetbrains.kotlin.backend.konan.KonanTarget +import org.jetbrains.kotlin.backend.konan.Distribution +import org.jetbrains.kotlin.backend.konan.TargetManager -// Name it Helper0 so that it get loaded. -class Helper0Disabled( - val konanHome: String, val host: KonanTarget, val target: KonanTarget) : Runnable { +// Name it CompilerHelper0 so that it get loaded. +class CompilerHelper0Disabled(val distribution: Distribution) : Runnable { override fun run() { - println("Running helper with $konanHome $host $target") + println("Running helper with ${distribution.konanHome} ${distribution.target} ${TargetManager.host}") + } +} + +// +// Example startup helper for Kotlin N stub generator. Compile to JAR and add it to +// Kotlin N classpath - it will get loaded and executed automatically. +// + +// Name it InteropHelper0 so that it get loaded. +class InteropHelper0Disabled(val dependenciesRoot: String, val dependencies: List) : Runnable { + override fun run() { + println("Running helper with $dependenciesRoot $dependencies") } } \ No newline at end of file diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanProperties.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanProperties.kt index 4dbc1996d4c..9cf8e8b49af 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanProperties.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanProperties.kt @@ -15,10 +15,13 @@ public class KonanProperties(val propertyFile: String) { } fun propertyString(key: String): String? = properties.getProperty(key) + fun propertyString(key: String, default: String) : String = properties.getProperty(key, default) fun propertyList(key: String): List { val value = properties.getProperty(key) return value?.split(' ') ?: listOf() } + + fun hasProperty(key: String): Boolean = properties.getProperty(key) != null } diff --git a/backend.native/konan.properties b/backend.native/konan.properties index a9c33d7c9cb..c67c08fe828 100644 --- a/backend.native/konan.properties +++ b/backend.native/konan.properties @@ -1,6 +1,7 @@ // TODO: Do we need a $variable substitution mechanism here? llvmVersion = 3.9.0 +dependenciesUrl = https://jetbrains.bintray.com/kotlin-native-dependencies // macbook arch.osx = x86_64 diff --git a/backend.native/tests/build.gradle b/backend.native/tests/build.gradle index c16e3142fd6..e37d5a72eed 100644 --- a/backend.native/tests/build.gradle +++ b/backend.native/tests/build.gradle @@ -25,8 +25,8 @@ dependencies { } } -def testOutputRoot = rootProject.file("test.output").absolutePath -def externalTestsDir = project.file("external") +ext.testOutputRoot = rootProject.file("test.output").absolutePath +ext.externalTestsDir = project.file("external") externalTestsDir.mkdirs() allprojects { diff --git a/build.gradle b/build.gradle index 8afbfb3bbcb..28cf3e99519 100644 --- a/build.gradle +++ b/build.gradle @@ -125,6 +125,7 @@ class PlatformInfo { task dist_compiler(type: Copy) { dependsOn ':backend.native:jars' + dependsOn ':tools:helpers:jar' destinationDir file('dist') @@ -174,6 +175,11 @@ task dist_compiler(type: Copy) { includeEmptyDirs = false } + // TODO: check if we use native libraries copied to dist (see above) or not. + from(project(':tools:helpers').file('build/libs')) { + into('konan/lib') + } + doLast { // MacOS System Integrity Protection strips DYLD_LIBRARY_PATH from // environment of all the binaries residing in /bin /usr/bin etc. diff --git a/buildSrc/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy b/buildSrc/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy index 7645e717753..7fbd98918a0 100644 --- a/buildSrc/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy +++ b/buildSrc/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy @@ -12,7 +12,7 @@ abstract class KonanTest extends JavaExec { def backendNative = project.project(":backend.native") def runtimeProject = project.project(":runtime") def dist = project.rootProject.file("dist") - def dependencies = project.findProject(":dependencies").file("all") + def dependenciesDir = project.findProject(":dependencies").file("all") def runtimeBc = new File("${dist.canonicalPath}/lib/runtime.bc").absolutePath def launcherBc = new File("${dist.canonicalPath}/lib/launcher.bc").absolutePath def startKtBc = new File("${dist.canonicalPath}/lib/start.kt.bc").absolutePath @@ -59,7 +59,7 @@ abstract class KonanTest extends JavaExec { void setJvmArgs(Iterable arguments) { super.setJvmArgs(arguments + "-Dkonan.home=${dist.canonicalPath}" + - "-Dkonan.dependencies=${dependencies.canonicalPat}", + "-Dkonan.dependencies=${dependenciesDir.canonicalPath}" + "-Djava.library.path=${dist.canonicalPath}/konan/nativelib") } diff --git a/cmd/interop b/cmd/interop index 35f18e3b3f2..f564eee1922 100644 --- a/cmd/interop +++ b/cmd/interop @@ -40,7 +40,6 @@ findHome() { KONAN_HOME="$(findHome)" NATIVE_LIB="${KONAN_HOME}/konan/nativelib" -DEPENDENCIES="${KONAN_HOME}/../dependencies/all" JAVA_OPTS="-ea \ -Djava.library.path=${NATIVE_LIB} \ -Dkonan.home=${KONAN_HOME}" @@ -49,7 +48,8 @@ STUB_GENERATOR_JAR="${KONAN_HOME}/konan/lib/StubGenerator.jar" KOTLIN_JAR="${KONAN_HOME}/konan/lib/kotlin-compiler.jar" INTEROP_INDEXER_JAR="${KONAN_HOME}/konan/lib/Indexer.jar" INTEROP_RUNTIME_JAR="${KONAN_HOME}/konan/lib/Runtime.jar" -INTEROP_CLASSPATH="$STUB_GENERATOR_JAR:$KOTLIN_JAR:$INTEROP_INDEXER_JAR:$INTEROP_RUNTIME_JAR" +HELPERS_JAR="${KONAN_HOME}/konan/lib/helpers.jar" +INTEROP_CLASSPATH="$STUB_GENERATOR_JAR:$KOTLIN_JAR:$INTEROP_INDEXER_JAR:$INTEROP_RUNTIME_JAR:$HELPERS_JAR" INTEROP_TOOL=org.jetbrains.kotlin.native.interop.gen.jvm.MainKt FLAVOR_ARG=-flavor:native diff --git a/cmd/konanc b/cmd/konanc index 4ac2d939e4c..9c05f1a1cbc 100755 --- a/cmd/konanc +++ b/cmd/konanc @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/usr/bin/env bash if [ -z "$JAVACMD" -a -n "$JAVA_HOME" -a -x "$JAVA_HOME/bin/java" ]; then JAVACMD="$JAVA_HOME/bin/java" @@ -52,8 +52,9 @@ KONAN_HOME="$(findHome)" KONAN_JAR="${KONAN_HOME}/konan/lib/backend.native.jar" KOTLIN_JAR="${KONAN_HOME}/konan/lib/kotlin-compiler.jar" INTEROP_JAR="${KONAN_HOME}/konan/lib/Runtime.jar" +HELPERS_JAR="${KONAN_HOME}/konan/lib/helpers.jar" NATIVE_LIB="${KONAN_HOME}/konan/nativelib" -KONAN_CLASSPATH="$KOTLIN_JAR:$INTEROP_JAR:$KONAN_JAR" +KONAN_CLASSPATH="$KOTLIN_JAR:$INTEROP_JAR:$KONAN_JAR:$HELPERS_JAR" KONAN_COMPILER=org.jetbrains.kotlin.cli.bc.K2NativeKt JAVA_OPTS=-ea diff --git a/settings.gradle b/settings.gradle index a4cb6759f92..2576a33e043 100644 --- a/settings.gradle +++ b/settings.gradle @@ -7,3 +7,4 @@ include ':backend.native' include ':runtime' include ':common' include ':backend.native:tests' +include ':tools:helpers' diff --git a/tools/helpers/build.gradle b/tools/helpers/build.gradle new file mode 100644 index 00000000000..52ddb8dded9 --- /dev/null +++ b/tools/helpers/build.gradle @@ -0,0 +1,128 @@ +import org.jetbrains.kotlin.NativeInteropPlugin +import org.jetbrains.kotlin.RunInteropKonanTest +import org.jetbrains.kotlin.RunKonanTest +import org.jetbrains.kotlin.TestFailedException + +import java.util.concurrent.atomic.AtomicInteger + +buildscript { + repositories { + mavenCentral() + } + + dependencies { + classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" + } +} + +repositories { + maven { + url 'http://oss.sonatype.org/content/repositories/snapshots' + } +} + +apply plugin: 'kotlin' +apply plugin: NativeInteropPlugin + +configurations { + cli_bc.extendsFrom compile +} + +dependencies { + compile project(path: ':backend.native', configuration: 'cli_bc') + cli_bc jar.outputs.files +} + +sourceSets { + testOutputLocal { + output.dir(rootProject.file("${findProject(":backend.native:tests").testOutputRoot}/tools/helpers")) + } +} + +def testDependencies = project.file("${sourceSets.testOutputLocal.output.dirs.singleFile.absolutePath}/dependencies") + +void prepareDependenciesDir(File testDependencies) { + project.delete testDependencies + testDependencies.mkdirs() + // We don't download big clang dependency in this test - only sysroots. + project.copy { + from project.findProject(":dependencies").file("all") + into testDependencies + include "clang*.tar.gz/**" + } +} + +kotlinNativeInterop { + sysstat { + pkg 'sysstat' + headers 'sys/stat.h' + flavor 'native' + } +} + +task testInteropHelper(type: RunInteropKonanTest) { + dependsOn jar + dependsOn ':dist' + + dependenciesDir = testDependencies + goldValue = "0\n0\n" + source = "testData/interop.kt" + interop = 'sysstat' + + doFirst { + prepareDependenciesDir(testDependencies) + } +} + +task testCompilerHelper(type: RunKonanTest) { + dependenciesDir = testDependencies + source = "testData/main.kt" + + doFirst { + prepareDependenciesDir(testDependencies) + } +} + +// Simple test for parallel compiler execution +task testParallel(type: DefaultTask) { + for (int i=0; i<4; i++) { + task("runParallel$i", type: RunKonanTest) { + dependenciesDir = testDependencies + source = "testData/main.kt" + } + } + + doFirst { + prepareDependenciesDir(testDependencies) + } + + doLast { + List threads = new ArrayList<>() + def downloadCnt = new AtomicInteger(0) + tasks.withType(RunKonanTest).matching { it.name.startsWith("runParallel") }.each { task -> + threads.add(Thread.start { + task.executeTest() + // The helper prints messages when it is downloading. + // We check that there is only one downloading using compilation logs. + if (file("${task.buildExePath()}.compilation.log").text.contains("Download dependency")) { + downloadCnt.incrementAndGet() + } + }) + } + threads.each { it.join() } + if (downloadCnt.intValue() != 1) { + throw new TestFailedException("Actual downloads: ${downloadCnt.intValue()}, expected: 1") + } + } +} + +task testHelpers { + dependsOn testInteropHelper + dependsOn testCompilerHelper + dependsOn testParallel + + doLast { + delete testDependencies + } +} + diff --git a/tools/helpers/src/main/kotlin/CompilerHelper0.kt b/tools/helpers/src/main/kotlin/CompilerHelper0.kt new file mode 100644 index 00000000000..c9f3f42ccc7 --- /dev/null +++ b/tools/helpers/src/main/kotlin/CompilerHelper0.kt @@ -0,0 +1,22 @@ +package org.jetbrains.kotlin.konan + +import org.jetbrains.kotlin.backend.konan.Distribution +import org.jetbrains.kotlin.backend.konan.KonanTarget +import org.jetbrains.kotlin.backend.konan.TargetManager +import java.io.File + +class CompilerHelper0(val dist: Distribution): Runnable { + + override fun run() { + val dependencies = mutableListOf() + dependencies.add(dist.sysRoot) + dependencies.add(dist.llvmHome) + if (dist.sysRoot != dist.targetSysRoot) { + dependencies.add(dist.targetSysRoot) + } + if (TargetManager.host == KonanTarget.LINUX) { + dependencies.add(dist.libGcc) + } + DependencyDownloader(File(dist.dependencies), dist.properties.properties, dependencies).run() + } +} \ No newline at end of file diff --git a/tools/helpers/src/main/kotlin/DependencyDownloader.kt b/tools/helpers/src/main/kotlin/DependencyDownloader.kt new file mode 100644 index 00000000000..274ee841bee --- /dev/null +++ b/tools/helpers/src/main/kotlin/DependencyDownloader.kt @@ -0,0 +1,85 @@ +package org.jetbrains.kotlin.konan + +import java.io.File +import java.io.RandomAccessFile +import java.net.URL +import java.nio.file.* +import java.util.* + +// TODO: Try to use some dependency management system (Ivy?) +// TODO: Show % and size during downloading +class DependencyDownloader(dependenciesRoot: File, val properties: Properties, val dependencies: List) { + + val dependenciesRoot = dependenciesRoot.apply { mkdirs() } + + val lockFile = File(dependenciesRoot, ".lock").apply { createNewFile() } + val listFile = File(dependenciesRoot, ".downloaded") + + val dependenciesUrl: String = + properties.getProperty("dependenciesUrl", "https://jetbrains.bintray.com/kotlin-native-dependencies") + + var isInfoShown = false + + private fun File.containsLine(line: String): Boolean { + if (!exists()) { + return false + } + var result = false + listFile.forEachLine { + if (!result && it == line) { + result = true + } + } + return result + } + + private fun processDependency(path: String) { + val depDir = File(path) + val depName = depDir.name + val inListFile = listFile.containsLine(depName) + if (inListFile && depDir.exists()) { + return + } + if (!isInfoShown) { + println("Downloading native dependencies (LLVM, sysroot etc). This is one-time action performing only for the first run of the compiler.") + isInfoShown = true + } + val downloaded = download(depName) + println("Extract dependency: $downloaded -> $depDir") + extract(downloaded) + if (!inListFile) { + listFile.appendText("$depName\n") + } + } + + private fun extract(tarGz: File) { + val tarProcess = ProcessBuilder().apply { + command("tar", "-x", "-f", "${tarGz.canonicalPath}") + directory(tarGz.parentFile) + }.start() + tarProcess.waitFor() + if (tarProcess.exitValue() != 0) { + throw RuntimeException("Cannot extract archive with dependency: ${tarGz.canonicalPath}") + } + } + + private fun download(dependencyName: String): File { + val to = File(dependenciesRoot.canonicalPath, "$dependencyName.tar.gz") + if (!to.exists()) { + val from = URL("$dependenciesUrl/$dependencyName.tar.gz") + println("Download dependency: $dependencyName from $from") + from.openStream().use { + Files.copy(it, Paths.get(to.toURI()), StandardCopyOption.REPLACE_EXISTING) + } + } + return to + } + + fun run() { + val systemLock = RandomAccessFile(lockFile, "rw").channel.lock() + dependencies.forEach { + processDependency(it) + } + systemLock.release() + } +} \ No newline at end of file diff --git a/tools/helpers/src/main/kotlin/InteropHelper0.kt b/tools/helpers/src/main/kotlin/InteropHelper0.kt new file mode 100644 index 00000000000..a00e7cc2a80 --- /dev/null +++ b/tools/helpers/src/main/kotlin/InteropHelper0.kt @@ -0,0 +1,18 @@ +package org.jetbrains.kotlin.konan + +import org.jetbrains.kotlin.backend.konan.KonanProperties +import java.io.File +import java.util.* + +class InteropHelper0( + val dependenciesRoot: String, + val propertiesFile: String, + val dependencies: List): Runnable { + + override fun run() = + DependencyDownloader( + File(dependenciesRoot), + Properties().apply { load(File(propertiesFile).inputStream()) }, + dependencies + ).run() +} diff --git a/tools/helpers/testData/interop.kt b/tools/helpers/testData/interop.kt new file mode 100644 index 00000000000..862a4f30b33 --- /dev/null +++ b/tools/helpers/testData/interop.kt @@ -0,0 +1,9 @@ +import sysstat.* +import kotlinx.cinterop.* + +fun main(args: Array) { + val statBuf = nativeHeap.alloc() + val res = stat("/", statBuf.ptr) + println(res) + println(statBuf.st_uid.value) +} \ No newline at end of file diff --git a/tools/helpers/testData/main.kt b/tools/helpers/testData/main.kt new file mode 100644 index 00000000000..96b607ab67d --- /dev/null +++ b/tools/helpers/testData/main.kt @@ -0,0 +1,3 @@ +fun main(args: Array) { + println(42) +} \ No newline at end of file