From a995fbcb59042c306cf84b57c96cc2326945c889 Mon Sep 17 00:00:00 2001 From: Alexander Gorshenev Date: Fri, 3 Mar 2017 16:07:57 +0300 Subject: [PATCH] This is a simple command line tool for interop. See INTEROP.md on how to use it. --- INTEROP.md | 25 ++++ .../kotlin/native/interop/gen/jvm/main.kt | 111 +++++++++++++----- backend.native/konan.properties | 4 +- build.gradle | 39 ++++++ .../kotlin/NativeInteropPlugin.groovy | 20 +--- cmd/interop | 63 ++++++++++ 6 files changed, 217 insertions(+), 45 deletions(-) create mode 100644 INTEROP.md create mode 100644 cmd/interop diff --git a/INTEROP.md b/INTEROP.md new file mode 100644 index 00000000000..d9e40ed90e0 --- /dev/null +++ b/INTEROP.md @@ -0,0 +1,25 @@ +# An example of interop tool use # + +Build the dependencies and the compiler (see README.md). + +Prepare stubs for the system sockets library: + + ./dist/bin/interop -def:backend.native/tests/interop/basics/sockets.def + +Compile the echo server: + + ./dist/bin/konanc backend.native/tests/interop/basics/echo_server.kt \ + sockets -nativelibrary socketsstubs.bc + +Run the server: + + ./program.kexe 3000 & + +Test the server by conecting to it, for example with telnet: + + telnet localhost 3000 + +Write something to console and watch server echoing it back. + +~~Quit telnet by pressing ctrl+] ctrl+D~~ + diff --git a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt index 2efd554357f..05ce1575224 100644 --- a/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt +++ b/Interop/StubGenerator/src/main/kotlin/org/jetbrains/kotlin/native/interop/gen/jvm/main.kt @@ -9,11 +9,7 @@ import java.lang.IllegalArgumentException import java.util.* fun main(args: Array) { - val llvmInstallPath = System.getProperty("llvmInstallPath")!! - - val ktGenRoot = args[0] - val nativeLibsDir = args[1] - val otherArgs = args.drop(2) + val konanHome = System.getProperty("konan.home")!! // TODO: remove OSX defaults. val substitutions = mapOf( @@ -21,7 +17,7 @@ fun main(args: Array) { "os" to (System.getenv("TARGET_OS") ?: detectHost()) ) - processLib(ktGenRoot, nativeLibsDir, llvmInstallPath, substitutions, otherArgs) + processLib(konanHome, substitutions, args.asList()) } private fun detectHost():String { @@ -88,41 +84,105 @@ private fun Properties.getSpaceSeparated(name: String): List { return this.getProperty(name)?.split(' ') ?: emptyList() } +private fun Properties.getOsSpecific(name: String): String? { + val host = detectHost() + return this.getProperty("$name.$host") +} + private fun List?.isTrue(): Boolean { // The rightmost wins, null != "true". return this?.last() == "true" } -private fun processLib(ktGenRoot: String, - nativeLibsDir: String, - llvmInstallPath: String, +private fun runCmd(command: Array, workDir: File, verbose: Boolean = false) { + if (verbose) println(command) + ProcessBuilder(*command) + .directory(workDir) + .inheritIO() + .runExpectingSuccess() +} + +private fun Properties.defaultCompilerOpts(dependencies: String): List { + val sysRootDir = this.getOsSpecific("sysRoot")!! + val sysRoot= "$dependencies/$sysRootDir" + val host = detectHost() + when (host) { + "osx" -> return listOf( + "-B$sysRoot/usr/bin", + "--sysroot=$sysRoot", + "-mmacosx-version-min=10.10") + "linux" -> { + val llvmHomeDir = this.getOsSpecific("llvmHome")!! + val llvmHome = "$dependencies/$llvmHomeDir" + val llvmVersion = this.getProperty("llvmVersion")!! + val gccToolChainDir = this.getOsSpecific("gccToolChain")!! + val gccToolChain= "$dependencies/$gccToolChainDir" +// StubGenerator passes the arguments to libclang which +// works not exactly the same way as the clang binary and +// (in particular) uses different default header search path. +// See e.g. http://lists.llvm.org/pipermail/cfe-dev/2013-November/033680.html +// We workaround the problem with -isystem flag below. + return listOf( + "-isystem", + "$llvmHome/lib/clang/$llvmVersion/include", + "--gcc-toolchain=$gccToolChain", + "-L$llvmHome/lib", + "-B$sysRoot/../bin", + "--sysroot=$sysRoot") + } + else -> error("Unexpected host: ${host}") + } +} + +private fun loadProperties(file: File?, substitutions: Map): Properties { + val result = Properties() + file?.bufferedReader()?.use { reader -> + result.load(reader) + } + substitute(result, substitutions) + return result +} + +private fun processLib(konanHome: String, substitutions: Map, - additionalArgs: List) { + commandArgs: List) { - val args = additionalArgs.groupBy ({ getArgPrefix(it)!! }, { dropPrefix(it)!! }) // TODO + val args = commandArgs.groupBy ({ getArgPrefix(it)!! }, { dropPrefix(it)!! }) // TODO + val userDir = System.getProperty("user.dir") + val ktGenRoot = args["-generated"]?.single() ?: userDir + val nativeLibsDir = args["-natives"]?.single() ?: userDir val platformName = args["-target"]?.single() ?: "jvm" val platform = KotlinPlatform.values().single { it.name.equals(platformName, ignoreCase = true) } val defFile = args["-def"]?.single()?.let { File(it) } + val config = loadProperties(defFile, substitutions) - val config = Properties() - defFile?.bufferedReader()?.use { reader -> - config.load(reader) - } - substitute(config, substitutions) + val konanFileName = args["-properties"]?.single() ?: + "${konanHome}/konan/konan.properties" + val konanFile = File(konanFileName) + val konanProperties = loadProperties(konanFile, mapOf()) + val llvmHome = konanProperties.getOsSpecific("llvmHome")!! + val dependencies = File("$konanHome/../dependencies/all").canonicalPath + val llvmInstallPath = "$dependencies/$llvmHome" val additionalHeaders = args["-h"].orEmpty() val additionalCompilerOpts = args["-copt"].orEmpty() val additionalLinkerOpts = args["-lopt"].orEmpty() val generateShims = args["-shims"].isTrue() + val verbose = args["-verbose"].isTrue() + val defaultOpts = konanProperties.defaultCompilerOpts(dependencies) val headerFiles = config.getSpaceSeparated("headers") + additionalHeaders - val compilerOpts = config.getSpaceSeparated("compilerOpts") + additionalCompilerOpts + val compilerOpts = + config.getSpaceSeparated("compilerOpts") + + defaultOpts + additionalCompilerOpts val compiler = "clang" val language = Language.C - val linkerOpts = config.getSpaceSeparated("linkerOpts").toTypedArray() + additionalLinkerOpts + val linkerOpts = + config.getSpaceSeparated("linkerOpts").toTypedArray() + + defaultOpts + additionalLinkerOpts val linker = args["-linker"]?.singleOrNull() ?: config.getProperty("linker") ?: "clang" val excludedFunctions = config.getSpaceSeparated("excludedFunctions").toSet() @@ -173,20 +233,14 @@ private fun processLib(ktGenRoot: String, *compilerArgsForJniIncludes, "-c", outCFile.path, "-o", outOFile.path) - ProcessBuilder(*compilerCmd) - .directory(workDir) - .inheritIO() - .runExpectingSuccess() + runCmd(compilerCmd, workDir, verbose) val outLib = nativeLibsDir + "/" + System.mapLibraryName(libName) val linkerCmd = arrayOf("$llvmInstallPath/bin/$linker", *linkerOpts, outOFile.path, "-shared", "-o", outLib, "-Wl,-flat_namespace,-undefined,dynamic_lookup") - ProcessBuilder(*linkerCmd) - .directory(workDir) - .inheritIO() - .runExpectingSuccess() + runCmd(linkerCmd, workDir, verbose) outOFile.delete() } else if (platform == KotlinPlatform.NATIVE) { @@ -195,10 +249,7 @@ private fun processLib(ktGenRoot: String, val compilerCmd = arrayOf("$llvmInstallPath/bin/$compiler", *compilerOpts.toTypedArray(), "-emit-llvm", "-c", outCFile.path, "-o", outLib) - ProcessBuilder(*compilerCmd) - .directory(workDir) - .inheritIO() - .runExpectingSuccess() + runCmd(compilerCmd, workDir, verbose) } outCFile.delete() diff --git a/backend.native/konan.properties b/backend.native/konan.properties index c42f4d44a93..fca614ad62d 100644 --- a/backend.native/konan.properties +++ b/backend.native/konan.properties @@ -1,5 +1,6 @@ -// TODO: utilize substitution mechanism from interop. +// TODO: Do we need a $variable substitution mechanism here? +llvmVersion = 3.9.0 // macbook arch.osx = x86_64 @@ -37,6 +38,7 @@ osVersionMin.osx-ios-sim = -ios_simulator_version_min 5.0.0 sysRoot.linux = target-gcc-toolchain-3-linux-x86-64/x86_64-unknown-linux-gnu/sysroot llvmHome.linux = clang+llvm-3.9.0-linux-x86-64 libGcc.linux = target-gcc-toolchain-3-linux-x86-64/lib/gcc/x86_64-unknown-linux-gnu/4.8.5 +gccToolChain.linux = target-gcc-toolchain-3-linux-x86-64 llvmLtoFlags.linux = -O3 -function-sections -exported-symbol=main llvmLlcFlags.linux = -march=x86-64 linkerKonanFlags.linux = -Bstatic -lstdc++ -Bdynamic -ldl -lm -lpthread diff --git a/build.gradle b/build.gradle index 9b1a89fd284..f469d0fd4df 100644 --- a/build.gradle +++ b/build.gradle @@ -124,6 +124,14 @@ task dist_compiler(type: Copy) { into('konan/lib') } + from(project('Interop').file('Indexer/build/libs')) { + into('konan/lib') + } + + from(project('Interop').file('StubGenerator/build/libs')) { + into('konan/lib') + } + from(project(':backend.native').file('build/external_jars')) { into('konan/lib') } @@ -132,16 +140,47 @@ task dist_compiler(type: Copy) { into('konan/nativelib') } + from(project(':Interop').file('Indexer/build/nativelibs')) { + into('konan/nativelib') + } + + from(project(':Interop').file('Runtime/build/nativelibs')) { + into('konan/nativelib') + } + from(file('cmd')) { fileMode(0755) include('konanc') include('kotlinc-native') + include('interop') into('bin') } from(project(':backend.native').file('konan.properties')) { into('konan') } + + from("$llvmDir/lib") { + into('konan/nativelib') + include '**/libclang.*' + includeEmptyDirs = false + } + + doLast { + // MacOS System Integrity Protection strips DYLD_LIBRARY_PATH from + // environment of all the binaries residing in /bin /usr/bin etc. + // That means /usr/bin/java can't use DYLD_LIBRARY_PATH to pick up + // rpath'ed libraries. + // See dyld(1), install_name_tool(1) for rpath details. + + if (isMac()) { + exec { + commandLine "install_name_tool" + args '-add_rpath', '@loader_path/', + 'dist/konan/nativelib/libclangstubs.dylib' + } + } + } } task list_dist(type: Exec) { diff --git a/buildSrc/src/main/groovy/org/jetbrains/kotlin/NativeInteropPlugin.groovy b/buildSrc/src/main/groovy/org/jetbrains/kotlin/NativeInteropPlugin.groovy index cf95d07fafa..43c14ab48f9 100644 --- a/buildSrc/src/main/groovy/org/jetbrains/kotlin/NativeInteropPlugin.groovy +++ b/buildSrc/src/main/groovy/org/jetbrains/kotlin/NativeInteropPlugin.groovy @@ -167,7 +167,7 @@ class NamedNativeInteropConfig implements Named { new File(project.findProject(":Interop:Indexer").buildDir, "nativelibs"), new File(project.findProject(":Interop:Runtime").buildDir, "nativelibs") ).asPath - systemProperties "llvmInstallPath" : project.llvmDir + systemProperties "konan.home": project.rootProject.file("dist") environment "LIBCLANG_DISABLE_CRASH_RECOVERY": "1" environment "DYLD_LIBRARY_PATH": "${project.llvmDir}/lib" environment "LD_LIBRARY_PATH": "${project.llvmDir}/lib" @@ -183,9 +183,12 @@ class NamedNativeInteropConfig implements Named { linkerOpts += linkFiles.files - args = [generatedSrcDir, nativeLibsDir] - + args "-properties:" + project.findProject(":backend.native").file("konan.properties") + args "-generated:" + generatedSrcDir + args "-natives:" + nativeLibsDir args "-target:" + this.target + // Uncomment to debug. + //args "-verbose:true" if (defFile != null) { args "-def:" + project.file(defFile) @@ -203,17 +206,6 @@ class NamedNativeInteropConfig implements Named { environment['PATH'] = project.files(project.clangPath).asPath + File.pathSeparator + environment['PATH'] - if (project.isLinux()) { - // StubGenerator passes the arguments to libclang which works not exactly the same way - // as the clang binary and (in particular) uses different default header search path. - // See e.g. http://lists.llvm.org/pipermail/cfe-dev/2013-November/033680.html - // Workaround the problem: - compilerOpts += ["-isystem", "${project.llvmDir}/lib/clang/${project.llvmVersion}/include"] - } - - compilerOpts += project.hostClangArgs - linkerOpts += project.hostClangArgs - args compilerOpts.collect { "-copt:$it" } args linkerOpts.collect { "-lopt:$it" } diff --git a/cmd/interop b/cmd/interop new file mode 100644 index 00000000000..82f960eb03a --- /dev/null +++ b/cmd/interop @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +if [ -z "$JAVACMD" -a -n "$JAVA_HOME" -a -x "$JAVA_HOME/bin/java" ]; then + JAVACMD="$JAVA_HOME/bin/java" +else + JAVACMD=java +fi +[ -n "$JAVACMD" ] || JAVACMD=java + +declare -a java_args +declare -a interop_args + +while [ $# -gt 0 ]; do + case "$1" in + -D*) + java_args=("${java_args[@]}" "$1") + shift + ;; + -J*) + java_args=("${java_args[@]}" "${1:2}") + shift + ;; + *) + interop_args=("${interop_args[@]}" "$1") + shift + ;; + esac +done + + +findHome() { + local source="${BASH_SOURCE[0]}" + while [ -h "$source" ] ; do + local linked="$(readlink "$source")" + local dir="$(cd -P $(dirname "$source") && cd -P $(dirname "$linked") && pwd)" + source="$dir/$(basename "$linked")" + done + (cd -P "$(dirname "$source")/.." && pwd) +} +KONAN_HOME="$(findHome)" + +NATIVE_LIB="${KONAN_HOME}/konan/nativelib" +DEPENDENCIES="${KONAN_HOME}/../dependencies/all" +JAVA_OPTS="-ea \ + -Djava.library.path=${NATIVE_LIB} \ + -Dkonan.home=${KONAN_HOME}" + +STUB_GENERATOR_JAR="${KONAN_HOME}/konan/lib/StubGenerator.jar" +KOTLIN_JAR="${KONAN_HOME}/konan/lib/kotlin-compiler.jar" +INTEROP_INDEXER_JAR="${KONAN_HOME}/konan/lib/Indexer.jar" +INTEROP_RUNTIME_JAR="${KONAN_HOME}/konan/lib/Runtime.jar" +INTEROP_CLASSPATH="$STUB_GENERATOR_JAR:$KOTLIN_JAR:$INTEROP_INDEXER_JAR:$INTEROP_RUNTIME_JAR" +INTEROP_TOOL=org.jetbrains.kotlin.native.interop.gen.jvm.MainKt + +TARGET_ARG=-target:native + +LIBCLANG_DISABLE_CRASH_RECOVERY=1 \ +$JAVACMD $JAVA_OPTS ${java_args[@]} \ + -cp $INTEROP_CLASSPATH \ + $INTEROP_TOOL \ + $TARGET_ARG ${interop_args[@]} \ + +