This is a simple command line tool for interop.

See INTEROP.md on how to use it.
2017-03-03 16:07:57 +03:00
parent 62e678a37a
commit a995fbcb59
6 changed files with 217 additions and 45 deletions
@@ -0,0 +1,25 @@
+# An example of interop tool use #
+
+Build the dependencies and the compiler (see README.md).
+
+Prepare stubs for the system sockets library:
+
+    ./dist/bin/interop -def:backend.native/tests/interop/basics/sockets.def
+
+Compile the echo server:
+
+    ./dist/bin/konanc backend.native/tests/interop/basics/echo_server.kt \
+        sockets -nativelibrary socketsstubs.bc 
+
+Run the server:
+
+    ./program.kexe 3000 &
+
+Test the server by conecting to it, for example with telnet:
+
+    telnet localhost 3000
+
+Write something to console and watch server echoing it back.
+
+~~Quit telnet by pressing ctrl+] ctrl+D~~
+
@@ -9,11 +9,7 @@ import java.lang.IllegalArgumentException
 import java.util.*

 fun main(args: Array<String>) {
-    val llvmInstallPath = System.getProperty("llvmInstallPath")!!
-
-    val ktGenRoot = args[0]
-    val nativeLibsDir = args[1]
-    val otherArgs = args.drop(2)
+    val konanHome = System.getProperty("konan.home")!!

    // TODO: remove OSX defaults.
    val substitutions = mapOf(
@@ -21,7 +17,7 @@ fun main(args: Array<String>) {
            "os" to (System.getenv("TARGET_OS") ?: detectHost())
    )

-    processLib(ktGenRoot, nativeLibsDir, llvmInstallPath, substitutions, otherArgs)
+    processLib(konanHome, substitutions, args.asList())
 }

 private fun detectHost():String {
@@ -88,41 +84,105 @@ private fun Properties.getSpaceSeparated(name: String): List<String> {
    return this.getProperty(name)?.split(' ') ?: emptyList()
 }

+private fun Properties.getOsSpecific(name: String): String? {
+    val host = detectHost()
+    return this.getProperty("$name.$host")
+}
+
 private fun List<String>?.isTrue(): Boolean {
    // The rightmost wins, null != "true".
    return this?.last() == "true"
 }

-private fun processLib(ktGenRoot: String,
-                       nativeLibsDir: String,
-                       llvmInstallPath: String,
+private fun runCmd(command: Array<String>, workDir: File, verbose: Boolean = false) {
+        if (verbose) println(command)
+        ProcessBuilder(*command)
+                .directory(workDir)
+                .inheritIO()
+                .runExpectingSuccess()
+}
+
+private fun Properties.defaultCompilerOpts(dependencies: String): List<String> {
+    val sysRootDir = this.getOsSpecific("sysRoot")!!
+    val sysRoot= "$dependencies/$sysRootDir"
+    val host = detectHost()
+    when (host) {
+        "osx" -> return listOf(
+            "-B$sysRoot/usr/bin",
+            "--sysroot=$sysRoot",
+            "-mmacosx-version-min=10.10")
+        "linux" -> {
+            val llvmHomeDir = this.getOsSpecific("llvmHome")!!
+            val llvmHome = "$dependencies/$llvmHomeDir"
+            val llvmVersion = this.getProperty("llvmVersion")!!
+            val gccToolChainDir = this.getOsSpecific("gccToolChain")!!
+            val gccToolChain= "$dependencies/$gccToolChainDir"
+// StubGenerator passes the arguments to libclang which 
+// works not exactly the same way as the clang binary and 
+// (in particular) uses different default header search path.
+// See e.g. http://lists.llvm.org/pipermail/cfe-dev/2013-November/033680.html
+// We workaround the problem with -isystem flag below.
+            return listOf(
+                "-isystem", 
+                "$llvmHome/lib/clang/$llvmVersion/include",
+                "--gcc-toolchain=$gccToolChain",
+                "-L$llvmHome/lib",
+                "-B$sysRoot/../bin",
+                "--sysroot=$sysRoot")
+        }
+        else -> error("Unexpected host: ${host}")
+    }
+}
+
+private fun loadProperties(file: File?, substitutions: Map<String, String>): Properties {
+    val result = Properties()
+    file?.bufferedReader()?.use { reader ->
+        result.load(reader)
+    }
+    substitute(result, substitutions)
+    return result
+}
+
+private fun processLib(konanHome: String,
                       substitutions: Map<String, String>,
-                       additionalArgs: List<String>) {
+                       commandArgs: List<String>) {

-    val args = additionalArgs.groupBy ({ getArgPrefix(it)!! }, { dropPrefix(it)!! }) // TODO
+    val args = commandArgs.groupBy ({ getArgPrefix(it)!! }, { dropPrefix(it)!! }) // TODO

+    val userDir = System.getProperty("user.dir")
+    val ktGenRoot = args["-generated"]?.single() ?: userDir
+    val nativeLibsDir = args["-natives"]?.single() ?: userDir
    val platformName = args["-target"]?.single() ?: "jvm"

    val platform = KotlinPlatform.values().single { it.name.equals(platformName, ignoreCase = true) }

    val defFile = args["-def"]?.single()?.let { File(it) }
+    val config = loadProperties(defFile, substitutions)

-    val config = Properties()
-    defFile?.bufferedReader()?.use { reader ->
-        config.load(reader)
-    }
-    substitute(config, substitutions)
+    val konanFileName = args["-properties"]?.single() ?:
+        "${konanHome}/konan/konan.properties"
+    val konanFile = File(konanFileName)
+    val konanProperties = loadProperties(konanFile, mapOf())

+    val llvmHome = konanProperties.getOsSpecific("llvmHome")!!
+    val dependencies = File("$konanHome/../dependencies/all").canonicalPath
+    val llvmInstallPath = "$dependencies/$llvmHome"
    val additionalHeaders = args["-h"].orEmpty()
    val additionalCompilerOpts = args["-copt"].orEmpty()
    val additionalLinkerOpts = args["-lopt"].orEmpty()
    val generateShims = args["-shims"].isTrue()
+    val verbose = args["-verbose"].isTrue()

+    val defaultOpts = konanProperties.defaultCompilerOpts(dependencies)
    val headerFiles = config.getSpaceSeparated("headers") + additionalHeaders
-    val compilerOpts = config.getSpaceSeparated("compilerOpts") + additionalCompilerOpts
+    val compilerOpts = 
+        config.getSpaceSeparated("compilerOpts") +
+        defaultOpts + additionalCompilerOpts 
    val compiler = "clang"
    val language = Language.C
-    val linkerOpts = config.getSpaceSeparated("linkerOpts").toTypedArray() + additionalLinkerOpts
+    val linkerOpts = 
+        config.getSpaceSeparated("linkerOpts").toTypedArray() +
+        defaultOpts + additionalLinkerOpts 
    val linker = args["-linker"]?.singleOrNull() ?: config.getProperty("linker") ?: "clang"
    val excludedFunctions = config.getSpaceSeparated("excludedFunctions").toSet()

@@ -173,20 +233,14 @@ private fun processLib(ktGenRoot: String,
                *compilerArgsForJniIncludes,
                "-c", outCFile.path, "-o", outOFile.path)

-        ProcessBuilder(*compilerCmd)
-                .directory(workDir)
-                .inheritIO()
-                .runExpectingSuccess()
+        runCmd(compilerCmd, workDir, verbose)

        val outLib = nativeLibsDir + "/" + System.mapLibraryName(libName)

        val linkerCmd = arrayOf("$llvmInstallPath/bin/$linker", *linkerOpts, outOFile.path, "-shared", "-o", outLib,
                "-Wl,-flat_namespace,-undefined,dynamic_lookup")

-        ProcessBuilder(*linkerCmd)
-                .directory(workDir)
-                .inheritIO()
-                .runExpectingSuccess()
+        runCmd(linkerCmd, workDir, verbose)

        outOFile.delete()
    } else if (platform == KotlinPlatform.NATIVE) {
@@ -195,10 +249,7 @@ private fun processLib(ktGenRoot: String,
        val compilerCmd = arrayOf("$llvmInstallPath/bin/$compiler", *compilerOpts.toTypedArray(),
                "-emit-llvm", "-c", outCFile.path, "-o", outLib)

-        ProcessBuilder(*compilerCmd)
-                .directory(workDir)
-                .inheritIO()
-                .runExpectingSuccess()
+        runCmd(compilerCmd, workDir, verbose)
    }

    outCFile.delete()
@@ -1,5 +1,6 @@

-// TODO: utilize substitution mechanism from interop.
+// TODO: Do we need a $variable substitution mechanism here?
+llvmVersion = 3.9.0

 // macbook
 arch.osx = x86_64
@@ -37,6 +38,7 @@ osVersionMin.osx-ios-sim = -ios_simulator_version_min 5.0.0
 sysRoot.linux = target-gcc-toolchain-3-linux-x86-64/x86_64-unknown-linux-gnu/sysroot
 llvmHome.linux = clang+llvm-3.9.0-linux-x86-64
 libGcc.linux = target-gcc-toolchain-3-linux-x86-64/lib/gcc/x86_64-unknown-linux-gnu/4.8.5
+gccToolChain.linux = target-gcc-toolchain-3-linux-x86-64
 llvmLtoFlags.linux = -O3 -function-sections -exported-symbol=main
 llvmLlcFlags.linux = -march=x86-64
 linkerKonanFlags.linux = -Bstatic -lstdc++ -Bdynamic -ldl -lm -lpthread
@@ -124,6 +124,14 @@ task dist_compiler(type: Copy) {
        into('konan/lib')
    }

+    from(project('Interop').file('Indexer/build/libs')) {
+        into('konan/lib')
+    }
+
+    from(project('Interop').file('StubGenerator/build/libs')) {
+        into('konan/lib')
+    }
+
    from(project(':backend.native').file('build/external_jars')) {
        into('konan/lib')
    }
@@ -132,16 +140,47 @@ task dist_compiler(type: Copy) {
        into('konan/nativelib')
    }

+    from(project(':Interop').file('Indexer/build/nativelibs')) {
+        into('konan/nativelib')
+    }
+
+    from(project(':Interop').file('Runtime/build/nativelibs')) {
+        into('konan/nativelib')
+    }
+
    from(file('cmd')) {
        fileMode(0755)
        include('konanc')
        include('kotlinc-native')
+        include('interop')
        into('bin')
    }

    from(project(':backend.native').file('konan.properties')) {
        into('konan')
    }
+
+    from("$llvmDir/lib") {
+        into('konan/nativelib')
+        include '**/libclang.*'
+        includeEmptyDirs = false
+    }
+
+    doLast {
+        // MacOS System Integrity Protection strips DYLD_LIBRARY_PATH from
+        // environment of all the binaries residing in /bin /usr/bin etc.
+        // That means /usr/bin/java can't use DYLD_LIBRARY_PATH to pick up
+        // rpath'ed libraries. 
+        // See dyld(1), install_name_tool(1) for rpath details.
+
+        if (isMac()) {
+            exec {
+                commandLine "install_name_tool"
+                args '-add_rpath', '@loader_path/', 
+                    'dist/konan/nativelib/libclangstubs.dylib'
+            }
+        }
+    }
 }

 task list_dist(type: Exec) {
@@ -167,7 +167,7 @@ class NamedNativeInteropConfig implements Named {
                    new File(project.findProject(":Interop:Indexer").buildDir, "nativelibs"),
                    new File(project.findProject(":Interop:Runtime").buildDir, "nativelibs")
            ).asPath
-            systemProperties "llvmInstallPath" : project.llvmDir
+            systemProperties "konan.home": project.rootProject.file("dist")
            environment "LIBCLANG_DISABLE_CRASH_RECOVERY": "1"
            environment "DYLD_LIBRARY_PATH": "${project.llvmDir}/lib"
            environment "LD_LIBRARY_PATH": "${project.llvmDir}/lib"
@@ -183,9 +183,12 @@ class NamedNativeInteropConfig implements Named {

                linkerOpts += linkFiles.files

-                args = [generatedSrcDir, nativeLibsDir]
-
+                args "-properties:" + project.findProject(":backend.native").file("konan.properties")
+                args "-generated:" + generatedSrcDir
+                args "-natives:" + nativeLibsDir
                args "-target:" + this.target
+                // Uncomment to debug.
+                //args "-verbose:true"

                if (defFile != null) {
                    args "-def:" + project.file(defFile)
@@ -203,17 +206,6 @@ class NamedNativeInteropConfig implements Named {
                environment['PATH'] = project.files(project.clangPath).asPath +
                        File.pathSeparator + environment['PATH']

-                if (project.isLinux()) {
-                    // StubGenerator passes the arguments to libclang which works not exactly the same way
-                    // as the clang binary and (in particular) uses different default header search path.
-                    // See e.g. http://lists.llvm.org/pipermail/cfe-dev/2013-November/033680.html
-                    // Workaround the problem:
-                    compilerOpts += ["-isystem", "${project.llvmDir}/lib/clang/${project.llvmVersion}/include"]
-                }
-
-                compilerOpts += project.hostClangArgs
-                linkerOpts   += project.hostClangArgs
-
                args compilerOpts.collect { "-copt:$it" }
                args linkerOpts.collect { "-lopt:$it" }

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash 
+
+if [ -z "$JAVACMD" -a -n "$JAVA_HOME" -a -x "$JAVA_HOME/bin/java" ]; then
+    JAVACMD="$JAVA_HOME/bin/java"
+else
+    JAVACMD=java
+fi
+[ -n "$JAVACMD" ] || JAVACMD=java
+
+declare -a java_args
+declare -a interop_args
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    -D*)
+      java_args=("${java_args[@]}" "$1")
+      shift
+      ;;
+    -J*)
+      java_args=("${java_args[@]}" "${1:2}")
+      shift
+      ;;
+     *)
+      interop_args=("${interop_args[@]}" "$1")
+      shift
+      ;;
+  esac
+done
+
+
+findHome() {
+    local source="${BASH_SOURCE[0]}"
+    while [ -h "$source" ] ; do
+	local linked="$(readlink "$source")"
+	local dir="$(cd -P $(dirname "$source") && cd -P $(dirname "$linked") && pwd)"
+	source="$dir/$(basename "$linked")"
+    done
+    (cd -P "$(dirname "$source")/.." && pwd)
+}
+KONAN_HOME="$(findHome)"
+
+NATIVE_LIB="${KONAN_HOME}/konan/nativelib"
+DEPENDENCIES="${KONAN_HOME}/../dependencies/all"
+JAVA_OPTS="-ea \
+    -Djava.library.path=${NATIVE_LIB} \
+    -Dkonan.home=${KONAN_HOME}"
+
+STUB_GENERATOR_JAR="${KONAN_HOME}/konan/lib/StubGenerator.jar"
+KOTLIN_JAR="${KONAN_HOME}/konan/lib/kotlin-compiler.jar"
+INTEROP_INDEXER_JAR="${KONAN_HOME}/konan/lib/Indexer.jar"
+INTEROP_RUNTIME_JAR="${KONAN_HOME}/konan/lib/Runtime.jar"
+INTEROP_CLASSPATH="$STUB_GENERATOR_JAR:$KOTLIN_JAR:$INTEROP_INDEXER_JAR:$INTEROP_RUNTIME_JAR"
+INTEROP_TOOL=org.jetbrains.kotlin.native.interop.gen.jvm.MainKt
+
+TARGET_ARG=-target:native
+
+LIBCLANG_DISABLE_CRASH_RECOVERY=1 \
+$JAVACMD $JAVA_OPTS ${java_args[@]} \
+    -cp $INTEROP_CLASSPATH \
+    $INTEROP_TOOL \
+    $TARGET_ARG ${interop_args[@]} \
+
+