Support Linux x64 and RaspberryPi cross-compilation on Windows and macOS hosts. (#2575)

RELEASE_NOTES.md

@@ -23,12 +23,12 @@ the following platforms:
 
  * Mac OS X 10.11 and later (x86-64), host and target (`-target macos`, default on macOS hosts)
  * Ubuntu Linux x86-64 (14.04, 16.04 and later), other Linux flavours may work as well, host and target
-   (`-target linux`, default on Linux hosts)
+   (`-target linux`, default on Linux hosts, hosted on Linux, Windows and macOS)
  * Microsoft Windows x86-64 (tested on Windows 7 and Windows 10), host and target (`-target mingw`,
    default on Windows hosts)
  * Apple iOS (armv7 and arm64 devices, x86 simulator), cross-compiled target
    (`-target ios_arm32|ios_arm64|ios_x64`), hosted on macOS
- * Linux arm32 hardfp, Raspberry Pi, cross-compiled target (`-target raspberrypi`), hosted on Linux
+ * Linux arm32 hardfp, Raspberry Pi, cross-compiled target (`-target raspberrypi`), hosted on Linux, Windows and macOS
  * Linux MIPS big endian, cross-compiled target (`-target mips`), hosted on Linux
  * Linux MIPS little endian, cross-compiled target (`-target mipsel`), hosted on Linux
  * Android arm32 and arm64 (`-target android_arm32|android_arm64`) target, hosted on Linux or macOS

konan/konan.properties

@@ -137,9 +137,28 @@ osVersionMin.ios_x64 = 9.0
 
 # Linux x86-64.
 llvmHome.linux_x64 = clang-llvm-6.0.1-linux-x86-64
+libffiDir.linux_x64 = libffi-3.2.1-2-linux-x86-64
+
 gccToolchain.linux_x64 = target-gcc-toolchain-3-linux-x86-64
 targetToolchain.linux_x64 = target-gcc-toolchain-3-linux-x86-64/x86_64-unknown-linux-gnu
-libffiDir.linux_x64 = libffi-3.2.1-2-linux-x86-64
+dependencies.linux_x64 = \
+    clang-llvm-6.0.1-linux-x86-64 \
+    target-gcc-toolchain-3-linux-x86-64 \
+    libffi-3.2.1-2-linux-x86-64
+
+gccToolchain.mingw_x64 = target-gcc-toolchain-3-linux-x86-64
+targetToolchain.mingw_x64-linux_x64 = msys2-mingw-w64-x86_64-gcc-7.3.0-clang-llvm-lld-6.0.1
+dependencies.mingw_x64-linux_x64 = \
+    msys2-mingw-w64-x86_64-gcc-7.3.0-clang-llvm-lld-6.0.1 \
+    libffi-3.2.1-mingw-w64-x86-64 \
+    target-gcc-toolchain-3-linux-x86-64
+
+gccToolchain.macos_x64 = target-gcc-toolchain-3-linux-x86-64
+targetToolchain.macos_x64-linux_x64 = clang-llvm-6.0.1-darwin-macos
+dependencies.macos_x64-linux_x64 = \
+    clang-llvm-6.0.1-darwin-macos \
+    libffi-3.2.1-3-darwin-macos \
+    target-gcc-toolchain-3-linux-x86-64
 
 quadruple.linux_x64 = x86_64-unknown-linux-gnu
 targetSysRoot.linux_x64 = target-gcc-toolchain-3-linux-x86-64/x86_64-unknown-linux-gnu/sysroot
@@ -151,7 +170,7 @@ llvmLtoNooptFlags.linux_x64 = -O1
 llvmLtoDynamicFlags.linux_x64 = -relocation-model=pic
 llvmLlcFlags.linux_x64 = -march=x86-64
 linkerKonanFlags.linux_x64 = -Bstatic -lstdc++ -Bdynamic -ldl -lm -lpthread \
-  --defsym __cxa_demangle=Konan_cxa_demangle
+  --defsym __cxa_demangle=Konan_cxa_demangle --no-threads
 linkerOptimizationFlags.linux_x64 = --gc-sections
 linkerNoDebugFlags.linux_x64 = -S
 linkerDynamicFlags.linux_x64 = -shared
@@ -160,18 +179,26 @@ dynamicLinker.linux_x64 = /lib64/ld-linux-x86-64.so.2
 entrySelector.linux_x64 = --defsym main=Konan_main
 # targetSysRoot relative
 abiSpecificLibraries.linux_x64 = ../lib64 lib64 usr/lib64
-dependencies.linux_x64 = \
-    clang-llvm-6.0.1-linux-x86-64 \
-    target-gcc-toolchain-3-linux-x86-64 \
-    libffi-3.2.1-2-linux-x86-64
 
 # Raspberry Pi
 targetToolchain.linux_x64-linux_arm32_hfp = target-gcc-toolchain-3-linux-x86-64/x86_64-unknown-linux-gnu
+targetToolchain.mingw_x64-linux_arm32_hfp = msys2-mingw-w64-x86_64-gcc-7.3.0-clang-llvm-lld-6.0.1
+targetToolchain.macos_x64-linux_arm32_hfp = clang-llvm-6.0.1-darwin-macos
 dependencies.linux_x64-linux_arm32_hfp = \
     clang-llvm-6.0.1-linux-x86-64 \
     target-gcc-toolchain-3-linux-x86-64 \
     target-sysroot-1-raspberrypi \
     libffi-3.2.1-2-linux-x86-64
+dependencies.mingw_x64-linux_arm32_hfp = \
+    msys2-mingw-w64-x86_64-gcc-7.3.0-clang-llvm-lld-6.0.1 \
+    libffi-3.2.1-mingw-w64-x86-64 \
+    target-gcc-toolchain-3-linux-x86-64 \
+    target-sysroot-1-raspberrypi
+dependencies.macos_x64-linux_arm32_hfp = \
+    clang-llvm-6.0.1-darwin-macos \
+    libffi-3.2.1-3-darwin-macos \
+    target-gcc-toolchain-3-linux-x86-64 \
+    target-sysroot-1-raspberrypi
 
 quadruple.linux_arm32_hfp = armv6-unknown-linux-gnueabihf
 entrySelector.linux_arm32_hfp = --defsym main=Konan_main

shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/ClangArgs.kt

@@ -28,7 +28,9 @@ class ClangArgs(private val configurables: Configurables) : Configurables by con
         get() {
             val result = when (target) {
                 KonanTarget.LINUX_X64 ->
-                    listOf("--sysroot=$absoluteTargetSysRoot")
+                    listOf("--sysroot=$absoluteTargetSysRoot") +
+                    if (target != host) listOf("-target", targetArg!!) else emptyList()
+
                 KonanTarget.LINUX_ARM32_HFP ->
                     listOf("-target", targetArg!!,
                             "-mfpu=vfp", "-mfloat-abi=hard",

shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/KonanTarget.kt

@@ -182,6 +182,8 @@ open class HostManager(protected val distribution: Distribution = Distribution()
                 ) + zephyrSubtargets
                 KonanTarget.MINGW_X64 -> listOf(
                     KonanTarget.MINGW_X64,
+                    KonanTarget.LINUX_X64,
+                    KonanTarget.LINUX_ARM32_HFP,
                     KonanTarget.WASM32
                 ) + zephyrSubtargets
                 KonanTarget.MACOS_X64 -> listOf(
@@ -189,6 +191,8 @@ open class HostManager(protected val distribution: Distribution = Distribution()
                     KonanTarget.IOS_ARM32,
                     KonanTarget.IOS_ARM64,
                     KonanTarget.IOS_X64,
+                    KonanTarget.LINUX_X64,
+                    KonanTarget.LINUX_ARM32_HFP,
                     KonanTarget.ANDROID_ARM32,
                     KonanTarget.ANDROID_ARM64,
                     KonanTarget.WASM32
@@ -242,9 +246,9 @@ open class HostManager(protected val distribution: Distribution = Distribution()
             else -> throw TargetSupportException("Unknown host target: ${host_os()} ${host_arch()}")
         }
 
-        val hostIsMac   = (host == KonanTarget.MACOS_X64)
+        val hostIsMac   = (host.family == Family.OSX)
-        val hostIsLinux = (host == KonanTarget.LINUX_X64)
+        val hostIsLinux = (host.family == Family.LINUX)
-        val hostIsMingw = (host == KonanTarget.MINGW_X64)
+        val hostIsMingw = (host.family == Family.MINGW)
 
         val hostSuffix get() = host.name
         @JvmStatic

shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/Linker.kt

@@ -30,6 +30,35 @@ enum class LinkerOutputKind {
     EXECUTABLE
 }
 
+// Here we take somewhat unexpected approach - we create the thin
+// library, and then repack it during post-link phase.
+// This way we ensure .a inputs are properly processed.
+private fun staticGnuArCommands(ar: String, executable: ExecutableFile,
+                                objectFiles: List<ObjectFile>, libraries: List<String>) = when {
+        HostManager.hostIsMingw -> {
+            val temp = executable.replace('/', '\\') + "__"
+            val arWindows = ar.replace('/', '\\')
+            listOf(
+                    Command(arWindows, "-rucT", temp).apply {
+                        +objectFiles
+                        +libraries
+                    },
+                    Command("cmd", "/c").apply {
+                        +"(echo create $executable & echo addlib ${temp} & echo save & echo end) | $arWindows -M"
+                    },
+                    Command("cmd", "/c", "del", "/q", temp))
+        }
+        HostManager.hostIsLinux || HostManager.hostIsMac -> listOf(
+                     Command(ar, "cqT", executable).apply {
+                        +objectFiles
+                        +libraries
+                     },
+                     Command("/bin/sh", "-c").apply {
+                        +"printf 'create $executable\\naddlib $executable\\nsave\\nend' | $ar -M"
+                     })
+        else -> TODO("Unsupported host ${HostManager.host}")
+    }
+
 // Use "clang -v -save-temps" to write linkCommand() method 
 // for another implementation of this class.
 abstract class LinkerFlags(val configurables: Configurables)
@@ -58,12 +87,6 @@ abstract class LinkerFlags(val configurables: Configurables)
         // Let's just pass them as absolute paths.
         return libraries
     }
-
-    protected fun postLinkGnuArCommand(ar: String, executable: ExecutableFile) =
-            Command("/bin/sh", "-c").apply {
-                +"printf 'create $executable\\naddlib $executable\\nsave\\nend' | $ar -M"
-            }
-
 }
 
 open class AndroidLinker(targetProperties: AndroidConfigurables)
@@ -81,17 +104,9 @@ open class AndroidLinker(targetProperties: AndroidConfigurables)
                               libraries: List<String>, linkerArgs: List<String>,
                               optimize: Boolean, debug: Boolean,
                               kind: LinkerOutputKind, outputDsymBundle: String): List<Command> {
-        if (kind == LinkerOutputKind.STATIC_LIBRARY) {
+        if (kind == LinkerOutputKind.STATIC_LIBRARY)
-            // Here we take somewhat unexpected approach - we create the thin
+            return staticGnuArCommands(ar, executable, objectFiles, libraries)
-            // library, and then repack it during post-link phase.
+
-            // This way we ensure .a inputs are properly processed.
-            return listOf(
-                    Command(ar, "cqT", executable).apply {
-                        +objectFiles
-                        +libraries
-                    },
-                    postLinkGnuArCommand(ar, executable))
-        }
         val dynamic = kind == LinkerOutputKind.DYNAMIC_LIBRARY
         // liblog.so must be linked in, as we use its functionality in runtime.
         return listOf(Command(clang).apply {
@@ -218,7 +233,8 @@ open class MacOSBasedLinker(targetProperties: AppleConfigurables)
 open class LinuxBasedLinker(targetProperties: LinuxBasedConfigurables)
     : LinkerFlags(targetProperties), LinuxBasedConfigurables by targetProperties {
 
-    private val ar = "$absoluteTargetToolchain/bin/ar"
+    private val ar = if (HostManager.hostIsMac) "$absoluteTargetToolchain/bin/llvm-ar"
+        else "$absoluteTargetToolchain/bin/ar"
     override val libGcc: String = "$absoluteTargetSysRoot/${super.libGcc}"
     private val linker = "$absoluteLlvmHome/bin/ld.lld"
     private val specificLibs = abiSpecificLibraries.map { "-L${absoluteTargetSysRoot}/$it" }
@@ -229,16 +245,9 @@ open class LinuxBasedLinker(targetProperties: LinuxBasedConfigurables)
                               libraries: List<String>, linkerArgs: List<String>,
                               optimize: Boolean, debug: Boolean,
                               kind: LinkerOutputKind, outputDsymBundle: String): List<Command> {
-        if (kind == LinkerOutputKind.STATIC_LIBRARY) {
+        if (kind == LinkerOutputKind.STATIC_LIBRARY)
-            // Here we take somewhat unexpected approach - we create the thin
+            return staticGnuArCommands(ar, executable, objectFiles, libraries)
-            // library, and then repack it during post-link phase.
+
-            // This way we ensure .a inputs are properly processed.
-            return listOf(Command(ar, "cqT", executable).apply {
-                           +objectFiles
-                           +libraries
-                          },
-                          postLinkGnuArCommand(ar, executable))
-        }
         val isMips = (configurables is LinuxMIPSConfigurables)
         val dynamic = kind == LinkerOutputKind.DYNAMIC_LIBRARY
         // TODO: Can we extract more to the konan.configurables?
@@ -279,7 +288,7 @@ open class LinuxBasedLinker(targetProperties: LinuxBasedConfigurables)
 open class MingwLinker(targetProperties: MingwConfigurables)
     : LinkerFlags(targetProperties), MingwConfigurables by targetProperties {
 
-    private val ar = "$absoluteTargetToolchain\\bin\\ar"
+    private val ar = "$absoluteTargetToolchain/bin/ar"
     private val linker = "$absoluteTargetToolchain/bin/clang++"
 
     override val useCompilerDriverAsLinker: Boolean get() = true
@@ -290,21 +299,9 @@ open class MingwLinker(targetProperties: MingwConfigurables)
                               libraries: List<String>, linkerArgs: List<String>,
                               optimize: Boolean, debug: Boolean,
                               kind: LinkerOutputKind, outputDsymBundle: String): List<Command> {
-        if (kind == LinkerOutputKind.STATIC_LIBRARY) {
+        if (kind == LinkerOutputKind.STATIC_LIBRARY)
-            // Here we take somewhat unexpected approach - we create the thin
+            return staticGnuArCommands(ar, executable, objectFiles, libraries)
-            // library, and then repack it during post-link phase.
+
-            // This way we ensure .a inputs are properly processed.
-            val temp = executable.replace('/', '\\') + "__"
-            return listOf(
-                    Command(ar, "-rucT", temp).apply {
-                        +objectFiles
-                        +libraries
-                    },
-                    Command("cmd", "/c").apply {
-                        +"(echo create $executable & echo addlib ${temp} & echo save & echo end) | $ar -M"
-                    },
-                    Command("cmd", "/c", "del", "/q", temp))
-        }
         val dynamic = kind == LinkerOutputKind.DYNAMIC_LIBRARY
         return listOf(Command(linker).apply {
             +listOf("-o", executable)