Refactor Android, android_x64 support. (#3306)

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/OptimizationPipeline.kt

@@ -59,9 +59,9 @@ private class LlvmPipelineConfiguration(context: Context) {
         KonanTarget.LINUX_ARM64 -> "generic"
         KonanTarget.ANDROID_ARM32 -> "arm7tdmi"
         KonanTarget.ANDROID_ARM64 -> "cortex-a57"
+        KonanTarget.ANDROID_X64 -> "x86-64"
         KonanTarget.LINUX_MIPS32 -> "mips32r2"
         KonanTarget.LINUX_MIPSEL32 -> "mips32r2"
-        KonanTarget.ANDROID_X64 -> TODO("implement me")
         KonanTarget.WATCHOS_ARM64 -> TODO("implement me")
         KonanTarget.WATCHOS_X64 -> TODO("implement me")
         KonanTarget.TVOS_ARM64 -> TODO("implement me")

konan/konan.properties

@@ -343,63 +343,88 @@ dynamicLinker.linux_mipsel32 = /lib/ld.so.1
 abiSpecificLibraries.linux_mipsel32 =
 
 # Android ARM32, based on NDK for android-21.
-targetToolchain.macos_x64-android_arm32 = target-toolchain-21-osx-android_arm32
+targetToolchain.macos_x64-android_arm32 = target-toolchain-1-osx-android_ndk
-# TODO: split dependencies to host-dependent and host-independent parts.
 dependencies.macos_x64-android_arm32 = \
     clang-llvm-6.0.1-darwin-macos \
-    target-sysroot-21-android_arm32 \
+    target-sysroot-1-android_ndk \
-    target-toolchain-21-osx-android_arm32 \
+    target-toolchain-1-osx-android_ndk \
     libffi-3.2.1-3-darwin-macos
-targetToolchain.linux_x64-android_arm32 = target-toolchain-21-linux-android_arm32
+targetToolchain.linux_x64-android_arm32 = target-toolchain-1-linux-android_ndk
 dependencies.linux_x64-android_arm32 = \
     clang-llvm-6.0.1-linux-x86-64 \
-    target-sysroot-21-android_arm32 \
+    target-sysroot-1-android_ndk \
-    target-toolchain-21-linux-android_arm32 \
+    target-toolchain-1-linux-android_ndk \
     libffi-3.2.1-2-linux-x86-64
-targetToolchain.mingw_x64-android_arm32 = target-toolchain-21-windows-android_arm32
+targetToolchain.mingw_x64-android_arm32 = target-toolchain-1-windows-android_ndk
 dependencies.mingw_x64-android_arm32 = \
     msys2-mingw-w64-x86_64-gcc-7.3.0-clang-llvm-lld-6.0.1-2 \
-    target-sysroot-21-android_arm32 \
+    target-sysroot-1-android_ndk \
-    target-toolchain-21-windows-android_arm32 \
+    target-toolchain-1-windows-android_ndk \
     libffi-3.2.1-mingw-w64-x86-64
 
 quadruple.android_arm32 = arm-linux-androideabi
-entrySelector.android_arm32 =  -Wl,--defsym -Wl,main=Konan_main
+entrySelector.android_arm32 = -Wl,--defsym -Wl,main=Konan_main
 clangFlags.android_arm32 = -cc1 -target-cpu arm7tdmi -emit-obj -disable-llvm-optzns -x ir -femulated-tls -mrelocation-model pic
 clangOptFlags.android_arm32 = -O3 -ffunction-sections
 linkerNoDebugFlags.android_arm32 = -Wl,-S
 clangNooptFlags.android_arm32 = -O1
-targetSysRoot.android_arm32 = target-sysroot-21-android_arm32
+targetSysRoot.android_arm32 = target-sysroot-1-android_ndk
-linkerKonanFlags.android_arm32 = -lm -latomic -lstdc++ -landroid
+linkerKonanFlags.android_arm32 = -lm -lstdc++ -landroid -llog -latomic
 
-# Android ARM64, based on NDK for android-21.
+# Android ARM64, based on NDK.
-targetToolchain.macos_x64-android_arm64 = target-toolchain-21-osx-android_arm64
+targetToolchain.macos_x64-android_x64 = target-toolchain-1-osx-android_ndk
-# TODO: split dependencies to host-dependent and host-independent parts.
+dependencies.macos_x64-android_x64 = \
-dependencies.macos_x64-android_arm64 = \
     clang-llvm-6.0.1-darwin-macos \
-    target-sysroot-21-android_arm64 \
+    target-sysroot-1-android_ndk \
-    target-toolchain-21-osx-android_arm64 \
+    target-toolchain-1-osx-android_ndk \
     libffi-3.2.1-3-darwin-macos
-targetToolchain.linux_x64-android_arm64 = target-toolchain-21-linux-android_arm64
+targetToolchain.linux_x64-android_x64 = target-toolchain-1-linux-android_ndk
-dependencies.linux_x64-android_arm64 = \
+dependencies.linux_x64-android_x64 = \
     clang-llvm-6.0.1-linux-x86-64 \
-    target-sysroot-21-android_arm64 \
+    target-sysroot-1-android_ndk \
-    target-toolchain-21-linux-android_arm64 \
+    target-toolchain-1-linux-android_ndk \
     libffi-3.2.1-2-linux-x86-64
-targetToolchain.mingw_x64-android_arm64 = target-toolchain-21-windows-android_arm64
+targetToolchain.mingw_x64-android_x64 = target-toolchain-1-windows-android_ndk
-dependencies.mingw_x64-android_arm64 = \
+dependencies.mingw_x64-android_x64 = \
     msys2-mingw-w64-x86_64-gcc-7.3.0-clang-llvm-lld-6.0.1-2 \
-    target-sysroot-21-android_arm64 \
+    target-sysroot-1-android_ndk \
-    target-toolchain-21-windows-android_arm64 \
+    target-toolchain-1-windows-android_ndk \
     libffi-3.2.1-mingw-w64-x86-64
 
 quadruple.android_arm64 = aarch64-linux-android
 entrySelector.android_arm64 =  -Wl,--defsym -Wl,main=Konan_main
 clangFlags.android_arm64 = -cc1 -target-cpu cortex-a57 -emit-obj -disable-llvm-passes -x ir -femulated-tls -mrelocation-model pic
-targetSysRoot.android_arm64 = target-sysroot-21-android_arm64
+targetSysRoot.android_arm64 = target-sysroot-1-android_ndk
-linkerKonanFlags.android_arm64 = -lm -latomic -lstdc++ -landroid
+linkerKonanFlags.android_arm64 = -lm -lstdc++ -landroid -llog -latomic
 linkerNoDebugFlags.android_arm64 = -Wl,-S
 
+# Android X64, based on NDK.
+targetToolchain.macos_x64-android_arm64 = target-toolchain-1-osx-android_ndk
+dependencies.macos_x64-android_arm64 = \
+    clang-llvm-6.0.1-darwin-macos \
+    target-sysroot-1-android_ndk \
+    target-toolchain-1-osx-android_ndk \
+    libffi-3.2.1-3-darwin-macos
+targetToolchain.linux_x64-android_arm64 = target-toolchain-1-linux-android_ndk
+dependencies.linux_x64-android_arm64 = \
+    clang-llvm-6.0.1-linux-x86-64 \
+    target-sysroot-1-android_ndk \
+    target-toolchain-1-linux-android_ndk \
+    libffi-3.2.1-2-linux-x86-64
+targetToolchain.mingw_x64-android_arm64 = target-toolchain-1-windows-android_ndk
+dependencies.mingw_x64-android_arm64 = \
+    msys2-mingw-w64-x86_64-gcc-7.3.0-clang-llvm-lld-6.0.1-2 \
+    target-sysroot-1-android_ndk \
+    target-toolchain-1-windows-android_ndk \
+    libffi-3.2.1-mingw-w64-x86-64
+
+quadruple.android_x64 = x86_64-linux-android
+entrySelector.android_x64 = -Wl,--defsym -Wl,main=Konan_main
+clangFlags.android_x64 = -cc1 -target-cpu x86-64 -emit-obj -disable-llvm-passes -x ir -femulated-tls -mrelocation-model pic
+targetSysRoot.android_x64 = target-sysroot-1-android_ndk
+linkerKonanFlags.android_x64 = -lm -lstdc++ -landroid -llog -latomic
+linkerNoDebugFlags.android_ax64 = -Wl,-S
+
 # Windows x86-64, based on mingw-w64.
 llvmHome.mingw_x64 = msys2-mingw-w64-x86_64-gcc-7.3.0-clang-llvm-lld-6.0.1-2
 targetToolchain.mingw_x64 = msys2-mingw-w64-x86_64-gcc-7.3.0-clang-llvm-lld-6.0.1-2
@@ -508,4 +533,4 @@ llcNooptFlags.wasm32 = -O1
 llcOptFlags.wasm32 = -O3
 llcDebugFlags.wasm32 = -O0
 # TODO: make explicit list of external symbols
-lld.wasm32 = --allow-undefined --no-entry --global-base=0 --no-threads
+lld.wasm32 = --allow-undefined --no-entry --global-base=0 --no-threads

platformLibs/src/platform/android/android.def

@@ -1,13 +1,28 @@
 depends = posix
 package = platform.android
-headers = jni.h android/api-level.h android/asset_manager.h android/asset_manager_jni.h android/bitmap.h \
+headers = jni.h stdbool.h \
-     android/configuration.h android/dlext.h android/input.h android/keycodes.h android/log.h \
+     android/NeuralNetworks.h \
-     android/looper.h android/native_activity.h android/native_window.h android/native_window_jni.h \
+     android/api-level.h android/asset_manager.h android/asset_manager_jni.h \
-     android/obb.h android/rect.h android/sensor.h android/storage_manager.h android/tts.h \
+     android/bitmap.h \
+     android/choreographer.h android/configuration.h \
+     android/data_space.h android/dlext.h \
+     android/fdsan.h android/font.h android/font_matcher.h \
+     android/hardware_buffer.h android/hardware_buffer_jni.h \
+     android/input.h \
+     android/keycodes.h \
+     android/log.h android/looper.h \
+     android/multinetwork.h \
+     android/native_activity.h android/native_window.h android/ndk-version.h \
+     android/obb.h \
+     android/rect.h \
+     android/sensor.h android/set_abort_message.h android/sharedmem.h android/sharedmem_jni.h \
+     android/storage_manager.h android/surface_texture.h android/surface_texture_jni.h \
+     android/sync.h android/system_fonts.h \
+     android/trace.h \
+     android/versioning.h \
      android/window.h
 
 headerFilter = **
-
 linkerOpts = -landroid -llog -ljnigraphics
 
 ---

platformLibs/src/platform/android/linux.def

@@ -1,12 +1,12 @@
 depends = posix
 package = platform.linux
 headers = byteswap.h elf.h endian.h features.h lastlog.h link.h \
-    malloc.h mntent.h nsswitch.h pathconf.h termio.h thread_db.h \
+    malloc.h mntent.h termio.h \
-    uchar.h util.h \
+    uchar.h \
     sys/epoll.h sys/inotify.h sys/klog.h sys/sendfile.h \
-    sys/sysconf.h sys/sysinfo.h sys/utime.h \
+    sys/sysconf.h sys/sysinfo.h \
-    net/if_ether.h net/if_packet.h netinet/ether.h netinet/in6.h netpacket/packet.h
+    linux/if_ether.h linux/utime.h \
-
+    net/if_packet.h netinet/ether.h netinet/in6.h netpacket/packet.h
 headers.android_arm32 = time64.h
 
 headerFilter = **

platformLibs/src/platform/android/media.def

@@ -3,4 +3,4 @@ package = platform.media
 headers = media/NdkMediaCodec.h media/NdkMediaCrypto.h media/NdkMediaDrm.h media/NdkMediaError.h \
     media/NdkMediaExtractor.h media/NdkMediaFormat.h media/NdkMediaMuxer.h
 headerFilter = media/**
-linkerOpts = -lmediandk
+linkerOpts = -lmediandk

platformLibs/src/platform/android/posix.def

@@ -3,15 +3,15 @@ package = platform.posix
 headers = alloca.h ar.h assert.h complex.h dirent.h dlfcn.h err.h errno.h fcntl.h \
     fenv.h fnmatch.h fts.h ftw.h getopt.h grp.h inttypes.h libgen.h limits.h \
     locale.h math.h memory.h netdb.h paths.h poll.h \
-    pthread.h pwd.h regex.h resolv.h sched.h search.h semaphore.h setjmp.h sgtty.h signal.h \
+    pthread.h pwd.h regex.h resolv.h sched.h search.h semaphore.h setjmp.h signal.h \
-    stdatomic.h stdint.h stdio.h stdlib.h string.h strings.h syslog.h termios.h \
+    stdatomic.h stdbool.h stdint.h stdio.h stdlib.h string.h strings.h syslog.h termios.h \
     time.h ucontext.h unistd.h utime.h utmp.h wchar.h wctype.h xlocale.h \
     net/ethernet.h net/if.h net/if_arp.h net/route.h \
     netinet/icmp6.h netinet/if_ether.h netinet/in.h netinet/in_systm.h \
     netinet/ip.h netinet/ip6.h netinet/ip_icmp.h netinet/tcp.h netinet/udp.h \
     sys/ioctl.h sys/ipc.h sys/mman.h sys/poll.h sys/ptrace.h \
     sys/queue.h sys/select.h sys/shm.h sys/stat.h \
-    sys/syslimits.h sys/time.h sys/times.h sys/utsname.h sys/wait.h
+    sys/time.h sys/times.h sys/utsname.h sys/wait.h
 
 linkerOpts = -ldl
 
@@ -42,4 +42,4 @@ static int init_sockets() {
 }
 
 static void deinit_sockets() {
-}
+}

runtime/src/launcher/cpp/androidLauncher.cpp

@@ -25,6 +25,7 @@
 
 #include <unistd.h>
 #include <pthread.h>
+#include <sys/errno.h>
 #include <sys/types.h>
 #include <sys/socket.h>
 

shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/ClangArgs.kt

@@ -18,6 +18,19 @@ package org.jetbrains.kotlin.konan.target
 
 import org.jetbrains.kotlin.konan.file.File
 
+internal object Android {
+    const val API = "21"
+    private val architectureMap = mapOf(
+            // KonanTarget.X86 to "x86",
+            KonanTarget.ANDROID_X64 to "x86_64",
+            KonanTarget.ANDROID_ARM32 to "arm",
+            KonanTarget.ANDROID_ARM64 to "arm64"
+    )
+
+    fun architectureDirForTarget(target: KonanTarget) =
+            "android-${API}/arch-${architectureMap.getValue(target)}"
+}
+
 class ClangArgs(private val configurables: Configurables) : Configurables by configurables {
 
     val targetArg = if (configurables is TargetableConfigurables)
@@ -72,22 +85,19 @@ class ClangArgs(private val configurables: Configurables) : Configurables by con
                 KonanTarget.IOS_X64 ->
                     listOf("-stdlib=libc++", "-isysroot", absoluteTargetSysRoot, "-miphoneos-version-min=9.0.0")
 
-                KonanTarget.ANDROID_ARM32 ->
+                KonanTarget.ANDROID_ARM32, KonanTarget.ANDROID_ARM64, KonanTarget.ANDROID_X64 -> {
-                    listOf("-target", targetArg!!,
+                    val clangTarget = targetArg!!
-                            "--sysroot=$absoluteTargetSysRoot",
+                    val architectureDir = Android.architectureDirForTarget(target)
-                            // HACKS!
+                    val toolchainSysroot = "$absoluteTargetToolchain/sysroot"
-                            "-I$absoluteTargetSysRoot/usr/include/c++/4.9.x",
+                    listOf("-target", clangTarget,
-                            "-I$absoluteTargetSysRoot/usr/include/c++/4.9.x/arm-linux-androideabi")
+                            "-D__ANDROID_API__=${Android.API}",
+                            "--sysroot=$absoluteTargetSysRoot/$architectureDir",
+                            "-I$toolchainSysroot/usr/include/c++/v1",
+                            "-I$toolchainSysroot/usr/include",
+                            "-I$toolchainSysroot/usr/include/$clangTarget")
+                }
 
-                KonanTarget.ANDROID_ARM64 ->
+                // By default WASM target forces `hidden` visibility which causes linkage problems.
-                    listOf("-target", targetArg!!,
-                            "--sysroot=$absoluteTargetSysRoot",
-                            // HACKS!
-                            "-I$absoluteTargetSysRoot/usr/include/c++/4.9.x",
-                            "-I$absoluteTargetSysRoot/usr/include/c++/4.9.x/aarch64-linux-android")
-
-                // By default wasm target forces `hidden` visibility which causes
-                // linkage problems.
                 KonanTarget.WASM32 ->
                     listOf("-target", targetArg!!,
                             "-fno-rtti",
@@ -103,7 +113,6 @@ class ClangArgs(private val configurables: Configurables) : Configurables by con
                             "-Xclang", "-isystem$absoluteTargetSysRoot/include/compat",
                             "-Xclang", "-isystem$absoluteTargetSysRoot/include/libc")
 
-                KonanTarget.ANDROID_X64 -> TODO("implement me")
                 KonanTarget.WATCHOS_ARM64 -> TODO("implement me")
                 KonanTarget.WATCHOS_X64 -> TODO("implement me")
                 KonanTarget.TVOS_ARM64 -> TODO("implement me")
@@ -239,6 +248,15 @@ class ClangArgs(private val configurables: Configurables) : Configurables by con
                         "-DKONAN_ARM64=1",
                         "-DKONAN_HAS_CXX11_EXCEPTION_FUNCTIONS=1")
 
+            KonanTarget.ANDROID_X64 ->
+                listOf("-D__ANDROID__",
+                        "-DUSE_GCC_UNWIND=1",
+                        "-DUSE_ELF_SYMBOLS=1",
+                        "-DELFSIZE=64",
+                        "-DKONAN_ANDROID=1",
+                        "-DKONAN_X64=1",
+                        "-DKONAN_HAS_CXX11_EXCEPTION_FUNCTIONS=1")
+
             KonanTarget.WASM32 ->
                 listOf("-DKONAN_WASM=1",
                         "-DKONAN_NO_FFI=1",
@@ -250,7 +268,6 @@ class ClangArgs(private val configurables: Configurables) : Configurables by con
                         "-DKONAN_NO_MEMMEM",
                         "-DKONAN_NO_CTORS_SECTION=1")
 
-            KonanTarget.ANDROID_X64 -> TODO("implement me")
             KonanTarget.WATCHOS_ARM64 -> TODO("implement me")
             KonanTarget.WATCHOS_X64 -> TODO("implement me")
             KonanTarget.TVOS_ARM64 -> TODO("implement me")

shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/ConfigurablesImpl.kt

@@ -44,13 +44,12 @@ fun loadConfigurables(target: KonanTarget, properties: Properties, baseDir: Stri
             LinuxMIPSConfigurablesImpl(target, properties, baseDir)
         KonanTarget.MACOS_X64, KonanTarget.IOS_ARM32, KonanTarget.IOS_ARM64, KonanTarget.IOS_X64 ->
             AppleConfigurablesImpl(target, properties, baseDir)
-        KonanTarget.ANDROID_ARM32, KonanTarget.ANDROID_ARM64 ->
+        KonanTarget.ANDROID_ARM32, KonanTarget.ANDROID_ARM64, KonanTarget.ANDROID_X64 ->
             AndroidConfigurablesImpl(target, properties, baseDir)
         KonanTarget.MINGW_X64, KonanTarget.MINGW_X86 ->
             MingwConfigurablesImpl(target, properties, baseDir)
         KonanTarget.WASM32 ->
             WasmConfigurablesImpl(target, properties, baseDir)
-        KonanTarget.ANDROID_X64 -> TODO("unimplemented: $target")
         KonanTarget.WATCHOS_ARM64, KonanTarget.WATCHOS_X64 -> TODO("unimplemented: $target")
         KonanTarget.TVOS_ARM64, KonanTarget.TVOS_X64 -> TODO("unimplemented: $target")
         is KonanTarget.ZEPHYR ->

shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/Linker.kt

@@ -108,13 +108,23 @@ open class AndroidLinker(targetProperties: AndroidConfigurables)
             return staticGnuArCommands(ar, executable, objectFiles, libraries)
 
         val dynamic = kind == LinkerOutputKind.DYNAMIC_LIBRARY
-        // liblog.so must be linked in, as we use its functionality in runtime.
+        val toolchainSysroot = "${absoluteTargetToolchain}/sysroot"
+        val architectureDir = Android.architectureDirForTarget(target)
+        val apiSysroot = "$absoluteTargetSysRoot/$architectureDir"
+        val clangTarget = targetArg!!
+        val libDirs = listOf(
+                "--sysroot=$apiSysroot",
+                if (target == KonanTarget.ANDROID_X64) "-L$apiSysroot/usr/lib64" else "-L$apiSysroot/usr/lib",
+                "-L$toolchainSysroot/usr/lib/$clangTarget/${Android.API}",
+                "-L$toolchainSysroot/usr/lib/$clangTarget")
         return listOf(Command(clang).apply {
             +"-o"
             +executable
             +"-fPIC"
             +"-shared"
-            +"-llog"
+            +"-target"
+            +targetArg!!
+            +libDirs
             +objectFiles
             if (optimize) +linkerOptimizationFlags
             if (!debug) +linkerNoDebugFlags
@@ -429,13 +439,12 @@ fun linker(configurables: Configurables): LinkerFlags =
                 LinuxBasedLinker(configurables as LinuxMIPSConfigurables)
             KonanTarget.MACOS_X64, KonanTarget.IOS_ARM32, KonanTarget.IOS_ARM64, KonanTarget.IOS_X64 ->
                 MacOSBasedLinker(configurables as AppleConfigurables)
-            KonanTarget.ANDROID_ARM32, KonanTarget.ANDROID_ARM64 ->
+            KonanTarget.ANDROID_ARM32, KonanTarget.ANDROID_ARM64, KonanTarget.ANDROID_X64 ->
                 AndroidLinker(configurables as AndroidConfigurables)
             KonanTarget.MINGW_X64, KonanTarget.MINGW_X86 ->
                 MingwLinker(configurables as MingwConfigurables)
             KonanTarget.WASM32 ->
                 WasmLinker(configurables as WasmConfigurables)
-            KonanTarget.ANDROID_X64 -> TODO("implement me")
             KonanTarget.WATCHOS_ARM64 -> TODO("implement me")
             KonanTarget.WATCHOS_X64 -> TODO("implement me")
             KonanTarget.TVOS_ARM64 -> TODO("implement me")

shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/Platform.kt

@@ -49,7 +49,7 @@ class PlatformManager(distribution: Distribution = Distribution(), experimental:
      * TODO: Don't forget to delete this field and replace all its usages to `enabled`.
      */
     val filteredOutEnabledButNotSupported
-        get() = enabled.filterNot { it == KonanTarget.ANDROID_X64 || it == KonanTarget.WATCHOS_X64 || it == KonanTarget.WATCHOS_ARM64 ||
+        get() = enabled.filterNot { it == KonanTarget.WATCHOS_X64 || it == KonanTarget.WATCHOS_ARM64 ||
                 it == KonanTarget.TVOS_X64 || it == KonanTarget.TVOS_ARM64}
 }