[KT-40584] Refactor LLVM dependency processing

Currently, Kotlin/Native is tightly tied to a single predefined LLVM distribution that is used by developers as well as users. We want to separate distribution into two (smaller for users and bigger for devs) and also provide a way for users to use their own LLVM distribution (e.g. system one).
This commit adds this user-dev separation, yet without actual update of distributions.
This commit is contained in:
Sergey Bogolepov
2020-08-19 14:23:39 +07:00
committed by Sergey Bogolepov
parent 1615817905
commit c409192b97
3 changed files with 53 additions and 67 deletions
+31 -60
View File
@@ -35,14 +35,28 @@ homeDependencyCache = .konan/cache
# Appendix that is used for smaller version of LLVM distributions.
reducedLlvmAppendix = compact
predefinedLlvmDistributions = \
clang-llvm-8.0.0-linux-x86-64 \
msys2-mingw-w64-x86_64-clang-llvm-lld-compiler_rt-8.0.1 \
clang-llvm-apple-8.0.0-darwin-macos
llvm.linux_x64.dev = clang-llvm-8.0.0-linux-x86-64
llvm.linux_x64.user = clang-llvm-8.0.0-linux-x86-64
llvm.mingw_x64.dev = msys2-mingw-w64-x86_64-clang-llvm-lld-compiler_rt-8.0.1
llvm.mingw_x64.user = msys2-mingw-w64-x86_64-clang-llvm-lld-compiler_rt-8.0.1
llvm.macos_x64.dev = clang-llvm-apple-8.0.0-darwin-macos
llvm.macos_x64.user = clang-llvm-apple-8.0.0-darwin-macos
clangDebugFlags = -O0
llvmVersion.linux_x64 = 8.0.0
llvmVersion.mingw_x64 = 8.0.1
# We follow Apple versioning convention here.
llvmVersion.macos_x64 = 8.0.0
# Mac OS X.
llvmHome.macos_x64 = clang-llvm-apple-8.0.0-darwin-macos
# Can be an absolute path instead of predefined value.
llvmHome.macos_x64 = $llvm.macos_x64.dev
targetToolchain.macos_x64 = target-toolchain-xcode_11_5-macos_x64
libffiDir.macos_x64 = libffi-3.2.1-3-darwin-macos
additionalToolsDir.macos_x64 = xcode-addon-xcode_11_5-macos_x64
@@ -66,8 +80,7 @@ osVersionMinFlagClang.macos_x64 = -mmacosx-version-min
osVersionMin.macos_x64 = 10.11
dependencies.macos_x64 = \
libffi-3.2.1-3-darwin-macos \
lldb-2-macos \
clang-llvm-apple-8.0.0-darwin-macos
lldb-2-macos
target-sysroot-xcode_11_5-macos_x64.default = \
remote:internal
@@ -82,8 +95,7 @@ xcode-addon-xcode_11_5-macos_x64.default = \
# Apple's 32-bit iOS.
targetToolchain.macos_x64-ios_arm32 = target-toolchain-xcode_11_5-macos_x64
dependencies.macos_x64-ios_arm32 = \
libffi-3.2.1-3-darwin-macos \
clang-llvm-apple-8.0.0-darwin-macos
libffi-3.2.1-3-darwin-macos
target-sysroot-xcode_11_5-ios_arm32.default = \
remote:internal
@@ -108,8 +120,7 @@ osVersionMin.ios_arm32 = 9.0
# Apple's 64-bit iOS.
targetToolchain.macos_x64-ios_arm64 = target-toolchain-xcode_11_5-macos_x64
dependencies.macos_x64-ios_arm64 = \
libffi-3.2.1-3-darwin-macos \
clang-llvm-apple-8.0.0-darwin-macos
libffi-3.2.1-3-darwin-macos
target-sysroot-xcode_11_5-ios_arm64.default = \
remote:internal
@@ -137,8 +148,7 @@ osVersionMin.ios_arm64 = 9.0
# Apple's iOS simulator.
targetToolchain.macos_x64-ios_x64 = target-toolchain-xcode_11_5-macos_x64
dependencies.macos_x64-ios_x64 = \
libffi-3.2.1-3-darwin-macos \
clang-llvm-apple-8.0.0-darwin-macos
libffi-3.2.1-3-darwin-macos
target-sysroot-xcode_11_5-ios_x64.default = \
remote:internal
@@ -163,8 +173,7 @@ osVersionMin.ios_x64 = 9.0
# Apple's tvOS simulator.
targetToolchain.macos_x64-tvos_x64 = target-toolchain-xcode_11_5-macos_x64
dependencies.macos_x64-tvos_x64 = \
libffi-3.2.1-3-darwin-macos \
clang-llvm-apple-8.0.0-darwin-macos
libffi-3.2.1-3-darwin-macos
target-sysroot-xcode_11_5-tvos_x64.default = \
remote:internal
@@ -189,8 +198,7 @@ osVersionMin.tvos_x64 = 9.0
# Apple's 64-bit tvOS.
targetToolchain.macos_x64-tvos_arm64 = target-toolchain-xcode_11_5-macos_x64
dependencies.macos_x64-tvos_arm64 = \
libffi-3.2.1-3-darwin-macos \
clang-llvm-apple-8.0.0-darwin-macos
libffi-3.2.1-3-darwin-macos
target-sysroot-xcode_11_5-tvos_arm64.default = \
remote:internal
@@ -215,8 +223,7 @@ osVersionMin.tvos_arm64 = 9.0
# watchOS armv7k
targetToolchain.macos_x64-watchos_arm32 = target-toolchain-xcode_11_5-macos_x64
dependencies.macos_x64-watchos_arm32 = \
libffi-3.2.1-3-darwin-macos \
clang-llvm-apple-8.0.0-darwin-macos
libffi-3.2.1-3-darwin-macos
target-sysroot-xcode_11_5-watchos_arm32.default = \
remote:internal
@@ -241,8 +248,7 @@ osVersionMin.watchos_arm32 = 5.0
# watchOS arm64_32
targetToolchain.macos_x64-watchos_arm64 = target-toolchain-xcode_11_5-macos_x64
dependencies.macos_x64-watchos_arm64 = \
libffi-3.2.1-3-darwin-macos \
clang-llvm-apple-8.0.0-darwin-macos
libffi-3.2.1-3-darwin-macos
target-sysroot-xcode_11_5-watchos_arm64.default = \
remote:internal
@@ -268,8 +274,7 @@ osVersionMin.watchos_arm64 = 5.0
# Apple's watchOS i386 simulator.
targetToolchain.macos_x64-watchos_x86 = target-toolchain-xcode_11_5-macos_x64
dependencies.macos_x64-watchos_x86 = \
libffi-3.2.1-3-darwin-macos \
clang-llvm-apple-8.0.0-darwin-macos
libffi-3.2.1-3-darwin-macos
target-sysroot-xcode_11_5-watchos_x86.default = \
remote:internal
@@ -296,13 +301,12 @@ osVersionMinFlagClang.watchos_x86 = -mwatchos-simulator-version-min
osVersionMin.watchos_x86 = 5.0
# Linux x86-64.
llvmHome.linux_x64 = clang-llvm-8.0.0-linux-x86-64
llvmHome.linux_x64 = $llvm.linux_x64.dev
libffiDir.linux_x64 = libffi-3.2.1-2-linux-x86-64
gccToolchain.linux_x64 = target-gcc-toolchain-3-linux-x86-64
targetToolchain.linux_x64 = target-gcc-toolchain-3-linux-x86-64/x86_64-unknown-linux-gnu
dependencies.linux_x64 = \
clang-llvm-8.0.0-linux-x86-64 \
target-gcc-toolchain-3-linux-x86-64 \
libffi-3.2.1-2-linux-x86-64 \
lldb-2-linux
@@ -310,14 +314,12 @@ dependencies.linux_x64 = \
gccToolchain.mingw_x64 = target-gcc-toolchain-3-linux-x86-64
targetToolchain.mingw_x64-linux_x64 = msys2-mingw-w64-x86_64-clang-llvm-lld-compiler_rt-8.0.1
dependencies.mingw_x64-linux_x64 = \
msys2-mingw-w64-x86_64-clang-llvm-lld-compiler_rt-8.0.1 \
libffi-3.2.1-mingw-w64-x86-64 \
target-gcc-toolchain-3-linux-x86-64
gccToolchain.macos_x64 = target-gcc-toolchain-3-linux-x86-64
targetToolchain.macos_x64-linux_x64 = clang-llvm-apple-8.0.0-darwin-macos
targetToolchain.macos_x64-linux_x64 = $llvmHome.macos_x64
dependencies.macos_x64-linux_x64 = \
clang-llvm-apple-8.0.0-darwin-macos \
libffi-3.2.1-3-darwin-macos \
target-gcc-toolchain-3-linux-x86-64
@@ -342,19 +344,16 @@ abiSpecificLibraries.linux_x64 = ../lib64 lib64 usr/lib64
# Raspberry Pi
targetToolchain.linux_x64-linux_arm32_hfp = target-gcc-toolchain-3-linux-x86-64/x86_64-unknown-linux-gnu
targetToolchain.mingw_x64-linux_arm32_hfp = msys2-mingw-w64-x86_64-clang-llvm-lld-compiler_rt-8.0.1
targetToolchain.macos_x64-linux_arm32_hfp = clang-llvm-apple-8.0.0-darwin-macos
targetToolchain.macos_x64-linux_arm32_hfp = $llvmHome.macos_x64
dependencies.linux_x64-linux_arm32_hfp = \
clang-llvm-8.0.0-linux-x86-64 \
target-gcc-toolchain-3-linux-x86-64 \
target-sysroot-2-raspberrypi \
libffi-3.2.1-2-linux-x86-64
dependencies.mingw_x64-linux_arm32_hfp = \
msys2-mingw-w64-x86_64-clang-llvm-lld-compiler_rt-8.0.1 \
libffi-3.2.1-mingw-w64-x86-64 \
target-gcc-toolchain-3-linux-x86-64 \
target-sysroot-2-raspberrypi
dependencies.macos_x64-linux_arm32_hfp = \
clang-llvm-apple-8.0.0-darwin-macos \
libffi-3.2.1-3-darwin-macos \
target-gcc-toolchain-3-linux-x86-64 \
target-sysroot-2-raspberrypi
@@ -384,19 +383,16 @@ abiSpecificLibraries.linux_arm32_hfp = \
# Linux arm64
targetToolchain.linux_x64-linux_arm64 = target-gcc-toolchain-3-linux-x86-64/x86_64-unknown-linux-gnu
targetToolchain.mingw_x64-linux_arm64 = msys2-mingw-w64-x86_64-clang-llvm-lld-compiler_rt-8.0.1
targetToolchain.macos_x64-linux_arm64 = clang-llvm-apple-8.0.0-darwin-macos
targetToolchain.macos_x64-linux_arm64 = $llvmHome.macos_x64
dependencies.linux_x64-linux_arm64 = \
clang-llvm-8.0.0-linux-x86-64 \
target-gcc-toolchain-3-linux-x86-64 \
target-sysroot-1-linux-glibc-arm64 \
libffi-3.2.1-2-linux-x86-64
dependencies.mingw_x64-linux_arm64 = \
msys2-mingw-w64-x86_64-clang-llvm-lld-compiler_rt-8.0.1 \
libffi-3.2.1-mingw-w64-x86-64 \
target-gcc-toolchain-3-linux-x86-64 \
target-sysroot-1-linux-glibc-arm64
dependencies.macos_x64-linux_arm64 = \
clang-llvm-apple-8.0.0-darwin-macos \
libffi-3.2.1-3-darwin-macos \
target-gcc-toolchain-3-linux-x86-64 \
target-sysroot-1-linux-glibc-arm64
@@ -426,7 +422,6 @@ abiSpecificLibraries.linux_arm64 = \
# MIPS
targetToolchain.linux_x64-linux_mips32 = target-gcc-toolchain-2-linux-mips/x86_64-unknown-linux-gnu
dependencies.linux_x64-linux_mips32 = \
clang-llvm-8.0.0-linux-x86-64 \
target-gcc-toolchain-2-linux-mips \
target-gcc-toolchain-3-linux-x86-64 \
target-sysroot-2-mips \
@@ -453,7 +448,6 @@ abiSpecificLibraries.linux_mips32 =
# MIPSel
targetToolchain.linux_x64-linux_mipsel32 = target-gcc-toolchain-2-linux-mips/x86_64-unknown-linux-gnu
dependencies.linux_x64-linux_mipsel32 = \
clang-llvm-8.0.0-linux-x86-64 \
target-gcc-toolchain-2-linux-mips \
target-gcc-toolchain-3-linux-x86-64 \
target-sysroot-2-mipsel \
@@ -479,19 +473,16 @@ abiSpecificLibraries.linux_mipsel32 =
# Android ARM32, based on NDK for android-21.
targetToolchain.macos_x64-android_arm32 = target-toolchain-2-osx-android_ndk
dependencies.macos_x64-android_arm32 = \
clang-llvm-apple-8.0.0-darwin-macos \
target-sysroot-1-android_ndk \
target-toolchain-2-osx-android_ndk \
libffi-3.2.1-3-darwin-macos
targetToolchain.linux_x64-android_arm32 = target-toolchain-2-linux-android_ndk
dependencies.linux_x64-android_arm32 = \
clang-llvm-8.0.0-linux-x86-64 \
target-sysroot-1-android_ndk \
target-toolchain-2-linux-android_ndk \
libffi-3.2.1-2-linux-x86-64
targetToolchain.mingw_x64-android_arm32 = target-toolchain-2-windows-android_ndk
dependencies.mingw_x64-android_arm32 = \
msys2-mingw-w64-x86_64-clang-llvm-lld-compiler_rt-8.0.1 \
target-sysroot-1-android_ndk \
target-toolchain-2-windows-android_ndk \
libffi-3.2.1-mingw-w64-x86-64
@@ -507,19 +498,16 @@ linkerKonanFlags.android_arm32 = -lm -lc++_static -lc++abi -landroid -llog -lato
# Android ARM64, based on NDK.
targetToolchain.macos_x64-android_arm64 = target-toolchain-2-osx-android_ndk
dependencies.macos_x64-android_arm64 = \
clang-llvm-apple-8.0.0-darwin-macos \
target-sysroot-1-android_ndk \
target-toolchain-2-osx-android_ndk \
libffi-3.2.1-3-darwin-macos
targetToolchain.linux_x64-android_arm64 = target-toolchain-2-linux-android_ndk
dependencies.linux_x64-android_arm64 = \
clang-llvm-8.0.0-linux-x86-64 \
target-sysroot-1-android_ndk \
target-toolchain-2-linux-android_ndk \
libffi-3.2.1-2-linux-x86-64
targetToolchain.mingw_x64-android_arm64 = target-toolchain-2-windows-android_ndk
dependencies.mingw_x64-android_arm64 = \
msys2-mingw-w64-x86_64-clang-llvm-lld-compiler_rt-8.0.1 \
target-sysroot-1-android_ndk \
target-toolchain-2-windows-android_ndk \
libffi-3.2.1-mingw-w64-x86-64
@@ -535,19 +523,16 @@ linkerNoDebugFlags.android_arm64 = -Wl,-S
# Android X86, based on NDK.
targetToolchain.macos_x64-android_x86 = target-toolchain-2-osx-android_ndk
dependencies.macos_x64-android_x86 = \
clang-llvm-apple-8.0.0-darwin-macos \
target-sysroot-1-android_ndk \
target-toolchain-2-osx-android_ndk \
libffi-3.2.1-3-darwin-macos
targetToolchain.linux_x64-android_x86 = target-toolchain-2-linux-android_ndk
dependencies.linux_x64-android_x86 = \
clang-llvm-8.0.0-linux-x86-64 \
target-sysroot-1-android_ndk \
target-toolchain-2-linux-android_ndk \
libffi-3.2.1-2-linux-x86-64
targetToolchain.mingw_x64-android_x86 = target-toolchain-2-windows-android_ndk
dependencies.mingw_x64-android_x86 = \
msys2-mingw-w64-x86_64-clang-llvm-lld-compiler_rt-8.0.1 \
target-sysroot-1-android_ndk \
target-toolchain-2-windows-android_ndk \
libffi-3.2.1-mingw-w64-x86-64
@@ -568,19 +553,16 @@ linkerNoDebugFlags.android_x86 = -Wl,-S
# Android X64, based on NDK.
targetToolchain.macos_x64-android_x64 = target-toolchain-2-osx-android_ndk
dependencies.macos_x64-android_x64 = \
clang-llvm-apple-8.0.0-darwin-macos \
target-sysroot-1-android_ndk \
target-toolchain-2-osx-android_ndk \
libffi-3.2.1-3-darwin-macos
targetToolchain.linux_x64-android_x64 = target-toolchain-2-linux-android_ndk
dependencies.linux_x64-android_x64 = \
clang-llvm-8.0.0-linux-x86-64 \
target-sysroot-1-android_ndk \
target-toolchain-2-linux-android_ndk \
libffi-3.2.1-2-linux-x86-64
targetToolchain.mingw_x64-android_x64 = target-toolchain-2-windows-android_ndk
dependencies.mingw_x64-android_x64 = \
msys2-mingw-w64-x86_64-clang-llvm-lld-compiler_rt-8.0.1 \
target-sysroot-1-android_ndk \
target-toolchain-2-windows-android_ndk \
libffi-3.2.1-mingw-w64-x86-64
@@ -595,23 +577,18 @@ linkerNoDebugFlags.android_x64 = -Wl,-S
# Windows x86-64, based on mingw-w64.
llvmHome.mingw_x64 = msys2-mingw-w64-x86_64-clang-llvm-lld-compiler_rt-8.0.1
llvmHome.mingw_x64 = $llvm.mingw_x64.dev
targetToolchain.mingw_x64 = msys2-mingw-w64-x86_64-clang-llvm-lld-compiler_rt-8.0.1
libffiDir.mingw_x64 = libffi-3.2.1-mingw-w64-x86-64
targetToolchain.linux_x64-mingw_x64 = msys2-mingw-w64-x86_64-clang-llvm-lld-compiler_rt-8.0.1
targetToolchain.macos_x64-mingw_x64 = msys2-mingw-w64-x86_64-clang-llvm-lld-compiler_rt-8.0.1
dependencies.mingw_x64 = \
msys2-mingw-w64-x86_64-clang-llvm-lld-compiler_rt-8.0.1 \
libffi-3.2.1-mingw-w64-x86-64 \
lldb-2-windows
dependencies.linux_x64-mingw_x64 = \
msys2-mingw-w64-x86_64-clang-llvm-lld-compiler_rt-8.0.1 \
clang-llvm-8.0.0-linux-x86-64 \
libffi-3.2.1-2-linux-x86-64
dependencies.macos_x64-mingw_x64 = \
msys2-mingw-w64-x86_64-clang-llvm-lld-compiler_rt-8.0.1 \
clang-llvm-apple-8.0.0-darwin-macos \
libffi-3.2.1-3-darwin-macos
quadruple.mingw_x64 = x86_64-w64-mingw32
@@ -634,17 +611,14 @@ linkerOptimizationFlags.mingw_x64 = -Wl,--gc-sections
targetToolchain.mingw_x64-mingw_x86 = msys2-mingw-w64-i686-clang-llvm-lld-compiler_rt-8.0.1
dependencies.mingw_x64-mingw_x86 = \
msys2-mingw-w64-i686-clang-llvm-lld-compiler_rt-8.0.1 \
msys2-mingw-w64-x86_64-clang-llvm-lld-compiler_rt-8.0.1 \
libffi-3.2.1-mingw-w64-x86-64
targetToolchain.linux_x64-mingw_x86 = msys2-mingw-w64-i686-clang-llvm-lld-compiler_rt-8.0.1
targetToolchain.macos_x64-mingw_x86 = msys2-mingw-w64-i686-clang-llvm-lld-compiler_rt-8.0.1
dependencies.linux_x64-mingw_x86 = \
msys2-mingw-w64-i686-clang-llvm-lld-compiler_rt-8.0.1 \
clang-llvm-8.0.0-linux-x86-64 \
libffi-3.2.1-2-linux-x86-64
dependencies.macos_x64-mingw_x86 = \
msys2-mingw-w64-i686-clang-llvm-lld-compiler_rt-8.0.1 \
clang-llvm-apple-8.0.0-darwin-macos \
libffi-3.2.1-3-darwin-macos
quadruple.mingw_x86 = i686-w64-mingw32
@@ -671,21 +645,18 @@ linkerOptimizationFlags.mingw_x86 = -Wl,--gc-sections
targetToolchain.macos_x64-wasm32 = target-toolchain-3-macos-wasm
dependencies.macos_x64-wasm32 = \
libffi-3.2.1-3-darwin-macos \
clang-llvm-apple-8.0.0-darwin-macos \
target-sysroot-4-embedded \
target-toolchain-3-macos-wasm
targetToolchain.linux_x64-wasm32 = target-toolchain-2-linux-wasm
dependencies.linux_x64-wasm32 = \
libffi-3.2.1-2-linux-x86-64 \
clang-llvm-8.0.0-linux-x86-64 \
target-sysroot-4-embedded \
target-toolchain-2-linux-wasm
targetToolchain.mingw_x64-wasm32 = target-toolchain-2-mingw-wasm
dependencies.mingw_x64-wasm32 = \
libffi-3.2.1-mingw-w64-x86-64 \
msys2-mingw-w64-x86_64-clang-llvm-lld-compiler_rt-8.0.1 \
target-sysroot-4-embedded \
target-toolchain-2-mingw-wasm
@@ -699,4 +670,4 @@ clangDebugFlags.wasm32 = -O0
lld.wasm32 = --allow-undefined --no-entry --global-base=0 --no-threads --export-dynamic
# The version of Kotlin/Native compiler
compilerVersion=@compilerVersion@
compilerVersion=@compilerVersion@
@@ -38,12 +38,27 @@ abstract class KonanPropertiesLoader(override val target: KonanTarget,
val properties: Properties,
private val baseDir: String? = null,
private val host: KonanTarget = HostManager.host) : Configurables {
open val dependencies get() = hostTargetList("dependencies")
private val predefinedLlvmDistributions: Set<String> =
properties.propertyList("predefinedLlvmDistributions").toSet()
private fun llvmDependencies(): List<String> {
// Store into variable to avoid repeated resolve.
val llvmHome = llvmHome
?: error("Undefined LLVM home!")
return when (llvmHome) {
in predefinedLlvmDistributions -> llvmHome
else -> null
}.let(::listOfNotNull)
}
open val dependencies: List<String>
get() = hostTargetList("dependencies") + llvmDependencies()
override fun downloadDependencies() {
dependencyProcessor!!.run()
}
// TODO: We may want to add caching to avoid repeated resolve.
override fun targetString(key: String): String?
= properties.targetString(key, target)
override fun targetList(key: String): List<String>
@@ -19,19 +19,19 @@ package org.jetbrains.kotlin.konan.properties
import org.jetbrains.kotlin.konan.target.*
fun Properties.hostString(name: String, host: KonanTarget): String?
= this.propertyString(name, host.name)
= this.resolvablePropertyString(name, host.name)
fun Properties.hostList(name: String, host: KonanTarget): List<String>
= this.propertyList(name, host.name)
= this.resolvablePropertyList(name, host.name)
fun Properties.targetString(name: String, target: KonanTarget): String?
= this.propertyString(name, target.name)
= this.resolvablePropertyString(name, target.name)
fun Properties.targetList(name: String, target: KonanTarget): List<String>
= this.propertyList(name, target.name)
= this.resolvablePropertyList(name, target.name)
fun Properties.hostTargetString(name: String, target: KonanTarget, host: KonanTarget): String?
= this.propertyString(name, hostTargetSuffix(host, target))
= this.resolvablePropertyString(name, hostTargetSuffix(host, target))
fun Properties.hostTargetList(name: String, target: KonanTarget, host: KonanTarget): List<String>
= this.propertyList(name, hostTargetSuffix(host, target))
= this.resolvablePropertyList(name, hostTargetSuffix(host, target))