From 38380f068633b3d97cd5aba3406b32a2d5a3b1a7 Mon Sep 17 00:00:00 2001 From: Sergey Bogolepov Date: Tue, 18 Feb 2020 16:33:42 +0700 Subject: [PATCH] Add task that makes LLVM dependency smaller Currently Kotlin/Native uses default LLVM distribution. While it is useful to have whole LLVM toolchain for development, it is huge pain for users, because dependency is needlessly big. The correct solution for this problem is to make own build script for LLVM, but it is a big task that make take a while. So for now we repack LLVM dependency without unnecessary files. --- dependencyPacker/build.gradle.kts | 92 +++++++++++++++++++++++++++ dependencyPacker/linux_llvm_blacklist | 6 ++ dependencyPacker/linux_llvm_whitelist | 28 ++++++++ dependencyPacker/macos_llvm_whitelist | 29 +++++++++ dependencyPacker/mingw_llvm_blacklist | 5 ++ dependencyPacker/mingw_llvm_whitelist | 39 ++++++++++++ konan/konan.properties | 3 + settings.gradle | 2 + 8 files changed, 204 insertions(+) create mode 100644 dependencyPacker/build.gradle.kts create mode 100644 dependencyPacker/linux_llvm_blacklist create mode 100644 dependencyPacker/linux_llvm_whitelist create mode 100644 dependencyPacker/macos_llvm_whitelist create mode 100644 dependencyPacker/mingw_llvm_blacklist create mode 100644 dependencyPacker/mingw_llvm_whitelist diff --git a/dependencyPacker/build.gradle.kts b/dependencyPacker/build.gradle.kts new file mode 100644 index 00000000000..ae00d0679e3 --- /dev/null +++ b/dependencyPacker/build.gradle.kts @@ -0,0 +1,92 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the LICENSE file. + */ +import org.jetbrains.kotlin.konan.properties.KonanPropertiesLoader +import org.jetbrains.kotlin.konan.util.DependencyProcessor +import org.jetbrains.kotlin.konan.target.KonanTarget +import org.jetbrains.kotlin.konan.target.Distribution +import org.jetbrains.kotlin.konan.target.presetName + +/** + * Creates an archive with LLVM distribution without files + * that are not required for Kotlin/Native. + * + * For example, it excludes binaries that are part of LLVM testing infrastructure. + */ +inline fun createLlvmPackingTask( + host: KonanTarget, + whitelist: File?, + blacklist: File?, + crossinline additionalConfiguration: T.() -> Unit +) { + val distribution = Distribution( + konanHomeOverride = rootProject.projectDir.absolutePath + ) + val reducedLlvmAppendix = distribution.properties + .getProperty("reducedLlvmAppendix") + val propertiesLoader = object : KonanPropertiesLoader( + host = host, + target = host, + properties = distribution.properties, + baseDir = DependencyProcessor.defaultDependenciesRoot.absolutePath + ) {} + + val hostName = host.presetName.capitalize() + val downloadTaskName = "downloadLlvmFor$hostName" + + tasks.create(downloadTaskName) { + doFirst { + propertiesLoader.downloadDependencies() + } + } + tasks.create("packLlvmFor$hostName") { + dependsOn(downloadTaskName) + additionalConfiguration() + + description = "Packs LLVM dependency for $hostName into archive" + group = "Distribution packing" + + // > When both inclusion and exclusion are used, + // > only files/directories that match at least one of the include patterns + // > and don't match any of the exclude patterns are used. + // + // See: https://ant.apache.org/manual/dirtasks.html + whitelist?.let { + it.readLines() + .filter { !it.startsWith("#") && it.isNotBlank() } + .forEach(this::include) + } + blacklist?.let { + it.readLines() + .filter { !it.startsWith("#") && it.isNotBlank() } + .forEach(this::exclude) + } + from(propertiesLoader.absoluteLlvmHome) + destinationDirectory.set(buildDir.resolve("artifacts")) + archiveBaseName.set(propertiesLoader.llvmHome) + archiveAppendix.set(reducedLlvmAppendix) + } +} + +createLlvmPackingTask( + KonanTarget.MACOS_X64, + whitelist = file("macos_llvm_whitelist"), + blacklist = null +) { + compression = Compression.GZIP +} + +createLlvmPackingTask( + KonanTarget.LINUX_X64, + whitelist = file("linux_llvm_whitelist"), + blacklist = file("linux_llvm_blacklist") +) { + compression = Compression.GZIP +} + +createLlvmPackingTask( + KonanTarget.MINGW_X64, + whitelist = file("mingw_llvm_whitelist"), + blacklist = file("mingw_llvm_blacklist") +) {} \ No newline at end of file diff --git a/dependencyPacker/linux_llvm_blacklist b/dependencyPacker/linux_llvm_blacklist new file mode 100644 index 00000000000..130477178c4 --- /dev/null +++ b/dependencyPacker/linux_llvm_blacklist @@ -0,0 +1,6 @@ +# Unused large files +lib/liblldb.so +lib/liblldb.so.8 +lib/libLTO.so +lib/libLTO.so.8 +lib/python2.7/site-packages/lldb/_lldb.so \ No newline at end of file diff --git a/dependencyPacker/linux_llvm_whitelist b/dependencyPacker/linux_llvm_whitelist new file mode 100644 index 00000000000..ffa1596665d --- /dev/null +++ b/dependencyPacker/linux_llvm_whitelist @@ -0,0 +1,28 @@ +# Compilation process +bin/clang +bin/clang++ +bin/clang-8 + +# Linkage +bin/dsymutil +bin/lld +bin/ld.lld +bin/wasm-ld +bin/llvm-ar + +# Useful utility +bin/llvm-dis +bin/c-index-test +bin/llvm-config + +# Coverage support +bin/llvm-cov +bin/llvm-profdata + +# Used to link runtime files +bin/llvm-link + +include/** +lib/** +libexec/** +share/** \ No newline at end of file diff --git a/dependencyPacker/macos_llvm_whitelist b/dependencyPacker/macos_llvm_whitelist new file mode 100644 index 00000000000..c7ec28a676a --- /dev/null +++ b/dependencyPacker/macos_llvm_whitelist @@ -0,0 +1,29 @@ +# Compilation process +bin/clang +bin/clang++ +bin/clang-8 + +# Linkage +bin/dsymutil +bin/lld +bin/ld.lld +bin/wasm-ld +bin/llvm-ar + +# Useful utility +bin/llvm-dis +bin/c-index-test +bin/llvm-config + +# Coverage support +bin/llvm-cov +bin/llvm-profdata + +# Used to link runtime files +bin/llvm-link + +include/** +lib/** +libexec/** +local/** +share/** \ No newline at end of file diff --git a/dependencyPacker/mingw_llvm_blacklist b/dependencyPacker/mingw_llvm_blacklist new file mode 100644 index 00000000000..ed0a4146b34 --- /dev/null +++ b/dependencyPacker/mingw_llvm_blacklist @@ -0,0 +1,5 @@ +lib/gcc/x86_64-w64-mingw32/9.2.0/cc1plus.exe +lib/gcc/x86_64-w64-mingw32/9.2.0/plugin/cc1.exe.a +lib/gcc/x86_64-w64-mingw32/9.2.0/plugin/cc1plus.exe.a +lib/gcc/x86_64-w64-mingw32/9.2.0/lto1.exe +lib/gcc/x86_64-w64-mingw32/9.2.0/cc1.exe diff --git a/dependencyPacker/mingw_llvm_whitelist b/dependencyPacker/mingw_llvm_whitelist new file mode 100644 index 00000000000..2d74d888343 --- /dev/null +++ b/dependencyPacker/mingw_llvm_whitelist @@ -0,0 +1,39 @@ +# Compilation process +bin/clang.exe +bin/clang++.exe +bin/clang-8.exe + +# Linkage +bin/lld.exe +bin/ld.lld.exe +bin/wasm-ld.exe +bin/llvm-ar.exe + +# Useful utility +bin/llvm-dis.exe +bin/llvm-config.exe + +# Coverage support +bin/llvm-cov.exe +bin/llvm-profdata.exe + +# Used to link runtime files +bin/llvm-link.exe + +# Used for samples +bin/windres.exe + +# Used by other binaries +bin/zlib1.dll +bin/libwinpthread-1.dll +bin/libgcc_s_seh-1.dll +bin/libstdc++-6.dll +bin/libz3.dll + +include/** +lib/** +etc/** +local/** +share/** +x86_64-w64-mingw32/** +libclang.dll \ No newline at end of file diff --git a/konan/konan.properties b/konan/konan.properties index d5be938af26..7b25f5b0ec7 100644 --- a/konan/konan.properties +++ b/konan/konan.properties @@ -32,6 +32,9 @@ downloadingAttempts = 10 downloadingAttemptIntervalMs = 3000 homeDependencyCache = .konan/cache +# Appendix that is used for smaller version of LLVM distributions. +reducedLlvmAppendix = compact + clangDebugFlags = -O0 llvmVersion.linux_x64 = 8.0.0 llvmVersion.mingw_x64 = 8.0.1 diff --git a/settings.gradle b/settings.gradle index 54a9affe879..149190137a6 100644 --- a/settings.gradle +++ b/settings.gradle @@ -30,6 +30,7 @@ include ':common' include ':backend.native:tests' include ':backend.native:debugger-tests' include ':utilities' +include 'dependencyPacker' include ':performance' include ':performance:ring' @@ -73,3 +74,4 @@ includeBuild 'build-tools' includeBuild 'extracted/konan.metadata' includeBuild 'extracted/konan.serializer' includeBuild 'tools/kotlin-native-gradle-plugin' +