Add task that makes LLVM dependency smaller

Currently Kotlin/Native uses default LLVM distribution.
While it is useful to have whole LLVM toolchain for development,
it is huge pain for users, because dependency is needlessly big.

The correct solution for this problem is to make own build script for LLVM,
but it is a big task that make take a while.

So for now we repack LLVM dependency without unnecessary files.

dependencyPacker/build.gradle.kts

@@ -0,0 +1,92 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+import org.jetbrains.kotlin.konan.properties.KonanPropertiesLoader
+import org.jetbrains.kotlin.konan.util.DependencyProcessor
+import org.jetbrains.kotlin.konan.target.KonanTarget
+import org.jetbrains.kotlin.konan.target.Distribution
+import org.jetbrains.kotlin.konan.target.presetName
+
+/**
+ * Creates an archive with LLVM distribution without files
+ * that are not required for Kotlin/Native.
+ *
+ * For example, it excludes binaries that are part of LLVM testing infrastructure.
+ */
+inline fun <reified T: AbstractArchiveTask> createLlvmPackingTask(
+        host: KonanTarget,
+        whitelist: File?,
+        blacklist: File?,
+        crossinline additionalConfiguration: T.() -> Unit
+) {
+    val distribution = Distribution(
+            konanHomeOverride = rootProject.projectDir.absolutePath
+    )
+    val reducedLlvmAppendix = distribution.properties
+            .getProperty("reducedLlvmAppendix")
+    val propertiesLoader = object : KonanPropertiesLoader(
+            host = host,
+            target = host,
+            properties = distribution.properties,
+            baseDir = DependencyProcessor.defaultDependenciesRoot.absolutePath
+    ) {}
+
+    val hostName = host.presetName.capitalize()
+    val downloadTaskName = "downloadLlvmFor$hostName"
+
+    tasks.create(downloadTaskName) {
+        doFirst {
+            propertiesLoader.downloadDependencies()
+        }
+    }
+    tasks.create<T>("packLlvmFor$hostName") {
+        dependsOn(downloadTaskName)
+        additionalConfiguration()
+
+        description = "Packs LLVM dependency for $hostName into archive"
+        group = "Distribution packing"
+
+        // > When both inclusion and exclusion are used,
+        // > only files/directories that match at least one of the include patterns
+        // > and don't match any of the exclude patterns are used.
+        //
+        // See: https://ant.apache.org/manual/dirtasks.html
+        whitelist?.let {
+            it.readLines()
+                    .filter { !it.startsWith("#") && it.isNotBlank() }
+                    .forEach(this::include)
+        }
+        blacklist?.let {
+            it.readLines()
+                    .filter { !it.startsWith("#") && it.isNotBlank() }
+                    .forEach(this::exclude)
+        }
+        from(propertiesLoader.absoluteLlvmHome)
+        destinationDirectory.set(buildDir.resolve("artifacts"))
+        archiveBaseName.set(propertiesLoader.llvmHome)
+        archiveAppendix.set(reducedLlvmAppendix)
+    }
+}
+
+createLlvmPackingTask<Tar>(
+        KonanTarget.MACOS_X64,
+        whitelist = file("macos_llvm_whitelist"),
+        blacklist = null
+) {
+    compression = Compression.GZIP
+}
+
+createLlvmPackingTask<Tar>(
+        KonanTarget.LINUX_X64,
+        whitelist = file("linux_llvm_whitelist"),
+        blacklist = file("linux_llvm_blacklist")
+) {
+    compression = Compression.GZIP
+}
+
+createLlvmPackingTask<Zip>(
+        KonanTarget.MINGW_X64,
+        whitelist = file("mingw_llvm_whitelist"),
+        blacklist = file("mingw_llvm_blacklist")
+) {}

dependencyPacker/linux_llvm_blacklist

@@ -0,0 +1,6 @@
+# Unused large files
+lib/liblldb.so
+lib/liblldb.so.8
+lib/libLTO.so
+lib/libLTO.so.8
+lib/python2.7/site-packages/lldb/_lldb.so

dependencyPacker/linux_llvm_whitelist

@@ -0,0 +1,28 @@
+# Compilation process
+bin/clang
+bin/clang++
+bin/clang-8
+
+# Linkage
+bin/dsymutil
+bin/lld
+bin/ld.lld
+bin/wasm-ld
+bin/llvm-ar
+
+# Useful utility
+bin/llvm-dis
+bin/c-index-test
+bin/llvm-config
+
+# Coverage support
+bin/llvm-cov
+bin/llvm-profdata
+
+# Used to link runtime files
+bin/llvm-link
+
+include/**
+lib/**
+libexec/**
+share/**

dependencyPacker/macos_llvm_whitelist

@@ -0,0 +1,29 @@
+# Compilation process
+bin/clang
+bin/clang++
+bin/clang-8
+
+# Linkage
+bin/dsymutil
+bin/lld
+bin/ld.lld
+bin/wasm-ld
+bin/llvm-ar
+
+# Useful utility
+bin/llvm-dis
+bin/c-index-test
+bin/llvm-config
+
+# Coverage support
+bin/llvm-cov
+bin/llvm-profdata
+
+# Used to link runtime files
+bin/llvm-link
+
+include/**
+lib/**
+libexec/**
+local/**
+share/**

dependencyPacker/mingw_llvm_blacklist

@@ -0,0 +1,5 @@
+lib/gcc/x86_64-w64-mingw32/9.2.0/cc1plus.exe
+lib/gcc/x86_64-w64-mingw32/9.2.0/plugin/cc1.exe.a
+lib/gcc/x86_64-w64-mingw32/9.2.0/plugin/cc1plus.exe.a
+lib/gcc/x86_64-w64-mingw32/9.2.0/lto1.exe
+lib/gcc/x86_64-w64-mingw32/9.2.0/cc1.exe

dependencyPacker/mingw_llvm_whitelist

@@ -0,0 +1,39 @@
+# Compilation process
+bin/clang.exe
+bin/clang++.exe
+bin/clang-8.exe
+
+# Linkage
+bin/lld.exe
+bin/ld.lld.exe
+bin/wasm-ld.exe
+bin/llvm-ar.exe
+
+# Useful utility
+bin/llvm-dis.exe
+bin/llvm-config.exe
+
+# Coverage support
+bin/llvm-cov.exe
+bin/llvm-profdata.exe
+
+# Used to link runtime files
+bin/llvm-link.exe
+
+# Used for samples
+bin/windres.exe
+
+# Used by other binaries
+bin/zlib1.dll
+bin/libwinpthread-1.dll
+bin/libgcc_s_seh-1.dll
+bin/libstdc++-6.dll
+bin/libz3.dll
+
+include/**
+lib/**
+etc/**
+local/**
+share/**
+x86_64-w64-mingw32/**
+libclang.dll

konan/konan.properties

@@ -32,6 +32,9 @@ downloadingAttempts = 10
 downloadingAttemptIntervalMs = 3000
 homeDependencyCache = .konan/cache
 
+# Appendix that is used for smaller version of LLVM distributions.
+reducedLlvmAppendix = compact
+
 clangDebugFlags = -O0
 llvmVersion.linux_x64 = 8.0.0
 llvmVersion.mingw_x64 = 8.0.1

settings.gradle

@@ -30,6 +30,7 @@ include ':common'
 include ':backend.native:tests'
 include ':backend.native:debugger-tests'
 include ':utilities'
+include 'dependencyPacker'
 
 include ':performance'
 include ':performance:ring'
@@ -73,3 +74,4 @@ includeBuild 'build-tools'
 includeBuild 'extracted/konan.metadata'
 includeBuild 'extracted/konan.serializer'
 includeBuild 'tools/kotlin-native-gradle-plugin'
+