From 75215fcb97b408ae84aa71fbd70bf75c7a79dded Mon Sep 17 00:00:00 2001 From: Sergey Bogolepov Date: Mon, 9 Aug 2021 11:37:07 +0700 Subject: [PATCH] [K/N] Option to build smaller LLVM distribution. Also add a few adjustments to `package.py`: * Enable threading in LLVM * Replace `--archive-path` with `--pack` to avoid naming divergence. --- kotlin-native/tools/llvm_builder/README.md | 26 +++++++-- kotlin-native/tools/llvm_builder/package.py | 65 +++++++++++++-------- 2 files changed, 62 insertions(+), 29 deletions(-) diff --git a/kotlin-native/tools/llvm_builder/README.md b/kotlin-native/tools/llvm_builder/README.md index ecea878cae5..fce0691bfc8 100644 --- a/kotlin-native/tools/llvm_builder/README.md +++ b/kotlin-native/tools/llvm_builder/README.md @@ -18,24 +18,42 @@ python3 package.py ``` It will create a `llvm-distribution` folder which contains LLVM distribution. Note that build will take a lot of time and all your machine cores. 😈 -See [HACKING.md](../../HACKING.md#using-different-llvm-distributions-as-part-of-kotlinnative-compilation-pipeline) on how to use freshly built distribution. +See [HACKING.md](../../HACKING.md#using-different-llvm-distributions-as-part-of-kotlinnative-compilation-pipeline) +on how to use freshly built distribution. + +### Building distribution for end-users +By default, `package.py` installs a lot of LLVM components that is not really required to use Kotlin/Native compiler. +To build only essential parts, run the following command: + +``` +python3 package.py --build-targets install-distribution --distribution-components $DISTRIBUTION_COMPONENTS +``` +Set of required `$DISTRIBUTION_COMPONENTS` is depends on OS: + +| OS | Distribution components | +| --- | --- | +|Windows | `clang libclang lld llvm-cov llvm-profdata llvm-ar clang-resource-headers` | +|macOS | `clang libclang lld llvm-cov llvm-profdata llvm-ar clang-resource-headers` | +|Linux | `clang libclang lld llvm-cov llvm-profdata llvm-ar clang-resource-headers compiler_rt` | ### Tuning Run `python3 package.py -h` to check how one can tune script's behavior. Some examples: -* `--archive-path` creates an archive (`zip` on Windows, `tar.gz` on macOS and Linux) with distribution +* `--pack` creates an archive (`zip` on Windows, `tar.gz` on macOS and Linux) with distribution alongside with its SHA256. * `--install-path` allows overriding distribution's output directory. * `--num-stages` specifies number of steps in build. Passing 2 or more makes bootstrap build which means that LLVM will build itself by using distribution from the previous step. * `--stage0` allows using existing LLVM toolchain for bootstrapping. +* `--build-targets` specifies what targets will be passed to Ninja. +* `--distribution-components` is a list of components that will be installed with `install-distribution` build target. ### Docker You can use Docker to build LLVM for Linux: ```shell docker build -t kotlin-llvm-builder . -docker run --rm -it -v :/output kotlin-llvm-builder --archive-path /output/llvm-11.1.0-linux-x64 +docker run --rm -it -v :/output kotlin-llvm-builder --install-path /output/llvm-11.1.0-linux-x64 --pack ``` @@ -43,5 +61,3 @@ docker run --rm -it -v :/output kotlin-llvm-builder --archive-path /o 1. Bootstrap build is not working on macOS for default git branch. 2. `libcxx` and `compiler-rt` are built as projects, not runtimes. 3. No way to run LLVM tests out of the box. -4. No way to override default `install` task out of the box. - diff --git a/kotlin-native/tools/llvm_builder/package.py b/kotlin-native/tools/llvm_builder/package.py index 17368d07780..f32bf051569 100644 --- a/kotlin-native/tools/llvm_builder/package.py +++ b/kotlin-native/tools/llvm_builder/package.py @@ -92,13 +92,31 @@ def construct_cmake_flags( install_path: str = None, projects: List[str] = None, runtimes: List[str] = None, - targets: List[str] = None + targets: List[str] = None, + distribution_components: List[str] = None ) -> List[str]: building_bootstrap = bootstrap_llvm_path is None c_compiler, cxx_compiler, linker, ar = None, None, None, None c_flags, cxx_flags, linker_flags = None, None, None + + cmake_args = [ + '-DCMAKE_BUILD_TYPE=Release', + '-DLLVM_ENABLE_ASSERTIONS=OFF', + '-DLLVM_ENABLE_TERMINFO=OFF', + '-DLLVM_INCLUDE_GO_TESTS=OFF', + '-DLLVM_ENABLE_Z3_SOLVER=OFF', + '-DCOMPILER_RT_BUILD_BUILTINS=ON', + '-DLLVM_ENABLE_THREADS=ON', + '-DLLVM_OPTIMIZED_TABLEGEN=ON' + ] if not building_bootstrap: + if distribution_components: + cmake_args.append('-DLLVM_DISTRIBUTION_COMPONENTS=' + ';'.join(distribution_components)) + # These links are actually copies on windows, so they're wasting precious disk space. + cmake_args.append("-DCLANG_LINKS_TO_CREATE=clang++") + cmake_args.append("-DLLD_SYMLINKS_TO_CREATE=ld.lld;wasm-ld") + if host_is_windows(): # CMake is not tolerant to backslashes c_compiler = f'{bootstrap_llvm_path}/bin/clang-cl.exe'.replace('\\', '/') @@ -120,14 +138,6 @@ def construct_cmake_flags( cxx_flags = ['-isysroot', isysroot, '-stdlib=libc++'] linker_flags = ['-stdlib=libc++'] - cmake_args = [ - '-DCMAKE_BUILD_TYPE=Release', - '-DLLVM_ENABLE_ASSERTIONS=OFF', - '-DLLVM_ENABLE_TERMINFO=OFF', - '-DLLVM_INCLUDE_GO_TESTS=OFF', - '-DLLVM_ENABLE_Z3_SOLVER=OFF', - '-DCOMPILER_RT_BUILD_BUILTINS=ON' - ] if host_is_darwin(): cmake_args.append('-DLLVM_ENABLE_LIBCXX=ON') @@ -186,7 +196,7 @@ def construct_cmake_flags( # Also not working for Linux and macOS because of signal chaining. # TODO: Enable after LLVM distribution patching. if not host_is_windows(): - cmake_args.append("-LLVM_BUILD_LLVM_DYLIB=OFF") + cmake_args.append("-DLLVM_BUILD_LLVM_DYLIB=OFF") cmake_args.append("-DLLVM_LINK_LLVM_DYLIB=OFF") return cmake_args @@ -221,11 +231,11 @@ def force_create_directory(parent, name) -> Path: def llvm_build_commands( - install_path, bootstrap_path, llvm_src, targets, ninja_target, projects, runtimes + install_path, bootstrap_path, llvm_src, targets, build_targets, projects, runtimes, distribution_components ) -> List[List[str]]: - cmake_flags = construct_cmake_flags(bootstrap_path, install_path, projects, runtimes, targets) + cmake_flags = construct_cmake_flags(bootstrap_path, install_path, projects, runtimes, targets, distribution_components) cmake_command = [cmake, "-G", "Ninja"] + cmake_flags + [os.path.join(llvm_src, "llvm")] - ninja_command = [ninja, ninja_target] + ninja_command = [ninja] + build_targets return [cmake_command, ninja_command] @@ -250,12 +260,18 @@ def default_num_stages(): def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="Build LLVM toochain for Kotlin/Native") + parser = argparse.ArgumentParser(description="Build LLVM toolchain for Kotlin/Native") # Output configuration. parser.add_argument("--install-path", type=str, default="llvm-distribution", required=False, - help="Where final LLVM distribution will be installed") - parser.add_argument("--archive-path", default=None, - help="Create an archive and its sha256 for final distribution at given path") + help="Where final LLVM distribution will be enabled") + parser.add_argument("--pack", action='store_true', + help="Create an archive and its sha256 for final distribution at `--install-path`") + parser.add_argument("--build-targets", default=["install"], + nargs="+", + help="What components should be installed") + parser.add_argument("--distribution-components", default=None, + nargs="+", + help="What components should be installed with `install-distribution` target") # Build configuration parser.add_argument("--stage0", type=str, default=None, help="Path to existing LLVM toolchain") @@ -280,7 +296,7 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--isysroot", type=str, default=None, help="(macOS only) Override path to macOS SDK") # Misc. - parser.add_argument("--save-temporary-files", type=bool, default=False, + parser.add_argument("--save-temporary-files", action='store_true', help="Should intermediate build results be saved?") return parser @@ -312,16 +328,16 @@ def build_distribution(args): else: # None targets means all available targets. targets = None - if building_final: install_path = args.install_path + build_targets = args.build_targets else: install_path = force_create_directory(current_dir, f"llvm-stage-{stage}") intermediate_build_results.append(install_path) + build_targets = ["install"] projects = ["clang", "lld", "libcxx", "libcxxabi", "compiler-rt"] runtimes = None - ninja_target = "install" build_dir = force_create_directory(current_dir, f"llvm-stage-{stage}-build") intermediate_build_results.append(build_dir) @@ -330,9 +346,10 @@ def build_distribution(args): bootstrap_path=absolute_path(bootstrap_path), llvm_src=absolute_path(args.llvm_src), targets=targets, - ninja_target=ninja_target, + build_targets=build_targets, projects=projects, - runtimes=runtimes + runtimes=runtimes, + distribution_components=args.distribution_components ) os.chdir(build_dir) @@ -404,8 +421,8 @@ def main(): temporary_llvm_repo = clone_llvm_repository(args.repo, args.branch, args.llvm_repo_destination) args.llvm_src = temporary_llvm_repo final_dist = build_distribution(args) - if args.archive_path is not None: - archive = create_archive(final_dist, args.archive_path) + if args.pack: + archive = create_archive(final_dist, args.install_path) create_checksum_file(archive, f"{archive}.sha256") if not args.save_temporary_files and temporary_llvm_repo is not None: print(f"Removing temporary directory: {temporary_llvm_repo}")