diff --git a/kotlin-native/tools/llvm_builder/README.md b/kotlin-native/tools/llvm_builder/README.md new file mode 100644 index 00000000000..da5ac9e43c5 --- /dev/null +++ b/kotlin-native/tools/llvm_builder/README.md @@ -0,0 +1,38 @@ +# LLVM Build Infrastructure + +[package.py](package.py) script automates downloading, building and packing of LLVM distribution used by Kotlin/Native. +### Requirements +* Check [LLVM requirements](https://llvm.org/docs/GettingStarted.html#requirements). Note that: + * Visual Studio 2017 is required on Windows + * Xcode is required on macOS +* [Python 3](https://www.python.org/) +* [ninja](https://ninja-build.org/) +* [git](https://git-scm.com/) +* [CMake](https://cmake.org/) + +### Usage + +Just run +``` +python3 package.py +``` +It will create a `llvm-distribution` folder which contains LLVM distribution. +Note that build will take a lot of time and all your machine cores. 😈 +See [HACKING.md](../../HACKING.md#using-different-llvm-distributions-as-part-of-kotlinnative-compilation-pipeline) on how to use freshly built distribution. + +### Tuning +Run `python3 package.py -h` to check how one can tune script's behavior. +Some examples: +* `--archive-path` creates an archive (`zip` on Windows, `tar.gz` on macOS and Linux) with distribution + alongside with its SHA256. +* `--install-path` allows overriding distribution's output directory. +* `--num-stages` specifies number of steps in build. Passing 2 or more makes bootstrap build which + means that LLVM will build itself by using distribution from the previous step. +* `--stage0` allows using existing LLVM toolchain for bootstrapping. + +### Known problems +1. Bootstrap build is not working on macOS for default git branch. +2. `libcxx` and `compiler-rt` are built as projects, not runtimes. +3. No way to run LLVM tests out of the box. +4. No way to override default `install` task out of the box. + diff --git a/kotlin-native/tools/llvm_builder/package.py b/kotlin-native/tools/llvm_builder/package.py new file mode 100644 index 00000000000..fa235d8bff3 --- /dev/null +++ b/kotlin-native/tools/llvm_builder/package.py @@ -0,0 +1,420 @@ +#!/usr/bin/python3 +# +# Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + +import argparse +import hashlib +import os +import shlex +import shutil +import subprocess +import sys +import urllib.request +from pathlib import Path +from typing import List + +vsdevcmd = None +isysroot = None + +ninja = 'ninja' +cmake = 'cmake' +git = 'git' + + +def absolute_path(path): + if path is not None: + # CMake is not tolerant to backslashes in path. + return os.path.abspath(path).replace('\\', '/') + else: + return None + + +def host_is_windows(): + return sys.platform == "win32" + + +def host_is_linux(): + return sys.platform == "linux" + + +def host_is_darwin(): + return sys.platform == "darwin" + + +def host_llvm_target(): + return "Native" + + +def host_default_compression(): + """ + Determine archive compression method based on current OS. + On Windows we use `zip` and `tar.gz` otherwise. + """ + if host_is_windows(): + return "zip" + else: + return "gztar" + + +def detect_xcode_sdk_path(): + """ + Get an absolute path to macOS SDK. + """ + return subprocess.check_output(['xcrun', '--show-sdk-path'], + universal_newlines=True).rstrip() + + +def detect_vsdevcmd(): + """ + Use vswhere (and download it, if needed) utility to find path to vsdevcmd.bat. + :return: path to vsdevcmd.bat + """ + vswhere = shutil.which('vswhere') + if vswhere is None: + print("Downloading vswhere utility to detect path to vsdevcmd.bat automatically") + vswhere_url = "https://github.com/microsoft/vswhere/releases/download/2.8.4/vswhere.exe" + urllib.request.urlretrieve(vswhere_url, 'vswhere.exe') + vswhere = shutil.which('vswhere') + if vswhere is None: + sys.exit("Failed to retrieve vswhere utility. Please provide path to vsdevcmd.bat with --vsdevcmd") + vswhere_args = [vswhere, '-prerelease', '-latest', '-property', 'installationPath'] + path_to_visual_studio = subprocess.check_output(vswhere_args, universal_newlines=True).rstrip() + vsdevcmd_path = os.path.join(path_to_visual_studio, "Common7", "Tools", "vsdevcmd.bat") + if not os.path.isfile(vsdevcmd_path): + sys.exit("vsdevcmd.bat is not found. Please provide path to vsdevcmd.bat with --vsdevcmd") + else: + print("Found vsdevcmd.bat: " + vsdevcmd_path) + return vsdevcmd_path + + +def construct_cmake_flags( + bootstrap_llvm_path: str = None, + install_path: str = None, + projects: List[str] = None, + runtimes: List[str] = None, + targets: List[str] = None +) -> List[str]: + building_bootstrap = bootstrap_llvm_path is None + + c_compiler, cxx_compiler, linker, ar = None, None, None, None + c_flags, cxx_flags, linker_flags = None, None, None + if not building_bootstrap: + if host_is_windows(): + # CMake is not tolerant to backslashes + c_compiler = f'{bootstrap_llvm_path}/bin/clang-cl.exe'.replace('\\', '/') + cxx_compiler = f'{bootstrap_llvm_path}/bin/clang-cl.exe'.replace('\\', '/') + linker = f'{bootstrap_llvm_path}/bin/lld-link.exe'.replace('\\', '/') + ar = f'{bootstrap_llvm_path}/bin/llvm-lib.exe'.replace('\\', '/') + elif host_is_linux(): + c_compiler = f'{bootstrap_llvm_path}/bin/clang' + cxx_compiler = f'{bootstrap_llvm_path}/bin/clang++' + linker = f'{bootstrap_llvm_path}/bin/ld.lld' + ar = f'{bootstrap_llvm_path}/bin/llvm-ar' + elif host_is_darwin(): + c_compiler = f'{bootstrap_llvm_path}/bin/clang' + cxx_compiler = f'{bootstrap_llvm_path}/bin/clang++' + # ld64.lld is not that good yet. + linker = None + ar = f'{bootstrap_llvm_path}/bin/llvm-ar' + c_flags = ['-isysroot', isysroot] + cxx_flags = ['-isysroot', isysroot, '-stdlib=libc++'] + linker_flags = ['-stdlib=libc++'] + + cmake_args = [ + '-DCMAKE_BUILD_TYPE=Release', + '-DLLVM_ENABLE_ASSERTIONS=OFF', + '-DLLVM_ENABLE_TERMINFO=OFF', + '-DLLVM_INCLUDE_GO_TESTS=OFF', + '-DLLVM_ENABLE_Z3_SOLVER=OFF', + '-DCOMPILER_RT_BUILD_BUILTINS=ON' + ] + + if host_is_darwin(): + cmake_args.append('-DLLVM_ENABLE_LIBCXX=ON') + if building_bootstrap: + # Don't waste time by doing unnecessary work for throwaway toolchain. + cmake_args.extend([ + '-DCOMPILER_RT_BUILD_CRT=OFF', + '-DCOMPILER_RT_BUILD_LIBFUZZER=OFF', + '-DCOMPILER_RT_BUILD_SANITIZERS=OFF', + '-DCOMPILER_RT_BUILD_XRAY=OFF', + '-DCOMPILER_RT_ENABLE_IOS=OFF', + '-DCOMPILER_RT_ENABLE_WATCHOS=OFF', + '-DCOMPILER_RT_ENABLE_TVOS=OFF', + ]) + else: + cmake_args.append('-DLIBCXX_USE_COMPILER_RT=ON') + + if install_path is not None: + cmake_args.append('-DCMAKE_INSTALL_PREFIX=' + install_path) + if targets is not None: + cmake_args.append('-DLLVM_TARGETS_TO_BUILD=' + ";".join(targets)) + if projects is not None: + cmake_args.append('-DLLVM_ENABLE_PROJECTS=' + ";".join(projects)) + if runtimes is not None: + cmake_args.append('-DLLVM_ENABLE_RUNTIMES=' + ";".join(runtimes)) + if c_compiler is not None: + cmake_args.append('-DCMAKE_C_COMPILER=' + c_compiler) + if cxx_compiler is not None: + cmake_args.append('-DCMAKE_CXX_COMPILER=' + cxx_compiler) + if linker is not None: + cmake_args.append('-DCMAKE_LINKER=' + linker) + if ar is not None: + cmake_args.append('-DCMAKE_AR=' + ar) + + if c_flags is not None: + cmake_args.append("-DCMAKE_C_FLAGS=" + ' '.join(c_flags)) + if cxx_flags is not None: + cmake_args.append("-DCMAKE_CXX_FLAGS=" + ' '.join(cxx_flags)) + if linker_flags is not None: + cmake_args.append('-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(linker_flags)) + cmake_args.append('-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(linker_flags)) + cmake_args.append('-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(linker_flags)) + + if host_is_windows(): + # Use MT to make distribution self-contained + # TODO: Consider -DCMAKE_INSTALL_UCRT_LIBRARIES=ON as an alternative + cmake_args.append('-DLLVM_USE_CRT_RELEASE=MT') + cmake_args.append('-DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded') + # We don't support PDB, so no need fir DIA. + cmake_args.append('-DLLVM_ENABLE_DIA_SDK=OFF') + + # Make distribution much smaller by linking to dynamic library + # instead of static linkage. + # Not working for Windows yet. + # + # Also not working for Linux and macOS because of signal chaining. + # TODO: Enable after LLVM distribution patching. + if not host_is_windows(): + cmake_args.append("-LLVM_BUILD_LLVM_DYLIB=OFF") + cmake_args.append("-DLLVM_LINK_LLVM_DYLIB=OFF") + + return cmake_args + + +def run_command(command: List[str]): + """ + Execute single command in terminal/cmd. + + Note that on Windows we prepare environment with vsdevcmd.bat. + """ + if host_is_windows(): + if vsdevcmd is None: + sys.exit("'VsDevCmd.bat' is not set!") + command = [vsdevcmd, "-arch=amd64", "&&"] + command + print("Running command: " + ' '.join(command)) + else: + command = [shlex.quote(arg) for arg in command] + command = ' '.join(command) + print("Running command: " + command) + + subprocess.run(command, shell=True, check=True) + + +def force_create_directory(parent, name) -> Path: + build_path = parent / name + print(f"Force-creating directory {build_path}") + if build_path.exists(): + shutil.rmtree(build_path) + os.mkdir(build_path) + return build_path + + +def llvm_build_commands( + install_path, bootstrap_path, llvm_src, targets, ninja_target, projects, runtimes +) -> List[List[str]]: + cmake_flags = construct_cmake_flags(bootstrap_path, install_path, projects, runtimes, targets) + cmake_command = [cmake, "-G", "Ninja"] + cmake_flags + [os.path.join(llvm_src, "llvm")] + ninja_command = [ninja, ninja_target] + return [cmake_command, ninja_command] + + +def clone_llvm_repository(repo, branch, llvm_repo_destination): + """ + Downloads a single commit from the given repository. + """ + if host_is_darwin(): + default_repo, default_branch = "https://github.com/apple/llvm-project", "apple/stable/20200108" + else: + default_repo, default_branch = "https://github.com/llvm/llvm-project", "release/11.x" + repo = default_repo if repo is None else repo + branch = default_branch if branch is None else branch + # Download only single commit because we don't need whole history just for building LLVM. + run_command([git, "clone", repo, "--branch", branch, "--depth", "1", "llvm-project"]) + return absolute_path(llvm_repo_destination) + + +def default_num_stages(): + if host_is_darwin(): + # Bootstrap build is not working for https://github.com/apple/llvm-project + apple/stable/20200108. + # So use single phase for now. + return 1 + else: + return 2 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Build LLVM toochain for Kotlin/Native") + # Output configuration. + parser.add_argument("--install-path", type=str, default="llvm-distribution", required=False, + help="Where final LLVM distribution will be installed") + parser.add_argument("--archive-path", default=None, + help="Create an archive and its sha256 for final distribution at given path") + # Build configuration + parser.add_argument("--stage0", type=str, default=None, + help="Path to existing LLVM toolchain") + parser.add_argument("--num-stages", type=int, default=default_num_stages(), + help="Number of stages in bootstrap.") + # LLVM sources. + parser.add_argument("--llvm-sources", dest="llvm_src", type=str, default=None, + help="Location of LLVM sources") + parser.add_argument("--repo", type=str, default=None) + parser.add_argument("--branch", type=str, default=None) + parser.add_argument("--llvm-repo-destination", type=str, default="llvm-project", + help="Where LLVM repository should be downloaded.") + # Environment setup. + parser.add_argument("--vsdevcmd", type=str, default=None, + help="(Windows only) Path to VsDevCmd.bat") + parser.add_argument("--ninja", type=str, default=None, + help="Override path to ninja") + parser.add_argument("--cmake", type=str, default=None, + help="Override path to cmake") + parser.add_argument("--git", type=str, default=None, + help="Override path to git") + parser.add_argument("--isysroot", type=str, default=None, + help="(macOS only) Override path to macOS SDK") + # Misc. + parser.add_argument("--save-temporary-files", type=bool, default=False, + help="Should intermediate build results be saved?") + return parser + + +def build_distribution(args): + """ + Performs (probably multistage) build of LLVM + and returns path to the final distribution. + """ + current_dir = Path().absolute() + num_stages = args.num_stages + bootstrap_path = args.stage0 + intermediate_build_results = [] + # Most likely, num_stages will be 1 or 2. + # 2 means bootstrap build: we build LLVM distribution (stage 1) + # that then compiles sources once again (stage 2). Thus, resulting + # distribution is (almost) independent from environment, which means + # reproducibility and less bugs. + # + # Sometimes it makes sense to generate yet another distribution to check + # that it is the same as built at stage 2 (so there is no non-determinism in LLVM). + for stage in range(1, num_stages + 1): + building_bootstrap = num_stages > 1 and stage == 1 + building_final = stage == num_stages + + if building_bootstrap: + # We only need a host target to start a bootstrap. + targets = [host_llvm_target()] + else: + # None targets means all available targets. + targets = None + + if building_final: + install_path = args.install_path + else: + install_path = force_create_directory(current_dir, f"llvm-stage-{stage}") + intermediate_build_results.append(install_path) + + projects = ["clang", "lld", "libcxx", "libcxxabi", "compiler-rt"] + runtimes = None + ninja_target = "install" + + build_dir = force_create_directory(current_dir, f"llvm-stage-{stage}-build") + intermediate_build_results.append(build_dir) + commands = llvm_build_commands( + install_path=absolute_path(install_path), + bootstrap_path=absolute_path(bootstrap_path), + llvm_src=absolute_path(args.llvm_src), + targets=targets, + ninja_target=ninja_target, + projects=projects, + runtimes=runtimes + ) + + os.chdir(build_dir) + for command in commands: + run_command(command) + os.chdir(current_dir) + bootstrap_path = install_path + + if not args.save_temporary_files: + for dir in intermediate_build_results: + print(f"Removing temporary directory: {dir}") + shutil.rmtree(dir) + + return absolute_path(args.install_path) + + +def create_archive(input_directory, output_path, compression=host_default_compression()) -> str: + print("Creating archive " + output_path + " from " + input_directory) + base_directory, archive_prefix = os.path.split(os.path.normpath(input_directory)) + return shutil.make_archive(output_path, compression, base_directory, archive_prefix) + + +def create_checksum_file(input_path, output_path): + chunk_size = 4096 + checksum = hashlib.sha256() + with open(input_path, "rb") as input_contents: + for chunk in iter(lambda: input_contents.read(chunk_size), b""): + checksum.update(chunk) + print(checksum.hexdigest(), file=open(output_path, "w")) + return True + + +def setup_environment(args): + """ + Setup globals that store information about script execution environment. + """ + global vsdevcmd, ninja, cmake, git, isysroot + # TODO: We probably can download some of these binaries ourselves. + if args.ninja: + ninja = args.ninja + elif shutil.which('ninja') is None: + sys.exit("'ninja' is not found. Install or provide via --ninja argument.") + if args.cmake: + cmake = args.cmake + elif shutil.which('cmake') is None: + sys.exit("'cmake' is not found. Install or provide via --cmake argument.") + if args.git: + git = args.git + elif shutil.which('git') is None: + sys.exit("'git' is not found. Install or provide via --git argument.") + if host_is_windows(): + if args.vsdevcmd: + vsdevcmd = args.vsdevcmd + else: + vsdevcmd = detect_vsdevcmd() + elif host_is_darwin(): + if args.isysroot: + isysroot = args.isysroot + else: + isysroot = detect_xcode_sdk_path() + + +def main(): + parser = build_parser() + args = parser.parse_args() + setup_environment(args) + temporary_llvm_repo = None + if args.llvm_src is None: + temporary_llvm_repo = clone_llvm_repository(args.repo, args.branch, args.llvm_repo_destination) + args.llvm_src = temporary_llvm_repo + final_dist = build_distribution(args) + if args.archive_path is not None: + archive = create_archive(final_dist, args.archive_path) + create_checksum_file(archive, f"{archive}.sha256") + if not args.save_temporary_files and temporary_llvm_repo is not None: + print(f"Removing temporary directory: {temporary_llvm_repo}") + shutil.rmtree(temporary_llvm_repo) + + +if __name__ == "__main__": + main()