[Native] Add initial version of LLVM buildscript
Previously there was no determined way to build LLVM for Kotlin/Native. This commits adds a Python script that allows to do so on all hosts.
This commit is contained in:
Sergey Bogolepov
@@ -0,0 +1,38 @@
|
||||
# LLVM Build Infrastructure
|
||||
|
||||
[package.py](package.py) script automates downloading, building and packing of LLVM distribution used by Kotlin/Native.
|
||||
### Requirements
|
||||
* Check [LLVM requirements](https://llvm.org/docs/GettingStarted.html#requirements). Note that:
|
||||
* Visual Studio 2017 is required on Windows
|
||||
* Xcode is required on macOS
|
||||
* [Python 3](https://www.python.org/)
|
||||
* [ninja](https://ninja-build.org/)
|
||||
* [git](https://git-scm.com/)
|
||||
* [CMake](https://cmake.org/)
|
||||
|
||||
### Usage
|
||||
|
||||
Just run
|
||||
```
|
||||
python3 package.py
|
||||
```
|
||||
It will create a `llvm-distribution` folder which contains LLVM distribution.
|
||||
Note that build will take a lot of time and all your machine cores. 😈
|
||||
See [HACKING.md](../../HACKING.md#using-different-llvm-distributions-as-part-of-kotlinnative-compilation-pipeline) on how to use freshly built distribution.
|
||||
|
||||
### Tuning
|
||||
Run `python3 package.py -h` to check how one can tune script's behavior.
|
||||
Some examples:
|
||||
* `--archive-path` creates an archive (`zip` on Windows, `tar.gz` on macOS and Linux) with distribution
|
||||
alongside with its SHA256.
|
||||
* `--install-path` allows overriding distribution's output directory.
|
||||
* `--num-stages` specifies number of steps in build. Passing 2 or more makes bootstrap build which
|
||||
means that LLVM will build itself by using distribution from the previous step.
|
||||
* `--stage0` allows using existing LLVM toolchain for bootstrapping.
|
||||
|
||||
### Known problems
|
||||
1. Bootstrap build is not working on macOS for default git branch.
|
||||
2. `libcxx` and `compiler-rt` are built as projects, not runtimes.
|
||||
3. No way to run LLVM tests out of the box.
|
||||
4. No way to override default `install` task out of the box.
|
||||
|
||||
@@ -0,0 +1,420 @@
|
||||
#!/usr/bin/python3
|
||||
#
|
||||
# Copyright 2010-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
vsdevcmd = None
|
||||
isysroot = None
|
||||
|
||||
ninja = 'ninja'
|
||||
cmake = 'cmake'
|
||||
git = 'git'
|
||||
|
||||
|
||||
def absolute_path(path):
|
||||
if path is not None:
|
||||
# CMake is not tolerant to backslashes in path.
|
||||
return os.path.abspath(path).replace('\\', '/')
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def host_is_windows():
|
||||
return sys.platform == "win32"
|
||||
|
||||
|
||||
def host_is_linux():
|
||||
return sys.platform == "linux"
|
||||
|
||||
|
||||
def host_is_darwin():
|
||||
return sys.platform == "darwin"
|
||||
|
||||
|
||||
def host_llvm_target():
|
||||
return "Native"
|
||||
|
||||
|
||||
def host_default_compression():
|
||||
"""
|
||||
Determine archive compression method based on current OS.
|
||||
On Windows we use `zip` and `tar.gz` otherwise.
|
||||
"""
|
||||
if host_is_windows():
|
||||
return "zip"
|
||||
else:
|
||||
return "gztar"
|
||||
|
||||
|
||||
def detect_xcode_sdk_path():
|
||||
"""
|
||||
Get an absolute path to macOS SDK.
|
||||
"""
|
||||
return subprocess.check_output(['xcrun', '--show-sdk-path'],
|
||||
universal_newlines=True).rstrip()
|
||||
|
||||
|
||||
def detect_vsdevcmd():
|
||||
"""
|
||||
Use vswhere (and download it, if needed) utility to find path to vsdevcmd.bat.
|
||||
:return: path to vsdevcmd.bat
|
||||
"""
|
||||
vswhere = shutil.which('vswhere')
|
||||
if vswhere is None:
|
||||
print("Downloading vswhere utility to detect path to vsdevcmd.bat automatically")
|
||||
vswhere_url = "https://github.com/microsoft/vswhere/releases/download/2.8.4/vswhere.exe"
|
||||
urllib.request.urlretrieve(vswhere_url, 'vswhere.exe')
|
||||
vswhere = shutil.which('vswhere')
|
||||
if vswhere is None:
|
||||
sys.exit("Failed to retrieve vswhere utility. Please provide path to vsdevcmd.bat with --vsdevcmd")
|
||||
vswhere_args = [vswhere, '-prerelease', '-latest', '-property', 'installationPath']
|
||||
path_to_visual_studio = subprocess.check_output(vswhere_args, universal_newlines=True).rstrip()
|
||||
vsdevcmd_path = os.path.join(path_to_visual_studio, "Common7", "Tools", "vsdevcmd.bat")
|
||||
if not os.path.isfile(vsdevcmd_path):
|
||||
sys.exit("vsdevcmd.bat is not found. Please provide path to vsdevcmd.bat with --vsdevcmd")
|
||||
else:
|
||||
print("Found vsdevcmd.bat: " + vsdevcmd_path)
|
||||
return vsdevcmd_path
|
||||
|
||||
|
||||
def construct_cmake_flags(
|
||||
bootstrap_llvm_path: str = None,
|
||||
install_path: str = None,
|
||||
projects: List[str] = None,
|
||||
runtimes: List[str] = None,
|
||||
targets: List[str] = None
|
||||
) -> List[str]:
|
||||
building_bootstrap = bootstrap_llvm_path is None
|
||||
|
||||
c_compiler, cxx_compiler, linker, ar = None, None, None, None
|
||||
c_flags, cxx_flags, linker_flags = None, None, None
|
||||
if not building_bootstrap:
|
||||
if host_is_windows():
|
||||
# CMake is not tolerant to backslashes
|
||||
c_compiler = f'{bootstrap_llvm_path}/bin/clang-cl.exe'.replace('\\', '/')
|
||||
cxx_compiler = f'{bootstrap_llvm_path}/bin/clang-cl.exe'.replace('\\', '/')
|
||||
linker = f'{bootstrap_llvm_path}/bin/lld-link.exe'.replace('\\', '/')
|
||||
ar = f'{bootstrap_llvm_path}/bin/llvm-lib.exe'.replace('\\', '/')
|
||||
elif host_is_linux():
|
||||
c_compiler = f'{bootstrap_llvm_path}/bin/clang'
|
||||
cxx_compiler = f'{bootstrap_llvm_path}/bin/clang++'
|
||||
linker = f'{bootstrap_llvm_path}/bin/ld.lld'
|
||||
ar = f'{bootstrap_llvm_path}/bin/llvm-ar'
|
||||
elif host_is_darwin():
|
||||
c_compiler = f'{bootstrap_llvm_path}/bin/clang'
|
||||
cxx_compiler = f'{bootstrap_llvm_path}/bin/clang++'
|
||||
# ld64.lld is not that good yet.
|
||||
linker = None
|
||||
ar = f'{bootstrap_llvm_path}/bin/llvm-ar'
|
||||
c_flags = ['-isysroot', isysroot]
|
||||
cxx_flags = ['-isysroot', isysroot, '-stdlib=libc++']
|
||||
linker_flags = ['-stdlib=libc++']
|
||||
|
||||
cmake_args = [
|
||||
'-DCMAKE_BUILD_TYPE=Release',
|
||||
'-DLLVM_ENABLE_ASSERTIONS=OFF',
|
||||
'-DLLVM_ENABLE_TERMINFO=OFF',
|
||||
'-DLLVM_INCLUDE_GO_TESTS=OFF',
|
||||
'-DLLVM_ENABLE_Z3_SOLVER=OFF',
|
||||
'-DCOMPILER_RT_BUILD_BUILTINS=ON'
|
||||
]
|
||||
|
||||
if host_is_darwin():
|
||||
cmake_args.append('-DLLVM_ENABLE_LIBCXX=ON')
|
||||
if building_bootstrap:
|
||||
# Don't waste time by doing unnecessary work for throwaway toolchain.
|
||||
cmake_args.extend([
|
||||
'-DCOMPILER_RT_BUILD_CRT=OFF',
|
||||
'-DCOMPILER_RT_BUILD_LIBFUZZER=OFF',
|
||||
'-DCOMPILER_RT_BUILD_SANITIZERS=OFF',
|
||||
'-DCOMPILER_RT_BUILD_XRAY=OFF',
|
||||
'-DCOMPILER_RT_ENABLE_IOS=OFF',
|
||||
'-DCOMPILER_RT_ENABLE_WATCHOS=OFF',
|
||||
'-DCOMPILER_RT_ENABLE_TVOS=OFF',
|
||||
])
|
||||
else:
|
||||
cmake_args.append('-DLIBCXX_USE_COMPILER_RT=ON')
|
||||
|
||||
if install_path is not None:
|
||||
cmake_args.append('-DCMAKE_INSTALL_PREFIX=' + install_path)
|
||||
if targets is not None:
|
||||
cmake_args.append('-DLLVM_TARGETS_TO_BUILD=' + ";".join(targets))
|
||||
if projects is not None:
|
||||
cmake_args.append('-DLLVM_ENABLE_PROJECTS=' + ";".join(projects))
|
||||
if runtimes is not None:
|
||||
cmake_args.append('-DLLVM_ENABLE_RUNTIMES=' + ";".join(runtimes))
|
||||
if c_compiler is not None:
|
||||
cmake_args.append('-DCMAKE_C_COMPILER=' + c_compiler)
|
||||
if cxx_compiler is not None:
|
||||
cmake_args.append('-DCMAKE_CXX_COMPILER=' + cxx_compiler)
|
||||
if linker is not None:
|
||||
cmake_args.append('-DCMAKE_LINKER=' + linker)
|
||||
if ar is not None:
|
||||
cmake_args.append('-DCMAKE_AR=' + ar)
|
||||
|
||||
if c_flags is not None:
|
||||
cmake_args.append("-DCMAKE_C_FLAGS=" + ' '.join(c_flags))
|
||||
if cxx_flags is not None:
|
||||
cmake_args.append("-DCMAKE_CXX_FLAGS=" + ' '.join(cxx_flags))
|
||||
if linker_flags is not None:
|
||||
cmake_args.append('-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(linker_flags))
|
||||
cmake_args.append('-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(linker_flags))
|
||||
cmake_args.append('-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(linker_flags))
|
||||
|
||||
if host_is_windows():
|
||||
# Use MT to make distribution self-contained
|
||||
# TODO: Consider -DCMAKE_INSTALL_UCRT_LIBRARIES=ON as an alternative
|
||||
cmake_args.append('-DLLVM_USE_CRT_RELEASE=MT')
|
||||
cmake_args.append('-DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded')
|
||||
# We don't support PDB, so no need fir DIA.
|
||||
cmake_args.append('-DLLVM_ENABLE_DIA_SDK=OFF')
|
||||
|
||||
# Make distribution much smaller by linking to dynamic library
|
||||
# instead of static linkage.
|
||||
# Not working for Windows yet.
|
||||
#
|
||||
# Also not working for Linux and macOS because of signal chaining.
|
||||
# TODO: Enable after LLVM distribution patching.
|
||||
if not host_is_windows():
|
||||
cmake_args.append("-LLVM_BUILD_LLVM_DYLIB=OFF")
|
||||
cmake_args.append("-DLLVM_LINK_LLVM_DYLIB=OFF")
|
||||
|
||||
return cmake_args
|
||||
|
||||
|
||||
def run_command(command: List[str]):
|
||||
"""
|
||||
Execute single command in terminal/cmd.
|
||||
|
||||
Note that on Windows we prepare environment with vsdevcmd.bat.
|
||||
"""
|
||||
if host_is_windows():
|
||||
if vsdevcmd is None:
|
||||
sys.exit("'VsDevCmd.bat' is not set!")
|
||||
command = [vsdevcmd, "-arch=amd64", "&&"] + command
|
||||
print("Running command: " + ' '.join(command))
|
||||
else:
|
||||
command = [shlex.quote(arg) for arg in command]
|
||||
command = ' '.join(command)
|
||||
print("Running command: " + command)
|
||||
|
||||
subprocess.run(command, shell=True, check=True)
|
||||
|
||||
|
||||
def force_create_directory(parent, name) -> Path:
|
||||
build_path = parent / name
|
||||
print(f"Force-creating directory {build_path}")
|
||||
if build_path.exists():
|
||||
shutil.rmtree(build_path)
|
||||
os.mkdir(build_path)
|
||||
return build_path
|
||||
|
||||
|
||||
def llvm_build_commands(
|
||||
install_path, bootstrap_path, llvm_src, targets, ninja_target, projects, runtimes
|
||||
) -> List[List[str]]:
|
||||
cmake_flags = construct_cmake_flags(bootstrap_path, install_path, projects, runtimes, targets)
|
||||
cmake_command = [cmake, "-G", "Ninja"] + cmake_flags + [os.path.join(llvm_src, "llvm")]
|
||||
ninja_command = [ninja, ninja_target]
|
||||
return [cmake_command, ninja_command]
|
||||
|
||||
|
||||
def clone_llvm_repository(repo, branch, llvm_repo_destination):
|
||||
"""
|
||||
Downloads a single commit from the given repository.
|
||||
"""
|
||||
if host_is_darwin():
|
||||
default_repo, default_branch = "https://github.com/apple/llvm-project", "apple/stable/20200108"
|
||||
else:
|
||||
default_repo, default_branch = "https://github.com/llvm/llvm-project", "release/11.x"
|
||||
repo = default_repo if repo is None else repo
|
||||
branch = default_branch if branch is None else branch
|
||||
# Download only single commit because we don't need whole history just for building LLVM.
|
||||
run_command([git, "clone", repo, "--branch", branch, "--depth", "1", "llvm-project"])
|
||||
return absolute_path(llvm_repo_destination)
|
||||
|
||||
|
||||
def default_num_stages():
|
||||
if host_is_darwin():
|
||||
# Bootstrap build is not working for https://github.com/apple/llvm-project + apple/stable/20200108.
|
||||
# So use single phase for now.
|
||||
return 1
|
||||
else:
|
||||
return 2
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Build LLVM toochain for Kotlin/Native")
|
||||
# Output configuration.
|
||||
parser.add_argument("--install-path", type=str, default="llvm-distribution", required=False,
|
||||
help="Where final LLVM distribution will be installed")
|
||||
parser.add_argument("--archive-path", default=None,
|
||||
help="Create an archive and its sha256 for final distribution at given path")
|
||||
# Build configuration
|
||||
parser.add_argument("--stage0", type=str, default=None,
|
||||
help="Path to existing LLVM toolchain")
|
||||
parser.add_argument("--num-stages", type=int, default=default_num_stages(),
|
||||
help="Number of stages in bootstrap.")
|
||||
# LLVM sources.
|
||||
parser.add_argument("--llvm-sources", dest="llvm_src", type=str, default=None,
|
||||
help="Location of LLVM sources")
|
||||
parser.add_argument("--repo", type=str, default=None)
|
||||
parser.add_argument("--branch", type=str, default=None)
|
||||
parser.add_argument("--llvm-repo-destination", type=str, default="llvm-project",
|
||||
help="Where LLVM repository should be downloaded.")
|
||||
# Environment setup.
|
||||
parser.add_argument("--vsdevcmd", type=str, default=None,
|
||||
help="(Windows only) Path to VsDevCmd.bat")
|
||||
parser.add_argument("--ninja", type=str, default=None,
|
||||
help="Override path to ninja")
|
||||
parser.add_argument("--cmake", type=str, default=None,
|
||||
help="Override path to cmake")
|
||||
parser.add_argument("--git", type=str, default=None,
|
||||
help="Override path to git")
|
||||
parser.add_argument("--isysroot", type=str, default=None,
|
||||
help="(macOS only) Override path to macOS SDK")
|
||||
# Misc.
|
||||
parser.add_argument("--save-temporary-files", type=bool, default=False,
|
||||
help="Should intermediate build results be saved?")
|
||||
return parser
|
||||
|
||||
|
||||
def build_distribution(args):
|
||||
"""
|
||||
Performs (probably multistage) build of LLVM
|
||||
and returns path to the final distribution.
|
||||
"""
|
||||
current_dir = Path().absolute()
|
||||
num_stages = args.num_stages
|
||||
bootstrap_path = args.stage0
|
||||
intermediate_build_results = []
|
||||
# Most likely, num_stages will be 1 or 2.
|
||||
# 2 means bootstrap build: we build LLVM distribution (stage 1)
|
||||
# that then compiles sources once again (stage 2). Thus, resulting
|
||||
# distribution is (almost) independent from environment, which means
|
||||
# reproducibility and less bugs.
|
||||
#
|
||||
# Sometimes it makes sense to generate yet another distribution to check
|
||||
# that it is the same as built at stage 2 (so there is no non-determinism in LLVM).
|
||||
for stage in range(1, num_stages + 1):
|
||||
building_bootstrap = num_stages > 1 and stage == 1
|
||||
building_final = stage == num_stages
|
||||
|
||||
if building_bootstrap:
|
||||
# We only need a host target to start a bootstrap.
|
||||
targets = [host_llvm_target()]
|
||||
else:
|
||||
# None targets means all available targets.
|
||||
targets = None
|
||||
|
||||
if building_final:
|
||||
install_path = args.install_path
|
||||
else:
|
||||
install_path = force_create_directory(current_dir, f"llvm-stage-{stage}")
|
||||
intermediate_build_results.append(install_path)
|
||||
|
||||
projects = ["clang", "lld", "libcxx", "libcxxabi", "compiler-rt"]
|
||||
runtimes = None
|
||||
ninja_target = "install"
|
||||
|
||||
build_dir = force_create_directory(current_dir, f"llvm-stage-{stage}-build")
|
||||
intermediate_build_results.append(build_dir)
|
||||
commands = llvm_build_commands(
|
||||
install_path=absolute_path(install_path),
|
||||
bootstrap_path=absolute_path(bootstrap_path),
|
||||
llvm_src=absolute_path(args.llvm_src),
|
||||
targets=targets,
|
||||
ninja_target=ninja_target,
|
||||
projects=projects,
|
||||
runtimes=runtimes
|
||||
)
|
||||
|
||||
os.chdir(build_dir)
|
||||
for command in commands:
|
||||
run_command(command)
|
||||
os.chdir(current_dir)
|
||||
bootstrap_path = install_path
|
||||
|
||||
if not args.save_temporary_files:
|
||||
for dir in intermediate_build_results:
|
||||
print(f"Removing temporary directory: {dir}")
|
||||
shutil.rmtree(dir)
|
||||
|
||||
return absolute_path(args.install_path)
|
||||
|
||||
|
||||
def create_archive(input_directory, output_path, compression=host_default_compression()) -> str:
|
||||
print("Creating archive " + output_path + " from " + input_directory)
|
||||
base_directory, archive_prefix = os.path.split(os.path.normpath(input_directory))
|
||||
return shutil.make_archive(output_path, compression, base_directory, archive_prefix)
|
||||
|
||||
|
||||
def create_checksum_file(input_path, output_path):
|
||||
chunk_size = 4096
|
||||
checksum = hashlib.sha256()
|
||||
with open(input_path, "rb") as input_contents:
|
||||
for chunk in iter(lambda: input_contents.read(chunk_size), b""):
|
||||
checksum.update(chunk)
|
||||
print(checksum.hexdigest(), file=open(output_path, "w"))
|
||||
return True
|
||||
|
||||
|
||||
def setup_environment(args):
|
||||
"""
|
||||
Setup globals that store information about script execution environment.
|
||||
"""
|
||||
global vsdevcmd, ninja, cmake, git, isysroot
|
||||
# TODO: We probably can download some of these binaries ourselves.
|
||||
if args.ninja:
|
||||
ninja = args.ninja
|
||||
elif shutil.which('ninja') is None:
|
||||
sys.exit("'ninja' is not found. Install or provide via --ninja argument.")
|
||||
if args.cmake:
|
||||
cmake = args.cmake
|
||||
elif shutil.which('cmake') is None:
|
||||
sys.exit("'cmake' is not found. Install or provide via --cmake argument.")
|
||||
if args.git:
|
||||
git = args.git
|
||||
elif shutil.which('git') is None:
|
||||
sys.exit("'git' is not found. Install or provide via --git argument.")
|
||||
if host_is_windows():
|
||||
if args.vsdevcmd:
|
||||
vsdevcmd = args.vsdevcmd
|
||||
else:
|
||||
vsdevcmd = detect_vsdevcmd()
|
||||
elif host_is_darwin():
|
||||
if args.isysroot:
|
||||
isysroot = args.isysroot
|
||||
else:
|
||||
isysroot = detect_xcode_sdk_path()
|
||||
|
||||
|
||||
def main():
|
||||
parser = build_parser()
|
||||
args = parser.parse_args()
|
||||
setup_environment(args)
|
||||
temporary_llvm_repo = None
|
||||
if args.llvm_src is None:
|
||||
temporary_llvm_repo = clone_llvm_repository(args.repo, args.branch, args.llvm_repo_destination)
|
||||
args.llvm_src = temporary_llvm_repo
|
||||
final_dist = build_distribution(args)
|
||||
if args.archive_path is not None:
|
||||
archive = create_archive(final_dist, args.archive_path)
|
||||
create_checksum_file(archive, f"{archive}.sha256")
|
||||
if not args.save_temporary_files and temporary_llvm_repo is not None:
|
||||
print(f"Removing temporary directory: {temporary_llvm_repo}")
|
||||
shutil.rmtree(temporary_llvm_repo)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user