azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[K/N] Option to build smaller LLVM distribution.

Browse Source 
Also add a few adjustments to `package.py`:
* Enable threading in LLVM
* Replace `--archive-path` with `--pack` to avoid naming divergence.
This commit is contained in:
Sergey Bogolepov
2021-08-09 11:37:07 +07:00
committed by Space
parent 74e26126df
commit 75215fcb97
2 changed files with 62 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kotlin-native/tools/llvm_builder/README.md
+21 -5
View File
@@ -18,24 +18,42 @@ python3 package.py
```
It will create a `llvm-distribution` folder which contains LLVM distribution.
Note that build will take a lot of time and all your machine cores. 😈
See [HACKING.md](../../HACKING.md#using-different-llvm-distributions-as-part-of-kotlinnative-compilation-pipeline) on how to use freshly built distribution.
See [HACKING.md](../../HACKING.md#using-different-llvm-distributions-as-part-of-kotlinnative-compilation-pipeline)
on how to use freshly built distribution.
### Building distribution for end-users
By default, `package.py` installs a lot of LLVM components that is not really required to use Kotlin/Native compiler.
To build only essential parts, run the following command:
```
python3 package.py --build-targets install-distribution --distribution-components $DISTRIBUTION_COMPONENTS
```
Set of required `$DISTRIBUTION_COMPONENTS` is depends on OS:
| OS | Distribution components |
| --- | --- |
|Windows | `clang libclang lld llvm-cov llvm-profdata llvm-ar clang-resource-headers` |
|macOS | `clang libclang lld llvm-cov llvm-profdata llvm-ar clang-resource-headers` |
|Linux | `clang libclang lld llvm-cov llvm-profdata llvm-ar clang-resource-headers compiler_rt` |
### Tuning
Run `python3 package.py -h` to check how one can tune script's behavior.
Some examples:
* `--archive-path` creates an archive (`zip` on Windows, `tar.gz` on macOS and Linux) with distribution
* `--pack` creates an archive (`zip` on Windows, `tar.gz` on macOS and Linux) with distribution
alongside with its SHA256.
* `--install-path` allows overriding distribution's output directory.
* `--num-stages` specifies number of steps in build. Passing 2 or more makes bootstrap build which
means that LLVM will build itself by using distribution from the previous step.
* `--stage0` allows using existing LLVM toolchain for bootstrapping.
* `--build-targets` specifies what targets will be passed to Ninja.
* `--distribution-components` is a list of components that will be installed with `install-distribution` build target.
### Docker
You can use Docker to build LLVM for Linux:
```shell
docker build -t kotlin-llvm-builder .
docker run --rm -it -v <HOST_PATH>:/output kotlin-llvm-builder --archive-path /output/llvm-11.1.0-linux-x64
docker run --rm -it -v <HOST_PATH>:/output kotlin-llvm-builder --install-path /output/llvm-11.1.0-linux-x64 --pack
```
@@ -43,5 +61,3 @@ docker run --rm -it -v <HOST_PATH>:/output kotlin-llvm-builder --archive-path /o
1. Bootstrap build is not working on macOS for default git branch.
2. `libcxx` and `compiler-rt` are built as projects, not runtimes.
3. No way to run LLVM tests out of the box.
4. No way to override default `install` task out of the box.
kotlin-native/tools/llvm_builder/package.py
+41 -24
View File
@@ -92,13 +92,31 @@ def construct_cmake_flags(
install_path: str = None,
projects: List[str] = None,
runtimes: List[str] = None,
targets: List[str] = None
targets: List[str] = None,
distribution_components: List[str] = None
) -> List[str]:
building_bootstrap = bootstrap_llvm_path is None
c_compiler, cxx_compiler, linker, ar = None, None, None, None
c_flags, cxx_flags, linker_flags = None, None, None
cmake_args = [
'-DCMAKE_BUILD_TYPE=Release',
'-DLLVM_ENABLE_ASSERTIONS=OFF',
'-DLLVM_ENABLE_TERMINFO=OFF',
'-DLLVM_INCLUDE_GO_TESTS=OFF',
'-DLLVM_ENABLE_Z3_SOLVER=OFF',
'-DCOMPILER_RT_BUILD_BUILTINS=ON',
'-DLLVM_ENABLE_THREADS=ON',
'-DLLVM_OPTIMIZED_TABLEGEN=ON'
]
if not building_bootstrap:
if distribution_components:
cmake_args.append('-DLLVM_DISTRIBUTION_COMPONENTS=' + ';'.join(distribution_components))
# These links are actually copies on windows, so they're wasting precious disk space.
cmake_args.append("-DCLANG_LINKS_TO_CREATE=clang++")
cmake_args.append("-DLLD_SYMLINKS_TO_CREATE=ld.lld;wasm-ld")
if host_is_windows():
# CMake is not tolerant to backslashes
c_compiler = f'{bootstrap_llvm_path}/bin/clang-cl.exe'.replace('\\', '/')
@@ -120,14 +138,6 @@ def construct_cmake_flags(
cxx_flags = ['-isysroot', isysroot, '-stdlib=libc++']
linker_flags = ['-stdlib=libc++']
cmake_args = [
'-DCMAKE_BUILD_TYPE=Release',
'-DLLVM_ENABLE_ASSERTIONS=OFF',
'-DLLVM_ENABLE_TERMINFO=OFF',
'-DLLVM_INCLUDE_GO_TESTS=OFF',
'-DLLVM_ENABLE_Z3_SOLVER=OFF',
'-DCOMPILER_RT_BUILD_BUILTINS=ON'
]
if host_is_darwin():
cmake_args.append('-DLLVM_ENABLE_LIBCXX=ON')
@@ -186,7 +196,7 @@ def construct_cmake_flags(
# Also not working for Linux and macOS because of signal chaining.
# TODO: Enable after LLVM distribution patching.
if not host_is_windows():
cmake_args.append("-LLVM_BUILD_LLVM_DYLIB=OFF")
cmake_args.append("-DLLVM_BUILD_LLVM_DYLIB=OFF")
cmake_args.append("-DLLVM_LINK_LLVM_DYLIB=OFF")
return cmake_args
@@ -221,11 +231,11 @@ def force_create_directory(parent, name) -> Path:
def llvm_build_commands(
install_path, bootstrap_path, llvm_src, targets, ninja_target, projects, runtimes
install_path, bootstrap_path, llvm_src, targets, build_targets, projects, runtimes, distribution_components
) -> List[List[str]]:
cmake_flags = construct_cmake_flags(bootstrap_path, install_path, projects, runtimes, targets)
cmake_flags = construct_cmake_flags(bootstrap_path, install_path, projects, runtimes, targets, distribution_components)
cmake_command = [cmake, "-G", "Ninja"] + cmake_flags + [os.path.join(llvm_src, "llvm")]
ninja_command = [ninja, ninja_target]
ninja_command = [ninja] + build_targets
return [cmake_command, ninja_command]
@@ -250,12 +260,18 @@ def default_num_stages():
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Build LLVM toochain for Kotlin/Native")
parser = argparse.ArgumentParser(description="Build LLVM toolchain for Kotlin/Native")
# Output configuration.
parser.add_argument("--install-path", type=str, default="llvm-distribution", required=False,
help="Where final LLVM distribution will be installed")
parser.add_argument("--archive-path", default=None,
help="Create an archive and its sha256 for final distribution at given path")
help="Where final LLVM distribution will be enabled")
parser.add_argument("--pack", action='store_true',
help="Create an archive and its sha256 for final distribution at `--install-path`")
parser.add_argument("--build-targets", default=["install"],
nargs="+",
help="What components should be installed")
parser.add_argument("--distribution-components", default=None,
nargs="+",
help="What components should be installed with `install-distribution` target")
# Build configuration
parser.add_argument("--stage0", type=str, default=None,
help="Path to existing LLVM toolchain")
@@ -280,7 +296,7 @@ def build_parser() -> argparse.ArgumentParser:
parser.add_argument("--isysroot", type=str, default=None,
help="(macOS only) Override path to macOS SDK")
# Misc.
parser.add_argument("--save-temporary-files", type=bool, default=False,
parser.add_argument("--save-temporary-files", action='store_true',
help="Should intermediate build results be saved?")
return parser
@@ -312,16 +328,16 @@ def build_distribution(args):
else:
# None targets means all available targets.
targets = None
if building_final:
install_path = args.install_path
build_targets = args.build_targets
else:
install_path = force_create_directory(current_dir, f"llvm-stage-{stage}")
intermediate_build_results.append(install_path)
build_targets = ["install"]
projects = ["clang", "lld", "libcxx", "libcxxabi", "compiler-rt"]
runtimes = None
ninja_target = "install"
build_dir = force_create_directory(current_dir, f"llvm-stage-{stage}-build")
intermediate_build_results.append(build_dir)
@@ -330,9 +346,10 @@ def build_distribution(args):
bootstrap_path=absolute_path(bootstrap_path),
llvm_src=absolute_path(args.llvm_src),
targets=targets,
ninja_target=ninja_target,
build_targets=build_targets,
projects=projects,
runtimes=runtimes
runtimes=runtimes,
distribution_components=args.distribution_components
)
os.chdir(build_dir)
@@ -404,8 +421,8 @@ def main():
temporary_llvm_repo = clone_llvm_repository(args.repo, args.branch, args.llvm_repo_destination)
args.llvm_src = temporary_llvm_repo
final_dist = build_distribution(args)
if args.archive_path is not None:
archive = create_archive(final_dist, args.archive_path)
if args.pack:
archive = create_archive(final_dist, args.install_path)
create_checksum_file(archive, f"{archive}.sha256")
if not args.save_temporary_files and temporary_llvm_repo is not None:
print(f"Removing temporary directory: {temporary_llvm_repo}")