*: fully hermetic builds and nix shell support
This change is a slightly more polished version of Serge's experiment:
- https://review.monogon.dev/c/monogon/+/1148
- https://bin.monogon.dev/pasta/sloth-parrot-ant
- https://bin.monogon.dev/pasta/eel-seal-wolf
There are two execution environments we have to support:
- Most builds run inside a sandbox, which is a Fedora
environment and does not require any host dependencies at all.
- Bazel itself and the tooling we require to bootstrap
the sandbox (mainly, Go and Proto toolchains). This has to
work directly on the host.
We first make the sandbox fully hermetic by setting
--experimental_use_hermetic_linux_sandbox, which set up an empty /
instead of mounting over individual directories, removing any remaining
host paths from the sandbox (except /proc and /dev/shm, which are
required by some toolchains). We also force static values for the shell,
$TMPDIR and $PATH, which would otherwise leak into the sandbox.
For the host, we use buildFHSUserEnv to build an environment which
supports our static toolchains, and well as a clean Bazel build
without all the nixpkgs patches which would otherwise break our custom
toolchains and sandbox implementation.
This allows us to use the exact same toolchains on NixOS and other
distros for perfect reproducibility.
Fixes https://github.com/monogon-dev/monogon/issues/174.
Fixes https://github.com/monogon-dev/monogon/issues/175.
Co-authored-by: Serge Bazanski <serge@monogon.tech>
Change-Id: I665471a45b315ce7e93ef16d9d056d7622886959
Reviewed-on: https://review.monogon.dev/c/monogon/+/1929
Tested-by: Jenkins CI
Reviewed-by: Serge Bazanski <serge@monogon.tech>
diff --git a/.bazelrc b/.bazelrc
index 9734f97..18253f1 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -3,11 +3,22 @@
build --incompatible_strict_action_env=true
# Run all spawns in our own hermetic sandbox sysroot.
-#build --experimental_use_hermetic_linux_sandbox
-# TODO: https://github.com/bazelbuild/rules_go/issues/1910
+build --experimental_use_hermetic_linux_sandbox
build --action_env=MONOGON_SANDBOX_DIGEST
import %workspace%/.bazelrc.sandbox
+# Hardwire all action envs to just use /usr/bin from the above sandbox. This is
+# necessary on NixOS Bazel builds, as they really like to inject /nix/store/*
+# paths otherwise. We also explicitly set it to /usr/bin only (no /bin) as
+# otherwise calling gcc from /bin/gcc breaks its own resolution of subordinate
+# commands (like cc1, as, etc.).
+build --action_env=PATH=/usr/bin
+build --host_action_env=PATH=/usr/bin
+
+# Make all shell run actions use /bin/bash instead of whatever the host might
+# have set. Again, looking at you, Bazel-on-NixOS.
+build --shell_executable=/bin/bash
+
# No local CPP toolchain resolution. In our sandbox root, it doesn't make sense -
# anything auto-detected during analysis stage is on the host instead of the sandbox.
# Sysroot rebuild is pure Go and doesn't need it either.
diff --git a/SETUP.md b/SETUP.md
index 16a2353..d48e894 100644
--- a/SETUP.md
+++ b/SETUP.md
@@ -32,7 +32,7 @@
- Ubuntu >= 20.04
- Debian >= 11
- RHEL / Alma / Rocky >= 8.4
-- NixOS >= 22.05 (see below)
+- NixOS >= 23.05 (see below)
You can use this snippet to install the official Bazelisk release binary to `/usr/local/bin`:
@@ -76,8 +76,11 @@
### NixOS
-Bazelisk and rules_go does not work on NixOS since its prebuilt binaries are dynamically linked,
-so we cannot bootstrap the sandbox. We recommend building in a "normal" container.
+We fully support building on NixOS, and we provide a `shell.nix` file to make it easy:
+
+ nix-shell --pure
+
+(just don't look at it too closely unless you want toolchain-themed nightmares)
## IntelliJ
diff --git a/shell.nix b/shell.nix
new file mode 100644
index 0000000..d64223f
--- /dev/null
+++ b/shell.nix
@@ -0,0 +1,87 @@
+# If you're on NixOS, use me! `nix-shell --pure`.
+with import (fetchTarball {
+ # nixpkgs 23.05 as of 2023/07/19
+ url = "https://github.com/NixOS/nixpkgs/archive/2fadc2426928c844054cd28fabe231ff26a70715.tar.gz";
+ sha256 = "sha256:06hpcqhaaqvd5gjcz2ps9lz6q2sf5fwgl5rwllpyl9x4g5g95ahv";
+}) {};
+let
+ wrapper = pkgs.writeScript "wrapper.sh"
+ ''
+ # Fancy colorful PS1 to make people notice easily they're in the Monogon Nix shell.
+ PS1='\[\033]0;\u/monogon:\w\007\]'
+ if type -P dircolors >/dev/null ; then
+ PS1+='\[\033[01;37m\]\u/monogon\[\033[01;36m\] \w \$\[\033[00m\] '
+ fi
+ export PS1
+
+ # Use Nix-provided cert store.
+ export NIX_SSL_CERT_FILE="${cacert}/etc/ssl/certs/ca-bundle.crt"
+ export SSL_CERT_FILE="${cacert}/etc/ssl/certs/ca-bundle.crt"
+
+ # Let some downstream machinery know we're on NixOS. This is used mostly to
+ # work around Bazel/NixOS interactions.
+ export MONOGON_NIXOS=yep
+
+ # Convince rules_go to use /bin/bash and not a NixOS store bash which has
+ # no idea how to resolve other things in the nix store once PATH is
+ # stripped by (host_)action_env.
+ export BAZEL_SH=/bin/bash
+
+ exec bash --noprofile --norc "$@"
+ '';
+in
+(pkgs.buildFHSUserEnv {
+ name = "monogon-nix";
+ targetPkgs = pkgs: with pkgs; [
+ git
+ (stdenv.mkDerivation {
+ name = "bazel";
+ src = builtins.fetchurl {
+ url = https://github.com/bazelbuild/bazel/releases/download/5.4.0/bazel-5.4.0-linux-x86_64;
+ sha256 = "1w58m1brwjfwsv48fmd66inry67m4vgb3bwvwmamhdv099v183jg";
+ };
+ unpackPhase = ''
+ true
+ '';
+ buildPhase = ''
+ mkdir -p $out/bin
+ cp $src $out/bin/.bazel-inner
+ chmod +x $out/bin/.bazel-inner
+
+ cat > $out/bin/bazel <<EOF
+ #!/usr/bin/bash
+ export BAZEL_REAL=$out/bin/.bazel-inner
+ function get_workspace_root() {
+ workspace_dir="\''${PWD}"
+ while [[ "\''${workspace_dir}" != / ]]; do
+ if [[ -e "\''${workspace_dir}/WORKSPACE" || -e "\''${workspace_dir}/WORKSPACE.bazel" ]]; then
+ readonly workspace_dir
+ return
+ fi
+ workspace_dir="$(dirname "\''${workspace_dir}")"
+ done
+ readonly workspace_dir=""
+ }
+
+ get_workspace_root
+ readonly wrapper="\''${workspace_dir}/tools/bazel"
+ if [ -f "\''${wrapper}" ]; then
+ exec -a "\$0" "\''${wrapper}" "\$@"
+ fi
+ exec -a "\$0" "\''${BAZEL_REAL}" "\$@"
+ EOF
+ chmod +x $out/bin/bazel
+ '';
+ dontStrip = true;
+ })
+ zlib
+ curl
+ gcc
+ binutils
+ openjdk11
+ patch
+ python3
+ ];
+ runScript = wrapper;
+}).env
+
diff --git a/tools/bazel b/tools/bazel
index 849bb9b..67de8e4 100755
--- a/tools/bazel
+++ b/tools/bazel
@@ -22,7 +22,8 @@
fi
# Recommend using Bazelisk instead of Bazel's "bazel.sh" wrapper.
- if [[ -z "${BAZELISK_SKIP_WRAPPER:-}" ]]; then
+ # Skip if we're inside the Nix shell (which uses a customized Bazel build).
+ if [[ -z "${BAZELISK_SKIP_WRAPPER:-}" && -z "${MONOGON_NIXOS:-}" ]]; then
echo "############################################################" >&2
echo "# Please use Bazelisk to build the Monorepo. Using Bazel #" >&2
echo "# directly may work, but is not recommended or supported. #" >&2
@@ -132,6 +133,7 @@
# Autogenerated by tools/bazel. Manual changes can result in stale caches.
# Modify the generator instead.
+# Mount directories from the generated Fedora sandbox root.
build --sandbox_add_mount_pair=${ROOT}/.bazeldnf/sandbox/default/root/etc:/etc
build --sandbox_add_mount_pair=${ROOT}/.bazeldnf/sandbox/default/root/usr:/usr
build --sandbox_add_mount_pair=${ROOT}/.bazeldnf/sandbox/default/root/var:/var
@@ -141,6 +143,19 @@
build --sandbox_add_mount_pair=${ROOT}/.bazeldnf/sandbox/default/root/lib:/lib
build --sandbox_add_mount_pair=${ROOT}/.bazeldnf/sandbox/default/root/bin:/bin
build --sandbox_add_mount_pair=${ROOT}/.bazeldnf/sandbox/default/root/sbin:/sbin
+
+# Needed for the Go SDK shipped by rules_go to resolve its own GOROOT via /proc/self/exe.
+build --sandbox_add_mount_pair=/proc
+
+# Needed for python's multiprocessing lock implementation
+# (_multiprocessing.SemLock for eg. mp.Queue), as used in EDK2's build system.
+build --sandbox_add_mount_pair=/dev/shm
+
+# Needed for qemu for tests.
+build --sandbox_add_mount_pair=/dev/kvm
+
+# Put a tmpfs on /tmp for better performance.
+build --sandbox_tmpfs_path=/tmp
EOF
echo "Done regenerating sysroot." >&2
@@ -168,4 +183,8 @@
# This is strictly necessary to guarantee correctness.
export MONOGON_SANDBOX_DIGEST="$(cat "${SANDBOX}/checksum")"
+# Ignore the host TMPDIR - it might be something funny like /run/user/1000,
+# we want it to be /tmp inside the sandbox.
+export TMPDIR=/tmp
+
exec -a "$0" "${BAZEL_REAL}" "$@"