third_party/nix: move overrides to toolchain-bundle derivation
We have multiple consumers of nixpkgs. The nix-shell for development
and our toolchain-bundle. To reduce the amount of applied overlays
in normal evaluation, we move all overrides/patches that are only
required for the toolchain bundle to its definition. Additionally
all small overrides get inlined as its actually more easy to read.
I also refactored the way the toolchain-bundle is constructed to make
it easier to extend.
Change-Id: If7daafb6de43d26a0b95d0248cfb8c573cc5bbbe
Reviewed-on: https://review.monogon.dev/c/monogon/+/4457
Reviewed-by: Leopold Schabel <leo@monogon.tech>
Tested-by: Jenkins CI
diff --git a/build/toolchain/toolchain-bundle/default.nix b/build/toolchain/toolchain-bundle/default.nix
index 794b6d2..1b016c5 100644
--- a/build/toolchain/toolchain-bundle/default.nix
+++ b/build/toolchain/toolchain-bundle/default.nix
@@ -1,53 +1,110 @@
-{ pkgs ? import ../../../third_party/nix/default.nix { } }: with pkgs;
-symlinkJoin {
+let
+ # We want our overrides to only apply when building for static environments.
+ conditionalOverlay = condition: overlay: (if condition then overlay else { });
+
+ pkgs = import ../../../third_party/nix/default.nix {
+ overlays = [
+ # Overrides for allowing static builds
+ (self: super: conditionalOverlay super.stdenv.hostPlatform.isStatic (with self; {
+ # A minimal version of qemu allowing for static builds.
+ qemu-minimal = self.callPackage ./pkgs/qemu { inherit super; };
+
+ # Static perl builds are a rabbit hole as they need patches
+ # and use of undocumented options. Check the derivation for more infos.
+ perl = self.callPackage ./pkgs/perl { inherit super; };
+
+ # Bison requires an override for not hardcoding nix paths.
+ bison = self.callPackage ./pkgs/bison { inherit super; };
+
+ # Provide a custom minimal version of util-linux
+ util-linux-minimal = super.util-linux.override (old: {
+ pamSupport = false;
+ ncursesSupport = false;
+ capabilitiesSupport = false;
+ systemdSupport = false;
+ translateManpages = false;
+ nlsSupport = false;
+ shadowSupport = false;
+ writeSupport = false;
+ });
+
+ # Revert "fixup" which hardcodes a nix path.
+ python3Minimal = super.python3Minimal.overrideAttrs (old: {
+ postPatch = old.postPatch + ''
+ substituteInPlace Lib/subprocess.py \
+ --replace-fail "'${bashNonInteractive}/bin/sh'" "'/bin/sh'"
+ '';
+ });
+
+ # Disable tests as they fail when static build.
+ diffutils = super.diffutils.overrideAttrs (_: {
+ doCheck = false;
+ doInstallCheck = false;
+ });
+
+ # vde2 currently doesn't build without these additional flags.
+ vde2 = super.vde2.overrideAttrs (oldAttrs: {
+ env.NIX_CFLAGS_COMPILE = (oldAttrs.NIX_CFLAGS_COMPILE or "") + " -Wno-error=int-conversion -Wno-error=implicit-function-declaration";
+ });
+ }))
+ ];
+
+ config.replaceCrossStdenv = { buildPackages, baseStdenv }:
+ (buildPackages.withCFlags [ "-fPIC" ]) baseStdenv;
+ };
+
+ # All platforms we want to build for.
+ mkPlatforms = platforms: with platforms; [
+ aarch64-multiplatform-musl
+ musl64
+ ];
+
+ # All packages that we want in our bundle.
+ mkPackages = platformPkgs: with platformPkgs; [
+ gnumake
+ flex
+ bison
+ lz4
+ busybox
+ findutils
+ bc
+ util-linux-minimal # custom pkg
+ perl
+ nasm
+ acpica-tools
+ patch
+ diffutils
+ qemu-minimal # custom pkg
+ m4
+ strace
+ python3Minimal
+ ];
+
+ mkPackagesEnv = platform: pkgs.buildEnv {
+ name = "toolchain-${platform.hostPlatform.config}";
+ paths = mkPackages platform.pkgsStatic;
+ };
+
+ mkBundle = platform: pkgs.stdenv.mkDerivation rec {
+ name = "toolchain-bundle-${platform.hostPlatform.config}";
+ buildInputs = [ pkgs.gnutar pkgs.zstd ];
+
+ phases = [ "buildPhase" ];
+ buildPhase =
+ let
+ merged = mkPackagesEnv platform;
+ in
+ ''
+ mkdir $out
+ tar --zstd --sort=name --hard-dereference -hcf $out/${name}.tar.zst -C ${merged} .
+ '';
+ };
+in
+with pkgs; symlinkJoin {
name = "toolchain";
paths =
let
- platforms = with pkgsCross; [
- aarch64-multiplatform-musl
- musl64
- ];
+ platforms = mkPlatforms pkgs.pkgsCross;
in
- map
- (platform: (
- let
- merged = buildEnv {
- name = "toolchain-env";
- paths = with platform.pkgsStatic; [
- gnumake
- flex
- bison
- lz4
- busybox
- findutils
- bc
- util-linux-minimal # custom pkg
- perl
- nasm
- acpica-tools
- patch
- diffutils
- qemu-minimal # custom pkg
- m4
- strace
- python3Minimal
- ];
- };
- in
- stdenv.mkDerivation rec {
- name = "toolchain-bundle";
- buildInputs = [ gnutar zstd ];
-
- phases = [ "buildPhase" "installPhase" ];
- buildPhase = ''
- tar --zstd --sort=name --hard-dereference -hcf bundle.tar.zst -C ${merged} .
- '';
-
- installPhase = ''
- mkdir $out
- mv bundle.tar.zst $out/${name}-${platform.hostPlatform.config}.tar.zst
- '';
- }
- ))
- platforms;
+ map mkBundle platforms;
}
diff --git a/build/toolchain/toolchain-bundle/pkgs/bison/BUILD.bazel b/build/toolchain/toolchain-bundle/pkgs/bison/BUILD.bazel
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/build/toolchain/toolchain-bundle/pkgs/bison/BUILD.bazel
diff --git a/build/toolchain/toolchain-bundle/pkgs/bison/default.nix b/build/toolchain/toolchain-bundle/pkgs/bison/default.nix
new file mode 100644
index 0000000..9132a30
--- /dev/null
+++ b/build/toolchain/toolchain-bundle/pkgs/bison/default.nix
@@ -0,0 +1,9 @@
+{ super, ... }:
+super.bison.overrideAttrs (_: {
+ # Check overrided file for more informations
+ postPatch = ''
+ cp ${./yacc.in} src/yacc.in
+ '';
+
+ env.BISON = "${builtins.placeholder "out"}/bin/bison";
+})
diff --git a/build/toolchain/toolchain-bundle/pkgs/bison/yacc.in b/build/toolchain/toolchain-bundle/pkgs/bison/yacc.in
new file mode 100644
index 0000000..b6ff366
--- /dev/null
+++ b/build/toolchain/toolchain-bundle/pkgs/bison/yacc.in
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+# The exec-prefix is used by YACC to find Bison at runtime.
+# Since we don't exactly know that, we have to depend on an
+# environment variable we set in Bazel.
+exec "$BISON" -y "$@"
\ No newline at end of file
diff --git a/build/toolchain/toolchain-bundle/pkgs/perl/BUILD.bazel b/build/toolchain/toolchain-bundle/pkgs/perl/BUILD.bazel
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/build/toolchain/toolchain-bundle/pkgs/perl/BUILD.bazel
diff --git a/build/toolchain/toolchain-bundle/pkgs/perl/default.nix b/build/toolchain/toolchain-bundle/pkgs/perl/default.nix
new file mode 100644
index 0000000..e8f5a0d
--- /dev/null
+++ b/build/toolchain/toolchain-bundle/pkgs/perl/default.nix
@@ -0,0 +1,28 @@
+{ lib, super, ... }@inputs:
+let
+ # Passthrough default configuration without our custom super attribute. Perl
+ # requires itself which breaks when we don't pass through the default attributes.
+ perl = super.perl.override (_: (lib.filterAttrs (name: _: name != "super") inputs));
+in
+perl.overrideAttrs (old: {
+ patches = old.patches ++ [
+ ./static_build.patch
+ ];
+
+ preConfigure = old.preConfigure + ''
+ cat >> config.over <<EOF
+ osvers="musllinux"
+ EOF
+ '';
+
+ configureFlags = old.configureFlags ++ [
+ "-Dotherlibdirs=.../../lib/perl5/${old.version}" # Tell perl to use a relative libdir
+ # 1. Why isn't this the default?
+ # 2. Apparently nobody uses this option, because it is missing the quotes inside the config_h.SH
+ # 3. Why should a variable called "procselfexe" be used with a different path than /proc/self/exe?
+ # 4. I really dislike perl. - fionera
+ "-Dprocselfexe=\"/proc/self/exe\""
+ ];
+
+ env.NIX_CFLAGS_COMPILE = (old.NIX_CFLAGS_COMPILE or "") + " -Wno-error=implicit-function-declaration";
+})
diff --git a/build/toolchain/toolchain-bundle/pkgs/perl/static_build.patch b/build/toolchain/toolchain-bundle/pkgs/perl/static_build.patch
new file mode 100644
index 0000000..8562a3f
--- /dev/null
+++ b/build/toolchain/toolchain-bundle/pkgs/perl/static_build.patch
@@ -0,0 +1,13 @@
+diff --git a/ext/re/Makefile.PL b/ext/re/Makefile.PL
+--- a/ext/re/Makefile.PL
++++ b/ext/re/Makefile.PL
+@@ -27,8 +27,9 @@
+ }
+ }
+
+ my $defines = '-DPERL_EXT_RE_BUILD -DPERL_EXT_RE_DEBUG -DPERL_EXT';
++$defines .= ' -DPERL_EXT_RE_STATIC';
+ my %args;
+ for my $arg (@ARGV) {
+ $args{$1} = $2 if $arg =~ /^(\w+)=(.*)$/;
+ }
diff --git a/build/toolchain/toolchain-bundle/pkgs/qemu/BUILD.bazel b/build/toolchain/toolchain-bundle/pkgs/qemu/BUILD.bazel
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/build/toolchain/toolchain-bundle/pkgs/qemu/BUILD.bazel
diff --git a/build/toolchain/toolchain-bundle/pkgs/qemu/default.nix b/build/toolchain/toolchain-bundle/pkgs/qemu/default.nix
new file mode 100644
index 0000000..81f8825
--- /dev/null
+++ b/build/toolchain/toolchain-bundle/pkgs/qemu/default.nix
@@ -0,0 +1,47 @@
+{ qemu_kvm, audit, ... }:
+let
+ qemuMinimal = qemu_kvm.override (old: {
+ hostCpuOnly = true;
+ vncSupport = true;
+
+ # Disable everything we don't need.
+ enableDocs = false;
+ ncursesSupport = false;
+ seccompSupport = false;
+ numaSupport = false;
+ alsaSupport = false;
+ pulseSupport = false;
+ pipewireSupport = false;
+ sdlSupport = false;
+ jackSupport = false;
+ gtkSupport = false;
+ smartcardSupport = false;
+ spiceSupport = false;
+ usbredirSupport = false;
+ xenSupport = false;
+ cephSupport = false;
+ glusterfsSupport = false;
+ openGLSupport = false;
+ rutabagaSupport = false;
+ virglSupport = false;
+ libiscsiSupport = false;
+ smbdSupport = false;
+ uringSupport = false;
+ canokeySupport = false;
+ capstoneSupport = false;
+ });
+in
+qemuMinimal.overrideAttrs (old: {
+ # Static build patch
+ # Based on https://github.com/NixOS/nixpkgs/pull/333923
+
+ patches = (old.patches ++ [
+ ./static_build_crc32c_duplicate_definition.patch
+ ]);
+
+ configureFlags = (builtins.filter (v: v != "--static") old.configureFlags) ++ [ "--disable-libcbor" ];
+ strictDeps = true;
+ # a private dependency of PAM which is not linked explicitly in static builds
+ buildInputs = old.buildInputs ++ [ audit ];
+ env.NIX_LDFLAGS = " -laudit ";
+})
diff --git a/build/toolchain/toolchain-bundle/pkgs/qemu/static_build_crc32c_duplicate_definition.patch b/build/toolchain/toolchain-bundle/pkgs/qemu/static_build_crc32c_duplicate_definition.patch
new file mode 100644
index 0000000..8e6c033
--- /dev/null
+++ b/build/toolchain/toolchain-bundle/pkgs/qemu/static_build_crc32c_duplicate_definition.patch
@@ -0,0 +1,117 @@
+commit 0ba0f342e2f3cb1d271d324d999d80d5c2834f2b
+Author: Guillaume Girol <symphorien+git@xlumurb.eu>
+Date: Sun Aug 11 12:00:00 2024 +0000
+
+ rename crc32c to a less generic name
+
+ when compiling qemu statically, this symbol clashes to one in libblkid.
+
+diff --git a/block/vhdx.c b/block/vhdx.c
+index 5aa1a13506..0dc9df217d 100644
+--- a/block/vhdx.c
++++ b/block/vhdx.c
+@@ -157,7 +157,7 @@ uint32_t vhdx_update_checksum(uint8_t *buf, size_t size, int crc_offset)
+ assert(size > (crc_offset + sizeof(crc)));
+
+ memset(buf + crc_offset, 0, sizeof(crc));
+- crc = crc32c(0xffffffff, buf, size);
++ crc = qemu_crc32c(0xffffffff, buf, size);
+ crc = cpu_to_le32(crc);
+ memcpy(buf + crc_offset, &crc, sizeof(crc));
+
+@@ -176,7 +176,7 @@ uint32_t vhdx_checksum_calc(uint32_t crc, uint8_t *buf, size_t size,
+ memset(buf + crc_offset, 0, sizeof(crc_orig));
+ }
+
+- crc_new = crc32c(crc, buf, size);
++ crc_new = qemu_crc32c(crc, buf, size);
+ if (crc_offset > 0) {
+ memcpy(buf + crc_offset, &crc_orig, sizeof(crc_orig));
+ }
+diff --git a/hw/net/net_rx_pkt.c b/hw/net/net_rx_pkt.c
+index 32e5f3f9cf..a53238e143 100644
+--- a/hw/net/net_rx_pkt.c
++++ b/hw/net/net_rx_pkt.c
+@@ -579,7 +579,7 @@ _net_rx_pkt_validate_sctp_sum(struct NetRxPkt *pkt)
+ return false;
+ }
+
+- calculated = crc32c(0xffffffff,
++ calculated = qemu_crc32c(0xffffffff,
+ (uint8_t *)vec->iov_base + off, vec->iov_len - off);
+ calculated = iov_crc32c(calculated ^ 0xffffffff, vec + 1, vec_len - 1);
+ valid = calculated == le32_to_cpu(original);
+diff --git a/include/qemu/crc32c.h b/include/qemu/crc32c.h
+index 88b4d2b3b3..b0f535c80e 100644
+--- a/include/qemu/crc32c.h
++++ b/include/qemu/crc32c.h
+@@ -29,7 +29,7 @@
+ #define QEMU_CRC32C_H
+
+
+-uint32_t crc32c(uint32_t crc, const uint8_t *data, unsigned int length);
++uint32_t qemu_crc32c(uint32_t crc, const uint8_t *data, unsigned int length);
+ uint32_t iov_crc32c(uint32_t crc, const struct iovec *iov, size_t iov_cnt);
+
+ #endif
+diff --git a/target/arm/helper.c b/target/arm/helper.c
+index 8fb4b474e8..cc5b2a8e99 100644
+--- a/target/arm/helper.c
++++ b/target/arm/helper.c
+@@ -12409,7 +12409,7 @@ uint32_t HELPER(crc32c)(uint32_t acc, uint32_t val, uint32_t bytes)
+ stl_le_p(buf, val);
+
+ /* Linux crc32c converts the output to one's complement. */
+- return crc32c(acc, buf, bytes) ^ 0xffffffff;
++ return qemu_crc32c(acc, buf, bytes) ^ 0xffffffff;
+ }
+
+ /*
+diff --git a/target/arm/tcg/helper-a64.c b/target/arm/tcg/helper-a64.c
+index c60d2a7ec9..d64912d4eb 100644
+--- a/target/arm/tcg/helper-a64.c
++++ b/target/arm/tcg/helper-a64.c
+@@ -514,7 +514,7 @@ uint64_t HELPER(crc32c_64)(uint64_t acc, uint64_t val, uint32_t bytes)
+ stq_le_p(buf, val);
+
+ /* Linux crc32c converts the output to one's complement. */
+- return crc32c(acc, buf, bytes) ^ 0xffffffff;
++ return qemu_crc32c(acc, buf, bytes) ^ 0xffffffff;
+ }
+
+ /*
+diff --git a/target/loongarch/tcg/op_helper.c b/target/loongarch/tcg/op_helper.c
+index fe79c62fa4..a90db6f4b9 100644
+--- a/target/loongarch/tcg/op_helper.c
++++ b/target/loongarch/tcg/op_helper.c
+@@ -77,7 +77,7 @@ target_ulong helper_crc32c(target_ulong val, target_ulong m, uint64_t sz)
+ target_ulong mask = ((sz * 8) == 64) ? -1ULL : ((1ULL << (sz * 8)) - 1);
+ m &= mask;
+ stq_le_p(buf, m);
+- return (int32_t) (crc32c(val, buf, sz) ^ 0xffffffff);
++ return (int32_t) (qemu_crc32c(val, buf, sz) ^ 0xffffffff);
+ }
+
+ target_ulong helper_cpucfg(CPULoongArchState *env, target_ulong rj)
+diff --git a/util/crc32c.c b/util/crc32c.c
+index ea7f345de8..2780a5c698 100644
+--- a/util/crc32c.c
++++ b/util/crc32c.c
+@@ -105,7 +105,7 @@ static const uint32_t crc32c_table[256] = {
+ };
+
+
+-uint32_t crc32c(uint32_t crc, const uint8_t *data, unsigned int length)
++uint32_t qemu_crc32c(uint32_t crc, const uint8_t *data, unsigned int length)
+ {
+ while (length--) {
+ crc = crc32c_table[(crc ^ *data++) & 0xFFL] ^ (crc >> 8);
+@@ -116,7 +116,7 @@ uint32_t crc32c(uint32_t crc, const uint8_t *data, unsigned int length)
+ uint32_t iov_crc32c(uint32_t crc, const struct iovec *iov, size_t iov_cnt)
+ {
+ while (iov_cnt--) {
+- crc = crc32c(crc, iov->iov_base, iov->iov_len) ^ 0xffffffff;
++ crc = qemu_crc32c(crc, iov->iov_base, iov->iov_len) ^ 0xffffffff;
+ iov++;
+ }
+ return crc ^ 0xffffffff;