third_party/nix: move overrides to toolchain-bundle derivation
We have multiple consumers of nixpkgs. The nix-shell for development
and our toolchain-bundle. To reduce the amount of applied overlays
in normal evaluation, we move all overrides/patches that are only
required for the toolchain bundle to its definition. Additionally
all small overrides get inlined as its actually more easy to read.
I also refactored the way the toolchain-bundle is constructed to make
it easier to extend.
Change-Id: If7daafb6de43d26a0b95d0248cfb8c573cc5bbbe
Reviewed-on: https://review.monogon.dev/c/monogon/+/4457
Reviewed-by: Leopold Schabel <leo@monogon.tech>
Tested-by: Jenkins CI
diff --git a/build/toolchain/toolchain-bundle/default.nix b/build/toolchain/toolchain-bundle/default.nix
index 794b6d2..1b016c5 100644
--- a/build/toolchain/toolchain-bundle/default.nix
+++ b/build/toolchain/toolchain-bundle/default.nix
@@ -1,53 +1,110 @@
-{ pkgs ? import ../../../third_party/nix/default.nix { } }: with pkgs;
-symlinkJoin {
+let
+ # We want our overrides to only apply when building for static environments.
+ conditionalOverlay = condition: overlay: (if condition then overlay else { });
+
+ pkgs = import ../../../third_party/nix/default.nix {
+ overlays = [
+ # Overrides for allowing static builds
+ (self: super: conditionalOverlay super.stdenv.hostPlatform.isStatic (with self; {
+ # A minimal version of qemu allowing for static builds.
+ qemu-minimal = self.callPackage ./pkgs/qemu { inherit super; };
+
+ # Static perl builds are a rabbit hole as they need patches
+ # and use of undocumented options. Check the derivation for more infos.
+ perl = self.callPackage ./pkgs/perl { inherit super; };
+
+ # Bison requires an override for not hardcoding nix paths.
+ bison = self.callPackage ./pkgs/bison { inherit super; };
+
+ # Provide a custom minimal version of util-linux
+ util-linux-minimal = super.util-linux.override (old: {
+ pamSupport = false;
+ ncursesSupport = false;
+ capabilitiesSupport = false;
+ systemdSupport = false;
+ translateManpages = false;
+ nlsSupport = false;
+ shadowSupport = false;
+ writeSupport = false;
+ });
+
+ # Revert "fixup" which hardcodes a nix path.
+ python3Minimal = super.python3Minimal.overrideAttrs (old: {
+ postPatch = old.postPatch + ''
+ substituteInPlace Lib/subprocess.py \
+ --replace-fail "'${bashNonInteractive}/bin/sh'" "'/bin/sh'"
+ '';
+ });
+
+ # Disable tests as they fail when static build.
+ diffutils = super.diffutils.overrideAttrs (_: {
+ doCheck = false;
+ doInstallCheck = false;
+ });
+
+ # vde2 currently doesn't build without these additional flags.
+ vde2 = super.vde2.overrideAttrs (oldAttrs: {
+ env.NIX_CFLAGS_COMPILE = (oldAttrs.NIX_CFLAGS_COMPILE or "") + " -Wno-error=int-conversion -Wno-error=implicit-function-declaration";
+ });
+ }))
+ ];
+
+ config.replaceCrossStdenv = { buildPackages, baseStdenv }:
+ (buildPackages.withCFlags [ "-fPIC" ]) baseStdenv;
+ };
+
+ # All platforms we want to build for.
+ mkPlatforms = platforms: with platforms; [
+ aarch64-multiplatform-musl
+ musl64
+ ];
+
+ # All packages that we want in our bundle.
+ mkPackages = platformPkgs: with platformPkgs; [
+ gnumake
+ flex
+ bison
+ lz4
+ busybox
+ findutils
+ bc
+ util-linux-minimal # custom pkg
+ perl
+ nasm
+ acpica-tools
+ patch
+ diffutils
+ qemu-minimal # custom pkg
+ m4
+ strace
+ python3Minimal
+ ];
+
+ mkPackagesEnv = platform: pkgs.buildEnv {
+ name = "toolchain-${platform.hostPlatform.config}";
+ paths = mkPackages platform.pkgsStatic;
+ };
+
+ mkBundle = platform: pkgs.stdenv.mkDerivation rec {
+ name = "toolchain-bundle-${platform.hostPlatform.config}";
+ buildInputs = [ pkgs.gnutar pkgs.zstd ];
+
+ phases = [ "buildPhase" ];
+ buildPhase =
+ let
+ merged = mkPackagesEnv platform;
+ in
+ ''
+ mkdir $out
+ tar --zstd --sort=name --hard-dereference -hcf $out/${name}.tar.zst -C ${merged} .
+ '';
+ };
+in
+with pkgs; symlinkJoin {
name = "toolchain";
paths =
let
- platforms = with pkgsCross; [
- aarch64-multiplatform-musl
- musl64
- ];
+ platforms = mkPlatforms pkgs.pkgsCross;
in
- map
- (platform: (
- let
- merged = buildEnv {
- name = "toolchain-env";
- paths = with platform.pkgsStatic; [
- gnumake
- flex
- bison
- lz4
- busybox
- findutils
- bc
- util-linux-minimal # custom pkg
- perl
- nasm
- acpica-tools
- patch
- diffutils
- qemu-minimal # custom pkg
- m4
- strace
- python3Minimal
- ];
- };
- in
- stdenv.mkDerivation rec {
- name = "toolchain-bundle";
- buildInputs = [ gnutar zstd ];
-
- phases = [ "buildPhase" "installPhase" ];
- buildPhase = ''
- tar --zstd --sort=name --hard-dereference -hcf bundle.tar.zst -C ${merged} .
- '';
-
- installPhase = ''
- mkdir $out
- mv bundle.tar.zst $out/${name}-${platform.hostPlatform.config}.tar.zst
- '';
- }
- ))
- platforms;
+ map mkBundle platforms;
}
diff --git a/third_party/nix/pkgs/bison/BUILD.bazel b/build/toolchain/toolchain-bundle/pkgs/bison/BUILD.bazel
similarity index 100%
rename from third_party/nix/pkgs/bison/BUILD.bazel
rename to build/toolchain/toolchain-bundle/pkgs/bison/BUILD.bazel
diff --git a/third_party/nix/pkgs/bison/default.nix b/build/toolchain/toolchain-bundle/pkgs/bison/default.nix
similarity index 61%
rename from third_party/nix/pkgs/bison/default.nix
rename to build/toolchain/toolchain-bundle/pkgs/bison/default.nix
index 8b2245e..9132a30 100644
--- a/third_party/nix/pkgs/bison/default.nix
+++ b/build/toolchain/toolchain-bundle/pkgs/bison/default.nix
@@ -1,6 +1,5 @@
-{ pkgs }: with pkgs;
-if (!stdenv.hostPlatform.isStatic) then bison else
-bison.overrideAttrs (old: {
+{ super, ... }:
+super.bison.overrideAttrs (_: {
# Check overrided file for more informations
postPatch = ''
cp ${./yacc.in} src/yacc.in
diff --git a/third_party/nix/pkgs/bison/yacc.in b/build/toolchain/toolchain-bundle/pkgs/bison/yacc.in
similarity index 100%
rename from third_party/nix/pkgs/bison/yacc.in
rename to build/toolchain/toolchain-bundle/pkgs/bison/yacc.in
diff --git a/third_party/nix/pkgs/perl/BUILD.bazel b/build/toolchain/toolchain-bundle/pkgs/perl/BUILD.bazel
similarity index 100%
rename from third_party/nix/pkgs/perl/BUILD.bazel
rename to build/toolchain/toolchain-bundle/pkgs/perl/BUILD.bazel
diff --git a/third_party/nix/pkgs/perl/default.nix b/build/toolchain/toolchain-bundle/pkgs/perl/default.nix
similarity index 72%
rename from third_party/nix/pkgs/perl/default.nix
rename to build/toolchain/toolchain-bundle/pkgs/perl/default.nix
index c2677b5..e8f5a0d 100644
--- a/third_party/nix/pkgs/perl/default.nix
+++ b/build/toolchain/toolchain-bundle/pkgs/perl/default.nix
@@ -1,5 +1,9 @@
-{ pkgs }: with pkgs;
-if (!stdenv.hostPlatform.isStatic) then perl else
+{ lib, super, ... }@inputs:
+let
+ # Passthrough default configuration without our custom super attribute. Perl
+ # requires itself which breaks when we don't pass through the default attributes.
+ perl = super.perl.override (_: (lib.filterAttrs (name: _: name != "super") inputs));
+in
perl.overrideAttrs (old: {
patches = old.patches ++ [
./static_build.patch
diff --git a/third_party/nix/pkgs/perl/static_build.patch b/build/toolchain/toolchain-bundle/pkgs/perl/static_build.patch
similarity index 100%
rename from third_party/nix/pkgs/perl/static_build.patch
rename to build/toolchain/toolchain-bundle/pkgs/perl/static_build.patch
diff --git a/third_party/nix/pkgs/qemu/BUILD.bazel b/build/toolchain/toolchain-bundle/pkgs/qemu/BUILD.bazel
similarity index 100%
rename from third_party/nix/pkgs/qemu/BUILD.bazel
rename to build/toolchain/toolchain-bundle/pkgs/qemu/BUILD.bazel
diff --git a/third_party/nix/pkgs/qemu/default.nix b/build/toolchain/toolchain-bundle/pkgs/qemu/default.nix
similarity index 94%
rename from third_party/nix/pkgs/qemu/default.nix
rename to build/toolchain/toolchain-bundle/pkgs/qemu/default.nix
index 5134cb7..81f8825 100644
--- a/third_party/nix/pkgs/qemu/default.nix
+++ b/build/toolchain/toolchain-bundle/pkgs/qemu/default.nix
@@ -1,5 +1,4 @@
-{ pkgs }: with pkgs;
-
+{ qemu_kvm, audit, ... }:
let
qemuMinimal = qemu_kvm.override (old: {
hostCpuOnly = true;
@@ -43,6 +42,6 @@
configureFlags = (builtins.filter (v: v != "--static") old.configureFlags) ++ [ "--disable-libcbor" ];
strictDeps = true;
# a private dependency of PAM which is not linked explicitly in static builds
- buildInputs = old.buildInputs ++ [ pkgs.audit ];
+ buildInputs = old.buildInputs ++ [ audit ];
env.NIX_LDFLAGS = " -laudit ";
})
diff --git a/third_party/nix/pkgs/qemu/static_build_crc32c_duplicate_definition.patch b/build/toolchain/toolchain-bundle/pkgs/qemu/static_build_crc32c_duplicate_definition.patch
similarity index 100%
rename from third_party/nix/pkgs/qemu/static_build_crc32c_duplicate_definition.patch
rename to build/toolchain/toolchain-bundle/pkgs/qemu/static_build_crc32c_duplicate_definition.patch
diff --git a/third_party/nix/default.nix b/third_party/nix/default.nix
index 726119e..f2650b3 100644
--- a/third_party/nix/default.nix
+++ b/third_party/nix/default.nix
@@ -1,27 +1,13 @@
-{ sources ? import ./sources.nix }:
+{ sources ? import ./sources.nix, overlays ? [ ], config ? { } }:
let
pkgs = import sources.nixpkgs
{
- overlays = [
+ overlays = overlays ++ [
(self: super: {
- qemu-minimal = import ./pkgs/qemu { pkgs = super; };
- diffutils = import ./pkgs/diffutils { pkgs = super; };
- util-linux-minimal = import ./pkgs/util-linux { pkgs = super; };
- bazel-unwrapped = import ./pkgs/bazel { pkgs = super; };
- perl = import ./pkgs/perl { pkgs = super; };
bazel_8 = self.callPackage ./pkgs/bazel_8/package.nix { };
- python3Minimal = import ./pkgs/python3 { pkgs = super; };
- bison = import ./pkgs/bison { pkgs = super; };
- })
- (self: super: {
- vde2 = super.vde2.overrideAttrs (oldAttrs: {
- env.NIX_CFLAGS_COMPILE = (oldAttrs.NIX_CFLAGS_COMPILE or "") + " -Wno-error=int-conversion -Wno-error=implicit-function-declaration";
- });
})
];
-
- config.replaceCrossStdenv = { buildPackages, baseStdenv }:
- (buildPackages.withCFlags [ "-fPIC" ]) baseStdenv;
+ config = config;
};
in
pkgs // {
diff --git a/third_party/nix/pkgs/diffutils/BUILD.bazel b/third_party/nix/pkgs/diffutils/BUILD.bazel
deleted file mode 100644
index e69de29..0000000
--- a/third_party/nix/pkgs/diffutils/BUILD.bazel
+++ /dev/null
diff --git a/third_party/nix/pkgs/diffutils/default.nix b/third_party/nix/pkgs/diffutils/default.nix
deleted file mode 100644
index bbb38e3..0000000
--- a/third_party/nix/pkgs/diffutils/default.nix
+++ /dev/null
@@ -1,39 +0,0 @@
-{ pkgs }: with pkgs;
-if (!stdenv.hostPlatform.isStatic) then diffutils else
-diffutils.overrideAttrs (old: {
- # Disable tests as they fail when static build.
-
- # FAIL: test-getopt-gnu
- #=====================
- #
- #test-getopt.h:661: assertion 'optind == 2' failed
- #FAIL test-getopt-gnu (exit status: 134)
- #
- #FAIL: test-getopt-posix
- #=======================
- #
- #test-getopt.h:661: assertion 'optind == 2' failed
- #FAIL test-getopt-posix (exit status: 134)
- #
- #FAIL: test-nl_langinfo-mt
- #=========================
- #
- #FAIL test-nl_langinfo-mt (exit status: 134)
- #
- #FAIL: test-random-mt
- #====================
- #
- #FAIL test-random-mt (exit status: 134)
- #
- #FAIL: test-setlocale_null-mt-one
- #================================
- #
- #FAIL test-setlocale_null-mt-one (exit status: 134)
- #
- #FAIL: test-setlocale_null-mt-all
- #================================
- #
- #FAIL test-setlocale_null-mt-all (exit status: 134)
- doCheck = false;
- doInstallCheck = false;
-})
diff --git a/third_party/nix/pkgs/python3/BUILD.bazel b/third_party/nix/pkgs/python3/BUILD.bazel
deleted file mode 100644
index e69de29..0000000
--- a/third_party/nix/pkgs/python3/BUILD.bazel
+++ /dev/null
diff --git a/third_party/nix/pkgs/python3/default.nix b/third_party/nix/pkgs/python3/default.nix
deleted file mode 100644
index 3102b88..0000000
--- a/third_party/nix/pkgs/python3/default.nix
+++ /dev/null
@@ -1,10 +0,0 @@
-{ pkgs }: with pkgs;
-# Only override for our actual build
-if (!stdenv.hostPlatform.isStatic) then python3Minimal else
-python3Minimal.overrideAttrs (old: {
- # Revert "fixup" which hardcodes a nix path.
- postPatch = old.postPatch + ''
- substituteInPlace Lib/subprocess.py \
- --replace-fail "'${bashNonInteractive}/bin/sh'" "'/bin/sh'"
- '';
-})
diff --git a/third_party/nix/pkgs/util-linux/BUILD.bazel b/third_party/nix/pkgs/util-linux/BUILD.bazel
deleted file mode 100644
index e69de29..0000000
--- a/third_party/nix/pkgs/util-linux/BUILD.bazel
+++ /dev/null
diff --git a/third_party/nix/pkgs/util-linux/default.nix b/third_party/nix/pkgs/util-linux/default.nix
deleted file mode 100644
index 1935cd3..0000000
--- a/third_party/nix/pkgs/util-linux/default.nix
+++ /dev/null
@@ -1,11 +0,0 @@
-{ pkgs }: with pkgs;
-util-linux.override (old: {
- pamSupport = false;
- ncursesSupport = false;
- capabilitiesSupport = false;
- systemdSupport = false;
- translateManpages = false;
- nlsSupport = false;
- shadowSupport = false;
- writeSupport = false;
-})