m/node: build Linux with modules
This introduces modules into our Linux build. I originally didn't want
to do this, this is why this wasn't done until now. But various things
in the kernel weren't set up for this, for example the AMD and Intel KVM
modules cannot both be loaded, only the first one loaded works. Also,
the Linux kernel cannot load firmware for built-in modules reliably as
the filesystem it tries to load it from is not always mounted first,
even if the kernel itself mounts it.
The firmware issue was brought up multiple times on LKML, but Linus is
of the opinion that the firmware should be next to the kernel module,
thus either built-in (not viable for licensing and size reasons) or the
modules need to be loadable and on the same filesystem as the firmware.
Thus unless we want to carry signifcant patches against the Kernel in a
deadlock-prone area, we are forced to adopt a design with loadable
modules (or ship everything twice in an initramfs which is also not
desirable).
The kernel config currently only has the modules as non-builtin which
require firmware, everything else has been left as-is. For boot-time
performance it would eventually be a good idea to move to a setup with
more modules once we're confident in the implementation and everything
can deal with late-loaded modules/devices.
As a drive-by fix this also moves the kernel builds to out-of-tree so
that we no longer pollute the source folder. Bazel protected us from
serious issues due to this, but it's still bad practice.
Change-Id: Iced8e12234565e5b7447e732716651e05e67d55b
Reviewed-on: https://review.monogon.dev/c/monogon/+/1791
Reviewed-by: Serge Bazanski <serge@monogon.tech>
Tested-by: Jenkins CI
diff --git a/metropolis/node/build/fwprune/BUILD.bazel b/metropolis/node/build/fwprune/BUILD.bazel
index 578b829..bc22665 100644
--- a/metropolis/node/build/fwprune/BUILD.bazel
+++ b/metropolis/node/build/fwprune/BUILD.bazel
@@ -7,7 +7,9 @@
visibility = ["//visibility:private"],
deps = [
"//metropolis/node/build/fsspec",
+ "//metropolis/pkg/kmod",
"@org_golang_google_protobuf//encoding/prototext",
+ "@org_golang_google_protobuf//proto",
],
)
diff --git a/metropolis/node/build/fwprune/def.bzl b/metropolis/node/build/fwprune/def.bzl
index 507532e..154009e 100644
--- a/metropolis/node/build/fwprune/def.bzl
+++ b/metropolis/node/build/fwprune/def.bzl
@@ -10,16 +10,32 @@
)
modinfo = ctx.attr.kernel[OutputGroupInfo].modinfo.to_list()[0]
+ modules = ctx.attr.kernel[OutputGroupInfo].modules.to_list()[0]
+
+ meta_out = ctx.actions.declare_file(ctx.label.name + "-meta.pb")
ctx.actions.run(
- outputs = [fsspec_out],
- inputs = [fwlist, modinfo, ctx.file.metadata] + ctx.files.firmware_files,
+ outputs = [fsspec_out, meta_out],
+ inputs = [fwlist, modinfo, modules, ctx.file.metadata] + ctx.files.firmware_files,
tools = [ctx.executable._fwprune],
executable = ctx.executable._fwprune,
- arguments = [modinfo.path, fwlist.path, ctx.file.metadata.path, fsspec_out.path],
+ arguments = [
+ "-modinfo",
+ modinfo.path,
+ "-modules",
+ modules.path,
+ "-firmware-file-list",
+ fwlist.path,
+ "-firmware-whence",
+ ctx.file.metadata.path,
+ "-out-meta",
+ meta_out.path,
+ "-out-fsspec",
+ fsspec_out.path,
+ ],
)
- return [DefaultInfo(files = depset([fsspec_out])), FSSpecInfo(spec = fsspec_out, referenced = ctx.files.firmware_files)]
+ return [DefaultInfo(files = depset([fsspec_out])), FSSpecInfo(spec = fsspec_out, referenced = ctx.files.firmware_files + [modules, meta_out])]
fsspec_linux_firmware = rule(
implementation = _fsspec_linux_firmware,
diff --git a/metropolis/node/build/fwprune/main.go b/metropolis/node/build/fwprune/main.go
index 861a2d0..c8c8e9c 100644
--- a/metropolis/node/build/fwprune/main.go
+++ b/metropolis/node/build/fwprune/main.go
@@ -5,82 +5,46 @@
package main
import (
- "bytes"
+ "debug/elf"
+ "flag"
+ "io/fs"
"log"
"os"
"path"
+ "path/filepath"
"regexp"
"sort"
"strings"
"google.golang.org/protobuf/encoding/prototext"
+ "google.golang.org/protobuf/proto"
"source.monogon.dev/metropolis/node/build/fsspec"
+ "source.monogon.dev/metropolis/pkg/kmod"
)
+// linkRegexp parses the Link: lines in the WHENCE file. This does not have
+// an official grammar, the regexp has been written in an approximation of
+// the original parsing algorithm at @linux-firmware//:copy_firmware.sh.
+var linkRegexp = regexp.MustCompile(`(?m:^Link:\s*([^\s]+)\s+->\s+([^\s+]+)\s*$)`)
+
var (
- // linkRegexp parses the Link: lines in the WHENCE file. This does not have
- // an official grammar, the regexp has been written in an approximation of
- // the original parsing algorithm at @linux-firmware//:copy_firmware.sh.
- linkRegexp = regexp.MustCompile(`(?m:^Link:\s*([^\s]+)\s+->\s+([^\s+]+)\s*$)`)
+ modinfoPath = flag.String("modinfo", "", "Path to the modules.builtin.modinfo file built with the kernel")
+ modulesPath = flag.String("modules", "", "Path to the directory containing the dynamically loaded kernel modules (.ko files)")
+ firmwareListPath = flag.String("firmware-file-list", "", "Path to a file containing a newline-separated list of paths to firmware files")
+ whenceFilePath = flag.String("firmware-whence", "", "Path to the linux-firmware WHENCE file containing aliases for firmware files")
+ outMetaPath = flag.String("out-meta", "", "Path where the resulting module metadata protobuf file should be created")
+ outFSSpecPath = flag.String("out-fsspec", "", "Path where the resulting fsspec should be created")
)
-// fwPaths returns a slice of filesystem paths relative to the root of the
-// linux-firmware repository, pointing at firmware files, according to contents
-// of the kernel build side effect: modules.builtin.modinfo.
-func fwPaths(mi []byte) []string {
- // Use a map pset to deduplicate firmware paths.
- pset := make(map[string]bool)
- // Get a slice of entries of the form "unix.license=GPL" from mi. Then extract
- // firmware information from it.
- entries := bytes.Split(mi, []byte{0})
- for _, entry := range entries {
- // Skip empty entries.
- if len(entry) == 0 {
- continue
- }
- // Parse the entries. Split each entry into a key-value pair, separated
- // by "=".
- kv := strings.SplitN(string(entry), "=", 2)
- key, value := kv[0], kv[1]
- // Split the key into a module.attribute] pair, such as "unix.license".
- ma := strings.SplitN(key, ".", 2)
- // Skip, if it's not a firmware entry, according to the attribute.
- if ma[1] != "firmware" {
- continue
- }
- // If it is though, value holds a firmware path.
- pset[value] = true
- }
- // Convert the deduplicated pset to a slice.
- pslice := make([]string, 0, len(pset))
- for p, _ := range pset {
- pslice = append(pslice, p)
- }
- sort.Strings(pslice)
- return pslice
-}
-
-// fwprune takes a modinfo file from the kernel and extracts a list of all
-// firmware files requested by all modules in that file. It then takes all
-// available firmware file paths (newline-separated in the firmwareList file)
-// and tries to match the requested file paths as a suffix of them.
-// For example if a module requests firmware foo/bar.bin and in the firmware list
-// there is a file at path build-out/x/y/foo/bar.bin it will use that file.
-// It also parses links from the linux-firmware metadata file and uses them
-// for matching requested firmware.
-// Finally it generates an fsspec placing each file under its requested path
-// under /lib/firmware.
func main() {
- if len(os.Args) != 5 {
- log.Fatal("Usage: fwprune modules.builtin.modinfo firmwareListPath metadataFilePath outSpec")
+ flag.Parse()
+ if *modinfoPath == "" || *modulesPath == "" || *firmwareListPath == "" ||
+ *whenceFilePath == "" || *outMetaPath == "" || *outFSSpecPath == "" {
+ log.Fatal("all flags are required and need to be provided")
}
- modinfo := os.Args[1]
- firmwareListPath := os.Args[2]
- metadataFilePath := os.Args[3]
- outSpec := os.Args[4]
- allFirmwareData, err := os.ReadFile(firmwareListPath)
+ allFirmwareData, err := os.ReadFile(*firmwareListPath)
if err != nil {
log.Fatalf("Failed to read firmware source list: %v", err)
}
@@ -89,16 +53,23 @@
// Create a look-up table of all possible suffixes to their full paths as
// this is much faster at O(n) than calling strings.HasSuffix for every
// possible combination which is O(n^2).
+ // For example a build output at out/a/b/c.bin will be entered into
+ // the suffix LUT as build as out/a/b/c.bin, a/b/c.bin, b/c.bin and c.bin.
+ // If the firmware then requests b/c.bin, the output path is contained in
+ // the suffix LUT.
suffixLUT := make(map[string]string)
for _, firmwarePath := range allFirmwarePaths {
pathParts := strings.Split(firmwarePath, string(os.PathSeparator))
for i := range pathParts {
- suffixLUT[path.Join(pathParts[i:len(pathParts)]...)] = firmwarePath
+ suffixLUT[path.Join(pathParts[i:]...)] = firmwarePath
}
}
+ // The linux-firmware repo contains a WHENCE file which contains (among
+ // other information) aliases for firmware which should be symlinked.
+ // Open this file and create a map of aliases in it.
linkMap := make(map[string]string)
- metadata, err := os.ReadFile(metadataFilePath)
+ metadata, err := os.ReadFile(*whenceFilePath)
if err != nil {
log.Fatalf("Failed to read metadata file: %v", err)
}
@@ -109,15 +80,90 @@
linkMap[link[1]] = link[2]
}
- // Get the firmware file paths used by modules according to modinfo data
- mi, err := os.ReadFile(modinfo)
+ // Collect module metadata (modinfo) from both built-in modules via the
+ // kbuild-generated metadata file as well as from the loadable modules by
+ // walking them.
+ var files []*fsspec.File
+ var symlinks []*fsspec.SymbolicLink
+
+ mi, err := os.Open(*modinfoPath)
if err != nil {
log.Fatalf("While reading modinfo: %v", err)
}
- fwp := fwPaths(mi)
+ modMeta, err := kmod.GetBuiltinModulesInfo(mi)
+ if err != nil {
+ log.Fatalf("Failed to read modules modinfo data: %v", err)
+ }
- var files []*fsspec.File
- var symlinks []*fsspec.SymbolicLink
+ err = filepath.WalkDir(*modulesPath, func(p string, d fs.DirEntry, err error) error {
+ if err != nil {
+ log.Fatal(err)
+ }
+ if d.IsDir() {
+ return nil
+ }
+ mod, err := elf.Open(p)
+ if err != nil {
+ log.Fatal(err)
+ }
+ defer mod.Close()
+ out, err := kmod.GetModuleInfo(mod)
+ if err != nil {
+ log.Fatal(err)
+ }
+ relPath, err := filepath.Rel(*modulesPath, p)
+ if err != nil {
+ return err
+ }
+ // Add path information for MakeMetaFromModuleInfo.
+ out["path"] = []string{relPath}
+ modMeta = append(modMeta, out)
+ files = append(files, &fsspec.File{
+ Path: path.Join("/lib/modules", relPath),
+ SourcePath: filepath.Join(*modulesPath, relPath),
+ Mode: 0555,
+ })
+ return nil
+ })
+ if err != nil {
+ log.Fatalf("Error walking modules: %v", err)
+ }
+
+ // Generate loading metadata from all known modules.
+ meta, err := kmod.MakeMetaFromModuleInfo(modMeta)
+ if err != nil {
+ log.Fatal(err)
+ }
+ metaRaw, err := proto.Marshal(meta)
+ if err != nil {
+ log.Fatal(err)
+ }
+ if err := os.WriteFile(*outMetaPath, metaRaw, 0640); err != nil {
+ log.Fatal(err)
+ }
+ files = append(files, &fsspec.File{
+ Path: "/lib/modules/meta.pb",
+ SourcePath: *outMetaPath,
+ Mode: 0444,
+ })
+
+ // Create set of all firmware paths required by modules
+ fwset := make(map[string]bool)
+ for _, m := range modMeta {
+ if len(m["path"]) == 0 && len(m.Firmware()) > 0 {
+ log.Fatalf("Module %v is built-in, but requires firmware. Linux does not support this in all configurations.", m.Name())
+ }
+ for _, fw := range m.Firmware() {
+ fwset[fw] = true
+ }
+ }
+
+ // Convert set to list and sort for determinism
+ fwp := make([]string, 0, len(fwset))
+ for p := range fwset {
+ fwp = append(fwp, p)
+ }
+ sort.Strings(fwp)
// This function is called for every requested firmware file and adds and
// resolves symlinks until it finds the target file and adds that too.
@@ -163,7 +209,7 @@
// Format output in a both human- and machine-readable form
marshalOpts := prototext.MarshalOptions{Multiline: true, Indent: " "}
fsspecRaw, err := marshalOpts.Marshal(&fsspec.FSSpec{File: files, SymbolicLink: symlinks})
- if err := os.WriteFile(outSpec, fsspecRaw, 0644); err != nil {
+ if err := os.WriteFile(*outFSSpecPath, fsspecRaw, 0644); err != nil {
log.Fatalf("failed writing output: %v", err)
}
}