m/n/b/fwprune: process links from metadata file
The linux-firmware repository has a metadata file called WHENCE which
contains mostly license and origin information, but critically it also
contains data for symbolic links which are not materialized inside the
repo itself. So we need to parse that file and create these symlinks
ourselves.
Change-Id: I9e6973e60d6f06e844dc879f658c9dd1913c432d
Reviewed-on: https://review.monogon.dev/c/monogon/+/555
Reviewed-by: Mateusz Zalega <mateusz@monogon.tech>
diff --git a/metropolis/node/BUILD.bazel b/metropolis/node/BUILD.bazel
index 1168cf5..8d5e644 100644
--- a/metropolis/node/BUILD.bazel
+++ b/metropolis/node/BUILD.bazel
@@ -27,6 +27,7 @@
name = "firmware",
firmware_files = ["@linux-firmware//:all_files"],
kernel = "//third_party/linux",
+ metadata = "@linux-firmware//:metadata",
)
cpio_ucode(
diff --git a/metropolis/node/build/fwprune/def.bzl b/metropolis/node/build/fwprune/def.bzl
index b43b1d1..507532e 100644
--- a/metropolis/node/build/fwprune/def.bzl
+++ b/metropolis/node/build/fwprune/def.bzl
@@ -13,10 +13,10 @@
ctx.actions.run(
outputs = [fsspec_out],
- inputs = [fwlist, modinfo] + ctx.files.firmware_files,
+ inputs = [fwlist, modinfo, ctx.file.metadata] + ctx.files.firmware_files,
tools = [ctx.executable._fwprune],
executable = ctx.executable._fwprune,
- arguments = [modinfo.path, fwlist.path, fsspec_out.path],
+ arguments = [modinfo.path, fwlist.path, ctx.file.metadata.path, fsspec_out.path],
)
return [DefaultInfo(files = depset([fsspec_out])), FSSpecInfo(spec = fsspec_out, referenced = ctx.files.firmware_files)]
@@ -36,6 +36,14 @@
be in here.
""",
),
+ "metadata": attr.label(
+ mandatory = True,
+ allow_single_file = True,
+ doc = """
+ The metadata file for the Linux firmware. Currently this is the WHENCE file at the root of the
+ linux-firmware repository. Used for resolving additional links.
+ """,
+ ),
"kernel": attr.label(
doc = """
Kernel for which firmware should be selected. Needs to have a modinfo OutputGroup.
diff --git a/metropolis/node/build/fwprune/main.go b/metropolis/node/build/fwprune/main.go
index 6ad2a93..861a2d0 100644
--- a/metropolis/node/build/fwprune/main.go
+++ b/metropolis/node/build/fwprune/main.go
@@ -8,7 +8,8 @@
"bytes"
"log"
"os"
- "path/filepath"
+ "path"
+ "regexp"
"sort"
"strings"
@@ -17,6 +18,13 @@
"source.monogon.dev/metropolis/node/build/fsspec"
)
+var (
+ // linkRegexp parses the Link: lines in the WHENCE file. This does not have
+ // an official grammar, the regexp has been written in an approximation of
+ // the original parsing algorithm at @linux-firmware//:copy_firmware.sh.
+ linkRegexp = regexp.MustCompile(`(?m:^Link:\s*([^\s]+)\s+->\s+([^\s+]+)\s*$)`)
+)
+
// fwPaths returns a slice of filesystem paths relative to the root of the
// linux-firmware repository, pointing at firmware files, according to contents
// of the kernel build side effect: modules.builtin.modinfo.
@@ -59,15 +67,18 @@
// and tries to match the requested file paths as a suffix of them.
// For example if a module requests firmware foo/bar.bin and in the firmware list
// there is a file at path build-out/x/y/foo/bar.bin it will use that file.
+// It also parses links from the linux-firmware metadata file and uses them
+// for matching requested firmware.
// Finally it generates an fsspec placing each file under its requested path
// under /lib/firmware.
func main() {
- if len(os.Args) != 4 {
- log.Fatal("Usage: fwprune modules.builtin.modinfo firmwareListPath outSpec")
+ if len(os.Args) != 5 {
+ log.Fatal("Usage: fwprune modules.builtin.modinfo firmwareListPath metadataFilePath outSpec")
}
modinfo := os.Args[1]
firmwareListPath := os.Args[2]
- outSpec := os.Args[3]
+ metadataFilePath := os.Args[3]
+ outSpec := os.Args[4]
allFirmwareData, err := os.ReadFile(firmwareListPath)
if err != nil {
@@ -82,10 +93,22 @@
for _, firmwarePath := range allFirmwarePaths {
pathParts := strings.Split(firmwarePath, string(os.PathSeparator))
for i := range pathParts {
- suffixLUT[filepath.Join(pathParts[i:len(pathParts)]...)] = firmwarePath
+ suffixLUT[path.Join(pathParts[i:len(pathParts)]...)] = firmwarePath
}
}
+ linkMap := make(map[string]string)
+ metadata, err := os.ReadFile(metadataFilePath)
+ if err != nil {
+ log.Fatalf("Failed to read metadata file: %v", err)
+ }
+ linksRaw := linkRegexp.FindAllStringSubmatch(string(metadata), -1)
+ for _, link := range linksRaw {
+ // For links we know the exact path referenced by kernel drives so
+ // a suffix LUT is unnecessary.
+ linkMap[link[1]] = link[2]
+ }
+
// Get the firmware file paths used by modules according to modinfo data
mi, err := os.ReadFile(modinfo)
if err != nil {
@@ -94,22 +117,52 @@
fwp := fwPaths(mi)
var files []*fsspec.File
+ var symlinks []*fsspec.SymbolicLink
- for _, p := range fwp {
+ // This function is called for every requested firmware file and adds and
+ // resolves symlinks until it finds the target file and adds that too.
+ populatedPaths := make(map[string]bool)
+ var chaseReference func(string)
+ chaseReference = func(p string) {
+ if populatedPaths[p] {
+ // Bail if path is already populated. Because of the DAG-like
+ // property of links in filesystems everything transitively pointed
+ // to by anything at this path has already been included.
+ return
+ }
+ placedPath := path.Join("/lib/firmware", p)
+ if linkTarget := linkMap[p]; linkTarget != "" {
+ symlinks = append(symlinks, &fsspec.SymbolicLink{
+ Path: placedPath,
+ TargetPath: linkTarget,
+ })
+ populatedPaths[placedPath] = true
+ // Symlinks are relative to their place, resolve them to be relative
+ // to the firmware root directory.
+ chaseReference(path.Join(path.Dir(p), linkTarget))
+ return
+ }
sourcePath := suffixLUT[p]
if sourcePath == "" {
// This should not be fatal as sometimes linux-firmware cannot
// ship all firmware usable by the kernel for mostly legal reasons.
log.Printf("WARNING: Requested firmware %q not found", p)
- continue
+ return
}
files = append(files, &fsspec.File{
- Path: filepath.Join("/lib/firmware", p),
+ Path: path.Join("/lib/firmware", p),
Mode: 0444,
SourcePath: sourcePath,
})
+ populatedPaths[path.Join("/lib/firmware", p)] = true
}
- fsspecRaw, err := prototext.Marshal(&fsspec.FSSpec{File: files})
+
+ for _, p := range fwp {
+ chaseReference(p)
+ }
+ // Format output in a both human- and machine-readable form
+ marshalOpts := prototext.MarshalOptions{Multiline: true, Indent: " "}
+ fsspecRaw, err := marshalOpts.Marshal(&fsspec.FSSpec{File: files, SymbolicLink: symlinks})
if err := os.WriteFile(outSpec, fsspecRaw, 0644); err != nil {
log.Fatalf("failed writing output: %v", err)
}