osbase/fat32: adopt structfs
Change the external interface of the FAT32 writer to take a
structfs.Tree instead of a FAT32-specific data structure. Producers of
file system data are no longer specific to FAT32.
With these changes, the blkio package becomes obsolete. The
LazyFileReader did not actually work as intended when used with
osbase/fat32, because fat32 copies data with io.CopyN and thus stops
reading before reaching EOF, so the LazyFileReader is never closed. The
new Blob interface requires the consumer to explicitly Open and Close.
Change-Id: I9a71a5f0bddf36ac38c656659e6dcfe520b88fb0
Reviewed-on: https://review.monogon.dev/c/monogon/+/4037
Reviewed-by: Lorenz Brun <lorenz@monogon.tech>
Tested-by: Jenkins CI
diff --git a/metropolis/cli/metroctl/BUILD.bazel b/metropolis/cli/metroctl/BUILD.bazel
index f4c5c15..bd7ab70 100644
--- a/metropolis/cli/metroctl/BUILD.bazel
+++ b/metropolis/cli/metroctl/BUILD.bazel
@@ -48,10 +48,9 @@
"//metropolis/node/core/rpc/resolver",
"//metropolis/proto/api",
"//metropolis/proto/common",
- "//osbase/blkio",
- "//osbase/fat32",
"//osbase/logtree",
"//osbase/logtree/proto",
+ "//osbase/structfs",
"//version",
"@com_github_adrg_xdg//:xdg",
"@com_github_schollz_progressbar_v3//:progressbar",
diff --git a/metropolis/cli/metroctl/cmd_install.go b/metropolis/cli/metroctl/cmd_install.go
index a34e470..1b9b8cb 100644
--- a/metropolis/cli/metroctl/cmd_install.go
+++ b/metropolis/cli/metroctl/cmd_install.go
@@ -4,7 +4,6 @@
package main
import (
- "bytes"
"context"
"crypto/ed25519"
_ "embed"
@@ -22,8 +21,7 @@
"source.monogon.dev/metropolis/cli/flagdefs"
"source.monogon.dev/metropolis/cli/metroctl/core"
common "source.monogon.dev/metropolis/node"
- "source.monogon.dev/osbase/blkio"
- "source.monogon.dev/osbase/fat32"
+ "source.monogon.dev/osbase/structfs"
)
var installCmd = &cobra.Command{
@@ -111,20 +109,15 @@
return params, nil
}
-func external(name, datafilePath string, flag *string) (fat32.SizedReader, error) {
+func external(name, datafilePath string, flag *string) (structfs.Blob, error) {
if flag == nil || *flag == "" {
rPath, err := runfiles.Rlocation(datafilePath)
if err != nil {
return nil, fmt.Errorf("no %s specified", name)
}
- df, err := os.ReadFile(rPath)
- if err != nil {
- return nil, fmt.Errorf("can't read file: %w", err)
- }
- return bytes.NewReader(df), nil
+ return structfs.OSPathBlob(rPath)
}
-
- f, err := blkio.NewFileReader(*flag)
+ f, err := structfs.OSPathBlob(*flag)
if err != nil {
return nil, fmt.Errorf("failed to open specified %s: %w", name, err)
}
diff --git a/metropolis/cli/metroctl/cmd_install_ssh.go b/metropolis/cli/metroctl/cmd_install_ssh.go
index efd1fd7..f0a5379 100644
--- a/metropolis/cli/metroctl/cmd_install_ssh.go
+++ b/metropolis/cli/metroctl/cmd_install_ssh.go
@@ -24,7 +24,7 @@
"google.golang.org/protobuf/proto"
"source.monogon.dev/go/net/ssh"
- "source.monogon.dev/osbase/fat32"
+ "source.monogon.dev/osbase/structfs"
)
var sshCmd = &cobra.Command{
@@ -130,14 +130,20 @@
return err
}
- barUploader := func(r fat32.SizedReader, targetPath string) {
+ barUploader := func(blob structfs.Blob, targetPath string) {
+ content, err := blob.Open()
+ if err != nil {
+ log.Fatalf("error while uploading %q: %v", targetPath, err)
+ }
+ defer content.Close()
+
bar := progressbar.DefaultBytes(
- r.Size(),
+ blob.Size(),
targetPath,
)
defer bar.Close()
- proxyReader := progressbar.NewReader(r, bar)
+ proxyReader := progressbar.NewReader(content, bar)
defer proxyReader.Close()
if err := conn.Upload(ctx, targetPath, &proxyReader); err != nil {
diff --git a/metropolis/cli/metroctl/core/BUILD.bazel b/metropolis/cli/metroctl/core/BUILD.bazel
index 3a599a7..73527d5 100644
--- a/metropolis/cli/metroctl/core/BUILD.bazel
+++ b/metropolis/cli/metroctl/core/BUILD.bazel
@@ -21,6 +21,7 @@
"//osbase/blockdev",
"//osbase/fat32",
"//osbase/gpt",
+ "//osbase/structfs",
"@io_k8s_client_go//pkg/apis/clientauthentication/v1:clientauthentication",
"@io_k8s_client_go//tools/clientcmd",
"@io_k8s_client_go//tools/clientcmd/api",
diff --git a/metropolis/cli/metroctl/core/install.go b/metropolis/cli/metroctl/core/install.go
index 190f66a..5d43a89 100644
--- a/metropolis/cli/metroctl/core/install.go
+++ b/metropolis/cli/metroctl/core/install.go
@@ -4,7 +4,6 @@
package core
import (
- "bytes"
"errors"
"fmt"
"math"
@@ -16,6 +15,7 @@
"source.monogon.dev/osbase/blockdev"
"source.monogon.dev/osbase/fat32"
"source.monogon.dev/osbase/gpt"
+ "source.monogon.dev/osbase/structfs"
)
type MakeInstallerImageArgs struct {
@@ -23,13 +23,13 @@
TargetPath string
// Reader for the installer EFI executable. Mandatory.
- Installer fat32.SizedReader
+ Installer structfs.Blob
// Optional NodeParameters to be embedded for use by the installer.
NodeParams *api.NodeParameters
// Optional Reader for a Metropolis bundle for use by the installer.
- Bundle fat32.SizedReader
+ Bundle structfs.Blob
}
// MakeInstallerImage generates an installer disk image containing a Table
@@ -40,7 +40,7 @@
return errors.New("installer is mandatory")
}
- espRoot := fat32.Inode{Attrs: fat32.AttrDirectory}
+ var espRoot structfs.Tree
// This needs to be a "Removable Media" according to the UEFI Specification
// V2.9 Section 3.5.1.1. This file is booted by any compliant UEFI firmware
@@ -54,7 +54,7 @@
if err != nil {
return fmt.Errorf("failed to marshal node params: %w", err)
}
- if err := espRoot.PlaceFile("metropolis-installer/nodeparams.pb", bytes.NewReader(nodeParamsRaw)); err != nil {
+ if err := espRoot.PlaceFile("metropolis-installer/nodeparams.pb", structfs.Bytes(nodeParamsRaw)); err != nil {
return err
}
}
diff --git a/metropolis/cli/takeover/BUILD.bazel b/metropolis/cli/takeover/BUILD.bazel
index fe58518..c262164 100644
--- a/metropolis/cli/takeover/BUILD.bazel
+++ b/metropolis/cli/takeover/BUILD.bazel
@@ -40,6 +40,7 @@
"//osbase/kexec",
"//osbase/net/dump",
"//osbase/net/proto",
+ "//osbase/structfs",
"//osbase/supervisor",
"@com_github_cavaliergopher_cpio//:cpio",
"@com_github_klauspost_compress//zstd",
diff --git a/metropolis/cli/takeover/e2e/BUILD.bazel b/metropolis/cli/takeover/e2e/BUILD.bazel
index b085947..ae4810b 100644
--- a/metropolis/cli/takeover/e2e/BUILD.bazel
+++ b/metropolis/cli/takeover/e2e/BUILD.bazel
@@ -22,6 +22,7 @@
deps = [
"//osbase/fat32",
"//osbase/freeport",
+ "//osbase/structfs",
"@io_bazel_rules_go//go/runfiles",
"@org_golang_x_crypto//ssh",
"@org_golang_x_crypto//ssh/agent",
diff --git a/metropolis/cli/takeover/e2e/main_test.go b/metropolis/cli/takeover/e2e/main_test.go
index 1ba5354..afce515 100644
--- a/metropolis/cli/takeover/e2e/main_test.go
+++ b/metropolis/cli/takeover/e2e/main_test.go
@@ -25,6 +25,7 @@
"source.monogon.dev/osbase/fat32"
"source.monogon.dev/osbase/freeport"
+ "source.monogon.dev/osbase/structfs"
)
var (
@@ -112,25 +113,16 @@
t.Fatal(err)
}
- rootInode := fat32.Inode{
- Attrs: fat32.AttrDirectory,
- Children: []*fat32.Inode{
- {
- Name: "user-data",
- Content: strings.NewReader("#cloud-config\n" + string(userData)),
- },
- {
- Name: "meta-data",
- Content: strings.NewReader(""),
- },
- },
+ root := structfs.Tree{
+ structfs.File("user-data", structfs.Bytes("#cloud-config\n"+string(userData))),
+ structfs.File("meta-data", structfs.Bytes("")),
}
cloudInitDataFile, err := os.CreateTemp("", "cidata*.img")
if err != nil {
t.Fatal(err)
}
defer os.Remove(cloudInitDataFile.Name())
- if err := fat32.WriteFS(cloudInitDataFile, rootInode, fat32.Options{Label: "cidata"}); err != nil {
+ if err := fat32.WriteFS(cloudInitDataFile, root, fat32.Options{Label: "cidata"}); err != nil {
t.Fatal(err)
}
diff --git a/metropolis/cli/takeover/install.go b/metropolis/cli/takeover/install.go
index 005b590..2b76095 100644
--- a/metropolis/cli/takeover/install.go
+++ b/metropolis/cli/takeover/install.go
@@ -5,10 +5,8 @@
import (
"archive/zip"
- "bytes"
_ "embed"
"fmt"
- "io/fs"
"os"
"path/filepath"
@@ -16,24 +14,28 @@
"source.monogon.dev/osbase/blockdev"
"source.monogon.dev/osbase/build/mkimage/osimage"
"source.monogon.dev/osbase/efivarfs"
+ "source.monogon.dev/osbase/structfs"
)
//go:embed metropolis/node/core/abloader/abloader.efi
var abloader []byte
-// FileSizedReader is a small adapter from fs.File to fs.SizedReader
-// Panics on Stat() failure, so should only be used with sources where Stat()
-// cannot fail.
-type FileSizedReader struct {
- fs.File
+// zipBlob looks up a file in a [zip.Reader] and adapts it to [structfs.Blob].
+func zipBlob(reader *zip.Reader, name string) (zipFileBlob, error) {
+ for _, file := range reader.File {
+ if file.Name == name {
+ return zipFileBlob{file}, nil
+ }
+ }
+ return zipFileBlob{}, fmt.Errorf("file %q not found", name)
}
-func (f FileSizedReader) Size() int64 {
- stat, err := f.Stat()
- if err != nil {
- panic(err)
- }
- return stat.Size()
+type zipFileBlob struct {
+ *zip.File
+}
+
+func (f zipFileBlob) Size() int64 {
+ return int64(f.File.UncompressedSize64)
}
// EnvInstallTarget environment variable which tells the takeover binary where
@@ -93,12 +95,12 @@
return nil, fmt.Errorf("failed to open root device: %w", err)
}
- efiPayload, err := bundle.Open("kernel_efi.efi")
+ efiPayload, err := zipBlob(bundle, "kernel_efi.efi")
if err != nil {
return nil, fmt.Errorf("invalid bundle: %w", err)
}
- systemImage, err := bundle.Open("verity_rootfs.img")
+ systemImage, err := zipBlob(bundle, "verity_rootfs.img")
if err != nil {
return nil, fmt.Errorf("invalid bundle: %w", err)
}
@@ -110,9 +112,9 @@
Data: 128,
},
SystemImage: systemImage,
- EFIPayload: FileSizedReader{efiPayload},
- ABLoader: bytes.NewReader(abloader),
- NodeParameters: bytes.NewReader(metropolisSpecRaw),
+ EFIPayload: efiPayload,
+ ABLoader: structfs.Bytes(abloader),
+ NodeParameters: structfs.Bytes(metropolisSpecRaw),
Output: rootDev,
}, nil
}
diff --git a/metropolis/installer/BUILD.bazel b/metropolis/installer/BUILD.bazel
index 5792326..fe80669 100644
--- a/metropolis/installer/BUILD.bazel
+++ b/metropolis/installer/BUILD.bazel
@@ -16,6 +16,7 @@
"//osbase/bringup",
"//osbase/build/mkimage/osimage",
"//osbase/efivarfs",
+ "//osbase/structfs",
"//osbase/supervisor",
"//osbase/sysfs",
"@org_golang_x_sys//unix",
diff --git a/metropolis/installer/main.go b/metropolis/installer/main.go
index e9956fd..216f99c 100644
--- a/metropolis/installer/main.go
+++ b/metropolis/installer/main.go
@@ -8,12 +8,10 @@
import (
"archive/zip"
- "bytes"
"context"
_ "embed"
"errors"
"fmt"
- "io/fs"
"os"
"path/filepath"
"strings"
@@ -25,6 +23,7 @@
"source.monogon.dev/osbase/bringup"
"source.monogon.dev/osbase/build/mkimage/osimage"
"source.monogon.dev/osbase/efivarfs"
+ "source.monogon.dev/osbase/structfs"
"source.monogon.dev/osbase/supervisor"
"source.monogon.dev/osbase/sysfs"
)
@@ -104,19 +103,22 @@
return suitable, nil
}
-// FileSizedReader is a small adapter from fs.File to fs.SizedReader
-// Panics on Stat() failure, so should only be used with sources where Stat()
-// cannot fail.
-type FileSizedReader struct {
- fs.File
+// zipBlob looks up a file in a [zip.Reader] and adapts it to [structfs.Blob].
+func zipBlob(reader *zip.Reader, name string) (zipFileBlob, error) {
+ for _, file := range reader.File {
+ if file.Name == name {
+ return zipFileBlob{file}, nil
+ }
+ }
+ return zipFileBlob{}, fmt.Errorf("file %q not found", name)
}
-func (f FileSizedReader) Size() int64 {
- stat, err := f.Stat()
- if err != nil {
- panic(err)
- }
- return stat.Size()
+type zipFileBlob struct {
+ *zip.File
+}
+
+func (f zipFileBlob) Size() int64 {
+ return int64(f.File.UncompressedSize64)
}
func main() {
@@ -162,7 +164,7 @@
return fmt.Errorf("while mounting the installer ESP: %w", err)
}
- nodeParameters, err := os.Open("/installer/metropolis-installer/nodeparams.pb")
+ nodeParameters, err := structfs.OSPathBlob("/installer/metropolis-installer/nodeparams.pb")
if err != nil {
return fmt.Errorf("failed to open node parameters from ESP: %w", err)
}
@@ -173,16 +175,14 @@
return fmt.Errorf("failed to open node bundle from ESP: %w", err)
}
defer bundle.Close()
- efiPayload, err := bundle.Open("kernel_efi.efi")
+ efiPayload, err := zipBlob(&bundle.Reader, "kernel_efi.efi")
if err != nil {
return fmt.Errorf("cannot open EFI payload in bundle: %w", err)
}
- defer efiPayload.Close()
- systemImage, err := bundle.Open("verity_rootfs.img")
+ systemImage, err := zipBlob(&bundle.Reader, "verity_rootfs.img")
if err != nil {
return fmt.Errorf("cannot open system image in bundle: %w", err)
}
- defer systemImage.Close()
// Build the osimage parameters.
installParams := osimage.Params{
@@ -198,9 +198,9 @@
Data: 128,
},
SystemImage: systemImage,
- EFIPayload: FileSizedReader{efiPayload},
- ABLoader: bytes.NewReader(abloader),
- NodeParameters: FileSizedReader{nodeParameters},
+ EFIPayload: efiPayload,
+ ABLoader: structfs.Bytes(abloader),
+ NodeParameters: nodeParameters,
}
// Calculate the minimum target size based on the installation parameters.
minSize := uint64((installParams.PartitionSize.ESP +
diff --git a/metropolis/installer/test/BUILD.bazel b/metropolis/installer/test/BUILD.bazel
index 7f7ad54..7b7828d 100644
--- a/metropolis/installer/test/BUILD.bazel
+++ b/metropolis/installer/test/BUILD.bazel
@@ -24,6 +24,7 @@
"//metropolis/proto/api",
"//osbase/build/mkimage/osimage",
"//osbase/cmd",
+ "//osbase/structfs",
"@com_github_diskfs_go_diskfs//:go-diskfs",
"@com_github_diskfs_go_diskfs//disk",
"@com_github_diskfs_go_diskfs//partition/gpt",
diff --git a/metropolis/installer/test/run_test.go b/metropolis/installer/test/run_test.go
index 766c839..cd87852 100644
--- a/metropolis/installer/test/run_test.go
+++ b/metropolis/installer/test/run_test.go
@@ -7,7 +7,6 @@
package installer
import (
- "bytes"
"context"
"fmt"
"log"
@@ -27,6 +26,7 @@
mctl "source.monogon.dev/metropolis/cli/metroctl/core"
"source.monogon.dev/osbase/build/mkimage/osimage"
"source.monogon.dev/osbase/cmd"
+ "source.monogon.dev/osbase/structfs"
)
var (
@@ -134,21 +134,21 @@
func TestMain(m *testing.M) {
installerImage = filepath.Join(os.Getenv("TEST_TMPDIR"), "installer.img")
- installer, err := os.ReadFile(xInstallerPath)
+ installer, err := structfs.OSPathBlob(xInstallerPath)
if err != nil {
log.Fatal(err)
}
- bundle, err := os.ReadFile(xBundlePath)
+ bundle, err := structfs.OSPathBlob(xBundlePath)
if err != nil {
log.Fatal(err)
}
iargs := mctl.MakeInstallerImageArgs{
- Installer: bytes.NewReader(installer),
+ Installer: installer,
TargetPath: installerImage,
NodeParams: &api.NodeParameters{},
- Bundle: bytes.NewReader(bundle),
+ Bundle: bundle,
}
if err := mctl.MakeInstallerImage(iargs); err != nil {
log.Fatalf("Couldn't create the installer image at %q: %v", installerImage, err)
diff --git a/metropolis/node/core/update/e2e/BUILD.bazel b/metropolis/node/core/update/e2e/BUILD.bazel
index 3f6aa66..83716d1 100644
--- a/metropolis/node/core/update/e2e/BUILD.bazel
+++ b/metropolis/node/core/update/e2e/BUILD.bazel
@@ -25,9 +25,9 @@
"xAbloaderPath": "$(rlocationpath //metropolis/node/core/abloader )",
},
deps = [
- "//osbase/blkio",
"//osbase/blockdev",
"//osbase/build/mkimage/osimage",
+ "//osbase/structfs",
"@io_bazel_rules_go//go/runfiles",
],
)
diff --git a/metropolis/node/core/update/e2e/e2e_test.go b/metropolis/node/core/update/e2e/e2e_test.go
index f7d0e36..4a1c6a2 100644
--- a/metropolis/node/core/update/e2e/e2e_test.go
+++ b/metropolis/node/core/update/e2e/e2e_test.go
@@ -21,9 +21,9 @@
"github.com/bazelbuild/rules_go/go/runfiles"
- "source.monogon.dev/osbase/blkio"
"source.monogon.dev/osbase/blockdev"
"source.monogon.dev/osbase/build/mkimage/osimage"
+ "source.monogon.dev/osbase/structfs"
)
var (
@@ -197,18 +197,16 @@
t.Cleanup(func() { os.Remove(rootDevPath) })
defer rootDisk.Close()
- boot, err := blkio.NewFileReader(xBootPath)
+ boot, err := structfs.OSPathBlob(xBootPath)
if err != nil {
t.Fatal(err)
}
- defer boot.Close()
- system, err := os.Open(xSystemXPath)
+ system, err := structfs.OSPathBlob(xSystemXPath)
if err != nil {
t.Fatal(err)
}
- defer system.Close()
- loader, err := blkio.NewFileReader(xAbloaderPath)
+ loader, err := structfs.OSPathBlob(xAbloaderPath)
if err != nil {
t.Fatal(err)
}