osbase/fat32: adopt structfs
Change the external interface of the FAT32 writer to take a
structfs.Tree instead of a FAT32-specific data structure. Producers of
file system data are no longer specific to FAT32.
With these changes, the blkio package becomes obsolete. The
LazyFileReader did not actually work as intended when used with
osbase/fat32, because fat32 copies data with io.CopyN and thus stops
reading before reaching EOF, so the LazyFileReader is never closed. The
new Blob interface requires the consumer to explicitly Open and Close.
Change-Id: I9a71a5f0bddf36ac38c656659e6dcfe520b88fb0
Reviewed-on: https://review.monogon.dev/c/monogon/+/4037
Reviewed-by: Lorenz Brun <lorenz@monogon.tech>
Tested-by: Jenkins CI
diff --git a/cloud/agent/BUILD.bazel b/cloud/agent/BUILD.bazel
index 7e7614d..3d342ae 100644
--- a/cloud/agent/BUILD.bazel
+++ b/cloud/agent/BUILD.bazel
@@ -27,6 +27,7 @@
"//osbase/pki",
"//osbase/scsi",
"//osbase/smbios",
+ "//osbase/structfs",
"//osbase/supervisor",
"@com_github_cenkalti_backoff_v4//:backoff",
"@com_github_mdlayher_ethtool//:ethtool",
diff --git a/cloud/agent/install.go b/cloud/agent/install.go
index cdc8fa5..914b0be 100644
--- a/cloud/agent/install.go
+++ b/cloud/agent/install.go
@@ -9,7 +9,6 @@
_ "embed"
"errors"
"fmt"
- "io/fs"
"net/http"
"os"
"path/filepath"
@@ -23,24 +22,28 @@
"source.monogon.dev/osbase/build/mkimage/osimage"
"source.monogon.dev/osbase/efivarfs"
npb "source.monogon.dev/osbase/net/proto"
+ "source.monogon.dev/osbase/structfs"
)
//go:embed metropolis/node/core/abloader/abloader.efi
var abloader []byte
-// FileSizedReader is a small adapter from fs.File to fs.SizedReader
-// Panics on Stat() failure, so should only be used with sources where Stat()
-// cannot fail.
-type FileSizedReader struct {
- fs.File
+// zipBlob looks up a file in a [zip.Reader] and adapts it to [structfs.Blob].
+func zipBlob(reader *zip.Reader, name string) (zipFileBlob, error) {
+ for _, file := range reader.File {
+ if file.Name == name {
+ return zipFileBlob{file}, nil
+ }
+ }
+ return zipFileBlob{}, fmt.Errorf("file %q not found", name)
}
-func (f FileSizedReader) Size() int64 {
- stat, err := f.Stat()
- if err != nil {
- panic(err)
- }
- return stat.Size()
+type zipFileBlob struct {
+ *zip.File
+}
+
+func (f zipFileBlob) Size() int64 {
+ return int64(f.File.UncompressedSize64)
}
// install dispatches OSInstallationRequests to the appropriate installer
@@ -115,16 +118,14 @@
if err != nil {
return fmt.Errorf("failed to open node bundle: %w", err)
}
- efiPayload, err := bundle.Open("kernel_efi.efi")
+ efiPayload, err := zipBlob(bundle, "kernel_efi.efi")
if err != nil {
return fmt.Errorf("invalid bundle: %w", err)
}
- defer efiPayload.Close()
- systemImage, err := bundle.Open("verity_rootfs.img")
+ systemImage, err := zipBlob(bundle, "verity_rootfs.img")
if err != nil {
return fmt.Errorf("invalid bundle: %w", err)
}
- defer systemImage.Close()
nodeParamsRaw, err := proto.Marshal(req.NodeParameters)
if err != nil {
@@ -143,9 +144,9 @@
Data: 128,
},
SystemImage: systemImage,
- EFIPayload: FileSizedReader{efiPayload},
- ABLoader: bytes.NewReader(abloader),
- NodeParameters: bytes.NewReader(nodeParamsRaw),
+ EFIPayload: efiPayload,
+ ABLoader: structfs.Bytes(abloader),
+ NodeParameters: structfs.Bytes(nodeParamsRaw),
Output: rootDev,
}
diff --git a/cloud/agent/takeover/e2e/BUILD.bazel b/cloud/agent/takeover/e2e/BUILD.bazel
index bfa7baa..b83b131 100644
--- a/cloud/agent/takeover/e2e/BUILD.bazel
+++ b/cloud/agent/takeover/e2e/BUILD.bazel
@@ -20,6 +20,7 @@
"//cloud/agent/api",
"//osbase/fat32",
"//osbase/freeport",
+ "//osbase/structfs",
"@com_github_pkg_sftp//:sftp",
"@io_bazel_rules_go//go/runfiles",
"@org_golang_google_protobuf//proto",
diff --git a/cloud/agent/takeover/e2e/main_test.go b/cloud/agent/takeover/e2e/main_test.go
index 9ab6769..756eb40 100644
--- a/cloud/agent/takeover/e2e/main_test.go
+++ b/cloud/agent/takeover/e2e/main_test.go
@@ -27,6 +27,7 @@
"source.monogon.dev/osbase/fat32"
"source.monogon.dev/osbase/freeport"
+ "source.monogon.dev/osbase/structfs"
)
var (
@@ -79,25 +80,16 @@
t.Fatal(err)
}
- rootInode := fat32.Inode{
- Attrs: fat32.AttrDirectory,
- Children: []*fat32.Inode{
- {
- Name: "user-data",
- Content: strings.NewReader("#cloud-config\n" + string(userData)),
- },
- {
- Name: "meta-data",
- Content: strings.NewReader(""),
- },
- },
+ root := structfs.Tree{
+ structfs.File("user-data", structfs.Bytes("#cloud-config\n"+string(userData))),
+ structfs.File("meta-data", structfs.Bytes("")),
}
cloudInitDataFile, err := os.CreateTemp("", "cidata*.img")
if err != nil {
t.Fatal(err)
}
defer os.Remove(cloudInitDataFile.Name())
- if err := fat32.WriteFS(cloudInitDataFile, rootInode, fat32.Options{Label: "cidata"}); err != nil {
+ if err := fat32.WriteFS(cloudInitDataFile, root, fat32.Options{Label: "cidata"}); err != nil {
t.Fatal(err)
}
diff --git a/metropolis/cli/metroctl/BUILD.bazel b/metropolis/cli/metroctl/BUILD.bazel
index f4c5c15..bd7ab70 100644
--- a/metropolis/cli/metroctl/BUILD.bazel
+++ b/metropolis/cli/metroctl/BUILD.bazel
@@ -48,10 +48,9 @@
"//metropolis/node/core/rpc/resolver",
"//metropolis/proto/api",
"//metropolis/proto/common",
- "//osbase/blkio",
- "//osbase/fat32",
"//osbase/logtree",
"//osbase/logtree/proto",
+ "//osbase/structfs",
"//version",
"@com_github_adrg_xdg//:xdg",
"@com_github_schollz_progressbar_v3//:progressbar",
diff --git a/metropolis/cli/metroctl/cmd_install.go b/metropolis/cli/metroctl/cmd_install.go
index a34e470..1b9b8cb 100644
--- a/metropolis/cli/metroctl/cmd_install.go
+++ b/metropolis/cli/metroctl/cmd_install.go
@@ -4,7 +4,6 @@
package main
import (
- "bytes"
"context"
"crypto/ed25519"
_ "embed"
@@ -22,8 +21,7 @@
"source.monogon.dev/metropolis/cli/flagdefs"
"source.monogon.dev/metropolis/cli/metroctl/core"
common "source.monogon.dev/metropolis/node"
- "source.monogon.dev/osbase/blkio"
- "source.monogon.dev/osbase/fat32"
+ "source.monogon.dev/osbase/structfs"
)
var installCmd = &cobra.Command{
@@ -111,20 +109,15 @@
return params, nil
}
-func external(name, datafilePath string, flag *string) (fat32.SizedReader, error) {
+func external(name, datafilePath string, flag *string) (structfs.Blob, error) {
if flag == nil || *flag == "" {
rPath, err := runfiles.Rlocation(datafilePath)
if err != nil {
return nil, fmt.Errorf("no %s specified", name)
}
- df, err := os.ReadFile(rPath)
- if err != nil {
- return nil, fmt.Errorf("can't read file: %w", err)
- }
- return bytes.NewReader(df), nil
+ return structfs.OSPathBlob(rPath)
}
-
- f, err := blkio.NewFileReader(*flag)
+ f, err := structfs.OSPathBlob(*flag)
if err != nil {
return nil, fmt.Errorf("failed to open specified %s: %w", name, err)
}
diff --git a/metropolis/cli/metroctl/cmd_install_ssh.go b/metropolis/cli/metroctl/cmd_install_ssh.go
index efd1fd7..f0a5379 100644
--- a/metropolis/cli/metroctl/cmd_install_ssh.go
+++ b/metropolis/cli/metroctl/cmd_install_ssh.go
@@ -24,7 +24,7 @@
"google.golang.org/protobuf/proto"
"source.monogon.dev/go/net/ssh"
- "source.monogon.dev/osbase/fat32"
+ "source.monogon.dev/osbase/structfs"
)
var sshCmd = &cobra.Command{
@@ -130,14 +130,20 @@
return err
}
- barUploader := func(r fat32.SizedReader, targetPath string) {
+ barUploader := func(blob structfs.Blob, targetPath string) {
+ content, err := blob.Open()
+ if err != nil {
+ log.Fatalf("error while uploading %q: %v", targetPath, err)
+ }
+ defer content.Close()
+
bar := progressbar.DefaultBytes(
- r.Size(),
+ blob.Size(),
targetPath,
)
defer bar.Close()
- proxyReader := progressbar.NewReader(r, bar)
+ proxyReader := progressbar.NewReader(content, bar)
defer proxyReader.Close()
if err := conn.Upload(ctx, targetPath, &proxyReader); err != nil {
diff --git a/metropolis/cli/metroctl/core/BUILD.bazel b/metropolis/cli/metroctl/core/BUILD.bazel
index 3a599a7..73527d5 100644
--- a/metropolis/cli/metroctl/core/BUILD.bazel
+++ b/metropolis/cli/metroctl/core/BUILD.bazel
@@ -21,6 +21,7 @@
"//osbase/blockdev",
"//osbase/fat32",
"//osbase/gpt",
+ "//osbase/structfs",
"@io_k8s_client_go//pkg/apis/clientauthentication/v1:clientauthentication",
"@io_k8s_client_go//tools/clientcmd",
"@io_k8s_client_go//tools/clientcmd/api",
diff --git a/metropolis/cli/metroctl/core/install.go b/metropolis/cli/metroctl/core/install.go
index 190f66a..5d43a89 100644
--- a/metropolis/cli/metroctl/core/install.go
+++ b/metropolis/cli/metroctl/core/install.go
@@ -4,7 +4,6 @@
package core
import (
- "bytes"
"errors"
"fmt"
"math"
@@ -16,6 +15,7 @@
"source.monogon.dev/osbase/blockdev"
"source.monogon.dev/osbase/fat32"
"source.monogon.dev/osbase/gpt"
+ "source.monogon.dev/osbase/structfs"
)
type MakeInstallerImageArgs struct {
@@ -23,13 +23,13 @@
TargetPath string
// Reader for the installer EFI executable. Mandatory.
- Installer fat32.SizedReader
+ Installer structfs.Blob
// Optional NodeParameters to be embedded for use by the installer.
NodeParams *api.NodeParameters
// Optional Reader for a Metropolis bundle for use by the installer.
- Bundle fat32.SizedReader
+ Bundle structfs.Blob
}
// MakeInstallerImage generates an installer disk image containing a Table
@@ -40,7 +40,7 @@
return errors.New("installer is mandatory")
}
- espRoot := fat32.Inode{Attrs: fat32.AttrDirectory}
+ var espRoot structfs.Tree
// This needs to be a "Removable Media" according to the UEFI Specification
// V2.9 Section 3.5.1.1. This file is booted by any compliant UEFI firmware
@@ -54,7 +54,7 @@
if err != nil {
return fmt.Errorf("failed to marshal node params: %w", err)
}
- if err := espRoot.PlaceFile("metropolis-installer/nodeparams.pb", bytes.NewReader(nodeParamsRaw)); err != nil {
+ if err := espRoot.PlaceFile("metropolis-installer/nodeparams.pb", structfs.Bytes(nodeParamsRaw)); err != nil {
return err
}
}
diff --git a/metropolis/cli/takeover/BUILD.bazel b/metropolis/cli/takeover/BUILD.bazel
index fe58518..c262164 100644
--- a/metropolis/cli/takeover/BUILD.bazel
+++ b/metropolis/cli/takeover/BUILD.bazel
@@ -40,6 +40,7 @@
"//osbase/kexec",
"//osbase/net/dump",
"//osbase/net/proto",
+ "//osbase/structfs",
"//osbase/supervisor",
"@com_github_cavaliergopher_cpio//:cpio",
"@com_github_klauspost_compress//zstd",
diff --git a/metropolis/cli/takeover/e2e/BUILD.bazel b/metropolis/cli/takeover/e2e/BUILD.bazel
index b085947..ae4810b 100644
--- a/metropolis/cli/takeover/e2e/BUILD.bazel
+++ b/metropolis/cli/takeover/e2e/BUILD.bazel
@@ -22,6 +22,7 @@
deps = [
"//osbase/fat32",
"//osbase/freeport",
+ "//osbase/structfs",
"@io_bazel_rules_go//go/runfiles",
"@org_golang_x_crypto//ssh",
"@org_golang_x_crypto//ssh/agent",
diff --git a/metropolis/cli/takeover/e2e/main_test.go b/metropolis/cli/takeover/e2e/main_test.go
index 1ba5354..afce515 100644
--- a/metropolis/cli/takeover/e2e/main_test.go
+++ b/metropolis/cli/takeover/e2e/main_test.go
@@ -25,6 +25,7 @@
"source.monogon.dev/osbase/fat32"
"source.monogon.dev/osbase/freeport"
+ "source.monogon.dev/osbase/structfs"
)
var (
@@ -112,25 +113,16 @@
t.Fatal(err)
}
- rootInode := fat32.Inode{
- Attrs: fat32.AttrDirectory,
- Children: []*fat32.Inode{
- {
- Name: "user-data",
- Content: strings.NewReader("#cloud-config\n" + string(userData)),
- },
- {
- Name: "meta-data",
- Content: strings.NewReader(""),
- },
- },
+ root := structfs.Tree{
+ structfs.File("user-data", structfs.Bytes("#cloud-config\n"+string(userData))),
+ structfs.File("meta-data", structfs.Bytes("")),
}
cloudInitDataFile, err := os.CreateTemp("", "cidata*.img")
if err != nil {
t.Fatal(err)
}
defer os.Remove(cloudInitDataFile.Name())
- if err := fat32.WriteFS(cloudInitDataFile, rootInode, fat32.Options{Label: "cidata"}); err != nil {
+ if err := fat32.WriteFS(cloudInitDataFile, root, fat32.Options{Label: "cidata"}); err != nil {
t.Fatal(err)
}
diff --git a/metropolis/cli/takeover/install.go b/metropolis/cli/takeover/install.go
index 005b590..2b76095 100644
--- a/metropolis/cli/takeover/install.go
+++ b/metropolis/cli/takeover/install.go
@@ -5,10 +5,8 @@
import (
"archive/zip"
- "bytes"
_ "embed"
"fmt"
- "io/fs"
"os"
"path/filepath"
@@ -16,24 +14,28 @@
"source.monogon.dev/osbase/blockdev"
"source.monogon.dev/osbase/build/mkimage/osimage"
"source.monogon.dev/osbase/efivarfs"
+ "source.monogon.dev/osbase/structfs"
)
//go:embed metropolis/node/core/abloader/abloader.efi
var abloader []byte
-// FileSizedReader is a small adapter from fs.File to fs.SizedReader
-// Panics on Stat() failure, so should only be used with sources where Stat()
-// cannot fail.
-type FileSizedReader struct {
- fs.File
+// zipBlob looks up a file in a [zip.Reader] and adapts it to [structfs.Blob].
+func zipBlob(reader *zip.Reader, name string) (zipFileBlob, error) {
+ for _, file := range reader.File {
+ if file.Name == name {
+ return zipFileBlob{file}, nil
+ }
+ }
+ return zipFileBlob{}, fmt.Errorf("file %q not found", name)
}
-func (f FileSizedReader) Size() int64 {
- stat, err := f.Stat()
- if err != nil {
- panic(err)
- }
- return stat.Size()
+type zipFileBlob struct {
+ *zip.File
+}
+
+func (f zipFileBlob) Size() int64 {
+ return int64(f.File.UncompressedSize64)
}
// EnvInstallTarget environment variable which tells the takeover binary where
@@ -93,12 +95,12 @@
return nil, fmt.Errorf("failed to open root device: %w", err)
}
- efiPayload, err := bundle.Open("kernel_efi.efi")
+ efiPayload, err := zipBlob(bundle, "kernel_efi.efi")
if err != nil {
return nil, fmt.Errorf("invalid bundle: %w", err)
}
- systemImage, err := bundle.Open("verity_rootfs.img")
+ systemImage, err := zipBlob(bundle, "verity_rootfs.img")
if err != nil {
return nil, fmt.Errorf("invalid bundle: %w", err)
}
@@ -110,9 +112,9 @@
Data: 128,
},
SystemImage: systemImage,
- EFIPayload: FileSizedReader{efiPayload},
- ABLoader: bytes.NewReader(abloader),
- NodeParameters: bytes.NewReader(metropolisSpecRaw),
+ EFIPayload: efiPayload,
+ ABLoader: structfs.Bytes(abloader),
+ NodeParameters: structfs.Bytes(metropolisSpecRaw),
Output: rootDev,
}, nil
}
diff --git a/metropolis/installer/BUILD.bazel b/metropolis/installer/BUILD.bazel
index 5792326..fe80669 100644
--- a/metropolis/installer/BUILD.bazel
+++ b/metropolis/installer/BUILD.bazel
@@ -16,6 +16,7 @@
"//osbase/bringup",
"//osbase/build/mkimage/osimage",
"//osbase/efivarfs",
+ "//osbase/structfs",
"//osbase/supervisor",
"//osbase/sysfs",
"@org_golang_x_sys//unix",
diff --git a/metropolis/installer/main.go b/metropolis/installer/main.go
index e9956fd..216f99c 100644
--- a/metropolis/installer/main.go
+++ b/metropolis/installer/main.go
@@ -8,12 +8,10 @@
import (
"archive/zip"
- "bytes"
"context"
_ "embed"
"errors"
"fmt"
- "io/fs"
"os"
"path/filepath"
"strings"
@@ -25,6 +23,7 @@
"source.monogon.dev/osbase/bringup"
"source.monogon.dev/osbase/build/mkimage/osimage"
"source.monogon.dev/osbase/efivarfs"
+ "source.monogon.dev/osbase/structfs"
"source.monogon.dev/osbase/supervisor"
"source.monogon.dev/osbase/sysfs"
)
@@ -104,19 +103,22 @@
return suitable, nil
}
-// FileSizedReader is a small adapter from fs.File to fs.SizedReader
-// Panics on Stat() failure, so should only be used with sources where Stat()
-// cannot fail.
-type FileSizedReader struct {
- fs.File
+// zipBlob looks up a file in a [zip.Reader] and adapts it to [structfs.Blob].
+func zipBlob(reader *zip.Reader, name string) (zipFileBlob, error) {
+ for _, file := range reader.File {
+ if file.Name == name {
+ return zipFileBlob{file}, nil
+ }
+ }
+ return zipFileBlob{}, fmt.Errorf("file %q not found", name)
}
-func (f FileSizedReader) Size() int64 {
- stat, err := f.Stat()
- if err != nil {
- panic(err)
- }
- return stat.Size()
+type zipFileBlob struct {
+ *zip.File
+}
+
+func (f zipFileBlob) Size() int64 {
+ return int64(f.File.UncompressedSize64)
}
func main() {
@@ -162,7 +164,7 @@
return fmt.Errorf("while mounting the installer ESP: %w", err)
}
- nodeParameters, err := os.Open("/installer/metropolis-installer/nodeparams.pb")
+ nodeParameters, err := structfs.OSPathBlob("/installer/metropolis-installer/nodeparams.pb")
if err != nil {
return fmt.Errorf("failed to open node parameters from ESP: %w", err)
}
@@ -173,16 +175,14 @@
return fmt.Errorf("failed to open node bundle from ESP: %w", err)
}
defer bundle.Close()
- efiPayload, err := bundle.Open("kernel_efi.efi")
+ efiPayload, err := zipBlob(&bundle.Reader, "kernel_efi.efi")
if err != nil {
return fmt.Errorf("cannot open EFI payload in bundle: %w", err)
}
- defer efiPayload.Close()
- systemImage, err := bundle.Open("verity_rootfs.img")
+ systemImage, err := zipBlob(&bundle.Reader, "verity_rootfs.img")
if err != nil {
return fmt.Errorf("cannot open system image in bundle: %w", err)
}
- defer systemImage.Close()
// Build the osimage parameters.
installParams := osimage.Params{
@@ -198,9 +198,9 @@
Data: 128,
},
SystemImage: systemImage,
- EFIPayload: FileSizedReader{efiPayload},
- ABLoader: bytes.NewReader(abloader),
- NodeParameters: FileSizedReader{nodeParameters},
+ EFIPayload: efiPayload,
+ ABLoader: structfs.Bytes(abloader),
+ NodeParameters: nodeParameters,
}
// Calculate the minimum target size based on the installation parameters.
minSize := uint64((installParams.PartitionSize.ESP +
diff --git a/metropolis/installer/test/BUILD.bazel b/metropolis/installer/test/BUILD.bazel
index 7f7ad54..7b7828d 100644
--- a/metropolis/installer/test/BUILD.bazel
+++ b/metropolis/installer/test/BUILD.bazel
@@ -24,6 +24,7 @@
"//metropolis/proto/api",
"//osbase/build/mkimage/osimage",
"//osbase/cmd",
+ "//osbase/structfs",
"@com_github_diskfs_go_diskfs//:go-diskfs",
"@com_github_diskfs_go_diskfs//disk",
"@com_github_diskfs_go_diskfs//partition/gpt",
diff --git a/metropolis/installer/test/run_test.go b/metropolis/installer/test/run_test.go
index 766c839..cd87852 100644
--- a/metropolis/installer/test/run_test.go
+++ b/metropolis/installer/test/run_test.go
@@ -7,7 +7,6 @@
package installer
import (
- "bytes"
"context"
"fmt"
"log"
@@ -27,6 +26,7 @@
mctl "source.monogon.dev/metropolis/cli/metroctl/core"
"source.monogon.dev/osbase/build/mkimage/osimage"
"source.monogon.dev/osbase/cmd"
+ "source.monogon.dev/osbase/structfs"
)
var (
@@ -134,21 +134,21 @@
func TestMain(m *testing.M) {
installerImage = filepath.Join(os.Getenv("TEST_TMPDIR"), "installer.img")
- installer, err := os.ReadFile(xInstallerPath)
+ installer, err := structfs.OSPathBlob(xInstallerPath)
if err != nil {
log.Fatal(err)
}
- bundle, err := os.ReadFile(xBundlePath)
+ bundle, err := structfs.OSPathBlob(xBundlePath)
if err != nil {
log.Fatal(err)
}
iargs := mctl.MakeInstallerImageArgs{
- Installer: bytes.NewReader(installer),
+ Installer: installer,
TargetPath: installerImage,
NodeParams: &api.NodeParameters{},
- Bundle: bytes.NewReader(bundle),
+ Bundle: bundle,
}
if err := mctl.MakeInstallerImage(iargs); err != nil {
log.Fatalf("Couldn't create the installer image at %q: %v", installerImage, err)
diff --git a/metropolis/node/core/update/e2e/BUILD.bazel b/metropolis/node/core/update/e2e/BUILD.bazel
index 3f6aa66..83716d1 100644
--- a/metropolis/node/core/update/e2e/BUILD.bazel
+++ b/metropolis/node/core/update/e2e/BUILD.bazel
@@ -25,9 +25,9 @@
"xAbloaderPath": "$(rlocationpath //metropolis/node/core/abloader )",
},
deps = [
- "//osbase/blkio",
"//osbase/blockdev",
"//osbase/build/mkimage/osimage",
+ "//osbase/structfs",
"@io_bazel_rules_go//go/runfiles",
],
)
diff --git a/metropolis/node/core/update/e2e/e2e_test.go b/metropolis/node/core/update/e2e/e2e_test.go
index f7d0e36..4a1c6a2 100644
--- a/metropolis/node/core/update/e2e/e2e_test.go
+++ b/metropolis/node/core/update/e2e/e2e_test.go
@@ -21,9 +21,9 @@
"github.com/bazelbuild/rules_go/go/runfiles"
- "source.monogon.dev/osbase/blkio"
"source.monogon.dev/osbase/blockdev"
"source.monogon.dev/osbase/build/mkimage/osimage"
+ "source.monogon.dev/osbase/structfs"
)
var (
@@ -197,18 +197,16 @@
t.Cleanup(func() { os.Remove(rootDevPath) })
defer rootDisk.Close()
- boot, err := blkio.NewFileReader(xBootPath)
+ boot, err := structfs.OSPathBlob(xBootPath)
if err != nil {
t.Fatal(err)
}
- defer boot.Close()
- system, err := os.Open(xSystemXPath)
+ system, err := structfs.OSPathBlob(xSystemXPath)
if err != nil {
t.Fatal(err)
}
- defer system.Close()
- loader, err := blkio.NewFileReader(xAbloaderPath)
+ loader, err := structfs.OSPathBlob(xAbloaderPath)
if err != nil {
t.Fatal(err)
}
diff --git a/osbase/blkio/BUILD.bazel b/osbase/blkio/BUILD.bazel
deleted file mode 100644
index 8b23a34..0000000
--- a/osbase/blkio/BUILD.bazel
+++ /dev/null
@@ -1,8 +0,0 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
-
-go_library(
- name = "blkio",
- srcs = ["blkio.go"],
- importpath = "source.monogon.dev/osbase/blkio",
- visibility = ["//visibility:public"],
-)
diff --git a/osbase/blkio/blkio.go b/osbase/blkio/blkio.go
deleted file mode 100644
index a4669ab..0000000
--- a/osbase/blkio/blkio.go
+++ /dev/null
@@ -1,86 +0,0 @@
-// Copyright The Monogon Project Authors.
-// SPDX-License-Identifier: Apache-2.0
-
-package blkio
-
-import (
- "fmt"
- "io"
- "os"
-)
-
-type ReaderWithSize struct {
- io.Reader
- size int64
-}
-
-// SizedReader is an io.Reader with a known size
-type SizedReader interface {
- io.Reader
- Size() int64
-}
-
-// NewSizedReader returns a SizedReader given a reader and a size.
-// The returned SizedReader is a ReaderWithSize.
-func NewSizedReader(r io.Reader, size int64) SizedReader {
- return &ReaderWithSize{r, size}
-}
-
-func (r *ReaderWithSize) Size() int64 {
- return r.size
-}
-
-// LazyFileReader implements a SizedReader which opens a file on first read
-// and closes it again after the reader has reached EOF.
-type LazyFileReader struct {
- name string
- size int64
- f *os.File
- done bool
-}
-
-func (r *LazyFileReader) init() error {
- f, err := os.Open(r.name)
- if err != nil {
- return fmt.Errorf("failed to open file for reading: %w", err)
- }
- r.f = f
- return nil
-}
-
-func (r *LazyFileReader) Size() int64 {
- return r.size
-}
-
-func (r *LazyFileReader) Read(b []byte) (n int, err error) {
- if r.done {
- return 0, io.EOF
- }
- if r.f == nil {
- if err = r.init(); err != nil {
- return
- }
- }
- n, err = r.f.Read(b)
- if err == io.EOF {
- r.done = true
- r.f.Close()
- }
- return
-}
-
-func (r *LazyFileReader) Close() {
- r.done = true
- r.f.Close()
-}
-
-func NewFileReader(name string) (*LazyFileReader, error) {
- info, err := os.Stat(name)
- if err != nil {
- return nil, fmt.Errorf("failed to stat: %w", err)
- }
- return &LazyFileReader{
- size: info.Size(),
- name: name,
- }, nil
-}
diff --git a/osbase/build/mkimage/BUILD.bazel b/osbase/build/mkimage/BUILD.bazel
index a822c62..aba561e 100644
--- a/osbase/build/mkimage/BUILD.bazel
+++ b/osbase/build/mkimage/BUILD.bazel
@@ -6,9 +6,9 @@
importpath = "source.monogon.dev/osbase/build/mkimage",
visibility = ["//visibility:private"],
deps = [
- "//osbase/blkio",
"//osbase/blockdev",
"//osbase/build/mkimage/osimage",
+ "//osbase/structfs",
],
)
diff --git a/osbase/build/mkimage/main.go b/osbase/build/mkimage/main.go
index 79e2920..bef7517 100644
--- a/osbase/build/mkimage/main.go
+++ b/osbase/build/mkimage/main.go
@@ -19,9 +19,9 @@
"log"
"os"
- "source.monogon.dev/osbase/blkio"
"source.monogon.dev/osbase/blockdev"
"source.monogon.dev/osbase/build/mkimage/osimage"
+ "source.monogon.dev/osbase/structfs"
)
func main() {
@@ -51,33 +51,30 @@
// Open the input files for osimage.Create, fill in reader objects and
// metadata in osimage.Params.
// Start with the EFI Payload the OS will boot from.
- p, err := blkio.NewFileReader(efiPayload)
+ var err error
+ cfg.EFIPayload, err = structfs.OSPathBlob(efiPayload)
if err != nil {
log.Fatalf("while opening the EFI payload at %q: %v", efiPayload, err)
}
- cfg.EFIPayload = p
- ab, err := blkio.NewFileReader(abLoaderPayload)
+ cfg.ABLoader, err = structfs.OSPathBlob(abLoaderPayload)
if err != nil {
log.Fatalf("while opening the abloader payload at %q: %v", abLoaderPayload, err)
}
- cfg.ABLoader = ab
// Attempt to open the system image if its path is set. In case the path
// isn't set, the system partition will still be created, but no
// contents will be written into it.
if systemImage != "" {
- img, err := os.Open(systemImage)
+ cfg.SystemImage, err = structfs.OSPathBlob(systemImage)
if err != nil {
log.Fatalf("while opening the system image at %q: %v", systemImage, err)
}
- defer img.Close()
- cfg.SystemImage = img
}
// Attempt to open the node parameters file if its path is set.
if nodeParams != "" {
- np, err := blkio.NewFileReader(nodeParams)
+ np, err := structfs.OSPathBlob(nodeParams)
if err != nil {
log.Fatalf("while opening node parameters at %q: %v", nodeParams, err)
}
diff --git a/osbase/build/mkimage/osimage/BUILD.bazel b/osbase/build/mkimage/osimage/BUILD.bazel
index cfcf096..f85fa20 100644
--- a/osbase/build/mkimage/osimage/BUILD.bazel
+++ b/osbase/build/mkimage/osimage/BUILD.bazel
@@ -10,6 +10,7 @@
"//osbase/efivarfs",
"//osbase/fat32",
"//osbase/gpt",
+ "//osbase/structfs",
"@com_github_google_uuid//:uuid",
],
)
diff --git a/osbase/build/mkimage/osimage/osimage.go b/osbase/build/mkimage/osimage/osimage.go
index 71b5499..4cd3952 100644
--- a/osbase/build/mkimage/osimage/osimage.go
+++ b/osbase/build/mkimage/osimage/osimage.go
@@ -16,6 +16,7 @@
"source.monogon.dev/osbase/efivarfs"
"source.monogon.dev/osbase/fat32"
"source.monogon.dev/osbase/gpt"
+ "source.monogon.dev/osbase/structfs"
)
var (
@@ -60,17 +61,17 @@
Output blockdev.BlockDev
// ABLoader provides the A/B loader which then loads the EFI loader for the
// correct slot.
- ABLoader fat32.SizedReader
+ ABLoader structfs.Blob
// EFIPayload provides contents of the EFI payload file. It must not be
// nil. This gets put into boot slot A.
- EFIPayload fat32.SizedReader
+ EFIPayload structfs.Blob
// SystemImage provides contents of the Metropolis system partition.
// If nil, no contents will be copied into the partition.
- SystemImage io.Reader
+ SystemImage structfs.Blob
// NodeParameters provides contents of the node parameters file. If nil,
// the node parameters file won't be created in the target ESP
// filesystem.
- NodeParameters fat32.SizedReader
+ NodeParameters structfs.Blob
// DiskGUID is a unique identifier of the image and a part of Table
// header. It's optional and can be left blank if the identifier is
// to be randomly generated. Setting it to a predetermined value can
@@ -86,7 +87,7 @@
type plan struct {
*Params
- rootInode fat32.Inode
+ efiRoot structfs.Tree
tbl *gpt.Table
efiPartition *gpt.Partition
systemPartitionA *gpt.Partition
@@ -100,7 +101,7 @@
// Ignore errors, this is only advisory.
i.Output.Discard(0, i.Output.BlockCount()*i.Output.BlockSize())
- if err := fat32.WriteFS(blockdev.NewRWS(i.efiPartition), i.rootInode, fat32.Options{
+ if err := fat32.WriteFS(blockdev.NewRWS(i.efiPartition), i.efiRoot, fat32.Options{
BlockSize: uint16(i.efiPartition.BlockSize()),
BlockCount: uint32(i.efiPartition.BlockCount()),
Label: "MNGN_BOOT",
@@ -108,9 +109,15 @@
return nil, fmt.Errorf("failed to write FAT32: %w", err)
}
- if _, err := io.Copy(blockdev.NewRWS(i.systemPartitionA), i.SystemImage); err != nil {
+ systemImage, err := i.SystemImage.Open()
+ if err != nil {
+ return nil, fmt.Errorf("failed to open system image: %w", err)
+ }
+ if _, err := io.Copy(blockdev.NewRWS(i.systemPartitionA), systemImage); err != nil {
+ systemImage.Close()
return nil, fmt.Errorf("failed to write system partition A: %w", err)
}
+ systemImage.Close()
if err := i.tbl.Write(); err != nil {
return nil, fmt.Errorf("failed to write Table: %w", err)
@@ -155,25 +162,22 @@
return nil, fmt.Errorf("failed to allocate ESP: %w", err)
}
- params.rootInode = fat32.Inode{
- Attrs: fat32.AttrDirectory,
- }
- if err := params.rootInode.PlaceFile(strings.TrimPrefix(EFIBootAPath, "/"), params.EFIPayload); err != nil {
+ if err := params.efiRoot.PlaceFile(strings.TrimPrefix(EFIBootAPath, "/"), params.EFIPayload); err != nil {
return nil, err
}
// Place the A/B loader at the EFI bootloader autodiscovery path.
- if err := params.rootInode.PlaceFile(strings.TrimPrefix(EFIPayloadPath, "/"), params.ABLoader); err != nil {
+ if err := params.efiRoot.PlaceFile(strings.TrimPrefix(EFIPayloadPath, "/"), params.ABLoader); err != nil {
return nil, err
}
if params.NodeParameters != nil {
- if err := params.rootInode.PlaceFile(nodeParamsPath, params.NodeParameters); err != nil {
+ if err := params.efiRoot.PlaceFile(nodeParamsPath, params.NodeParameters); err != nil {
return nil, err
}
}
// Try to layout the fat32 partition. If it detects that the disk is too
// small, an error will be returned.
- if _, err := fat32.SizeFS(params.rootInode, fat32.Options{
+ if _, err := fat32.SizeFS(params.efiRoot, fat32.Options{
BlockSize: uint16(params.efiPartition.BlockSize()),
BlockCount: uint32(params.efiPartition.BlockCount()),
Label: "MNGN_BOOT",
diff --git a/osbase/fat32/BUILD.bazel b/osbase/fat32/BUILD.bazel
index 1e0e909..ac3eb96 100644
--- a/osbase/fat32/BUILD.bazel
+++ b/osbase/fat32/BUILD.bazel
@@ -11,6 +11,7 @@
],
importpath = "source.monogon.dev/osbase/fat32",
visibility = ["//visibility:public"],
+ deps = ["//osbase/structfs"],
)
go_test(
@@ -28,6 +29,7 @@
"xFsckPath": "$(rlocationpath @com_github_dosfstools_dosfstools//:fsck )",
},
deps = [
+ "//osbase/structfs",
"@com_github_stretchr_testify//assert",
"@com_github_stretchr_testify//require",
"@io_bazel_rules_go//go/runfiles",
diff --git a/osbase/fat32/dos83.go b/osbase/fat32/dos83.go
index 1d988be..b8d219d 100644
--- a/osbase/fat32/dos83.go
+++ b/osbase/fat32/dos83.go
@@ -28,12 +28,12 @@
// well as valid ASCII
var validDOSName = regexp.MustCompile(`^^([A-Z0-9!#$%&'()@^_\x60{}~-]{0,8})(\.[A-Z0-9!#$%&'()-@^_\x60{}~-]{1,3})?$`)
-func makeUniqueDOSNames(inodes []*Inode) error {
+func makeUniqueDOSNames(nodes []*node) error {
taken := make(map[[11]byte]bool)
- var lossyNameInodes []*Inode
+ var lossyNameNodes []*node
// Make two passes to ensure that names can always be passed through even
// if they would conflict with a generated name.
- for _, i := range inodes {
+ for _, i := range nodes {
for j := range i.dosName {
i.dosName[j] = ' '
}
@@ -54,7 +54,7 @@
taken[i.dosName] = true
continue
}
- lossyNameInodes = append(lossyNameInodes, i)
+ lossyNameNodes = append(lossyNameNodes, i)
}
// Willfully ignore the recommended short name generation algorithm as it
// requires tons of bookkeeping and doesn't result in stable names so
@@ -63,7 +63,7 @@
// of that because of long file name entries), so 4 hex characters
// guarantee uniqueness, regardless of the rest of name.
var nameIdx int
- for _, i := range lossyNameInodes {
+ for _, i := range lossyNameNodes {
nameUpper := strings.ToUpper(i.Name)
dotParts := strings.Split(nameUpper, ".")
for j := range dotParts {
diff --git a/osbase/fat32/fat32.go b/osbase/fat32/fat32.go
index e5c2e58..340bb45 100644
--- a/osbase/fat32/fat32.go
+++ b/osbase/fat32/fat32.go
@@ -13,9 +13,10 @@
"io/fs"
"math"
"math/bits"
- "strings"
"time"
"unicode/utf16"
+
+ "source.monogon.dev/osbase/structfs"
)
// This package contains multiple references to the FAT32 specification, called
@@ -33,7 +34,7 @@
// as large as it needs to be.
BlockCount uint32
- // Human-readable filesystem label. Maximum 10 bytes (gets cut off), should
+ // Human-readable filesystem label. Maximum 11 bytes (gets cut off), should
// be uppercase alphanumeric.
Label string
@@ -42,13 +43,7 @@
ID uint32
}
-// SizedReader is an io.Reader with a known size
-type SizedReader interface {
- io.Reader
- Size() int64
-}
-
-// Attribute is a bitset of flags set on an inode.
+// Attribute is a bitset of flags set on a directory entry.
// See also the spec page 24
type Attribute uint8
@@ -59,39 +54,51 @@
AttrHidden Attribute = 0x02
// AttrSystem indicates that this is an operating system file.
AttrSystem Attribute = 0x04
- // AttrDirectory indicates that this is a directory and not a file.
- AttrDirectory Attribute = 0x10
+ // attrVolumeID indicates that this is a special directory entry which
+ // contains the volume label.
+ attrVolumeID Attribute = 0x08
+ // attrDirectory indicates that this is a directory and not a file.
+ attrDirectory Attribute = 0x10
// AttrArchive canonically indicates that a file has been created/modified
// since the last backup. Its use in practice is inconsistent.
AttrArchive Attribute = 0x20
)
-// Inode is file or directory on the FAT32 filesystem. Note that the concept
-// of an inode doesn't really exist on FAT32, its directories are just special
-// files.
-type Inode struct {
- // Name of the file or directory (not including its path)
- Name string
- // Time the file or directory was last modified
- ModTime time.Time
+// DirEntrySys contains additional directory entry fields which are specific to
+// FAT32. To set these fields, the Sys field of a [structfs.Node] can be set to
+// a pointer to this or to a struct which embeds it.
+type DirEntrySys struct {
// Time the file or directory was created
CreateTime time.Time
// Attributes
Attrs Attribute
- // Children of this directory (only valid when Attrs has AttrDirectory set)
- Children []*Inode
- // Content of this file
- // Only valid when Attrs doesn't have AttrDirectory set.
- Content SizedReader
+}
- // Filled out on placement and write-out
- startCluster int
- parent *Inode
+func (d *DirEntrySys) FAT32() *DirEntrySys {
+ return d
+}
+
+// DirEntrySysAccessor is used to access [DirEntrySys] instead of directly type
+// asserting the struct, to allow for embedding.
+type DirEntrySysAccessor interface {
+ FAT32() *DirEntrySys
+}
+
+// node is a file or directory on the FAT32 filesystem. It wraps a
+// [structfs.Node] and holds additional fields which are filled during planning.
+type node struct {
+ *structfs.Node
dosName [11]byte
+ createTime time.Time
+ attrs Attribute
+ parent *node
+ children []*node
+ size uint32
+ startCluster int
}
// Number of LFN entries + normal entry (all 32 bytes)
-func (i Inode) metaSize() (int64, error) {
+func (i node) metaSize() (int64, error) {
fileNameUTF16 := utf16.Encode([]rune(i.Name))
// VFAT file names are null-terminated
fileNameUTF16 = append(fileNameUTF16, 0x00)
@@ -112,9 +119,9 @@
return sum
}
-// writeMeta writes information about this inode into the contents of the parent
-// inode.
-func (i Inode) writeMeta(w io.Writer) error {
+// writeMeta writes information about this node into the contents of the parent
+// node.
+func (i node) writeMeta(w io.Writer) error {
fileNameUTF16 := utf16.Encode([]rune(i.Name))
// VFAT file names are null-terminated
fileNameUTF16 = append(fileNameUTF16, 0x00)
@@ -153,21 +160,15 @@
return err
}
}
- selfSize, err := i.dataSize()
- if err != nil {
- return err
- }
- if selfSize >= 4*1024*1024*1024 {
- return errors.New("single file size exceeds 4GiB which is prohibited in FAT32")
- }
- if i.Attrs&AttrDirectory != 0 {
+ selfSize := i.size
+ if i.attrs&attrDirectory != 0 {
selfSize = 0 // Directories don't have an explicit size
}
date, t, _ := timeToMsDosTime(i.ModTime)
- cdate, ctime, ctens := timeToMsDosTime(i.CreateTime)
+ cdate, ctime, ctens := timeToMsDosTime(i.createTime)
if err := binary.Write(w, binary.LittleEndian, &dirEntry{
DOSName: i.dosName,
- Attributes: uint8(i.Attrs),
+ Attributes: uint8(i.attrs),
CreationTenMilli: ctens,
CreationTime: ctime,
CreationDate: cdate,
@@ -175,27 +176,27 @@
LastWrittenToTime: t,
LastWrittenToDate: date,
FirstClusterLow: uint16(i.startCluster & 0xffff),
- FileSize: uint32(selfSize),
+ FileSize: selfSize,
}); err != nil {
return err
}
return nil
}
-// writeData writes the contents of this inode (including possible metadata
+// writeData writes the contents of this node (including possible metadata
// of its children, but not its children's data)
-func (i Inode) writeData(w io.Writer, volumeLabel [11]byte) error {
- if i.Attrs&AttrDirectory != 0 {
+func (i node) writeData(w io.Writer, volumeLabel [11]byte) error {
+ if i.attrs&attrDirectory != 0 {
if i.parent == nil {
if err := binary.Write(w, binary.LittleEndian, &dirEntry{
DOSName: volumeLabel,
- Attributes: 0x08, // Volume ID, internal use only
+ Attributes: uint8(attrVolumeID),
}); err != nil {
return err
}
} else {
date, t, _ := timeToMsDosTime(i.ModTime)
- cdate, ctime, ctens := timeToMsDosTime(i.CreateTime)
+ cdate, ctime, ctens := timeToMsDosTime(i.createTime)
if err := binary.Write(w, binary.LittleEndian, &dirEntry{
DOSName: [11]byte{'.', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '},
CreationDate: cdate,
@@ -203,7 +204,7 @@
CreationTenMilli: ctens,
LastWrittenToTime: t,
LastWrittenToDate: date,
- Attributes: uint8(i.Attrs),
+ Attributes: uint8(i.attrs),
FirstClusterHigh: uint16(i.startCluster >> 16),
FirstClusterLow: uint16(i.startCluster & 0xffff),
}); err != nil {
@@ -224,93 +225,56 @@
CreationTenMilli: ctens,
LastWrittenToTime: t,
LastWrittenToDate: date,
- Attributes: uint8(AttrDirectory),
+ Attributes: uint8(attrDirectory),
FirstClusterHigh: uint16(startCluster >> 16),
FirstClusterLow: uint16(startCluster & 0xffff),
}); err != nil {
return err
}
}
- err := makeUniqueDOSNames(i.Children)
- if err != nil {
- return err
- }
- for _, c := range i.Children {
+ for _, c := range i.children {
if err := c.writeMeta(w); err != nil {
return err
}
}
} else {
- if _, err := io.CopyN(w, i.Content, i.Content.Size()); err != nil {
+ content, err := i.Content.Open()
+ if err != nil {
+ return err
+ }
+ defer content.Close()
+ if _, err := io.CopyN(w, content, int64(i.size)); err != nil {
return err
}
}
return nil
}
-func (i Inode) dataSize() (int64, error) {
- if i.Attrs&AttrDirectory != 0 {
- var size int64
- if i.parent != nil {
- // Dot and dotdot directories
- size += 2 * 32
- } else {
- // Volume ID
- size += 1 * 32
- }
- for _, c := range i.Children {
- cs, err := c.metaSize()
- if err != nil {
- return 0, err
- }
- size += cs
- }
- if size > 2*1024*1024 {
- return 0, errors.New("directory contains > 2MiB of metadata which is prohibited in FAT32")
- }
- return size, nil
+func (i node) dirSize() (uint32, error) {
+ var size int64
+ if i.parent != nil {
+ // Dot and dotdot directories
+ size += 2 * 32
} else {
- return i.Content.Size(), nil
+ // Volume ID
+ size += 1 * 32
}
-}
-
-func (i *Inode) PlaceFile(path string, reader SizedReader) error {
- pathParts := strings.Split(path, "/")
- inodeRef := i
- for j, part := range pathParts {
- var childExists bool
- for _, child := range inodeRef.Children {
- if strings.EqualFold(child.Name, part) {
- inodeRef = child
- childExists = true
- break
- }
+ for _, c := range i.children {
+ cs, err := c.metaSize()
+ if err != nil {
+ return 0, err
}
- if j == len(pathParts)-1 { // Is last path part (i.e. file name)
- if childExists {
- return &fs.PathError{Path: path, Err: fs.ErrExist, Op: "create"}
- }
- newInode := &Inode{
- Name: part,
- Content: reader,
- }
- inodeRef.Children = append(inodeRef.Children, newInode)
- return nil
- } else if !childExists {
- newInode := &Inode{
- Name: part,
- Attrs: AttrDirectory,
- }
- inodeRef.Children = append(inodeRef.Children, newInode)
- inodeRef = newInode
- }
+ size += cs
}
- panic("unreachable")
+ if size > 2*1024*1024 {
+ return 0, errors.New("directory contains > 2MiB of metadata which is prohibited in FAT32")
+ }
+ return uint32(size), nil
}
type planningState struct {
- // List of inodes in filesystem layout order
- orderedInodes []*Inode
+ // List of nodes in filesystem layout order
+ orderedNodes []*node
// File Allocation Table
fat []uint32
// Size of a single cluster in the FAT in bytes
@@ -335,16 +299,54 @@
return allocStartCluster
}
-func (i *Inode) placeRecursively(p *planningState) error {
- selfDataSize, err := i.dataSize()
- if err != nil {
- return fmt.Errorf("%s: %w", i.Name, err)
+func (i *node) placeRecursively(p *planningState) error {
+ if i.Mode.IsDir() {
+ for _, c := range i.Node.Children {
+ node := &node{
+ Node: c,
+ createTime: c.ModTime,
+ parent: i,
+ }
+ if sys, ok := c.Sys.(DirEntrySysAccessor); ok {
+ sys := sys.FAT32()
+ node.attrs = sys.Attrs & (AttrReadOnly | AttrHidden | AttrSystem | AttrArchive)
+ if !sys.CreateTime.IsZero() {
+ node.createTime = sys.CreateTime
+ }
+ }
+ switch {
+ case c.Mode.IsRegular():
+ size := c.Content.Size()
+ if size < 0 {
+ return fmt.Errorf("%s: negative file size", c.Name)
+ }
+ if size >= 4*1024*1024*1024 {
+ return fmt.Errorf("%s: single file size exceeds 4GiB which is prohibited in FAT32", c.Name)
+ }
+ node.size = uint32(size)
+ if len(c.Children) != 0 {
+ return fmt.Errorf("%s: file cannot have children", c.Name)
+ }
+ case c.Mode.IsDir():
+ node.attrs |= attrDirectory
+ default:
+ return fmt.Errorf("%s: unsupported file type %s", c.Name, c.Mode.Type().String())
+ }
+ i.children = append(i.children, node)
+ }
+ err := makeUniqueDOSNames(i.children)
+ if err != nil {
+ return err
+ }
+ i.size, err = i.dirSize()
+ if err != nil {
+ return fmt.Errorf("%s: %w", i.Name, err)
+ }
}
- i.startCluster = p.allocBytes(selfDataSize)
- p.orderedInodes = append(p.orderedInodes, i)
- for _, c := range i.Children {
- c.parent = i
- err = c.placeRecursively(p)
+ i.startCluster = p.allocBytes(int64(i.size))
+ p.orderedNodes = append(p.orderedNodes, i)
+ for _, c := range i.children {
+ err := c.placeRecursively(p)
if err != nil {
return fmt.Errorf("%s/%w", i.Name, err)
}
@@ -352,10 +354,9 @@
return nil
}
-// WriteFS writes a filesystem described by a root inode and its children to a
-// given io.Writer.
-func WriteFS(w io.Writer, rootInode Inode, opts Options) error {
- bs, fsi, p, err := prepareFS(&opts, rootInode)
+// WriteFS writes a filesystem described by a tree to a given io.Writer.
+func WriteFS(w io.Writer, root structfs.Tree, opts Options) error {
+ bs, fsi, p, err := prepareFS(&opts, root)
if err != nil {
return err
}
@@ -405,9 +406,9 @@
}
}
- for _, i := range p.orderedInodes {
+ for _, i := range p.orderedNodes {
if err := i.writeData(wb, bs.Label); err != nil {
- return fmt.Errorf("failed to write inode %q: %w", i.Name, err)
+ return fmt.Errorf("failed to write contents of %q: %w", i.Name, err)
}
if err := wb.FinishBlock(int64(opts.BlockSize)*int64(bs.BlocksPerCluster), true); err != nil {
return err
@@ -420,7 +421,7 @@
return nil
}
-func prepareFS(opts *Options, rootInode Inode) (*bootSector, *fsinfo, *planningState, error) {
+func prepareFS(opts *Options, root structfs.Tree) (*bootSector, *fsinfo, *planningState, error) {
if opts.BlockSize == 0 {
opts.BlockSize = 512
}
@@ -437,9 +438,6 @@
}
opts.ID = binary.BigEndian.Uint32(buf[:])
}
- if rootInode.Attrs&AttrDirectory == 0 {
- return nil, nil, nil, errors.New("root inode must be a directory (i.e. have AttrDirectory set)")
- }
bs := bootSector{
// Assembled x86_32 machine code corresponding to
// jmp $
@@ -499,7 +497,14 @@
}
// First two clusters are special
p.fat = append(p.fat, 0x0fffff00|uint32(bs.MediaCode), 0x0fffffff)
- err := rootInode.placeRecursively(&p)
+ rootNode := &node{
+ Node: &structfs.Node{
+ Mode: fs.ModeDir,
+ Children: root,
+ },
+ attrs: attrDirectory,
+ }
+ err := rootNode.placeRecursively(&p)
if err != nil {
return nil, nil, nil, err
}
@@ -514,7 +519,7 @@
p.fat = append(p.fat, fatFree)
}
- bs.RootClusterNumber = uint32(rootInode.startCluster)
+ bs.RootClusterNumber = uint32(rootNode.startCluster)
bs.BlocksPerFAT = uint32(binary.Size(p.fat)+int(opts.BlockSize)-1) / uint32(opts.BlockSize)
occupiedBlocks := uint32(bs.ReservedBlocks) + (uint32(len(p.fat)-2) * uint32(bs.BlocksPerCluster)) + bs.BlocksPerFAT*uint32(bs.NumFATs)
@@ -552,10 +557,10 @@
}
// SizeFS returns the number of blocks required to hold the filesystem defined
-// by rootInode and opts. This can be used for sizing calculations before
-// calling WriteFS.
-func SizeFS(rootInode Inode, opts Options) (int64, error) {
- bs, _, _, err := prepareFS(&opts, rootInode)
+// by root and opts. This can be used for sizing calculations before calling
+// WriteFS.
+func SizeFS(root structfs.Tree, opts Options) (int64, error) {
+ bs, _, _, err := prepareFS(&opts, root)
if err != nil {
return 0, err
}
diff --git a/osbase/fat32/fsck_test.go b/osbase/fat32/fsck_test.go
index d4908be..5043003 100644
--- a/osbase/fat32/fsck_test.go
+++ b/osbase/fat32/fsck_test.go
@@ -14,6 +14,8 @@
"time"
"github.com/bazelbuild/rules_go/go/runfiles"
+
+ "source.monogon.dev/osbase/structfs"
)
var (
@@ -39,14 +41,14 @@
}
}
-func testWithFsck(t *testing.T, rootInode Inode, opts Options) {
+func testWithFsck(t *testing.T, root structfs.Tree, opts Options) {
t.Helper()
testFile, err := os.CreateTemp("", "fat32-fsck-test")
if err != nil {
t.Fatal(err)
}
defer os.Remove(testFile.Name())
- sizeBlocks, err := SizeFS(rootInode, opts)
+ sizeBlocks, err := SizeFS(root, opts)
if err != nil {
t.Fatalf("failed to calculate size: %v", err)
}
@@ -62,7 +64,7 @@
t.Fatalf("seek failed: %v", err)
}
- if err := WriteFS(testFile, rootInode, opts); err != nil {
+ if err := WriteFS(testFile, root, opts); err != nil {
t.Fatalf("failed to write test FS: %v", err)
}
// Run fsck non-interactively (-n), disallow spaces in short file names (-S)
@@ -89,11 +91,7 @@
for _, blockSize := range []uint16{512, 4096, 32768} {
for _, fixed := range []string{"", "Fixed"} {
t.Run(fmt.Sprintf("BlockSize%d%v", blockSize, fixed), func(t *testing.T) {
- rootInode := Inode{
- Attrs: AttrDirectory,
- ModTime: time.Date(2022, 03, 04, 5, 6, 7, 8, time.UTC),
- CreateTime: time.Date(2022, 03, 04, 5, 6, 7, 8, time.UTC),
- }
+ var root structfs.Tree
files := []struct {
name string
path string
@@ -105,7 +103,7 @@
{"LargeFile", "test1/largefile.txt", largeString.String()},
}
for _, c := range files {
- err := rootInode.PlaceFile(c.path, strings.NewReader(c.content))
+ err := root.PlaceFile(c.path, structfs.Bytes(c.content))
if err != nil {
t.Errorf("failed to place file: %v", err)
}
@@ -115,7 +113,7 @@
// Use a block count that is slightly higher than the minimum
opts.BlockCount = 67000
}
- testWithFsck(t, rootInode, opts)
+ testWithFsck(t, root, opts)
})
}
}
@@ -125,19 +123,18 @@
if os.Getenv("IN_KTEST") == "true" {
t.Skip("In ktest")
}
- rootInode := Inode{
- Attrs: AttrDirectory,
- ModTime: time.Date(2022, 03, 04, 5, 6, 7, 8, time.UTC),
- }
+ var root structfs.Tree
for i := 0; i < (32*1024)-2; i++ {
- rootInode.Children = append(rootInode.Children, &Inode{
+ root = append(root, &structfs.Node{
Name: fmt.Sprintf("test%d", i),
- Content: strings.NewReader("random test content"),
- // Add some random attributes
- Attrs: AttrHidden | AttrSystem,
- // And a random ModTime
+ Content: structfs.Bytes("random test content"),
+ // Add a random ModTime
ModTime: time.Date(2022, 03, 04, 5, 6, 7, 8, time.UTC),
+ Sys: &DirEntrySys{
+ // Add some random attributes
+ Attrs: AttrHidden | AttrSystem,
+ },
})
}
- testWithFsck(t, rootInode, Options{ID: 1234, Label: "TEST"})
+ testWithFsck(t, root, Options{ID: 1234, Label: "TEST"})
}
diff --git a/osbase/fat32/linux_test.go b/osbase/fat32/linux_test.go
index ffe7672..002ac2c 100644
--- a/osbase/fat32/linux_test.go
+++ b/osbase/fat32/linux_test.go
@@ -4,9 +4,9 @@
package fat32
import (
- "bytes"
"fmt"
"io"
+ "io/fs"
"math/rand"
"os"
"strings"
@@ -16,6 +16,8 @@
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sys/unix"
+
+ "source.monogon.dev/osbase/structfs"
)
func TestKernelInterop(t *testing.T) {
@@ -25,7 +27,7 @@
type testCase struct {
name string
- setup func(root *Inode) error
+ setup func() structfs.Tree
validate func(t *testing.T) error
}
@@ -43,14 +45,15 @@
tests := []testCase{
{
name: "SimpleFolder",
- setup: func(root *Inode) error {
- root.Children = []*Inode{{
- Name: "testdir",
- Attrs: AttrDirectory,
- CreateTime: testTimestamp1,
- ModTime: testTimestamp2,
+ setup: func() structfs.Tree {
+ return structfs.Tree{{
+ Name: "testdir",
+ Mode: fs.ModeDir,
+ ModTime: testTimestamp2,
+ Sys: &DirEntrySys{
+ CreateTime: testTimestamp1,
+ },
}}
- return nil
},
validate: func(t *testing.T) error {
var stat unix.Statx_t
@@ -81,14 +84,15 @@
},
{
name: "SimpleFile",
- setup: func(root *Inode) error {
- root.Children = []*Inode{{
- Name: "testfile",
- CreateTime: testTimestamp3,
- ModTime: testTimestamp4,
- Content: strings.NewReader(testContent1),
+ setup: func() structfs.Tree {
+ return structfs.Tree{{
+ Name: "testfile",
+ ModTime: testTimestamp4,
+ Sys: &DirEntrySys{
+ CreateTime: testTimestamp3,
+ },
+ Content: structfs.Bytes(testContent1),
}}
- return nil
},
validate: func(t *testing.T) error {
var stat unix.Statx_t
@@ -118,20 +122,23 @@
},
{
name: "FolderHierarchy",
- setup: func(i *Inode) error {
- i.Children = []*Inode{{
- Name: "l1",
- Attrs: AttrDirectory,
- CreateTime: testTimestamp1,
- ModTime: testTimestamp2,
- Children: []*Inode{{
- Name: "l2",
- Attrs: AttrDirectory,
+ setup: func() structfs.Tree {
+ return structfs.Tree{{
+ Name: "l1",
+ Mode: fs.ModeDir,
+ ModTime: testTimestamp2,
+ Sys: &DirEntrySys{
CreateTime: testTimestamp1,
- ModTime: testTimestamp2,
+ },
+ Children: structfs.Tree{{
+ Name: "l2",
+ Mode: fs.ModeDir,
+ ModTime: testTimestamp2,
+ Sys: &DirEntrySys{
+ CreateTime: testTimestamp1,
+ },
}},
}}
- return nil
},
validate: func(t *testing.T) error {
dirInfo, err := os.ReadDir("/dut/l1")
@@ -149,14 +156,13 @@
},
{
name: "LargeFile",
- setup: func(i *Inode) error {
+ setup: func() structfs.Tree {
content := make([]byte, 6500)
io.ReadFull(rand.New(rand.NewSource(1)), content)
- i.Children = []*Inode{{
+ return structfs.Tree{{
Name: "test.bin",
- Content: bytes.NewReader(content),
+ Content: structfs.Bytes(content),
}}
- return nil
},
validate: func(t *testing.T) error {
var stat unix.Stat_t
@@ -176,12 +182,11 @@
},
{
name: "Unicode",
- setup: func(i *Inode) error {
- i.Children = []*Inode{{
+ setup: func() structfs.Tree {
+ return structfs.Tree{{
Name: "✨😂", // Really exercise that UTF-16 conversion
- Content: strings.NewReader("😂"),
+ Content: structfs.Bytes("😂"),
}}
- return nil
},
validate: func(t *testing.T) error {
file, err := os.Open("/dut/✨😂")
@@ -197,25 +202,26 @@
t.Fatalf("Failed to open unicode file: %v (available files: %v)", err, strings.Join(availableFileNames, ", "))
}
defer file.Close()
- contents, err := io.ReadAll(file)
- if err != nil {
- t.Errorf("Wrong content: expected %x, got %x", []byte("😂"), contents)
- }
+ expected := []byte("😂")
+ actual, err := io.ReadAll(file)
+ assert.NoError(t, err, "failed to read test file")
+ assert.Equal(t, expected, actual, "content not identical")
return nil
},
},
{
name: "MultipleMetaClusters",
- setup: func(root *Inode) error {
+ setup: func() structfs.Tree {
// Only test up to 2048 files as Linux gets VERY slow if going
// up to the maximum of approximately 32K
+ var root structfs.Tree
for i := 0; i < 2048; i++ {
- root.Children = append(root.Children, &Inode{
+ root = append(root, &structfs.Node{
Name: fmt.Sprintf("verylongtestfilename%d", i),
- Content: strings.NewReader("random test content"),
+ Content: structfs.Bytes("random test content"),
})
}
- return nil
+ return root
},
validate: func(t *testing.T) error {
files, err := os.ReadDir("/dut")
@@ -245,13 +251,8 @@
t.Fatalf("failed to get ramdisk block size: %v", err)
}
defer file.Close()
- rootInode := Inode{
- Attrs: AttrDirectory,
- }
- if err := test.setup(&rootInode); err != nil {
- t.Fatalf("setup failed: %v", err)
- }
- if err := WriteFS(file, rootInode, Options{
+ root := test.setup()
+ if err := WriteFS(file, root, Options{
ID: 1234,
Label: "KTEST",
BlockSize: uint16(blockSize),