Switch Metropolis to EROFS-based root filesystem
This gets rid of the old large initramfs and switches to an EROFS-based root
filesystem. It also drops the copy & remount compatibility code. As this filesystem is
properly read-only and not just ephemeral, this also brings various changes to the code
to make systems compatible with that.
Test Plan: Covered by E2E tests, also manually smoke-tested.
X-Origin-Diff: phab/D696
GitOrigin-RevId: 037f2b8253e7cff8435cc79771fad05f53670ff0
diff --git a/metropolis/node/core/localstorage/crypt/blockdev.go b/metropolis/node/core/localstorage/crypt/blockdev.go
index 86d5381..7f874b7 100644
--- a/metropolis/node/core/localstorage/crypt/blockdev.go
+++ b/metropolis/node/core/localstorage/crypt/blockdev.go
@@ -79,12 +79,14 @@
}
for partNumber, part := range table.Partitions {
if part.Type == EFIPartitionType {
- if err := unix.Mknod(ESPDevicePath, 0600|unix.S_IFBLK, int(unix.Mkdev(uint32(majorDev), uint32(partNumber+1)))); err != nil {
+ err := unix.Mknod(ESPDevicePath, 0600|unix.S_IFBLK, int(unix.Mkdev(uint32(majorDev), uint32(partNumber+1))))
+ if err != nil && !os.IsExist(err) {
return fmt.Errorf("failed to create device node for ESP partition: %w", err)
}
}
if part.Type == NodeDataPartitionType {
- if err := unix.Mknod(NodeDataCryptPath, 0600|unix.S_IFBLK, int(unix.Mkdev(uint32(majorDev), uint32(partNumber+1)))); err != nil {
+ err := unix.Mknod(NodeDataCryptPath, 0600|unix.S_IFBLK, int(unix.Mkdev(uint32(majorDev), uint32(partNumber+1))))
+ if err != nil && !os.IsExist(err) {
return fmt.Errorf("failed to create device node for Metropolis node encrypted data partition: %w", err)
}
}
diff --git a/metropolis/node/core/localstorage/directory_data.go b/metropolis/node/core/localstorage/directory_data.go
index 9be4fc1..01e2e07 100644
--- a/metropolis/node/core/localstorage/directory_data.go
+++ b/metropolis/node/core/localstorage/directory_data.go
@@ -18,7 +18,6 @@
import (
"fmt"
- "os"
"os/exec"
"golang.org/x/sys/unix"
@@ -139,10 +138,6 @@
}
func (d *DataDirectory) mount() error {
- if err := os.Mkdir(d.FullPath(), 0755); err != nil {
- return fmt.Errorf("making data directory: %w", err)
- }
-
if err := unix.Mount("/dev/data", d.FullPath(), "xfs", unix.MS_NOEXEC|unix.MS_NODEV, "pquota"); err != nil {
return fmt.Errorf("mounting data directory: %w", err)
}
diff --git a/metropolis/node/core/localstorage/directory_root.go b/metropolis/node/core/localstorage/directory_root.go
index ac1a453..c3a49cb 100644
--- a/metropolis/node/core/localstorage/directory_root.go
+++ b/metropolis/node/core/localstorage/directory_root.go
@@ -39,28 +39,26 @@
return fmt.Errorf("MakeBlockDevices: %w", err)
}
- if err := os.Mkdir(r.ESP.FullPath(), 0755); err != nil {
- return fmt.Errorf("making ESP directory: %w", err)
- }
-
if err := unix.Mount(crypt.ESPDevicePath, r.ESP.FullPath(), "vfat", unix.MS_NOEXEC|unix.MS_NODEV|unix.MS_SYNC, ""); err != nil {
return fmt.Errorf("mounting ESP partition: %w", err)
}
r.Data.canMount = true
- if err := os.Mkdir(r.Tmp.FullPath(), 0777); err != nil {
- return fmt.Errorf("making /tmp directory: %w", err)
- }
-
if err := unix.Mount("tmpfs", r.Tmp.FullPath(), "tmpfs", unix.MS_NOEXEC|unix.MS_NODEV, ""); err != nil {
return fmt.Errorf("mounting /tmp: %w", err)
}
+ if err := unix.Mount("tmpfs", r.Ephemeral.FullPath(), "tmpfs", unix.MS_NODEV, ""); err != nil {
+ return fmt.Errorf("mounting /ephemeral: %v", err)
+ }
+
+ if err := unix.Mount("tmpfs", r.Run.FullPath(), "tmpfs", unix.MS_NOEXEC|unix.MS_NODEV, ""); err != nil {
+ return fmt.Errorf("mounting /run: %w", err)
+ }
+
// TODO(q3k): do this automatically?
for _, d := range []declarative.DirectoryPlacement{
- r.Etc,
- r.Ephemeral,
r.Ephemeral.Consensus,
r.Ephemeral.Containerd, r.Ephemeral.Containerd.Tmp, r.Ephemeral.Containerd.RunSC, r.Ephemeral.Containerd.IPAM,
r.Ephemeral.FlexvolumePlugins,
diff --git a/metropolis/node/core/localstorage/storage.go b/metropolis/node/core/localstorage/storage.go
index 110513c..73d33e1 100644
--- a/metropolis/node/core/localstorage/storage.go
+++ b/metropolis/node/core/localstorage/storage.go
@@ -142,8 +142,8 @@
type EtcDirectory struct {
declarative.Directory
- Hosts declarative.File `file:"hosts"`
- MachineID declarative.File `file:"machine-id"`
+ Hosts declarative.File `file:"hosts"` // Symlinked to /ephemeral/hosts, baked into the erofs system image
+ MachineID declarative.File `file:"machine-id"` // Symlinked to /ephemeral/machine-id, baked into the erofs system image
}
type EphemeralDirectory struct {
@@ -151,6 +151,8 @@
Consensus EphemeralConsensusDirectory `dir:"consensus"`
Containerd EphemeralContainerdDirectory `dir:"containerd"`
FlexvolumePlugins declarative.Directory `dir:"flexvolume_plugins"`
+ Hosts declarative.File `file:"hosts"`
+ MachineID declarative.File `file:"machine-id"`
}
type EphemeralConsensusDirectory struct {