m/c/metroctl/core: add frontend-independent metroctl support pkg
This adds metroctl/core, a package which contains parts of metroctl
which do significant amounts of work beyond just providing a CLI for
them.
This package is intended to be used for integrating with functions
provided by metroctl, for example for using them in integration tests
or writing other frontends providing functionality similar to metroctl
(like a GUI or webapp).
Change-Id: I8a56bfbefce8d18c6c9be3349e3c7a15a699d009
Reviewed-on: https://review.monogon.dev/c/monogon/+/411
Reviewed-by: Mateusz Zalega <mateusz@monogon.tech>
Vouch-Run-CI: Mateusz Zalega <mateusz@monogon.tech>
diff --git a/build/fietsje/deps_monogon.go b/build/fietsje/deps_monogon.go
index fc81bf5..f192b6c 100644
--- a/build/fietsje/deps_monogon.go
+++ b/build/fietsje/deps_monogon.go
@@ -62,8 +62,11 @@
p.collectOverride("github.com/mattn/go-shellwords", "v1.0.11")
// Used by //metropolis/build/mkimage
- p.collect("github.com/diskfs/go-diskfs", "v1.0.0").use(
+ p.collect("github.com/diskfs/go-diskfs", "v1.2.0").use(
"gopkg.in/djherbis/times.v1",
+ "github.com/pkg/xattr",
+ "github.com/pierrec/lz4",
+ "github.com/ulikunitz/xz",
)
// Used by //metropolis/build/genosrelease
diff --git a/metropolis/cli/metroctl/core/BUILD.bazel b/metropolis/cli/metroctl/core/BUILD.bazel
new file mode 100644
index 0000000..3c59706
--- /dev/null
+++ b/metropolis/cli/metroctl/core/BUILD.bazel
@@ -0,0 +1,19 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+ name = "go_default_library",
+ srcs = [
+ "core.go",
+ "install.go",
+ ],
+ importpath = "source.monogon.dev/metropolis/cli/metroctl/core",
+ visibility = ["//visibility:public"],
+ deps = [
+ "//metropolis/proto/api:go_default_library",
+ "@com_github_diskfs_go_diskfs//:go_default_library",
+ "@com_github_diskfs_go_diskfs//disk:go_default_library",
+ "@com_github_diskfs_go_diskfs//filesystem:go_default_library",
+ "@com_github_diskfs_go_diskfs//partition/gpt:go_default_library",
+ "@org_golang_google_protobuf//proto:go_default_library",
+ ],
+)
diff --git a/metropolis/cli/metroctl/core/core.go b/metropolis/cli/metroctl/core/core.go
new file mode 100644
index 0000000..545a408
--- /dev/null
+++ b/metropolis/cli/metroctl/core/core.go
@@ -0,0 +1,6 @@
+// Package core contains parts of metroctl which do significant amounts of work
+// beyond just providing a CLI for them.
+// This package is intended to be used for integrating with functions provided
+// by metroctl, for example for using them in integration tests or writing other
+// frontends providing functionality similar to metroctl (like a GUI or webapp).
+package core
diff --git a/metropolis/cli/metroctl/core/install.go b/metropolis/cli/metroctl/core/install.go
new file mode 100644
index 0000000..35b49ea
--- /dev/null
+++ b/metropolis/cli/metroctl/core/install.go
@@ -0,0 +1,166 @@
+package core
+
+import (
+ "errors"
+ "fmt"
+ "io"
+ "os"
+
+ "github.com/diskfs/go-diskfs"
+ "github.com/diskfs/go-diskfs/disk"
+ "github.com/diskfs/go-diskfs/filesystem"
+ "github.com/diskfs/go-diskfs/partition/gpt"
+ "google.golang.org/protobuf/proto"
+ "source.monogon.dev/metropolis/proto/api"
+)
+
+func mibToSectors(size uint64, logicalBlockSize int64) uint64 {
+ return (size * 1024 * 1024) / uint64(logicalBlockSize)
+}
+
+// Mask for aligning values to 1MiB boundaries. Go complains if you shift
+// 1 bits out of the value in a constant so the construction is a bit
+// convoluted.
+const align1MiBMask = (1<<44 - 1) << 20
+
+const MiB = 1024 * 1024
+
+type MakeInstallerImageArgs struct {
+ // Path to either a file or a disk which will contain the installer data.
+ TargetPath string
+
+ // Reader for the installer EFI executable. Mandatory.
+ Installer io.Reader
+ InstallerSize uint64
+
+ // Optional NodeParameters to be embedded for use by the installer.
+ NodeParams *api.NodeParameters
+
+ // Optional Reader for a Metropolis bundle for use by the installer.
+ Bundle io.Reader
+ BundleSize uint64
+}
+
+// MakeInstallerImage generates an installer disk image containing a GPT
+// partition table and a single FAT32 partition with an installer and optionally
+// with a bundle and/or Node Parameters.
+func MakeInstallerImage(args MakeInstallerImageArgs) error {
+ if args.Installer == nil {
+ return errors.New("Installer is mandatory")
+ }
+ if args.InstallerSize == 0 {
+ return errors.New("InstallerSize needs to be valid (>0)")
+ }
+ if args.Bundle != nil && args.BundleSize == 0 {
+ return errors.New("if a Bundle is passed BundleSize needs to be valid (>0)")
+ }
+
+ var err error
+ var nodeParamsRaw []byte
+ if args.NodeParams != nil {
+ nodeParamsRaw, err = proto.Marshal(args.NodeParams)
+ if err != nil {
+ return fmt.Errorf("failed to marshal node params: %w", err)
+ }
+ }
+
+ var img *disk.Disk
+ // The following section is a bit ugly, it would technically be nicer to
+ // just pack all clusters of the FAT32 together, figure out how many were
+ // needed at the end and truncate the partition there. But that would
+ // require writing a new FAT32 writer, the effort to do that is in no way
+ // proportional to its advantages. So let's just do some conservative
+ // calculations on how much space we need and call it a day.
+
+ // ~4MiB FAT32 headers, 1MiB alignment overhead (bitmask drops up to 1MiB),
+ // 5% filesystem overhead
+ partitionSizeBytes := (uint64(float32(5*MiB+args.BundleSize+args.InstallerSize+uint64(len(nodeParamsRaw))) * 1.05)) & align1MiBMask
+ // FAT32 has a minimum partition size of 32MiB, so clamp the lower partition
+ // size to that.
+ if partitionSizeBytes < 32*MiB {
+ partitionSizeBytes = 32 * MiB
+ }
+ // If creating an image, create it with minimal size, i.e. 1MiB at each
+ // end for partitioning metadata and alignment.
+ // 1MiB alignment is used as that will essentially guarantee that any
+ // partition is aligned to whatever internal block size is used by the
+ // storage device. Especially flash-based storage likes to use much bigger
+ // blocks than advertised as sectors which can degrade performance when
+ // partitions are misaligned.
+ calculatedImageBytes := 2*MiB + partitionSizeBytes
+
+ if _, err = os.Stat(args.TargetPath); os.IsNotExist(err) {
+ img, err = diskfs.Create(args.TargetPath, int64(calculatedImageBytes), diskfs.Raw)
+ } else {
+ img, err = diskfs.Open(args.TargetPath)
+ }
+ if err != nil {
+ return fmt.Errorf("failed to create/open target: %w", err)
+ }
+ defer img.File.Close()
+ // This has an edge case where the data would technically fit but our 5%
+ // overhead are too conservative. But it is very rare and I don't really
+ // trust diskfs to generate good errors when it overflows so we'll just
+ // reject early.
+ if uint64(img.Size) < calculatedImageBytes {
+ return errors.New("target too small, data won't fit")
+ }
+
+ gptTable := &gpt.Table{
+ LogicalSectorSize: int(img.LogicalBlocksize),
+ PhysicalSectorSize: int(img.PhysicalBlocksize),
+ ProtectiveMBR: true,
+ Partitions: []*gpt.Partition{
+ {
+ Type: gpt.EFISystemPartition,
+ Name: "MetropolisInstaller",
+ Start: mibToSectors(1, img.LogicalBlocksize),
+ Size: partitionSizeBytes,
+ },
+ },
+ }
+ if err := img.Partition(gptTable); err != nil {
+ return fmt.Errorf("failed to partition target: %w", err)
+ }
+ fs, err := img.CreateFilesystem(disk.FilesystemSpec{Partition: 1, FSType: filesystem.TypeFat32, VolumeLabel: "METRO_INST"})
+ if err != nil {
+ return fmt.Errorf("failed to create target filesystem: %w", err)
+ }
+
+ // Create EFI partition structure.
+ for _, dir := range []string{"/EFI", "/EFI/BOOT", "/EFI/metropolis-installer"} {
+ if err := fs.Mkdir(dir); err != nil {
+ panic(err)
+ }
+ }
+ // This needs to be a "Removable Media" according to the UEFI Specification
+ // V2.9 Section 3.5.1.1. This file is booted by any compliant UEFI firmware
+ // in absence of another bootable boot entry.
+ installerFile, err := fs.OpenFile("/EFI/BOOT/BOOTx64.EFI", os.O_CREATE|os.O_RDWR)
+ if err != nil {
+ panic(err)
+ }
+ if _, err := io.Copy(installerFile, args.Installer); err != nil {
+ return fmt.Errorf("failed to copy installer file: %w", err)
+ }
+ if args.NodeParams != nil {
+ nodeParamsFile, err := fs.OpenFile("/EFI/metropolis-installer/nodeparams.pb", os.O_CREATE|os.O_RDWR)
+ if err != nil {
+ panic(err)
+ }
+ _, err = nodeParamsFile.Write(nodeParamsRaw)
+ if err != nil {
+ return fmt.Errorf("failed to write node params: %w", err)
+ }
+ }
+ if args.Bundle != nil {
+ bundleFile, err := fs.OpenFile("/EFI/metropolis-installer/bundle.bin", os.O_CREATE|os.O_RDWR)
+ if err != nil {
+ panic(err)
+ }
+ if _, err := io.Copy(bundleFile, args.Bundle); err != nil {
+ return fmt.Errorf("failed to copy bundle: %w", err)
+ }
+ }
+ return nil
+}
diff --git a/third_party/go/repositories.bzl b/third_party/go/repositories.bzl
index c134586..9e7f712 100644
--- a/third_party/go/repositories.bzl
+++ b/third_party/go/repositories.bzl
@@ -551,8 +551,8 @@
go_repository(
name = "com_github_diskfs_go_diskfs",
importpath = "github.com/diskfs/go-diskfs",
- version = "v1.0.0",
- sum = "h1:sLQnXItICiYgiHcYNNujKT9kOKnk7diOvZGEKvxrwpc=",
+ version = "v1.2.0",
+ sum = "h1:Ow4xorEDw1VNYKbC+SA/qQNwi5gWIwdKUxmUcLFST24=",
build_extra_args = [
"-go_naming_convention=go_default_library",
"-go_naming_convention_external=go_default_library",
@@ -1742,6 +1742,16 @@
],
)
go_repository(
+ name = "com_github_pierrec_lz4",
+ importpath = "github.com/pierrec/lz4",
+ version = "v2.3.0+incompatible",
+ sum = "h1:CZzRn4Ut9GbUkHlQ7jqBXeZQV41ZSKWFc302ZU6lUTk=",
+ build_extra_args = [
+ "-go_naming_convention=go_default_library",
+ "-go_naming_convention_external=go_default_library",
+ ],
+ )
+ go_repository(
name = "com_github_pkg_errors",
importpath = "github.com/pkg/errors",
version = "v0.9.1",
@@ -1752,6 +1762,16 @@
],
)
go_repository(
+ name = "com_github_pkg_xattr",
+ importpath = "github.com/pkg/xattr",
+ version = "v0.4.1",
+ sum = "h1:dhclzL6EqOXNaPDWqoeb9tIxATfBSmjqL0b4DpSjwRw=",
+ build_extra_args = [
+ "-go_naming_convention=go_default_library",
+ "-go_naming_convention_external=go_default_library",
+ ],
+ )
+ go_repository(
name = "com_github_pquerna_cachecontrol",
importpath = "github.com/pquerna/cachecontrol",
version = "v0.0.0-20171018203845-0dec1b30a021",
@@ -2026,6 +2046,16 @@
],
)
go_repository(
+ name = "com_github_ulikunitz_xz",
+ importpath = "github.com/ulikunitz/xz",
+ version = "v0.5.6",
+ sum = "h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8=",
+ build_extra_args = [
+ "-go_naming_convention=go_default_library",
+ "-go_naming_convention_external=go_default_library",
+ ],
+ )
+ go_repository(
name = "com_github_urfave_cli",
importpath = "github.com/urfave/cli",
version = "v1.22.1",
diff --git a/third_party/go/shelf.pb.text b/third_party/go/shelf.pb.text
index b3f3007..f4a2ed1 100644
--- a/third_party/go/shelf.pb.text
+++ b/third_party/go/shelf.pb.text
@@ -699,6 +699,13 @@
semver: "v1.0.0"
>
entry: <
+ import_path: "github.com/diskfs/go-diskfs"
+ version: "v1.2.0"
+ bazel_name: "com_github_diskfs_go_diskfs"
+ sum: "h1:Ow4xorEDw1VNYKbC+SA/qQNwi5gWIwdKUxmUcLFST24="
+ semver: "v1.2.0"
+>
+entry: <
import_path: "github.com/dnstap/golang-dnstap"
version: "v0.2.0"
bazel_name: "com_github_dnstap_golang_dnstap"
@@ -2064,6 +2071,13 @@
semver: "v0.0.0-20180202154549-b0b1615b78e5"
>
entry: <
+ import_path: "github.com/pierrec/lz4"
+ version: "v2.3.0+incompatible"
+ bazel_name: "com_github_pierrec_lz4"
+ sum: "h1:CZzRn4Ut9GbUkHlQ7jqBXeZQV41ZSKWFc302ZU6lUTk="
+ semver: "v2.3.0+incompatible"
+>
+entry: <
import_path: "github.com/pkg/errors"
version: "ba968bfe8b2f7e042a574c888954fccecfa385b4"
bazel_name: "com_github_pkg_errors"
@@ -2078,6 +2092,13 @@
semver: "v0.9.1"
>
entry: <
+ import_path: "github.com/pkg/xattr"
+ version: "v0.4.1"
+ bazel_name: "com_github_pkg_xattr"
+ sum: "h1:dhclzL6EqOXNaPDWqoeb9tIxATfBSmjqL0b4DpSjwRw="
+ semver: "v0.4.1"
+>
+entry: <
import_path: "github.com/pquerna/cachecontrol"
version: "v0.0.0-20171018203845-0dec1b30a021"
bazel_name: "com_github_pquerna_cachecontrol"
@@ -2400,6 +2421,13 @@
semver: "v7.0.0+incompatible"
>
entry: <
+ import_path: "github.com/ulikunitz/xz"
+ version: "v0.5.6"
+ bazel_name: "com_github_ulikunitz_xz"
+ sum: "h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8="
+ semver: "v0.5.6"
+>
+entry: <
import_path: "github.com/urfave/cli"
version: "bfe2e925cfb6d44b40ad3a779165ea7e8aff9212"
bazel_name: "com_github_urfave_cli"