m: {cli,installer}: runfiles through datafile.MustGet
This implements datafile, a small library to more ergonomically resolve
Bazel runfiles:
1. It also works in cases where a tool doesn't run through `bazel run`.
2. It provides a MustGet wrapper which returns already read bytes and
removes some boilerplate at the callsite.
3. It allows us to extend the library in the future to prepare special
'self-contained' builds of some binaries, for example to bundle the
installer kernel in metroctl.
We then use this library to simplify the installer and installer tests.
In the installer, we technically remove the ability to specify arbitrary
kernels/bundles on the command line, but is this functionality actually
useful?
Change-Id: I46155b9951729c810e0d36930b470edfdfd82943
Reviewed-on: https://review.monogon.dev/c/monogon/+/484
Reviewed-by: Mateusz Zalega <mateusz@monogon.tech>
diff --git a/metropolis/cli/metroctl/BUILD.bazel b/metropolis/cli/metroctl/BUILD.bazel
index 6a311c8..f9b8b51 100644
--- a/metropolis/cli/metroctl/BUILD.bazel
+++ b/metropolis/cli/metroctl/BUILD.bazel
@@ -16,6 +16,7 @@
deps = [
"//metropolis/cli/metroctl/core:go_default_library",
"//metropolis/cli/pkg/context:go_default_library",
+ "//metropolis/cli/pkg/datafile:go_default_library",
"//metropolis/node:go_default_library",
"//metropolis/node/core/rpc:go_default_library",
"//metropolis/proto/api:go_default_library",
diff --git a/metropolis/cli/metroctl/install.go b/metropolis/cli/metroctl/install.go
index 10fc9f5..d2ca453 100644
--- a/metropolis/cli/metroctl/install.go
+++ b/metropolis/cli/metroctl/install.go
@@ -1,6 +1,7 @@
package main
import (
+ "bytes"
"crypto/ed25519"
"crypto/rand"
"encoding/pem"
@@ -12,6 +13,7 @@
"github.com/spf13/cobra"
"source.monogon.dev/metropolis/cli/metroctl/core"
+ "source.monogon.dev/metropolis/cli/pkg/datafile"
"source.monogon.dev/metropolis/proto/api"
)
@@ -20,57 +22,20 @@
Use: "install",
}
-// install flags
-var installer *string
-var bundle *string
-
var genusbCmd = &cobra.Command{
- Use: "genusb target --installer=inst.efi --bundle=bundle.bin",
+ Use: "genusb target",
Short: "Generates a Metropolis installer disk or image.",
- Example: "metroctl install genusb /dev/sdx --installer=installer_x86_64.efi --bundle=metropolis_dev_x86_64.tar.xz",
+ Example: "metroctl install genusb /dev/sdx",
Args: cobra.ExactArgs(1), // One positional argument: the target
Run: doGenUSB,
}
-// If useInTreeArtifacts is true metroctl should use a bundle and installer
-// directly from the build tree. It is automatically set to true if metroctl is
-// running under bazel run. Specifying either one manually still overrides
-// the in-tree artifacts.
-var useInTreeArtifacts = os.Getenv("BUILD_WORKSPACE_DIRECTORY") != ""
-
-var inTreeInstaller = "metropolis/node/installer/kernel.efi"
-var inTreeBundle = "metropolis/node/node.zip"
-
// A PEM block type for a Metropolis initial owner private key
const ownerKeyType = "METROPOLIS INITIAL OWNER PRIVATE KEY"
func doGenUSB(cmd *cobra.Command, args []string) {
- if useInTreeArtifacts && *installer == "" {
- installer = &inTreeInstaller
- }
- if useInTreeArtifacts && *bundle == "" {
- bundle = &inTreeBundle
- }
- installerFile, err := os.Open(*installer)
- if err != nil {
- log.Fatalf("Failed to open installer: %v", err)
- }
- installerFileStat, err := installerFile.Stat()
- if err != nil {
- log.Fatalf("Failed to stat installer: %v", err)
- }
- var bundleFile *os.File
- var bundleFileStat os.FileInfo
- if bundle != nil && *bundle != "" {
- bundleFile, err = os.Open(*bundle)
- if err != nil {
- log.Fatalf("Failed to open bundle: %v", err)
- }
- bundleFileStat, err = bundleFile.Stat()
- if err != nil {
- log.Fatalf("Failed to stat bundle: %v", err)
- }
- }
+ installer := datafile.MustGet("metropolis/node/installer/kernel.efi")
+ bundle := datafile.MustGet("metropolis/node/node.zip")
// TODO(lorenz): Have a key management story for this
if err := os.MkdirAll(filepath.Join(xdg.ConfigHome, "metroctl"), 0700); err != nil {
@@ -118,16 +83,14 @@
installerImageArgs := core.MakeInstallerImageArgs{
TargetPath: args[0],
- Installer: installerFile,
- InstallerSize: uint64(installerFileStat.Size()),
+ Installer: bytes.NewBuffer(installer),
+ InstallerSize: uint64(len(installer)),
NodeParams: params,
+ Bundle: bytes.NewBuffer(bundle),
+ BundleSize: uint64(len(bundle)),
}
- if bundleFile != nil {
- installerImageArgs.Bundle = bundleFile
- installerImageArgs.BundleSize = uint64(bundleFileStat.Size())
- }
-
+ log.Printf("Generating installer image (this can take a while, see issues/92).")
if err := core.MakeInstallerImage(installerImageArgs); err != nil {
log.Fatalf("Failed to create installer: %v", err)
}
@@ -136,7 +99,4 @@
func init() {
rootCmd.AddCommand(installCmd)
installCmd.AddCommand(genusbCmd)
-
- bundle = installCmd.PersistentFlags().StringP("bundle", "b", "", "Metropolis bundle file to use")
- installer = installCmd.PersistentFlags().StringP("installer", "i", "", "Metropolis installer file to use")
}
diff --git a/metropolis/cli/pkg/datafile/BUILD.bazel b/metropolis/cli/pkg/datafile/BUILD.bazel
new file mode 100644
index 0000000..8628d9a
--- /dev/null
+++ b/metropolis/cli/pkg/datafile/BUILD.bazel
@@ -0,0 +1,9 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+ name = "go_default_library",
+ srcs = ["datafile.go"],
+ importpath = "source.monogon.dev/metropolis/cli/pkg/datafile",
+ visibility = ["//visibility:public"],
+ deps = ["@io_bazel_rules_go//go/tools/bazel:go_default_library"],
+)
diff --git a/metropolis/cli/pkg/datafile/datafile.go b/metropolis/cli/pkg/datafile/datafile.go
new file mode 100644
index 0000000..a65780d
--- /dev/null
+++ b/metropolis/cli/pkg/datafile/datafile.go
@@ -0,0 +1,99 @@
+// datafile provides an abstraction for accessing files passed through the data
+// attribute in a Bazel build rule.
+//
+// It thinly wraps around the Bazel/Go runfile library (to allow running from
+// outside `bazel run`).
+package datafile
+
+import (
+ "bufio"
+ "fmt"
+ "log"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/bazelbuild/rules_go/go/tools/bazel"
+)
+
+// parseManifest takes a bazel runfile MANIFEST and parses it into a map from
+// workspace-relative path to absolute path, flattening all workspaces into a
+// single tree.
+func parseManifest(path string) (map[string]string, error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return nil, fmt.Errorf("could not open MANIFEST: %v", err)
+ }
+ defer f.Close()
+
+ manifest := make(map[string]string)
+ scanner := bufio.NewScanner(f)
+ for scanner.Scan() {
+ parts := strings.Split(scanner.Text(), " ")
+ if len(parts) != 2 {
+ continue
+ }
+ fpathParts := strings.Split(parts[0], string(os.PathSeparator))
+ fpath := strings.Join(fpathParts[1:], string(os.PathSeparator))
+ manifest[fpath] = parts[1]
+ }
+ return manifest, nil
+}
+
+// resolveRunfile tries to resolve a workspace-relative file path into an
+// absolute path with the use of bazel runfiles, through either the original
+// Bazel/Go runfile integration or a wrapper that also supports running from
+// outside `bazel run`.
+func resolveRunfile(path string) (string, error) {
+ var errEx error
+ ep, err := os.Executable()
+ if err == nil {
+ rfdir := ep + ".runfiles"
+ mfpath := filepath.Join(rfdir, "MANIFEST")
+ if stat, err := os.Stat(rfdir); err == nil && stat.IsDir() {
+ // We have a runfiles directory, parse MANIFEST and resolve files this way.
+ manifest, err := parseManifest(mfpath)
+ if err == nil {
+ tpath := manifest[path]
+ if tpath == "" {
+ errEx = fmt.Errorf("not in MANIFEST")
+ } else {
+ return tpath, err
+ }
+ } else {
+ errEx = err
+ }
+ } else {
+ errEx = err
+ }
+ }
+
+ // Try runfiles just in case.
+ rf, errRF := bazel.Runfile(path)
+ if errRF == nil {
+ return rf, nil
+ }
+ return "", fmt.Errorf("could not resolve via executable location (%v) and runfile resolution failed: %v", errEx, errRF)
+}
+
+// Get tries to read a workspace-relative file path through the use of Bazel
+// runfiles, including for cases when executables are running outside `bazel
+// run`.
+func Get(path string) ([]byte, error) {
+ rfpath, err := resolveRunfile(path)
+ if err != nil {
+ return nil, err
+ }
+ return os.ReadFile(rfpath)
+}
+
+// MustGet either successfully resolves a file through Get() or logs an error
+// (through the stdlib log library) and stops execution. This should thus only
+// be used in binaries which use the log library.
+func MustGet(path string) []byte {
+ res, err := Get(path)
+ if err != nil {
+ log.Fatalf("Could not get datafile %s: %v", path, err)
+ }
+ return res
+}
diff --git a/metropolis/node/installer/main.go b/metropolis/node/installer/main.go
index 075dbb5..79a78fe 100644
--- a/metropolis/node/installer/main.go
+++ b/metropolis/node/installer/main.go
@@ -30,6 +30,7 @@
"syscall"
"golang.org/x/sys/unix"
+
"source.monogon.dev/metropolis/node/build/mkimage/osimage"
"source.monogon.dev/metropolis/pkg/efivarfs"
"source.monogon.dev/metropolis/pkg/sysfs"
diff --git a/metropolis/test/installer/BUILD.bazel b/metropolis/test/installer/BUILD.bazel
index df62c1c..eeb14df 100644
--- a/metropolis/test/installer/BUILD.bazel
+++ b/metropolis/test/installer/BUILD.bazel
@@ -21,13 +21,13 @@
visibility = ["//visibility:private"],
deps = [
"//metropolis/cli/metroctl/core:go_default_library",
+ "//metropolis/cli/pkg/datafile:go_default_library",
"//metropolis/node/build/mkimage/osimage:go_default_library",
"//metropolis/pkg/logbuffer:go_default_library",
"//metropolis/proto/api:go_default_library",
"@com_github_diskfs_go_diskfs//:go_default_library",
"@com_github_diskfs_go_diskfs//disk:go_default_library",
"@com_github_diskfs_go_diskfs//partition/gpt:go_default_library",
- "@io_bazel_rules_go//go/tools/bazel:go_default_library",
],
)
diff --git a/metropolis/test/installer/main.go b/metropolis/test/installer/main.go
index b36c70d..0a5e5aa 100644
--- a/metropolis/test/installer/main.go
+++ b/metropolis/test/installer/main.go
@@ -31,12 +31,12 @@
"syscall"
"testing"
- "github.com/bazelbuild/rules_go/go/tools/bazel"
diskfs "github.com/diskfs/go-diskfs"
"github.com/diskfs/go-diskfs/disk"
"github.com/diskfs/go-diskfs/partition/gpt"
mctl "source.monogon.dev/metropolis/cli/metroctl/core"
+ "source.monogon.dev/metropolis/cli/pkg/datafile"
"source.monogon.dev/metropolis/node/build/mkimage/osimage"
"source.monogon.dev/metropolis/pkg/logbuffer"
"source.monogon.dev/metropolis/proto/api"
@@ -45,12 +45,6 @@
// Each variable in this block points to either a test dependency or a side
// effect. These variables are initialized in TestMain using Bazel.
var (
- // installerEFIPayload is a filesystem path pointing at the unified kernel
- // image dependency.
- installerEFIPayload string
- // testOSBundle is a filesystem path pointing at the Metropolis installation
- // bundle.
- testOSBundle string
// installerImage is a filesystem path pointing at the installer image that
// is generated during the test, and is removed afterwards.
installerImage string
@@ -165,64 +159,18 @@
}
func TestMain(m *testing.M) {
- // Initialize global variables holding filesystem paths pointing to runtime
- // dependencies and side effects.
- paths := []struct {
- // res is a pointer to the global variable initialized.
- res *string
- // dep states whether the path should be resolved as a dependency, rather
- // than a side effect.
- dep bool
- // src is a source path, based on which res is resolved. In case of
- // dependencies it must be a path relative to the repository root. For
- // side effects, it must be just a filename.
- src string
- }{
- {&installerEFIPayload, true, "metropolis/test/installer/kernel.efi"},
- {&testOSBundle, true, "metropolis/test/installer/testos/testos_bundle.zip"},
- {&installerImage, false, "installer.img"},
- {&nodeStorage, false, "stor.img"},
- }
- for _, p := range paths {
- if p.dep {
- res, err := bazel.Runfile(p.src)
- if err != nil {
- log.Fatal(err)
- }
- *p.res = res
- } else {
- od := os.Getenv("TEST_TMPDIR")
- // If od is empty, just use the working directory, which is set according
- // to the rundir attribute of go_test.
- *p.res = filepath.Join(od, p.src)
- }
- }
+ installerImage = filepath.Join(os.Getenv("TEST_TMPDIR"), "installer.img")
+ nodeStorage = filepath.Join(os.Getenv("TEST_TMPDIR"), "stor.img")
- // Build the installer image with metroctl, given the EFI executable
- // generated by Metropolis buildsystem.
- installer, err := os.Open(installerEFIPayload)
- if err != nil {
- log.Fatalf("Couldn't open the installer EFI executable at %q: %v", installerEFIPayload, err)
- }
- info, err := installer.Stat()
- if err != nil {
- log.Fatalf("Couldn't stat the installer EFI executable: %v", err)
- }
- bundle, err := os.Open(testOSBundle)
- if err != nil {
- log.Fatalf("Failed to open TestOS bundle: %v", err)
- }
- bundleStat, err := bundle.Stat()
- if err != nil {
- log.Fatalf("Failed to stat() TestOS bundle: %v", err)
- }
+ installer := datafile.MustGet("metropolis/test/installer/kernel.efi")
+ bundle := datafile.MustGet("metropolis/test/installer/testos/testos_bundle.zip")
iargs := mctl.MakeInstallerImageArgs{
- Installer: installer,
- InstallerSize: uint64(info.Size()),
+ Installer: bytes.NewBuffer(installer),
+ InstallerSize: uint64(len(installer)),
TargetPath: installerImage,
NodeParams: &api.NodeParameters{},
- Bundle: bundle,
- BundleSize: uint64(bundleStat.Size()),
+ Bundle: bytes.NewBuffer(bundle),
+ BundleSize: uint64(len(bundle)),
}
if err := mctl.MakeInstallerImage(iargs); err != nil {
log.Fatalf("Couldn't create the installer image at %q: %v", installerImage, err)