metropolis/test: use localregistry
This removes everything but the preseed test image from the preseed
image pool, instead opting to serve all test image via localregistry.
The registry API is served from a dedicated IP inside the virtual
network and forwarded to an ephemeral listener on the host. The relevant
infrastructure is added to the launch package.
As it is required to add configuration to containerd for this registry
anyways as it does not and should not have TLS we take that opportunity
to give it a descriptive name (test.monogon.internal).
Visibilities of images are also adjusted as they are now referenced much
closer to their point of use.
Against main this saves 51MiB in bundle size (289MiB -> 238MiB).
Change-Id: I31f732eb8c4ccec486204f35e3635b588fd9c85b
Reviewed-on: https://review.monogon.dev/c/monogon/+/1927
Tested-by: Jenkins CI
Reviewed-by: Leopold Schabel <leo@monogon.tech>
diff --git a/metropolis/test/launch/cluster/cluster.go b/metropolis/test/launch/cluster/cluster.go
index 5b39f67..ea69d96 100644
--- a/metropolis/test/launch/cluster/cluster.go
+++ b/metropolis/test/launch/cluster/cluster.go
@@ -16,6 +16,7 @@
"fmt"
"io"
"net"
+ "net/http"
"os"
"os/exec"
"path"
@@ -40,6 +41,7 @@
"source.monogon.dev/metropolis/node/core/identity"
"source.monogon.dev/metropolis/node/core/rpc"
"source.monogon.dev/metropolis/node/core/rpc/resolver"
+ "source.monogon.dev/metropolis/pkg/localregistry"
apb "source.monogon.dev/metropolis/proto/api"
cpb "source.monogon.dev/metropolis/proto/common"
"source.monogon.dev/metropolis/test/launch"
@@ -148,7 +150,7 @@
// Create the TPM state directory and initialize all files required by swtpm.
tpmt := filepath.Join(stdp, "tpm")
- if err := os.Mkdir(tpmt, 0755); err != nil {
+ if err := os.Mkdir(tpmt, 0o755); err != nil {
return nil, fmt.Errorf("while creating the TPM directory: %w", err)
}
tpms, err := datafile.ResolveRunfile("metropolis/node/tpm")
@@ -275,7 +277,8 @@
tpmSocketPath := filepath.Join(r.sd, "tpm-socket")
fwVarPath := filepath.Join(r.ld, "OVMF_VARS.fd")
storagePath := filepath.Join(r.ld, "node.img")
- qemuArgs := []string{"-machine", "q35", "-accel", "kvm", "-nographic", "-nodefaults", "-m", "4096",
+ qemuArgs := []string{
+ "-machine", "q35", "-accel", "kvm", "-nographic", "-nodefaults", "-m", "4096",
"-cpu", "host", "-smp", "sockets=1,cpus=1,cores=2,threads=2,maxcpus=4",
"-drive", "if=pflash,format=raw,readonly,file=external/edk2/OVMF_CODE.fd",
"-drive", "if=pflash,format=raw,file=" + fwVarPath,
@@ -286,7 +289,8 @@
"-tpmdev", "emulator,id=tpm0,chardev=chrtpm",
"-device", "tpm-tis,tpmdev=tpm0",
"-device", "virtio-rng-pci",
- "-serial", "stdio"}
+ "-serial", "stdio",
+ }
if !options.AllowReboot {
qemuArgs = append(qemuArgs, "-no-reboot")
@@ -298,7 +302,7 @@
if err != nil {
return fmt.Errorf("failed to encode node paraeters: %w", err)
}
- if err := os.WriteFile(parametersPath, parametersRaw, 0644); err != nil {
+ if err := os.WriteFile(parametersPath, parametersRaw, 0o644); err != nil {
return fmt.Errorf("failed to write node parameters: %w", err)
}
qemuArgs = append(qemuArgs, "-fw_cfg", "name=dev.monogon.metropolis/parameters.pb,file="+parametersPath)
@@ -495,6 +499,11 @@
// bootstrapping them. The nodes' address information in Cluster.Nodes will be
// incomplete.
LeaveNodesNew bool
+
+ // Optional local registry which will be made available to the cluster to
+ // pull images from. This is a more efficient alternative to preseeding all
+ // images used for testing.
+ LocalRegistry *localregistry.Server
}
// Cluster is the running Metropolis cluster launched using the LaunchCluster
@@ -638,7 +647,7 @@
}
func NewSerialFileLogger(p string) (io.ReadWriter, error) {
- f, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE, 0600)
+ f, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE, 0o600)
if err != nil {
return nil, err
}
@@ -688,7 +697,7 @@
// Make a list of channels that will be populated by all running node qemu
// processes.
done := make([]chan error, opts.NumNodes)
- for i, _ := range done {
+ for i := range done {
done[i] = make(chan error, 1)
}
@@ -732,6 +741,31 @@
done[0] <- err
}()
+ localRegistryAddr := net.TCPAddr{
+ IP: net.IPv4(10, 42, 0, 82),
+ Port: 5000,
+ }
+
+ var guestSvcMap launch.GuestServiceMap
+ if opts.LocalRegistry != nil {
+ l, err := net.ListenTCP("tcp", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1)})
+ if err != nil {
+ ctxC()
+ return nil, fmt.Errorf("failed to create TCP listener for local registry: %w", err)
+ }
+ s := http.Server{
+ Handler: opts.LocalRegistry,
+ }
+ go s.Serve(l)
+ go func() {
+ <-ctxT.Done()
+ s.Close()
+ }()
+ guestSvcMap = launch.GuestServiceMap{
+ &localRegistryAddr: *l.Addr().(*net.TCPAddr),
+ }
+ }
+
// Launch nanoswitch.
portMap, err := launch.ConflictFreePortMap(ClusterPorts)
if err != nil {
@@ -757,6 +791,7 @@
InitramfsPath: "metropolis/test/nanoswitch/initramfs.cpio.lz4",
ExtraNetworkInterfaces: switchPorts,
PortMap: portMap,
+ GuestServiceMap: guestSvcMap,
SerialPort: serialPort,
PcapDump: path.Join(ld, "nanoswitch.pcap"),
}); err != nil {
@@ -1108,7 +1143,7 @@
}
host := net.JoinHostPort(c.NodeIDs[0], node.KubernetesAPIWrappedPort.PortString())
- var clientConfig = rest.Config{
+ clientConfig := rest.Config{
Host: host,
TLSClientConfig: rest.TLSClientConfig{
// TODO(q3k): use CA certificate