metropolis/test: use localregistry
This removes everything but the preseed test image from the preseed
image pool, instead opting to serve all test image via localregistry.
The registry API is served from a dedicated IP inside the virtual
network and forwarded to an ephemeral listener on the host. The relevant
infrastructure is added to the launch package.
As it is required to add configuration to containerd for this registry
anyways as it does not and should not have TLS we take that opportunity
to give it a descriptive name (test.monogon.internal).
Visibilities of images are also adjusted as they are now referenced much
closer to their point of use.
Against main this saves 51MiB in bundle size (289MiB -> 238MiB).
Change-Id: I31f732eb8c4ccec486204f35e3635b588fd9c85b
Reviewed-on: https://review.monogon.dev/c/monogon/+/1927
Tested-by: Jenkins CI
Reviewed-by: Leopold Schabel <leo@monogon.tech>
diff --git a/metropolis/test/launch/cluster/BUILD.bazel b/metropolis/test/launch/cluster/BUILD.bazel
index a2dcc52..08d4cfe 100644
--- a/metropolis/test/launch/cluster/BUILD.bazel
+++ b/metropolis/test/launch/cluster/BUILD.bazel
@@ -25,6 +25,7 @@
"//metropolis/node/core/identity",
"//metropolis/node/core/rpc",
"//metropolis/node/core/rpc/resolver",
+ "//metropolis/pkg/localregistry",
"//metropolis/pkg/logbuffer",
"//metropolis/proto/api",
"//metropolis/proto/common",
diff --git a/metropolis/test/launch/cluster/cluster.go b/metropolis/test/launch/cluster/cluster.go
index 5b39f67..ea69d96 100644
--- a/metropolis/test/launch/cluster/cluster.go
+++ b/metropolis/test/launch/cluster/cluster.go
@@ -16,6 +16,7 @@
"fmt"
"io"
"net"
+ "net/http"
"os"
"os/exec"
"path"
@@ -40,6 +41,7 @@
"source.monogon.dev/metropolis/node/core/identity"
"source.monogon.dev/metropolis/node/core/rpc"
"source.monogon.dev/metropolis/node/core/rpc/resolver"
+ "source.monogon.dev/metropolis/pkg/localregistry"
apb "source.monogon.dev/metropolis/proto/api"
cpb "source.monogon.dev/metropolis/proto/common"
"source.monogon.dev/metropolis/test/launch"
@@ -148,7 +150,7 @@
// Create the TPM state directory and initialize all files required by swtpm.
tpmt := filepath.Join(stdp, "tpm")
- if err := os.Mkdir(tpmt, 0755); err != nil {
+ if err := os.Mkdir(tpmt, 0o755); err != nil {
return nil, fmt.Errorf("while creating the TPM directory: %w", err)
}
tpms, err := datafile.ResolveRunfile("metropolis/node/tpm")
@@ -275,7 +277,8 @@
tpmSocketPath := filepath.Join(r.sd, "tpm-socket")
fwVarPath := filepath.Join(r.ld, "OVMF_VARS.fd")
storagePath := filepath.Join(r.ld, "node.img")
- qemuArgs := []string{"-machine", "q35", "-accel", "kvm", "-nographic", "-nodefaults", "-m", "4096",
+ qemuArgs := []string{
+ "-machine", "q35", "-accel", "kvm", "-nographic", "-nodefaults", "-m", "4096",
"-cpu", "host", "-smp", "sockets=1,cpus=1,cores=2,threads=2,maxcpus=4",
"-drive", "if=pflash,format=raw,readonly,file=external/edk2/OVMF_CODE.fd",
"-drive", "if=pflash,format=raw,file=" + fwVarPath,
@@ -286,7 +289,8 @@
"-tpmdev", "emulator,id=tpm0,chardev=chrtpm",
"-device", "tpm-tis,tpmdev=tpm0",
"-device", "virtio-rng-pci",
- "-serial", "stdio"}
+ "-serial", "stdio",
+ }
if !options.AllowReboot {
qemuArgs = append(qemuArgs, "-no-reboot")
@@ -298,7 +302,7 @@
if err != nil {
return fmt.Errorf("failed to encode node paraeters: %w", err)
}
- if err := os.WriteFile(parametersPath, parametersRaw, 0644); err != nil {
+ if err := os.WriteFile(parametersPath, parametersRaw, 0o644); err != nil {
return fmt.Errorf("failed to write node parameters: %w", err)
}
qemuArgs = append(qemuArgs, "-fw_cfg", "name=dev.monogon.metropolis/parameters.pb,file="+parametersPath)
@@ -495,6 +499,11 @@
// bootstrapping them. The nodes' address information in Cluster.Nodes will be
// incomplete.
LeaveNodesNew bool
+
+ // Optional local registry which will be made available to the cluster to
+ // pull images from. This is a more efficient alternative to preseeding all
+ // images used for testing.
+ LocalRegistry *localregistry.Server
}
// Cluster is the running Metropolis cluster launched using the LaunchCluster
@@ -638,7 +647,7 @@
}
func NewSerialFileLogger(p string) (io.ReadWriter, error) {
- f, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE, 0600)
+ f, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE, 0o600)
if err != nil {
return nil, err
}
@@ -688,7 +697,7 @@
// Make a list of channels that will be populated by all running node qemu
// processes.
done := make([]chan error, opts.NumNodes)
- for i, _ := range done {
+ for i := range done {
done[i] = make(chan error, 1)
}
@@ -732,6 +741,31 @@
done[0] <- err
}()
+ localRegistryAddr := net.TCPAddr{
+ IP: net.IPv4(10, 42, 0, 82),
+ Port: 5000,
+ }
+
+ var guestSvcMap launch.GuestServiceMap
+ if opts.LocalRegistry != nil {
+ l, err := net.ListenTCP("tcp", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1)})
+ if err != nil {
+ ctxC()
+ return nil, fmt.Errorf("failed to create TCP listener for local registry: %w", err)
+ }
+ s := http.Server{
+ Handler: opts.LocalRegistry,
+ }
+ go s.Serve(l)
+ go func() {
+ <-ctxT.Done()
+ s.Close()
+ }()
+ guestSvcMap = launch.GuestServiceMap{
+ &localRegistryAddr: *l.Addr().(*net.TCPAddr),
+ }
+ }
+
// Launch nanoswitch.
portMap, err := launch.ConflictFreePortMap(ClusterPorts)
if err != nil {
@@ -757,6 +791,7 @@
InitramfsPath: "metropolis/test/nanoswitch/initramfs.cpio.lz4",
ExtraNetworkInterfaces: switchPorts,
PortMap: portMap,
+ GuestServiceMap: guestSvcMap,
SerialPort: serialPort,
PcapDump: path.Join(ld, "nanoswitch.pcap"),
}); err != nil {
@@ -1108,7 +1143,7 @@
}
host := net.JoinHostPort(c.NodeIDs[0], node.KubernetesAPIWrappedPort.PortString())
- var clientConfig = rest.Config{
+ clientConfig := rest.Config{
Host: host,
TLSClientConfig: rest.TLSClientConfig{
// TODO(q3k): use CA certificate
diff --git a/metropolis/test/launch/launch.go b/metropolis/test/launch/launch.go
index cd51549..a048cef 100644
--- a/metropolis/test/launch/launch.go
+++ b/metropolis/test/launch/launch.go
@@ -114,6 +114,21 @@
return portMap, nil
}
+// GuestServiceMap maps an IP/port combination inside the virtual guest network
+// to a TCPAddr reachable by the host. If the guest connects to the virtual
+// address/port, this connection gets forwarded to the host.
+type GuestServiceMap map[*net.TCPAddr]net.TCPAddr
+
+// ToQemuForwards generates QEMU guestfwd values (https://qemu.weilnetz.de/doc/qemu-
+// doc.html#:~:text=guestfwd=) for all mapped addresses.
+func (p GuestServiceMap) ToQemuForwards() []string {
+ var guestfwdOptions []string
+ for guestAddr, hostAddr := range p {
+ guestfwdOptions = append(guestfwdOptions, fmt.Sprintf("tcp:%s-tcp:%s", guestAddr.String(), hostAddr.String()))
+ }
+ return guestfwdOptions
+}
+
// NewSocketPair creates a new socket pair. By connecting both ends to different
// instances you can connect them with a virtual "network cable". The ends can be
// passed into the ConnectToSocket option.
@@ -165,6 +180,10 @@
// network interface.
PortMap PortMap
+ // GuestServiceMap contains TCP services made available in the guest virtual
+ // network which are running on the host.
+ GuestServiceMap GuestServiceMap
+
// DisableHostNetworkInterface disables the SLIRP-backed host network interface
// that is normally the first network interface. If this is set PortMap is ignored.
// Mostly useful for speeding up QEMU's startup time for tests.
@@ -257,6 +276,9 @@
if opts.PortMap != nil {
qemuNetConfig["hostfwd"] = opts.PortMap.ToQemuForwards()
}
+ if opts.GuestServiceMap != nil {
+ qemuNetConfig["guestfwd"] = opts.GuestServiceMap.ToQemuForwards()
+ }
baseArgs = append(baseArgs, "-netdev", qemuNetConfig.ToOption(qemuNetType),
"-device", "virtio-net-device,netdev=usernet0,mac="+HostInterfaceMAC.String())