metropolis/proto: EnrolmentConfig -> NodeParameters
This starts off the move to a node configuration API conforming to
the lifecycle management design document.
Instead of an Enrolment Config used only to join an existing cluster, we
move to a NodeParameters proto that must always be given to a node if
it's supposed to either bootstrap a new cluster or join an existing one.
This links the existing cluster management code (and its state machine)
to work with this file. However, that state machine will be removed very
soon, anyway.
We also remove everything related to golden tickets.
This breaks multi-node tests.
X-Origin-Diff: phab/D710
GitOrigin-RevId: f22615fbccab975f2d5e6928bdc7387ab3aa5714
diff --git a/metropolis/test/e2e/main_test.go b/metropolis/test/e2e/main_test.go
index 71713cc..3e4ffb0 100644
--- a/metropolis/test/e2e/main_test.go
+++ b/metropolis/test/e2e/main_test.go
@@ -82,7 +82,15 @@
procExit := make(chan struct{})
go func() {
- if err := launch.Launch(ctx, launch.Options{Ports: portMap, SerialPort: os.Stdout}); err != nil {
+ if err := launch.Launch(ctx, launch.Options{
+ Ports: portMap,
+ SerialPort: os.Stdout,
+ NodeParameters: &apb.NodeParameters{
+ Cluster: &apb.NodeParameters_ClusterBootstrap_{
+ ClusterBootstrap: &apb.NodeParameters_ClusterBootstrap{},
+ },
+ },
+ }); err != nil {
panic(err)
}
close(procExit)
diff --git a/metropolis/test/launch/cli/launch-multi2/BUILD.bazel b/metropolis/test/launch/cli/launch-multi2/BUILD.bazel
index 91efdd7..51118cf 100644
--- a/metropolis/test/launch/cli/launch-multi2/BUILD.bazel
+++ b/metropolis/test/launch/cli/launch-multi2/BUILD.bazel
@@ -5,14 +5,7 @@
srcs = ["main.go"],
importpath = "source.monogon.dev/metropolis/test/launch/cli/launch-multi2",
visibility = ["//visibility:private"],
- deps = [
- "//metropolis/node:go_default_library",
- "//metropolis/pkg/logbuffer:go_default_library",
- "//metropolis/proto/api:go_default_library",
- "//metropolis/test/launch:go_default_library",
- "@com_github_grpc_ecosystem_go_grpc_middleware//retry:go_default_library",
- "@org_golang_google_grpc//:go_default_library",
- ],
+ deps = ["//metropolis/pkg/logbuffer:go_default_library"],
)
go_binary(
diff --git a/metropolis/test/launch/cli/launch-multi2/main.go b/metropolis/test/launch/cli/launch-multi2/main.go
index a2e00bc..d6c5f05 100644
--- a/metropolis/test/launch/cli/launch-multi2/main.go
+++ b/metropolis/test/launch/cli/launch-multi2/main.go
@@ -17,22 +17,12 @@
package main
import (
- "context"
"fmt"
"io"
"log"
"os"
- "os/signal"
- "syscall"
- "time"
- grpcretry "github.com/grpc-ecosystem/go-grpc-middleware/retry"
- "google.golang.org/grpc"
-
- common "source.monogon.dev/metropolis/node"
"source.monogon.dev/metropolis/pkg/logbuffer"
- apb "source.monogon.dev/metropolis/proto/api"
- "source.monogon.dev/metropolis/test/launch"
)
// prefixedStdout is a os.Stdout proxy that prefixes every line with a constant
@@ -57,77 +47,5 @@
}
func main() {
- sigs := make(chan os.Signal, 1)
- signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
- ctx, cancel := context.WithCancel(context.Background())
- go func() {
- <-sigs
- cancel()
- }()
- sw0, vm0, err := launch.NewSocketPair()
- if err != nil {
- log.Fatalf("Failed to create network pipe: %v\n", err)
- }
- sw1, vm1, err := launch.NewSocketPair()
- if err != nil {
- log.Fatalf("Failed to create network pipe: %v\n", err)
- }
-
- go func() {
- if err := launch.Launch(ctx, launch.Options{
- ConnectToSocket: vm0,
- SerialPort: prefixedStdout("1| "),
- }); err != nil {
- log.Fatalf("Failed to launch vm0: %v", err)
- }
- }()
- nanoswitchPortMap := make(launch.PortMap)
- identityPorts := []uint16{
- common.ExternalServicePort,
- common.DebugServicePort,
- common.KubernetesAPIPort,
- }
- for _, port := range identityPorts {
- nanoswitchPortMap[port] = port
- }
- go func() {
- opts := []grpcretry.CallOption{
- grpcretry.WithBackoff(grpcretry.BackoffExponential(100 * time.Millisecond)),
- }
- conn, err := nanoswitchPortMap.DialGRPC(common.DebugServicePort, grpc.WithInsecure(),
- grpc.WithUnaryInterceptor(grpcretry.UnaryClientInterceptor(opts...)))
- if err != nil {
- panic(err)
- }
- defer conn.Close()
- debug := apb.NewNodeDebugServiceClient(conn)
- res, err := debug.GetGoldenTicket(ctx, &apb.GetGoldenTicketRequest{
- // HACK: this is assigned by DHCP, and we assume that everything goes well.
- ExternalIp: "10.1.0.3",
- }, grpcretry.WithMax(10))
- if err != nil {
- log.Fatalf("Failed to get golden ticket: %v", err)
- }
-
- ec := &apb.EnrolmentConfig{
- GoldenTicket: res.Ticket,
- }
-
- if err := launch.Launch(ctx, launch.Options{
- ConnectToSocket: vm1,
- EnrolmentConfig: ec,
- SerialPort: prefixedStdout("2| "),
- }); err != nil {
- log.Fatalf("Failed to launch vm1: %v", err)
- }
- }()
- if err := launch.RunMicroVM(ctx, &launch.MicroVMOptions{
- SerialPort: os.Stdout,
- KernelPath: "metropolis/test/ktest/linux-testing.elf",
- InitramfsPath: "metropolis/test/nanoswitch/initramfs.lz4",
- ExtraNetworkInterfaces: []*os.File{sw0, sw1},
- PortMap: nanoswitchPortMap,
- }); err != nil {
- log.Fatalf("Failed to launch nanoswitch: %v", err)
- }
+ log.Fatal("unimplemented")
}
diff --git a/metropolis/test/launch/cli/launch/BUILD.bazel b/metropolis/test/launch/cli/launch/BUILD.bazel
index 43c02b6..824b2ff 100644
--- a/metropolis/test/launch/cli/launch/BUILD.bazel
+++ b/metropolis/test/launch/cli/launch/BUILD.bazel
@@ -5,7 +5,10 @@
srcs = ["main.go"],
importpath = "source.monogon.dev/metropolis/test/launch/cli/launch",
visibility = ["//visibility:private"],
- deps = ["//metropolis/test/launch:go_default_library"],
+ deps = [
+ "//metropolis/proto/api:go_default_library",
+ "//metropolis/test/launch:go_default_library",
+ ],
)
go_binary(
diff --git a/metropolis/test/launch/cli/launch/main.go b/metropolis/test/launch/cli/launch/main.go
index a855f73..cb85c88 100644
--- a/metropolis/test/launch/cli/launch/main.go
+++ b/metropolis/test/launch/cli/launch/main.go
@@ -23,6 +23,7 @@
"os/signal"
"syscall"
+ apb "source.monogon.dev/metropolis/proto/api"
"source.monogon.dev/metropolis/test/launch"
)
@@ -34,7 +35,15 @@
<-sigs
cancel()
}()
- if err := launch.Launch(ctx, launch.Options{Ports: launch.IdentityPortMap(launch.NodePorts), SerialPort: os.Stdout}); err != nil {
+ if err := launch.Launch(ctx, launch.Options{
+ Ports: launch.IdentityPortMap(launch.NodePorts),
+ SerialPort: os.Stdout,
+ NodeParameters: &apb.NodeParameters{
+ Cluster: &apb.NodeParameters_ClusterBootstrap_{
+ ClusterBootstrap: &apb.NodeParameters_ClusterBootstrap{},
+ },
+ },
+ }); err != nil {
if err == ctx.Err() {
return
}
diff --git a/metropolis/test/launch/launch.go b/metropolis/test/launch/launch.go
index 6e6891a..3a444ef 100644
--- a/metropolis/test/launch/launch.go
+++ b/metropolis/test/launch/launch.go
@@ -130,8 +130,8 @@
// It can be set to an existing file descriptor (like os.Stdout/os.Stderr) or any Go structure implementing this interface.
SerialPort io.ReadWriter
- // EnrolmentConfig is passed into the VM and subsequently used for bootstrapping if no enrolment config is built-in
- EnrolmentConfig *apb.EnrolmentConfig
+ // NodeParameters is passed into the VM and subsequently used for bootstrapping or registering into a cluster.
+ NodeParameters *apb.NodeParameters
}
// NodePorts is the list of ports a fully operational Metropolis node listens on
@@ -254,16 +254,16 @@
qemuArgs = append(qemuArgs, "-no-reboot")
}
- if options.EnrolmentConfig != nil {
- enrolmentConfigPath := filepath.Join(tempDir, "enrolment.pb")
- enrolmentConfigRaw, err := proto.Marshal(options.EnrolmentConfig)
+ if options.NodeParameters != nil {
+ parametersPath := filepath.Join(tempDir, "parameters.pb")
+ parametersRaw, err := proto.Marshal(options.NodeParameters)
if err != nil {
- return fmt.Errorf("failed to encode enrolment config: %w", err)
+ return fmt.Errorf("failed to encode node paraeters: %w", err)
}
- if err := ioutil.WriteFile(enrolmentConfigPath, enrolmentConfigRaw, 0644); err != nil {
- return fmt.Errorf("failed to write enrolment config: %w", err)
+ if err := ioutil.WriteFile(parametersPath, parametersRaw, 0644); err != nil {
+ return fmt.Errorf("failed to write node parameters: %w", err)
}
- qemuArgs = append(qemuArgs, "-fw_cfg", "name=dev.monogon.metropolis/enrolment.pb,file="+enrolmentConfigPath)
+ qemuArgs = append(qemuArgs, "-fw_cfg", "name=dev.monogon.metropolis/parameters.pb,file="+parametersPath)
}
// Start TPM emulator as a subprocess
@@ -498,7 +498,15 @@
}
go func() {
- if err := Launch(ctx, Options{ConnectToSocket: vmPorts[0]}); err != nil {
+ if err := Launch(ctx, Options{
+ ConnectToSocket: vmPorts[0],
+ NodeParameters: &apb.NodeParameters{
+ Cluster: &apb.NodeParameters_ClusterBootstrap_{
+ ClusterBootstrap: &apb.NodeParameters_ClusterBootstrap{},
+ },
+ },
+ }); err != nil {
+
// Launch() only terminates when QEMU has terminated. At that point our function probably doesn't run anymore
// so we have no way of communicating the error back up, so let's just log it. Also a failure in launching
// VMs should be very visible by the unavailability of the clients we return.
@@ -532,23 +540,7 @@
debug := apb.NewNodeDebugServiceClient(conn)
if opts.NumNodes == 2 {
- res, err := debug.GetGoldenTicket(ctx, &apb.GetGoldenTicketRequest{
- // HACK: this is assigned by DHCP, and we assume that everything goes well.
- ExternalIp: "10.1.0.3",
- }, grpcretry.WithMax(10))
- if err != nil {
- return nil, nil, fmt.Errorf("failed to get golden ticket: %w", err)
- }
-
- ec := &apb.EnrolmentConfig{
- GoldenTicket: res.Ticket,
- }
-
- go func() {
- if err := Launch(ctx, Options{ConnectToSocket: vmPorts[1], EnrolmentConfig: ec}); err != nil {
- log.Printf("Failed to launch vm1: %v", err)
- }
- }()
+ return nil, nil, fmt.Errorf("multinode unimplemented")
}
return debug, portMap, nil