m/{cli,test/launch}: integrate launch/cluster with metroctl
This makes test-launch2 (and possibly later any other code that uses the
launch/cluster library) tell the user that they can connect to the newly
launched cluster using metroctl, either by using specific flags, or
using a wrapper script, or using kubectl.
Change-Id: I54035ee02f3cbab3d17f46b1f1685b91aab275a9
Reviewed-on: https://review.monogon.dev/c/monogon/+/1373
Tested-by: Jenkins CI
Reviewed-by: Leopold Schabel <leo@monogon.tech>
diff --git a/metropolis/test/launch/cli/launch-multi2/BUILD.bazel b/metropolis/test/launch/cli/launch-multi2/BUILD.bazel
index e56d131..90440eb 100644
--- a/metropolis/test/launch/cli/launch-multi2/BUILD.bazel
+++ b/metropolis/test/launch/cli/launch-multi2/BUILD.bazel
@@ -7,6 +7,7 @@
importpath = "source.monogon.dev/metropolis/test/launch/cli/launch-multi2",
visibility = ["//visibility:private"],
deps = [
+ "//metropolis/cli/metroctl/core",
"//metropolis/cli/pkg/context",
"//metropolis/test/launch/cluster",
],
@@ -15,12 +16,7 @@
go_binary(
name = "launch-multi2_bin",
data = [
- "//metropolis/node:image",
- "//metropolis/node:swtpm_data",
- "//metropolis/test/ktest:linux-testing",
- "//metropolis/test/nanoswitch:initramfs",
- "//third_party/edk2:firmware",
- "@com_github_bonzini_qboot//:qboot-bin",
+ "//metropolis/cli/metroctl",
],
embed = [":launch-multi2_lib"],
visibility = ["//:__pkg__"],
diff --git a/metropolis/test/launch/cli/launch-multi2/main.go b/metropolis/test/launch/cli/launch-multi2/main.go
index 70c3745..46770f2 100644
--- a/metropolis/test/launch/cli/launch-multi2/main.go
+++ b/metropolis/test/launch/cli/launch-multi2/main.go
@@ -20,6 +20,7 @@
"context"
"log"
+ metroctl "source.monogon.dev/metropolis/cli/metroctl/core"
clicontext "source.monogon.dev/metropolis/cli/pkg/context"
"source.monogon.dev/metropolis/test/launch/cluster"
)
@@ -32,7 +33,33 @@
if err != nil {
log.Fatalf("LaunchCluster: %v", err)
}
+
+ mpath, err := cluster.MetroctlRunfilePath()
+ if err != nil {
+ log.Fatalf("MetroctlRunfilePath: %v", err)
+ }
+ wpath, err := cl.MakeMetroctlWrapper()
+ if err != nil {
+ log.Fatalf("MakeWrapper: %v", err)
+ }
+
+ apiservers, err := cl.KubernetesControllerNodeAddresses(ctx)
+ if err != nil {
+ log.Fatalf("Could not get Kubernetes controller nodes: %v", err)
+ }
+ if len(apiservers) < 1 {
+ log.Fatalf("Cluster has no Kubernetes controller nodes")
+ }
+
+ configName := "launch-multi2"
+ if err := metroctl.InstallKubeletConfig(mpath, cl.ConnectOptions(), configName, apiservers[0]); err != nil {
+ log.Fatalf("InstallKubeletConfig: %v", err)
+ }
+
log.Printf("Launch: Cluster running!")
+ log.Printf(" To access cluster use: metroctl %s", cl.MetroctlFlags())
+ log.Printf(" Or use this handy wrapper: %s", wpath)
+ log.Printf(" To access Kubernetes, use kubectl --context=%s", configName)
<-ctx.Done()
cl.Close()
diff --git a/metropolis/test/launch/cluster/BUILD.bazel b/metropolis/test/launch/cluster/BUILD.bazel
index a0e82b4..e81ff7c 100644
--- a/metropolis/test/launch/cluster/BUILD.bazel
+++ b/metropolis/test/launch/cluster/BUILD.bazel
@@ -5,6 +5,7 @@
srcs = [
"cluster.go",
"insecure_key.go",
+ "metroctl.go",
"prefixed_stdio.go",
],
data = [
@@ -18,6 +19,7 @@
importpath = "source.monogon.dev/metropolis/test/launch/cluster",
visibility = ["//visibility:public"],
deps = [
+ "//metropolis/cli/metroctl/core",
"//metropolis/cli/pkg/datafile",
"//metropolis/node",
"//metropolis/node/core/identity",
@@ -28,6 +30,7 @@
"//metropolis/proto/common",
"//metropolis/test/launch",
"@com_github_cenkalti_backoff_v4//:backoff",
+ "@com_github_kballard_go_shellquote//:go-shellquote",
"@org_golang_google_grpc//:go_default_library",
"@org_golang_google_grpc//codes",
"@org_golang_google_grpc//status",
diff --git a/metropolis/test/launch/cluster/cluster.go b/metropolis/test/launch/cluster/cluster.go
index a0ad82f..5ea25b5 100644
--- a/metropolis/test/launch/cluster/cluster.go
+++ b/metropolis/test/launch/cluster/cluster.go
@@ -7,6 +7,7 @@
import (
"bytes"
"context"
+ "crypto/ed25519"
"crypto/rand"
"crypto/tls"
"errors"
@@ -28,6 +29,7 @@
"google.golang.org/grpc/status"
"google.golang.org/protobuf/proto"
+ metroctl "source.monogon.dev/metropolis/cli/metroctl/core"
"source.monogon.dev/metropolis/cli/pkg/datafile"
"source.monogon.dev/metropolis/node"
"source.monogon.dev/metropolis/node/core/identity"
@@ -508,7 +510,8 @@
// used to facilitate communication between QEMU and swtpm. It's different
// from launchDir, and anchored nearer the file system root, due to the
// socket path length limitation imposed by the kernel.
- socketDir string
+ socketDir string
+ metroctlDir string
// socksDialer is used by DialNode to establish connections to nodes via the
// SOCKS server ran by nanoswitch.
@@ -624,12 +627,19 @@
}
// Create the launch directory.
- ld, err := os.MkdirTemp(os.Getenv("TEST_TMPDIR"), "cluster*")
+ ld, err := os.MkdirTemp(os.Getenv("TEST_TMPDIR"), "cluster-*")
if err != nil {
return nil, fmt.Errorf("failed to create the launch directory: %w", err)
}
- // Create the socket directory.
- sd, err := os.MkdirTemp("/tmp", "cluster*")
+ // Create the metroctl config directory. We keep it in /tmp because in some
+ // scenarios it's end-user visible and we want it short.
+ md, err := os.MkdirTemp("/tmp", "metroctl-*")
+ if err != nil {
+ return nil, fmt.Errorf("failed to create the metroctl directory: %w", err)
+ }
+
+ // Create the socket directory. We keep it in /tmp because of socket path limits.
+ sd, err := os.MkdirTemp("/tmp", "cluster-*")
if err != nil {
return nil, fmt.Errorf("failed to create the socket directory: %w", err)
}
@@ -722,6 +732,16 @@
return nil, err
}
+ // Write credentials to the metroctl directory.
+ if err := metroctl.WriteOwnerKey(md, cert.PrivateKey.(ed25519.PrivateKey)); err != nil {
+ ctxC()
+ return nil, fmt.Errorf("could not write owner key: %w", err)
+ }
+ if err := metroctl.WriteOwnerCertificate(md, cert.Certificate[0]); err != nil {
+ ctxC()
+ return nil, fmt.Errorf("could not write owner certificate: %w", err)
+ }
+
// Set up a partially initialized cluster instance, to be filled in in the
// later steps.
cluster := &Cluster{
@@ -734,10 +754,11 @@
firstNode.ID,
},
- nodesDone: done,
- nodeOpts: nodeOpts,
- launchDir: ld,
- socketDir: sd,
+ nodesDone: done,
+ nodeOpts: nodeOpts,
+ launchDir: ld,
+ socketDir: sd,
+ metroctlDir: md,
socksDialer: socksDialer,
@@ -970,6 +991,7 @@
launch.Log("Cluster: removing nodes' state files.")
os.RemoveAll(c.launchDir)
os.RemoveAll(c.socketDir)
+ os.RemoveAll(c.metroctlDir)
launch.Log("Cluster: done")
return multierr.Combine(errs...)
}
@@ -999,3 +1021,37 @@
addr = net.JoinHostPort(node.ManagementAddress, port)
return c.socksDialer.Dial("tcp", addr)
}
+
+// KubernetesControllerNodeAddresses returns the list of IP addresses of nodes
+// which are currently Kubernetes controllers, ie. run an apiserver. This list
+// might be empty if no node is currently configured with the
+// 'KubernetesController' node.
+func (c *Cluster) KubernetesControllerNodeAddresses(ctx context.Context) ([]string, error) {
+ curC, err := c.CuratorClient()
+ if err != nil {
+ return nil, err
+ }
+ mgmt := apb.NewManagementClient(curC)
+ srv, err := mgmt.GetNodes(ctx, &apb.GetNodesRequest{
+ Filter: "has(node.roles.kubernetes_controller)",
+ })
+ if err != nil {
+ return nil, err
+ }
+ defer srv.CloseSend()
+ var res []string
+ for {
+ n, err := srv.Recv()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return nil, err
+ }
+ if n.Status == nil || n.Status.ExternalAddress == "" {
+ continue
+ }
+ res = append(res, n.Status.ExternalAddress)
+ }
+ return res, nil
+}
diff --git a/metropolis/test/launch/cluster/metroctl.go b/metropolis/test/launch/cluster/metroctl.go
new file mode 100644
index 0000000..da77fe5
--- /dev/null
+++ b/metropolis/test/launch/cluster/metroctl.go
@@ -0,0 +1,73 @@
+package cluster
+
+import (
+ "fmt"
+ "net"
+ "os"
+ "path"
+ "sort"
+
+ "github.com/kballard/go-shellquote"
+
+ metroctl "source.monogon.dev/metropolis/cli/metroctl/core"
+ "source.monogon.dev/metropolis/cli/pkg/datafile"
+)
+
+const metroctlRunfile = "metropolis/cli/metroctl/metroctl_/metroctl"
+
+// MetroctlRunfilePath returns the absolute path to the metroctl binary available
+// if the built target depends on //metropolis/cli/metroctl. Otherwise, an error
+// is returned.
+func MetroctlRunfilePath() (string, error) {
+ path, err := datafile.ResolveRunfile(metroctlRunfile)
+ if err != nil {
+ return "", fmt.Errorf("//metropolis/cli/metroctl not found in runfiles, did you include it as a data dependency? error: %w", err)
+ }
+ return path, nil
+}
+
+// ConnectOptions returns metroctl.ConnectOptions that describe connectivity to
+// the launched cluster.
+func (c *Cluster) ConnectOptions() *metroctl.ConnectOptions {
+ // Use all metropolis nodes as endpoints. That's fine, metroctl's resolver will
+ // figure out what to actually use.
+ var endpoints []string
+ for _, n := range c.Nodes {
+ endpoints = append(endpoints, n.ManagementAddress)
+ }
+ sort.Strings(endpoints)
+ return &metroctl.ConnectOptions{
+ ConfigPath: c.metroctlDir,
+ ProxyServer: net.JoinHostPort("127.0.0.1", fmt.Sprintf("%d", c.Ports[SOCKSPort])),
+ Endpoints: endpoints,
+ }
+}
+
+// MetroctlFlags return stringified flags to pass to a metroctl binary to connect
+// to the launched cluster.
+func (c *Cluster) MetroctlFlags() string {
+ return shellquote.Join(c.ConnectOptions().ToFlags()...)
+}
+
+// MakeMetroctlWrapper builds and returns the path to a shell script which calls
+// metroctl (from //metropolis/cli/metroctl, which must be included as a data
+// dependency of the built target) with all the required flags to connect to the
+// launched cluster.
+func (c *Cluster) MakeMetroctlWrapper() (string, error) {
+ mpath, err := MetroctlRunfilePath()
+ if err != nil {
+ return "", err
+ }
+ wpath := path.Join(c.metroctlDir, "metroctl.sh")
+
+ // Don't create wrapper if it already exists.
+ if _, err := os.Stat(wpath); err == nil {
+ return wpath, nil
+ }
+
+ wrapper := fmt.Sprintf("#!/usr/bin/env bash\nexec %s %s \"$@\"", mpath, c.MetroctlFlags())
+ if err := os.WriteFile(wpath, []byte(wrapper), 0555); err != nil {
+ return "", fmt.Errorf("could not write wrapper: %w", err)
+ }
+ return wpath, nil
+}