m/{cli,test/launch}: integrate launch/cluster with metroctl
This makes test-launch2 (and possibly later any other code that uses the
launch/cluster library) tell the user that they can connect to the newly
launched cluster using metroctl, either by using specific flags, or
using a wrapper script, or using kubectl.
Change-Id: I54035ee02f3cbab3d17f46b1f1685b91aab275a9
Reviewed-on: https://review.monogon.dev/c/monogon/+/1373
Tested-by: Jenkins CI
Reviewed-by: Leopold Schabel <leo@monogon.tech>
diff --git a/go.mod b/go.mod
index ff2d8ed..37b8525 100644
--- a/go.mod
+++ b/go.mod
@@ -92,6 +92,7 @@
github.com/improbable-eng/grpc-web v0.15.0
github.com/insomniacslk/dhcp v0.0.0-20220119180841-3c283ff8b7dd
github.com/joho/godotenv v1.4.0
+ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
github.com/kevinburke/go-bindata v3.23.0+incompatible
github.com/kyleconroy/sqlc v1.15.0
github.com/lib/pq v1.10.6
diff --git a/go.sum b/go.sum
index 279bd7f..c05e7b0 100644
--- a/go.sum
+++ b/go.sum
@@ -1400,6 +1400,7 @@
github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/kevinburke/go-bindata v3.23.0+incompatible h1:rqNOXZlqrYhMVVAsQx8wuc+LaA73YcfbQ407wAykyS8=
github.com/kevinburke/go-bindata v3.23.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM=
diff --git a/metropolis/cli/metroctl/core/config.go b/metropolis/cli/metroctl/core/config.go
index d62bf2e..9714421 100644
--- a/metropolis/cli/metroctl/core/config.go
+++ b/metropolis/cli/metroctl/core/config.go
@@ -7,12 +7,16 @@
"encoding/pem"
"errors"
"fmt"
+ "net"
+ "net/url"
"os"
"path/filepath"
clientauthentication "k8s.io/client-go/pkg/apis/clientauthentication/v1"
"k8s.io/client-go/tools/clientcmd"
clientapi "k8s.io/client-go/tools/clientcmd/api"
+
+ "source.monogon.dev/metropolis/node"
)
const (
@@ -129,24 +133,33 @@
return
}
-// InstallK8SWrapper configures the current user's kubectl to connect to a
-// Kubernetes cluster as defined by server (Metropolis wrapped APIServer
-// endpoint), proxyURL (optional proxy URL) and metroctlPath (binary managing
-// credentials for this cluster, and used to implement the client-side part of
-// the Metropolis-wrapped APIServer protocol). The configuration will be saved to
-// the 'configName' context in kubectl.
-func InstallK8SWrapper(metroctlPath, configName, server, proxyURL string) error {
+// InstallKubeletConfig modifies the default kubelet kubeconfig of the host
+// system to be able to connect via a metroctl (and an associated ConnectOptions)
+// to a Kubernetes apiserver at IP address/hostname 'server'.
+//
+// The kubelet's kubeconfig changes will be limited to contexts/configs/... named
+// configName. The configName context will be made the default context only if
+// there is no other default context in the current subconfig.
+//
+// Kubeconfigs can only take a single Kubernetes server address, so this function
+// similarly only allows you to specify only a single server address.
+func InstallKubeletConfig(metroctlPath string, opts *ConnectOptions, configName, server string) error {
ca := clientcmd.NewDefaultPathOptions()
config, err := ca.GetStartingConfig()
if err != nil {
return fmt.Errorf("getting initial config failed: %w", err)
}
+ args := []string{
+ "k8scredplugin",
+ }
+ args = append(args, opts.ToFlags()...)
+
config.AuthInfos[configName] = &clientapi.AuthInfo{
Exec: &clientapi.ExecConfig{
APIVersion: clientauthentication.SchemeGroupVersion.String(),
Command: metroctlPath,
- Args: []string{"k8scredplugin"},
+ Args: args,
InstallHint: `Authenticating to Metropolis clusters requires metroctl to be present.
Running metroctl takeownership creates this entry and either points to metroctl as a command in
PATH if metroctl is in PATH at that time or to the absolute path to metroctl at that time.
@@ -156,14 +169,17 @@
},
}
+ var u url.URL
+ u.Scheme = "https"
+ u.Host = net.JoinHostPort(server, node.KubernetesAPIWrappedPort.PortString())
config.Clusters[configName] = &clientapi.Cluster{
// MVP: This is insecure, but making this work would be wasted effort
// as all of it will be replaced by the identity system.
// TODO(issues/144): adjust cluster endpoints once have functioning roles
// implemented.
InsecureSkipTLSVerify: true,
- Server: server,
- ProxyURL: proxyURL,
+ Server: u.String(),
+ ProxyURL: opts.ProxyURL(),
}
config.Contexts[configName] = &clientapi.Context{
@@ -183,3 +199,61 @@
}
return nil
}
+
+// ConnectOptions define how to reach a Metropolis cluster from metroctl.
+//
+// This structure can be built directly. All unset fields mean 'default'. It can
+// then be used to generate the equivalent flags to passs to metroctl.
+//
+// Nil pointers to ConnectOptions are equivalent to an empty ConneectOptions when
+// methods on it are called.
+type ConnectOptions struct {
+ // ConfigPath is the path at which the metroctl configuration/credentials live.
+ // If not set, the default will be used.
+ ConfigPath string
+ // ProxyServer is a host:port pair that indicates the metropolis cluster should
+ // be reached via the given SOCKS5 proxy. If not set, the cluster can be reached
+ // directly from the host networking stack.
+ ProxyServer string
+ // Endpoints are the IP addresses/hostnames (without port part) of the Metropolis
+ // instances that metroctl should use to establish connectivity to a cluster.
+ // These instances should have the ControlPlane role set.
+ Endpoints []string
+}
+
+// ToFlags returns the metroctl flags corresponding to the options described by
+// this ConnectionOptions struct.
+func (c *ConnectOptions) ToFlags() []string {
+ var res []string
+
+ if c == nil {
+ return res
+ }
+
+ if c.ConfigPath != "" {
+ res = append(res, "--config", c.ConfigPath)
+ }
+ if c.ProxyServer != "" {
+ res = append(res, "--proxy", c.ProxyServer)
+ }
+ for _, ep := range c.Endpoints {
+ res = append(res, "--endpoints", ep)
+ }
+
+ return res
+}
+
+// ProxyURL returns a kubeconfig-compatible URL of the proxy server configured by
+// ConnectOptions, or an empty string if not set.
+func (c *ConnectOptions) ProxyURL() string {
+ if c == nil {
+ return ""
+ }
+ if c.ProxyServer == "" {
+ return ""
+ }
+ var u url.URL
+ u.Scheme = "socks5"
+ u.Host = c.ProxyServer
+ return u.String()
+}
diff --git a/metropolis/cli/metroctl/k8scredplugin.go b/metropolis/cli/metroctl/k8scredplugin.go
index 605bc87..9b3226e 100644
--- a/metropolis/cli/metroctl/k8scredplugin.go
+++ b/metropolis/cli/metroctl/k8scredplugin.go
@@ -20,8 +20,9 @@
Long: `This implements a Kubernetes client-go credential plugin to
authenticate client-go based callers including kubectl against a Metropolis
cluster. This should never be directly called by end users.`,
- Args: cobra.ExactArgs(0),
- Run: doK8sCredPlugin,
+ Args: cobra.ExactArgs(0),
+ Hidden: true,
+ Run: doK8sCredPlugin,
}
func doK8sCredPlugin(cmd *cobra.Command, args []string) {
diff --git a/metropolis/cli/metroctl/main.go b/metropolis/cli/metroctl/main.go
index 8714332..19b378d 100644
--- a/metropolis/cli/metroctl/main.go
+++ b/metropolis/cli/metroctl/main.go
@@ -6,6 +6,8 @@
"github.com/adrg/xdg"
"github.com/spf13/cobra"
+
+ "source.monogon.dev/metropolis/cli/metroctl/core"
)
// rootCmd represents the base command when called without any subcommands
@@ -58,3 +60,13 @@
func main() {
cobra.CheckErr(rootCmd.Execute())
}
+
+// connectOptions returns core.ConnectOptions as defined by the metroctl flags
+// currently set.
+func connectOptions() *core.ConnectOptions {
+ return &core.ConnectOptions{
+ ProxyServer: flags.proxyAddr,
+ Endpoints: flags.clusterEndpoints,
+ ConfigPath: flags.configPath,
+ }
+}
diff --git a/metropolis/cli/metroctl/takeownership.go b/metropolis/cli/metroctl/takeownership.go
index e0b2f79..2c37340 100644
--- a/metropolis/cli/metroctl/takeownership.go
+++ b/metropolis/cli/metroctl/takeownership.go
@@ -68,10 +68,10 @@
log.Fatalf("Failed to create kubectl entry as metroctl is neither in PATH nor can its absolute path be determined: %v", err)
}
}
- // TODO(issues/144): adjust cluster endpoints once have functioning roles
- // implemented.
- server := "https://" + net.JoinHostPort(flags.clusterEndpoints[0], node.KubernetesAPIWrappedPort.PortString())
- if err := core.InstallK8SWrapper(metroctlPath, "metroctl", server, ""); err != nil {
+ // TODO(q3k, issues/144): this only works as long as all nodes are kubernetes controller
+ // nodes. This won't be the case for too long. Figure this out.
+ apiserver := "https://" + net.JoinHostPort(flags.clusterEndpoints[0], node.KubernetesAPIWrappedPort.PortString())
+ if err := core.InstallKubeletConfig(metroctlPath, connectOptions(), "metroctl", apiserver); err != nil {
log.Fatalf("Failed to install metroctl/k8s integration: %v", err)
}
log.Println("Success! kubeconfig is set up. You can now run kubectl --context=metropolis ... to access the Kubernetes cluster.")
diff --git a/metropolis/test/launch/cli/launch-multi2/BUILD.bazel b/metropolis/test/launch/cli/launch-multi2/BUILD.bazel
index e56d131..90440eb 100644
--- a/metropolis/test/launch/cli/launch-multi2/BUILD.bazel
+++ b/metropolis/test/launch/cli/launch-multi2/BUILD.bazel
@@ -7,6 +7,7 @@
importpath = "source.monogon.dev/metropolis/test/launch/cli/launch-multi2",
visibility = ["//visibility:private"],
deps = [
+ "//metropolis/cli/metroctl/core",
"//metropolis/cli/pkg/context",
"//metropolis/test/launch/cluster",
],
@@ -15,12 +16,7 @@
go_binary(
name = "launch-multi2_bin",
data = [
- "//metropolis/node:image",
- "//metropolis/node:swtpm_data",
- "//metropolis/test/ktest:linux-testing",
- "//metropolis/test/nanoswitch:initramfs",
- "//third_party/edk2:firmware",
- "@com_github_bonzini_qboot//:qboot-bin",
+ "//metropolis/cli/metroctl",
],
embed = [":launch-multi2_lib"],
visibility = ["//:__pkg__"],
diff --git a/metropolis/test/launch/cli/launch-multi2/main.go b/metropolis/test/launch/cli/launch-multi2/main.go
index 70c3745..46770f2 100644
--- a/metropolis/test/launch/cli/launch-multi2/main.go
+++ b/metropolis/test/launch/cli/launch-multi2/main.go
@@ -20,6 +20,7 @@
"context"
"log"
+ metroctl "source.monogon.dev/metropolis/cli/metroctl/core"
clicontext "source.monogon.dev/metropolis/cli/pkg/context"
"source.monogon.dev/metropolis/test/launch/cluster"
)
@@ -32,7 +33,33 @@
if err != nil {
log.Fatalf("LaunchCluster: %v", err)
}
+
+ mpath, err := cluster.MetroctlRunfilePath()
+ if err != nil {
+ log.Fatalf("MetroctlRunfilePath: %v", err)
+ }
+ wpath, err := cl.MakeMetroctlWrapper()
+ if err != nil {
+ log.Fatalf("MakeWrapper: %v", err)
+ }
+
+ apiservers, err := cl.KubernetesControllerNodeAddresses(ctx)
+ if err != nil {
+ log.Fatalf("Could not get Kubernetes controller nodes: %v", err)
+ }
+ if len(apiservers) < 1 {
+ log.Fatalf("Cluster has no Kubernetes controller nodes")
+ }
+
+ configName := "launch-multi2"
+ if err := metroctl.InstallKubeletConfig(mpath, cl.ConnectOptions(), configName, apiservers[0]); err != nil {
+ log.Fatalf("InstallKubeletConfig: %v", err)
+ }
+
log.Printf("Launch: Cluster running!")
+ log.Printf(" To access cluster use: metroctl %s", cl.MetroctlFlags())
+ log.Printf(" Or use this handy wrapper: %s", wpath)
+ log.Printf(" To access Kubernetes, use kubectl --context=%s", configName)
<-ctx.Done()
cl.Close()
diff --git a/metropolis/test/launch/cluster/BUILD.bazel b/metropolis/test/launch/cluster/BUILD.bazel
index a0e82b4..e81ff7c 100644
--- a/metropolis/test/launch/cluster/BUILD.bazel
+++ b/metropolis/test/launch/cluster/BUILD.bazel
@@ -5,6 +5,7 @@
srcs = [
"cluster.go",
"insecure_key.go",
+ "metroctl.go",
"prefixed_stdio.go",
],
data = [
@@ -18,6 +19,7 @@
importpath = "source.monogon.dev/metropolis/test/launch/cluster",
visibility = ["//visibility:public"],
deps = [
+ "//metropolis/cli/metroctl/core",
"//metropolis/cli/pkg/datafile",
"//metropolis/node",
"//metropolis/node/core/identity",
@@ -28,6 +30,7 @@
"//metropolis/proto/common",
"//metropolis/test/launch",
"@com_github_cenkalti_backoff_v4//:backoff",
+ "@com_github_kballard_go_shellquote//:go-shellquote",
"@org_golang_google_grpc//:go_default_library",
"@org_golang_google_grpc//codes",
"@org_golang_google_grpc//status",
diff --git a/metropolis/test/launch/cluster/cluster.go b/metropolis/test/launch/cluster/cluster.go
index a0ad82f..5ea25b5 100644
--- a/metropolis/test/launch/cluster/cluster.go
+++ b/metropolis/test/launch/cluster/cluster.go
@@ -7,6 +7,7 @@
import (
"bytes"
"context"
+ "crypto/ed25519"
"crypto/rand"
"crypto/tls"
"errors"
@@ -28,6 +29,7 @@
"google.golang.org/grpc/status"
"google.golang.org/protobuf/proto"
+ metroctl "source.monogon.dev/metropolis/cli/metroctl/core"
"source.monogon.dev/metropolis/cli/pkg/datafile"
"source.monogon.dev/metropolis/node"
"source.monogon.dev/metropolis/node/core/identity"
@@ -508,7 +510,8 @@
// used to facilitate communication between QEMU and swtpm. It's different
// from launchDir, and anchored nearer the file system root, due to the
// socket path length limitation imposed by the kernel.
- socketDir string
+ socketDir string
+ metroctlDir string
// socksDialer is used by DialNode to establish connections to nodes via the
// SOCKS server ran by nanoswitch.
@@ -624,12 +627,19 @@
}
// Create the launch directory.
- ld, err := os.MkdirTemp(os.Getenv("TEST_TMPDIR"), "cluster*")
+ ld, err := os.MkdirTemp(os.Getenv("TEST_TMPDIR"), "cluster-*")
if err != nil {
return nil, fmt.Errorf("failed to create the launch directory: %w", err)
}
- // Create the socket directory.
- sd, err := os.MkdirTemp("/tmp", "cluster*")
+ // Create the metroctl config directory. We keep it in /tmp because in some
+ // scenarios it's end-user visible and we want it short.
+ md, err := os.MkdirTemp("/tmp", "metroctl-*")
+ if err != nil {
+ return nil, fmt.Errorf("failed to create the metroctl directory: %w", err)
+ }
+
+ // Create the socket directory. We keep it in /tmp because of socket path limits.
+ sd, err := os.MkdirTemp("/tmp", "cluster-*")
if err != nil {
return nil, fmt.Errorf("failed to create the socket directory: %w", err)
}
@@ -722,6 +732,16 @@
return nil, err
}
+ // Write credentials to the metroctl directory.
+ if err := metroctl.WriteOwnerKey(md, cert.PrivateKey.(ed25519.PrivateKey)); err != nil {
+ ctxC()
+ return nil, fmt.Errorf("could not write owner key: %w", err)
+ }
+ if err := metroctl.WriteOwnerCertificate(md, cert.Certificate[0]); err != nil {
+ ctxC()
+ return nil, fmt.Errorf("could not write owner certificate: %w", err)
+ }
+
// Set up a partially initialized cluster instance, to be filled in in the
// later steps.
cluster := &Cluster{
@@ -734,10 +754,11 @@
firstNode.ID,
},
- nodesDone: done,
- nodeOpts: nodeOpts,
- launchDir: ld,
- socketDir: sd,
+ nodesDone: done,
+ nodeOpts: nodeOpts,
+ launchDir: ld,
+ socketDir: sd,
+ metroctlDir: md,
socksDialer: socksDialer,
@@ -970,6 +991,7 @@
launch.Log("Cluster: removing nodes' state files.")
os.RemoveAll(c.launchDir)
os.RemoveAll(c.socketDir)
+ os.RemoveAll(c.metroctlDir)
launch.Log("Cluster: done")
return multierr.Combine(errs...)
}
@@ -999,3 +1021,37 @@
addr = net.JoinHostPort(node.ManagementAddress, port)
return c.socksDialer.Dial("tcp", addr)
}
+
+// KubernetesControllerNodeAddresses returns the list of IP addresses of nodes
+// which are currently Kubernetes controllers, ie. run an apiserver. This list
+// might be empty if no node is currently configured with the
+// 'KubernetesController' node.
+func (c *Cluster) KubernetesControllerNodeAddresses(ctx context.Context) ([]string, error) {
+ curC, err := c.CuratorClient()
+ if err != nil {
+ return nil, err
+ }
+ mgmt := apb.NewManagementClient(curC)
+ srv, err := mgmt.GetNodes(ctx, &apb.GetNodesRequest{
+ Filter: "has(node.roles.kubernetes_controller)",
+ })
+ if err != nil {
+ return nil, err
+ }
+ defer srv.CloseSend()
+ var res []string
+ for {
+ n, err := srv.Recv()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return nil, err
+ }
+ if n.Status == nil || n.Status.ExternalAddress == "" {
+ continue
+ }
+ res = append(res, n.Status.ExternalAddress)
+ }
+ return res, nil
+}
diff --git a/metropolis/test/launch/cluster/metroctl.go b/metropolis/test/launch/cluster/metroctl.go
new file mode 100644
index 0000000..da77fe5
--- /dev/null
+++ b/metropolis/test/launch/cluster/metroctl.go
@@ -0,0 +1,73 @@
+package cluster
+
+import (
+ "fmt"
+ "net"
+ "os"
+ "path"
+ "sort"
+
+ "github.com/kballard/go-shellquote"
+
+ metroctl "source.monogon.dev/metropolis/cli/metroctl/core"
+ "source.monogon.dev/metropolis/cli/pkg/datafile"
+)
+
+const metroctlRunfile = "metropolis/cli/metroctl/metroctl_/metroctl"
+
+// MetroctlRunfilePath returns the absolute path to the metroctl binary available
+// if the built target depends on //metropolis/cli/metroctl. Otherwise, an error
+// is returned.
+func MetroctlRunfilePath() (string, error) {
+ path, err := datafile.ResolveRunfile(metroctlRunfile)
+ if err != nil {
+ return "", fmt.Errorf("//metropolis/cli/metroctl not found in runfiles, did you include it as a data dependency? error: %w", err)
+ }
+ return path, nil
+}
+
+// ConnectOptions returns metroctl.ConnectOptions that describe connectivity to
+// the launched cluster.
+func (c *Cluster) ConnectOptions() *metroctl.ConnectOptions {
+ // Use all metropolis nodes as endpoints. That's fine, metroctl's resolver will
+ // figure out what to actually use.
+ var endpoints []string
+ for _, n := range c.Nodes {
+ endpoints = append(endpoints, n.ManagementAddress)
+ }
+ sort.Strings(endpoints)
+ return &metroctl.ConnectOptions{
+ ConfigPath: c.metroctlDir,
+ ProxyServer: net.JoinHostPort("127.0.0.1", fmt.Sprintf("%d", c.Ports[SOCKSPort])),
+ Endpoints: endpoints,
+ }
+}
+
+// MetroctlFlags return stringified flags to pass to a metroctl binary to connect
+// to the launched cluster.
+func (c *Cluster) MetroctlFlags() string {
+ return shellquote.Join(c.ConnectOptions().ToFlags()...)
+}
+
+// MakeMetroctlWrapper builds and returns the path to a shell script which calls
+// metroctl (from //metropolis/cli/metroctl, which must be included as a data
+// dependency of the built target) with all the required flags to connect to the
+// launched cluster.
+func (c *Cluster) MakeMetroctlWrapper() (string, error) {
+ mpath, err := MetroctlRunfilePath()
+ if err != nil {
+ return "", err
+ }
+ wpath := path.Join(c.metroctlDir, "metroctl.sh")
+
+ // Don't create wrapper if it already exists.
+ if _, err := os.Stat(wpath); err == nil {
+ return wpath, nil
+ }
+
+ wrapper := fmt.Sprintf("#!/usr/bin/env bash\nexec %s %s \"$@\"", mpath, c.MetroctlFlags())
+ if err := os.WriteFile(wpath, []byte(wrapper), 0555); err != nil {
+ return "", fmt.Errorf("could not write wrapper: %w", err)
+ }
+ return wpath, nil
+}