m/n/kubernetes: implement Metropolis authenticating proxy
This implements an authenticating proxy for K8s which can authenticate
Metropolis credentials and passes the extracted identity information
back to the Kubernetes API server. It currently only handles user
authentication, machine-to-machine authentication is still done by the
API server itself. It also adds a role binding to allow full access
to the owner as we do not have an identity system yet.
Change-Id: I02043924bb7ce7a1acdb826dad2d27a4c2008136
Reviewed-on: https://review.monogon.dev/c/monogon/+/509
Reviewed-by: Sergiusz Bazanski <serge@monogon.tech>
diff --git a/metropolis/test/e2e/BUILD.bazel b/metropolis/test/e2e/BUILD.bazel
index 7f8571f..6630db8 100644
--- a/metropolis/test/e2e/BUILD.bazel
+++ b/metropolis/test/e2e/BUILD.bazel
@@ -9,14 +9,14 @@
importpath = "source.monogon.dev/metropolis/test/e2e",
visibility = ["//metropolis/test:__subpackages__"],
deps = [
- "//metropolis/proto/api:go_default_library",
+ "//metropolis/test/launch/cluster:go_default_library",
"@io_k8s_api//apps/v1:go_default_library",
"@io_k8s_api//core/v1:go_default_library",
"@io_k8s_apimachinery//pkg/api/resource:go_default_library",
"@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
"@io_k8s_apimachinery//pkg/util/intstr:go_default_library",
"@io_k8s_client_go//kubernetes:go_default_library",
- "@io_k8s_client_go//tools/clientcmd:go_default_library",
+ "@io_k8s_client_go//rest:go_default_library",
],
)
diff --git a/metropolis/test/e2e/k8s_cts/main.go b/metropolis/test/e2e/k8s_cts/main.go
index 46b99b4..15be332 100644
--- a/metropolis/test/e2e/k8s_cts/main.go
+++ b/metropolis/test/e2e/k8s_cts/main.go
@@ -106,7 +106,7 @@
log.Fatalf("Failed to launch cluster: %v", err)
}
log.Println("Cluster initialized")
- clientSet, err := e2e.GetKubeClientSet(ctx, cl.Debug, cl.Ports[common.KubernetesAPIPort])
+ clientSet, err := e2e.GetKubeClientSet(cl, cl.Ports[common.KubernetesAPIWrappedPort])
if err != nil {
log.Fatalf("Failed to get clientSet: %v", err)
}
diff --git a/metropolis/test/e2e/kubernetes_helpers.go b/metropolis/test/e2e/kubernetes_helpers.go
index 3c1f95a..ec12ca0 100644
--- a/metropolis/test/e2e/kubernetes_helpers.go
+++ b/metropolis/test/e2e/kubernetes_helpers.go
@@ -17,10 +17,9 @@
package e2e
import (
- "context"
- "errors"
+ "crypto/x509"
+ "encoding/pem"
"fmt"
- "time"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
@@ -28,44 +27,31 @@
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/client-go/kubernetes"
- "k8s.io/client-go/tools/clientcmd"
+ "k8s.io/client-go/rest"
- apb "source.monogon.dev/metropolis/proto/api"
+ "source.monogon.dev/metropolis/test/launch/cluster"
)
-// GetKubeClientSet gets a Kubeconfig from the debug API and creates a K8s
-// ClientSet using it. The identity used has the system:masters group and thus
-// has RBAC access to everything.
-func GetKubeClientSet(ctx context.Context, client apb.NodeDebugServiceClient, port uint16) (kubernetes.Interface, error) {
- var lastErr = errors.New("context canceled before any operation completed")
- for {
- reqT, cancel := context.WithTimeout(ctx, 5*time.Second)
- defer cancel()
- res, err := client.GetDebugKubeconfig(reqT, &apb.GetDebugKubeconfigRequest{Id: "debug-user", Groups: []string{"system:masters"}})
- if err == nil {
- rawClientConfig, err := clientcmd.NewClientConfigFromBytes([]byte(res.DebugKubeconfig))
- if err != nil {
- return nil, err // Invalid Kubeconfigs are immediately fatal
- }
-
- clientConfig, err := rawClientConfig.ClientConfig()
- clientConfig.Host = fmt.Sprintf("localhost:%v", port)
- clientSet, err := kubernetes.NewForConfig(clientConfig)
- if err != nil {
- return nil, err
- }
- return clientSet, nil
- }
- if err != nil && err == ctx.Err() {
- return nil, lastErr
- }
- lastErr = err
- select {
- case <-ctx.Done():
- return nil, lastErr
- case <-time.After(1 * time.Second):
- }
+// GetKubeClientSet gets a Kubernetes client set accessing the Metropolis
+// Kubernetes authenticating proxy using the cluster owner identity.
+// It currently has access to everything (i.e. the cluster-admin role)
+// via the owner-admin binding.
+func GetKubeClientSet(cluster *cluster.Cluster, port uint16) (kubernetes.Interface, error) {
+ pkcs8Key, err := x509.MarshalPKCS8PrivateKey(cluster.Owner.PrivateKey)
+ if err != nil {
+ // We explicitly pass an Ed25519 private key in, so this can't happen
+ panic(err)
}
+ var clientConfig = rest.Config{
+ Host: fmt.Sprintf("localhost:%v", port),
+ TLSClientConfig: rest.TLSClientConfig{
+ ServerName: "kubernetes.default.svc.cluster.local",
+ Insecure: true,
+ CertData: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cluster.Owner.Certificate[0]}),
+ KeyData: pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8Key}),
+ },
+ }
+ return kubernetes.NewForConfig(&clientConfig)
}
// makeTestDeploymentSpec generates a Deployment spec for a single pod running
diff --git a/metropolis/test/e2e/main_test.go b/metropolis/test/e2e/main_test.go
index 72fa619..201ba92 100644
--- a/metropolis/test/e2e/main_test.go
+++ b/metropolis/test/e2e/main_test.go
@@ -114,11 +114,9 @@
return nil
})
})
- t.Run("Get Kubernetes Debug Kubeconfig", func(t *testing.T) {
+ t.Run("Get Kubernetes client", func(t *testing.T) {
t.Parallel()
- selfCtx, cancel := context.WithTimeout(ctx, largeTestTimeout)
- defer cancel()
- clientSet, err := GetKubeClientSet(selfCtx, cluster.Debug, cluster.Ports[common.KubernetesAPIPort])
+ clientSet, err := GetKubeClientSet(cluster, cluster.Ports[common.KubernetesAPIWrappedPort])
if err != nil {
t.Fatal(err)
}
diff --git a/metropolis/test/launch/cluster/cluster.go b/metropolis/test/launch/cluster/cluster.go
index 5b1f3dc..9b32f97 100644
--- a/metropolis/test/launch/cluster/cluster.go
+++ b/metropolis/test/launch/cluster/cluster.go
@@ -70,6 +70,7 @@
node.DebugServicePort,
node.KubernetesAPIPort,
+ node.KubernetesAPIWrappedPort,
node.CuratorServicePort,
node.DebuggerPort,
}
@@ -286,6 +287,7 @@
node.DebugServicePort,
node.KubernetesAPIPort,
+ node.KubernetesAPIWrappedPort,
}
// ClusterOptions contains all options for launching a Metropolis cluster.
diff --git a/metropolis/test/nanoswitch/nanoswitch.go b/metropolis/test/nanoswitch/nanoswitch.go
index 6d0b6bb..de04a42 100644
--- a/metropolis/test/nanoswitch/nanoswitch.go
+++ b/metropolis/test/nanoswitch/nanoswitch.go
@@ -311,6 +311,7 @@
supervisor.Run(ctx, "proxy-cur1", userspaceProxy(net.IPv4(10, 1, 0, 2), common.CuratorServicePort))
supervisor.Run(ctx, "proxy-dbg1", userspaceProxy(net.IPv4(10, 1, 0, 2), common.DebugServicePort))
supervisor.Run(ctx, "proxy-k8s-api1", userspaceProxy(net.IPv4(10, 1, 0, 2), common.KubernetesAPIPort))
+ supervisor.Run(ctx, "proxy-k8s-api-wrapped1", userspaceProxy(net.IPv4(10, 1, 0, 2), common.KubernetesAPIWrappedPort))
supervisor.Signal(ctx, supervisor.SignalHealthy)
supervisor.Signal(ctx, supervisor.SignalDone)
return nil