m/n/kubernetes: implement Metropolis authenticating proxy
This implements an authenticating proxy for K8s which can authenticate
Metropolis credentials and passes the extracted identity information
back to the Kubernetes API server. It currently only handles user
authentication, machine-to-machine authentication is still done by the
API server itself. It also adds a role binding to allow full access
to the owner as we do not have an identity system yet.
Change-Id: I02043924bb7ce7a1acdb826dad2d27a4c2008136
Reviewed-on: https://review.monogon.dev/c/monogon/+/509
Reviewed-by: Sergiusz Bazanski <serge@monogon.tech>
diff --git a/metropolis/node/kubernetes/BUILD.bazel b/metropolis/node/kubernetes/BUILD.bazel
index a162bcc..2a35c4b 100644
--- a/metropolis/node/kubernetes/BUILD.bazel
+++ b/metropolis/node/kubernetes/BUILD.bazel
@@ -19,6 +19,7 @@
"//metropolis/node/core/localstorage:go_default_library",
"//metropolis/node/core/network:go_default_library",
"//metropolis/node/core/network/dns:go_default_library",
+ "//metropolis/node/kubernetes/authproxy:go_default_library",
"//metropolis/node/kubernetes/clusternet:go_default_library",
"//metropolis/node/kubernetes/nfproxy:go_default_library",
"//metropolis/node/kubernetes/pki:go_default_library",
diff --git a/metropolis/node/kubernetes/apiserver.go b/metropolis/node/kubernetes/apiserver.go
index 39105b2..cd4ff60 100644
--- a/metropolis/node/kubernetes/apiserver.go
+++ b/metropolis/node/kubernetes/apiserver.go
@@ -109,7 +109,7 @@
pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: s.aggregationClientCert})),
args.FileOpt("--proxy-client-key-file", "aggregation-client-key.pem",
pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: s.aggregationClientKey})),
- "--requestheader-allowed-names=front-proxy-client",
+ "--requestheader-allowed-names=front-proxy-client,metropolis-auth-proxy-client",
args.FileOpt("--requestheader-client-ca-file", "aggregation-ca.pem",
pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: s.aggregationCA})),
"--requestheader-extra-headers-prefix=X-Remote-Extra-",
diff --git a/metropolis/node/kubernetes/authproxy/BUILD.bazel b/metropolis/node/kubernetes/authproxy/BUILD.bazel
new file mode 100644
index 0000000..965e8ad
--- /dev/null
+++ b/metropolis/node/kubernetes/authproxy/BUILD.bazel
@@ -0,0 +1,15 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+ name = "go_default_library",
+ srcs = ["authproxy.go"],
+ importpath = "source.monogon.dev/metropolis/node/kubernetes/authproxy",
+ visibility = ["//visibility:public"],
+ deps = [
+ "//metropolis/node:go_default_library",
+ "//metropolis/node/core/identity:go_default_library",
+ "//metropolis/node/kubernetes/pki:go_default_library",
+ "//metropolis/pkg/supervisor:go_default_library",
+ "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
+ ],
+)
diff --git a/metropolis/node/kubernetes/authproxy/authproxy.go b/metropolis/node/kubernetes/authproxy/authproxy.go
new file mode 100644
index 0000000..0477b88
--- /dev/null
+++ b/metropolis/node/kubernetes/authproxy/authproxy.go
@@ -0,0 +1,151 @@
+// Package authproxy implements an authenticating proxy in front of the K8s
+// API server converting Metropolis credentials into authentication headers.
+package authproxy
+
+import (
+ "context"
+ "crypto/tls"
+ "crypto/x509"
+ "encoding/json"
+ "fmt"
+ "net"
+ "net/http"
+ "net/http/httputil"
+ "net/url"
+ "strings"
+ "time"
+
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+ "source.monogon.dev/metropolis/node"
+ "source.monogon.dev/metropolis/node/core/identity"
+ "source.monogon.dev/metropolis/node/kubernetes/pki"
+ "source.monogon.dev/metropolis/pkg/supervisor"
+)
+
+type Service struct {
+ // KPKI is a reference to the Kubernetes PKI
+ KPKI *pki.PKI
+ // Node contains the node identity
+ Node *identity.Node
+}
+
+func (s *Service) getTLSCert(ctx context.Context, name pki.KubeCertificateName) (*tls.Certificate, error) {
+ cert, key, err := s.KPKI.Certificate(ctx, name)
+ if err != nil {
+ return nil, fmt.Errorf("could not load certificate %q from PKI: %w", name, err)
+ }
+ parsedKey, err := x509.ParsePKCS8PrivateKey(key)
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse key for cert %q: %w", name, err)
+ }
+ return &tls.Certificate{
+ Certificate: [][]byte{cert},
+ PrivateKey: parsedKey,
+ }, nil
+}
+
+func respondWithK8sStatus(w http.ResponseWriter, status *metav1.Status) error {
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(int(status.Code))
+ return json.NewEncoder(w).Encode(status)
+}
+
+func (s *Service) Run(ctx context.Context) error {
+ logger := supervisor.Logger(ctx)
+
+ k8sCAs := x509.NewCertPool()
+ cert, _, err := s.KPKI.Certificate(ctx, pki.IdCA)
+ parsedCert, err := x509.ParseCertificate(cert)
+ if err != nil {
+ return fmt.Errorf("failed to parse K8s CA certificate: %w", err)
+ }
+ k8sCAs.AddCert(parsedCert)
+
+ clientCert, err := s.getTLSCert(ctx, pki.MetropolisAuthProxyClient)
+ if err != nil {
+ return err
+ }
+
+ proxy := httputil.NewSingleHostReverseProxy(&url.URL{
+ Host: net.JoinHostPort("localhost", node.KubernetesAPIPort.PortString()),
+ Scheme: "https",
+ })
+ proxy.Transport = &http.Transport{
+ DialContext: (&net.Dialer{
+ Timeout: 30 * time.Second,
+ KeepAlive: 30 * time.Second,
+ }).DialContext,
+ TLSClientConfig: &tls.Config{
+ RootCAs: k8sCAs,
+ Certificates: []tls.Certificate{*clientCert},
+ NextProtos: []string{"h2", "http/1.1"},
+ },
+ ForceAttemptHTTP2: true,
+ MaxIdleConns: 100,
+ IdleConnTimeout: 90 * time.Second,
+ TLSHandshakeTimeout: 10 * time.Second,
+ ExpectContinueTimeout: 1 * time.Second,
+ }
+ proxy.ErrorHandler = func(w http.ResponseWriter, req *http.Request, err error) {
+ logger.Infof("Proxy error: %v", err)
+ respondWithK8sStatus(w, &metav1.Status{
+ Status: metav1.StatusFailure,
+ Code: http.StatusBadGateway,
+ Reason: metav1.StatusReasonServiceUnavailable,
+ Message: "authproxy could not reach apiserver",
+ })
+ }
+
+ serverCert, err := s.getTLSCert(ctx, pki.APIServer)
+ if err != nil {
+ return err
+ }
+ clientCAs := x509.NewCertPool()
+ clientCAs.AddCert(s.Node.ClusterCA())
+ server := &http.Server{
+ Addr: ":" + node.KubernetesAPIWrappedPort.PortString(),
+ TLSConfig: &tls.Config{
+ MinVersion: tls.VersionTLS12,
+ NextProtos: []string{"h2", "http/1.1"},
+ ClientAuth: tls.RequireAndVerifyClientCert,
+ ClientCAs: clientCAs,
+ Certificates: []tls.Certificate{*serverCert},
+ },
+ // Limits match @io_k8s_apiserver/pkg/server:secure_serving.go Serve()
+ MaxHeaderBytes: 1 << 20,
+ IdleTimeout: 90 * time.Second,
+ ReadHeaderTimeout: 32 * time.Second,
+
+ Handler: http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+ // Guaranteed to exist because of RequireAndVerifyClientCert
+ clientCert := req.TLS.VerifiedChains[0][0]
+ clientIdentity, err := identity.VerifyUserInCluster(clientCert, s.Node.ClusterCA())
+ if err != nil {
+ respondWithK8sStatus(rw, &metav1.Status{
+ Status: metav1.StatusFailure,
+ Code: http.StatusUnauthorized,
+ Reason: metav1.StatusReasonUnauthorized,
+ Message: fmt.Sprintf("Metropolis authentication failed: %v", err),
+ })
+ return
+ }
+ // Clone the request as otherwise modifying it is not allowed
+ newReq := req.Clone(req.Context())
+ // Drop any X-Remote headers to prevent injection
+ for k := range newReq.Header {
+ if strings.HasPrefix(http.CanonicalHeaderKey(k), http.CanonicalHeaderKey("X-Remote-")) {
+ newReq.Header.Del(k)
+ }
+ }
+ newReq.Header.Set("X-Remote-User", clientIdentity)
+ newReq.Header.Set("X-Remote-Group", "")
+
+ proxy.ServeHTTP(rw, newReq)
+ }),
+ }
+ go server.ListenAndServeTLS("", "")
+ logger.Info("K8s AuthProxy running")
+ <-ctx.Done()
+ return server.Close()
+}
diff --git a/metropolis/node/kubernetes/pki/kubernetes.go b/metropolis/node/kubernetes/pki/kubernetes.go
index 0c795f2..542c614 100644
--- a/metropolis/node/kubernetes/pki/kubernetes.go
+++ b/metropolis/node/kubernetes/pki/kubernetes.go
@@ -76,6 +76,9 @@
// https://kubernetes.io/docs/tasks/extend-kubernetes/configure-aggregation-layer/#ca-reusage-and-conflicts
AggregationCA KubeCertificateName = "aggregation-ca"
FrontProxyClient KubeCertificateName = "front-proxy-client"
+ // The Metropolis authentication proxy needs to be able to proxy requests
+ // and assert the established identity to the Kubernetes API server.
+ MetropolisAuthProxyClient KubeCertificateName = "metropolis-auth-proxy-client"
)
const (
@@ -149,6 +152,7 @@
Mode: opki.CertificateManaged,
}
make(AggregationCA, FrontProxyClient, opki.Client("front-proxy-client", nil))
+ make(AggregationCA, MetropolisAuthProxyClient, opki.Client("metropolis-auth-proxy-client", nil))
return &pki
}
diff --git a/metropolis/node/kubernetes/reconciler/resources_rbac.go b/metropolis/node/kubernetes/reconciler/resources_rbac.go
index 15386a6..0976ba5 100644
--- a/metropolis/node/kubernetes/reconciler/resources_rbac.go
+++ b/metropolis/node/kubernetes/reconciler/resources_rbac.go
@@ -28,6 +28,7 @@
clusterRolePSPDefault = builtinRBACName("psp-default")
clusterRoleBindingDefaultPSP = builtinRBACName("default-psp-for-sa")
clusterRoleBindingAPIServerKubeletClient = builtinRBACName("apiserver-kubelet-client")
+ clusterRoleBindingOwnerAdmin = builtinRBACName("owner-admin")
)
type resourceClusterRoles struct {
@@ -150,5 +151,27 @@
},
},
},
+ clusterRoleBindingOwnerAdmin: &rbac.ClusterRoleBinding{
+ ObjectMeta: meta.ObjectMeta{
+ Name: clusterRoleBindingOwnerAdmin,
+ Labels: builtinLabels(nil),
+ Annotations: map[string]string{
+ "kubernetes.io/description": "This binding grants the Metropolis Cluster owner access to the " +
+ "cluster-admin role on Kubernetes.",
+ },
+ },
+ RoleRef: rbac.RoleRef{
+ APIGroup: rbac.GroupName,
+ Kind: "ClusterRole",
+ Name: "cluster-admin",
+ },
+ Subjects: []rbac.Subject{
+ {
+ APIGroup: rbac.GroupName,
+ Kind: "User",
+ Name: "owner",
+ },
+ },
+ },
}
}
diff --git a/metropolis/node/kubernetes/service.go b/metropolis/node/kubernetes/service.go
index 1af7607..03be33c 100644
--- a/metropolis/node/kubernetes/service.go
+++ b/metropolis/node/kubernetes/service.go
@@ -32,6 +32,7 @@
"source.monogon.dev/metropolis/node/core/localstorage"
"source.monogon.dev/metropolis/node/core/network"
"source.monogon.dev/metropolis/node/core/network/dns"
+ "source.monogon.dev/metropolis/node/kubernetes/authproxy"
"source.monogon.dev/metropolis/node/kubernetes/clusternet"
"source.monogon.dev/metropolis/node/kubernetes/nfproxy"
"source.monogon.dev/metropolis/node/kubernetes/pki"
@@ -173,6 +174,11 @@
KubeletDirectory: &s.c.Root.Data.Kubernetes.Kubelet,
}
+ authProxy := authproxy.Service{
+ KPKI: s.c.KPKI,
+ Node: s.c.Node,
+ }
+
for _, sub := range []struct {
name string
runnable supervisor.Runnable
@@ -185,6 +191,7 @@
{"clusternet", clusternet.Run},
{"nfproxy", nfproxy.Run},
{"kvmdeviceplugin", kvmDevicePlugin.Run},
+ {"authproxy", authProxy.Run},
} {
err := supervisor.Run(ctx, sub.name, sub.runnable)
if err != nil {
diff --git a/metropolis/node/ports.go b/metropolis/node/ports.go
index e7adfb7..c90e7dc 100644
--- a/metropolis/node/ports.go
+++ b/metropolis/node/ports.go
@@ -37,6 +37,9 @@
// KubernetesAPIPort is the TCP port on which the Kubernetes API is
// exposed.
KubernetesAPIPort Port = 6443
+ // KubernetesAPIWrappedPort is the TCP port on which the Metropolis
+ // authenticating proxy for the Kubernetes API is exposed.
+ KubernetesAPIWrappedPort Port = 6444
// DebuggerPort is the port on which the delve debugger runs (on debug
// builds only). Not to be confused with DebugServicePort.
DebuggerPort Port = 2345
@@ -54,6 +57,8 @@
return "wireguard"
case KubernetesAPIPort:
return "kubernetes-api"
+ case KubernetesAPIWrappedPort:
+ return "kubernetes-api-wrapped"
case DebuggerPort:
return "delve"
}
diff --git a/metropolis/test/e2e/BUILD.bazel b/metropolis/test/e2e/BUILD.bazel
index 7f8571f..6630db8 100644
--- a/metropolis/test/e2e/BUILD.bazel
+++ b/metropolis/test/e2e/BUILD.bazel
@@ -9,14 +9,14 @@
importpath = "source.monogon.dev/metropolis/test/e2e",
visibility = ["//metropolis/test:__subpackages__"],
deps = [
- "//metropolis/proto/api:go_default_library",
+ "//metropolis/test/launch/cluster:go_default_library",
"@io_k8s_api//apps/v1:go_default_library",
"@io_k8s_api//core/v1:go_default_library",
"@io_k8s_apimachinery//pkg/api/resource:go_default_library",
"@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
"@io_k8s_apimachinery//pkg/util/intstr:go_default_library",
"@io_k8s_client_go//kubernetes:go_default_library",
- "@io_k8s_client_go//tools/clientcmd:go_default_library",
+ "@io_k8s_client_go//rest:go_default_library",
],
)
diff --git a/metropolis/test/e2e/k8s_cts/main.go b/metropolis/test/e2e/k8s_cts/main.go
index 46b99b4..15be332 100644
--- a/metropolis/test/e2e/k8s_cts/main.go
+++ b/metropolis/test/e2e/k8s_cts/main.go
@@ -106,7 +106,7 @@
log.Fatalf("Failed to launch cluster: %v", err)
}
log.Println("Cluster initialized")
- clientSet, err := e2e.GetKubeClientSet(ctx, cl.Debug, cl.Ports[common.KubernetesAPIPort])
+ clientSet, err := e2e.GetKubeClientSet(cl, cl.Ports[common.KubernetesAPIWrappedPort])
if err != nil {
log.Fatalf("Failed to get clientSet: %v", err)
}
diff --git a/metropolis/test/e2e/kubernetes_helpers.go b/metropolis/test/e2e/kubernetes_helpers.go
index 3c1f95a..ec12ca0 100644
--- a/metropolis/test/e2e/kubernetes_helpers.go
+++ b/metropolis/test/e2e/kubernetes_helpers.go
@@ -17,10 +17,9 @@
package e2e
import (
- "context"
- "errors"
+ "crypto/x509"
+ "encoding/pem"
"fmt"
- "time"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
@@ -28,44 +27,31 @@
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/client-go/kubernetes"
- "k8s.io/client-go/tools/clientcmd"
+ "k8s.io/client-go/rest"
- apb "source.monogon.dev/metropolis/proto/api"
+ "source.monogon.dev/metropolis/test/launch/cluster"
)
-// GetKubeClientSet gets a Kubeconfig from the debug API and creates a K8s
-// ClientSet using it. The identity used has the system:masters group and thus
-// has RBAC access to everything.
-func GetKubeClientSet(ctx context.Context, client apb.NodeDebugServiceClient, port uint16) (kubernetes.Interface, error) {
- var lastErr = errors.New("context canceled before any operation completed")
- for {
- reqT, cancel := context.WithTimeout(ctx, 5*time.Second)
- defer cancel()
- res, err := client.GetDebugKubeconfig(reqT, &apb.GetDebugKubeconfigRequest{Id: "debug-user", Groups: []string{"system:masters"}})
- if err == nil {
- rawClientConfig, err := clientcmd.NewClientConfigFromBytes([]byte(res.DebugKubeconfig))
- if err != nil {
- return nil, err // Invalid Kubeconfigs are immediately fatal
- }
-
- clientConfig, err := rawClientConfig.ClientConfig()
- clientConfig.Host = fmt.Sprintf("localhost:%v", port)
- clientSet, err := kubernetes.NewForConfig(clientConfig)
- if err != nil {
- return nil, err
- }
- return clientSet, nil
- }
- if err != nil && err == ctx.Err() {
- return nil, lastErr
- }
- lastErr = err
- select {
- case <-ctx.Done():
- return nil, lastErr
- case <-time.After(1 * time.Second):
- }
+// GetKubeClientSet gets a Kubernetes client set accessing the Metropolis
+// Kubernetes authenticating proxy using the cluster owner identity.
+// It currently has access to everything (i.e. the cluster-admin role)
+// via the owner-admin binding.
+func GetKubeClientSet(cluster *cluster.Cluster, port uint16) (kubernetes.Interface, error) {
+ pkcs8Key, err := x509.MarshalPKCS8PrivateKey(cluster.Owner.PrivateKey)
+ if err != nil {
+ // We explicitly pass an Ed25519 private key in, so this can't happen
+ panic(err)
}
+ var clientConfig = rest.Config{
+ Host: fmt.Sprintf("localhost:%v", port),
+ TLSClientConfig: rest.TLSClientConfig{
+ ServerName: "kubernetes.default.svc.cluster.local",
+ Insecure: true,
+ CertData: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cluster.Owner.Certificate[0]}),
+ KeyData: pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8Key}),
+ },
+ }
+ return kubernetes.NewForConfig(&clientConfig)
}
// makeTestDeploymentSpec generates a Deployment spec for a single pod running
diff --git a/metropolis/test/e2e/main_test.go b/metropolis/test/e2e/main_test.go
index 72fa619..201ba92 100644
--- a/metropolis/test/e2e/main_test.go
+++ b/metropolis/test/e2e/main_test.go
@@ -114,11 +114,9 @@
return nil
})
})
- t.Run("Get Kubernetes Debug Kubeconfig", func(t *testing.T) {
+ t.Run("Get Kubernetes client", func(t *testing.T) {
t.Parallel()
- selfCtx, cancel := context.WithTimeout(ctx, largeTestTimeout)
- defer cancel()
- clientSet, err := GetKubeClientSet(selfCtx, cluster.Debug, cluster.Ports[common.KubernetesAPIPort])
+ clientSet, err := GetKubeClientSet(cluster, cluster.Ports[common.KubernetesAPIWrappedPort])
if err != nil {
t.Fatal(err)
}
diff --git a/metropolis/test/launch/cluster/cluster.go b/metropolis/test/launch/cluster/cluster.go
index 5b1f3dc..9b32f97 100644
--- a/metropolis/test/launch/cluster/cluster.go
+++ b/metropolis/test/launch/cluster/cluster.go
@@ -70,6 +70,7 @@
node.DebugServicePort,
node.KubernetesAPIPort,
+ node.KubernetesAPIWrappedPort,
node.CuratorServicePort,
node.DebuggerPort,
}
@@ -286,6 +287,7 @@
node.DebugServicePort,
node.KubernetesAPIPort,
+ node.KubernetesAPIWrappedPort,
}
// ClusterOptions contains all options for launching a Metropolis cluster.
diff --git a/metropolis/test/nanoswitch/nanoswitch.go b/metropolis/test/nanoswitch/nanoswitch.go
index 6d0b6bb..de04a42 100644
--- a/metropolis/test/nanoswitch/nanoswitch.go
+++ b/metropolis/test/nanoswitch/nanoswitch.go
@@ -311,6 +311,7 @@
supervisor.Run(ctx, "proxy-cur1", userspaceProxy(net.IPv4(10, 1, 0, 2), common.CuratorServicePort))
supervisor.Run(ctx, "proxy-dbg1", userspaceProxy(net.IPv4(10, 1, 0, 2), common.DebugServicePort))
supervisor.Run(ctx, "proxy-k8s-api1", userspaceProxy(net.IPv4(10, 1, 0, 2), common.KubernetesAPIPort))
+ supervisor.Run(ctx, "proxy-k8s-api-wrapped1", userspaceProxy(net.IPv4(10, 1, 0, 2), common.KubernetesAPIWrappedPort))
supervisor.Signal(ctx, supervisor.SignalHealthy)
supervisor.Signal(ctx, supervisor.SignalDone)
return nil