m/n/kubernetes: factor out cluster domain
This removes the hardcoded Kubernetes cluster domain and pushes it out
to a single place at the root of the Kubernetes supervisor tree.
This will later be aligned with the cluster domain specified in the
identity design document, currently this does not change any behavior.
It also removes a bogous SAN from the Kubernetes API server certificate
(kubernetes.default.svc.cluster) for which there is no corresponding
search path.
Change-Id: I30b8907a7b846415f5002c09a24d2d37930a9cd1
Reviewed-on: https://review.monogon.dev/c/monogon/+/773
Tested-by: Jenkins CI
Reviewed-by: Sergiusz Bazanski <serge@monogon.tech>
diff --git a/metropolis/node/core/roleserve/worker_kubernetes.go b/metropolis/node/core/roleserve/worker_kubernetes.go
index 06e6735..904e4f5 100644
--- a/metropolis/node/core/roleserve/worker_kubernetes.go
+++ b/metropolis/node/core/roleserve/worker_kubernetes.go
@@ -166,8 +166,12 @@
return fmt.Errorf("failed to start containerd service: %w", err)
}
+ // TODO(lorenz): Align this with the global cluster domain once it
+ // exists.
+ clusterDomain := "cluster.local"
+
// Start building Kubernetes service...
- pki := kpki.New(supervisor.Logger(ctx), kkv)
+ pki := kpki.New(supervisor.Logger(ctx), kkv, clusterDomain)
kubeSvc := kubernetes.New(kubernetes.Config{
Node: &d.membership.credentials.Node,
@@ -182,9 +186,10 @@
// That's a /16.
Mask: net.IPMask{0xff, 0xff, 0x00, 0x00},
},
- KPKI: pki,
- Root: s.storageRoot,
- Network: s.network,
+ ClusterDomain: clusterDomain,
+ KPKI: pki,
+ Root: s.storageRoot,
+ Network: s.network,
})
// Start Kubernetes.
if err := supervisor.Run(ctx, "kubernetes", kubeSvc.Run); err != nil {
diff --git a/metropolis/node/kubernetes/kubelet.go b/metropolis/node/kubernetes/kubelet.go
index d966e5d..31357ab 100644
--- a/metropolis/node/kubernetes/kubelet.go
+++ b/metropolis/node/kubernetes/kubelet.go
@@ -38,6 +38,7 @@
type kubeletService struct {
NodeName string
ClusterDNS []net.IP
+ ClusterDomain string
KubeletDirectory *localstorage.DataKubernetesKubeletDirectory
EphemeralDirectory *localstorage.EphemeralDirectory
Output io.Writer
@@ -92,7 +93,7 @@
},
},
// TODO(q3k): move reconciler.False to a generic package, fix the following references.
- ClusterDomain: "cluster.local", // cluster.local is hardcoded in the certificate too currently
+ ClusterDomain: s.ClusterDomain,
EnableControllerAttachDetach: reconciler.False(),
HairpinMode: "none",
MakeIPTablesUtilChains: reconciler.False(), // We don't have iptables
diff --git a/metropolis/node/kubernetes/pki/kubernetes.go b/metropolis/node/kubernetes/pki/kubernetes.go
index 1a14f99..ef046a2 100644
--- a/metropolis/node/kubernetes/pki/kubernetes.go
+++ b/metropolis/node/kubernetes/pki/kubernetes.go
@@ -100,7 +100,7 @@
Certificates map[KubeCertificateName]*opki.Certificate
}
-func New(l logtree.LeveledLogger, kv clientv3.KV) *PKI {
+func New(l logtree.LeveledLogger, kv clientv3.KV, clusterDomain string) *PKI {
pki := PKI{
namespace: opki.Namespaced(etcdPrefix),
logger: l,
@@ -130,8 +130,7 @@
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
- "kubernetes.default.svc.cluster",
- "kubernetes.default.svc.cluster.local",
+ "kubernetes.default.svc." + clusterDomain,
"localhost",
},
// TODO(q3k): add service network internal apiserver address
diff --git a/metropolis/node/kubernetes/service.go b/metropolis/node/kubernetes/service.go
index 03be33c..ff0f55c 100644
--- a/metropolis/node/kubernetes/service.go
+++ b/metropolis/node/kubernetes/service.go
@@ -45,6 +45,7 @@
type Config struct {
ServiceIPRange net.IPNet
ClusterNet net.IPNet
+ ClusterDomain string
KPKI *pki.PKI
Root *localstorage.Root
@@ -121,6 +122,7 @@
kubelet := kubeletService{
NodeName: s.c.Node.ID(),
ClusterDNS: []net.IP{address},
+ ClusterDomain: s.c.ClusterDomain,
KubeletDirectory: &s.c.Root.Data.Kubernetes.Kubelet,
EphemeralDirectory: &s.c.Root.Ephemeral,
KPKI: s.c.KPKI,
@@ -200,7 +202,7 @@
}
supervisor.Logger(ctx).Info("Registering K8s CoreDNS")
- clusterDNSDirective := dns.NewKubernetesDirective("cluster.local", masterKubeconfig)
+ clusterDNSDirective := dns.NewKubernetesDirective(s.c.ClusterDomain, masterKubeconfig)
s.c.Network.ConfigureDNS(clusterDNSDirective)
supervisor.Signal(ctx, supervisor.SignalHealthy)
diff --git a/metropolis/test/e2e/kubernetes_helpers.go b/metropolis/test/e2e/kubernetes_helpers.go
index 44fa660..2e53970 100644
--- a/metropolis/test/e2e/kubernetes_helpers.go
+++ b/metropolis/test/e2e/kubernetes_helpers.go
@@ -45,7 +45,7 @@
var clientConfig = rest.Config{
Host: fmt.Sprintf("localhost:%v", port),
TLSClientConfig: rest.TLSClientConfig{
- ServerName: "kubernetes.default.svc.cluster.local",
+ ServerName: "kubernetes.default.svc",
Insecure: true,
CertData: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cluster.Owner.Certificate[0]}),
KeyData: pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8Key}),