m/n/kubernetes: introduce feature gate infra
This introduces centralized infrastructure to control feature gates in K8s.
It includes a test to make sure that we do not keep outdated flags in there.
Change-Id: Ife251cbd5210bc8b3757bb3829e91bcdb2e6fdfb
Reviewed-on: https://review.monogon.dev/c/monogon/+/3664
Reviewed-by: Tim Windelschmidt <tim@monogon.tech>
Tested-by: Jenkins CI
diff --git a/metropolis/node/kubernetes/BUILD.bazel b/metropolis/node/kubernetes/BUILD.bazel
index 787a9c7..9f51ba0 100644
--- a/metropolis/node/kubernetes/BUILD.bazel
+++ b/metropolis/node/kubernetes/BUILD.bazel
@@ -1,4 +1,4 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "kubernetes",
@@ -7,6 +7,7 @@
"apiserver.go",
"controller-manager.go",
"csi.go",
+ "feature_gates.go",
"kubelet.go",
"labelmaker.go",
"provisioner.go",
@@ -62,6 +63,7 @@
"@io_k8s_client_go//tools/record",
"@io_k8s_client_go//tools/reference",
"@io_k8s_client_go//util/workqueue",
+ "@io_k8s_component_base//featuregate",
"@io_k8s_kubelet//config/v1beta1",
"@io_k8s_kubelet//pkg/apis/pluginregistration/v1:pluginregistration",
"@io_k8s_kubernetes//plugin/pkg/admission/security/podsecurity",
@@ -73,3 +75,13 @@
"@org_golang_x_sys//unix",
],
)
+
+go_test(
+ name = "kubernetes_test",
+ srcs = ["feature_gates_test.go"],
+ embed = [":kubernetes"],
+ deps = [
+ "@io_k8s_apiserver//pkg/util/feature",
+ "@io_k8s_component_base//featuregate",
+ ],
+)
diff --git a/metropolis/node/kubernetes/apiserver.go b/metropolis/node/kubernetes/apiserver.go
index e4df4a9..476c8bf 100644
--- a/metropolis/node/kubernetes/apiserver.go
+++ b/metropolis/node/kubernetes/apiserver.go
@@ -186,6 +186,7 @@
pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: s.serverKey})),
args.FileOpt("--admission-control-config-file", "admission-control.json", admissionConfigRaw),
"--allow-privileged=true",
+ extraFeatureGates.AsFlag(),
)
if args.Error() != nil {
return err
diff --git a/metropolis/node/kubernetes/controller-manager.go b/metropolis/node/kubernetes/controller-manager.go
index 0a49ce1..b1fdfc6 100644
--- a/metropolis/node/kubernetes/controller-manager.go
+++ b/metropolis/node/kubernetes/controller-manager.go
@@ -91,6 +91,7 @@
// This is intentionally empty, but if unset it tries to mkdir it
// in the usual place, generating an error.
"--flex-volume-plugin-dir=/kubernetes/conf/flexvolume-plugins",
+ extraFeatureGates.AsFlag(),
)
if args.Error() != nil {
diff --git a/metropolis/node/kubernetes/feature_gates.go b/metropolis/node/kubernetes/feature_gates.go
new file mode 100644
index 0000000..06d970f
--- /dev/null
+++ b/metropolis/node/kubernetes/feature_gates.go
@@ -0,0 +1,35 @@
+package kubernetes
+
+import (
+ "fmt"
+ "strings"
+
+ "k8s.io/component-base/featuregate"
+)
+
+type featureGates map[featuregate.Feature]bool
+
+// AsFlag returns the feature gates as a --feature-gate flag.
+func (fgs featureGates) AsFlag() string {
+ var strb strings.Builder
+ strb.WriteString("--feature-gates=")
+ i := 0
+ for f, en := range fgs {
+ fmt.Fprintf(&strb, "%s=%v", string(f), en)
+ if i++; i != len(fgs) {
+ strb.WriteByte(',')
+ }
+ }
+ return strb.String()
+}
+
+// AsConfigObject returns the feature gates as a plain map for K8s configs.
+func (fgs featureGates) AsMap() map[string]bool {
+ out := make(map[string]bool)
+ for f, en := range fgs {
+ out[string(f)] = en
+ }
+ return out
+}
+
+var extraFeatureGates = featureGates{}
diff --git a/metropolis/node/kubernetes/feature_gates_test.go b/metropolis/node/kubernetes/feature_gates_test.go
new file mode 100644
index 0000000..3cccd14
--- /dev/null
+++ b/metropolis/node/kubernetes/feature_gates_test.go
@@ -0,0 +1,35 @@
+package kubernetes
+
+import (
+ "testing"
+
+ utilfeature "k8s.io/apiserver/pkg/util/feature"
+ "k8s.io/component-base/featuregate"
+)
+
+func TestFeatureGateDefaults(t *testing.T) {
+ for f, en := range extraFeatureGates {
+ if utilfeature.DefaultFeatureGate.Enabled(f) == en {
+ t.Errorf("Feature gate %q is already %v by default, remove it from extraFeatureGates", string(f), en)
+ }
+ }
+}
+
+func TestAsFlags(t *testing.T) {
+ for _, c := range []struct {
+ name string
+ fg featureGates
+ expected string
+ }{
+ {"None", featureGates{}, "--feature-gates="},
+ {"Single", featureGates{featuregate.Feature("Test"): true}, "--feature-gates=Test=true"},
+ {"Multiple", featureGates{featuregate.Feature("Test"): true, featuregate.Feature("Test2"): false}, "--feature-gates=Test=true,Test2=false"},
+ } {
+ t.Run(c.name, func(t *testing.T) {
+ got := c.fg.AsFlag()
+ if got != c.expected {
+ t.Errorf("Expected %q, got %q", c.expected, got)
+ }
+ })
+ }
+}
diff --git a/metropolis/node/kubernetes/kubelet.go b/metropolis/node/kubernetes/kubelet.go
index 2c46080..9845b3b 100644
--- a/metropolis/node/kubernetes/kubelet.go
+++ b/metropolis/node/kubernetes/kubelet.go
@@ -118,8 +118,9 @@
VolumePluginDir: s.EphemeralDirectory.FlexvolumePlugins.FullPath(),
// Currently we allocate a /24 per node, so we can have a maximum of
// 253 pods per node.
- MaxPods: 253,
- PodLogsDir: "/data/kubelet/logs",
+ MaxPods: 253,
+ PodLogsDir: "/data/kubelet/logs",
+ FeatureGates: extraFeatureGates.AsMap(),
}
}
diff --git a/metropolis/node/kubernetes/scheduler.go b/metropolis/node/kubernetes/scheduler.go
index cfa338a..d058970 100644
--- a/metropolis/node/kubernetes/scheduler.go
+++ b/metropolis/node/kubernetes/scheduler.go
@@ -67,6 +67,7 @@
pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: config.serverKey})),
args.FileOpt("--client-ca-file", "root-ca.pem",
pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: config.rootCA})),
+ extraFeatureGates.AsFlag(),
)
if args.Error() != nil {
return fmt.Errorf("failed to use fileargs: %w", err)