m/n/k/reconciler: implement leader election
Before this change, the reconciler runs on all Kubernetes controllers.
When we are in a rolling upgrade of the cluster where a reconciled
object changes, this will cause the old and new versions of the
reconciler to fight each other, constantly updating the object back and
forth.
Now, the reconciler is elected among nodes of the latest release. The
status of the reconciliation is communicated to all Kubernetes
controllers through a new key-value in etcd.
Additionally, compatibility constraints can be expressed by changing the
constants minReconcilerRelease and minApiserverRelease, allowing
reconciliation to happen in a controlled way that ensures compatibility
even during rolling upgrades.
Change-Id: Iaf7c27702bd9809a13d47bcf041b71438353bef2
Reviewed-on: https://review.monogon.dev/c/monogon/+/3062
Tested-by: Jenkins CI
Reviewed-by: Serge Bazanski <serge@monogon.tech>
diff --git a/metropolis/node/kubernetes/reconciler/BUILD.bazel b/metropolis/node/kubernetes/reconciler/BUILD.bazel
index 306c273..caa239a 100644
--- a/metropolis/node/kubernetes/reconciler/BUILD.bazel
+++ b/metropolis/node/kubernetes/reconciler/BUILD.bazel
@@ -4,6 +4,7 @@
name = "reconciler",
srcs = [
"reconciler.go",
+ "reconciler_status.go",
"resources_csi.go",
"resources_rbac.go",
"resources_runtimeclass.go",
@@ -12,19 +13,48 @@
importpath = "source.monogon.dev/metropolis/node/kubernetes/reconciler",
visibility = ["//metropolis/node:__subpackages__"],
deps = [
+ "//metropolis/node/core/consensus/client",
+ "//metropolis/node/core/curator",
+ "//metropolis/node/core/curator/proto/private",
+ "//metropolis/pkg/event/etcd",
+ "//metropolis/pkg/event/memory",
"//metropolis/pkg/supervisor",
+ "//metropolis/version",
+ "//version",
+ "//version/spec",
+ "@com_github_cenkalti_backoff_v4//:backoff",
+ "@io_etcd_go_etcd_api_v3//mvccpb",
+ "@io_etcd_go_etcd_client_v3//:client",
+ "@io_etcd_go_etcd_client_v3//concurrency",
"@io_k8s_api//core/v1:core",
"@io_k8s_api//node/v1:node",
"@io_k8s_api//rbac/v1:rbac",
"@io_k8s_api//storage/v1:storage",
"@io_k8s_apimachinery//pkg/apis/meta/v1:meta",
"@io_k8s_client_go//kubernetes",
+ "@org_golang_google_protobuf//proto",
],
)
go_test(
name = "reconciler_test",
- srcs = ["reconciler_test.go"],
+ srcs = [
+ "reconciler_status_test.go",
+ "reconciler_test.go",
+ ],
embed = [":reconciler"],
- deps = ["@io_k8s_apimachinery//pkg/apis/meta/v1:meta"],
+ deps = [
+ "//metropolis/node/core/consensus/client",
+ "//metropolis/node/core/curator",
+ "//metropolis/node/core/curator/proto/private",
+ "//metropolis/pkg/supervisor",
+ "//metropolis/proto/common",
+ "//metropolis/version",
+ "//version",
+ "//version/spec",
+ "@io_etcd_go_etcd_tests_v3//integration",
+ "@io_k8s_apimachinery//pkg/apis/meta/v1:meta",
+ "@io_k8s_client_go//kubernetes/fake",
+ "@org_golang_google_protobuf//proto",
+ ],
)