m/n/k/reconciler: implement leader election
Before this change, the reconciler runs on all Kubernetes controllers.
When we are in a rolling upgrade of the cluster where a reconciled
object changes, this will cause the old and new versions of the
reconciler to fight each other, constantly updating the object back and
forth.
Now, the reconciler is elected among nodes of the latest release. The
status of the reconciliation is communicated to all Kubernetes
controllers through a new key-value in etcd.
Additionally, compatibility constraints can be expressed by changing the
constants minReconcilerRelease and minApiserverRelease, allowing
reconciliation to happen in a controlled way that ensures compatibility
even during rolling upgrades.
Change-Id: Iaf7c27702bd9809a13d47bcf041b71438353bef2
Reviewed-on: https://review.monogon.dev/c/monogon/+/3062
Tested-by: Jenkins CI
Reviewed-by: Serge Bazanski <serge@monogon.tech>
diff --git a/metropolis/node/kubernetes/reconciler/reconciler.go b/metropolis/node/kubernetes/reconciler/reconciler.go
index 2d2bfda..4ea2d84 100644
--- a/metropolis/node/kubernetes/reconciler/reconciler.go
+++ b/metropolis/node/kubernetes/reconciler/reconciler.go
@@ -29,12 +29,9 @@
import (
"context"
"fmt"
- "time"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
-
- "source.monogon.dev/metropolis/pkg/supervisor"
)
// True is a sad workaround for all the pointer booleans in K8s specs
@@ -120,7 +117,7 @@
}
}
-func ReconcileAll(ctx context.Context, clientSet kubernetes.Interface) error {
+func reconcileAll(ctx context.Context, clientSet kubernetes.Interface) error {
resources := allResources(clientSet)
for name, resource := range resources {
err := reconcile(ctx, resource)
@@ -131,26 +128,6 @@
return nil
}
-func Maintain(clientSet kubernetes.Interface) supervisor.Runnable {
- return func(ctx context.Context) error {
- log := supervisor.Logger(ctx)
- supervisor.Signal(ctx, supervisor.SignalHealthy)
- t := time.NewTicker(10 * time.Second)
- defer t.Stop()
- for {
- select {
- case <-t.C:
- err := ReconcileAll(ctx, clientSet)
- if err != nil {
- log.Warning(err)
- }
- case <-ctx.Done():
- return nil
- }
- }
- }
-}
-
func reconcile(ctx context.Context, r resource) error {
present, err := r.List(ctx)
if err != nil {