m/test/e2e: Implement TestE2ECoreHA
This is a basic test which exercises a TPM-sealed three-node cluster by
performing a rolling restart of control plane nodes.
Change-Id: Ic6f46192d8ccba1ef7a767988cf5a216beb5a4c6
Reviewed-on: https://review.monogon.dev/c/monogon/+/2884
Reviewed-by: Lorenz Brun <lorenz@monogon.tech>
Tested-by: Jenkins CI
diff --git a/metropolis/test/launch/cluster/cluster.go b/metropolis/test/launch/cluster/cluster.go
index b1bb072..49c60dc 100644
--- a/metropolis/test/launch/cluster/cluster.go
+++ b/metropolis/test/launch/cluster/cluster.go
@@ -37,6 +37,7 @@
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
+ ipb "source.monogon.dev/metropolis/node/core/curator/proto/api"
apb "source.monogon.dev/metropolis/proto/api"
cpb "source.monogon.dev/metropolis/proto/common"
@@ -1310,3 +1311,60 @@
})
return err
}
+
+// MakeConsensusMember adds the ConsensusMember role to a node by ID.
+func (c *Cluster) MakeConsensusMember(ctx context.Context, id string) error {
+ curC, err := c.CuratorClient()
+ if err != nil {
+ return err
+ }
+ mgmt := apb.NewManagementClient(curC)
+ cur := ipb.NewCuratorClient(curC)
+
+ tr := true
+ launch.Log("Cluster: %s: adding ConsensusMember", id)
+ bo := backoff.NewExponentialBackOff()
+ bo.MaxElapsedTime = 10 * time.Second
+
+ backoff.Retry(func() error {
+ _, err = mgmt.UpdateNodeRoles(ctx, &apb.UpdateNodeRolesRequest{
+ Node: &apb.UpdateNodeRolesRequest_Id{
+ Id: id,
+ },
+ ConsensusMember: &tr,
+ })
+ if err != nil {
+ launch.Log("Cluster: %s: UpdateNodeRoles failed: %v", id, err)
+ }
+ return err
+ }, backoff.WithContext(bo, ctx))
+ if err != nil {
+ return err
+ }
+
+ launch.Log("Cluster: %s: waiting for learner/full members...", id)
+
+ learner := false
+ for {
+ res, err := cur.GetConsensusStatus(ctx, &ipb.GetConsensusStatusRequest{})
+ if err != nil {
+ return fmt.Errorf("GetConsensusStatus: %w", err)
+ }
+ for _, member := range res.EtcdMember {
+ if member.Id != id {
+ continue
+ }
+ switch member.Status {
+ case ipb.GetConsensusStatusResponse_EtcdMember_STATUS_LEARNER:
+ if !learner {
+ learner = true
+ launch.Log("Cluster: %s: became a learner, waiting for full member...", id)
+ }
+ case ipb.GetConsensusStatusResponse_EtcdMember_STATUS_FULL:
+ launch.Log("Cluster: %s: became a full member", id)
+ return nil
+ }
+ }
+ time.Sleep(100 * time.Millisecond)
+ }
+}