m/n/core: implement GetClusterInfo
This implements Management.GetClusterInfo which is used to retrieve a
ClusterDirectory. This in turn will be used by nodes that wish to
register into a cluster.
This could've been skipped and instead Curator.Watch could've been used.
However, the Curator service is only really (currently) intended to be
used by node-to-node communications. To keep with the current design, we
implement a separate RPC, but we should maybe reconsider if this
separation makes sense.
Change-Id: Ie9d475731f4faafdc51a2aa51a1582ee1a259fd2
Reviewed-on: https://review.monogon.dev/c/monogon/+/340
Reviewed-by: Lorenz Brun <lorenz@monogon.tech>
diff --git a/metropolis/node/core/curator/impl_leader_management.go b/metropolis/node/core/curator/impl_leader_management.go
index 0f6b55c..aa2d54c 100644
--- a/metropolis/node/core/curator/impl_leader_management.go
+++ b/metropolis/node/core/curator/impl_leader_management.go
@@ -1,8 +1,10 @@
package curator
import (
+ "bytes"
"context"
"crypto/rand"
+ "sort"
"go.etcd.io/etcd/clientv3"
"google.golang.org/grpc/codes"
@@ -11,6 +13,7 @@
ppb "source.monogon.dev/metropolis/node/core/curator/proto/private"
apb "source.monogon.dev/metropolis/proto/api"
+ cpb "source.monogon.dev/metropolis/proto/common"
)
type leaderManagement struct {
@@ -74,3 +77,58 @@
Ticket: ticketBytes,
}, nil
}
+
+// GetClusterInfo implements Curator.GetClusterInfo, which returns summary
+// information about the Metropolis cluster.
+func (l *leaderManagement) GetClusterInfo(ctx context.Context, req *apb.GetClusterInfoRequest) (*apb.GetClusterInfoResponse, error) {
+ res, err := l.txnAsLeader(ctx, nodeEtcdPrefix.Range())
+ if err != nil {
+ return nil, status.Errorf(codes.Unavailable, "could not retrieve list of nodes: %v", err)
+ }
+
+ // Sort nodes by public key, filter out Up, use top 15 in cluster directory
+ // (limited to an arbitrary amount that doesn't overload callers with
+ // unnecesssary information).
+ //
+ // MVP: this should be formalized and possibly re-designed/engineered.
+ kvs := res.Responses[0].GetResponseRange().Kvs
+ var nodes []*Node
+ for _, kv := range kvs {
+ node, err := nodeUnmarshal(kv.Value)
+ if err != nil {
+ // TODO(q3k): log this
+ continue
+ }
+ if node.state != cpb.NodeState_NODE_STATE_UP {
+ continue
+ }
+ nodes = append(nodes, node)
+ }
+ sort.Slice(nodes, func(i, j int) bool {
+ return bytes.Compare(nodes[i].pubkey, nodes[j].pubkey) < 0
+ })
+ if len(nodes) > 15 {
+ nodes = nodes[:15]
+ }
+
+ // Build cluster directory.
+ directory := &cpb.ClusterDirectory{
+ Nodes: make([]*cpb.ClusterDirectory_Node, len(nodes)),
+ }
+ for i, node := range nodes {
+ var addresses []*cpb.ClusterDirectory_Node_Address
+ if node.status != nil && node.status.ExternalAddress != "" {
+ addresses = append(addresses, &cpb.ClusterDirectory_Node_Address{
+ Host: node.status.ExternalAddress,
+ })
+ }
+ directory.Nodes[i] = &cpb.ClusterDirectory_Node{
+ PublicKey: node.pubkey,
+ Addresses: addresses,
+ }
+ }
+
+ return &apb.GetClusterInfoResponse{
+ ClusterDirectory: directory,
+ }, nil
+}
diff --git a/metropolis/node/core/curator/impl_leader_test.go b/metropolis/node/core/curator/impl_leader_test.go
index 4318a1f..059c444 100644
--- a/metropolis/node/core/curator/impl_leader_test.go
+++ b/metropolis/node/core/curator/impl_leader_test.go
@@ -252,3 +252,46 @@
t.Errorf("UpdateNodeStatus for other node (%q vs local %q) succeeded, should have failed", cl.localNodeID, cl.otherNodeID)
}
}
+
+// TestManagementClusterInfo exercises GetClusterInfo after setting a status.
+func TestMangementClusterInfo(t *testing.T) {
+ cl := fakeLeader(t)
+ defer cl.cancel()
+
+ mgmt := apb.NewManagementClient(cl.mgmtConn)
+ curator := ipb.NewCuratorClient(cl.localNodeConn)
+
+ ctx, ctxC := context.WithCancel(context.Background())
+ defer ctxC()
+
+ // Update status to set an external address.
+ _, err := curator.UpdateNodeStatus(ctx, &ipb.UpdateNodeStatusRequest{
+ NodeId: cl.localNodeID,
+ Status: &cpb.NodeStatus{
+ ExternalAddress: "192.0.2.10",
+ },
+ })
+ if err != nil {
+ t.Fatalf("UpdateNodeStatus: %v", err)
+ }
+
+ // Retrieve cluster info and make sure it's as expected.
+ res, err := mgmt.GetClusterInfo(ctx, &apb.GetClusterInfoRequest{})
+ if err != nil {
+ t.Fatalf("GetClusterInfo failed: %v", err)
+ }
+
+ nodes := res.ClusterDirectory.Nodes
+ if want, got := 1, len(nodes); want != got {
+ t.Fatalf("ClusterDirectory.Nodes contains %d elements, wanted %d", want, got)
+ }
+ node := nodes[0]
+
+ // Address should match address set from status.
+ if want, got := 1, len(node.Addresses); want != got {
+ t.Fatalf("ClusterDirectory.Nodes[0].Addresses has %d elements, wanted %d", want, got)
+ }
+ if want, got := "192.0.2.10", node.Addresses[0].Host; want != got {
+ t.Fatalf("Nodes[0].Addresses[0].Host is %q, wanted %q", want, got)
+ }
+}
diff --git a/metropolis/node/core/curator/listener.go b/metropolis/node/core/curator/listener.go
index 76bc1a5..c4f0062 100644
--- a/metropolis/node/core/curator/listener.go
+++ b/metropolis/node/core/curator/listener.go
@@ -374,3 +374,12 @@
})
return
}
+
+func (l *listener) GetClusterInfo(ctx context.Context, req *apb.GetClusterInfoRequest) (res *apb.GetClusterInfoResponse, err error) {
+ err = l.callImpl(ctx, func(ctx context.Context, impl rpc.ClusterExternalServices) error {
+ var err2 error
+ res, err2 = impl.GetClusterInfo(ctx, req)
+ return err2
+ })
+ return
+}
diff --git a/metropolis/proto/api/BUILD.bazel b/metropolis/proto/api/BUILD.bazel
index ef8885d..9f46fb4 100644
--- a/metropolis/proto/api/BUILD.bazel
+++ b/metropolis/proto/api/BUILD.bazel
@@ -11,7 +11,10 @@
"management.proto",
],
visibility = ["//visibility:public"],
- deps = ["//metropolis/proto/ext:ext_proto"],
+ deps = [
+ "//metropolis/proto/common:common_proto",
+ "//metropolis/proto/ext:ext_proto",
+ ],
)
go_proto_library(
@@ -20,7 +23,10 @@
importpath = "source.monogon.dev/metropolis/proto/api",
proto = ":api_proto",
visibility = ["//visibility:public"],
- deps = ["//metropolis/proto/ext:go_default_library"],
+ deps = [
+ "//metropolis/proto/common:go_default_library",
+ "//metropolis/proto/ext:go_default_library",
+ ],
)
go_library(
diff --git a/metropolis/proto/api/management.proto b/metropolis/proto/api/management.proto
index bbd81f8..309fb19 100644
--- a/metropolis/proto/api/management.proto
+++ b/metropolis/proto/api/management.proto
@@ -2,6 +2,7 @@
package metropolis.proto.api;
option go_package = "source.monogon.dev/metropolis/proto/api";
+import "metropolis/proto/common/common.proto";
import "metropolis/proto/ext/authorization.proto";
// Management service available to Cluster Managers.
@@ -17,6 +18,15 @@
need: PERMISSION_GET_REGISTER_TICKET
};
}
+ // GetClusterInfo retrieves publicly available summary information about
+ // this cluster, notably data required for nodes to register into a cluster
+ // or join it (other than the Register Ticket, which is gated by an
+ // additional permission).
+ rpc GetClusterInfo(GetClusterInfoRequest) returns (GetClusterInfoResponse) {
+ option (metropolis.proto.ext.authorization) = {
+ need: PERMISSION_READ_CLUSTER_STATUS
+ };
+ }
}
message GetRegisterTicketRequest {
@@ -26,3 +36,12 @@
// Opaque bytes that comprise the RegisterTicket.
bytes ticket = 1;
}
+
+message GetClusterInfoRequest {
+}
+
+message GetClusterInfoResponse {
+ // cluster_directory contains information about individual nodes in the
+ // cluster that can be used to dial the cluster's services.
+ metropolis.proto.common.ClusterDirectory cluster_directory = 1;
+}
diff --git a/metropolis/proto/common/common.proto b/metropolis/proto/common/common.proto
index 5a49520..63e6cfb 100644
--- a/metropolis/proto/common/common.proto
+++ b/metropolis/proto/common/common.proto
@@ -127,11 +127,7 @@
message Address {
string host = 1;
};
- repeated Address addesses = 2;
+ repeated Address addresses = 2;
};
repeated Node nodes = 1;
}
-
-message ClusterIdentity {
- bytes ca_fingerprint = 1;
-}