m/n/core: factor out gRPC/TLS into rpc and identity libraries
This is an annoying large change, which started its life as me pulling
the 'let's add tests for authentication' thread, and ended up in
unifying a whole bunch of dispersed logic under two new libraries.
Notable changes:
- m/n/core/identity now contains the NodeCertificate (now called Node)
and NodeCredentials types. These used to exist in the cluster code,
but were factored out to prevent loops between the curator, the
cluster enrolment logic, and other code. They can now be shared by
nearly all of the node code, removing the need for some conversions
between subsystems/packages.
- Alongside Node{,Credentials} types, the identity package contains
code that creates x509 certificate templates and verifies x509
certificates, and has functions specific to nodes and users - not
clients and servers. This allows moving most of the rest of
certificate checking code into a single set of functions, and allows
us to test this logic thoroughly.
- pki.{Client,Server,CA} are not used by the node core code anymore,
and can now be moved to kubernetes-specific code (as that was their
original purpose and that's their only current use).
- m/n/core/rpc has been refactored to deduplicate code between the
local/external gRPC servers and unary/stream interceptors for these
servers, also allowing for more thorough testing and unified
behaviour between all.
- A PeerInfo structure is now injected into all gRPC handlers, and is
unified to contain information both about nodes, users, and possibly
unauthenticated callers.
- The AAA.Escrow implementation now makes use of PeerInfo in order to
retrieve the client's certificate, instead of rolling its own logic.
- The EphemeralClusterCredentials test helper has been moved to the rpc
library, and now returns identity objects, allowing for simplified
test code (less juggling of bare public keys and
{x509,tls}.Certificate objects).
Change-Id: I9284966b4f18c0d7628167ca3168b4b4037808c1
Reviewed-on: https://review.monogon.dev/c/monogon/+/325
Reviewed-by: Lorenz Brun <lorenz@monogon.tech>
diff --git a/metropolis/node/core/cluster/BUILD.bazel b/metropolis/node/core/cluster/BUILD.bazel
index 322a337..2d3e813 100644
--- a/metropolis/node/core/cluster/BUILD.bazel
+++ b/metropolis/node/core/cluster/BUILD.bazel
@@ -1,11 +1,10 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = [
"cluster.go",
"cluster_bootstrap.go",
- "node.go",
"status.go",
"watcher.go",
],
@@ -15,6 +14,7 @@
"//metropolis/node/core/consensus:go_default_library",
"//metropolis/node/core/consensus/client:go_default_library",
"//metropolis/node/core/curator:go_default_library",
+ "//metropolis/node/core/identity:go_default_library",
"//metropolis/node/core/localstorage:go_default_library",
"//metropolis/node/core/network:go_default_library",
"//metropolis/pkg/event:go_default_library",
@@ -26,10 +26,3 @@
"@org_golang_google_protobuf//proto:go_default_library",
],
)
-
-go_test(
- name = "go_default_test",
- srcs = ["node_test.go"],
- embed = [":go_default_library"],
- deps = ["//metropolis/node/core/curator:go_default_library"],
-)
diff --git a/metropolis/node/core/cluster/cluster_bootstrap.go b/metropolis/node/core/cluster/cluster_bootstrap.go
index 2905d17..2b3b333 100644
--- a/metropolis/node/core/cluster/cluster_bootstrap.go
+++ b/metropolis/node/core/cluster/cluster_bootstrap.go
@@ -26,6 +26,7 @@
"source.monogon.dev/metropolis/node/core/consensus"
"source.monogon.dev/metropolis/node/core/curator"
+ "source.monogon.dev/metropolis/node/core/identity"
"source.monogon.dev/metropolis/pkg/supervisor"
apb "source.monogon.dev/metropolis/proto/api"
cpb "source.monogon.dev/metropolis/proto/common"
@@ -127,7 +128,7 @@
// Using the short-circuited credentials from the curator, build our
// NodeCredentials. That, and the public part of the credentials
// (NodeCertificate) are the primary output of the cluster manager.
- creds, err := NewNodeCredentials(priv, nodeCert, caCert)
+ creds, err := identity.NewNodeCredentials(priv, nodeCert, caCert)
if err != nil {
return fmt.Errorf("failed to use newly bootstrapped node credentials: %w", err)
}
diff --git a/metropolis/node/core/cluster/node.go b/metropolis/node/core/cluster/node.go
deleted file mode 100644
index af5b654..0000000
--- a/metropolis/node/core/cluster/node.go
+++ /dev/null
@@ -1,154 +0,0 @@
-package cluster
-
-import (
- "crypto/ed25519"
- "crypto/subtle"
- "crypto/tls"
- "crypto/x509"
- "fmt"
-
- "source.monogon.dev/metropolis/node/core/curator"
- "source.monogon.dev/metropolis/node/core/localstorage"
-)
-
-// NodeCertificate is the public part of the credentials of a node. They are
-// emitted for a node by the cluster CA contained within the curator.
-type NodeCertificate struct {
- node *x509.Certificate
- ca *x509.Certificate
-}
-
-// ClusterCA returns the CA certificate of the cluster for which this
-// NodeCertificate is emitted.
-func (n *NodeCertificate) ClusterCA() *x509.Certificate {
- return n.ca
-}
-
-// NodeCredentials are the public and private part of the credentials of a node.
-//
-// It represents all the data necessary for a node to authenticate over mTLS to
-// other nodes and the rest of the cluster.
-//
-// It must never be made available to any node other than the node it has been
-// emitted for.
-type NodeCredentials struct {
- NodeCertificate
- private ed25519.PrivateKey
-}
-
-// NewNodeCertificate wraps a pair CA and node DER-encoded certificates into
-// NodeCertificate, ensuring the given certificate data is valid and compatible
-// Metropolis assumptions.
-//
-// It does _not_ verify that the given CA is a known/trusted Metropolis CA for a
-// running cluster.
-func NewNodeCertificate(cert, ca []byte) (*NodeCertificate, error) {
- certParsed, err := x509.ParseCertificate(cert)
- if err != nil {
- return nil, fmt.Errorf("could not parse node certificate: %w", err)
- }
- caCertParsed, err := x509.ParseCertificate(ca)
- if err != nil {
- return nil, fmt.Errorf("could not parse ca certificate: %w", err)
- }
-
- // Ensure both CA and node certs use ED25519.
- if certParsed.PublicKeyAlgorithm != x509.Ed25519 {
- return nil, fmt.Errorf("node certificate must use ED25519, is %s", certParsed.PublicKeyAlgorithm.String())
- }
- if pub, ok := certParsed.PublicKey.(ed25519.PublicKey); !ok || len(pub) != ed25519.PublicKeySize {
- return nil, fmt.Errorf("node certificate ED25519 key invalid")
- }
- if caCertParsed.PublicKeyAlgorithm != x509.Ed25519 {
- return nil, fmt.Errorf("CA certificate must use ED25519, is %s", caCertParsed.PublicKeyAlgorithm.String())
- }
- if pub, ok := caCertParsed.PublicKey.(ed25519.PublicKey); !ok || len(pub) != ed25519.PublicKeySize {
- return nil, fmt.Errorf("CA certificate ED25519 key invalid")
- }
-
- // Ensure that the certificate is signed by the CA certificate.
- if err := certParsed.CheckSignatureFrom(caCertParsed); err != nil {
- return nil, fmt.Errorf("certificate not signed by given CA: %w", err)
- }
-
- // Ensure that the certificate has the node's calculated ID in its DNS names.
- found := false
- nid := curator.NodeID(certParsed.PublicKey.(ed25519.PublicKey))
- for _, n := range certParsed.DNSNames {
- if n == nid {
- found = true
- break
- }
- }
- if !found {
- return nil, fmt.Errorf("calculated node ID %q not found in node certificate's DNS names (%v)", nid, certParsed.DNSNames)
- }
-
- return &NodeCertificate{
- node: certParsed,
- ca: caCertParsed,
- }, nil
-}
-
-// NewNodeCredentials wraps a pair of CA and node DER-encoded certificates plus
-// a private key into NodeCredentials, ensuring that the given data is valid and
-// compatible with Metropolis assumptions.
-//
-// It does _not_ verify that the given CA is a known/trusted Metropolis CA for a
-// running cluster.
-func NewNodeCredentials(priv, cert, ca []byte) (*NodeCredentials, error) {
- nc, err := NewNodeCertificate(cert, ca)
- if err != nil {
- return nil, err
- }
-
- // Ensure that the private key is a valid length.
- if want, got := ed25519.PrivateKeySize, len(priv); want != got {
- return nil, fmt.Errorf("private key is not the correct length, wanted %d, got %d", want, got)
- }
-
- // Ensure that the given private key matches the given public key.
- if want, got := ed25519.PrivateKey(priv).Public().(ed25519.PublicKey), nc.PublicKey(); subtle.ConstantTimeCompare(want, got) != 1 {
- return nil, fmt.Errorf("public key does not match private key")
- }
-
- return &NodeCredentials{
- NodeCertificate: *nc,
- private: ed25519.PrivateKey(priv),
- }, nil
-}
-
-// Save stores the given node credentials in local storage.
-func (c *NodeCredentials) Save(d *localstorage.PKIDirectory) error {
- if err := d.CACertificate.Write(c.ca.Raw, 0400); err != nil {
- return fmt.Errorf("when writing CA certificate: %w", err)
- }
- if err := d.Certificate.Write(c.node.Raw, 0400); err != nil {
- return fmt.Errorf("when writing node certificate: %w", err)
- }
- if err := d.Key.Write(c.private, 0400); err != nil {
- return fmt.Errorf("when writing node private key: %w", err)
- }
- return nil
-}
-
-// PublicKey returns the ED25519 public key corresponding to this node's
-// certificate/credentials.
-func (nc *NodeCertificate) PublicKey() ed25519.PublicKey {
- // Safe: we have ensured that the given certificate has an ed25519 public key on
- // NewNodeCertificate.
- return nc.node.PublicKey.(ed25519.PublicKey)
-}
-
-// ID returns the canonical ID/name of the node for which this
-// certificate/credentials were emitted.
-func (nc *NodeCertificate) ID() string {
- return curator.NodeID(nc.PublicKey())
-}
-
-func (nc *NodeCredentials) TLSCredentials() tls.Certificate {
- return tls.Certificate{
- Certificate: [][]byte{nc.node.Raw},
- PrivateKey: nc.private,
- }
-}
diff --git a/metropolis/node/core/cluster/node_test.go b/metropolis/node/core/cluster/node_test.go
deleted file mode 100644
index 079d4dc..0000000
--- a/metropolis/node/core/cluster/node_test.go
+++ /dev/null
@@ -1,105 +0,0 @@
-package cluster
-
-import (
- "crypto/ed25519"
- "crypto/rand"
- "crypto/x509"
- "crypto/x509/pkix"
- "math/big"
- "testing"
- "time"
-
- "source.monogon.dev/metropolis/node/core/curator"
-)
-
-type alterCert func(t *x509.Certificate)
-
-func createPKI(t *testing.T, fca, fnode alterCert) (caCertBytes, nodeCertBytes, nodePriv []byte) {
- t.Helper()
-
- caPub, caPriv, err := ed25519.GenerateKey(rand.Reader)
- if err != nil {
- t.Fatalf("GenerateKey: %v", err)
- }
- var nodePub ed25519.PublicKey
- nodePub, nodePriv, err = ed25519.GenerateKey(rand.Reader)
- if err != nil {
- t.Fatalf("GenerateKey: %v", err)
- }
-
- caTemplate := &x509.Certificate{
- SerialNumber: big.NewInt(1),
- Subject: pkix.Name{CommonName: "CA"},
- IsCA: true,
- KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature,
- ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageOCSPSigning},
- NotBefore: time.Now(),
- NotAfter: time.Unix(253402300799, 0),
- BasicConstraintsValid: true,
- }
- fca(caTemplate)
-
- caCertBytes, err = x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, caPub, caPriv)
- if err != nil {
- t.Fatalf("CreateCertificate (CA): %v", err)
- }
- caCert, err := x509.ParseCertificate(caCertBytes)
- if err != nil {
- t.Fatalf("ParseCertificate (CA): %v", err)
- }
-
- nodeTemplate := &x509.Certificate{
- SerialNumber: big.NewInt(2),
- Subject: pkix.Name{},
- KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
- NotBefore: time.Now(),
- NotAfter: time.Unix(253402300799, 0),
- DNSNames: []string{curator.NodeID(nodePub)},
- }
- fnode(nodeTemplate)
-
- nodeCertBytes, err = x509.CreateCertificate(rand.Reader, nodeTemplate, caCert, nodePub, caPriv)
- if err != nil {
- t.Fatalf("CreateCertificate (node): %v", err)
- }
-
- return
-}
-
-// TestNodeCertificateX509 exercises X509 validity checks performed by
-// NewNodeCertificate.
-func TestNodeCertificateX509(t *testing.T) {
- for i, te := range []struct {
- fca alterCert
- fnode alterCert
- success bool
- }{
- // Case 0: everything should work.
- {
- func(ca *x509.Certificate) {},
- func(n *x509.Certificate) {},
- true,
- },
- // Case 1: CA must be IsCA
- {
- func(ca *x509.Certificate) { ca.IsCA = false },
- func(n *x509.Certificate) {},
- false,
- },
- // Case 2: node must have its ID as a DNS name.
- {
- func(ca *x509.Certificate) {},
- func(n *x509.Certificate) { n.DNSNames = []string{"node"} },
- false,
- },
- } {
- caCert, nodeCert, nodePriv := createPKI(t, te.fca, te.fnode)
- _, err := NewNodeCredentials(nodePriv, nodeCert, caCert)
- if te.success && err != nil {
- t.Fatalf("Case %d: NewNodeCredentials failed: %v", i, err)
- }
- if !te.success && err == nil {
- t.Fatalf("Case %d: NewNodeCredentials succeeded, wanted failure", i)
- }
- }
-}
diff --git a/metropolis/node/core/cluster/status.go b/metropolis/node/core/cluster/status.go
index 3f99567..3dbfb56 100644
--- a/metropolis/node/core/cluster/status.go
+++ b/metropolis/node/core/cluster/status.go
@@ -5,6 +5,7 @@
"fmt"
"source.monogon.dev/metropolis/node/core/consensus/client"
+ "source.monogon.dev/metropolis/node/core/identity"
cpb "source.monogon.dev/metropolis/proto/common"
)
@@ -26,7 +27,7 @@
// Credentials used for the node to authenticate to the Curator and other
// cluster services.
- Credentials *NodeCredentials
+ Credentials *identity.NodeCredentials
}
// ConsensusUser is the to-level user of an etcd client in Metropolis node
diff --git a/metropolis/node/core/curator/BUILD.bazel b/metropolis/node/core/curator/BUILD.bazel
index 40f7740..22e237f 100644
--- a/metropolis/node/core/curator/BUILD.bazel
+++ b/metropolis/node/core/curator/BUILD.bazel
@@ -22,6 +22,7 @@
"//metropolis/node/core/consensus/client:go_default_library",
"//metropolis/node/core/curator/proto/api:go_default_library",
"//metropolis/node/core/curator/proto/private:go_default_library",
+ "//metropolis/node/core/identity:go_default_library",
"//metropolis/node/core/localstorage:go_default_library",
"//metropolis/node/core/rpc:go_default_library",
"//metropolis/pkg/combinectx:go_default_library",
@@ -36,8 +37,6 @@
"@io_etcd_go_etcd//clientv3/concurrency:go_default_library",
"@org_golang_google_grpc//:go_default_library",
"@org_golang_google_grpc//codes:go_default_library",
- "@org_golang_google_grpc//credentials:go_default_library",
- "@org_golang_google_grpc//peer:go_default_library",
"@org_golang_google_grpc//status:go_default_library",
"@org_golang_google_protobuf//proto:go_default_library",
"@org_golang_x_sys//unix:go_default_library",
@@ -55,18 +54,17 @@
embed = [":go_default_library"],
deps = [
"//metropolis/node/core/consensus/client:go_default_library",
+ "//metropolis/node/core/identity:go_default_library",
"//metropolis/node/core/localstorage:go_default_library",
"//metropolis/node/core/localstorage/declarative:go_default_library",
"//metropolis/node/core/rpc:go_default_library",
"//metropolis/pkg/event/memory:go_default_library",
- "//metropolis/pkg/pki:go_default_library",
"//metropolis/pkg/supervisor:go_default_library",
"//metropolis/proto/api:go_default_library",
"@io_etcd_go_etcd//clientv3:go_default_library",
"@io_etcd_go_etcd//integration:go_default_library",
"@org_golang_google_grpc//:go_default_library",
"@org_golang_google_grpc//codes:go_default_library",
- "@org_golang_google_grpc//credentials:go_default_library",
"@org_golang_google_grpc//status:go_default_library",
"@org_golang_google_grpc//test/bufconn:go_default_library",
],
diff --git a/metropolis/node/core/curator/bootstrap.go b/metropolis/node/core/curator/bootstrap.go
index 98d176a..e1add0d 100644
--- a/metropolis/node/core/curator/bootstrap.go
+++ b/metropolis/node/core/curator/bootstrap.go
@@ -10,6 +10,7 @@
"source.monogon.dev/metropolis/node/core/consensus/client"
ppb "source.monogon.dev/metropolis/node/core/curator/proto/private"
+ "source.monogon.dev/metropolis/node/core/identity"
"source.monogon.dev/metropolis/pkg/pki"
)
@@ -27,7 +28,7 @@
// cluster. It can only be called by cluster bootstrap code. It returns the
// generated x509 CA and node certificates.
func BootstrapNodeCredentials(ctx context.Context, etcd client.Namespaced, pubkey ed25519.PublicKey) (ca, node []byte, err error) {
- id := NodeID(pubkey)
+ id := identity.NodeID(pubkey)
ca, err = pkiCA.Ensure(ctx, etcd)
if err != nil {
@@ -37,7 +38,7 @@
nodeCert := &pki.Certificate{
Namespace: &pkiNamespace,
Issuer: pkiCA,
- Template: pki.Server([]string{id}, nil),
+ Template: identity.NodeCertificate(pubkey),
Mode: pki.CertificateExternal,
PublicKey: pubkey,
Name: fmt.Sprintf("node-%s", id),
diff --git a/metropolis/node/core/curator/curator.go b/metropolis/node/core/curator/curator.go
index 35b066e..6d16d62 100644
--- a/metropolis/node/core/curator/curator.go
+++ b/metropolis/node/core/curator/curator.go
@@ -13,8 +13,6 @@
import (
"context"
- "crypto/tls"
- "crypto/x509"
"errors"
"fmt"
"time"
@@ -25,6 +23,7 @@
"source.monogon.dev/metropolis/node/core/consensus/client"
ppb "source.monogon.dev/metropolis/node/core/curator/proto/private"
+ "source.monogon.dev/metropolis/node/core/identity"
"source.monogon.dev/metropolis/node/core/localstorage"
"source.monogon.dev/metropolis/node/core/rpc"
"source.monogon.dev/metropolis/pkg/event"
@@ -34,12 +33,12 @@
// Config is the configuration of the curator.
type Config struct {
+ // NodeCredentials are the identity credentials for the node that is running
+ // this curator.
+ NodeCredentials *identity.NodeCredentials
// Etcd is an etcd client in which all curator storage and leader election
// will be kept.
Etcd client.Namespaced
- // NodeID is the ID of the node that this curator will run on. It's used to
- // populate the leader election lock.
- NodeID string
// LeaderTTL is the timeout on the lease used to perform leader election.
// Any active leader must continue updating its lease at least this often,
// or the lease (and leadership) will be lost.
@@ -50,15 +49,6 @@
// Directory is the curator ephemeral directory in which the curator will
// store its local domain socket for connections from the node.
Directory *localstorage.EphemeralCuratorDirectory
-
- // ServerCredentials is the TLS certificate/key of the node that the curator
- // will use to run public gRPC services. It should be signed by
- // ClusterCACertificate.
- ServerCredentials tls.Certificate
- // ClusterCACertificate is the cluster's CA certificate. It will be used to
- // authenticate the client certificates of incoming connections to the public
- // gRPC services.
- ClusterCACertificate *x509.Certificate
}
// Service is the Curator service. See the package-level documentation for more
@@ -137,7 +127,7 @@
// LeaderElectionValue from private/storage.proto.
func (c *Config) buildLockValue(ttl int) ([]byte, error) {
v := &ppb.LeaderElectionValue{
- NodeId: c.NodeID,
+ NodeId: c.NodeCredentials.ID(),
Ttl: uint64(ttl),
}
bytes, err := proto.Marshal(v)
@@ -255,7 +245,7 @@
for {
s, err := w.get(ctx)
if err != nil {
- supervisor.Logger(ctx).Warningf("Election watcher existing: get(): %w", err)
+ supervisor.Logger(ctx).Warningf("Election watcher exiting: get(): %v", err)
return
}
if l := s.leader; l != nil {
@@ -270,11 +260,8 @@
// providing the Curator API to consumers, dispatching to either a locally
// running leader, or forwarding to a remotely running leader.
lis := listener{
- directory: s.config.Directory,
- ServerSecurity: rpc.ServerSecurity{
- NodeCredentials: s.config.ServerCredentials,
- ClusterCACertificate: s.config.ClusterCACertificate,
- },
+ directory: s.config.Directory,
+ node: s.config.NodeCredentials,
electionWatch: s.electionWatch,
etcd: s.config.Etcd,
dispatchC: make(chan dispatchRequest),
@@ -306,5 +293,6 @@
}
func (s *Service) DialCluster(ctx context.Context) (*grpc.ClientConn, error) {
- return grpc.DialContext(ctx, fmt.Sprintf("unix://%s", s.config.Directory.ClientSocket.FullPath()), grpc.WithInsecure())
+ remote := fmt.Sprintf("unix://%s", s.config.Directory.ClientSocket.FullPath())
+ return rpc.NewNodeClient(remote)
}
diff --git a/metropolis/node/core/curator/curator_test.go b/metropolis/node/core/curator/curator_test.go
index c67fc99..af87c9e 100644
--- a/metropolis/node/core/curator/curator_test.go
+++ b/metropolis/node/core/curator/curator_test.go
@@ -12,8 +12,10 @@
"go.etcd.io/etcd/integration"
"source.monogon.dev/metropolis/node/core/consensus/client"
+ "source.monogon.dev/metropolis/node/core/identity"
"source.monogon.dev/metropolis/node/core/localstorage"
"source.monogon.dev/metropolis/node/core/localstorage/declarative"
+ "source.monogon.dev/metropolis/node/core/rpc"
"source.monogon.dev/metropolis/pkg/supervisor"
)
@@ -61,7 +63,7 @@
// newDut creates a new dut harness for a curator instance, connected to a given
// etcd endpoint.
-func newDut(ctx context.Context, t *testing.T, endpoint string) *dut {
+func newDut(ctx context.Context, t *testing.T, endpoint string, n *identity.NodeCredentials) *dut {
t.Helper()
// Create new etcd client to the given endpoint.
cli, err := clientv3.New(clientv3.Config{
@@ -86,15 +88,14 @@
t.Fatalf("PlaceFS: %v", err)
}
- id := fmt.Sprintf("test-%s", endpoint)
svc := New(Config{
- Etcd: client.NewLocal(cli),
- NodeID: id,
- LeaderTTL: time.Second,
- Directory: &dir,
+ Etcd: client.NewLocal(cli),
+ NodeCredentials: n,
+ LeaderTTL: time.Second,
+ Directory: &dir,
})
- if err := supervisor.Run(ctx, id, svc.Run); err != nil {
- t.Fatalf("Run %s: %v", id, err)
+ if err := supervisor.Run(ctx, n.ID(), svc.Run); err != nil {
+ t.Fatalf("Run %s: %v", n.ID(), err)
}
return &dut{
endpoint: endpoint,
@@ -220,10 +221,11 @@
}
// Start a new supervisor in which we create all curator DUTs.
+ ephemeral := rpc.NewEphemeralClusterCredentials(t, 3)
dutC := make(chan *dut)
supervisor.TestHarness(t, func(ctx context.Context) error {
- for e, _ := range endpointToNum {
- dutC <- newDut(ctx, t, e)
+ for e, n := range endpointToNum {
+ dutC <- newDut(ctx, t, e, ephemeral.Nodes[n])
}
close(dutC)
supervisor.Signal(ctx, supervisor.SignalHealthy)
@@ -256,7 +258,7 @@
// these have changed when we switch to another leader and back.
key := dss[leaderEndpoint].leader.lockKey
rev := dss[leaderEndpoint].leader.lockRev
- leaderNodeID := duts[leaderEndpoint].instance.config.NodeID
+ leaderNodeID := duts[leaderEndpoint].instance.config.NodeCredentials.ID()
leaderNum := endpointToNum[leaderEndpoint]
// Ensure the leader/follower data in the electionStatus are as expected.
@@ -316,7 +318,7 @@
// Get new leader's key and rev.
newKey := dss[newLeaderEndpoint].leader.lockKey
newRev := dss[newLeaderEndpoint].leader.lockRev
- newLeaderNodeID := duts[newLeaderEndpoint].instance.config.NodeID
+ newLeaderNodeID := duts[newLeaderEndpoint].instance.config.NodeCredentials.ID()
if leaderEndpoint == newLeaderEndpoint {
t.Errorf("leader endpoint didn't change (%q -> %q)", leaderEndpoint, newLeaderEndpoint)
diff --git a/metropolis/node/core/curator/impl_leader_aaa.go b/metropolis/node/core/curator/impl_leader_aaa.go
index 2f8124a..dd0d546 100644
--- a/metropolis/node/core/curator/impl_leader_aaa.go
+++ b/metropolis/node/core/curator/impl_leader_aaa.go
@@ -7,12 +7,12 @@
"errors"
"google.golang.org/grpc/codes"
- "google.golang.org/grpc/credentials"
- "google.golang.org/grpc/peer"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/proto"
ppb "source.monogon.dev/metropolis/node/core/curator/proto/private"
+ "source.monogon.dev/metropolis/node/core/identity"
+ "source.monogon.dev/metropolis/node/core/rpc"
"source.monogon.dev/metropolis/pkg/pki"
apb "source.monogon.dev/metropolis/proto/api"
)
@@ -26,37 +26,6 @@
leadership
}
-// pubkeyFromGRPC returns the ed25519 public key presented by the client in any
-// client certificate for a gRPC call. If no certificate is presented, nil is
-// returned. If the connection is insecure or the client presented some invalid
-// certificate configuration, a gRPC status is returned that can be directly
-// passed to the client. Otherwise, the public key is returned.
-//
-// SECURITY: the public key is not verified to be authorized to perform any
-// action,just to be a valid ed25519 key.
-func pubkeyFromGRPC(ctx context.Context) (ed25519.PublicKey, error) {
- p, ok := peer.FromContext(ctx)
- if !ok {
- return nil, status.Error(codes.Unavailable, "could not retrieve peer info")
- }
- tlsInfo, ok := p.AuthInfo.(credentials.TLSInfo)
- if !ok {
- return nil, status.Error(codes.Unauthenticated, "connection not secure")
- }
- count := len(tlsInfo.State.PeerCertificates)
- if count == 0 {
- return nil, nil
- }
- if count > 1 {
- return nil, status.Errorf(codes.Unauthenticated, "exactly one client certificate must be sent (got %d)", count)
- }
- pk, ok := tlsInfo.State.PeerCertificates[0].PublicKey.(ed25519.PublicKey)
- if !ok {
- return nil, status.Errorf(codes.Unauthenticated, "client certificate must be for ed25519 key")
- }
- return pk, nil
-}
-
// getOwnerPubkey returns the public key of the configured owner of the cluster.
//
// MVP: this should be turned into a proper user/entity system.
@@ -90,6 +59,13 @@
// certificate which can be used to perform further management actions.
func (a *leaderAAA) Escrow(srv apb.AAA_EscrowServer) error {
ctx := srv.Context()
+ peerInfo := rpc.GetPeerInfo(ctx)
+ if peerInfo == nil {
+ return status.Error(codes.Unauthenticated, "no PeerInfo available")
+ }
+ if peerInfo.Unauthenticated == nil {
+ return status.Error(codes.InvalidArgument, "connection is already authenticated")
+ }
// Receive Parameters from client. This tells us what identity the client wants
// from us.
@@ -117,13 +93,7 @@
// TODO(q3k) The AAA proto doesn't really have a proof kind for this, for now we
// go with REFRESH_CERTIFICATE. We should either make the AAA proto explicitly
// handle this as a special KIND.
- pk, err := pubkeyFromGRPC(ctx)
- if err != nil {
- // If an error occurred, it's either because the connection is not secured by
- // TLS, or an invalid certificate was presented (ie. more then one cert, or a
- // non-ed25519 cert). Fail as per AAA proto.
- return err
- }
+ pk := peerInfo.Unauthenticated.SelfSignedPublicKey
if pk == nil {
// No cert was presented, respond with REFRESH_CERTIFICATE request.
err := srv.Send(&apb.EscrowFromServer{
@@ -159,7 +129,7 @@
oc := pki.Certificate{
Namespace: &pkiNamespace,
Issuer: pkiCA,
- Template: pki.Client("owner", nil),
+ Template: identity.UserCertificate("owner"),
Name: "owner",
Mode: pki.CertificateExternal,
PublicKey: pk,
diff --git a/metropolis/node/core/curator/impl_leader_test.go b/metropolis/node/core/curator/impl_leader_test.go
index eb1db38..593830d 100644
--- a/metropolis/node/core/curator/impl_leader_test.go
+++ b/metropolis/node/core/curator/impl_leader_test.go
@@ -3,34 +3,28 @@
import (
"bytes"
"context"
- "crypto/tls"
- "crypto/x509"
- "net"
"testing"
"go.etcd.io/etcd/integration"
"google.golang.org/grpc"
- "google.golang.org/grpc/credentials"
"google.golang.org/grpc/test/bufconn"
"source.monogon.dev/metropolis/node/core/consensus/client"
"source.monogon.dev/metropolis/node/core/rpc"
- "source.monogon.dev/metropolis/pkg/pki"
apb "source.monogon.dev/metropolis/proto/api"
)
// fakeLeader creates a curatorLeader without any underlying leader election, in
-// its own etcd namespace. It starts a gRPC listener of its public services
-// implementation and returns a client to it.
+// its own etcd namespace. It starts public and local gRPC listeners and returns
+// clients to them.
//
-// The entire gRPC layer is encrypted, authenticated and authorized in the same
-// way as by the full Curator codebase running in Metropolis. An ephemeral
-// cluster CA and node/manager credentials are created, and are used to
-// establish a secure channel when creating the gRPC listener and client.
+// The gRPC listeners are replicated to behave as when running the Curator
+// within Metropolis, so all calls performed will be authenticated and encrypted
+// the same way.
//
// This is used to test functionality of the individual curatorLeader RPC
// implementations without the overhead of having to wait for a leader election.
-func fakeLeader(t *testing.T) (grpc.ClientConnInterface, context.CancelFunc) {
+func fakeLeader(t *testing.T) fakeLeaderData {
t.Helper()
// Set up context whose cancel function will be returned to the user for
// terminating all harnesses started by this function.
@@ -69,57 +63,96 @@
// Build a test cluster PKI and node/manager certificates, and create the
// listener security parameters which will authenticate incoming requests.
- node, manager, ca := pki.EphemeralClusterCredentials(t)
- sec := &rpc.ServerSecurity{
- NodeCredentials: node,
- ClusterCACertificate: ca,
+ ephemeral := rpc.NewEphemeralClusterCredentials(t, 1)
+ nodeCredentials := ephemeral.Nodes[0]
+
+ cNode := NewNodeForBootstrap(nil, nodeCredentials.PublicKey())
+ // Inject new node into leader, using curator bootstrap functionality.
+ if err := BootstrapFinish(ctx, cl, &cNode, nodeCredentials.PublicKey()); err != nil {
+ t.Fatalf("could not finish node bootstrap: %v", err)
}
+ // Create security interceptors for both gRPC listeners.
+ externalSec := &rpc.ExternalServerSecurity{
+ NodeCredentials: nodeCredentials,
+ }
+ localSec := &rpc.LocalServerSecurity{
+ Node: &nodeCredentials.Node,
+ }
// Create a curator gRPC server which performs authentication as per the created
// listenerSecurity and is backed by the created leader.
- srv := sec.SetupPublicGRPC(leader)
+ externalSrv := externalSec.SetupExternalGRPC(leader)
+ localSrv := localSec.SetupLocalGRPC(leader)
// The gRPC server will listen on an internal 'loopback' buffer.
- lis := bufconn.Listen(1024 * 1024)
+ externalLis := bufconn.Listen(1024 * 1024)
+ localLis := bufconn.Listen(1024 * 1024)
go func() {
- if err := srv.Serve(lis); err != nil {
+ if err := externalSrv.Serve(externalLis); err != nil {
t.Fatalf("GRPC serve failed: %v", err)
}
}()
+ go func() {
+ if err := localSrv.Serve(localLis); err != nil {
+ t.Fatalf("GRPC serve failed: %v", err)
+ }
+ }()
+
// Stop the gRPC server on context cancel.
go func() {
<-ctx.Done()
- srv.Stop()
+ externalSrv.Stop()
+ localSrv.Stop()
}()
// Create an authenticated manager gRPC client.
- // TODO(q3k): factor this out to its own library, alongside the code in //metropolis/test/e2e/client.go.
- pool := x509.NewCertPool()
- pool.AddCert(ca)
- gclCreds := credentials.NewTLS(&tls.Config{
- Certificates: []tls.Certificate{manager},
- RootCAs: pool,
- })
- gcl, err := grpc.Dial("test-server", grpc.WithContextDialer(func(_ context.Context, _ string) (net.Conn, error) {
- return lis.Dial()
- }), grpc.WithTransportCredentials(gclCreds))
+ mcl, err := rpc.NewAuthenticatedClientTest(externalLis, ephemeral.Manager, ephemeral.CA)
+ if err != nil {
+ t.Fatalf("Dialing external GRPC failed: %v", err)
+ }
+
+ // Create a locally authenticated node gRPC client.
+ lcl, err := rpc.NewNodeClientTest(localLis)
if err != nil {
t.Fatalf("Dialing local GRPC failed: %v", err)
}
- // Close the client on context cancel.
+
+ // Close the clients on context cancel.
go func() {
<-ctx.Done()
- gcl.Close()
+ mcl.Close()
+ lcl.Close()
}()
- return gcl, ctxC
+ return fakeLeaderData{
+ mgmtConn: mcl,
+ localNodeConn: lcl,
+ localNodeID: nodeCredentials.ID(),
+ cancel: ctxC,
+ }
+}
+
+// fakeLeaderData is returned by fakeLeader and contains information about the
+// newly created leader and connections to its gRPC listeners.
+type fakeLeaderData struct {
+ // mgmtConn is a gRPC connection to the leader's public gRPC interface,
+ // authenticated as a cluster manager.
+ mgmtConn grpc.ClientConnInterface
+ // localNodeConn is a gRPC connection to the leader's internal/local node gRPC
+ // interface, which usually runs on a domain socket and is only available to
+ // other Metropolis node code.
+ localNodeConn grpc.ClientConnInterface
+ // localNodeID is the NodeID of the fake node that the leader is running on.
+ localNodeID string
+ // cancel shuts down the fake leader and all client connections.
+ cancel context.CancelFunc
}
// TestManagementRegisterTicket exercises the Management.GetRegisterTicket RPC.
func TestManagementRegisterTicket(t *testing.T) {
- cl, cancel := fakeLeader(t)
- defer cancel()
+ cl := fakeLeader(t)
+ defer cl.cancel()
- mgmt := apb.NewManagementClient(cl)
+ mgmt := apb.NewManagementClient(cl.mgmtConn)
ctx, ctxC := context.WithCancel(context.Background())
defer ctxC()
diff --git a/metropolis/node/core/curator/listener.go b/metropolis/node/core/curator/listener.go
index c49eab3..d21d951 100644
--- a/metropolis/node/core/curator/listener.go
+++ b/metropolis/node/core/curator/listener.go
@@ -13,6 +13,7 @@
"source.monogon.dev/metropolis/node"
"source.monogon.dev/metropolis/node/core/consensus/client"
cpb "source.monogon.dev/metropolis/node/core/curator/proto/api"
+ "source.monogon.dev/metropolis/node/core/identity"
"source.monogon.dev/metropolis/node/core/localstorage"
"source.monogon.dev/metropolis/node/core/rpc"
"source.monogon.dev/metropolis/pkg/combinectx"
@@ -38,8 +39,7 @@
// some calls might not be idempotent and the caller is better equipped to know
// when to retry.
type listener struct {
- rpc.ServerSecurity
-
+ node *identity.NodeCredentials
// etcd is a client to the locally running consensus (etcd) server which is used
// both for storing lock/leader election status and actual Curator data.
etcd client.Namespaced
@@ -123,7 +123,7 @@
// context cancel function for ctx, or nil if ctx is nil.
ctxC *context.CancelFunc
// active Curator implementation, or nil if not yet set up.
- impl rpc.ClusterServices
+ impl rpc.ClusterExternalServices
}
// switchTo switches the activeTarget over to a Curator implementation as per
@@ -163,7 +163,7 @@
ctx context.Context
// impl is the CuratorServer implementation to which RPCs should be directed
// according to the dispatcher.
- impl rpc.ClusterServices
+ impl rpc.ClusterExternalServices
}
// dispatch contacts the dispatcher to retrieve an up-to-date listenerTarget.
@@ -197,10 +197,12 @@
return fmt.Errorf("when starting dispatcher: %w", err)
}
- srvLocal := grpc.NewServer()
- cpb.RegisterCuratorServer(srvLocal, l)
-
- srvPublic := l.SetupPublicGRPC(l)
+ es := rpc.ExternalServerSecurity{
+ NodeCredentials: l.node,
+ }
+ ls := rpc.LocalServerSecurity{
+ Node: &l.node.Node,
+ }
err := supervisor.Run(ctx, "local", func(ctx context.Context) error {
lisLocal, err := net.ListenUnix("unix", &net.UnixAddr{Name: l.directory.ClientSocket.FullPath(), Net: "unix"})
@@ -209,25 +211,25 @@
}
defer lisLocal.Close()
- runnable := supervisor.GRPCServer(srvLocal, lisLocal, true)
+ runnable := supervisor.GRPCServer(ls.SetupLocalGRPC(l), lisLocal, true)
return runnable(ctx)
})
if err != nil {
return fmt.Errorf("while starting local gRPC listener: %w", err)
}
- err = supervisor.Run(ctx, "public", func(ctx context.Context) error {
- lisPublic, err := net.Listen("tcp", fmt.Sprintf(":%d", node.CuratorServicePort))
+ err = supervisor.Run(ctx, "external", func(ctx context.Context) error {
+ lisExternal, err := net.Listen("tcp", fmt.Sprintf(":%d", node.CuratorServicePort))
if err != nil {
- return fmt.Errorf("failed to listen on public curator socket: %w", err)
+ return fmt.Errorf("failed to listen on external curator socket: %w", err)
}
- defer lisPublic.Close()
+ defer lisExternal.Close()
- runnable := supervisor.GRPCServer(srvPublic, lisPublic, true)
+ runnable := supervisor.GRPCServer(es.SetupExternalGRPC(l), lisExternal, true)
return runnable(ctx)
})
if err != nil {
- return fmt.Errorf("while starting public gRPC listener: %w", err)
+ return fmt.Errorf("while starting external gRPC listener: %w", err)
}
supervisor.Logger(ctx).Info("Listeners started.")
@@ -249,7 +251,7 @@
// returned are either returned directly or converted to an UNAVAILABLE status
// if the error is as a result of the context being canceled due to the
// implementation switching.
-type implOperation func(ctx context.Context, impl rpc.ClusterServices) error
+type implOperation func(ctx context.Context, impl rpc.ClusterExternalServices) error
// callImpl gets the newest listenerTarget from the dispatcher, combines the
// given context with the context of the listenerTarget implementation and calls
@@ -309,7 +311,7 @@
}
func (l *listener) Watch(req *cpb.WatchRequest, srv cpb.Curator_WatchServer) error {
- proxy := func(ctx context.Context, impl rpc.ClusterServices) error {
+ proxy := func(ctx context.Context, impl rpc.ClusterExternalServices) error {
return impl.Watch(req, &curatorWatchServer{
ServerStream: srv,
ctx: ctx,
@@ -340,7 +342,7 @@
}
func (l *listener) Escrow(srv apb.AAA_EscrowServer) error {
- return l.callImpl(srv.Context(), func(ctx context.Context, impl rpc.ClusterServices) error {
+ return l.callImpl(srv.Context(), func(ctx context.Context, impl rpc.ClusterExternalServices) error {
return impl.Escrow(&aaaEscrowServer{
ServerStream: srv,
ctx: ctx,
@@ -349,7 +351,7 @@
}
func (l *listener) GetRegisterTicket(ctx context.Context, req *apb.GetRegisterTicketRequest) (res *apb.GetRegisterTicketResponse, err error) {
- err = l.callImpl(ctx, func(ctx context.Context, impl rpc.ClusterServices) error {
+ err = l.callImpl(ctx, func(ctx context.Context, impl rpc.ClusterExternalServices) error {
var err2 error
res, err2 = impl.GetRegisterTicket(ctx, req)
return err2
diff --git a/metropolis/node/core/curator/listener_test.go b/metropolis/node/core/curator/listener_test.go
index 4644f6c..fad7e92 100644
--- a/metropolis/node/core/curator/listener_test.go
+++ b/metropolis/node/core/curator/listener_test.go
@@ -41,6 +41,9 @@
// Create test event value.
var val memory.Value
+ eph := rpc.NewEphemeralClusterCredentials(t, 1)
+ creds := eph.Nodes[0]
+
// Create DUT listener.
l := &listener{
etcd: nil,
@@ -51,6 +54,7 @@
}
},
dispatchC: make(chan dispatchRequest),
+ node: creds,
}
// Start listener under supervisor.
@@ -71,7 +75,7 @@
// Check that canceling the request unblocks a pending dispatched call.
errC := make(chan error)
go func() {
- errC <- l.callImpl(ctxR, func(ctx context.Context, impl rpc.ClusterServices) error {
+ errC <- l.callImpl(ctxR, func(ctx context.Context, impl rpc.ClusterExternalServices) error {
<-ctx.Done()
return ctx.Err()
})
@@ -85,7 +89,7 @@
// Check that switching implementations unblocks a pending dispatched call.
scheduledC := make(chan struct{})
go func() {
- errC <- l.callImpl(ctx, func(ctx context.Context, impl rpc.ClusterServices) error {
+ errC <- l.callImpl(ctx, func(ctx context.Context, impl rpc.ClusterExternalServices) error {
close(scheduledC)
<-ctx.Done()
return ctx.Err()
diff --git a/metropolis/node/core/curator/state_node.go b/metropolis/node/core/curator/state_node.go
index 4e2535f..e0763c4 100644
--- a/metropolis/node/core/curator/state_node.go
+++ b/metropolis/node/core/curator/state_node.go
@@ -18,7 +18,6 @@
import (
"context"
- "encoding/hex"
"fmt"
"net"
"strings"
@@ -27,6 +26,7 @@
"google.golang.org/protobuf/proto"
ppb "source.monogon.dev/metropolis/node/core/curator/proto/private"
+ "source.monogon.dev/metropolis/node/core/identity"
"source.monogon.dev/metropolis/node/core/localstorage"
"source.monogon.dev/metropolis/pkg/supervisor"
cpb "source.monogon.dev/metropolis/proto/common"
@@ -94,22 +94,9 @@
type NodeRoleKubernetesWorker struct {
}
-// nodeIDBare returns the `{pubkeyHash}` part of the node ID.
-func nodeIDBare(pub []byte) string {
- return hex.EncodeToString(pub[:16])
-}
-
-// NodeID returns the name of this node, which is `metropolis-{pubkeyHash}`.
-// This name should be the primary way to refer to Metropoils nodes within a
-// cluster, and is guaranteed to be unique by relying on cryptographic
-// randomness.
-func NodeID(pub []byte) string {
- return fmt.Sprintf("metropolis-%s", nodeIDBare(pub))
-}
-
// ID returns the name of this node. See NodeID for more information.
func (n *Node) ID() string {
- return NodeID(n.pubkey)
+ return identity.NodeID(n.pubkey)
}
func (n *Node) String() string {
@@ -183,7 +170,7 @@
if err := ephemeral.Hosts.Write([]byte(strings.Join(hosts, "\n")), 0644); err != nil {
return fmt.Errorf("failed to write /ephemeral/hosts: %w", err)
}
- if err := ephemeral.MachineID.Write([]byte(nodeIDBare(n.pubkey)), 0644); err != nil {
+ if err := ephemeral.MachineID.Write([]byte(identity.NodeIDBare(n.pubkey)), 0644); err != nil {
return fmt.Errorf("failed to write /ephemeral/machine-id: %w", err)
}
diff --git a/metropolis/node/core/curator/state_pki.go b/metropolis/node/core/curator/state_pki.go
index 5c217c5..2384158 100644
--- a/metropolis/node/core/curator/state_pki.go
+++ b/metropolis/node/core/curator/state_pki.go
@@ -1,6 +1,7 @@
package curator
import (
+ "source.monogon.dev/metropolis/node/core/identity"
"source.monogon.dev/metropolis/pkg/pki"
)
@@ -13,7 +14,7 @@
pkiCA = &pki.Certificate{
Namespace: &pkiNamespace,
Issuer: pki.SelfSigned,
- Template: pki.CA("Metropolis Cluster CA"),
+ Template: identity.CACertificate("Metropolis Cluster CA"),
Name: "cluster-ca",
}
)
diff --git a/metropolis/node/core/identity/BUILD.bazel b/metropolis/node/core/identity/BUILD.bazel
new file mode 100644
index 0000000..c5b481a
--- /dev/null
+++ b/metropolis/node/core/identity/BUILD.bazel
@@ -0,0 +1,18 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+ name = "go_default_library",
+ srcs = [
+ "certificates.go",
+ "identity.go",
+ ],
+ importpath = "source.monogon.dev/metropolis/node/core/identity",
+ visibility = ["//visibility:public"],
+ deps = ["//metropolis/node/core/localstorage:go_default_library"],
+)
+
+go_test(
+ name = "go_default_test",
+ srcs = ["certificates_test.go"],
+ embed = [":go_default_library"],
+)
diff --git a/metropolis/node/core/identity/certificates.go b/metropolis/node/core/identity/certificates.go
new file mode 100644
index 0000000..95b7e0d
--- /dev/null
+++ b/metropolis/node/core/identity/certificates.go
@@ -0,0 +1,169 @@
+package identity
+
+import (
+ "crypto/ed25519"
+ "crypto/x509"
+ "crypto/x509/pkix"
+ "fmt"
+ "math/big"
+)
+
+// UserCertificate makes a Metropolis-compatible user certificate template.
+func UserCertificate(identity string) x509.Certificate {
+ return x509.Certificate{
+ Subject: pkix.Name{
+ CommonName: identity,
+ },
+ KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+ ExtKeyUsage: []x509.ExtKeyUsage{
+ x509.ExtKeyUsageClientAuth,
+ },
+ }
+}
+
+// NodeCertificate makes a Metropolis-compatible node certificate template.
+func NodeCertificate(pubkey ed25519.PublicKey) x509.Certificate {
+ return x509.Certificate{
+ Subject: pkix.Name{
+ CommonName: NodeID(pubkey),
+ },
+ KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+ ExtKeyUsage: []x509.ExtKeyUsage{
+ // Note: node certificates are also effectively being used to perform client
+ // authentication to other node certificates, but they don't have the ClientAuth
+ // bit set. Instead, Metropolis uses the ClientAuth and ServerAuth bits
+ // exclusively to distinguish Metropolis nodes from Metropolis users.
+ x509.ExtKeyUsageServerAuth,
+ },
+ // We populate the Node's ID (metropolis-xxxx) as the DNS name for this
+ // certificate for ease of use within Metropolis, where the local DNS setup
+ // allows each node's IP address to be resolvable through the Node's ID.
+ DNSNames: []string{
+ NodeID(pubkey),
+ },
+ }
+}
+
+// CA makes a Metropolis-compatible CA certificate template.
+//
+// cn is a human-readable string that can be used to distinguish Metropolis
+// clusters, if needed. It is not machine-parsed, instead only signature
+// verification and CA pinning is performed.
+func CACertificate(cn string) x509.Certificate {
+ return x509.Certificate{
+ SerialNumber: big.NewInt(1),
+ Subject: pkix.Name{
+ CommonName: cn,
+ },
+ IsCA: true,
+ KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature,
+ ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageOCSPSigning},
+ }
+}
+
+// VerifyInCluster ensures that the given certificate has been signed by a CA
+// certificate and are both certificates emitted for ed25519 keypairs.
+//
+// The subject certificate's public key is returned if verification is
+// successful, and error is returned otherwise.
+func VerifyInCluster(cert, ca *x509.Certificate) (ed25519.PublicKey, error) {
+ // Ensure ca certificate uses ED25519 keypair.
+ if _, ok := ca.PublicKey.(ed25519.PublicKey); !ok {
+ return nil, fmt.Errorf("ca certificate not issued for ed25519 keypair")
+ }
+
+ // Ensure subject cert is signed by ca.
+ if err := cert.CheckSignatureFrom(ca); err != nil {
+ return nil, fmt.Errorf("signature veritifcation failed: %w", err)
+ }
+
+ // Ensure subject certificate is _not_ CA. CAs (cluster or possibly
+ // intermediaries) are not supposed to either directly serve traffic or perform
+ // client actions on the cluster.
+ if cert.IsCA {
+ return nil, fmt.Errorf("subject certificate is a CA")
+ }
+
+ // Extract subject ED25519 public key.
+ pubkey, ok := cert.PublicKey.(ed25519.PublicKey)
+ if !ok {
+ return nil, fmt.Errorf("certificate not issued for ed25519 keypair")
+ }
+
+ return pubkey, nil
+}
+
+// VerifyNodeInCluster ensures that a given certificate is a Metropolis node
+// certificate emitted by a given Metropolis CA.
+//
+// The node's public key is returned if verification is successful, and error is
+// returned otherwise.
+func VerifyNodeInCluster(node, ca *x509.Certificate) (ed25519.PublicKey, error) {
+ pk, err := VerifyInCluster(node, ca)
+ if err != nil {
+ return nil, err
+ }
+
+ // Ensure certificate has ServerAuth bit, thereby marking it as a node certificate.
+ found := false
+ for _, ku := range node.ExtKeyUsage {
+ if ku == x509.ExtKeyUsageServerAuth {
+ found = true
+ break
+ }
+ }
+ if !found {
+ return nil, fmt.Errorf("not a node certificate (missing ServerAuth key usage)")
+ }
+
+ id := NodeID(pk)
+
+ // Ensure node ID is present in Subject.CommonName and at least one DNS name.
+ if node.Subject.CommonName != id {
+ return nil, fmt.Errorf("node ID not found in CommonName")
+ }
+
+ found = false
+ for _, n := range node.DNSNames {
+ if n == id {
+ found = true
+ break
+ }
+ }
+ if !found {
+ return nil, fmt.Errorf("node ID not found in DNSNames")
+ }
+
+ return pk, nil
+}
+
+// VerifyUserInCluster ensures that a given certificate is a Metropolis user
+// certificate emitted by a given Metropolis CA.
+//
+// The user certificate's identity is returned if verification is successful,
+// and error is returned otherwise.
+func VerifyUserInCluster(user, ca *x509.Certificate) (string, error) {
+ _, err := VerifyInCluster(user, ca)
+ if err != nil {
+ return "", err
+ }
+
+ // Ensure certificate has ClientAuth bit, thereby marking it as a user certificate.
+ found := false
+ for _, ku := range user.ExtKeyUsage {
+ if ku == x509.ExtKeyUsageClientAuth {
+ found = true
+ break
+ }
+ }
+ if !found {
+ return "", fmt.Errorf("not a user certificate (missing ClientAuth key usage)")
+ }
+
+ // Extract identity from CommonName, ensure set.
+ identity := user.Subject.CommonName
+ if identity == "" {
+ return "", fmt.Errorf("CommonName not set")
+ }
+ return identity, nil
+}
diff --git a/metropolis/node/core/identity/certificates_test.go b/metropolis/node/core/identity/certificates_test.go
new file mode 100644
index 0000000..f96f517
--- /dev/null
+++ b/metropolis/node/core/identity/certificates_test.go
@@ -0,0 +1,180 @@
+package identity
+
+import (
+ "crypto/ed25519"
+ "crypto/rand"
+ "crypto/x509"
+ "math/big"
+ "testing"
+ "time"
+)
+
+// alterCert is used by test code to slightly alter certificates before they get
+// signed.
+type alterCert func(t *x509.Certificate)
+
+// basic is the bare minimum for ceritifcates to be properly issued over what
+// {CA,User,Node}Certificate return. The equivalent logic is present in the pki
+// codebase, we replicate it here because we don't use pki.
+func basic(t *x509.Certificate) {
+ t.SerialNumber = big.NewInt(1)
+ t.NotBefore = time.Now()
+ t.NotAfter = time.Unix(253402300799, 0)
+ t.BasicConstraintsValid = true
+}
+
+func noop(_ *x509.Certificate) {}
+
+// createPKI builds a minimum viable cluster PKI. We do not use
+// EphemeralClusterCredentials because we want to test the behaviour of the
+// certificate verification code when the certificate templates are slightly
+// altered, including in ways that the pki could would normally prevent us
+// from doing.
+func createPKI(t *testing.T, fca, fnode, fuser alterCert) (caCertBytes, nodeCertBytes, userCertBytes []byte) {
+ t.Helper()
+
+ caPub, caPriv, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatalf("GenerateKey: %v", err)
+ }
+ nodePub, _, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatalf("GenerateKey: %v", err)
+ }
+ userPub, _, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatalf("GenerateKey: %v", err)
+ }
+
+ caTemplate := CACertificate("test metropolis CA")
+ basic(&caTemplate)
+ fca(&caTemplate)
+
+ caCertBytes, err = x509.CreateCertificate(rand.Reader, &caTemplate, &caTemplate, caPub, caPriv)
+ if err != nil {
+ t.Fatalf("CreateCertificate (CA): %v", err)
+ }
+ caCert, err := x509.ParseCertificate(caCertBytes)
+ if err != nil {
+ t.Fatalf("ParseCertificate (CA): %v", err)
+ }
+
+ nodeTemplate := NodeCertificate(nodePub)
+ basic(&nodeTemplate)
+ fnode(&nodeTemplate)
+ nodeCertBytes, err = x509.CreateCertificate(rand.Reader, &nodeTemplate, caCert, nodePub, caPriv)
+ if err != nil {
+ t.Fatalf("CreateCertificate (node): %v", err)
+ }
+
+ userTemplate := UserCertificate("test")
+ basic(&userTemplate)
+ fuser(&userTemplate)
+ userCertBytes, err = x509.CreateCertificate(rand.Reader, &userTemplate, caCert, userPub, caPriv)
+ if err != nil {
+ t.Fatalf("CreateCertificate (node): %v", err)
+ }
+
+ return
+}
+
+func TestCertificates(t *testing.T) {
+ for i, te := range []struct {
+ fca alterCert
+ fnode alterCert
+ fuser alterCert
+ successNode bool
+ successUser bool
+ }{
+ // Case 0: everything should work.
+ {
+ noop,
+ noop,
+ noop,
+ true, true,
+ },
+ // Case 1: CA must be IsCA
+ {
+ func(ca *x509.Certificate) { ca.IsCA = false },
+ noop,
+ noop,
+ false, false,
+ },
+ // Case 2: node must not have IsCA set
+ {
+ noop,
+ func(n *x509.Certificate) { n.IsCA = true },
+ noop,
+ false, true,
+ },
+ // Case 3: user must not have IsCA set
+ {
+ noop,
+ noop,
+ func(u *x509.Certificate) { u.IsCA = true },
+ true, false,
+ },
+ // Case 4: node must have its ID as a DNS name.
+ {
+ noop,
+ func(n *x509.Certificate) { n.DNSNames = []string{"node"} },
+ noop,
+ false, true,
+ },
+ // Case 5: node must have its ID as CommoNName.
+ {
+ noop,
+ func(n *x509.Certificate) { n.Subject.CommonName = "node" },
+ noop,
+ false, true,
+ },
+ // Case 6: user must have CommonName set.
+ {
+ noop,
+ noop,
+ func(u *x509.Certificate) { u.Subject.CommonName = "" },
+ true, false,
+ },
+ } {
+ caCert, nodeCert, userCert := createPKI(t, te.fca, te.fnode, te.fuser)
+ caCertParsed, err := x509.ParseCertificate(caCert)
+ if err != nil {
+ t.Fatalf("Case %d: ParseCertificate(ca): %v", i, err)
+ }
+ nodeCertParsed, err := x509.ParseCertificate(nodeCert)
+ if err != nil {
+ t.Fatalf("Case %d: ParseCertificate(node): %v", i, err)
+ }
+ userCertParsed, err := x509.ParseCertificate(userCert)
+ if err != nil {
+ t.Fatalf("Case %d: ParseCertificate(node): %v", i, err)
+ }
+
+ // Check node certificate as node certificate. Should succeed iff successNode.
+ _, err = VerifyNodeInCluster(nodeCertParsed, caCertParsed)
+ if te.successNode && err != nil {
+ t.Errorf("Case %d: VerifyNodeInCluster failed: %v", i, err)
+ }
+ if !te.successNode && err == nil {
+ t.Errorf("Case %d: VerifyNodeInCluster succeeded, wanted failure", i)
+ }
+
+ // Check user certificate as user certificate. Should succeed iff successUser.
+ _, err = VerifyUserInCluster(userCertParsed, caCertParsed)
+ if te.successUser && err != nil {
+ t.Errorf("Case %d: VerifyUserInCluster failed: %v", i, err)
+ }
+ if !te.successUser && err == nil {
+ t.Errorf("Case %d: VerifyUserInCluster succeeded, wanted failure", i)
+ }
+
+ // Check user certificate as node certificate. Should always fail.
+ if _, err := VerifyNodeInCluster(userCertParsed, caCertParsed); err == nil {
+ t.Errorf("Case %d: User certificate erroneously verified as node ceritficate", i)
+ }
+ // Check node certificate as user certificate. Should always fail.
+ if _, err := VerifyUserInCluster(nodeCertParsed, caCertParsed); err == nil {
+ t.Errorf("Case %d: Node certificate erroneously verified as user ceritficate", i)
+ }
+ }
+}
diff --git a/metropolis/node/core/identity/identity.go b/metropolis/node/core/identity/identity.go
new file mode 100644
index 0000000..862e794
--- /dev/null
+++ b/metropolis/node/core/identity/identity.go
@@ -0,0 +1,138 @@
+package identity
+
+import (
+ "crypto/ed25519"
+ "crypto/subtle"
+ "crypto/tls"
+ "crypto/x509"
+ "encoding/hex"
+ "fmt"
+
+ "source.monogon.dev/metropolis/node/core/localstorage"
+)
+
+// Node is the public part of the credentials of a node. They are
+// emitted for a node by the cluster CA contained within the curator.
+type Node struct {
+ node *x509.Certificate
+ ca *x509.Certificate
+}
+
+// NewNode wraps a pair CA and node DER-encoded certificates into
+// Node, ensuring the given certificate data is valid and compatible
+// with Metropolis assumptions.
+func NewNode(cert, ca []byte) (*Node, error) {
+ certParsed, err := x509.ParseCertificate(cert)
+ if err != nil {
+ return nil, fmt.Errorf("could not parse node certificate: %w", err)
+ }
+ caCertParsed, err := x509.ParseCertificate(ca)
+ if err != nil {
+ return nil, fmt.Errorf("could not parse ca certificate: %w", err)
+ }
+
+ if _, err := VerifyNodeInCluster(certParsed, caCertParsed); err != nil {
+ return nil, fmt.Errorf("could not node certificate within cluster CA: %w", err)
+ }
+
+ return &Node{
+ node: certParsed,
+ ca: caCertParsed,
+ }, nil
+}
+
+// PublicKey returns the Ed25519 public key corresponding to this node's
+// certificate/credentials.
+func (n *Node) PublicKey() ed25519.PublicKey {
+ // Safe: we have ensured that the given certificate has an Ed25519 public key on
+ // NewNode.
+ return n.node.PublicKey.(ed25519.PublicKey)
+}
+
+// ClusterCA returns the CA certificate of the cluster for which this
+// Node is emitted.
+func (n *Node) ClusterCA() *x509.Certificate {
+ return n.ca
+}
+
+// ID returns the canonical ID/name of the node for which this
+// certificate/credentials were emitted.
+func (n *Node) ID() string {
+ return NodeID(n.PublicKey())
+}
+
+func (n *Node) Certificate() *x509.Certificate {
+ return n.node
+}
+
+// NodeCredentials are the public and private part of the credentials of a node.
+//
+// It represents all the data necessary for a node to authenticate over mTLS to
+// other nodes and the rest of the cluster.
+//
+// It must never be made available to any node other than the node it has been
+// emitted for.
+type NodeCredentials struct {
+ Node
+ private ed25519.PrivateKey
+}
+
+// NewNodeCredentials wraps a pair of CA and node DER-encoded certificates plus
+// a private key into NodeCredentials, ensuring that the given data is valid and
+// compatible with Metropolis assumptions.
+func NewNodeCredentials(priv, cert, ca []byte) (*NodeCredentials, error) {
+ nc, err := NewNode(cert, ca)
+ if err != nil {
+ return nil, err
+ }
+
+ // Ensure that the private key is a valid length.
+ if want, got := ed25519.PrivateKeySize, len(priv); want != got {
+ return nil, fmt.Errorf("private key is not the correct length, wanted %d, got %d", want, got)
+ }
+
+ // Ensure that the given private key matches the given public key.
+ if want, got := ed25519.PrivateKey(priv).Public().(ed25519.PublicKey), nc.PublicKey(); subtle.ConstantTimeCompare(want, got) != 1 {
+ return nil, fmt.Errorf("public key does not match private key")
+ }
+
+ return &NodeCredentials{
+ Node: *nc,
+ private: priv,
+ }, nil
+}
+
+func (n *NodeCredentials) TLSCredentials() tls.Certificate {
+ return tls.Certificate{
+ Leaf: n.node,
+ Certificate: [][]byte{n.node.Raw},
+ PrivateKey: n.private,
+ }
+}
+
+// Save stores the given node credentials in local storage.
+func (n *NodeCredentials) Save(d *localstorage.PKIDirectory) error {
+ if err := d.CACertificate.Write(n.ca.Raw, 0400); err != nil {
+ return fmt.Errorf("when writing CA certificate: %w", err)
+ }
+ if err := d.Certificate.Write(n.node.Raw, 0400); err != nil {
+ return fmt.Errorf("when writing node certificate: %w", err)
+ }
+ if err := d.Key.Write(n.private, 0400); err != nil {
+ return fmt.Errorf("when writing node private key: %w", err)
+ }
+ return nil
+}
+
+// NodeIDBare returns the `{pubkeyHash}` part of the node ID.
+func NodeIDBare(pub []byte) string {
+ return hex.EncodeToString(pub[:16])
+}
+
+// NodeID returns the name of this node, which is `metropolis-{pubkeyHash}`.
+// This name should be the primary way to refer to Metropoils nodes within a
+// cluster, and is guaranteed to be unique by relying on cryptographic
+// randomness.
+func NodeID(pub []byte) string {
+ return fmt.Sprintf("metropolis-%s", NodeIDBare(pub))
+}
diff --git a/metropolis/node/core/main.go b/metropolis/node/core/main.go
index 9ed2beb..fa768c2 100644
--- a/metropolis/node/core/main.go
+++ b/metropolis/node/core/main.go
@@ -181,13 +181,11 @@
// management of the cluster.
// In the future, this will only be started on nodes that run etcd.
c := curator.New(curator.Config{
- Etcd: ckv,
- NodeID: status.Credentials.ID(),
+ Etcd: ckv,
+ NodeCredentials: status.Credentials,
// TODO(q3k): make this configurable?
- LeaderTTL: time.Second * 5,
- Directory: &root.Ephemeral.Curator,
- ServerCredentials: status.Credentials.TLSCredentials(),
- ClusterCACertificate: status.Credentials.ClusterCA(),
+ LeaderTTL: time.Second * 5,
+ Directory: &root.Ephemeral.Curator,
})
if err := supervisor.Run(ctx, "curator", c.Run); err != nil {
close(trapdoor)
diff --git a/metropolis/node/core/rpc/BUILD.bazel b/metropolis/node/core/rpc/BUILD.bazel
index df03356..d281a92 100644
--- a/metropolis/node/core/rpc/BUILD.bazel
+++ b/metropolis/node/core/rpc/BUILD.bazel
@@ -1,15 +1,20 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = [
"client.go",
+ "methodinfo.go",
+ "peerinfo.go",
"server.go",
+ "server_authentication.go",
+ "testhelpers.go",
],
importpath = "source.monogon.dev/metropolis/node/core/rpc",
visibility = ["//visibility:public"],
deps = [
"//metropolis/node/core/curator/proto/api:go_default_library",
+ "//metropolis/node/core/identity:go_default_library",
"//metropolis/pkg/pki:go_default_library",
"//metropolis/proto/api:go_default_library",
"//metropolis/proto/ext:go_default_library",
@@ -18,8 +23,23 @@
"@org_golang_google_grpc//credentials:go_default_library",
"@org_golang_google_grpc//peer:go_default_library",
"@org_golang_google_grpc//status:go_default_library",
+ "@org_golang_google_grpc//test/bufconn:go_default_library",
"@org_golang_google_protobuf//proto:go_default_library",
"@org_golang_google_protobuf//reflect/protoreflect:go_default_library",
"@org_golang_google_protobuf//reflect/protoregistry:go_default_library",
],
)
+
+go_test(
+ name = "go_default_test",
+ srcs = ["server_authentication_test.go"],
+ embed = [":go_default_library"],
+ deps = [
+ "//metropolis/node/core/curator/proto/api:go_default_library",
+ "//metropolis/proto/api:go_default_library",
+ "//metropolis/proto/ext:go_default_library",
+ "@org_golang_google_grpc//codes:go_default_library",
+ "@org_golang_google_grpc//status:go_default_library",
+ "@org_golang_google_grpc//test/bufconn:go_default_library",
+ ],
+)
diff --git a/metropolis/node/core/rpc/client.go b/metropolis/node/core/rpc/client.go
index cc48f95..10d2545 100644
--- a/metropolis/node/core/rpc/client.go
+++ b/metropolis/node/core/rpc/client.go
@@ -8,15 +8,37 @@
"crypto/x509"
"fmt"
"math/big"
+ "net"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
+ "google.golang.org/grpc/test/bufconn"
+ "source.monogon.dev/metropolis/node/core/identity"
"source.monogon.dev/metropolis/pkg/pki"
apb "source.monogon.dev/metropolis/proto/api"
)
+type verifyPeerCertificate func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error
+
+func verifyClusterCertificate(ca *x509.Certificate) verifyPeerCertificate {
+ return func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
+ if len(rawCerts) != 1 {
+ return fmt.Errorf("server presented %d certificates, wanted exactly one", len(rawCerts))
+ }
+ serverCert, err := x509.ParseCertificate(rawCerts[0])
+ if err != nil {
+ return fmt.Errorf("server presented unparseable certificate: %w", err)
+ }
+ if _, err := identity.VerifyNodeInCluster(serverCert, ca); err != nil {
+ return fmt.Errorf("node certificate verification failed: %w", err)
+ }
+
+ return nil
+ }
+}
+
// NewEphemeralClient dials a cluster's services using just a self-signed
// certificate and can be used to then escrow real cluster credentials for the
// owner.
@@ -26,18 +48,16 @@
// 'real' client certificate (yet). Current users include users of AAA.Escrow
// and new nodes Registering into the Cluster.
//
-// If ca is given, the other side of the connection is verified to be served by
-// a node presenting a certificate signed by that CA. Otherwise, no
-// verification of the other side is performed (however, any attacker
-// impersonating the cluster cannot use the escrowed credentials as the private
-// key is never passed to the server).
-func NewEphemeralClient(remote string, private ed25519.PrivateKey, ca *x509.Certificate) (*grpc.ClientConn, error) {
+// If 'ca' is given, the remote side will be cryptographically verified to be a
+// node that's part of the cluster represented by the ca. Otherwise, no
+// verification is performed and this function is unsafe.
+func NewEphemeralClient(remote string, private ed25519.PrivateKey, ca *x509.Certificate, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
template := x509.Certificate{
SerialNumber: big.NewInt(1),
NotBefore: time.Now(),
NotAfter: pki.UnknownNotAfter,
- KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+ KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
BasicConstraintsValid: true,
}
@@ -49,53 +69,13 @@
Certificate: [][]byte{certificateBytes},
PrivateKey: private,
}
- creds := credentials.NewTLS(&tls.Config{
- Certificates: []tls.Certificate{
- certificate,
- },
- InsecureSkipVerify: true,
- VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
- if len(rawCerts) < 1 {
- return fmt.Errorf("server presented no certificate")
- }
- certs := make([]*x509.Certificate, len(rawCerts))
- for i, rawCert := range rawCerts {
- cert, err := x509.ParseCertificate(rawCert)
- if err != nil {
- return fmt.Errorf("could not parse server certificate %d: %v", i, err)
- }
- certs[i] = cert
- }
+ return NewAuthenticatedClient(remote, certificate, ca, opts...)
+}
- if ca != nil {
- // CA given, perform full chain verification.
- roots := x509.NewCertPool()
- roots.AddCert(ca)
- opts := x509.VerifyOptions{
- Roots: roots,
- Intermediates: x509.NewCertPool(),
- }
- for _, cert := range certs[1:] {
- opts.Intermediates.AddCert(cert)
- }
- _, err := certs[0].Verify(opts)
- if err != nil {
- return err
- }
- }
-
- // Regardless of CA given, ensure that the leaf certificate has the
- // right ExtKeyUsage.
- for _, ku := range certs[0].ExtKeyUsage {
- if ku == x509.ExtKeyUsageServerAuth {
- return nil
- }
- }
- return fmt.Errorf("server presented a certificate without server auth ext key usage")
- },
- })
-
- return grpc.Dial(remote, grpc.WithTransportCredentials(creds))
+func NewEphemeralClientTest(listener *bufconn.Listener, private ed25519.PrivateKey, ca *x509.Certificate) (*grpc.ClientConn, error) {
+ return NewEphemeralClient("local", private, ca, grpc.WithContextDialer(func(_ context.Context, _ string) (net.Conn, error) {
+ return listener.Dial()
+ }))
}
// RetrieveOwnerCertificates uses AAA.Escrow to retrieve a cluster manager
@@ -129,3 +109,38 @@
PrivateKey: private,
}, nil
}
+
+// NewAuthenticatedClient dials a cluster's services using the given TLS
+// credentials (either user or node credentials).
+//
+// If 'ca' is given, the remote side will be cryptographically verified to be a
+// node that's part of the cluster represented by the ca. Otherwise, no
+// verification is performed and this function is unsafe.
+func NewAuthenticatedClient(remote string, cert tls.Certificate, ca *x509.Certificate, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
+ config := &tls.Config{
+ Certificates: []tls.Certificate{cert},
+ InsecureSkipVerify: true,
+ }
+ if ca != nil {
+ config.VerifyPeerCertificate = verifyClusterCertificate(ca)
+ }
+ opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(config)))
+ return grpc.Dial(remote, opts...)
+}
+
+func NewNodeClient(remote string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
+ opts = append(opts, grpc.WithInsecure())
+ return grpc.Dial(remote, opts...)
+}
+
+func NewAuthenticatedClientTest(listener *bufconn.Listener, cert tls.Certificate, ca *x509.Certificate) (*grpc.ClientConn, error) {
+ return NewAuthenticatedClient("local", cert, ca, grpc.WithContextDialer(func(_ context.Context, _ string) (net.Conn, error) {
+ return listener.Dial()
+ }))
+}
+
+func NewNodeClientTest(listener *bufconn.Listener) (*grpc.ClientConn, error) {
+ return NewNodeClient("local", grpc.WithContextDialer(func(_ context.Context, _ string) (net.Conn, error) {
+ return listener.Dial()
+ }))
+}
diff --git a/metropolis/node/core/rpc/methodinfo.go b/metropolis/node/core/rpc/methodinfo.go
new file mode 100644
index 0000000..0a597fa
--- /dev/null
+++ b/metropolis/node/core/rpc/methodinfo.go
@@ -0,0 +1,86 @@
+package rpc
+
+import (
+ "fmt"
+ "regexp"
+
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+ "google.golang.org/protobuf/proto"
+ "google.golang.org/protobuf/reflect/protoreflect"
+ "google.golang.org/protobuf/reflect/protoregistry"
+
+ epb "source.monogon.dev/metropolis/proto/ext"
+)
+
+// methodInfo is the parsed information for a given RPC method, as configured by
+// the metropolis.common.ext.authorization extension.
+type methodInfo struct {
+ // unauthenticated is true if the method is defined as 'unauthenticated', ie.
+ // that all requests should be passed to the gRPC handler without any
+ // authentication or authorization performed.
+ unauthenticated bool
+ // need is a map of permissions that the caller needs to have in order to be
+ // allowed to call this method. If not empty, unauthenticated cannot be set to
+ // true.
+ need map[epb.Permission]bool
+}
+
+var (
+ // reMethodName matches a /some.service/Method string from
+ // {Stream,Unary}ServerInfo.FullMethod.
+ reMethodName = regexp.MustCompile(`^/([^/]+)/([^/.]+)$`)
+)
+
+// getMethodInfo returns the methodInfo for a given method name, as retrieved
+// from grpc.{Stream,Unary}ServerInfo.FullMethod, or nil if the method could not
+// be found.
+//
+// SECURITY: If the given method does not have any
+// metropolis.common.ext.authorization annotations, a methodInfo which requires
+// authorization but no permissions is returned, defaulting to a mildly secure
+// default of a method that can be called by any authenticated user.
+func getMethodInfo(methodName string) (*methodInfo, error) {
+ m := reMethodName.FindStringSubmatch(methodName)
+ if len(m) != 3 {
+ return nil, status.Errorf(codes.InvalidArgument, "invalid method name %q", methodName)
+ }
+ // Convert /foo.bar/Method to foo.bar.Method, which is used by the protoregistry.
+ methodName = fmt.Sprintf("%s.%s", m[1], m[2])
+ desc, err := protoregistry.GlobalFiles.FindDescriptorByName(protoreflect.FullName(methodName))
+ if err != nil {
+ return nil, status.Errorf(codes.InvalidArgument, "could not retrieve descriptor for method: %v", err)
+ }
+ method, ok := desc.(protoreflect.MethodDescriptor)
+ if !ok {
+ return nil, status.Error(codes.InvalidArgument, "querying method name did not yield a MethodDescriptor")
+ }
+
+ // Get authorization extension, defaults to no options set.
+ if !proto.HasExtension(method.Options(), epb.E_Authorization) {
+ return nil, status.Errorf(codes.Internal, "method does not provide Authorization extension, failing safe")
+ }
+ authz, ok := proto.GetExtension(method.Options(), epb.E_Authorization).(*epb.Authorization)
+ if !ok {
+ return nil, status.Errorf(codes.Internal, "method contains Authorization extension with wrong type, failing safe")
+ }
+ if authz == nil {
+ return nil, status.Errorf(codes.Internal, "method contains nil Authorization extension, failing safe")
+ }
+
+ // If unauthenticated connections are allowed, return immediately.
+ if authz.AllowUnauthenticated && len(authz.Need) == 0 {
+ return &methodInfo{
+ unauthenticated: true,
+ }, nil
+ }
+
+ // Otherwise, return needed permissions.
+ res := &methodInfo{
+ need: make(map[epb.Permission]bool),
+ }
+ for _, n := range authz.Need {
+ res.need[n] = true
+ }
+ return res, nil
+}
diff --git a/metropolis/node/core/rpc/peerinfo.go b/metropolis/node/core/rpc/peerinfo.go
new file mode 100644
index 0000000..a4d685c
--- /dev/null
+++ b/metropolis/node/core/rpc/peerinfo.go
@@ -0,0 +1,129 @@
+package rpc
+
+import (
+ "context"
+ "fmt"
+
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+
+ epb "source.monogon.dev/metropolis/proto/ext"
+)
+
+type Permissions map[epb.Permission]bool
+
+// PeerInfo represents the Metropolis-level information about the remote side
+// of a gRPC RPC, ie. about the calling client in server handlers and about the
+// handling server in client code.
+//
+// Exactly one of {Node, User, Unauthenticated} will be non-nil.
+type PeerInfo struct {
+ // Node is the information about a peer Node, and identifies that the other side
+ // of the connection is either a Node servicng gRPC requests for a cluster, or a
+ // Node connecting to a gRPC service.
+ Node *PeerInfoNode
+ // User is the information about a peer User, and identifies that the other side
+ // of the connection is a Metropolis user or manager (eg. owner). This will only
+ // be set in service handlers, as users cannot serve gRPC connections.
+ User *PeerInfoUser
+ // Unauthenticated is set for incoming gRPC connections which that have the
+ // Unauthenticated authorization extension set to true, and mark that the other
+ // side of the connection has not been verified at all.
+ Unauthenticated *PeerInfoUnauthenticated
+}
+
+// PeerInfoNode contains information about a Node on the other side of a gRPC
+// connection.
+type PeerInfoNode struct {
+ // PublicKey is the ED25519 public key bytes of the node.
+ PublicKey []byte
+
+ // Permissions are the set of permissions this node has.
+ Permissions Permissions
+}
+
+// PeerInfoUser contains information about a user on the other side of a gRPC
+// connection.
+type PeerInfoUser struct {
+ // Identity is an opaque identifier for the user. MVP: Currently this is always
+ // "manager".
+ Identity string
+}
+
+type PeerInfoUnauthenticated struct {
+ // SelfSignedPublicKey is the ED25519 public key bytes of the other side of the
+ // connection, if that side presented a self-signed certificate to prove control
+ // of a private key corresponding to this public key. If it did not present a
+ // self-signed certificate that can be parsed for such a key, this will be nil.
+ //
+ // This can be used by code with expects Unauthenticated RPCs but wants to
+ // authenticate the connection based on ownership of some keypair, for example
+ // in the AAA.Escrow method.
+ SelfSignedPublicKey []byte
+}
+
+// GetPeerInfo returns the PeerInfo of the peer of a gRPC connection, or nil if
+// this connection does not carry any PeerInfo.
+func GetPeerInfo(ctx context.Context) *PeerInfo {
+ if pi, ok := ctx.Value(peerInfoKey).(*PeerInfo); ok {
+ return pi
+ }
+ return nil
+}
+
+func (p *PeerInfo) CheckPermissions(need Permissions) error {
+ if p.Unauthenticated != nil {
+ // This generally shouldn't happen, as unauthenticated users shouldn't be
+ // allowed to reach this part of the code - methods with Need != nil will not be
+ // processed as unauthenticated for security, and will instead act as
+ // authenticated methods and reject unauthenticated connections.
+ for _, v := range need {
+ if v {
+ return status.Error(codes.Unauthenticated, "unauthenticated connection")
+ }
+ }
+ return nil
+ } else if p.User != nil {
+ // MVP: all permissions are granted to all users.
+ // TODO(q3k): check authz.Need once we have a user/identity system implemented.
+ return nil
+ } else if p.Node != nil {
+ for n, v := range need {
+ if v && !p.Node.Permissions[n] {
+ return status.Errorf(codes.PermissionDenied, "node missing %s permission", n.String())
+ }
+ }
+ return nil
+ }
+
+ return fmt.Errorf("invalid PeerInfo: neither Unauthenticated, User nor Node is set")
+}
+
+type peerInfoKeyType string
+
+// peerInfoKey is the context key for storing PeerInfo.
+const peerInfoKey = peerInfoKeyType("peerInfo")
+
+// apply returns the given context with itself stored under a unique key, that
+// can be later retrieved via GetPeerInfo.
+func (p *PeerInfo) apply(ctx context.Context) context.Context {
+ return context.WithValue(ctx, peerInfoKey, p)
+}
+
+// peerInfoServerStream is a grpc.ServerStream wrapper which contains some
+// PeerInfo, and returns it as part of the Context() of the ServerStream.
+type peerInfoServerStream struct {
+ grpc.ServerStream
+ pi *PeerInfo
+}
+
+func (p *peerInfoServerStream) Context() context.Context {
+ return p.pi.apply(p.ServerStream.Context())
+}
+
+// serverStream wraps a grpc.ServerStream with a structure that attaches this
+// PeerInfo in all contexts returned by Context().
+func (p *PeerInfo) serverStream(ss grpc.ServerStream) grpc.ServerStream {
+ return &peerInfoServerStream{ss, p}
+}
diff --git a/metropolis/node/core/rpc/server.go b/metropolis/node/core/rpc/server.go
index 3d6917d..70a0a74 100644
--- a/metropolis/node/core/rpc/server.go
+++ b/metropolis/node/core/rpc/server.go
@@ -1,165 +1,37 @@
package rpc
import (
- "context"
- "crypto/tls"
- "crypto/x509"
- "strings"
-
- "google.golang.org/grpc"
- "google.golang.org/grpc/codes"
- "google.golang.org/grpc/credentials"
- "google.golang.org/grpc/peer"
- "google.golang.org/grpc/status"
- "google.golang.org/protobuf/proto"
- "google.golang.org/protobuf/reflect/protoreflect"
- "google.golang.org/protobuf/reflect/protoregistry"
-
cpb "source.monogon.dev/metropolis/node/core/curator/proto/api"
apb "source.monogon.dev/metropolis/proto/api"
epb "source.monogon.dev/metropolis/proto/ext"
)
-// ClusterServices is the interface containing all gRPC services that a
-// Metropolis Cluster implements on its public interface. With the current
-// implementaiton of Metropolis, this is all implemented by the Curator.
-type ClusterServices interface {
+var (
+ // nodePermissions are the set of metropolis.common.ext.authorization
+ // permissions automatically given to nodes when connecting to curator gRPC
+ // services, either locally or remotely.
+ nodePermissions = Permissions{
+ epb.Permission_PERMISSION_READ_CLUSTER_STATUS: true,
+ }
+)
+
+// ClusterExternalServices is the interface containing all gRPC services that a
+// Metropolis Cluster implements on its external interface. With the current
+// implementation of Metropolis, this is all implemented by the Curator.
+type ClusterExternalServices interface {
cpb.CuratorServer
apb.AAAServer
apb.ManagementServer
}
-// ServerSecurity are the security options of a RPC server that will run
-// ClusterServices on a Metropolis node. It contains all the data for the
-// server implementation to authenticate itself to the clients and authenticate
-// and authorize clients connecting to it.
-type ServerSecurity struct {
- // NodeCredentials is the TLS certificate/key of the node that the server
- // implementation is running on. It should be signed by
- // ClusterCACertificate.
- NodeCredentials tls.Certificate
- // ClusterCACertificate is the cluster's CA certificate. It will be used to
- // authenticate the client certificates of incoming gRPC connections.
- ClusterCACertificate *x509.Certificate
+// ClusterInternalServices is the interface containing all gRPC services that a
+// Metropolis Cluster implements on its internal interface. Currently this is
+// just the Curator service.
+type ClusterInternalServices interface {
+ cpb.CuratorServer
}
-// SetupPublicGRPC returns a grpc.Server ready to listen and serve all public
-// gRPC APIs that the cluster server implementation should run, with all calls
-// being authenticated and authorized based on the data in ServerSecurity. The
-// argument 'impls' is the object implementing the gRPC APIs.
-//
-// This effectively configures gRPC interceptors that verify
-// metropolis.proto.ext.authorizaton options and authenticate/authorize
-// incoming connections. It also runs the gRPC server with the correct TLS
-// settings for authenticating itself to callers.
-func (l *ServerSecurity) SetupPublicGRPC(impls ClusterServices) *grpc.Server {
- publicCreds := credentials.NewTLS(&tls.Config{
- Certificates: []tls.Certificate{l.NodeCredentials},
- ClientAuth: tls.RequestClientCert,
- })
-
- s := grpc.NewServer(
- grpc.Creds(publicCreds),
- grpc.UnaryInterceptor(l.unaryInterceptor),
- grpc.StreamInterceptor(l.streamInterceptor),
- )
- cpb.RegisterCuratorServer(s, impls)
- apb.RegisterAAAServer(s, impls)
- apb.RegisterManagementServer(s, impls)
- return s
-}
-
-// authorize performs an authorization check for the given gRPC context
-// (containing peer information) and given RPC method name (as obtained from
-// FullMethodName in {Unary,Stream}ServerInfo). The actual authorization
-// requirements per method are retrieved from the Authorization protobuf
-// option applied to the RPC method.
-//
-// If the peer (as retrieved from the context) is authorized to run this method,
-// no error is returned. Otherwise, a gRPC status is returned outlining the
-// reason the authorization being rejected.
-func (l *ServerSecurity) authorize(ctx context.Context, methodName string) error {
- if !strings.HasPrefix(methodName, "/") {
- return status.Errorf(codes.InvalidArgument, "invalid method name %q", methodName)
- }
- methodName = strings.ReplaceAll(methodName[1:], "/", ".")
- desc, err := protoregistry.GlobalFiles.FindDescriptorByName(protoreflect.FullName(methodName))
- if err != nil {
- return status.Errorf(codes.InvalidArgument, "could not retrieve descriptor for method: %v", err)
- }
- method, ok := desc.(protoreflect.MethodDescriptor)
- if !ok {
- return status.Error(codes.InvalidArgument, "querying method name did not yield a MethodDescriptor")
- }
-
- // Get authorization extension, defaults to no options set.
- authz, ok := proto.GetExtension(method.Options(), epb.E_Authorization).(*epb.Authorization)
- if !ok || authz == nil {
- authz = &epb.Authorization{}
- }
-
- // If unauthenticated connections are allowed, let them through immediately.
- if authz.AllowUnauthenticated && len(authz.Need) == 0 {
- return nil
- }
-
- // Otherwise, we check that the other side of the connection is authenticated
- // using a valid cluster CA client certificate.
- p, ok := peer.FromContext(ctx)
- if !ok {
- return status.Error(codes.Unavailable, "could not retrive peer info")
- }
- tlsInfo, ok := p.AuthInfo.(credentials.TLSInfo)
- if !ok {
- return status.Error(codes.Unauthenticated, "connection not secure")
- }
- count := len(tlsInfo.State.PeerCertificates)
- if count == 0 {
- return status.Errorf(codes.Unauthenticated, "no client certificate presented")
- }
- if count > 1 {
- return status.Errorf(codes.Unauthenticated, "exactly one client certificate must be sent (got %d)", count)
- }
- pCert := tlsInfo.State.PeerCertificates[0]
-
- // Ensure that the certificate is signed by the cluster CA.
- if err := pCert.CheckSignatureFrom(l.ClusterCACertificate); err != nil {
- return status.Errorf(codes.Unauthenticated, "invalid client certificate: %v", err)
- }
- // Ensure that the certificate is a client certificate.
- // TODO(q3k): synchronize this with //metropolis/pkg/pki Client()/Server()/...
- isClient := false
- for _, ku := range pCert.ExtKeyUsage {
- if ku == x509.ExtKeyUsageClientAuth {
- isClient = true
- break
- }
- }
- if !isClient {
- return status.Error(codes.PermissionDenied, "presented certificate is not a client certificate")
- }
-
- // MVP: all permissions are granted to all users.
- // TODO(q3k): check authz.Need once we have a user/identity system implemented.
- return nil
-}
-
-// streamInterceptor is a gRPC server stream interceptor that performs
-// authentication and authorization of incoming RPCs based on the Authorization
-// option set on each method.
-func (l *ServerSecurity) streamInterceptor(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
- if err := l.authorize(ss.Context(), info.FullMethod); err != nil {
- return err
- }
- return handler(srv, ss)
-}
-
-// unaryInterceptor is a gRPC server unary interceptor that performs
-// authentication and authorization of incoming RPCs based on the Authorization
-// option set on each method.
-func (l *ServerSecurity) unaryInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
- if err := l.authorize(ctx, info.FullMethod); err != nil {
- return nil, err
- }
- return handler(ctx, req)
+type ClusterServices interface {
+ ClusterExternalServices
+ ClusterInternalServices
}
diff --git a/metropolis/node/core/rpc/server_authentication.go b/metropolis/node/core/rpc/server_authentication.go
new file mode 100644
index 0000000..6ae2618
--- /dev/null
+++ b/metropolis/node/core/rpc/server_authentication.go
@@ -0,0 +1,268 @@
+package rpc
+
+import (
+ "context"
+ "crypto/ed25519"
+ "crypto/tls"
+ "crypto/x509"
+
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/credentials"
+ "google.golang.org/grpc/peer"
+ "google.golang.org/grpc/status"
+
+ cpb "source.monogon.dev/metropolis/node/core/curator/proto/api"
+ "source.monogon.dev/metropolis/node/core/identity"
+ apb "source.monogon.dev/metropolis/proto/api"
+)
+
+// authenticationStrategy is implemented by {Local,External}ServerSecurity to
+// share logic between the two implementations.
+type authenticationStrategy interface {
+ // getPeerInfo will be called by the stream and unary gRPC server interceptors
+ // to authenticate incoming gRPC calls. It's given the gRPC context of the call
+ // (therefore allowing access to information about the underlying gRPC
+ // transport), and should return a PeerInfo structure describing the
+ // authenticated other end of the connection, or a gRPC status if the other
+ // side could not be successfully authenticated.
+ //
+ // The returned PeerInfo will then be used to perform authorization checks based
+ // on the configured authentication of a given gRPC method, as described by the
+ // metropolis.proto.ext.authorization extension. The same PeerInfo will then be
+ // available to the gRPC handler for this method by retrieving it from the
+ // context (via GetPeerInfo).
+ getPeerInfo(ctx context.Context) (*PeerInfo, error)
+
+ // getPeerInfoUnauthenticated is an equivalent to getPeerInfo, but called by the
+ // interceptors when a method is marked as 'unauthenticated'. The implementation
+ // should return a PeerInfo containing Unauthenticated, potentially populating
+ // it with UnauthenticatedPublicKey if such a public key could be retrieved.
+ getPeerInfoUnauthenticated(ctx context.Context) (*PeerInfo, error)
+}
+
+// stream implements the gRPC StreamInterceptor interface for use with
+// grpc.NewServer, based on an authenticationStrategy.
+func streamInterceptor(a authenticationStrategy) grpc.StreamServerInterceptor {
+ return func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
+ pi, err := check(ss.Context(), a, info.FullMethod)
+ if err != nil {
+ return err
+ }
+ return handler(srv, pi.serverStream(ss))
+ }
+}
+
+// unaryInterceptor implements the gRPC UnaryInterceptor interface for use with
+// grpc.NewServer, based on an authenticationStrategy.
+func unaryInterceptor(a authenticationStrategy) grpc.UnaryServerInterceptor {
+ return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
+ pi, err := check(ctx, a, info.FullMethod)
+ if err != nil {
+ return nil, err
+ }
+ return handler(pi.apply(ctx), req)
+ }
+}
+
+// check is called by the unary and server interceptors to perform
+// authentication and authorization checks for a given RPC, calling the
+// serverInterceptors' authenticate function if needed.
+func check(ctx context.Context, a authenticationStrategy, methodName string) (*PeerInfo, error) {
+ mi, err := getMethodInfo(methodName)
+ if err != nil {
+ return nil, err
+ }
+
+ if mi.unauthenticated {
+ return a.getPeerInfoUnauthenticated(ctx)
+ }
+
+ pi, err := a.getPeerInfo(ctx)
+ if err != nil {
+ return nil, err
+ }
+ if err := pi.CheckPermissions(mi.need); err != nil {
+ return nil, err
+ }
+ return pi, nil
+}
+
+// ServerSecurity are the security options of a RPC server that will run
+// ClusterServices on a Metropolis node. It contains all the data for the
+// server implementation to authenticate itself to the clients and authenticate
+// and authorize clients connecting to it.
+//
+// It implements authenticationStrategy.
+type ExternalServerSecurity struct {
+ // NodeCredentials which will be used to run the gRPC server, and whose CA
+ // certificate will be used to authenticate incoming requests.
+ NodeCredentials *identity.NodeCredentials
+
+ // nodePermissions is used by tests to inject the permissions available to a
+ // node. When not set, it defaults to the global nodePermissions map.
+ nodePermissions Permissions
+}
+
+// SetupExternalGRPC returns a grpc.Server ready to listen and serve all external
+// gRPC APIs that the cluster server implementation should run, with all calls
+// being authenticated and authorized based on the data in ServerSecurity. The
+// argument 'impls' is the object implementing the gRPC APIs.
+//
+// This effectively configures gRPC interceptors that verify
+// metropolis.proto.ext.authorization options and authenticate/authorize
+// incoming connections. It also runs the gRPC server with the correct TLS
+// settings for authenticating itself to callers.
+func (l *ExternalServerSecurity) SetupExternalGRPC(impls ClusterExternalServices) *grpc.Server {
+ externalCreds := credentials.NewTLS(&tls.Config{
+ Certificates: []tls.Certificate{l.NodeCredentials.TLSCredentials()},
+ ClientAuth: tls.RequestClientCert,
+ })
+
+ s := grpc.NewServer(
+ grpc.Creds(externalCreds),
+ grpc.UnaryInterceptor(unaryInterceptor(l)),
+ grpc.StreamInterceptor(streamInterceptor(l)),
+ )
+ cpb.RegisterCuratorServer(s, impls)
+ apb.RegisterAAAServer(s, impls)
+ apb.RegisterManagementServer(s, impls)
+ return s
+}
+
+func (l *ExternalServerSecurity) getPeerInfo(ctx context.Context) (*PeerInfo, error) {
+ cert, err := getPeerCertificate(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ // Ensure that the certificate is signed by the cluster CA.
+ if err := cert.CheckSignatureFrom(l.NodeCredentials.ClusterCA()); err != nil {
+ return nil, status.Errorf(codes.Unauthenticated, "certificate not signed by cluster CA: %v", err)
+ }
+
+ nodepk, errNode := identity.VerifyNodeInCluster(cert, l.NodeCredentials.ClusterCA())
+ if errNode == nil {
+ // This is a Metropolis node.
+ np := l.nodePermissions
+ if np == nil {
+ np = nodePermissions
+ }
+ return &PeerInfo{
+ Node: &PeerInfoNode{
+ PublicKey: nodepk,
+ Permissions: np,
+ },
+ }, nil
+ }
+
+ userid, errUser := identity.VerifyUserInCluster(cert, l.NodeCredentials.ClusterCA())
+ if errUser == nil {
+ // This is a Metropolis user/manager.
+ return &PeerInfo{
+ User: &PeerInfoUser{
+ Identity: userid,
+ },
+ }, nil
+ }
+
+ // Could not parse as either node or user certificate.
+ return nil, status.Errorf(codes.Unauthenticated, "presented certificate is neither user certificate (%v) nor node certificate (%v)", errUser, errNode)
+}
+
+func (l *ExternalServerSecurity) getPeerInfoUnauthenticated(ctx context.Context) (*PeerInfo, error) {
+ res := PeerInfo{
+ Unauthenticated: &PeerInfoUnauthenticated{},
+ }
+
+ // If peer presented a valid self-signed certificate, attach that to the
+ // Unauthenticated struct.
+ cert, err := getPeerCertificate(ctx)
+ if err == nil {
+ if err := cert.CheckSignature(cert.SignatureAlgorithm, cert.RawTBSCertificate, cert.Signature); err != nil {
+ return nil, status.Errorf(codes.Unauthenticated, "presented certificate must be self-signed (check error: %v)", err)
+ }
+ res.Unauthenticated.SelfSignedPublicKey = cert.PublicKey.(ed25519.PublicKey)
+ }
+
+ return &res, nil
+}
+
+// getPeerCertificate returns the x509 certificate associated with the given
+// gRPC connection's context and ensures that it is a certificate for an Ed25519
+// keypair. The certificate is _not_ checked against the cluster CA.
+//
+// A gRPC status is returned if the certificate is invalid / unauthenticated for
+// any reason.
+func getPeerCertificate(ctx context.Context) (*x509.Certificate, error) {
+ p, ok := peer.FromContext(ctx)
+ if !ok {
+ return nil, status.Error(codes.Unavailable, "could not retrive peer info")
+ }
+ tlsInfo, ok := p.AuthInfo.(credentials.TLSInfo)
+ if !ok {
+ return nil, status.Error(codes.Unauthenticated, "connection not secure")
+ }
+ count := len(tlsInfo.State.PeerCertificates)
+ if count == 0 {
+ return nil, status.Errorf(codes.Unauthenticated, "no client certificate presented")
+ }
+ if count > 1 {
+ return nil, status.Errorf(codes.Unauthenticated, "exactly one client certificate must be sent (got %d)", count)
+ }
+ cert := tlsInfo.State.PeerCertificates[0]
+ if _, ok := cert.PublicKey.(ed25519.PublicKey); !ok {
+ return nil, status.Errorf(codes.Unauthenticated, "certificate must be issued for an ED25519 keypair")
+ }
+
+ return cert, nil
+}
+
+// LocalServerSecurity are the security options of an RPC server that will run
+// the Curator service over a local domain socket. When set up using
+// LocalServerSecurity, all incoming RPCs will be authenticated as coming from
+// the node that this service is running on.
+//
+// It implements authenticationStrategy.
+type LocalServerSecurity struct {
+ // Node for which the gRPC server will authenticate all incoming requests as
+ // originating from.
+ Node *identity.Node
+
+ // nodePermissions is used by tests to inject the permissions available to a
+ // node. When not set, it defaults to the global nodePermissions map.
+ nodePermissions Permissions
+}
+
+// SetupLocalGRPC returns a grpc.Server ready to listen on a local domain socket
+// and serve the Curator service. All incoming RPCs will be authenticated as
+// originating from the node for which LocalServerSecurity has been configured.
+func (l *LocalServerSecurity) SetupLocalGRPC(impls ClusterInternalServices) *grpc.Server {
+ s := grpc.NewServer(
+ grpc.UnaryInterceptor(unaryInterceptor(l)),
+ grpc.StreamInterceptor(streamInterceptor(l)),
+ )
+ cpb.RegisterCuratorServer(s, impls)
+ return s
+}
+
+func (l *LocalServerSecurity) getPeerInfo(_ context.Context) (*PeerInfo, error) {
+ // Local connections are always node connections.
+ np := l.nodePermissions
+ if np == nil {
+ np = nodePermissions
+ }
+ return &PeerInfo{
+ Node: &PeerInfoNode{
+ PublicKey: l.Node.PublicKey(),
+ Permissions: np,
+ },
+ }, nil
+}
+
+func (l *LocalServerSecurity) getPeerInfoUnauthenticated(_ context.Context) (*PeerInfo, error) {
+ // This shouldn't happen - why would we call unauthenticated methods locally?
+ // This can be implemented, but doesn't really make sense. For now, assume this
+ // is a programming error. This can be changed if needed.
+ return nil, status.Errorf(codes.Unauthenticated, "unauthenticated methods not supported over local connections")
+}
diff --git a/metropolis/node/core/rpc/server_authentication_test.go b/metropolis/node/core/rpc/server_authentication_test.go
new file mode 100644
index 0000000..d383150
--- /dev/null
+++ b/metropolis/node/core/rpc/server_authentication_test.go
@@ -0,0 +1,171 @@
+package rpc
+
+import (
+ "context"
+ "crypto/ed25519"
+ "crypto/rand"
+ "testing"
+
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+ "google.golang.org/grpc/test/bufconn"
+
+ cpb "source.monogon.dev/metropolis/node/core/curator/proto/api"
+ apb "source.monogon.dev/metropolis/proto/api"
+ epb "source.monogon.dev/metropolis/proto/ext"
+)
+
+// testImplementations implements ClusterServices by returning 'unimplementd'
+// for every RPC call.
+type testImplementation struct {
+ cpb.UnimplementedCuratorServer
+ apb.UnimplementedAAAServer
+ apb.UnimplementedManagementServer
+}
+
+// TestExternalServerSecurity ensures that the unary interceptor of the
+// ServerSecurity structure works, and authenticates/authorizes incoming RPCs as
+// expected.
+func TestExternalServerSecurity(t *testing.T) {
+ ctx, ctxC := context.WithCancel(context.Background())
+ defer ctxC()
+
+ eph := NewEphemeralClusterCredentials(t, 1)
+ permissions := make(Permissions)
+ for k, v := range nodePermissions {
+ permissions[k] = v
+ }
+ ss := ExternalServerSecurity{
+ NodeCredentials: eph.Nodes[0],
+ nodePermissions: permissions,
+ }
+
+ impl := &testImplementation{}
+ srv := ss.SetupExternalGRPC(impl)
+ lis := bufconn.Listen(1024 * 1024)
+ go func() {
+ if err := srv.Serve(lis); err != nil {
+ t.Fatalf("GRPC serve failed: %v", err)
+ }
+ }()
+ defer lis.Close()
+ defer srv.Stop()
+
+ // Authenticate as manager externally, ensure that GetRegisterTicket runs.
+ cl, err := NewAuthenticatedClientTest(lis, eph.Manager, eph.CA)
+ if err != nil {
+ t.Fatalf("NewAuthenticatedClient: %v", err)
+ }
+ defer cl.Close()
+ mgmt := apb.NewManagementClient(cl)
+ _, err = mgmt.GetRegisterTicket(ctx, &apb.GetRegisterTicketRequest{})
+ if s, ok := status.FromError(err); !ok || s.Code() != codes.Unimplemented {
+ t.Errorf("GetRegisterTicket returned %v, wanted codes.Unimplemented", err)
+ }
+
+ // Authenticate as node externally, ensure that GetRegisterTicket is refused
+ // (this is because nodes miss the GET_REGISTER_TICKET permissions).
+ cl, err = NewAuthenticatedClientTest(lis, eph.Nodes[0].TLSCredentials(), eph.CA)
+ if err != nil {
+ t.Fatalf("NewAuthenticatedClient: %v", err)
+ }
+ defer cl.Close()
+ mgmt = apb.NewManagementClient(cl)
+ _, err = mgmt.GetRegisterTicket(ctx, &apb.GetRegisterTicketRequest{})
+ if s, ok := status.FromError(err); !ok || s.Code() != codes.PermissionDenied {
+ t.Errorf("GetRegisterTicket (by external node) returned %v, wanted codes.PermissionDenied", err)
+ }
+
+ // Give the node GET_REGISTER_TICKET permissions and try again. This should pass.
+ permissions[epb.Permission_PERMISSION_GET_REGISTER_TICKET] = true
+ _, err = mgmt.GetRegisterTicket(ctx, &apb.GetRegisterTicketRequest{})
+ if s, ok := status.FromError(err); !ok || s.Code() != codes.Unimplemented {
+ t.Errorf("GetRegisterTicket returned %v, wanted codes.Unimplemented", err)
+ }
+ permissions[epb.Permission_PERMISSION_GET_REGISTER_TICKET] = false
+
+ // Authenticate with an ephemeral/self-signed certificate, ensure that
+ // GetRegisterTicket is refused (this is because GetRegisterTicket requires an
+ // authenticated connection).
+ _, sk, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatalf("GenerateKey: %v", err)
+ }
+ cl, err = NewEphemeralClientTest(lis, sk, eph.CA)
+ if err != nil {
+ t.Fatalf("NewEphemeralClient: %v", err)
+ }
+ defer cl.Close()
+ mgmt = apb.NewManagementClient(cl)
+ _, err = mgmt.GetRegisterTicket(ctx, &apb.GetRegisterTicketRequest{})
+ if s, ok := status.FromError(err); !ok || s.Code() != codes.Unauthenticated {
+ t.Errorf("GetRegisterTicket (by ephemeral cert) returned %v, wanted codes.Unauthenticated", err)
+ }
+}
+
+// TestLocalServerSecurity ensures that the unary interceptor of the
+// LocalServerSecurity structure works, and authenticates/authorizes incoming
+// RPCs as expected.
+func TestLocalServerSecurity(t *testing.T) {
+ ctx, ctxC := context.WithCancel(context.Background())
+ defer ctxC()
+
+ eph := NewEphemeralClusterCredentials(t, 1)
+
+ permissions := make(Permissions)
+ for k, v := range nodePermissions {
+ permissions[k] = v
+ }
+
+ ls := LocalServerSecurity{
+ Node: &eph.Nodes[0].Node,
+ nodePermissions: permissions,
+ }
+
+ impl := &testImplementation{}
+ srv := ls.SetupLocalGRPC(impl)
+ lis := bufconn.Listen(1024 * 1024)
+ go func() {
+ if err := srv.Serve(lis); err != nil {
+ t.Fatalf("GRPC serve failed: %v", err)
+ }
+ }()
+ defer lis.Close()
+ defer srv.Stop()
+
+ // Nodes should have access to Curator.Watch.
+ cl, err := NewNodeClientTest(lis)
+ if err != nil {
+ t.Fatalf("NewAuthenticatedClient: %v", err)
+ }
+ defer cl.Close()
+
+ curator := cpb.NewCuratorClient(cl)
+ req := &cpb.WatchRequest{
+ Kind: &cpb.WatchRequest_NodeInCluster_{
+ NodeInCluster: &cpb.WatchRequest_NodeInCluster{
+ NodeId: eph.Nodes[0].ID(),
+ },
+ },
+ }
+ w, err := curator.Watch(ctx, req)
+ if err != nil {
+ t.Fatalf("Watch: %v", err)
+ }
+ _, err = w.Recv()
+ if s, ok := status.FromError(err); !ok || s.Code() != codes.Unimplemented {
+ t.Errorf("Watch (by local node) returned %v, wanted codes.Unimplemented", err)
+ }
+
+ // Take away the node's PERMISSION_READ_CLUSTER_STATUS permissions and try
+ // again. This should fail.
+ permissions[epb.Permission_PERMISSION_READ_CLUSTER_STATUS] = false
+ w, err = curator.Watch(ctx, req)
+ if err != nil {
+ t.Fatalf("Watch: %v", err)
+ }
+ _, err = w.Recv()
+ if s, ok := status.FromError(err); !ok || s.Code() != codes.PermissionDenied {
+ t.Errorf("Watch (by local node after removing permission) returned %v, wanted codes.PermissionDenied", err)
+ }
+}
diff --git a/metropolis/node/core/rpc/testhelpers.go b/metropolis/node/core/rpc/testhelpers.go
new file mode 100644
index 0000000..93e4b46
--- /dev/null
+++ b/metropolis/node/core/rpc/testhelpers.go
@@ -0,0 +1,100 @@
+package rpc
+
+import (
+ "context"
+ "crypto/ed25519"
+ "crypto/rand"
+ "crypto/tls"
+ "crypto/x509"
+ "testing"
+
+ "source.monogon.dev/metropolis/node/core/identity"
+ "source.monogon.dev/metropolis/pkg/pki"
+)
+
+// NewEphemeralClusterCredentials creates a set of TLS certificates for use in a
+// test Metropolis cluster. These are a CA certificate, a Manager certificate
+// and an arbitrary amount of Node certificates (per the nodes argument).
+//
+// All of these are ephemeral, ie. not stored anywhere - including the CA
+// certificate. This function is for use by tests which want to bring up a
+// minimum set of PKI credentials for a fake Metropolis cluster.
+func NewEphemeralClusterCredentials(t *testing.T, nodes int) *EphemeralClusterCredentials {
+ ctx := context.Background()
+ t.Helper()
+
+ ns := pki.Namespaced("unused")
+ caCert := pki.Certificate{
+ Namespace: &ns,
+ Issuer: pki.SelfSigned,
+ Template: identity.CACertificate("test cluster ca"),
+ Mode: pki.CertificateEphemeral,
+ }
+ caBytes, err := caCert.Ensure(ctx, nil)
+ if err != nil {
+ t.Fatalf("Could not ensure CA certificate: %v", err)
+ }
+ ca, err := x509.ParseCertificate(caBytes)
+ if err != nil {
+ t.Fatalf("Could not parse new CA certificate: %v", err)
+ }
+
+ managerCert := pki.Certificate{
+ Namespace: &ns,
+ Issuer: &caCert,
+ Template: identity.UserCertificate("owner"),
+ Mode: pki.CertificateEphemeral,
+ }
+ managerBytes, err := managerCert.Ensure(ctx, nil)
+ if err != nil {
+ t.Fatalf("Could not ensure manager certificate: %v", err)
+ }
+ res := &EphemeralClusterCredentials{
+ Nodes: make([]*identity.NodeCredentials, nodes),
+ Manager: tls.Certificate{
+ Certificate: [][]byte{managerBytes},
+ PrivateKey: managerCert.PrivateKey,
+ },
+ CA: ca,
+ }
+
+ for i := 0; i < nodes; i++ {
+ npk, npr, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatalf("Could not generate node keypair: %v", err)
+ }
+ nodeCert := pki.Certificate{
+ Namespace: &ns,
+ Issuer: &caCert,
+ Template: identity.NodeCertificate(npk),
+ Mode: pki.CertificateEphemeral,
+ PublicKey: npk,
+ Name: "",
+ }
+ nodeBytes, err := nodeCert.Ensure(ctx, nil)
+ if err != nil {
+ t.Fatalf("Could not ensure node certificate: %v", err)
+ }
+ node, err := identity.NewNodeCredentials(npr, nodeBytes, caBytes)
+ if err != nil {
+ t.Fatalf("Could not build node credentials: %v", err)
+ }
+ res.Nodes[i] = node
+ }
+
+ return res
+}
+
+// EphemeralClusterCredentials are TLS/PKI credentials for use in a Metropolis
+// test cluster.
+type EphemeralClusterCredentials struct {
+ // Nodes are the node credentials for the cluster. Each contains a private
+ // key and x509 certificate authenticating the bearer as a Metropolis node.
+ Nodes []*identity.NodeCredentials
+ // Manager TLS certificate for the cluster. Contains a private key and x509
+ // certificate authenticating the bearer as a Metropolis manager.
+ Manager tls.Certificate
+ // CA is the x509 certificate of the CA certificate for the cluster. Manager and
+ // Node certificates are signed by this CA.
+ CA *x509.Certificate
+}