Attestation & Identity & Global Unlock & Enrolment
This changes the node startup sequence significantly. Now the following three startup procedures replace the old setup/join mechanic:
* If no enrolment config is present, automatically bootstrap a new cluster and become master for it.
* If an enrolment config with an enrolment token is present, register with the NodeManagementService.
* If an enrolment config without an enrolment token is present, attempt a normal cluster unlock.
It also completely revamps the GRPC management services:
* NodeManagementService is a master-only service that deals with other nodes and has a cluster-wide identity
* NodeService is only available in unlocked state and keyed with the node identity
* ClusterManagement is now a master-only service that's been spun out of the main NMS since they have very different authentication models and also deals with EnrolmentConfigs
The TPM support library has also been extended by:
* Lots of integrity attestation and verification functions
* Built-in AK management
* Some advanced policy-based authentication stuff
Also contains various enhancements to the network service to make everything work in a proper multi-node environment.
Lots of old code has been thrown out.
Test Plan: Passed a full manual test of all three startup modes (bootstrap, enrolment and normal unlock) including automated EnrolmentConfig generation and consumption in a dual-node configuration on swtpm / OVMF.
Bug: T499
X-Origin-Diff: phab/D291
GitOrigin-RevId: d53755c828218b1df83a1d7ad252c7b3231abca8
diff --git a/core/internal/api/server.go b/core/internal/api/server.go
index efd0be5..4e1e5fa 100644
--- a/core/internal/api/server.go
+++ b/core/internal/api/server.go
@@ -17,23 +17,36 @@
package api
import (
+ "context"
+ "crypto/ed25519"
+ "crypto/rand"
+ "crypto/tls"
+ "crypto/x509"
+ "crypto/x509/pkix"
+ "errors"
"fmt"
+ "math/big"
+ "net"
+ "time"
+
+ "git.monogon.dev/source/nexantic.git/core/generated/api"
schema "git.monogon.dev/source/nexantic.git/core/generated/api"
"git.monogon.dev/source/nexantic.git/core/internal/common"
"git.monogon.dev/source/nexantic.git/core/internal/common/service"
"git.monogon.dev/source/nexantic.git/core/internal/consensus"
+ "go.etcd.io/etcd/clientv3"
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/reflection"
- "net"
+ "google.golang.org/grpc/credentials"
)
type (
Server struct {
*service.BaseService
- setupService common.SetupService
- grpcServer *grpc.Server
+ grpcServer *grpc.Server
+ externalGrpcServer *grpc.Server
consensusService *consensus.Service
@@ -41,49 +54,186 @@
}
Config struct {
- Port uint16
}
)
-func NewApiServer(config *Config, logger *zap.Logger, setupService common.SetupService, consensusService *consensus.Service) (*Server, error) {
+var (
+ // From RFC 5280 Section 4.1.2.5
+ unknownNotAfter = time.Unix(253402300799, 0)
+)
+
+func NewApiServer(config *Config, logger *zap.Logger, consensusService *consensus.Service) (*Server, error) {
s := &Server{
config: config,
- setupService: setupService,
consensusService: consensusService,
}
s.BaseService = service.NewBaseService("api", logger, s)
- grpcServer := grpc.NewServer()
- schema.RegisterClusterManagementServer(grpcServer, s)
- schema.RegisterSetupServiceServer(grpcServer, s)
-
- reflection.Register(grpcServer)
-
- s.grpcServer = grpcServer
-
return s, nil
}
+func (s *Server) getStore() clientv3.KV {
+ // Cannot be moved to initialization because an internal reference will be nil
+ return s.consensusService.GetStore("api", "")
+}
+
+// BootstrapNewClusterHook creates the necessary key material for the API Servers and stores it in
+// the consensus service. It also creates a node entry for the initial node.
+func (s *Server) BootstrapNewClusterHook(initNodeReq *api.NewNodeInfo) error {
+ serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 127)
+ serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+ if err != nil {
+ return fmt.Errorf("Failed to generate serial number: %w", err)
+ }
+
+ pubKey, privKeyRaw, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ return err
+ }
+ privkey, err := x509.MarshalPKCS8PrivateKey(privKeyRaw)
+ if err != nil {
+ return err
+ }
+
+ // This has no SANs because it authenticates by public key, not by name
+ masterCert := &x509.Certificate{
+ SerialNumber: serialNumber,
+ Subject: pkix.Name{
+ CommonName: "Smalltown Master",
+ },
+ IsCA: false,
+ BasicConstraintsValid: true,
+ NotBefore: time.Now(),
+ NotAfter: unknownNotAfter,
+ // Certificate is used both as server & client
+ ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
+ }
+ cert, err := x509.CreateCertificate(rand.Reader, masterCert, masterCert, pubKey, privKeyRaw)
+ if err != nil {
+ return err
+ }
+ store := s.getStore()
+ if _, err := store.Put(context.Background(), "master.der", string(cert)); err != nil {
+ return err
+ }
+ if _, err := store.Put(context.Background(), "master-key.der", string(privkey)); err != nil {
+ return err
+ }
+
+ // TODO: Further integrity providers need to be plumbed in here
+ node, err := s.TPM2BootstrapNode(initNodeReq)
+ if err != nil {
+ return err
+ }
+
+ if err := s.registerNewNode(node); err != nil {
+ return err
+ }
+ return nil
+}
+
+// GetMasterCert gets the master certificate in X.509 DER form
+// This is mainly used to issue enrolment configs
+func (s *Server) GetMasterCert() ([]byte, error) {
+ store := s.getStore()
+ res, err := store.Get(context.Background(), "master.der")
+ if err != nil {
+ return []byte{}, err
+ }
+ if len(res.Kvs) != 1 {
+ return []byte{}, errors.New("master certificate not found")
+ }
+ certRaw := res.Kvs[0].Value
+ return certRaw, nil
+}
+
+// TODO(lorenz): Move consensus/certificate interaction into a utility, is now duplicated too often
+func (s *Server) loadMasterCert() (*tls.Certificate, error) {
+
+ store := s.getStore()
+ var tlsCert tls.Certificate
+ res, err := store.Get(context.Background(), "master.der")
+ if err != nil {
+ return nil, err
+ }
+ if len(res.Kvs) != 1 {
+ return nil, errors.New("master certificate not found")
+ }
+ certRaw := res.Kvs[0].Value
+
+ tlsCert.Certificate = append(tlsCert.Certificate, certRaw)
+ tlsCert.Leaf, err = x509.ParseCertificate(certRaw)
+
+ res, err = store.Get(context.Background(), "master-key.der")
+ if err != nil {
+ return nil, err
+ }
+ if len(res.Kvs) != 1 {
+ return nil, errors.New("master certificate not found")
+ }
+ keyRaw := res.Kvs[0].Value
+ key, err := x509.ParsePKCS8PrivateKey(keyRaw)
+ if err != nil {
+ return nil, fmt.Errorf("failed to load master private key: %w", err)
+ }
+ edKey, ok := key.(ed25519.PrivateKey)
+ if !ok {
+ return nil, errors.New("invalid private key")
+ }
+ tlsCert.PrivateKey = edKey
+ return &tlsCert, nil
+}
+
func (s *Server) OnStart() error {
- listenHost := fmt.Sprintf(":%d", s.config.Port)
- lis, err := net.Listen("tcp", listenHost)
+ masterListenHost := fmt.Sprintf(":%d", common.MasterServicePort)
+ lis, err := net.Listen("tcp", masterListenHost)
if err != nil {
s.Logger.Fatal("failed to listen", zap.Error(err))
}
+ externalListeneHost := fmt.Sprintf(":%d", common.ExternalServicePort)
+ externalListener, err := net.Listen("tcp", externalListeneHost)
+ if err != nil {
+ s.Logger.Fatal("failed to listen", zap.Error(err))
+ }
+
+ masterCert, err := s.loadMasterCert()
+ if err != nil {
+ s.Logger.Error("Failed to load Master Service Key Material: %w", zap.Error(err))
+ return err
+ }
+
+ masterTransportCredentials := credentials.NewServerTLSFromCert(masterCert)
+
+ masterGrpcServer := grpc.NewServer(grpc.Creds(masterTransportCredentials))
+ clusterManagementGrpcServer := grpc.NewServer()
+ schema.RegisterClusterManagementServer(clusterManagementGrpcServer, s)
+ schema.RegisterNodeManagementServiceServer(masterGrpcServer, s)
+
+ reflection.Register(masterGrpcServer)
+
+ s.grpcServer = masterGrpcServer
+ s.externalGrpcServer = clusterManagementGrpcServer
+
go func() {
err = s.grpcServer.Serve(lis)
s.Logger.Error("API server failed", zap.Error(err))
}()
- s.Logger.Info("gRPC listening", zap.String("host", listenHost))
+ go func() {
+ err = s.externalGrpcServer.Serve(externalListener)
+ s.Logger.Error("API server failed", zap.Error(err))
+ }()
+
+ s.Logger.Info("gRPC listening", zap.String("host", masterListenHost))
return nil
}
func (s *Server) OnStop() error {
s.grpcServer.Stop()
+ s.externalGrpcServer.Stop()
return nil
}