Attestation & Identity & Global Unlock & Enrolment
This changes the node startup sequence significantly. Now the following three startup procedures replace the old setup/join mechanic:
* If no enrolment config is present, automatically bootstrap a new cluster and become master for it.
* If an enrolment config with an enrolment token is present, register with the NodeManagementService.
* If an enrolment config without an enrolment token is present, attempt a normal cluster unlock.
It also completely revamps the GRPC management services:
* NodeManagementService is a master-only service that deals with other nodes and has a cluster-wide identity
* NodeService is only available in unlocked state and keyed with the node identity
* ClusterManagement is now a master-only service that's been spun out of the main NMS since they have very different authentication models and also deals with EnrolmentConfigs
The TPM support library has also been extended by:
* Lots of integrity attestation and verification functions
* Built-in AK management
* Some advanced policy-based authentication stuff
Also contains various enhancements to the network service to make everything work in a proper multi-node environment.
Lots of old code has been thrown out.
Test Plan: Passed a full manual test of all three startup modes (bootstrap, enrolment and normal unlock) including automated EnrolmentConfig generation and consumption in a dual-node configuration on swtpm / OVMF.
Bug: T499
X-Origin-Diff: phab/D291
GitOrigin-RevId: d53755c828218b1df83a1d7ad252c7b3231abca8
diff --git a/core/internal/api/BUILD.bazel b/core/internal/api/BUILD.bazel
index e7ec6f8..e862340 100644
--- a/core/internal/api/BUILD.bazel
+++ b/core/internal/api/BUILD.bazel
@@ -4,19 +4,24 @@
name = "go_default_library",
srcs = [
"cluster.go",
+ "enrolment.go",
+ "nodemanagement.go",
+ "nodes.go",
"server.go",
- "setup.go",
],
importpath = "git.monogon.dev/source/nexantic.git/core/internal/api",
visibility = ["//core:__subpackages__"],
deps = [
"//core/api/api:go_default_library",
"//core/internal/common:go_default_library",
- "//core/internal/common/grpc:go_default_library",
"//core/internal/common/service:go_default_library",
"//core/internal/consensus:go_default_library",
+ "//core/pkg/tpm:go_default_library",
+ "@com_github_gogo_protobuf//proto:go_default_library",
+ "@io_etcd_go_etcd//clientv3:go_default_library",
"@org_golang_google_grpc//:go_default_library",
"@org_golang_google_grpc//codes:go_default_library",
+ "@org_golang_google_grpc//credentials:go_default_library",
"@org_golang_google_grpc//reflection:go_default_library",
"@org_golang_google_grpc//status:go_default_library",
"@org_uber_go_zap//:go_default_library",
diff --git a/core/internal/api/cluster.go b/core/internal/api/cluster.go
index d2c18c3..dc794b3 100644
--- a/core/internal/api/cluster.go
+++ b/core/internal/api/cluster.go
@@ -19,112 +19,97 @@
import (
"context"
"crypto/rand"
- "encoding/hex"
- "fmt"
+ "encoding/base64"
+ "io"
+
+ "git.monogon.dev/source/nexantic.git/core/generated/api"
schema "git.monogon.dev/source/nexantic.git/core/generated/api"
- "git.monogon.dev/source/nexantic.git/core/internal/common/grpc"
+ "github.com/gogo/protobuf/proto"
+ "go.etcd.io/etcd/clientv3"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"go.uber.org/zap"
)
-var (
- ErrAttestationFailed = status.Error(codes.PermissionDenied, "attestation failed")
-)
-
func (s *Server) AddNode(ctx context.Context, req *schema.AddNodeRequest) (*schema.AddNodeResponse, error) {
- // Setup API client
- c, err := grpc.NewSmalltownAPIClient(fmt.Sprintf("%s:%d", req.Addr, s.config.Port))
- if err != nil {
- return nil, err
- }
-
- // Check attestation
- nonce := make([]byte, 20)
- _, err = rand.Read(nonce)
- if err != nil {
- s.Logger.Error("Nonce generation failed", zap.Error(err))
- return nil, status.Error(codes.Unavailable, "nonce generation failed")
- }
- hexNonce := hex.EncodeToString(nonce)
-
- aRes, err := c.Setup.Attest(ctx, &schema.AttestRequest{
- Challenge: hexNonce,
- })
- if err != nil {
- s := status.Convert(err)
- return nil, status.Errorf(s.Code(), "attestation failed: %v", s.Message())
- }
-
- //TODO(hendrik): Verify response
- if aRes.Response != hexNonce {
- return nil, ErrAttestationFailed
- }
-
- consensusCerts, err := s.consensusService.IssueCertificate(req.Addr)
- if err != nil {
- // Errors from IssueCertificate are always treated as internal
- s.Logger.Error("Node certificate issuance failed", zap.String("addr", req.Addr), zap.Error(err))
- return nil, status.Error(codes.Internal, "could not issue node certificate")
- }
-
- // TODO(leo): fetch remote hostname rather than using the addr
- name := req.Addr
-
- // Add new node to local etcd cluster.
- memberID, err := s.consensusService.AddMember(ctx, name, fmt.Sprintf("https://%s:%d", req.Addr, s.config.Port))
- if err != nil {
- return nil, status.Errorf(codes.Unavailable, "failed to add node to etcd cluster: %v", err)
- }
-
- s.Logger.Info("Added new node to consensus cluster; sending cluster join request to node",
- zap.String("addr", req.Addr), zap.Uint16("port", s.config.Port))
-
- // Send JoinCluster request to new node to make it join.
- _, err = c.Setup.JoinCluster(ctx, &schema.JoinClusterRequest{
- InitialCluster: s.consensusService.GetInitialClusterString(),
- ProvisioningToken: req.ProvisioningToken,
- Certs: consensusCerts,
- })
- if err != nil {
- errRevoke := s.consensusService.RevokeCertificate(req.Addr)
- if errRevoke != nil {
- s.Logger.Error("Failed to revoke a certificate after rollback - potential security risk", zap.Error(errRevoke))
- }
- // Revert etcd add member - might fail if consensus cannot be established.
- errRemove := s.consensusService.RemoveMember(ctx, memberID)
- if errRemove != nil || errRevoke != nil {
- return nil, fmt.Errorf("rollback failed after failed provisioning; err=%v; err_rb=%v; err_revoke=%v", err, errRemove, errRevoke)
- }
- return nil, status.Errorf(codes.Unavailable, "failed to join etcd cluster with node: %v", err)
- }
- s.Logger.Info("Fully provisioned new node",
- zap.String("host", req.Addr),
- zap.Uint16("apiPort", s.config.Port),
- zap.Uint64("member_id", memberID))
-
- return &schema.AddNodeResponse{}, nil
+ return nil, status.Error(codes.Unimplemented, "Unimplemented")
}
-func (s *Server) RemoveNode(context.Context, *schema.RemoveNodeRequest) (*schema.RemoveNodeResponse, error) {
- return nil, status.Error(codes.Unimplemented, "unimplemented")
+func (s *Server) RemoveNode(ctx context.Context, req *schema.RemoveNodeRequest) (*schema.RemoveNodeRequest, error) {
+ return nil, status.Error(codes.Unimplemented, "Unimplemented")
}
-func (s *Server) ListNodes(context.Context, *schema.ListNodesRequest) (*schema.ListNodesResponse, error) {
- nodes := s.consensusService.GetNodes()
- resNodes := make([]*schema.Node, len(nodes))
-
- for i, node := range nodes {
- resNodes[i] = &schema.Node{
- Id: node.ID,
- Name: node.Name,
- Address: node.Address,
- Synced: node.Synced,
+func (s *Server) ListNodes(ctx context.Context, req *schema.ListNodesRequest) (*schema.ListNodesResponse, error) {
+ store := s.getStore()
+ res, err := store.Get(ctx, "nodes/", clientv3.WithPrefix())
+ if err != nil {
+ return nil, status.Error(codes.Unavailable, "Consensus unavailable")
+ }
+ var resNodes []*api.Node
+ for _, nodeEntry := range res.Kvs {
+ var node api.Node
+ if err := proto.Unmarshal(nodeEntry.Value, &node); err != nil {
+ s.Logger.Error("Encountered invalid node data", zap.Error(err))
+ return nil, status.Error(codes.Internal, "Invalid data")
}
+ // Zero out Global Unlock Key, it's never supposed to leave the cluster
+ node.GlobalUnlockKey = []byte{}
+
+ resNodes = append(resNodes, &node)
}
return &schema.ListNodesResponse{
Nodes: resNodes,
}, nil
}
+
+func (s *Server) ListEnrolmentConfigs(ctx context.Context, req *api.ListEnrolmentConfigsRequest) (*api.ListEnrolmentConfigsResponse, error) {
+ return nil, status.Error(codes.Unimplemented, "Unimplemented")
+}
+
+func (s *Server) NewEnrolmentConfig(ctx context.Context, req *api.NewEnrolmentConfigRequest) (*api.NewEnrolmentConfigResponse, error) {
+ store := s.getStore()
+ token := make([]byte, 32)
+ if _, err := io.ReadFull(rand.Reader, token); err != nil {
+ return nil, status.Error(codes.Unavailable, "failed to get randonmess")
+ }
+ nodes, err := store.Get(ctx, "nodes/", clientv3.WithPrefix())
+ if err != nil {
+ return nil, status.Error(codes.Unavailable, "consensus unavailable")
+ }
+ var masterIPs [][]byte
+ for _, nodeKV := range nodes.Kvs {
+ var node api.Node
+ if err := proto.Unmarshal(nodeKV.Value, &node); err != nil {
+ return nil, status.Error(codes.Internal, "invalid node")
+ }
+ if node.State == api.Node_MASTER {
+ masterIPs = append(masterIPs, node.Address)
+ }
+ }
+ masterCert, err := s.GetMasterCert()
+ if err != nil {
+ return nil, status.Error(codes.Unavailable, "consensus unavailable")
+ }
+
+ enrolmentConfig := &api.EnrolmentConfig{
+ EnrolmentSecret: token,
+ MasterIps: masterIPs,
+ MastersCert: masterCert,
+ }
+ enrolmentConfigRaw, err := proto.Marshal(enrolmentConfig)
+ if err != nil {
+ return nil, status.Error(codes.Internal, "failed to encode config")
+ }
+ if _, err := store.Put(ctx, "enrolments/"+base64.RawURLEncoding.EncodeToString(token), string(enrolmentConfigRaw)); err != nil {
+ return nil, status.Error(codes.Unavailable, "consensus unavailable")
+ }
+ return &schema.NewEnrolmentConfigResponse{
+ EnrolmentConfig: enrolmentConfig,
+ }, nil
+}
+
+func (s *Server) RemoveEnrolmentConfig(ctx context.Context, req *api.RemoveEnrolmentConfigRequest) (*api.RemoveEnrolmentConfigResponse, error) {
+ return nil, status.Error(codes.Unimplemented, "Unimplemented")
+}
diff --git a/core/internal/api/enrolment.go b/core/internal/api/enrolment.go
new file mode 100644
index 0000000..976b0f2
--- /dev/null
+++ b/core/internal/api/enrolment.go
@@ -0,0 +1,55 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+ "context"
+ "encoding/base64"
+ "errors"
+ "fmt"
+
+ "git.monogon.dev/source/nexantic.git/core/generated/api"
+ "github.com/gogo/protobuf/proto"
+ "go.etcd.io/etcd/clientv3"
+)
+
+const enrolmentPrefix = "enrolments/"
+
+var errNotExists = errors.New("not found")
+
+type EnrolmentStore struct {
+ backend clientv3.KV
+}
+
+func (s *EnrolmentStore) GetBySecret(ctx context.Context, secret []byte) (*api.EnrolmentConfig, error) {
+
+ res, err := s.backend.Get(ctx, enrolmentPrefix+base64.RawURLEncoding.EncodeToString(secret))
+ if err != nil {
+ return nil, fmt.Errorf("failed to query consensus: %w", err)
+ }
+ if res.Count == 0 {
+ return nil, errNotExists
+ } else if res.Count > 1 {
+ panic("more than one value for the same key, bailing")
+ }
+ rawVal := res.Kvs[0].Value
+ var config *api.EnrolmentConfig
+ if err := proto.Unmarshal(rawVal, config); err != nil {
+ return nil, err
+ }
+ return config, nil
+}
diff --git a/core/internal/api/nodemanagement.go b/core/internal/api/nodemanagement.go
new file mode 100644
index 0000000..4268a0f
--- /dev/null
+++ b/core/internal/api/nodemanagement.go
@@ -0,0 +1,279 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+ "bytes"
+ "context"
+ "crypto/ed25519"
+ "crypto/rand"
+ "crypto/sha256"
+ "crypto/subtle"
+ "crypto/x509"
+ "encoding/base64"
+ "errors"
+ "fmt"
+ "io"
+
+ "git.monogon.dev/source/nexantic.git/core/generated/api"
+ "git.monogon.dev/source/nexantic.git/core/pkg/tpm"
+ "github.com/gogo/protobuf/proto"
+ "go.etcd.io/etcd/clientv3"
+ "go.uber.org/zap"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+)
+
+const nodesPrefix = "nodes/"
+const enrolmentsPrefix = "enrolments/"
+
+func nodeId(idCert []byte) (string, error) {
+ // Currently we only identify nodes by ID key
+ cert, err := x509.ParseCertificate(idCert)
+ if err != nil {
+ return "", err
+ }
+ pubKey, ok := cert.PublicKey.(ed25519.PublicKey)
+ if !ok {
+ return "", errors.New("invalid node identity certificate")
+ }
+
+ return "smalltown-" + base64.RawStdEncoding.EncodeToString([]byte(pubKey)), nil
+}
+
+func (s *Server) registerNewNode(node *api.Node) error {
+ nodeRaw, err := proto.Marshal(node)
+ if err != nil {
+ return err
+ }
+
+ nodeID, err := nodeId(node.IdCert)
+ if err != nil {
+ return err
+ }
+
+ key := nodesPrefix + nodeID
+
+ // Overwriting nodes is a BadIdea(TM), so make this a Compare-and-Swap
+ res, err := s.getStore().Txn(context.Background()).If(
+ clientv3.Compare(clientv3.CreateRevision(key), "=", 0),
+ ).Then(
+ clientv3.OpPut(key, string(nodeRaw)),
+ ).Commit()
+ if err != nil {
+ return fmt.Errorf("failed to store new node: %w", err)
+ }
+ if !res.Succeeded {
+ s.Logger.Warn("double-registration of node attempted", zap.String("node", nodeID))
+ }
+ return nil
+}
+
+func (s *Server) TPM2BootstrapNode(newNodeInfo *api.NewNodeInfo) (*api.Node, error) {
+ akPublic, err := tpm.GetAKPublic()
+ if err != nil {
+ return nil, err
+ }
+ ekPubkey, ekCert, err := tpm.GetEKPublic()
+ if err != nil {
+ return nil, err
+ }
+ return &api.Node{
+ Address: newNodeInfo.Ip,
+ Integrity: &api.Node_Tpm2{Tpm2: &api.NodeTPM2{
+ AkPub: akPublic,
+ EkCert: ekCert,
+ EkPubkey: ekPubkey,
+ }},
+ GlobalUnlockKey: newNodeInfo.GlobalUnlockKey,
+ IdCert: newNodeInfo.IdCert,
+ State: api.Node_MASTER,
+ }, nil
+}
+
+func (s *Server) TPM2Unlock(unlockServer api.NodeManagementService_TPM2UnlockServer) error {
+ nonce := make([]byte, 32)
+ if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
+ return status.Error(codes.Unavailable, "failed to get randonmess")
+ }
+ if err := unlockServer.Send(&api.TPM2UnlockFlowResponse{
+ Stage: &api.TPM2UnlockFlowResponse_UnlockInit{
+ UnlockInit: &api.TPM2UnlockInit{
+ Nonce: nonce,
+ },
+ },
+ }); err != nil {
+ return err
+ }
+ unlockReqContainer, err := unlockServer.Recv()
+ if err != nil {
+ return err
+ }
+ unlockReqVariant, ok := unlockReqContainer.Stage.(*api.TPM2UnlockFlowRequeset_UnlockRequest)
+ if !ok {
+ return status.Errorf(codes.InvalidArgument, "protocol violation")
+ }
+ unlockRequest := unlockReqVariant.UnlockRequest
+
+ store := s.getStore()
+ // This is safe, etcd does not do relative paths
+ path := nodesPrefix + unlockRequest.NodeId
+ nodeRes, err := store.Get(unlockServer.Context(), path)
+ if err != nil {
+ return status.Error(codes.Unavailable, "consensus request failed")
+ }
+ if nodeRes.Count == 0 {
+ return status.Error(codes.NotFound, "this node does not exist")
+ } else if nodeRes.Count > 1 {
+ panic("invariant violation: more than one node with the same id")
+ }
+ nodeRaw := nodeRes.Kvs[0].Value
+ var node api.Node
+ if err := proto.Unmarshal(nodeRaw, &node); err != nil {
+ s.Logger.Error("Failed to decode node", zap.Error(err))
+ return status.Error(codes.Internal, "invalid node")
+ }
+
+ nodeTPM2, ok := node.Integrity.(*api.Node_Tpm2)
+ if !ok {
+ return status.Error(codes.InvalidArgument, "node not integrity-protected with TPM2")
+ }
+
+ validQuote, err := tpm.VerifyAttestPlatform(nonce, nodeTPM2.Tpm2.AkPub, unlockRequest.Quote, unlockRequest.QuoteSignature)
+ if err != nil {
+ return status.Error(codes.PermissionDenied, "invalid quote")
+ }
+
+ pcrHash := sha256.New()
+ for _, pcr := range unlockRequest.Pcrs {
+ pcrHash.Write(pcr)
+ }
+ expectedPCRHash := pcrHash.Sum(nil)
+
+ if !bytes.Equal(validQuote.AttestedQuoteInfo.PCRDigest, expectedPCRHash) {
+ return status.Error(codes.InvalidArgument, "the quote's PCR hash does not match the supplied PCRs")
+ }
+
+ // TODO: Plug in policy engine to decide if the unlock should actually happen
+
+ return unlockServer.Send(&api.TPM2UnlockFlowResponse{Stage: &api.TPM2UnlockFlowResponse_UnlockResponse{
+ UnlockResponse: &api.TPM2UnlockResponse{
+ GlobalUnlockKey: node.GlobalUnlockKey,
+ },
+ }})
+}
+
+func (s *Server) NewTPM2NodeRegister(registerServer api.NodeManagementService_NewTPM2NodeRegisterServer) error {
+ registerReqContainer, err := registerServer.Recv()
+ if err != nil {
+ return err
+ }
+ registerReqVariant, ok := registerReqContainer.Stage.(*api.TPM2FlowRequest_Register)
+ if !ok {
+ return status.Error(codes.InvalidArgument, "protocol violation")
+ }
+ registerReq := registerReqVariant.Register
+
+ challengeNonce := make([]byte, 32)
+ if _, err := io.ReadFull(rand.Reader, challengeNonce); err != nil {
+ return status.Error(codes.Unavailable, "failed to get randonmess")
+ }
+ challenge, challengeBlob, err := tpm.MakeAKChallenge(registerReq.EkPubkey, registerReq.AkPublic, challengeNonce)
+ if err != nil {
+ return status.Errorf(codes.InvalidArgument, "failed to challenge AK: %v", err)
+ }
+ nonce := make([]byte, 32)
+ if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
+ return status.Error(codes.Unavailable, "failed to get randonmess")
+ }
+ if err := registerServer.Send(&api.TPM2FlowResponse{Stage: &api.TPM2FlowResponse_AttestRequest{AttestRequest: &api.TPM2AttestRequest{
+ AkChallenge: challenge,
+ AkChallengeSecret: challengeBlob,
+ QuoteNonce: nonce,
+ }}}); err != nil {
+ return err
+ }
+ attestationResContainer, err := registerServer.Recv()
+ if err != nil {
+ return err
+ }
+ attestResVariant, ok := attestationResContainer.Stage.(*api.TPM2FlowRequest_AttestResponse)
+ if !ok {
+ return status.Error(codes.InvalidArgument, "protocol violation")
+ }
+ attestRes := attestResVariant.AttestResponse
+
+ if subtle.ConstantTimeCompare(attestRes.AkChallengeSolution, challengeNonce) != 1 {
+ return status.Error(codes.InvalidArgument, "invalid challenge response")
+ }
+
+ validQuote, err := tpm.VerifyAttestPlatform(nonce, registerReq.AkPublic, attestRes.Quote, attestRes.QuoteSignature)
+ if err != nil {
+ return status.Error(codes.PermissionDenied, "invalid quote")
+ }
+
+ pcrHash := sha256.New()
+ for _, pcr := range attestRes.Pcrs {
+ pcrHash.Write(pcr)
+ }
+ expectedPCRHash := pcrHash.Sum(nil)
+
+ if !bytes.Equal(validQuote.AttestedQuoteInfo.PCRDigest, expectedPCRHash) {
+ return status.Error(codes.InvalidArgument, "the quote's PCR hash does not match the supplied PCRs")
+ }
+
+ newNodeInfoContainer, err := registerServer.Recv()
+ newNodeInfoVariant, ok := newNodeInfoContainer.Stage.(*api.TPM2FlowRequest_NewNodeInfo)
+ newNodeInfo := newNodeInfoVariant.NewNodeInfo
+
+ store := s.getStore()
+ res, err := store.Get(registerServer.Context(), "enrolments/"+base64.RawURLEncoding.EncodeToString(newNodeInfo.EnrolmentConfig.EnrolmentSecret))
+ if err != nil {
+ return status.Error(codes.Unavailable, "Consensus unavailable")
+ }
+ if res.Count == 0 {
+ return status.Error(codes.PermissionDenied, "Invalid enrolment secret")
+ } else if res.Count > 1 {
+ panic("more than one value for the same key, bailing")
+ }
+ rawVal := res.Kvs[0].Value
+ var config api.EnrolmentConfig
+ if err := proto.Unmarshal(rawVal, &config); err != nil {
+ return err
+ }
+
+ // TODO: Plug in policy engine here
+
+ node := api.Node{
+ Address: newNodeInfo.Ip,
+ Integrity: &api.Node_Tpm2{Tpm2: &api.NodeTPM2{
+ AkPub: registerReq.AkPublic,
+ EkCert: registerReq.EkCert,
+ EkPubkey: registerReq.EkPubkey,
+ }},
+ GlobalUnlockKey: newNodeInfo.GlobalUnlockKey,
+ IdCert: newNodeInfo.IdCert,
+ State: api.Node_UNININITALIZED,
+ }
+
+ if err := s.registerNewNode(&node); err != nil {
+ s.Logger.Error("failed to register a node", zap.Error(err))
+ return status.Error(codes.Internal, "failed to register node")
+ }
+
+ return nil
+}
diff --git a/core/internal/api/nodes.go b/core/internal/api/nodes.go
new file mode 100644
index 0000000..da3cbc4
--- /dev/null
+++ b/core/internal/api/nodes.go
@@ -0,0 +1,17 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
diff --git a/core/internal/api/server.go b/core/internal/api/server.go
index efd0be5..4e1e5fa 100644
--- a/core/internal/api/server.go
+++ b/core/internal/api/server.go
@@ -17,23 +17,36 @@
package api
import (
+ "context"
+ "crypto/ed25519"
+ "crypto/rand"
+ "crypto/tls"
+ "crypto/x509"
+ "crypto/x509/pkix"
+ "errors"
"fmt"
+ "math/big"
+ "net"
+ "time"
+
+ "git.monogon.dev/source/nexantic.git/core/generated/api"
schema "git.monogon.dev/source/nexantic.git/core/generated/api"
"git.monogon.dev/source/nexantic.git/core/internal/common"
"git.monogon.dev/source/nexantic.git/core/internal/common/service"
"git.monogon.dev/source/nexantic.git/core/internal/consensus"
+ "go.etcd.io/etcd/clientv3"
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/reflection"
- "net"
+ "google.golang.org/grpc/credentials"
)
type (
Server struct {
*service.BaseService
- setupService common.SetupService
- grpcServer *grpc.Server
+ grpcServer *grpc.Server
+ externalGrpcServer *grpc.Server
consensusService *consensus.Service
@@ -41,49 +54,186 @@
}
Config struct {
- Port uint16
}
)
-func NewApiServer(config *Config, logger *zap.Logger, setupService common.SetupService, consensusService *consensus.Service) (*Server, error) {
+var (
+ // From RFC 5280 Section 4.1.2.5
+ unknownNotAfter = time.Unix(253402300799, 0)
+)
+
+func NewApiServer(config *Config, logger *zap.Logger, consensusService *consensus.Service) (*Server, error) {
s := &Server{
config: config,
- setupService: setupService,
consensusService: consensusService,
}
s.BaseService = service.NewBaseService("api", logger, s)
- grpcServer := grpc.NewServer()
- schema.RegisterClusterManagementServer(grpcServer, s)
- schema.RegisterSetupServiceServer(grpcServer, s)
-
- reflection.Register(grpcServer)
-
- s.grpcServer = grpcServer
-
return s, nil
}
+func (s *Server) getStore() clientv3.KV {
+ // Cannot be moved to initialization because an internal reference will be nil
+ return s.consensusService.GetStore("api", "")
+}
+
+// BootstrapNewClusterHook creates the necessary key material for the API Servers and stores it in
+// the consensus service. It also creates a node entry for the initial node.
+func (s *Server) BootstrapNewClusterHook(initNodeReq *api.NewNodeInfo) error {
+ serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 127)
+ serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+ if err != nil {
+ return fmt.Errorf("Failed to generate serial number: %w", err)
+ }
+
+ pubKey, privKeyRaw, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ return err
+ }
+ privkey, err := x509.MarshalPKCS8PrivateKey(privKeyRaw)
+ if err != nil {
+ return err
+ }
+
+ // This has no SANs because it authenticates by public key, not by name
+ masterCert := &x509.Certificate{
+ SerialNumber: serialNumber,
+ Subject: pkix.Name{
+ CommonName: "Smalltown Master",
+ },
+ IsCA: false,
+ BasicConstraintsValid: true,
+ NotBefore: time.Now(),
+ NotAfter: unknownNotAfter,
+ // Certificate is used both as server & client
+ ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
+ }
+ cert, err := x509.CreateCertificate(rand.Reader, masterCert, masterCert, pubKey, privKeyRaw)
+ if err != nil {
+ return err
+ }
+ store := s.getStore()
+ if _, err := store.Put(context.Background(), "master.der", string(cert)); err != nil {
+ return err
+ }
+ if _, err := store.Put(context.Background(), "master-key.der", string(privkey)); err != nil {
+ return err
+ }
+
+ // TODO: Further integrity providers need to be plumbed in here
+ node, err := s.TPM2BootstrapNode(initNodeReq)
+ if err != nil {
+ return err
+ }
+
+ if err := s.registerNewNode(node); err != nil {
+ return err
+ }
+ return nil
+}
+
+// GetMasterCert gets the master certificate in X.509 DER form
+// This is mainly used to issue enrolment configs
+func (s *Server) GetMasterCert() ([]byte, error) {
+ store := s.getStore()
+ res, err := store.Get(context.Background(), "master.der")
+ if err != nil {
+ return []byte{}, err
+ }
+ if len(res.Kvs) != 1 {
+ return []byte{}, errors.New("master certificate not found")
+ }
+ certRaw := res.Kvs[0].Value
+ return certRaw, nil
+}
+
+// TODO(lorenz): Move consensus/certificate interaction into a utility, is now duplicated too often
+func (s *Server) loadMasterCert() (*tls.Certificate, error) {
+
+ store := s.getStore()
+ var tlsCert tls.Certificate
+ res, err := store.Get(context.Background(), "master.der")
+ if err != nil {
+ return nil, err
+ }
+ if len(res.Kvs) != 1 {
+ return nil, errors.New("master certificate not found")
+ }
+ certRaw := res.Kvs[0].Value
+
+ tlsCert.Certificate = append(tlsCert.Certificate, certRaw)
+ tlsCert.Leaf, err = x509.ParseCertificate(certRaw)
+
+ res, err = store.Get(context.Background(), "master-key.der")
+ if err != nil {
+ return nil, err
+ }
+ if len(res.Kvs) != 1 {
+ return nil, errors.New("master certificate not found")
+ }
+ keyRaw := res.Kvs[0].Value
+ key, err := x509.ParsePKCS8PrivateKey(keyRaw)
+ if err != nil {
+ return nil, fmt.Errorf("failed to load master private key: %w", err)
+ }
+ edKey, ok := key.(ed25519.PrivateKey)
+ if !ok {
+ return nil, errors.New("invalid private key")
+ }
+ tlsCert.PrivateKey = edKey
+ return &tlsCert, nil
+}
+
func (s *Server) OnStart() error {
- listenHost := fmt.Sprintf(":%d", s.config.Port)
- lis, err := net.Listen("tcp", listenHost)
+ masterListenHost := fmt.Sprintf(":%d", common.MasterServicePort)
+ lis, err := net.Listen("tcp", masterListenHost)
if err != nil {
s.Logger.Fatal("failed to listen", zap.Error(err))
}
+ externalListeneHost := fmt.Sprintf(":%d", common.ExternalServicePort)
+ externalListener, err := net.Listen("tcp", externalListeneHost)
+ if err != nil {
+ s.Logger.Fatal("failed to listen", zap.Error(err))
+ }
+
+ masterCert, err := s.loadMasterCert()
+ if err != nil {
+ s.Logger.Error("Failed to load Master Service Key Material: %w", zap.Error(err))
+ return err
+ }
+
+ masterTransportCredentials := credentials.NewServerTLSFromCert(masterCert)
+
+ masterGrpcServer := grpc.NewServer(grpc.Creds(masterTransportCredentials))
+ clusterManagementGrpcServer := grpc.NewServer()
+ schema.RegisterClusterManagementServer(clusterManagementGrpcServer, s)
+ schema.RegisterNodeManagementServiceServer(masterGrpcServer, s)
+
+ reflection.Register(masterGrpcServer)
+
+ s.grpcServer = masterGrpcServer
+ s.externalGrpcServer = clusterManagementGrpcServer
+
go func() {
err = s.grpcServer.Serve(lis)
s.Logger.Error("API server failed", zap.Error(err))
}()
- s.Logger.Info("gRPC listening", zap.String("host", listenHost))
+ go func() {
+ err = s.externalGrpcServer.Serve(externalListener)
+ s.Logger.Error("API server failed", zap.Error(err))
+ }()
+
+ s.Logger.Info("gRPC listening", zap.String("host", masterListenHost))
return nil
}
func (s *Server) OnStop() error {
s.grpcServer.Stop()
+ s.externalGrpcServer.Stop()
return nil
}
diff --git a/core/internal/api/setup.go b/core/internal/api/setup.go
deleted file mode 100644
index f317534..0000000
--- a/core/internal/api/setup.go
+++ /dev/null
@@ -1,65 +0,0 @@
-// Copyright 2020 The Monogon Project Authors.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package api
-
-import (
- "context"
- "errors"
- "fmt"
-
- schema "git.monogon.dev/source/nexantic.git/core/generated/api"
-)
-
-const (
- MinNameLength = 3
-)
-
-var (
- ErrInvalidProvisioningToken = errors.New("invalid provisioning token")
- ErrInvalidNameLength = fmt.Errorf("name must be at least %d characters long", MinNameLength)
-)
-
-func (s *Server) Setup(c context.Context, r *schema.SetupRequest) (*schema.SetupResponse, error) {
- return &schema.SetupResponse{}, nil
-}
-
-func (s *Server) BootstrapNewCluster(context.Context, *schema.BootstrapNewClusterRequest) (*schema.BootstrapNewClusterResponse, error) {
- err := s.setupService.SetupNewCluster()
- return &schema.BootstrapNewClusterResponse{}, err
-}
-
-func (s *Server) JoinCluster(ctx context.Context, req *schema.JoinClusterRequest) (*schema.JoinClusterResponse, error) {
- // Verify provisioning token
- if s.setupService.GetJoinClusterToken() != req.ProvisioningToken {
- return nil, ErrInvalidProvisioningToken
- }
-
- // Join cluster
- err := s.setupService.JoinCluster(req.InitialCluster, req.Certs)
- if err != nil {
- return nil, err
- }
-
- return &schema.JoinClusterResponse{}, nil
-}
-
-func (s *Server) Attest(c context.Context, r *schema.AttestRequest) (*schema.AttestResponse, error) {
- // TODO implement
- return &schema.AttestResponse{
- Response: r.Challenge,
- }, nil
-}