Attestation & Identity & Global Unlock & Enrolment
This changes the node startup sequence significantly. Now the following three startup procedures replace the old setup/join mechanic:
* If no enrolment config is present, automatically bootstrap a new cluster and become master for it.
* If an enrolment config with an enrolment token is present, register with the NodeManagementService.
* If an enrolment config without an enrolment token is present, attempt a normal cluster unlock.
It also completely revamps the GRPC management services:
* NodeManagementService is a master-only service that deals with other nodes and has a cluster-wide identity
* NodeService is only available in unlocked state and keyed with the node identity
* ClusterManagement is now a master-only service that's been spun out of the main NMS since they have very different authentication models and also deals with EnrolmentConfigs
The TPM support library has also been extended by:
* Lots of integrity attestation and verification functions
* Built-in AK management
* Some advanced policy-based authentication stuff
Also contains various enhancements to the network service to make everything work in a proper multi-node environment.
Lots of old code has been thrown out.
Test Plan: Passed a full manual test of all three startup modes (bootstrap, enrolment and normal unlock) including automated EnrolmentConfig generation and consumption in a dual-node configuration on swtpm / OVMF.
Bug: T499
X-Origin-Diff: phab/D291
GitOrigin-RevId: d53755c828218b1df83a1d7ad252c7b3231abca8
diff --git a/core/internal/api/BUILD.bazel b/core/internal/api/BUILD.bazel
index e7ec6f8..e862340 100644
--- a/core/internal/api/BUILD.bazel
+++ b/core/internal/api/BUILD.bazel
@@ -4,19 +4,24 @@
name = "go_default_library",
srcs = [
"cluster.go",
+ "enrolment.go",
+ "nodemanagement.go",
+ "nodes.go",
"server.go",
- "setup.go",
],
importpath = "git.monogon.dev/source/nexantic.git/core/internal/api",
visibility = ["//core:__subpackages__"],
deps = [
"//core/api/api:go_default_library",
"//core/internal/common:go_default_library",
- "//core/internal/common/grpc:go_default_library",
"//core/internal/common/service:go_default_library",
"//core/internal/consensus:go_default_library",
+ "//core/pkg/tpm:go_default_library",
+ "@com_github_gogo_protobuf//proto:go_default_library",
+ "@io_etcd_go_etcd//clientv3:go_default_library",
"@org_golang_google_grpc//:go_default_library",
"@org_golang_google_grpc//codes:go_default_library",
+ "@org_golang_google_grpc//credentials:go_default_library",
"@org_golang_google_grpc//reflection:go_default_library",
"@org_golang_google_grpc//status:go_default_library",
"@org_uber_go_zap//:go_default_library",
diff --git a/core/internal/api/cluster.go b/core/internal/api/cluster.go
index d2c18c3..dc794b3 100644
--- a/core/internal/api/cluster.go
+++ b/core/internal/api/cluster.go
@@ -19,112 +19,97 @@
import (
"context"
"crypto/rand"
- "encoding/hex"
- "fmt"
+ "encoding/base64"
+ "io"
+
+ "git.monogon.dev/source/nexantic.git/core/generated/api"
schema "git.monogon.dev/source/nexantic.git/core/generated/api"
- "git.monogon.dev/source/nexantic.git/core/internal/common/grpc"
+ "github.com/gogo/protobuf/proto"
+ "go.etcd.io/etcd/clientv3"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"go.uber.org/zap"
)
-var (
- ErrAttestationFailed = status.Error(codes.PermissionDenied, "attestation failed")
-)
-
func (s *Server) AddNode(ctx context.Context, req *schema.AddNodeRequest) (*schema.AddNodeResponse, error) {
- // Setup API client
- c, err := grpc.NewSmalltownAPIClient(fmt.Sprintf("%s:%d", req.Addr, s.config.Port))
- if err != nil {
- return nil, err
- }
-
- // Check attestation
- nonce := make([]byte, 20)
- _, err = rand.Read(nonce)
- if err != nil {
- s.Logger.Error("Nonce generation failed", zap.Error(err))
- return nil, status.Error(codes.Unavailable, "nonce generation failed")
- }
- hexNonce := hex.EncodeToString(nonce)
-
- aRes, err := c.Setup.Attest(ctx, &schema.AttestRequest{
- Challenge: hexNonce,
- })
- if err != nil {
- s := status.Convert(err)
- return nil, status.Errorf(s.Code(), "attestation failed: %v", s.Message())
- }
-
- //TODO(hendrik): Verify response
- if aRes.Response != hexNonce {
- return nil, ErrAttestationFailed
- }
-
- consensusCerts, err := s.consensusService.IssueCertificate(req.Addr)
- if err != nil {
- // Errors from IssueCertificate are always treated as internal
- s.Logger.Error("Node certificate issuance failed", zap.String("addr", req.Addr), zap.Error(err))
- return nil, status.Error(codes.Internal, "could not issue node certificate")
- }
-
- // TODO(leo): fetch remote hostname rather than using the addr
- name := req.Addr
-
- // Add new node to local etcd cluster.
- memberID, err := s.consensusService.AddMember(ctx, name, fmt.Sprintf("https://%s:%d", req.Addr, s.config.Port))
- if err != nil {
- return nil, status.Errorf(codes.Unavailable, "failed to add node to etcd cluster: %v", err)
- }
-
- s.Logger.Info("Added new node to consensus cluster; sending cluster join request to node",
- zap.String("addr", req.Addr), zap.Uint16("port", s.config.Port))
-
- // Send JoinCluster request to new node to make it join.
- _, err = c.Setup.JoinCluster(ctx, &schema.JoinClusterRequest{
- InitialCluster: s.consensusService.GetInitialClusterString(),
- ProvisioningToken: req.ProvisioningToken,
- Certs: consensusCerts,
- })
- if err != nil {
- errRevoke := s.consensusService.RevokeCertificate(req.Addr)
- if errRevoke != nil {
- s.Logger.Error("Failed to revoke a certificate after rollback - potential security risk", zap.Error(errRevoke))
- }
- // Revert etcd add member - might fail if consensus cannot be established.
- errRemove := s.consensusService.RemoveMember(ctx, memberID)
- if errRemove != nil || errRevoke != nil {
- return nil, fmt.Errorf("rollback failed after failed provisioning; err=%v; err_rb=%v; err_revoke=%v", err, errRemove, errRevoke)
- }
- return nil, status.Errorf(codes.Unavailable, "failed to join etcd cluster with node: %v", err)
- }
- s.Logger.Info("Fully provisioned new node",
- zap.String("host", req.Addr),
- zap.Uint16("apiPort", s.config.Port),
- zap.Uint64("member_id", memberID))
-
- return &schema.AddNodeResponse{}, nil
+ return nil, status.Error(codes.Unimplemented, "Unimplemented")
}
-func (s *Server) RemoveNode(context.Context, *schema.RemoveNodeRequest) (*schema.RemoveNodeResponse, error) {
- return nil, status.Error(codes.Unimplemented, "unimplemented")
+func (s *Server) RemoveNode(ctx context.Context, req *schema.RemoveNodeRequest) (*schema.RemoveNodeRequest, error) {
+ return nil, status.Error(codes.Unimplemented, "Unimplemented")
}
-func (s *Server) ListNodes(context.Context, *schema.ListNodesRequest) (*schema.ListNodesResponse, error) {
- nodes := s.consensusService.GetNodes()
- resNodes := make([]*schema.Node, len(nodes))
-
- for i, node := range nodes {
- resNodes[i] = &schema.Node{
- Id: node.ID,
- Name: node.Name,
- Address: node.Address,
- Synced: node.Synced,
+func (s *Server) ListNodes(ctx context.Context, req *schema.ListNodesRequest) (*schema.ListNodesResponse, error) {
+ store := s.getStore()
+ res, err := store.Get(ctx, "nodes/", clientv3.WithPrefix())
+ if err != nil {
+ return nil, status.Error(codes.Unavailable, "Consensus unavailable")
+ }
+ var resNodes []*api.Node
+ for _, nodeEntry := range res.Kvs {
+ var node api.Node
+ if err := proto.Unmarshal(nodeEntry.Value, &node); err != nil {
+ s.Logger.Error("Encountered invalid node data", zap.Error(err))
+ return nil, status.Error(codes.Internal, "Invalid data")
}
+ // Zero out Global Unlock Key, it's never supposed to leave the cluster
+ node.GlobalUnlockKey = []byte{}
+
+ resNodes = append(resNodes, &node)
}
return &schema.ListNodesResponse{
Nodes: resNodes,
}, nil
}
+
+func (s *Server) ListEnrolmentConfigs(ctx context.Context, req *api.ListEnrolmentConfigsRequest) (*api.ListEnrolmentConfigsResponse, error) {
+ return nil, status.Error(codes.Unimplemented, "Unimplemented")
+}
+
+func (s *Server) NewEnrolmentConfig(ctx context.Context, req *api.NewEnrolmentConfigRequest) (*api.NewEnrolmentConfigResponse, error) {
+ store := s.getStore()
+ token := make([]byte, 32)
+ if _, err := io.ReadFull(rand.Reader, token); err != nil {
+ return nil, status.Error(codes.Unavailable, "failed to get randonmess")
+ }
+ nodes, err := store.Get(ctx, "nodes/", clientv3.WithPrefix())
+ if err != nil {
+ return nil, status.Error(codes.Unavailable, "consensus unavailable")
+ }
+ var masterIPs [][]byte
+ for _, nodeKV := range nodes.Kvs {
+ var node api.Node
+ if err := proto.Unmarshal(nodeKV.Value, &node); err != nil {
+ return nil, status.Error(codes.Internal, "invalid node")
+ }
+ if node.State == api.Node_MASTER {
+ masterIPs = append(masterIPs, node.Address)
+ }
+ }
+ masterCert, err := s.GetMasterCert()
+ if err != nil {
+ return nil, status.Error(codes.Unavailable, "consensus unavailable")
+ }
+
+ enrolmentConfig := &api.EnrolmentConfig{
+ EnrolmentSecret: token,
+ MasterIps: masterIPs,
+ MastersCert: masterCert,
+ }
+ enrolmentConfigRaw, err := proto.Marshal(enrolmentConfig)
+ if err != nil {
+ return nil, status.Error(codes.Internal, "failed to encode config")
+ }
+ if _, err := store.Put(ctx, "enrolments/"+base64.RawURLEncoding.EncodeToString(token), string(enrolmentConfigRaw)); err != nil {
+ return nil, status.Error(codes.Unavailable, "consensus unavailable")
+ }
+ return &schema.NewEnrolmentConfigResponse{
+ EnrolmentConfig: enrolmentConfig,
+ }, nil
+}
+
+func (s *Server) RemoveEnrolmentConfig(ctx context.Context, req *api.RemoveEnrolmentConfigRequest) (*api.RemoveEnrolmentConfigResponse, error) {
+ return nil, status.Error(codes.Unimplemented, "Unimplemented")
+}
diff --git a/core/internal/api/enrolment.go b/core/internal/api/enrolment.go
new file mode 100644
index 0000000..976b0f2
--- /dev/null
+++ b/core/internal/api/enrolment.go
@@ -0,0 +1,55 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+ "context"
+ "encoding/base64"
+ "errors"
+ "fmt"
+
+ "git.monogon.dev/source/nexantic.git/core/generated/api"
+ "github.com/gogo/protobuf/proto"
+ "go.etcd.io/etcd/clientv3"
+)
+
+const enrolmentPrefix = "enrolments/"
+
+var errNotExists = errors.New("not found")
+
+type EnrolmentStore struct {
+ backend clientv3.KV
+}
+
+func (s *EnrolmentStore) GetBySecret(ctx context.Context, secret []byte) (*api.EnrolmentConfig, error) {
+
+ res, err := s.backend.Get(ctx, enrolmentPrefix+base64.RawURLEncoding.EncodeToString(secret))
+ if err != nil {
+ return nil, fmt.Errorf("failed to query consensus: %w", err)
+ }
+ if res.Count == 0 {
+ return nil, errNotExists
+ } else if res.Count > 1 {
+ panic("more than one value for the same key, bailing")
+ }
+ rawVal := res.Kvs[0].Value
+ var config *api.EnrolmentConfig
+ if err := proto.Unmarshal(rawVal, config); err != nil {
+ return nil, err
+ }
+ return config, nil
+}
diff --git a/core/internal/api/nodemanagement.go b/core/internal/api/nodemanagement.go
new file mode 100644
index 0000000..4268a0f
--- /dev/null
+++ b/core/internal/api/nodemanagement.go
@@ -0,0 +1,279 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+ "bytes"
+ "context"
+ "crypto/ed25519"
+ "crypto/rand"
+ "crypto/sha256"
+ "crypto/subtle"
+ "crypto/x509"
+ "encoding/base64"
+ "errors"
+ "fmt"
+ "io"
+
+ "git.monogon.dev/source/nexantic.git/core/generated/api"
+ "git.monogon.dev/source/nexantic.git/core/pkg/tpm"
+ "github.com/gogo/protobuf/proto"
+ "go.etcd.io/etcd/clientv3"
+ "go.uber.org/zap"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+)
+
+const nodesPrefix = "nodes/"
+const enrolmentsPrefix = "enrolments/"
+
+func nodeId(idCert []byte) (string, error) {
+ // Currently we only identify nodes by ID key
+ cert, err := x509.ParseCertificate(idCert)
+ if err != nil {
+ return "", err
+ }
+ pubKey, ok := cert.PublicKey.(ed25519.PublicKey)
+ if !ok {
+ return "", errors.New("invalid node identity certificate")
+ }
+
+ return "smalltown-" + base64.RawStdEncoding.EncodeToString([]byte(pubKey)), nil
+}
+
+func (s *Server) registerNewNode(node *api.Node) error {
+ nodeRaw, err := proto.Marshal(node)
+ if err != nil {
+ return err
+ }
+
+ nodeID, err := nodeId(node.IdCert)
+ if err != nil {
+ return err
+ }
+
+ key := nodesPrefix + nodeID
+
+ // Overwriting nodes is a BadIdea(TM), so make this a Compare-and-Swap
+ res, err := s.getStore().Txn(context.Background()).If(
+ clientv3.Compare(clientv3.CreateRevision(key), "=", 0),
+ ).Then(
+ clientv3.OpPut(key, string(nodeRaw)),
+ ).Commit()
+ if err != nil {
+ return fmt.Errorf("failed to store new node: %w", err)
+ }
+ if !res.Succeeded {
+ s.Logger.Warn("double-registration of node attempted", zap.String("node", nodeID))
+ }
+ return nil
+}
+
+func (s *Server) TPM2BootstrapNode(newNodeInfo *api.NewNodeInfo) (*api.Node, error) {
+ akPublic, err := tpm.GetAKPublic()
+ if err != nil {
+ return nil, err
+ }
+ ekPubkey, ekCert, err := tpm.GetEKPublic()
+ if err != nil {
+ return nil, err
+ }
+ return &api.Node{
+ Address: newNodeInfo.Ip,
+ Integrity: &api.Node_Tpm2{Tpm2: &api.NodeTPM2{
+ AkPub: akPublic,
+ EkCert: ekCert,
+ EkPubkey: ekPubkey,
+ }},
+ GlobalUnlockKey: newNodeInfo.GlobalUnlockKey,
+ IdCert: newNodeInfo.IdCert,
+ State: api.Node_MASTER,
+ }, nil
+}
+
+func (s *Server) TPM2Unlock(unlockServer api.NodeManagementService_TPM2UnlockServer) error {
+ nonce := make([]byte, 32)
+ if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
+ return status.Error(codes.Unavailable, "failed to get randonmess")
+ }
+ if err := unlockServer.Send(&api.TPM2UnlockFlowResponse{
+ Stage: &api.TPM2UnlockFlowResponse_UnlockInit{
+ UnlockInit: &api.TPM2UnlockInit{
+ Nonce: nonce,
+ },
+ },
+ }); err != nil {
+ return err
+ }
+ unlockReqContainer, err := unlockServer.Recv()
+ if err != nil {
+ return err
+ }
+ unlockReqVariant, ok := unlockReqContainer.Stage.(*api.TPM2UnlockFlowRequeset_UnlockRequest)
+ if !ok {
+ return status.Errorf(codes.InvalidArgument, "protocol violation")
+ }
+ unlockRequest := unlockReqVariant.UnlockRequest
+
+ store := s.getStore()
+ // This is safe, etcd does not do relative paths
+ path := nodesPrefix + unlockRequest.NodeId
+ nodeRes, err := store.Get(unlockServer.Context(), path)
+ if err != nil {
+ return status.Error(codes.Unavailable, "consensus request failed")
+ }
+ if nodeRes.Count == 0 {
+ return status.Error(codes.NotFound, "this node does not exist")
+ } else if nodeRes.Count > 1 {
+ panic("invariant violation: more than one node with the same id")
+ }
+ nodeRaw := nodeRes.Kvs[0].Value
+ var node api.Node
+ if err := proto.Unmarshal(nodeRaw, &node); err != nil {
+ s.Logger.Error("Failed to decode node", zap.Error(err))
+ return status.Error(codes.Internal, "invalid node")
+ }
+
+ nodeTPM2, ok := node.Integrity.(*api.Node_Tpm2)
+ if !ok {
+ return status.Error(codes.InvalidArgument, "node not integrity-protected with TPM2")
+ }
+
+ validQuote, err := tpm.VerifyAttestPlatform(nonce, nodeTPM2.Tpm2.AkPub, unlockRequest.Quote, unlockRequest.QuoteSignature)
+ if err != nil {
+ return status.Error(codes.PermissionDenied, "invalid quote")
+ }
+
+ pcrHash := sha256.New()
+ for _, pcr := range unlockRequest.Pcrs {
+ pcrHash.Write(pcr)
+ }
+ expectedPCRHash := pcrHash.Sum(nil)
+
+ if !bytes.Equal(validQuote.AttestedQuoteInfo.PCRDigest, expectedPCRHash) {
+ return status.Error(codes.InvalidArgument, "the quote's PCR hash does not match the supplied PCRs")
+ }
+
+ // TODO: Plug in policy engine to decide if the unlock should actually happen
+
+ return unlockServer.Send(&api.TPM2UnlockFlowResponse{Stage: &api.TPM2UnlockFlowResponse_UnlockResponse{
+ UnlockResponse: &api.TPM2UnlockResponse{
+ GlobalUnlockKey: node.GlobalUnlockKey,
+ },
+ }})
+}
+
+func (s *Server) NewTPM2NodeRegister(registerServer api.NodeManagementService_NewTPM2NodeRegisterServer) error {
+ registerReqContainer, err := registerServer.Recv()
+ if err != nil {
+ return err
+ }
+ registerReqVariant, ok := registerReqContainer.Stage.(*api.TPM2FlowRequest_Register)
+ if !ok {
+ return status.Error(codes.InvalidArgument, "protocol violation")
+ }
+ registerReq := registerReqVariant.Register
+
+ challengeNonce := make([]byte, 32)
+ if _, err := io.ReadFull(rand.Reader, challengeNonce); err != nil {
+ return status.Error(codes.Unavailable, "failed to get randonmess")
+ }
+ challenge, challengeBlob, err := tpm.MakeAKChallenge(registerReq.EkPubkey, registerReq.AkPublic, challengeNonce)
+ if err != nil {
+ return status.Errorf(codes.InvalidArgument, "failed to challenge AK: %v", err)
+ }
+ nonce := make([]byte, 32)
+ if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
+ return status.Error(codes.Unavailable, "failed to get randonmess")
+ }
+ if err := registerServer.Send(&api.TPM2FlowResponse{Stage: &api.TPM2FlowResponse_AttestRequest{AttestRequest: &api.TPM2AttestRequest{
+ AkChallenge: challenge,
+ AkChallengeSecret: challengeBlob,
+ QuoteNonce: nonce,
+ }}}); err != nil {
+ return err
+ }
+ attestationResContainer, err := registerServer.Recv()
+ if err != nil {
+ return err
+ }
+ attestResVariant, ok := attestationResContainer.Stage.(*api.TPM2FlowRequest_AttestResponse)
+ if !ok {
+ return status.Error(codes.InvalidArgument, "protocol violation")
+ }
+ attestRes := attestResVariant.AttestResponse
+
+ if subtle.ConstantTimeCompare(attestRes.AkChallengeSolution, challengeNonce) != 1 {
+ return status.Error(codes.InvalidArgument, "invalid challenge response")
+ }
+
+ validQuote, err := tpm.VerifyAttestPlatform(nonce, registerReq.AkPublic, attestRes.Quote, attestRes.QuoteSignature)
+ if err != nil {
+ return status.Error(codes.PermissionDenied, "invalid quote")
+ }
+
+ pcrHash := sha256.New()
+ for _, pcr := range attestRes.Pcrs {
+ pcrHash.Write(pcr)
+ }
+ expectedPCRHash := pcrHash.Sum(nil)
+
+ if !bytes.Equal(validQuote.AttestedQuoteInfo.PCRDigest, expectedPCRHash) {
+ return status.Error(codes.InvalidArgument, "the quote's PCR hash does not match the supplied PCRs")
+ }
+
+ newNodeInfoContainer, err := registerServer.Recv()
+ newNodeInfoVariant, ok := newNodeInfoContainer.Stage.(*api.TPM2FlowRequest_NewNodeInfo)
+ newNodeInfo := newNodeInfoVariant.NewNodeInfo
+
+ store := s.getStore()
+ res, err := store.Get(registerServer.Context(), "enrolments/"+base64.RawURLEncoding.EncodeToString(newNodeInfo.EnrolmentConfig.EnrolmentSecret))
+ if err != nil {
+ return status.Error(codes.Unavailable, "Consensus unavailable")
+ }
+ if res.Count == 0 {
+ return status.Error(codes.PermissionDenied, "Invalid enrolment secret")
+ } else if res.Count > 1 {
+ panic("more than one value for the same key, bailing")
+ }
+ rawVal := res.Kvs[0].Value
+ var config api.EnrolmentConfig
+ if err := proto.Unmarshal(rawVal, &config); err != nil {
+ return err
+ }
+
+ // TODO: Plug in policy engine here
+
+ node := api.Node{
+ Address: newNodeInfo.Ip,
+ Integrity: &api.Node_Tpm2{Tpm2: &api.NodeTPM2{
+ AkPub: registerReq.AkPublic,
+ EkCert: registerReq.EkCert,
+ EkPubkey: registerReq.EkPubkey,
+ }},
+ GlobalUnlockKey: newNodeInfo.GlobalUnlockKey,
+ IdCert: newNodeInfo.IdCert,
+ State: api.Node_UNININITALIZED,
+ }
+
+ if err := s.registerNewNode(&node); err != nil {
+ s.Logger.Error("failed to register a node", zap.Error(err))
+ return status.Error(codes.Internal, "failed to register node")
+ }
+
+ return nil
+}
diff --git a/core/internal/api/nodes.go b/core/internal/api/nodes.go
new file mode 100644
index 0000000..da3cbc4
--- /dev/null
+++ b/core/internal/api/nodes.go
@@ -0,0 +1,17 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
diff --git a/core/internal/api/server.go b/core/internal/api/server.go
index efd0be5..4e1e5fa 100644
--- a/core/internal/api/server.go
+++ b/core/internal/api/server.go
@@ -17,23 +17,36 @@
package api
import (
+ "context"
+ "crypto/ed25519"
+ "crypto/rand"
+ "crypto/tls"
+ "crypto/x509"
+ "crypto/x509/pkix"
+ "errors"
"fmt"
+ "math/big"
+ "net"
+ "time"
+
+ "git.monogon.dev/source/nexantic.git/core/generated/api"
schema "git.monogon.dev/source/nexantic.git/core/generated/api"
"git.monogon.dev/source/nexantic.git/core/internal/common"
"git.monogon.dev/source/nexantic.git/core/internal/common/service"
"git.monogon.dev/source/nexantic.git/core/internal/consensus"
+ "go.etcd.io/etcd/clientv3"
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/reflection"
- "net"
+ "google.golang.org/grpc/credentials"
)
type (
Server struct {
*service.BaseService
- setupService common.SetupService
- grpcServer *grpc.Server
+ grpcServer *grpc.Server
+ externalGrpcServer *grpc.Server
consensusService *consensus.Service
@@ -41,49 +54,186 @@
}
Config struct {
- Port uint16
}
)
-func NewApiServer(config *Config, logger *zap.Logger, setupService common.SetupService, consensusService *consensus.Service) (*Server, error) {
+var (
+ // From RFC 5280 Section 4.1.2.5
+ unknownNotAfter = time.Unix(253402300799, 0)
+)
+
+func NewApiServer(config *Config, logger *zap.Logger, consensusService *consensus.Service) (*Server, error) {
s := &Server{
config: config,
- setupService: setupService,
consensusService: consensusService,
}
s.BaseService = service.NewBaseService("api", logger, s)
- grpcServer := grpc.NewServer()
- schema.RegisterClusterManagementServer(grpcServer, s)
- schema.RegisterSetupServiceServer(grpcServer, s)
-
- reflection.Register(grpcServer)
-
- s.grpcServer = grpcServer
-
return s, nil
}
+func (s *Server) getStore() clientv3.KV {
+ // Cannot be moved to initialization because an internal reference will be nil
+ return s.consensusService.GetStore("api", "")
+}
+
+// BootstrapNewClusterHook creates the necessary key material for the API Servers and stores it in
+// the consensus service. It also creates a node entry for the initial node.
+func (s *Server) BootstrapNewClusterHook(initNodeReq *api.NewNodeInfo) error {
+ serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 127)
+ serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+ if err != nil {
+ return fmt.Errorf("Failed to generate serial number: %w", err)
+ }
+
+ pubKey, privKeyRaw, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ return err
+ }
+ privkey, err := x509.MarshalPKCS8PrivateKey(privKeyRaw)
+ if err != nil {
+ return err
+ }
+
+ // This has no SANs because it authenticates by public key, not by name
+ masterCert := &x509.Certificate{
+ SerialNumber: serialNumber,
+ Subject: pkix.Name{
+ CommonName: "Smalltown Master",
+ },
+ IsCA: false,
+ BasicConstraintsValid: true,
+ NotBefore: time.Now(),
+ NotAfter: unknownNotAfter,
+ // Certificate is used both as server & client
+ ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
+ }
+ cert, err := x509.CreateCertificate(rand.Reader, masterCert, masterCert, pubKey, privKeyRaw)
+ if err != nil {
+ return err
+ }
+ store := s.getStore()
+ if _, err := store.Put(context.Background(), "master.der", string(cert)); err != nil {
+ return err
+ }
+ if _, err := store.Put(context.Background(), "master-key.der", string(privkey)); err != nil {
+ return err
+ }
+
+ // TODO: Further integrity providers need to be plumbed in here
+ node, err := s.TPM2BootstrapNode(initNodeReq)
+ if err != nil {
+ return err
+ }
+
+ if err := s.registerNewNode(node); err != nil {
+ return err
+ }
+ return nil
+}
+
+// GetMasterCert gets the master certificate in X.509 DER form
+// This is mainly used to issue enrolment configs
+func (s *Server) GetMasterCert() ([]byte, error) {
+ store := s.getStore()
+ res, err := store.Get(context.Background(), "master.der")
+ if err != nil {
+ return []byte{}, err
+ }
+ if len(res.Kvs) != 1 {
+ return []byte{}, errors.New("master certificate not found")
+ }
+ certRaw := res.Kvs[0].Value
+ return certRaw, nil
+}
+
+// TODO(lorenz): Move consensus/certificate interaction into a utility, is now duplicated too often
+func (s *Server) loadMasterCert() (*tls.Certificate, error) {
+
+ store := s.getStore()
+ var tlsCert tls.Certificate
+ res, err := store.Get(context.Background(), "master.der")
+ if err != nil {
+ return nil, err
+ }
+ if len(res.Kvs) != 1 {
+ return nil, errors.New("master certificate not found")
+ }
+ certRaw := res.Kvs[0].Value
+
+ tlsCert.Certificate = append(tlsCert.Certificate, certRaw)
+ tlsCert.Leaf, err = x509.ParseCertificate(certRaw)
+
+ res, err = store.Get(context.Background(), "master-key.der")
+ if err != nil {
+ return nil, err
+ }
+ if len(res.Kvs) != 1 {
+ return nil, errors.New("master certificate not found")
+ }
+ keyRaw := res.Kvs[0].Value
+ key, err := x509.ParsePKCS8PrivateKey(keyRaw)
+ if err != nil {
+ return nil, fmt.Errorf("failed to load master private key: %w", err)
+ }
+ edKey, ok := key.(ed25519.PrivateKey)
+ if !ok {
+ return nil, errors.New("invalid private key")
+ }
+ tlsCert.PrivateKey = edKey
+ return &tlsCert, nil
+}
+
func (s *Server) OnStart() error {
- listenHost := fmt.Sprintf(":%d", s.config.Port)
- lis, err := net.Listen("tcp", listenHost)
+ masterListenHost := fmt.Sprintf(":%d", common.MasterServicePort)
+ lis, err := net.Listen("tcp", masterListenHost)
if err != nil {
s.Logger.Fatal("failed to listen", zap.Error(err))
}
+ externalListeneHost := fmt.Sprintf(":%d", common.ExternalServicePort)
+ externalListener, err := net.Listen("tcp", externalListeneHost)
+ if err != nil {
+ s.Logger.Fatal("failed to listen", zap.Error(err))
+ }
+
+ masterCert, err := s.loadMasterCert()
+ if err != nil {
+ s.Logger.Error("Failed to load Master Service Key Material: %w", zap.Error(err))
+ return err
+ }
+
+ masterTransportCredentials := credentials.NewServerTLSFromCert(masterCert)
+
+ masterGrpcServer := grpc.NewServer(grpc.Creds(masterTransportCredentials))
+ clusterManagementGrpcServer := grpc.NewServer()
+ schema.RegisterClusterManagementServer(clusterManagementGrpcServer, s)
+ schema.RegisterNodeManagementServiceServer(masterGrpcServer, s)
+
+ reflection.Register(masterGrpcServer)
+
+ s.grpcServer = masterGrpcServer
+ s.externalGrpcServer = clusterManagementGrpcServer
+
go func() {
err = s.grpcServer.Serve(lis)
s.Logger.Error("API server failed", zap.Error(err))
}()
- s.Logger.Info("gRPC listening", zap.String("host", listenHost))
+ go func() {
+ err = s.externalGrpcServer.Serve(externalListener)
+ s.Logger.Error("API server failed", zap.Error(err))
+ }()
+
+ s.Logger.Info("gRPC listening", zap.String("host", masterListenHost))
return nil
}
func (s *Server) OnStop() error {
s.grpcServer.Stop()
+ s.externalGrpcServer.Stop()
return nil
}
diff --git a/core/internal/api/setup.go b/core/internal/api/setup.go
deleted file mode 100644
index f317534..0000000
--- a/core/internal/api/setup.go
+++ /dev/null
@@ -1,65 +0,0 @@
-// Copyright 2020 The Monogon Project Authors.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package api
-
-import (
- "context"
- "errors"
- "fmt"
-
- schema "git.monogon.dev/source/nexantic.git/core/generated/api"
-)
-
-const (
- MinNameLength = 3
-)
-
-var (
- ErrInvalidProvisioningToken = errors.New("invalid provisioning token")
- ErrInvalidNameLength = fmt.Errorf("name must be at least %d characters long", MinNameLength)
-)
-
-func (s *Server) Setup(c context.Context, r *schema.SetupRequest) (*schema.SetupResponse, error) {
- return &schema.SetupResponse{}, nil
-}
-
-func (s *Server) BootstrapNewCluster(context.Context, *schema.BootstrapNewClusterRequest) (*schema.BootstrapNewClusterResponse, error) {
- err := s.setupService.SetupNewCluster()
- return &schema.BootstrapNewClusterResponse{}, err
-}
-
-func (s *Server) JoinCluster(ctx context.Context, req *schema.JoinClusterRequest) (*schema.JoinClusterResponse, error) {
- // Verify provisioning token
- if s.setupService.GetJoinClusterToken() != req.ProvisioningToken {
- return nil, ErrInvalidProvisioningToken
- }
-
- // Join cluster
- err := s.setupService.JoinCluster(req.InitialCluster, req.Certs)
- if err != nil {
- return nil, err
- }
-
- return &schema.JoinClusterResponse{}, nil
-}
-
-func (s *Server) Attest(c context.Context, r *schema.AttestRequest) (*schema.AttestResponse, error) {
- // TODO implement
- return &schema.AttestResponse{
- Response: r.Challenge,
- }, nil
-}
diff --git a/core/internal/common/BUILD.bazel b/core/internal/common/BUILD.bazel
index 1adbfd8..8bd89d6 100644
--- a/core/internal/common/BUILD.bazel
+++ b/core/internal/common/BUILD.bazel
@@ -5,5 +5,4 @@
srcs = ["setup.go"],
importpath = "git.monogon.dev/source/nexantic.git/core/internal/common",
visibility = ["//:__subpackages__"],
- deps = ["//core/api/api:go_default_library"],
)
diff --git a/core/internal/common/grpc/grpc.go b/core/internal/common/grpc/grpc.go
index e9cfed0..97fc4c8 100644
--- a/core/internal/common/grpc/grpc.go
+++ b/core/internal/common/grpc/grpc.go
@@ -27,7 +27,6 @@
conn *grpc.ClientConn
Cluster api.ClusterManagementClient
- Setup api.SetupServiceClient
}
)
@@ -42,7 +41,6 @@
// Setup all client connections
s.Cluster = api.NewClusterManagementClient(conn)
- s.Setup = api.NewSetupServiceClient(conn)
return s, nil
}
diff --git a/core/internal/common/setup.go b/core/internal/common/setup.go
index 1124d27..e745e54 100644
--- a/core/internal/common/setup.go
+++ b/core/internal/common/setup.go
@@ -16,27 +16,23 @@
package common
-import "git.monogon.dev/source/nexantic.git/core/generated/api"
-
-// TODO(leo): merge api and node packages and get rid of this extra layer of indirection?
-
type (
- SetupService interface {
- CurrentState() SmalltownState
- GetJoinClusterToken() string
- SetupNewCluster() error
- EnterJoinClusterMode() error
- JoinCluster(initialCluster string, certs *api.ConsensusCertificates) error
- }
-
SmalltownState string
)
const (
- // Node is unprovisioned and waits for Setup to be called.
- StateSetupMode SmalltownState = "setup"
- // Setup() has been called, node waits for a JoinCluster or BootstrapCluster call.
- StateClusterJoinMode SmalltownState = "join"
+ // These are here to prevent depdendency loops
+ NodeServicePort = 7835
+ ConsensusPort = 7834
+ MasterServicePort = 7833
+ ExternalServicePort = 7836
+)
+
+const (
+ // Node is provisioning a new cluster with itself as a master
+ StateNewClusterMode SmalltownState = "setup"
+ // Node is enrolling itself and waiting to be adopted
+ StateEnrollMode SmalltownState = "join"
// Node is fully provisioned.
- StateConfigured SmalltownState = "configured"
+ StateJoined SmalltownState = "enrolled"
)
diff --git a/core/internal/consensus/BUILD.bazel b/core/internal/consensus/BUILD.bazel
index 5fa33ac..53b118d 100644
--- a/core/internal/consensus/BUILD.bazel
+++ b/core/internal/consensus/BUILD.bazel
@@ -7,6 +7,7 @@
visibility = ["//:__subpackages__"],
deps = [
"//core/api/api:go_default_library",
+ "//core/internal/common:go_default_library",
"//core/internal/common/service:go_default_library",
"//core/internal/consensus/ca:go_default_library",
"@com_github_pkg_errors//:go_default_library",
diff --git a/core/internal/consensus/consensus.go b/core/internal/consensus/consensus.go
index 6fa6210..5a0bead 100644
--- a/core/internal/consensus/consensus.go
+++ b/core/internal/consensus/consensus.go
@@ -33,6 +33,7 @@
"strings"
"time"
+ "git.monogon.dev/source/nexantic.git/core/internal/common"
"git.monogon.dev/source/nexantic.git/core/internal/common/service"
"git.monogon.dev/source/nexantic.git/core/generated/api"
@@ -91,7 +92,6 @@
NewCluster bool
ExternalHost string
ListenHost string
- ListenPort uint16
}
Member struct {
@@ -143,13 +143,13 @@
cfg.LCUrls = []url.URL{*listenerURL}
// Advertise Peer URLs
- apURL, err := url.Parse(fmt.Sprintf("https://%s:%d", s.config.ExternalHost, s.config.ListenPort))
+ apURL, err := url.Parse(fmt.Sprintf("https://%s:%d", s.config.ExternalHost, common.ConsensusPort))
if err != nil {
return fmt.Errorf("invalid external_host or listen_port: %w", err)
}
// Listen Peer URLs
- lpURL, err := url.Parse(fmt.Sprintf("https://%s:%d", s.config.ListenHost, s.config.ListenPort))
+ lpURL, err := url.Parse(fmt.Sprintf("https://%s:%d", s.config.ListenHost, common.ConsensusPort))
if err != nil {
return fmt.Errorf("invalid listen_host or listen_port: %w", err)
}
diff --git a/core/internal/integrity/BUILD.bazel b/core/internal/integrity/BUILD.bazel
new file mode 100644
index 0000000..cb551cc
--- /dev/null
+++ b/core/internal/integrity/BUILD.bazel
@@ -0,0 +1,14 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+ name = "go_default_library",
+ srcs = ["common.go"],
+ importpath = "git.monogon.dev/source/nexantic.git/core/internal/integrity",
+ visibility = ["//core:__subpackages__"],
+ deps = [
+ "//core/api/api:go_default_library",
+ "//core/internal/common:go_default_library",
+ "@org_golang_google_grpc//:go_default_library",
+ "@org_golang_google_grpc//credentials:go_default_library",
+ ],
+)
diff --git a/core/internal/integrity/common.go b/core/internal/integrity/common.go
new file mode 100644
index 0000000..f92c008
--- /dev/null
+++ b/core/internal/integrity/common.go
@@ -0,0 +1,81 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package integrity
+
+import (
+ "bytes"
+ "crypto/tls"
+ "crypto/x509"
+ "errors"
+ "fmt"
+ "net"
+ "strings"
+
+ "git.monogon.dev/source/nexantic.git/core/generated/api"
+ "git.monogon.dev/source/nexantic.git/core/internal/common"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/credentials"
+)
+
+// Agent specifices the interface which every integrity agent needs to fulfill
+type Agent interface {
+ // Initialize needs to be called once and initializes the systems required to maintain integrity
+ // on the given platform.
+ // nodeCert is a X.509 DER certificate which identifies the node once it's unlocked. This is
+ // required to bind the node certificate (which is only available when the node is unlocked) to
+ // the integrity subsystem used to attest said node.
+ // Initialize returns the cryptographic identity that it's bound to.
+ Initialize(newNode api.NewNodeInfo, enrolment api.EnrolmentConfig) (string, error)
+
+ // Unlock performs all required actions to assure the integrity of the platform and retrieves
+ // the unlock key in a secure manner
+ Unlock(enrolment api.EnrolmentConfig) ([]byte, error)
+}
+
+// DialNMS creates a secure GRPC connection to the NodeManagementService
+func DialNMS(enrolment api.EnrolmentConfig) (*grpc.ClientConn, error) {
+ var targets []string
+ for _, target := range enrolment.MasterIps {
+ targets = append(targets, fmt.Sprintf("%v:%v", net.IP(target), common.MasterServicePort))
+ }
+ cert, err := x509.ParseCertificate(enrolment.MastersCert)
+ if err != nil {
+ return nil, err
+ }
+ mastersPool := x509.NewCertPool()
+ mastersPool.AddCert(cert)
+
+ secureTransport := &tls.Config{
+ InsecureSkipVerify: true,
+ // Critical function, please review any changes with care
+ // TODO(lorenz): Actively check that this actually provides the security guarantees that we need
+ VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
+ for _, cert := range rawCerts {
+ // X.509 certificates in DER can be compared like this since DER has a unique representation
+ // for each certificate.
+ if bytes.Equal(cert, enrolment.MastersCert) {
+ return nil
+ }
+ }
+ return errors.New("failed to find authorized NMS certificate")
+ },
+ MinVersion: tls.VersionTLS13,
+ }
+ secureTransportCreds := credentials.NewTLS(secureTransport)
+
+ return grpc.Dial(strings.Join(targets, ","), grpc.WithTransportCredentials(secureTransportCreds))
+}
diff --git a/core/internal/integrity/tpm2/BUILD.bazel b/core/internal/integrity/tpm2/BUILD.bazel
new file mode 100644
index 0000000..c4c77bb
--- /dev/null
+++ b/core/internal/integrity/tpm2/BUILD.bazel
@@ -0,0 +1,13 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+ name = "go_default_library",
+ srcs = ["tpm2.go"],
+ importpath = "git.monogon.dev/source/nexantic.git/core/internal/integrity/tpm2",
+ visibility = ["//core:__subpackages__"],
+ deps = [
+ "//core/api/api:go_default_library",
+ "//core/internal/integrity:go_default_library",
+ "//core/pkg/tpm:go_default_library",
+ ],
+)
diff --git a/core/internal/integrity/tpm2/tpm2.go b/core/internal/integrity/tpm2/tpm2.go
new file mode 100644
index 0000000..8497562
--- /dev/null
+++ b/core/internal/integrity/tpm2/tpm2.go
@@ -0,0 +1,155 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tpm2
+
+import (
+ "context"
+ "errors"
+ "fmt"
+
+ "git.monogon.dev/source/nexantic.git/core/generated/api"
+ "git.monogon.dev/source/nexantic.git/core/internal/integrity"
+ "git.monogon.dev/source/nexantic.git/core/pkg/tpm"
+)
+
+type TPM2Agent struct {
+}
+
+func (a *TPM2Agent) Initialize(newNode api.NewNodeInfo, enrolment api.EnrolmentConfig) error {
+ nmsConn, err := integrity.DialNMS(enrolment)
+ nmsClient := api.NewNodeManagementServiceClient(nmsConn)
+ ekPub, ekCert, err := tpm.GetEKPublic()
+ if err != nil {
+ return fmt.Errorf("failed to generate EK: %w", err)
+ }
+
+ akPub, err := tpm.GetAKPublic()
+ if err != nil {
+ return fmt.Errorf("failed to generate AK: %w", err)
+ }
+
+ registerSession, err := nmsClient.NewTPM2NodeRegister(context.Background())
+ if err != nil {
+ return fmt.Errorf("failed to open registration session: %w", err)
+ }
+ defer registerSession.CloseSend()
+ if err := registerSession.Send(&api.TPM2FlowRequest{
+ Stage: &api.TPM2FlowRequest_Register{
+ Register: &api.TPM2RegisterRequest{
+ AkPublic: akPub,
+ EkPubkey: ekPub,
+ EkCert: ekCert,
+ },
+ },
+ }); err != nil {
+ return fmt.Errorf("failed to send registration: %w", err)
+ }
+
+ res1, err := registerSession.Recv()
+ if err != nil {
+ return fmt.Errorf("failed to receive attest request: %w", err)
+ }
+ attestReqContainer, ok := res1.Stage.(*api.TPM2FlowResponse_AttestRequest)
+ if !ok {
+ return errors.New("protocol violation: after RegisterRequest expected AttestRequest")
+ }
+ attestReq := attestReqContainer.AttestRequest
+ solution, err := tpm.SolveAKChallenge(attestReq.AkChallenge, attestReq.AkChallengeSecret)
+ if err != nil {
+ return fmt.Errorf("failed to solve AK challenge: %w", err)
+ }
+ pcrs, err := tpm.GetPCRs()
+ if err != nil {
+ return fmt.Errorf("failed to get SRTM PCRs: %w", err)
+ }
+ quote, quoteSig, err := tpm.AttestPlatform(attestReq.QuoteNonce)
+ if err != nil {
+ return fmt.Errorf("failed Quote operation: %w", err)
+ }
+ if err := registerSession.Send(&api.TPM2FlowRequest{
+ Stage: &api.TPM2FlowRequest_AttestResponse{
+ AttestResponse: &api.TPM2AttestResponse{
+ AkChallengeSolution: solution,
+ Pcrs: pcrs,
+ Quote: quote,
+ QuoteSignature: quoteSig,
+ },
+ },
+ }); err != nil {
+ return fmt.Errorf("failed to send AttestResponse: %w", err)
+ }
+ if err := registerSession.Send(&api.TPM2FlowRequest{
+ Stage: &api.TPM2FlowRequest_NewNodeInfo{
+ NewNodeInfo: &newNode,
+ },
+ }); err != nil {
+ return fmt.Errorf("failed to send NewNodeInfo: %w", err)
+ }
+ return nil
+}
+
+// Unlock attests the node state to the remote NMS and asks it for the global unlock key
+func (a *TPM2Agent) Unlock(enrolment api.EnrolmentConfig) ([]byte, error) {
+ nmsConn, err := integrity.DialNMS(enrolment)
+ if err != nil {
+ return []byte{}, err
+ }
+ nmsClient := api.NewNodeManagementServiceClient(nmsConn)
+ unlockClient, err := nmsClient.TPM2Unlock(context.Background())
+ if err != nil {
+ return []byte{}, err
+ }
+ defer unlockClient.CloseSend()
+ unlockInitContainer, err := unlockClient.Recv()
+ if err != nil {
+ return []byte{}, err
+ }
+ unlockInitVariant, ok := unlockInitContainer.Stage.(*api.TPM2UnlockFlowResponse_UnlockInit)
+ if !ok {
+ return []byte{}, errors.New("TPM2Unlock protocol violation")
+ }
+ unlockInit := unlockInitVariant.UnlockInit
+ quote, sig, err := tpm.AttestPlatform(unlockInit.Nonce)
+ if err != nil {
+ return []byte{}, fmt.Errorf("failed to attest platform: %w", err)
+ }
+ pcrs, err := tpm.GetPCRs()
+ if err != nil {
+ return []byte{}, fmt.Errorf("failed to get PCRs from TPM: %w", err)
+ }
+ if err := unlockClient.Send(&api.TPM2UnlockFlowRequeset{Stage: &api.TPM2UnlockFlowRequeset_UnlockRequest{
+ UnlockRequest: &api.TPM2UnlockRequest{
+ Pcrs: pcrs,
+ Quote: quote,
+ QuoteSignature: sig,
+ NodeId: enrolment.NodeId,
+ },
+ }}); err != nil {
+ return []byte{}, err
+ }
+ unlockResponseContainer, err := unlockClient.Recv()
+ if err != nil {
+ return []byte{}, err
+ }
+ unlockResponseVariant, ok := unlockResponseContainer.Stage.(*api.TPM2UnlockFlowResponse_UnlockResponse)
+ if !ok {
+ return []byte{}, errors.New("violated TPM2Unlock protocol")
+ }
+ unlockResponse := unlockResponseVariant.UnlockResponse
+
+ return unlockResponse.GlobalUnlockKey, nil
+}
diff --git a/core/internal/network/BUILD.bazel b/core/internal/network/BUILD.bazel
index e7f7d55..3208ea8 100644
--- a/core/internal/network/BUILD.bazel
+++ b/core/internal/network/BUILD.bazel
@@ -7,6 +7,7 @@
visibility = ["//:__subpackages__"],
deps = [
"//core/internal/common/service:go_default_library",
+ "@com_github_insomniacslk_dhcp//dhcpv4:go_default_library",
"@com_github_insomniacslk_dhcp//dhcpv4/nclient4:go_default_library",
"@com_github_vishvananda_netlink//:go_default_library",
"@org_golang_x_sys//unix:go_default_library",
diff --git a/core/internal/network/main.go b/core/internal/network/main.go
index 0888de7..04ab159 100644
--- a/core/internal/network/main.go
+++ b/core/internal/network/main.go
@@ -21,9 +21,12 @@
"fmt"
"net"
"os"
+ "sync"
+ "time"
"git.monogon.dev/source/nexantic.git/core/internal/common/service"
+ "github.com/insomniacslk/dhcp/dhcpv4"
"github.com/insomniacslk/dhcp/dhcpv4/nclient4"
"github.com/vishvananda/netlink"
"go.uber.org/zap"
@@ -39,6 +42,9 @@
*service.BaseService
config Config
dhcp4Client *nclient4.Client
+ ip *net.IP
+ ipNotify chan struct{}
+ lock sync.Mutex
}
type Config struct {
@@ -46,7 +52,8 @@
func NewNetworkService(config Config, logger *zap.Logger) (*Service, error) {
s := &Service{
- config: config,
+ config: config,
+ ipNotify: make(chan struct{}),
}
s.BaseService = service.NewBaseService("network", logger, s)
return s, nil
@@ -105,20 +112,60 @@
if err != nil {
panic(err)
}
- _, ack, err := client.Request(context.Background())
- if err != nil {
- panic(err)
+ var ack *dhcpv4.DHCPv4
+ for {
+ dhcpCtx, dhcpCtxCancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer dhcpCtxCancel()
+ _, ack, err = client.Request(dhcpCtx)
+ if err == nil {
+ break
+ }
+ s.Logger.Info("DHCP request failed", zap.Error(err))
}
s.Logger.Info("Network service got IP", zap.String("ip", ack.YourIPAddr.String()))
if err := setResolvconf(ack.DNS(), []string{}); err != nil {
s.Logger.Warn("Failed to set resolvconf", zap.Error(err))
}
+
+ s.lock.Lock()
+ s.ip = &ack.YourIPAddr
+ s.lock.Unlock()
+loop:
+ for {
+ select {
+ case s.ipNotify <- struct{}{}:
+ default:
+ break loop
+ }
+ }
+
if err := addNetworkRoutes(iface, net.IPNet{IP: ack.YourIPAddr, Mask: ack.SubnetMask()}, ack.GatewayIPAddr); err != nil {
s.Logger.Warn("Failed to add routes", zap.Error(err))
}
return nil
}
+// GetIP returns the current IP (and optionally waits for one to be assigned)
+func (s *Service) GetIP(wait bool) *net.IP {
+ s.lock.Lock()
+ if !wait {
+ ip := s.ip
+ s.lock.Unlock()
+ return ip
+ }
+
+ for {
+ if s.ip != nil {
+ ip := s.ip
+ s.lock.Unlock()
+ return ip
+ }
+ s.lock.Unlock()
+ <-s.ipNotify
+ s.lock.Lock()
+ }
+}
+
func (s *Service) OnStart() error {
s.Logger.Info("Starting network service")
links, err := netlink.LinkList()
diff --git a/core/internal/node/BUILD.bazel b/core/internal/node/BUILD.bazel
index 529c6ee..d96c0e6 100644
--- a/core/internal/node/BUILD.bazel
+++ b/core/internal/node/BUILD.bazel
@@ -13,10 +13,15 @@
"//core/internal/api:go_default_library",
"//core/internal/common:go_default_library",
"//core/internal/consensus:go_default_library",
+ "//core/internal/integrity/tpm2:go_default_library",
"//core/internal/kubernetes:go_default_library",
+ "//core/internal/network:go_default_library",
"//core/internal/storage:go_default_library",
- "@com_github_google_uuid//:go_default_library",
+ "@com_github_cenkalti_backoff_v4//:go_default_library",
+ "@com_github_gogo_protobuf//proto:go_default_library",
+ "@org_golang_google_grpc//:go_default_library",
"@org_golang_google_grpc//codes:go_default_library",
+ "@org_golang_google_grpc//credentials:go_default_library",
"@org_golang_google_grpc//status:go_default_library",
"@org_uber_go_zap//:go_default_library",
],
diff --git a/core/internal/node/main.go b/core/internal/node/main.go
index 40aa8b3..b1d74d6 100644
--- a/core/internal/node/main.go
+++ b/core/internal/node/main.go
@@ -17,34 +17,61 @@
package node
import (
+ "bytes"
+ "crypto/ed25519"
+ "crypto/rand"
+ "crypto/sha512"
+ "crypto/tls"
+ "crypto/x509"
+ "crypto/x509/pkix"
+ "encoding/base64"
+ "errors"
"flag"
+ "fmt"
+ "io/ioutil"
+ "math/big"
+ "net"
+ "time"
+ "os"
+
+ apipb "git.monogon.dev/source/nexantic.git/core/generated/api"
"git.monogon.dev/source/nexantic.git/core/internal/api"
"git.monogon.dev/source/nexantic.git/core/internal/common"
"git.monogon.dev/source/nexantic.git/core/internal/consensus"
+ "git.monogon.dev/source/nexantic.git/core/internal/integrity/tpm2"
"git.monogon.dev/source/nexantic.git/core/internal/kubernetes"
+ "git.monogon.dev/source/nexantic.git/core/internal/network"
"git.monogon.dev/source/nexantic.git/core/internal/storage"
- "os"
+ "github.com/cenkalti/backoff/v4"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/credentials"
- "github.com/google/uuid"
+ "github.com/gogo/protobuf/proto"
"go.uber.org/zap"
)
+var (
+ // From RFC 5280 Section 4.1.2.5
+ unknownNotAfter = time.Unix(253402300799, 0)
+)
+
type (
SmalltownNode struct {
Api *api.Server
Consensus *consensus.Service
Storage *storage.Manager
Kubernetes *kubernetes.Service
+ Network *network.Service
- logger *zap.Logger
- state common.SmalltownState
- joinToken string
- hostname string
+ logger *zap.Logger
+ state common.SmalltownState
+ hostname string
+ enrolmentConfig *apipb.EnrolmentConfig
}
)
-func NewSmalltownNode(logger *zap.Logger, apiPort, consensusPort uint16) (*SmalltownNode, error) {
+func NewSmalltownNode(logger *zap.Logger) (*SmalltownNode, error) {
flag.Parse()
logger.Info("Creating Smalltown node")
@@ -53,17 +80,32 @@
panic(err)
}
+ networkService, err := network.NewNetworkService(network.Config{}, logger.With(zap.String("component", "network")))
+ if err != nil {
+ panic(err)
+ }
+
+ if err := networkService.Start(); err != nil {
+ logger.Panic("Failed to start network service", zap.Error(err))
+ }
+
storageManager, err := storage.Initialize(logger.With(zap.String("component", "storage")))
if err != nil {
logger.Error("Failed to initialize storage manager", zap.Error(err))
return nil, err
}
+ externalIP := networkService.GetIP(true)
+ if externalIP == nil {
+ logger.Panic("Waited for IP but didn't get one")
+ }
+
+ // Important to know if the GetIP above hangs
+ logger.Info("Node has IP", zap.String("ip", externalIP.String()))
consensusService, err := consensus.NewConsensusService(consensus.Config{
Name: hostname,
- ListenPort: consensusPort,
ListenHost: "0.0.0.0",
- ExternalHost: "10.0.2.15", // TODO: Once Multi-Node setups are actually used, this needs to be corrected
+ ExternalHost: externalIP.String(),
}, logger.With(zap.String("module", "consensus")))
if err != nil {
return nil, err
@@ -72,13 +114,12 @@
s := &SmalltownNode{
Consensus: consensusService,
Storage: storageManager,
+ Network: networkService,
logger: logger,
hostname: hostname,
}
- apiService, err := api.NewApiServer(&api.Config{
- Port: apiPort,
- }, logger.With(zap.String("module", "api")), s, s.Consensus)
+ apiService, err := api.NewApiServer(&api.Config{}, logger.With(zap.String("module", "api")), s.Consensus)
if err != nil {
return nil, err
}
@@ -95,51 +136,330 @@
func (s *SmalltownNode) Start() error {
s.logger.Info("Starting Smalltown node")
- if s.Consensus.IsProvisioned() {
- s.logger.Info("Consensus is provisioned")
- err := s.startFull()
- if err != nil {
- return err
- }
- } else {
- s.logger.Info("Consensus is not provisioned, starting provisioning...")
- err := s.startForSetup()
- if err != nil {
- return err
- }
+ // TODO(lorenz): Abstracting enrolment sounds like a good idea, but ends up being painful
+ // because of things like storage access. I'm keeping it this way until the more complex
+ // enrolment procedures are fleshed out. This is also a bit panic()-happy, but there is really
+ // no good way out of an invalid enrolment configuration.
+ enrolmentPath, err := s.Storage.GetPathInPlace(storage.PlaceESP, "enrolment.pb")
+ if err != nil {
+ s.logger.Panic("ESP configuration partition not available", zap.Error(err))
}
+ enrolmentConfigRaw, err := ioutil.ReadFile(enrolmentPath)
+ if err == nil {
+ // We have an enrolment file, let's check its contents
+ var enrolmentConfig apipb.EnrolmentConfig
+ if err := proto.Unmarshal(enrolmentConfigRaw, &enrolmentConfig); err != nil {
+ s.logger.Panic("Invalid enrolment configuration provided", zap.Error(err))
+ }
+ s.enrolmentConfig = &enrolmentConfig
+ // The enrolment secret is only zeroed after
+ if len(enrolmentConfig.EnrolmentSecret) == 0 {
+ return s.startFull()
+ }
+ return s.startEnrolling()
+ } else if os.IsNotExist(err) {
+ // This is ok like this, once a new cluster has been set up the initial node also generates
+ // its own enrolment config
+ return s.startForSetup()
+ }
+ // Unknown error reading enrolment config (disk issues/invalid configuration format/...)
+ s.logger.Panic("Invalid enrolment configuration provided", zap.Error(err))
+ panic("Unreachable")
+}
+
+func (s *SmalltownNode) startEnrolling() error {
+ s.logger.Info("Initializing subsystems for enrolment")
+ s.state = common.StateEnrollMode
+
+ nodeInfo, nodeID, err := s.InitializeNode()
+ if err != nil {
+ return err
+ }
+
+ // We only support TPM2 at the moment, any abstractions here would be premature
+ trustAgent := tpm2.TPM2Agent{}
+
+ initializeOp := func() error {
+ if err := trustAgent.Initialize(*nodeInfo, *s.enrolmentConfig); err != nil {
+ s.logger.Warn("Failed to initialize integrity backend", zap.Error(err))
+ return err
+ }
+ return nil
+ }
+
+ if err := backoff.Retry(initializeOp, getIntegrityBackoff()); err != nil {
+ panic("invariant violated: integrity initialization retry can never fail")
+ }
+
+ enrolmentPath, err := s.Storage.GetPathInPlace(storage.PlaceESP, "enrolment.pb")
+ if err != nil {
+ panic(err)
+ }
+
+ s.enrolmentConfig.EnrolmentSecret = []byte{}
+ s.enrolmentConfig.NodeId = nodeID
+
+ enrolmentConfigRaw, err := proto.Marshal(s.enrolmentConfig)
+ if err != nil {
+ panic(err)
+ }
+ if err := ioutil.WriteFile(enrolmentPath, enrolmentConfigRaw, 0600); err != nil {
+ return err
+ }
+ s.logger.Info("Node successfully enrolled")
return nil
}
func (s *SmalltownNode) startForSetup() error {
- s.logger.Info("Initializing subsystems for setup mode")
- s.state = common.StateSetupMode
- s.joinToken = uuid.New().String()
-
- err := s.Api.Start()
+ s.logger.Info("Setting up a new cluster")
+ initData, nodeID, err := s.InitializeNode()
if err != nil {
- s.logger.Error("Failed to start the API service", zap.Error(err))
return err
}
- return nil
-}
+ if err := s.initNodeAPI(); err != nil {
+ return err
+ }
-func (s *SmalltownNode) startFull() error {
- s.logger.Info("Initializing subsystems for production")
- s.state = common.StateConfigured
-
- err := s.SetupBackend()
+ dataPath, err := s.Storage.GetPathInPlace(storage.PlaceData, "etcd")
if err != nil {
return err
}
+ // Spin up etcd
+ config := s.Consensus.GetConfig()
+ config.NewCluster = true
+ config.Name = s.hostname
+ config.DataDir = dataPath
+ s.Consensus.SetConfig(config)
+
+ // Generate the cluster CA and store it to local storage.
+ if err := s.Consensus.PrecreateCA(); err != nil {
+ return err
+ }
+
err = s.Consensus.Start()
if err != nil {
return err
}
+ // Now that the cluster is up and running, we can persist the CA to the cluster.
+ if err := s.Consensus.InjectCA(); err != nil {
+ return err
+ }
+
+ if err := s.Api.BootstrapNewClusterHook(initData); err != nil {
+ return err
+ }
+
+ if err := s.Kubernetes.NewCluster(); err != nil {
+ return err
+ }
+
+ if err := s.Kubernetes.Start(); err != nil {
+ return err
+ }
+
+ if err := s.Api.Start(); err != nil {
+ s.logger.Error("Failed to start the API service", zap.Error(err))
+ return err
+ }
+
+ enrolmentPath, err := s.Storage.GetPathInPlace(storage.PlaceESP, "enrolment.pb")
+ if err != nil {
+ panic(err)
+ }
+
+ masterCert, err := s.Api.GetMasterCert()
+ if err != nil {
+ return err
+ }
+
+ enrolmentConfig := &apipb.EnrolmentConfig{
+ EnrolmentSecret: []byte{}, // First node is always already enrolled
+ MastersCert: masterCert,
+ MasterIps: [][]byte{[]byte(*s.Network.GetIP(true))},
+ NodeId: nodeID,
+ }
+ enrolmentConfigRaw, err := proto.Marshal(enrolmentConfig)
+ if err != nil {
+ panic(err)
+ }
+ if err := ioutil.WriteFile(enrolmentPath, enrolmentConfigRaw, 0600); err != nil {
+ return err
+ }
+ masterCertFingerprint := sha512.Sum512_256(masterCert)
+ s.logger.Info("New Smalltown cluster successfully bootstrapped", zap.Binary("fingerprint", masterCertFingerprint[:]))
+
+ return nil
+}
+
+func (s *SmalltownNode) generateNodeID() ([]byte, string, error) {
+ serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 127)
+ serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+ if err != nil {
+ return []byte{}, "", fmt.Errorf("Failed to generate serial number: %w", err)
+ }
+
+ pubKey, privKeyRaw, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ return []byte{}, "", err
+ }
+ privkey, err := x509.MarshalPKCS8PrivateKey(privKeyRaw)
+ if err != nil {
+ return []byte{}, "", err
+ }
+
+ nodeKeyPath, err := s.Storage.GetPathInPlace(storage.PlaceData, "node-key.der")
+ if err != nil {
+ return []byte{}, "", err
+ }
+
+ if err := ioutil.WriteFile(nodeKeyPath, privkey, 0600); err != nil {
+ return []byte{}, "", fmt.Errorf("failed to write node key: %w", err)
+ }
+
+ name := "smalltown-" + base64.RawStdEncoding.EncodeToString([]byte(pubKey))
+
+ // This has no SANs because it authenticates by public key, not by name
+ nodeCert := &x509.Certificate{
+ SerialNumber: serialNumber,
+ Subject: pkix.Name{
+ // We identify nodes by their ID public keys (not hashed since a strong hash is longer and serves no benefit)
+ CommonName: name,
+ },
+ IsCA: false,
+ BasicConstraintsValid: true,
+ NotBefore: time.Now(),
+ NotAfter: unknownNotAfter,
+ // Certificate is used both as server & client
+ ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
+ }
+ cert, err := x509.CreateCertificate(rand.Reader, nodeCert, nodeCert, pubKey, privKeyRaw)
+ if err != nil {
+ return []byte{}, "", err
+ }
+
+ nodeCertPath, err := s.Storage.GetPathInPlace(storage.PlaceData, "node.der")
+ if err != nil {
+ return []byte{}, "", err
+ }
+
+ if err := ioutil.WriteFile(nodeCertPath, cert, 0600); err != nil {
+ return []byte{}, "", fmt.Errorf("failed to write node cert: %w", err)
+ }
+ return cert, name, nil
+}
+
+func (s *SmalltownNode) initNodeAPI() error {
+ certPath, err := s.Storage.GetPathInPlace(storage.PlaceData, "node.der")
+ if err != nil {
+ s.logger.Panic("Invariant violated: Data is available once this is called")
+ }
+ keyPath, err := s.Storage.GetPathInPlace(storage.PlaceData, "node-key.der")
+ if err != nil {
+ s.logger.Panic("Invariant violated: Data is available once this is called")
+ }
+
+ certRaw, err := ioutil.ReadFile(certPath)
+ if err != nil {
+ return err
+ }
+ privKeyRaw, err := ioutil.ReadFile(keyPath)
+ if err != nil {
+ return err
+ }
+
+ var nodeID tls.Certificate
+
+ cert, err := x509.ParseCertificate(certRaw)
+ if err != nil {
+ return err
+ }
+
+ privKey, err := x509.ParsePKCS8PrivateKey(privKeyRaw)
+ if err != nil {
+ return err
+ }
+
+ nodeID.Certificate = [][]byte{certRaw}
+ nodeID.PrivateKey = privKey
+ nodeID.Leaf = cert
+
+ secureTransport := &tls.Config{
+ Certificates: []tls.Certificate{nodeID},
+ ClientAuth: tls.RequireAndVerifyClientCert,
+ InsecureSkipVerify: true,
+ // Critical function, please review any changes with care
+ // TODO(lorenz): Actively check that this actually provides the security guarantees that we need
+ VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
+ for _, cert := range rawCerts {
+ // X.509 certificates in DER can be compared like this since DER has a unique representation
+ // for each certificate.
+ if bytes.Equal(cert, s.enrolmentConfig.MastersCert) {
+ return nil
+ }
+ }
+ return errors.New("failed to find authorized NMS certificate")
+ },
+ MinVersion: tls.VersionTLS13,
+ }
+ secureTransportCreds := credentials.NewTLS(secureTransport)
+
+ masterListenHost := fmt.Sprintf(":%d", common.NodeServicePort)
+ lis, err := net.Listen("tcp", masterListenHost)
+ if err != nil {
+ s.logger.Fatal("failed to listen", zap.Error(err))
+ }
+
+ nodeGRPCServer := grpc.NewServer(grpc.Creds(secureTransportCreds))
+ apipb.RegisterNodeServiceServer(nodeGRPCServer, s)
+ go func() {
+ if err := nodeGRPCServer.Serve(lis); err != nil {
+ panic(err) // Can only happen during initialization and is always fatal
+ }
+ }()
+ return nil
+}
+
+func getIntegrityBackoff() *backoff.ExponentialBackOff {
+ unlockBackoff := backoff.NewExponentialBackOff()
+ unlockBackoff.MaxElapsedTime = time.Duration(0)
+ unlockBackoff.InitialInterval = 5 * time.Second
+ unlockBackoff.MaxInterval = 5 * time.Minute
+ return unlockBackoff
+}
+
+func (s *SmalltownNode) startFull() error {
+ s.logger.Info("Initializing subsystems for production")
+ s.state = common.StateJoined
+
+ trustAgent := tpm2.TPM2Agent{}
+ unlockOp := func() error {
+ unlockKey, err := trustAgent.Unlock(*s.enrolmentConfig)
+ if err != nil {
+ s.logger.Warn("Failed to unlock", zap.Error(err))
+ return err
+ }
+ if err := s.Storage.MountData(unlockKey); err != nil {
+ s.logger.Panic("Failed to mount storage", zap.Error(err))
+ return err
+ }
+ return nil
+ }
+
+ if err := backoff.Retry(unlockOp, getIntegrityBackoff()); err != nil {
+ s.logger.Panic("Invariant violated: Unlock retry can never fail")
+ }
+
+ s.initNodeAPI()
+
+ err := s.Consensus.Start()
+ if err != nil {
+ return err
+ }
+
err = s.Api.Start()
if err != nil {
s.logger.Error("Failed to start the API service", zap.Error(err))
@@ -158,9 +478,3 @@
s.logger.Info("Stopping Smalltown node")
return nil
}
-
-func (s *SmalltownNode) SetupBackend() error {
- s.logger.Debug("Creating trust backend")
-
- return nil
-}
diff --git a/core/internal/node/setup.go b/core/internal/node/setup.go
index 285046a..23eb24c 100644
--- a/core/internal/node/setup.go
+++ b/core/internal/node/setup.go
@@ -17,12 +17,15 @@
package node
import (
+ "context"
+
"git.monogon.dev/source/nexantic.git/core/generated/api"
"git.monogon.dev/source/nexantic.git/core/internal/common"
- "git.monogon.dev/source/nexantic.git/core/internal/storage"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
+ "errors"
+
"go.uber.org/zap"
)
@@ -38,107 +41,58 @@
return s.state
}
-func (s *SmalltownNode) GetJoinClusterToken() string {
- return s.joinToken
-}
-
-func (s *SmalltownNode) SetupNewCluster() error {
- if s.state == common.StateConfigured {
- return ErrAlreadySetup
- }
- dataPath, err := s.Storage.GetPathInPlace(storage.PlaceData, "etcd")
- if err == storage.ErrNotInitialized {
- return ErrStorageNotInitialized
- } else if err != nil {
- return err
- }
-
- s.logger.Info("Setting up a new cluster")
- s.logger.Info("Provisioning consensus")
-
- // Make sure etcd is not yet provisioned
- if s.Consensus.IsProvisioned() {
- return ErrConsensusAlreadyProvisioned
- }
-
- // Spin up etcd
- config := s.Consensus.GetConfig()
- config.NewCluster = true
- config.Name = s.hostname
- config.DataDir = dataPath
- s.Consensus.SetConfig(config)
-
- // Generate the cluster CA and store it to local storage.
- if err := s.Consensus.PrecreateCA(); err != nil {
- return err
- }
-
- err = s.Consensus.Start()
+// InitializeNode contains functionality that needs to be executed regardless of what the node does
+// later on
+func (s *SmalltownNode) InitializeNode() (*api.NewNodeInfo, string, error) {
+ globalUnlockKey, err := s.Storage.InitializeData()
if err != nil {
- return err
+ return nil, "", err
}
- // Now that the cluster is up and running, we can persist the CA to the cluster.
- if err := s.Consensus.InjectCA(); err != nil {
- return err
- }
+ nodeIP := s.Network.GetIP(true)
- if err := s.Kubernetes.NewCluster(); err != nil {
- return err
- }
-
- if err := s.Kubernetes.Start(); err != nil {
- return err
- }
-
- // Change system state
- s.state = common.StateConfigured
-
- s.logger.Info("New Cluster set up. Node is now fully operational")
-
- return nil
-}
-
-func (s *SmalltownNode) EnterJoinClusterMode() error {
- if s.state == common.StateConfigured {
- return ErrAlreadySetup
- }
- s.state = common.StateClusterJoinMode
-
- s.logger.Info("Node is now in the cluster join mode")
-
- return nil
-}
-
-func (s *SmalltownNode) JoinCluster(clusterString string, certs *api.ConsensusCertificates) error {
- if s.state != common.StateClusterJoinMode {
- return ErrNotInJoinMode
- }
-
- s.logger.Info("Joining cluster", zap.String("cluster", clusterString))
-
- err := s.SetupBackend()
+ nodeCert, nodeID, err := s.generateNodeID()
if err != nil {
- return err
+ return nil, "", err
}
+ return &api.NewNodeInfo{
+ EnrolmentConfig: s.enrolmentConfig,
+ Ip: []byte(*nodeIP),
+ IdCert: nodeCert,
+ GlobalUnlockKey: globalUnlockKey,
+ }, nodeID, nil
+}
+
+func (s *SmalltownNode) JoinCluster(context context.Context, req *api.JoinClusterRequest) (*api.JoinClusterResponse, error) {
+ if s.state != common.StateEnrollMode {
+ return nil, ErrNotInJoinMode
+ }
+
+ s.logger.Info("Joining Consenus")
+
config := s.Consensus.GetConfig()
config.Name = s.hostname
- config.InitialCluster = clusterString
+ config.InitialCluster = "default" // Clusters can't cross-join anyways due to cryptography
s.Consensus.SetConfig(config)
- if err := s.Consensus.WriteCertificateFiles(certs); err != nil {
- return err
+ var err error
+ if err != nil {
+ s.logger.Warn("Invalid JoinCluster request", zap.Error(err))
+ return nil, errors.New("invalid join request")
+ }
+ if err := s.Consensus.WriteCertificateFiles(req.Certs); err != nil {
+ return nil, err
}
// Start consensus
err = s.Consensus.Start()
if err != nil {
- return err
+ return nil, err
}
- s.state = common.StateConfigured
+ s.state = common.StateJoined
s.logger.Info("Joined cluster. Node is now syncing.")
- return nil
+ return &api.JoinClusterResponse{}, nil
}
diff --git a/core/internal/storage/data.go b/core/internal/storage/data.go
index 80af4c9..2b2251a 100644
--- a/core/internal/storage/data.go
+++ b/core/internal/storage/data.go
@@ -18,13 +18,13 @@
import (
"fmt"
- "git.monogon.dev/source/nexantic.git/core/pkg/tpm"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"sync"
+ "git.monogon.dev/source/nexantic.git/core/pkg/tpm"
"go.uber.org/zap"
"golang.org/x/sys/unix"
)
@@ -65,72 +65,99 @@
manager.mutex.Lock()
defer manager.mutex.Unlock()
- sealedKeyFile, err := os.Open(etcdSealedKeyLocation)
- if os.IsNotExist(err) {
- logger.Info("Initializing encrypted storage, this might take a while...")
- go manager.initializeData()
- } else if err != nil {
- return nil, err
- } else {
- sealedKey, err := ioutil.ReadAll(sealedKeyFile)
- sealedKeyFile.Close()
- if err != nil {
- return nil, err
- }
- key, err := tpm.Unseal(sealedKey)
- if err != nil {
- return nil, err
- }
- if err := MapEncryptedBlockDevice("data", SmalltownDataCryptPath, key); err != nil {
- return nil, err
- }
- if err := manager.mountData(); err != nil {
- return nil, err
- }
- logger.Info("Mounted encrypted storage")
- }
return manager, nil
}
-func (s *Manager) initializeData() {
- key, err := tpm.GenerateSafeKey(256 / 8)
+var keySize uint16 = 256 / 8
+
+// MountData mounts the Smalltown data partition with the given global unlock key. It automatically
+// unseals the local unlock key from the TPM.
+func (s *Manager) MountData(globalUnlockKey []byte) error {
+ localPath, err := s.GetPathInPlace(PlaceESP, "local_unlock.bin")
if err != nil {
- s.logger.Error("Failed to generate master key", zap.Error(err))
- s.initializationError = fmt.Errorf("Failed to generate master key: %w", err)
- return
+ return fmt.Errorf("failed to find ESP mount: %w", err)
}
- sealedKey, err := tpm.Seal(key, tpm.FullSystemPCRs)
+ localUnlockBlob, err := ioutil.ReadFile(localPath)
if err != nil {
- s.logger.Error("Failed to seal master key", zap.Error(err))
- s.initializationError = fmt.Errorf("Failed to seal master key: %w", err)
- return
+ return fmt.Errorf("failed to read local unlock file from ESP: %w", err)
}
+ localUnlockKey, err := tpm.Unseal(localUnlockBlob)
+ if err != nil {
+ return fmt.Errorf("failed to unseal local unlock key: %w", err)
+ }
+
+ key := make([]byte, keySize)
+ for i := uint16(0); i < keySize; i++ {
+ key[i] = localUnlockKey[i] ^ globalUnlockKey[i]
+ }
+
+ if err := MapEncryptedBlockDevice("data", SmalltownDataCryptPath, key); err != nil {
+ return err
+ }
+ if err := s.mountData(); err != nil {
+ return err
+ }
+ s.mutex.Lock()
+ s.dataReady = true
+ s.mutex.Unlock()
+ s.logger.Info("Mounted encrypted storage")
+ return nil
+}
+
+// InitializeData initializes the Smalltown data partition and returns the global unlock key. It seals
+// the local portion into the TPM and stores the blob on the ESP. This is a potentially slow
+// operation since it touches the whole partition.
+func (s *Manager) InitializeData() ([]byte, error) {
+ localUnlockKey, err := tpm.GenerateSafeKey(keySize)
+ if err != nil {
+ return []byte{}, fmt.Errorf("failed to generate safe key: %w", err)
+ }
+ globalUnlockKey, err := tpm.GenerateSafeKey(keySize)
+ if err != nil {
+ return []byte{}, fmt.Errorf("failed to generate safe key: %w", err)
+ }
+
+ localUnlockBlob, err := tpm.Seal(localUnlockKey, tpm.SecureBootPCRs)
+ if err != nil {
+ return []byte{}, fmt.Errorf("failed to seal local unlock key: %w", err)
+ }
+
+ // The actual key is generated by XORing together the localUnlockKey and the globalUnlockKey
+ // This provides us with a mathematical guarantee that the resulting key cannot be recovered
+ // whithout knowledge of both parts.
+ key := make([]byte, keySize)
+ for i := uint16(0); i < keySize; i++ {
+ key[i] = localUnlockKey[i] ^ globalUnlockKey[i]
+ }
+
if err := InitializeEncryptedBlockDevice("data", SmalltownDataCryptPath, key); err != nil {
s.logger.Error("Failed to initialize encrypted block device", zap.Error(err))
- s.initializationError = fmt.Errorf("Failed to initialize encrypted block device: %w", err)
- return
+ return []byte{}, fmt.Errorf("failed to initialize encrypted block device: %w", err)
}
mkfsCmd := exec.Command("/bin/mkfs.xfs", "-qf", "/dev/data")
if _, err := mkfsCmd.Output(); err != nil {
s.logger.Error("Failed to format encrypted block device", zap.Error(err))
- s.initializationError = fmt.Errorf("Failed to format encrypted block device: %w", err)
- return
- }
- // This file is the marker if the partition has
- if err := ioutil.WriteFile(etcdSealedKeyLocation, sealedKey, 0600); err != nil {
- panic(err)
+ return []byte{}, fmt.Errorf("failed to format encrypted block device: %w", err)
}
if err := s.mountData(); err != nil {
- s.initializationError = err
- return
+ return []byte{}, err
}
s.mutex.Lock()
s.dataReady = true
s.mutex.Unlock()
+ localPath, err := s.GetPathInPlace(PlaceESP, "local_unlock.bin")
+ if err != nil {
+ return []byte{}, fmt.Errorf("failed to find ESP mount: %w", err)
+ }
+ if err := ioutil.WriteFile(localPath, localUnlockBlob, 0600); err != nil {
+ return []byte{}, fmt.Errorf("failed to write local unlock file to ESP: %w", err)
+ }
+
s.logger.Info("Initialized encrypted storage")
+ return globalUnlockKey, nil
}
func (s *Manager) mountData() error {