Attestation & Identity & Global Unlock & Enrolment
This changes the node startup sequence significantly. Now the following three startup procedures replace the old setup/join mechanic:
* If no enrolment config is present, automatically bootstrap a new cluster and become master for it.
* If an enrolment config with an enrolment token is present, register with the NodeManagementService.
* If an enrolment config without an enrolment token is present, attempt a normal cluster unlock.
It also completely revamps the GRPC management services:
* NodeManagementService is a master-only service that deals with other nodes and has a cluster-wide identity
* NodeService is only available in unlocked state and keyed with the node identity
* ClusterManagement is now a master-only service that's been spun out of the main NMS since they have very different authentication models and also deals with EnrolmentConfigs
The TPM support library has also been extended by:
* Lots of integrity attestation and verification functions
* Built-in AK management
* Some advanced policy-based authentication stuff
Also contains various enhancements to the network service to make everything work in a proper multi-node environment.
Lots of old code has been thrown out.
Test Plan: Passed a full manual test of all three startup modes (bootstrap, enrolment and normal unlock) including automated EnrolmentConfig generation and consumption in a dual-node configuration on swtpm / OVMF.
Bug: T499
X-Origin-Diff: phab/D291
GitOrigin-RevId: d53755c828218b1df83a1d7ad252c7b3231abca8
diff --git a/core/api/api/BUILD.bazel b/core/api/api/BUILD.bazel
index d28e60c..61e4fe7 100644
--- a/core/api/api/BUILD.bazel
+++ b/core/api/api/BUILD.bazel
@@ -5,7 +5,6 @@
name = "api_proto",
srcs = ["schema.proto"],
visibility = ["//visibility:public"],
- deps = ["//core/api/common:common_proto"],
)
go_proto_library(
@@ -14,7 +13,6 @@
importpath = "git.monogon.dev/source/nexantic.git/core/generated/api",
proto = ":api_proto",
visibility = ["//visibility:public"],
- deps = ["//core/api/common:go_default_library"],
)
go_library(
diff --git a/core/api/api/schema.proto b/core/api/api/schema.proto
index dd774e6..7b4bdaf 100644
--- a/core/api/api/schema.proto
+++ b/core/api/api/schema.proto
@@ -17,164 +17,225 @@
syntax = "proto3";
package api;
-import "core/api/common/main.proto";
-
option go_package = "git.monogon.dev/source/nexantic.git/core/generated/api";
// TODO(leo): A "cluster" in terms of this API is an etcd cluster. We have
// since realized that we will need multiple kinds of nodes in a Smalltown cluster
// (like worker nodes), which aren't etcd members. This API is pretty strongly
-// coupled to etcd at this point. How do we handle cluster membership for workers?
+// coupled to etcd at this point. How do we handle cluster membership for
+// workers?
// The ClusterManagement service is used by an authenticated administrative user
// to manage node membership in an existing Smalltown cluster.
service ClusterManagement {
- // Add a node to the cluster, subject to successful remote attestation.
- rpc AddNode (AddNodeRequest) returns (AddNodeResponse) {
+ // Add a node to the cluster, subject to successful remote attestation.
+ rpc AddNode(AddNodeRequest) returns (AddNodeResponse) {}
- }
+ // Remove a node from the cluster.
+ rpc RemoveNode(RemoveNodeRequest) returns (RemoveNodeRequest) {}
- // Remove a node from the cluster.
- rpc RemoveNode (RemoveNodeRequest) returns (RemoveNodeResponse) {
+ // List all cluster nodes
+ rpc ListNodes(ListNodesRequest) returns (ListNodesResponse) {}
- }
+ // NewEnrolmentConfig generates a new enrolment config for adding new nodes to
+ // the cluster
+ rpc NewEnrolmentConfig(NewEnrolmentConfigRequest)
+ returns (NewEnrolmentConfigResponse) {}
- // List all cluster nodes
- rpc ListNodes (ListNodesRequest) returns (ListNodesResponse) {
+ rpc ListEnrolmentConfigs(ListEnrolmentConfigsRequest)
+ returns (ListEnrolmentConfigsResponse) {}
- }
+ rpc RemoveEnrolmentConfig(RemoveEnrolmentConfigRequest)
+ returns (RemoveEnrolmentConfigResponse) {}
}
-// SetupService manages a single node's lifecycle, and it called either by an administrative
-// user while bootstrapping the cluster, or by existing nodes in a cluster.
-service SetupService {
- // Setup bootstraps an unprovisioned node and selects its bootstrapping mode
- // (either joining an existing cluster, or creating a new one).
- rpc Setup (SetupRequest) returns (SetupResponse) {
+message NewEnrolmentConfigRequest { string name = 1; }
+message NewEnrolmentConfigResponse { EnrolmentConfig enrolment_config = 1; }
- }
-
- // BootstrapCluster is called by an administrative user to bootstrap the first node
- // of a Smalltown cluster.
- rpc BootstrapNewCluster (BootstrapNewClusterRequest) returns (BootstrapNewClusterResponse) {
-
- }
-
- // JoinCluster can be called by another Smalltown node when the node has been put into
- // JOIN_CLUSTER mode using Setup. This request sets up all necessary config variables,
- // joins the consensus and puts the node into production state.
- rpc JoinCluster (JoinClusterRequest) returns (JoinClusterResponse) {
-
- }
-
- // Attest is called by an existing cluster node to verify a node's remote
- // attestation identity.
- //
- // This is not yet implemented, but at least the following values will be signed:
- //
- // - the node's trust backend and other configuration
- // (such that the new node can verify whether it is in an acceptable state)
- //
- // - the set of PCRs we use for sealing, which includes the firmware and secure boot state
- // (see pkg/tpm/tpm.go)
- rpc Attest (AttestRequest) returns (AttestResponse) {
-
- }
+message ListEnrolmentConfigsRequest {}
+message ListEnrolmentConfigsResponse {
+ repeated EnrolmentConfig enrolment_config = 1;
}
-message SetupRequest {
- // Hostname for the new node
- // TODO(leo): how will we handle hostnames? do we let the customer choose them? etc.
- string nodeName = 1;
- // Trust backend to be used. Right now, we support just one kind of trust
- // backend (our internal one), but at some point, we might support external
- // key management hardware. It has to be configured this early since it would
- // also store cluster secrets used during provisioning and setup.
- smalltown.common.TrustBackend trustBackend = 2;
+message RemoveEnrolmentConfigRequest {
+ // TODO(lorenz): How do we want to remove EnrolmentConfigs?
+}
+message RemoveEnrolmentConfigResponse {}
+
+// NodeService runs on all nodes and allows active masters to perform things
+// like attestation or grab other system state. Callers are authenticated via
+// TLS using the certificate from the EnrolmentConfig. Any client needs to
+// authenticate the node it's talking to by getting the public key from the
+// consensus service to verify against.
+service NodeService {
+ rpc JoinCluster(JoinClusterRequest) returns (JoinClusterResponse) {}
}
-message SetupResponse {
- // provisioningToken is a secret key that establishes a mutual trust-on-first-use
- // relationship between the cluster and the new node (after passing attestation checks).
- string provisioningToken = 1;
+// NodeManagementService runs on all masters, is identified by the
+// NodeManagementService TLS certificate and is where nodes report to when they
+// initially join or are restarted and need to be unlocked again.
+service NodeManagementService {
+ // NewTPM2NodeRegister is called by a node as soon as it is properly
+ // initialized. Then any number of policies can determine weather and when to
+ // add the node to the cluster.
+ //
+ // The idea behind this is that we just deliver everything we have trust-wise
+ // and then it's up to the customer or his policies to either adopt this node
+ // or not since TPM trust hierarchies are a mess in general.
+ rpc NewTPM2NodeRegister(stream TPM2FlowRequest)
+ returns (stream TPM2FlowResponse) {}
+
+ // Nodes that were rebooted request their global unlock secret from here.
+ rpc TPM2Unlock(stream TPM2UnlockFlowRequeset)
+ returns (stream TPM2UnlockFlowResponse) {}
+}
+
+/*
+This flow needs to run in a single TLS session, so we force that with
+bidirectional streaming. It works like this New Node NodeManagementService
+TPM2RegisterRequest ------>
+ <------ TPM2AttestRequest
+TPM2AttestResponse ------>
+NewNodeInfo ------>
+*/
+
+message TPM2FlowRequest {
+ oneof Stage {
+ TPM2RegisterRequest register = 1;
+ TPM2AttestResponse attest_response = 2;
+ NewNodeInfo new_node_info = 3;
+ }
+}
+
+message TPM2FlowResponse {
+ oneof Stage { TPM2AttestRequest attest_request = 1; }
+}
+
+// EnrolmentConfig is attached to an installation payload
+message EnrolmentConfig {
+ bytes enrolment_secret = 1;
+ bytes masters_cert = 2; // X.509 DER certificate of the NodeManagement service
+ repeated bytes master_ips = 3; // IPs where the NodeManagement service runs
+ // Filled in by node after it is enrolled
+ string node_id = 4;
+}
+
+message TPM2RegisterRequest {
+ bytes ak_public = 9; // AK public portion, TPM2_PUBLIC
+ bytes ek_pubkey = 5; // TPM EK public key, PKIX
+ bytes ek_cert = 6; // TPM EK certificate, X.509 DER (only if available in TPM)
+}
+
+message TPM2AttestRequest {
+ bytes ak_challenge = 1;
+ bytes ak_challenge_secret = 2;
+ bytes quote_nonce = 3;
+}
+
+message TPM2AttestResponse {
+ bytes quote = 1;
+ bytes quote_signature = 4;
+
+ bytes ak_challenge_solution = 2;
+ repeated bytes pcrs = 3; // All 16 SHA256 SRTM PCRs in order
+}
+
+message NewNodeInfo {
+ EnrolmentConfig enrolment_config = 1;
+
+ bytes ip = 11; // IP of the node
+
+ bytes id_cert = 4; // ID certificate, X.509 DER
+
+ // Part of the encryption key for cluster unlock (32 byte), to be XOR'ed with
+ // the node-local part on the TPM
+ bytes global_unlock_key = 7;
+}
+
+message TPM2UnlockInit { bytes nonce = 1; }
+
+message TPM2UnlockRequest {
+ string node_id = 4;
+ repeated bytes pcrs = 1;
+ bytes quote = 2;
+ bytes quote_signature = 3;
+}
+
+message TPM2UnlockResponse { bytes global_unlock_key = 1; }
+
+message TPM2UnlockFlowRequeset {
+ oneof Stage { TPM2UnlockRequest unlock_request = 1; }
+}
+
+message TPM2UnlockFlowResponse {
+ oneof Stage {
+ TPM2UnlockInit unlock_init = 1;
+ TPM2UnlockResponse unlock_response = 2;
+ }
}
// ConsensusCertificates is a node's individual etcd certificates.
// When provisioning a new node, the existing node sends the new node
// its certificates after authenticating it.
message ConsensusCertificates {
- bytes ca = 1;
- bytes crl = 2;
- bytes cert = 3;
- bytes key = 4;
+ bytes ca = 1;
+ bytes crl = 2;
+ bytes cert = 3;
+ bytes key = 4;
}
message JoinClusterRequest {
- // The callee's provisioningToken. Knowledge of this token authenticates the caller.
- string provisioningToken = 1;
- // Cluster bootstrap URI for etcd. The caller will set this to the
- // list of existing nodes in the cluster. This value is only used during bootstrap.
- string initialCluster = 2;
- // New node's etcd client certificates
- ConsensusCertificates certs = 3;
+ // Cluster bootstrap URI for etcd. The caller will set this to the
+ // list of existing nodes in the cluster. This value is only used during
+ // bootstrap.
+ string initialCluster = 2;
+ // New node's etcd client certificates
+ ConsensusCertificates certs = 3;
}
-message JoinClusterResponse {
-
-}
-
-message BootstrapNewClusterRequest {
-
-}
-
-message BootstrapNewClusterResponse {
-
-}
-
-message AttestRequest {
- string challenge = 1;
-}
-
-message AttestResponse {
- string response = 1;
-}
+message JoinClusterResponse {}
message AddNodeRequest {
- // New node's address to connect to.
- // TODO(leo): Is this always an IP address?
- string addr = 1;
- // New node's provisioning token.
- string provisioningToken = 4;
+ string node_id = 1;
+ // TODO: Add things like role
}
-message AddNodeResponse {
+message AddNodeResponse {}
+message RemoveNodeRequest {}
+
+message RemoveNodeResponse {}
+
+message ListNodesRequest {}
+
+message ListNodesResponse { repeated Node nodes = 1; }
+
+message NodeTPM2 {
+ bytes ak_pub = 1; // TPM2T_PUBLIC
+ bytes ek_pubkey = 2; // PKIX DER
+ bytes ek_cert = 3; // X.509 DER
}
-message RemoveNodeRequest {
-
-}
-
-message RemoveNodeResponse {
-
-}
-
-message ListNodesRequest {
-
-}
-
-message ListNodesResponse {
- repeated Node nodes = 1;
-}
-
-// Node describes a single node's etcd membership state
+// Node describes a single node's state
message Node {
- // etcd member ID
- uint64 id = 1;
- // etcd member name
- string name = 2;
- // etcd peer URL
- string address = 3;
- // Whether the etcd member is synced with the cluster.
- bool synced = 4;
-}
+ bytes id_cert = 5;
+ bytes global_unlock_key = 7;
+ bytes address = 3;
+ enum State {
+ UNININITALIZED = 0; // WARNING: In this state the node has not been adopted and thus cannot be fully trusted
+ MASTER = 1; // A full master node with Consensus, NMS & Kubernetes control plane
+ WORKER = 2; // A worker node with just Kubelet and supporting services
+ }
+ State state = 9;
+
+ oneof integrity { NodeTPM2 tpm2 = 6; }
+
+ // etcd State (might later be moved to a separate type)
+
+ // etcd member ID
+ uint64 id = 1;
+ // etcd member name
+ string name = 2;
+ // Whether the etcd member is synced with the cluster.
+ bool synced = 4;
+}
\ No newline at end of file
diff --git a/core/cmd/init/main.go b/core/cmd/init/main.go
index 902008d..2219a35 100644
--- a/core/cmd/init/main.go
+++ b/core/cmd/init/main.go
@@ -18,13 +18,14 @@
import (
"fmt"
- "git.monogon.dev/source/nexantic.git/core/internal/network"
- "git.monogon.dev/source/nexantic.git/core/internal/node"
- "git.monogon.dev/source/nexantic.git/core/pkg/tpm"
"os"
"os/signal"
"runtime/debug"
+ "git.monogon.dev/source/nexantic.git/core/internal/network"
+ "git.monogon.dev/source/nexantic.git/core/internal/node"
+ "git.monogon.dev/source/nexantic.git/core/pkg/tpm"
+
"go.uber.org/zap"
"golang.org/x/sys/unix"
)
@@ -84,7 +85,7 @@
logger.Panic("Failed to start network service", zap.Error(err))
}
- nodeInstance, err := node.NewSmalltownNode(logger, apiPort, consensusPort)
+ nodeInstance, err := node.NewSmalltownNode(logger)
if err != nil {
panic(err)
}
diff --git a/core/cmd/mkenrolment/BUILD.bazel b/core/cmd/mkenrolment/BUILD.bazel
new file mode 100644
index 0000000..14db892
--- /dev/null
+++ b/core/cmd/mkenrolment/BUILD.bazel
@@ -0,0 +1,19 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+ name = "go_default_library",
+ srcs = ["main.go"],
+ importpath = "git.monogon.dev/source/nexantic.git/core/cmd/mkenrolment",
+ visibility = ["//visibility:private"],
+ deps = [
+ "//core/api/api:go_default_library",
+ "@com_github_gogo_protobuf//proto:go_default_library",
+ "@org_golang_google_grpc//:go_default_library",
+ ],
+)
+
+go_binary(
+ name = "mkenrolment",
+ embed = [":go_default_library"],
+ visibility = ["//visibility:public"],
+)
diff --git a/core/cmd/mkenrolment/main.go b/core/cmd/mkenrolment/main.go
new file mode 100644
index 0000000..a3254e0
--- /dev/null
+++ b/core/cmd/mkenrolment/main.go
@@ -0,0 +1,45 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "context"
+ "io/ioutil"
+ "os"
+
+ "git.monogon.dev/source/nexantic.git/core/generated/api"
+ "github.com/gogo/protobuf/proto"
+ "google.golang.org/grpc"
+)
+
+func main() {
+ conn, err := grpc.Dial(os.Args[1], grpc.WithInsecure())
+ if err != nil {
+ panic(err)
+ }
+ defer conn.Close()
+ cmc := api.NewClusterManagementClient(conn)
+ res, err := cmc.NewEnrolmentConfig(context.Background(), &api.NewEnrolmentConfigRequest{
+ Name: "test",
+ })
+
+ if err != nil {
+ panic(err)
+ }
+ enrolmentConfigRaw, err := proto.Marshal(res.EnrolmentConfig)
+ ioutil.WriteFile("enrolment.pb", enrolmentConfigRaw, 0644)
+}
diff --git a/core/cmd/mkimage/main.go b/core/cmd/mkimage/main.go
index f733e78..0803c80 100644
--- a/core/cmd/mkimage/main.go
+++ b/core/cmd/mkimage/main.go
@@ -31,9 +31,10 @@
var SmalltownDataPartition gpt.Type = gpt.Type("9eeec464-6885-414a-b278-4305c51f7966")
var (
- efiPayloadPath = flag.String("efi", "", "UEFI payload")
- outputPath = flag.String("out", "", "Output disk image")
- initramfsPath = flag.String("initramfs", "", "External initramfs [optional]")
+ efiPayloadPath = flag.String("efi", "", "UEFI payload")
+ outputPath = flag.String("out", "", "Output disk image")
+ initramfsPath = flag.String("initramfs", "", "External initramfs [optional]")
+ enrolmentCredentialsPath = flag.String("enrolment-credentials", "", "Enrolment credentials [optional]")
)
func mibToSectors(size uint64) uint64 {
@@ -135,6 +136,23 @@
os.Exit(1)
}
}
+ if *enrolmentCredentialsPath != "" {
+ enrolmentCredentials, err := fs.OpenFile("/EFI/smalltown/enrolment.pb", os.O_CREATE|os.O_RDWR)
+ enrolmentCredentialsSrc, err := os.Open(*enrolmentCredentialsPath)
+ if err != nil {
+ fmt.Printf("Failed to open enrolment credentials for reading: %v", err)
+ os.Exit(1)
+ }
+ enrolmentCredentialsFull, err := ioutil.ReadAll(enrolmentCredentialsSrc)
+ if err != nil {
+ fmt.Printf("Failed to read enrolment credentials: %v", err)
+ os.Exit(1)
+ }
+ if _, err := enrolmentCredentials.Write(enrolmentCredentialsFull); err != nil {
+ fmt.Printf("Failed to write enrolment credentials")
+ os.Exit(1)
+ }
+ }
if err := diskImg.File.Close(); err != nil {
fmt.Printf("Failed to write image: %v", err)
os.Exit(1)
diff --git a/core/internal/api/BUILD.bazel b/core/internal/api/BUILD.bazel
index e7ec6f8..e862340 100644
--- a/core/internal/api/BUILD.bazel
+++ b/core/internal/api/BUILD.bazel
@@ -4,19 +4,24 @@
name = "go_default_library",
srcs = [
"cluster.go",
+ "enrolment.go",
+ "nodemanagement.go",
+ "nodes.go",
"server.go",
- "setup.go",
],
importpath = "git.monogon.dev/source/nexantic.git/core/internal/api",
visibility = ["//core:__subpackages__"],
deps = [
"//core/api/api:go_default_library",
"//core/internal/common:go_default_library",
- "//core/internal/common/grpc:go_default_library",
"//core/internal/common/service:go_default_library",
"//core/internal/consensus:go_default_library",
+ "//core/pkg/tpm:go_default_library",
+ "@com_github_gogo_protobuf//proto:go_default_library",
+ "@io_etcd_go_etcd//clientv3:go_default_library",
"@org_golang_google_grpc//:go_default_library",
"@org_golang_google_grpc//codes:go_default_library",
+ "@org_golang_google_grpc//credentials:go_default_library",
"@org_golang_google_grpc//reflection:go_default_library",
"@org_golang_google_grpc//status:go_default_library",
"@org_uber_go_zap//:go_default_library",
diff --git a/core/internal/api/cluster.go b/core/internal/api/cluster.go
index d2c18c3..dc794b3 100644
--- a/core/internal/api/cluster.go
+++ b/core/internal/api/cluster.go
@@ -19,112 +19,97 @@
import (
"context"
"crypto/rand"
- "encoding/hex"
- "fmt"
+ "encoding/base64"
+ "io"
+
+ "git.monogon.dev/source/nexantic.git/core/generated/api"
schema "git.monogon.dev/source/nexantic.git/core/generated/api"
- "git.monogon.dev/source/nexantic.git/core/internal/common/grpc"
+ "github.com/gogo/protobuf/proto"
+ "go.etcd.io/etcd/clientv3"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"go.uber.org/zap"
)
-var (
- ErrAttestationFailed = status.Error(codes.PermissionDenied, "attestation failed")
-)
-
func (s *Server) AddNode(ctx context.Context, req *schema.AddNodeRequest) (*schema.AddNodeResponse, error) {
- // Setup API client
- c, err := grpc.NewSmalltownAPIClient(fmt.Sprintf("%s:%d", req.Addr, s.config.Port))
- if err != nil {
- return nil, err
- }
-
- // Check attestation
- nonce := make([]byte, 20)
- _, err = rand.Read(nonce)
- if err != nil {
- s.Logger.Error("Nonce generation failed", zap.Error(err))
- return nil, status.Error(codes.Unavailable, "nonce generation failed")
- }
- hexNonce := hex.EncodeToString(nonce)
-
- aRes, err := c.Setup.Attest(ctx, &schema.AttestRequest{
- Challenge: hexNonce,
- })
- if err != nil {
- s := status.Convert(err)
- return nil, status.Errorf(s.Code(), "attestation failed: %v", s.Message())
- }
-
- //TODO(hendrik): Verify response
- if aRes.Response != hexNonce {
- return nil, ErrAttestationFailed
- }
-
- consensusCerts, err := s.consensusService.IssueCertificate(req.Addr)
- if err != nil {
- // Errors from IssueCertificate are always treated as internal
- s.Logger.Error("Node certificate issuance failed", zap.String("addr", req.Addr), zap.Error(err))
- return nil, status.Error(codes.Internal, "could not issue node certificate")
- }
-
- // TODO(leo): fetch remote hostname rather than using the addr
- name := req.Addr
-
- // Add new node to local etcd cluster.
- memberID, err := s.consensusService.AddMember(ctx, name, fmt.Sprintf("https://%s:%d", req.Addr, s.config.Port))
- if err != nil {
- return nil, status.Errorf(codes.Unavailable, "failed to add node to etcd cluster: %v", err)
- }
-
- s.Logger.Info("Added new node to consensus cluster; sending cluster join request to node",
- zap.String("addr", req.Addr), zap.Uint16("port", s.config.Port))
-
- // Send JoinCluster request to new node to make it join.
- _, err = c.Setup.JoinCluster(ctx, &schema.JoinClusterRequest{
- InitialCluster: s.consensusService.GetInitialClusterString(),
- ProvisioningToken: req.ProvisioningToken,
- Certs: consensusCerts,
- })
- if err != nil {
- errRevoke := s.consensusService.RevokeCertificate(req.Addr)
- if errRevoke != nil {
- s.Logger.Error("Failed to revoke a certificate after rollback - potential security risk", zap.Error(errRevoke))
- }
- // Revert etcd add member - might fail if consensus cannot be established.
- errRemove := s.consensusService.RemoveMember(ctx, memberID)
- if errRemove != nil || errRevoke != nil {
- return nil, fmt.Errorf("rollback failed after failed provisioning; err=%v; err_rb=%v; err_revoke=%v", err, errRemove, errRevoke)
- }
- return nil, status.Errorf(codes.Unavailable, "failed to join etcd cluster with node: %v", err)
- }
- s.Logger.Info("Fully provisioned new node",
- zap.String("host", req.Addr),
- zap.Uint16("apiPort", s.config.Port),
- zap.Uint64("member_id", memberID))
-
- return &schema.AddNodeResponse{}, nil
+ return nil, status.Error(codes.Unimplemented, "Unimplemented")
}
-func (s *Server) RemoveNode(context.Context, *schema.RemoveNodeRequest) (*schema.RemoveNodeResponse, error) {
- return nil, status.Error(codes.Unimplemented, "unimplemented")
+func (s *Server) RemoveNode(ctx context.Context, req *schema.RemoveNodeRequest) (*schema.RemoveNodeRequest, error) {
+ return nil, status.Error(codes.Unimplemented, "Unimplemented")
}
-func (s *Server) ListNodes(context.Context, *schema.ListNodesRequest) (*schema.ListNodesResponse, error) {
- nodes := s.consensusService.GetNodes()
- resNodes := make([]*schema.Node, len(nodes))
-
- for i, node := range nodes {
- resNodes[i] = &schema.Node{
- Id: node.ID,
- Name: node.Name,
- Address: node.Address,
- Synced: node.Synced,
+func (s *Server) ListNodes(ctx context.Context, req *schema.ListNodesRequest) (*schema.ListNodesResponse, error) {
+ store := s.getStore()
+ res, err := store.Get(ctx, "nodes/", clientv3.WithPrefix())
+ if err != nil {
+ return nil, status.Error(codes.Unavailable, "Consensus unavailable")
+ }
+ var resNodes []*api.Node
+ for _, nodeEntry := range res.Kvs {
+ var node api.Node
+ if err := proto.Unmarshal(nodeEntry.Value, &node); err != nil {
+ s.Logger.Error("Encountered invalid node data", zap.Error(err))
+ return nil, status.Error(codes.Internal, "Invalid data")
}
+ // Zero out Global Unlock Key, it's never supposed to leave the cluster
+ node.GlobalUnlockKey = []byte{}
+
+ resNodes = append(resNodes, &node)
}
return &schema.ListNodesResponse{
Nodes: resNodes,
}, nil
}
+
+func (s *Server) ListEnrolmentConfigs(ctx context.Context, req *api.ListEnrolmentConfigsRequest) (*api.ListEnrolmentConfigsResponse, error) {
+ return nil, status.Error(codes.Unimplemented, "Unimplemented")
+}
+
+func (s *Server) NewEnrolmentConfig(ctx context.Context, req *api.NewEnrolmentConfigRequest) (*api.NewEnrolmentConfigResponse, error) {
+ store := s.getStore()
+ token := make([]byte, 32)
+ if _, err := io.ReadFull(rand.Reader, token); err != nil {
+ return nil, status.Error(codes.Unavailable, "failed to get randonmess")
+ }
+ nodes, err := store.Get(ctx, "nodes/", clientv3.WithPrefix())
+ if err != nil {
+ return nil, status.Error(codes.Unavailable, "consensus unavailable")
+ }
+ var masterIPs [][]byte
+ for _, nodeKV := range nodes.Kvs {
+ var node api.Node
+ if err := proto.Unmarshal(nodeKV.Value, &node); err != nil {
+ return nil, status.Error(codes.Internal, "invalid node")
+ }
+ if node.State == api.Node_MASTER {
+ masterIPs = append(masterIPs, node.Address)
+ }
+ }
+ masterCert, err := s.GetMasterCert()
+ if err != nil {
+ return nil, status.Error(codes.Unavailable, "consensus unavailable")
+ }
+
+ enrolmentConfig := &api.EnrolmentConfig{
+ EnrolmentSecret: token,
+ MasterIps: masterIPs,
+ MastersCert: masterCert,
+ }
+ enrolmentConfigRaw, err := proto.Marshal(enrolmentConfig)
+ if err != nil {
+ return nil, status.Error(codes.Internal, "failed to encode config")
+ }
+ if _, err := store.Put(ctx, "enrolments/"+base64.RawURLEncoding.EncodeToString(token), string(enrolmentConfigRaw)); err != nil {
+ return nil, status.Error(codes.Unavailable, "consensus unavailable")
+ }
+ return &schema.NewEnrolmentConfigResponse{
+ EnrolmentConfig: enrolmentConfig,
+ }, nil
+}
+
+func (s *Server) RemoveEnrolmentConfig(ctx context.Context, req *api.RemoveEnrolmentConfigRequest) (*api.RemoveEnrolmentConfigResponse, error) {
+ return nil, status.Error(codes.Unimplemented, "Unimplemented")
+}
diff --git a/core/internal/api/enrolment.go b/core/internal/api/enrolment.go
new file mode 100644
index 0000000..976b0f2
--- /dev/null
+++ b/core/internal/api/enrolment.go
@@ -0,0 +1,55 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+ "context"
+ "encoding/base64"
+ "errors"
+ "fmt"
+
+ "git.monogon.dev/source/nexantic.git/core/generated/api"
+ "github.com/gogo/protobuf/proto"
+ "go.etcd.io/etcd/clientv3"
+)
+
+const enrolmentPrefix = "enrolments/"
+
+var errNotExists = errors.New("not found")
+
+type EnrolmentStore struct {
+ backend clientv3.KV
+}
+
+func (s *EnrolmentStore) GetBySecret(ctx context.Context, secret []byte) (*api.EnrolmentConfig, error) {
+
+ res, err := s.backend.Get(ctx, enrolmentPrefix+base64.RawURLEncoding.EncodeToString(secret))
+ if err != nil {
+ return nil, fmt.Errorf("failed to query consensus: %w", err)
+ }
+ if res.Count == 0 {
+ return nil, errNotExists
+ } else if res.Count > 1 {
+ panic("more than one value for the same key, bailing")
+ }
+ rawVal := res.Kvs[0].Value
+ var config *api.EnrolmentConfig
+ if err := proto.Unmarshal(rawVal, config); err != nil {
+ return nil, err
+ }
+ return config, nil
+}
diff --git a/core/internal/api/nodemanagement.go b/core/internal/api/nodemanagement.go
new file mode 100644
index 0000000..4268a0f
--- /dev/null
+++ b/core/internal/api/nodemanagement.go
@@ -0,0 +1,279 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+ "bytes"
+ "context"
+ "crypto/ed25519"
+ "crypto/rand"
+ "crypto/sha256"
+ "crypto/subtle"
+ "crypto/x509"
+ "encoding/base64"
+ "errors"
+ "fmt"
+ "io"
+
+ "git.monogon.dev/source/nexantic.git/core/generated/api"
+ "git.monogon.dev/source/nexantic.git/core/pkg/tpm"
+ "github.com/gogo/protobuf/proto"
+ "go.etcd.io/etcd/clientv3"
+ "go.uber.org/zap"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+)
+
+const nodesPrefix = "nodes/"
+const enrolmentsPrefix = "enrolments/"
+
+func nodeId(idCert []byte) (string, error) {
+ // Currently we only identify nodes by ID key
+ cert, err := x509.ParseCertificate(idCert)
+ if err != nil {
+ return "", err
+ }
+ pubKey, ok := cert.PublicKey.(ed25519.PublicKey)
+ if !ok {
+ return "", errors.New("invalid node identity certificate")
+ }
+
+ return "smalltown-" + base64.RawStdEncoding.EncodeToString([]byte(pubKey)), nil
+}
+
+func (s *Server) registerNewNode(node *api.Node) error {
+ nodeRaw, err := proto.Marshal(node)
+ if err != nil {
+ return err
+ }
+
+ nodeID, err := nodeId(node.IdCert)
+ if err != nil {
+ return err
+ }
+
+ key := nodesPrefix + nodeID
+
+ // Overwriting nodes is a BadIdea(TM), so make this a Compare-and-Swap
+ res, err := s.getStore().Txn(context.Background()).If(
+ clientv3.Compare(clientv3.CreateRevision(key), "=", 0),
+ ).Then(
+ clientv3.OpPut(key, string(nodeRaw)),
+ ).Commit()
+ if err != nil {
+ return fmt.Errorf("failed to store new node: %w", err)
+ }
+ if !res.Succeeded {
+ s.Logger.Warn("double-registration of node attempted", zap.String("node", nodeID))
+ }
+ return nil
+}
+
+func (s *Server) TPM2BootstrapNode(newNodeInfo *api.NewNodeInfo) (*api.Node, error) {
+ akPublic, err := tpm.GetAKPublic()
+ if err != nil {
+ return nil, err
+ }
+ ekPubkey, ekCert, err := tpm.GetEKPublic()
+ if err != nil {
+ return nil, err
+ }
+ return &api.Node{
+ Address: newNodeInfo.Ip,
+ Integrity: &api.Node_Tpm2{Tpm2: &api.NodeTPM2{
+ AkPub: akPublic,
+ EkCert: ekCert,
+ EkPubkey: ekPubkey,
+ }},
+ GlobalUnlockKey: newNodeInfo.GlobalUnlockKey,
+ IdCert: newNodeInfo.IdCert,
+ State: api.Node_MASTER,
+ }, nil
+}
+
+func (s *Server) TPM2Unlock(unlockServer api.NodeManagementService_TPM2UnlockServer) error {
+ nonce := make([]byte, 32)
+ if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
+ return status.Error(codes.Unavailable, "failed to get randonmess")
+ }
+ if err := unlockServer.Send(&api.TPM2UnlockFlowResponse{
+ Stage: &api.TPM2UnlockFlowResponse_UnlockInit{
+ UnlockInit: &api.TPM2UnlockInit{
+ Nonce: nonce,
+ },
+ },
+ }); err != nil {
+ return err
+ }
+ unlockReqContainer, err := unlockServer.Recv()
+ if err != nil {
+ return err
+ }
+ unlockReqVariant, ok := unlockReqContainer.Stage.(*api.TPM2UnlockFlowRequeset_UnlockRequest)
+ if !ok {
+ return status.Errorf(codes.InvalidArgument, "protocol violation")
+ }
+ unlockRequest := unlockReqVariant.UnlockRequest
+
+ store := s.getStore()
+ // This is safe, etcd does not do relative paths
+ path := nodesPrefix + unlockRequest.NodeId
+ nodeRes, err := store.Get(unlockServer.Context(), path)
+ if err != nil {
+ return status.Error(codes.Unavailable, "consensus request failed")
+ }
+ if nodeRes.Count == 0 {
+ return status.Error(codes.NotFound, "this node does not exist")
+ } else if nodeRes.Count > 1 {
+ panic("invariant violation: more than one node with the same id")
+ }
+ nodeRaw := nodeRes.Kvs[0].Value
+ var node api.Node
+ if err := proto.Unmarshal(nodeRaw, &node); err != nil {
+ s.Logger.Error("Failed to decode node", zap.Error(err))
+ return status.Error(codes.Internal, "invalid node")
+ }
+
+ nodeTPM2, ok := node.Integrity.(*api.Node_Tpm2)
+ if !ok {
+ return status.Error(codes.InvalidArgument, "node not integrity-protected with TPM2")
+ }
+
+ validQuote, err := tpm.VerifyAttestPlatform(nonce, nodeTPM2.Tpm2.AkPub, unlockRequest.Quote, unlockRequest.QuoteSignature)
+ if err != nil {
+ return status.Error(codes.PermissionDenied, "invalid quote")
+ }
+
+ pcrHash := sha256.New()
+ for _, pcr := range unlockRequest.Pcrs {
+ pcrHash.Write(pcr)
+ }
+ expectedPCRHash := pcrHash.Sum(nil)
+
+ if !bytes.Equal(validQuote.AttestedQuoteInfo.PCRDigest, expectedPCRHash) {
+ return status.Error(codes.InvalidArgument, "the quote's PCR hash does not match the supplied PCRs")
+ }
+
+ // TODO: Plug in policy engine to decide if the unlock should actually happen
+
+ return unlockServer.Send(&api.TPM2UnlockFlowResponse{Stage: &api.TPM2UnlockFlowResponse_UnlockResponse{
+ UnlockResponse: &api.TPM2UnlockResponse{
+ GlobalUnlockKey: node.GlobalUnlockKey,
+ },
+ }})
+}
+
+func (s *Server) NewTPM2NodeRegister(registerServer api.NodeManagementService_NewTPM2NodeRegisterServer) error {
+ registerReqContainer, err := registerServer.Recv()
+ if err != nil {
+ return err
+ }
+ registerReqVariant, ok := registerReqContainer.Stage.(*api.TPM2FlowRequest_Register)
+ if !ok {
+ return status.Error(codes.InvalidArgument, "protocol violation")
+ }
+ registerReq := registerReqVariant.Register
+
+ challengeNonce := make([]byte, 32)
+ if _, err := io.ReadFull(rand.Reader, challengeNonce); err != nil {
+ return status.Error(codes.Unavailable, "failed to get randonmess")
+ }
+ challenge, challengeBlob, err := tpm.MakeAKChallenge(registerReq.EkPubkey, registerReq.AkPublic, challengeNonce)
+ if err != nil {
+ return status.Errorf(codes.InvalidArgument, "failed to challenge AK: %v", err)
+ }
+ nonce := make([]byte, 32)
+ if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
+ return status.Error(codes.Unavailable, "failed to get randonmess")
+ }
+ if err := registerServer.Send(&api.TPM2FlowResponse{Stage: &api.TPM2FlowResponse_AttestRequest{AttestRequest: &api.TPM2AttestRequest{
+ AkChallenge: challenge,
+ AkChallengeSecret: challengeBlob,
+ QuoteNonce: nonce,
+ }}}); err != nil {
+ return err
+ }
+ attestationResContainer, err := registerServer.Recv()
+ if err != nil {
+ return err
+ }
+ attestResVariant, ok := attestationResContainer.Stage.(*api.TPM2FlowRequest_AttestResponse)
+ if !ok {
+ return status.Error(codes.InvalidArgument, "protocol violation")
+ }
+ attestRes := attestResVariant.AttestResponse
+
+ if subtle.ConstantTimeCompare(attestRes.AkChallengeSolution, challengeNonce) != 1 {
+ return status.Error(codes.InvalidArgument, "invalid challenge response")
+ }
+
+ validQuote, err := tpm.VerifyAttestPlatform(nonce, registerReq.AkPublic, attestRes.Quote, attestRes.QuoteSignature)
+ if err != nil {
+ return status.Error(codes.PermissionDenied, "invalid quote")
+ }
+
+ pcrHash := sha256.New()
+ for _, pcr := range attestRes.Pcrs {
+ pcrHash.Write(pcr)
+ }
+ expectedPCRHash := pcrHash.Sum(nil)
+
+ if !bytes.Equal(validQuote.AttestedQuoteInfo.PCRDigest, expectedPCRHash) {
+ return status.Error(codes.InvalidArgument, "the quote's PCR hash does not match the supplied PCRs")
+ }
+
+ newNodeInfoContainer, err := registerServer.Recv()
+ newNodeInfoVariant, ok := newNodeInfoContainer.Stage.(*api.TPM2FlowRequest_NewNodeInfo)
+ newNodeInfo := newNodeInfoVariant.NewNodeInfo
+
+ store := s.getStore()
+ res, err := store.Get(registerServer.Context(), "enrolments/"+base64.RawURLEncoding.EncodeToString(newNodeInfo.EnrolmentConfig.EnrolmentSecret))
+ if err != nil {
+ return status.Error(codes.Unavailable, "Consensus unavailable")
+ }
+ if res.Count == 0 {
+ return status.Error(codes.PermissionDenied, "Invalid enrolment secret")
+ } else if res.Count > 1 {
+ panic("more than one value for the same key, bailing")
+ }
+ rawVal := res.Kvs[0].Value
+ var config api.EnrolmentConfig
+ if err := proto.Unmarshal(rawVal, &config); err != nil {
+ return err
+ }
+
+ // TODO: Plug in policy engine here
+
+ node := api.Node{
+ Address: newNodeInfo.Ip,
+ Integrity: &api.Node_Tpm2{Tpm2: &api.NodeTPM2{
+ AkPub: registerReq.AkPublic,
+ EkCert: registerReq.EkCert,
+ EkPubkey: registerReq.EkPubkey,
+ }},
+ GlobalUnlockKey: newNodeInfo.GlobalUnlockKey,
+ IdCert: newNodeInfo.IdCert,
+ State: api.Node_UNININITALIZED,
+ }
+
+ if err := s.registerNewNode(&node); err != nil {
+ s.Logger.Error("failed to register a node", zap.Error(err))
+ return status.Error(codes.Internal, "failed to register node")
+ }
+
+ return nil
+}
diff --git a/core/internal/api/nodes.go b/core/internal/api/nodes.go
new file mode 100644
index 0000000..da3cbc4
--- /dev/null
+++ b/core/internal/api/nodes.go
@@ -0,0 +1,17 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
diff --git a/core/internal/api/server.go b/core/internal/api/server.go
index efd0be5..4e1e5fa 100644
--- a/core/internal/api/server.go
+++ b/core/internal/api/server.go
@@ -17,23 +17,36 @@
package api
import (
+ "context"
+ "crypto/ed25519"
+ "crypto/rand"
+ "crypto/tls"
+ "crypto/x509"
+ "crypto/x509/pkix"
+ "errors"
"fmt"
+ "math/big"
+ "net"
+ "time"
+
+ "git.monogon.dev/source/nexantic.git/core/generated/api"
schema "git.monogon.dev/source/nexantic.git/core/generated/api"
"git.monogon.dev/source/nexantic.git/core/internal/common"
"git.monogon.dev/source/nexantic.git/core/internal/common/service"
"git.monogon.dev/source/nexantic.git/core/internal/consensus"
+ "go.etcd.io/etcd/clientv3"
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/reflection"
- "net"
+ "google.golang.org/grpc/credentials"
)
type (
Server struct {
*service.BaseService
- setupService common.SetupService
- grpcServer *grpc.Server
+ grpcServer *grpc.Server
+ externalGrpcServer *grpc.Server
consensusService *consensus.Service
@@ -41,49 +54,186 @@
}
Config struct {
- Port uint16
}
)
-func NewApiServer(config *Config, logger *zap.Logger, setupService common.SetupService, consensusService *consensus.Service) (*Server, error) {
+var (
+ // From RFC 5280 Section 4.1.2.5
+ unknownNotAfter = time.Unix(253402300799, 0)
+)
+
+func NewApiServer(config *Config, logger *zap.Logger, consensusService *consensus.Service) (*Server, error) {
s := &Server{
config: config,
- setupService: setupService,
consensusService: consensusService,
}
s.BaseService = service.NewBaseService("api", logger, s)
- grpcServer := grpc.NewServer()
- schema.RegisterClusterManagementServer(grpcServer, s)
- schema.RegisterSetupServiceServer(grpcServer, s)
-
- reflection.Register(grpcServer)
-
- s.grpcServer = grpcServer
-
return s, nil
}
+func (s *Server) getStore() clientv3.KV {
+ // Cannot be moved to initialization because an internal reference will be nil
+ return s.consensusService.GetStore("api", "")
+}
+
+// BootstrapNewClusterHook creates the necessary key material for the API Servers and stores it in
+// the consensus service. It also creates a node entry for the initial node.
+func (s *Server) BootstrapNewClusterHook(initNodeReq *api.NewNodeInfo) error {
+ serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 127)
+ serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+ if err != nil {
+ return fmt.Errorf("Failed to generate serial number: %w", err)
+ }
+
+ pubKey, privKeyRaw, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ return err
+ }
+ privkey, err := x509.MarshalPKCS8PrivateKey(privKeyRaw)
+ if err != nil {
+ return err
+ }
+
+ // This has no SANs because it authenticates by public key, not by name
+ masterCert := &x509.Certificate{
+ SerialNumber: serialNumber,
+ Subject: pkix.Name{
+ CommonName: "Smalltown Master",
+ },
+ IsCA: false,
+ BasicConstraintsValid: true,
+ NotBefore: time.Now(),
+ NotAfter: unknownNotAfter,
+ // Certificate is used both as server & client
+ ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
+ }
+ cert, err := x509.CreateCertificate(rand.Reader, masterCert, masterCert, pubKey, privKeyRaw)
+ if err != nil {
+ return err
+ }
+ store := s.getStore()
+ if _, err := store.Put(context.Background(), "master.der", string(cert)); err != nil {
+ return err
+ }
+ if _, err := store.Put(context.Background(), "master-key.der", string(privkey)); err != nil {
+ return err
+ }
+
+ // TODO: Further integrity providers need to be plumbed in here
+ node, err := s.TPM2BootstrapNode(initNodeReq)
+ if err != nil {
+ return err
+ }
+
+ if err := s.registerNewNode(node); err != nil {
+ return err
+ }
+ return nil
+}
+
+// GetMasterCert gets the master certificate in X.509 DER form
+// This is mainly used to issue enrolment configs
+func (s *Server) GetMasterCert() ([]byte, error) {
+ store := s.getStore()
+ res, err := store.Get(context.Background(), "master.der")
+ if err != nil {
+ return []byte{}, err
+ }
+ if len(res.Kvs) != 1 {
+ return []byte{}, errors.New("master certificate not found")
+ }
+ certRaw := res.Kvs[0].Value
+ return certRaw, nil
+}
+
+// TODO(lorenz): Move consensus/certificate interaction into a utility, is now duplicated too often
+func (s *Server) loadMasterCert() (*tls.Certificate, error) {
+
+ store := s.getStore()
+ var tlsCert tls.Certificate
+ res, err := store.Get(context.Background(), "master.der")
+ if err != nil {
+ return nil, err
+ }
+ if len(res.Kvs) != 1 {
+ return nil, errors.New("master certificate not found")
+ }
+ certRaw := res.Kvs[0].Value
+
+ tlsCert.Certificate = append(tlsCert.Certificate, certRaw)
+ tlsCert.Leaf, err = x509.ParseCertificate(certRaw)
+
+ res, err = store.Get(context.Background(), "master-key.der")
+ if err != nil {
+ return nil, err
+ }
+ if len(res.Kvs) != 1 {
+ return nil, errors.New("master certificate not found")
+ }
+ keyRaw := res.Kvs[0].Value
+ key, err := x509.ParsePKCS8PrivateKey(keyRaw)
+ if err != nil {
+ return nil, fmt.Errorf("failed to load master private key: %w", err)
+ }
+ edKey, ok := key.(ed25519.PrivateKey)
+ if !ok {
+ return nil, errors.New("invalid private key")
+ }
+ tlsCert.PrivateKey = edKey
+ return &tlsCert, nil
+}
+
func (s *Server) OnStart() error {
- listenHost := fmt.Sprintf(":%d", s.config.Port)
- lis, err := net.Listen("tcp", listenHost)
+ masterListenHost := fmt.Sprintf(":%d", common.MasterServicePort)
+ lis, err := net.Listen("tcp", masterListenHost)
if err != nil {
s.Logger.Fatal("failed to listen", zap.Error(err))
}
+ externalListeneHost := fmt.Sprintf(":%d", common.ExternalServicePort)
+ externalListener, err := net.Listen("tcp", externalListeneHost)
+ if err != nil {
+ s.Logger.Fatal("failed to listen", zap.Error(err))
+ }
+
+ masterCert, err := s.loadMasterCert()
+ if err != nil {
+ s.Logger.Error("Failed to load Master Service Key Material: %w", zap.Error(err))
+ return err
+ }
+
+ masterTransportCredentials := credentials.NewServerTLSFromCert(masterCert)
+
+ masterGrpcServer := grpc.NewServer(grpc.Creds(masterTransportCredentials))
+ clusterManagementGrpcServer := grpc.NewServer()
+ schema.RegisterClusterManagementServer(clusterManagementGrpcServer, s)
+ schema.RegisterNodeManagementServiceServer(masterGrpcServer, s)
+
+ reflection.Register(masterGrpcServer)
+
+ s.grpcServer = masterGrpcServer
+ s.externalGrpcServer = clusterManagementGrpcServer
+
go func() {
err = s.grpcServer.Serve(lis)
s.Logger.Error("API server failed", zap.Error(err))
}()
- s.Logger.Info("gRPC listening", zap.String("host", listenHost))
+ go func() {
+ err = s.externalGrpcServer.Serve(externalListener)
+ s.Logger.Error("API server failed", zap.Error(err))
+ }()
+
+ s.Logger.Info("gRPC listening", zap.String("host", masterListenHost))
return nil
}
func (s *Server) OnStop() error {
s.grpcServer.Stop()
+ s.externalGrpcServer.Stop()
return nil
}
diff --git a/core/internal/api/setup.go b/core/internal/api/setup.go
deleted file mode 100644
index f317534..0000000
--- a/core/internal/api/setup.go
+++ /dev/null
@@ -1,65 +0,0 @@
-// Copyright 2020 The Monogon Project Authors.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package api
-
-import (
- "context"
- "errors"
- "fmt"
-
- schema "git.monogon.dev/source/nexantic.git/core/generated/api"
-)
-
-const (
- MinNameLength = 3
-)
-
-var (
- ErrInvalidProvisioningToken = errors.New("invalid provisioning token")
- ErrInvalidNameLength = fmt.Errorf("name must be at least %d characters long", MinNameLength)
-)
-
-func (s *Server) Setup(c context.Context, r *schema.SetupRequest) (*schema.SetupResponse, error) {
- return &schema.SetupResponse{}, nil
-}
-
-func (s *Server) BootstrapNewCluster(context.Context, *schema.BootstrapNewClusterRequest) (*schema.BootstrapNewClusterResponse, error) {
- err := s.setupService.SetupNewCluster()
- return &schema.BootstrapNewClusterResponse{}, err
-}
-
-func (s *Server) JoinCluster(ctx context.Context, req *schema.JoinClusterRequest) (*schema.JoinClusterResponse, error) {
- // Verify provisioning token
- if s.setupService.GetJoinClusterToken() != req.ProvisioningToken {
- return nil, ErrInvalidProvisioningToken
- }
-
- // Join cluster
- err := s.setupService.JoinCluster(req.InitialCluster, req.Certs)
- if err != nil {
- return nil, err
- }
-
- return &schema.JoinClusterResponse{}, nil
-}
-
-func (s *Server) Attest(c context.Context, r *schema.AttestRequest) (*schema.AttestResponse, error) {
- // TODO implement
- return &schema.AttestResponse{
- Response: r.Challenge,
- }, nil
-}
diff --git a/core/internal/common/BUILD.bazel b/core/internal/common/BUILD.bazel
index 1adbfd8..8bd89d6 100644
--- a/core/internal/common/BUILD.bazel
+++ b/core/internal/common/BUILD.bazel
@@ -5,5 +5,4 @@
srcs = ["setup.go"],
importpath = "git.monogon.dev/source/nexantic.git/core/internal/common",
visibility = ["//:__subpackages__"],
- deps = ["//core/api/api:go_default_library"],
)
diff --git a/core/internal/common/grpc/grpc.go b/core/internal/common/grpc/grpc.go
index e9cfed0..97fc4c8 100644
--- a/core/internal/common/grpc/grpc.go
+++ b/core/internal/common/grpc/grpc.go
@@ -27,7 +27,6 @@
conn *grpc.ClientConn
Cluster api.ClusterManagementClient
- Setup api.SetupServiceClient
}
)
@@ -42,7 +41,6 @@
// Setup all client connections
s.Cluster = api.NewClusterManagementClient(conn)
- s.Setup = api.NewSetupServiceClient(conn)
return s, nil
}
diff --git a/core/internal/common/setup.go b/core/internal/common/setup.go
index 1124d27..e745e54 100644
--- a/core/internal/common/setup.go
+++ b/core/internal/common/setup.go
@@ -16,27 +16,23 @@
package common
-import "git.monogon.dev/source/nexantic.git/core/generated/api"
-
-// TODO(leo): merge api and node packages and get rid of this extra layer of indirection?
-
type (
- SetupService interface {
- CurrentState() SmalltownState
- GetJoinClusterToken() string
- SetupNewCluster() error
- EnterJoinClusterMode() error
- JoinCluster(initialCluster string, certs *api.ConsensusCertificates) error
- }
-
SmalltownState string
)
const (
- // Node is unprovisioned and waits for Setup to be called.
- StateSetupMode SmalltownState = "setup"
- // Setup() has been called, node waits for a JoinCluster or BootstrapCluster call.
- StateClusterJoinMode SmalltownState = "join"
+ // These are here to prevent depdendency loops
+ NodeServicePort = 7835
+ ConsensusPort = 7834
+ MasterServicePort = 7833
+ ExternalServicePort = 7836
+)
+
+const (
+ // Node is provisioning a new cluster with itself as a master
+ StateNewClusterMode SmalltownState = "setup"
+ // Node is enrolling itself and waiting to be adopted
+ StateEnrollMode SmalltownState = "join"
// Node is fully provisioned.
- StateConfigured SmalltownState = "configured"
+ StateJoined SmalltownState = "enrolled"
)
diff --git a/core/internal/consensus/BUILD.bazel b/core/internal/consensus/BUILD.bazel
index 5fa33ac..53b118d 100644
--- a/core/internal/consensus/BUILD.bazel
+++ b/core/internal/consensus/BUILD.bazel
@@ -7,6 +7,7 @@
visibility = ["//:__subpackages__"],
deps = [
"//core/api/api:go_default_library",
+ "//core/internal/common:go_default_library",
"//core/internal/common/service:go_default_library",
"//core/internal/consensus/ca:go_default_library",
"@com_github_pkg_errors//:go_default_library",
diff --git a/core/internal/consensus/consensus.go b/core/internal/consensus/consensus.go
index 6fa6210..5a0bead 100644
--- a/core/internal/consensus/consensus.go
+++ b/core/internal/consensus/consensus.go
@@ -33,6 +33,7 @@
"strings"
"time"
+ "git.monogon.dev/source/nexantic.git/core/internal/common"
"git.monogon.dev/source/nexantic.git/core/internal/common/service"
"git.monogon.dev/source/nexantic.git/core/generated/api"
@@ -91,7 +92,6 @@
NewCluster bool
ExternalHost string
ListenHost string
- ListenPort uint16
}
Member struct {
@@ -143,13 +143,13 @@
cfg.LCUrls = []url.URL{*listenerURL}
// Advertise Peer URLs
- apURL, err := url.Parse(fmt.Sprintf("https://%s:%d", s.config.ExternalHost, s.config.ListenPort))
+ apURL, err := url.Parse(fmt.Sprintf("https://%s:%d", s.config.ExternalHost, common.ConsensusPort))
if err != nil {
return fmt.Errorf("invalid external_host or listen_port: %w", err)
}
// Listen Peer URLs
- lpURL, err := url.Parse(fmt.Sprintf("https://%s:%d", s.config.ListenHost, s.config.ListenPort))
+ lpURL, err := url.Parse(fmt.Sprintf("https://%s:%d", s.config.ListenHost, common.ConsensusPort))
if err != nil {
return fmt.Errorf("invalid listen_host or listen_port: %w", err)
}
diff --git a/core/internal/integrity/BUILD.bazel b/core/internal/integrity/BUILD.bazel
new file mode 100644
index 0000000..cb551cc
--- /dev/null
+++ b/core/internal/integrity/BUILD.bazel
@@ -0,0 +1,14 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+ name = "go_default_library",
+ srcs = ["common.go"],
+ importpath = "git.monogon.dev/source/nexantic.git/core/internal/integrity",
+ visibility = ["//core:__subpackages__"],
+ deps = [
+ "//core/api/api:go_default_library",
+ "//core/internal/common:go_default_library",
+ "@org_golang_google_grpc//:go_default_library",
+ "@org_golang_google_grpc//credentials:go_default_library",
+ ],
+)
diff --git a/core/internal/integrity/common.go b/core/internal/integrity/common.go
new file mode 100644
index 0000000..f92c008
--- /dev/null
+++ b/core/internal/integrity/common.go
@@ -0,0 +1,81 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package integrity
+
+import (
+ "bytes"
+ "crypto/tls"
+ "crypto/x509"
+ "errors"
+ "fmt"
+ "net"
+ "strings"
+
+ "git.monogon.dev/source/nexantic.git/core/generated/api"
+ "git.monogon.dev/source/nexantic.git/core/internal/common"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/credentials"
+)
+
+// Agent specifices the interface which every integrity agent needs to fulfill
+type Agent interface {
+ // Initialize needs to be called once and initializes the systems required to maintain integrity
+ // on the given platform.
+ // nodeCert is a X.509 DER certificate which identifies the node once it's unlocked. This is
+ // required to bind the node certificate (which is only available when the node is unlocked) to
+ // the integrity subsystem used to attest said node.
+ // Initialize returns the cryptographic identity that it's bound to.
+ Initialize(newNode api.NewNodeInfo, enrolment api.EnrolmentConfig) (string, error)
+
+ // Unlock performs all required actions to assure the integrity of the platform and retrieves
+ // the unlock key in a secure manner
+ Unlock(enrolment api.EnrolmentConfig) ([]byte, error)
+}
+
+// DialNMS creates a secure GRPC connection to the NodeManagementService
+func DialNMS(enrolment api.EnrolmentConfig) (*grpc.ClientConn, error) {
+ var targets []string
+ for _, target := range enrolment.MasterIps {
+ targets = append(targets, fmt.Sprintf("%v:%v", net.IP(target), common.MasterServicePort))
+ }
+ cert, err := x509.ParseCertificate(enrolment.MastersCert)
+ if err != nil {
+ return nil, err
+ }
+ mastersPool := x509.NewCertPool()
+ mastersPool.AddCert(cert)
+
+ secureTransport := &tls.Config{
+ InsecureSkipVerify: true,
+ // Critical function, please review any changes with care
+ // TODO(lorenz): Actively check that this actually provides the security guarantees that we need
+ VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
+ for _, cert := range rawCerts {
+ // X.509 certificates in DER can be compared like this since DER has a unique representation
+ // for each certificate.
+ if bytes.Equal(cert, enrolment.MastersCert) {
+ return nil
+ }
+ }
+ return errors.New("failed to find authorized NMS certificate")
+ },
+ MinVersion: tls.VersionTLS13,
+ }
+ secureTransportCreds := credentials.NewTLS(secureTransport)
+
+ return grpc.Dial(strings.Join(targets, ","), grpc.WithTransportCredentials(secureTransportCreds))
+}
diff --git a/core/internal/integrity/tpm2/BUILD.bazel b/core/internal/integrity/tpm2/BUILD.bazel
new file mode 100644
index 0000000..c4c77bb
--- /dev/null
+++ b/core/internal/integrity/tpm2/BUILD.bazel
@@ -0,0 +1,13 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+ name = "go_default_library",
+ srcs = ["tpm2.go"],
+ importpath = "git.monogon.dev/source/nexantic.git/core/internal/integrity/tpm2",
+ visibility = ["//core:__subpackages__"],
+ deps = [
+ "//core/api/api:go_default_library",
+ "//core/internal/integrity:go_default_library",
+ "//core/pkg/tpm:go_default_library",
+ ],
+)
diff --git a/core/internal/integrity/tpm2/tpm2.go b/core/internal/integrity/tpm2/tpm2.go
new file mode 100644
index 0000000..8497562
--- /dev/null
+++ b/core/internal/integrity/tpm2/tpm2.go
@@ -0,0 +1,155 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tpm2
+
+import (
+ "context"
+ "errors"
+ "fmt"
+
+ "git.monogon.dev/source/nexantic.git/core/generated/api"
+ "git.monogon.dev/source/nexantic.git/core/internal/integrity"
+ "git.monogon.dev/source/nexantic.git/core/pkg/tpm"
+)
+
+type TPM2Agent struct {
+}
+
+func (a *TPM2Agent) Initialize(newNode api.NewNodeInfo, enrolment api.EnrolmentConfig) error {
+ nmsConn, err := integrity.DialNMS(enrolment)
+ nmsClient := api.NewNodeManagementServiceClient(nmsConn)
+ ekPub, ekCert, err := tpm.GetEKPublic()
+ if err != nil {
+ return fmt.Errorf("failed to generate EK: %w", err)
+ }
+
+ akPub, err := tpm.GetAKPublic()
+ if err != nil {
+ return fmt.Errorf("failed to generate AK: %w", err)
+ }
+
+ registerSession, err := nmsClient.NewTPM2NodeRegister(context.Background())
+ if err != nil {
+ return fmt.Errorf("failed to open registration session: %w", err)
+ }
+ defer registerSession.CloseSend()
+ if err := registerSession.Send(&api.TPM2FlowRequest{
+ Stage: &api.TPM2FlowRequest_Register{
+ Register: &api.TPM2RegisterRequest{
+ AkPublic: akPub,
+ EkPubkey: ekPub,
+ EkCert: ekCert,
+ },
+ },
+ }); err != nil {
+ return fmt.Errorf("failed to send registration: %w", err)
+ }
+
+ res1, err := registerSession.Recv()
+ if err != nil {
+ return fmt.Errorf("failed to receive attest request: %w", err)
+ }
+ attestReqContainer, ok := res1.Stage.(*api.TPM2FlowResponse_AttestRequest)
+ if !ok {
+ return errors.New("protocol violation: after RegisterRequest expected AttestRequest")
+ }
+ attestReq := attestReqContainer.AttestRequest
+ solution, err := tpm.SolveAKChallenge(attestReq.AkChallenge, attestReq.AkChallengeSecret)
+ if err != nil {
+ return fmt.Errorf("failed to solve AK challenge: %w", err)
+ }
+ pcrs, err := tpm.GetPCRs()
+ if err != nil {
+ return fmt.Errorf("failed to get SRTM PCRs: %w", err)
+ }
+ quote, quoteSig, err := tpm.AttestPlatform(attestReq.QuoteNonce)
+ if err != nil {
+ return fmt.Errorf("failed Quote operation: %w", err)
+ }
+ if err := registerSession.Send(&api.TPM2FlowRequest{
+ Stage: &api.TPM2FlowRequest_AttestResponse{
+ AttestResponse: &api.TPM2AttestResponse{
+ AkChallengeSolution: solution,
+ Pcrs: pcrs,
+ Quote: quote,
+ QuoteSignature: quoteSig,
+ },
+ },
+ }); err != nil {
+ return fmt.Errorf("failed to send AttestResponse: %w", err)
+ }
+ if err := registerSession.Send(&api.TPM2FlowRequest{
+ Stage: &api.TPM2FlowRequest_NewNodeInfo{
+ NewNodeInfo: &newNode,
+ },
+ }); err != nil {
+ return fmt.Errorf("failed to send NewNodeInfo: %w", err)
+ }
+ return nil
+}
+
+// Unlock attests the node state to the remote NMS and asks it for the global unlock key
+func (a *TPM2Agent) Unlock(enrolment api.EnrolmentConfig) ([]byte, error) {
+ nmsConn, err := integrity.DialNMS(enrolment)
+ if err != nil {
+ return []byte{}, err
+ }
+ nmsClient := api.NewNodeManagementServiceClient(nmsConn)
+ unlockClient, err := nmsClient.TPM2Unlock(context.Background())
+ if err != nil {
+ return []byte{}, err
+ }
+ defer unlockClient.CloseSend()
+ unlockInitContainer, err := unlockClient.Recv()
+ if err != nil {
+ return []byte{}, err
+ }
+ unlockInitVariant, ok := unlockInitContainer.Stage.(*api.TPM2UnlockFlowResponse_UnlockInit)
+ if !ok {
+ return []byte{}, errors.New("TPM2Unlock protocol violation")
+ }
+ unlockInit := unlockInitVariant.UnlockInit
+ quote, sig, err := tpm.AttestPlatform(unlockInit.Nonce)
+ if err != nil {
+ return []byte{}, fmt.Errorf("failed to attest platform: %w", err)
+ }
+ pcrs, err := tpm.GetPCRs()
+ if err != nil {
+ return []byte{}, fmt.Errorf("failed to get PCRs from TPM: %w", err)
+ }
+ if err := unlockClient.Send(&api.TPM2UnlockFlowRequeset{Stage: &api.TPM2UnlockFlowRequeset_UnlockRequest{
+ UnlockRequest: &api.TPM2UnlockRequest{
+ Pcrs: pcrs,
+ Quote: quote,
+ QuoteSignature: sig,
+ NodeId: enrolment.NodeId,
+ },
+ }}); err != nil {
+ return []byte{}, err
+ }
+ unlockResponseContainer, err := unlockClient.Recv()
+ if err != nil {
+ return []byte{}, err
+ }
+ unlockResponseVariant, ok := unlockResponseContainer.Stage.(*api.TPM2UnlockFlowResponse_UnlockResponse)
+ if !ok {
+ return []byte{}, errors.New("violated TPM2Unlock protocol")
+ }
+ unlockResponse := unlockResponseVariant.UnlockResponse
+
+ return unlockResponse.GlobalUnlockKey, nil
+}
diff --git a/core/internal/network/BUILD.bazel b/core/internal/network/BUILD.bazel
index e7f7d55..3208ea8 100644
--- a/core/internal/network/BUILD.bazel
+++ b/core/internal/network/BUILD.bazel
@@ -7,6 +7,7 @@
visibility = ["//:__subpackages__"],
deps = [
"//core/internal/common/service:go_default_library",
+ "@com_github_insomniacslk_dhcp//dhcpv4:go_default_library",
"@com_github_insomniacslk_dhcp//dhcpv4/nclient4:go_default_library",
"@com_github_vishvananda_netlink//:go_default_library",
"@org_golang_x_sys//unix:go_default_library",
diff --git a/core/internal/network/main.go b/core/internal/network/main.go
index 0888de7..04ab159 100644
--- a/core/internal/network/main.go
+++ b/core/internal/network/main.go
@@ -21,9 +21,12 @@
"fmt"
"net"
"os"
+ "sync"
+ "time"
"git.monogon.dev/source/nexantic.git/core/internal/common/service"
+ "github.com/insomniacslk/dhcp/dhcpv4"
"github.com/insomniacslk/dhcp/dhcpv4/nclient4"
"github.com/vishvananda/netlink"
"go.uber.org/zap"
@@ -39,6 +42,9 @@
*service.BaseService
config Config
dhcp4Client *nclient4.Client
+ ip *net.IP
+ ipNotify chan struct{}
+ lock sync.Mutex
}
type Config struct {
@@ -46,7 +52,8 @@
func NewNetworkService(config Config, logger *zap.Logger) (*Service, error) {
s := &Service{
- config: config,
+ config: config,
+ ipNotify: make(chan struct{}),
}
s.BaseService = service.NewBaseService("network", logger, s)
return s, nil
@@ -105,20 +112,60 @@
if err != nil {
panic(err)
}
- _, ack, err := client.Request(context.Background())
- if err != nil {
- panic(err)
+ var ack *dhcpv4.DHCPv4
+ for {
+ dhcpCtx, dhcpCtxCancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer dhcpCtxCancel()
+ _, ack, err = client.Request(dhcpCtx)
+ if err == nil {
+ break
+ }
+ s.Logger.Info("DHCP request failed", zap.Error(err))
}
s.Logger.Info("Network service got IP", zap.String("ip", ack.YourIPAddr.String()))
if err := setResolvconf(ack.DNS(), []string{}); err != nil {
s.Logger.Warn("Failed to set resolvconf", zap.Error(err))
}
+
+ s.lock.Lock()
+ s.ip = &ack.YourIPAddr
+ s.lock.Unlock()
+loop:
+ for {
+ select {
+ case s.ipNotify <- struct{}{}:
+ default:
+ break loop
+ }
+ }
+
if err := addNetworkRoutes(iface, net.IPNet{IP: ack.YourIPAddr, Mask: ack.SubnetMask()}, ack.GatewayIPAddr); err != nil {
s.Logger.Warn("Failed to add routes", zap.Error(err))
}
return nil
}
+// GetIP returns the current IP (and optionally waits for one to be assigned)
+func (s *Service) GetIP(wait bool) *net.IP {
+ s.lock.Lock()
+ if !wait {
+ ip := s.ip
+ s.lock.Unlock()
+ return ip
+ }
+
+ for {
+ if s.ip != nil {
+ ip := s.ip
+ s.lock.Unlock()
+ return ip
+ }
+ s.lock.Unlock()
+ <-s.ipNotify
+ s.lock.Lock()
+ }
+}
+
func (s *Service) OnStart() error {
s.Logger.Info("Starting network service")
links, err := netlink.LinkList()
diff --git a/core/internal/node/BUILD.bazel b/core/internal/node/BUILD.bazel
index 529c6ee..d96c0e6 100644
--- a/core/internal/node/BUILD.bazel
+++ b/core/internal/node/BUILD.bazel
@@ -13,10 +13,15 @@
"//core/internal/api:go_default_library",
"//core/internal/common:go_default_library",
"//core/internal/consensus:go_default_library",
+ "//core/internal/integrity/tpm2:go_default_library",
"//core/internal/kubernetes:go_default_library",
+ "//core/internal/network:go_default_library",
"//core/internal/storage:go_default_library",
- "@com_github_google_uuid//:go_default_library",
+ "@com_github_cenkalti_backoff_v4//:go_default_library",
+ "@com_github_gogo_protobuf//proto:go_default_library",
+ "@org_golang_google_grpc//:go_default_library",
"@org_golang_google_grpc//codes:go_default_library",
+ "@org_golang_google_grpc//credentials:go_default_library",
"@org_golang_google_grpc//status:go_default_library",
"@org_uber_go_zap//:go_default_library",
],
diff --git a/core/internal/node/main.go b/core/internal/node/main.go
index 40aa8b3..b1d74d6 100644
--- a/core/internal/node/main.go
+++ b/core/internal/node/main.go
@@ -17,34 +17,61 @@
package node
import (
+ "bytes"
+ "crypto/ed25519"
+ "crypto/rand"
+ "crypto/sha512"
+ "crypto/tls"
+ "crypto/x509"
+ "crypto/x509/pkix"
+ "encoding/base64"
+ "errors"
"flag"
+ "fmt"
+ "io/ioutil"
+ "math/big"
+ "net"
+ "time"
+ "os"
+
+ apipb "git.monogon.dev/source/nexantic.git/core/generated/api"
"git.monogon.dev/source/nexantic.git/core/internal/api"
"git.monogon.dev/source/nexantic.git/core/internal/common"
"git.monogon.dev/source/nexantic.git/core/internal/consensus"
+ "git.monogon.dev/source/nexantic.git/core/internal/integrity/tpm2"
"git.monogon.dev/source/nexantic.git/core/internal/kubernetes"
+ "git.monogon.dev/source/nexantic.git/core/internal/network"
"git.monogon.dev/source/nexantic.git/core/internal/storage"
- "os"
+ "github.com/cenkalti/backoff/v4"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/credentials"
- "github.com/google/uuid"
+ "github.com/gogo/protobuf/proto"
"go.uber.org/zap"
)
+var (
+ // From RFC 5280 Section 4.1.2.5
+ unknownNotAfter = time.Unix(253402300799, 0)
+)
+
type (
SmalltownNode struct {
Api *api.Server
Consensus *consensus.Service
Storage *storage.Manager
Kubernetes *kubernetes.Service
+ Network *network.Service
- logger *zap.Logger
- state common.SmalltownState
- joinToken string
- hostname string
+ logger *zap.Logger
+ state common.SmalltownState
+ hostname string
+ enrolmentConfig *apipb.EnrolmentConfig
}
)
-func NewSmalltownNode(logger *zap.Logger, apiPort, consensusPort uint16) (*SmalltownNode, error) {
+func NewSmalltownNode(logger *zap.Logger) (*SmalltownNode, error) {
flag.Parse()
logger.Info("Creating Smalltown node")
@@ -53,17 +80,32 @@
panic(err)
}
+ networkService, err := network.NewNetworkService(network.Config{}, logger.With(zap.String("component", "network")))
+ if err != nil {
+ panic(err)
+ }
+
+ if err := networkService.Start(); err != nil {
+ logger.Panic("Failed to start network service", zap.Error(err))
+ }
+
storageManager, err := storage.Initialize(logger.With(zap.String("component", "storage")))
if err != nil {
logger.Error("Failed to initialize storage manager", zap.Error(err))
return nil, err
}
+ externalIP := networkService.GetIP(true)
+ if externalIP == nil {
+ logger.Panic("Waited for IP but didn't get one")
+ }
+
+ // Important to know if the GetIP above hangs
+ logger.Info("Node has IP", zap.String("ip", externalIP.String()))
consensusService, err := consensus.NewConsensusService(consensus.Config{
Name: hostname,
- ListenPort: consensusPort,
ListenHost: "0.0.0.0",
- ExternalHost: "10.0.2.15", // TODO: Once Multi-Node setups are actually used, this needs to be corrected
+ ExternalHost: externalIP.String(),
}, logger.With(zap.String("module", "consensus")))
if err != nil {
return nil, err
@@ -72,13 +114,12 @@
s := &SmalltownNode{
Consensus: consensusService,
Storage: storageManager,
+ Network: networkService,
logger: logger,
hostname: hostname,
}
- apiService, err := api.NewApiServer(&api.Config{
- Port: apiPort,
- }, logger.With(zap.String("module", "api")), s, s.Consensus)
+ apiService, err := api.NewApiServer(&api.Config{}, logger.With(zap.String("module", "api")), s.Consensus)
if err != nil {
return nil, err
}
@@ -95,51 +136,330 @@
func (s *SmalltownNode) Start() error {
s.logger.Info("Starting Smalltown node")
- if s.Consensus.IsProvisioned() {
- s.logger.Info("Consensus is provisioned")
- err := s.startFull()
- if err != nil {
- return err
- }
- } else {
- s.logger.Info("Consensus is not provisioned, starting provisioning...")
- err := s.startForSetup()
- if err != nil {
- return err
- }
+ // TODO(lorenz): Abstracting enrolment sounds like a good idea, but ends up being painful
+ // because of things like storage access. I'm keeping it this way until the more complex
+ // enrolment procedures are fleshed out. This is also a bit panic()-happy, but there is really
+ // no good way out of an invalid enrolment configuration.
+ enrolmentPath, err := s.Storage.GetPathInPlace(storage.PlaceESP, "enrolment.pb")
+ if err != nil {
+ s.logger.Panic("ESP configuration partition not available", zap.Error(err))
}
+ enrolmentConfigRaw, err := ioutil.ReadFile(enrolmentPath)
+ if err == nil {
+ // We have an enrolment file, let's check its contents
+ var enrolmentConfig apipb.EnrolmentConfig
+ if err := proto.Unmarshal(enrolmentConfigRaw, &enrolmentConfig); err != nil {
+ s.logger.Panic("Invalid enrolment configuration provided", zap.Error(err))
+ }
+ s.enrolmentConfig = &enrolmentConfig
+ // The enrolment secret is only zeroed after
+ if len(enrolmentConfig.EnrolmentSecret) == 0 {
+ return s.startFull()
+ }
+ return s.startEnrolling()
+ } else if os.IsNotExist(err) {
+ // This is ok like this, once a new cluster has been set up the initial node also generates
+ // its own enrolment config
+ return s.startForSetup()
+ }
+ // Unknown error reading enrolment config (disk issues/invalid configuration format/...)
+ s.logger.Panic("Invalid enrolment configuration provided", zap.Error(err))
+ panic("Unreachable")
+}
+
+func (s *SmalltownNode) startEnrolling() error {
+ s.logger.Info("Initializing subsystems for enrolment")
+ s.state = common.StateEnrollMode
+
+ nodeInfo, nodeID, err := s.InitializeNode()
+ if err != nil {
+ return err
+ }
+
+ // We only support TPM2 at the moment, any abstractions here would be premature
+ trustAgent := tpm2.TPM2Agent{}
+
+ initializeOp := func() error {
+ if err := trustAgent.Initialize(*nodeInfo, *s.enrolmentConfig); err != nil {
+ s.logger.Warn("Failed to initialize integrity backend", zap.Error(err))
+ return err
+ }
+ return nil
+ }
+
+ if err := backoff.Retry(initializeOp, getIntegrityBackoff()); err != nil {
+ panic("invariant violated: integrity initialization retry can never fail")
+ }
+
+ enrolmentPath, err := s.Storage.GetPathInPlace(storage.PlaceESP, "enrolment.pb")
+ if err != nil {
+ panic(err)
+ }
+
+ s.enrolmentConfig.EnrolmentSecret = []byte{}
+ s.enrolmentConfig.NodeId = nodeID
+
+ enrolmentConfigRaw, err := proto.Marshal(s.enrolmentConfig)
+ if err != nil {
+ panic(err)
+ }
+ if err := ioutil.WriteFile(enrolmentPath, enrolmentConfigRaw, 0600); err != nil {
+ return err
+ }
+ s.logger.Info("Node successfully enrolled")
return nil
}
func (s *SmalltownNode) startForSetup() error {
- s.logger.Info("Initializing subsystems for setup mode")
- s.state = common.StateSetupMode
- s.joinToken = uuid.New().String()
-
- err := s.Api.Start()
+ s.logger.Info("Setting up a new cluster")
+ initData, nodeID, err := s.InitializeNode()
if err != nil {
- s.logger.Error("Failed to start the API service", zap.Error(err))
return err
}
- return nil
-}
+ if err := s.initNodeAPI(); err != nil {
+ return err
+ }
-func (s *SmalltownNode) startFull() error {
- s.logger.Info("Initializing subsystems for production")
- s.state = common.StateConfigured
-
- err := s.SetupBackend()
+ dataPath, err := s.Storage.GetPathInPlace(storage.PlaceData, "etcd")
if err != nil {
return err
}
+ // Spin up etcd
+ config := s.Consensus.GetConfig()
+ config.NewCluster = true
+ config.Name = s.hostname
+ config.DataDir = dataPath
+ s.Consensus.SetConfig(config)
+
+ // Generate the cluster CA and store it to local storage.
+ if err := s.Consensus.PrecreateCA(); err != nil {
+ return err
+ }
+
err = s.Consensus.Start()
if err != nil {
return err
}
+ // Now that the cluster is up and running, we can persist the CA to the cluster.
+ if err := s.Consensus.InjectCA(); err != nil {
+ return err
+ }
+
+ if err := s.Api.BootstrapNewClusterHook(initData); err != nil {
+ return err
+ }
+
+ if err := s.Kubernetes.NewCluster(); err != nil {
+ return err
+ }
+
+ if err := s.Kubernetes.Start(); err != nil {
+ return err
+ }
+
+ if err := s.Api.Start(); err != nil {
+ s.logger.Error("Failed to start the API service", zap.Error(err))
+ return err
+ }
+
+ enrolmentPath, err := s.Storage.GetPathInPlace(storage.PlaceESP, "enrolment.pb")
+ if err != nil {
+ panic(err)
+ }
+
+ masterCert, err := s.Api.GetMasterCert()
+ if err != nil {
+ return err
+ }
+
+ enrolmentConfig := &apipb.EnrolmentConfig{
+ EnrolmentSecret: []byte{}, // First node is always already enrolled
+ MastersCert: masterCert,
+ MasterIps: [][]byte{[]byte(*s.Network.GetIP(true))},
+ NodeId: nodeID,
+ }
+ enrolmentConfigRaw, err := proto.Marshal(enrolmentConfig)
+ if err != nil {
+ panic(err)
+ }
+ if err := ioutil.WriteFile(enrolmentPath, enrolmentConfigRaw, 0600); err != nil {
+ return err
+ }
+ masterCertFingerprint := sha512.Sum512_256(masterCert)
+ s.logger.Info("New Smalltown cluster successfully bootstrapped", zap.Binary("fingerprint", masterCertFingerprint[:]))
+
+ return nil
+}
+
+func (s *SmalltownNode) generateNodeID() ([]byte, string, error) {
+ serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 127)
+ serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+ if err != nil {
+ return []byte{}, "", fmt.Errorf("Failed to generate serial number: %w", err)
+ }
+
+ pubKey, privKeyRaw, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ return []byte{}, "", err
+ }
+ privkey, err := x509.MarshalPKCS8PrivateKey(privKeyRaw)
+ if err != nil {
+ return []byte{}, "", err
+ }
+
+ nodeKeyPath, err := s.Storage.GetPathInPlace(storage.PlaceData, "node-key.der")
+ if err != nil {
+ return []byte{}, "", err
+ }
+
+ if err := ioutil.WriteFile(nodeKeyPath, privkey, 0600); err != nil {
+ return []byte{}, "", fmt.Errorf("failed to write node key: %w", err)
+ }
+
+ name := "smalltown-" + base64.RawStdEncoding.EncodeToString([]byte(pubKey))
+
+ // This has no SANs because it authenticates by public key, not by name
+ nodeCert := &x509.Certificate{
+ SerialNumber: serialNumber,
+ Subject: pkix.Name{
+ // We identify nodes by their ID public keys (not hashed since a strong hash is longer and serves no benefit)
+ CommonName: name,
+ },
+ IsCA: false,
+ BasicConstraintsValid: true,
+ NotBefore: time.Now(),
+ NotAfter: unknownNotAfter,
+ // Certificate is used both as server & client
+ ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
+ }
+ cert, err := x509.CreateCertificate(rand.Reader, nodeCert, nodeCert, pubKey, privKeyRaw)
+ if err != nil {
+ return []byte{}, "", err
+ }
+
+ nodeCertPath, err := s.Storage.GetPathInPlace(storage.PlaceData, "node.der")
+ if err != nil {
+ return []byte{}, "", err
+ }
+
+ if err := ioutil.WriteFile(nodeCertPath, cert, 0600); err != nil {
+ return []byte{}, "", fmt.Errorf("failed to write node cert: %w", err)
+ }
+ return cert, name, nil
+}
+
+func (s *SmalltownNode) initNodeAPI() error {
+ certPath, err := s.Storage.GetPathInPlace(storage.PlaceData, "node.der")
+ if err != nil {
+ s.logger.Panic("Invariant violated: Data is available once this is called")
+ }
+ keyPath, err := s.Storage.GetPathInPlace(storage.PlaceData, "node-key.der")
+ if err != nil {
+ s.logger.Panic("Invariant violated: Data is available once this is called")
+ }
+
+ certRaw, err := ioutil.ReadFile(certPath)
+ if err != nil {
+ return err
+ }
+ privKeyRaw, err := ioutil.ReadFile(keyPath)
+ if err != nil {
+ return err
+ }
+
+ var nodeID tls.Certificate
+
+ cert, err := x509.ParseCertificate(certRaw)
+ if err != nil {
+ return err
+ }
+
+ privKey, err := x509.ParsePKCS8PrivateKey(privKeyRaw)
+ if err != nil {
+ return err
+ }
+
+ nodeID.Certificate = [][]byte{certRaw}
+ nodeID.PrivateKey = privKey
+ nodeID.Leaf = cert
+
+ secureTransport := &tls.Config{
+ Certificates: []tls.Certificate{nodeID},
+ ClientAuth: tls.RequireAndVerifyClientCert,
+ InsecureSkipVerify: true,
+ // Critical function, please review any changes with care
+ // TODO(lorenz): Actively check that this actually provides the security guarantees that we need
+ VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
+ for _, cert := range rawCerts {
+ // X.509 certificates in DER can be compared like this since DER has a unique representation
+ // for each certificate.
+ if bytes.Equal(cert, s.enrolmentConfig.MastersCert) {
+ return nil
+ }
+ }
+ return errors.New("failed to find authorized NMS certificate")
+ },
+ MinVersion: tls.VersionTLS13,
+ }
+ secureTransportCreds := credentials.NewTLS(secureTransport)
+
+ masterListenHost := fmt.Sprintf(":%d", common.NodeServicePort)
+ lis, err := net.Listen("tcp", masterListenHost)
+ if err != nil {
+ s.logger.Fatal("failed to listen", zap.Error(err))
+ }
+
+ nodeGRPCServer := grpc.NewServer(grpc.Creds(secureTransportCreds))
+ apipb.RegisterNodeServiceServer(nodeGRPCServer, s)
+ go func() {
+ if err := nodeGRPCServer.Serve(lis); err != nil {
+ panic(err) // Can only happen during initialization and is always fatal
+ }
+ }()
+ return nil
+}
+
+func getIntegrityBackoff() *backoff.ExponentialBackOff {
+ unlockBackoff := backoff.NewExponentialBackOff()
+ unlockBackoff.MaxElapsedTime = time.Duration(0)
+ unlockBackoff.InitialInterval = 5 * time.Second
+ unlockBackoff.MaxInterval = 5 * time.Minute
+ return unlockBackoff
+}
+
+func (s *SmalltownNode) startFull() error {
+ s.logger.Info("Initializing subsystems for production")
+ s.state = common.StateJoined
+
+ trustAgent := tpm2.TPM2Agent{}
+ unlockOp := func() error {
+ unlockKey, err := trustAgent.Unlock(*s.enrolmentConfig)
+ if err != nil {
+ s.logger.Warn("Failed to unlock", zap.Error(err))
+ return err
+ }
+ if err := s.Storage.MountData(unlockKey); err != nil {
+ s.logger.Panic("Failed to mount storage", zap.Error(err))
+ return err
+ }
+ return nil
+ }
+
+ if err := backoff.Retry(unlockOp, getIntegrityBackoff()); err != nil {
+ s.logger.Panic("Invariant violated: Unlock retry can never fail")
+ }
+
+ s.initNodeAPI()
+
+ err := s.Consensus.Start()
+ if err != nil {
+ return err
+ }
+
err = s.Api.Start()
if err != nil {
s.logger.Error("Failed to start the API service", zap.Error(err))
@@ -158,9 +478,3 @@
s.logger.Info("Stopping Smalltown node")
return nil
}
-
-func (s *SmalltownNode) SetupBackend() error {
- s.logger.Debug("Creating trust backend")
-
- return nil
-}
diff --git a/core/internal/node/setup.go b/core/internal/node/setup.go
index 285046a..23eb24c 100644
--- a/core/internal/node/setup.go
+++ b/core/internal/node/setup.go
@@ -17,12 +17,15 @@
package node
import (
+ "context"
+
"git.monogon.dev/source/nexantic.git/core/generated/api"
"git.monogon.dev/source/nexantic.git/core/internal/common"
- "git.monogon.dev/source/nexantic.git/core/internal/storage"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
+ "errors"
+
"go.uber.org/zap"
)
@@ -38,107 +41,58 @@
return s.state
}
-func (s *SmalltownNode) GetJoinClusterToken() string {
- return s.joinToken
-}
-
-func (s *SmalltownNode) SetupNewCluster() error {
- if s.state == common.StateConfigured {
- return ErrAlreadySetup
- }
- dataPath, err := s.Storage.GetPathInPlace(storage.PlaceData, "etcd")
- if err == storage.ErrNotInitialized {
- return ErrStorageNotInitialized
- } else if err != nil {
- return err
- }
-
- s.logger.Info("Setting up a new cluster")
- s.logger.Info("Provisioning consensus")
-
- // Make sure etcd is not yet provisioned
- if s.Consensus.IsProvisioned() {
- return ErrConsensusAlreadyProvisioned
- }
-
- // Spin up etcd
- config := s.Consensus.GetConfig()
- config.NewCluster = true
- config.Name = s.hostname
- config.DataDir = dataPath
- s.Consensus.SetConfig(config)
-
- // Generate the cluster CA and store it to local storage.
- if err := s.Consensus.PrecreateCA(); err != nil {
- return err
- }
-
- err = s.Consensus.Start()
+// InitializeNode contains functionality that needs to be executed regardless of what the node does
+// later on
+func (s *SmalltownNode) InitializeNode() (*api.NewNodeInfo, string, error) {
+ globalUnlockKey, err := s.Storage.InitializeData()
if err != nil {
- return err
+ return nil, "", err
}
- // Now that the cluster is up and running, we can persist the CA to the cluster.
- if err := s.Consensus.InjectCA(); err != nil {
- return err
- }
+ nodeIP := s.Network.GetIP(true)
- if err := s.Kubernetes.NewCluster(); err != nil {
- return err
- }
-
- if err := s.Kubernetes.Start(); err != nil {
- return err
- }
-
- // Change system state
- s.state = common.StateConfigured
-
- s.logger.Info("New Cluster set up. Node is now fully operational")
-
- return nil
-}
-
-func (s *SmalltownNode) EnterJoinClusterMode() error {
- if s.state == common.StateConfigured {
- return ErrAlreadySetup
- }
- s.state = common.StateClusterJoinMode
-
- s.logger.Info("Node is now in the cluster join mode")
-
- return nil
-}
-
-func (s *SmalltownNode) JoinCluster(clusterString string, certs *api.ConsensusCertificates) error {
- if s.state != common.StateClusterJoinMode {
- return ErrNotInJoinMode
- }
-
- s.logger.Info("Joining cluster", zap.String("cluster", clusterString))
-
- err := s.SetupBackend()
+ nodeCert, nodeID, err := s.generateNodeID()
if err != nil {
- return err
+ return nil, "", err
}
+ return &api.NewNodeInfo{
+ EnrolmentConfig: s.enrolmentConfig,
+ Ip: []byte(*nodeIP),
+ IdCert: nodeCert,
+ GlobalUnlockKey: globalUnlockKey,
+ }, nodeID, nil
+}
+
+func (s *SmalltownNode) JoinCluster(context context.Context, req *api.JoinClusterRequest) (*api.JoinClusterResponse, error) {
+ if s.state != common.StateEnrollMode {
+ return nil, ErrNotInJoinMode
+ }
+
+ s.logger.Info("Joining Consenus")
+
config := s.Consensus.GetConfig()
config.Name = s.hostname
- config.InitialCluster = clusterString
+ config.InitialCluster = "default" // Clusters can't cross-join anyways due to cryptography
s.Consensus.SetConfig(config)
- if err := s.Consensus.WriteCertificateFiles(certs); err != nil {
- return err
+ var err error
+ if err != nil {
+ s.logger.Warn("Invalid JoinCluster request", zap.Error(err))
+ return nil, errors.New("invalid join request")
+ }
+ if err := s.Consensus.WriteCertificateFiles(req.Certs); err != nil {
+ return nil, err
}
// Start consensus
err = s.Consensus.Start()
if err != nil {
- return err
+ return nil, err
}
- s.state = common.StateConfigured
+ s.state = common.StateJoined
s.logger.Info("Joined cluster. Node is now syncing.")
- return nil
+ return &api.JoinClusterResponse{}, nil
}
diff --git a/core/internal/storage/data.go b/core/internal/storage/data.go
index 80af4c9..2b2251a 100644
--- a/core/internal/storage/data.go
+++ b/core/internal/storage/data.go
@@ -18,13 +18,13 @@
import (
"fmt"
- "git.monogon.dev/source/nexantic.git/core/pkg/tpm"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"sync"
+ "git.monogon.dev/source/nexantic.git/core/pkg/tpm"
"go.uber.org/zap"
"golang.org/x/sys/unix"
)
@@ -65,72 +65,99 @@
manager.mutex.Lock()
defer manager.mutex.Unlock()
- sealedKeyFile, err := os.Open(etcdSealedKeyLocation)
- if os.IsNotExist(err) {
- logger.Info("Initializing encrypted storage, this might take a while...")
- go manager.initializeData()
- } else if err != nil {
- return nil, err
- } else {
- sealedKey, err := ioutil.ReadAll(sealedKeyFile)
- sealedKeyFile.Close()
- if err != nil {
- return nil, err
- }
- key, err := tpm.Unseal(sealedKey)
- if err != nil {
- return nil, err
- }
- if err := MapEncryptedBlockDevice("data", SmalltownDataCryptPath, key); err != nil {
- return nil, err
- }
- if err := manager.mountData(); err != nil {
- return nil, err
- }
- logger.Info("Mounted encrypted storage")
- }
return manager, nil
}
-func (s *Manager) initializeData() {
- key, err := tpm.GenerateSafeKey(256 / 8)
+var keySize uint16 = 256 / 8
+
+// MountData mounts the Smalltown data partition with the given global unlock key. It automatically
+// unseals the local unlock key from the TPM.
+func (s *Manager) MountData(globalUnlockKey []byte) error {
+ localPath, err := s.GetPathInPlace(PlaceESP, "local_unlock.bin")
if err != nil {
- s.logger.Error("Failed to generate master key", zap.Error(err))
- s.initializationError = fmt.Errorf("Failed to generate master key: %w", err)
- return
+ return fmt.Errorf("failed to find ESP mount: %w", err)
}
- sealedKey, err := tpm.Seal(key, tpm.FullSystemPCRs)
+ localUnlockBlob, err := ioutil.ReadFile(localPath)
if err != nil {
- s.logger.Error("Failed to seal master key", zap.Error(err))
- s.initializationError = fmt.Errorf("Failed to seal master key: %w", err)
- return
+ return fmt.Errorf("failed to read local unlock file from ESP: %w", err)
}
+ localUnlockKey, err := tpm.Unseal(localUnlockBlob)
+ if err != nil {
+ return fmt.Errorf("failed to unseal local unlock key: %w", err)
+ }
+
+ key := make([]byte, keySize)
+ for i := uint16(0); i < keySize; i++ {
+ key[i] = localUnlockKey[i] ^ globalUnlockKey[i]
+ }
+
+ if err := MapEncryptedBlockDevice("data", SmalltownDataCryptPath, key); err != nil {
+ return err
+ }
+ if err := s.mountData(); err != nil {
+ return err
+ }
+ s.mutex.Lock()
+ s.dataReady = true
+ s.mutex.Unlock()
+ s.logger.Info("Mounted encrypted storage")
+ return nil
+}
+
+// InitializeData initializes the Smalltown data partition and returns the global unlock key. It seals
+// the local portion into the TPM and stores the blob on the ESP. This is a potentially slow
+// operation since it touches the whole partition.
+func (s *Manager) InitializeData() ([]byte, error) {
+ localUnlockKey, err := tpm.GenerateSafeKey(keySize)
+ if err != nil {
+ return []byte{}, fmt.Errorf("failed to generate safe key: %w", err)
+ }
+ globalUnlockKey, err := tpm.GenerateSafeKey(keySize)
+ if err != nil {
+ return []byte{}, fmt.Errorf("failed to generate safe key: %w", err)
+ }
+
+ localUnlockBlob, err := tpm.Seal(localUnlockKey, tpm.SecureBootPCRs)
+ if err != nil {
+ return []byte{}, fmt.Errorf("failed to seal local unlock key: %w", err)
+ }
+
+ // The actual key is generated by XORing together the localUnlockKey and the globalUnlockKey
+ // This provides us with a mathematical guarantee that the resulting key cannot be recovered
+ // whithout knowledge of both parts.
+ key := make([]byte, keySize)
+ for i := uint16(0); i < keySize; i++ {
+ key[i] = localUnlockKey[i] ^ globalUnlockKey[i]
+ }
+
if err := InitializeEncryptedBlockDevice("data", SmalltownDataCryptPath, key); err != nil {
s.logger.Error("Failed to initialize encrypted block device", zap.Error(err))
- s.initializationError = fmt.Errorf("Failed to initialize encrypted block device: %w", err)
- return
+ return []byte{}, fmt.Errorf("failed to initialize encrypted block device: %w", err)
}
mkfsCmd := exec.Command("/bin/mkfs.xfs", "-qf", "/dev/data")
if _, err := mkfsCmd.Output(); err != nil {
s.logger.Error("Failed to format encrypted block device", zap.Error(err))
- s.initializationError = fmt.Errorf("Failed to format encrypted block device: %w", err)
- return
- }
- // This file is the marker if the partition has
- if err := ioutil.WriteFile(etcdSealedKeyLocation, sealedKey, 0600); err != nil {
- panic(err)
+ return []byte{}, fmt.Errorf("failed to format encrypted block device: %w", err)
}
if err := s.mountData(); err != nil {
- s.initializationError = err
- return
+ return []byte{}, err
}
s.mutex.Lock()
s.dataReady = true
s.mutex.Unlock()
+ localPath, err := s.GetPathInPlace(PlaceESP, "local_unlock.bin")
+ if err != nil {
+ return []byte{}, fmt.Errorf("failed to find ESP mount: %w", err)
+ }
+ if err := ioutil.WriteFile(localPath, localUnlockBlob, 0600); err != nil {
+ return []byte{}, fmt.Errorf("failed to write local unlock file to ESP: %w", err)
+ }
+
s.logger.Info("Initialized encrypted storage")
+ return globalUnlockKey, nil
}
func (s *Manager) mountData() error {
diff --git a/core/pkg/tpm/BUILD.bazel b/core/pkg/tpm/BUILD.bazel
index 6803e8a..c39055f 100644
--- a/core/pkg/tpm/BUILD.bazel
+++ b/core/pkg/tpm/BUILD.bazel
@@ -2,13 +2,17 @@
go_library(
name = "go_default_library",
- srcs = ["tpm.go"],
+ srcs = [
+ "credactivation_compat.go",
+ "tpm.go",
+ ],
importpath = "git.monogon.dev/source/nexantic.git/core/pkg/tpm",
visibility = ["//visibility:public"],
deps = [
"//core/pkg/sysfs:go_default_library",
"@com_github_gogo_protobuf//proto:go_default_library",
"@com_github_google_go_tpm//tpm2:go_default_library",
+ "@com_github_google_go_tpm//tpmutil:go_default_library",
"@com_github_google_go_tpm_tools//proto:go_default_library",
"@com_github_google_go_tpm_tools//tpm2tools:go_default_library",
"@com_github_pkg_errors//:go_default_library",
diff --git a/core/pkg/tpm/credactivation_compat.go b/core/pkg/tpm/credactivation_compat.go
new file mode 100644
index 0000000..0a848d2
--- /dev/null
+++ b/core/pkg/tpm/credactivation_compat.go
@@ -0,0 +1,121 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tpm
+
+// This file is adapted from github.com/google/go-tpm/tpm2/credactivation which outputs broken
+// challenges for unknown reasons. They use u16 length-delimited outputs for the challenge blobs
+// which is incorrect.
+// TODO(lorenz): I'll eventually deal with this upstream, but for now just fix it here (it's not that)
+// much code after all.
+
+import (
+ "crypto/aes"
+ "crypto/cipher"
+ "crypto/hmac"
+ "crypto/rsa"
+ "fmt"
+ "io"
+
+ "github.com/google/go-tpm/tpm2"
+ "github.com/google/go-tpm/tpmutil"
+)
+
+const (
+ labelIdentity = "IDENTITY"
+ labelStorage = "STORAGE"
+ labelIntegrity = "INTEGRITY"
+)
+
+func generateRSA(aik *tpm2.HashValue, pub *rsa.PublicKey, symBlockSize int, secret []byte, rnd io.Reader) ([]byte, []byte, error) {
+ newAIKHash, err := aik.Alg.HashConstructor()
+ if err != nil {
+ return nil, nil, err
+ }
+
+ // The seed length should match the keysize used by the EKs symmetric cipher.
+ // For typical RSA EKs, this will be 128 bits (16 bytes).
+ // Spec: TCG 2.0 EK Credential Profile revision 14, section 2.1.5.1.
+ seed := make([]byte, symBlockSize)
+ if _, err := io.ReadFull(rnd, seed); err != nil {
+ return nil, nil, fmt.Errorf("generating seed: %v", err)
+ }
+
+ // Encrypt the seed value using the provided public key.
+ // See annex B, section 10.4 of the TPM specification revision 2 part 1.
+ label := append([]byte(labelIdentity), 0)
+ encSecret, err := rsa.EncryptOAEP(newAIKHash(), rnd, pub, seed, label)
+ if err != nil {
+ return nil, nil, fmt.Errorf("generating encrypted seed: %v", err)
+ }
+
+ // Generate the encrypted credential by convolving the seed with the digest of
+ // the AIK, and using the result as the key to encrypt the secret.
+ // See section 24.4 of TPM 2.0 specification, part 1.
+ aikNameEncoded, err := aik.Encode()
+ if err != nil {
+ return nil, nil, fmt.Errorf("encoding aikName: %v", err)
+ }
+ symmetricKey, err := tpm2.KDFa(aik.Alg, seed, labelStorage, aikNameEncoded, nil, len(seed)*8)
+ if err != nil {
+ return nil, nil, fmt.Errorf("generating symmetric key: %v", err)
+ }
+ c, err := aes.NewCipher(symmetricKey)
+ if err != nil {
+ return nil, nil, fmt.Errorf("symmetric cipher setup: %v", err)
+ }
+ cv, err := tpmutil.Pack(tpmutil.U16Bytes(secret))
+ if err != nil {
+ return nil, nil, fmt.Errorf("generating cv (TPM2B_Digest): %v", err)
+ }
+
+ // IV is all null bytes. encIdentity represents the encrypted credential.
+ encIdentity := make([]byte, len(cv))
+ cipher.NewCFBEncrypter(c, make([]byte, len(symmetricKey))).XORKeyStream(encIdentity, cv)
+
+ // Generate the integrity HMAC, which is used to protect the integrity of the
+ // encrypted structure.
+ // See section 24.5 of the TPM specification revision 2 part 1.
+ macKey, err := tpm2.KDFa(aik.Alg, seed, labelIntegrity, nil, nil, newAIKHash().Size()*8)
+ if err != nil {
+ return nil, nil, fmt.Errorf("generating HMAC key: %v", err)
+ }
+
+ mac := hmac.New(newAIKHash, macKey)
+ mac.Write(encIdentity)
+ mac.Write(aikNameEncoded)
+ integrityHMAC := mac.Sum(nil)
+
+ idObject := &tpm2.IDObject{
+ IntegrityHMAC: integrityHMAC,
+ EncIdentity: encIdentity,
+ }
+ id, err := tpmutil.Pack(idObject)
+ if err != nil {
+ return nil, nil, fmt.Errorf("encoding IDObject: %v", err)
+ }
+
+ packedID, err := tpmutil.Pack(id)
+ if err != nil {
+ return nil, nil, fmt.Errorf("packing id: %v", err)
+ }
+ packedEncSecret, err := tpmutil.Pack(encSecret)
+ if err != nil {
+ return nil, nil, fmt.Errorf("packing encSecret: %v", err)
+ }
+
+ return packedID, packedEncSecret, nil
+}
diff --git a/core/pkg/tpm/tpm.go b/core/pkg/tpm/tpm.go
index e3973d1..bb92289 100644
--- a/core/pkg/tpm/tpm.go
+++ b/core/pkg/tpm/tpm.go
@@ -17,19 +17,26 @@
package tpm
import (
+ "bytes"
+ "crypto"
"crypto/rand"
+ "crypto/rsa"
+ "crypto/x509"
"fmt"
- "git.monogon.dev/source/nexantic.git/core/pkg/sysfs"
"io"
"os"
"path/filepath"
"strconv"
"sync"
+ "time"
+
+ "git.monogon.dev/source/nexantic.git/core/pkg/sysfs"
"github.com/gogo/protobuf/proto"
tpmpb "github.com/google/go-tpm-tools/proto"
"github.com/google/go-tpm-tools/tpm2tools"
"github.com/google/go-tpm/tpm2"
+ "github.com/google/go-tpm/tpmutil"
"github.com/pkg/errors"
"go.uber.org/zap"
"golang.org/x/sys/unix"
@@ -52,18 +59,18 @@
// FirmwarePCRs are alle PCRs that contain the firmware measurements
// See https://trustedcomputinggroup.org/wp-content/uploads/TCG_EFI_Platform_1_22_Final_-v15.pdf
FirmwarePCRs = []int{
- 0, // platform firmware
- 2, // option ROM code
- 3, // option ROM configuration and data
+ 0, // platform firmware
+ 2, // option ROM code
+ 3, // option ROM configuration and data
}
// FullSystemPCRs are all PCRs that contain any measurements up to the currently running EFI payload.
FullSystemPCRs = []int{
- 0, // platform firmware
- 1, // host platform configuration
- 2, // option ROM code
- 3, // option ROM configuration and data
- 4, // EFI payload
+ 0, // platform firmware
+ 1, // host platform configuration
+ 2, // option ROM code
+ 3, // option ROM configuration and data
+ 4, // EFI payload
}
// Using FullSystemPCRs is the most secure, but also the most brittle option since updating the EFI
@@ -84,6 +91,13 @@
)
var (
+ numSRTMPCRs = 16
+ srtmPCRs = tpm2.PCRSelection{Hash: tpm2.AlgSHA256, PCRs: []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}}
+ // TCG Trusted Platform Module Library Level 00 Revision 0.99 Table 6
+ tpmGeneratedValue = uint32(0xff544347)
+)
+
+var (
// ErrNotExists is returned when no TPMs are available in the system
ErrNotExists = errors.New("no TPMs found")
// ErrNotInitialized is returned when this package was not initialized successfully
@@ -101,6 +115,10 @@
type TPM struct {
logger *zap.Logger
device io.ReadWriteCloser
+
+ // We keep the AK loaded since it's used fairly often and deriving it is expensive
+ akHandleCache tpmutil.Handle
+ akPublicKey crypto.PublicKey
}
// Initialize finds and opens the TPM (if any). If there is no TPM available it returns
@@ -230,3 +248,301 @@
}
return unsealedData, nil
}
+
+// Standard AK template for RSA2048 non-duplicatable restricted signing for attestation
+var akTemplate = tpm2.Public{
+ Type: tpm2.AlgRSA,
+ NameAlg: tpm2.AlgSHA256,
+ Attributes: tpm2.FlagSignerDefault,
+ RSAParameters: &tpm2.RSAParams{
+ Sign: &tpm2.SigScheme{
+ Alg: tpm2.AlgRSASSA,
+ Hash: tpm2.AlgSHA256,
+ },
+ KeyBits: 2048,
+ },
+}
+
+func loadAK() error {
+ var err error
+ // Rationale: The AK is a EK-equivalent key and used only for attestation. Using a non-primary
+ // key here would require us to store the wrapped version somewhere, which is inconvenient.
+ // This being a primary key in the Endorsement hierarchy means that it can always be recreated
+ // and can never be "destroyed". Under our security model this is of no concern since we identify
+ // a node by its IK (Identity Key) which we can destroy.
+ tpm.akHandleCache, tpm.akPublicKey, err = tpm2.CreatePrimary(tpm.device, tpm2.HandleEndorsement,
+ tpm2.PCRSelection{}, "", "", akTemplate)
+ return err
+}
+
+// Process documented in TCG EK Credential Profile 2.2.1
+func loadEK() (tpmutil.Handle, crypto.PublicKey, error) {
+ // The EK is a primary key which is supposed to be certified by the manufacturer of the TPM.
+ // Its public attributes are standardized in TCG EK Credential Profile 2.0 Table 1. These need
+ // to match exactly or we aren't getting the key the manufacturere signed. tpm2tools contains
+ // such a template already, so we're using that instead of redoing it ourselves.
+ // This ignores the more complicated ways EKs can be specified, the additional stuff you can do
+ // is just absolutely crazy (see 2.2.1.2 onward)
+ return tpm2.CreatePrimary(tpm.device, tpm2.HandleEndorsement,
+ tpm2.PCRSelection{}, "", "", tpm2tools.DefaultEKTemplateRSA())
+}
+
+// GetAKPublic gets the TPM2T_PUBLIC of the AK key
+func GetAKPublic() ([]byte, error) {
+ lock.Lock()
+ defer lock.Unlock()
+ if tpm == nil {
+ return []byte{}, ErrNotInitialized
+ }
+ if tpm.akHandleCache == tpmutil.Handle(0) {
+ if err := loadAK(); err != nil {
+ return []byte{}, fmt.Errorf("failed to load AK primary key: %w", err)
+ }
+ }
+ public, _, _, err := tpm2.ReadPublic(tpm.device, tpm.akHandleCache)
+ if err != nil {
+ return []byte{}, err
+ }
+ return public.Encode()
+}
+
+// TCG TPM v2.0 Provisioning Guidance v1.0 7.8 Table 2 and
+// TCG EK Credential Profile v2.1 2.2.1.4 de-facto Standard for Windows
+// These are both non-normative and reference Windows 10 documentation that's no longer available :(
+// But in practice this is what people are using, so if it's normative or not doesn't really matter
+const ekCertHandle = 0x01c00002
+
+// GetEKPublic gets the public key and (if available) Certificate of the EK
+func GetEKPublic() ([]byte, []byte, error) {
+ lock.Lock()
+ defer lock.Unlock()
+ if tpm == nil {
+ return []byte{}, []byte{}, ErrNotInitialized
+ }
+ ekHandle, publicRaw, err := loadEK()
+ if err != nil {
+ return []byte{}, []byte{}, fmt.Errorf("failed to load EK primary key: %w", err)
+ }
+ defer tpm2.FlushContext(tpm.device, ekHandle)
+ // Don't question the use of HandleOwner, that's the Standard™
+ ekCertRaw, err := tpm2.NVReadEx(tpm.device, ekCertHandle, tpm2.HandleOwner, "", 0)
+ if err != nil {
+ return []byte{}, []byte{}, err
+ }
+
+ publicKey, err := x509.MarshalPKIXPublicKey(publicRaw)
+ if err != nil {
+ return []byte{}, []byte{}, err
+ }
+
+ return publicKey, ekCertRaw, nil
+}
+
+// MakeAKChallenge generates a challenge for TPM residency and attributes of the AK
+func MakeAKChallenge(ekPubKey, akPub []byte, nonce []byte) ([]byte, []byte, error) {
+ ekPubKeyData, err := x509.ParsePKIXPublicKey(ekPubKey)
+ if err != nil {
+ return []byte{}, []byte{}, fmt.Errorf("failed to decode EK pubkey: %w", err)
+ }
+ akPubData, err := tpm2.DecodePublic(akPub)
+ if err != nil {
+ return []byte{}, []byte{}, fmt.Errorf("failed to decode AK public part: %w", err)
+ }
+ // Make sure we're attesting the right attributes (in particular Restricted)
+ if !akPubData.MatchesTemplate(akTemplate) {
+ return []byte{}, []byte{}, errors.New("the key being challenged is not a valid AK")
+ }
+ akName, err := akPubData.Name()
+ if err != nil {
+ return []byte{}, []byte{}, fmt.Errorf("failed to derive AK name: %w", err)
+ }
+ return generateRSA(akName.Digest, ekPubKeyData.(*rsa.PublicKey), 16, nonce, rand.Reader)
+}
+
+// SolveAKChallenge solves a challenge for TPM residency of the AK
+func SolveAKChallenge(credBlob, secretChallenge []byte) ([]byte, error) {
+ lock.Lock()
+ defer lock.Unlock()
+ if tpm == nil {
+ return []byte{}, ErrNotInitialized
+ }
+ if tpm.akHandleCache == tpmutil.Handle(0) {
+ if err := loadAK(); err != nil {
+ return []byte{}, fmt.Errorf("failed to load AK primary key: %w", err)
+ }
+ }
+
+ ekHandle, _, err := loadEK()
+ if err != nil {
+ return []byte{}, fmt.Errorf("failed to load EK: %w", err)
+ }
+ defer tpm2.FlushContext(tpm.device, ekHandle)
+
+ // This is necessary since the EK requires an endorsement handle policy in its session
+ // For us this is stupid because we keep all hierarchies open anyways since a) we cannot safely
+ // store secrets on the OS side pre-global unlock and b) it makes no sense in this security model
+ // since an uncompromised host OS will not let an untrusted entity attest as itself and a
+ // compromised OS can either not pass PCR policy checks or the game's already over (you
+ // successfully runtime-exploited a production Smalltown Core)
+ endorsementSession, _, err := tpm2.StartAuthSession(
+ tpm.device,
+ tpm2.HandleNull,
+ tpm2.HandleNull,
+ make([]byte, 16),
+ nil,
+ tpm2.SessionPolicy,
+ tpm2.AlgNull,
+ tpm2.AlgSHA256)
+ if err != nil {
+ panic(err)
+ }
+ defer tpm2.FlushContext(tpm.device, endorsementSession)
+
+ _, err = tpm2.PolicySecret(tpm.device, tpm2.HandleEndorsement, tpm2.AuthCommand{Session: tpm2.HandlePasswordSession, Attributes: tpm2.AttrContinueSession}, endorsementSession, nil, nil, nil, 0)
+ if err != nil {
+ return []byte{}, fmt.Errorf("failed to make a policy secret session: %w", err)
+ }
+
+ for {
+ solution, err := tpm2.ActivateCredentialUsingAuth(tpm.device, []tpm2.AuthCommand{
+ {Session: tpm2.HandlePasswordSession, Attributes: tpm2.AttrContinueSession}, // Use standard no-password authentication
+ {Session: endorsementSession, Attributes: tpm2.AttrContinueSession}, // Use a full policy session for the EK
+ }, tpm.akHandleCache, ekHandle, credBlob, secretChallenge)
+ if warn, ok := err.(tpm2.Warning); ok && warn.Code == tpm2.RCRetry {
+ time.Sleep(100 * time.Millisecond)
+ continue
+ }
+ return solution, err
+ }
+}
+
+// FlushTransientHandles flushes all sessions and non-persistent handles
+func FlushTransientHandles() error {
+ lock.Lock()
+ defer lock.Unlock()
+ if tpm == nil {
+ return ErrNotInitialized
+ }
+ flushHandleTypes := []tpm2.HandleType{tpm2.HandleTypeTransient, tpm2.HandleTypeLoadedSession, tpm2.HandleTypeSavedSession}
+ for _, handleType := range flushHandleTypes {
+ handles, err := tpm2tools.Handles(tpm.device, handleType)
+ if err != nil {
+ return err
+ }
+ for _, handle := range handles {
+ if err := tpm2.FlushContext(tpm.device, handle); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+// AttestPlatform performs a PCR quote using the AK and returns the quote and its signature
+func AttestPlatform(nonce []byte) ([]byte, []byte, error) {
+ lock.Lock()
+ defer lock.Unlock()
+ if tpm == nil {
+ return []byte{}, []byte{}, ErrNotInitialized
+ }
+ if tpm.akHandleCache == tpmutil.Handle(0) {
+ if err := loadAK(); err != nil {
+ return []byte{}, []byte{}, fmt.Errorf("failed to load AK primary key: %w", err)
+ }
+ }
+ // We only care about SHA256 since SHA1 is weak. This is supported on at least GCE and
+ // Intel / AMD fTPM, which is good enough for now. Alg is null because that would just hash the
+ // nonce, which is dumb.
+ quote, signature, err := tpm2.Quote(tpm.device, tpm.akHandleCache, "", "", nonce, srtmPCRs,
+ tpm2.AlgNull)
+ if err != nil {
+ return []byte{}, []byte{}, fmt.Errorf("failed to quote PCRs: %w", err)
+ }
+ return quote, signature.RSA.Signature, err
+}
+
+// VerifyAttestPlatform verifies a given attestation. You can rely on all data coming back as being
+// from the TPM on which the AK is bound to.
+func VerifyAttestPlatform(nonce, akPub, quote, signature []byte) (*tpm2.AttestationData, error) {
+ hash := crypto.SHA256.New()
+ hash.Write(quote)
+
+ akPubData, err := tpm2.DecodePublic(akPub)
+ if err != nil {
+ return nil, fmt.Errorf("invalid AK: %w", err)
+ }
+ akPublicKey, err := akPubData.Key()
+ if err != nil {
+ return nil, fmt.Errorf("invalid AK: %w", err)
+ }
+ akRSAKey, ok := akPublicKey.(*rsa.PublicKey)
+ if !ok {
+ return nil, errors.New("invalid AK: invalid key type")
+ }
+
+ if err := rsa.VerifyPKCS1v15(akRSAKey, crypto.SHA256, hash.Sum(nil), signature); err != nil {
+ return nil, err
+ }
+
+ quoteData, err := tpm2.DecodeAttestationData(quote)
+ if err != nil {
+ return nil, err
+ }
+ // quoteData.Magic works together with the TPM's Restricted key attribute. If this attribute is set
+ // (which it needs to be for the AK to be considered valid) the TPM will not sign external data
+ // having this prefix with such a key. Only data that originates inside the TPM like quotes and
+ // key certifications can have this prefix and sill be signed by a restricted key. This check
+ // is thus vital, otherwise somebody can just feed the TPM an arbitrary attestation to sign with
+ // its AK and this function will happily accept the forged attestation.
+ if quoteData.Magic != tpmGeneratedValue {
+ return nil, errors.New("invalid TPM quote: data marker for internal data not set - forged attestation")
+ }
+ if quoteData.Type != tpm2.TagAttestQuote {
+ return nil, errors.New("invalid TPM qoute: not a TPM quote")
+ }
+ if !bytes.Equal(quoteData.ExtraData, nonce) {
+ return nil, errors.New("invalid TPM quote: wrong nonce")
+ }
+
+ return quoteData, nil
+}
+
+// GetPCRs returns all SRTM PCRs in-order
+func GetPCRs() ([][]byte, error) {
+ lock.Lock()
+ defer lock.Unlock()
+ if tpm == nil {
+ return [][]byte{}, ErrNotInitialized
+ }
+ pcrs := make([][]byte, numSRTMPCRs)
+
+ // The TPM can (and most do) return partial results. Let's just retry as many times as we have
+ // PCRs since each read should return at least one PCR.
+readLoop:
+ for i := 0; i < numSRTMPCRs; i++ {
+ sel := tpm2.PCRSelection{Hash: tpm2.AlgSHA256}
+ for pcrN := 0; pcrN < numSRTMPCRs; pcrN++ {
+ if len(pcrs[pcrN]) == 0 {
+ sel.PCRs = append(sel.PCRs, pcrN)
+ }
+ }
+
+ readPCRs, err := tpm2.ReadPCRs(tpm.device, sel)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read PCRs: %w", err)
+ }
+
+ for pcrN, pcr := range readPCRs {
+ pcrs[pcrN] = pcr
+ }
+ for _, pcr := range pcrs {
+ // If at least one PCR is still not read, continue
+ if len(pcr) == 0 {
+ continue readLoop
+ }
+ }
+ break
+ }
+
+ return pcrs, nil
+}
diff --git a/core/scripts/launch.sh b/core/scripts/launch.sh
index 730f60b..bc91f0c 100755
--- a/core/scripts/launch.sh
+++ b/core/scripts/launch.sh
@@ -13,7 +13,7 @@
-drive if=pflash,format=raw,readonly,file=external/edk2/OVMF_CODE.fd \
-drive if=pflash,format=raw,snapshot=on,file=external/edk2/OVMF_VARS.fd \
-drive if=virtio,format=raw,snapshot=on,cache=unsafe,file=core/smalltown.img \
- -netdev user,id=net0,hostfwd=tcp::7833-:7833,hostfwd=tcp::7834-:7834,hostfwd=tcp::6443-:6443 \
+ -netdev user,id=net0,hostfwd=tcp::7833-:7833,hostfwd=tcp::7834-:7834,hostfwd=tcp::6443-:6443,hostfwd=tcp::7835-:7835 \
-device virtio-net-pci,netdev=net0 \
-chardev socket,id=chrtpm,path=tpm-socket \
-tpmdev emulator,id=tpm0,chardev=chrtpm \
diff --git a/core/scripts/test_boot.sh b/core/scripts/test_boot.sh
index 3b6674e..3b4b57d 100755
--- a/core/scripts/test_boot.sh
+++ b/core/scripts/test_boot.sh
@@ -27,12 +27,6 @@
exit 1
}
-print_stderr "Calling api.SetupService.Setup\n"
-system "grpc_cli --channel_creds_type=insecure call localhost:7833 api.SetupService.Setup -json_input '{\"nodeName\": \"node-1\"}'"
-
-print_stderr "Calling api.SetupService.BootstrapNewCluster\n"
-system "grpc_cli --channel_creds_type=insecure call localhost:7833 api.SetupService.BootstrapNewCluster ''"
-
# Make an educated guess if the control plane came up
expect -timeout 3 "\n" {
exp_continue