cluster/internal: remove old cluster enrolment/integrity/management
This had to be done sooner or later, as it has been woefully
underdesigned. A lot of the TPM2 code will make a comeback, but keeping
this around (and buildable) right now is too painful. Once we get
multi-node clusters again, and properly design node/cluster lifecycle,
we'll add integrity/attestation support back in.
Test Plan: this should fail. part of a larger stack. D590 is the first tip of the stack that should work.
X-Origin-Diff: phab/D587
GitOrigin-RevId: e8a43906a767aa4cb66b051027d619ce364269e7
diff --git a/core/cmd/mkenrolment/BUILD.bazel b/core/cmd/mkenrolment/BUILD.bazel
deleted file mode 100644
index 14db892..0000000
--- a/core/cmd/mkenrolment/BUILD.bazel
+++ /dev/null
@@ -1,19 +0,0 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
-
-go_library(
- name = "go_default_library",
- srcs = ["main.go"],
- importpath = "git.monogon.dev/source/nexantic.git/core/cmd/mkenrolment",
- visibility = ["//visibility:private"],
- deps = [
- "//core/api/api:go_default_library",
- "@com_github_gogo_protobuf//proto:go_default_library",
- "@org_golang_google_grpc//:go_default_library",
- ],
-)
-
-go_binary(
- name = "mkenrolment",
- embed = [":go_default_library"],
- visibility = ["//visibility:public"],
-)
diff --git a/core/cmd/mkenrolment/main.go b/core/cmd/mkenrolment/main.go
deleted file mode 100644
index 8b29489..0000000
--- a/core/cmd/mkenrolment/main.go
+++ /dev/null
@@ -1,46 +0,0 @@
-// Copyright 2020 The Monogon Project Authors.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
- "context"
- "io/ioutil"
- "os"
-
- "github.com/gogo/protobuf/proto"
- "google.golang.org/grpc"
-
- "git.monogon.dev/source/nexantic.git/core/generated/api"
-)
-
-func main() {
- conn, err := grpc.Dial(os.Args[1], grpc.WithInsecure())
- if err != nil {
- panic(err)
- }
- defer conn.Close()
- cmc := api.NewClusterManagementClient(conn)
- res, err := cmc.NewEnrolmentConfig(context.Background(), &api.NewEnrolmentConfigRequest{
- Name: "test",
- })
-
- if err != nil {
- panic(err)
- }
- enrolmentConfigRaw, err := proto.Marshal(res.EnrolmentConfig)
- ioutil.WriteFile("enrolment.pb", enrolmentConfigRaw, 0644)
-}
diff --git a/core/internal/api/BUILD.bazel b/core/internal/api/BUILD.bazel
deleted file mode 100644
index 2f25fe6..0000000
--- a/core/internal/api/BUILD.bazel
+++ /dev/null
@@ -1,30 +0,0 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
-
-go_library(
- name = "go_default_library",
- srcs = [
- "cluster.go",
- "enrolment.go",
- "nodemanagement.go",
- "nodes.go",
- "server.go",
- ],
- importpath = "git.monogon.dev/source/nexantic.git/core/internal/api",
- visibility = ["//core:__subpackages__"],
- deps = [
- "//core/api/api:go_default_library",
- "//core/internal/common:go_default_library",
- "//core/internal/common/service:go_default_library",
- "//core/internal/consensus:go_default_library",
- "//core/pkg/tpm:go_default_library",
- "@com_github_gogo_protobuf//proto:go_default_library",
- "@com_github_grpc_ecosystem_go_grpc_middleware//retry:go_default_library",
- "@io_etcd_go_etcd//clientv3:go_default_library",
- "@org_golang_google_grpc//:go_default_library",
- "@org_golang_google_grpc//codes:go_default_library",
- "@org_golang_google_grpc//credentials:go_default_library",
- "@org_golang_google_grpc//reflection:go_default_library",
- "@org_golang_google_grpc//status:go_default_library",
- "@org_uber_go_zap//:go_default_library",
- ],
-)
diff --git a/core/internal/api/cluster.go b/core/internal/api/cluster.go
deleted file mode 100644
index 67fcd83..0000000
--- a/core/internal/api/cluster.go
+++ /dev/null
@@ -1,116 +0,0 @@
-// Copyright 2020 The Monogon Project Authors.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package api
-
-import (
- "context"
- "crypto/rand"
- "encoding/hex"
- "io"
-
- "github.com/gogo/protobuf/proto"
- "go.etcd.io/etcd/clientv3"
- "google.golang.org/grpc/codes"
- "google.golang.org/grpc/status"
-
- "git.monogon.dev/source/nexantic.git/core/generated/api"
- schema "git.monogon.dev/source/nexantic.git/core/generated/api"
-
- "go.uber.org/zap"
-)
-
-func (s *Server) AddNode(ctx context.Context, req *schema.AddNodeRequest) (*schema.AddNodeResponse, error) {
- return nil, status.Error(codes.Unimplemented, "Unimplemented")
-}
-
-func (s *Server) RemoveNode(ctx context.Context, req *schema.RemoveNodeRequest) (*schema.RemoveNodeRequest, error) {
- return nil, status.Error(codes.Unimplemented, "Unimplemented")
-}
-
-func (s *Server) ListNodes(ctx context.Context, req *schema.ListNodesRequest) (*schema.ListNodesResponse, error) {
- store := s.getStore()
- res, err := store.Get(ctx, "nodes/", clientv3.WithPrefix())
- if err != nil {
- return nil, status.Error(codes.Unavailable, "Consensus unavailable")
- }
- var resNodes []*api.Node
- for _, nodeEntry := range res.Kvs {
- var node api.Node
- if err := proto.Unmarshal(nodeEntry.Value, &node); err != nil {
- s.Logger.Error("Encountered invalid node data", zap.Error(err))
- return nil, status.Error(codes.Internal, "Invalid data")
- }
- // Zero out Global Unlock Key, it's never supposed to leave the cluster
- node.GlobalUnlockKey = []byte{}
-
- resNodes = append(resNodes, &node)
- }
-
- return &schema.ListNodesResponse{
- Nodes: resNodes,
- }, nil
-}
-
-func (s *Server) ListEnrolmentConfigs(ctx context.Context, req *api.ListEnrolmentConfigsRequest) (*api.ListEnrolmentConfigsResponse, error) {
- return nil, status.Error(codes.Unimplemented, "Unimplemented")
-}
-
-func (s *Server) NewEnrolmentConfig(ctx context.Context, req *api.NewEnrolmentConfigRequest) (*api.NewEnrolmentConfigResponse, error) {
- store := s.getStore()
- token := make([]byte, 32)
- if _, err := io.ReadFull(rand.Reader, token); err != nil {
- return nil, status.Error(codes.Unavailable, "failed to get randonmess")
- }
- nodes, err := store.Get(ctx, "nodes/", clientv3.WithPrefix())
- if err != nil {
- return nil, status.Error(codes.Unavailable, "consensus unavailable")
- }
- var masterIPs [][]byte
- for _, nodeKV := range nodes.Kvs {
- var node api.Node
- if err := proto.Unmarshal(nodeKV.Value, &node); err != nil {
- return nil, status.Error(codes.Internal, "invalid node")
- }
- if node.State == api.Node_MASTER {
- masterIPs = append(masterIPs, node.Address)
- }
- }
- masterCert, err := s.GetMasterCert()
- if err != nil {
- return nil, status.Error(codes.Unavailable, "consensus unavailable")
- }
-
- enrolmentConfig := &api.EnrolmentConfig{
- EnrolmentSecret: token,
- MasterIps: masterIPs,
- MastersCert: masterCert,
- }
- enrolmentConfigRaw, err := proto.Marshal(enrolmentConfig)
- if err != nil {
- return nil, status.Error(codes.Internal, "failed to encode config")
- }
- if _, err := store.Put(ctx, "enrolments/"+hex.EncodeToString(token), string(enrolmentConfigRaw)); err != nil {
- return nil, status.Error(codes.Unavailable, "consensus unavailable")
- }
- return &schema.NewEnrolmentConfigResponse{
- EnrolmentConfig: enrolmentConfig,
- }, nil
-}
-
-func (s *Server) RemoveEnrolmentConfig(ctx context.Context, req *api.RemoveEnrolmentConfigRequest) (*api.RemoveEnrolmentConfigResponse, error) {
- return nil, status.Error(codes.Unimplemented, "Unimplemented")
-}
diff --git a/core/internal/api/enrolment.go b/core/internal/api/enrolment.go
deleted file mode 100644
index eb892ae..0000000
--- a/core/internal/api/enrolment.go
+++ /dev/null
@@ -1,56 +0,0 @@
-// Copyright 2020 The Monogon Project Authors.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package api
-
-import (
- "context"
- "encoding/hex"
- "errors"
- "fmt"
-
- "github.com/gogo/protobuf/proto"
- "go.etcd.io/etcd/clientv3"
-
- "git.monogon.dev/source/nexantic.git/core/generated/api"
-)
-
-const enrolmentPrefix = "enrolments/"
-
-var errNotExists = errors.New("not found")
-
-type EnrolmentStore struct {
- backend clientv3.KV
-}
-
-func (s *EnrolmentStore) GetBySecret(ctx context.Context, secret []byte) (*api.EnrolmentConfig, error) {
-
- res, err := s.backend.Get(ctx, enrolmentPrefix+hex.EncodeToString(secret))
- if err != nil {
- return nil, fmt.Errorf("failed to query consensus: %w", err)
- }
- if res.Count == 0 {
- return nil, errNotExists
- } else if res.Count > 1 {
- panic("more than one value for the same key, bailing")
- }
- rawVal := res.Kvs[0].Value
- var config *api.EnrolmentConfig
- if err := proto.Unmarshal(rawVal, config); err != nil {
- return nil, err
- }
- return config, nil
-}
diff --git a/core/internal/api/nodemanagement.go b/core/internal/api/nodemanagement.go
deleted file mode 100644
index 4bc4659..0000000
--- a/core/internal/api/nodemanagement.go
+++ /dev/null
@@ -1,354 +0,0 @@
-// Copyright 2020 The Monogon Project Authors.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package api
-
-import (
- "bytes"
- "context"
- "crypto/ed25519"
- "crypto/rand"
- "crypto/sha256"
- "crypto/subtle"
- "crypto/tls"
- "crypto/x509"
- "encoding/hex"
- "errors"
- "fmt"
- "io"
- "net"
- "time"
-
- "github.com/gogo/protobuf/proto"
- grpcretry "github.com/grpc-ecosystem/go-grpc-middleware/retry"
- "go.etcd.io/etcd/clientv3"
- "go.uber.org/zap"
- "google.golang.org/grpc"
- "google.golang.org/grpc/codes"
- "google.golang.org/grpc/credentials"
- "google.golang.org/grpc/status"
-
- "git.monogon.dev/source/nexantic.git/core/generated/api"
- "git.monogon.dev/source/nexantic.git/core/internal/common"
- "git.monogon.dev/source/nexantic.git/core/pkg/tpm"
-)
-
-const nodesPrefix = "nodes/"
-const enrolmentsPrefix = "enrolments/"
-
-func nodeId(idCert []byte) (string, error) {
- // Currently we only identify nodes by ID key
- cert, err := x509.ParseCertificate(idCert)
- if err != nil {
- return "", err
- }
- pubKey, ok := cert.PublicKey.(ed25519.PublicKey)
- if !ok {
- return "", errors.New("invalid node identity certificate")
- }
-
- return common.NameFromIDKey(pubKey), nil
-}
-
-func (s *Server) registerNewNode(node *api.Node) error {
- nodeRaw, err := proto.Marshal(node)
- if err != nil {
- return err
- }
-
- nodeID, err := nodeId(node.IdCert)
- if err != nil {
- return err
- }
-
- key := nodesPrefix + nodeID
-
- // Overwriting nodes is a BadIdea(TM), so make this a Compare-and-Swap
- res, err := s.getStore().Txn(context.Background()).If(
- clientv3.Compare(clientv3.CreateRevision(key), "=", 0),
- ).Then(
- clientv3.OpPut(key, string(nodeRaw)),
- ).Commit()
- if err != nil {
- return fmt.Errorf("failed to store new node: %w", err)
- }
- if !res.Succeeded {
- s.Logger.Warn("double-registration of node attempted", zap.String("node", nodeID))
- }
- return nil
-}
-
-func (s *Server) TPM2BootstrapNode(newNodeInfo *api.NewNodeInfo) (*api.Node, error) {
- akPublic, err := tpm.GetAKPublic()
- if err != nil {
- return nil, err
- }
- ekPubkey, ekCert, err := tpm.GetEKPublic()
- if err != nil {
- return nil, err
- }
- return &api.Node{
- Address: newNodeInfo.Ip,
- Integrity: &api.Node_Tpm2{Tpm2: &api.NodeTPM2{
- AkPub: akPublic,
- EkCert: ekCert,
- EkPubkey: ekPubkey,
- }},
- GlobalUnlockKey: newNodeInfo.GlobalUnlockKey,
- IdCert: newNodeInfo.IdCert,
- State: api.Node_MASTER,
- }, nil
-}
-
-func (s *Server) TPM2Unlock(unlockServer api.NodeManagementService_TPM2UnlockServer) error {
- nonce := make([]byte, 32)
- if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
- return status.Error(codes.Unavailable, "failed to get randomness")
- }
- if err := unlockServer.Send(&api.TPM2UnlockFlowResponse{
- Stage: &api.TPM2UnlockFlowResponse_UnlockInit{
- UnlockInit: &api.TPM2UnlockInit{
- Nonce: nonce,
- },
- },
- }); err != nil {
- return err
- }
- unlockReqContainer, err := unlockServer.Recv()
- if err != nil {
- return err
- }
- unlockReqVariant, ok := unlockReqContainer.Stage.(*api.TPM2UnlockFlowRequeset_UnlockRequest)
- if !ok {
- return status.Errorf(codes.InvalidArgument, "protocol violation")
- }
- unlockRequest := unlockReqVariant.UnlockRequest
-
- store := s.getStore()
- // This is safe, etcd does not do relative paths
- path := nodesPrefix + unlockRequest.NodeId
- nodeRes, err := store.Get(unlockServer.Context(), path)
- if err != nil {
- return status.Error(codes.Unavailable, "consensus request failed")
- }
- if nodeRes.Count == 0 {
- return status.Error(codes.NotFound, "this node does not exist")
- } else if nodeRes.Count > 1 {
- panic("invariant violation: more than one node with the same id")
- }
- nodeRaw := nodeRes.Kvs[0].Value
- var node api.Node
- if err := proto.Unmarshal(nodeRaw, &node); err != nil {
- s.Logger.Error("Failed to decode node", zap.Error(err))
- return status.Error(codes.Internal, "invalid node")
- }
-
- nodeTPM2, ok := node.Integrity.(*api.Node_Tpm2)
- if !ok {
- return status.Error(codes.InvalidArgument, "node not integrity-protected with TPM2")
- }
-
- validQuote, err := tpm.VerifyAttestPlatform(nonce, nodeTPM2.Tpm2.AkPub, unlockRequest.Quote, unlockRequest.QuoteSignature)
- if err != nil {
- return status.Error(codes.PermissionDenied, "invalid quote")
- }
-
- pcrHash := sha256.New()
- for _, pcr := range unlockRequest.Pcrs {
- pcrHash.Write(pcr)
- }
- expectedPCRHash := pcrHash.Sum(nil)
-
- if !bytes.Equal(validQuote.AttestedQuoteInfo.PCRDigest, expectedPCRHash) {
- return status.Error(codes.InvalidArgument, "the quote's PCR hash does not match the supplied PCRs")
- }
-
- // TODO: Plug in policy engine to decide if the unlock should actually happen
-
- return unlockServer.Send(&api.TPM2UnlockFlowResponse{Stage: &api.TPM2UnlockFlowResponse_UnlockResponse{
- UnlockResponse: &api.TPM2UnlockResponse{
- GlobalUnlockKey: node.GlobalUnlockKey,
- },
- }})
-}
-
-func (s *Server) dialNode(ctx context.Context, node *api.Node) (api.NodeServiceClient, error) {
- masterID, err := s.loadMasterCert()
- if err != nil {
- return nil, err
- }
-
- secureTransport := &tls.Config{
- Certificates: []tls.Certificate{*masterID},
- InsecureSkipVerify: true,
- // Critical function, please review any changes with care
- // TODO(lorenz): Actively check that this actually provides the security guarantees that we need
- VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
- for _, cert := range rawCerts {
- // X.509 certificates in DER can be compared like this since DER has a unique representation
- // for each certificate.
- if bytes.Equal(cert, node.IdCert) {
- return nil
- }
- }
- return errors.New("failed to find authorized Node certificate")
- },
- MinVersion: tls.VersionTLS13,
- }
- addr := net.IP(node.Address)
- opts := []grpcretry.CallOption{
- grpcretry.WithBackoff(grpcretry.BackoffExponential(100 * time.Millisecond)),
- }
- clientCreds := grpc.WithTransportCredentials(credentials.NewTLS(secureTransport))
- clientConn, err := grpc.DialContext(ctx, fmt.Sprintf("%v:%v", addr, common.NodeServicePort), clientCreds,
- grpc.WithUnaryInterceptor(grpcretry.UnaryClientInterceptor(opts...)))
- if err != nil {
- return nil, fmt.Errorf("failed to dial node service: %w", err)
- }
- return api.NewNodeServiceClient(clientConn), nil
-}
-
-func (s *Server) NewTPM2NodeRegister(registerServer api.NodeManagementService_NewTPM2NodeRegisterServer) error {
- registerReqContainer, err := registerServer.Recv()
- if err != nil {
- return err
- }
- registerReqVariant, ok := registerReqContainer.Stage.(*api.TPM2FlowRequest_Register)
- if !ok {
- return status.Error(codes.InvalidArgument, "protocol violation")
- }
- registerReq := registerReqVariant.Register
-
- challengeNonce := make([]byte, 32)
- if _, err := io.ReadFull(rand.Reader, challengeNonce); err != nil {
- return status.Error(codes.Unavailable, "failed to get randomness")
- }
- challenge, challengeBlob, err := tpm.MakeAKChallenge(registerReq.EkPubkey, registerReq.AkPublic, challengeNonce)
- if err != nil {
- return status.Errorf(codes.InvalidArgument, "failed to challenge AK: %v", err)
- }
- nonce := make([]byte, 32)
- if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
- return status.Error(codes.Unavailable, "failed to get randomness")
- }
- if err := registerServer.Send(&api.TPM2FlowResponse{Stage: &api.TPM2FlowResponse_AttestRequest{AttestRequest: &api.TPM2AttestRequest{
- AkChallenge: challenge,
- AkChallengeSecret: challengeBlob,
- QuoteNonce: nonce,
- }}}); err != nil {
- return err
- }
- attestationResContainer, err := registerServer.Recv()
- if err != nil {
- return err
- }
- attestResVariant, ok := attestationResContainer.Stage.(*api.TPM2FlowRequest_AttestResponse)
- if !ok {
- return status.Error(codes.InvalidArgument, "protocol violation")
- }
- attestRes := attestResVariant.AttestResponse
-
- if subtle.ConstantTimeCompare(attestRes.AkChallengeSolution, challengeNonce) != 1 {
- return status.Error(codes.InvalidArgument, "invalid challenge response")
- }
-
- validQuote, err := tpm.VerifyAttestPlatform(nonce, registerReq.AkPublic, attestRes.Quote, attestRes.QuoteSignature)
- if err != nil {
- return status.Error(codes.PermissionDenied, "invalid quote")
- }
-
- pcrHash := sha256.New()
- for _, pcr := range attestRes.Pcrs {
- pcrHash.Write(pcr)
- }
- expectedPCRHash := pcrHash.Sum(nil)
-
- if !bytes.Equal(validQuote.AttestedQuoteInfo.PCRDigest, expectedPCRHash) {
- return status.Error(codes.InvalidArgument, "the quote's PCR hash does not match the supplied PCRs")
- }
-
- newNodeInfoContainer, err := registerServer.Recv()
- newNodeInfoVariant, ok := newNodeInfoContainer.Stage.(*api.TPM2FlowRequest_NewNodeInfo)
- newNodeInfo := newNodeInfoVariant.NewNodeInfo
-
- store := s.getStore()
- res, err := store.Get(registerServer.Context(), "enrolments/"+hex.EncodeToString(newNodeInfo.EnrolmentConfig.EnrolmentSecret))
- if err != nil {
- return status.Error(codes.Unavailable, "Consensus unavailable")
- }
- if res.Count == 0 {
- return status.Error(codes.PermissionDenied, "Invalid enrolment secret")
- } else if res.Count > 1 {
- panic("more than one value for the same key, bailing")
- }
- rawVal := res.Kvs[0].Value
- var config api.EnrolmentConfig
- if err := proto.Unmarshal(rawVal, &config); err != nil {
- return err
- }
-
- // TODO: Plug in policy engine here
- idCert, err := x509.ParseCertificate(newNodeInfo.IdCert)
- if err != nil {
- return err
- }
- nodeIdPubKey, ok := idCert.PublicKey.(ed25519.PublicKey)
- if !ok || len(nodeIdPubKey) != ed25519.PublicKeySize {
- return status.Error(codes.InvalidArgument, "Invalid ID certificate public key")
- }
-
- node := api.Node{
- Name: common.NameFromIDKey(nodeIdPubKey),
- Address: newNodeInfo.Ip,
- Integrity: &api.Node_Tpm2{Tpm2: &api.NodeTPM2{
- AkPub: registerReq.AkPublic,
- EkCert: registerReq.EkCert,
- EkPubkey: registerReq.EkPubkey,
- }},
- GlobalUnlockKey: newNodeInfo.GlobalUnlockKey,
- IdCert: newNodeInfo.IdCert,
- State: api.Node_MASTER,
- }
-
- if err := s.registerNewNode(&node); err != nil {
- s.Logger.Error("failed to register a node", zap.Error(err))
- return status.Error(codes.Internal, "failed to register node")
- }
-
- go func() {
- ctx := context.Background()
- nodeClient, err := s.dialNode(ctx, &node)
- if err != nil {
- s.Logger.Warn("Failed to join newly enrolled node", zap.Error(err))
- return
- }
- newCerts, initialCluster, err := s.consensusService.ProvisionMember(node.Name, node.Address)
- if err != nil {
- s.Logger.Warn("Failed to join newly enrolled node", zap.Error(err))
- return
- }
- _, err = nodeClient.JoinCluster(ctx, &api.JoinClusterRequest{
- InitialCluster: initialCluster,
- Certs: newCerts,
- }, grpcretry.WithMax(10))
- if err != nil {
- s.Logger.Warn("Failed to join newly enrolled node", zap.Error(err))
- return
- }
- }()
-
- return nil
-}
diff --git a/core/internal/api/nodes.go b/core/internal/api/nodes.go
deleted file mode 100644
index da3cbc4..0000000
--- a/core/internal/api/nodes.go
+++ /dev/null
@@ -1,17 +0,0 @@
-// Copyright 2020 The Monogon Project Authors.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package api
diff --git a/core/internal/api/server.go b/core/internal/api/server.go
deleted file mode 100644
index 88f33f9..0000000
--- a/core/internal/api/server.go
+++ /dev/null
@@ -1,240 +0,0 @@
-// Copyright 2020 The Monogon Project Authors.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package api
-
-import (
- "context"
- "crypto/ed25519"
- "crypto/rand"
- "crypto/tls"
- "crypto/x509"
- "crypto/x509/pkix"
- "errors"
- "fmt"
- "math/big"
- "net"
- "time"
-
- "go.etcd.io/etcd/clientv3"
- "go.uber.org/zap"
- "google.golang.org/grpc"
- "google.golang.org/grpc/credentials"
- "google.golang.org/grpc/reflection"
-
- "git.monogon.dev/source/nexantic.git/core/generated/api"
- schema "git.monogon.dev/source/nexantic.git/core/generated/api"
- "git.monogon.dev/source/nexantic.git/core/internal/common"
- "git.monogon.dev/source/nexantic.git/core/internal/common/service"
- "git.monogon.dev/source/nexantic.git/core/internal/consensus"
-)
-
-type (
- Server struct {
- *service.BaseService
-
- grpcServer *grpc.Server
- externalGrpcServer *grpc.Server
-
- consensusService *consensus.Service
-
- config *Config
- }
-
- Config struct {
- }
-)
-
-var (
- // From RFC 5280 Section 4.1.2.5
- unknownNotAfter = time.Unix(253402300799, 0)
-)
-
-func NewApiServer(config *Config, logger *zap.Logger, consensusService *consensus.Service) (*Server, error) {
- s := &Server{
- config: config,
- consensusService: consensusService,
- }
-
- s.BaseService = service.NewBaseService("api", logger, s)
-
- return s, nil
-}
-
-func (s *Server) getStore() clientv3.KV {
- // Cannot be moved to initialization because an internal reference will be nil
- return s.consensusService.GetStore("api", "")
-}
-
-// BootstrapNewClusterHook creates the necessary key material for the API Servers and stores it in
-// the consensus service. It also creates a node entry for the initial node.
-func (s *Server) BootstrapNewClusterHook(initNodeReq *api.NewNodeInfo) error {
- serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 127)
- serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
- if err != nil {
- return fmt.Errorf("Failed to generate serial number: %w", err)
- }
-
- pubKey, privKeyRaw, err := ed25519.GenerateKey(rand.Reader)
- if err != nil {
- return err
- }
- privkey, err := x509.MarshalPKCS8PrivateKey(privKeyRaw)
- if err != nil {
- return err
- }
-
- // This has no SANs because it authenticates by public key, not by name
- masterCert := &x509.Certificate{
- SerialNumber: serialNumber,
- Subject: pkix.Name{
- CommonName: "Smalltown Master",
- },
- IsCA: false,
- BasicConstraintsValid: true,
- NotBefore: time.Now(),
- NotAfter: unknownNotAfter,
- // Certificate is used both as server & client
- ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
- }
- cert, err := x509.CreateCertificate(rand.Reader, masterCert, masterCert, pubKey, privKeyRaw)
- if err != nil {
- return err
- }
- store := s.getStore()
- if _, err := store.Put(context.Background(), "master.der", string(cert)); err != nil {
- return err
- }
- if _, err := store.Put(context.Background(), "master-key.der", string(privkey)); err != nil {
- return err
- }
-
- // TODO: Further integrity providers need to be plumbed in here
- node, err := s.TPM2BootstrapNode(initNodeReq)
- if err != nil {
- return err
- }
-
- if err := s.registerNewNode(node); err != nil {
- return err
- }
- return nil
-}
-
-// GetMasterCert gets the master certificate in X.509 DER form
-// This is mainly used to issue enrolment configs
-func (s *Server) GetMasterCert() ([]byte, error) {
- store := s.getStore()
- res, err := store.Get(context.Background(), "master.der")
- if err != nil {
- return []byte{}, err
- }
- if len(res.Kvs) != 1 {
- return []byte{}, errors.New("master certificate not found")
- }
- certRaw := res.Kvs[0].Value
- return certRaw, nil
-}
-
-// TODO(lorenz): Move consensus/certificate interaction into a utility, is now duplicated too often
-func (s *Server) loadMasterCert() (*tls.Certificate, error) {
-
- store := s.getStore()
- var tlsCert tls.Certificate
- res, err := store.Get(context.Background(), "master.der")
- if err != nil {
- return nil, err
- }
- if len(res.Kvs) != 1 {
- return nil, errors.New("master certificate not found")
- }
- certRaw := res.Kvs[0].Value
-
- tlsCert.Certificate = append(tlsCert.Certificate, certRaw)
- tlsCert.Leaf, err = x509.ParseCertificate(certRaw)
-
- res, err = store.Get(context.Background(), "master-key.der")
- if err != nil {
- return nil, err
- }
- if len(res.Kvs) != 1 {
- return nil, errors.New("master certificate not found")
- }
- keyRaw := res.Kvs[0].Value
- key, err := x509.ParsePKCS8PrivateKey(keyRaw)
- if err != nil {
- return nil, fmt.Errorf("failed to load master private key: %w", err)
- }
- edKey, ok := key.(ed25519.PrivateKey)
- if !ok {
- return nil, errors.New("invalid private key")
- }
- tlsCert.PrivateKey = edKey
- return &tlsCert, nil
-}
-
-func (s *Server) OnStart() error {
- masterListenHost := fmt.Sprintf(":%d", common.MasterServicePort)
- lis, err := net.Listen("tcp", masterListenHost)
- if err != nil {
- s.Logger.Fatal("failed to listen", zap.Error(err))
- }
-
- externalListeneHost := fmt.Sprintf(":%d", common.ExternalServicePort)
- externalListener, err := net.Listen("tcp", externalListeneHost)
- if err != nil {
- s.Logger.Fatal("failed to listen", zap.Error(err))
- }
-
- masterCert, err := s.loadMasterCert()
- if err != nil {
- s.Logger.Error("Failed to load Master Service Key Material: %w", zap.Error(err))
- return err
- }
-
- masterTransportCredentials := credentials.NewServerTLSFromCert(masterCert)
-
- masterGrpcServer := grpc.NewServer(grpc.Creds(masterTransportCredentials))
- clusterManagementGrpcServer := grpc.NewServer()
- schema.RegisterClusterManagementServer(clusterManagementGrpcServer, s)
- schema.RegisterNodeManagementServiceServer(masterGrpcServer, s)
-
- reflection.Register(masterGrpcServer)
-
- s.grpcServer = masterGrpcServer
- s.externalGrpcServer = clusterManagementGrpcServer
-
- go func() {
- err = s.grpcServer.Serve(lis)
- s.Logger.Error("API server failed", zap.Error(err))
- }()
-
- go func() {
- err = s.externalGrpcServer.Serve(externalListener)
- s.Logger.Error("API server failed", zap.Error(err))
- }()
-
- s.Logger.Info("gRPC listening", zap.String("host", masterListenHost))
-
- return nil
-}
-
-func (s *Server) OnStop() error {
- s.grpcServer.Stop()
- s.externalGrpcServer.Stop()
-
- return nil
-}
diff --git a/core/internal/common/grpc/BUILD.bazel b/core/internal/common/grpc/BUILD.bazel
deleted file mode 100644
index 85661c6..0000000
--- a/core/internal/common/grpc/BUILD.bazel
+++ /dev/null
@@ -1,12 +0,0 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
-
-go_library(
- name = "go_default_library",
- srcs = ["grpc.go"],
- importpath = "git.monogon.dev/source/nexantic.git/core/internal/common/grpc",
- visibility = ["//:__subpackages__"],
- deps = [
- "//core/api/api:go_default_library",
- "@org_golang_google_grpc//:go_default_library",
- ],
-)
diff --git a/core/internal/common/grpc/grpc.go b/core/internal/common/grpc/grpc.go
deleted file mode 100644
index 97fc4c8..0000000
--- a/core/internal/common/grpc/grpc.go
+++ /dev/null
@@ -1,50 +0,0 @@
-// Copyright 2020 The Monogon Project Authors.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package grpc
-
-import (
- "git.monogon.dev/source/nexantic.git/core/generated/api"
-
- "google.golang.org/grpc"
-)
-
-type (
- SmalltownClient struct {
- conn *grpc.ClientConn
-
- Cluster api.ClusterManagementClient
- }
-)
-
-func NewSmalltownAPIClient(address string) (*SmalltownClient, error) {
- s := &SmalltownClient{}
-
- conn, err := grpc.Dial(address, grpc.WithInsecure())
- if err != nil {
- return nil, err
- }
- s.conn = conn
-
- // Setup all client connections
- s.Cluster = api.NewClusterManagementClient(conn)
-
- return s, nil
-}
-
-func (s *SmalltownClient) Close() error {
- return s.conn.Close()
-}
diff --git a/core/internal/integrity/BUILD.bazel b/core/internal/integrity/BUILD.bazel
deleted file mode 100644
index cb551cc..0000000
--- a/core/internal/integrity/BUILD.bazel
+++ /dev/null
@@ -1,14 +0,0 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
-
-go_library(
- name = "go_default_library",
- srcs = ["common.go"],
- importpath = "git.monogon.dev/source/nexantic.git/core/internal/integrity",
- visibility = ["//core:__subpackages__"],
- deps = [
- "//core/api/api:go_default_library",
- "//core/internal/common:go_default_library",
- "@org_golang_google_grpc//:go_default_library",
- "@org_golang_google_grpc//credentials:go_default_library",
- ],
-)
diff --git a/core/internal/integrity/common.go b/core/internal/integrity/common.go
deleted file mode 100644
index 52196ce..0000000
--- a/core/internal/integrity/common.go
+++ /dev/null
@@ -1,83 +0,0 @@
-// Copyright 2020 The Monogon Project Authors.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package integrity
-
-import (
- "bytes"
- "crypto/tls"
- "crypto/x509"
- "errors"
- "fmt"
- "net"
- "strings"
-
- "google.golang.org/grpc"
- "google.golang.org/grpc/credentials"
-
- "git.monogon.dev/source/nexantic.git/core/generated/api"
- "git.monogon.dev/source/nexantic.git/core/internal/common"
-)
-
-// Agent specifices the interface which every integrity agent needs to fulfill
-// TODO: This interface is not yet used, we call the TPM2 agent directly.
-type Agent interface {
- // Initialize needs to be called once and initializes the systems required to maintain integrity
- // on the given platform.
- // nodeCert is a X.509 DER certificate which identifies the node once it's unlocked. This is
- // required to bind the node certificate (which is only available when the node is unlocked) to
- // the integrity subsystem used to attest said node.
- // Initialize returns the cryptographic identity that it's bound to.
- Initialize(newNode api.NewNodeInfo, enrolment api.EnrolmentConfig) (string, error)
-
- // Unlock performs all required actions to assure the integrity of the platform and securely retrieves
- // the unlock key.
- Unlock(enrolment api.EnrolmentConfig) ([]byte, error)
-}
-
-// DialNMS creates a secure GRPC connection to the NodeManagementService
-func DialNMS(enrolment api.EnrolmentConfig) (*grpc.ClientConn, error) {
- var targets []string
- for _, target := range enrolment.MasterIps {
- targets = append(targets, fmt.Sprintf("%v:%v", net.IP(target), common.MasterServicePort))
- }
- cert, err := x509.ParseCertificate(enrolment.MastersCert)
- if err != nil {
- return nil, err
- }
- mastersPool := x509.NewCertPool()
- mastersPool.AddCert(cert)
-
- secureTransport := &tls.Config{
- InsecureSkipVerify: true,
- // Critical function, please review any changes with care
- // TODO(lorenz): Actively check that this actually provides the security guarantees that we need
- VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
- for _, cert := range rawCerts {
- // X.509 certificates in DER can be compared like this since DER has a unique representation
- // for each certificate.
- if bytes.Equal(cert, enrolment.MastersCert) {
- return nil
- }
- }
- return errors.New("failed to find authorized NMS certificate")
- },
- MinVersion: tls.VersionTLS13,
- }
- secureTransportCreds := credentials.NewTLS(secureTransport)
-
- return grpc.Dial(strings.Join(targets, ","), grpc.WithTransportCredentials(secureTransportCreds))
-}
diff --git a/core/internal/integrity/tpm2/BUILD.bazel b/core/internal/integrity/tpm2/BUILD.bazel
deleted file mode 100644
index c4c77bb..0000000
--- a/core/internal/integrity/tpm2/BUILD.bazel
+++ /dev/null
@@ -1,13 +0,0 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
-
-go_library(
- name = "go_default_library",
- srcs = ["tpm2.go"],
- importpath = "git.monogon.dev/source/nexantic.git/core/internal/integrity/tpm2",
- visibility = ["//core:__subpackages__"],
- deps = [
- "//core/api/api:go_default_library",
- "//core/internal/integrity:go_default_library",
- "//core/pkg/tpm:go_default_library",
- ],
-)
diff --git a/core/internal/integrity/tpm2/tpm2.go b/core/internal/integrity/tpm2/tpm2.go
deleted file mode 100644
index 8497562..0000000
--- a/core/internal/integrity/tpm2/tpm2.go
+++ /dev/null
@@ -1,155 +0,0 @@
-// Copyright 2020 The Monogon Project Authors.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package tpm2
-
-import (
- "context"
- "errors"
- "fmt"
-
- "git.monogon.dev/source/nexantic.git/core/generated/api"
- "git.monogon.dev/source/nexantic.git/core/internal/integrity"
- "git.monogon.dev/source/nexantic.git/core/pkg/tpm"
-)
-
-type TPM2Agent struct {
-}
-
-func (a *TPM2Agent) Initialize(newNode api.NewNodeInfo, enrolment api.EnrolmentConfig) error {
- nmsConn, err := integrity.DialNMS(enrolment)
- nmsClient := api.NewNodeManagementServiceClient(nmsConn)
- ekPub, ekCert, err := tpm.GetEKPublic()
- if err != nil {
- return fmt.Errorf("failed to generate EK: %w", err)
- }
-
- akPub, err := tpm.GetAKPublic()
- if err != nil {
- return fmt.Errorf("failed to generate AK: %w", err)
- }
-
- registerSession, err := nmsClient.NewTPM2NodeRegister(context.Background())
- if err != nil {
- return fmt.Errorf("failed to open registration session: %w", err)
- }
- defer registerSession.CloseSend()
- if err := registerSession.Send(&api.TPM2FlowRequest{
- Stage: &api.TPM2FlowRequest_Register{
- Register: &api.TPM2RegisterRequest{
- AkPublic: akPub,
- EkPubkey: ekPub,
- EkCert: ekCert,
- },
- },
- }); err != nil {
- return fmt.Errorf("failed to send registration: %w", err)
- }
-
- res1, err := registerSession.Recv()
- if err != nil {
- return fmt.Errorf("failed to receive attest request: %w", err)
- }
- attestReqContainer, ok := res1.Stage.(*api.TPM2FlowResponse_AttestRequest)
- if !ok {
- return errors.New("protocol violation: after RegisterRequest expected AttestRequest")
- }
- attestReq := attestReqContainer.AttestRequest
- solution, err := tpm.SolveAKChallenge(attestReq.AkChallenge, attestReq.AkChallengeSecret)
- if err != nil {
- return fmt.Errorf("failed to solve AK challenge: %w", err)
- }
- pcrs, err := tpm.GetPCRs()
- if err != nil {
- return fmt.Errorf("failed to get SRTM PCRs: %w", err)
- }
- quote, quoteSig, err := tpm.AttestPlatform(attestReq.QuoteNonce)
- if err != nil {
- return fmt.Errorf("failed Quote operation: %w", err)
- }
- if err := registerSession.Send(&api.TPM2FlowRequest{
- Stage: &api.TPM2FlowRequest_AttestResponse{
- AttestResponse: &api.TPM2AttestResponse{
- AkChallengeSolution: solution,
- Pcrs: pcrs,
- Quote: quote,
- QuoteSignature: quoteSig,
- },
- },
- }); err != nil {
- return fmt.Errorf("failed to send AttestResponse: %w", err)
- }
- if err := registerSession.Send(&api.TPM2FlowRequest{
- Stage: &api.TPM2FlowRequest_NewNodeInfo{
- NewNodeInfo: &newNode,
- },
- }); err != nil {
- return fmt.Errorf("failed to send NewNodeInfo: %w", err)
- }
- return nil
-}
-
-// Unlock attests the node state to the remote NMS and asks it for the global unlock key
-func (a *TPM2Agent) Unlock(enrolment api.EnrolmentConfig) ([]byte, error) {
- nmsConn, err := integrity.DialNMS(enrolment)
- if err != nil {
- return []byte{}, err
- }
- nmsClient := api.NewNodeManagementServiceClient(nmsConn)
- unlockClient, err := nmsClient.TPM2Unlock(context.Background())
- if err != nil {
- return []byte{}, err
- }
- defer unlockClient.CloseSend()
- unlockInitContainer, err := unlockClient.Recv()
- if err != nil {
- return []byte{}, err
- }
- unlockInitVariant, ok := unlockInitContainer.Stage.(*api.TPM2UnlockFlowResponse_UnlockInit)
- if !ok {
- return []byte{}, errors.New("TPM2Unlock protocol violation")
- }
- unlockInit := unlockInitVariant.UnlockInit
- quote, sig, err := tpm.AttestPlatform(unlockInit.Nonce)
- if err != nil {
- return []byte{}, fmt.Errorf("failed to attest platform: %w", err)
- }
- pcrs, err := tpm.GetPCRs()
- if err != nil {
- return []byte{}, fmt.Errorf("failed to get PCRs from TPM: %w", err)
- }
- if err := unlockClient.Send(&api.TPM2UnlockFlowRequeset{Stage: &api.TPM2UnlockFlowRequeset_UnlockRequest{
- UnlockRequest: &api.TPM2UnlockRequest{
- Pcrs: pcrs,
- Quote: quote,
- QuoteSignature: sig,
- NodeId: enrolment.NodeId,
- },
- }}); err != nil {
- return []byte{}, err
- }
- unlockResponseContainer, err := unlockClient.Recv()
- if err != nil {
- return []byte{}, err
- }
- unlockResponseVariant, ok := unlockResponseContainer.Stage.(*api.TPM2UnlockFlowResponse_UnlockResponse)
- if !ok {
- return []byte{}, errors.New("violated TPM2Unlock protocol")
- }
- unlockResponse := unlockResponseVariant.UnlockResponse
-
- return unlockResponse.GlobalUnlockKey, nil
-}
diff --git a/core/internal/node/BUILD.bazel b/core/internal/node/BUILD.bazel
deleted file mode 100644
index 48afed0..0000000
--- a/core/internal/node/BUILD.bazel
+++ /dev/null
@@ -1,31 +0,0 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
-
-go_library(
- name = "go_default_library",
- srcs = [
- "debug.go",
- "main.go",
- "setup.go",
- ],
- importpath = "git.monogon.dev/source/nexantic.git/core/internal/node",
- visibility = ["//:__subpackages__"],
- deps = [
- "//core/api/api:go_default_library",
- "//core/internal/api:go_default_library",
- "//core/internal/common:go_default_library",
- "//core/internal/consensus:go_default_library",
- "//core/internal/containerd:go_default_library",
- "//core/internal/integrity/tpm2:go_default_library",
- "//core/internal/kubernetes:go_default_library",
- "//core/internal/network:go_default_library",
- "//core/internal/storage:go_default_library",
- "@com_github_cenkalti_backoff_v4//:go_default_library",
- "@com_github_gogo_protobuf//proto:go_default_library",
- "@org_golang_google_grpc//:go_default_library",
- "@org_golang_google_grpc//codes:go_default_library",
- "@org_golang_google_grpc//credentials:go_default_library",
- "@org_golang_google_grpc//status:go_default_library",
- "@org_golang_x_sys//unix:go_default_library",
- "@org_uber_go_zap//:go_default_library",
- ],
-)
diff --git a/core/internal/node/debug.go b/core/internal/node/debug.go
deleted file mode 100644
index 2ed3896..0000000
--- a/core/internal/node/debug.go
+++ /dev/null
@@ -1,92 +0,0 @@
-// Copyright 2020 The Monogon Project Authors.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package node
-
-// Implements a debug gRPC service for testing and introspection
-// This is attached to the SmalltownNode because most other services are instantiated there and thus are accessible
-// from there. Have a look at //core/cmd/dbg if you need to interact with this from a CLI.
-
-import (
- "context"
- "math"
-
- "google.golang.org/grpc/codes"
- "google.golang.org/grpc/status"
-
- schema "git.monogon.dev/source/nexantic.git/core/generated/api"
- "git.monogon.dev/source/nexantic.git/core/internal/storage"
-)
-
-func (s *SmalltownNode) GetDebugKubeconfig(ctx context.Context, req *schema.GetDebugKubeconfigRequest) (*schema.GetDebugKubeconfigResponse, error) {
- return s.Kubernetes.GetDebugKubeconfig(ctx, req)
-}
-
-// GetComponentLogs gets various logbuffers from binaries we call. This function just deals with the first path component,
-// delegating the rest to the service-specific handlers.
-func (s *SmalltownNode) GetComponentLogs(ctx context.Context, req *schema.GetComponentLogsRequest) (*schema.GetComponentLogsResponse, error) {
- if len(req.ComponentPath) < 1 {
- return nil, status.Error(codes.InvalidArgument, "component_path needs to contain at least one part")
- }
- linesToRead := int(req.TailLines)
- if linesToRead == 0 {
- linesToRead = math.MaxInt32
- }
- var lines []string
- var err error
- switch req.ComponentPath[0] {
- case "containerd":
- if len(req.ComponentPath) < 2 {
- lines = s.Containerd.Log.ReadLinesTruncated(linesToRead, "...")
- } else if req.ComponentPath[1] == "runsc" {
- lines = s.Containerd.RunscLog.ReadLinesTruncated(linesToRead, "...")
- }
- case "kube":
- if len(req.ComponentPath) < 2 {
- return nil, status.Error(codes.NotFound, "Component not found")
- }
- lines, err = s.Kubernetes.GetComponentLogs(req.ComponentPath[1], linesToRead)
- if err != nil {
- return nil, status.Error(codes.NotFound, "Component not found")
- }
- default:
- return nil, status.Error(codes.NotFound, "component not found")
- }
- return &schema.GetComponentLogsResponse{Line: lines}, nil
-}
-
-// GetCondition checks for various conditions exposed by different services. Mostly intended for testing. If you need
-// to make sure something is available in an E2E test, consider adding a condition here.
-func (s *SmalltownNode) GetCondition(ctx context.Context, req *schema.GetConditionRequest) (*schema.GetConditionResponse, error) {
- var ok bool
- switch req.Name {
- case "IPAssigned":
- ip, err := s.Network.GetIP(ctx, false)
- if err == nil && ip != nil {
- ok = true
- }
- case "DataAvailable":
- _, err := s.Storage.GetPathInPlace(storage.PlaceData, "test")
- if err == nil {
- ok = true
- }
- default:
- return nil, status.Errorf(codes.NotFound, "condition %v not found", req.Name)
- }
- return &schema.GetConditionResponse{
- Ok: ok,
- }, nil
-}
diff --git a/core/internal/node/main.go b/core/internal/node/main.go
deleted file mode 100644
index 8c40e9f..0000000
--- a/core/internal/node/main.go
+++ /dev/null
@@ -1,547 +0,0 @@
-// Copyright 2020 The Monogon Project Authors.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package node
-
-import (
- "bytes"
- "context"
- "crypto/ed25519"
- "crypto/rand"
- "crypto/sha512"
- "crypto/tls"
- "crypto/x509"
- "crypto/x509/pkix"
- "errors"
- "flag"
- "fmt"
- "io/ioutil"
- "math/big"
- "net"
- "os"
- "strings"
- "time"
-
- "github.com/cenkalti/backoff/v4"
- "github.com/gogo/protobuf/proto"
- "go.uber.org/zap"
- "golang.org/x/sys/unix"
- "google.golang.org/grpc"
- "google.golang.org/grpc/credentials"
-
- apipb "git.monogon.dev/source/nexantic.git/core/generated/api"
- "git.monogon.dev/source/nexantic.git/core/internal/api"
- "git.monogon.dev/source/nexantic.git/core/internal/common"
- "git.monogon.dev/source/nexantic.git/core/internal/consensus"
- "git.monogon.dev/source/nexantic.git/core/internal/containerd"
- "git.monogon.dev/source/nexantic.git/core/internal/integrity/tpm2"
- "git.monogon.dev/source/nexantic.git/core/internal/kubernetes"
- "git.monogon.dev/source/nexantic.git/core/internal/network"
- "git.monogon.dev/source/nexantic.git/core/internal/storage"
-)
-
-var (
- // From RFC 5280 Section 4.1.2.5
- unknownNotAfter = time.Unix(253402300799, 0)
-)
-
-type (
- SmalltownNode struct {
- Api *api.Server
- Consensus *consensus.Service
- Storage *storage.Manager
- Kubernetes *kubernetes.Service
- Containerd *containerd.Service
- Network *network.Service
-
- logger *zap.Logger
- state common.SmalltownState
- hostname string
- enrolmentConfig *apipb.EnrolmentConfig
-
- debugServer *grpc.Server
- }
-)
-
-func NewSmalltownNode(logger *zap.Logger, ntwk *network.Service, strg *storage.Manager) (*SmalltownNode, error) {
- flag.Parse()
- logger.Info("Creating Smalltown node")
- ctx := context.Background()
-
- hostname, err := os.Hostname()
- if err != nil {
- panic(err)
- }
-
- // Wait for IP adddress...
- ctxT, ctxTC := context.WithTimeout(ctx, time.Second*10)
- defer ctxTC()
- externalIP, err := ntwk.GetIP(ctxT, true)
- if err != nil {
- logger.Panic("Could not get IP address", zap.Error(err))
- }
-
- // Important to know if the GetIP above hangs
- logger.Info("Node has IP", zap.String("ip", externalIP.String()))
-
- consensusService, err := consensus.NewConsensusService(consensus.Config{
- Name: hostname,
- ListenHost: "0.0.0.0",
- ExternalHost: externalIP.String(),
- }, logger.With(zap.String("module", "consensus")))
- if err != nil {
- return nil, err
- }
-
- containerdService, err := containerd.New()
- if err != nil {
- return nil, err
- }
-
- s := &SmalltownNode{
- Consensus: consensusService,
- Containerd: containerdService,
- Storage: strg,
- Network: ntwk,
- logger: logger,
- hostname: hostname,
- }
-
- apiService, err := api.NewApiServer(&api.Config{}, logger.With(zap.String("module", "api")), s.Consensus)
- if err != nil {
- return nil, err
- }
-
- s.Api = apiService
-
- s.Kubernetes = kubernetes.New(logger.With(zap.String("module", "kubernetes")), consensusService, strg)
-
- s.debugServer = grpc.NewServer()
- apipb.RegisterNodeDebugServiceServer(s.debugServer, s)
-
- logger.Info("Created SmalltownNode")
-
- return s, nil
-}
-
-func (s *SmalltownNode) Start(ctx context.Context) error {
- s.logger.Info("Starting Smalltown node")
-
- s.startDebugSvc()
-
- // TODO(lorenz): Abstracting enrolment sounds like a good idea, but ends up being painful
- // because of things like storage access. I'm keeping it this way until the more complex
- // enrolment procedures are fleshed out. This is also a bit panic()-happy, but there is really
- // no good way out of an invalid enrolment configuration.
- enrolmentPath, err := s.Storage.GetPathInPlace(storage.PlaceESP, "enrolment.pb")
- if err != nil {
- s.logger.Panic("ESP configuration partition not available", zap.Error(err))
- }
- enrolmentConfigRaw, err := ioutil.ReadFile(enrolmentPath)
- if os.IsNotExist(err) {
- enrolmentConfigRaw, err = ioutil.ReadFile("/sys/firmware/qemu_fw_cfg/by_name/com.nexantic.smalltown/enrolment.pb/raw")
- }
- if err == nil {
- // We have an enrolment file, let's check its contents
- var enrolmentConfig apipb.EnrolmentConfig
- if err := proto.Unmarshal(enrolmentConfigRaw, &enrolmentConfig); err != nil {
- s.logger.Panic("Invalid enrolment configuration provided", zap.Error(err))
- }
- s.enrolmentConfig = &enrolmentConfig
- // The enrolment secret is only zeroed after
- if len(enrolmentConfig.EnrolmentSecret) == 0 {
- return s.startFull()
- }
- return s.startEnrolling(ctx)
- } else if os.IsNotExist(err) {
- // This is ok like this, once a new cluster has been set up the initial node also generates
- // its own enrolment config
- return s.startForSetup(ctx)
- }
- // Unknown error reading enrolment config (disk issues/invalid configuration format/...)
- s.logger.Panic("Invalid enrolment configuration provided", zap.Error(err))
- panic("Unreachable")
-}
-
-func (s *SmalltownNode) startDebugSvc() {
- debugListenHost := fmt.Sprintf(":%v", common.DebugServicePort)
- debugListener, err := net.Listen("tcp", debugListenHost)
- if err != nil {
- s.logger.Fatal("failed to listen", zap.Error(err))
- }
-
- go func() {
- if err := s.debugServer.Serve(debugListener); err != nil {
- s.logger.Fatal("failed to serve", zap.Error(err))
- }
- }()
-}
-
-func (s *SmalltownNode) initHostname() error {
- if err := unix.Sethostname([]byte(s.hostname)); err != nil {
- return err
- }
- if err := ioutil.WriteFile("/etc/hosts", []byte(fmt.Sprintf("%v %v", "127.0.0.1", s.hostname)), 0644); err != nil {
- return err
- }
- return ioutil.WriteFile("/etc/machine-id", []byte(strings.TrimPrefix(s.hostname, "smalltown-")), 0644)
-}
-
-func (s *SmalltownNode) startEnrolling(ctx context.Context) error {
- s.logger.Info("Initializing subsystems for enrolment")
- s.state = common.StateEnrollMode
-
- nodeInfo, nodeID, err := s.InitializeNode(ctx)
- if err != nil {
- return err
- }
-
- s.hostname = nodeID
- if err := s.initHostname(); err != nil {
- return err
- }
-
- if err := s.initNodeAPI(); err != nil {
- return err
- }
-
- // We only support TPM2 at the moment, any abstractions here would be premature
- trustAgent := tpm2.TPM2Agent{}
-
- initializeOp := func() error {
- if err := trustAgent.Initialize(*nodeInfo, *s.enrolmentConfig); err != nil {
- s.logger.Warn("Failed to initialize integrity backend", zap.Error(err))
- return err
- }
- return nil
- }
-
- if err := backoff.Retry(initializeOp, getIntegrityBackoff()); err != nil {
- panic("invariant violated: integrity initialization retry can never fail")
- }
-
- enrolmentPath, err := s.Storage.GetPathInPlace(storage.PlaceESP, "enrolment.pb")
- if err != nil {
- panic(err)
- }
-
- s.enrolmentConfig.EnrolmentSecret = []byte{}
- s.enrolmentConfig.NodeId = nodeID
-
- enrolmentConfigRaw, err := proto.Marshal(s.enrolmentConfig)
- if err != nil {
- panic(err)
- }
- if err := ioutil.WriteFile(enrolmentPath, enrolmentConfigRaw, 0600); err != nil {
- return err
- }
- s.logger.Info("Node successfully enrolled")
-
- return nil
-}
-
-func (s *SmalltownNode) startForSetup(ctx context.Context) error {
- s.logger.Info("Setting up a new cluster")
- initData, nodeID, err := s.InitializeNode(ctx)
- if err != nil {
- return err
- }
- s.hostname = nodeID
- if err := s.initHostname(); err != nil {
- return err
- }
-
- if err := s.initNodeAPI(); err != nil {
- return err
- }
-
- // TODO: Use supervisor.Run for this
- go s.Containerd.Run()(context.TODO())
-
- dataPath, err := s.Storage.GetPathInPlace(storage.PlaceData, "etcd")
- if err != nil {
- return err
- }
-
- // Spin up etcd
- config := s.Consensus.GetConfig()
- config.NewCluster = true
- config.Name = s.hostname
- config.DataDir = dataPath
- s.Consensus.SetConfig(config)
-
- // Generate the cluster CA and store it to local storage.
- extIP, err := s.Network.GetIP(ctx, true)
- if err != nil {
- return err
- }
- if err := s.Consensus.PrecreateCA(*extIP); err != nil {
- return err
- }
-
- err = s.Consensus.Start()
- if err != nil {
- return err
- }
-
- // Now that the cluster is up and running, we can persist the CA to the cluster.
- if err := s.Consensus.InjectCA(); err != nil {
- return err
- }
-
- if err := s.Api.BootstrapNewClusterHook(initData); err != nil {
- return err
- }
-
- if err := s.Kubernetes.NewCluster(); err != nil {
- return err
- }
-
- if err := s.Kubernetes.Start(); err != nil {
- return err
- }
-
- if err := s.Api.Start(); err != nil {
- s.logger.Error("Failed to start the API service", zap.Error(err))
- return err
- }
-
- enrolmentPath, err := s.Storage.GetPathInPlace(storage.PlaceESP, "enrolment.pb")
- if err != nil {
- panic(err)
- }
-
- masterCert, err := s.Api.GetMasterCert()
- if err != nil {
- return err
- }
-
- ip, err := s.Network.GetIP(ctx, true)
- if err != nil {
- return fmt.Errorf("could not get node IP: %v", err)
- }
- enrolmentConfig := &apipb.EnrolmentConfig{
- EnrolmentSecret: []byte{}, // First node is always already enrolled
- MastersCert: masterCert,
- MasterIps: [][]byte{[]byte(*ip)},
- NodeId: nodeID,
- }
- enrolmentConfigRaw, err := proto.Marshal(enrolmentConfig)
- if err != nil {
- panic(err)
- }
- if err := ioutil.WriteFile(enrolmentPath, enrolmentConfigRaw, 0600); err != nil {
- return err
- }
- masterCertFingerprint := sha512.Sum512_256(masterCert)
- s.logger.Info("New Smalltown cluster successfully bootstrapped", zap.Binary("fingerprint", masterCertFingerprint[:]))
-
- return nil
-}
-
-func (s *SmalltownNode) generateNodeID() ([]byte, string, error) {
- serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 127)
- serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
- if err != nil {
- return []byte{}, "", fmt.Errorf("Failed to generate serial number: %w", err)
- }
-
- pubKey, privKeyRaw, err := ed25519.GenerateKey(rand.Reader)
- if err != nil {
- return []byte{}, "", err
- }
- privkey, err := x509.MarshalPKCS8PrivateKey(privKeyRaw)
- if err != nil {
- return []byte{}, "", err
- }
-
- nodeKeyPath, err := s.Storage.GetPathInPlace(storage.PlaceData, "node-key.der")
- if err != nil {
- return []byte{}, "", err
- }
-
- if err := ioutil.WriteFile(nodeKeyPath, privkey, 0600); err != nil {
- return []byte{}, "", fmt.Errorf("failed to write node key: %w", err)
- }
-
- name := common.NameFromIDKey(pubKey)
-
- // This has no SANs because it authenticates by public key, not by name
- nodeCert := &x509.Certificate{
- SerialNumber: serialNumber,
- Subject: pkix.Name{
- // We identify nodes by their ID public keys (not hashed since a strong hash is longer and serves no benefit)
- CommonName: name,
- },
- IsCA: false,
- BasicConstraintsValid: true,
- NotBefore: time.Now(),
- NotAfter: unknownNotAfter,
- // Certificate is used both as server & client
- ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
- }
- cert, err := x509.CreateCertificate(rand.Reader, nodeCert, nodeCert, pubKey, privKeyRaw)
- if err != nil {
- return []byte{}, "", err
- }
-
- nodeCertPath, err := s.Storage.GetPathInPlace(storage.PlaceData, "node.der")
- if err != nil {
- return []byte{}, "", err
- }
-
- if err := ioutil.WriteFile(nodeCertPath, cert, 0600); err != nil {
- return []byte{}, "", fmt.Errorf("failed to write node cert: %w", err)
- }
- return cert, name, nil
-}
-
-func (s *SmalltownNode) initNodeAPI() error {
- certPath, err := s.Storage.GetPathInPlace(storage.PlaceData, "node.der")
- if err != nil {
- s.logger.Panic("Invariant violated: Data is available once this is called")
- }
- keyPath, err := s.Storage.GetPathInPlace(storage.PlaceData, "node-key.der")
- if err != nil {
- s.logger.Panic("Invariant violated: Data is available once this is called")
- }
-
- certRaw, err := ioutil.ReadFile(certPath)
- if err != nil {
- return err
- }
- privKeyRaw, err := ioutil.ReadFile(keyPath)
- if err != nil {
- return err
- }
-
- var nodeID tls.Certificate
-
- cert, err := x509.ParseCertificate(certRaw)
- if err != nil {
- return err
- }
-
- privKey, err := x509.ParsePKCS8PrivateKey(privKeyRaw)
- if err != nil {
- return err
- }
-
- nodeID.Certificate = [][]byte{certRaw}
- nodeID.PrivateKey = privKey
- nodeID.Leaf = cert
-
- secureTransport := &tls.Config{
- Certificates: []tls.Certificate{nodeID},
- ClientAuth: tls.RequestClientCert,
- InsecureSkipVerify: true,
- // Critical function, please review any changes with care
- // TODO(lorenz): Actively check that this actually provides the security guarantees that we need
- VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
- for _, cert := range rawCerts {
- // X.509 certificates in DER can be compared like this since DER has a unique representation
- // for each certificate.
- if bytes.Equal(cert, s.enrolmentConfig.MastersCert) {
- return nil
- }
- }
- s.logger.Warn("Rejecting NodeService connection with no trusted client certificate")
- return errors.New("failed to find authorized NMS certificate")
- },
- MinVersion: tls.VersionTLS13,
- }
- secureTransportCreds := credentials.NewTLS(secureTransport)
-
- masterListenHost := fmt.Sprintf(":%d", common.NodeServicePort)
- lis, err := net.Listen("tcp", masterListenHost)
- if err != nil {
- s.logger.Fatal("failed to listen", zap.Error(err))
- }
-
- nodeGRPCServer := grpc.NewServer(grpc.Creds(secureTransportCreds))
- apipb.RegisterNodeServiceServer(nodeGRPCServer, s)
- go func() {
- if err := nodeGRPCServer.Serve(lis); err != nil {
- panic(err) // Can only happen during initialization and is always fatal
- }
- }()
-
- return nil
-}
-
-func getIntegrityBackoff() *backoff.ExponentialBackOff {
- unlockBackoff := backoff.NewExponentialBackOff()
- unlockBackoff.MaxElapsedTime = time.Duration(0)
- unlockBackoff.InitialInterval = 5 * time.Second
- unlockBackoff.MaxInterval = 5 * time.Minute
- return unlockBackoff
-}
-
-func (s *SmalltownNode) startFull() error {
- s.logger.Info("Initializing subsystems for production")
- s.state = common.StateJoined
-
- s.hostname = s.enrolmentConfig.NodeId
- if err := s.initHostname(); err != nil {
- return err
- }
-
- trustAgent := tpm2.TPM2Agent{}
- unlockOp := func() error {
- unlockKey, err := trustAgent.Unlock(*s.enrolmentConfig)
- if err != nil {
- s.logger.Warn("Failed to unlock", zap.Error(err))
- return err
- }
- if err := s.Storage.MountData(unlockKey); err != nil {
- s.logger.Panic("Failed to mount storage", zap.Error(err))
- return err
- }
- return nil
- }
-
- if err := backoff.Retry(unlockOp, getIntegrityBackoff()); err != nil {
- s.logger.Panic("Invariant violated: Unlock retry can never fail")
- }
-
- s.initNodeAPI()
-
- // TODO: Use supervisor.Run for this
- go s.Containerd.Run()(context.TODO())
-
- err := s.Consensus.Start()
- if err != nil {
- return err
- }
-
- err = s.Api.Start()
- if err != nil {
- s.logger.Error("Failed to start the API service", zap.Error(err))
- return err
- }
-
- err = s.Kubernetes.Start()
- if err != nil {
- s.logger.Error("Failed to start the Kubernetes Service", zap.Error(err))
- }
-
- return nil
-}
-
-func (s *SmalltownNode) Stop() error {
- s.logger.Info("Stopping Smalltown node")
- return nil
-}
diff --git a/core/internal/node/setup.go b/core/internal/node/setup.go
deleted file mode 100644
index a9e841c..0000000
--- a/core/internal/node/setup.go
+++ /dev/null
@@ -1,110 +0,0 @@
-// Copyright 2020 The Monogon Project Authors.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package node
-
-import (
- "context"
- "fmt"
- "os"
-
- "git.monogon.dev/source/nexantic.git/core/internal/storage"
-
- "google.golang.org/grpc/codes"
- "google.golang.org/grpc/status"
-
- "git.monogon.dev/source/nexantic.git/core/generated/api"
- "git.monogon.dev/source/nexantic.git/core/internal/common"
-)
-
-var (
- ErrConsensusAlreadyProvisioned = status.Error(codes.FailedPrecondition, "consensus is already provisioned; make sure the data folder is empty")
- ErrAlreadySetup = status.Error(codes.FailedPrecondition, "node is already set up")
- ErrNotInJoinMode = status.Error(codes.FailedPrecondition, "node is not in the cluster join mode")
- ErrTrustNotInitialized = status.Error(codes.FailedPrecondition, "trust backend not initialized")
- ErrStorageNotInitialized = status.Error(codes.FailedPrecondition, "storage not initialized")
-)
-
-func (s *SmalltownNode) CurrentState() common.SmalltownState {
- return s.state
-}
-
-// InitializeNode contains functionality that needs to be executed regardless of what the node does
-// later on
-func (s *SmalltownNode) InitializeNode(ctx context.Context) (*api.NewNodeInfo, string, error) {
- globalUnlockKey, err := s.Storage.InitializeData()
- if err != nil {
- return nil, "", err
- }
-
- nodeIP, err := s.Network.GetIP(ctx, true)
- if err != nil {
- return nil, "", fmt.Errorf("could not get IP: %v", err)
- }
-
- nodeCert, nodeID, err := s.generateNodeID()
- if err != nil {
- return nil, "", err
- }
-
- return &api.NewNodeInfo{
- EnrolmentConfig: s.enrolmentConfig,
- Ip: *nodeIP,
- IdCert: nodeCert,
- GlobalUnlockKey: globalUnlockKey,
- }, nodeID, nil
-}
-
-func (s *SmalltownNode) JoinCluster(ctx context.Context, req *api.JoinClusterRequest) (*api.JoinClusterResponse, error) {
- if s.state != common.StateEnrollMode {
- return nil, ErrNotInJoinMode
- }
-
- s.logger.Info("Joining Consenus")
-
- dataPath, err := s.Storage.GetPathInPlace(storage.PlaceData, "etcd")
- if err != nil {
- return nil, status.Errorf(codes.Unavailable, "Data partition not available: %v", err)
- }
-
- if err := os.MkdirAll(dataPath, 0600); err != nil {
- return nil, status.Errorf(codes.Internal, "Cannot create path on data partition: %v", err)
- }
-
- config := s.Consensus.GetConfig()
- config.Name = s.hostname
- config.InitialCluster = req.InitialCluster
- config.DataDir = dataPath
- s.Consensus.SetConfig(config)
-
- if err := s.Consensus.WriteCertificateFiles(req.Certs); err != nil {
- return nil, err
- }
-
- // Start consensus
- err = s.Consensus.Start()
- if err != nil {
- return nil, err
- }
-
- s.state = common.StateJoined
- go s.Containerd.Run()(context.TODO())
- s.Kubernetes.Start()
-
- s.logger.Info("Joined cluster. Node is now syncing.")
-
- return &api.JoinClusterResponse{}, nil
-}