Add nanoswitch and cluster testing
Adds nanoswitch and the `switched-multi2` launch target to launch two Smalltown instances on a switched
network and enroll them into a single cluster. Nanoswitch contains a Linux bridge and a minimal DHCP server
and connects to the two Smalltown instances over virtual Ethernet cables. Also moves out the DHCP client into
a package since nanoswitch needs it.
Test Plan:
Manually tested using `bazel run //:launch -- switched-multi2` and observing that the second VM
(whose serial port is mapped to stdout) prints that it is enrolled. Also validated by `bazel run //core/cmd/dbg -- kubectl get node -o wide` returning two ready nodes.
X-Origin-Diff: phab/D572
GitOrigin-RevId: 9f6e2b3d8268749dd81588205646ae3976ad14b3
diff --git a/core/internal/node/main.go b/core/internal/node/main.go
index 2cf88f4..8c40e9f 100644
--- a/core/internal/node/main.go
+++ b/core/internal/node/main.go
@@ -25,7 +25,6 @@
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
- "encoding/hex"
"errors"
"flag"
"fmt"
@@ -215,6 +214,10 @@
return err
}
+ if err := s.initNodeAPI(); err != nil {
+ return err
+ }
+
// We only support TPM2 at the moment, any abstractions here would be premature
trustAgent := tpm2.TPM2Agent{}
@@ -281,7 +284,11 @@
s.Consensus.SetConfig(config)
// Generate the cluster CA and store it to local storage.
- if err := s.Consensus.PrecreateCA(); err != nil {
+ extIP, err := s.Network.GetIP(ctx, true)
+ if err != nil {
+ return err
+ }
+ if err := s.Consensus.PrecreateCA(*extIP); err != nil {
return err
}
@@ -370,7 +377,7 @@
return []byte{}, "", fmt.Errorf("failed to write node key: %w", err)
}
- name := "smalltown-" + hex.EncodeToString([]byte(pubKey[:16]))
+ name := common.NameFromIDKey(pubKey)
// This has no SANs because it authenticates by public key, not by name
nodeCert := &x509.Certificate{
@@ -439,7 +446,7 @@
secureTransport := &tls.Config{
Certificates: []tls.Certificate{nodeID},
- ClientAuth: tls.RequireAndVerifyClientCert,
+ ClientAuth: tls.RequestClientCert,
InsecureSkipVerify: true,
// Critical function, please review any changes with care
// TODO(lorenz): Actively check that this actually provides the security guarantees that we need
@@ -451,6 +458,7 @@
return nil
}
}
+ s.logger.Warn("Rejecting NodeService connection with no trusted client certificate")
return errors.New("failed to find authorized NMS certificate")
},
MinVersion: tls.VersionTLS13,
@@ -470,6 +478,7 @@
panic(err) // Can only happen during initialization and is always fatal
}
}()
+
return nil
}
diff --git a/core/internal/node/setup.go b/core/internal/node/setup.go
index cbbfd4d..a9e841c 100644
--- a/core/internal/node/setup.go
+++ b/core/internal/node/setup.go
@@ -18,10 +18,11 @@
import (
"context"
- "errors"
"fmt"
+ "os"
- "go.uber.org/zap"
+ "git.monogon.dev/source/nexantic.git/core/internal/storage"
+
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
@@ -61,28 +62,34 @@
return &api.NewNodeInfo{
EnrolmentConfig: s.enrolmentConfig,
- Ip: []byte(*nodeIP),
+ Ip: *nodeIP,
IdCert: nodeCert,
GlobalUnlockKey: globalUnlockKey,
}, nodeID, nil
}
-func (s *SmalltownNode) JoinCluster(context context.Context, req *api.JoinClusterRequest) (*api.JoinClusterResponse, error) {
+func (s *SmalltownNode) JoinCluster(ctx context.Context, req *api.JoinClusterRequest) (*api.JoinClusterResponse, error) {
if s.state != common.StateEnrollMode {
return nil, ErrNotInJoinMode
}
s.logger.Info("Joining Consenus")
+ dataPath, err := s.Storage.GetPathInPlace(storage.PlaceData, "etcd")
+ if err != nil {
+ return nil, status.Errorf(codes.Unavailable, "Data partition not available: %v", err)
+ }
+
+ if err := os.MkdirAll(dataPath, 0600); err != nil {
+ return nil, status.Errorf(codes.Internal, "Cannot create path on data partition: %v", err)
+ }
+
config := s.Consensus.GetConfig()
config.Name = s.hostname
- config.InitialCluster = "default" // Clusters can't cross-join anyways due to cryptography
+ config.InitialCluster = req.InitialCluster
+ config.DataDir = dataPath
s.Consensus.SetConfig(config)
- var err error
- if err != nil {
- s.logger.Warn("Invalid JoinCluster request", zap.Error(err))
- return nil, errors.New("invalid join request")
- }
+
if err := s.Consensus.WriteCertificateFiles(req.Certs); err != nil {
return nil, err
}
@@ -94,6 +101,8 @@
}
s.state = common.StateJoined
+ go s.Containerd.Run()(context.TODO())
+ s.Kubernetes.Start()
s.logger.Info("Joined cluster. Node is now syncing.")