Added bootstrap CA
This adds a self-contained CA for bootstrapping and securing etcd
using certificates of infinite duration and a CRL for near-instant
revocation.
The bootstrapping problem is addressed by first
generating the CA and issuing initial certificates and then
injecting them once the consensus system is up and running.
All files are also kept on the encrypted persistent data store to
prevent the same bootstrapping problem when the node is already
initialized. The CRL is synchronized using a sync loop on every
node running the consensus service and distributed inside that.
The CA uses Ed25519-based cryptography and identifies the
hosts by their external hostname.
Test Plan:
Initial bootstrapping manually tested on a single node using a
manual gRPC call for Setup() and openssl s_client for connecting
to etcd.
X-Origin-Diff: phab/D233
GitOrigin-RevId: bd67818b5b649b13e0c098e480059ef990826542
diff --git a/core/internal/consensus/consensus.go b/core/internal/consensus/consensus.go
index e1f59d6..d87a506 100644
--- a/core/internal/consensus/consensus.go
+++ b/core/internal/consensus/consensus.go
@@ -17,9 +17,24 @@
package consensus
import (
+ "bytes"
"context"
+ "crypto/x509"
+ "encoding/hex"
+ "encoding/pem"
"fmt"
+ "io/ioutil"
+ "math/rand"
+ "net/url"
+ "os"
+ "path"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "git.monogon.dev/source/nexantic.git/core/generated/api"
"git.monogon.dev/source/nexantic.git/core/internal/common"
+ "git.monogon.dev/source/nexantic.git/core/internal/consensus/ca"
"github.com/pkg/errors"
"go.etcd.io/etcd/clientv3"
"go.etcd.io/etcd/clientv3/namespace"
@@ -28,9 +43,7 @@
"go.etcd.io/etcd/pkg/types"
"go.etcd.io/etcd/proxy/grpcproxy/adapter"
"go.uber.org/zap"
- "net/url"
- "os"
- "strings"
+ "golang.org/x/sys/unix"
)
const (
@@ -38,13 +51,25 @@
DefaultLogger = "zap"
)
+const (
+ CAPath = "ca.pem"
+ CertPath = "cert.pem"
+ KeyPath = "cert-key.pem"
+ CRLPath = "ca-crl.der"
+ CRLSwapPath = "ca-crl.der.swp"
+)
+
type (
Service struct {
*common.BaseService
- etcd *embed.Etcd
- kv clientv3.KV
- ready bool
+ etcd *embed.Etcd
+ kv clientv3.KV
+ ready bool
+ bootstrapCA *ca.CA
+ bootstrapCert []byte
+ watchCRLTicker *time.Ticker
+ lastCRL []byte
config *Config
}
@@ -84,15 +109,27 @@
cfg := embed.NewConfig()
+ cfg.PeerTLSInfo.CertFile = filepath.Join(s.config.DataDir, CertPath)
+ cfg.PeerTLSInfo.KeyFile = filepath.Join(s.config.DataDir, KeyPath)
+ cfg.PeerTLSInfo.TrustedCAFile = filepath.Join(s.config.DataDir, CAPath)
+ cfg.PeerTLSInfo.ClientCertAuth = true
+ cfg.PeerTLSInfo.CRLFile = filepath.Join(s.config.DataDir, CRLPath)
+
+ lastCRL, err := ioutil.ReadFile(cfg.PeerTLSInfo.CRLFile)
+ if err != nil {
+ return fmt.Errorf("failed to read etcd CRL: %w", err)
+ }
+ s.lastCRL = lastCRL
+
// Reset LCUrls because we don't want to expose any client
cfg.LCUrls = nil
- apURL, err := url.Parse(fmt.Sprintf("http://%s:%d", s.config.ExternalHost, s.config.ListenPort))
+ apURL, err := url.Parse(fmt.Sprintf("https://%s:%d", s.config.ExternalHost, s.config.ListenPort))
if err != nil {
return errors.Wrap(err, "invalid external_host or listen_port")
}
- lpURL, err := url.Parse(fmt.Sprintf("http://%s:%d", s.config.ListenHost, s.config.ListenPort))
+ lpURL, err := url.Parse(fmt.Sprintf("https://%s:%d", s.config.ListenHost, s.config.ListenPort))
if err != nil {
return errors.Wrap(err, "invalid listen_host or listen_port")
}
@@ -134,10 +171,213 @@
// Inject kv client
s.kv = clientv3.NewKVFromKVClient(adapter.KvServerToKvClient(s.etcd.Server), nil)
+ // Start CRL watcher
+ go s.watchCRL()
+
return nil
}
+func (s *Service) SetupCertificates(certs *api.ConsensusCertificates) error {
+ if err := ioutil.WriteFile(filepath.Join(s.config.DataDir, CRLPath), certs.Crl, 0600); err != nil {
+ return err
+ }
+ if err := ioutil.WriteFile(filepath.Join(s.config.DataDir, CertPath),
+ pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certs.Cert}), 0600); err != nil {
+ return err
+ }
+ if err := ioutil.WriteFile(filepath.Join(s.config.DataDir, KeyPath),
+ pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: certs.Key}), 0600); err != nil {
+ return err
+ }
+ if err := ioutil.WriteFile(filepath.Join(s.config.DataDir, CAPath),
+ pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certs.Ca}), 0600); err != nil {
+ return err
+ }
+ return nil
+}
+
+func (s *Service) PrecreateCA() error {
+ // Provision an etcd CA
+ etcdRootCA, err := ca.New("Smalltown etcd Root CA")
+ if err != nil {
+ return err
+ }
+ cert, privkey, err := etcdRootCA.IssueCertificate(s.config.ExternalHost)
+ if err != nil {
+ return fmt.Errorf("failed to self-issue a certificate: %w", err)
+ }
+ if err := os.MkdirAll(s.config.DataDir, 0700); err != nil {
+ return fmt.Errorf("failed to create consensus data dir: %w", err)
+ }
+ // Preserve certificate for later injection
+ s.bootstrapCert = cert
+ if err := s.SetupCertificates(&api.ConsensusCertificates{
+ Ca: etcdRootCA.CACertRaw,
+ Crl: etcdRootCA.CRLRaw,
+ Cert: cert,
+ Key: privkey,
+ }); err != nil {
+ return fmt.Errorf("failed to setup certificates: %w", err)
+ }
+ s.bootstrapCA = etcdRootCA
+ return nil
+}
+
+const (
+ caPathEtcd = "/etcd-ca/ca.der"
+ caKeyPathEtcd = "/etcd-ca/ca-key.der"
+ crlPathEtcd = "/etcd-ca/crl.der"
+ certPrefixEtcd = "/etcd-ca/certs"
+)
+
+func (s *Service) InjectCA() error {
+ if _, err := s.kv.Put(context.Background(), caPathEtcd, string(s.bootstrapCA.CACertRaw)); err != nil {
+ return err
+ }
+ // TODO: Should be wrapped by the master key
+ if _, err := s.kv.Put(context.Background(), caKeyPathEtcd, string([]byte(*s.bootstrapCA.PrivateKey))); err != nil {
+ return err
+ }
+ if _, err := s.kv.Put(context.Background(), crlPathEtcd, string(s.bootstrapCA.CRLRaw)); err != nil {
+ return err
+ }
+ certVal, err := x509.ParseCertificate(s.bootstrapCert)
+ if err != nil {
+ return err
+ }
+ serial := hex.EncodeToString(certVal.SerialNumber.Bytes())
+ if _, err := s.kv.Put(context.Background(), path.Join(certPrefixEtcd, serial), string(s.bootstrapCert)); err != nil {
+ return fmt.Errorf("failed to persist certificate: %w", err)
+ }
+ // Clear out bootstrap CA after injecting
+ s.bootstrapCA = nil
+ s.bootstrapCert = []byte{}
+ return nil
+}
+
+func (s *Service) etcdGetSingle(path string) ([]byte, int64, error) {
+ res, err := s.kv.Get(context.Background(), path)
+ if err != nil {
+ return nil, -1, fmt.Errorf("failed to get key from etcd: %w", err)
+ }
+ if len(res.Kvs) != 1 {
+ return nil, -1, errors.New("key not available")
+ }
+ return res.Kvs[0].Value, res.Kvs[0].ModRevision, nil
+}
+
+func (s *Service) takeCAOnline() (*ca.CA, int64, error) {
+ // TODO: Technically this could be done in a single request, but it's more logic
+ caCert, _, err := s.etcdGetSingle(caPathEtcd)
+ if err != nil {
+ return nil, -1, fmt.Errorf("failed to get CA certificate from etcd: %w", err)
+ }
+ caKey, _, err := s.etcdGetSingle(caKeyPathEtcd)
+ if err != nil {
+ return nil, -1, fmt.Errorf("failed to get CA key from etcd: %w", err)
+ }
+ // TODO: Unwrap CA key once wrapping is implemented
+ crl, crlRevision, err := s.etcdGetSingle(crlPathEtcd)
+ if err != nil {
+ return nil, -1, fmt.Errorf("failed to get CRL from etcd: %w", err)
+ }
+ idCA, err := ca.FromCertificates(caCert, caKey, crl)
+ if err != nil {
+ return nil, -1, fmt.Errorf("failed to take CA online: %w", err)
+ }
+ return idCA, crlRevision, nil
+}
+
+func (s *Service) IssueCertificate(hostname string) (*api.ConsensusCertificates, error) {
+ idCA, _, err := s.takeCAOnline()
+ if err != nil {
+ return nil, err
+ }
+ cert, key, err := idCA.IssueCertificate(hostname)
+ if err != nil {
+ return nil, fmt.Errorf("failed to issue certificate: %w", err)
+ }
+ certVal, err := x509.ParseCertificate(cert)
+ if err != nil {
+ return nil, err
+ }
+ serial := hex.EncodeToString(certVal.SerialNumber.Bytes())
+ if _, err := s.kv.Put(context.Background(), path.Join(certPrefixEtcd, serial), string(cert)); err != nil {
+ return nil, fmt.Errorf("failed to persist certificate: %w", err)
+ }
+ return &api.ConsensusCertificates{
+ Ca: idCA.CACertRaw,
+ Cert: cert,
+ Crl: idCA.CRLRaw,
+ Key: key,
+ }, nil
+}
+
+func (s *Service) RevokeCertificate(hostname string) error {
+ rand.Seed(time.Now().UnixNano())
+ for {
+ idCA, crlRevision, err := s.takeCAOnline()
+ if err != nil {
+ return err
+ }
+ allIssuedCerts, err := s.kv.Get(context.Background(), certPrefixEtcd, clientv3.WithPrefix())
+ for _, cert := range allIssuedCerts.Kvs {
+ certVal, err := x509.ParseCertificate(cert.Value)
+ if err != nil {
+ s.Logger.Error("Failed to parse previously issued certificate, this is a security risk", zap.Error(err))
+ continue
+ }
+ for _, dnsName := range certVal.DNSNames {
+ if dnsName == hostname {
+ // Revoke this
+ if err := idCA.Revoke(certVal.SerialNumber); err != nil {
+ // We need to fail if any single revocation fails otherwise outer applications
+ // have no chance of calling this safely
+ return err
+ }
+ }
+ }
+ }
+ cmp := clientv3.Compare(clientv3.ModRevision(crlPathEtcd), "=", crlRevision)
+ op := clientv3.OpPut(crlPathEtcd, string(idCA.CRLRaw))
+ res, err := s.kv.Txn(context.Background()).If(cmp).Then(op).Commit()
+ if err != nil {
+ return fmt.Errorf("failed to persist new CRL in etcd: %w", err)
+ }
+ if res.Succeeded { // Transaction has succeeded
+ break
+ }
+ // Sleep a random duration between 0 and 300ms to reduce serialization failures
+ time.Sleep(time.Duration(rand.Intn(300)) * time.Millisecond)
+ }
+ return nil
+}
+
+func (s *Service) watchCRL() {
+ // TODO: Change etcd client to WatchableKV and make this an actual watch
+ // This needs changes in more places, so leaving it now
+ s.watchCRLTicker = time.NewTicker(30 * time.Second)
+ for range s.watchCRLTicker.C {
+ crl, _, err := s.etcdGetSingle(crlPathEtcd)
+ if err != nil {
+ s.Logger.Warn("Failed to check for new CRL", zap.Error(err))
+ continue
+ }
+ // This is cryptographic material but not secret, so no constant time compare necessary here
+ if !bytes.Equal(crl, s.lastCRL) {
+ if err := ioutil.WriteFile(filepath.Join(s.config.DataDir, CRLSwapPath), crl, 0600); err != nil {
+ s.Logger.Warn("Failed to write updated CRL", zap.Error(err))
+ }
+ // This uses unix.Rename to guarantee a particular atomic update behavior
+ if err := unix.Rename(filepath.Join(s.config.DataDir, CRLSwapPath), filepath.Join(s.config.DataDir, CRLPath)); err != nil {
+ s.Logger.Warn("Failed to atomically swap updated CRL", zap.Error(err))
+ }
+ }
+ }
+}
+
func (s *Service) OnStop() error {
+ s.watchCRLTicker.Stop()
s.etcd.Close()
return nil