Added bootstrap CA
This adds a self-contained CA for bootstrapping and securing etcd
using certificates of infinite duration and a CRL for near-instant
revocation.
The bootstrapping problem is addressed by first
generating the CA and issuing initial certificates and then
injecting them once the consensus system is up and running.
All files are also kept on the encrypted persistent data store to
prevent the same bootstrapping problem when the node is already
initialized. The CRL is synchronized using a sync loop on every
node running the consensus service and distributed inside that.
The CA uses Ed25519-based cryptography and identifies the
hosts by their external hostname.
Test Plan:
Initial bootstrapping manually tested on a single node using a
manual gRPC call for Setup() and openssl s_client for connecting
to etcd.
X-Origin-Diff: phab/D233
GitOrigin-RevId: bd67818b5b649b13e0c098e480059ef990826542
diff --git a/core/api/api/schema.proto b/core/api/api/schema.proto
index d6721a0..238cfce 100644
--- a/core/api/api/schema.proto
+++ b/core/api/api/schema.proto
@@ -86,6 +86,13 @@
string provisioningToken = 1;
}
+message ConsensusCertificates {
+ bytes ca = 1;
+ bytes crl = 2;
+ bytes cert = 3;
+ bytes key = 4;
+}
+
message ProvisionClusterRequest {
string provisioningToken = 1;
@@ -94,6 +101,7 @@
string externalHost = 4;
smalltown.common.TrustBackend trustBackend = 5;
bytes storeKey = 6;
+ ConsensusCertificates certs = 7;
}
message ProvisionClusterResponse {
diff --git a/core/internal/api/cluster.go b/core/internal/api/cluster.go
index 32a5691..64a757f 100644
--- a/core/internal/api/cluster.go
+++ b/core/internal/api/cluster.go
@@ -60,8 +60,13 @@
return nil, ErrAttestationFailed
}
+ consensusCerts, err := s.consensusService.IssueCertificate(req.Host)
+ if err != nil {
+ return nil, err
+ }
+
// Provision cluster info locally
- memberID, err := s.consensusService.AddMember(ctx, req.Name, fmt.Sprintf("http://%s:%d", req.Host, req.ConsensusPort))
+ memberID, err := s.consensusService.AddMember(ctx, req.Name, fmt.Sprintf("https://%s:%d", req.Host, req.ConsensusPort))
if err != nil {
return nil, err
}
@@ -77,12 +82,17 @@
ExternalHost: req.Host,
NodeName: req.Name,
TrustBackend: req.TrustBackend,
+ Certs: consensusCerts,
})
if err != nil {
+ err3 := s.consensusService.RevokeCertificate(req.Host)
+ if err3 != nil {
+ s.Logger.Error("Failed to revoke a certificate after rollback, potential security risk", zap.Error(err3))
+ }
// Revert Consensus add member - might fail if consensus cannot be established
err2 := s.consensusService.RemoveMember(ctx, memberID)
- if err2 != nil {
- return nil, fmt.Errorf("Rollback failed after failed provisioning; err=%v; err_rb=%v", err, err2)
+ if err2 != nil || err3 != nil {
+ return nil, fmt.Errorf("Rollback failed after failed provisioning; err=%v; err_rb=%v; err_revoke=%v", err, err2, err3)
}
return nil, err
}
diff --git a/core/internal/api/setup.go b/core/internal/api/setup.go
index 943f203..6aeda40 100644
--- a/core/internal/api/setup.go
+++ b/core/internal/api/setup.go
@@ -20,6 +20,7 @@
"context"
"errors"
"fmt"
+
schema "git.monogon.dev/source/nexantic.git/core/generated/api"
)
@@ -87,7 +88,7 @@
}
// Join cluster
- err := s.setupService.JoinCluster(req.NodeName, req.InitialCluster, req.ExternalHost)
+ err := s.setupService.JoinCluster(req.NodeName, req.InitialCluster, req.ExternalHost, req.Certs)
if err != nil {
return nil, err
}
diff --git a/core/internal/common/setup.go b/core/internal/common/setup.go
index fd70d0a..331d29a 100644
--- a/core/internal/common/setup.go
+++ b/core/internal/common/setup.go
@@ -16,13 +16,15 @@
package common
+import "git.monogon.dev/source/nexantic.git/core/generated/api"
+
type (
SetupService interface {
CurrentState() SmalltownState
GetJoinClusterToken() string
SetupNewCluster(name string, externalHost string) error
EnterJoinClusterMode() error
- JoinCluster(name string, clusterString string, externalHost string) error
+ JoinCluster(name string, clusterString string, externalHost string, certs *api.ConsensusCertificates) error
}
SmalltownState string
diff --git a/core/internal/consensus/BUILD.bazel b/core/internal/consensus/BUILD.bazel
index 72d73b4..c1c6989 100644
--- a/core/internal/consensus/BUILD.bazel
+++ b/core/internal/consensus/BUILD.bazel
@@ -6,7 +6,9 @@
importpath = "git.monogon.dev/source/nexantic.git/core/internal/consensus",
visibility = ["//:__subpackages__"],
deps = [
+ "//core/api/api:go_default_library",
"//core/internal/common:go_default_library",
+ "//core/internal/consensus/ca:go_default_library",
"@com_github_pkg_errors//:go_default_library",
"@io_etcd_go_etcd//clientv3:go_default_library",
"@io_etcd_go_etcd//clientv3/namespace:go_default_library",
@@ -14,6 +16,7 @@
"@io_etcd_go_etcd//etcdserver/api/membership:go_default_library",
"@io_etcd_go_etcd//pkg/types:go_default_library",
"@io_etcd_go_etcd//proxy/grpcproxy/adapter:go_default_library",
+ "@org_golang_x_sys//unix:go_default_library",
"@org_uber_go_zap//:go_default_library",
],
)
diff --git a/core/internal/consensus/ca/BUILD.bazel b/core/internal/consensus/ca/BUILD.bazel
new file mode 100644
index 0000000..5d50bfb
--- /dev/null
+++ b/core/internal/consensus/ca/BUILD.bazel
@@ -0,0 +1,11 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+ name = "go_default_library",
+ srcs = [
+ "ca.go",
+ "compat.go",
+ ],
+ importpath = "git.monogon.dev/source/nexantic.git/core/internal/consensus/ca",
+ visibility = ["//:__subpackages__"],
+)
diff --git a/core/internal/consensus/ca/ca.go b/core/internal/consensus/ca/ca.go
new file mode 100644
index 0000000..925f030
--- /dev/null
+++ b/core/internal/consensus/ca/ca.go
@@ -0,0 +1,197 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ca
+
+import (
+ "crypto"
+ "crypto/ed25519"
+ "crypto/rand"
+ "crypto/sha1"
+ "crypto/x509"
+ "crypto/x509/pkix"
+ "encoding/asn1"
+ "errors"
+ "fmt"
+ "math/big"
+ "time"
+)
+
+var (
+ // From RFC 5280 Section 4.1.2.5
+ unknownNotAfter = time.Unix(253402300799, 0)
+)
+
+type CA struct {
+ // TODO: Potentially protect the key with memguard
+ PrivateKey *ed25519.PrivateKey
+ CACert *x509.Certificate
+ CACertRaw []byte
+ CRLRaw []byte
+ Revoked []pkix.RevokedCertificate
+}
+
+// Workaround for https://github.com/golang/go/issues/26676 in Go's crypto/x509. Specifically Go
+// violates Section 4.2.1.2 of RFC 5280 without this. Should eventually be redundant.
+//
+// Taken from https://github.com/FiloSottile/mkcert/blob/master/cert.go#L295 written by one of Go's
+// crypto engineers
+func calculateSKID(pubKey crypto.PublicKey) ([]byte, error) {
+ spkiASN1, err := x509.MarshalPKIXPublicKey(pubKey)
+ if err != nil {
+ return nil, err
+ }
+
+ var spki struct {
+ Algorithm pkix.AlgorithmIdentifier
+ SubjectPublicKey asn1.BitString
+ }
+ _, err = asn1.Unmarshal(spkiASN1, &spki)
+ if err != nil {
+ return nil, err
+ }
+ skid := sha1.Sum(spki.SubjectPublicKey.Bytes)
+ return skid[:], nil
+}
+
+func New(name string) (*CA, error) {
+ pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ panic(err)
+ }
+
+ serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 127)
+ serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+ if err != nil {
+ return nil, fmt.Errorf("Failed to generate serial number: %w", err)
+ }
+
+ skid, err := calculateSKID(pubKey)
+ if err != nil {
+ return nil, err
+ }
+
+ caCert := &x509.Certificate{
+ SerialNumber: serialNumber,
+ Subject: pkix.Name{
+ CommonName: name,
+ },
+ IsCA: true,
+ BasicConstraintsValid: true,
+ NotBefore: time.Now(),
+ NotAfter: unknownNotAfter,
+ KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature,
+ ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageOCSPSigning},
+ AuthorityKeyId: skid,
+ SubjectKeyId: skid,
+ }
+
+ caCertRaw, err := x509.CreateCertificate(rand.Reader, caCert, caCert, pubKey, privKey)
+ if err != nil {
+ return nil, fmt.Errorf("Failed to create root certificate: %w", err)
+ }
+
+ ca := &CA{
+ PrivateKey: &privKey,
+ CACertRaw: caCertRaw,
+ CACert: caCert,
+ }
+ if ca.reissueCRL() != nil {
+ return nil, fmt.Errorf("failed to create initial CRL: %w", err)
+ }
+
+ return ca, nil
+}
+
+func FromCertificates(caCert []byte, caKey []byte, crl []byte) (*CA, error) {
+ if len(caKey) != ed25519.PrivateKeySize {
+ return nil, errors.New("Invalid CA private key size")
+ }
+ privateKey := ed25519.PrivateKey(caKey)
+
+ caCertVal, err := x509.ParseCertificate(caCert)
+ if err != nil {
+ return nil, err
+ }
+ crlVal, err := x509.ParseCRL(crl)
+ if err != nil {
+ return nil, err
+ }
+ return &CA{
+ PrivateKey: &privateKey,
+ CACertRaw: caCert,
+ CACert: caCertVal,
+ Revoked: crlVal.TBSCertList.RevokedCertificates,
+ CRLRaw: crl,
+ }, nil
+}
+
+func (ca *CA) IssueCertificate(hostname string) (cert []byte, privkey []byte, err error) {
+ serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 127)
+ serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+ if err != nil {
+ err = fmt.Errorf("Failed to generate serial number: %w", err)
+ return
+ }
+
+ pubKey, privKeyRaw, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ return
+ }
+ privkey, err = x509.MarshalPKCS8PrivateKey(privKeyRaw)
+ if err != nil {
+ return
+ }
+
+ etcdCert := &x509.Certificate{
+ SerialNumber: serialNumber,
+ Subject: pkix.Name{
+ CommonName: hostname,
+ OrganizationalUnit: []string{"etcd"},
+ },
+ IsCA: false,
+ BasicConstraintsValid: true,
+ NotBefore: time.Now(),
+ NotAfter: unknownNotAfter,
+ ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
+ DNSNames: []string{hostname},
+ }
+ cert, err = x509.CreateCertificate(rand.Reader, etcdCert, ca.CACert, pubKey, ca.PrivateKey)
+ return
+}
+
+func (ca *CA) reissueCRL() error {
+ compatCert := CompatCertificate(*ca.CACert)
+ newCRL, err := compatCert.CreateCRL(rand.Reader, ca.PrivateKey, ca.Revoked, time.Now(), unknownNotAfter)
+ if err != nil {
+ return err
+ }
+ ca.CRLRaw = newCRL
+ return nil
+}
+
+func (ca *CA) Revoke(serial *big.Int) error {
+ for _, revokedCert := range ca.Revoked {
+ if revokedCert.SerialNumber.Cmp(serial) == 0 {
+ return nil // Already revoked
+ }
+ }
+ ca.Revoked = append(ca.Revoked, pkix.RevokedCertificate{
+ SerialNumber: serial,
+ RevocationTime: time.Now(),
+ })
+ return ca.reissueCRL()
+}
diff --git a/core/internal/consensus/ca/compat.go b/core/internal/consensus/ca/compat.go
new file mode 100644
index 0000000..9c1f4a6
--- /dev/null
+++ b/core/internal/consensus/ca/compat.go
@@ -0,0 +1,110 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// I've fixed this upstream, compat is going away once
+// https://go-review.googlesource.com/c/go/+/204046 hits stable
+package ca
+
+import (
+ "crypto"
+ "crypto/x509"
+ "crypto/x509/pkix"
+ "encoding/asn1"
+ "errors"
+ "io"
+ "time"
+)
+
+// Workaround for Go not supporting Ed25519 CRLs
+type CompatCertificate x509.Certificate
+
+var oidExtensionAuthorityKeyId = []int{2, 5, 29, 35}
+var oidSignatureEd25519 = asn1.ObjectIdentifier{1, 3, 101, 112}
+
+func signingParamsForPublicKey(pub interface{}, requestedSigAlgo x509.SignatureAlgorithm) (hashFunc crypto.Hash, sigAlgo pkix.AlgorithmIdentifier, err error) {
+ sigAlgo.Algorithm = oidSignatureEd25519
+ return
+}
+
+// RFC 5280, 4.2.1.1
+type authKeyId struct {
+ Id []byte `asn1:"optional,tag:0"`
+}
+
+// CreateCRL returns a DER encoded CRL, signed by this Certificate, that
+// contains the given list of revoked certificates.
+func (c *CompatCertificate) CreateCRL(rand io.Reader, priv interface{}, revokedCerts []pkix.RevokedCertificate, now, expiry time.Time) (crlBytes []byte, err error) {
+ key, ok := priv.(crypto.Signer)
+ if !ok {
+ return nil, errors.New("x509: certificate private key does not implement crypto.Signer")
+ }
+
+ hashFunc, signatureAlgorithm, err := signingParamsForPublicKey(key.Public(), 0)
+ if err != nil {
+ return nil, err
+ }
+
+ // Force revocation times to UTC per RFC 5280.
+ revokedCertsUTC := make([]pkix.RevokedCertificate, len(revokedCerts))
+ for i, rc := range revokedCerts {
+ rc.RevocationTime = rc.RevocationTime.UTC()
+ revokedCertsUTC[i] = rc
+ }
+
+ tbsCertList := pkix.TBSCertificateList{
+ Version: 1,
+ Signature: signatureAlgorithm,
+ Issuer: c.Subject.ToRDNSequence(),
+ ThisUpdate: now.UTC(),
+ NextUpdate: expiry.UTC(),
+ RevokedCertificates: revokedCertsUTC,
+ }
+
+ // Authority Key Id
+ if len(c.SubjectKeyId) > 0 {
+ var aki pkix.Extension
+ aki.Id = oidExtensionAuthorityKeyId
+ aki.Value, err = asn1.Marshal(authKeyId{Id: c.SubjectKeyId})
+ if err != nil {
+ return
+ }
+ tbsCertList.Extensions = append(tbsCertList.Extensions, aki)
+ }
+
+ tbsCertListContents, err := asn1.Marshal(tbsCertList)
+ if err != nil {
+ return
+ }
+
+ signed := tbsCertListContents
+ if hashFunc != 0 {
+ h := hashFunc.New()
+ h.Write(signed)
+ signed = h.Sum(nil)
+ }
+
+ var signature []byte
+ signature, err = key.Sign(rand, signed, hashFunc)
+ if err != nil {
+ return
+ }
+
+ return asn1.Marshal(pkix.CertificateList{
+ TBSCertList: tbsCertList,
+ SignatureAlgorithm: signatureAlgorithm,
+ SignatureValue: asn1.BitString{Bytes: signature, BitLength: len(signature) * 8},
+ })
+}
diff --git a/core/internal/consensus/consensus.go b/core/internal/consensus/consensus.go
index e1f59d6..d87a506 100644
--- a/core/internal/consensus/consensus.go
+++ b/core/internal/consensus/consensus.go
@@ -17,9 +17,24 @@
package consensus
import (
+ "bytes"
"context"
+ "crypto/x509"
+ "encoding/hex"
+ "encoding/pem"
"fmt"
+ "io/ioutil"
+ "math/rand"
+ "net/url"
+ "os"
+ "path"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "git.monogon.dev/source/nexantic.git/core/generated/api"
"git.monogon.dev/source/nexantic.git/core/internal/common"
+ "git.monogon.dev/source/nexantic.git/core/internal/consensus/ca"
"github.com/pkg/errors"
"go.etcd.io/etcd/clientv3"
"go.etcd.io/etcd/clientv3/namespace"
@@ -28,9 +43,7 @@
"go.etcd.io/etcd/pkg/types"
"go.etcd.io/etcd/proxy/grpcproxy/adapter"
"go.uber.org/zap"
- "net/url"
- "os"
- "strings"
+ "golang.org/x/sys/unix"
)
const (
@@ -38,13 +51,25 @@
DefaultLogger = "zap"
)
+const (
+ CAPath = "ca.pem"
+ CertPath = "cert.pem"
+ KeyPath = "cert-key.pem"
+ CRLPath = "ca-crl.der"
+ CRLSwapPath = "ca-crl.der.swp"
+)
+
type (
Service struct {
*common.BaseService
- etcd *embed.Etcd
- kv clientv3.KV
- ready bool
+ etcd *embed.Etcd
+ kv clientv3.KV
+ ready bool
+ bootstrapCA *ca.CA
+ bootstrapCert []byte
+ watchCRLTicker *time.Ticker
+ lastCRL []byte
config *Config
}
@@ -84,15 +109,27 @@
cfg := embed.NewConfig()
+ cfg.PeerTLSInfo.CertFile = filepath.Join(s.config.DataDir, CertPath)
+ cfg.PeerTLSInfo.KeyFile = filepath.Join(s.config.DataDir, KeyPath)
+ cfg.PeerTLSInfo.TrustedCAFile = filepath.Join(s.config.DataDir, CAPath)
+ cfg.PeerTLSInfo.ClientCertAuth = true
+ cfg.PeerTLSInfo.CRLFile = filepath.Join(s.config.DataDir, CRLPath)
+
+ lastCRL, err := ioutil.ReadFile(cfg.PeerTLSInfo.CRLFile)
+ if err != nil {
+ return fmt.Errorf("failed to read etcd CRL: %w", err)
+ }
+ s.lastCRL = lastCRL
+
// Reset LCUrls because we don't want to expose any client
cfg.LCUrls = nil
- apURL, err := url.Parse(fmt.Sprintf("http://%s:%d", s.config.ExternalHost, s.config.ListenPort))
+ apURL, err := url.Parse(fmt.Sprintf("https://%s:%d", s.config.ExternalHost, s.config.ListenPort))
if err != nil {
return errors.Wrap(err, "invalid external_host or listen_port")
}
- lpURL, err := url.Parse(fmt.Sprintf("http://%s:%d", s.config.ListenHost, s.config.ListenPort))
+ lpURL, err := url.Parse(fmt.Sprintf("https://%s:%d", s.config.ListenHost, s.config.ListenPort))
if err != nil {
return errors.Wrap(err, "invalid listen_host or listen_port")
}
@@ -134,10 +171,213 @@
// Inject kv client
s.kv = clientv3.NewKVFromKVClient(adapter.KvServerToKvClient(s.etcd.Server), nil)
+ // Start CRL watcher
+ go s.watchCRL()
+
return nil
}
+func (s *Service) SetupCertificates(certs *api.ConsensusCertificates) error {
+ if err := ioutil.WriteFile(filepath.Join(s.config.DataDir, CRLPath), certs.Crl, 0600); err != nil {
+ return err
+ }
+ if err := ioutil.WriteFile(filepath.Join(s.config.DataDir, CertPath),
+ pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certs.Cert}), 0600); err != nil {
+ return err
+ }
+ if err := ioutil.WriteFile(filepath.Join(s.config.DataDir, KeyPath),
+ pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: certs.Key}), 0600); err != nil {
+ return err
+ }
+ if err := ioutil.WriteFile(filepath.Join(s.config.DataDir, CAPath),
+ pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certs.Ca}), 0600); err != nil {
+ return err
+ }
+ return nil
+}
+
+func (s *Service) PrecreateCA() error {
+ // Provision an etcd CA
+ etcdRootCA, err := ca.New("Smalltown etcd Root CA")
+ if err != nil {
+ return err
+ }
+ cert, privkey, err := etcdRootCA.IssueCertificate(s.config.ExternalHost)
+ if err != nil {
+ return fmt.Errorf("failed to self-issue a certificate: %w", err)
+ }
+ if err := os.MkdirAll(s.config.DataDir, 0700); err != nil {
+ return fmt.Errorf("failed to create consensus data dir: %w", err)
+ }
+ // Preserve certificate for later injection
+ s.bootstrapCert = cert
+ if err := s.SetupCertificates(&api.ConsensusCertificates{
+ Ca: etcdRootCA.CACertRaw,
+ Crl: etcdRootCA.CRLRaw,
+ Cert: cert,
+ Key: privkey,
+ }); err != nil {
+ return fmt.Errorf("failed to setup certificates: %w", err)
+ }
+ s.bootstrapCA = etcdRootCA
+ return nil
+}
+
+const (
+ caPathEtcd = "/etcd-ca/ca.der"
+ caKeyPathEtcd = "/etcd-ca/ca-key.der"
+ crlPathEtcd = "/etcd-ca/crl.der"
+ certPrefixEtcd = "/etcd-ca/certs"
+)
+
+func (s *Service) InjectCA() error {
+ if _, err := s.kv.Put(context.Background(), caPathEtcd, string(s.bootstrapCA.CACertRaw)); err != nil {
+ return err
+ }
+ // TODO: Should be wrapped by the master key
+ if _, err := s.kv.Put(context.Background(), caKeyPathEtcd, string([]byte(*s.bootstrapCA.PrivateKey))); err != nil {
+ return err
+ }
+ if _, err := s.kv.Put(context.Background(), crlPathEtcd, string(s.bootstrapCA.CRLRaw)); err != nil {
+ return err
+ }
+ certVal, err := x509.ParseCertificate(s.bootstrapCert)
+ if err != nil {
+ return err
+ }
+ serial := hex.EncodeToString(certVal.SerialNumber.Bytes())
+ if _, err := s.kv.Put(context.Background(), path.Join(certPrefixEtcd, serial), string(s.bootstrapCert)); err != nil {
+ return fmt.Errorf("failed to persist certificate: %w", err)
+ }
+ // Clear out bootstrap CA after injecting
+ s.bootstrapCA = nil
+ s.bootstrapCert = []byte{}
+ return nil
+}
+
+func (s *Service) etcdGetSingle(path string) ([]byte, int64, error) {
+ res, err := s.kv.Get(context.Background(), path)
+ if err != nil {
+ return nil, -1, fmt.Errorf("failed to get key from etcd: %w", err)
+ }
+ if len(res.Kvs) != 1 {
+ return nil, -1, errors.New("key not available")
+ }
+ return res.Kvs[0].Value, res.Kvs[0].ModRevision, nil
+}
+
+func (s *Service) takeCAOnline() (*ca.CA, int64, error) {
+ // TODO: Technically this could be done in a single request, but it's more logic
+ caCert, _, err := s.etcdGetSingle(caPathEtcd)
+ if err != nil {
+ return nil, -1, fmt.Errorf("failed to get CA certificate from etcd: %w", err)
+ }
+ caKey, _, err := s.etcdGetSingle(caKeyPathEtcd)
+ if err != nil {
+ return nil, -1, fmt.Errorf("failed to get CA key from etcd: %w", err)
+ }
+ // TODO: Unwrap CA key once wrapping is implemented
+ crl, crlRevision, err := s.etcdGetSingle(crlPathEtcd)
+ if err != nil {
+ return nil, -1, fmt.Errorf("failed to get CRL from etcd: %w", err)
+ }
+ idCA, err := ca.FromCertificates(caCert, caKey, crl)
+ if err != nil {
+ return nil, -1, fmt.Errorf("failed to take CA online: %w", err)
+ }
+ return idCA, crlRevision, nil
+}
+
+func (s *Service) IssueCertificate(hostname string) (*api.ConsensusCertificates, error) {
+ idCA, _, err := s.takeCAOnline()
+ if err != nil {
+ return nil, err
+ }
+ cert, key, err := idCA.IssueCertificate(hostname)
+ if err != nil {
+ return nil, fmt.Errorf("failed to issue certificate: %w", err)
+ }
+ certVal, err := x509.ParseCertificate(cert)
+ if err != nil {
+ return nil, err
+ }
+ serial := hex.EncodeToString(certVal.SerialNumber.Bytes())
+ if _, err := s.kv.Put(context.Background(), path.Join(certPrefixEtcd, serial), string(cert)); err != nil {
+ return nil, fmt.Errorf("failed to persist certificate: %w", err)
+ }
+ return &api.ConsensusCertificates{
+ Ca: idCA.CACertRaw,
+ Cert: cert,
+ Crl: idCA.CRLRaw,
+ Key: key,
+ }, nil
+}
+
+func (s *Service) RevokeCertificate(hostname string) error {
+ rand.Seed(time.Now().UnixNano())
+ for {
+ idCA, crlRevision, err := s.takeCAOnline()
+ if err != nil {
+ return err
+ }
+ allIssuedCerts, err := s.kv.Get(context.Background(), certPrefixEtcd, clientv3.WithPrefix())
+ for _, cert := range allIssuedCerts.Kvs {
+ certVal, err := x509.ParseCertificate(cert.Value)
+ if err != nil {
+ s.Logger.Error("Failed to parse previously issued certificate, this is a security risk", zap.Error(err))
+ continue
+ }
+ for _, dnsName := range certVal.DNSNames {
+ if dnsName == hostname {
+ // Revoke this
+ if err := idCA.Revoke(certVal.SerialNumber); err != nil {
+ // We need to fail if any single revocation fails otherwise outer applications
+ // have no chance of calling this safely
+ return err
+ }
+ }
+ }
+ }
+ cmp := clientv3.Compare(clientv3.ModRevision(crlPathEtcd), "=", crlRevision)
+ op := clientv3.OpPut(crlPathEtcd, string(idCA.CRLRaw))
+ res, err := s.kv.Txn(context.Background()).If(cmp).Then(op).Commit()
+ if err != nil {
+ return fmt.Errorf("failed to persist new CRL in etcd: %w", err)
+ }
+ if res.Succeeded { // Transaction has succeeded
+ break
+ }
+ // Sleep a random duration between 0 and 300ms to reduce serialization failures
+ time.Sleep(time.Duration(rand.Intn(300)) * time.Millisecond)
+ }
+ return nil
+}
+
+func (s *Service) watchCRL() {
+ // TODO: Change etcd client to WatchableKV and make this an actual watch
+ // This needs changes in more places, so leaving it now
+ s.watchCRLTicker = time.NewTicker(30 * time.Second)
+ for range s.watchCRLTicker.C {
+ crl, _, err := s.etcdGetSingle(crlPathEtcd)
+ if err != nil {
+ s.Logger.Warn("Failed to check for new CRL", zap.Error(err))
+ continue
+ }
+ // This is cryptographic material but not secret, so no constant time compare necessary here
+ if !bytes.Equal(crl, s.lastCRL) {
+ if err := ioutil.WriteFile(filepath.Join(s.config.DataDir, CRLSwapPath), crl, 0600); err != nil {
+ s.Logger.Warn("Failed to write updated CRL", zap.Error(err))
+ }
+ // This uses unix.Rename to guarantee a particular atomic update behavior
+ if err := unix.Rename(filepath.Join(s.config.DataDir, CRLSwapPath), filepath.Join(s.config.DataDir, CRLPath)); err != nil {
+ s.Logger.Warn("Failed to atomically swap updated CRL", zap.Error(err))
+ }
+ }
+ }
+}
+
func (s *Service) OnStop() error {
+ s.watchCRLTicker.Stop()
s.etcd.Close()
return nil
diff --git a/core/internal/node/BUILD.bazel b/core/internal/node/BUILD.bazel
index 0596269..763df36 100644
--- a/core/internal/node/BUILD.bazel
+++ b/core/internal/node/BUILD.bazel
@@ -9,6 +9,7 @@
importpath = "git.monogon.dev/source/nexantic.git/core/internal/node",
visibility = ["//:__subpackages__"],
deps = [
+ "//core/api/api:go_default_library",
"//core/internal/api:go_default_library",
"//core/internal/common:go_default_library",
"//core/internal/consensus:go_default_library",
diff --git a/core/internal/node/setup.go b/core/internal/node/setup.go
index 28585dd..5d8953d 100644
--- a/core/internal/node/setup.go
+++ b/core/internal/node/setup.go
@@ -17,9 +17,11 @@
package node
import (
+ "git.monogon.dev/source/nexantic.git/core/generated/api"
"git.monogon.dev/source/nexantic.git/core/internal/common"
"errors"
+
"go.uber.org/zap"
)
@@ -67,11 +69,19 @@
config.DataDir = dataPath
s.Consensus.SetConfig(config)
+ if err := s.Consensus.PrecreateCA(); err != nil {
+ return err
+ }
+
err = s.Consensus.Start()
if err != nil {
return err
}
+ if err := s.Consensus.InjectCA(); err != nil {
+ return err
+ }
+
// Change system state
s.state = common.StateConfigured
@@ -91,7 +101,7 @@
return nil
}
-func (s *SmalltownNode) JoinCluster(name string, clusterString string, externalHost string) error {
+func (s *SmalltownNode) JoinCluster(name string, clusterString string, externalHost string, certs *api.ConsensusCertificates) error {
if s.state != common.StateClusterJoinMode {
return ErrNotInJoinMode
}
@@ -108,6 +118,9 @@
config.InitialCluster = clusterString
config.ExternalHost = externalHost
s.Consensus.SetConfig(config)
+ if err := s.Consensus.SetupCertificates(certs); err != nil {
+ return err
+ }
// Start consensus
err = s.Consensus.Start()