m/node/kubernetes/pki: refactor out CA functionality
This factors out all non-k8s-specific CA functionality from
metropolis/node/kubernetes/pki into metropolis/pkg/pki.
This will allow us to re-use the same PKI-in-CA system to issue
certificates for the Metropolis cluster and nodes.
We also drive-by change some Kubernetes/PKI interactions to make things
cleaner. Notably, this implements Certificate.Mount to return a
fileargs.FileArgs containing all the files neede to use this
Certificate.
Test Plan: covered by current e2e tests. An etcd harness to test this independently would be nice, though.
X-Origin-Diff: phab/D709
GitOrigin-RevId: bdc9ff215b94c9192f65c6da8935fe2818fd14ad
diff --git a/metropolis/node/core/main.go b/metropolis/node/core/main.go
index 3cf75ce..0f6ebd1 100644
--- a/metropolis/node/core/main.go
+++ b/metropolis/node/core/main.go
@@ -197,7 +197,7 @@
// Ensure Kubernetes PKI objects exist in etcd.
kpkiKV := m.ConsensusKV("cluster", "kpki")
- kpki := pki.NewKubernetes(lt.MustLeveledFor("pki.kubernetes"), kpkiKV)
+ kpki := pki.New(lt.MustLeveledFor("pki.kubernetes"), kpkiKV)
if err := kpki.EnsureAll(ctx); err != nil {
return fmt.Errorf("failed to ensure kubernetes PKI present: %w", err)
}
diff --git a/metropolis/node/kubernetes/BUILD.bazel b/metropolis/node/kubernetes/BUILD.bazel
index 8f00410..ada30c9 100644
--- a/metropolis/node/kubernetes/BUILD.bazel
+++ b/metropolis/node/kubernetes/BUILD.bazel
@@ -16,7 +16,6 @@
deps = [
"//metropolis/node:go_default_library",
"//metropolis/node/core/localstorage:go_default_library",
- "//metropolis/node/core/localstorage/declarative:go_default_library",
"//metropolis/node/core/network/dns:go_default_library",
"//metropolis/node/kubernetes/clusternet:go_default_library",
"//metropolis/node/kubernetes/nfproxy:go_default_library",
@@ -25,6 +24,7 @@
"//metropolis/pkg/fileargs:go_default_library",
"//metropolis/pkg/fsquota:go_default_library",
"//metropolis/pkg/logtree:go_default_library",
+ "//metropolis/pkg/pki:go_default_library",
"//metropolis/pkg/supervisor:go_default_library",
"//metropolis/proto/api:go_default_library",
"@com_github_container_storage_interface_spec//lib/go/csi:go_default_library",
diff --git a/metropolis/node/kubernetes/apiserver.go b/metropolis/node/kubernetes/apiserver.go
index 2757fb2..704b8df 100644
--- a/metropolis/node/kubernetes/apiserver.go
+++ b/metropolis/node/kubernetes/apiserver.go
@@ -31,7 +31,7 @@
)
type apiserverService struct {
- KPKI *pki.KubernetesPKI
+ KPKI *pki.PKI
AdvertiseAddress net.IP
ServiceIPRange net.IPNet
EphemeralConsensusDirectory *localstorage.EphemeralConsensusDirectory
@@ -55,7 +55,7 @@
name pki.KubeCertificateName
}{
{&s.idCA, nil, pki.IdCA},
- {&s.kubeletClientCert, &s.kubeletClientKey, pki.KubeletClient},
+ {&s.kubeletClientCert, &s.kubeletClientKey, pki.APIServerKubeletClient},
{&s.aggregationCA, nil, pki.AggregationCA},
{&s.aggregationClientCert, &s.aggregationClientKey, pki.FrontProxyClient},
{&s.serverCert, &s.serverKey, pki.APIServer},
diff --git a/metropolis/node/kubernetes/controller-manager.go b/metropolis/node/kubernetes/controller-manager.go
index c54112c..546321c 100644
--- a/metropolis/node/kubernetes/controller-manager.go
+++ b/metropolis/node/kubernetes/controller-manager.go
@@ -38,7 +38,7 @@
serverKey []byte
}
-func getPKIControllerManagerConfig(ctx context.Context, kpki *pki.KubernetesPKI) (*controllerManagerConfig, error) {
+func getPKIControllerManagerConfig(ctx context.Context, kpki *pki.PKI) (*controllerManagerConfig, error) {
var config controllerManagerConfig
var err error
config.rootCA, _, err = kpki.Certificate(ctx, pki.IdCA)
diff --git a/metropolis/node/kubernetes/kubelet.go b/metropolis/node/kubernetes/kubelet.go
index 741dba8..0cacaef 100644
--- a/metropolis/node/kubernetes/kubelet.go
+++ b/metropolis/node/kubernetes/kubelet.go
@@ -19,7 +19,6 @@
import (
"context"
"encoding/json"
- "encoding/pem"
"fmt"
"io"
"net"
@@ -29,10 +28,10 @@
kubeletconfig "k8s.io/kubelet/config/v1beta1"
"source.monogon.dev/metropolis/node/core/localstorage"
- "source.monogon.dev/metropolis/node/core/localstorage/declarative"
"source.monogon.dev/metropolis/node/kubernetes/pki"
"source.monogon.dev/metropolis/node/kubernetes/reconciler"
"source.monogon.dev/metropolis/pkg/fileargs"
+ opki "source.monogon.dev/metropolis/pkg/pki"
"source.monogon.dev/metropolis/pkg/supervisor"
)
@@ -42,42 +41,32 @@
KubeletDirectory *localstorage.DataKubernetesKubeletDirectory
EphemeralDirectory *localstorage.EphemeralDirectory
Output io.Writer
- KPKI *pki.KubernetesPKI
+ KPKI *pki.PKI
+
+ mount *opki.FilesystemCertificate
+ mountKubeconfigPath string
}
func (s *kubeletService) createCertificates(ctx context.Context) error {
- identity := fmt.Sprintf("system:node:%s", s.NodeName)
-
- ca := s.KPKI.Certificates[pki.IdCA]
- cacert, _, err := ca.Ensure(ctx, s.KPKI.KV)
+ server, client, err := s.KPKI.VolatileKubelet(ctx, s.NodeName)
if err != nil {
- return fmt.Errorf("could not ensure ca certificate: %w", err)
+ return fmt.Errorf("when generating local kubelet credentials: %w", err)
}
- kubeconfig, err := pki.New(ca, "", pki.Client(identity, []string{"system:nodes"})).Kubeconfig(ctx, s.KPKI.KV)
+ clientKubeconfig, err := pki.Kubeconfig(ctx, s.KPKI.KV, client)
if err != nil {
- return fmt.Errorf("could not create volatile kubelet client cert: %w", err)
+ return fmt.Errorf("when generating kubeconfig: %w", err)
}
- cert, key, err := pki.New(ca, "", pki.Server([]string{s.NodeName}, nil)).Ensure(ctx, s.KPKI.KV)
+ // Use a single fileargs mount for server certificate and client kubeconfig.
+ mounted, err := server.Mount(ctx, s.KPKI.KV)
if err != nil {
- return fmt.Errorf("could not create volatile kubelet server cert: %w", err)
+ return fmt.Errorf("could not mount kubelet cert dir: %w", err)
}
+ // mounted is closed by Run() on process exit.
- // TODO(q3k): this should probably become its own function //metropolis/node/kubernetes/pki.
- for _, el := range []struct {
- target declarative.FilePlacement
- data []byte
- }{
- {s.KubeletDirectory.Kubeconfig, kubeconfig},
- {s.KubeletDirectory.PKI.CACertificate, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cacert})},
- {s.KubeletDirectory.PKI.Certificate, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert})},
- {s.KubeletDirectory.PKI.Key, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: key})},
- } {
- if err := el.target.Write(el.data, 0400); err != nil {
- return fmt.Errorf("could not write %v: %w", el.target, err)
- }
- }
+ s.mount = mounted
+ s.mountKubeconfigPath = mounted.ArgPath("kubeconfig", clientKubeconfig)
return nil
}
@@ -93,13 +82,13 @@
Kind: "KubeletConfiguration",
APIVersion: kubeletconfig.GroupName + "/v1beta1",
},
- TLSCertFile: s.KubeletDirectory.PKI.Certificate.FullPath(),
- TLSPrivateKeyFile: s.KubeletDirectory.PKI.Key.FullPath(),
+ TLSCertFile: s.mount.CertPath,
+ TLSPrivateKeyFile: s.mount.KeyPath,
TLSMinVersion: "VersionTLS13",
ClusterDNS: clusterDNS,
Authentication: kubeletconfig.KubeletAuthentication{
X509: kubeletconfig.KubeletX509Authentication{
- ClientCAFile: s.KubeletDirectory.PKI.CACertificate.FullPath(),
+ ClientCAFile: s.mount.CACertPath,
},
},
// TODO(q3k): move reconciler.False to a generic package, fix the following references.
@@ -123,6 +112,7 @@
if err := s.createCertificates(ctx); err != nil {
return fmt.Errorf("when creating certificates: %w", err)
}
+ defer s.mount.Close()
configRaw, err := json.Marshal(s.configure())
if err != nil {
@@ -137,7 +127,7 @@
fargs.FileOpt("--config", "config.json", configRaw),
"--container-runtime=remote",
fmt.Sprintf("--container-runtime-endpoint=unix://%s", s.EphemeralDirectory.Containerd.ClientSocket.FullPath()),
- fmt.Sprintf("--kubeconfig=%s", s.KubeletDirectory.Kubeconfig.FullPath()),
+ fmt.Sprintf("--kubeconfig=%s", s.mountKubeconfigPath),
fmt.Sprintf("--root-dir=%s", s.KubeletDirectory.FullPath()),
)
cmd.Env = []string{"PATH=/kubernetes/bin"}
diff --git a/metropolis/node/kubernetes/pki/BUILD.bazel b/metropolis/node/kubernetes/pki/BUILD.bazel
index 5dcfe78..cfbba0c 100644
--- a/metropolis/node/kubernetes/pki/BUILD.bazel
+++ b/metropolis/node/kubernetes/pki/BUILD.bazel
@@ -2,16 +2,13 @@
go_library(
name = "go_default_library",
- srcs = [
- "ca.go",
- "certificate.go",
- "kubernetes.go",
- ],
+ srcs = ["kubernetes.go"],
importpath = "source.monogon.dev/metropolis/node/kubernetes/pki",
visibility = ["//metropolis/node:__subpackages__"],
deps = [
"//metropolis/node:go_default_library",
"//metropolis/pkg/logtree:go_default_library",
+ "//metropolis/pkg/pki:go_default_library",
"@io_etcd_go_etcd//clientv3:go_default_library",
"@io_k8s_client_go//tools/clientcmd:go_default_library",
"@io_k8s_client_go//tools/clientcmd/api:go_default_library",
diff --git a/metropolis/node/kubernetes/pki/ca.go b/metropolis/node/kubernetes/pki/ca.go
deleted file mode 100644
index 64453cd..0000000
--- a/metropolis/node/kubernetes/pki/ca.go
+++ /dev/null
@@ -1,151 +0,0 @@
-// Copyright 2020 The Monogon Project Authors.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package pki
-
-import (
- "context"
- "crypto"
- "crypto/ed25519"
- "crypto/rand"
- "crypto/sha1"
- "crypto/x509"
- "crypto/x509/pkix"
- "encoding/asn1"
- "fmt"
- "math/big"
- "time"
-
- "go.etcd.io/etcd/clientv3"
-)
-
-// Issuer is a CA that can issue certificates. Two issuers are currently implemented:
-// - SelfSigned, which will generated a certificate signed by its corresponding private key.
-// - Certificate, which will use another existing Certificate as a CA.
-type Issuer interface {
- // CACertificate returns the DER-encoded x509 certificate of the CA that will sign certificates when Issue is
- // called, or nil if this is self-signing issuer.
- CACertificate(ctx context.Context, kv clientv3.KV) ([]byte, error)
- // Issue will generate a key and certificate signed by the Issuer. The returned certificate is x509 DER-encoded,
- // while the key is a bare ed25519 key.
- Issue(ctx context.Context, template x509.Certificate, kv clientv3.KV) (cert, key []byte, err error)
-}
-
-var (
- // From RFC 5280 Section 4.1.2.5
- unknownNotAfter = time.Unix(253402300799, 0)
-)
-
-// Workaround for https://github.com/golang/go/issues/26676 in Go's crypto/x509. Specifically Go
-// violates Section 4.2.1.2 of RFC 5280 without this.
-// Fixed for 1.15 in https://go-review.googlesource.com/c/go/+/227098/.
-//
-// Taken from https://github.com/FiloSottile/mkcert/blob/master/cert.go#L295 written by one of Go's
-// crypto engineers
-func calculateSKID(pubKey crypto.PublicKey) ([]byte, error) {
- spkiASN1, err := x509.MarshalPKIXPublicKey(pubKey)
- if err != nil {
- return nil, err
- }
-
- var spki struct {
- Algorithm pkix.AlgorithmIdentifier
- SubjectPublicKey asn1.BitString
- }
- _, err = asn1.Unmarshal(spkiASN1, &spki)
- if err != nil {
- return nil, err
- }
- skid := sha1.Sum(spki.SubjectPublicKey.Bytes)
- return skid[:], nil
-}
-
-// issueCertificate is a generic low level certificate-and-key issuance function. If ca or cakey is null, the
-// certificate will be self-signed. The returned certificate is DER-encoded, while the returned key is internal.
-func issueCertificate(template x509.Certificate, ca *x509.Certificate, caKey interface{}) (cert, key []byte, err error) {
- pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
- if err != nil {
- panic(err)
- }
-
- serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 127)
- serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
- if err != nil {
- err = fmt.Errorf("failed to generate serial number: %w", err)
- return
- }
-
- skid, err := calculateSKID(pubKey)
- if err != nil {
- return []byte{}, privKey, err
- }
-
- template.SerialNumber = serialNumber
- template.NotBefore = time.Now()
- template.NotAfter = unknownNotAfter
- template.BasicConstraintsValid = true
- template.SubjectKeyId = skid
-
- // Set the AuthorityKeyID to the SKID of the signing certificate (or self, if self-signing).
- if ca != nil && caKey != nil {
- template.AuthorityKeyId = ca.AuthorityKeyId
- } else {
- template.AuthorityKeyId = template.SubjectKeyId
- }
-
- if ca == nil || caKey == nil {
- ca = &template
- caKey = privKey
- }
-
- caCertRaw, err := x509.CreateCertificate(rand.Reader, &template, ca, pubKey, caKey)
- return caCertRaw, privKey, err
-}
-
-type selfSigned struct{}
-
-func (s *selfSigned) Issue(ctx context.Context, template x509.Certificate, kv clientv3.KV) (cert, key []byte, err error) {
- return issueCertificate(template, nil, nil)
-}
-
-func (s *selfSigned) CACertificate(ctx context.Context, kv clientv3.KV) ([]byte, error) {
- return nil, nil
-}
-
-var (
- // SelfSigned is an Issuer that generates self-signed certificates.
- SelfSigned = &selfSigned{}
-)
-
-func (c *Certificate) Issue(ctx context.Context, template x509.Certificate, kv clientv3.KV) (cert, key []byte, err error) {
- caCert, caKey, err := c.ensure(ctx, kv)
- if err != nil {
- return nil, nil, fmt.Errorf("could not ensure CA certificate %q exists: %w", c.name, err)
- }
-
- ca, err := x509.ParseCertificate(caCert)
- if err != nil {
- return nil, nil, fmt.Errorf("could not parse CA certificate: %w", err)
- }
- // Ensure only one level of CAs exist, and that they are created explicitly.
- template.IsCA = false
- return issueCertificate(template, ca, ed25519.PrivateKey(caKey))
-}
-
-func (c *Certificate) CACertificate(ctx context.Context, kv clientv3.KV) ([]byte, error) {
- cert, _, err := c.ensure(ctx, kv)
- return cert, err
-}
diff --git a/metropolis/node/kubernetes/pki/certificate.go b/metropolis/node/kubernetes/pki/certificate.go
deleted file mode 100644
index da7d301..0000000
--- a/metropolis/node/kubernetes/pki/certificate.go
+++ /dev/null
@@ -1,192 +0,0 @@
-// Copyright 2020 The Monogon Project Authors.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package pki
-
-import (
- "context"
- "crypto/ed25519"
- "crypto/x509"
- "crypto/x509/pkix"
- "fmt"
- "net"
-
- "go.etcd.io/etcd/clientv3"
-)
-
-// Certificate is the promise of a Certificate being available to the caller. In this case, Certificate refers to a
-// pair of x509 certificate and corresponding private key.
-// Certificates can be stored in etcd, and their issuers might also be store on etcd. As such, this type's methods
-// contain references to an etcd KV client.
-// This Certificate type is agnostic to usage, but mostly geared towards Kubernetes certificates.
-type Certificate struct {
- // issuer is the Issuer that will generate this certificate if one doesn't yet exist or etcd, or the requested
- // certificate is volatile (not to be stored on etcd).
- issuer Issuer
- // name is a unique key for storing the certificate in etcd. If empty, certificate is 'volatile', will not be stored
- // on etcd, and every .Ensure() call will generate a new pair.
- name string
- // template is an x509 certificate definition that will be used to generate the certificate when issuing it.
- template x509.Certificate
-}
-
-const (
- // etcdPrefix is where all the PKI data is stored in etcd.
- etcdPrefix = "/kube-pki/"
-)
-
-func etcdPath(f string, args ...interface{}) string {
- return etcdPrefix + fmt.Sprintf(f, args...)
-}
-
-// New creates a new Certificate, or to be more precise, a promise that a certificate will exist once Ensure is called.
-// Issuer must be a valid certificate issuer (SelfSigned or another Certificate). Name must be unique among all
-// certificates, or empty (which will cause the certificate to be volatile, ie. not stored in etcd).
-func New(issuer Issuer, name string, template x509.Certificate) *Certificate {
- return &Certificate{
- issuer: issuer,
- name: name,
- template: template,
- }
-}
-
-// Client makes a Kubernetes PKI-compatible client certificate template.
-// Directly derived from Kubernetes PKI requirements documented at
-// https://kubernetes.io/docs/setup/best-practices/certificates/#configure-certificates-manually
-func Client(identity string, groups []string) x509.Certificate {
- return x509.Certificate{
- Subject: pkix.Name{
- CommonName: identity,
- Organization: groups,
- },
- KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
- ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
- }
-}
-
-// Server makes a Kubernetes PKI-compatible server certificate template.
-func Server(dnsNames []string, ips []net.IP) x509.Certificate {
- return x509.Certificate{
- Subject: pkix.Name{},
- KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
- ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
- DNSNames: dnsNames,
- IPAddresses: ips,
- }
-}
-
-// CA makes a Certificate that can sign other certificates.
-func CA(cn string) x509.Certificate {
- return x509.Certificate{
- Subject: pkix.Name{
- CommonName: cn,
- },
- IsCA: true,
- KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature,
- ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageOCSPSigning},
- }
-}
-
-func (c *Certificate) etcdPaths() (cert, key string) {
- return etcdPath("%s-cert.der", c.name), etcdPath("%s-key.der", c.name)
-}
-
-// ensure returns a DER-encoded x509 certificate and internally encoded bare ed25519 key for a given Certificate,
-// in memory (if volatile), loading it from etcd, or creating and saving it on etcd if needed.
-// This function is safe to call in parallel from multiple etcd clients (including across machines), but it will error
-// in case a concurrent certificate generation happens. These errors are, however, safe to retry - as long as all the
-// certificate creators (ie., Metropolis nodes) run the same version of this code.
-// TODO(q3k): in the future, this should be handled better - especially as we introduce new certificates, or worse,
-// change the issuance chain. As a stopgap measure, an explicit per-certificate or even global lock can be implemented.
-// And, even before that, we can handle concurrency errors in a smarter way.
-func (c *Certificate) ensure(ctx context.Context, kv clientv3.KV) (cert, key []byte, err error) {
- if c.name == "" {
- // Volatile certificate - generate.
- // TODO(q3k): cache internally?
- cert, key, err = c.issuer.Issue(ctx, c.template, kv)
- if err != nil {
- err = fmt.Errorf("failed to issue: %w", err)
- return
- }
- return
- }
-
- certPath, keyPath := c.etcdPaths()
-
- // Try loading certificate and key from etcd.
- certRes, err := kv.Get(ctx, certPath)
- if err != nil {
- err = fmt.Errorf("failed to get certificate from etcd: %w", err)
- return
- }
- keyRes, err := kv.Get(ctx, keyPath)
- if err != nil {
- err = fmt.Errorf("failed to get key from etcd: %w", err)
- return
- }
-
- if len(certRes.Kvs) == 1 && len(keyRes.Kvs) == 1 {
- // Certificate and key exists in etcd, return that.
- cert = certRes.Kvs[0].Value
- key = keyRes.Kvs[0].Value
-
- err = nil
- // TODO(q3k): check for expiration
- return
- }
-
- // No certificate found - issue one.
- cert, key, err = c.issuer.Issue(ctx, c.template, kv)
- if err != nil {
- err = fmt.Errorf("failed to issue: %w", err)
- return
- }
-
- // Save to etcd in transaction. This ensures that no partial writes happen, and that we haven't been raced to the
- // save.
- res, err := kv.Txn(ctx).
- If(
- clientv3.Compare(clientv3.CreateRevision(certPath), "=", 0),
- clientv3.Compare(clientv3.CreateRevision(keyPath), "=", 0),
- ).
- Then(
- clientv3.OpPut(certPath, string(cert)),
- clientv3.OpPut(keyPath, string(key)),
- ).Commit()
- if err != nil {
- err = fmt.Errorf("failed to write newly issued certificate: %w", err)
- } else if !res.Succeeded {
- err = fmt.Errorf("certificate issuance transaction failed: concurrent write")
- }
-
- return
-}
-
-// Ensure returns an x509 DER-encoded (but not PEM-encoded) certificate and key for a given Certificate.
-// If the certificate is volatile, each call to Ensure will cause a new certificate to be generated.
-// Otherwise, it will be retrieved from etcd, or generated and stored there if needed.
-func (c *Certificate) Ensure(ctx context.Context, kv clientv3.KV) (cert, key []byte, err error) {
- cert, key, err = c.ensure(ctx, kv)
- if err != nil {
- return nil, nil, err
- }
- key, err = x509.MarshalPKCS8PrivateKey(ed25519.PrivateKey(key))
- if err != nil {
- err = fmt.Errorf("could not marshal private key (data corruption?): %w", err)
- return
- }
- return cert, key, err
-}
diff --git a/metropolis/node/kubernetes/pki/kubernetes.go b/metropolis/node/kubernetes/pki/kubernetes.go
index 2e3490b..467f718 100644
--- a/metropolis/node/kubernetes/pki/kubernetes.go
+++ b/metropolis/node/kubernetes/pki/kubernetes.go
@@ -14,6 +14,13 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+// package pki builds upon metropolis/pkg/pki/ to provide an
+// etcd-backed implementation of all x509 PKI Certificates/CAs required to run
+// Kubernetes.
+// Most elements of the PKI are 'static' long-standing certificates/credentials
+// stored within etcd. However, this package also provides a method to generate
+// 'volatile' (in-memory) certificates/credentials for per-node Kubelets and
+// any client certificates.
package pki
import (
@@ -31,10 +38,12 @@
common "source.monogon.dev/metropolis/node"
"source.monogon.dev/metropolis/pkg/logtree"
+ opki "source.monogon.dev/metropolis/pkg/pki"
)
-// KubeCertificateName is an enum-like unique name of a static Kubernetes certificate. The value of the name is used
-// as the unique part of an etcd path where the certificate and key are stored.
+// KubeCertificateName is an enum-like unique name of a static Kubernetes
+// certificate. The value of the name is used as the unique part of an etcd
+// path where the certificate and key are stored.
type KubeCertificateName string
const (
@@ -44,8 +53,8 @@
// Kubernetes apiserver server certificate.
APIServer KubeCertificateName = "apiserver"
- // Kubelet client certificate, used to authenticate to the apiserver.
- KubeletClient KubeCertificateName = "kubelet-client"
+ // APIServer client certificate used to authenticate to kubelets.
+ APIServerKubeletClient KubeCertificateName = "apiserver-kubelet-client"
// Kubernetes Controller manager client certificate, used to authenticate to the apiserver.
ControllerManagerClient KubeCertificateName = "controller-manager-client"
@@ -68,32 +77,37 @@
)
const (
+ // etcdPrefix is where all the PKI data is stored in etcd.
+ etcdPrefix = "/kube-pki/"
// serviceAccountKeyName is the etcd path part that is used to store the ServiceAccount authentication secret.
// This is not a certificate, just an RSA key.
serviceAccountKeyName = "service-account-privkey"
)
-// KubernetesPKI manages all PKI resources required to run Kubernetes on Metropolis. It contains all static certificates,
-// which can be retrieved, or be used to generate Kubeconfigs from.
-type KubernetesPKI struct {
+// PKI manages all PKI resources required to run Kubernetes on Metropolis. It
+// contains all static certificates, which can be retrieved, or be used to
+// generate Kubeconfigs from.
+type PKI struct {
+ namespace opki.Namespace
logger logtree.LeveledLogger
KV clientv3.KV
- Certificates map[KubeCertificateName]*Certificate
+ Certificates map[KubeCertificateName]*opki.Certificate
}
-func NewKubernetes(l logtree.LeveledLogger, kv clientv3.KV) *KubernetesPKI {
- pki := KubernetesPKI{
+func New(l logtree.LeveledLogger, kv clientv3.KV) *PKI {
+ pki := PKI{
+ namespace: opki.Namespaced(etcdPrefix),
logger: l,
KV: kv,
- Certificates: make(map[KubeCertificateName]*Certificate),
+ Certificates: make(map[KubeCertificateName]*opki.Certificate),
}
make := func(i, name KubeCertificateName, template x509.Certificate) {
- pki.Certificates[name] = New(pki.Certificates[i], string(name), template)
+ pki.Certificates[name] = pki.namespace.New(pki.Certificates[i], string(name), template)
}
- pki.Certificates[IdCA] = New(SelfSigned, string(IdCA), CA("Metropolis Kubernetes ID CA"))
- make(IdCA, APIServer, Server(
+ pki.Certificates[IdCA] = pki.namespace.New(opki.SelfSigned, string(IdCA), opki.CA("Metropolis Kubernetes ID CA"))
+ make(IdCA, APIServer, opki.Server(
[]string{
"kubernetes",
"kubernetes.default",
@@ -104,21 +118,21 @@
},
[]net.IP{{10, 0, 255, 1}, {127, 0, 0, 1}}, // TODO(q3k): add service network internal apiserver address
))
- make(IdCA, KubeletClient, Client("metropolis:apiserver-kubelet-client", nil))
- make(IdCA, ControllerManagerClient, Client("system:kube-controller-manager", nil))
- make(IdCA, ControllerManager, Server([]string{"kube-controller-manager.local"}, nil))
- make(IdCA, SchedulerClient, Client("system:kube-scheduler", nil))
- make(IdCA, Scheduler, Server([]string{"kube-scheduler.local"}, nil))
- make(IdCA, Master, Client("metropolis:master", []string{"system:masters"}))
+ make(IdCA, APIServerKubeletClient, opki.Client("metropolis:apiserver-kubelet-client", nil))
+ make(IdCA, ControllerManagerClient, opki.Client("system:kube-controller-manager", nil))
+ make(IdCA, ControllerManager, opki.Server([]string{"kube-controller-manager.local"}, nil))
+ make(IdCA, SchedulerClient, opki.Client("system:kube-scheduler", nil))
+ make(IdCA, Scheduler, opki.Server([]string{"kube-scheduler.local"}, nil))
+ make(IdCA, Master, opki.Client("metropolis:master", []string{"system:masters"}))
- pki.Certificates[AggregationCA] = New(SelfSigned, string(AggregationCA), CA("Metropolis OpenAPI Aggregation CA"))
- make(AggregationCA, FrontProxyClient, Client("front-proxy-client", nil))
+ pki.Certificates[AggregationCA] = pki.namespace.New(opki.SelfSigned, string(AggregationCA), opki.CA("Metropolis OpenAPI Aggregation CA"))
+ make(AggregationCA, FrontProxyClient, opki.Client("front-proxy-client", nil))
return &pki
}
// EnsureAll ensures that all static certificates (and the serviceaccount key) are present on etcd.
-func (k *KubernetesPKI) EnsureAll(ctx context.Context) error {
+func (k *PKI) EnsureAll(ctx context.Context) error {
for n, v := range k.Certificates {
k.logger.Infof("Ensuring %s exists", string(n))
_, _, err := v.Ensure(ctx, k.KV)
@@ -135,18 +149,18 @@
// Kubeconfig generates a kubeconfig blob for a given certificate name. The same lifetime semantics as in .Certificate
// apply.
-func (k *KubernetesPKI) Kubeconfig(ctx context.Context, name KubeCertificateName) ([]byte, error) {
+func (k *PKI) Kubeconfig(ctx context.Context, name KubeCertificateName) ([]byte, error) {
c, ok := k.Certificates[name]
if !ok {
return nil, fmt.Errorf("no certificate %q", name)
}
- return c.Kubeconfig(ctx, k.KV)
+ return Kubeconfig(ctx, k.KV, c)
}
// Certificate retrieves an x509 DER-encoded (but not PEM-wrapped) key and certificate for a given certificate name.
// If the requested certificate is volatile, it will be created on demand. Otherwise it will be created on etcd (if not
// present), and retrieved from there.
-func (k *KubernetesPKI) Certificate(ctx context.Context, name KubeCertificateName) (cert, key []byte, err error) {
+func (k *PKI) Certificate(ctx context.Context, name KubeCertificateName) (cert, key []byte, err error) {
c, ok := k.Certificates[name]
if !ok {
return nil, nil, fmt.Errorf("no certificate %q", name)
@@ -155,7 +169,7 @@
}
// Kubeconfig generates a kubeconfig blob for this certificate. The same lifetime semantics as in .Ensure apply.
-func (c *Certificate) Kubeconfig(ctx context.Context, kv clientv3.KV) ([]byte, error) {
+func Kubeconfig(ctx context.Context, kv clientv3.KV, c *opki.Certificate) ([]byte, error) {
cert, key, err := c.Ensure(ctx, kv)
if err != nil {
@@ -167,7 +181,7 @@
cluster := configapi.NewCluster()
cluster.Server = fmt.Sprintf("https://127.0.0.1:%v", common.KubernetesAPIPort)
- ca, err := c.issuer.CACertificate(ctx, kv)
+ ca, err := c.Issuer.CACertificate(ctx, kv)
if err != nil {
return nil, fmt.Errorf("could not get CA certificate: %w", err)
}
@@ -190,13 +204,12 @@
return clientcmd.Write(*kubeconfig)
}
-// ServiceAccountKey retrieves (and possible generates and stores on etcd) the Kubernetes service account key. The
+// ServiceAccountKey retrieves (and possibly generates and stores on etcd) the Kubernetes service account key. The
// returned data is ready to be used by Kubernetes components (in PKIX form).
-func (k *KubernetesPKI) ServiceAccountKey(ctx context.Context) ([]byte, error) {
+func (k *PKI) ServiceAccountKey(ctx context.Context) ([]byte, error) {
// TODO(q3k): this should be abstracted away once we abstract away etcd access into a library with try-or-create
// semantics.
-
- path := etcdPath("%s.der", serviceAccountKeyName)
+ path := fmt.Sprintf("%s%s.der", etcdPrefix, serviceAccountKeyName)
// Try loading key from etcd.
keyRes, err := k.KV.Get(ctx, path)
@@ -226,3 +239,29 @@
}
return key, nil
}
+
+// VolatileKubelet returns a pair of server/client ceritficates for the Kubelet
+// to use. The certificates are volatile, meaning they are not stored in etcd,
+// and instead are regenerated any time this function is called.
+func (k *PKI) VolatileKubelet(ctx context.Context, name string) (server *opki.Certificate, client *opki.Certificate, err error) {
+ name = fmt.Sprintf("system:node:%s", name)
+ err = k.EnsureAll(ctx)
+ if err != nil {
+ err = fmt.Errorf("could not ensure certificates exist: %w", err)
+ }
+ kubeCA := k.Certificates[IdCA]
+ server = k.namespace.New(kubeCA, "", opki.Server([]string{name}, nil))
+ client = k.namespace.New(kubeCA, "", opki.Client(name, []string{"system:nodes"}))
+ return
+}
+
+// VolatileClient returns a client certificate for Kubernetes clients to use.
+// The generated certificate will place the user in the given groups, and with
+// a given identiy as the certificate's CN.
+func (k *PKI) VolatileClient(ctx context.Context, identity string, groups []string) (*opki.Certificate, error) {
+ if err := k.EnsureAll(ctx); err != nil {
+ return nil, fmt.Errorf("could not ensure certificates exist: %w", err)
+ }
+ kubeCA := k.Certificates[IdCA]
+ return k.namespace.New(kubeCA, "", opki.Client(identity, groups)), nil
+}
diff --git a/metropolis/node/kubernetes/scheduler.go b/metropolis/node/kubernetes/scheduler.go
index adb77ea..15410d6 100644
--- a/metropolis/node/kubernetes/scheduler.go
+++ b/metropolis/node/kubernetes/scheduler.go
@@ -33,7 +33,7 @@
serverKey []byte
}
-func getPKISchedulerConfig(ctx context.Context, kpki *pki.KubernetesPKI) (*schedulerConfig, error) {
+func getPKISchedulerConfig(ctx context.Context, kpki *pki.PKI) (*schedulerConfig, error) {
var config schedulerConfig
var err error
config.serverCert, config.serverKey, err = kpki.Certificate(ctx, pki.Scheduler)
diff --git a/metropolis/node/kubernetes/service.go b/metropolis/node/kubernetes/service.go
index 2f9b18e..8d0c795 100644
--- a/metropolis/node/kubernetes/service.go
+++ b/metropolis/node/kubernetes/service.go
@@ -44,7 +44,7 @@
ServiceIPRange net.IPNet
ClusterNet net.IPNet
- KPKI *pki.KubernetesPKI
+ KPKI *pki.PKI
Root *localstorage.Root
CorednsRegistrationChan chan *dns.ExtraDirective
}
@@ -168,10 +168,13 @@
// GetDebugKubeconfig issues a kubeconfig for an arbitrary given identity. Useful for debugging and testing.
func (s *Service) GetDebugKubeconfig(ctx context.Context, request *apb.GetDebugKubeconfigRequest) (*apb.GetDebugKubeconfigResponse, error) {
- ca := s.c.KPKI.Certificates[pki.IdCA]
- debugKubeconfig, err := pki.New(ca, "", pki.Client(request.Id, request.Groups)).Kubeconfig(ctx, s.c.KPKI.KV)
+ client, err := s.c.KPKI.VolatileClient(ctx, request.Id, request.Groups)
+ if err != nil {
+ return nil, status.Errorf(codes.Unavailable, "Failed to get volatile client certificate: %v", err)
+ }
+ kubeconfig, err := pki.Kubeconfig(ctx, s.c.KPKI.KV, client)
if err != nil {
return nil, status.Errorf(codes.Unavailable, "Failed to generate kubeconfig: %v", err)
}
- return &apb.GetDebugKubeconfigResponse{DebugKubeconfig: string(debugKubeconfig)}, nil
+ return &apb.GetDebugKubeconfigResponse{DebugKubeconfig: string(kubeconfig)}, nil
}