core/internal/kubernetes: refactor PKI fully
We move ad-hoc certificate/key creation to a little declarative,
future-inspired API.
The API is split into two distinct layers:
- an etcd-backed managed certificate storage that understands server
certificates, client certificates and CAs
- a Kubernetes PKI object, that understands what certificates are
needed to bring up a cluster
This allows for deduplicated path names in etcd, some semantic
information about available certificates, and is in general groundwork
for some future improvements, like:
- a slightly higher level etcd 'data store' api, with
less-stringly-typed paths
- simplification of service startup code (there's a bunch of cleanups
that can be still done in core/internal/kubernetes wrt. to
certificate marshaling to the filesystem, etc)
Test Plan: covered by existing tests - but this should also now be nicely testable in isolation!
X-Origin-Diff: phab/D564
GitOrigin-RevId: a58620c37ac064a15b7db106b7a5cbe9bd0b7cd0
diff --git a/core/internal/kubernetes/BUILD.bazel b/core/internal/kubernetes/BUILD.bazel
index f3304cc..97387df 100644
--- a/core/internal/kubernetes/BUILD.bazel
+++ b/core/internal/kubernetes/BUILD.bazel
@@ -4,7 +4,6 @@
name = "go_default_library",
srcs = [
"apiserver.go",
- "auth.go",
"controller-manager.go",
"csi.go",
"kubelet.go",
@@ -19,6 +18,7 @@
"//core/internal/common:go_default_library",
"//core/internal/common/supervisor:go_default_library",
"//core/internal/consensus:go_default_library",
+ "//core/internal/kubernetes/pki:go_default_library",
"//core/internal/kubernetes/reconciler:go_default_library",
"//core/internal/storage:go_default_library",
"//core/pkg/fileargs:go_default_library",
@@ -39,7 +39,6 @@
"@io_k8s_client_go//kubernetes/typed/core/v1:go_default_library",
"@io_k8s_client_go//tools/cache:go_default_library",
"@io_k8s_client_go//tools/clientcmd:go_default_library",
- "@io_k8s_client_go//tools/clientcmd/api:go_default_library",
"@io_k8s_client_go//tools/record:go_default_library",
"@io_k8s_client_go//tools/reference:go_default_library",
"@io_k8s_client_go//util/workqueue:go_default_library",
diff --git a/core/internal/kubernetes/apiserver.go b/core/internal/kubernetes/apiserver.go
index 9bc32f3..0a740dd 100644
--- a/core/internal/kubernetes/apiserver.go
+++ b/core/internal/kubernetes/apiserver.go
@@ -19,19 +19,17 @@
import (
"context"
"encoding/pem"
- "errors"
"fmt"
"io"
"net"
"os/exec"
- "path"
"git.monogon.dev/source/nexantic.git/core/internal/common"
+ "git.monogon.dev/source/nexantic.git/core/internal/common/supervisor"
+ "git.monogon.dev/source/nexantic.git/core/internal/kubernetes/pki"
+ "git.monogon.dev/source/nexantic.git/core/pkg/fileargs"
"go.etcd.io/etcd/clientv3"
-
- "git.monogon.dev/source/nexantic.git/core/internal/common/supervisor"
- "git.monogon.dev/source/nexantic.git/core/pkg/fileargs"
)
type apiserverConfig struct {
@@ -49,22 +47,37 @@
serverKey []byte
}
-func getPKIApiserverConfig(consensusKV clientv3.KV) (*apiserverConfig, error) {
+func getPKIApiserverConfig(ctx context.Context, kv clientv3.KV, kpki *pki.KubernetesPKI) (*apiserverConfig, error) {
var config apiserverConfig
+
+ for _, el := range []struct {
+ targetCert *[]byte
+ targetKey *[]byte
+ name pki.KubeCertificateName
+ }{
+ {&config.idCA, nil, pki.IdCA},
+ {&config.kubeletClientCert, &config.kubeletClientKey, pki.KubeletClient},
+ {&config.aggregationCA, nil, pki.AggregationCA},
+ {&config.aggregationClientCert, &config.aggregationClientKey, pki.FrontProxyClient},
+ {&config.serverCert, &config.serverKey, pki.APIServer},
+ } {
+ cert, key, err := kpki.Certificate(ctx, el.name, kv)
+ if err != nil {
+ return nil, fmt.Errorf("could not load certificate %q from PKI: %w", el.name, err)
+ }
+ if el.targetCert != nil {
+ *el.targetCert = cert
+ }
+ if el.targetKey != nil {
+ *el.targetKey = key
+ }
+ }
+
var err error
- config.idCA, _, err = getCert(consensusKV, "id-ca")
- config.kubeletClientCert, config.kubeletClientKey, err = getCert(consensusKV, "kubelet-client")
- config.aggregationCA, _, err = getCert(consensusKV, "aggregation-ca")
- config.aggregationClientCert, config.aggregationClientKey, err = getCert(consensusKV, "front-proxy-client")
- config.serverCert, config.serverKey, err = getCert(consensusKV, "apiserver")
- saPrivkey, err := consensusKV.Get(context.Background(), path.Join(etcdPath, "service-account-privkey.der"))
+ config.serviceAccountPrivKey, err = kpki.ServiceAccountKey(ctx, kv)
if err != nil {
- return nil, fmt.Errorf("failed to get serviceaccount privkey: %w", err)
+ return nil, fmt.Errorf("could not load serviceaccount privkey: %w", err)
}
- if len(saPrivkey.Kvs) != 1 {
- return nil, errors.New("failed to get serviceaccount privkey: not found")
- }
- config.serviceAccountPrivKey = saPrivkey.Kvs[0].Value
return &config, nil
}
diff --git a/core/internal/kubernetes/auth.go b/core/internal/kubernetes/auth.go
deleted file mode 100644
index fe2fe59..0000000
--- a/core/internal/kubernetes/auth.go
+++ /dev/null
@@ -1,399 +0,0 @@
-// Copyright 2020 The Monogon Project Authors.
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package kubernetes
-
-import (
- "context"
- "crypto"
- "crypto/ed25519"
- "crypto/rand"
- "crypto/rsa"
- "crypto/sha1"
- "crypto/x509"
- "crypto/x509/pkix"
- "encoding/asn1"
- "encoding/pem"
- "fmt"
- "math/big"
- "net"
- "os"
- "path"
- "time"
-
- "git.monogon.dev/source/nexantic.git/core/internal/common"
-
- "go.etcd.io/etcd/clientv3"
- "k8s.io/client-go/tools/clientcmd"
- configapi "k8s.io/client-go/tools/clientcmd/api"
-)
-
-const (
- etcdPath = "/kube-pki/"
-)
-
-var (
- // From RFC 5280 Section 4.1.2.5
- unknownNotAfter = time.Unix(253402300799, 0)
-)
-
-// Directly derived from Kubernetes PKI requirements documented at
-// https://kubernetes.io/docs/setup/best-practices/certificates/#configure-certificates-manually
-func clientCertTemplate(identity string, groups []string) x509.Certificate {
- return x509.Certificate{
- Subject: pkix.Name{
- CommonName: identity,
- Organization: groups,
- },
- KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
- ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
- }
-}
-func serverCertTemplate(dnsNames []string, ips []net.IP) x509.Certificate {
- return x509.Certificate{
- Subject: pkix.Name{},
- KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
- ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
- DNSNames: dnsNames,
- IPAddresses: ips,
- }
-}
-
-// Workaround for https://github.com/golang/go/issues/26676 in Go's crypto/x509. Specifically Go
-// violates Section 4.2.1.2 of RFC 5280 without this.
-// Fixed for 1.15 in https://go-review.googlesource.com/c/go/+/227098/.
-//
-// Taken from https://github.com/FiloSottile/mkcert/blob/master/cert.go#L295 written by one of Go's
-// crypto engineers
-func calculateSKID(pubKey crypto.PublicKey) ([]byte, error) {
- spkiASN1, err := x509.MarshalPKIXPublicKey(pubKey)
- if err != nil {
- return nil, err
- }
-
- var spki struct {
- Algorithm pkix.AlgorithmIdentifier
- SubjectPublicKey asn1.BitString
- }
- _, err = asn1.Unmarshal(spkiASN1, &spki)
- if err != nil {
- return nil, err
- }
- skid := sha1.Sum(spki.SubjectPublicKey.Bytes)
- return skid[:], nil
-}
-
-func newCA(name string) ([]byte, ed25519.PrivateKey, error) {
- pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
- if err != nil {
- panic(err)
- }
-
- serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 127)
- serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
- if err != nil {
- return []byte{}, privKey, fmt.Errorf("Failed to generate serial number: %w", err)
- }
-
- skid, err := calculateSKID(pubKey)
- if err != nil {
- return []byte{}, privKey, err
- }
-
- caCert := &x509.Certificate{
- SerialNumber: serialNumber,
- Subject: pkix.Name{
- CommonName: name,
- },
- IsCA: true,
- BasicConstraintsValid: true,
- NotBefore: time.Now(),
- NotAfter: unknownNotAfter,
- KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature,
- ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageOCSPSigning},
- AuthorityKeyId: skid,
- SubjectKeyId: skid,
- }
-
- caCertRaw, err := x509.CreateCertificate(rand.Reader, caCert, caCert, pubKey, privKey)
- return caCertRaw, privKey, err
-}
-
-func storeCert(consensusKV clientv3.KV, name string, cert []byte, key []byte) error {
- certPath := path.Join(etcdPath, fmt.Sprintf("%v-cert.der", name))
- keyPath := path.Join(etcdPath, fmt.Sprintf("%v-key.der", name))
- if _, err := consensusKV.Put(context.Background(), certPath, string(cert)); err != nil {
- return fmt.Errorf("failed to store certificate: %w", err)
- }
- if _, err := consensusKV.Put(context.Background(), keyPath, string(key)); err != nil {
- return fmt.Errorf("failed to store key: %w", err)
- }
- return nil
-}
-
-func getCert(consensusKV clientv3.KV, name string) (cert []byte, key []byte, err error) {
- certPath := path.Join(etcdPath, fmt.Sprintf("%v-cert.der", name))
- keyPath := path.Join(etcdPath, fmt.Sprintf("%v-key.der", name))
- certRes, err := consensusKV.Get(context.Background(), certPath)
- if err != nil {
- err = fmt.Errorf("failed to get certificate: %w", err)
- return
- }
- keyRes, err := consensusKV.Get(context.Background(), keyPath)
- if err != nil {
- err = fmt.Errorf("failed to get certificate: %w", err)
- return
- }
- if len(certRes.Kvs) != 1 || len(keyRes.Kvs) != 1 {
- err = fmt.Errorf("failed to find certificate %v", name)
- return
- }
- cert = certRes.Kvs[0].Value
- key = keyRes.Kvs[0].Value
- return
-}
-
-func getSingle(consensusKV clientv3.KV, name string) ([]byte, error) {
- res, err := consensusKV.Get(context.Background(), path.Join(etcdPath, name))
- if err != nil {
- return []byte{}, fmt.Errorf("failed to get PKI item: %w", err)
- }
- if len(res.Kvs) != 1 {
- return []byte{}, fmt.Errorf("failed to find PKI item %v", name)
- }
- return res.Kvs[0].Value, nil
-}
-
-// newCluster initializes the whole PKI for Kubernetes. It issues a single certificate per control
-// plane service since it assumes that etcd is already a secure place to store data. This removes
-// the need for revocation and makes the logic much simpler. Thus PKI data can NEVER be stored
-// outside of etcd or other secure storage locations. All PKI data is stored in DER form and not
-// PEM encoded since that would require more logic to deal with it.
-func newCluster(consensusKV clientv3.KV) error {
- // This whole issuance procedure is pretty repetitive, but abstracts badly because a lot of it
- // is subtly different.
- idCA, idKey, err := newCA("Smalltown Kubernetes ID CA")
- if err != nil {
- return fmt.Errorf("failed to create Kubernetes ID CA: %w", err)
- }
- if err := storeCert(consensusKV, "id-ca", idCA, idKey); err != nil {
- return err
- }
- aggregationCA, aggregationKey, err := newCA("Smalltown OpenAPI Aggregation CA")
- if err != nil {
- return fmt.Errorf("failed to create OpenAPI Aggregation CA: %w", err)
- }
- if err := storeCert(consensusKV, "aggregation-ca", aggregationCA, aggregationKey); err != nil {
- return err
- }
-
- // ServiceAccounts don't support ed25519 yet, so use RSA (better side-channel resistance than ECDSA)
- serviceAccountPrivKeyRaw, err := rsa.GenerateKey(rand.Reader, 2048)
- if err != nil {
- panic(err)
- }
- serviceAccountPrivKey, err := x509.MarshalPKCS8PrivateKey(serviceAccountPrivKeyRaw)
- if err != nil {
- panic(err) // Always a programmer error
- }
- _, err = consensusKV.Put(context.Background(), path.Join(etcdPath, "service-account-privkey.der"),
- string(serviceAccountPrivKey))
- if err != nil {
- return fmt.Errorf("failed to store service-account-privkey.der: %w", err)
- }
-
- apiserverCert, apiserverKey, err := issueCertificate(
- serverCertTemplate([]string{
- "kubernetes",
- "kubernetes.default",
- "kubernetes.default.svc",
- "kubernetes.default.svc.cluster",
- "kubernetes.default.svc.cluster.local",
- "localhost",
- }, []net.IP{{127, 0, 0, 1}}, // TODO: Add service internal IP
- ),
- idCA, idKey,
- )
- if err != nil {
- return fmt.Errorf("failed to issue certificate for apiserver: %w", err)
- }
- if err := storeCert(consensusKV, "apiserver", apiserverCert, apiserverKey); err != nil {
- return err
- }
-
- kubeletClientCert, kubeletClientKey, err := issueCertificate(
- clientCertTemplate("smalltown:apiserver-kubelet-client", []string{}),
- idCA, idKey,
- )
- if err != nil {
- return fmt.Errorf("failed to issue certificate for kubelet client: %w", err)
- }
- if err := storeCert(consensusKV, "kubelet-client", kubeletClientCert, kubeletClientKey); err != nil {
- return err
- }
-
- frontProxyClientCert, frontProxyClientKey, err := issueCertificate(
- clientCertTemplate("front-proxy-client", []string{}),
- aggregationCA, aggregationKey,
- )
- if err != nil {
- return fmt.Errorf("failed to issue certificate for OpenAPI frontend: %w", err)
- }
- if err := storeCert(consensusKV, "front-proxy-client", frontProxyClientCert, frontProxyClientKey); err != nil {
- return err
- }
-
- controllerManagerClientCert, controllerManagerClientKey, err := issueCertificate(
- clientCertTemplate("system:kube-controller-manager", []string{}),
- idCA, idKey,
- )
- if err != nil {
- return fmt.Errorf("failed to issue certificate for controller-manager client: %w", err)
- }
-
- controllerManagerKubeconfig, err := makeLocalKubeconfig(idCA, controllerManagerClientCert,
- controllerManagerClientKey)
- if err != nil {
- return fmt.Errorf("failed to create kubeconfig for controller-manager: %w", err)
- }
-
- _, err = consensusKV.Put(context.Background(), path.Join(etcdPath, "controller-manager.kubeconfig"),
- string(controllerManagerKubeconfig))
- if err != nil {
- return fmt.Errorf("failed to store controller-manager kubeconfig: %w", err)
- }
-
- controllerManagerCert, controllerManagerKey, err := issueCertificate(
- serverCertTemplate([]string{"kube-controller-manager.local"}, []net.IP{}),
- idCA, idKey,
- )
- if err != nil {
- return fmt.Errorf("failed to issue certificate for controller-manager: %w", err)
- }
- if err := storeCert(consensusKV, "controller-manager", controllerManagerCert, controllerManagerKey); err != nil {
- return err
- }
-
- schedulerClientCert, schedulerClientKey, err := issueCertificate(
- clientCertTemplate("system:kube-scheduler", []string{}),
- idCA, idKey,
- )
- if err != nil {
- return fmt.Errorf("failed to issue certificate for scheduler client: %w", err)
- }
-
- schedulerKubeconfig, err := makeLocalKubeconfig(idCA, schedulerClientCert, schedulerClientKey)
- if err != nil {
- return fmt.Errorf("failed to create kubeconfig for scheduler: %w", err)
- }
-
- _, err = consensusKV.Put(context.Background(), path.Join(etcdPath, "scheduler.kubeconfig"),
- string(schedulerKubeconfig))
- if err != nil {
- return fmt.Errorf("failed to store controller-manager kubeconfig: %w", err)
- }
-
- schedulerCert, schedulerKey, err := issueCertificate(
- serverCertTemplate([]string{"kube-scheduler.local"}, []net.IP{}),
- idCA, idKey,
- )
- if err != nil {
- return fmt.Errorf("failed to issue certificate for scheduler: %w", err)
- }
- if err := storeCert(consensusKV, "scheduler", schedulerCert, schedulerKey); err != nil {
- return err
- }
-
- masterClientCert, masterClientKey, err := issueCertificate(
- clientCertTemplate("smalltown:master", []string{"system:masters"}),
- idCA, idKey,
- )
- if err != nil {
- return fmt.Errorf("failed to issue certificate for master client: %w", err)
- }
-
- masterClientKubeconfig, err := makeLocalKubeconfig(idCA, masterClientCert,
- masterClientKey)
- if err != nil {
- return fmt.Errorf("failed to create kubeconfig for master client: %w", err)
- }
-
- _, err = consensusKV.Put(context.Background(), path.Join(etcdPath, "master.kubeconfig"),
- string(masterClientKubeconfig))
- if err != nil {
- return fmt.Errorf("failed to store master kubeconfig: %w", err)
- }
-
- hostname, err := os.Hostname()
- if err != nil {
- return err
- }
- if err := bootstrapLocalKubelet(consensusKV, hostname); err != nil {
- return err
- }
-
- return nil
-}
-
-func issueCertificate(template x509.Certificate, caCert []byte, privateKey interface{}) (cert []byte, privkey []byte, err error) {
- serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 127)
- serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
- if err != nil {
- err = fmt.Errorf("Failed to generate serial number: %w", err)
- return
- }
-
- caCertObj, err := x509.ParseCertificate(caCert)
- if err != nil {
- err = fmt.Errorf("failed to parse CA certificate: %w", err)
- }
-
- pubKey, privKeyRaw, err := ed25519.GenerateKey(rand.Reader)
- if err != nil {
- return
- }
- privkey, err = x509.MarshalPKCS8PrivateKey(privKeyRaw)
- if err != nil {
- return
- }
-
- template.SerialNumber = serialNumber
- template.IsCA = false
- template.BasicConstraintsValid = true
- template.NotBefore = time.Now()
- template.NotAfter = unknownNotAfter
-
- cert, err = x509.CreateCertificate(rand.Reader, &template, caCertObj, pubKey, privateKey)
- return
-}
-
-func makeLocalKubeconfig(ca, cert, key []byte) ([]byte, error) {
- kubeconfig := configapi.NewConfig()
- cluster := configapi.NewCluster()
- cluster.Server = fmt.Sprintf("https://127.0.0.1:%v", common.KubernetesAPIPort)
- cluster.CertificateAuthorityData = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ca})
- kubeconfig.Clusters["default"] = cluster
- authInfo := configapi.NewAuthInfo()
- authInfo.ClientCertificateData = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert})
- authInfo.ClientKeyData = pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: key})
- kubeconfig.AuthInfos["default"] = authInfo
- ctx := configapi.NewContext()
- ctx.Cluster = "default"
- ctx.AuthInfo = "default"
- kubeconfig.Contexts["default"] = ctx
- kubeconfig.CurrentContext = "default"
- return clientcmd.Write(*kubeconfig)
-}
diff --git a/core/internal/kubernetes/controller-manager.go b/core/internal/kubernetes/controller-manager.go
index 20d4605..0934ae1 100644
--- a/core/internal/kubernetes/controller-manager.go
+++ b/core/internal/kubernetes/controller-manager.go
@@ -27,6 +27,7 @@
"go.etcd.io/etcd/clientv3"
"git.monogon.dev/source/nexantic.git/core/internal/common/supervisor"
+ "git.monogon.dev/source/nexantic.git/core/internal/kubernetes/pki"
"git.monogon.dev/source/nexantic.git/core/pkg/fileargs"
)
@@ -40,22 +41,22 @@
serverKey []byte
}
-func getPKIControllerManagerConfig(consensusKV clientv3.KV) (*controllerManagerConfig, error) {
+func getPKIControllerManagerConfig(ctx context.Context, kv clientv3.KV, kpki *pki.KubernetesPKI) (*controllerManagerConfig, error) {
var config controllerManagerConfig
var err error
- config.rootCA, _, err = getCert(consensusKV, "id-ca")
+ config.rootCA, _, err = kpki.Certificate(ctx, pki.IdCA, kv)
if err != nil {
return nil, fmt.Errorf("failed to get ID root CA: %w", err)
}
- config.serverCert, config.serverKey, err = getCert(consensusKV, "controller-manager")
+ config.serverCert, config.serverKey, err = kpki.Certificate(ctx, pki.ControllerManager, kv)
if err != nil {
return nil, fmt.Errorf("failed to get controller-manager serving certificate: %w", err)
}
- config.serviceAccountPrivKey, err = getSingle(consensusKV, "service-account-privkey.der")
+ config.serviceAccountPrivKey, err = kpki.ServiceAccountKey(ctx, kv)
if err != nil {
return nil, fmt.Errorf("failed to get serviceaccount privkey: %w", err)
}
- config.kubeConfig, err = getSingle(consensusKV, "controller-manager.kubeconfig")
+ config.kubeConfig, err = kpki.Kubeconfig(ctx, pki.ControllerManagerClient, kv)
if err != nil {
return nil, fmt.Errorf("failed to get controller-manager kubeconfig: %w", err)
}
diff --git a/core/internal/kubernetes/kubelet.go b/core/internal/kubernetes/kubelet.go
index 3b0d966..639e891 100644
--- a/core/internal/kubernetes/kubelet.go
+++ b/core/internal/kubernetes/kubelet.go
@@ -18,7 +18,6 @@
import (
"context"
- "crypto/ed25519"
"encoding/json"
"encoding/pem"
"fmt"
@@ -28,52 +27,64 @@
"os"
"os/exec"
+ "go.etcd.io/etcd/clientv3"
+
"git.monogon.dev/source/nexantic.git/core/internal/common/supervisor"
+ "git.monogon.dev/source/nexantic.git/core/internal/kubernetes/pki"
"git.monogon.dev/source/nexantic.git/core/internal/kubernetes/reconciler"
"git.monogon.dev/source/nexantic.git/core/pkg/fileargs"
- "go.etcd.io/etcd/clientv3"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubeletconfig "k8s.io/kubelet/config/v1beta1"
)
+var (
+ kubeletRoot = "/data/kubernetes"
+ kubeletKubeconfig = kubeletRoot + "/kubelet.kubeconfig"
+ kubeletCACert = kubeletRoot + "/ca.crt"
+ kubeletCert = kubeletRoot + "/kubelet.crt"
+ kubeletKey = kubeletRoot + "/kubelet.key"
+)
+
type KubeletSpec struct {
clusterDNS []net.IP
}
-func bootstrapLocalKubelet(consensusKV clientv3.KV, nodeName string) error {
- idCA, idKeyRaw, err := getCert(consensusKV, "id-ca")
+func createKubeletConfig(ctx context.Context, kv clientv3.KV, kpki *pki.KubernetesPKI, nodeName string) error {
+ identity := fmt.Sprintf("system:node:%s", nodeName)
+
+ ca := kpki.Certificates[pki.IdCA]
+ cacert, _, err := ca.Ensure(ctx, kv)
if err != nil {
- return err
- }
- idKey := ed25519.PrivateKey(idKeyRaw)
- cert, key, err := issueCertificate(clientCertTemplate("system:node:"+nodeName, []string{"system:nodes"}), idCA, idKey)
- if err != nil {
- return err
- }
- kubeconfig, err := makeLocalKubeconfig(idCA, cert, key)
- if err != nil {
- return err
+ return fmt.Errorf("could not ensure ca certificate: %w", err)
}
- serverCert, serverKey, err := issueCertificate(serverCertTemplate([]string{nodeName}, []net.IP{}), idCA, idKey)
+ kubeconfig, err := pki.New(ca, "", pki.Client(identity, []string{"system:nodes"})).Kubeconfig(ctx, kv)
if err != nil {
- return err
+ return fmt.Errorf("could not create volatile kubelet client cert: %w", err)
}
- if err := os.MkdirAll("/data/kubernetes", 0755); err != nil {
- return err
+
+ cert, key, err := pki.New(ca, "volatile", pki.Server([]string{nodeName}, nil)).Ensure(ctx, kv)
+ if err != nil {
+ return fmt.Errorf("could not create volatile kubelet server cert: %w", err)
}
- if err := ioutil.WriteFile("/data/kubernetes/kubelet.kubeconfig", kubeconfig, 0400); err != nil {
- return err
+
+ if err := os.MkdirAll(kubeletRoot, 0755); err != nil {
+ return fmt.Errorf("could not create kubelet root directory: %w", err)
}
- if err := ioutil.WriteFile("/data/kubernetes/ca.crt", pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: idCA}), 0400); err != nil {
- return err
- }
- if err := ioutil.WriteFile("/data/kubernetes/kubelet.crt", pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: serverCert}), 0400); err != nil {
- return err
- }
- if err := ioutil.WriteFile("/data/kubernetes/kubelet.key", pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: serverKey}), 0400); err != nil {
- return err
+ // TODO(q3k): this should probably become its own function //core/internal/kubernetes/pki.
+ for _, el := range []struct {
+ target string
+ data []byte
+ }{
+ {kubeletKubeconfig, kubeconfig},
+ {kubeletCACert, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cacert})},
+ {kubeletCert, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert})},
+ {kubeletKey, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: key})},
+ } {
+ if err := ioutil.WriteFile(el.target, el.data, 0400); err != nil {
+ return fmt.Errorf("could not write %q: %w", el.target, err)
+ }
}
return nil
diff --git a/core/internal/kubernetes/pki/BUILD.bazel b/core/internal/kubernetes/pki/BUILD.bazel
new file mode 100644
index 0000000..e188bfa
--- /dev/null
+++ b/core/internal/kubernetes/pki/BUILD.bazel
@@ -0,0 +1,19 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+ name = "go_default_library",
+ srcs = [
+ "ca.go",
+ "certificate.go",
+ "kubernetes.go",
+ ],
+ importpath = "git.monogon.dev/source/nexantic.git/core/internal/kubernetes/pki",
+ visibility = ["//core:__subpackages__"],
+ deps = [
+ "//core/internal/common:go_default_library",
+ "@io_etcd_go_etcd//clientv3:go_default_library",
+ "@io_k8s_client_go//tools/clientcmd:go_default_library",
+ "@io_k8s_client_go//tools/clientcmd/api:go_default_library",
+ "@org_uber_go_zap//:go_default_library",
+ ],
+)
diff --git a/core/internal/kubernetes/pki/ca.go b/core/internal/kubernetes/pki/ca.go
new file mode 100644
index 0000000..64453cd
--- /dev/null
+++ b/core/internal/kubernetes/pki/ca.go
@@ -0,0 +1,151 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pki
+
+import (
+ "context"
+ "crypto"
+ "crypto/ed25519"
+ "crypto/rand"
+ "crypto/sha1"
+ "crypto/x509"
+ "crypto/x509/pkix"
+ "encoding/asn1"
+ "fmt"
+ "math/big"
+ "time"
+
+ "go.etcd.io/etcd/clientv3"
+)
+
+// Issuer is a CA that can issue certificates. Two issuers are currently implemented:
+// - SelfSigned, which will generated a certificate signed by its corresponding private key.
+// - Certificate, which will use another existing Certificate as a CA.
+type Issuer interface {
+ // CACertificate returns the DER-encoded x509 certificate of the CA that will sign certificates when Issue is
+ // called, or nil if this is self-signing issuer.
+ CACertificate(ctx context.Context, kv clientv3.KV) ([]byte, error)
+ // Issue will generate a key and certificate signed by the Issuer. The returned certificate is x509 DER-encoded,
+ // while the key is a bare ed25519 key.
+ Issue(ctx context.Context, template x509.Certificate, kv clientv3.KV) (cert, key []byte, err error)
+}
+
+var (
+ // From RFC 5280 Section 4.1.2.5
+ unknownNotAfter = time.Unix(253402300799, 0)
+)
+
+// Workaround for https://github.com/golang/go/issues/26676 in Go's crypto/x509. Specifically Go
+// violates Section 4.2.1.2 of RFC 5280 without this.
+// Fixed for 1.15 in https://go-review.googlesource.com/c/go/+/227098/.
+//
+// Taken from https://github.com/FiloSottile/mkcert/blob/master/cert.go#L295 written by one of Go's
+// crypto engineers
+func calculateSKID(pubKey crypto.PublicKey) ([]byte, error) {
+ spkiASN1, err := x509.MarshalPKIXPublicKey(pubKey)
+ if err != nil {
+ return nil, err
+ }
+
+ var spki struct {
+ Algorithm pkix.AlgorithmIdentifier
+ SubjectPublicKey asn1.BitString
+ }
+ _, err = asn1.Unmarshal(spkiASN1, &spki)
+ if err != nil {
+ return nil, err
+ }
+ skid := sha1.Sum(spki.SubjectPublicKey.Bytes)
+ return skid[:], nil
+}
+
+// issueCertificate is a generic low level certificate-and-key issuance function. If ca or cakey is null, the
+// certificate will be self-signed. The returned certificate is DER-encoded, while the returned key is internal.
+func issueCertificate(template x509.Certificate, ca *x509.Certificate, caKey interface{}) (cert, key []byte, err error) {
+ pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ panic(err)
+ }
+
+ serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 127)
+ serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+ if err != nil {
+ err = fmt.Errorf("failed to generate serial number: %w", err)
+ return
+ }
+
+ skid, err := calculateSKID(pubKey)
+ if err != nil {
+ return []byte{}, privKey, err
+ }
+
+ template.SerialNumber = serialNumber
+ template.NotBefore = time.Now()
+ template.NotAfter = unknownNotAfter
+ template.BasicConstraintsValid = true
+ template.SubjectKeyId = skid
+
+ // Set the AuthorityKeyID to the SKID of the signing certificate (or self, if self-signing).
+ if ca != nil && caKey != nil {
+ template.AuthorityKeyId = ca.AuthorityKeyId
+ } else {
+ template.AuthorityKeyId = template.SubjectKeyId
+ }
+
+ if ca == nil || caKey == nil {
+ ca = &template
+ caKey = privKey
+ }
+
+ caCertRaw, err := x509.CreateCertificate(rand.Reader, &template, ca, pubKey, caKey)
+ return caCertRaw, privKey, err
+}
+
+type selfSigned struct{}
+
+func (s *selfSigned) Issue(ctx context.Context, template x509.Certificate, kv clientv3.KV) (cert, key []byte, err error) {
+ return issueCertificate(template, nil, nil)
+}
+
+func (s *selfSigned) CACertificate(ctx context.Context, kv clientv3.KV) ([]byte, error) {
+ return nil, nil
+}
+
+var (
+ // SelfSigned is an Issuer that generates self-signed certificates.
+ SelfSigned = &selfSigned{}
+)
+
+func (c *Certificate) Issue(ctx context.Context, template x509.Certificate, kv clientv3.KV) (cert, key []byte, err error) {
+ caCert, caKey, err := c.ensure(ctx, kv)
+ if err != nil {
+ return nil, nil, fmt.Errorf("could not ensure CA certificate %q exists: %w", c.name, err)
+ }
+
+ ca, err := x509.ParseCertificate(caCert)
+ if err != nil {
+ return nil, nil, fmt.Errorf("could not parse CA certificate: %w", err)
+ }
+ // Ensure only one level of CAs exist, and that they are created explicitly.
+ template.IsCA = false
+ return issueCertificate(template, ca, ed25519.PrivateKey(caKey))
+}
+
+func (c *Certificate) CACertificate(ctx context.Context, kv clientv3.KV) ([]byte, error) {
+ cert, _, err := c.ensure(ctx, kv)
+ return cert, err
+}
diff --git a/core/internal/kubernetes/pki/certificate.go b/core/internal/kubernetes/pki/certificate.go
new file mode 100644
index 0000000..e0dea0d
--- /dev/null
+++ b/core/internal/kubernetes/pki/certificate.go
@@ -0,0 +1,179 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pki
+
+import (
+ "context"
+ "crypto/ed25519"
+ "crypto/x509"
+ "crypto/x509/pkix"
+ "fmt"
+ "net"
+
+ "go.etcd.io/etcd/clientv3"
+)
+
+// Certificate is the promise of a Certificate being available to the caller. In this case, Certificate refers to a
+// pair of x509 certificate and corresponding private key.
+// Certificates can be stored in etcd, and their issuers might also be store on etcd. As such, this type's methods
+// contain references to an etcd KV client.
+// This Certificate type is agnostic to usage, but mostly geared towards Kubernetes certificates.
+type Certificate struct {
+ // issuer is the Issuer that will generate this certificate if one doesn't yet exist or etcd, or the requested
+ // certificate is volatile (not to be stored on etcd).
+ issuer Issuer
+ // name is a unique key for storing the certificate in etcd. If empty, certificate is 'volatile', will not be stored
+ // on etcd, and every .Ensure() call will generate a new pair.
+ name string
+ // template is an x509 certificate definition that will be used to generate the certificate when issuing it.
+ template x509.Certificate
+}
+
+const (
+ // etcdPrefix is where all the PKI data is stored in etcd.
+ etcdPrefix = "/kube-pki/"
+)
+
+func etcdPath(f string, args ...interface{}) string {
+ return etcdPrefix + fmt.Sprintf(f, args...)
+}
+
+// New creates a new Certificate, or to be more precise, a promise that a certificate will exist once Ensure is called.
+// Issuer must be a valid certificate issuer (SelfSigned or another Certificate). Name must be unique among all
+// certificates, or empty (which will cause the certificate to be volatile, ie. not stored in etcd).
+func New(issuer Issuer, name string, template x509.Certificate) *Certificate {
+ return &Certificate{
+ issuer: issuer,
+ name: name,
+ template: template,
+ }
+}
+
+// Client makes a Kubernetes PKI-compatible client certificate template.
+// Directly derived from Kubernetes PKI requirements documented at
+// https://kubernetes.io/docs/setup/best-practices/certificates/#configure-certificates-manually
+func Client(identity string, groups []string) x509.Certificate {
+ return x509.Certificate{
+ Subject: pkix.Name{
+ CommonName: identity,
+ Organization: groups,
+ },
+ KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+ ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+ }
+}
+
+// Server makes a Kubernetes PKI-compatible server certificate template.
+func Server(dnsNames []string, ips []net.IP) x509.Certificate {
+ return x509.Certificate{
+ Subject: pkix.Name{},
+ KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+ ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+ DNSNames: dnsNames,
+ IPAddresses: ips,
+ }
+}
+
+// CA makes a Certificate that can sign other certificates.
+func CA(cn string) x509.Certificate {
+ return x509.Certificate{
+ Subject: pkix.Name{
+ CommonName: cn,
+ },
+ IsCA: true,
+ KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature,
+ ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageOCSPSigning},
+ }
+}
+
+func (c *Certificate) etcdPaths() (cert, key string) {
+ return etcdPath("%s-cert.der", c.name), etcdPath("%s-key.der", c.name)
+}
+
+// ensure returns a DER-encoded x509 certificate and internally encoded bare ed25519 key for a given Certificate,
+// in memory (if volatile), loading it from etcd, or creating and saving it on etcd if needed.
+func (c *Certificate) ensure(ctx context.Context, kv clientv3.KV) (cert, key []byte, err error) {
+ if c.name == "" {
+ // Volatile certificate - generate.
+ // TODO(q3k): cache internally?
+ cert, key, err = c.issuer.Issue(ctx, c.template, kv)
+ if err != nil {
+ err = fmt.Errorf("failed to issue: %w", err)
+ return
+ }
+ return
+ }
+
+ certPath, keyPath := c.etcdPaths()
+
+ // Try loading certificate and key from etcd.
+ certRes, err := kv.Get(ctx, certPath)
+ if err != nil {
+ err = fmt.Errorf("failed to get certificate from etcd: %w", err)
+ return
+ }
+ keyRes, err := kv.Get(ctx, keyPath)
+ if err != nil {
+ err = fmt.Errorf("failed to get key from etcd: %w", err)
+ return
+ }
+
+ if len(certRes.Kvs) == 1 && len(keyRes.Kvs) == 1 {
+ // Certificate and key exists in etcd, return that.
+ cert = certRes.Kvs[0].Value
+ key = keyRes.Kvs[0].Value
+
+ err = nil
+ // TODO(q3k): check for expiration
+ return
+ }
+
+ // No certificate found - issue one.
+ cert, key, err = c.issuer.Issue(ctx, c.template, kv)
+ if err != nil {
+ err = fmt.Errorf("failed to issue: %w", err)
+ return
+ }
+
+ // Save to etcd in transaction. This ensures that no partial writes happen.
+ _, err = kv.Txn(ctx).
+ Then(
+ clientv3.OpPut(certPath, string(cert)),
+ clientv3.OpPut(keyPath, string(key)),
+ ).Commit()
+ if err != nil {
+ err = fmt.Errorf("failed to write newly issued certificate: %w", err)
+ }
+
+ return
+}
+
+// Ensure returns an x509 DER-encoded (but not PEM-encoded) certificate and key for a given Certificate.
+// If the certificate is volatile, each call to Ensure will cause a new certificate to be generated.
+// Otherwise, it will be retrieved from etcd, or generated and stored there if needed.
+func (c *Certificate) Ensure(ctx context.Context, kv clientv3.KV) (cert, key []byte, err error) {
+ cert, key, err = c.ensure(ctx, kv)
+ if err != nil {
+ return nil, nil, err
+ }
+ key, err = x509.MarshalPKCS8PrivateKey(ed25519.PrivateKey(key))
+ if err != nil {
+ err = fmt.Errorf("could not marshal private key (data corruption?): %w", err)
+ return
+ }
+ return cert, key, err
+}
diff --git a/core/internal/kubernetes/pki/kubernetes.go b/core/internal/kubernetes/pki/kubernetes.go
new file mode 100644
index 0000000..ed70b87
--- /dev/null
+++ b/core/internal/kubernetes/pki/kubernetes.go
@@ -0,0 +1,227 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pki
+
+import (
+ "context"
+ "crypto/rand"
+ "crypto/rsa"
+ "crypto/x509"
+ "encoding/pem"
+ "fmt"
+ "net"
+
+ "go.uber.org/zap"
+
+ "go.etcd.io/etcd/clientv3"
+ "k8s.io/client-go/tools/clientcmd"
+ configapi "k8s.io/client-go/tools/clientcmd/api"
+
+ "git.monogon.dev/source/nexantic.git/core/internal/common"
+)
+
+// KubeCertificateName is an enum-like unique name of a static Kubernetes certificate. The value of the name is used
+// as the unique part of an etcd path where the certificate and key are stored.
+type KubeCertificateName string
+
+const (
+ // The main Kubernetes CA, used to authenticate API consumers, and servers.
+ IdCA KubeCertificateName = "id-ca"
+
+ // Kubernetes apiserver server certificate.
+ APIServer KubeCertificateName = "apiserver"
+
+ // Kubelet client certificate, used to authenticate to the apiserver.
+ KubeletClient KubeCertificateName = "kubelet-client"
+
+ // Kubernetes Controller manager client certificate, used to authenticate to the apiserver.
+ ControllerManagerClient KubeCertificateName = "controller-manager-client"
+ // Kubernetes Controller manager server certificate, used to run its HTTP server.
+ ControllerManager KubeCertificateName = "controller-manager"
+
+ // Kubernetes Scheduler client certificate, used to authenticate to the apiserver.
+ SchedulerClient KubeCertificateName = "scheduler-client"
+ // Kubernetes scheduler server certificate, used to run its HTTP server.
+ Scheduler KubeCertificateName = "scheduler"
+
+ // Root-on-kube (system:masters) client certificate. Used to control the apiserver (and resources) by Smalltown
+ // internally.
+ Master KubeCertificateName = "master"
+
+ // OpenAPI Kubernetes Aggregation CA.
+ // See: https://kubernetes.io/docs/tasks/extend-kubernetes/configure-aggregation-layer/#ca-reusage-and-conflicts
+ AggregationCA KubeCertificateName = "aggregation-ca"
+ FrontProxyClient KubeCertificateName = "front-proxy-client"
+)
+
+const (
+ // serviceAccountKeyName is the etcd path part that is used to store the ServiceAccount authentication secret.
+ // This is not a certificate, just an RSA key.
+ serviceAccountKeyName = "service-account-privkey"
+)
+
+// KubernetesPKI manages all PKI resources required to run Kubernetes on Smalltown. It contains all static certificates,
+// which can be retrieved, or be used to generate Kubeconfigs from.
+type KubernetesPKI struct {
+ logger *zap.Logger
+ Certificates map[KubeCertificateName]*Certificate
+}
+
+func NewKubernetes(l *zap.Logger) *KubernetesPKI {
+ pki := KubernetesPKI{
+ logger: l,
+ Certificates: make(map[KubeCertificateName]*Certificate),
+ }
+
+ make := func(i, name KubeCertificateName, template x509.Certificate) {
+ pki.Certificates[name] = New(pki.Certificates[i], string(name), template)
+ }
+
+ pki.Certificates[IdCA] = New(SelfSigned, string(IdCA), CA("Smalltown Kubernetes ID CA"))
+ make(IdCA, APIServer, Server(
+ []string{
+ "kubernetes",
+ "kubernetes.default",
+ "kubernetes.default.svc",
+ "kubernetes.default.svc.cluster",
+ "kubernetes.default.svc.cluster.local",
+ "localhost",
+ },
+ []net.IP{{127, 0, 0, 1}}, // TODO(q3k): add service network internal apiserver address
+ ))
+ make(IdCA, KubeletClient, Client("smalltown:apiserver-kubelet-client", nil))
+ make(IdCA, ControllerManagerClient, Client("system:kube-controller-manager", nil))
+ make(IdCA, ControllerManager, Server([]string{"kube-controller-manager.local"}, nil))
+ make(IdCA, SchedulerClient, Client("system:kube-scheduler", nil))
+ make(IdCA, Scheduler, Server([]string{"kube-scheduler.local"}, nil))
+ make(IdCA, Master, Client("smalltown:master", []string{"system:masters"}))
+
+ pki.Certificates[AggregationCA] = New(SelfSigned, string(AggregationCA), CA("Smalltown OpenAPI Aggregation CA"))
+ make(AggregationCA, FrontProxyClient, Client("front-proxy-client", nil))
+
+ return &pki
+}
+
+// EnsureAll ensures that all static certificates (and the serviceaccount key) are present on etcd.
+func (k *KubernetesPKI) EnsureAll(ctx context.Context, kv clientv3.KV) error {
+ for n, v := range k.Certificates {
+ k.logger.Info("ensureing certificate existence", zap.String("name", string(n)))
+ _, _, err := v.Ensure(ctx, kv)
+ if err != nil {
+ return fmt.Errorf("could not ensure certificate %q exists: %w", n, err)
+ }
+ }
+ _, err := k.ServiceAccountKey(ctx, kv)
+ if err != nil {
+ return fmt.Errorf("could not ensure service account key exists: %w", err)
+ }
+ return nil
+}
+
+// Kubeconfig generates a kubeconfig blob for a given certificate name. The same lifetime semantics as in .Certificate
+// apply.
+func (k *KubernetesPKI) Kubeconfig(ctx context.Context, name KubeCertificateName, kv clientv3.KV) ([]byte, error) {
+ c, ok := k.Certificates[name]
+ if !ok {
+ return nil, fmt.Errorf("no certificate %q", name)
+ }
+ return c.Kubeconfig(ctx, kv)
+}
+
+// Certificate retrieves an x509 DER-encoded (but not PEM-wrapped) key and certificate for a given certificate name.
+// If the requested certificate is volatile, it will be created on demand. Otherwise it will be created on etcd (if not
+// present), and retrieved from there.
+func (k *KubernetesPKI) Certificate(ctx context.Context, name KubeCertificateName, kv clientv3.KV) (cert, key []byte, err error) {
+ c, ok := k.Certificates[name]
+ if !ok {
+ return nil, nil, fmt.Errorf("no certificate %q", name)
+ }
+ return c.Ensure(ctx, kv)
+}
+
+// Kubeconfig generates a kubeconfig blob for this certificate. The same lifetime semantics as in .Ensure apply.
+func (c *Certificate) Kubeconfig(ctx context.Context, kv clientv3.KV) ([]byte, error) {
+
+ cert, key, err := c.Ensure(ctx, kv)
+ if err != nil {
+ return nil, fmt.Errorf("could not ensure certificate exists: %w", err)
+ }
+
+ kubeconfig := configapi.NewConfig()
+
+ cluster := configapi.NewCluster()
+ cluster.Server = fmt.Sprintf("https://127.0.0.1:%v", common.KubernetesAPIPort)
+
+ ca, err := c.issuer.CACertificate(ctx, kv)
+ if err != nil {
+ return nil, fmt.Errorf("could not get CA certificate: %w", err)
+ }
+ if ca != nil {
+ cluster.CertificateAuthorityData = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ca})
+ }
+ kubeconfig.Clusters["default"] = cluster
+
+ authInfo := configapi.NewAuthInfo()
+ authInfo.ClientCertificateData = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert})
+ authInfo.ClientKeyData = pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: key})
+ kubeconfig.AuthInfos["default"] = authInfo
+
+ ct := configapi.NewContext()
+ ct.Cluster = "default"
+ ct.AuthInfo = "default"
+ kubeconfig.Contexts["default"] = ct
+
+ kubeconfig.CurrentContext = "default"
+ return clientcmd.Write(*kubeconfig)
+}
+
+// ServiceAccountKey retrieves (and possible generates and stores on etcd) the Kubernetes service account key. The
+// returned data is ready to be used by Kubernetes components (in PKIX form).
+func (k *KubernetesPKI) ServiceAccountKey(ctx context.Context, kv clientv3.KV) ([]byte, error) {
+ // TODO(q3k): this should be abstracted away once we abstract away etcd access into a library with try-or-create
+ // semantics.
+
+ path := etcdPath("%s.der", serviceAccountKeyName)
+
+ // Try loading key from etcd.
+ keyRes, err := kv.Get(ctx, path)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get key from etcd: %w", err)
+ }
+
+ if len(keyRes.Kvs) == 1 {
+ // Certificate and key exists in etcd, return that.
+ return keyRes.Kvs[0].Value, nil
+ }
+
+ // No key found - generate one.
+ keyRaw, err := rsa.GenerateKey(rand.Reader, 2048)
+ if err != nil {
+ panic(err)
+ }
+ key, err := x509.MarshalPKCS8PrivateKey(keyRaw)
+ if err != nil {
+ panic(err) // Always a programmer error
+ }
+
+ // Save to etcd.
+ _, err = kv.Put(ctx, path, string(key))
+ if err != nil {
+ err = fmt.Errorf("failed to write newly generated key: %w", err)
+ }
+ return key, nil
+}
diff --git a/core/internal/kubernetes/provisioner.go b/core/internal/kubernetes/provisioner.go
index c227fdf..c864715 100644
--- a/core/internal/kubernetes/provisioner.go
+++ b/core/internal/kubernetes/provisioner.go
@@ -24,10 +24,9 @@
"os"
"path/filepath"
- "git.monogon.dev/source/nexantic.git/core/internal/common/supervisor"
-
"go.uber.org/zap"
+ "git.monogon.dev/source/nexantic.git/core/internal/common/supervisor"
"git.monogon.dev/source/nexantic.git/core/internal/storage"
"git.monogon.dev/source/nexantic.git/core/pkg/fsquota"
diff --git a/core/internal/kubernetes/scheduler.go b/core/internal/kubernetes/scheduler.go
index d3ee20b..35b1e64 100644
--- a/core/internal/kubernetes/scheduler.go
+++ b/core/internal/kubernetes/scheduler.go
@@ -26,6 +26,7 @@
"go.etcd.io/etcd/clientv3"
"git.monogon.dev/source/nexantic.git/core/internal/common/supervisor"
+ "git.monogon.dev/source/nexantic.git/core/internal/kubernetes/pki"
"git.monogon.dev/source/nexantic.git/core/pkg/fileargs"
)
@@ -35,14 +36,14 @@
serverKey []byte
}
-func getPKISchedulerConfig(consensusKV clientv3.KV) (*schedulerConfig, error) {
+func getPKISchedulerConfig(ctx context.Context, kv clientv3.KV, kpki *pki.KubernetesPKI) (*schedulerConfig, error) {
var config schedulerConfig
var err error
- config.serverCert, config.serverKey, err = getCert(consensusKV, "scheduler")
+ config.serverCert, config.serverKey, err = kpki.Certificate(ctx, pki.Scheduler, kv)
if err != nil {
return nil, fmt.Errorf("failed to get scheduler serving certificate: %w", err)
}
- config.kubeConfig, err = getSingle(consensusKV, "scheduler.kubeconfig")
+ config.kubeConfig, err = kpki.Kubeconfig(ctx, pki.SchedulerClient, kv)
if err != nil {
return nil, fmt.Errorf("failed to get scheduler kubeconfig: %w", err)
}
diff --git a/core/internal/kubernetes/service.go b/core/internal/kubernetes/service.go
index b2d340e..826e2d8 100644
--- a/core/internal/kubernetes/service.go
+++ b/core/internal/kubernetes/service.go
@@ -18,9 +18,10 @@
import (
"context"
- "crypto/ed25519"
"errors"
+ "fmt"
"net"
+ "os"
"time"
"k8s.io/client-go/informers"
@@ -35,6 +36,7 @@
schema "git.monogon.dev/source/nexantic.git/core/generated/api"
"git.monogon.dev/source/nexantic.git/core/internal/common/supervisor"
"git.monogon.dev/source/nexantic.git/core/internal/consensus"
+ "git.monogon.dev/source/nexantic.git/core/internal/kubernetes/pki"
"git.monogon.dev/source/nexantic.git/core/internal/kubernetes/reconciler"
"git.monogon.dev/source/nexantic.git/core/internal/storage"
"git.monogon.dev/source/nexantic.git/core/pkg/logbuffer"
@@ -47,25 +49,32 @@
}
type Service struct {
- consensusService *consensus.Service
- storageService *storage.Manager
- logger *zap.Logger
+ consensusService *consensus.Service
+ storageService *storage.Manager
+ logger *zap.Logger
+
apiserverLogs *logbuffer.LogBuffer
controllerManagerLogs *logbuffer.LogBuffer
schedulerLogs *logbuffer.LogBuffer
kubeletLogs *logbuffer.LogBuffer
+
+ kpki *pki.KubernetesPKI
}
func New(logger *zap.Logger, consensusService *consensus.Service, storageService *storage.Manager) *Service {
s := &Service{
- consensusService: consensusService,
- storageService: storageService,
- logger: logger,
+ consensusService: consensusService,
+ storageService: storageService,
+ logger: logger,
+
apiserverLogs: logbuffer.New(5000, 16384),
controllerManagerLogs: logbuffer.New(5000, 16384),
schedulerLogs: logbuffer.New(5000, 16384),
kubeletLogs: logbuffer.New(5000, 16384),
+
+ kpki: pki.NewKubernetes(logger.Named("pki")),
}
+
return s
}
@@ -74,7 +83,9 @@
}
func (s *Service) NewCluster() error {
- return newCluster(s.getKV())
+ // TODO(q3k): this needs to be passed by the caller.
+ ctx := context.TODO()
+ return s.kpki.EnsureAll(ctx, s.getKV())
}
// GetComponentLogs grabs logs from various Kubernetes binaries
@@ -98,16 +109,8 @@
if !s.consensusService.IsReady() {
return nil, status.Error(codes.Unavailable, "Consensus not ready yet")
}
- idCA, idKeyRaw, err := getCert(s.getKV(), "id-ca")
- idKey := ed25519.PrivateKey(idKeyRaw)
- if err != nil {
- return nil, status.Errorf(codes.Unavailable, "Failed to load ID CA: %v", err)
- }
- debugCert, debugKey, err := issueCertificate(clientCertTemplate(request.Id, request.Groups), idCA, idKey)
- if err != nil {
- return nil, status.Errorf(codes.Unavailable, "Failed to issue certs for kubeconfig: %v\n", err)
- }
- debugKubeconfig, err := makeLocalKubeconfig(idCA, debugCert, debugKey)
+ ca := s.kpki.Certificates[pki.IdCA]
+ debugKubeconfig, err := pki.New(ca, "", pki.Client(request.Id, request.Groups)).Kubeconfig(ctx, s.getKV())
if err != nil {
return nil, status.Errorf(codes.Unavailable, "Failed to generate kubeconfig: %v", err)
}
@@ -135,60 +138,65 @@
},
}
consensusKV := s.getKV()
- apiserverConfig, err := getPKIApiserverConfig(consensusKV)
+ apiserverConfig, err := getPKIApiserverConfig(ctx, consensusKV, s.kpki)
if err != nil {
- return err
+ return fmt.Errorf("could not generate apiserver pki config: %w", err)
}
apiserverConfig.advertiseAddress = config.AdvertiseAddress
apiserverConfig.serviceIPRange = config.ServiceIPRange
- controllerManagerConfig, err := getPKIControllerManagerConfig(consensusKV)
+ controllerManagerConfig, err := getPKIControllerManagerConfig(ctx, consensusKV, s.kpki)
if err != nil {
- return err
+ return fmt.Errorf("could not generate controller manager pki config: %w", err)
}
controllerManagerConfig.clusterNet = config.ClusterNet
- schedulerConfig, err := getPKISchedulerConfig(consensusKV)
+ schedulerConfig, err := getPKISchedulerConfig(ctx, consensusKV, s.kpki)
if err != nil {
- return err
+ return fmt.Errorf("could not generate scheduler pki config: %w", err)
}
- masterKubeconfig, err := getSingle(consensusKV, "master.kubeconfig")
+ masterKubeconfig, err := s.kpki.Kubeconfig(ctx, pki.Master, consensusKV)
if err != nil {
- return err
+ return fmt.Errorf("could not generate master kubeconfig: %w", err)
}
rawClientConfig, err := clientcmd.NewClientConfigFromBytes(masterKubeconfig)
if err != nil {
- return err
+ return fmt.Errorf("could not generate kubernetes client config: %w", err)
}
clientConfig, err := rawClientConfig.ClientConfig()
clientSet, err := kubernetes.NewForConfig(clientConfig)
if err != nil {
- return err
+ return fmt.Errorf("could not generate kubernetes client: %w", err)
}
informerFactory := informers.NewSharedInformerFactory(clientSet, 5*time.Minute)
- if err := supervisor.Run(ctx, "apiserver", runAPIServer(*apiserverConfig, s.apiserverLogs)); err != nil {
- return err
+ hostname, err := os.Hostname()
+ if err != nil {
+ return fmt.Errorf("failed to get hostname: %w", err)
}
- if err := supervisor.Run(ctx, "controller-manager", runControllerManager(*controllerManagerConfig, s.controllerManagerLogs)); err != nil {
- return err
+ err = createKubeletConfig(ctx, consensusKV, s.kpki, hostname)
+ if err != nil {
+ return fmt.Errorf("could not created kubelet config: %w", err)
}
- if err := supervisor.Run(ctx, "scheduler", runScheduler(*schedulerConfig, s.schedulerLogs)); err != nil {
- return err
- }
- if err := supervisor.Run(ctx, "kubelet", runKubelet(&KubeletSpec{}, s.kubeletLogs)); err != nil {
- return err
- }
- if err := supervisor.Run(ctx, "reconciler", reconciler.Run(clientSet)); err != nil {
- return err
- }
- if err := supervisor.Run(ctx, "csi-plugin", runCSIPlugin(s.storageService)); err != nil {
- return err
- }
- if err := supervisor.Run(ctx, "pv-provisioner", runCSIProvisioner(s.storageService, clientSet, informerFactory)); err != nil {
- return err
+
+ for _, sub := range []struct {
+ name string
+ runnable supervisor.Runnable
+ }{
+ {"apiserver", runAPIServer(*apiserverConfig, s.apiserverLogs)},
+ {"controller-manager", runControllerManager(*controllerManagerConfig, s.controllerManagerLogs)},
+ {"scheduler", runScheduler(*schedulerConfig, s.schedulerLogs)},
+ {"kubelet", runKubelet(&KubeletSpec{}, s.kubeletLogs)},
+ {"reconciler", reconciler.Run(clientSet)},
+ {"csi-plugin", runCSIPlugin(s.storageService)},
+ {"pv-provisioner", runCSIProvisioner(s.storageService, clientSet, informerFactory)},
+ } {
+ err := supervisor.Run(ctx, sub.name, sub.runnable)
+ if err != nil {
+ return fmt.Errorf("could not run sub-service %q: %w", sub.name, err)
+ }
}
supervisor.Signal(ctx, supervisor.SignalHealthy)