core/internal: move containerd and kubernetes to localstorage
This moves the last users of the old 'storage' library onto 'localstorage'. We move a lot of 'runtime' directories to a single `/ephemeral` root. This could be called `/run`, but that might imply FHS compliance - which we don't have, nor want to have.
We also slightly refactor Kubernetes services to be a bit nicer to spawn. But generally, this is a pure refactor, with no functional changes.
Test Plan: this should fail. part of a larger stack. D590 is the first tip of the stack that should work.
X-Origin-Diff: phab/D589
GitOrigin-RevId: d2a7c0bb52c2a7c753199221c609e03474936c22
diff --git a/core/internal/kubernetes/apiserver.go b/core/internal/kubernetes/apiserver.go
index 26c258a..58c3d1e 100644
--- a/core/internal/kubernetes/apiserver.go
+++ b/core/internal/kubernetes/apiserver.go
@@ -27,14 +27,17 @@
"git.monogon.dev/source/nexantic.git/core/internal/common"
"git.monogon.dev/source/nexantic.git/core/internal/common/supervisor"
"git.monogon.dev/source/nexantic.git/core/internal/kubernetes/pki"
+ "git.monogon.dev/source/nexantic.git/core/internal/localstorage"
"git.monogon.dev/source/nexantic.git/core/pkg/fileargs"
-
- "go.etcd.io/etcd/clientv3"
)
-type apiserverConfig struct {
- advertiseAddress net.IP
- serviceIPRange net.IPNet
+type apiserverService struct {
+ KPKI *pki.KubernetesPKI
+ AdvertiseAddress net.IP
+ ServiceIPRange net.IPNet
+ Output io.Writer
+ EphemeralConsensusDirectory *localstorage.EphemeralConsensusDirectory
+
// All PKI-related things are in DER
idCA []byte
kubeletClientCert []byte
@@ -47,23 +50,21 @@
serverKey []byte
}
-func getPKIApiserverConfig(ctx context.Context, kv clientv3.KV, kpki *pki.KubernetesPKI) (*apiserverConfig, error) {
- var config apiserverConfig
-
+func (s *apiserverService) loadPKI(ctx context.Context) error {
for _, el := range []struct {
targetCert *[]byte
targetKey *[]byte
name pki.KubeCertificateName
}{
- {&config.idCA, nil, pki.IdCA},
- {&config.kubeletClientCert, &config.kubeletClientKey, pki.KubeletClient},
- {&config.aggregationCA, nil, pki.AggregationCA},
- {&config.aggregationClientCert, &config.aggregationClientKey, pki.FrontProxyClient},
- {&config.serverCert, &config.serverKey, pki.APIServer},
+ {&s.idCA, nil, pki.IdCA},
+ {&s.kubeletClientCert, &s.kubeletClientKey, pki.KubeletClient},
+ {&s.aggregationCA, nil, pki.AggregationCA},
+ {&s.aggregationClientCert, &s.aggregationClientKey, pki.FrontProxyClient},
+ {&s.serverCert, &s.serverKey, pki.APIServer},
} {
- cert, key, err := kpki.Certificate(ctx, el.name, kv)
+ cert, key, err := s.KPKI.Certificate(ctx, el.name)
if err != nil {
- return nil, fmt.Errorf("could not load certificate %q from PKI: %w", el.name, err)
+ return fmt.Errorf("could not load certificate %q from PKI: %w", el.name, err)
}
if el.targetCert != nil {
*el.targetCert = cert
@@ -74,62 +75,63 @@
}
var err error
- config.serviceAccountPrivKey, err = kpki.ServiceAccountKey(ctx, kv)
+ s.serviceAccountPrivKey, err = s.KPKI.ServiceAccountKey(ctx)
if err != nil {
- return nil, fmt.Errorf("could not load serviceaccount privkey: %w", err)
+ return fmt.Errorf("could not load serviceaccount privkey: %w", err)
}
- return &config, nil
+ return nil
}
-func runAPIServer(config apiserverConfig, output io.Writer) supervisor.Runnable {
- return func(ctx context.Context) error {
- args, err := fileargs.New()
- if err != nil {
- panic(err) // If this fails, something is very wrong. Just crash.
- }
- defer args.Close()
- cmd := exec.CommandContext(ctx, "/kubernetes/bin/kube", "kube-apiserver",
- fmt.Sprintf("--advertise-address=%v", config.advertiseAddress.String()),
- "--authorization-mode=Node,RBAC",
- args.FileOpt("--client-ca-file", "client-ca.pem",
- pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: config.idCA})),
- "--enable-admission-plugins=NodeRestriction,PodSecurityPolicy",
- "--enable-aggregator-routing=true",
- "--insecure-port=0",
- fmt.Sprintf("--secure-port=%v", common.KubernetesAPIPort),
- // Due to the magic of GRPC this really needs four slashes and a :0
- fmt.Sprintf("--etcd-servers=%v", "unix:////consensus/listener.sock:0"),
- args.FileOpt("--kubelet-client-certificate", "kubelet-client-cert.pem",
- pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: config.kubeletClientCert})),
- args.FileOpt("--kubelet-client-key", "kubelet-client-key.pem",
- pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: config.kubeletClientKey})),
- "--kubelet-preferred-address-types=InternalIP",
- args.FileOpt("--proxy-client-cert-file", "aggregation-client-cert.pem",
- pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: config.aggregationClientCert})),
- args.FileOpt("--proxy-client-key-file", "aggregation-client-key.pem",
- pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: config.aggregationClientKey})),
- "--requestheader-allowed-names=front-proxy-client",
- args.FileOpt("--requestheader-client-ca-file", "aggregation-ca.pem",
- pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: config.aggregationCA})),
- "--requestheader-extra-headers-prefix=X-Remote-Extra-",
- "--requestheader-group-headers=X-Remote-Group",
- "--requestheader-username-headers=X-Remote-User",
- args.FileOpt("--service-account-key-file", "service-account-pubkey.pem",
- pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: config.serviceAccountPrivKey})),
- fmt.Sprintf("--service-cluster-ip-range=%v", config.serviceIPRange.String()),
- args.FileOpt("--tls-cert-file", "server-cert.pem",
- pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: config.serverCert})),
- args.FileOpt("--tls-private-key-file", "server-key.pem",
- pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: config.serverKey})),
- )
- if args.Error() != nil {
- return err
- }
- cmd.Stdout = output
- cmd.Stderr = output
- supervisor.Signal(ctx, supervisor.SignalHealthy)
- err = cmd.Run()
- fmt.Fprintf(output, "apiserver stopped: %v\n", err)
+func (s *apiserverService) Run(ctx context.Context) error {
+ if err := s.loadPKI(ctx); err != nil {
+ return fmt.Errorf("loading PKI data failed: %w", err)
+ }
+ args, err := fileargs.New()
+ if err != nil {
+ panic(err) // If this fails, something is very wrong. Just crash.
+ }
+ defer args.Close()
+
+ cmd := exec.CommandContext(ctx, "/kubernetes/bin/kube", "kube-apiserver",
+ fmt.Sprintf("--advertise-address=%v", s.AdvertiseAddress.String()),
+ "--authorization-mode=Node,RBAC",
+ args.FileOpt("--client-ca-file", "client-ca.pem",
+ pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: s.idCA})),
+ "--enable-admission-plugins=NodeRestriction,PodSecurityPolicy",
+ "--enable-aggregator-routing=true",
+ "--insecure-port=0",
+ fmt.Sprintf("--secure-port=%v", common.KubernetesAPIPort),
+ fmt.Sprintf("--etcd-servers=unix:///%s:0", s.EphemeralConsensusDirectory.ClientSocket.FullPath()),
+ args.FileOpt("--kubelet-client-certificate", "kubelet-client-cert.pem",
+ pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: s.kubeletClientCert})),
+ args.FileOpt("--kubelet-client-key", "kubelet-client-key.pem",
+ pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: s.kubeletClientKey})),
+ "--kubelet-preferred-address-types=InternalIP",
+ args.FileOpt("--proxy-client-cert-file", "aggregation-client-cert.pem",
+ pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: s.aggregationClientCert})),
+ args.FileOpt("--proxy-client-key-file", "aggregation-client-key.pem",
+ pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: s.aggregationClientKey})),
+ "--requestheader-allowed-names=front-proxy-client",
+ args.FileOpt("--requestheader-client-ca-file", "aggregation-ca.pem",
+ pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: s.aggregationCA})),
+ "--requestheader-extra-headers-prefix=X-Remote-Extra-",
+ "--requestheader-group-headers=X-Remote-Group",
+ "--requestheader-username-headers=X-Remote-User",
+ args.FileOpt("--service-account-key-file", "service-account-pubkey.pem",
+ pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: s.serviceAccountPrivKey})),
+ fmt.Sprintf("--service-cluster-ip-range=%v", s.ServiceIPRange.String()),
+ args.FileOpt("--tls-cert-file", "server-cert.pem",
+ pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: s.serverCert})),
+ args.FileOpt("--tls-private-key-file", "server-key.pem",
+ pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: s.serverKey})),
+ )
+ if args.Error() != nil {
return err
}
+ cmd.Stdout = s.Output
+ cmd.Stderr = s.Output
+ supervisor.Signal(ctx, supervisor.SignalHealthy)
+ err = cmd.Run()
+ fmt.Fprintf(s.Output, "apiserver stopped: %v\n", err)
+ return err
}