Initial Kubernetes Control Plane
This adds a minimum viable Kubernetes Control Plane consisting of a
kube-apiserver, kube-controller-manager and kube-scheduler. It contains
two small CAs for Kubernetes Identity management based on shared
certificates and contains changes for exposing etcd via UNIX socket
so that the apiserver can talk to it.
Test Plan:
Tested by manually calling Setup() and observing subsequent logs and
connecting to the API server.
Bug: T485
X-Origin-Diff: phab/D271
GitOrigin-RevId: e56f3e50eb9d33ea291289faa1aac3bebdeb3346
diff --git a/core/internal/kubernetes/BUILD.bazel b/core/internal/kubernetes/BUILD.bazel
new file mode 100644
index 0000000..0fa4dc4
--- /dev/null
+++ b/core/internal/kubernetes/BUILD.bazel
@@ -0,0 +1,23 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+ name = "go_default_library",
+ srcs = [
+ "apiserver.go",
+ "auth.go",
+ "controller-manager.go",
+ "scheduler.go",
+ "service.go",
+ ],
+ importpath = "git.monogon.dev/source/nexantic.git/core/internal/kubernetes",
+ visibility = ["//core:__subpackages__"],
+ deps = [
+ "//core/internal/common/service:go_default_library",
+ "//core/internal/consensus:go_default_library",
+ "//core/pkg/fileargs:go_default_library",
+ "@io_etcd_go_etcd//clientv3:go_default_library",
+ "@kubernetes//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library",
+ "@kubernetes//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library",
+ "@org_uber_go_zap//:go_default_library",
+ ],
+)
diff --git a/core/internal/kubernetes/apiserver.go b/core/internal/kubernetes/apiserver.go
new file mode 100644
index 0000000..c2664a6
--- /dev/null
+++ b/core/internal/kubernetes/apiserver.go
@@ -0,0 +1,113 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kubernetes
+
+import (
+ "context"
+ "encoding/pem"
+ "errors"
+ "fmt"
+ "net"
+ "os"
+ "os/exec"
+ "path"
+
+ "git.monogon.dev/source/nexantic.git/core/pkg/fileargs"
+ "go.etcd.io/etcd/clientv3"
+)
+
+type apiserverConfig struct {
+ advertiseAddress net.IP
+ serviceIPRange net.IPNet
+ // All PKI-related things are in DER
+ idCA []byte
+ kubeletClientCert []byte
+ kubeletClientKey []byte
+ aggregationCA []byte
+ aggregationClientCert []byte
+ aggregationClientKey []byte
+ serviceAccountPrivKey []byte // In PKIX form
+ serverCert []byte
+ serverKey []byte
+}
+
+func getPKIApiserverConfig(consensusKV clientv3.KV) (*apiserverConfig, error) {
+ var config apiserverConfig
+ var err error
+ config.idCA, _, err = getCert(consensusKV, "id-ca")
+ config.kubeletClientCert, config.kubeletClientKey, err = getCert(consensusKV, "kubelet-client")
+ config.aggregationCA, _, err = getCert(consensusKV, "aggregation-ca")
+ config.aggregationClientCert, config.aggregationClientKey, err = getCert(consensusKV, "front-proxy-client")
+ config.serverCert, config.serverKey, err = getCert(consensusKV, "apiserver")
+ saPrivkey, err := consensusKV.Get(context.Background(), path.Join(etcdPath, "service-account-privkey.der"))
+ if err != nil {
+ return nil, fmt.Errorf("failed to get serviceaccount privkey: %w", err)
+ }
+ if len(saPrivkey.Kvs) != 1 {
+ return nil, errors.New("failed to get serviceaccount privkey: not found")
+ }
+ config.serviceAccountPrivKey = saPrivkey.Kvs[0].Value
+ return &config, nil
+}
+
+func runAPIServer(config apiserverConfig) error {
+ args, err := fileargs.New()
+ if err != nil {
+ panic(err) // If this fails, something is very wrong. Just crash.
+ }
+ defer args.Close()
+ cmd := exec.Command("/bin/kube-controlplane", "kube-apiserver",
+ fmt.Sprintf("--advertise-address=%v", config.advertiseAddress.String()),
+ "--authorization-mode=Node,RBAC",
+ args.FileOpt("--client-ca-file", "client-ca.pem",
+ pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: config.idCA})),
+ "--enable-admission-plugins=NodeRestriction,PodSecurityPolicy",
+ "--enable-aggregator-routing=true",
+ "--insecure-port=0",
+ // Due to the magic of GRPC this really needs four slashes and a :0
+ fmt.Sprintf("--etcd-servers=%v", "unix:////consensus/listener.sock:0"),
+ args.FileOpt("--kubelet-client-certificate", "kubelet-client-cert.pem",
+ pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: config.kubeletClientCert})),
+ args.FileOpt("--kubelet-client-key", "kubelet-client-key.pem",
+ pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: config.kubeletClientKey})),
+ "--kubelet-preferred-address-types=InternalIP",
+ args.FileOpt("--proxy-client-cert-file", "aggregation-client-cert.pem",
+ pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: config.aggregationClientCert})),
+ args.FileOpt("--proxy-client-key-file", "aggregation-client-key.pem",
+ pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: config.aggregationClientKey})),
+ "--requestheader-allowed-names=front-proxy-client",
+ args.FileOpt("--requestheader-client-ca-file", "aggregation-ca.pem",
+ pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: config.aggregationCA})),
+ "--requestheader-extra-headers-prefix=X-Remote-Extra-",
+ "--requestheader-group-headers=X-Remote-Group",
+ "--requestheader-username-headers=X-Remote-User",
+ args.FileOpt("--service-account-key-file", "service-account-pubkey.pem",
+ pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: config.serviceAccountPrivKey})),
+ fmt.Sprintf("--service-cluster-ip-range=%v", config.serviceIPRange.String()),
+ args.FileOpt("--tls-cert-file", "server-cert.pem",
+ pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: config.serverCert})),
+ args.FileOpt("--tls-private-key-file", "server-key.pem",
+ pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: config.serverKey})),
+ )
+ if args.Error() != nil {
+ return err
+ }
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ err = cmd.Run()
+ return err
+}
diff --git a/core/internal/kubernetes/auth.go b/core/internal/kubernetes/auth.go
new file mode 100644
index 0000000..afc51c1
--- /dev/null
+++ b/core/internal/kubernetes/auth.go
@@ -0,0 +1,367 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kubernetes
+
+import (
+ "context"
+ "crypto"
+ "crypto/ed25519"
+ "crypto/rand"
+ "crypto/rsa"
+ "crypto/sha1"
+ "crypto/x509"
+ "crypto/x509/pkix"
+ "encoding/asn1"
+ "encoding/pem"
+ "fmt"
+ "math/big"
+ "net"
+ "path"
+ "time"
+
+ "go.etcd.io/etcd/clientv3"
+ "k8s.io/client-go/tools/clientcmd"
+ configapi "k8s.io/client-go/tools/clientcmd/api"
+)
+
+const (
+ etcdPath = "/kube-pki/"
+)
+
+var (
+ // From RFC 5280 Section 4.1.2.5
+ unknownNotAfter = time.Unix(253402300799, 0)
+)
+
+// Directly derived from Kubernetes PKI requirements documented at
+// https://kubernetes.io/docs/setup/best-practices/certificates/#configure-certificates-manually
+func clientCertTemplate(identity string, groups []string) x509.Certificate {
+ return x509.Certificate{
+ Subject: pkix.Name{
+ CommonName: identity,
+ Organization: groups,
+ },
+ KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+ ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+ }
+}
+func serverCertTemplate(dnsNames []string, ips []net.IP) x509.Certificate {
+ return x509.Certificate{
+ Subject: pkix.Name{},
+ KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+ ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+ DNSNames: dnsNames,
+ IPAddresses: ips,
+ }
+}
+
+// Workaround for https://github.com/golang/go/issues/26676 in Go's crypto/x509. Specifically Go
+// violates Section 4.2.1.2 of RFC 5280 without this. Should eventually be redundant.
+//
+// Taken from https://github.com/FiloSottile/mkcert/blob/master/cert.go#L295 written by one of Go's
+// crypto engineers
+func calculateSKID(pubKey crypto.PublicKey) ([]byte, error) {
+ spkiASN1, err := x509.MarshalPKIXPublicKey(pubKey)
+ if err != nil {
+ return nil, err
+ }
+
+ var spki struct {
+ Algorithm pkix.AlgorithmIdentifier
+ SubjectPublicKey asn1.BitString
+ }
+ _, err = asn1.Unmarshal(spkiASN1, &spki)
+ if err != nil {
+ return nil, err
+ }
+ skid := sha1.Sum(spki.SubjectPublicKey.Bytes)
+ return skid[:], nil
+}
+
+func newCA(name string) ([]byte, ed25519.PrivateKey, error) {
+ pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ panic(err)
+ }
+
+ serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 127)
+ serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+ if err != nil {
+ return []byte{}, privKey, fmt.Errorf("Failed to generate serial number: %w", err)
+ }
+
+ skid, err := calculateSKID(pubKey)
+ if err != nil {
+ return []byte{}, privKey, err
+ }
+
+ caCert := &x509.Certificate{
+ SerialNumber: serialNumber,
+ Subject: pkix.Name{
+ CommonName: name,
+ },
+ IsCA: true,
+ BasicConstraintsValid: true,
+ NotBefore: time.Now(),
+ NotAfter: unknownNotAfter,
+ KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature,
+ ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageOCSPSigning},
+ AuthorityKeyId: skid,
+ SubjectKeyId: skid,
+ }
+
+ caCertRaw, err := x509.CreateCertificate(rand.Reader, caCert, caCert, pubKey, privKey)
+ return caCertRaw, privKey, err
+}
+
+func storeCert(consensusKV clientv3.KV, name string, cert []byte, key []byte) error {
+ certPath := path.Join(etcdPath, fmt.Sprintf("%v-cert.der", name))
+ keyPath := path.Join(etcdPath, fmt.Sprintf("%v-key.der", name))
+ if _, err := consensusKV.Put(context.Background(), certPath, string(cert)); err != nil {
+ return fmt.Errorf("failed to store certificate: %w", err)
+ }
+ if _, err := consensusKV.Put(context.Background(), keyPath, string(key)); err != nil {
+ return fmt.Errorf("failed to store key: %w", err)
+ }
+ return nil
+}
+
+func getCert(consensusKV clientv3.KV, name string) (cert []byte, key []byte, err error) {
+ certPath := path.Join(etcdPath, fmt.Sprintf("%v-cert.der", name))
+ keyPath := path.Join(etcdPath, fmt.Sprintf("%v-key.der", name))
+ certRes, err := consensusKV.Get(context.Background(), certPath)
+ if err != nil {
+ err = fmt.Errorf("failed to get certificate: %w", err)
+ return
+ }
+ keyRes, err := consensusKV.Get(context.Background(), keyPath)
+ if err != nil {
+ err = fmt.Errorf("failed to get certificate: %w", err)
+ return
+ }
+ if len(certRes.Kvs) != 1 || len(keyRes.Kvs) != 1 {
+ err = fmt.Errorf("failed to find certificate %v", name)
+ return
+ }
+ cert = certRes.Kvs[0].Value
+ key = keyRes.Kvs[0].Value
+ return
+}
+
+func getSingle(consensusKV clientv3.KV, name string) ([]byte, error) {
+ res, err := consensusKV.Get(context.Background(), path.Join(etcdPath, name))
+ if err != nil {
+ return []byte{}, fmt.Errorf("failed to get PKI item: %w", err)
+ }
+ if len(res.Kvs) != 1 {
+ return []byte{}, fmt.Errorf("failed to find PKI item %v", name)
+ }
+ return res.Kvs[0].Value, nil
+}
+
+// newCluster initializes the whole PKI for Kubernetes. It issues a single certificate per control
+// plane service since it assumes that etcd is already a secure place to store data. This removes
+// the need for revocation and makes the logic much simpler. Thus PKI data can NEVER be stored
+// outside of etcd or other secure storage locations. All PKI data is stored in DER form and not
+// PEM encoded since that would require more logic to deal with it.
+func newCluster(consensusKV clientv3.KV) error {
+ // This whole issuance procedure is pretty repetitive, but abstracts badly because a lot of it
+ // is subtly different.
+ idCA, idKey, err := newCA("Smalltown Kubernetes ID CA")
+ if err != nil {
+ return fmt.Errorf("failed to create Kubernetes ID CA: %w", err)
+ }
+ if err := storeCert(consensusKV, "id-ca", idCA, idKey); err != nil {
+ return err
+ }
+ aggregationCA, aggregationKey, err := newCA("Smalltown OpenAPI Aggregation CA")
+ if err != nil {
+ return fmt.Errorf("failed to create OpenAPI Aggregation CA: %w", err)
+ }
+ if err := storeCert(consensusKV, "aggregation-ca", aggregationCA, aggregationKey); err != nil {
+ return err
+ }
+
+ // ServiceAccounts don't support ed25519 yet, so use RSA (better side-channel resistance than ECDSA)
+ serviceAccountPrivKeyRaw, err := rsa.GenerateKey(rand.Reader, 2048)
+ if err != nil {
+ panic(err)
+ }
+ serviceAccountPrivKey, err := x509.MarshalPKCS8PrivateKey(serviceAccountPrivKeyRaw)
+ if err != nil {
+ panic(err) // Always a programmer error
+ }
+ _, err = consensusKV.Put(context.Background(), path.Join(etcdPath, "service-account-privkey.der"),
+ string(serviceAccountPrivKey))
+ if err != nil {
+ return fmt.Errorf("failed to store service-account-privkey.der: %w", err)
+ }
+
+ apiserverCert, apiserverKey, err := issueCertificate(
+ serverCertTemplate([]string{
+ "kubernetes",
+ "kubernetes.default",
+ "kubernetes.default.svc",
+ "kubernetes.default.svc.cluster",
+ "kubernetes.default.svc.cluster.local",
+ "localhost",
+ }, []net.IP{{127, 0, 0, 1}}, // TODO: Add service internal IP
+ ),
+ idCA, idKey,
+ )
+ if err != nil {
+ return fmt.Errorf("failed to issue certificate for apiserver: %w", err)
+ }
+ if err := storeCert(consensusKV, "apiserver", apiserverCert, apiserverKey); err != nil {
+ return err
+ }
+
+ kubeletClientCert, kubeletClientKey, err := issueCertificate(
+ clientCertTemplate("kube-apiserver-kubelet-client", []string{"system:masters"}),
+ idCA, idKey,
+ )
+ if err != nil {
+ return fmt.Errorf("failed to issue certificate for kubelet client: %w", err)
+ }
+ if err := storeCert(consensusKV, "kubelet-client", kubeletClientCert, kubeletClientKey); err != nil {
+ return err
+ }
+
+ frontProxyClientCert, frontProxyClientKey, err := issueCertificate(
+ clientCertTemplate("front-proxy-client", []string{}),
+ aggregationCA, aggregationKey,
+ )
+ if err != nil {
+ return fmt.Errorf("failed to issue certificate for OpenAPI frontend: %w", err)
+ }
+ if err := storeCert(consensusKV, "front-proxy-client", frontProxyClientCert, frontProxyClientKey); err != nil {
+ return err
+ }
+
+ controllerManagerClientCert, controllerManagerClientKey, err := issueCertificate(
+ clientCertTemplate("system:kube-controller-manager", []string{}),
+ idCA, idKey,
+ )
+ if err != nil {
+ return fmt.Errorf("failed to issue certificate for controller-manager client: %w", err)
+ }
+
+ controllerManagerKubeconfig, err := makeLocalKubeconfig(idCA, controllerManagerClientCert,
+ controllerManagerClientKey)
+ if err != nil {
+ return fmt.Errorf("failed to create kubeconfig for controller-manager: %w", err)
+ }
+
+ _, err = consensusKV.Put(context.Background(), path.Join(etcdPath, "controller-manager.kubeconfig"),
+ string(controllerManagerKubeconfig))
+ if err != nil {
+ return fmt.Errorf("failed to store controller-manager kubeconfig: %w", err)
+ }
+
+ controllerManagerCert, controllerManagerKey, err := issueCertificate(
+ serverCertTemplate([]string{"kube-controller-manager.local"}, []net.IP{}),
+ idCA, idKey,
+ )
+ if err != nil {
+ return fmt.Errorf("failed to issue certificate for controller-manager: %w", err)
+ }
+ if err := storeCert(consensusKV, "controller-manager", controllerManagerCert, controllerManagerKey); err != nil {
+ return err
+ }
+
+ schedulerClientCert, schedulerClientKey, err := issueCertificate(
+ clientCertTemplate("system:kube-scheduler", []string{}),
+ idCA, idKey,
+ )
+ if err != nil {
+ return fmt.Errorf("failed to issue certificate for scheduler client: %w", err)
+ }
+
+ schedulerKubeconfig, err := makeLocalKubeconfig(idCA, schedulerClientCert, schedulerClientKey)
+ if err != nil {
+ return fmt.Errorf("failed to create kubeconfig for scheduler: %w", err)
+ }
+
+ _, err = consensusKV.Put(context.Background(), path.Join(etcdPath, "scheduler.kubeconfig"),
+ string(schedulerKubeconfig))
+ if err != nil {
+ return fmt.Errorf("failed to store controller-manager kubeconfig: %w", err)
+ }
+
+ schedulerCert, schedulerKey, err := issueCertificate(
+ serverCertTemplate([]string{"kube-scheduler.local"}, []net.IP{}),
+ idCA, idKey,
+ )
+ if err != nil {
+ return fmt.Errorf("failed to issue certificate for scheduler: %w", err)
+ }
+ if err := storeCert(consensusKV, "scheduler", schedulerCert, schedulerKey); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func issueCertificate(template x509.Certificate, caCert []byte, privateKey interface{}) (cert []byte, privkey []byte, err error) {
+ serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 127)
+ serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+ if err != nil {
+ err = fmt.Errorf("Failed to generate serial number: %w", err)
+ return
+ }
+
+ caCertObj, err := x509.ParseCertificate(caCert)
+ if err != nil {
+ err = fmt.Errorf("failed to parse CA certificate: %w", err)
+ }
+
+ pubKey, privKeyRaw, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ return
+ }
+ privkey, err = x509.MarshalPKCS8PrivateKey(privKeyRaw)
+ if err != nil {
+ return
+ }
+
+ template.SerialNumber = serialNumber
+ template.IsCA = false
+ template.BasicConstraintsValid = true
+ template.NotBefore = time.Now()
+ template.NotAfter = unknownNotAfter
+
+ cert, err = x509.CreateCertificate(rand.Reader, &template, caCertObj, pubKey, privateKey)
+ return
+}
+
+func makeLocalKubeconfig(ca, cert, key []byte) ([]byte, error) {
+ kubeconfig := configapi.NewConfig()
+ cluster := configapi.NewCluster()
+ cluster.Server = "https://localhost:6443"
+ cluster.CertificateAuthorityData = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ca})
+ kubeconfig.Clusters["default"] = cluster
+ authInfo := configapi.NewAuthInfo()
+ authInfo.ClientCertificateData = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert})
+ authInfo.ClientKeyData = pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: key})
+ kubeconfig.AuthInfos["default"] = authInfo
+ ctx := configapi.NewContext()
+ ctx.Cluster = "default"
+ ctx.AuthInfo = "default"
+ kubeconfig.Contexts["default"] = ctx
+ kubeconfig.CurrentContext = "default"
+ return clientcmd.Write(*kubeconfig)
+}
diff --git a/core/internal/kubernetes/controller-manager.go b/core/internal/kubernetes/controller-manager.go
new file mode 100644
index 0000000..fab0036
--- /dev/null
+++ b/core/internal/kubernetes/controller-manager.go
@@ -0,0 +1,88 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kubernetes
+
+import (
+ "encoding/pem"
+ "fmt"
+ "net"
+ "os"
+ "os/exec"
+
+ "git.monogon.dev/source/nexantic.git/core/pkg/fileargs"
+ "go.etcd.io/etcd/clientv3"
+)
+
+type controllerManagerConfig struct {
+ clusterNet net.IPNet
+ // All PKI-related things are in DER
+ kubeConfig []byte
+ rootCA []byte
+ serviceAccountPrivKey []byte // In PKCS#8 form
+ serverCert []byte
+ serverKey []byte
+}
+
+func getPKIControllerManagerConfig(consensusKV clientv3.KV) (*controllerManagerConfig, error) {
+ var config controllerManagerConfig
+ var err error
+ config.rootCA, _, err = getCert(consensusKV, "id-ca")
+ if err != nil {
+ return nil, fmt.Errorf("failed to get ID root CA: %w", err)
+ }
+ config.serverCert, config.serverKey, err = getCert(consensusKV, "controller-manager")
+ if err != nil {
+ return nil, fmt.Errorf("failed to get controller-manager serving certificate: %w", err)
+ }
+ config.serviceAccountPrivKey, err = getSingle(consensusKV, "service-account-privkey.der")
+ if err != nil {
+ return nil, fmt.Errorf("failed to get serviceaccount privkey: %w", err)
+ }
+ config.kubeConfig, err = getSingle(consensusKV, "controller-manager.kubeconfig")
+ if err != nil {
+ return nil, fmt.Errorf("failed to get controller-manager kubeconfig: %w", err)
+ }
+ return &config, nil
+}
+
+func runControllerManager(config controllerManagerConfig) error {
+ args, err := fileargs.New()
+ if err != nil {
+ panic(err) // If this fails, something is very wrong. Just crash.
+ }
+ defer args.Close()
+ cmd := exec.Command("/bin/kube-controlplane", "kube-controller-manager",
+ args.FileOpt("--kubeconfig", "kubeconfig", config.kubeConfig),
+ args.FileOpt("--service-account-private-key-file", "service-account-privkey.pem",
+ pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: config.serviceAccountPrivKey})),
+ args.FileOpt("--root-ca-file", "root-ca.pem",
+ pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: config.rootCA})),
+ "--port=0", // Kill insecure serving
+ "--use-service-account-credentials=true", // Enables things like PSP enforcement
+ fmt.Sprintf("--cluster-cidr=%v", config.clusterNet.String()),
+ args.FileOpt("--tls-cert-file", "server-cert.pem",
+ pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: config.serverCert})),
+ args.FileOpt("--tls-private-key-file", "server-key.pem",
+ pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: config.serverKey})),
+ )
+ if args.Error() != nil {
+ return fmt.Errorf("failed to use fileargs: %w", err)
+ }
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ return cmd.Run()
+}
diff --git a/core/internal/kubernetes/scheduler.go b/core/internal/kubernetes/scheduler.go
new file mode 100644
index 0000000..baaebc5
--- /dev/null
+++ b/core/internal/kubernetes/scheduler.go
@@ -0,0 +1,69 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kubernetes
+
+import (
+ "encoding/pem"
+ "fmt"
+ "os"
+ "os/exec"
+
+ "git.monogon.dev/source/nexantic.git/core/pkg/fileargs"
+ "go.etcd.io/etcd/clientv3"
+)
+
+type schedulerConfig struct {
+ kubeConfig []byte
+ serverCert []byte
+ serverKey []byte
+}
+
+func getPKISchedulerConfig(consensusKV clientv3.KV) (*schedulerConfig, error) {
+ var config schedulerConfig
+ var err error
+ config.serverCert, config.serverKey, err = getCert(consensusKV, "scheduler")
+ if err != nil {
+ return nil, fmt.Errorf("failed to get scheduler serving certificate: %w", err)
+ }
+ config.kubeConfig, err = getSingle(consensusKV, "scheduler.kubeconfig")
+ if err != nil {
+ return nil, fmt.Errorf("failed to get scheduler kubeconfig: %w", err)
+ }
+ return &config, nil
+}
+
+func runScheduler(config schedulerConfig) error {
+ args, err := fileargs.New()
+ if err != nil {
+ panic(err) // If this fails, something is very wrong. Just crash.
+ }
+ defer args.Close()
+ cmd := exec.Command("/bin/kube-controlplane", "kube-scheduler",
+ args.FileOpt("--kubeconfig", "kubeconfig", config.kubeConfig),
+ "--port=0", // Kill insecure serving
+ args.FileOpt("--tls-cert-file", "server-cert.pem",
+ pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: config.serverCert})),
+ args.FileOpt("--tls-private-key-file", "server-key.pem",
+ pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: config.serverKey})),
+ )
+ if args.Error() != nil {
+ return fmt.Errorf("failed to use fileargs: %w", err)
+ }
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ return cmd.Run()
+}
diff --git a/core/internal/kubernetes/service.go b/core/internal/kubernetes/service.go
new file mode 100644
index 0000000..57f2822
--- /dev/null
+++ b/core/internal/kubernetes/service.go
@@ -0,0 +1,103 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kubernetes
+
+import (
+ "errors"
+ "net"
+
+ "git.monogon.dev/source/nexantic.git/core/internal/common/service"
+ "git.monogon.dev/source/nexantic.git/core/internal/consensus"
+ "go.etcd.io/etcd/clientv3"
+ "go.uber.org/zap"
+)
+
+type Config struct {
+ AdvertiseAddress net.IP
+ ServiceIPRange net.IPNet
+ ClusterNet net.IPNet
+}
+
+type Service struct {
+ *service.BaseService
+ consensusService *consensus.Service
+ logger *zap.Logger
+}
+
+func New(logger *zap.Logger, consensusService *consensus.Service) *Service {
+ s := &Service{
+ consensusService: consensusService,
+ logger: logger,
+ }
+ s.BaseService = service.NewBaseService("kubernetes", logger, s)
+ return s
+}
+
+func (s *Service) getKV() clientv3.KV {
+ return s.consensusService.GetStore("kubernetes", "")
+}
+
+func (s *Service) NewCluster() error {
+ return newCluster(s.getKV())
+}
+
+func (s *Service) OnStart() error {
+ config := Config{
+ AdvertiseAddress: net.IP{10, 0, 2, 15}, // Depends on networking
+ ServiceIPRange: net.IPNet{ // TODO: Decide if configurable / final value
+ IP: net.IP{192, 168, 188, 0},
+ Mask: net.IPMask{0xff, 0xff, 0xff, 0x00}, // /24, but Go stores as a literal mask
+ },
+ ClusterNet: net.IPNet{
+ IP: net.IP{192, 168, 188, 0},
+ Mask: net.IPMask{0xff, 0xff, 0xfd, 0x00}, // /22
+ },
+ }
+ consensusKV := s.getKV()
+ apiserverConfig, err := getPKIApiserverConfig(consensusKV)
+ if err != nil {
+ return err
+ }
+ apiserverConfig.advertiseAddress = config.AdvertiseAddress
+ apiserverConfig.serviceIPRange = config.ServiceIPRange
+ controllerManagerConfig, err := getPKIControllerManagerConfig(consensusKV)
+ if err != nil {
+ return err
+ }
+ controllerManagerConfig.clusterNet = config.ClusterNet
+ schedulerConfig, err := getPKISchedulerConfig(consensusKV)
+ if err != nil {
+ return err
+ }
+
+ go func() {
+ runAPIServer(*apiserverConfig)
+ }()
+ go func() {
+ runControllerManager(*controllerManagerConfig)
+ }()
+ go func() {
+ runScheduler(*schedulerConfig)
+ }()
+
+ return nil
+}
+
+func (s *Service) OnStop() error {
+ // Requires advanced process management and not necessary for MVP
+ return errors.New("Not implemented")
+}