Add Kubernetes Worker and infrastructure
Adds Kubernetes Kubelet with patches for syscall-based mounting and
syscall-based (and much faster) metrics. fsquota patches have been
deferred to a further revision (for robust emptyDir capacity isolation).
Changes encoding of the node ID to hex since Base64-URL is not supported
as a character set for K8s names. Also adds `/etc/machine-id` and
`/etc/os-release` since Kubernetes wants them. `os-release` is generated
by stamping, `machine-id` is the hex-encoded node ID derived from the
public key.
Also includes a primitive reconciler which automatically ensures a set of
built-in Kubernetes objects are always present. Currently this includes
a PSP and some basic RBAC policies that are elementary to proper cluster
operations.
Adds an additional gRPC service (NodeDebugService) to cleanly
communicate with external debug and test tooling. It supports reading
from logbuffers for all externally-run components, checking conditions
(for replacing log matching in testing and debugging) and getting
debug credentials for the Kubernetes cluster.
A small utility (dbg) is provided that interfaces with NodeDebugService
and provides access to its functions from the CLI. It also incorporates
a kubectl wrapper which directly grabs credentials from the Debug API
and passes them to kubectl
(e.g. `bazel run //core/cmd/dbg -- kubectl describe node`).
Test Plan:
Manually tested.
Kubernetes:
`bazel run //core/cmd/dbg -- kubectl create -f test.yml`
Checked that pods run, logs are accessible and exec works.
Reading buffers:
`bazel run //core/cmd/dbg -- logs containerd`
Outputs containerd logs in the right order.
Automated testing is in the works, but has been deferred to a future
revision because this one is already too big again.
X-Origin-Diff: phab/D525
GitOrigin-RevId: 0fbfa0c433de405526c7f09ef10c466896331328
diff --git a/core/internal/kubernetes/BUILD.bazel b/core/internal/kubernetes/BUILD.bazel
index e9b0573..534bf6e 100644
--- a/core/internal/kubernetes/BUILD.bazel
+++ b/core/internal/kubernetes/BUILD.bazel
@@ -6,18 +6,30 @@
"apiserver.go",
"auth.go",
"controller-manager.go",
+ "kubelet.go",
+ "reconcile.go",
"scheduler.go",
"service.go",
],
importpath = "git.monogon.dev/source/nexantic.git/core/internal/kubernetes",
visibility = ["//core:__subpackages__"],
deps = [
+ "//core/api/api:go_default_library",
"//core/internal/common/service:go_default_library",
"//core/internal/consensus:go_default_library",
"//core/pkg/fileargs:go_default_library",
+ "//core/pkg/logbuffer:go_default_library",
"@io_etcd_go_etcd//clientv3:go_default_library",
+ "@io_k8s_api//core/v1:go_default_library",
+ "@io_k8s_api//policy/v1beta1:go_default_library",
+ "@io_k8s_api//rbac/v1:go_default_library",
+ "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
+ "@io_k8s_client_go//kubernetes:go_default_library",
"@io_k8s_client_go//tools/clientcmd:go_default_library",
"@io_k8s_client_go//tools/clientcmd/api:go_default_library",
+ "@io_k8s_kubelet//config/v1beta1:go_default_library",
+ "@org_golang_google_grpc//codes:go_default_library",
+ "@org_golang_google_grpc//status:go_default_library",
"@org_uber_go_zap//:go_default_library",
],
)
diff --git a/core/internal/kubernetes/apiserver.go b/core/internal/kubernetes/apiserver.go
index 858fdb1..ac43035 100644
--- a/core/internal/kubernetes/apiserver.go
+++ b/core/internal/kubernetes/apiserver.go
@@ -21,8 +21,8 @@
"encoding/pem"
"errors"
"fmt"
+ "go.uber.org/zap"
"net"
- "os"
"os/exec"
"path"
@@ -65,13 +65,13 @@
return &config, nil
}
-func runAPIServer(config apiserverConfig) error {
+func (s *Service) runAPIServer(ctx context.Context, config apiserverConfig) error {
args, err := fileargs.New()
if err != nil {
panic(err) // If this fails, something is very wrong. Just crash.
}
defer args.Close()
- cmd := exec.Command("/bin/kube-controlplane", "kube-apiserver",
+ cmd := exec.CommandContext(ctx, "/kubernetes/bin/kube", "kube-apiserver",
fmt.Sprintf("--advertise-address=%v", config.advertiseAddress.String()),
"--authorization-mode=Node,RBAC",
args.FileOpt("--client-ca-file", "client-ca.pem",
@@ -85,7 +85,7 @@
pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: config.kubeletClientCert})),
args.FileOpt("--kubelet-client-key", "kubelet-client-key.pem",
pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: config.kubeletClientKey})),
- "--kubelet-preferred-address-types=InternalIP",
+ "--kubelet-preferred-address-types=Hostname",
args.FileOpt("--proxy-client-cert-file", "aggregation-client-cert.pem",
pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: config.aggregationClientCert})),
args.FileOpt("--proxy-client-key-file", "aggregation-client-key.pem",
@@ -107,8 +107,14 @@
if args.Error() != nil {
return err
}
- cmd.Stdout = os.Stdout
- cmd.Stderr = os.Stderr
+ cmd.Stdout = s.apiserverLogs
+ cmd.Stderr = s.apiserverLogs
err = cmd.Run()
+ fmt.Fprintf(s.apiserverLogs, "apiserver stopped: %v\n", err)
+ if ctx.Err() == context.Canceled {
+ s.logger.Info("apiserver stopped", zap.Error(err))
+ } else {
+ s.logger.Warn("apiserver stopped unexpectedly", zap.Error(err))
+ }
return err
}
diff --git a/core/internal/kubernetes/auth.go b/core/internal/kubernetes/auth.go
index 0095bc4..25e2e4b 100644
--- a/core/internal/kubernetes/auth.go
+++ b/core/internal/kubernetes/auth.go
@@ -30,6 +30,7 @@
"fmt"
"math/big"
"net"
+ "os"
"path"
"time"
@@ -232,7 +233,7 @@
}
kubeletClientCert, kubeletClientKey, err := issueCertificate(
- clientCertTemplate("kube-apiserver-kubelet-client", []string{"system:masters"}),
+ clientCertTemplate("smalltown:apiserver-kubelet-client", []string{}),
idCA, idKey,
)
if err != nil {
@@ -314,6 +315,34 @@
return err
}
+ masterClientCert, masterClientKey, err := issueCertificate(
+ clientCertTemplate("smalltown:master", []string{"system:masters"}),
+ idCA, idKey,
+ )
+ if err != nil {
+ return fmt.Errorf("failed to issue certificate for master client: %w", err)
+ }
+
+ masterClientKubeconfig, err := makeLocalKubeconfig(idCA, masterClientCert,
+ masterClientKey)
+ if err != nil {
+ return fmt.Errorf("failed to create kubeconfig for master client: %w", err)
+ }
+
+ _, err = consensusKV.Put(context.Background(), path.Join(etcdPath, "master.kubeconfig"),
+ string(masterClientKubeconfig))
+ if err != nil {
+ return fmt.Errorf("failed to store master kubeconfig: %w", err)
+ }
+
+ hostname, err := os.Hostname()
+ if err != nil {
+ return err
+ }
+ if err := bootstrapLocalKubelet(consensusKV, hostname); err != nil {
+ return err
+ }
+
return nil
}
diff --git a/core/internal/kubernetes/controller-manager.go b/core/internal/kubernetes/controller-manager.go
index 1146a14..a67f6fd 100644
--- a/core/internal/kubernetes/controller-manager.go
+++ b/core/internal/kubernetes/controller-manager.go
@@ -17,10 +17,11 @@
package kubernetes
import (
+ "context"
"encoding/pem"
"fmt"
+ "go.uber.org/zap"
"net"
- "os"
"os/exec"
"go.etcd.io/etcd/clientv3"
@@ -60,13 +61,13 @@
return &config, nil
}
-func runControllerManager(config controllerManagerConfig) error {
+func (s *Service) runControllerManager(ctx context.Context, config controllerManagerConfig) error {
args, err := fileargs.New()
if err != nil {
panic(err) // If this fails, something is very wrong. Just crash.
}
defer args.Close()
- cmd := exec.Command("/bin/kube-controlplane", "kube-controller-manager",
+ cmd := exec.CommandContext(ctx, "/kubernetes/bin/kube", "kube-controller-manager",
args.FileOpt("--kubeconfig", "kubeconfig", config.kubeConfig),
args.FileOpt("--service-account-private-key-file", "service-account-privkey.pem",
pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: config.serviceAccountPrivKey})),
@@ -83,7 +84,14 @@
if args.Error() != nil {
return fmt.Errorf("failed to use fileargs: %w", err)
}
- cmd.Stdout = os.Stdout
- cmd.Stderr = os.Stderr
- return cmd.Run()
+ cmd.Stdout = s.controllerManagerLogs
+ cmd.Stderr = s.controllerManagerLogs
+ err = cmd.Run()
+ fmt.Fprintf(s.controllerManagerLogs, "controller-manager stopped: %v\n", err)
+ if ctx.Err() == context.Canceled {
+ s.logger.Info("controller-manager stopped", zap.Error(err))
+ } else {
+ s.logger.Warn("controller-manager stopped unexpectedly", zap.Error(err))
+ }
+ return err
}
diff --git a/core/internal/kubernetes/kubelet.go b/core/internal/kubernetes/kubelet.go
new file mode 100644
index 0000000..b7d8157
--- /dev/null
+++ b/core/internal/kubernetes/kubelet.go
@@ -0,0 +1,141 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kubernetes
+
+import (
+ "context"
+ "crypto/ed25519"
+ "encoding/json"
+ "encoding/pem"
+ "fmt"
+ "go.uber.org/zap"
+ "io/ioutil"
+ "os"
+ "os/exec"
+
+ "net"
+
+ "git.monogon.dev/source/nexantic.git/core/pkg/fileargs"
+ "go.etcd.io/etcd/clientv3"
+ v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/kubelet/config/v1beta1"
+)
+
+type KubeletSpec struct {
+ clusterDNS []net.IP
+}
+
+func bootstrapLocalKubelet(consensusKV clientv3.KV, nodeName string) error {
+ idCA, idKeyRaw, err := getCert(consensusKV, "id-ca")
+ if err != nil {
+ return err
+ }
+ idKey := ed25519.PrivateKey(idKeyRaw)
+ cert, key, err := issueCertificate(clientCertTemplate("system:node:"+nodeName, []string{"system:nodes"}), idCA, idKey)
+ if err != nil {
+ return err
+ }
+ kubeconfig, err := makeLocalKubeconfig(idCA, cert, key)
+ if err != nil {
+ return err
+ }
+
+ serverCert, serverKey, err := issueCertificate(serverCertTemplate([]string{nodeName}, []net.IP{}), idCA, idKey)
+ if err != nil {
+ return err
+ }
+ if err := os.MkdirAll("/data/kubernetes", 0755); err != nil {
+ return err
+ }
+ if err := ioutil.WriteFile("/data/kubernetes/kubelet.kubeconfig", kubeconfig, 0400); err != nil {
+ return err
+ }
+ if err := ioutil.WriteFile("/data/kubernetes/ca.crt", pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: idCA}), 0400); err != nil {
+ return err
+ }
+ if err := ioutil.WriteFile("/data/kubernetes/kubelet.crt", pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: serverCert}), 0400); err != nil {
+ return err
+ }
+ if err := ioutil.WriteFile("/data/kubernetes/kubelet.key", pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: serverKey}), 0400); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *Service) runKubelet(ctx context.Context, spec *KubeletSpec) error {
+ fargs, err := fileargs.New()
+ if err != nil {
+ return err
+ }
+ var clusterDNS []string
+ for _, dnsIP := range spec.clusterDNS {
+ clusterDNS = append(clusterDNS, dnsIP.String())
+ }
+
+ kubeletConf := &v1beta1.KubeletConfiguration{
+ TypeMeta: v1.TypeMeta{
+ Kind: "KubeletConfiguration",
+ APIVersion: v1beta1.GroupName + "/v1beta1",
+ },
+ TLSCertFile: "/data/kubernetes/kubelet.crt",
+ TLSPrivateKeyFile: "/data/kubernetes/kubelet.key",
+ TLSMinVersion: "VersionTLS13",
+ ClusterDNS: clusterDNS,
+ Authentication: v1beta1.KubeletAuthentication{
+ X509: v1beta1.KubeletX509Authentication{
+ ClientCAFile: "/data/kubernetes/ca.crt",
+ },
+ },
+ ClusterDomain: "cluster.local",
+ EnableControllerAttachDetach: False(),
+ HairpinMode: "none",
+ MakeIPTablesUtilChains: False(), // We don't have iptables
+ FailSwapOn: False(), // Our kernel doesn't have swap enabled which breaks Kubelet's detection
+ KubeReserved: map[string]string{
+ "cpu": "200m",
+ "memory": "300Mi",
+ },
+ // We're not going to use this, but let's make it point to a known-empty directory in case anybody manages to
+ // trigger it.
+ VolumePluginDir: "/kubernetes/conf/flexvolume-plugins",
+ }
+
+ configRaw, err := json.Marshal(kubeletConf)
+ if err != nil {
+ return err
+ }
+ cmd := exec.CommandContext(ctx, "/kubernetes/bin/kube", "kubelet",
+ fargs.FileOpt("--config", "config.json", configRaw),
+ "--container-runtime=remote",
+ "--container-runtime-endpoint=unix:///containerd/run/containerd.sock",
+ "--kubeconfig=/data/kubernetes/kubelet.kubeconfig",
+ "--root-dir=/data/kubernetes/kubelet",
+ )
+ cmd.Env = []string{"PATH=/kubernetes/bin"}
+ cmd.Stdout = s.kubeletLogs
+ cmd.Stderr = s.kubeletLogs
+
+ err = cmd.Run()
+ fmt.Fprintf(s.kubeletLogs, "kubelet stopped: %v\n", err)
+ if ctx.Err() == context.Canceled {
+ s.logger.Info("kubelet stopped", zap.Error(err))
+ } else {
+ s.logger.Warn("kubelet stopped unexpectedly", zap.Error(err))
+ }
+ return err
+}
diff --git a/core/internal/kubernetes/reconcile.go b/core/internal/kubernetes/reconcile.go
new file mode 100644
index 0000000..cf991ce
--- /dev/null
+++ b/core/internal/kubernetes/reconcile.go
@@ -0,0 +1,313 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// The reconciler ensures that a base set of K8s resources is always available in the cluster. These are necessary to
+// ensure correct out-of-the-box functionality. All resources containing the smalltown.com/builtin=true label are assumed
+// to be managed by the reconciler.
+// It currently does not revert modifications made by admins, it is planned to create an admission plugin prohibiting
+// such modifications to resources with the smalltown.com/builtin label to deal with that problem. This would also solve a
+// potential issue where you could delete resources just by adding the smalltown.com/builtin=true label.
+package kubernetes
+
+import (
+ "context"
+ "time"
+
+ "go.uber.org/zap"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/api/policy/v1beta1"
+ rbacv1 "k8s.io/api/rbac/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/client-go/kubernetes"
+ "k8s.io/client-go/tools/clientcmd"
+)
+
+const builtinRBACPrefix = "smalltown:"
+
+// Sad workaround for all the pointer booleans in K8s specs
+func True() *bool {
+ val := true
+ return &val
+}
+func False() *bool {
+ val := false
+ return &val
+}
+
+func rbac(name string) string {
+ return builtinRBACPrefix + name
+}
+
+// Extended from https://github.com/kubernetes/kubernetes/blob/master/cluster/gce/addons/podsecuritypolicies/unprivileged-addon.yaml
+var builtinPSPs = []*v1beta1.PodSecurityPolicy{
+ {
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "default",
+ Labels: map[string]string{
+ "smalltown.com/builtin": "true",
+ },
+ Annotations: map[string]string{
+ "kubernetes.io/description": "This default PSP allows the creation of pods using features that are" +
+ " generally considered safe against any sort of escape.",
+ },
+ },
+ Spec: v1beta1.PodSecurityPolicySpec{
+ AllowPrivilegeEscalation: True(),
+ AllowedCapabilities: []corev1.Capability{ // runc's default list of allowed capabilities
+ "SETPCAP",
+ "MKNOD",
+ "AUDIT_WRITE",
+ "CHOWN",
+ "NET_RAW",
+ "DAC_OVERRIDE",
+ "FOWNER",
+ "FSETID",
+ "KILL",
+ "SETGID",
+ "SETUID",
+ "NET_BIND_SERVICE",
+ "SYS_CHROOT",
+ "SETFCAP",
+ },
+ HostNetwork: false,
+ HostIPC: false,
+ HostPID: false,
+ FSGroup: v1beta1.FSGroupStrategyOptions{
+ Rule: v1beta1.FSGroupStrategyRunAsAny,
+ },
+ RunAsUser: v1beta1.RunAsUserStrategyOptions{
+ Rule: v1beta1.RunAsUserStrategyRunAsAny,
+ },
+ SELinux: v1beta1.SELinuxStrategyOptions{
+ Rule: v1beta1.SELinuxStrategyRunAsAny,
+ },
+ SupplementalGroups: v1beta1.SupplementalGroupsStrategyOptions{
+ Rule: v1beta1.SupplementalGroupsStrategyRunAsAny,
+ },
+ Volumes: []v1beta1.FSType{ // Volumes considered safe to use
+ v1beta1.ConfigMap,
+ v1beta1.EmptyDir,
+ v1beta1.Projected,
+ v1beta1.Secret,
+ v1beta1.DownwardAPI,
+ v1beta1.PersistentVolumeClaim,
+ },
+ },
+ },
+}
+
+var builtinClusterRoles = []*rbacv1.ClusterRole{
+ {
+ ObjectMeta: metav1.ObjectMeta{
+ Name: rbac("psp-default"),
+ Annotations: map[string]string{
+ "kubernetes.io/description": "This role grants access to the \"default\" PSP.",
+ },
+ },
+ Rules: []rbacv1.PolicyRule{
+ {
+ APIGroups: []string{"policy"},
+ Resources: []string{"podsecuritypolicies"},
+ ResourceNames: []string{"default"},
+ Verbs: []string{"use"},
+ },
+ },
+ },
+}
+
+var builtinClusterRoleBindings = []*rbacv1.ClusterRoleBinding{
+ {
+ ObjectMeta: metav1.ObjectMeta{
+ Name: rbac("default-psp-for-sa"),
+ Annotations: map[string]string{
+ "kubernetes.io/description": "This binding grants every service account access to the \"default\" PSP. " +
+ "Creation of Pods is still restricted by other RBAC roles. Otherwise no pods (unprivileged or not) " +
+ "can be created.",
+ },
+ },
+ RoleRef: rbacv1.RoleRef{
+ APIGroup: rbacv1.GroupName,
+ Kind: "ClusterRole",
+ Name: rbac("psp-default"),
+ },
+ Subjects: []rbacv1.Subject{
+ {
+ APIGroup: rbacv1.GroupName,
+ Kind: "Group",
+ Name: "system:serviceaccounts",
+ },
+ },
+ },
+ {
+ ObjectMeta: metav1.ObjectMeta{
+ Name: rbac("apiserver-kubelet-client"),
+ Annotations: map[string]string{
+ "kubernetes.io/description": "This binding grants the apiserver access to the kubelets. This enables " +
+ "lots of built-in functionality like reading logs or forwarding ports via the API.",
+ },
+ },
+ RoleRef: rbacv1.RoleRef{
+ APIGroup: rbacv1.GroupName,
+ Kind: "ClusterRole",
+ Name: "system:kubelet-api-admin",
+ },
+ Subjects: []rbacv1.Subject{
+ {
+ APIGroup: rbacv1.GroupName,
+ Kind: "User",
+ Name: "smalltown:apiserver-kubelet-client",
+ },
+ },
+ },
+}
+
+func runReconciler(ctx context.Context, masterKubeconfig []byte, log *zap.Logger) error {
+ rawClientConfig, err := clientcmd.NewClientConfigFromBytes(masterKubeconfig)
+ if err != nil {
+ return err
+ }
+
+ clientConfig, err := rawClientConfig.ClientConfig()
+ clientset, err := kubernetes.NewForConfig(clientConfig)
+ if err != nil {
+ return err
+ }
+ t := time.NewTicker(10 * time.Second)
+ for {
+ err = reconcile(ctx, clientset)
+ select {
+ case <-t.C:
+ err = reconcile(ctx, clientset)
+ if err != nil {
+ log.Warn("Failed to reconcile built-in resources", zap.Error(err))
+ }
+ case <-ctx.Done():
+ return nil
+ }
+ }
+}
+
+func reconcile(ctx context.Context, clientset *kubernetes.Clientset) error {
+ if err := reconcilePSPs(ctx, clientset); err != nil {
+ return err
+ }
+ if err := reconcileClusterRoles(ctx, clientset); err != nil {
+ return err
+ }
+ if err := reconcileClusterRoleBindings(ctx, clientset); err != nil {
+ return err
+ }
+ return nil
+}
+
+func reconcilePSPs(ctx context.Context, clientset *kubernetes.Clientset) error {
+ pspClient := clientset.PolicyV1beta1().PodSecurityPolicies()
+ availablePSPs, err := pspClient.List(ctx, metav1.ListOptions{
+ LabelSelector: "smalltown.com/builtin=true",
+ })
+ if err != nil {
+ return err
+ }
+ availablePSPMap := make(map[string]struct{})
+ for _, psp := range availablePSPs.Items {
+ availablePSPMap[psp.Name] = struct{}{}
+ }
+ expectedPSPMap := make(map[string]*v1beta1.PodSecurityPolicy)
+ for _, psp := range builtinPSPs {
+ expectedPSPMap[psp.Name] = psp
+ }
+ for pspName, psp := range expectedPSPMap {
+ if _, ok := availablePSPMap[pspName]; !ok {
+ if _, err := pspClient.Create(ctx, psp, metav1.CreateOptions{}); err != nil {
+ return err
+ }
+ }
+ }
+ for pspName, _ := range availablePSPMap {
+ if _, ok := expectedPSPMap[pspName]; !ok {
+ if err := pspClient.Delete(ctx, pspName, metav1.DeleteOptions{}); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+func reconcileClusterRoles(ctx context.Context, clientset *kubernetes.Clientset) error {
+ crClient := clientset.RbacV1().ClusterRoles()
+ availableCRs, err := crClient.List(ctx, metav1.ListOptions{
+ LabelSelector: "smalltown.com/builtin=true",
+ })
+ if err != nil {
+ return err
+ }
+ availableCRMap := make(map[string]struct{})
+ for _, cr := range availableCRs.Items {
+ availableCRMap[cr.Name] = struct{}{}
+ }
+ expectedCRMap := make(map[string]*rbacv1.ClusterRole)
+ for _, cr := range builtinClusterRoles {
+ expectedCRMap[cr.Name] = cr
+ }
+ for crName, psp := range expectedCRMap {
+ if _, ok := availableCRMap[crName]; !ok {
+ if _, err := crClient.Create(ctx, psp, metav1.CreateOptions{}); err != nil {
+ return err
+ }
+ }
+ }
+ for crName, _ := range availableCRMap {
+ if _, ok := expectedCRMap[crName]; !ok {
+ if err := crClient.Delete(ctx, crName, metav1.DeleteOptions{}); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+func reconcileClusterRoleBindings(ctx context.Context, clientset *kubernetes.Clientset) error {
+ crbClient := clientset.RbacV1().ClusterRoleBindings()
+ availableCRBs, err := crbClient.List(ctx, metav1.ListOptions{
+ LabelSelector: "smalltown.com/builtin=true",
+ })
+ if err != nil {
+ return err
+ }
+ availableCRBMap := make(map[string]struct{})
+ for _, crb := range availableCRBs.Items {
+ availableCRBMap[crb.Name] = struct{}{}
+ }
+ expectedCRBMap := make(map[string]*rbacv1.ClusterRoleBinding)
+ for _, crb := range builtinClusterRoleBindings {
+ expectedCRBMap[crb.Name] = crb
+ }
+ for crbName, psp := range expectedCRBMap {
+ if _, ok := availableCRBMap[crbName]; !ok {
+ if _, err := crbClient.Create(ctx, psp, metav1.CreateOptions{}); err != nil {
+ return err
+ }
+ }
+ }
+ for crbName, _ := range availableCRBMap {
+ if _, ok := expectedCRBMap[crbName]; !ok {
+ if err := crbClient.Delete(ctx, crbName, metav1.DeleteOptions{}); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
diff --git a/core/internal/kubernetes/scheduler.go b/core/internal/kubernetes/scheduler.go
index ac21588..75dea97 100644
--- a/core/internal/kubernetes/scheduler.go
+++ b/core/internal/kubernetes/scheduler.go
@@ -17,12 +17,13 @@
package kubernetes
import (
+ "context"
"encoding/pem"
"fmt"
- "os"
"os/exec"
"go.etcd.io/etcd/clientv3"
+ "go.uber.org/zap"
"git.monogon.dev/source/nexantic.git/core/pkg/fileargs"
)
@@ -47,13 +48,13 @@
return &config, nil
}
-func runScheduler(config schedulerConfig) error {
+func (s *Service) runScheduler(ctx context.Context, config schedulerConfig) error {
args, err := fileargs.New()
if err != nil {
panic(err) // If this fails, something is very wrong. Just crash.
}
defer args.Close()
- cmd := exec.Command("/bin/kube-controlplane", "kube-scheduler",
+ cmd := exec.CommandContext(ctx, "/kubernetes/bin/kube", "kube-scheduler",
args.FileOpt("--kubeconfig", "kubeconfig", config.kubeConfig),
"--port=0", // Kill insecure serving
args.FileOpt("--tls-cert-file", "server-cert.pem",
@@ -64,7 +65,14 @@
if args.Error() != nil {
return fmt.Errorf("failed to use fileargs: %w", err)
}
- cmd.Stdout = os.Stdout
- cmd.Stderr = os.Stderr
- return cmd.Run()
+ cmd.Stdout = s.schedulerLogs
+ cmd.Stderr = s.schedulerLogs
+ err = cmd.Run()
+ fmt.Fprintf(s.schedulerLogs, "scheduler stopped: %v\n", err)
+ if ctx.Err() == context.Canceled {
+ s.logger.Info("scheduler stopped", zap.Error(err))
+ } else {
+ s.logger.Warn("scheduler stopped unexpectedly", zap.Error(err))
+ }
+ return err
}
diff --git a/core/internal/kubernetes/service.go b/core/internal/kubernetes/service.go
index 9d653b4..5e28292 100644
--- a/core/internal/kubernetes/service.go
+++ b/core/internal/kubernetes/service.go
@@ -17,9 +17,18 @@
package kubernetes
import (
+ "context"
+ "crypto/ed25519"
"errors"
+ "fmt"
"net"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+
+ schema "git.monogon.dev/source/nexantic.git/core/generated/api"
+ "git.monogon.dev/source/nexantic.git/core/pkg/logbuffer"
+
"go.etcd.io/etcd/clientv3"
"go.uber.org/zap"
@@ -35,14 +44,22 @@
type Service struct {
*service.BaseService
- consensusService *consensus.Service
- logger *zap.Logger
+ consensusService *consensus.Service
+ logger *zap.Logger
+ apiserverLogs *logbuffer.LogBuffer
+ controllerManagerLogs *logbuffer.LogBuffer
+ schedulerLogs *logbuffer.LogBuffer
+ kubeletLogs *logbuffer.LogBuffer
}
func New(logger *zap.Logger, consensusService *consensus.Service) *Service {
s := &Service{
- consensusService: consensusService,
- logger: logger,
+ consensusService: consensusService,
+ logger: logger,
+ apiserverLogs: logbuffer.New(5000, 16384),
+ controllerManagerLogs: logbuffer.New(5000, 16384),
+ schedulerLogs: logbuffer.New(5000, 16384),
+ kubeletLogs: logbuffer.New(5000, 16384),
}
s.BaseService = service.NewBaseService("kubernetes", logger, s)
return s
@@ -56,6 +73,40 @@
return newCluster(s.getKV())
}
+// GetComponentLogs grabs logs from various Kubernetes binaries
+func (s *Service) GetComponentLogs(component string, n int) ([]string, error) {
+ switch component {
+ case "apiserver":
+ return s.apiserverLogs.ReadLinesTruncated(n, "..."), nil
+ case "controller-manager":
+ return s.controllerManagerLogs.ReadLinesTruncated(n, "..."), nil
+ case "scheduler":
+ return s.schedulerLogs.ReadLinesTruncated(n, "..."), nil
+ case "kubelet":
+ return s.kubeletLogs.ReadLinesTruncated(n, "..."), nil
+ default:
+ return []string{}, errors.New("component not available")
+ }
+}
+
+// GetDebugKubeconfig issues a kubeconfig for an arbitrary given identity. Useful for debugging and testing.
+func (s *Service) GetDebugKubeconfig(ctx context.Context, request *schema.GetDebugKubeconfigRequest) (*schema.GetDebugKubeconfigResponse, error) {
+ idCA, idKeyRaw, err := getCert(s.getKV(), "id-ca")
+ idKey := ed25519.PrivateKey(idKeyRaw)
+ if err != nil {
+ return nil, status.Errorf(codes.Unavailable, "Failed to load ID CA: %v", err)
+ }
+ debugCert, debugKey, err := issueCertificate(clientCertTemplate(request.Id, request.Groups), idCA, idKey)
+ if err != nil {
+ return nil, status.Errorf(codes.Unavailable, "Failed to issue certs for kubeconfig: %v\n", err)
+ }
+ debugKubeconfig, err := makeLocalKubeconfig(idCA, debugCert, debugKey)
+ if err != nil {
+ return nil, status.Errorf(codes.Unavailable, "Failed to generate kubeconfig: %v", err)
+ }
+ return &schema.GetDebugKubeconfigResponse{DebugKubeconfig: string(debugKubeconfig)}, nil
+}
+
func (s *Service) OnStart() error {
config := Config{
AdvertiseAddress: net.IP{10, 0, 2, 15}, // Depends on networking
@@ -85,14 +136,30 @@
return err
}
+ masterKubeconfig, err := getSingle(consensusKV, "master.kubeconfig")
+ if err != nil {
+ return err
+ }
+
+ // TODO(lorenz): Once internal/node is part of the supervisor tree, these should all be supervisor runnables
go func() {
- runAPIServer(*apiserverConfig)
+ s.runAPIServer(context.TODO(), *apiserverConfig)
}()
go func() {
- runControllerManager(*controllerManagerConfig)
+ s.runControllerManager(context.TODO(), *controllerManagerConfig)
}()
go func() {
- runScheduler(*schedulerConfig)
+ s.runScheduler(context.TODO(), *schedulerConfig)
+ }()
+
+ go func() {
+ if err := s.runKubelet(context.TODO(), &KubeletSpec{}); err != nil {
+ fmt.Printf("Failed to launch kubelet: %v\n", err)
+ }
+ }()
+
+ go func() {
+ go runReconciler(context.TODO(), masterKubeconfig, s.logger)
}()
return nil