Lorenz Brun | 878f5f9 | 2020-05-12 16:15:39 +0200 | [diff] [blame] | 1 | // Copyright 2020 The Monogon Project Authors. |
| 2 | // |
| 3 | // SPDX-License-Identifier: Apache-2.0 |
| 4 | // |
| 5 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | // you may not use this file except in compliance with the License. |
| 7 | // You may obtain a copy of the License at |
| 8 | // |
| 9 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | // |
| 11 | // Unless required by applicable law or agreed to in writing, software |
| 12 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | // See the License for the specific language governing permissions and |
| 15 | // limitations under the License. |
| 16 | |
| 17 | package kubernetes |
| 18 | |
| 19 | import ( |
| 20 | "context" |
Lorenz Brun | 878f5f9 | 2020-05-12 16:15:39 +0200 | [diff] [blame] | 21 | "encoding/json" |
Lorenz Brun | 878f5f9 | 2020-05-12 16:15:39 +0200 | [diff] [blame] | 22 | "fmt" |
Lorenz Brun | 8e3b8fc | 2020-05-19 14:29:40 +0200 | [diff] [blame] | 23 | "io" |
Lorenz Brun | 8e3b8fc | 2020-05-19 14:29:40 +0200 | [diff] [blame] | 24 | "net" |
Lorenz Brun | 878f5f9 | 2020-05-12 16:15:39 +0200 | [diff] [blame] | 25 | "os/exec" |
| 26 | |
Lorenz Brun | 878f5f9 | 2020-05-12 16:15:39 +0200 | [diff] [blame] | 27 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
Lorenz Brun | 8e3b8fc | 2020-05-19 14:29:40 +0200 | [diff] [blame] | 28 | kubeletconfig "k8s.io/kubelet/config/v1beta1" |
Serge Bazanski | 77cb6c5 | 2020-12-19 00:09:22 +0100 | [diff] [blame] | 29 | |
Serge Bazanski | 31370b0 | 2021-01-07 16:31:14 +0100 | [diff] [blame] | 30 | "source.monogon.dev/metropolis/node/core/localstorage" |
Serge Bazanski | 31370b0 | 2021-01-07 16:31:14 +0100 | [diff] [blame] | 31 | "source.monogon.dev/metropolis/node/kubernetes/pki" |
| 32 | "source.monogon.dev/metropolis/node/kubernetes/reconciler" |
| 33 | "source.monogon.dev/metropolis/pkg/fileargs" |
Serge Bazanski | 9411f7c | 2021-03-10 13:12:53 +0100 | [diff] [blame] | 34 | opki "source.monogon.dev/metropolis/pkg/pki" |
Serge Bazanski | 31370b0 | 2021-01-07 16:31:14 +0100 | [diff] [blame] | 35 | "source.monogon.dev/metropolis/pkg/supervisor" |
Lorenz Brun | 878f5f9 | 2020-05-12 16:15:39 +0200 | [diff] [blame] | 36 | ) |
| 37 | |
Serge Bazanski | c2c7ad9 | 2020-07-13 17:20:09 +0200 | [diff] [blame] | 38 | type kubeletService struct { |
| 39 | NodeName string |
| 40 | ClusterDNS []net.IP |
| 41 | KubeletDirectory *localstorage.DataKubernetesKubeletDirectory |
| 42 | EphemeralDirectory *localstorage.EphemeralDirectory |
| 43 | Output io.Writer |
Serge Bazanski | 9411f7c | 2021-03-10 13:12:53 +0100 | [diff] [blame] | 44 | KPKI *pki.PKI |
| 45 | |
| 46 | mount *opki.FilesystemCertificate |
| 47 | mountKubeconfigPath string |
Lorenz Brun | 878f5f9 | 2020-05-12 16:15:39 +0200 | [diff] [blame] | 48 | } |
| 49 | |
Serge Bazanski | c2c7ad9 | 2020-07-13 17:20:09 +0200 | [diff] [blame] | 50 | func (s *kubeletService) createCertificates(ctx context.Context) error { |
Serge Bazanski | 9411f7c | 2021-03-10 13:12:53 +0100 | [diff] [blame] | 51 | server, client, err := s.KPKI.VolatileKubelet(ctx, s.NodeName) |
Serge Bazanski | 71f7a56 | 2020-06-22 16:37:28 +0200 | [diff] [blame] | 52 | if err != nil { |
Serge Bazanski | 9411f7c | 2021-03-10 13:12:53 +0100 | [diff] [blame] | 53 | return fmt.Errorf("when generating local kubelet credentials: %w", err) |
Serge Bazanski | 71f7a56 | 2020-06-22 16:37:28 +0200 | [diff] [blame] | 54 | } |
| 55 | |
Serge Bazanski | 9411f7c | 2021-03-10 13:12:53 +0100 | [diff] [blame] | 56 | clientKubeconfig, err := pki.Kubeconfig(ctx, s.KPKI.KV, client) |
Serge Bazanski | 71f7a56 | 2020-06-22 16:37:28 +0200 | [diff] [blame] | 57 | if err != nil { |
Serge Bazanski | 9411f7c | 2021-03-10 13:12:53 +0100 | [diff] [blame] | 58 | return fmt.Errorf("when generating kubeconfig: %w", err) |
Serge Bazanski | 71f7a56 | 2020-06-22 16:37:28 +0200 | [diff] [blame] | 59 | } |
Serge Bazanski | dbfc638 | 2020-06-19 20:35:43 +0200 | [diff] [blame] | 60 | |
Serge Bazanski | 9411f7c | 2021-03-10 13:12:53 +0100 | [diff] [blame] | 61 | // Use a single fileargs mount for server certificate and client kubeconfig. |
| 62 | mounted, err := server.Mount(ctx, s.KPKI.KV) |
Serge Bazanski | dbfc638 | 2020-06-19 20:35:43 +0200 | [diff] [blame] | 63 | if err != nil { |
Serge Bazanski | 9411f7c | 2021-03-10 13:12:53 +0100 | [diff] [blame] | 64 | return fmt.Errorf("could not mount kubelet cert dir: %w", err) |
Serge Bazanski | 71f7a56 | 2020-06-22 16:37:28 +0200 | [diff] [blame] | 65 | } |
Serge Bazanski | 9411f7c | 2021-03-10 13:12:53 +0100 | [diff] [blame] | 66 | // mounted is closed by Run() on process exit. |
Serge Bazanski | dbfc638 | 2020-06-19 20:35:43 +0200 | [diff] [blame] | 67 | |
Serge Bazanski | 9411f7c | 2021-03-10 13:12:53 +0100 | [diff] [blame] | 68 | s.mount = mounted |
| 69 | s.mountKubeconfigPath = mounted.ArgPath("kubeconfig", clientKubeconfig) |
Serge Bazanski | 71f7a56 | 2020-06-22 16:37:28 +0200 | [diff] [blame] | 70 | |
| 71 | return nil |
| 72 | } |
| 73 | |
Serge Bazanski | c2c7ad9 | 2020-07-13 17:20:09 +0200 | [diff] [blame] | 74 | func (s *kubeletService) configure() *kubeletconfig.KubeletConfiguration { |
| 75 | var clusterDNS []string |
| 76 | for _, dnsIP := range s.ClusterDNS { |
| 77 | clusterDNS = append(clusterDNS, dnsIP.String()) |
| 78 | } |
Lorenz Brun | 878f5f9 | 2020-05-12 16:15:39 +0200 | [diff] [blame] | 79 | |
Serge Bazanski | c2c7ad9 | 2020-07-13 17:20:09 +0200 | [diff] [blame] | 80 | return &kubeletconfig.KubeletConfiguration{ |
| 81 | TypeMeta: v1.TypeMeta{ |
| 82 | Kind: "KubeletConfiguration", |
| 83 | APIVersion: kubeletconfig.GroupName + "/v1beta1", |
| 84 | }, |
Serge Bazanski | 9411f7c | 2021-03-10 13:12:53 +0100 | [diff] [blame] | 85 | TLSCertFile: s.mount.CertPath, |
| 86 | TLSPrivateKeyFile: s.mount.KeyPath, |
Serge Bazanski | c2c7ad9 | 2020-07-13 17:20:09 +0200 | [diff] [blame] | 87 | TLSMinVersion: "VersionTLS13", |
| 88 | ClusterDNS: clusterDNS, |
| 89 | Authentication: kubeletconfig.KubeletAuthentication{ |
| 90 | X509: kubeletconfig.KubeletX509Authentication{ |
Serge Bazanski | 9411f7c | 2021-03-10 13:12:53 +0100 | [diff] [blame] | 91 | ClientCAFile: s.mount.CACertPath, |
Lorenz Brun | 878f5f9 | 2020-05-12 16:15:39 +0200 | [diff] [blame] | 92 | }, |
Serge Bazanski | c2c7ad9 | 2020-07-13 17:20:09 +0200 | [diff] [blame] | 93 | }, |
| 94 | // TODO(q3k): move reconciler.False to a generic package, fix the following references. |
| 95 | ClusterDomain: "cluster.local", // cluster.local is hardcoded in the certificate too currently |
| 96 | EnableControllerAttachDetach: reconciler.False(), |
| 97 | HairpinMode: "none", |
| 98 | MakeIPTablesUtilChains: reconciler.False(), // We don't have iptables |
| 99 | FailSwapOn: reconciler.False(), // Our kernel doesn't have swap enabled which breaks Kubelet's detection |
| 100 | KubeReserved: map[string]string{ |
| 101 | "cpu": "200m", |
| 102 | "memory": "300Mi", |
| 103 | }, |
Lorenz Brun | 0db90ba | 2020-04-06 14:04:52 +0200 | [diff] [blame] | 104 | |
Serge Bazanski | c2c7ad9 | 2020-07-13 17:20:09 +0200 | [diff] [blame] | 105 | // We're not going to use this, but let's make it point to a known-empty directory in case anybody manages to |
| 106 | // trigger it. |
| 107 | VolumePluginDir: s.EphemeralDirectory.FlexvolumePlugins.FullPath(), |
| 108 | } |
| 109 | } |
Lorenz Brun | 878f5f9 | 2020-05-12 16:15:39 +0200 | [diff] [blame] | 110 | |
Serge Bazanski | c2c7ad9 | 2020-07-13 17:20:09 +0200 | [diff] [blame] | 111 | func (s *kubeletService) Run(ctx context.Context) error { |
| 112 | if err := s.createCertificates(ctx); err != nil { |
| 113 | return fmt.Errorf("when creating certificates: %w", err) |
| 114 | } |
Serge Bazanski | 9411f7c | 2021-03-10 13:12:53 +0100 | [diff] [blame] | 115 | defer s.mount.Close() |
Lorenz Brun | 8e3b8fc | 2020-05-19 14:29:40 +0200 | [diff] [blame] | 116 | |
Serge Bazanski | c2c7ad9 | 2020-07-13 17:20:09 +0200 | [diff] [blame] | 117 | configRaw, err := json.Marshal(s.configure()) |
| 118 | if err != nil { |
| 119 | return fmt.Errorf("when marshaling kubelet configuration: %w", err) |
| 120 | } |
| 121 | |
| 122 | fargs, err := fileargs.New() |
| 123 | if err != nil { |
Lorenz Brun | 878f5f9 | 2020-05-12 16:15:39 +0200 | [diff] [blame] | 124 | return err |
| 125 | } |
Serge Bazanski | c2c7ad9 | 2020-07-13 17:20:09 +0200 | [diff] [blame] | 126 | cmd := exec.CommandContext(ctx, "/kubernetes/bin/kube", "kubelet", |
| 127 | fargs.FileOpt("--config", "config.json", configRaw), |
| 128 | "--container-runtime=remote", |
| 129 | fmt.Sprintf("--container-runtime-endpoint=unix://%s", s.EphemeralDirectory.Containerd.ClientSocket.FullPath()), |
Serge Bazanski | 9411f7c | 2021-03-10 13:12:53 +0100 | [diff] [blame] | 130 | fmt.Sprintf("--kubeconfig=%s", s.mountKubeconfigPath), |
Serge Bazanski | c2c7ad9 | 2020-07-13 17:20:09 +0200 | [diff] [blame] | 131 | fmt.Sprintf("--root-dir=%s", s.KubeletDirectory.FullPath()), |
| 132 | ) |
| 133 | cmd.Env = []string{"PATH=/kubernetes/bin"} |
Serge Bazanski | 0560429 | 2021-03-12 17:47:21 +0100 | [diff] [blame] | 134 | return supervisor.RunCommand(ctx, cmd, supervisor.ParseKLog()) |
Lorenz Brun | 878f5f9 | 2020-05-12 16:15:39 +0200 | [diff] [blame] | 135 | } |