Lorenz Brun | 6e8f69c | 2019-11-18 10:44:24 +0100 | [diff] [blame] | 1 | // Copyright 2020 The Monogon Project Authors. |
| 2 | // |
| 3 | // SPDX-License-Identifier: Apache-2.0 |
| 4 | // |
| 5 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | // you may not use this file except in compliance with the License. |
| 7 | // You may obtain a copy of the License at |
| 8 | // |
| 9 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | // |
| 11 | // Unless required by applicable law or agreed to in writing, software |
| 12 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | // See the License for the specific language governing permissions and |
| 15 | // limitations under the License. |
| 16 | |
| 17 | package kubernetes |
| 18 | |
| 19 | import ( |
Lorenz Brun | 878f5f9 | 2020-05-12 16:15:39 +0200 | [diff] [blame] | 20 | "context" |
| 21 | "crypto/ed25519" |
Lorenz Brun | 6e8f69c | 2019-11-18 10:44:24 +0100 | [diff] [blame] | 22 | "errors" |
| 23 | "net" |
Lorenz Brun | b15abad | 2020-04-16 11:17:12 +0200 | [diff] [blame] | 24 | "time" |
| 25 | |
| 26 | "k8s.io/client-go/informers" |
| 27 | "k8s.io/client-go/tools/clientcmd" |
Lorenz Brun | 6e8f69c | 2019-11-18 10:44:24 +0100 | [diff] [blame] | 28 | |
Lorenz Brun | 8e3b8fc | 2020-05-19 14:29:40 +0200 | [diff] [blame] | 29 | "go.etcd.io/etcd/clientv3" |
| 30 | "go.uber.org/zap" |
Lorenz Brun | 878f5f9 | 2020-05-12 16:15:39 +0200 | [diff] [blame] | 31 | "google.golang.org/grpc/codes" |
| 32 | "google.golang.org/grpc/status" |
Lorenz Brun | b15abad | 2020-04-16 11:17:12 +0200 | [diff] [blame] | 33 | "k8s.io/client-go/kubernetes" |
Lorenz Brun | 878f5f9 | 2020-05-12 16:15:39 +0200 | [diff] [blame] | 34 | |
| 35 | schema "git.monogon.dev/source/nexantic.git/core/generated/api" |
Lorenz Brun | 8e3b8fc | 2020-05-19 14:29:40 +0200 | [diff] [blame] | 36 | "git.monogon.dev/source/nexantic.git/core/internal/common/supervisor" |
Hendrik Hofstadt | 8efe51e | 2020-02-28 12:53:41 +0100 | [diff] [blame] | 37 | "git.monogon.dev/source/nexantic.git/core/internal/consensus" |
Serge Bazanski | e6030f6 | 2020-06-03 17:52:59 +0200 | [diff] [blame] | 38 | "git.monogon.dev/source/nexantic.git/core/internal/kubernetes/reconciler" |
Lorenz Brun | 0db90ba | 2020-04-06 14:04:52 +0200 | [diff] [blame] | 39 | "git.monogon.dev/source/nexantic.git/core/internal/storage" |
Lorenz Brun | 8e3b8fc | 2020-05-19 14:29:40 +0200 | [diff] [blame] | 40 | "git.monogon.dev/source/nexantic.git/core/pkg/logbuffer" |
Lorenz Brun | 6e8f69c | 2019-11-18 10:44:24 +0100 | [diff] [blame] | 41 | ) |
| 42 | |
| 43 | type Config struct { |
| 44 | AdvertiseAddress net.IP |
| 45 | ServiceIPRange net.IPNet |
| 46 | ClusterNet net.IPNet |
| 47 | } |
| 48 | |
| 49 | type Service struct { |
Lorenz Brun | 878f5f9 | 2020-05-12 16:15:39 +0200 | [diff] [blame] | 50 | consensusService *consensus.Service |
Lorenz Brun | 0db90ba | 2020-04-06 14:04:52 +0200 | [diff] [blame] | 51 | storageService *storage.Manager |
Lorenz Brun | 878f5f9 | 2020-05-12 16:15:39 +0200 | [diff] [blame] | 52 | logger *zap.Logger |
| 53 | apiserverLogs *logbuffer.LogBuffer |
| 54 | controllerManagerLogs *logbuffer.LogBuffer |
| 55 | schedulerLogs *logbuffer.LogBuffer |
| 56 | kubeletLogs *logbuffer.LogBuffer |
Lorenz Brun | 6e8f69c | 2019-11-18 10:44:24 +0100 | [diff] [blame] | 57 | } |
| 58 | |
Lorenz Brun | 0db90ba | 2020-04-06 14:04:52 +0200 | [diff] [blame] | 59 | func New(logger *zap.Logger, consensusService *consensus.Service, storageService *storage.Manager) *Service { |
Lorenz Brun | 6e8f69c | 2019-11-18 10:44:24 +0100 | [diff] [blame] | 60 | s := &Service{ |
Lorenz Brun | 878f5f9 | 2020-05-12 16:15:39 +0200 | [diff] [blame] | 61 | consensusService: consensusService, |
Lorenz Brun | 0db90ba | 2020-04-06 14:04:52 +0200 | [diff] [blame] | 62 | storageService: storageService, |
Lorenz Brun | 878f5f9 | 2020-05-12 16:15:39 +0200 | [diff] [blame] | 63 | logger: logger, |
| 64 | apiserverLogs: logbuffer.New(5000, 16384), |
| 65 | controllerManagerLogs: logbuffer.New(5000, 16384), |
| 66 | schedulerLogs: logbuffer.New(5000, 16384), |
| 67 | kubeletLogs: logbuffer.New(5000, 16384), |
Lorenz Brun | 6e8f69c | 2019-11-18 10:44:24 +0100 | [diff] [blame] | 68 | } |
Lorenz Brun | 6e8f69c | 2019-11-18 10:44:24 +0100 | [diff] [blame] | 69 | return s |
| 70 | } |
| 71 | |
| 72 | func (s *Service) getKV() clientv3.KV { |
| 73 | return s.consensusService.GetStore("kubernetes", "") |
| 74 | } |
| 75 | |
| 76 | func (s *Service) NewCluster() error { |
| 77 | return newCluster(s.getKV()) |
| 78 | } |
| 79 | |
Lorenz Brun | 878f5f9 | 2020-05-12 16:15:39 +0200 | [diff] [blame] | 80 | // GetComponentLogs grabs logs from various Kubernetes binaries |
| 81 | func (s *Service) GetComponentLogs(component string, n int) ([]string, error) { |
| 82 | switch component { |
| 83 | case "apiserver": |
| 84 | return s.apiserverLogs.ReadLinesTruncated(n, "..."), nil |
| 85 | case "controller-manager": |
| 86 | return s.controllerManagerLogs.ReadLinesTruncated(n, "..."), nil |
| 87 | case "scheduler": |
| 88 | return s.schedulerLogs.ReadLinesTruncated(n, "..."), nil |
| 89 | case "kubelet": |
| 90 | return s.kubeletLogs.ReadLinesTruncated(n, "..."), nil |
| 91 | default: |
| 92 | return []string{}, errors.New("component not available") |
| 93 | } |
| 94 | } |
| 95 | |
| 96 | // GetDebugKubeconfig issues a kubeconfig for an arbitrary given identity. Useful for debugging and testing. |
| 97 | func (s *Service) GetDebugKubeconfig(ctx context.Context, request *schema.GetDebugKubeconfigRequest) (*schema.GetDebugKubeconfigResponse, error) { |
Lorenz Brun | fc5dbc6 | 2020-05-28 12:18:07 +0200 | [diff] [blame] | 98 | if !s.consensusService.IsReady() { |
| 99 | return nil, status.Error(codes.Unavailable, "Consensus not ready yet") |
| 100 | } |
Lorenz Brun | 878f5f9 | 2020-05-12 16:15:39 +0200 | [diff] [blame] | 101 | idCA, idKeyRaw, err := getCert(s.getKV(), "id-ca") |
| 102 | idKey := ed25519.PrivateKey(idKeyRaw) |
| 103 | if err != nil { |
| 104 | return nil, status.Errorf(codes.Unavailable, "Failed to load ID CA: %v", err) |
| 105 | } |
| 106 | debugCert, debugKey, err := issueCertificate(clientCertTemplate(request.Id, request.Groups), idCA, idKey) |
| 107 | if err != nil { |
| 108 | return nil, status.Errorf(codes.Unavailable, "Failed to issue certs for kubeconfig: %v\n", err) |
| 109 | } |
| 110 | debugKubeconfig, err := makeLocalKubeconfig(idCA, debugCert, debugKey) |
| 111 | if err != nil { |
| 112 | return nil, status.Errorf(codes.Unavailable, "Failed to generate kubeconfig: %v", err) |
| 113 | } |
| 114 | return &schema.GetDebugKubeconfigResponse{DebugKubeconfig: string(debugKubeconfig)}, nil |
| 115 | } |
| 116 | |
Lorenz Brun | 8e3b8fc | 2020-05-19 14:29:40 +0200 | [diff] [blame] | 117 | func (s *Service) Start() error { |
| 118 | // TODO(lorenz): This creates a new supervision tree, it should instead attach to the root one. But for that SmalltownNode needs |
| 119 | // to be ported to supervisor. |
| 120 | supervisor.New(context.TODO(), s.logger, s.Run()) |
Lorenz Brun | 6e8f69c | 2019-11-18 10:44:24 +0100 | [diff] [blame] | 121 | return nil |
| 122 | } |
| 123 | |
Lorenz Brun | 8e3b8fc | 2020-05-19 14:29:40 +0200 | [diff] [blame] | 124 | func (s *Service) Run() supervisor.Runnable { |
| 125 | return func(ctx context.Context) error { |
| 126 | config := Config{ |
| 127 | AdvertiseAddress: net.IP{10, 0, 2, 15}, // Depends on networking |
| 128 | ServiceIPRange: net.IPNet{ // TODO: Decide if configurable / final value |
| 129 | IP: net.IP{192, 168, 188, 0}, |
| 130 | Mask: net.IPMask{0xff, 0xff, 0xff, 0x00}, // /24, but Go stores as a literal mask |
| 131 | }, |
| 132 | ClusterNet: net.IPNet{ |
| 133 | IP: net.IP{192, 168, 188, 0}, |
| 134 | Mask: net.IPMask{0xff, 0xff, 0xfd, 0x00}, // /22 |
| 135 | }, |
| 136 | } |
| 137 | consensusKV := s.getKV() |
| 138 | apiserverConfig, err := getPKIApiserverConfig(consensusKV) |
| 139 | if err != nil { |
| 140 | return err |
| 141 | } |
| 142 | apiserverConfig.advertiseAddress = config.AdvertiseAddress |
| 143 | apiserverConfig.serviceIPRange = config.ServiceIPRange |
| 144 | controllerManagerConfig, err := getPKIControllerManagerConfig(consensusKV) |
| 145 | if err != nil { |
| 146 | return err |
| 147 | } |
| 148 | controllerManagerConfig.clusterNet = config.ClusterNet |
| 149 | schedulerConfig, err := getPKISchedulerConfig(consensusKV) |
| 150 | if err != nil { |
| 151 | return err |
| 152 | } |
| 153 | |
| 154 | masterKubeconfig, err := getSingle(consensusKV, "master.kubeconfig") |
| 155 | if err != nil { |
| 156 | return err |
| 157 | } |
| 158 | |
Lorenz Brun | b15abad | 2020-04-16 11:17:12 +0200 | [diff] [blame] | 159 | rawClientConfig, err := clientcmd.NewClientConfigFromBytes(masterKubeconfig) |
| 160 | if err != nil { |
| 161 | return err |
| 162 | } |
| 163 | |
| 164 | clientConfig, err := rawClientConfig.ClientConfig() |
| 165 | clientSet, err := kubernetes.NewForConfig(clientConfig) |
| 166 | if err != nil { |
| 167 | return err |
| 168 | } |
| 169 | |
| 170 | informerFactory := informers.NewSharedInformerFactory(clientSet, 5*time.Minute) |
| 171 | |
Lorenz Brun | 8e3b8fc | 2020-05-19 14:29:40 +0200 | [diff] [blame] | 172 | if err := supervisor.Run(ctx, "apiserver", runAPIServer(*apiserverConfig, s.apiserverLogs)); err != nil { |
| 173 | return err |
| 174 | } |
| 175 | if err := supervisor.Run(ctx, "controller-manager", runControllerManager(*controllerManagerConfig, s.controllerManagerLogs)); err != nil { |
| 176 | return err |
| 177 | } |
| 178 | if err := supervisor.Run(ctx, "scheduler", runScheduler(*schedulerConfig, s.schedulerLogs)); err != nil { |
| 179 | return err |
| 180 | } |
Lorenz Brun | 8e3b8fc | 2020-05-19 14:29:40 +0200 | [diff] [blame] | 181 | if err := supervisor.Run(ctx, "kubelet", runKubelet(&KubeletSpec{}, s.kubeletLogs)); err != nil { |
| 182 | return err |
| 183 | } |
Serge Bazanski | e6030f6 | 2020-06-03 17:52:59 +0200 | [diff] [blame] | 184 | if err := supervisor.Run(ctx, "reconciler", reconciler.Run(clientSet)); err != nil { |
Lorenz Brun | 8e3b8fc | 2020-05-19 14:29:40 +0200 | [diff] [blame] | 185 | return err |
| 186 | } |
Lorenz Brun | 0db90ba | 2020-04-06 14:04:52 +0200 | [diff] [blame] | 187 | if err := supervisor.Run(ctx, "csi-plugin", runCSIPlugin(s.storageService)); err != nil { |
| 188 | return err |
| 189 | } |
Lorenz Brun | b15abad | 2020-04-16 11:17:12 +0200 | [diff] [blame] | 190 | if err := supervisor.Run(ctx, "pv-provisioner", runCSIProvisioner(s.storageService, clientSet, informerFactory)); err != nil { |
| 191 | return err |
| 192 | } |
| 193 | |
Lorenz Brun | 8e3b8fc | 2020-05-19 14:29:40 +0200 | [diff] [blame] | 194 | supervisor.Signal(ctx, supervisor.SignalHealthy) |
| 195 | supervisor.Signal(ctx, supervisor.SignalDone) |
| 196 | return nil |
| 197 | } |
Lorenz Brun | 6e8f69c | 2019-11-18 10:44:24 +0100 | [diff] [blame] | 198 | } |