blob: 6f13a3a742ecb0d4178a5e9c7fed2c7956522747 [file] [log] [blame]
Lorenz Brunae0d90d2019-09-05 17:53:56 +02001// Copyright 2020 The Monogon Project Authors.
2//
3// SPDX-License-Identifier: Apache-2.0
4//
5// Licensed under the Apache License, Version 2.0 (the "License");
6// you may not use this file except in compliance with the License.
7// You may obtain a copy of the License at
8//
9// http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing, software
12// distributed under the License is distributed on an "AS IS" BASIS,
13// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14// See the License for the specific language governing permissions and
15// limitations under the License.
16
17package main
18
19import (
Serge Bazanskicdb8c782020-02-17 12:34:02 +010020 "context"
Serge Bazanski57b43752020-07-13 19:17:48 +020021 "crypto/ed25519"
22 "crypto/rand"
23 "crypto/x509"
Lorenz Brundd8c80e2019-10-07 16:19:49 +020024 "fmt"
Serge Bazanski1ebd1e12020-07-13 19:17:16 +020025 "log"
Serge Bazanski57b43752020-07-13 19:17:48 +020026 "math/big"
Serge Bazanski1ebd1e12020-07-13 19:17:16 +020027 "net"
Lorenz Brunae0d90d2019-09-05 17:53:56 +020028 "os"
Lorenz Brunae0d90d2019-09-05 17:53:56 +020029 "os/signal"
Lorenz Brunf95909d2019-09-11 19:48:26 +020030 "runtime/debug"
Serge Bazanski76003f82021-06-17 16:39:01 +020031 "time"
Lorenz Brunae0d90d2019-09-05 17:53:56 +020032
Lorenz Brunae0d90d2019-09-05 17:53:56 +020033 "golang.org/x/sys/unix"
Serge Bazanski1ebd1e12020-07-13 19:17:16 +020034 "google.golang.org/grpc"
Serge Bazanski31370b02021-01-07 16:31:14 +010035 common "source.monogon.dev/metropolis/node"
36 "source.monogon.dev/metropolis/node/core/cluster"
Serge Bazanski76003f82021-06-17 16:39:01 +020037 "source.monogon.dev/metropolis/node/core/curator"
Serge Bazanski31370b02021-01-07 16:31:14 +010038 "source.monogon.dev/metropolis/node/core/localstorage"
39 "source.monogon.dev/metropolis/node/core/localstorage/declarative"
40 "source.monogon.dev/metropolis/node/core/network"
Serge Bazanski31370b02021-01-07 16:31:14 +010041 "source.monogon.dev/metropolis/node/kubernetes"
42 "source.monogon.dev/metropolis/node/kubernetes/containerd"
43 "source.monogon.dev/metropolis/node/kubernetes/pki"
44 "source.monogon.dev/metropolis/pkg/logtree"
45 "source.monogon.dev/metropolis/pkg/supervisor"
46 "source.monogon.dev/metropolis/pkg/tpm"
47 apb "source.monogon.dev/metropolis/proto/api"
Lorenz Brunae0d90d2019-09-05 17:53:56 +020048)
49
Serge Bazanski1ebd1e12020-07-13 19:17:16 +020050var (
Serge Bazanski216fe7b2021-05-21 18:36:16 +020051 // kubernetesConfig is the static/global part of the Kubernetes service
52 // configuration. In the future, this might be configurable by loading it
53 // from the EnrolmentConfig. Fow now, it's static and same across all
54 // clusters.
Serge Bazanski1ebd1e12020-07-13 19:17:16 +020055 kubernetesConfig = kubernetes.Config{
56 ServiceIPRange: net.IPNet{ // TODO(q3k): Decide if configurable / final value
Lorenz Brunca24cfa2020-08-18 13:49:37 +020057 IP: net.IP{10, 0, 255, 1},
Serge Bazanski1ebd1e12020-07-13 19:17:16 +020058 Mask: net.IPMask{0xff, 0xff, 0xff, 0x00}, // /24, but Go stores as a literal mask
59 },
60 ClusterNet: net.IPNet{
61 IP: net.IP{10, 0, 0, 0},
62 Mask: net.IPMask{0xff, 0xff, 0x00, 0x00}, // /16
63 },
64 }
Leopold Schabela4516f92019-12-04 20:27:05 +000065)
66
Lorenz Brunae0d90d2019-09-05 17:53:56 +020067func main() {
Lorenz Brunf95909d2019-09-11 19:48:26 +020068 defer func() {
69 if r := recover(); r != nil {
70 fmt.Println("Init panicked:", r)
71 debug.PrintStack()
72 }
73 unix.Sync()
Leopold Schabel68c58752019-11-14 21:00:59 +010074 // TODO(lorenz): Switch this to Reboot when init panics are less likely
Serge Bazanski216fe7b2021-05-21 18:36:16 +020075 // Best effort, nothing we can do if this fails except printing the
76 // error to the console.
Leopold Schabel68c58752019-11-14 21:00:59 +010077 if err := unix.Reboot(unix.LINUX_REBOOT_CMD_POWER_OFF); err != nil {
78 panic(fmt.Sprintf("failed to halt node: %v\n", err))
79 }
Lorenz Brunf95909d2019-09-11 19:48:26 +020080 }()
Serge Bazanskic7359672020-10-30 16:38:57 +010081
Serge Bazanski662b5b32020-12-21 13:49:00 +010082 // Set up logger for Metropolis. Currently logs everything to stderr.
Serge Bazanskic7359672020-10-30 16:38:57 +010083 lt := logtree.New()
84 reader, err := lt.Read("", logtree.WithChildren(), logtree.WithStream())
Lorenz Brunae0d90d2019-09-05 17:53:56 +020085 if err != nil {
Serge Bazanskic7359672020-10-30 16:38:57 +010086 panic(fmt.Errorf("could not set up root log reader: %v", err))
Lorenz Brunae0d90d2019-09-05 17:53:56 +020087 }
Serge Bazanskic7359672020-10-30 16:38:57 +010088 go func() {
89 for {
90 p := <-reader.Stream
Serge Bazanskib0272182020-11-02 18:39:44 +010091 fmt.Fprintf(os.Stderr, "%s\n", p.String())
Serge Bazanskic7359672020-10-30 16:38:57 +010092 }
93 }()
94
95 // Initial logger. Used until we get to a supervisor.
96 logger := lt.MustLeveledFor("init")
Serge Bazanski581b0bd2020-03-12 13:36:43 +010097
Lorenz Brun3a99c592021-01-26 19:57:21 +010098 // Set up basic mounts
99 err = setupMounts(logger)
Serge Bazanski581b0bd2020-03-12 13:36:43 +0100100 if err != nil {
Lorenz Brun3a99c592021-01-26 19:57:21 +0100101 panic(fmt.Errorf("could not set up basic mounts: %w", err))
Serge Bazanski581b0bd2020-03-12 13:36:43 +0100102 }
103
Serge Bazanski216fe7b2021-05-21 18:36:16 +0200104 // Linux kernel default is 4096 which is far too low. Raise it to 1M which
105 // is what gVisor suggests.
Lorenz Brun878f5f92020-05-12 16:15:39 +0200106 if err := unix.Setrlimit(unix.RLIMIT_NOFILE, &unix.Rlimit{Cur: 1048576, Max: 1048576}); err != nil {
Serge Bazanskic7359672020-10-30 16:38:57 +0100107 logger.Fatalf("Failed to raise rlimits: %v", err)
Lorenz Brun878f5f92020-05-12 16:15:39 +0200108 }
109
Serge Bazanski662b5b32020-12-21 13:49:00 +0100110 logger.Info("Starting Metropolis node init")
Lorenz Brunae0d90d2019-09-05 17:53:56 +0200111
Lorenz Brunae0d90d2019-09-05 17:53:56 +0200112 signalChannel := make(chan os.Signal, 2)
113 signal.Notify(signalChannel)
114
Serge Bazanskic7359672020-10-30 16:38:57 +0100115 if err := tpm.Initialize(logger); err != nil {
116 logger.Fatalf("Failed to initialize TPM 2.0: %v", err)
Lorenz Brunae0d90d2019-09-05 17:53:56 +0200117 }
118
Serge Bazanskid8af5bf2021-03-16 13:38:29 +0100119 networkSvc := network.New()
Leopold Schabel68c58752019-11-14 21:00:59 +0100120
Serge Bazanski216fe7b2021-05-21 18:36:16 +0200121 // This function initializes a headless Delve if this is a debug build or
122 // does nothing if it's not
Lorenz Brun70f65b22020-07-08 17:02:47 +0200123 initializeDebugger(networkSvc)
124
Serge Bazanski1ebd1e12020-07-13 19:17:16 +0200125 // Prepare local storage.
126 root := &localstorage.Root{}
127 if err := declarative.PlaceFS(root, "/"); err != nil {
128 panic(fmt.Errorf("when placing root FS: %w", err))
129 }
130
Serge Bazanski216fe7b2021-05-21 18:36:16 +0200131 // trapdoor is a channel used to signal to the init service that a very
132 // low-level, unrecoverable failure occured. This causes a GURU MEDITATION
133 // ERROR visible to the end user.
Serge Bazanski1ebd1e12020-07-13 19:17:16 +0200134 trapdoor := make(chan struct{})
135
136 // Make context for supervisor. We cancel it when we reach the trapdoor.
137 ctxS, ctxC := context.WithCancel(context.Background())
138
Serge Bazanski216fe7b2021-05-21 18:36:16 +0200139 // Start root initialization code as a supervisor one-shot runnable. This
140 // means waiting for the network, starting the cluster manager, and then
141 // starting all services related to the node's roles.
Serge Bazanski1ebd1e12020-07-13 19:17:16 +0200142 // TODO(q3k): move this to a separate 'init' service.
Serge Bazanskic7359672020-10-30 16:38:57 +0100143 supervisor.New(ctxS, func(ctx context.Context) error {
Serge Bazanski1ebd1e12020-07-13 19:17:16 +0200144 logger := supervisor.Logger(ctx)
145
146 // Start storage and network - we need this to get anything else done.
147 if err := root.Start(ctx); err != nil {
148 return fmt.Errorf("cannot start root FS: %w", err)
149 }
Serge Bazanskib1b742f2020-03-24 13:58:19 +0100150 if err := supervisor.Run(ctx, "network", networkSvc.Run); err != nil {
Serge Bazanski1ebd1e12020-07-13 19:17:16 +0200151 return fmt.Errorf("when starting network: %w", err)
Serge Bazanskib1b742f2020-03-24 13:58:19 +0100152 }
Lorenz Brunf95909d2019-09-11 19:48:26 +0200153
Serge Bazanski216fe7b2021-05-21 18:36:16 +0200154 // Start cluster manager. This kicks off cluster membership machinery,
155 // which will either start a new cluster, enroll into one or join one.
Serge Bazanski1ebd1e12020-07-13 19:17:16 +0200156 m := cluster.NewManager(root, networkSvc)
157 if err := supervisor.Run(ctx, "enrolment", m.Run); err != nil {
158 return fmt.Errorf("when starting enrolment: %w", err)
159 }
160
Serge Bazanskia105db52021-04-12 19:57:46 +0200161 // Wait until the node finds a home in the new cluster.
162 watcher := m.Watch()
163 status, err := watcher.GetHome(ctx)
Serge Bazanski42e61c62021-03-18 15:07:18 +0100164 if err != nil {
Serge Bazanski1ebd1e12020-07-13 19:17:16 +0200165 close(trapdoor)
Serge Bazanskia105db52021-04-12 19:57:46 +0200166 return fmt.Errorf("new couldn't find home in new cluster, aborting: %w", err)
Serge Bazanski1ebd1e12020-07-13 19:17:16 +0200167 }
168
Serge Bazanski76003f82021-06-17 16:39:01 +0200169 // Start cluster curator.
170 kv, err := status.ConsensusClient(cluster.ConsensusUserCurator)
171 if err != nil {
172 return fmt.Errorf("failed to retrieve consensus curator client: %w", err)
173 }
174 c := curator.New(curator.Config{
175 Etcd: kv,
Serge Bazanskia959cbd2021-06-17 15:56:51 +0200176 NodeID: status.Credentials.ID(),
Serge Bazanski76003f82021-06-17 16:39:01 +0200177 // TODO(q3k): make this configurable?
178 LeaderTTL: time.Second * 5,
179 Directory: &root.Ephemeral.Curator,
180 })
181 if err := supervisor.Run(ctx, "curator", c.Run); err != nil {
182 return fmt.Errorf("when starting curator: %w", err)
183 }
184
Serge Bazanski216fe7b2021-05-21 18:36:16 +0200185 // We are now in a cluster. We can thus access our 'node' object and
186 // start all services that we should be running.
Serge Bazanski1ebd1e12020-07-13 19:17:16 +0200187
Serge Bazanski1ebd1e12020-07-13 19:17:16 +0200188 logger.Info("Enrolment success, continuing startup.")
Serge Bazanskia959cbd2021-06-17 15:56:51 +0200189
190 // HACK: always start k8s worker, this is being currently refactored
191 // and will get fixed in review/188.
192 logger.Info("Starting Kubernetes worker services...")
193
194 kkv, err := status.ConsensusClient(cluster.ConsensusUserKubernetesPKI)
195 if err != nil {
196 return fmt.Errorf("failed to retrieve consensus kubernetes PKI client: %w", err)
Serge Bazanski1ebd1e12020-07-13 19:17:16 +0200197 }
198
Serge Bazanskia959cbd2021-06-17 15:56:51 +0200199 // Ensure Kubernetes PKI objects exist in etcd.
200 kpki := pki.New(lt.MustLeveledFor("pki.kubernetes"), kkv)
201 if err := kpki.EnsureAll(ctx); err != nil {
202 return fmt.Errorf("failed to ensure kubernetes PKI present: %w", err)
203 }
Serge Bazanski1ebd1e12020-07-13 19:17:16 +0200204
Serge Bazanskia959cbd2021-06-17 15:56:51 +0200205 containerdSvc := &containerd.Service{
206 EphemeralVolume: &root.Ephemeral.Containerd,
207 }
208 if err := supervisor.Run(ctx, "containerd", containerdSvc.Run); err != nil {
209 return fmt.Errorf("failed to start containerd service: %w", err)
210 }
Serge Bazanskia105db52021-04-12 19:57:46 +0200211
Serge Bazanskia959cbd2021-06-17 15:56:51 +0200212 kubernetesConfig.KPKI = kpki
213 kubernetesConfig.Root = root
214 kubernetesConfig.Network = networkSvc
215 kubeSvc := kubernetes.New(kubernetesConfig)
216 if err := supervisor.Run(ctx, "kubernetes", kubeSvc.Run); err != nil {
217 return fmt.Errorf("failed to start kubernetes service: %w", err)
Serge Bazanski1ebd1e12020-07-13 19:17:16 +0200218 }
219
220 // Start the node debug service.
Serge Bazanski1ebd1e12020-07-13 19:17:16 +0200221 dbg := &debugService{
222 cluster: m,
Serge Bazanskib0272182020-11-02 18:39:44 +0100223 logtree: lt,
Serge Bazanski1ebd1e12020-07-13 19:17:16 +0200224 kubernetes: kubeSvc,
Lorenz Brun09c275b2021-03-30 12:47:09 +0200225 traceLock: make(chan struct{}, 1),
Serge Bazanski1ebd1e12020-07-13 19:17:16 +0200226 }
227 dbgSrv := grpc.NewServer()
228 apb.RegisterNodeDebugServiceServer(dbgSrv, dbg)
229 dbgLis, err := net.Listen("tcp", fmt.Sprintf(":%d", common.DebugServicePort))
Serge Bazanskib1b742f2020-03-24 13:58:19 +0100230 if err != nil {
Serge Bazanski1ebd1e12020-07-13 19:17:16 +0200231 return fmt.Errorf("failed to listen on debug service: %w", err)
232 }
233 if err := supervisor.Run(ctx, "debug", supervisor.GRPCServer(dbgSrv, dbgLis, false)); err != nil {
234 return fmt.Errorf("failed to start debug service: %w", err)
Serge Bazanskib1b742f2020-03-24 13:58:19 +0100235 }
Lorenz Brunae0d90d2019-09-05 17:53:56 +0200236
Serge Bazanskib1b742f2020-03-24 13:58:19 +0100237 supervisor.Signal(ctx, supervisor.SignalHealthy)
Serge Bazanski1ebd1e12020-07-13 19:17:16 +0200238 supervisor.Signal(ctx, supervisor.SignalDone)
239 return nil
Serge Bazanskic7359672020-10-30 16:38:57 +0100240 }, supervisor.WithExistingLogtree(lt))
Serge Bazanskib1b742f2020-03-24 13:58:19 +0100241
Serge Bazanski1ebd1e12020-07-13 19:17:16 +0200242 // We're PID1, so orphaned processes get reparented to us to clean up
243 for {
244 select {
245 case <-trapdoor:
Serge Bazanski216fe7b2021-05-21 18:36:16 +0200246 // If the trapdoor got closed, we got stuck early enough in the
247 // boot process that we can't do anything about it. Display a
248 // generic error message until we handle error conditions better.
Serge Bazanski1ebd1e12020-07-13 19:17:16 +0200249 ctxC()
250 log.Printf(" ########################")
251 log.Printf(" # GURU MEDIATION ERROR #")
252 log.Printf(" ########################")
253 log.Printf("")
Serge Bazanski662b5b32020-12-21 13:49:00 +0100254 log.Printf("Metropolis encountered an uncorrectable error and this node must be")
255 log.Printf("restarted.")
256 log.Printf("")
Serge Bazanski1ebd1e12020-07-13 19:17:16 +0200257 log.Printf("(Error condition: init trapdoor closed)")
258 log.Printf("")
259 select {}
260
261 case sig := <-signalChannel:
262 switch sig {
263 case unix.SIGCHLD:
264 var status unix.WaitStatus
265 var rusage unix.Rusage
266 for {
267 res, err := unix.Wait4(-1, &status, unix.WNOHANG, &rusage)
268 if err != nil && err != unix.ECHILD {
Serge Bazanskic7359672020-10-30 16:38:57 +0100269 logger.Errorf("Failed to wait on orphaned child: %v", err)
Serge Bazanski1ebd1e12020-07-13 19:17:16 +0200270 break
Serge Bazanskib1b742f2020-03-24 13:58:19 +0100271 }
Serge Bazanski1ebd1e12020-07-13 19:17:16 +0200272 if res <= 0 {
273 break
274 }
Lorenz Brunae0d90d2019-09-05 17:53:56 +0200275 }
Serge Bazanski1ebd1e12020-07-13 19:17:16 +0200276 case unix.SIGURG:
Serge Bazanski216fe7b2021-05-21 18:36:16 +0200277 // Go 1.14 introduced asynchronous preemption, which uses
278 // SIGURG.
279 // In order not to break backwards compatibility in the
280 // unlikely case of an application actually using SIGURG on its
281 // own, they're not filtering them.
Serge Bazanski1ebd1e12020-07-13 19:17:16 +0200282 // (https://github.com/golang/go/issues/37942)
Serge Bazanskic7359672020-10-30 16:38:57 +0100283 logger.V(5).Info("Ignoring SIGURG")
Serge Bazanski1ebd1e12020-07-13 19:17:16 +0200284 // TODO(lorenz): We can probably get more than just SIGCHLD as init, but I can't think
285 // of any others right now, just log them in case we hit any of them.
286 default:
Serge Bazanskic7359672020-10-30 16:38:57 +0100287 logger.Warningf("Got unexpected signal %s", sig.String())
Lorenz Brunae0d90d2019-09-05 17:53:56 +0200288 }
Lorenz Brunae0d90d2019-09-05 17:53:56 +0200289 }
Serge Bazanski1ebd1e12020-07-13 19:17:16 +0200290 }
Lorenz Brunae0d90d2019-09-05 17:53:56 +0200291}
Serge Bazanski57b43752020-07-13 19:17:48 +0200292
Serge Bazanski216fe7b2021-05-21 18:36:16 +0200293// nodeCertificate creates a node key/certificate for a foreign node. This is
294// duplicated code with localstorage's PKIDirectory EnsureSelfSigned, but is
295// temporary (and specific to 'golden tickets').
Serge Bazanski57b43752020-07-13 19:17:48 +0200296func (s *debugService) nodeCertificate() (cert, key []byte, err error) {
297 pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
298 if err != nil {
299 err = fmt.Errorf("failed to generate key: %w", err)
300 return
301 }
302
303 key, err = x509.MarshalPKCS8PrivateKey(privKey)
304 if err != nil {
305 err = fmt.Errorf("failed to marshal key: %w", err)
306 return
307 }
308
309 serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 127)
310 serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
311 if err != nil {
312 err = fmt.Errorf("failed to generate serial number: %w", err)
313 return
314 }
315
316 template := localstorage.CertificateForNode(pubKey)
317 template.SerialNumber = serialNumber
318
319 cert, err = x509.CreateCertificate(rand.Reader, &template, &template, pubKey, privKey)
320 if err != nil {
321 err = fmt.Errorf("could not sign certificate: %w", err)
322 return
323 }
324 return
325}