core/internal: move containerd and kubernetes to localstorage
This moves the last users of the old 'storage' library onto 'localstorage'. We move a lot of 'runtime' directories to a single `/ephemeral` root. This could be called `/run`, but that might imply FHS compliance - which we don't have, nor want to have.
We also slightly refactor Kubernetes services to be a bit nicer to spawn. But generally, this is a pure refactor, with no functional changes.
Test Plan: this should fail. part of a larger stack. D590 is the first tip of the stack that should work.
X-Origin-Diff: phab/D589
GitOrigin-RevId: d2a7c0bb52c2a7c753199221c609e03474936c22
diff --git a/core/internal/containerd/BUILD.bazel b/core/internal/containerd/BUILD.bazel
index a1deae0..77262b5 100644
--- a/core/internal/containerd/BUILD.bazel
+++ b/core/internal/containerd/BUILD.bazel
@@ -6,9 +6,8 @@
importpath = "git.monogon.dev/source/nexantic.git/core/internal/containerd",
visibility = ["//core:__subpackages__"],
deps = [
- "//core/internal/common/supervisor:go_default_library",
+ "//core/internal/localstorage:go_default_library",
"//core/pkg/logbuffer:go_default_library",
- "@org_golang_x_sys//unix:go_default_library",
],
)
diff --git a/core/internal/containerd/config.toml b/core/internal/containerd/config.toml
index 415391a..75d0a69 100644
--- a/core/internal/containerd/config.toml
+++ b/core/internal/containerd/config.toml
@@ -1,13 +1,13 @@
version = 2
root = "/data/containerd"
-state = "/containerd/run"
+state = "/ephemeral/containerd"
plugin_dir = ""
disabled_plugins = []
required_plugins = []
oom_score = 0
[grpc]
- address = "/containerd/run/containerd.sock"
+ address = "/ephemeral/containerd/client.sock"
tcp_address = ""
tcp_tls_cert = ""
tcp_tls_key = ""
diff --git a/core/internal/containerd/main.go b/core/internal/containerd/main.go
index 9abc976..289efe7 100644
--- a/core/internal/containerd/main.go
+++ b/core/internal/containerd/main.go
@@ -24,63 +24,54 @@
"os/exec"
"time"
- "git.monogon.dev/source/nexantic.git/core/internal/common/supervisor"
-
+ "git.monogon.dev/source/nexantic.git/core/internal/localstorage"
"git.monogon.dev/source/nexantic.git/core/pkg/logbuffer"
-
- "golang.org/x/sys/unix"
)
-const runscLogsFIFOPath = "/containerd/run/runsc-logs.fifo"
-
type Service struct {
+ EphemeralVolume *localstorage.EphemeralContainerdDirectory
+
Log *logbuffer.LogBuffer
RunscLog *logbuffer.LogBuffer
}
-func New() (*Service, error) {
- return &Service{Log: logbuffer.New(5000, 16384), RunscLog: logbuffer.New(5000, 16384)}, nil
-}
+func (s *Service) Run(ctx context.Context) error {
+ if s.Log == nil {
+ s.Log = logbuffer.New(5000, 16384)
+ }
+ if s.RunscLog == nil {
+ s.RunscLog = logbuffer.New(5000, 16384)
+ }
-func (s *Service) Run() supervisor.Runnable {
- return func(ctx context.Context) error {
- cmd := exec.CommandContext(ctx, "/containerd/bin/containerd", "--config", "/containerd/conf/config.toml")
- cmd.Stdout = s.Log
- cmd.Stderr = s.Log
- cmd.Env = []string{"PATH=/containerd/bin", "TMPDIR=/containerd/run/tmp"}
+ cmd := exec.CommandContext(ctx, "/containerd/bin/containerd", "--config", "/containerd/conf/config.toml")
+ cmd.Stdout = s.Log
+ cmd.Stderr = s.Log
+ cmd.Env = []string{"PATH=/containerd/bin", "TMPDIR=" + s.EphemeralVolume.Tmp.FullPath()}
- if err := unix.Mount("tmpfs", "/containerd/run", "tmpfs", 0, ""); err != nil {
- panic(err)
- }
- if err := os.MkdirAll("/containerd/run/tmp", 0755); err != nil {
- panic(err)
- }
-
- runscFifo, err := os.OpenFile(runscLogsFIFOPath, os.O_CREATE|os.O_RDONLY, os.ModeNamedPipe|0777)
- if err != nil {
- return err
- }
- go func() {
- for {
- n, err := io.Copy(s.RunscLog, runscFifo)
- if n == 0 && err == nil {
- // Hack because pipes/FIFOs can return zero reads when nobody is writing. To avoid busy-looping,
- // sleep a bit before retrying. This does not loose data since the FIFO internal buffer will
- // stall writes when it becomes full. 10ms maximum stall in a non-latency critical process (reading
- // debug logs) is not an issue for us.
- time.Sleep(10 * time.Millisecond)
- } else if err != nil {
- // TODO: Use supervisor.Logger() and Error() before exiting. Should never happen.
- fmt.Println(err)
- return // It's likely that this will busy-loop printing errors if it encounters one, so bail
- }
- }
- }()
-
- // TODO(lorenz): Healthcheck against CRI RuntimeService.Status() and SignalHealthy
-
- err = cmd.Run()
- fmt.Fprintf(s.Log, "containerd stopped: %v\n", err)
+ runscFifo, err := os.OpenFile(s.EphemeralVolume.RunSCLogsFIFO.FullPath(), os.O_CREATE|os.O_RDONLY, os.ModeNamedPipe|0777)
+ if err != nil {
return err
}
+ go func() {
+ for {
+ n, err := io.Copy(s.RunscLog, runscFifo)
+ if n == 0 && err == nil {
+ // Hack because pipes/FIFOs can return zero reads when nobody is writing. To avoid busy-looping,
+ // sleep a bit before retrying. This does not loose data since the FIFO internal buffer will
+ // stall writes when it becomes full. 10ms maximum stall in a non-latency critical process (reading
+ // debug logs) is not an issue for us.
+ time.Sleep(10 * time.Millisecond)
+ } else if err != nil {
+ // TODO: Use supervisor.Logger() and Error() before exiting. Should never happen.
+ fmt.Println(err)
+ return // It's likely that this will busy-loop printing errors if it encounters one, so bail
+ }
+ }
+ }()
+
+ // TODO(lorenz): Healthcheck against CRI RuntimeService.Status() and SignalHealthy
+
+ err = cmd.Run()
+ fmt.Fprintf(s.Log, "containerd stopped: %v\n", err)
+ return err
}
diff --git a/core/internal/containerd/runsc.toml b/core/internal/containerd/runsc.toml
index 15126b9..4fe0751 100644
--- a/core/internal/containerd/runsc.toml
+++ b/core/internal/containerd/runsc.toml
@@ -1,6 +1,6 @@
-root = "/containerd/run/runsc"
+root = "/ephemeral/containerd/runsc"
[runsc_config]
debug = "true"
-debug-log = "/containerd/run/runsc-logs.fifo"
-panic-log = "/containerd/run/runsc-logs.fifo"
-log = "/containerd/run/runsc-logs.fifo"
+debug-log = "/ephemeral/containerd/runsc-logs.fifo"
+panic-log = "/ephemeral/containerd/runsc-logs.fifo"
+log = "/ephemeral/containerd/runsc-logs.fifo"