Add support for gVisor logging
Test Plan: Started Container using `bazel run //core/cmd/dbg -- kubectl run -i busybox --image=busybox test`, then observed logs using `bazel run //core/cmd/dbg logs containerd.runsc`
X-Origin-Diff: phab/D527
GitOrigin-RevId: 10dfa1704cbc18becc2005e7b38cc881e6ec50b5
diff --git a/core/internal/containerd/main.go b/core/internal/containerd/main.go
index 77e9156..9abc976 100644
--- a/core/internal/containerd/main.go
+++ b/core/internal/containerd/main.go
@@ -19,21 +19,27 @@
import (
"context"
"fmt"
- "git.monogon.dev/source/nexantic.git/core/internal/common/supervisor"
+ "io"
"os"
"os/exec"
+ "time"
+
+ "git.monogon.dev/source/nexantic.git/core/internal/common/supervisor"
"git.monogon.dev/source/nexantic.git/core/pkg/logbuffer"
"golang.org/x/sys/unix"
)
+const runscLogsFIFOPath = "/containerd/run/runsc-logs.fifo"
+
type Service struct {
- Log *logbuffer.LogBuffer
+ Log *logbuffer.LogBuffer
+ RunscLog *logbuffer.LogBuffer
}
func New() (*Service, error) {
- return &Service{Log: logbuffer.New(5000, 16384)}, nil
+ return &Service{Log: logbuffer.New(5000, 16384), RunscLog: logbuffer.New(5000, 16384)}, nil
}
func (s *Service) Run() supervisor.Runnable {
@@ -50,9 +56,30 @@
panic(err)
}
+ runscFifo, err := os.OpenFile(runscLogsFIFOPath, os.O_CREATE|os.O_RDONLY, os.ModeNamedPipe|0777)
+ if err != nil {
+ return err
+ }
+ go func() {
+ for {
+ n, err := io.Copy(s.RunscLog, runscFifo)
+ if n == 0 && err == nil {
+ // Hack because pipes/FIFOs can return zero reads when nobody is writing. To avoid busy-looping,
+ // sleep a bit before retrying. This does not loose data since the FIFO internal buffer will
+ // stall writes when it becomes full. 10ms maximum stall in a non-latency critical process (reading
+ // debug logs) is not an issue for us.
+ time.Sleep(10 * time.Millisecond)
+ } else if err != nil {
+ // TODO: Use supervisor.Logger() and Error() before exiting. Should never happen.
+ fmt.Println(err)
+ return // It's likely that this will busy-loop printing errors if it encounters one, so bail
+ }
+ }
+ }()
+
// TODO(lorenz): Healthcheck against CRI RuntimeService.Status() and SignalHealthy
- err := cmd.Run()
+ err = cmd.Run()
fmt.Fprintf(s.Log, "containerd stopped: %v\n", err)
return err
}
diff --git a/core/internal/containerd/runsc.toml b/core/internal/containerd/runsc.toml
index 52d846f..15126b9 100644
--- a/core/internal/containerd/runsc.toml
+++ b/core/internal/containerd/runsc.toml
@@ -1,5 +1,6 @@
root = "/containerd/run/runsc"
[runsc_config]
-debug = "false"
-# Setting intentionally left here in case anybody needs it since it is hard to find
-#debug-log = "/containerd/run/runsc-logs/"
\ No newline at end of file
+debug = "true"
+debug-log = "/containerd/run/runsc-logs.fifo"
+panic-log = "/containerd/run/runsc-logs.fifo"
+log = "/containerd/run/runsc-logs.fifo"
diff --git a/core/internal/node/debug.go b/core/internal/node/debug.go
index 1d91ad6..2ed3896 100644
--- a/core/internal/node/debug.go
+++ b/core/internal/node/debug.go
@@ -49,7 +49,11 @@
var err error
switch req.ComponentPath[0] {
case "containerd":
- lines = s.Containerd.Log.ReadLinesTruncated(linesToRead, "...")
+ if len(req.ComponentPath) < 2 {
+ lines = s.Containerd.Log.ReadLinesTruncated(linesToRead, "...")
+ } else if req.ComponentPath[1] == "runsc" {
+ lines = s.Containerd.RunscLog.ReadLinesTruncated(linesToRead, "...")
+ }
case "kube":
if len(req.ComponentPath) < 2 {
return nil, status.Error(codes.NotFound, "Component not found")