metropolis/node: use concise logging in tty0/ttyS1
Change-Id: I28ad4418a05a7962d61160583b2e210f73ff8c51
Reviewed-on: https://review.monogon.dev/c/monogon/+/1360
Reviewed-by: Leopold Schabel <leo@monogon.tech>
Tested-by: Jenkins CI
diff --git a/metropolis/node/core/main.go b/metropolis/node/core/main.go
index b36f6ce..0d13b4b 100644
--- a/metropolis/node/core/main.go
+++ b/metropolis/node/core/main.go
@@ -21,6 +21,7 @@
"fmt"
"io"
"os"
+ "strings"
"time"
"golang.org/x/sys/unix"
@@ -49,19 +50,23 @@
// Set up logger for Metropolis. Currently logs everything to /dev/tty0 and
// /dev/ttyS0.
- consoles := []string{"/dev/tty0", "/dev/ttyS0"}
- // Logtree readers that will be used to retrieve data from the root logtree and
- // write them to consoles. We keep a reference to them as we want to be able to
- // close them after a fatal error, allowing us to clearly output an error to the
- // user without the console being clobbered by logtree logs.
- var readers []*logtree.LogReader
+ consoles := []console{
+ {
+ path: "/dev/tty0",
+ maxWidth: 80,
+ },
+ {
+ path: "/dev/ttyS0",
+ maxWidth: 120,
+ },
+ }
// Alternative channel that crash handling writes to, and which gets distributed
// to the consoles.
crash := make(chan string)
// Open up consoles and set up logging from logtree and crash channel.
- for _, p := range consoles {
- f, err := os.OpenFile(p, os.O_WRONLY, 0)
+ for _, console := range consoles {
+ f, err := os.OpenFile(console.path, os.O_WRONLY, 0)
if err != nil {
continue
}
@@ -69,18 +74,20 @@
if err != nil {
panic(fmt.Errorf("could not set up root log reader: %v", err))
}
- readers = append(readers, reader)
- go func(path string, f io.Writer) {
+ console.reader = reader
+ go func(path string, maxWidth int, f io.Writer) {
fmt.Fprintf(f, "\nMetropolis: this is %s. Verbose node logs follow.\n\n", path)
for {
select {
case p := <-reader.Stream:
- fmt.Fprintf(f, "%s\n", p.String())
+ if consoleFilter(p) {
+ fmt.Fprintf(f, "%s\n", p.ConciseString(logtree.MetropolisShortenDict, maxWidth))
+ }
case s := <-crash:
fmt.Fprintf(f, "%s\n", s)
}
}
- }(p, f)
+ }(console.path, console.maxWidth, f)
}
// Initialize persistent panic handler early
@@ -197,9 +204,9 @@
ctxC()
time.Sleep(time.Second)
// After a bit, kill all console log readers.
- for _, r := range readers {
- r.Close()
- r.Stream = nil
+ for _, console := range consoles {
+ console.reader.Close()
+ console.reader.Stream = nil
}
// Wait for final logs to flush to console...
time.Sleep(time.Second)
@@ -211,3 +218,44 @@
// Return to minit, which will reboot this node.
os.Exit(1)
}
+
+// consoleFilter is used to filter out some uselessly verbose logs from the
+// console.
+//
+// This should be limited to external services, our internal services should
+// instead just have good logging by default.
+func consoleFilter(p *logtree.LogEntry) bool {
+ if p.Raw != nil {
+ return false
+ }
+ if p.Leveled == nil {
+ return false
+ }
+ s := string(p.DN)
+ if strings.HasPrefix(s, "root.role.controlplane.launcher.consensus.etcd") {
+ return p.Leveled.Severity().AtLeast(logtree.WARNING)
+ }
+ // TODO(q3k): turn off RPC traces instead
+ if strings.HasPrefix(s, "root.role.controlplane.launcher.curator.listener.rpc") {
+ return false
+ }
+ if strings.HasPrefix(s, "root.role.kubernetes.run.kubernetes.networked.kubelet") {
+ return p.Leveled.Severity().AtLeast(logtree.WARNING)
+ }
+ if strings.HasPrefix(s, "root.role.kubernetes.run.kubernetes.networked.apiserver") {
+ return p.Leveled.Severity().AtLeast(logtree.WARNING)
+ }
+ if strings.HasPrefix(s, "root.role.kubernetes.run.kubernetes.controller-manager") {
+ return p.Leveled.Severity().AtLeast(logtree.WARNING)
+ }
+ if strings.HasPrefix(s, "root.role.kubernetes.run.kubernetes.scheduler") {
+ return p.Leveled.Severity().AtLeast(logtree.WARNING)
+ }
+ return true
+}
+
+type console struct {
+ path string
+ maxWidth int
+ reader *logtree.LogReader
+}
diff --git a/metropolis/node/core/panichandler.go b/metropolis/node/core/panichandler.go
index fe0e2e1..3f17ffc 100644
--- a/metropolis/node/core/panichandler.go
+++ b/metropolis/node/core/panichandler.go
@@ -20,6 +20,7 @@
// This hooks into a global variable which is checked by runtime.write and used
// instead of runtime.write1 if populated.
+//
//go:linkname overrideWrite runtime.overrideWrite
var overrideWrite func(fd uintptr, p unsafe.Pointer, n int32) int32
@@ -30,6 +31,7 @@
// runtime.write1, just with a hardcoded file descriptor and using the assembly
// function unix.RawSyscall to not get a dependency on Go's calling convention
// and needing an implementation for every architecture.
+//
//go:nosplit
func runtimeWrite(fd uintptr, p unsafe.Pointer, n int32) int32 {
// Only redirect writes to stderr.
@@ -55,7 +57,7 @@
const runtimeLogPath = "/esp/core_runtime.log"
-func initPanicHandler(lt *logtree.LogTree, consoles []string) {
+func initPanicHandler(lt *logtree.LogTree, consoles []console) {
rl := lt.MustRawFor("panichandler")
l := lt.MustLeveledFor("panichandler")
@@ -82,11 +84,11 @@
runtimeFds = append(runtimeFds, fd)
}
- for _, s := range consoles {
- fd, err := unix.Open(s, os.O_WRONLY, 0)
+ for _, console := range consoles {
+ fd, err := unix.Open(console.path, os.O_WRONLY, 0)
if err == nil {
runtimeFds = append(runtimeFds, fd)
- l.Infof("Panic console: %s", s)
+ l.Infof("Panic console: %s", console.path)
}
}