m/pkg/supervisor: log instances of runnables pending restart
This can be helpful to debug stuck runnables that cannot restart due to
some of their children not restarting yet.
We should probably also keep a list of 'stuck' runnables and expose them
via some introspection API?
Change-Id: Ia6219f6e721987b0746cb5cd0e5f11c4edc01cc6
Reviewed-on: https://review.monogon.dev/c/monogon/+/415
Reviewed-by: Leopold Schabel <leo@nexantic.com>
diff --git a/metropolis/pkg/supervisor/supervisor_processor.go b/metropolis/pkg/supervisor/supervisor_processor.go
index 5fa759e..b205a0b 100644
--- a/metropolis/pkg/supervisor/supervisor_processor.go
+++ b/metropolis/pkg/supervisor/supervisor_processor.go
@@ -334,8 +334,10 @@
// they're ready/restartable. All of the node's children must be
// restartable in order for this node to be restartable.
childrenReady := true
+ var childrenNotReady []string
for _, c := range cur.children {
if !ready[c.dn()] {
+ childrenNotReady = append(childrenNotReady, c.dn())
childrenReady = false
break
}
@@ -353,6 +355,10 @@
curReady = true
}
+ if cur.state == nodeStateDead && !childrenReady {
+ s.ilogger.Warningf("Not restarting %s: children not ready to be restarted: %v", curDn, childrenNotReady)
+ }
+
// Note down that we have an opinion on this node, and note that
// opinion down.
visited[curDn] = true