cloud/bmaas: implement webug
Webug (pronounced: /wɛbʌɡ/, not /wiːbʌɡ/) is a web debug interface for
BMDB, inspired by the great web debug interfaces of old.
It uses the new BMDB reflection API to access most
machine/tag/work/backoff information, plus sqlc queries to access
session information.
Change-Id: If0e65b6fc33ad92baef9c6d98333f90a02efa1b3
Reviewed-on: https://review.monogon.dev/c/monogon/+/1132
Reviewed-by: Lorenz Brun <lorenz@monogon.tech>
Tested-by: Jenkins CI
diff --git a/cloud/bmaas/bmdb/connection.go b/cloud/bmaas/bmdb/connection.go
index c07bf15..acf226f 100644
--- a/cloud/bmaas/bmdb/connection.go
+++ b/cloud/bmaas/bmdb/connection.go
@@ -5,6 +5,7 @@
"database/sql"
"fmt"
+ "github.com/google/uuid"
"k8s.io/klog/v2"
"source.monogon.dev/cloud/bmaas/bmdb/model"
@@ -78,3 +79,16 @@
func (c *Connection) Reflect(ctx context.Context) (*reflection.Schema, error) {
return reflection.Reflect(ctx, c.db)
}
+
+// ListHistoryOf retrieves a full audit history of a machine, sorted
+// chronologically. It can be read without a session / transaction for debugging
+// purposes.
+func (c *Connection) ListHistoryOf(ctx context.Context, machine uuid.UUID) ([]model.WorkHistory, error) {
+ return model.New(c.db).ListHistoryOf(ctx, machine)
+}
+
+// GetSession retrieves all information about a session. It can be read without a
+// session/transaction for debugging purposes.
+func (c *Connection) GetSession(ctx context.Context, session uuid.UUID) ([]model.Session, error) {
+ return model.New(c.db).GetSession(ctx, session)
+}
diff --git a/cloud/bmaas/bmdb/model/queries_base.sql b/cloud/bmaas/bmdb/model/queries_base.sql
index 8257fb8..4119f94 100644
--- a/cloud/bmaas/bmdb/model/queries_base.sql
+++ b/cloud/bmaas/bmdb/model/queries_base.sql
@@ -67,4 +67,10 @@
SELECT *
FROM work_history
WHERE machine_id = $1
-ORDER BY timestamp ASC;
\ No newline at end of file
+ORDER BY timestamp ASC;
+
+-- name: GetSession :many
+-- Retrieve session information by session ID.
+SELECT *
+FROM sessions
+WHERE session_id = $1;
\ No newline at end of file
diff --git a/cloud/bmaas/bmdb/webug/BUILD.bazel b/cloud/bmaas/bmdb/webug/BUILD.bazel
new file mode 100644
index 0000000..172fd99
--- /dev/null
+++ b/cloud/bmaas/bmdb/webug/BUILD.bazel
@@ -0,0 +1,28 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+ name = "webug",
+ srcs = [
+ "functions.go",
+ "views.go",
+ "webug.go",
+ ],
+ embedsrcs = [
+ "templates/base.html",
+ "templates/fragment_tag.html",
+ "templates/fragment_tag_default.html",
+ "templates/fragment_tag_provided.html",
+ "templates/machines.html",
+ "templates/machine.html",
+ ],
+ importpath = "source.monogon.dev/cloud/bmaas/bmdb/webug",
+ visibility = ["//visibility:public"],
+ deps = [
+ "//cloud/bmaas/bmdb",
+ "//cloud/bmaas/bmdb/model",
+ "//cloud/bmaas/bmdb/reflection",
+ "@com_github_cenkalti_backoff_v4//:backoff",
+ "@com_github_google_uuid//:uuid",
+ "@io_k8s_klog_v2//:klog",
+ ],
+)
diff --git a/cloud/bmaas/bmdb/webug/functions.go b/cloud/bmaas/bmdb/webug/functions.go
new file mode 100644
index 0000000..4c5c625
--- /dev/null
+++ b/cloud/bmaas/bmdb/webug/functions.go
@@ -0,0 +1,36 @@
+package webug
+
+import (
+ "strings"
+)
+
+var (
+ // templateFuncs are helper functions accessible to the rendered templates.
+ templateFuncs = map[string]any{
+ // summarizeError attempts to make a Go-style "foo: bar: baz" error short by
+ // using some ugly heuristics. This is currently used to show a shorter error
+ // message in the backoff column of the machine list.
+ //
+ // TODO(q3k): fix backoff causes to be less verbose and nuke this.
+ "summarizeError": func(in string) string {
+ parts := strings.Split(in, ": ")
+ for i, p := range parts {
+ // Attempt to strip some common error prefixes.
+ if strings.HasPrefix(p, "failed to ") {
+ continue
+ }
+ if strings.HasPrefix(p, "when ") {
+ continue
+ }
+ if strings.HasPrefix(p, "while ") {
+ continue
+ }
+ // If we had some prefixes stripped but suddenly reached a part that is not
+ // prefixed
+ return "[...] " + strings.Join(parts[i:], ": ")
+ }
+ // If we stripped every single segment then just return the whole thing.
+ return in
+ },
+ }
+)
diff --git a/cloud/bmaas/bmdb/webug/templates/base.html b/cloud/bmaas/bmdb/webug/templates/base.html
new file mode 100644
index 0000000..3b8df8e
--- /dev/null
+++ b/cloud/bmaas/bmdb/webug/templates/base.html
@@ -0,0 +1,90 @@
+<!DOCTYPE html>
+<meta charset="utf-8">
+<title>BMDB webug</title>
+<style>
+ body {
+ font-family: sans-serif;
+ background: #fff;
+ }
+
+ /* Logotype. */
+ h1 {
+ clear: both;
+ padding-left: 1em;
+ padding-top: 0.5em;
+ }
+ h1 a {
+ text-decoration: none;
+ }
+ h1 a, h1 a:visited, h1 a:hover, h1 a:active {
+ color: inherit;
+ }
+ h1 span.red {
+ background-color: red;
+ color: white;
+ padding: 0.1em;
+ border-radius: 0.4em;
+ }
+ h1 span.info {
+ font-size: 0.5em;
+ font-weight: normal;
+ font-style: italic;
+ }
+
+ /* Section headers. */
+ h2 {
+ clear: both;
+ width: 100%;
+ text-align: center;
+ font-size: 120%;
+ background: #eeeeff;
+ }
+
+ /* Stylish tables. */
+ table, th, td {
+ background-color: #eee;
+ padding: 0.2em 0.4em 0.2em 0.4em;
+ }
+ table th {
+ background-color: #c0c0c0;
+ }
+ table {
+ background-color: #fff;
+ border-spacing: 0.2em;
+ }
+
+ /* Colouring of the Work History log in machine.html. */
+ tr.EventFailed td, tr.EventCanceled td {
+ background-color: #f8e8e8;
+ }
+ tr.EventFinished td {
+ background-color: #e8f8e8;
+ }
+
+ /* Generic font style tags for any element. */
+ .small {
+ font-size: 0.8em;
+ }
+ .faint {
+ color: #333;
+ }
+ .mono {
+ font-family: monospace;
+ }
+ .error {
+ color: #f00;
+ }
+
+ /* For simple column style layouts. */
+ .vsplit {
+ display: flex;
+ flex-direction: row;
+ flex-wrap: nowrap;
+ align-items: stretch;
+ }
+ .column {
+ flex-grow: 1;
+ padding: 0.5em;
+ }
+</style>
+<h1><a href="/">we<span class="red">bug</span></a> <span class="info">BMDB at {{ .BMDBAddress }} schema {{ .BMDBSchema }}</span></h1>
\ No newline at end of file
diff --git a/cloud/bmaas/bmdb/webug/templates/fragment_tag.html b/cloud/bmaas/bmdb/webug/templates/fragment_tag.html
new file mode 100644
index 0000000..f7ee320
--- /dev/null
+++ b/cloud/bmaas/bmdb/webug/templates/fragment_tag.html
@@ -0,0 +1,5 @@
+{{- if eq .Type.Name "Provided" -}}
+ {{- template "fragment_tag_provided.html" . -}}
+{{- else -}}
+ {{- template "fragment_tag_default.html" . -}}
+{{- end -}}
diff --git a/cloud/bmaas/bmdb/webug/templates/fragment_tag_default.html b/cloud/bmaas/bmdb/webug/templates/fragment_tag_default.html
new file mode 100644
index 0000000..464d517
--- /dev/null
+++ b/cloud/bmaas/bmdb/webug/templates/fragment_tag_default.html
@@ -0,0 +1 @@
+<b>{{ .Type.Name }}</b>(...)
diff --git a/cloud/bmaas/bmdb/webug/templates/fragment_tag_provided.html b/cloud/bmaas/bmdb/webug/templates/fragment_tag_provided.html
new file mode 100644
index 0000000..776a63d
--- /dev/null
+++ b/cloud/bmaas/bmdb/webug/templates/fragment_tag_provided.html
@@ -0,0 +1,3 @@
+{{- $provider := (.Field "provider").HumanValue }}
+{{- $pid := (.Field "provider_id").HumanValue }}
+<b>{{ .Type.Name }}</b>({{- $provider }}, <a href="provider/{{ $provider }}/{{ $pid }}" style="font-family: mono">{{ $pid }}</a>)
diff --git a/cloud/bmaas/bmdb/webug/templates/machine.html b/cloud/bmaas/bmdb/webug/templates/machine.html
new file mode 100644
index 0000000..1d90014
--- /dev/null
+++ b/cloud/bmaas/bmdb/webug/templates/machine.html
@@ -0,0 +1,176 @@
+{{ template "base.html" .Base }}
+<h2>Machine {{ .Machine.ID }}</h2>
+
+{{ $sessions := .Sessions }}
+
+<table>
+ <tr>
+ <td><b>Machine ID</b></td>
+ <td class="mono">{{ .Machine.ID }}</td>
+ </tr>
+ <tr>
+ <td><b>Created</b></td>
+ <td>{{ .Machine.Created }}</td>
+ </tr>
+ <tr>
+ <td><b>Active Backoffs</b></td>
+ <td>{{ len .Machine.ActiveBackoffs }}</td>
+ </tr>
+ <tr>
+ <td><b>Active Work</b></td>
+ <td>{{ len .Machine.Work }}</td>
+ </tr>
+</table>
+
+<div class="vsplit">
+ <div class="column">
+ <h2>Tags</h2>
+ {{ range $name, $tag := .Machine.Tags }}
+ <table>
+ <tr>
+ <th colspan="2">
+ {{ template "fragment_tag.html" $tag }}
+ </th>
+ </tr>
+ {{ range $tag.Fields }}
+ <tr>
+ <td>
+ <b>{{ .Type.NativeName }}:</b>
+ </td>
+ <td class="mono">
+ {{ .HumanValue }}
+ </td>
+ </tr>
+ {{ end }}
+ </table>
+ {{ else }}
+ <i>No tags.</i>
+ {{ end }}
+ <h2>Work</h2>
+ {{ range $name, $work := .Machine.Work }}
+ <table>
+ <tr>
+ <th colspan="3">
+ <b>{{ $work.Process }}</b>
+ </th>
+ </tr>
+ <tr>
+ <td><b>Process:</b></td>
+ <td class="mono" colspan="2">
+ {{ $work.Process }}
+ </td>
+ </tr>
+ {{ $sessionOrErr := index $sessions $name }}
+ {{ if ne $sessionOrErr.Error "" }}
+ <tr>
+ <td colspan="3" class="error">
+ Could not retrieve session information: {{ $sessionOrErr.Error }}
+ </td>
+ </tr>
+ {{ else }}
+ {{ $session := $sessionOrErr.Session }}
+ <tr>
+ <td rowspan="5" style="vertical-align: top;"><b>Session</b></td>
+ <td><b>ID:</b></td>
+ <td class="mono" colspan="2">
+ <a href="/session/{{ $session.SessionID }}">{{ $session.SessionID }}</a>
+ </td>
+ </tr>
+ <tr>
+ <td><b>Component:</b></td>
+ <td class="mono">{{ $session.SessionComponentName }}</td>
+ </tr>
+ <tr>
+ <td><b>Runtime:</b></td>
+ <td class="mono">{{ $session.SessionRuntimeInfo }}</td>
+ </tr>
+ <tr>
+ <td><b>Created At:</b></td>
+ <td>{{ $session.SessionCreatedAt }}</td>
+ </tr>
+ <tr>
+ <td><b>Liveness:</b></td>
+ <td>Interval {{ $session.SessionIntervalSeconds }}s, deadline {{ $session.SessionDeadline }}</td>
+ </tr>
+ {{ end }}
+ </table>
+ {{ else }}
+ <i>No active work.</i>
+ {{ end }}
+ <h2>Backoffs</h2>
+ <h3>Active</h3>
+ {{ range $name, $backoff := .Machine.ActiveBackoffs }}
+ <table>
+ <tr>
+ <th colspan="2">
+ <b>{{ $backoff.Process }}</b>
+ </th>
+ </tr>
+ <tr>
+ <td><b>Process:</b></td>
+ <td class="mono">{{ $backoff.Process }}</td>
+ </tr>
+ <tr>
+ <td><b>Until:</b></td>
+ <td class="mono">{{ $backoff.Until }}</td>
+ </tr>
+ <tr>
+ <td><b>Cause:</b></td>
+ <td class="mono">{{ $backoff.Cause }}</td>
+ </tr>
+ </table>
+ {{ else }}
+ <i>No active backoffs.</i>
+ {{ end }}
+ <h3>Expired</h3>
+ {{ range $name, $backoff := .Machine.ExpiredBackoffs }}
+ <table style="opacity: 0.4">
+ <tr>
+ <th colspan="2">
+ <b>{{ $backoff.Process }}</b>
+ </th>
+ </tr>
+ <tr>
+ <td><b>Process:</b></td>
+ <td class="mono">{{ $backoff.Process }}</td>
+ </tr>
+ <tr>
+ <td><b>Until:</b></td>
+ <td class="mono">{{ $backoff.Until }}</td>
+ </tr>
+ <tr>
+ <td><b>Cause:</b></td>
+ <td class="mono">{{ $backoff.Cause }}</td>
+ </tr>
+ </table>
+ {{ else }}
+ <i>No expired backoffs.</i>
+ {{ end }}
+ </div>
+ <div class="column">
+ <h2>Work History</h2>
+ {{ if ne .HistoryError "" }}
+ <b class="error">Unavailable: {{ .HistoryError }}</b>
+ {{ else }}
+ <i>Note: reverse chronological order.</i>
+ <table>
+ <tr>
+ <th>Time</th>
+ <th>Process</th>
+ <th>Event</th>
+ </tr>
+ {{ range .History }}
+ <tr class="Event{{.Event}}">
+ <td>{{ .Timestamp }}</td>
+ <td><b>{{ .Process }}</b></td>
+ {{ if eq .Event "Failed" }}
+ <td>{{ .Event }}: <span class="mono">{{ .FailedCause.String }}</span></td>
+ {{ else }}
+ <td>{{ .Event }}</td>
+ {{ end }}
+ </tr>
+ {{ end }}
+ </table>
+ {{ end }}
+ </div>
+</div>
\ No newline at end of file
diff --git a/cloud/bmaas/bmdb/webug/templates/machines.html b/cloud/bmaas/bmdb/webug/templates/machines.html
new file mode 100644
index 0000000..d8658a1
--- /dev/null
+++ b/cloud/bmaas/bmdb/webug/templates/machines.html
@@ -0,0 +1,36 @@
+{{ template "base.html" .Base }}
+<h2>Machine List</h2>
+<p>
+ Click on a Machine ID to explore it further. Commonly viewed tags are expanded.
+</p>
+<table>
+ <tr>
+ <th>Machine ID</th>
+ <th>Work</th>
+ <th>Backoffs</th>
+ <th>Tags</th>
+ </tr>
+ {{ range .Machines -}}
+ <tr>
+ <td class="mono"><a href="/machine/{{ .ID }}">{{ .ID }}</a></td>
+ <td>
+ {{- range $process, $work := .Work -}}
+ <b><a href="/session/{{ $work.SessionID }}">{{ $process }}</a></b>
+ {{- end -}}
+ </td>
+ <td>
+ {{- range $process, $backoff := .Backoffs -}}
+ <b>{{ $backoff.Process }}</b>(<span class="small">{{ summarizeError .Cause }}</span>)
+ {{- end -}}
+ </td>
+ <td>
+ {{- range $name, $tag := .Tags -}}
+ {{- template "fragment_tag.html" $tag -}}
+ {{- end -}}
+ </td>
+ </tr>
+ {{ end -}}
+</table>
+<p class="small faint mono">
+ {{ .NMachines }} rows, rendered in {{ .RenderTime }}. Query: {{ .Query }}
+</p>
\ No newline at end of file
diff --git a/cloud/bmaas/bmdb/webug/views.go b/cloud/bmaas/bmdb/webug/views.go
new file mode 100644
index 0000000..7e6d91f
--- /dev/null
+++ b/cloud/bmaas/bmdb/webug/views.go
@@ -0,0 +1,171 @@
+package webug
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "time"
+
+ "github.com/google/uuid"
+ "k8s.io/klog/v2"
+
+ "source.monogon.dev/cloud/bmaas/bmdb/model"
+ "source.monogon.dev/cloud/bmaas/bmdb/reflection"
+)
+
+// baseParams are passed to all rendered templates, and are consumed by tags in
+// templates/base.html.
+type baseParams struct {
+ // Address to display in page header.
+ BMDBAddress string
+ // Schema version to display in page header.
+ BMDBSchema string
+}
+
+// makeBase builds baseParams from information about the current connection.
+func (s *server) makeBase() baseParams {
+ address := fmt.Sprintf("%s@%s", s.conn.DatabaseName, s.conn.Address)
+ if s.conn.InMemory {
+ address += " (in memory)"
+ }
+ return baseParams{
+ BMDBAddress: address,
+ BMDBSchema: s.curSchema().Version,
+ }
+}
+
+// viewMachines renders a list of all machines in the BMDB.
+func (s *server) viewMachines(w http.ResponseWriter, r *http.Request, args ...string) {
+ start := time.Now()
+ res, err := s.curSchema().GetMachines(r.Context(), &reflection.GetMachinesOpts{Strict: true})
+ if err != nil {
+ w.WriteHeader(http.StatusInternalServerError)
+ fmt.Fprintf(w, "could not dump BMDB: %v", err)
+ return
+ }
+ duration := time.Since(start)
+
+ type params struct {
+ Base baseParams
+ Query string
+ Machines []*reflection.Machine
+ NMachines int
+ RenderTime time.Duration
+ }
+ err = templates.ExecuteTemplate(w, "machines.html", ¶ms{
+ Base: s.makeBase(),
+ Query: res.Query,
+ Machines: res.Data,
+ NMachines: len(res.Data),
+ RenderTime: duration,
+ })
+ if err != nil {
+ klog.Errorf("Template rendering failed: %v", err)
+ }
+}
+
+// viewMachineDetail renders a detailed page for a single machine.
+func (s *server) viewMachineDetail(w http.ResponseWriter, r *http.Request, args ...string) {
+ mid, err := uuid.Parse(args[0])
+ if err != nil {
+ w.WriteHeader(http.StatusUnprocessableEntity)
+ fmt.Fprintf(w, "invalid machine UUID")
+ return
+ }
+
+ opts := reflection.GetMachinesOpts{
+ FilterMachine: &mid,
+ Strict: true,
+ ExpiredBackoffs: true,
+ }
+ res, err := s.curSchema().GetMachines(r.Context(), &opts)
+ if err != nil {
+ w.WriteHeader(http.StatusInternalServerError)
+ fmt.Fprintf(w, "could not dump BMDB: %v", err)
+ return
+ }
+ if len(res.Data) == 0 {
+ w.WriteHeader(http.StatusNotFound)
+ fmt.Fprintf(w, "machine not found")
+ return
+ }
+ machine := res.Data[0]
+
+ // Params to pass to template.
+ type sessionOrError struct {
+ Session *model.Session
+ Error string
+ }
+ type params struct {
+ Base baseParams
+ Machine *reflection.Machine
+
+ HistoryError string
+ History []model.WorkHistory
+
+ Sessions map[string]sessionOrError
+ }
+ p := params{
+ Base: s.makeBase(),
+ Machine: machine,
+ Sessions: make(map[string]sessionOrError),
+ }
+
+ // History retrieval is performed with strict consistency guarantees, and thus
+ // might block. Make sure we don't block the entire page.
+ subQueriesCtx, subQueriesCtxC := context.WithTimeout(r.Context(), time.Second)
+ defer subQueriesCtxC()
+ history, err := s.conn.ListHistoryOf(subQueriesCtx, mid)
+ if err != nil {
+ p.HistoryError = err.Error()
+ }
+
+ // Same for sessions.
+ for name, work := range machine.Work {
+ sessions, err := s.conn.GetSession(subQueriesCtx, work.SessionID)
+ if err != nil {
+ p.Sessions[name] = sessionOrError{Error: err.Error()}
+ } else {
+ if len(sessions) == 0 {
+ // This can happen if the session literally just disappeared.
+ //
+ // TODO(q3k): put all of these operations in a DB TX so that we don't end up with
+ // possible inconsistencies?
+ p.Sessions[name] = sessionOrError{Error: "not found"}
+ continue
+ }
+ p.Sessions[name] = sessionOrError{Session: &sessions[0]}
+ }
+ }
+
+ p.History = make([]model.WorkHistory, len(history))
+ for i := 0; i < len(history); i += 1 {
+ p.History[i] = history[len(history)-(i+1)]
+ }
+ if err := templates.ExecuteTemplate(w, "machine.html", &p); err != nil {
+ klog.Errorf("Template rendering failed: %v", err)
+ }
+}
+
+// viewProviderRedirects redirects a given provider and provider_id into a
+// provider's web portal for more detailed information about an underlying
+// machine.
+func (s *server) viewProviderRedirect(w http.ResponseWriter, r *http.Request, args ...string) {
+ providerUrls := map[string]string{
+ "Equinix": "https://console.equinix.com/devices/%s/overview",
+ }
+ if providerUrls[args[0]] == "" {
+ w.WriteHeader(http.StatusNotFound)
+ fmt.Fprintf(w, "Usage: /provider/Equinix/<id>")
+ return
+ }
+ url := fmt.Sprintf(providerUrls[args[0]], args[1])
+ http.Redirect(w, r, url, http.StatusFound)
+}
+
+// viewSession shows detailed information about a BMDB session.
+func (s *server) viewSession(w http.ResponseWriter, r *http.Request, args ...string) {
+ // TODO(q3k): implement this once we add session info to work history so that
+ // this can actually display something useful.
+ fmt.Fprintf(w, "underconstruction.gif")
+}
diff --git a/cloud/bmaas/bmdb/webug/webug.go b/cloud/bmaas/bmdb/webug/webug.go
new file mode 100644
index 0000000..91888d5
--- /dev/null
+++ b/cloud/bmaas/bmdb/webug/webug.go
@@ -0,0 +1,174 @@
+// Package webug implements a web-based debug/troubleshooting/introspection
+// system for the BMDB. It's optimized for use by developers and trained
+// operators, prioritizing information density, fast navigation and heavy
+// interlinking.
+package webug
+
+import (
+ "context"
+ "embed"
+ "flag"
+ "fmt"
+ "html/template"
+ "net/http"
+ "regexp"
+ "sync"
+ "time"
+
+ "github.com/cenkalti/backoff/v4"
+ "k8s.io/klog/v2"
+
+ "source.monogon.dev/cloud/bmaas/bmdb"
+ "source.monogon.dev/cloud/bmaas/bmdb/reflection"
+)
+
+var (
+ //go:embed templates/*html
+ templateFS embed.FS
+ templates = template.Must(template.New("base.html").Funcs(templateFuncs).ParseFS(templateFS, "templates/*"))
+)
+
+// server holds the state of an active webug interface.
+type server struct {
+ // connection pool to BMDB.
+ conn *bmdb.Connection
+ // schema retrieved from BMDB.
+ schema *reflection.Schema
+ // muSchema locks schema for updates.
+ muSchema sync.RWMutex
+}
+
+// curSchema returns the current cached BMDB schema.
+func (s *server) curSchema() *reflection.Schema {
+ s.muSchema.Lock()
+ defer s.muSchema.Unlock()
+ return s.schema
+}
+
+// schemaWorker runs a background goroutine which attempts to update the server's
+// cached BMDB schema every hour.
+func (s *server) schemaWorker(ctx context.Context) {
+ t := time.NewTicker(time.Hour)
+ defer t.Stop()
+
+ for {
+ // Wait for the timer to tick, or for the context to expire.
+ select {
+ case <-ctx.Done():
+ klog.Infof("Schema fetch worker exiting: %v", ctx.Err())
+ return
+ case <-t.C:
+ }
+
+ // Time to check the schema. Do that in an exponential backoff loop until
+ // successful.
+ bo := backoff.NewExponentialBackOff()
+ bo.MaxElapsedTime = 0
+ var schema *reflection.Schema
+ err := backoff.Retry(func() error {
+ var err error
+ schema, err = s.conn.Reflect(ctx)
+ if err != nil {
+ klog.Warningf("Failed to fetch new schema: %v", err)
+ return err
+ }
+ return nil
+ }, backoff.WithContext(bo, ctx))
+ // This will only happen due to context expiration.
+ if err != nil {
+ klog.Errorf("Giving up on schema fetch: %v", err)
+ continue
+ }
+
+ // Swap the current schema if necessary.
+ cur := s.curSchema().Version
+ new := schema.Version
+ if cur != new {
+ klog.Infof("Got new schema: %s -> %s", cur, new)
+ s.muSchema.Lock()
+ s.schema = schema
+ s.muSchema.Unlock()
+ }
+ }
+}
+
+// Register webug on an HTTP mux, using a BMDB connection pool.
+//
+// The given context will be used not only to time out the registration call, but
+// also used to run a BMDB schema fetching goroutine that will attempt to fetch
+// newer versions of the schema every hour.
+//
+// This is a low-level function useful when tying webug into an existing web
+// application. If you just want to run webug on a separate port that's
+// configured by flags, use Config and Config.RegisterFlags.
+func Register(ctx context.Context, conn *bmdb.Connection, mux *http.ServeMux) error {
+ schema, err := conn.Reflect(ctx)
+ if err != nil {
+ return fmt.Errorf("could not get BMDB schema for webug: %w", err)
+ }
+ s := server{
+ conn: conn,
+ schema: schema,
+ }
+ go s.schemaWorker(ctx)
+
+ type route struct {
+ pattern *regexp.Regexp
+ handler func(w http.ResponseWriter, r *http.Request, args ...string)
+ }
+
+ routes := []route{
+ {regexp.MustCompile(`^/$`), s.viewMachines},
+ {regexp.MustCompile(`^/machine/([a-fA-F0-9\-]+)$`), s.viewMachineDetail},
+ {regexp.MustCompile(`^/provider/([^/]+)/([^/]+)$`), s.viewProviderRedirect},
+ {regexp.MustCompile(`^/session/([^/]+)`), s.viewSession},
+ }
+
+ mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+ for _, route := range routes {
+ match := route.pattern.FindStringSubmatch(r.URL.Path)
+ if match == nil {
+ continue
+ }
+ route.handler(w, r, match[1:]...)
+ return
+ }
+ http.NotFound(w, r)
+ })
+ return nil
+}
+
+// Config describes the webug interface configuration. This should be embedded
+// inside your component's Config object.
+//
+// To configure, either set values or call RegisterFlags before flag.Parse.
+//
+// To run after configuration, call Start.
+type Config struct {
+ // If set, start a webug interface on an HTTP listener bound to the given address.
+ WebugListenAddress string
+}
+
+// RegisterFlags for webug interface.
+func (c *Config) RegisterFlags() {
+ flag.StringVar(&c.WebugListenAddress, "webug_listen_address", "", "Address to start BMDB webug on. If not set, webug will not be started.")
+}
+
+// Start the webug interface in the foreground if enabled. The returned error
+// will be either a configuration/connection error returned as soon as possible,
+// or a context expiration error.
+//
+// The given context will be used for all connections from the webug interface to
+// the given BMDB connection.
+func (c *Config) Start(ctx context.Context, conn *bmdb.Connection) error {
+ if c.WebugListenAddress == "" {
+ return nil
+ }
+ mux := http.NewServeMux()
+ if err := Register(ctx, conn, mux); err != nil {
+ return err
+ }
+
+ klog.Infof("Webug listening at %s", c.WebugListenAddress)
+ return http.ListenAndServe(c.WebugListenAddress, mux)
+}
diff --git a/cloud/bmaas/server/BUILD.bazel b/cloud/bmaas/server/BUILD.bazel
index 2c96a03..aa4d898 100644
--- a/cloud/bmaas/server/BUILD.bazel
+++ b/cloud/bmaas/server/BUILD.bazel
@@ -11,6 +11,7 @@
deps = [
"//cloud/bmaas/bmdb",
"//cloud/bmaas/bmdb/model",
+ "//cloud/bmaas/bmdb/webug",
"//cloud/bmaas/server/api",
"//cloud/lib/component",
"//metropolis/node/core/rpc",
diff --git a/cloud/bmaas/server/server.go b/cloud/bmaas/server/server.go
index 97fb393..57496d5 100644
--- a/cloud/bmaas/server/server.go
+++ b/cloud/bmaas/server/server.go
@@ -12,6 +12,7 @@
"k8s.io/klog/v2"
"source.monogon.dev/cloud/bmaas/bmdb"
+ "source.monogon.dev/cloud/bmaas/bmdb/webug"
apb "source.monogon.dev/cloud/bmaas/server/api"
"source.monogon.dev/cloud/lib/component"
)
@@ -19,6 +20,7 @@
type Config struct {
Component component.ComponentConfig
BMDB bmdb.BMDB
+ Webug webug.Config
// PublicListenAddress is the address at which the 'public' (agent-facing) gRPC
// server listener will run.
@@ -39,6 +41,7 @@
c.BMDB.ComponentName = "srv"
c.BMDB.RuntimeInfo = runtimeInfo()
c.BMDB.Database.RegisterFlags("bmdb")
+ c.Webug.RegisterFlags()
flag.StringVar(&c.PublicListenAddress, "srv_public_grpc_listen_address", ":8080", "Address to listen at for public/user gRPC connections for bmdbsrv")
}
@@ -108,4 +111,9 @@
s.bmdb = conn
s.startInternalGRPC(ctx)
s.startPublic(ctx)
+ go func() {
+ if err := s.Config.Webug.Start(ctx, conn); err != nil && err != ctx.Err() {
+ klog.Exitf("Failed to start webug: %v", err)
+ }
+ }()
}
diff --git a/cloud/shepherd/equinix/manager/server/BUILD.bazel b/cloud/shepherd/equinix/manager/server/BUILD.bazel
index 8593004..a8431fe 100644
--- a/cloud/shepherd/equinix/manager/server/BUILD.bazel
+++ b/cloud/shepherd/equinix/manager/server/BUILD.bazel
@@ -7,6 +7,7 @@
visibility = ["//visibility:private"],
deps = [
"//cloud/bmaas/bmdb",
+ "//cloud/bmaas/bmdb/webug",
"//cloud/lib/component",
"//cloud/shepherd/equinix/manager",
"//cloud/shepherd/equinix/wrapngo",
diff --git a/cloud/shepherd/equinix/manager/server/main.go b/cloud/shepherd/equinix/manager/server/main.go
index 567ebd7..550eea3 100644
--- a/cloud/shepherd/equinix/manager/server/main.go
+++ b/cloud/shepherd/equinix/manager/server/main.go
@@ -10,6 +10,7 @@
"k8s.io/klog"
"source.monogon.dev/cloud/bmaas/bmdb"
+ "source.monogon.dev/cloud/bmaas/bmdb/webug"
"source.monogon.dev/cloud/lib/component"
"source.monogon.dev/cloud/shepherd/equinix/manager"
"source.monogon.dev/cloud/shepherd/equinix/wrapngo"
@@ -24,6 +25,7 @@
AgentConfig manager.AgentConfig
ProvisionerConfig manager.ProvisionerConfig
InitializerConfig manager.InitializerConfig
+ WebugConfig webug.Config
API wrapngo.Opts
}
@@ -46,6 +48,7 @@
c.AgentConfig.RegisterFlags()
c.ProvisionerConfig.RegisterFlags()
c.InitializerConfig.RegisterFlags()
+ c.WebugConfig.RegisterFlags()
c.API.RegisterFlags()
}
@@ -90,6 +93,7 @@
if err != nil {
klog.Exitf("Failed to open BMDB connection: %v", err)
}
+
go func() {
err = provisioner.Run(ctx, conn)
if err != nil {
@@ -102,6 +106,11 @@
klog.Exit(err)
}
}()
+ go func() {
+ if err := c.WebugConfig.Start(ctx, conn); err != nil && err != ctx.Err() {
+ klog.Exitf("Failed to start webug: %v", err)
+ }
+ }()
<-ctx.Done()
}