cloud: enable prometheus server
This makes cloud components start a Prometheus server on a separate port
(:4243 by default) using either gRPC TLS credentials or plain HTTP.
The Prometheus server currently only export the standard Go/Process
metrics that the default Prometheus library exports.
Change-Id: I9f2ae20c34446c0e10a946d4a93251764f5d2fce
Reviewed-on: https://review.monogon.dev/c/monogon/+/1599
Reviewed-by: Leopold Schabel <leo@monogon.tech>
Tested-by: Jenkins CI
diff --git a/cloud/lib/component/BUILD.bazel b/cloud/lib/component/BUILD.bazel
index d705997..dcbe0fe 100644
--- a/cloud/lib/component/BUILD.bazel
+++ b/cloud/lib/component/BUILD.bazel
@@ -18,6 +18,9 @@
"@com_github_golang_migrate_migrate_v4//database/cockroachdb",
"@com_github_golang_migrate_migrate_v4//source",
"@com_github_lib_pq//:pq",
+ "@com_github_prometheus_client_golang//prometheus",
+ "@com_github_prometheus_client_golang//prometheus/collectors",
+ "@com_github_prometheus_client_golang//prometheus/promhttp",
"@io_k8s_klog_v2//:klog",
"@org_golang_google_grpc//:go_default_library",
"@org_golang_google_grpc//credentials",
diff --git a/cloud/lib/component/component.go b/cloud/lib/component/component.go
index 9337d52..1acd961 100644
--- a/cloud/lib/component/component.go
+++ b/cloud/lib/component/component.go
@@ -5,13 +5,19 @@
package component
import (
+ "context"
"crypto/tls"
"crypto/x509"
"flag"
+ "net"
+ "net/http"
"os"
"path/filepath"
"github.com/adrg/xdg"
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/collectors"
+ "github.com/prometheus/client_golang/prometheus/promhttp"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"k8s.io/klog/v2"
@@ -45,6 +51,16 @@
// ComponentName is the name of this component, which should be [a-z0-9+]. It's
// used to prefix all flags set by the Configuration.
ComponentName string
+
+ // PrometheusListenAddress is the address on which the component should serve
+ // Prometheus metrics.
+ PrometheusListenAddress string
+ // PrometheusInsecure enables serving Prometheus metrics without any TLS, running
+ // a plain HTTP listener. If disabled, Prometheus metrics are served using the
+ // same PKI setup as the components' gRPC server.
+ PrometheusInsecure bool
+
+ prometheusRegistry *prometheus.Registry
}
// RegisterFlags registers the component configuration to be provided by flags.
@@ -54,6 +70,8 @@
flag.StringVar(&c.GRPCCertificatePath, componentName+"_grpc_certificate_path", "", "Path to gRPC server/client certificate for "+componentName)
flag.StringVar(&c.GRPCCAPath, componentName+"_grpc_ca_certificate_path", "", "Path to gRPC CA certificate for "+componentName)
flag.StringVar(&c.GRPCListenAddress, componentName+"_grpc_listen_address", ":4242", "Address to listen at for gRPC connections for "+componentName)
+ flag.StringVar(&c.PrometheusListenAddress, componentName+"_prometheus_listen_address", ":4243", "Address to listen at for Prometheus connections for "+componentName)
+ flag.BoolVar(&c.PrometheusInsecure, componentName+"_prometheus_insecure", false, "Serve plain HTTP prometheus without mTLS. If not set, main gRPC TLS credentials/certificates are used")
flag.BoolVar(&c.DevCerts, componentName+"_dev_certs", false, "Use developer certificates (autogenerated) for "+componentName)
flag.StringVar(&c.DevCertsPath, componentName+"_dev_certs_path", filepath.Join(xdg.ConfigHome, "monogon-dev-certs"), "Path for storing developer certificates")
@@ -61,9 +79,7 @@
c.ComponentName = componentName
}
-// GRPCServerOptions returns pre-built grpc.ServerOptions that this component
-// should use to serve internal gRPC.
-func (c *ComponentConfig) GRPCServerOptions() []grpc.ServerOption {
+func (c *ComponentConfig) getTLSConfig() *tls.Config {
var certPath, keyPath, caPath string
if c.DevCerts {
// Use devcerts if requested.
@@ -97,13 +113,66 @@
if err != nil {
klog.Exitf("Could not load GRPC TLS keypair: %v", err)
}
- tlsConf := &tls.Config{
+ return &tls.Config{
Certificates: []tls.Certificate{pair},
ClientAuth: tls.RequireAndVerifyClientCert,
ClientCAs: certPool,
}
+}
+
+// PrometheusRegistry returns this component's singleton Prometheus registry,
+// creating it as needed. This method is not goroutine-safe, and should only be
+// called during the setup process of the Component.
+func (c *ComponentConfig) PrometheusRegistry() *prometheus.Registry {
+ if c.prometheusRegistry == nil {
+ c.prometheusRegistry = prometheus.NewRegistry()
+ c.prometheusRegistry.Register(collectors.NewGoCollector())
+ c.prometheusRegistry.Register(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
+ }
+ return c.prometheusRegistry
+}
+
+// StartPrometheus starts a Prometheus metrics server in a goroutine. It will
+// serve any metrics that have been registered with the registry returned by
+// PrometheusRegistry.
+func (c *ComponentConfig) StartPrometheus(ctx context.Context) {
+ reg := c.PrometheusRegistry()
+
+ mux := http.NewServeMux()
+ mux.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))
+
+ var lis net.Listener
+ var err error
+
+ if c.PrometheusInsecure {
+ lis, err = net.Listen("tcp", c.PrometheusListenAddress)
+ } else {
+ lis, err = tls.Listen("tcp", c.PrometheusListenAddress, c.getTLSConfig())
+ }
+ if err != nil {
+ klog.Exitf("Could not listen on prometheus address: %v", err)
+ }
+
+ srv := http.Server{
+ Handler: mux,
+ }
+ go func() {
+ klog.Infof("Prometheus listening on %s", lis.Addr())
+ if err := srv.Serve(lis); err != nil && ctx.Err() == nil {
+ klog.Exitf("Prometheus serve failed: %v", err)
+ }
+ }()
+ go func() {
+ <-ctx.Done()
+ srv.Close()
+ }()
+}
+
+// GRPCServerOptions returns pre-built grpc.ServerOptions that this component
+// should use to serve internal gRPC.
+func (c *ComponentConfig) GRPCServerOptions() []grpc.ServerOption {
return []grpc.ServerOption{
- grpc.Creds(credentials.NewTLS(tlsConf)),
+ grpc.Creds(credentials.NewTLS(c.getTLSConfig())),
}
}