cloud/{apigw,lib/component}: add cockroachdb client, sample schema
This sets up some boilerplate to connect to CockroachDB servers,
including test in-memory servers.
We also add a first pass apigw user table schema, as the first user of
this new functionality. We exercise that, in turn, in a test.
We also rename component.Configuration to component.ComponentConfig.
There's a stutter in there, but it makes sense with
component.CockroachConfig alongside.
Change-Id: I76691146b87ce135d60db179b3f51eee16525df7
Reviewed-on: https://review.monogon.dev/c/monogon/+/912
Reviewed-by: Leopold Schabel <leo@monogon.tech>
Vouch-Run-CI: Leopold Schabel <leo@monogon.tech>
Tested-by: Jenkins CI
diff --git a/cloud/apigw/model/BUILD.bazel b/cloud/apigw/model/BUILD.bazel
new file mode 100644
index 0000000..081d488
--- /dev/null
+++ b/cloud/apigw/model/BUILD.bazel
@@ -0,0 +1,28 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("//build/sqlc:sqlc.bzl", "sqlc_go_library")
+
+sqlc_go_library(
+ name = "sqlc_model",
+ dialect = "cockroachdb",
+ importpath = "source.monogon.dev/cloud/apigw/model",
+ migrations = glob(["migrations/*sql"]),
+ queries = [
+ "queries.sql",
+ ],
+)
+
+go_library(
+ name = "model",
+ srcs = ["migrations.go"],
+ embed = [
+ ":sqlc_model", # keep
+ ],
+ embedsrcs = glob(["migrations/*sql"]),
+ importpath = "source.monogon.dev/cloud/apigw/model",
+ visibility = ["//visibility:public"],
+ deps = [
+ "@com_github_golang_migrate_migrate_v4//source",
+ "@com_github_golang_migrate_migrate_v4//source/iofs",
+ "@com_github_google_uuid//:uuid", # keep
+ ],
+)
diff --git a/cloud/apigw/model/migrations.go b/cloud/apigw/model/migrations.go
new file mode 100644
index 0000000..2c07768
--- /dev/null
+++ b/cloud/apigw/model/migrations.go
@@ -0,0 +1,15 @@
+package model
+
+import (
+ "embed"
+
+ "github.com/golang-migrate/migrate/v4/source"
+ "github.com/golang-migrate/migrate/v4/source/iofs"
+)
+
+//go:embed migrations/*.sql
+var migrationData embed.FS
+
+func MigrationsSource() (source.Driver, error) {
+ return iofs.New(migrationData, "migrations")
+}
diff --git a/cloud/apigw/model/migrations/1663155947_initial.down.sql b/cloud/apigw/model/migrations/1663155947_initial.down.sql
new file mode 100644
index 0000000..032d6cf
--- /dev/null
+++ b/cloud/apigw/model/migrations/1663155947_initial.down.sql
@@ -0,0 +1 @@
+DROP TABLE accounts;
\ No newline at end of file
diff --git a/cloud/apigw/model/migrations/1663155947_initial.up.sql b/cloud/apigw/model/migrations/1663155947_initial.up.sql
new file mode 100644
index 0000000..4812e00
--- /dev/null
+++ b/cloud/apigw/model/migrations/1663155947_initial.up.sql
@@ -0,0 +1,17 @@
+CREATE TABLE accounts (
+ -- Internal account ID. Never changes.
+ account_id UUID NOT NULL DEFAULT gen_random_uuid() PRIMARY KEY,
+
+ -- Identity used to tied this account to OIDC.
+ -- OpenID Connect Core, 2. ID Token: “It MUST NOT exceed 255 ASCII
+ -- characters in length”.
+ account_oidc_sub STRING(255) NOT NULL UNIQUE,
+
+ --- Copy/cache of user data retrieved from OIDC IdP on login. Currently this
+ --- is only updated on first login, but we should find a way to trigger
+ --- a re-retrieval.
+ -- Display name preferred by user.
+ -- Self-limiting ourselves to 255 unicode codepoints here. This is also
+ -- supposedly what keycloak also defaults to for user attributes.
+ account_display_name STRING(255) NOT NULL
+);
\ No newline at end of file
diff --git a/cloud/apigw/model/queries.sql b/cloud/apigw/model/queries.sql
new file mode 100644
index 0000000..564f91d
--- /dev/null
+++ b/cloud/apigw/model/queries.sql
@@ -0,0 +1,13 @@
+-- name: GetAccountByOIDC :many
+SELECT
+ accounts.*
+FROM accounts
+WHERE account_oidc_sub = $1;
+
+-- name: InitializeAccountFromOIDC :one
+INSERT INTO accounts (
+ account_oidc_sub, account_display_name
+) VALUES (
+ $1, $2
+)
+RETURNING *;
\ No newline at end of file
diff --git a/cloud/apigw/server/BUILD.bazel b/cloud/apigw/server/BUILD.bazel
index 2444267..246efff 100644
--- a/cloud/apigw/server/BUILD.bazel
+++ b/cloud/apigw/server/BUILD.bazel
@@ -7,6 +7,7 @@
visibility = ["//visibility:public"],
deps = [
"//cloud/api",
+ "//cloud/apigw/model",
"//cloud/lib/component",
"@com_github_improbable_eng_grpc_web//go/grpcweb",
"@io_k8s_klog_v2//:klog",
@@ -21,9 +22,13 @@
go_test(
name = "server_test",
srcs = ["server_test.go"],
+ data = [
+ "@cockroach",
+ ],
embed = [":server"],
deps = [
"//cloud/api",
+ "//cloud/apigw/model",
"//cloud/lib/component",
"@org_golang_google_grpc//codes",
"@org_golang_google_protobuf//proto",
diff --git a/cloud/apigw/server/server.go b/cloud/apigw/server/server.go
index b3b80bf..a068e84 100644
--- a/cloud/apigw/server/server.go
+++ b/cloud/apigw/server/server.go
@@ -15,13 +15,15 @@
"k8s.io/klog/v2"
apb "source.monogon.dev/cloud/api"
+ "source.monogon.dev/cloud/apigw/model"
"source.monogon.dev/cloud/lib/component"
)
// Config is the main configuration of the apigw server. It's usually populated
// from flags via RegisterFlags, but can also be set manually (eg. in tests).
type Config struct {
- component.Configuration
+ Component component.ComponentConfig
+ Database component.CockroachConfig
PublicListenAddress string
}
@@ -29,7 +31,8 @@
// RegisterFlags registers the component configuration to be provided by flags.
// This must be called exactly once before then calling flags.Parse().
func (c *Config) RegisterFlags() {
- c.Configuration.RegisterFlags("apigw")
+ c.Component.RegisterFlags("apigw")
+ c.Database.RegisterFlags("apigw_db")
flag.StringVar(&c.PublicListenAddress, "apigw_public_grpc_listen_address", ":8080", "Address to listen at for public/user gRPC connections for apigw")
}
@@ -51,8 +54,8 @@
}
func (s *Server) startInternalGRPC(ctx context.Context) {
- g := grpc.NewServer(s.Config.GRPCServerOptions()...)
- lis, err := net.Listen("tcp", s.Config.GRPCListenAddress)
+ g := grpc.NewServer(s.Config.Component.GRPCServerOptions()...)
+ lis, err := net.Listen("tcp", s.Config.Component.GRPCListenAddress)
if err != nil {
klog.Exitf("Could not listen: %v", err)
}
@@ -97,6 +100,20 @@
// Start runs the two listeners of the server. The process will fail (via
// klog.Exit) if any of the listeners/servers fail to start.
func (s *Server) Start(ctx context.Context) {
+ if s.Config.Database.Migrations == nil {
+ klog.Infof("Using default migrations source.")
+ m, err := model.MigrationsSource()
+ if err != nil {
+ klog.Exitf("failed to prepare migrations source: %w", err)
+ }
+ s.Config.Database.Migrations = m
+ }
+
+ klog.Infof("Running migrations...")
+ if err := s.Config.Database.MigrateUp(); err != nil {
+ klog.Exitf("Migrations failed: %v", err)
+ }
+ klog.Infof("Migrations done.")
s.startInternalGRPC(ctx)
s.startPublic(ctx)
}
diff --git a/cloud/apigw/server/server_test.go b/cloud/apigw/server/server_test.go
index 16cdc07..704de36 100644
--- a/cloud/apigw/server/server_test.go
+++ b/cloud/apigw/server/server_test.go
@@ -13,22 +13,29 @@
"google.golang.org/protobuf/proto"
apb "source.monogon.dev/cloud/api"
+ "source.monogon.dev/cloud/apigw/model"
"source.monogon.dev/cloud/lib/component"
)
-// TestPublicSimple ensures the public grpc-web listener is working.
-func TestPublicSimple(t *testing.T) {
- s := Server{
+func dut() *Server {
+ return &Server{
Config: Config{
- Configuration: component.Configuration{
+ Component: component.ComponentConfig{
GRPCListenAddress: ":0",
DevCerts: true,
DevCertsPath: "/tmp/foo",
},
+ Database: component.CockroachConfig{
+ InMemory: true,
+ },
PublicListenAddress: ":0",
},
}
+}
+// TestPublicSimple ensures the public grpc-web listener is working.
+func TestPublicSimple(t *testing.T) {
+ s := dut()
ctx := context.Background()
s.Start(ctx)
@@ -72,3 +79,48 @@
t.Errorf("Wanted message %q, got %q", want, got)
}
}
+
+// TestUserSimple makes sure we can add and retrieve users. This is a low-level
+// test which mostly exercises the machinery to bring up a working database in
+// tests.
+func TestUserSimple(t *testing.T) {
+ s := dut()
+ ctx := context.Background()
+ s.Start(ctx)
+
+ db, err := s.Config.Database.Connect()
+ if err != nil {
+ t.Fatalf("Connecting to the database failed: %v", err)
+ }
+ q := model.New(db)
+
+ // Start out with no account by sub 'test'.
+ accounts, err := q.GetAccountByOIDC(ctx, "test")
+ if err != nil {
+ t.Fatalf("Retrieving accounts failed: %v", err)
+ }
+ if want, got := 0, len(accounts); want != got {
+ t.Fatalf("Expected no accounts at first, got %d", got)
+ }
+
+ // Create a new test account for sub 'test'.
+ _, err = q.InitializeAccountFromOIDC(ctx, model.InitializeAccountFromOIDCParams{
+ AccountOidcSub: "test",
+ AccountDisplayName: "Test User",
+ })
+ if err != nil {
+ t.Fatalf("Creating new account failed: %v", err)
+ }
+
+ // Expect this account to be available now.
+ accounts, err = q.GetAccountByOIDC(ctx, "test")
+ if err != nil {
+ t.Fatalf("Retrieving accounts failed: %v", err)
+ }
+ if want, got := 1, len(accounts); want != got {
+ t.Fatalf("Expected exactly one account after creation, got %d", got)
+ }
+ if want, got := "Test User", accounts[0].AccountDisplayName; want != got {
+ t.Fatalf("Expected to read back display name %q, got %q", want, got)
+ }
+}
diff --git a/cloud/lib/component/BUILD.bazel b/cloud/lib/component/BUILD.bazel
index a97d770..d705997 100644
--- a/cloud/lib/component/BUILD.bazel
+++ b/cloud/lib/component/BUILD.bazel
@@ -4,13 +4,20 @@
name = "component",
srcs = [
"component.go",
+ "crdb.go",
"devcerts.go",
],
importpath = "source.monogon.dev/cloud/lib/component",
visibility = ["//visibility:public"],
deps = [
+ "//metropolis/cli/pkg/datafile",
"//metropolis/pkg/pki",
"@com_github_adrg_xdg//:xdg",
+ "@com_github_cockroachdb_cockroach_go_v2//testserver",
+ "@com_github_golang_migrate_migrate_v4//:migrate",
+ "@com_github_golang_migrate_migrate_v4//database/cockroachdb",
+ "@com_github_golang_migrate_migrate_v4//source",
+ "@com_github_lib_pq//:pq",
"@io_k8s_klog_v2//:klog",
"@org_golang_google_grpc//:go_default_library",
"@org_golang_google_grpc//credentials",
diff --git a/cloud/lib/component/component.go b/cloud/lib/component/component.go
index 4353bdf..831d099 100644
--- a/cloud/lib/component/component.go
+++ b/cloud/lib/component/component.go
@@ -17,8 +17,11 @@
"k8s.io/klog/v2"
)
-// Configuration is the common configuration of a component.
-type Configuration struct {
+// ComponentConfig is the common configuration of a component. It's
+// supposed to be instantiated within a Configuration struct of a component.
+//
+// It can be configured by flags (via RegisterFlags) or manually (eg. in tests).
+type ComponentConfig struct {
// GRPCKeyPath is the filesystem path of the x509 key used to serve internal
// gRPC traffic.
GRPCKeyPath string
@@ -46,7 +49,7 @@
// RegisterFlags registers the component configuration to be provided by flags.
// This must be called exactly once before then calling flags.Parse().
-func (c *Configuration) RegisterFlags(componentName string) {
+func (c *ComponentConfig) RegisterFlags(componentName string) {
flag.StringVar(&c.GRPCKeyPath, componentName+"_grpc_key_path", "", "Path to gRPC server/client key for "+componentName)
flag.StringVar(&c.GRPCCertificatePath, componentName+"_grpc_certificate_path", "", "Path to gRPC server/client certificate for "+componentName)
flag.StringVar(&c.GRPCCAPath, componentName+"_grpc_ca_certificate_path", "", "Path to gRPC CA certificate for "+componentName)
@@ -60,7 +63,7 @@
// GRPCServerOptions returns pre-built grpc.ServerOptions that this component
// should use to serve internal gRPC.
-func (c *Configuration) GRPCServerOptions() []grpc.ServerOption {
+func (c *ComponentConfig) GRPCServerOptions() []grpc.ServerOption {
var certPath, keyPath, caPath string
if c.DevCerts {
// Use devcerts if requested.
diff --git a/cloud/lib/component/crdb.go b/cloud/lib/component/crdb.go
new file mode 100644
index 0000000..bff98f0
--- /dev/null
+++ b/cloud/lib/component/crdb.go
@@ -0,0 +1,144 @@
+package component
+
+import (
+ "database/sql"
+ "flag"
+ "net/url"
+ "os"
+ "sync"
+
+ "github.com/cockroachdb/cockroach-go/v2/testserver"
+ "github.com/golang-migrate/migrate/v4"
+ _ "github.com/golang-migrate/migrate/v4/database/cockroachdb"
+ "github.com/golang-migrate/migrate/v4/source"
+ _ "github.com/lib/pq"
+ "k8s.io/klog/v2"
+
+ "source.monogon.dev/metropolis/cli/pkg/datafile"
+)
+
+// CockroachConfig is the common configuration of a components' connection to
+// CockroachDB. It's supposed to be instantiated within a Configuration struct
+// of a component.
+//
+// It can be configured by flags (via RegisterFlags) or manually (eg. in tests).
+type CockroachConfig struct {
+ // Migrations is the go-migrate source of migrations for this database. Usually
+ // this can be taken from a go-embedded set of migration files.
+ Migrations source.Driver
+
+ // EndpointHost is the host part of the endpoint address of the database server.
+ EndpointHost string
+ // TLSKeyPath is the filesystem path of the x509 key used to authenticate to the
+ // database server.
+ TLSKeyPath string
+ // TLSKeyPath is the filesystem path of the x509 certificate used to
+ // authenticate to the database server.
+ TLSCertificatePath string
+ // TLSCACertificatePath is the filesystem path of the x509 CA certificate used
+ // to verify the database server's certificate.
+ TLSCACertificatePath string
+ // UserName is the username to be used on the database server.
+ UserName string
+ // UserName is the database name to be used on the database server.
+ DatabaseName string
+
+ // InMemory indicates that an in-memory CockroachDB instance should be used.
+ // Data will be lost after the component shuts down.
+ InMemory bool
+
+ // mu guards inMemoryInstance.
+ mu sync.Mutex
+ // inMemoryInstance is populated with a CockroachDB test server handle when
+ // InMemory is set and Connect()/MigrateUp() is called.
+ inMemoryInstance testserver.TestServer
+}
+
+// RegisterFlags registers the connection configuration to be provided by flags.
+// This must be called exactly once before then calling flags.Parse().
+func (c *CockroachConfig) RegisterFlags(prefix string) {
+ flag.StringVar(&c.EndpointHost, prefix+"_endpoint_host", "", "Host of CockroachDB endpoint for "+prefix)
+ flag.StringVar(&c.TLSKeyPath, prefix+"_tls_key_path", "", "Path to CockroachDB TLS client key for "+prefix)
+ flag.StringVar(&c.TLSCertificatePath, prefix+"_tls_certificate_path", "", "Path to CockroachDB TLS client certificate for "+prefix)
+ flag.StringVar(&c.TLSCACertificatePath, prefix+"_tls_ca_certificate_path", "", "Path to CockroachDB CA certificate for "+prefix)
+ flag.StringVar(&c.UserName, prefix+"_user_name", prefix, "CockroachDB user name for "+prefix)
+ flag.StringVar(&c.DatabaseName, prefix+"_database_name", prefix, "CockroachDB database name for "+prefix)
+ flag.BoolVar(&c.InMemory, prefix+"_eat_my_data", false, "Use in-memory CockroachDB for "+prefix+". Warning: Data will be lost at process shutdown!")
+}
+
+// startInMemory starts an in-memory cockroachdb server as a subprocess, and
+// returns a DSN that connects to the newly created database.
+func (c *CockroachConfig) startInMemory(scheme string) string {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+
+ klog.Warningf("STARTING IN-MEMORY COCKROACHDB FOR TESTS")
+ klog.Warningf("ALL DATA WILL BE LOST AFTER SERVER SHUTDOWN!")
+
+ if c.inMemoryInstance == nil {
+ opts := []testserver.TestServerOpt{
+ testserver.SecureOpt(),
+ }
+ if path, err := datafile.ResolveRunfile("external/cockroach/cockroach"); err == nil {
+ opts = append(opts, testserver.CockroachBinaryPathOpt(path))
+ } else {
+ if os.Getenv("TEST_TMPDIR") != "" {
+ klog.Exitf("In test which requires in-memory cockroachdb, but @cockroach//:cockroach missing as a dependency. Failing.")
+ }
+ klog.Warningf("CockroachDB in-memory database requested, but not available as a build dependency. Trying to download it...")
+ }
+
+ inst, err := testserver.NewTestServer(opts...)
+ if err != nil {
+ klog.Exitf("Failed to create crdb test server: %v", err)
+ }
+ c.inMemoryInstance = inst
+ }
+
+ u := *c.inMemoryInstance.PGURL()
+ u.Scheme = scheme
+ return u.String()
+}
+
+// buildDSN returns a DSN to the configured database connection with a given DSN
+// scheme. The scheme will usually be 'postgres' or 'cockroach', depending on
+// whether it's used for lib/pq or for golang-migrate.
+func (c *CockroachConfig) buildDSN(scheme string) string {
+ if c.InMemory {
+ return c.startInMemory(scheme)
+ }
+
+ query := make(url.Values)
+ query.Set("sslmode", "verify-full")
+ query.Set("sslcert", c.TLSCertificatePath)
+ query.Set("sslkey", c.TLSKeyPath)
+ query.Set("sslrootcert", c.TLSCACertificatePath)
+ u := url.URL{
+ Scheme: scheme,
+ User: url.User(c.UserName),
+ Host: c.EndpointHost,
+ Path: c.DatabaseName,
+ RawQuery: query.Encode(),
+ }
+ return u.String()
+}
+
+// Connect returns a working *sql.DB handle to the database described by this
+// CockroachConfig.
+func (d *CockroachConfig) Connect() (*sql.DB, error) {
+ dsn := d.buildDSN("postgres")
+ klog.Infof("Connecting to %s...", dsn)
+ return sql.Open("postgres", d.buildDSN("postgres"))
+}
+
+// MigrateUp performs all possible migrations upwards for the database described
+// by this CockroachConfig.
+func (d *CockroachConfig) MigrateUp() error {
+ dsn := d.buildDSN("cockroachdb")
+ klog.Infof("Running migrations on %s...", dsn)
+ m, err := migrate.NewWithSourceInstance("iofs", d.Migrations, dsn)
+ if err != nil {
+ return err
+ }
+ return m.Up()
+}
diff --git a/cloud/lib/component/devcerts.go b/cloud/lib/component/devcerts.go
index af1e3d3..4d8668d 100644
--- a/cloud/lib/component/devcerts.go
+++ b/cloud/lib/component/devcerts.go
@@ -18,7 +18,7 @@
// GetDevCerts returns paths to this component's development certificate, key
// and CA, or panics if unavailable.
-func (c *Configuration) GetDevCerts() (certPath, keyPath, caPath string) {
+func (c *ComponentConfig) GetDevCerts() (certPath, keyPath, caPath string) {
klog.Infof("Using developer certificates at %s", c.DevCertsPath)
caPath = c.ensureDevCA()
@@ -30,7 +30,7 @@
// component and returns paths to them. This data is either read from disk if it
// already exists, or is generated when this function is called. If any problem
// occurs, the code panics.
-func (c *Configuration) ensureDevComponent() (certPath, keyPath string) {
+func (c *ComponentConfig) ensureDevComponent() (certPath, keyPath string) {
caKeyPath := c.DevCertsPath + "/ca.key"
caCertPath := c.DevCertsPath + "/ca.cert"
@@ -113,7 +113,7 @@
// paths to them. This data is either read from disk if it already exists, or is
// generated when this function is called. If any problem occurs, the code
// panics.
-func (c *Configuration) ensureDevCA() (caCertPath string) {
+func (c *ComponentConfig) ensureDevCA() (caCertPath string) {
caKeyPath := c.DevCertsPath + "/ca.key"
caCertPath = c.DevCertsPath + "/ca.cert"