cloud/{apigw,lib/component}: add cockroachdb client, sample schema
This sets up some boilerplate to connect to CockroachDB servers,
including test in-memory servers.
We also add a first pass apigw user table schema, as the first user of
this new functionality. We exercise that, in turn, in a test.
We also rename component.Configuration to component.ComponentConfig.
There's a stutter in there, but it makes sense with
component.CockroachConfig alongside.
Change-Id: I76691146b87ce135d60db179b3f51eee16525df7
Reviewed-on: https://review.monogon.dev/c/monogon/+/912
Reviewed-by: Leopold Schabel <leo@monogon.tech>
Vouch-Run-CI: Leopold Schabel <leo@monogon.tech>
Tested-by: Jenkins CI
diff --git a/cloud/apigw/model/BUILD.bazel b/cloud/apigw/model/BUILD.bazel
new file mode 100644
index 0000000..081d488
--- /dev/null
+++ b/cloud/apigw/model/BUILD.bazel
@@ -0,0 +1,28 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("//build/sqlc:sqlc.bzl", "sqlc_go_library")
+
+sqlc_go_library(
+ name = "sqlc_model",
+ dialect = "cockroachdb",
+ importpath = "source.monogon.dev/cloud/apigw/model",
+ migrations = glob(["migrations/*sql"]),
+ queries = [
+ "queries.sql",
+ ],
+)
+
+go_library(
+ name = "model",
+ srcs = ["migrations.go"],
+ embed = [
+ ":sqlc_model", # keep
+ ],
+ embedsrcs = glob(["migrations/*sql"]),
+ importpath = "source.monogon.dev/cloud/apigw/model",
+ visibility = ["//visibility:public"],
+ deps = [
+ "@com_github_golang_migrate_migrate_v4//source",
+ "@com_github_golang_migrate_migrate_v4//source/iofs",
+ "@com_github_google_uuid//:uuid", # keep
+ ],
+)
diff --git a/cloud/apigw/model/migrations.go b/cloud/apigw/model/migrations.go
new file mode 100644
index 0000000..2c07768
--- /dev/null
+++ b/cloud/apigw/model/migrations.go
@@ -0,0 +1,15 @@
+package model
+
+import (
+ "embed"
+
+ "github.com/golang-migrate/migrate/v4/source"
+ "github.com/golang-migrate/migrate/v4/source/iofs"
+)
+
+//go:embed migrations/*.sql
+var migrationData embed.FS
+
+func MigrationsSource() (source.Driver, error) {
+ return iofs.New(migrationData, "migrations")
+}
diff --git a/cloud/apigw/model/migrations/1663155947_initial.down.sql b/cloud/apigw/model/migrations/1663155947_initial.down.sql
new file mode 100644
index 0000000..032d6cf
--- /dev/null
+++ b/cloud/apigw/model/migrations/1663155947_initial.down.sql
@@ -0,0 +1 @@
+DROP TABLE accounts;
\ No newline at end of file
diff --git a/cloud/apigw/model/migrations/1663155947_initial.up.sql b/cloud/apigw/model/migrations/1663155947_initial.up.sql
new file mode 100644
index 0000000..4812e00
--- /dev/null
+++ b/cloud/apigw/model/migrations/1663155947_initial.up.sql
@@ -0,0 +1,17 @@
+CREATE TABLE accounts (
+ -- Internal account ID. Never changes.
+ account_id UUID NOT NULL DEFAULT gen_random_uuid() PRIMARY KEY,
+
+ -- Identity used to tied this account to OIDC.
+ -- OpenID Connect Core, 2. ID Token: “It MUST NOT exceed 255 ASCII
+ -- characters in length”.
+ account_oidc_sub STRING(255) NOT NULL UNIQUE,
+
+ --- Copy/cache of user data retrieved from OIDC IdP on login. Currently this
+ --- is only updated on first login, but we should find a way to trigger
+ --- a re-retrieval.
+ -- Display name preferred by user.
+ -- Self-limiting ourselves to 255 unicode codepoints here. This is also
+ -- supposedly what keycloak also defaults to for user attributes.
+ account_display_name STRING(255) NOT NULL
+);
\ No newline at end of file
diff --git a/cloud/apigw/model/queries.sql b/cloud/apigw/model/queries.sql
new file mode 100644
index 0000000..564f91d
--- /dev/null
+++ b/cloud/apigw/model/queries.sql
@@ -0,0 +1,13 @@
+-- name: GetAccountByOIDC :many
+SELECT
+ accounts.*
+FROM accounts
+WHERE account_oidc_sub = $1;
+
+-- name: InitializeAccountFromOIDC :one
+INSERT INTO accounts (
+ account_oidc_sub, account_display_name
+) VALUES (
+ $1, $2
+)
+RETURNING *;
\ No newline at end of file
diff --git a/cloud/apigw/server/BUILD.bazel b/cloud/apigw/server/BUILD.bazel
index 2444267..246efff 100644
--- a/cloud/apigw/server/BUILD.bazel
+++ b/cloud/apigw/server/BUILD.bazel
@@ -7,6 +7,7 @@
visibility = ["//visibility:public"],
deps = [
"//cloud/api",
+ "//cloud/apigw/model",
"//cloud/lib/component",
"@com_github_improbable_eng_grpc_web//go/grpcweb",
"@io_k8s_klog_v2//:klog",
@@ -21,9 +22,13 @@
go_test(
name = "server_test",
srcs = ["server_test.go"],
+ data = [
+ "@cockroach",
+ ],
embed = [":server"],
deps = [
"//cloud/api",
+ "//cloud/apigw/model",
"//cloud/lib/component",
"@org_golang_google_grpc//codes",
"@org_golang_google_protobuf//proto",
diff --git a/cloud/apigw/server/server.go b/cloud/apigw/server/server.go
index b3b80bf..a068e84 100644
--- a/cloud/apigw/server/server.go
+++ b/cloud/apigw/server/server.go
@@ -15,13 +15,15 @@
"k8s.io/klog/v2"
apb "source.monogon.dev/cloud/api"
+ "source.monogon.dev/cloud/apigw/model"
"source.monogon.dev/cloud/lib/component"
)
// Config is the main configuration of the apigw server. It's usually populated
// from flags via RegisterFlags, but can also be set manually (eg. in tests).
type Config struct {
- component.Configuration
+ Component component.ComponentConfig
+ Database component.CockroachConfig
PublicListenAddress string
}
@@ -29,7 +31,8 @@
// RegisterFlags registers the component configuration to be provided by flags.
// This must be called exactly once before then calling flags.Parse().
func (c *Config) RegisterFlags() {
- c.Configuration.RegisterFlags("apigw")
+ c.Component.RegisterFlags("apigw")
+ c.Database.RegisterFlags("apigw_db")
flag.StringVar(&c.PublicListenAddress, "apigw_public_grpc_listen_address", ":8080", "Address to listen at for public/user gRPC connections for apigw")
}
@@ -51,8 +54,8 @@
}
func (s *Server) startInternalGRPC(ctx context.Context) {
- g := grpc.NewServer(s.Config.GRPCServerOptions()...)
- lis, err := net.Listen("tcp", s.Config.GRPCListenAddress)
+ g := grpc.NewServer(s.Config.Component.GRPCServerOptions()...)
+ lis, err := net.Listen("tcp", s.Config.Component.GRPCListenAddress)
if err != nil {
klog.Exitf("Could not listen: %v", err)
}
@@ -97,6 +100,20 @@
// Start runs the two listeners of the server. The process will fail (via
// klog.Exit) if any of the listeners/servers fail to start.
func (s *Server) Start(ctx context.Context) {
+ if s.Config.Database.Migrations == nil {
+ klog.Infof("Using default migrations source.")
+ m, err := model.MigrationsSource()
+ if err != nil {
+ klog.Exitf("failed to prepare migrations source: %w", err)
+ }
+ s.Config.Database.Migrations = m
+ }
+
+ klog.Infof("Running migrations...")
+ if err := s.Config.Database.MigrateUp(); err != nil {
+ klog.Exitf("Migrations failed: %v", err)
+ }
+ klog.Infof("Migrations done.")
s.startInternalGRPC(ctx)
s.startPublic(ctx)
}
diff --git a/cloud/apigw/server/server_test.go b/cloud/apigw/server/server_test.go
index 16cdc07..704de36 100644
--- a/cloud/apigw/server/server_test.go
+++ b/cloud/apigw/server/server_test.go
@@ -13,22 +13,29 @@
"google.golang.org/protobuf/proto"
apb "source.monogon.dev/cloud/api"
+ "source.monogon.dev/cloud/apigw/model"
"source.monogon.dev/cloud/lib/component"
)
-// TestPublicSimple ensures the public grpc-web listener is working.
-func TestPublicSimple(t *testing.T) {
- s := Server{
+func dut() *Server {
+ return &Server{
Config: Config{
- Configuration: component.Configuration{
+ Component: component.ComponentConfig{
GRPCListenAddress: ":0",
DevCerts: true,
DevCertsPath: "/tmp/foo",
},
+ Database: component.CockroachConfig{
+ InMemory: true,
+ },
PublicListenAddress: ":0",
},
}
+}
+// TestPublicSimple ensures the public grpc-web listener is working.
+func TestPublicSimple(t *testing.T) {
+ s := dut()
ctx := context.Background()
s.Start(ctx)
@@ -72,3 +79,48 @@
t.Errorf("Wanted message %q, got %q", want, got)
}
}
+
+// TestUserSimple makes sure we can add and retrieve users. This is a low-level
+// test which mostly exercises the machinery to bring up a working database in
+// tests.
+func TestUserSimple(t *testing.T) {
+ s := dut()
+ ctx := context.Background()
+ s.Start(ctx)
+
+ db, err := s.Config.Database.Connect()
+ if err != nil {
+ t.Fatalf("Connecting to the database failed: %v", err)
+ }
+ q := model.New(db)
+
+ // Start out with no account by sub 'test'.
+ accounts, err := q.GetAccountByOIDC(ctx, "test")
+ if err != nil {
+ t.Fatalf("Retrieving accounts failed: %v", err)
+ }
+ if want, got := 0, len(accounts); want != got {
+ t.Fatalf("Expected no accounts at first, got %d", got)
+ }
+
+ // Create a new test account for sub 'test'.
+ _, err = q.InitializeAccountFromOIDC(ctx, model.InitializeAccountFromOIDCParams{
+ AccountOidcSub: "test",
+ AccountDisplayName: "Test User",
+ })
+ if err != nil {
+ t.Fatalf("Creating new account failed: %v", err)
+ }
+
+ // Expect this account to be available now.
+ accounts, err = q.GetAccountByOIDC(ctx, "test")
+ if err != nil {
+ t.Fatalf("Retrieving accounts failed: %v", err)
+ }
+ if want, got := 1, len(accounts); want != got {
+ t.Fatalf("Expected exactly one account after creation, got %d", got)
+ }
+ if want, got := "Test User", accounts[0].AccountDisplayName; want != got {
+ t.Fatalf("Expected to read back display name %q, got %q", want, got)
+ }
+}