build/sqlc: implement simple sqlc/bindata rules
This is a first pass at implementing rules to generate sqlc stubs from
.sql files and embed relevant migration files into bindata.
This is not yet used. The Go API is subject to change - especially the
way migrations are laid out in the generated package will probably
change.
Change-Id: I0873031603957a176ad4664c3b10768b791e0dd5
Reviewed-on: https://review.monogon.dev/c/monogon/+/884
Tested-by: Jenkins CI
Reviewed-by: Leopold Schabel <leo@monogon.tech>
diff --git a/build/sqlc/BUILD.bazel b/build/sqlc/BUILD.bazel
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/build/sqlc/BUILD.bazel
diff --git a/build/sqlc/README.md b/build/sqlc/README.md
new file mode 100644
index 0000000..72d9a86
--- /dev/null
+++ b/build/sqlc/README.md
@@ -0,0 +1,50 @@
+Bazel rules for sqlc
+===
+
+This is a set of rules which uses [sqlc](https://github.com/kyleconroy/sqlc) to generate Go code (types and functions) based on a SQL schema/migrations and a list of queries to be turned into functions.
+
+It also embeds the migrations using [bindata](https://github.com/kevinburke/go-bindata).
+
+Usage
+---
+
+In an empty directory (eg. monogon/foo/bar/model), create:
+
+ - Migration files, eg. `1662395623_foo.up.sql` and `1662395623_foo.down.sql` containing CREATE TABLE and DROP TABLE statements respectively.
+ - A query file, containing SQL queries annotated with function names and return values (see [official docs](https://docs.sqlc.dev/en/stable/tutorials/getting-started-postgresql.html) for a sample `query.sql` file).
+ - A `BUILD.bazel` file containing:
+
+```
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("//build/sqlc:sqlc.bzl", "sqlc_go_library")
+
+sqlc_go_library(
+ name = "sqlc_model",
+ importpath = "source.monogon.dev/foo/bar/model",
+ migrations = [
+ "1662395623_foo.up.sql",
+ "1662395623_foo.down.sql",
+ # More migrations can be created by provising larger timestamp values.
+ ],
+ queries = [
+ "queries.sql",
+ ],
+ dialect = "cockroachdb",
+)
+
+go_library(
+ name = "model",
+ importpath = "source.monogon.dev/foo/bar/model",
+ embed = [":sqlc_model"],
+ deps = [
+ # Might need this for CockroachDB UUID types.
+ "@com_github_google_uuid//:uuid",
+ ],
+)
+```
+
+The built `go_library ` will contain sqlc functions corresponding to queries defined in `queries.sql` and structures corresponding to database tables (and query parameters/results).
+
+To list the generated files for inspection/debugging, `bazel aquery //foo/bar:sqlc_model` and find files named `db.go`, `model.go` and `queries.sql.go` (or similar, depending on how your query file(s) are named).
+
+TODO(q3k): document migrations (and probably move them to a subpackage).
\ No newline at end of file
diff --git a/build/sqlc/sqlc.bzl b/build/sqlc/sqlc.bzl
new file mode 100644
index 0000000..3083fd3
--- /dev/null
+++ b/build/sqlc/sqlc.bzl
@@ -0,0 +1,182 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_context")
+
+# _parse_migrations takes a list of golang-migrate compatible schema files (in
+# the format <timestamp>_some_description_{up,down}.sql) and splits them into
+# 'up' and 'down' dictionaries, each a map from timestamp to underlying file.
+#
+# It also does some checks on the provided file names, making sure that
+# golang-migrate will parse them correctly.
+def _parse_migrations(files):
+ uppers = {}
+ downers = {}
+
+ # Ensure filename fits golang-migrate format, sort into 'up' and 'down' files.
+ for file in files:
+ if not file.basename.endswith(".up.sql") and not file.basename.endswith(".down.sql"):
+ fail("migration %s must end woth .{up,down}.sql" % file.basename)
+ if len(file.basename.split('.')) != 3:
+ fail("migration %s must not contain any . other than in .{up,down}.sql extension" % file.basename)
+ first = file.basename.split('.')[0]
+ if len(first.split('_')) < 2:
+ fail("migration %s must be in <timestamp>_<name>.{up,down}.sql format" % file.basename)
+ timestamp = first.split('_')[0]
+ if not timestamp.isdigit():
+ fail("migration %s must be in <timestamp>_<name>.{up,down}.sql format" % file.basename)
+ timestamp = int(timestamp)
+ if timestamp < 1662136250:
+ fail("migration %s must be in <timestamp>_<name>.{up,down}.sql format" % file.basename)
+
+ if file.basename.endswith('.up.sql'):
+ if timestamp in uppers:
+ fail("migration %s conflicts with %s" % [file.basename, uppers[timestamp].basename])
+ uppers[timestamp] = file
+ if file.basename.endswith('.down.sql'):
+ if timestamp in downers:
+ fail("migration %s conflicts with %s" % [file.basename, downers[timestamp].basename])
+ downers[timestamp] = file
+
+ # Check each 'up' has a corresponding 'down', and vice-versa.
+ for timestamp, up in uppers.items():
+ if timestamp not in downers:
+ fail("%s has no corresponding 'down' migration" % up.basename)
+ if downers[timestamp].basename.replace('down.sql', 'up.sql') != up.basename:
+ fail("%s has no corresponding 'down' migration" % up.basename)
+ for timestamp, down in downers.items():
+ if timestamp not in uppers:
+ fail("%s has no corresponding 'up' migration" % down.basename)
+ if uppers[timestamp].basename.replace('up.sql', 'down.sql') != down.basename:
+ fail("%s has no corresponding 'up' migration" % down.basename)
+
+ return uppers, downers
+
+def _sqlc_go_library(ctx):
+ go = go_context(ctx)
+
+ importpath_parts = ctx.attr.importpath.split("/")
+ package_name = importpath_parts[-1]
+
+ # Split migrations into 'up' and 'down'. Only pass 'up' to sqlc. Use both
+ # to generate golang-migrate compatible bindata.
+ uppers, downers = _parse_migrations(ctx.files.migrations)
+
+ # Make sure given queries have no repeating basenames. This ensures clean
+ # mapping source SQL file name and generated Go file.
+ query_basenames = []
+ for query in ctx.files.queries:
+ if query.basename in query_basenames:
+ fail("duplicate %s base name in query files" % query.basename)
+ query_basenames.append(query.basename)
+
+ # Go files generated by sqlc.
+ sqlc_go_sources = [
+ # db.go and models.go always exist.
+ ctx.actions.declare_file("db.go"),
+ ctx.actions.declare_file("models.go"),
+ ]
+ # For every query file, basename.go is also generated.
+ for basename in query_basenames:
+ sqlc_go_sources.append(ctx.actions.declare_file(basename + ".go"))
+
+ migrations_source = ctx.actions.declare_file("migrations.go")
+
+ # Cockroachdb is PostgreSQL with some extra overrides to fix Go/SQL type
+ # mappings.
+ overrides = []
+ if ctx.attr.dialect == "cockroachdb":
+ overrides = [
+ # INT is 64-bit in cockroachdb (32-bit in postgres).
+ { "go_type": "int64", "db_type": "pg_catalog.int4" },
+ ]
+
+ config = ctx.actions.declare_file("_config.yaml")
+ # All paths in config are relative to the config file. However, Bazel paths
+ # are relative to the execution root/CWD. To make things work regardless of
+ # config file placement, we prepend all config paths with a `../../ ...`
+ # path walk that makes the path be execroot relative again.
+ config_walk = '../' * config.path.count('/')
+ config_data = json.encode({
+ "version": 2,
+ "sql": [
+ {
+ "schema": [config_walk + up.path for up in uppers.values()],
+ "queries": [config_walk + query.path for query in ctx.files.queries],
+ "engine": "postgresql",
+ "gen": {
+ "go": {
+ "package": package_name,
+ "out": config_walk + sqlc_go_sources[0].dirname,
+ "overrides": overrides,
+ },
+ },
+ },
+ ],
+ })
+ ctx.actions.write(config, config_data)
+
+ # Generate types/functions using sqlc.
+ ctx.actions.run(
+ mnemonic = "SqlcGen",
+ executable = ctx.executable._sqlc,
+ arguments = [
+ "generate",
+ "-f", config.path,
+ ],
+ inputs = [
+ config
+ ] + uppers.values() + ctx.files.queries,
+ outputs = sqlc_go_sources,
+ )
+
+ # Generate migrations bindata.
+ ctx.actions.run(
+ mnemonic = "MigrationsEmbed",
+ executable = ctx.executable._bindata,
+ arguments = [
+ "-pkg", package_name,
+ "-prefix", ctx.label.workspace_root,
+ "-o", migrations_source.path,
+ ] + [up.path for up in uppers.values()] + [down.path for down in downers.values()],
+ inputs = uppers.values() + downers.values(),
+ outputs = [migrations_source],
+ )
+
+ library = go.new_library(go, srcs = sqlc_go_sources + [migrations_source], importparth = ctx.attr.importpath)
+ source = go.library_to_source(go, ctx.attr, library, ctx.coverage_instrumented())
+ return [
+ library,
+ source,
+ OutputGroupInfo(go_generated_srcs = depset(library.srcs)),
+ ]
+
+
+sqlc_go_library = rule(
+ implementation = _sqlc_go_library,
+ attrs = {
+ "migrations": attr.label_list(
+ allow_files = True,
+ ),
+ "queries": attr.label_list(
+ allow_files = True,
+ ),
+ "importpath": attr.string(
+ mandatory = True,
+ ),
+ "dialect": attr.string(
+ mandatory = True,
+ values = ["postgresql", "cockroachdb"],
+ ),
+ "_sqlc": attr.label(
+ default = Label("@com_github_kyleconroy_sqlc//cmd/sqlc"),
+ allow_single_file = True,
+ executable = True,
+ cfg = "exec",
+ ),
+ "_bindata": attr.label(
+ default = Label("@com_github_kevinburke_go_bindata//go-bindata"),
+ allow_single_file = True,
+ executable = True,
+ cfg = "exec",
+ ),
+ },
+ toolchains = ["@io_bazel_rules_go//go:toolchain"],
+)
diff --git a/build/sqlc/tool-dep/gomod-generated-placeholder.go b/build/sqlc/tool-dep/gomod-generated-placeholder.go
index 924235c..7db97d2 100644
--- a/build/sqlc/tool-dep/gomod-generated-placeholder.go
+++ b/build/sqlc/tool-dep/gomod-generated-placeholder.go
@@ -6,6 +6,7 @@
package main
import (
+ _ "github.com/kevinburke/go-bindata"
_ "github.com/kyleconroy/sqlc/pkg/cli"
)
diff --git a/go.mod b/go.mod
index 98ff875..3a839f8 100644
--- a/go.mod
+++ b/go.mod
@@ -82,6 +82,7 @@
github.com/google/uuid v1.3.0
github.com/insomniacslk/dhcp v0.0.0-20220119180841-3c283ff8b7dd
github.com/joho/godotenv v1.4.0
+ github.com/kevinburke/go-bindata v3.23.0+incompatible
github.com/kyleconroy/sqlc v1.15.0
github.com/mattn/go-shellwords v1.0.12
github.com/mdlayher/raw v0.1.0
diff --git a/go.sum b/go.sum
index 63a103f..9263021 100644
--- a/go.sum
+++ b/go.sum
@@ -1288,6 +1288,8 @@
github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
+github.com/kevinburke/go-bindata v3.23.0+incompatible h1:rqNOXZlqrYhMVVAsQx8wuc+LaA73YcfbQ407wAykyS8=
+github.com/kevinburke/go-bindata v3.23.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM=
github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
diff --git a/third_party/go/repositories.bzl b/third_party/go/repositories.bzl
index 84e0bab..b1ae98b 100644
--- a/third_party/go/repositories.bzl
+++ b/third_party/go/repositories.bzl
@@ -2862,6 +2862,12 @@
sum = "h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=",
version = "v1.16.1",
)
+ go_repository(
+ name = "com_github_kevinburke_go_bindata",
+ importpath = "github.com/kevinburke/go-bindata",
+ sum = "h1:rqNOXZlqrYhMVVAsQx8wuc+LaA73YcfbQ407wAykyS8=",
+ version = "v3.23.0+incompatible",
+ )
go_repository(
name = "com_github_kevinburke_ssh_config",