third_party/sandboxroot: add mirror tool
This mirrors the sandbox RPMs into a GCS bucket any time we regenerate
it. Hopefully this stops the constant barrage of random 404s when Fedora
just happened to bump a library and all the mirrors lost its previous
version.
This tool is currently specific to our bazeldnf-based sandboxroot setup,
but could be extended to mirror all of our dependencies at some point.
As our mirror is the last in the list, it should only be used when a
file is missing from other mirrors. In the future, we should have some
job that alerts us when too many of our deps are missing from upstream
mirrors.
Change-Id: I08ccbdf99ec868363918e30f3d2ae94f463e045f
Reviewed-on: https://review.monogon.dev/c/monogon/+/1473
Tested-by: Jenkins CI
Reviewed-by: Leopold Schabel <leo@monogon.tech>
diff --git a/third_party/sandboxroot/mirror/bazeldnf.go b/third_party/sandboxroot/mirror/bazeldnf.go
new file mode 100644
index 0000000..4f85581
--- /dev/null
+++ b/third_party/sandboxroot/mirror/bazeldnf.go
@@ -0,0 +1,69 @@
+package main
+
+import (
+ "fmt"
+
+ "go.starlark.net/starlark"
+)
+
+// getBazelDNFFiles parses third_party/sandboxroot/repositories.bzl (at the given
+// path) into a list of rpmDefs. It does so by loading the .bzl file into a
+// minimal starlark interpreter that emulates enough of the Bazel internal API to
+// get things going.
+func getBazelDNFFiles(path string) ([]*rpmDef, error) {
+ var res []*rpmDef
+
+ // rpm will be called any time the Starlark code calls rpm() from
+ // @bazeldnf//:deps.bzl.
+ rpm := func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+ var name, sha256 starlark.String
+ var urls *starlark.List
+ if err := starlark.UnpackArgs("rpm", args, kwargs, "name", &name, "sha256", &sha256, "urls", &urls); err != nil {
+ return nil, err
+ }
+ it := urls.Iterate()
+ defer it.Done()
+
+ var urlsS []string
+ var url starlark.Value
+ for it.Next(&url) {
+ if url.Type() != "string" {
+ return nil, fmt.Errorf("urls must be a list of strings")
+ }
+ urlS := url.(starlark.String)
+ urlsS = append(urlsS, urlS.GoString())
+ }
+
+ ext, err := newRPMDef(name.GoString(), sha256.GoString(), urlsS)
+ if err != nil {
+ return nil, fmt.Errorf("invalid rpm: %v", err)
+ }
+ res = append(res, ext)
+ return starlark.None, nil
+ }
+
+ thread := &starlark.Thread{
+ Name: "fakebazel",
+ Load: func(thread *starlark.Thread, module string) (starlark.StringDict, error) {
+ switch module {
+ case "@bazeldnf//:deps.bzl":
+ return map[string]starlark.Value{
+ "rpm": starlark.NewBuiltin("rpm", rpm),
+ }, nil
+ }
+ return nil, fmt.Errorf("not implemented in fakebazel")
+ },
+ }
+ globals, err := starlark.ExecFile(thread, path, nil, nil)
+ if err != nil {
+ return nil, fmt.Errorf("executing failed: %w", err)
+ }
+ if !globals.Has("sandbox_dependencies") {
+ return nil, fmt.Errorf("does not contain sandbox_dupendencies")
+ }
+ _, err = starlark.Call(thread, globals["sandbox_dependencies"], nil, nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to call sandbox_dependencies: %w", err)
+ }
+ return res, nil
+}