osbase/build/mkoci/index: add package
This adds the mkoci/index package, which contains a Bazel rule for
building a multi-platform OCI index for an image.
rules_oci also has an image index rule, but it is not suitable for us.
That rule tries to read the platform from the container image config,
meaning it only works for container images. The rule here works for
arbitrary OCI artifact types. To make this work, the rule which
generates the image must put the platform into the descriptor in
index.json.
Because the index is not for any specific platform, there is a new "all"
platform where the index is generated.
Change-Id: I4ab1b87609d10b77c2f7fc42ee427f87d9f5ddc2
Reviewed-on: https://review.monogon.dev/c/monogon/+/4476
Reviewed-by: Tim Windelschmidt <tim@monogon.tech>
Tested-by: Jenkins CI
diff --git a/osbase/build/mkoci/index/BUILD.bazel b/osbase/build/mkoci/index/BUILD.bazel
new file mode 100644
index 0000000..246ae4d
--- /dev/null
+++ b/osbase/build/mkoci/index/BUILD.bazel
@@ -0,0 +1,20 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+ name = "index_lib",
+ srcs = ["main.go"],
+ importpath = "source.monogon.dev/osbase/build/mkoci/index",
+ visibility = ["//visibility:private"],
+ deps = [
+ "//osbase/oci",
+ "@com_github_opencontainers_go_digest//:go-digest",
+ "@com_github_opencontainers_image_spec//specs-go",
+ "@com_github_opencontainers_image_spec//specs-go/v1:specs-go",
+ ],
+)
+
+go_binary(
+ name = "mkoci_index",
+ embed = [":index_lib"],
+ visibility = ["//visibility:public"],
+)
diff --git a/osbase/build/mkoci/index/def.bzl b/osbase/build/mkoci/index/def.bzl
new file mode 100644
index 0000000..a8a82b6
--- /dev/null
+++ b/osbase/build/mkoci/index/def.bzl
@@ -0,0 +1,97 @@
+def _multi_platform_transition_impl(_settings, attr):
+ return {
+ str(platform): {"//command_line_option:platforms": str(platform)}
+ for platform in attr.platforms
+ }
+
+_multi_platform_transition = transition(
+ implementation = _multi_platform_transition_impl,
+ inputs = [],
+ outputs = ["//command_line_option:platforms"],
+)
+
+def _platform_independent_transition_impl(_settings, _attr):
+ return {"//command_line_option:platforms": "//build/platforms:all"}
+
+_platform_independent_transition = transition(
+ implementation = _platform_independent_transition_impl,
+ inputs = [],
+ outputs = ["//command_line_option:platforms"],
+)
+
+def _oci_index_impl(ctx):
+ inputs = []
+ transitive_runfiles = []
+ args = ctx.actions.args()
+
+ for platform in ctx.attr.platforms:
+ # Use ctx.split_attr because for ctx.attr, the order is unspecified.
+ image = ctx.split_attr.src[str(platform.label)]
+ files = image[DefaultInfo].files.to_list()
+ if len(files) != 1:
+ fail("image does not have exactly one directory: {}", files)
+ file = files[0]
+ if not file.is_directory:
+ fail("image is not a directory: {}", file)
+ inputs.append(file)
+ args.add("-image", file.path)
+ transitive_runfiles.append(image[DefaultInfo].default_runfiles)
+
+ output = ctx.actions.declare_directory(ctx.label.name)
+ args.add("-out", output.path)
+
+ ctx.actions.run(
+ mnemonic = "MkOCIIndex",
+ executable = ctx.executable._mkoci_index,
+ arguments = [args],
+ inputs = inputs,
+ outputs = [output],
+ )
+
+ # The inputs are referenced by symlinks.
+ runfiles = ctx.runfiles(files = inputs)
+
+ # Also merge the runfiles of the input images, in case they already use symlinks.
+ runfiles = runfiles.merge_all(transitive_runfiles)
+ return [DefaultInfo(
+ files = depset([output]),
+ runfiles = runfiles,
+ )]
+
+oci_index = rule(
+ cfg = _platform_independent_transition,
+ implementation = _oci_index_impl,
+ doc = """
+ Build a multi-platform OCI index. This rule works with arbitrary image
+ types, as it does not attempt to parse the image config.
+
+ Since the index is not for a specific platform, it is transitioned to
+ the platform-independent platform.
+ """,
+ attrs = {
+ "src": attr.label(
+ doc = """
+ OCI image, stored in an OCI layout directory. The descriptor in
+ the index.json should include platform information, as the
+ descriptor is copied as is into the generated index.
+ """,
+ mandatory = True,
+ allow_files = True,
+ cfg = _multi_platform_transition,
+ ),
+ "platforms": attr.label_list(
+ doc = """
+ A list of platforms for which the OCI image is built and added to the index.
+ """,
+ mandatory = True,
+ providers = [platform_common.PlatformInfo],
+ ),
+
+ # Tool
+ "_mkoci_index": attr.label(
+ default = Label("//osbase/build/mkoci/index:mkoci_index"),
+ executable = True,
+ cfg = "exec",
+ ),
+ },
+)
diff --git a/osbase/build/mkoci/index/main.go b/osbase/build/mkoci/index/main.go
new file mode 100644
index 0000000..d2bcd67
--- /dev/null
+++ b/osbase/build/mkoci/index/main.go
@@ -0,0 +1,143 @@
+// Copyright The Monogon Project Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "crypto/sha256"
+ "encoding/json"
+ "flag"
+ "fmt"
+ "log"
+ "os"
+ "path/filepath"
+
+ "github.com/opencontainers/go-digest"
+ ocispec "github.com/opencontainers/image-spec/specs-go"
+ ocispecv1 "github.com/opencontainers/image-spec/specs-go/v1"
+
+ "source.monogon.dev/osbase/oci"
+)
+
+var (
+ outPath = flag.String("out", "", "Output OCI Image Layout directory path")
+)
+
+func addImage(outPath string, path string, haveBlob map[digest.Digest]bool) (*ocispecv1.Descriptor, error) {
+ index, err := oci.ReadLayoutIndex(path)
+ if err != nil {
+ return nil, err
+ }
+ if len(index.Manifest.Manifests) == 0 {
+ return nil, fmt.Errorf("index.json contains no manifests")
+ }
+ if len(index.Manifest.Manifests) != 1 {
+ return nil, fmt.Errorf("index.json files containing multiple manifests are not supported")
+ }
+ manifestDescriptor := &index.Manifest.Manifests[0]
+
+ image, err := oci.AsImage(index.Ref(manifestDescriptor))
+ if err != nil {
+ return nil, err
+ }
+
+ // Create symlinks to blobs
+ descriptors := []ocispecv1.Descriptor{*manifestDescriptor, image.Manifest.Config}
+ descriptors = append(descriptors, image.Manifest.Layers...)
+ for _, descriptor := range descriptors {
+ if haveBlob[descriptor.Digest] {
+ continue
+ }
+ haveBlob[descriptor.Digest] = true
+
+ algorithm, encoded, err := oci.ParseDigest(string(descriptor.Digest))
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse digest: %w", err)
+ }
+ srcPath := filepath.Join(path, "blobs", algorithm, encoded)
+ destDir := filepath.Join(outPath, "blobs", algorithm)
+ destPath := filepath.Join(outPath, "blobs", algorithm, encoded)
+ relPath, err := filepath.Rel(destDir, srcPath)
+ if err != nil {
+ return nil, err
+ }
+ err = os.Symlink(relPath, destPath)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ return manifestDescriptor, nil
+}
+
+func main() {
+ var images []string
+ flag.Func("image", "OCI image path", func(path string) error {
+ images = append(images, path)
+ return nil
+ })
+ flag.Parse()
+
+ // Create blobs directory.
+ blobsPath := filepath.Join(*outPath, "blobs", "sha256")
+ err := os.MkdirAll(blobsPath, 0755)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ haveBlob := make(map[digest.Digest]bool)
+ index := ocispecv1.Index{
+ Versioned: ocispec.Versioned{SchemaVersion: 2},
+ MediaType: ocispecv1.MediaTypeImageIndex,
+ Manifests: []ocispecv1.Descriptor{},
+ }
+ for _, path := range images {
+ descriptor, err := addImage(*outPath, path, haveBlob)
+ if err != nil {
+ log.Fatalf("Failed to add image %q: %v", path, err)
+ }
+ index.Manifests = append(index.Manifests, *descriptor)
+ }
+
+ // Write the index manifest.
+ indexBytes, err := json.MarshalIndent(index, "", "\t")
+ if err != nil {
+ log.Fatalf("Failed to marshal index manifest: %v", err)
+ }
+ indexBytes = append(indexBytes, '\n')
+ indexHash := fmt.Sprintf("%x", sha256.Sum256(indexBytes))
+ err = os.WriteFile(filepath.Join(blobsPath, indexHash), indexBytes, 0644)
+ if err != nil {
+ log.Fatalf("Failed to write index manifest: %v", err)
+ }
+
+ // Write the entry-point index.
+ topIndex := ocispecv1.Index{
+ Versioned: ocispec.Versioned{SchemaVersion: 2},
+ MediaType: ocispecv1.MediaTypeImageIndex,
+ Manifests: []ocispecv1.Descriptor{{
+ MediaType: ocispecv1.MediaTypeImageIndex,
+ Digest: digest.NewDigestFromEncoded(digest.SHA256, indexHash),
+ Size: int64(len(indexBytes)),
+ }},
+ }
+ topIndexBytes, err := json.MarshalIndent(topIndex, "", "\t")
+ if err != nil {
+ log.Fatalf("Failed to marshal entry-point index: %v", err)
+ }
+ topIndexBytes = append(topIndexBytes, '\n')
+ err = os.WriteFile(filepath.Join(*outPath, "index.json"), topIndexBytes, 0644)
+ if err != nil {
+ log.Fatalf("Failed to write entry-point index: %v", err)
+ }
+
+ // Write the oci-layout marker file.
+ err = os.WriteFile(
+ filepath.Join(*outPath, "oci-layout"),
+ []byte(`{"imageLayoutVersion": "1.0.0"}`+"\n"),
+ 0644,
+ )
+ if err != nil {
+ log.Fatalf("Failed to write oci-layout file: %v", err)
+ }
+}