Implement image preseeding
This pulls in the infrastructure to build OCI bundles with Bazel and adds a loader to
load them into containerd at runtime.
Test Plan: New E2E test using a simple hello world Go image.
Bug: T793
X-Origin-Diff: phab/D585
GitOrigin-RevId: 3bc5e35a89a80a9683778ced72cc79e2d0b684ed
diff --git a/core/BUILD b/core/BUILD
index 5328671..ea4b70f 100644
--- a/core/BUILD
+++ b/core/BUILD
@@ -38,6 +38,9 @@
"//core/internal/containerd:config.toml": "/containerd/conf/config.toml",
"//core/internal/containerd:cnispec.gojson": "/containerd/conf/cnispec.gojson",
+ # Containerd preseed bundles
+ "//core/tests/e2e/preseedtest:preseedtest.tar": "/containerd/preseed/k8s.io/preseedtest.tar",
+
# CNI Plugins
"@com_github_containernetworking_plugins//plugins/main/loopback": "/containerd/bin/cni/loopback",
"@com_github_containernetworking_plugins//plugins/main/ptp": "/containerd/bin/cni/ptp",
diff --git a/core/internal/containerd/BUILD.bazel b/core/internal/containerd/BUILD.bazel
index 77262b5..16269d1 100644
--- a/core/internal/containerd/BUILD.bazel
+++ b/core/internal/containerd/BUILD.bazel
@@ -6,8 +6,12 @@
importpath = "git.monogon.dev/source/nexantic.git/core/internal/containerd",
visibility = ["//core:__subpackages__"],
deps = [
+ "//core/internal/common/supervisor:go_default_library",
"//core/internal/localstorage:go_default_library",
"//core/pkg/logbuffer:go_default_library",
+ "@com_github_containerd_containerd//:go_default_library",
+ "@com_github_containerd_containerd//namespaces:go_default_library",
+ "@org_uber_go_zap//:go_default_library",
],
)
diff --git a/core/internal/containerd/main.go b/core/internal/containerd/main.go
index 289efe7..0c6d497 100644
--- a/core/internal/containerd/main.go
+++ b/core/internal/containerd/main.go
@@ -20,19 +20,30 @@
"context"
"fmt"
"io"
+ "io/ioutil"
"os"
"os/exec"
+ "path/filepath"
"time"
+ ctr "github.com/containerd/containerd"
+ "github.com/containerd/containerd/namespaces"
+ "go.uber.org/zap"
+
"git.monogon.dev/source/nexantic.git/core/internal/localstorage"
+
+ "git.monogon.dev/source/nexantic.git/core/internal/common/supervisor"
"git.monogon.dev/source/nexantic.git/core/pkg/logbuffer"
)
+const (
+ preseedNamespacesDir = "/containerd/preseed/"
+)
+
type Service struct {
EphemeralVolume *localstorage.EphemeralContainerdDirectory
-
- Log *logbuffer.LogBuffer
- RunscLog *logbuffer.LogBuffer
+ Log *logbuffer.LogBuffer
+ RunscLog *logbuffer.LogBuffer
}
func (s *Service) Run(ctx context.Context) error {
@@ -43,6 +54,8 @@
s.RunscLog = logbuffer.New(5000, 16384)
}
+ logger := supervisor.Logger(ctx)
+
cmd := exec.CommandContext(ctx, "/containerd/bin/containerd", "--config", "/containerd/conf/config.toml")
cmd.Stdout = s.Log
cmd.Stderr = s.Log
@@ -62,16 +75,76 @@
// debug logs) is not an issue for us.
time.Sleep(10 * time.Millisecond)
} else if err != nil {
- // TODO: Use supervisor.Logger() and Error() before exiting. Should never happen.
- fmt.Println(err)
+ logger.Error("gVisor log pump failed, stopping it", zap.Error(err))
return // It's likely that this will busy-loop printing errors if it encounters one, so bail
}
}
}()
- // TODO(lorenz): Healthcheck against CRI RuntimeService.Status() and SignalHealthy
+ if err := supervisor.Run(ctx, "preseed", s.runPreseed); err != nil {
+ return fmt.Errorf("failed to start preseed runnable: %w", err)
+ }
+ supervisor.Signal(ctx, supervisor.SignalHealthy)
err = cmd.Run()
fmt.Fprintf(s.Log, "containerd stopped: %v\n", err)
return err
}
+
+// runPreseed loads OCI bundles in tar form from preseedNamespacesDir into containerd at startup.
+// This can be run multiple times, containerd will automatically dedup the layers.
+// containerd uses namespaces to keep images (and everything else) separate so to define where the images will be loaded
+// to they need to be in a folder named after the namespace they should be loaded into.
+// containerd's CRI plugin (which is built as part of containerd) uses a hardcoded namespace ("k8s.io") for everything
+// accessed through CRI, so if an image should be available on K8s it needs to be in that namespace.
+// As an example if image helloworld should be loaded for use with Kubernetes, the OCI bundle needs to be at
+// <preseedNamespacesDir>/k8s.io/helloworld.tar. No tagging beyond what's in the bundle is performed.
+func (s *Service) runPreseed(ctx context.Context) error {
+ client, err := ctr.New(s.EphemeralVolume.ClientSocket.FullPath())
+ if err != nil {
+ return fmt.Errorf("failed to connect to containerd: %w", err)
+ }
+ logger := supervisor.Logger(ctx)
+ preseedNamespaceDirs, err := ioutil.ReadDir(preseedNamespacesDir)
+ if err != nil {
+ return fmt.Errorf("failed to open preseed dir: %w", err)
+ }
+ for _, dir := range preseedNamespaceDirs {
+ if !dir.IsDir() {
+ logger.Warn("Non-Directory found in preseed folder, ignoring", zap.String("name", dir.Name()))
+ continue
+ }
+ namespace := dir.Name()
+ images, err := ioutil.ReadDir(filepath.Join(preseedNamespacesDir, namespace))
+ if err != nil {
+ return fmt.Errorf("failed to list namespace preseed directory for ns \"%v\": %w", namespace, err)
+ }
+ ctxWithNS := namespaces.WithNamespace(ctx, namespace)
+ for _, image := range images {
+ if image.IsDir() {
+ logger.Warn("Directory found in preseed namespaced folder, ignoring", zap.String("name", image.Name()))
+ continue
+ }
+ imageFile, err := os.Open(filepath.Join(preseedNamespacesDir, namespace, image.Name()))
+ if err != nil {
+ return fmt.Errorf("failed to open preseed image \"%v\": %w", image.Name(), err)
+ }
+ // defer in this loop is fine since we're never going to preseed more than ~1M images which is where our
+ // file descriptor limit is.
+ defer imageFile.Close()
+ importedImages, err := client.Import(ctxWithNS, imageFile)
+ if err != nil {
+ return fmt.Errorf("failed to import preseed image: %w", err)
+ }
+ var importedImageNames []string
+ for _, img := range importedImages {
+ importedImageNames = append(importedImageNames, img.Name)
+ }
+ logger.Info("Successfully imported preseeded bundle into containerd",
+ zap.String("namespace", namespace), zap.Strings("images", importedImageNames))
+ }
+ }
+ supervisor.Signal(ctx, supervisor.SignalHealthy)
+ supervisor.Signal(ctx, supervisor.SignalDone)
+ return nil
+}
diff --git a/core/tests/e2e/main_test.go b/core/tests/e2e/main_test.go
index faae520..99cfdff 100644
--- a/core/tests/e2e/main_test.go
+++ b/core/tests/e2e/main_test.go
@@ -171,6 +171,37 @@
return fmt.Errorf("pod is not ready: %v", events.Items[0].Message)
}
})
+ testEventual(t, "Pod with preseeded image", ctx, smallTestTimeout, func(ctx context.Context) error {
+ _, err := clientSet.CoreV1().Pods("default").Create(ctx, &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "preseed-test-1",
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{{
+ Name: "preseed-test-1",
+ ImagePullPolicy: corev1.PullNever,
+ Image: "bazel/core/tests/e2e/preseedtest:preseedtest",
+ }},
+ RestartPolicy: corev1.RestartPolicyNever,
+ },
+ }, metav1.CreateOptions{})
+ return err
+ })
+ testEventual(t, "Pod with preseeded image is completed", ctx, largeTestTimeout, func(ctx context.Context) error {
+ pod, err := clientSet.CoreV1().Pods("default").Get(ctx, "preseed-test-1", metav1.GetOptions{})
+ if err != nil {
+ return fmt.Errorf("failed to get pod: %w", err)
+ }
+ if pod.Status.Phase == corev1.PodSucceeded {
+ return nil
+ }
+ events, err := clientSet.CoreV1().Events("default").List(ctx, metav1.ListOptions{FieldSelector: fmt.Sprintf("involvedObject.name=%s,involvedObject.namespace=default", pod.Name)})
+ if err != nil || len(events.Items) == 0 {
+ return fmt.Errorf("pod is not ready: %v", pod.Status.Phase)
+ } else {
+ return fmt.Errorf("pod is not ready: %v", events.Items[len(events.Items)-1].Message)
+ }
+ })
})
})
diff --git a/core/tests/e2e/preseedtest/BUILD.bazel b/core/tests/e2e/preseedtest/BUILD.bazel
new file mode 100644
index 0000000..9536846
--- /dev/null
+++ b/core/tests/e2e/preseedtest/BUILD.bazel
@@ -0,0 +1,16 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_docker//go:image.bzl", "go_image")
+
+go_library(
+ name = "go_default_library",
+ srcs = ["main.go"],
+ importpath = "git.monogon.dev/source/nexantic.git/core/tests/e2e/preseedtest",
+ visibility = ["//visibility:private"],
+)
+
+go_image(
+ name = "preseedtest",
+ embed = [":go_default_library"],
+ pure = "on",
+ visibility = ["//visibility:public"],
+)
diff --git a/core/tests/e2e/preseedtest/main.go b/core/tests/e2e/preseedtest/main.go
new file mode 100644
index 0000000..ceb3898
--- /dev/null
+++ b/core/tests/e2e/preseedtest/main.go
@@ -0,0 +1,23 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import "fmt"
+
+func main() {
+ fmt.Println("Hello world from preseeded image")
+}