Make containerd work with read-only root
This makes containerd work with a read-only root. There were a few config mistakes on our side which
caused it to write to the rootfs (mostly leftovers from the switch to /ephemeral) and a semi-hardcoded path
in /var/lib/cni from containernetworking/cni. This is technically configurable, but it would require patching
three different repos (see diff message) and getting all of them to agree to take the change and wait for
it to propagate to all repos (containerd is known to be slow to release stuff). So let's just hack in
this one-line diff for the time being.
Test Plan: Should be covered by existing tests
X-Origin-Diff: phab/D694
GitOrigin-RevId: 0e8f5dbfb216539c16e64130af9fe1023722ae1b
diff --git a/build/fietsje/deps_containerd.go b/build/fietsje/deps_containerd.go
index 09db846..6154960 100644
--- a/build/fietsje/deps_containerd.go
+++ b/build/fietsje/deps_containerd.go
@@ -36,7 +36,6 @@
"github.com/containerd/imgcrypt",
"github.com/containers/ocicrypt",
"github.com/containerd/typeurl",
- "github.com/containernetworking/cni",
"github.com/coreos/go-systemd/v22",
"github.com/cpuguy83/go-md2man/v2",
"github.com/davecgh/go-spew",
@@ -86,10 +85,12 @@
"gopkg.in/yaml.v2",
"k8s.io/klog/v2",
"sigs.k8s.io/yaml",
+ ).with(disabledProtoBuild, patches("containerd-netns-statedir.patch")).use(
+ "github.com/containerd/cri",
).with(disabledProtoBuild).use(
"github.com/Microsoft/hcsshim",
"github.com/containerd/cgroups",
- "github.com/containerd/cri",
+
"github.com/gogo/googleapis",
).with(buildTags("selinux")).use(
"github.com/opencontainers/selinux",
@@ -98,7 +99,9 @@
"ttrpc-hacks.patch",
)).use(
"github.com/containerd/ttrpc",
- ).replace(
+ ).with(patches(
+ "cni-fix-cachepath.patch",
+ )).use("github.com/containernetworking/cni").replace(
// ttrpc is broken by go protobuf v2, this is a tentative PR that's
// not yet merged by upstream.
// See: https://github.com/containerd/ttrpc/pull/67
diff --git a/metropolis/node/BUILD.bazel b/metropolis/node/BUILD.bazel
index 96c075c..5c0934f 100644
--- a/metropolis/node/BUILD.bazel
+++ b/metropolis/node/BUILD.bazel
@@ -20,7 +20,7 @@
name = "initramfs",
extra_dirs = [
"/kubernetes/conf/flexvolume-plugins",
- "/containerd/run",
+ "/containerd/plugins",
],
files = {
"//metropolis/node/core": "/init",
diff --git a/metropolis/node/core/localstorage/storage.go b/metropolis/node/core/localstorage/storage.go
index 0f98209..110513c 100644
--- a/metropolis/node/core/localstorage/storage.go
+++ b/metropolis/node/core/localstorage/storage.go
@@ -49,6 +49,8 @@
Ephemeral EphemeralDirectory `dir:"ephemeral"`
// FHS-standard /tmp directory, used by ioutil.TempFile.
Tmp TmpDirectory `dir:"tmp"`
+ // FHS-standard /run directory. Used by various services.
+ Run RunDirectory `dir:"run"`
}
type PKIDirectory struct {
@@ -163,8 +165,17 @@
Tmp declarative.Directory `dir:"tmp"`
RunSC declarative.Directory `dir:"runsc"`
IPAM declarative.Directory `dir:"ipam"`
+ CNI declarative.Directory `dir:"cni"`
+ CNICache declarative.Directory `dir:"cni-cache"` // Hardcoded @com_github_containernetworking_cni via patch
}
type TmpDirectory struct {
declarative.Directory
}
+
+type RunDirectory struct {
+ declarative.Directory
+ // Hardcoded in @com_github_containerd_containerd//pkg/process:utils.go and
+ // @com_github_containerd_containerd//runtime/v2/shim:util_unix.go
+ Containerd declarative.Directory `dir:"containerd"`
+}
diff --git a/metropolis/node/kubernetes/containerd/cnispec.gojson b/metropolis/node/kubernetes/containerd/cnispec.gojson
index 0057036..d703ded 100644
--- a/metropolis/node/kubernetes/containerd/cnispec.gojson
+++ b/metropolis/node/kubernetes/containerd/cnispec.gojson
@@ -8,7 +8,7 @@
"mtu": 1420,
"ipam": {
"type": "host-local",
- "dataDir": "/containerd/run/ipam",
+ "dataDir": "/ephemeral/containerd/ipam",
"ranges": [
{{range $i, $range := .PodCIDRRanges}}{{if $i}},
{{end}}[
diff --git a/metropolis/node/kubernetes/containerd/config.toml b/metropolis/node/kubernetes/containerd/config.toml
index f8c7fb1..da2bed7 100644
--- a/metropolis/node/kubernetes/containerd/config.toml
+++ b/metropolis/node/kubernetes/containerd/config.toml
@@ -3,7 +3,7 @@
state = "/ephemeral/containerd"
plugin_dir = ""
disabled_plugins = []
-required_plugins = []
+required_plugins = ["io.containerd.grpc.v1.cri"]
oom_score = 0
[grpc]
@@ -95,7 +95,7 @@
base_runtime_spec = ""
[plugins."io.containerd.grpc.v1.cri".cni]
bin_dir = "/containerd/bin/cni"
- conf_dir = "/containerd/conf/cni"
+ conf_dir = "/ephemeral/containerd/cni"
max_conf_num = 0
conf_template = "/containerd/conf/cnispec.gojson"
[plugins."io.containerd.grpc.v1.cri".registry]
diff --git a/third_party/go/patches/cni-fix-cachepath.patch b/third_party/go/patches/cni-fix-cachepath.patch
new file mode 100644
index 0000000..65b30aa
--- /dev/null
+++ b/third_party/go/patches/cni-fix-cachepath.patch
@@ -0,0 +1,44 @@
+Copyright 2020 The Monogon Project Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+From 0b2583e76ac9f9675bbd539485918c96da830d21 Mon Sep 17 00:00:00 2001
+From: Lorenz Brun <lorenz@brun.one>
+Date: Mon, 25 Jan 2021 18:20:01 +0100
+Subject: [PATCH] Point CacheDir to the correct location for Metropolis
+
+This is arguably an ugly hack, but they hardcoded it and the fastest way to
+access anything resembling a config is through three different repos:
+containernetworking/cni -> containerd/go-cni -> containerd/cri ->
+containerd/containerd.
+---
+ libcni/api.go | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/libcni/api.go b/libcni/api.go
+index 7e52bd8..7f3dfe6 100644
+--- a/libcni/api.go
++++ b/libcni/api.go
+@@ -30,7 +30,7 @@ import (
+ )
+
+ var (
+- CacheDir = "/var/lib/cni"
++ CacheDir = "/ephemeral/containerd/cni-cache"
+ )
+
+ const (
+--
+2.25.1
+
diff --git a/third_party/go/patches/containerd-netns-statedir.patch b/third_party/go/patches/containerd-netns-statedir.patch
new file mode 100644
index 0000000..a693eb7
--- /dev/null
+++ b/third_party/go/patches/containerd-netns-statedir.patch
@@ -0,0 +1,96 @@
+Copyright 2020 The Monogon Project Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+From 3e7a8cebf9d40487adc7d4a22b5c628add5e7eac Mon Sep 17 00:00:00 2001
+From: Lorenz Brun <lorenz@nexantic.com>
+Date: Wed, 27 Jan 2021 13:05:30 +0100
+Subject: [PATCH] Move netns directory into StateDir
+
+---
+ pkg/netns/netns_unix.go | 12 +++++-------
+ pkg/server/sandbox_run.go | 3 ++-
+ 2 files changed, 7 insertions(+), 8 deletions(-)
+
+diff --git a/pkg/netns/netns_unix.go b/pkg/netns/netns_unix.go
+index 7449e235..b31716cb 100644
+--- a/pkg/netns/netns_unix.go
++++ b/pkg/netns/netns_unix.go
+@@ -48,14 +48,12 @@ import (
+ osinterface "github.com/containerd/cri/pkg/os"
+ )
+
+-const nsRunDir = "/var/run/netns"
+-
+ // Some of the following functions are migrated from
+ // https://github.com/containernetworking/plugins/blob/master/pkg/testutils/netns_linux.go
+
+ // newNS creates a new persistent (bind-mounted) network namespace and returns the
+ // path to the network namespace.
+-func newNS() (nsPath string, err error) {
++func newNS(baseDir string) (nsPath string, err error) {
+ b := make([]byte, 16)
+ if _, err := rand.Reader.Read(b); err != nil {
+ return "", errors.Wrap(err, "failed to generate random netns name")
+@@ -64,13 +62,13 @@ func newNS() (nsPath string, err error) {
+ // Create the directory for mounting network namespaces
+ // This needs to be a shared mountpoint in case it is mounted in to
+ // other namespaces (containers)
+- if err := os.MkdirAll(nsRunDir, 0755); err != nil {
++ if err := os.MkdirAll(baseDir, 0755); err != nil {
+ return "", err
+ }
+
+ // create an empty file at the mount point
+ nsName := fmt.Sprintf("cni-%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
+- nsPath = path.Join(nsRunDir, nsName)
++ nsPath = path.Join(baseDir, nsName)
+ mountPointFd, err := os.Create(nsPath)
+ if err != nil {
+ return "", err
+@@ -164,8 +162,8 @@ type NetNS struct {
+ }
+
+ // NewNetNS creates a network namespace.
+-func NewNetNS() (*NetNS, error) {
+- path, err := newNS()
++func NewNetNS(baseDir string) (*NetNS, error) {
++ path, err := newNS(baseDir)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to setup netns")
+ }
+diff --git a/pkg/server/sandbox_run.go b/pkg/server/sandbox_run.go
+index dd4c51e3..32a2d6e8 100644
+--- a/pkg/server/sandbox_run.go
++++ b/pkg/server/sandbox_run.go
+@@ -19,6 +19,7 @@ package server
+ import (
+ "encoding/json"
+ "math"
++ "path/filepath"
+ goruntime "runtime"
+ "strings"
+
+@@ -117,7 +118,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
+ // handle. NetNSPath in sandbox metadata and NetNS is non empty only for non host network
+ // namespaces. If the pod is in host network namespace then both are empty and should not
+ // be used.
+- sandbox.NetNS, err = netns.NewNetNS()
++ sandbox.NetNS, err = netns.NewNetNS(filepath.Join(c.config.StateDir, "netns"))
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to create network namespace for sandbox %q", id)
+ }
+--
+2.25.1
+
diff --git a/third_party/go/repositories.bzl b/third_party/go/repositories.bzl
index a517fe1..efab698 100644
--- a/third_party/go/repositories.bzl
+++ b/third_party/go/repositories.bzl
@@ -312,6 +312,10 @@
version = "v1.19.1-0.20201126003523-adc0b6a578ed",
sum = "h1:M2yIwrNSafh4rW/yXAiAlSqpydW7vjvDjZ0ClMb+EMQ=",
build_file_proto_mode = "disable",
+ patches = [
+ "//third_party/go/patches:containerd-netns-statedir.patch",
+ ],
+ patch_args = ["-p1"],
build_extra_args = [
"-go_naming_convention=go_default_library",
"-go_naming_convention_external=go_default_library",
@@ -383,6 +387,10 @@
importpath = "github.com/containernetworking/cni",
version = "v0.8.0",
sum = "h1:BT9lpgGoH4jw3lFC7Odz2prU5ruiYKcgAjMCbgybcKI=",
+ patches = [
+ "//third_party/go/patches:cni-fix-cachepath.patch",
+ ],
+ patch_args = ["-p1"],
build_extra_args = [
"-go_naming_convention=go_default_library",
"-go_naming_convention_external=go_default_library",