Make containerd work with read-only root
This makes containerd work with a read-only root. There were a few config mistakes on our side which
caused it to write to the rootfs (mostly leftovers from the switch to /ephemeral) and a semi-hardcoded path
in /var/lib/cni from containernetworking/cni. This is technically configurable, but it would require patching
three different repos (see diff message) and getting all of them to agree to take the change and wait for
it to propagate to all repos (containerd is known to be slow to release stuff). So let's just hack in
this one-line diff for the time being.
Test Plan: Should be covered by existing tests
X-Origin-Diff: phab/D694
GitOrigin-RevId: 0e8f5dbfb216539c16e64130af9fe1023722ae1b
diff --git a/metropolis/node/BUILD.bazel b/metropolis/node/BUILD.bazel
index 96c075c..5c0934f 100644
--- a/metropolis/node/BUILD.bazel
+++ b/metropolis/node/BUILD.bazel
@@ -20,7 +20,7 @@
name = "initramfs",
extra_dirs = [
"/kubernetes/conf/flexvolume-plugins",
- "/containerd/run",
+ "/containerd/plugins",
],
files = {
"//metropolis/node/core": "/init",
diff --git a/metropolis/node/core/localstorage/storage.go b/metropolis/node/core/localstorage/storage.go
index 0f98209..110513c 100644
--- a/metropolis/node/core/localstorage/storage.go
+++ b/metropolis/node/core/localstorage/storage.go
@@ -49,6 +49,8 @@
Ephemeral EphemeralDirectory `dir:"ephemeral"`
// FHS-standard /tmp directory, used by ioutil.TempFile.
Tmp TmpDirectory `dir:"tmp"`
+ // FHS-standard /run directory. Used by various services.
+ Run RunDirectory `dir:"run"`
}
type PKIDirectory struct {
@@ -163,8 +165,17 @@
Tmp declarative.Directory `dir:"tmp"`
RunSC declarative.Directory `dir:"runsc"`
IPAM declarative.Directory `dir:"ipam"`
+ CNI declarative.Directory `dir:"cni"`
+ CNICache declarative.Directory `dir:"cni-cache"` // Hardcoded @com_github_containernetworking_cni via patch
}
type TmpDirectory struct {
declarative.Directory
}
+
+type RunDirectory struct {
+ declarative.Directory
+ // Hardcoded in @com_github_containerd_containerd//pkg/process:utils.go and
+ // @com_github_containerd_containerd//runtime/v2/shim:util_unix.go
+ Containerd declarative.Directory `dir:"containerd"`
+}
diff --git a/metropolis/node/kubernetes/containerd/cnispec.gojson b/metropolis/node/kubernetes/containerd/cnispec.gojson
index 0057036..d703ded 100644
--- a/metropolis/node/kubernetes/containerd/cnispec.gojson
+++ b/metropolis/node/kubernetes/containerd/cnispec.gojson
@@ -8,7 +8,7 @@
"mtu": 1420,
"ipam": {
"type": "host-local",
- "dataDir": "/containerd/run/ipam",
+ "dataDir": "/ephemeral/containerd/ipam",
"ranges": [
{{range $i, $range := .PodCIDRRanges}}{{if $i}},
{{end}}[
diff --git a/metropolis/node/kubernetes/containerd/config.toml b/metropolis/node/kubernetes/containerd/config.toml
index f8c7fb1..da2bed7 100644
--- a/metropolis/node/kubernetes/containerd/config.toml
+++ b/metropolis/node/kubernetes/containerd/config.toml
@@ -3,7 +3,7 @@
state = "/ephemeral/containerd"
plugin_dir = ""
disabled_plugins = []
-required_plugins = []
+required_plugins = ["io.containerd.grpc.v1.cri"]
oom_score = 0
[grpc]
@@ -95,7 +95,7 @@
base_runtime_spec = ""
[plugins."io.containerd.grpc.v1.cri".cni]
bin_dir = "/containerd/bin/cni"
- conf_dir = "/containerd/conf/cni"
+ conf_dir = "/ephemeral/containerd/cni"
max_conf_num = 0
conf_template = "/containerd/conf/cnispec.gojson"
[plugins."io.containerd.grpc.v1.cri".registry]