Add Kubernetes Worker and infrastructure
Adds Kubernetes Kubelet with patches for syscall-based mounting and
syscall-based (and much faster) metrics. fsquota patches have been
deferred to a further revision (for robust emptyDir capacity isolation).
Changes encoding of the node ID to hex since Base64-URL is not supported
as a character set for K8s names. Also adds `/etc/machine-id` and
`/etc/os-release` since Kubernetes wants them. `os-release` is generated
by stamping, `machine-id` is the hex-encoded node ID derived from the
public key.
Also includes a primitive reconciler which automatically ensures a set of
built-in Kubernetes objects are always present. Currently this includes
a PSP and some basic RBAC policies that are elementary to proper cluster
operations.
Adds an additional gRPC service (NodeDebugService) to cleanly
communicate with external debug and test tooling. It supports reading
from logbuffers for all externally-run components, checking conditions
(for replacing log matching in testing and debugging) and getting
debug credentials for the Kubernetes cluster.
A small utility (dbg) is provided that interfaces with NodeDebugService
and provides access to its functions from the CLI. It also incorporates
a kubectl wrapper which directly grabs credentials from the Debug API
and passes them to kubectl
(e.g. `bazel run //core/cmd/dbg -- kubectl describe node`).
Test Plan:
Manually tested.
Kubernetes:
`bazel run //core/cmd/dbg -- kubectl create -f test.yml`
Checked that pods run, logs are accessible and exec works.
Reading buffers:
`bazel run //core/cmd/dbg -- logs containerd`
Outputs containerd logs in the right order.
Automated testing is in the works, but has been deferred to a future
revision because this one is already too big again.
X-Origin-Diff: phab/D525
GitOrigin-RevId: 0fbfa0c433de405526c7f09ef10c466896331328
diff --git a/BUILD b/BUILD
index 1575183..8da7909 100644
--- a/BUILD
+++ b/BUILD
@@ -3,7 +3,6 @@
load("@io_bazel_rules_go//go:def.bzl", "go_path", "nogo")
# gazelle:prefix git.monogon.dev/source/nexantic.git
-# gazelle:exclude core/cmd/kube-controlplane
gazelle(name = "gazelle")
fietsje(name = "fietsje")
diff --git a/build/fietsje/deps_kubernetes.go b/build/fietsje/deps_kubernetes.go
index 5fa623c..76a1f24 100644
--- a/build/fietsje/deps_kubernetes.go
+++ b/build/fietsje/deps_kubernetes.go
@@ -22,12 +22,19 @@
"k8s.io/kubernetes", "v1.19.0-alpha.2",
buildTags("providerless"),
disabledProtoBuild,
- patches("k8s-kubernetes.patch", "k8s-kubernetes-build.patch"),
+ patches(
+ "k8s-kubernetes.patch",
+ "k8s-kubernetes-build.patch",
+ "k8s-native-metrics.patch",
+ "k8s-use-native.patch",
+ ),
).inject(
// repo infra, not requested by k8s, but used with bazel
"k8s.io/repo-infra", "df02ded38f9506e5bbcbf21702034b4fef815f2f",
).with(patches("k8s-client-go.patch", "k8s-client-go-build.patch")).use(
"k8s.io/client-go",
+ ).with(patches("k8s-native-mounter.patch")).use(
+ "k8s.io/utils",
).use(
"k8s.io/cli-runtime",
"k8s.io/client-go",
@@ -137,7 +144,6 @@
"k8s.io/gengo",
"k8s.io/heapster",
"k8s.io/kube-openapi",
- "k8s.io/utils",
"sigs.k8s.io/apiserver-network-proxy/konnectivity-client",
"sigs.k8s.io/kustomize",
"sigs.k8s.io/structured-merge-diff/v3",
diff --git a/build/fietsje/deps_sqlboiler.go b/build/fietsje/deps_sqlboiler.go
index 3b167a8..dd0cb22 100644
--- a/build/fietsje/deps_sqlboiler.go
+++ b/build/fietsje/deps_sqlboiler.go
@@ -52,6 +52,7 @@
"github.com/mattn/go-sqlite3",
"github.com/mitchellh/cli",
"github.com/posener/complete",
+ "github.com/joho/godotenv",
"gopkg.in/gorp.v1",
)
}
diff --git a/build/fietsje/main.go b/build/fietsje/main.go
index 373080a..7aa020b 100644
--- a/build/fietsje/main.go
+++ b/build/fietsje/main.go
@@ -89,6 +89,7 @@
p.collect("github.com/diskfs/go-diskfs", "v1.0.0").use(
"gopkg.in/djherbis/times.v1",
)
+
// used by //build/bindata
p.collect("github.com/kevinburke/go-bindata", "v3.16.0")
diff --git a/build/print-workspace-status.sh b/build/print-workspace-status.sh
index bf098d0..fcb38b3 100755
--- a/build/print-workspace-status.sh
+++ b/build/print-workspace-status.sh
@@ -1,12 +1,13 @@
#!/usr/bin/env bash
# Workspace status used for build stamping.
-# This is currently only consumed by //third_party/go/kubernetes_version_def.bzl.
-
set -o errexit
set -o nounset
set -o pipefail
+# TODO: Figure out how to version Smalltown
+SIGNOS_VERSION=1.0.0-dev
+
KUBERNETES_gitTreeState="clean"
if [ ! -z "$(git status --porcelain)" ]; then
KUBERNETES_gitTreeState="dirty"
@@ -26,4 +27,5 @@
KUBERNETES_buildDate $(date \
${SOURCE_DATE_EPOCH:+"--date=@${SOURCE_DATE_EPOCH}"} \
-u +'%Y-%m-%dT%H:%M:%SZ')
+STABLE_SIGNOS_version $SIGNOS_VERSION
EOF
diff --git a/core/BUILD b/core/BUILD
index 4df9777..ef4b86a 100644
--- a/core/BUILD
+++ b/core/BUILD
@@ -2,7 +2,7 @@
name = "initramfs",
srcs = [
"//core/cmd/init",
- "//core/cmd/kube-controlplane",
+ "//core/cmd/kube",
"//third_party/xfsprogs:mkfs.xfs",
"@io_k8s_kubernetes//cmd/kubelet:_kubelet-pure",
"@com_github_containerd_containerd//cmd/containerd",
@@ -18,6 +18,7 @@
"//core/internal/containerd:config.toml",
"//core/internal/containerd:runsc.toml",
"@cacerts//file",
+ ":os-release-info",
],
outs = [
"initramfs.cpio.lz4",
@@ -27,15 +28,20 @@
dir /dev 0755 0 0
nod /dev/console 0600 0 0 c 5 1
nod /dev/null 0644 0 0 c 1 3
+nod /dev/kmsg 0644 0 0 c 1 11
nod /dev/ptmx 0644 0 0 c 5 2
file /init $(location //core/cmd/init) 0755 0 0
dir /etc 0755 0 0
+file /etc/os-release $(location :os-release-info) 0644 0 0
dir /etc/ssl 0755 0 0
file /etc/ssl/cert.pem $(location @cacerts//file) 0444 0 0
dir /bin 0755 0 0
file /bin/mkfs.xfs $(location //third_party/xfsprogs:mkfs.xfs) 0755 0 0
-file /bin/kube-controlplane $(location //core/cmd/kube-controlplane) 0755 0 0
-file /bin/kubelet $(location @io_k8s_kubernetes//cmd/kubelet:_kubelet-pure) 0755 0 0
+dir /kubernetes 0755 0 0
+dir /kubernetes/bin 0755 0 0
+file /kubernetes/bin/kube $(location //core/cmd/kube) 0755 0 0
+dir /kubernetes/conf 0755 0 0
+dir /kubernetes/conf/flexvolume-plugins 0755 0 0
dir /containerd 0755 0 0
dir /containerd/bin 0755 0 0
file /containerd/bin/containerd $(location @com_github_containerd_containerd//cmd/containerd) 0755 0 0
@@ -119,3 +125,12 @@
""",
visibility = ["//visibility:public"],
)
+
+load("//core/build/genosrelease:defs.bzl", "os_release")
+
+os_release(
+ name = "os-release-info",
+ os_id = "smalltown",
+ os_name = "Smalltown",
+ stamp_var = "STABLE_SIGNOS_version",
+)
diff --git a/core/api/api/schema.proto b/core/api/api/schema.proto
index 3f1b7e3..d614740 100644
--- a/core/api/api/schema.proto
+++ b/core/api/api/schema.proto
@@ -81,6 +81,47 @@
}
}
+message GetDebugKubeconfigRequest {
+ string id = 1; // Kubernetes identity (user)
+ repeated string groups = 2; // Kubernetes groups
+}
+
+message GetDebugKubeconfigResponse {
+ string debug_kubeconfig = 1;
+}
+
+message GetComponentLogsRequest {
+ // For supported paths see core/internal/node/debug.go
+ repeated string component_path = 1;
+ uint32 tail_lines = 2; // 0 = whole ring buffer
+}
+
+message GetComponentLogsResponse {
+ repeated string line = 1;
+}
+
+message GetConditionRequest {
+ string name = 1;
+}
+
+message GetConditionResponse {
+ bool ok = 1;
+}
+// NodeDebugService exposes debug and testing endpoints that allow introspection into a running Smalltown instance.
+// It is not authenticated and will be disabled in production. It is currently consumed by core/cmd/dbg and
+// by tests. For exact documentation of the available parameters please look at core/internal/node/debug.go.
+service NodeDebugService {
+ // GetDebugKubeconfig issues kubeconfigs with arbitrary identities and groups for debugging
+ rpc GetDebugKubeconfig(GetDebugKubeconfigRequest) returns (GetDebugKubeconfigResponse) {
+ }
+ // GetComponentLogs dumps various log ringbuffers for binaries that we run.
+ rpc GetComponentLogs(GetComponentLogsRequest) returns (GetComponentLogsResponse) {
+ }
+ // GetCondition gives the current status of various conditions inside Smalltown. Mainly used for testing.
+ rpc GetCondition(GetConditionRequest) returns (GetConditionResponse) {
+ }
+}
+
// NodeManagementService runs on all masters, is identified by the
// NodeManagementService TLS certificate. It is used by nodes to
// initially join the cluster or subsequently request the cluster unlock secret
diff --git a/core/build/genosrelease/BUILD.bazel b/core/build/genosrelease/BUILD.bazel
new file mode 100644
index 0000000..4cdc2ee
--- /dev/null
+++ b/core/build/genosrelease/BUILD.bazel
@@ -0,0 +1,15 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+ name = "go_default_library",
+ srcs = ["main.go"],
+ importpath = "git.monogon.dev/source/nexantic.git/core/build/genosrelease",
+ visibility = ["//visibility:private"],
+ deps = ["@com_github_joho_godotenv//:go_default_library"],
+)
+
+go_binary(
+ name = "genosrelease",
+ embed = [":go_default_library"],
+ visibility = ["//visibility:public"],
+)
diff --git a/core/build/genosrelease/defs.bzl b/core/build/genosrelease/defs.bzl
new file mode 100644
index 0000000..d6190d9
--- /dev/null
+++ b/core/build/genosrelease/defs.bzl
@@ -0,0 +1,54 @@
+# Copyright 2020 The Monogon Project Authors.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+def _os_release_impl(ctx):
+ ctx.actions.run(
+ mnemonic = "GenOSRelease",
+ progress_message = "Generating os-release",
+ inputs = [ctx.info_file],
+ outputs = [ctx.outputs.out],
+ executable = ctx.executable._genosrelease,
+ arguments = [
+ "-status_file",
+ ctx.info_file.path,
+ "-out_file",
+ ctx.outputs.out.path,
+ "-stamp_var",
+ ctx.attr.stamp_var,
+ "-name",
+ ctx.attr.os_name,
+ "-id",
+ ctx.attr.os_id,
+ ],
+ )
+
+os_release = rule(
+ implementation = _os_release_impl,
+ attrs = {
+ "os_name": attr.string(mandatory = True),
+ "os_id": attr.string(mandatory = True),
+ "stamp_var": attr.string(mandatory = True),
+ "_genosrelease": attr.label(
+ default = Label("//core/build/genosrelease"),
+ cfg = "host",
+ executable = True,
+ allow_files = True,
+ ),
+ },
+ outputs = {
+ "out": "os-release",
+ },
+)
diff --git a/core/build/genosrelease/main.go b/core/build/genosrelease/main.go
new file mode 100644
index 0000000..2344f19
--- /dev/null
+++ b/core/build/genosrelease/main.go
@@ -0,0 +1,78 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// genosrelease provides rudimentary support to generate os-release files following the freedesktop spec
+// (https://www.freedesktop.org/software/systemd/man/os-release.html) from arguments and stamping
+package main
+
+import (
+ "flag"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "strings"
+
+ "github.com/joho/godotenv"
+)
+
+var (
+ flagStatusFile = flag.String("status_file", "", "path to bazel workspace status file")
+ flagOutFile = flag.String("out_file", "os-release", "path to os-release output file")
+ flagStampVar = flag.String("stamp_var", "", "variable to use as version from the workspace status file")
+ flagName = flag.String("name", "", "name parameter (see freedesktop spec)")
+ flagID = flag.String("id", "", "id parameter (see freedesktop spec)")
+)
+
+func main() {
+ flag.Parse()
+ statusFileContent, err := ioutil.ReadFile(*flagStatusFile)
+ if err != nil {
+ fmt.Printf("Failed to open bazel workspace status file: %v\n", err)
+ os.Exit(1)
+ }
+ statusVars := make(map[string]string)
+ for _, line := range strings.Split(string(statusFileContent), "\n") {
+ line = strings.TrimSpace(line)
+ parts := strings.Fields(line)
+ if len(parts) != 2 {
+ continue
+ }
+ statusVars[parts[0]] = parts[1]
+ }
+
+ smalltownVersion, ok := statusVars[*flagStampVar]
+ if !ok {
+ fmt.Printf("%v key not set in bazel workspace status file\n", *flagStampVar)
+ os.Exit(1)
+ }
+ // As specified by https://www.freedesktop.org/software/systemd/man/os-release.html
+ osReleaseVars := map[string]string{
+ "NAME": *flagName,
+ "ID": *flagID,
+ "VERSION": smalltownVersion,
+ "VERSION_ID": smalltownVersion,
+ "PRETTY_NAME": *flagName + " " + smalltownVersion,
+ }
+ osReleaseContent, err := godotenv.Marshal(osReleaseVars)
+ if err != nil {
+ fmt.Printf("Failed to encode os-release file: %v\n", err)
+ os.Exit(1)
+ }
+ if err := ioutil.WriteFile(*flagOutFile, []byte(osReleaseContent), 0644); err != nil {
+ fmt.Printf("Failed to write os-release file: %v\n", err)
+ os.Exit(1)
+ }
+}
diff --git a/core/cmd/dbg/BUILD.bazel b/core/cmd/dbg/BUILD.bazel
new file mode 100644
index 0000000..7088ea1
--- /dev/null
+++ b/core/cmd/dbg/BUILD.bazel
@@ -0,0 +1,23 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+ name = "go_default_library",
+ srcs = ["main.go"],
+ importpath = "git.monogon.dev/source/nexantic.git/core/cmd/dbg",
+ visibility = ["//visibility:private"],
+ deps = [
+ "//core/api/api:go_default_library",
+ "@com_github_spf13_pflag//:go_default_library",
+ "@io_k8s_component_base//cli/flag:go_default_library",
+ "@io_k8s_kubectl//pkg/cmd/plugin:go_default_library",
+ "@io_k8s_kubectl//pkg/util/logs:go_default_library",
+ "@io_k8s_kubernetes//pkg/kubectl/cmd:go_default_library",
+ "@org_golang_google_grpc//:go_default_library",
+ ],
+)
+
+go_binary(
+ name = "dbg",
+ embed = [":go_default_library"],
+ visibility = ["//visibility:public"],
+)
diff --git a/core/cmd/dbg/main.go b/core/cmd/dbg/main.go
new file mode 100644
index 0000000..44803ec
--- /dev/null
+++ b/core/cmd/dbg/main.go
@@ -0,0 +1,125 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "io/ioutil"
+ "math/rand"
+ "os"
+ "strings"
+ "time"
+
+ "github.com/spf13/pflag"
+ "google.golang.org/grpc"
+ cliflag "k8s.io/component-base/cli/flag"
+ "k8s.io/kubectl/pkg/cmd/plugin"
+ "k8s.io/kubectl/pkg/util/logs"
+ "k8s.io/kubernetes/pkg/kubectl/cmd"
+
+ apipb "git.monogon.dev/source/nexantic.git/core/generated/api"
+)
+
+func main() {
+ // Hardcode localhost since this should never be used to interface with a production node because of missing
+ // encryption & authentication
+ grpcClient, err := grpc.Dial("localhost:7837", grpc.WithInsecure())
+ if err != nil {
+ fmt.Printf("Failed to dial debug service (is it running): %v\n", err)
+ }
+ debugClient := apipb.NewNodeDebugServiceClient(grpcClient)
+ if len(os.Args) < 2 {
+ fmt.Println("Please specify a subcommand")
+ os.Exit(1)
+ }
+
+ logsCmd := flag.NewFlagSet("logs", flag.ExitOnError)
+ logsTailN := logsCmd.Uint("tail", 0, "Get last n lines (0 = whole buffer)")
+ logsCmd.Usage = func() {
+ fmt.Fprintf(os.Stderr, "Usage: %s %s [options] component_path\n", os.Args[0], os.Args[1])
+ flag.PrintDefaults()
+
+ fmt.Fprintf(os.Stderr, "Example:\n %s %s --tail 5 kube.apiserver\n", os.Args[0], os.Args[1])
+ }
+ conditionCmd := flag.NewFlagSet("condition", flag.ExitOnError)
+ conditionCmd.Usage = func() {
+ fmt.Fprintf(os.Stderr, "Usage: %s %s [options] component_path\n", os.Args[0], os.Args[1])
+ flag.PrintDefaults()
+
+ fmt.Fprintf(os.Stderr, "Example:\n %s %s IPAssigned\n", os.Args[0], os.Args[1])
+ }
+ switch os.Args[1] {
+ case "logs":
+ logsCmd.Parse(os.Args[2:])
+ componentPath := strings.Split(logsCmd.Arg(0), ".")
+ res, err := debugClient.GetComponentLogs(context.Background(), &apipb.GetComponentLogsRequest{ComponentPath: componentPath, TailLines: uint32(*logsTailN)})
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Failed to get logs: %v\n", err)
+ os.Exit(1)
+ }
+ for _, line := range res.Line {
+ fmt.Println(line)
+ }
+ return
+ case "condition":
+ conditionCmd.Parse(os.Args[2:])
+ condition := conditionCmd.Arg(0)
+ res, err := debugClient.GetCondition(context.Background(), &apipb.GetConditionRequest{Name: condition})
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Failed to get condition: %v\n", err)
+ os.Exit(1)
+ }
+ fmt.Println(res.Ok)
+ case "kubectl":
+ // Always get a kubeconfig with cluster-admin (group system:masters), kubectl itself can impersonate
+ kubeconfigFile, err := ioutil.TempFile("", "dbg_kubeconfig")
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Failed to create kubeconfig temp file: %v\n", err)
+ os.Exit(1)
+ }
+ defer kubeconfigFile.Close()
+ defer os.Remove(kubeconfigFile.Name())
+
+ res, err := debugClient.GetDebugKubeconfig(context.Background(), &apipb.GetDebugKubeconfigRequest{Id: "debug-user", Groups: []string{"system:masters"}})
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Failed to get kubeconfig: %v\n", err)
+ os.Exit(1)
+ }
+ if _, err := kubeconfigFile.WriteString(res.DebugKubeconfig); err != nil {
+ fmt.Fprintf(os.Stderr, "Failed to write kubeconfig: %v\n", err)
+ os.Exit(1)
+ }
+
+ // This magic sets up everything as if this was just the kubectl binary. It sets the KUBECONFIG environment
+ // variable so that it knows where the Kubeconfig is located and forcibly overwrites the arguments so that
+ // the "wrapper" arguments are not visible to its flags parser. The base code is straight from
+ // https://github.com/kubernetes/kubernetes/blob/master/cmd/kubectl/kubectl.go
+ os.Setenv("KUBECONFIG", kubeconfigFile.Name())
+ rand.Seed(time.Now().UnixNano())
+ pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)
+ pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
+ logs.InitLogs()
+ defer logs.FlushLogs()
+ command := cmd.NewDefaultKubectlCommandWithArgs(cmd.NewDefaultPluginHandler(plugin.ValidPluginFilenamePrefixes), os.Args[2:], os.Stdin, os.Stdout, os.Stderr)
+ command.SetArgs(os.Args[2:])
+ if err := command.Execute(); err != nil {
+ os.Exit(1)
+ }
+ }
+}
diff --git a/core/cmd/init/main.go b/core/cmd/init/main.go
index d1e2f87..42770a9 100644
--- a/core/cmd/init/main.go
+++ b/core/cmd/init/main.go
@@ -62,6 +62,11 @@
panic(fmt.Errorf("could not remount root: %w", err))
}
+ // Linux kernel default is 4096 which is far too low. Raise it to 1M which is what gVisor suggests.
+ if err := unix.Setrlimit(unix.RLIMIT_NOFILE, &unix.Rlimit{Cur: 1048576, Max: 1048576}); err != nil {
+ logger.Panic("Failed to raise rlimits", zap.Error(err))
+ }
+
logger.Info("Starting Smalltown Init")
signalChannel := make(chan os.Signal, 2)
diff --git a/core/cmd/kube-controlplane/BUILD b/core/cmd/kube/BUILD
similarity index 61%
rename from core/cmd/kube-controlplane/BUILD
rename to core/cmd/kube/BUILD
index c9049e4..b1a22fe 100644
--- a/core/cmd/kube-controlplane/BUILD
+++ b/core/cmd/kube/BUILD
@@ -1,43 +1,26 @@
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
-# This is a hack to make go modules ignore all of kube-controlplane since Kubernetes is not importable
-# and we still need normal go mod tooling to work. Instead we're just depending on our own Kubernetes
-# which is already being built with Bazel and thus works fine as a dependency.
-
-genrule(
- name = "hack_ignore",
- srcs = [
- "main.go",
- ],
- outs = [
- "main_patched.go",
- ],
- cmd = """
- sed '/+build ignore/d' $(location main.go) > "$@"
- """,
- visibility = ["//visibility:public"],
-)
-
go_library(
name = "go_default_library",
- srcs = [":main_patched.go"],
- importpath = "git.monogon.dev/source/nexantic.git/core/cmd/kubemaster",
+ srcs = ["main.go"],
+ importpath = "git.monogon.dev/source/nexantic.git/core/cmd/kube",
visibility = ["//visibility:private"],
deps = [
- "@io_k8s_kubernetes//cmd/kube-apiserver/app:go_default_library",
- "@io_k8s_kubernetes//cmd/kube-controller-manager/app:go_default_library",
- "@io_k8s_kubernetes//cmd/kube-scheduler/app:go_default_library",
+ "@com_github_spf13_cobra//:go_default_library",
+ "@com_github_spf13_pflag//:go_default_library",
"@io_k8s_component_base//cli/flag:go_default_library",
"@io_k8s_component_base//logs:go_default_library",
"@io_k8s_component_base//metrics/prometheus/restclient:go_default_library",
"@io_k8s_component_base//metrics/prometheus/version:go_default_library",
- "@com_github_spf13_cobra//:go_default_library",
- "@com_github_spf13_pflag//:go_default_library",
+ "@io_k8s_kubernetes//cmd/kube-apiserver/app:go_default_library",
+ "@io_k8s_kubernetes//cmd/kube-controller-manager/app:go_default_library",
+ "@io_k8s_kubernetes//cmd/kube-scheduler/app:go_default_library",
+ "@io_k8s_kubernetes//cmd/kubelet/app:go_default_library",
],
)
go_binary(
- name = "kube-controlplane",
+ name = "kube",
embed = [":go_default_library"],
pure = "on",
visibility = ["//visibility:public"],
diff --git a/core/cmd/kube-controlplane/main.go b/core/cmd/kube/main.go
similarity index 94%
rename from core/cmd/kube-controlplane/main.go
rename to core/cmd/kube/main.go
index 854e742..3b4ac08 100644
--- a/core/cmd/kube-controlplane/main.go
+++ b/core/cmd/kube/main.go
@@ -14,8 +14,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-//+build ignore
-
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
@@ -50,6 +48,7 @@
kubeapiserver "k8s.io/kubernetes/cmd/kube-apiserver/app"
kubecontrollermanager "k8s.io/kubernetes/cmd/kube-controller-manager/app"
kubescheduler "k8s.io/kubernetes/cmd/kube-scheduler/app"
+ kubelet "k8s.io/kubernetes/cmd/kubelet/app"
)
func main() {
@@ -95,16 +94,18 @@
apiserver := func() *cobra.Command { return kubeapiserver.NewAPIServerCommand() }
controller := func() *cobra.Command { return kubecontrollermanager.NewControllerManagerCommand() }
scheduler := func() *cobra.Command { return kubescheduler.NewSchedulerCommand() }
+ kubelet := func() *cobra.Command { return kubelet.NewKubeletCommand() }
commandFns := []func() *cobra.Command{
apiserver,
controller,
scheduler,
+ kubelet,
}
cmd := &cobra.Command{
- Use: "kube-controlplane",
- Short: "Combines all Kubernetes Control Plane components in a single binary",
+ Use: "kube",
+ Short: "Combines all Kubernetes components in a single binary",
Run: func(cmd *cobra.Command, args []string) {
if len(args) != 0 {
cmd.Help()
diff --git a/core/cmd/mkimage/main.go b/core/cmd/mkimage/main.go
index ddf8813..1238c04 100644
--- a/core/cmd/mkimage/main.go
+++ b/core/cmd/mkimage/main.go
@@ -35,6 +35,7 @@
outputPath = flag.String("out", "", "Output disk image")
initramfsPath = flag.String("initramfs", "", "External initramfs [optional]")
enrolmentCredentialsPath = flag.String("enrolment-credentials", "", "Enrolment credentials [optional]")
+ dataPartitionSizeMiB = flag.Uint64("data-partition-size", 2048, "Override the data partition size (default 2048 MiB)")
)
func mibToSectors(size uint64) uint64 {
@@ -71,7 +72,7 @@
Type: SmalltownDataPartition,
Name: "SIGNOS-DATA",
Start: mibToSectors(256),
- End: mibToSectors(2560) - 1,
+ End: mibToSectors(*dataPartitionSizeMiB+256) - 1,
},
},
}
diff --git a/core/internal/api/cluster.go b/core/internal/api/cluster.go
index fb2c982..67fcd83 100644
--- a/core/internal/api/cluster.go
+++ b/core/internal/api/cluster.go
@@ -19,7 +19,7 @@
import (
"context"
"crypto/rand"
- "encoding/base64"
+ "encoding/hex"
"io"
"github.com/gogo/protobuf/proto"
@@ -103,7 +103,7 @@
if err != nil {
return nil, status.Error(codes.Internal, "failed to encode config")
}
- if _, err := store.Put(ctx, "enrolments/"+base64.RawURLEncoding.EncodeToString(token), string(enrolmentConfigRaw)); err != nil {
+ if _, err := store.Put(ctx, "enrolments/"+hex.EncodeToString(token), string(enrolmentConfigRaw)); err != nil {
return nil, status.Error(codes.Unavailable, "consensus unavailable")
}
return &schema.NewEnrolmentConfigResponse{
diff --git a/core/internal/api/enrolment.go b/core/internal/api/enrolment.go
index 6bf65f4..eb892ae 100644
--- a/core/internal/api/enrolment.go
+++ b/core/internal/api/enrolment.go
@@ -18,7 +18,7 @@
import (
"context"
- "encoding/base64"
+ "encoding/hex"
"errors"
"fmt"
@@ -38,7 +38,7 @@
func (s *EnrolmentStore) GetBySecret(ctx context.Context, secret []byte) (*api.EnrolmentConfig, error) {
- res, err := s.backend.Get(ctx, enrolmentPrefix+base64.RawURLEncoding.EncodeToString(secret))
+ res, err := s.backend.Get(ctx, enrolmentPrefix+hex.EncodeToString(secret))
if err != nil {
return nil, fmt.Errorf("failed to query consensus: %w", err)
}
diff --git a/core/internal/api/nodemanagement.go b/core/internal/api/nodemanagement.go
index f193d5c..0a3614e 100644
--- a/core/internal/api/nodemanagement.go
+++ b/core/internal/api/nodemanagement.go
@@ -24,7 +24,7 @@
"crypto/sha256"
"crypto/subtle"
"crypto/x509"
- "encoding/base64"
+ "encoding/hex"
"errors"
"fmt"
"io"
@@ -53,7 +53,7 @@
return "", errors.New("invalid node identity certificate")
}
- return "smalltown-" + base64.RawStdEncoding.EncodeToString([]byte(pubKey)), nil
+ return "smalltown-" + hex.EncodeToString([]byte(pubKey[:16])), nil
}
func (s *Server) registerNewNode(node *api.Node) error {
@@ -242,7 +242,7 @@
newNodeInfo := newNodeInfoVariant.NewNodeInfo
store := s.getStore()
- res, err := store.Get(registerServer.Context(), "enrolments/"+base64.RawURLEncoding.EncodeToString(newNodeInfo.EnrolmentConfig.EnrolmentSecret))
+ res, err := store.Get(registerServer.Context(), "enrolments/"+hex.EncodeToString(newNodeInfo.EnrolmentConfig.EnrolmentSecret))
if err != nil {
return status.Error(codes.Unavailable, "Consensus unavailable")
}
diff --git a/core/internal/common/setup.go b/core/internal/common/setup.go
index e745e54..7a268ae 100644
--- a/core/internal/common/setup.go
+++ b/core/internal/common/setup.go
@@ -26,6 +26,7 @@
ConsensusPort = 7834
MasterServicePort = 7833
ExternalServicePort = 7836
+ DebugServicePort = 7837
)
const (
diff --git a/core/internal/consensus/ca/ca.go b/core/internal/consensus/ca/ca.go
index ce9a840..5952b6f 100644
--- a/core/internal/consensus/ca/ca.go
+++ b/core/internal/consensus/ca/ca.go
@@ -53,7 +53,8 @@
}
// Workaround for https://github.com/golang/go/issues/26676 in Go's crypto/x509. Specifically Go
-// violates Section 4.2.1.2 of RFC 5280 without this. Should eventually be redundant.
+// violates Section 4.2.1.2 of RFC 5280 without this.
+// Fixed for 1.15 in https://go-review.googlesource.com/c/go/+/227098/.
//
// Taken from https://github.com/FiloSottile/mkcert/blob/master/cert.go#L295 written by one of Go's
// crypto engineers (BSD 3-clause).
diff --git a/core/internal/containerd/BUILD.bazel b/core/internal/containerd/BUILD.bazel
index dd7cf6d..56c2822 100644
--- a/core/internal/containerd/BUILD.bazel
+++ b/core/internal/containerd/BUILD.bazel
@@ -6,6 +6,7 @@
importpath = "git.monogon.dev/source/nexantic.git/core/internal/containerd",
visibility = ["//core:__subpackages__"],
deps = [
+ "//core/internal/common/supervisor:go_default_library",
"//core/pkg/logbuffer:go_default_library",
"@org_golang_x_sys//unix:go_default_library",
],
diff --git a/core/internal/containerd/main.go b/core/internal/containerd/main.go
index f4952e4..77e9156 100644
--- a/core/internal/containerd/main.go
+++ b/core/internal/containerd/main.go
@@ -18,6 +18,8 @@
import (
"context"
+ "fmt"
+ "git.monogon.dev/source/nexantic.git/core/internal/common/supervisor"
"os"
"os/exec"
@@ -26,23 +28,32 @@
"golang.org/x/sys/unix"
)
-// Implements supervisor.Runnable for later integration
+type Service struct {
+ Log *logbuffer.LogBuffer
+}
-func RunContainerd(ctx context.Context) error {
- containerdLog := logbuffer.New(1000, 16384)
- cmd := exec.CommandContext(ctx, "/containerd/bin/containerd", "--config", "/containerd/conf/config.toml")
- cmd.Stdout = containerdLog
- cmd.Stderr = containerdLog
- cmd.Env = []string{"PATH=/containerd/bin", "TMPDIR=/containerd/run/tmp"}
+func New() (*Service, error) {
+ return &Service{Log: logbuffer.New(5000, 16384)}, nil
+}
- if err := unix.Mount("tmpfs", "/containerd/run", "tmpfs", 0, ""); err != nil {
- panic(err)
+func (s *Service) Run() supervisor.Runnable {
+ return func(ctx context.Context) error {
+ cmd := exec.CommandContext(ctx, "/containerd/bin/containerd", "--config", "/containerd/conf/config.toml")
+ cmd.Stdout = s.Log
+ cmd.Stderr = s.Log
+ cmd.Env = []string{"PATH=/containerd/bin", "TMPDIR=/containerd/run/tmp"}
+
+ if err := unix.Mount("tmpfs", "/containerd/run", "tmpfs", 0, ""); err != nil {
+ panic(err)
+ }
+ if err := os.MkdirAll("/containerd/run/tmp", 0755); err != nil {
+ panic(err)
+ }
+
+ // TODO(lorenz): Healthcheck against CRI RuntimeService.Status() and SignalHealthy
+
+ err := cmd.Run()
+ fmt.Fprintf(s.Log, "containerd stopped: %v\n", err)
+ return err
}
- if err := os.MkdirAll("/containerd/run/tmp", 0755); err != nil {
- panic(err)
- }
-
- // TODO(lorenz): Healthcheck against cri.Status() RPC
-
- return cmd.Run()
}
diff --git a/core/internal/kubernetes/BUILD.bazel b/core/internal/kubernetes/BUILD.bazel
index e9b0573..534bf6e 100644
--- a/core/internal/kubernetes/BUILD.bazel
+++ b/core/internal/kubernetes/BUILD.bazel
@@ -6,18 +6,30 @@
"apiserver.go",
"auth.go",
"controller-manager.go",
+ "kubelet.go",
+ "reconcile.go",
"scheduler.go",
"service.go",
],
importpath = "git.monogon.dev/source/nexantic.git/core/internal/kubernetes",
visibility = ["//core:__subpackages__"],
deps = [
+ "//core/api/api:go_default_library",
"//core/internal/common/service:go_default_library",
"//core/internal/consensus:go_default_library",
"//core/pkg/fileargs:go_default_library",
+ "//core/pkg/logbuffer:go_default_library",
"@io_etcd_go_etcd//clientv3:go_default_library",
+ "@io_k8s_api//core/v1:go_default_library",
+ "@io_k8s_api//policy/v1beta1:go_default_library",
+ "@io_k8s_api//rbac/v1:go_default_library",
+ "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
+ "@io_k8s_client_go//kubernetes:go_default_library",
"@io_k8s_client_go//tools/clientcmd:go_default_library",
"@io_k8s_client_go//tools/clientcmd/api:go_default_library",
+ "@io_k8s_kubelet//config/v1beta1:go_default_library",
+ "@org_golang_google_grpc//codes:go_default_library",
+ "@org_golang_google_grpc//status:go_default_library",
"@org_uber_go_zap//:go_default_library",
],
)
diff --git a/core/internal/kubernetes/apiserver.go b/core/internal/kubernetes/apiserver.go
index 858fdb1..ac43035 100644
--- a/core/internal/kubernetes/apiserver.go
+++ b/core/internal/kubernetes/apiserver.go
@@ -21,8 +21,8 @@
"encoding/pem"
"errors"
"fmt"
+ "go.uber.org/zap"
"net"
- "os"
"os/exec"
"path"
@@ -65,13 +65,13 @@
return &config, nil
}
-func runAPIServer(config apiserverConfig) error {
+func (s *Service) runAPIServer(ctx context.Context, config apiserverConfig) error {
args, err := fileargs.New()
if err != nil {
panic(err) // If this fails, something is very wrong. Just crash.
}
defer args.Close()
- cmd := exec.Command("/bin/kube-controlplane", "kube-apiserver",
+ cmd := exec.CommandContext(ctx, "/kubernetes/bin/kube", "kube-apiserver",
fmt.Sprintf("--advertise-address=%v", config.advertiseAddress.String()),
"--authorization-mode=Node,RBAC",
args.FileOpt("--client-ca-file", "client-ca.pem",
@@ -85,7 +85,7 @@
pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: config.kubeletClientCert})),
args.FileOpt("--kubelet-client-key", "kubelet-client-key.pem",
pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: config.kubeletClientKey})),
- "--kubelet-preferred-address-types=InternalIP",
+ "--kubelet-preferred-address-types=Hostname",
args.FileOpt("--proxy-client-cert-file", "aggregation-client-cert.pem",
pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: config.aggregationClientCert})),
args.FileOpt("--proxy-client-key-file", "aggregation-client-key.pem",
@@ -107,8 +107,14 @@
if args.Error() != nil {
return err
}
- cmd.Stdout = os.Stdout
- cmd.Stderr = os.Stderr
+ cmd.Stdout = s.apiserverLogs
+ cmd.Stderr = s.apiserverLogs
err = cmd.Run()
+ fmt.Fprintf(s.apiserverLogs, "apiserver stopped: %v\n", err)
+ if ctx.Err() == context.Canceled {
+ s.logger.Info("apiserver stopped", zap.Error(err))
+ } else {
+ s.logger.Warn("apiserver stopped unexpectedly", zap.Error(err))
+ }
return err
}
diff --git a/core/internal/kubernetes/auth.go b/core/internal/kubernetes/auth.go
index 0095bc4..25e2e4b 100644
--- a/core/internal/kubernetes/auth.go
+++ b/core/internal/kubernetes/auth.go
@@ -30,6 +30,7 @@
"fmt"
"math/big"
"net"
+ "os"
"path"
"time"
@@ -232,7 +233,7 @@
}
kubeletClientCert, kubeletClientKey, err := issueCertificate(
- clientCertTemplate("kube-apiserver-kubelet-client", []string{"system:masters"}),
+ clientCertTemplate("smalltown:apiserver-kubelet-client", []string{}),
idCA, idKey,
)
if err != nil {
@@ -314,6 +315,34 @@
return err
}
+ masterClientCert, masterClientKey, err := issueCertificate(
+ clientCertTemplate("smalltown:master", []string{"system:masters"}),
+ idCA, idKey,
+ )
+ if err != nil {
+ return fmt.Errorf("failed to issue certificate for master client: %w", err)
+ }
+
+ masterClientKubeconfig, err := makeLocalKubeconfig(idCA, masterClientCert,
+ masterClientKey)
+ if err != nil {
+ return fmt.Errorf("failed to create kubeconfig for master client: %w", err)
+ }
+
+ _, err = consensusKV.Put(context.Background(), path.Join(etcdPath, "master.kubeconfig"),
+ string(masterClientKubeconfig))
+ if err != nil {
+ return fmt.Errorf("failed to store master kubeconfig: %w", err)
+ }
+
+ hostname, err := os.Hostname()
+ if err != nil {
+ return err
+ }
+ if err := bootstrapLocalKubelet(consensusKV, hostname); err != nil {
+ return err
+ }
+
return nil
}
diff --git a/core/internal/kubernetes/controller-manager.go b/core/internal/kubernetes/controller-manager.go
index 1146a14..a67f6fd 100644
--- a/core/internal/kubernetes/controller-manager.go
+++ b/core/internal/kubernetes/controller-manager.go
@@ -17,10 +17,11 @@
package kubernetes
import (
+ "context"
"encoding/pem"
"fmt"
+ "go.uber.org/zap"
"net"
- "os"
"os/exec"
"go.etcd.io/etcd/clientv3"
@@ -60,13 +61,13 @@
return &config, nil
}
-func runControllerManager(config controllerManagerConfig) error {
+func (s *Service) runControllerManager(ctx context.Context, config controllerManagerConfig) error {
args, err := fileargs.New()
if err != nil {
panic(err) // If this fails, something is very wrong. Just crash.
}
defer args.Close()
- cmd := exec.Command("/bin/kube-controlplane", "kube-controller-manager",
+ cmd := exec.CommandContext(ctx, "/kubernetes/bin/kube", "kube-controller-manager",
args.FileOpt("--kubeconfig", "kubeconfig", config.kubeConfig),
args.FileOpt("--service-account-private-key-file", "service-account-privkey.pem",
pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: config.serviceAccountPrivKey})),
@@ -83,7 +84,14 @@
if args.Error() != nil {
return fmt.Errorf("failed to use fileargs: %w", err)
}
- cmd.Stdout = os.Stdout
- cmd.Stderr = os.Stderr
- return cmd.Run()
+ cmd.Stdout = s.controllerManagerLogs
+ cmd.Stderr = s.controllerManagerLogs
+ err = cmd.Run()
+ fmt.Fprintf(s.controllerManagerLogs, "controller-manager stopped: %v\n", err)
+ if ctx.Err() == context.Canceled {
+ s.logger.Info("controller-manager stopped", zap.Error(err))
+ } else {
+ s.logger.Warn("controller-manager stopped unexpectedly", zap.Error(err))
+ }
+ return err
}
diff --git a/core/internal/kubernetes/kubelet.go b/core/internal/kubernetes/kubelet.go
new file mode 100644
index 0000000..b7d8157
--- /dev/null
+++ b/core/internal/kubernetes/kubelet.go
@@ -0,0 +1,141 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kubernetes
+
+import (
+ "context"
+ "crypto/ed25519"
+ "encoding/json"
+ "encoding/pem"
+ "fmt"
+ "go.uber.org/zap"
+ "io/ioutil"
+ "os"
+ "os/exec"
+
+ "net"
+
+ "git.monogon.dev/source/nexantic.git/core/pkg/fileargs"
+ "go.etcd.io/etcd/clientv3"
+ v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/kubelet/config/v1beta1"
+)
+
+type KubeletSpec struct {
+ clusterDNS []net.IP
+}
+
+func bootstrapLocalKubelet(consensusKV clientv3.KV, nodeName string) error {
+ idCA, idKeyRaw, err := getCert(consensusKV, "id-ca")
+ if err != nil {
+ return err
+ }
+ idKey := ed25519.PrivateKey(idKeyRaw)
+ cert, key, err := issueCertificate(clientCertTemplate("system:node:"+nodeName, []string{"system:nodes"}), idCA, idKey)
+ if err != nil {
+ return err
+ }
+ kubeconfig, err := makeLocalKubeconfig(idCA, cert, key)
+ if err != nil {
+ return err
+ }
+
+ serverCert, serverKey, err := issueCertificate(serverCertTemplate([]string{nodeName}, []net.IP{}), idCA, idKey)
+ if err != nil {
+ return err
+ }
+ if err := os.MkdirAll("/data/kubernetes", 0755); err != nil {
+ return err
+ }
+ if err := ioutil.WriteFile("/data/kubernetes/kubelet.kubeconfig", kubeconfig, 0400); err != nil {
+ return err
+ }
+ if err := ioutil.WriteFile("/data/kubernetes/ca.crt", pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: idCA}), 0400); err != nil {
+ return err
+ }
+ if err := ioutil.WriteFile("/data/kubernetes/kubelet.crt", pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: serverCert}), 0400); err != nil {
+ return err
+ }
+ if err := ioutil.WriteFile("/data/kubernetes/kubelet.key", pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: serverKey}), 0400); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *Service) runKubelet(ctx context.Context, spec *KubeletSpec) error {
+ fargs, err := fileargs.New()
+ if err != nil {
+ return err
+ }
+ var clusterDNS []string
+ for _, dnsIP := range spec.clusterDNS {
+ clusterDNS = append(clusterDNS, dnsIP.String())
+ }
+
+ kubeletConf := &v1beta1.KubeletConfiguration{
+ TypeMeta: v1.TypeMeta{
+ Kind: "KubeletConfiguration",
+ APIVersion: v1beta1.GroupName + "/v1beta1",
+ },
+ TLSCertFile: "/data/kubernetes/kubelet.crt",
+ TLSPrivateKeyFile: "/data/kubernetes/kubelet.key",
+ TLSMinVersion: "VersionTLS13",
+ ClusterDNS: clusterDNS,
+ Authentication: v1beta1.KubeletAuthentication{
+ X509: v1beta1.KubeletX509Authentication{
+ ClientCAFile: "/data/kubernetes/ca.crt",
+ },
+ },
+ ClusterDomain: "cluster.local",
+ EnableControllerAttachDetach: False(),
+ HairpinMode: "none",
+ MakeIPTablesUtilChains: False(), // We don't have iptables
+ FailSwapOn: False(), // Our kernel doesn't have swap enabled which breaks Kubelet's detection
+ KubeReserved: map[string]string{
+ "cpu": "200m",
+ "memory": "300Mi",
+ },
+ // We're not going to use this, but let's make it point to a known-empty directory in case anybody manages to
+ // trigger it.
+ VolumePluginDir: "/kubernetes/conf/flexvolume-plugins",
+ }
+
+ configRaw, err := json.Marshal(kubeletConf)
+ if err != nil {
+ return err
+ }
+ cmd := exec.CommandContext(ctx, "/kubernetes/bin/kube", "kubelet",
+ fargs.FileOpt("--config", "config.json", configRaw),
+ "--container-runtime=remote",
+ "--container-runtime-endpoint=unix:///containerd/run/containerd.sock",
+ "--kubeconfig=/data/kubernetes/kubelet.kubeconfig",
+ "--root-dir=/data/kubernetes/kubelet",
+ )
+ cmd.Env = []string{"PATH=/kubernetes/bin"}
+ cmd.Stdout = s.kubeletLogs
+ cmd.Stderr = s.kubeletLogs
+
+ err = cmd.Run()
+ fmt.Fprintf(s.kubeletLogs, "kubelet stopped: %v\n", err)
+ if ctx.Err() == context.Canceled {
+ s.logger.Info("kubelet stopped", zap.Error(err))
+ } else {
+ s.logger.Warn("kubelet stopped unexpectedly", zap.Error(err))
+ }
+ return err
+}
diff --git a/core/internal/kubernetes/reconcile.go b/core/internal/kubernetes/reconcile.go
new file mode 100644
index 0000000..cf991ce
--- /dev/null
+++ b/core/internal/kubernetes/reconcile.go
@@ -0,0 +1,313 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// The reconciler ensures that a base set of K8s resources is always available in the cluster. These are necessary to
+// ensure correct out-of-the-box functionality. All resources containing the smalltown.com/builtin=true label are assumed
+// to be managed by the reconciler.
+// It currently does not revert modifications made by admins, it is planned to create an admission plugin prohibiting
+// such modifications to resources with the smalltown.com/builtin label to deal with that problem. This would also solve a
+// potential issue where you could delete resources just by adding the smalltown.com/builtin=true label.
+package kubernetes
+
+import (
+ "context"
+ "time"
+
+ "go.uber.org/zap"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/api/policy/v1beta1"
+ rbacv1 "k8s.io/api/rbac/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/client-go/kubernetes"
+ "k8s.io/client-go/tools/clientcmd"
+)
+
+const builtinRBACPrefix = "smalltown:"
+
+// Sad workaround for all the pointer booleans in K8s specs
+func True() *bool {
+ val := true
+ return &val
+}
+func False() *bool {
+ val := false
+ return &val
+}
+
+func rbac(name string) string {
+ return builtinRBACPrefix + name
+}
+
+// Extended from https://github.com/kubernetes/kubernetes/blob/master/cluster/gce/addons/podsecuritypolicies/unprivileged-addon.yaml
+var builtinPSPs = []*v1beta1.PodSecurityPolicy{
+ {
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "default",
+ Labels: map[string]string{
+ "smalltown.com/builtin": "true",
+ },
+ Annotations: map[string]string{
+ "kubernetes.io/description": "This default PSP allows the creation of pods using features that are" +
+ " generally considered safe against any sort of escape.",
+ },
+ },
+ Spec: v1beta1.PodSecurityPolicySpec{
+ AllowPrivilegeEscalation: True(),
+ AllowedCapabilities: []corev1.Capability{ // runc's default list of allowed capabilities
+ "SETPCAP",
+ "MKNOD",
+ "AUDIT_WRITE",
+ "CHOWN",
+ "NET_RAW",
+ "DAC_OVERRIDE",
+ "FOWNER",
+ "FSETID",
+ "KILL",
+ "SETGID",
+ "SETUID",
+ "NET_BIND_SERVICE",
+ "SYS_CHROOT",
+ "SETFCAP",
+ },
+ HostNetwork: false,
+ HostIPC: false,
+ HostPID: false,
+ FSGroup: v1beta1.FSGroupStrategyOptions{
+ Rule: v1beta1.FSGroupStrategyRunAsAny,
+ },
+ RunAsUser: v1beta1.RunAsUserStrategyOptions{
+ Rule: v1beta1.RunAsUserStrategyRunAsAny,
+ },
+ SELinux: v1beta1.SELinuxStrategyOptions{
+ Rule: v1beta1.SELinuxStrategyRunAsAny,
+ },
+ SupplementalGroups: v1beta1.SupplementalGroupsStrategyOptions{
+ Rule: v1beta1.SupplementalGroupsStrategyRunAsAny,
+ },
+ Volumes: []v1beta1.FSType{ // Volumes considered safe to use
+ v1beta1.ConfigMap,
+ v1beta1.EmptyDir,
+ v1beta1.Projected,
+ v1beta1.Secret,
+ v1beta1.DownwardAPI,
+ v1beta1.PersistentVolumeClaim,
+ },
+ },
+ },
+}
+
+var builtinClusterRoles = []*rbacv1.ClusterRole{
+ {
+ ObjectMeta: metav1.ObjectMeta{
+ Name: rbac("psp-default"),
+ Annotations: map[string]string{
+ "kubernetes.io/description": "This role grants access to the \"default\" PSP.",
+ },
+ },
+ Rules: []rbacv1.PolicyRule{
+ {
+ APIGroups: []string{"policy"},
+ Resources: []string{"podsecuritypolicies"},
+ ResourceNames: []string{"default"},
+ Verbs: []string{"use"},
+ },
+ },
+ },
+}
+
+var builtinClusterRoleBindings = []*rbacv1.ClusterRoleBinding{
+ {
+ ObjectMeta: metav1.ObjectMeta{
+ Name: rbac("default-psp-for-sa"),
+ Annotations: map[string]string{
+ "kubernetes.io/description": "This binding grants every service account access to the \"default\" PSP. " +
+ "Creation of Pods is still restricted by other RBAC roles. Otherwise no pods (unprivileged or not) " +
+ "can be created.",
+ },
+ },
+ RoleRef: rbacv1.RoleRef{
+ APIGroup: rbacv1.GroupName,
+ Kind: "ClusterRole",
+ Name: rbac("psp-default"),
+ },
+ Subjects: []rbacv1.Subject{
+ {
+ APIGroup: rbacv1.GroupName,
+ Kind: "Group",
+ Name: "system:serviceaccounts",
+ },
+ },
+ },
+ {
+ ObjectMeta: metav1.ObjectMeta{
+ Name: rbac("apiserver-kubelet-client"),
+ Annotations: map[string]string{
+ "kubernetes.io/description": "This binding grants the apiserver access to the kubelets. This enables " +
+ "lots of built-in functionality like reading logs or forwarding ports via the API.",
+ },
+ },
+ RoleRef: rbacv1.RoleRef{
+ APIGroup: rbacv1.GroupName,
+ Kind: "ClusterRole",
+ Name: "system:kubelet-api-admin",
+ },
+ Subjects: []rbacv1.Subject{
+ {
+ APIGroup: rbacv1.GroupName,
+ Kind: "User",
+ Name: "smalltown:apiserver-kubelet-client",
+ },
+ },
+ },
+}
+
+func runReconciler(ctx context.Context, masterKubeconfig []byte, log *zap.Logger) error {
+ rawClientConfig, err := clientcmd.NewClientConfigFromBytes(masterKubeconfig)
+ if err != nil {
+ return err
+ }
+
+ clientConfig, err := rawClientConfig.ClientConfig()
+ clientset, err := kubernetes.NewForConfig(clientConfig)
+ if err != nil {
+ return err
+ }
+ t := time.NewTicker(10 * time.Second)
+ for {
+ err = reconcile(ctx, clientset)
+ select {
+ case <-t.C:
+ err = reconcile(ctx, clientset)
+ if err != nil {
+ log.Warn("Failed to reconcile built-in resources", zap.Error(err))
+ }
+ case <-ctx.Done():
+ return nil
+ }
+ }
+}
+
+func reconcile(ctx context.Context, clientset *kubernetes.Clientset) error {
+ if err := reconcilePSPs(ctx, clientset); err != nil {
+ return err
+ }
+ if err := reconcileClusterRoles(ctx, clientset); err != nil {
+ return err
+ }
+ if err := reconcileClusterRoleBindings(ctx, clientset); err != nil {
+ return err
+ }
+ return nil
+}
+
+func reconcilePSPs(ctx context.Context, clientset *kubernetes.Clientset) error {
+ pspClient := clientset.PolicyV1beta1().PodSecurityPolicies()
+ availablePSPs, err := pspClient.List(ctx, metav1.ListOptions{
+ LabelSelector: "smalltown.com/builtin=true",
+ })
+ if err != nil {
+ return err
+ }
+ availablePSPMap := make(map[string]struct{})
+ for _, psp := range availablePSPs.Items {
+ availablePSPMap[psp.Name] = struct{}{}
+ }
+ expectedPSPMap := make(map[string]*v1beta1.PodSecurityPolicy)
+ for _, psp := range builtinPSPs {
+ expectedPSPMap[psp.Name] = psp
+ }
+ for pspName, psp := range expectedPSPMap {
+ if _, ok := availablePSPMap[pspName]; !ok {
+ if _, err := pspClient.Create(ctx, psp, metav1.CreateOptions{}); err != nil {
+ return err
+ }
+ }
+ }
+ for pspName, _ := range availablePSPMap {
+ if _, ok := expectedPSPMap[pspName]; !ok {
+ if err := pspClient.Delete(ctx, pspName, metav1.DeleteOptions{}); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+func reconcileClusterRoles(ctx context.Context, clientset *kubernetes.Clientset) error {
+ crClient := clientset.RbacV1().ClusterRoles()
+ availableCRs, err := crClient.List(ctx, metav1.ListOptions{
+ LabelSelector: "smalltown.com/builtin=true",
+ })
+ if err != nil {
+ return err
+ }
+ availableCRMap := make(map[string]struct{})
+ for _, cr := range availableCRs.Items {
+ availableCRMap[cr.Name] = struct{}{}
+ }
+ expectedCRMap := make(map[string]*rbacv1.ClusterRole)
+ for _, cr := range builtinClusterRoles {
+ expectedCRMap[cr.Name] = cr
+ }
+ for crName, psp := range expectedCRMap {
+ if _, ok := availableCRMap[crName]; !ok {
+ if _, err := crClient.Create(ctx, psp, metav1.CreateOptions{}); err != nil {
+ return err
+ }
+ }
+ }
+ for crName, _ := range availableCRMap {
+ if _, ok := expectedCRMap[crName]; !ok {
+ if err := crClient.Delete(ctx, crName, metav1.DeleteOptions{}); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+func reconcileClusterRoleBindings(ctx context.Context, clientset *kubernetes.Clientset) error {
+ crbClient := clientset.RbacV1().ClusterRoleBindings()
+ availableCRBs, err := crbClient.List(ctx, metav1.ListOptions{
+ LabelSelector: "smalltown.com/builtin=true",
+ })
+ if err != nil {
+ return err
+ }
+ availableCRBMap := make(map[string]struct{})
+ for _, crb := range availableCRBs.Items {
+ availableCRBMap[crb.Name] = struct{}{}
+ }
+ expectedCRBMap := make(map[string]*rbacv1.ClusterRoleBinding)
+ for _, crb := range builtinClusterRoleBindings {
+ expectedCRBMap[crb.Name] = crb
+ }
+ for crbName, psp := range expectedCRBMap {
+ if _, ok := availableCRBMap[crbName]; !ok {
+ if _, err := crbClient.Create(ctx, psp, metav1.CreateOptions{}); err != nil {
+ return err
+ }
+ }
+ }
+ for crbName, _ := range availableCRBMap {
+ if _, ok := expectedCRBMap[crbName]; !ok {
+ if err := crbClient.Delete(ctx, crbName, metav1.DeleteOptions{}); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
diff --git a/core/internal/kubernetes/scheduler.go b/core/internal/kubernetes/scheduler.go
index ac21588..75dea97 100644
--- a/core/internal/kubernetes/scheduler.go
+++ b/core/internal/kubernetes/scheduler.go
@@ -17,12 +17,13 @@
package kubernetes
import (
+ "context"
"encoding/pem"
"fmt"
- "os"
"os/exec"
"go.etcd.io/etcd/clientv3"
+ "go.uber.org/zap"
"git.monogon.dev/source/nexantic.git/core/pkg/fileargs"
)
@@ -47,13 +48,13 @@
return &config, nil
}
-func runScheduler(config schedulerConfig) error {
+func (s *Service) runScheduler(ctx context.Context, config schedulerConfig) error {
args, err := fileargs.New()
if err != nil {
panic(err) // If this fails, something is very wrong. Just crash.
}
defer args.Close()
- cmd := exec.Command("/bin/kube-controlplane", "kube-scheduler",
+ cmd := exec.CommandContext(ctx, "/kubernetes/bin/kube", "kube-scheduler",
args.FileOpt("--kubeconfig", "kubeconfig", config.kubeConfig),
"--port=0", // Kill insecure serving
args.FileOpt("--tls-cert-file", "server-cert.pem",
@@ -64,7 +65,14 @@
if args.Error() != nil {
return fmt.Errorf("failed to use fileargs: %w", err)
}
- cmd.Stdout = os.Stdout
- cmd.Stderr = os.Stderr
- return cmd.Run()
+ cmd.Stdout = s.schedulerLogs
+ cmd.Stderr = s.schedulerLogs
+ err = cmd.Run()
+ fmt.Fprintf(s.schedulerLogs, "scheduler stopped: %v\n", err)
+ if ctx.Err() == context.Canceled {
+ s.logger.Info("scheduler stopped", zap.Error(err))
+ } else {
+ s.logger.Warn("scheduler stopped unexpectedly", zap.Error(err))
+ }
+ return err
}
diff --git a/core/internal/kubernetes/service.go b/core/internal/kubernetes/service.go
index 9d653b4..5e28292 100644
--- a/core/internal/kubernetes/service.go
+++ b/core/internal/kubernetes/service.go
@@ -17,9 +17,18 @@
package kubernetes
import (
+ "context"
+ "crypto/ed25519"
"errors"
+ "fmt"
"net"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+
+ schema "git.monogon.dev/source/nexantic.git/core/generated/api"
+ "git.monogon.dev/source/nexantic.git/core/pkg/logbuffer"
+
"go.etcd.io/etcd/clientv3"
"go.uber.org/zap"
@@ -35,14 +44,22 @@
type Service struct {
*service.BaseService
- consensusService *consensus.Service
- logger *zap.Logger
+ consensusService *consensus.Service
+ logger *zap.Logger
+ apiserverLogs *logbuffer.LogBuffer
+ controllerManagerLogs *logbuffer.LogBuffer
+ schedulerLogs *logbuffer.LogBuffer
+ kubeletLogs *logbuffer.LogBuffer
}
func New(logger *zap.Logger, consensusService *consensus.Service) *Service {
s := &Service{
- consensusService: consensusService,
- logger: logger,
+ consensusService: consensusService,
+ logger: logger,
+ apiserverLogs: logbuffer.New(5000, 16384),
+ controllerManagerLogs: logbuffer.New(5000, 16384),
+ schedulerLogs: logbuffer.New(5000, 16384),
+ kubeletLogs: logbuffer.New(5000, 16384),
}
s.BaseService = service.NewBaseService("kubernetes", logger, s)
return s
@@ -56,6 +73,40 @@
return newCluster(s.getKV())
}
+// GetComponentLogs grabs logs from various Kubernetes binaries
+func (s *Service) GetComponentLogs(component string, n int) ([]string, error) {
+ switch component {
+ case "apiserver":
+ return s.apiserverLogs.ReadLinesTruncated(n, "..."), nil
+ case "controller-manager":
+ return s.controllerManagerLogs.ReadLinesTruncated(n, "..."), nil
+ case "scheduler":
+ return s.schedulerLogs.ReadLinesTruncated(n, "..."), nil
+ case "kubelet":
+ return s.kubeletLogs.ReadLinesTruncated(n, "..."), nil
+ default:
+ return []string{}, errors.New("component not available")
+ }
+}
+
+// GetDebugKubeconfig issues a kubeconfig for an arbitrary given identity. Useful for debugging and testing.
+func (s *Service) GetDebugKubeconfig(ctx context.Context, request *schema.GetDebugKubeconfigRequest) (*schema.GetDebugKubeconfigResponse, error) {
+ idCA, idKeyRaw, err := getCert(s.getKV(), "id-ca")
+ idKey := ed25519.PrivateKey(idKeyRaw)
+ if err != nil {
+ return nil, status.Errorf(codes.Unavailable, "Failed to load ID CA: %v", err)
+ }
+ debugCert, debugKey, err := issueCertificate(clientCertTemplate(request.Id, request.Groups), idCA, idKey)
+ if err != nil {
+ return nil, status.Errorf(codes.Unavailable, "Failed to issue certs for kubeconfig: %v\n", err)
+ }
+ debugKubeconfig, err := makeLocalKubeconfig(idCA, debugCert, debugKey)
+ if err != nil {
+ return nil, status.Errorf(codes.Unavailable, "Failed to generate kubeconfig: %v", err)
+ }
+ return &schema.GetDebugKubeconfigResponse{DebugKubeconfig: string(debugKubeconfig)}, nil
+}
+
func (s *Service) OnStart() error {
config := Config{
AdvertiseAddress: net.IP{10, 0, 2, 15}, // Depends on networking
@@ -85,14 +136,30 @@
return err
}
+ masterKubeconfig, err := getSingle(consensusKV, "master.kubeconfig")
+ if err != nil {
+ return err
+ }
+
+ // TODO(lorenz): Once internal/node is part of the supervisor tree, these should all be supervisor runnables
go func() {
- runAPIServer(*apiserverConfig)
+ s.runAPIServer(context.TODO(), *apiserverConfig)
}()
go func() {
- runControllerManager(*controllerManagerConfig)
+ s.runControllerManager(context.TODO(), *controllerManagerConfig)
}()
go func() {
- runScheduler(*schedulerConfig)
+ s.runScheduler(context.TODO(), *schedulerConfig)
+ }()
+
+ go func() {
+ if err := s.runKubelet(context.TODO(), &KubeletSpec{}); err != nil {
+ fmt.Printf("Failed to launch kubelet: %v\n", err)
+ }
+ }()
+
+ go func() {
+ go runReconciler(context.TODO(), masterKubeconfig, s.logger)
}()
return nil
diff --git a/core/internal/node/BUILD.bazel b/core/internal/node/BUILD.bazel
index d96c0e6..48afed0 100644
--- a/core/internal/node/BUILD.bazel
+++ b/core/internal/node/BUILD.bazel
@@ -3,6 +3,7 @@
go_library(
name = "go_default_library",
srcs = [
+ "debug.go",
"main.go",
"setup.go",
],
@@ -13,6 +14,7 @@
"//core/internal/api:go_default_library",
"//core/internal/common:go_default_library",
"//core/internal/consensus:go_default_library",
+ "//core/internal/containerd:go_default_library",
"//core/internal/integrity/tpm2:go_default_library",
"//core/internal/kubernetes:go_default_library",
"//core/internal/network:go_default_library",
@@ -23,6 +25,7 @@
"@org_golang_google_grpc//codes:go_default_library",
"@org_golang_google_grpc//credentials:go_default_library",
"@org_golang_google_grpc//status:go_default_library",
+ "@org_golang_x_sys//unix:go_default_library",
"@org_uber_go_zap//:go_default_library",
],
)
diff --git a/core/internal/node/debug.go b/core/internal/node/debug.go
new file mode 100644
index 0000000..1d91ad6
--- /dev/null
+++ b/core/internal/node/debug.go
@@ -0,0 +1,88 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+// Implements a debug gRPC service for testing and introspection
+// This is attached to the SmalltownNode because most other services are instantiated there and thus are accessible
+// from there. Have a look at //core/cmd/dbg if you need to interact with this from a CLI.
+
+import (
+ "context"
+ "math"
+
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+
+ schema "git.monogon.dev/source/nexantic.git/core/generated/api"
+ "git.monogon.dev/source/nexantic.git/core/internal/storage"
+)
+
+func (s *SmalltownNode) GetDebugKubeconfig(ctx context.Context, req *schema.GetDebugKubeconfigRequest) (*schema.GetDebugKubeconfigResponse, error) {
+ return s.Kubernetes.GetDebugKubeconfig(ctx, req)
+}
+
+// GetComponentLogs gets various logbuffers from binaries we call. This function just deals with the first path component,
+// delegating the rest to the service-specific handlers.
+func (s *SmalltownNode) GetComponentLogs(ctx context.Context, req *schema.GetComponentLogsRequest) (*schema.GetComponentLogsResponse, error) {
+ if len(req.ComponentPath) < 1 {
+ return nil, status.Error(codes.InvalidArgument, "component_path needs to contain at least one part")
+ }
+ linesToRead := int(req.TailLines)
+ if linesToRead == 0 {
+ linesToRead = math.MaxInt32
+ }
+ var lines []string
+ var err error
+ switch req.ComponentPath[0] {
+ case "containerd":
+ lines = s.Containerd.Log.ReadLinesTruncated(linesToRead, "...")
+ case "kube":
+ if len(req.ComponentPath) < 2 {
+ return nil, status.Error(codes.NotFound, "Component not found")
+ }
+ lines, err = s.Kubernetes.GetComponentLogs(req.ComponentPath[1], linesToRead)
+ if err != nil {
+ return nil, status.Error(codes.NotFound, "Component not found")
+ }
+ default:
+ return nil, status.Error(codes.NotFound, "component not found")
+ }
+ return &schema.GetComponentLogsResponse{Line: lines}, nil
+}
+
+// GetCondition checks for various conditions exposed by different services. Mostly intended for testing. If you need
+// to make sure something is available in an E2E test, consider adding a condition here.
+func (s *SmalltownNode) GetCondition(ctx context.Context, req *schema.GetConditionRequest) (*schema.GetConditionResponse, error) {
+ var ok bool
+ switch req.Name {
+ case "IPAssigned":
+ ip, err := s.Network.GetIP(ctx, false)
+ if err == nil && ip != nil {
+ ok = true
+ }
+ case "DataAvailable":
+ _, err := s.Storage.GetPathInPlace(storage.PlaceData, "test")
+ if err == nil {
+ ok = true
+ }
+ default:
+ return nil, status.Errorf(codes.NotFound, "condition %v not found", req.Name)
+ }
+ return &schema.GetConditionResponse{
+ Ok: ok,
+ }, nil
+}
diff --git a/core/internal/node/main.go b/core/internal/node/main.go
index b0674d2..4041cb8 100644
--- a/core/internal/node/main.go
+++ b/core/internal/node/main.go
@@ -25,14 +25,16 @@
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
- "encoding/base64"
+ "encoding/hex"
"errors"
"flag"
"fmt"
+ "git.monogon.dev/source/nexantic.git/core/internal/containerd"
"io/ioutil"
"math/big"
"net"
"os"
+ "strings"
"time"
apipb "git.monogon.dev/source/nexantic.git/core/generated/api"
@@ -43,6 +45,7 @@
"git.monogon.dev/source/nexantic.git/core/internal/kubernetes"
"git.monogon.dev/source/nexantic.git/core/internal/network"
"git.monogon.dev/source/nexantic.git/core/internal/storage"
+ "golang.org/x/sys/unix"
"github.com/cenkalti/backoff/v4"
"github.com/gogo/protobuf/proto"
@@ -62,12 +65,15 @@
Consensus *consensus.Service
Storage *storage.Manager
Kubernetes *kubernetes.Service
+ Containerd *containerd.Service
Network *network.Service
logger *zap.Logger
state common.SmalltownState
hostname string
enrolmentConfig *apipb.EnrolmentConfig
+
+ debugServer *grpc.Server
}
)
@@ -101,12 +107,18 @@
return nil, err
}
+ containerdService, err := containerd.New()
+ if err != nil {
+ return nil, err
+ }
+
s := &SmalltownNode{
- Consensus: consensusService,
- Storage: strg,
- Network: ntwk,
- logger: logger,
- hostname: hostname,
+ Consensus: consensusService,
+ Containerd: containerdService,
+ Storage: strg,
+ Network: ntwk,
+ logger: logger,
+ hostname: hostname,
}
apiService, err := api.NewApiServer(&api.Config{}, logger.With(zap.String("module", "api")), s.Consensus)
@@ -118,6 +130,9 @@
s.Kubernetes = kubernetes.New(logger.With(zap.String("module", "kubernetes")), consensusService)
+ s.debugServer = grpc.NewServer()
+ apipb.RegisterNodeDebugServiceServer(s.debugServer, s)
+
logger.Info("Created SmalltownNode")
return s, nil
@@ -126,6 +141,8 @@
func (s *SmalltownNode) Start(ctx context.Context) error {
s.logger.Info("Starting Smalltown node")
+ s.startDebugSvc()
+
// TODO(lorenz): Abstracting enrolment sounds like a good idea, but ends up being painful
// because of things like storage access. I'm keeping it this way until the more complex
// enrolment procedures are fleshed out. This is also a bit panic()-happy, but there is really
@@ -157,6 +174,30 @@
panic("Unreachable")
}
+func (s *SmalltownNode) startDebugSvc() {
+ debugListenHost := fmt.Sprintf(":%v", common.DebugServicePort)
+ debugListener, err := net.Listen("tcp", debugListenHost)
+ if err != nil {
+ s.logger.Fatal("failed to listen", zap.Error(err))
+ }
+
+ go func() {
+ if err := s.debugServer.Serve(debugListener); err != nil {
+ s.logger.Fatal("failed to serve", zap.Error(err))
+ }
+ }()
+}
+
+func (s *SmalltownNode) initHostname() error {
+ if err := unix.Sethostname([]byte(s.hostname)); err != nil {
+ return err
+ }
+ if err := ioutil.WriteFile("/etc/hosts", []byte(fmt.Sprintf("%v %v", "127.0.0.1", s.hostname)), 0644); err != nil {
+ return err
+ }
+ return ioutil.WriteFile("/etc/machine-id", []byte(strings.TrimPrefix(s.hostname, "smalltown-")), 0644)
+}
+
func (s *SmalltownNode) startEnrolling(ctx context.Context) error {
s.logger.Info("Initializing subsystems for enrolment")
s.state = common.StateEnrollMode
@@ -166,6 +207,11 @@
return err
}
+ s.hostname = nodeID
+ if err := s.initHostname(); err != nil {
+ return err
+ }
+
// We only support TPM2 at the moment, any abstractions here would be premature
trustAgent := tpm2.TPM2Agent{}
@@ -207,11 +253,18 @@
if err != nil {
return err
}
+ s.hostname = nodeID
+ if err := s.initHostname(); err != nil {
+ return err
+ }
if err := s.initNodeAPI(); err != nil {
return err
}
+ // TODO: Use supervisor.Run for this
+ go s.Containerd.Run()(context.TODO())
+
dataPath, err := s.Storage.GetPathInPlace(storage.PlaceData, "etcd")
if err != nil {
return err
@@ -314,7 +367,7 @@
return []byte{}, "", fmt.Errorf("failed to write node key: %w", err)
}
- name := "smalltown-" + base64.RawStdEncoding.EncodeToString([]byte(pubKey))
+ name := "smalltown-" + hex.EncodeToString([]byte(pubKey[:16]))
// This has no SANs because it authenticates by public key, not by name
nodeCert := &x509.Certificate{
@@ -429,6 +482,11 @@
s.logger.Info("Initializing subsystems for production")
s.state = common.StateJoined
+ s.hostname = s.enrolmentConfig.NodeId
+ if err := s.initHostname(); err != nil {
+ return err
+ }
+
trustAgent := tpm2.TPM2Agent{}
unlockOp := func() error {
unlockKey, err := trustAgent.Unlock(*s.enrolmentConfig)
@@ -449,6 +507,9 @@
s.initNodeAPI()
+ // TODO: Use supervisor.Run for this
+ go s.Containerd.Run()(context.TODO())
+
err := s.Consensus.Start()
if err != nil {
return err
diff --git a/core/scripts/launch.sh b/core/scripts/launch.sh
index d4ab0bb..3fb3b57 100755
--- a/core/scripts/launch.sh
+++ b/core/scripts/launch.sh
@@ -13,7 +13,7 @@
-drive if=pflash,format=raw,readonly,file=external/edk2/OVMF_CODE.fd \
-drive if=pflash,format=raw,snapshot=on,file=external/edk2/OVMF_VARS.fd \
-drive if=virtio,format=raw,snapshot=on,cache=unsafe,file=core/smalltown.img \
- -netdev user,id=net0,net=10.42.0.0/24,dhcpstart=10.42.0.10,hostfwd=tcp::7833-:7833,hostfwd=tcp::7834-:7834,hostfwd=tcp::6443-:6443,hostfwd=tcp::7835-:7835 \
+ -netdev user,id=net0,net=10.42.0.0/24,dhcpstart=10.42.0.10,hostfwd=tcp::7833-:7833,hostfwd=tcp::7834-:7834,hostfwd=tcp::6443-:6443,hostfwd=tcp::7835-:7835,hostfwd=tcp::7837-:7837 \
-device virtio-net-pci,netdev=net0 \
-chardev socket,id=chrtpm,path=tpm-socket \
-tpmdev emulator,id=tpm0,chardev=chrtpm \
diff --git a/third_party/go/patches/k8s-native-metrics.patch b/third_party/go/patches/k8s-native-metrics.patch
new file mode 100644
index 0000000..9568764
--- /dev/null
+++ b/third_party/go/patches/k8s-native-metrics.patch
@@ -0,0 +1,274 @@
+Copyright 2020 The Monogon Project Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+From b16e57cc52a437465bbd12c24fb05fe5790afe1d Mon Sep 17 00:00:00 2001
+From: Lorenz Brun <lorenz@brun.one>
+Date: Tue, 17 Mar 2020 21:41:08 +0100
+Subject: [PATCH 2/3] Add a native volume metrics implementation
+
+---
+ pkg/volume/BUILD | 3 +
+ pkg/volume/metrics_native.go | 101 +++++++++++++++++++++++++++++
+ pkg/volume/metrics_native_test.go | 102 ++++++++++++++++++++++++++++++
+ 3 files changed, 206 insertions(+)
+ create mode 100644 pkg/volume/metrics_native.go
+ create mode 100644 pkg/volume/metrics_native_test.go
+
+diff --git a/pkg/volume/BUILD b/pkg/volume/BUILD
+index 720b13406dc..b6e4b7e6d6f 100644
+--- a/pkg/volume/BUILD
++++ b/pkg/volume/BUILD
+@@ -7,6 +7,7 @@ go_library(
+ "metrics_cached.go",
+ "metrics_du.go",
+ "metrics_errors.go",
++ "metrics_native.go",
+ "metrics_nil.go",
+ "metrics_statfs.go",
+ "noop_expandable_plugin.go",
+@@ -35,6 +36,7 @@ go_library(
+ "@io_k8s_client_go//tools/cache:go_default_library",
+ "@io_k8s_client_go//tools/record:go_default_library",
+ "@io_k8s_cloud_provider//:go_default_library",
++ "@org_golang_x_sys//unix:go_default_library",
+ "@io_k8s_klog//:go_default_library",
+ "@io_k8s_utils//exec:go_default_library",
+ "@io_k8s_utils//mount:go_default_library",
+@@ -55,6 +57,7 @@ go_test(
+ name = "go_default_test",
+ srcs = [
+ "metrics_du_test.go",
++ "metrics_native_test.go",
+ "metrics_nil_test.go",
+ "metrics_statfs_test.go",
+ "plugins_test.go",
+diff --git a/pkg/volume/metrics_native.go b/pkg/volume/metrics_native.go
+new file mode 100644
+index 00000000000..3934b946f2e
+--- /dev/null
++++ b/pkg/volume/metrics_native.go
+@@ -0,0 +1,101 @@
++/*
++Copyright 2020 The Kubernetes Authors.
++
++Licensed under the Apache License, Version 2.0 (the "License");
++you may not use this file except in compliance with the License.
++You may obtain a copy of the License at
++
++ http://www.apache.org/licenses/LICENSE-2.0
++
++Unless required by applicable law or agreed to in writing, software
++distributed under the License is distributed on an "AS IS" BASIS,
++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++See the License for the specific language governing permissions and
++limitations under the License.
++*/
++
++package volume
++
++import (
++ "os"
++ "path/filepath"
++
++ "golang.org/x/sys/unix"
++ "k8s.io/apimachinery/pkg/api/resource"
++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
++)
++
++var _ MetricsProvider = &metricsNative{}
++
++// MetricsNative represents a MetricsProvider that calculates the volume metrics with either a quota
++// or by walking the path using syscalls.
++type metricsNative struct {
++ path string
++}
++
++// NewMetricsNative returns a new metricsNative provider
++func NewMetricsNative(path string) MetricsProvider {
++ return &metricsNative{path}
++}
++
++// GetMetrics returns an empty Metrics and an error.
++// See MetricsProvider.GetMetrics
++func (m *metricsNative) GetMetrics() (*Metrics, error) {
++ var inodesCount int64
++ var bytesCount int64
++
++ var getMetricsRecursive func(path string) error
++ getMetricsRecursive = func(path string) error {
++ var stat unix.Stat_t
++ err := unix.Lstat(path, &stat)
++ if os.IsNotExist(err) {
++ return nil
++ } else if err != nil {
++ return err
++ }
++ // TODO: This double-counts hardlinks
++ bytesCount += stat.Blocks * 512
++ inodesCount++
++ if stat.Mode&unix.S_IFDIR != 0 && stat.Mode&unix.S_IFLNK == 0 {
++ fd, err := os.Open(path)
++ if os.IsNotExist(err) {
++ return nil
++ } else if err != nil {
++ return err
++ }
++ // We manually close fd before recursing, otherwise we have too many FDs open
++
++ entries, err := fd.Readdirnames(0)
++ if err != nil {
++ fd.Close()
++ return err
++ }
++ fd.Close()
++ for _, entry := range entries {
++ if err := getMetricsRecursive(filepath.Join(path, entry)); err != nil {
++ return err
++ }
++ }
++ }
++ return nil
++ }
++
++ if err := getMetricsRecursive(m.path); err != nil {
++ return &Metrics{}, err
++ }
++
++ var statfs unix.Statfs_t
++ if err := unix.Statfs(m.path, &statfs); err != nil {
++ return &Metrics{}, err
++ }
++
++ return &Metrics{
++ Time: metav1.Now(),
++ Used: resource.NewQuantity(bytesCount, resource.BinarySI),
++ InodesUsed: resource.NewQuantity(inodesCount, resource.BinarySI),
++ Available: resource.NewQuantity(int64(statfs.Bavail)*statfs.Bsize, resource.BinarySI),
++ Capacity: resource.NewQuantity(int64(statfs.Blocks)*statfs.Bsize, resource.BinarySI),
++ Inodes: resource.NewQuantity(int64(statfs.Files)*statfs.Bsize, resource.BinarySI),
++ InodesFree: resource.NewQuantity(int64(statfs.Ffree)*statfs.Bsize, resource.BinarySI),
++ }, nil
++}
+diff --git a/pkg/volume/metrics_native_test.go b/pkg/volume/metrics_native_test.go
+new file mode 100644
+index 00000000000..2d5546591ce
+--- /dev/null
++++ b/pkg/volume/metrics_native_test.go
+@@ -0,0 +1,102 @@
++// +build linux
++
++/*
++Copyright 2015 The Kubernetes Authors.
++
++Licensed under the Apache License, Version 2.0 (the "License");
++you may not use this file except in compliance with the License.
++You may obtain a copy of the License at
++
++ http://www.apache.org/licenses/LICENSE-2.0
++
++Unless required by applicable law or agreed to in writing, software
++distributed under the License is distributed on an "AS IS" BASIS,
++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++See the License for the specific language governing permissions and
++limitations under the License.
++*/
++
++package volume_test
++
++import (
++ "io/ioutil"
++ "os"
++ "path/filepath"
++ "testing"
++
++ utiltesting "k8s.io/client-go/util/testing"
++ . "k8s.io/kubernetes/pkg/volume"
++ volumetest "k8s.io/kubernetes/pkg/volume/testing"
++)
++
++// TestMetricsNativeGetCapacity tests that MetricsNative can read disk usage
++// for path
++func TestMetricsNativeGetCapacity(t *testing.T) {
++ tmpDir, err := utiltesting.MkTmpdir("metrics_du_test")
++ if err != nil {
++ t.Fatalf("Can't make a tmp dir: %v", err)
++ }
++ defer os.RemoveAll(tmpDir)
++ metrics := NewMetricsNative(tmpDir)
++
++ expectedEmptyDirUsage, err := volumetest.FindEmptyDirectoryUsageOnTmpfs()
++ if err != nil {
++ t.Errorf("Unexpected error finding expected empty directory usage on tmpfs: %v", err)
++ }
++
++ actual, err := metrics.GetMetrics()
++ if err != nil {
++ t.Errorf("Unexpected error when calling GetMetrics %v", err)
++ }
++ if e, a := expectedEmptyDirUsage.Value(), actual.Used.Value(); e != a {
++ t.Errorf("Unexpected value for empty directory; expected %v, got %v", e, a)
++ }
++
++ // TODO(pwittroc): Figure out a way to test these values for correctness, maybe by formatting and mounting a file
++ // as a filesystem
++ if a := actual.Capacity.Value(); a <= 0 {
++ t.Errorf("Expected Capacity %d to be greater than 0.", a)
++ }
++ if a := actual.Available.Value(); a <= 0 {
++ t.Errorf("Expected Available %d to be greater than 0.", a)
++ }
++
++ // Write a file in a directory and expect Used to increase
++ os.MkdirAll(filepath.Join(tmpDir, "d1"), 0755)
++ ioutil.WriteFile(filepath.Join(tmpDir, "d1", "f1"), []byte("Hello World"), os.ModeTemporary)
++ actual, err = metrics.GetMetrics()
++ if err != nil {
++ t.Errorf("Unexpected error when calling GetMetrics %v", err)
++ }
++ if e, a := (2*expectedEmptyDirUsage.Value() + getExpectedBlockSize(filepath.Join(tmpDir, "d1", "f1"))), actual.Used.Value(); e != a {
++ t.Errorf("Unexpected Used for directory with file. Expected %v, got %d.", e, a)
++ }
++}
++
++// TestMetricsNativeRequireInit tests that if MetricsNative is not initialized with a path, GetMetrics
++// returns an error
++func TestMetricsNativeRequirePath(t *testing.T) {
++ metrics := NewMetricsNative("")
++ actual, err := metrics.GetMetrics()
++ expected := &Metrics{}
++ if !volumetest.MetricsEqualIgnoreTimestamp(actual, expected) {
++ t.Errorf("Expected empty Metrics from uninitialized MetricsNative, actual %v", *actual)
++ }
++ if err == nil {
++ t.Errorf("Expected error when calling GetMetrics on uninitialized MetricsNative, actual nil")
++ }
++}
++
++// TestMetricsNativeRealDirectory tests that if MetricsNative is initialized to a non-existent path, GetMetrics
++// returns an error
++func TestMetricsNativeRequireRealDirectory(t *testing.T) {
++ metrics := NewMetricsNative("/not/a/real/directory")
++ actual, err := metrics.GetMetrics()
++ expected := &Metrics{}
++ if !volumetest.MetricsEqualIgnoreTimestamp(actual, expected) {
++ t.Errorf("Expected empty Metrics from incorrectly initialized MetricsNative, actual %v", *actual)
++ }
++ if err == nil {
++ t.Errorf("Expected error when calling GetMetrics on incorrectly initialized MetricsNative, actual nil")
++ }
++}
+--
+2.25.1
+
diff --git a/third_party/go/patches/k8s-native-mounter.patch b/third_party/go/patches/k8s-native-mounter.patch
new file mode 100644
index 0000000..5e95bfd
--- /dev/null
+++ b/third_party/go/patches/k8s-native-mounter.patch
@@ -0,0 +1,249 @@
+Copyright 2020 The Monogon Project Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+From 8335005ed1983ca5ac036af15dd04b8717898c35 Mon Sep 17 00:00:00 2001
+From: Lorenz Brun <lorenz@brun.one>
+Date: Mon, 16 Mar 2020 22:13:08 +0100
+Subject: [PATCH 1/3] Provide native mounter implementation for Linux
+
+---
+ BUILD.bazel | 2 +
+ mount/mount_linux.go | 141 ++++++++++++++++++++++-
+ 2 files changed, 139 insertions(+), 4 deletions(-)
+
+diff --git a/mount/BUILD b/mount/BUILD.bazel
+index bef3ec2cf55..6f997103dac 100644
+--- a/mount/BUILD.bazel
++++ b/mount/BUILD.bazel
+@@ -21,6 +21,7 @@ go_library(
+ "//exec:go_default_library",
+ ] + select({
+ "@io_bazel_rules_go//go/platform:android": [
++ "@org_golang_x_sys//unix:go_default_library",
+ "//io:go_default_library",
+ ],
+ "@io_bazel_rules_go//go/platform:darwin": [
+@@ -36,6 +37,7 @@ go_library(
+ "//io:go_default_library",
+ ],
+ "@io_bazel_rules_go//go/platform:linux": [
++ "@org_golang_x_sys//unix:go_default_library",
+ "//io:go_default_library",
+ ],
+ "@io_bazel_rules_go//go/platform:nacl": [
+diff --git a/mount/mount_linux.go b/mount/mount_linux.go
+index 41f69efe3f0..01182684653 100644
+--- a/mount/mount_linux.go
++++ b/mount/mount_linux.go
+@@ -20,6 +20,7 @@ package mount
+
+ import (
+ "fmt"
++ "io/ioutil"
+ "os"
+ "os/exec"
+ "path/filepath"
+@@ -27,6 +28,7 @@ import (
+ "strings"
+ "syscall"
+
++ "golang.org/x/sys/unix"
+ "k8s.io/klog"
+ utilexec "k8s.io/utils/exec"
+ utilio "k8s.io/utils/io"
+@@ -49,8 +51,10 @@ const (
+ // for the linux platform. This implementation assumes that the
+ // kubelet is running in the host's root mount namespace.
+ type Mounter struct {
+- mounterPath string
+- withSystemd bool
++ mounterPath string
++ withSystemd bool
++ withLinuxUtils bool
++ nativeSupportedFstypes map[string]struct{}
+ }
+
+ // New returns a mount.Interface for the current system.
+@@ -58,8 +62,10 @@ type Mounter struct {
+ // mounterPath allows using an alternative to `/bin/mount` for mounting.
+ func New(mounterPath string) Interface {
+ return &Mounter{
+- mounterPath: mounterPath,
+- withSystemd: detectSystemd(),
++ mounterPath: mounterPath,
++ withSystemd: detectSystemd(),
++ withLinuxUtils: detectLinuxUtils(),
++ nativeSupportedFstypes: detectNativeSupportedFstypes(),
+ }
+ }
+
+@@ -78,6 +84,29 @@ func (mounter *Mounter) Mount(source string, target string, fstype string, optio
+ // method should be used by callers that pass sensitive material (like
+ // passwords) as mount options.
+ func (mounter *Mounter) MountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error {
++ if !mounter.withLinuxUtils {
++ flags, pflags, fsoptions := parseMountOptions(options)
++ if len(pflags) > 0 {
++ return fmt.Errorf("the native mounter is active and does not support mount propagation at the moment")
++ }
++
++ if !mounter.nativeSupportsFstype(fstype) && flags&unix.MS_BIND == 0 {
++ return fmt.Errorf("the native mounter is active and cannot mount filesystems of type \"%v\"", fstype)
++ }
++
++ if flags&unix.MS_BIND != 0 && flags & ^uintptr(unix.MS_BIND) != 0 {
++ if err := unix.Mount(source, target, "", unix.MS_BIND, ""); err != nil {
++ return fmt.Errorf("bind pre-mount failed: %w", err)
++ }
++ flags |= unix.MS_REMOUNT
++ }
++
++ if err := unix.Mount(source, target, fstype, flags, fsoptions); err != nil {
++ return fmt.Errorf("failed to mount filesystem: %w", err)
++ }
++ return nil
++ }
++
+ // Path to mounter binary if containerized mounter is needed. Otherwise, it is set to empty.
+ // All Linux distros are expected to be shipped with a mount utility that a support bind mounts.
+ mounterPath := ""
+@@ -102,6 +131,80 @@ func (mounter *Mounter) MountSensitive(source string, target string, fstype stri
+ return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, options, sensitiveOptions)
+ }
+
++// nativeSupportsFstype checks if the native mounter can mount the given fstype
++func (mounter *Mounter) nativeSupportsFstype(fstype string) bool {
++ _, ok := mounter.nativeSupportedFstypes[fstype]
++ return ok
++}
++
++// parseMountOptions parses the string and returns the flags, propagation
++// flags and any mount data that it contains.
++// Taken from libcontainer/specconv/spec_linux.go (Apache 2.0) and modified
++func parseMountOptions(options []string) (uintptr, []uintptr, string) {
++ var (
++ flag uintptr
++ pgflag []uintptr
++ data []string
++ )
++ flags := map[string]struct {
++ clear bool
++ flag uintptr
++ }{
++ "async": {true, syscall.MS_SYNCHRONOUS},
++ "atime": {true, syscall.MS_NOATIME},
++ "bind": {false, syscall.MS_BIND},
++ "defaults": {false, 0},
++ "dev": {true, syscall.MS_NODEV},
++ "diratime": {true, syscall.MS_NODIRATIME},
++ "dirsync": {false, syscall.MS_DIRSYNC},
++ "exec": {true, syscall.MS_NOEXEC},
++ "mand": {false, syscall.MS_MANDLOCK},
++ "noatime": {false, syscall.MS_NOATIME},
++ "nodev": {false, syscall.MS_NODEV},
++ "nodiratime": {false, syscall.MS_NODIRATIME},
++ "noexec": {false, syscall.MS_NOEXEC},
++ "nomand": {true, syscall.MS_MANDLOCK},
++ "norelatime": {true, syscall.MS_RELATIME},
++ "nostrictatime": {true, syscall.MS_STRICTATIME},
++ "nosuid": {false, syscall.MS_NOSUID},
++ "rbind": {false, syscall.MS_BIND | syscall.MS_REC},
++ "relatime": {false, syscall.MS_RELATIME},
++ "remount": {false, syscall.MS_REMOUNT},
++ "ro": {false, syscall.MS_RDONLY},
++ "rw": {true, syscall.MS_RDONLY},
++ "strictatime": {false, syscall.MS_STRICTATIME},
++ "suid": {true, syscall.MS_NOSUID},
++ "sync": {false, syscall.MS_SYNCHRONOUS},
++ }
++ propagationFlags := map[string]uintptr{
++ "private": syscall.MS_PRIVATE,
++ "shared": syscall.MS_SHARED,
++ "slave": syscall.MS_SLAVE,
++ "unbindable": syscall.MS_UNBINDABLE,
++ "rprivate": syscall.MS_PRIVATE | syscall.MS_REC,
++ "rshared": syscall.MS_SHARED | syscall.MS_REC,
++ "rslave": syscall.MS_SLAVE | syscall.MS_REC,
++ "runbindable": syscall.MS_UNBINDABLE | syscall.MS_REC,
++ }
++ for _, o := range options {
++ // If the option does not exist in the flags table or the flag
++ // is not supported on the platform,
++ // then it is a data value for a specific fs type
++ if f, exists := flags[o]; exists && f.flag != 0 {
++ if f.clear {
++ flag &= ^f.flag
++ } else {
++ flag |= f.flag
++ }
++ } else if f, exists := propagationFlags[o]; exists && f != 0 {
++ pgflag = append(pgflag, f)
++ } else {
++ data = append(data, o)
++ }
++ }
++ return flag, pgflag, strings.Join(data, ",")
++}
++
+ // doMount runs the mount command. mounterPath is the path to mounter binary if containerized mounter is used.
+ // sensitiveOptions is an extention of options except they will not be logged (because they may contain sensitive material)
+ func (mounter *Mounter) doMount(mounterPath string, mountCmd string, source string, target string, fstype string, options []string, sensitiveOptions []string) error {
+@@ -179,6 +282,30 @@ func detectSystemd() bool {
+ return true
+ }
+
++// detectLinuxUtils detects if the host operating system has the mount and unmount commands present
++func detectLinuxUtils() bool {
++ _, err := exec.LookPath("mount")
++ return err == nil
++}
++
++func detectNativeSupportedFstypes() map[string]struct{} {
++ nativeSupportedFstypes := make(map[string]struct{})
++ filesystemsRaw, err := ioutil.ReadFile("/proc/filesystems")
++ if err != nil {
++ return nativeSupportedFstypes
++ }
++ filesystemLines := strings.Split(string(filesystemsRaw), "\n")
++ for _, line := range filesystemLines {
++ fields := strings.Fields(line)
++ if len(fields) != 2 {
++ continue
++ }
++ filesystem := fields[1]
++ nativeSupportedFstypes[filesystem] = struct{}{}
++ }
++ return nativeSupportedFstypes
++}
++
+ // MakeMountArgs makes the arguments to the mount(8) command.
+ // options MUST not contain sensitive material (like passwords).
+ func MakeMountArgs(source, target, fstype string, options []string) (mountArgs []string) {
+@@ -236,6 +363,12 @@ func AddSystemdScopeSensitive(systemdRunPath, mountName, command string, args []
+ // Unmount unmounts the target.
+ func (mounter *Mounter) Unmount(target string) error {
+ klog.V(4).Infof("Unmounting %s", target)
++ if !mounter.withLinuxUtils {
++ if err := unix.Unmount(target, unix.UMOUNT_NOFOLLOW); err != nil {
++ return fmt.Errorf("unmount failed: %v", err)
++ }
++ return nil
++ }
+ command := exec.Command("umount", target)
+ output, err := command.CombinedOutput()
+ if err != nil {
+--
+2.25.1
+
diff --git a/third_party/go/patches/k8s-use-native.patch b/third_party/go/patches/k8s-use-native.patch
new file mode 100644
index 0000000..61001da
--- /dev/null
+++ b/third_party/go/patches/k8s-use-native.patch
@@ -0,0 +1,153 @@
+Copyright 2020 The Monogon Project Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+From ee4a7df588550ee5cbc3b8419e1ce185a8abb302 Mon Sep 17 00:00:00 2001
+From: Lorenz Brun <lorenz@brun.one>
+Date: Tue, 17 Mar 2020 22:07:24 +0100
+Subject: [PATCH 3/3] Use StatFS and Native volume metrics instead of du
+
+---
+ pkg/kubelet/stats/log_metrics_provider.go | 2 +-
+ pkg/volume/configmap/configmap.go | 4 ++--
+ pkg/volume/downwardapi/downwardapi.go | 4 ++--
+ pkg/volume/emptydir/empty_dir.go | 4 ++--
+ pkg/volume/projected/projected.go | 4 ++--
+ pkg/volume/secret/secret.go | 4 ++--
+ 6 files changed, 11 insertions(+), 11 deletions(-)
+
+diff --git a/pkg/kubelet/stats/log_metrics_provider.go b/pkg/kubelet/stats/log_metrics_provider.go
+index 4a53eef74a3..ff87fec5ec3 100644
+--- a/pkg/kubelet/stats/log_metrics_provider.go
++++ b/pkg/kubelet/stats/log_metrics_provider.go
+@@ -33,5 +33,5 @@ func NewLogMetricsService() LogMetricsService {
+ }
+
+ func (l logMetrics) createLogMetricsProvider(path string) volume.MetricsProvider {
+- return volume.NewMetricsDu(path)
++ return volume.NewMetricsNative(path)
+ }
+diff --git a/pkg/volume/configmap/configmap.go b/pkg/volume/configmap/configmap.go
+index 0e74dd0a1d8..430d739aab7 100644
+--- a/pkg/volume/configmap/configmap.go
++++ b/pkg/volume/configmap/configmap.go
+@@ -97,7 +97,7 @@ func (plugin *configMapPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts v
+ pod.UID,
+ plugin,
+ plugin.host.GetMounter(plugin.GetPluginName()),
+- volume.NewCachedMetrics(volume.NewMetricsDu(getPath(pod.UID, spec.Name(), plugin.host))),
++ volume.NewCachedMetrics(volume.NewMetricsNative(getPath(pod.UID, spec.Name(), plugin.host))),
+ },
+ source: *spec.Volume.ConfigMap,
+ pod: *pod,
+@@ -113,7 +113,7 @@ func (plugin *configMapPlugin) NewUnmounter(volName string, podUID types.UID) (v
+ podUID,
+ plugin,
+ plugin.host.GetMounter(plugin.GetPluginName()),
+- volume.NewCachedMetrics(volume.NewMetricsDu(getPath(podUID, volName, plugin.host))),
++ volume.NewCachedMetrics(volume.NewMetricsNative(getPath(podUID, volName, plugin.host))),
+ },
+ }, nil
+ }
+diff --git a/pkg/volume/downwardapi/downwardapi.go b/pkg/volume/downwardapi/downwardapi.go
+index a1779c0dac9..f0a0f99b318 100644
+--- a/pkg/volume/downwardapi/downwardapi.go
++++ b/pkg/volume/downwardapi/downwardapi.go
+@@ -99,7 +99,7 @@ func (plugin *downwardAPIPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts
+ pod: pod,
+ podUID: pod.UID,
+ plugin: plugin,
+- MetricsProvider: volume.NewCachedMetrics(volume.NewMetricsDu(getPath(pod.UID, spec.Name(), plugin.host))),
++ MetricsProvider: volume.NewCachedMetrics(volume.NewMetricsStatFS(getPath(pod.UID, spec.Name(), plugin.host))),
+ }
+ return &downwardAPIVolumeMounter{
+ downwardAPIVolume: v,
+@@ -114,7 +114,7 @@ func (plugin *downwardAPIPlugin) NewUnmounter(volName string, podUID types.UID)
+ volName: volName,
+ podUID: podUID,
+ plugin: plugin,
+- MetricsProvider: volume.NewCachedMetrics(volume.NewMetricsDu(getPath(podUID, volName, plugin.host))),
++ MetricsProvider: volume.NewCachedMetrics(volume.NewMetricsStatFS(getPath(podUID, volName, plugin.host))),
+ },
+ }, nil
+ }
+diff --git a/pkg/volume/emptydir/empty_dir.go b/pkg/volume/emptydir/empty_dir.go
+index 0a25d2b684c..5dc83b90c5b 100644
+--- a/pkg/volume/emptydir/empty_dir.go
++++ b/pkg/volume/emptydir/empty_dir.go
+@@ -121,7 +121,7 @@ func (plugin *emptyDirPlugin) newMounterInternal(spec *volume.Spec, pod *v1.Pod,
+ mounter: mounter,
+ mountDetector: mountDetector,
+ plugin: plugin,
+- MetricsProvider: volume.NewMetricsDu(getPath(pod.UID, spec.Name(), plugin.host)),
++ MetricsProvider: volume.NewMetricsNative(getPath(pod.UID, spec.Name(), plugin.host)),
+ }, nil
+ }
+
+@@ -138,7 +138,7 @@ func (plugin *emptyDirPlugin) newUnmounterInternal(volName string, podUID types.
+ mounter: mounter,
+ mountDetector: mountDetector,
+ plugin: plugin,
+- MetricsProvider: volume.NewMetricsDu(getPath(podUID, volName, plugin.host)),
++ MetricsProvider: volume.NewMetricsNative(getPath(podUID, volName, plugin.host)),
+ }
+ return ed, nil
+ }
+diff --git a/pkg/volume/projected/projected.go b/pkg/volume/projected/projected.go
+index 0f65a97610c..890f9c1c7bc 100644
+--- a/pkg/volume/projected/projected.go
++++ b/pkg/volume/projected/projected.go
+@@ -114,7 +114,7 @@ func (plugin *projectedPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts v
+ sources: spec.Volume.Projected.Sources,
+ podUID: pod.UID,
+ plugin: plugin,
+- MetricsProvider: volume.NewCachedMetrics(volume.NewMetricsDu(getPath(pod.UID, spec.Name(), plugin.host))),
++ MetricsProvider: volume.NewCachedMetrics(volume.NewMetricsStatFS(getPath(pod.UID, spec.Name(), plugin.host))),
+ },
+ source: *spec.Volume.Projected,
+ pod: pod,
+@@ -128,7 +128,7 @@ func (plugin *projectedPlugin) NewUnmounter(volName string, podUID types.UID) (v
+ volName: volName,
+ podUID: podUID,
+ plugin: plugin,
+- MetricsProvider: volume.NewCachedMetrics(volume.NewMetricsDu(getPath(podUID, volName, plugin.host))),
++ MetricsProvider: volume.NewCachedMetrics(volume.NewMetricsStatFS(getPath(podUID, volName, plugin.host))),
+ },
+ }, nil
+ }
+diff --git a/pkg/volume/secret/secret.go b/pkg/volume/secret/secret.go
+index a195c59ddd8..4c290cb8f24 100644
+--- a/pkg/volume/secret/secret.go
++++ b/pkg/volume/secret/secret.go
+@@ -100,7 +100,7 @@ func (plugin *secretPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts volu
+ pod.UID,
+ plugin,
+ plugin.host.GetMounter(plugin.GetPluginName()),
+- volume.NewCachedMetrics(volume.NewMetricsDu(getPath(pod.UID, spec.Name(), plugin.host))),
++ volume.NewCachedMetrics(volume.NewMetricsStatFS(getPath(pod.UID, spec.Name(), plugin.host))),
+ },
+ source: *spec.Volume.Secret,
+ pod: *pod,
+@@ -116,7 +116,7 @@ func (plugin *secretPlugin) NewUnmounter(volName string, podUID types.UID) (volu
+ podUID,
+ plugin,
+ plugin.host.GetMounter(plugin.GetPluginName()),
+- volume.NewCachedMetrics(volume.NewMetricsDu(getPath(podUID, volName, plugin.host))),
++ volume.NewCachedMetrics(volume.NewMetricsStatFS(getPath(podUID, volName, plugin.host))),
+ },
+ }, nil
+ }
+--
+2.25.1
+
diff --git a/third_party/go/repositories.bzl b/third_party/go/repositories.bzl
index 00e92e2..c129802 100644
--- a/third_party/go/repositories.bzl
+++ b/third_party/go/repositories.bzl
@@ -828,6 +828,12 @@
sum = "h1:742eGXur0715JMq73aD95/FU0XpVKXqNuTnEfXsLOYQ=",
)
go_repository(
+ name = "com_github_joho_godotenv",
+ importpath = "github.com/joho/godotenv",
+ version = "v1.3.0",
+ sum = "h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=",
+ )
+ go_repository(
name = "com_github_jonboulle_clockwork",
importpath = "github.com/jonboulle/clockwork",
version = "v0.1.0",
@@ -1667,6 +1673,8 @@
patches = [
"//third_party/go/patches:k8s-kubernetes.patch",
"//third_party/go/patches:k8s-kubernetes-build.patch",
+ "//third_party/go/patches:k8s-native-metrics.patch",
+ "//third_party/go/patches:k8s-use-native.patch",
],
patch_args = ["-p1"],
)
@@ -1700,6 +1708,10 @@
importpath = "k8s.io/utils",
version = "v0.0.0-20200324210504-a9aa75ae1b89",
sum = "h1:d4vVOjXm687F1iLSP2q3lyPPuyvTUt3aVoBpi2DqRsU=",
+ patches = [
+ "//third_party/go/patches:k8s-native-mounter.patch",
+ ],
+ patch_args = ["-p1"],
)
go_repository(
name = "io_k8s_sigs_apiserver_network_proxy_konnectivity_client",
diff --git a/third_party/go/shelf.pb.text b/third_party/go/shelf.pb.text
index 2fb4394..1673d67 100644
--- a/third_party/go/shelf.pb.text
+++ b/third_party/go/shelf.pb.text
@@ -972,6 +972,13 @@
semver: "v0.0.0-20160618110441-2cf9dc699c56"
>
entry: <
+ import_path: "github.com/joho/godotenv"
+ version: "v1.3.0"
+ bazel_name: "com_github_joho_godotenv"
+ sum: "h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc="
+ semver: "v1.3.0"
+>
+entry: <
import_path: "github.com/jonboulle/clockwork"
version: "v0.1.0"
bazel_name: "com_github_jonboulle_clockwork"