Add Kubernetes Worker and infrastructure
Adds Kubernetes Kubelet with patches for syscall-based mounting and
syscall-based (and much faster) metrics. fsquota patches have been
deferred to a further revision (for robust emptyDir capacity isolation).
Changes encoding of the node ID to hex since Base64-URL is not supported
as a character set for K8s names. Also adds `/etc/machine-id` and
`/etc/os-release` since Kubernetes wants them. `os-release` is generated
by stamping, `machine-id` is the hex-encoded node ID derived from the
public key.
Also includes a primitive reconciler which automatically ensures a set of
built-in Kubernetes objects are always present. Currently this includes
a PSP and some basic RBAC policies that are elementary to proper cluster
operations.
Adds an additional gRPC service (NodeDebugService) to cleanly
communicate with external debug and test tooling. It supports reading
from logbuffers for all externally-run components, checking conditions
(for replacing log matching in testing and debugging) and getting
debug credentials for the Kubernetes cluster.
A small utility (dbg) is provided that interfaces with NodeDebugService
and provides access to its functions from the CLI. It also incorporates
a kubectl wrapper which directly grabs credentials from the Debug API
and passes them to kubectl
(e.g. `bazel run //core/cmd/dbg -- kubectl describe node`).
Test Plan:
Manually tested.
Kubernetes:
`bazel run //core/cmd/dbg -- kubectl create -f test.yml`
Checked that pods run, logs are accessible and exec works.
Reading buffers:
`bazel run //core/cmd/dbg -- logs containerd`
Outputs containerd logs in the right order.
Automated testing is in the works, but has been deferred to a future
revision because this one is already too big again.
X-Origin-Diff: phab/D525
GitOrigin-RevId: 0fbfa0c433de405526c7f09ef10c466896331328
diff --git a/core/cmd/dbg/BUILD.bazel b/core/cmd/dbg/BUILD.bazel
new file mode 100644
index 0000000..7088ea1
--- /dev/null
+++ b/core/cmd/dbg/BUILD.bazel
@@ -0,0 +1,23 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+ name = "go_default_library",
+ srcs = ["main.go"],
+ importpath = "git.monogon.dev/source/nexantic.git/core/cmd/dbg",
+ visibility = ["//visibility:private"],
+ deps = [
+ "//core/api/api:go_default_library",
+ "@com_github_spf13_pflag//:go_default_library",
+ "@io_k8s_component_base//cli/flag:go_default_library",
+ "@io_k8s_kubectl//pkg/cmd/plugin:go_default_library",
+ "@io_k8s_kubectl//pkg/util/logs:go_default_library",
+ "@io_k8s_kubernetes//pkg/kubectl/cmd:go_default_library",
+ "@org_golang_google_grpc//:go_default_library",
+ ],
+)
+
+go_binary(
+ name = "dbg",
+ embed = [":go_default_library"],
+ visibility = ["//visibility:public"],
+)
diff --git a/core/cmd/dbg/main.go b/core/cmd/dbg/main.go
new file mode 100644
index 0000000..44803ec
--- /dev/null
+++ b/core/cmd/dbg/main.go
@@ -0,0 +1,125 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "io/ioutil"
+ "math/rand"
+ "os"
+ "strings"
+ "time"
+
+ "github.com/spf13/pflag"
+ "google.golang.org/grpc"
+ cliflag "k8s.io/component-base/cli/flag"
+ "k8s.io/kubectl/pkg/cmd/plugin"
+ "k8s.io/kubectl/pkg/util/logs"
+ "k8s.io/kubernetes/pkg/kubectl/cmd"
+
+ apipb "git.monogon.dev/source/nexantic.git/core/generated/api"
+)
+
+func main() {
+ // Hardcode localhost since this should never be used to interface with a production node because of missing
+ // encryption & authentication
+ grpcClient, err := grpc.Dial("localhost:7837", grpc.WithInsecure())
+ if err != nil {
+ fmt.Printf("Failed to dial debug service (is it running): %v\n", err)
+ }
+ debugClient := apipb.NewNodeDebugServiceClient(grpcClient)
+ if len(os.Args) < 2 {
+ fmt.Println("Please specify a subcommand")
+ os.Exit(1)
+ }
+
+ logsCmd := flag.NewFlagSet("logs", flag.ExitOnError)
+ logsTailN := logsCmd.Uint("tail", 0, "Get last n lines (0 = whole buffer)")
+ logsCmd.Usage = func() {
+ fmt.Fprintf(os.Stderr, "Usage: %s %s [options] component_path\n", os.Args[0], os.Args[1])
+ flag.PrintDefaults()
+
+ fmt.Fprintf(os.Stderr, "Example:\n %s %s --tail 5 kube.apiserver\n", os.Args[0], os.Args[1])
+ }
+ conditionCmd := flag.NewFlagSet("condition", flag.ExitOnError)
+ conditionCmd.Usage = func() {
+ fmt.Fprintf(os.Stderr, "Usage: %s %s [options] component_path\n", os.Args[0], os.Args[1])
+ flag.PrintDefaults()
+
+ fmt.Fprintf(os.Stderr, "Example:\n %s %s IPAssigned\n", os.Args[0], os.Args[1])
+ }
+ switch os.Args[1] {
+ case "logs":
+ logsCmd.Parse(os.Args[2:])
+ componentPath := strings.Split(logsCmd.Arg(0), ".")
+ res, err := debugClient.GetComponentLogs(context.Background(), &apipb.GetComponentLogsRequest{ComponentPath: componentPath, TailLines: uint32(*logsTailN)})
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Failed to get logs: %v\n", err)
+ os.Exit(1)
+ }
+ for _, line := range res.Line {
+ fmt.Println(line)
+ }
+ return
+ case "condition":
+ conditionCmd.Parse(os.Args[2:])
+ condition := conditionCmd.Arg(0)
+ res, err := debugClient.GetCondition(context.Background(), &apipb.GetConditionRequest{Name: condition})
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Failed to get condition: %v\n", err)
+ os.Exit(1)
+ }
+ fmt.Println(res.Ok)
+ case "kubectl":
+ // Always get a kubeconfig with cluster-admin (group system:masters), kubectl itself can impersonate
+ kubeconfigFile, err := ioutil.TempFile("", "dbg_kubeconfig")
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Failed to create kubeconfig temp file: %v\n", err)
+ os.Exit(1)
+ }
+ defer kubeconfigFile.Close()
+ defer os.Remove(kubeconfigFile.Name())
+
+ res, err := debugClient.GetDebugKubeconfig(context.Background(), &apipb.GetDebugKubeconfigRequest{Id: "debug-user", Groups: []string{"system:masters"}})
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Failed to get kubeconfig: %v\n", err)
+ os.Exit(1)
+ }
+ if _, err := kubeconfigFile.WriteString(res.DebugKubeconfig); err != nil {
+ fmt.Fprintf(os.Stderr, "Failed to write kubeconfig: %v\n", err)
+ os.Exit(1)
+ }
+
+ // This magic sets up everything as if this was just the kubectl binary. It sets the KUBECONFIG environment
+ // variable so that it knows where the Kubeconfig is located and forcibly overwrites the arguments so that
+ // the "wrapper" arguments are not visible to its flags parser. The base code is straight from
+ // https://github.com/kubernetes/kubernetes/blob/master/cmd/kubectl/kubectl.go
+ os.Setenv("KUBECONFIG", kubeconfigFile.Name())
+ rand.Seed(time.Now().UnixNano())
+ pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)
+ pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
+ logs.InitLogs()
+ defer logs.FlushLogs()
+ command := cmd.NewDefaultKubectlCommandWithArgs(cmd.NewDefaultPluginHandler(plugin.ValidPluginFilenamePrefixes), os.Args[2:], os.Stdin, os.Stdout, os.Stderr)
+ command.SetArgs(os.Args[2:])
+ if err := command.Execute(); err != nil {
+ os.Exit(1)
+ }
+ }
+}
diff --git a/core/cmd/init/main.go b/core/cmd/init/main.go
index d1e2f87..42770a9 100644
--- a/core/cmd/init/main.go
+++ b/core/cmd/init/main.go
@@ -62,6 +62,11 @@
panic(fmt.Errorf("could not remount root: %w", err))
}
+ // Linux kernel default is 4096 which is far too low. Raise it to 1M which is what gVisor suggests.
+ if err := unix.Setrlimit(unix.RLIMIT_NOFILE, &unix.Rlimit{Cur: 1048576, Max: 1048576}); err != nil {
+ logger.Panic("Failed to raise rlimits", zap.Error(err))
+ }
+
logger.Info("Starting Smalltown Init")
signalChannel := make(chan os.Signal, 2)
diff --git a/core/cmd/kube-controlplane/BUILD b/core/cmd/kube/BUILD
similarity index 61%
rename from core/cmd/kube-controlplane/BUILD
rename to core/cmd/kube/BUILD
index c9049e4..b1a22fe 100644
--- a/core/cmd/kube-controlplane/BUILD
+++ b/core/cmd/kube/BUILD
@@ -1,43 +1,26 @@
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
-# This is a hack to make go modules ignore all of kube-controlplane since Kubernetes is not importable
-# and we still need normal go mod tooling to work. Instead we're just depending on our own Kubernetes
-# which is already being built with Bazel and thus works fine as a dependency.
-
-genrule(
- name = "hack_ignore",
- srcs = [
- "main.go",
- ],
- outs = [
- "main_patched.go",
- ],
- cmd = """
- sed '/+build ignore/d' $(location main.go) > "$@"
- """,
- visibility = ["//visibility:public"],
-)
-
go_library(
name = "go_default_library",
- srcs = [":main_patched.go"],
- importpath = "git.monogon.dev/source/nexantic.git/core/cmd/kubemaster",
+ srcs = ["main.go"],
+ importpath = "git.monogon.dev/source/nexantic.git/core/cmd/kube",
visibility = ["//visibility:private"],
deps = [
- "@io_k8s_kubernetes//cmd/kube-apiserver/app:go_default_library",
- "@io_k8s_kubernetes//cmd/kube-controller-manager/app:go_default_library",
- "@io_k8s_kubernetes//cmd/kube-scheduler/app:go_default_library",
+ "@com_github_spf13_cobra//:go_default_library",
+ "@com_github_spf13_pflag//:go_default_library",
"@io_k8s_component_base//cli/flag:go_default_library",
"@io_k8s_component_base//logs:go_default_library",
"@io_k8s_component_base//metrics/prometheus/restclient:go_default_library",
"@io_k8s_component_base//metrics/prometheus/version:go_default_library",
- "@com_github_spf13_cobra//:go_default_library",
- "@com_github_spf13_pflag//:go_default_library",
+ "@io_k8s_kubernetes//cmd/kube-apiserver/app:go_default_library",
+ "@io_k8s_kubernetes//cmd/kube-controller-manager/app:go_default_library",
+ "@io_k8s_kubernetes//cmd/kube-scheduler/app:go_default_library",
+ "@io_k8s_kubernetes//cmd/kubelet/app:go_default_library",
],
)
go_binary(
- name = "kube-controlplane",
+ name = "kube",
embed = [":go_default_library"],
pure = "on",
visibility = ["//visibility:public"],
diff --git a/core/cmd/kube-controlplane/main.go b/core/cmd/kube/main.go
similarity index 94%
rename from core/cmd/kube-controlplane/main.go
rename to core/cmd/kube/main.go
index 854e742..3b4ac08 100644
--- a/core/cmd/kube-controlplane/main.go
+++ b/core/cmd/kube/main.go
@@ -14,8 +14,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-//+build ignore
-
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
@@ -50,6 +48,7 @@
kubeapiserver "k8s.io/kubernetes/cmd/kube-apiserver/app"
kubecontrollermanager "k8s.io/kubernetes/cmd/kube-controller-manager/app"
kubescheduler "k8s.io/kubernetes/cmd/kube-scheduler/app"
+ kubelet "k8s.io/kubernetes/cmd/kubelet/app"
)
func main() {
@@ -95,16 +94,18 @@
apiserver := func() *cobra.Command { return kubeapiserver.NewAPIServerCommand() }
controller := func() *cobra.Command { return kubecontrollermanager.NewControllerManagerCommand() }
scheduler := func() *cobra.Command { return kubescheduler.NewSchedulerCommand() }
+ kubelet := func() *cobra.Command { return kubelet.NewKubeletCommand() }
commandFns := []func() *cobra.Command{
apiserver,
controller,
scheduler,
+ kubelet,
}
cmd := &cobra.Command{
- Use: "kube-controlplane",
- Short: "Combines all Kubernetes Control Plane components in a single binary",
+ Use: "kube",
+ Short: "Combines all Kubernetes components in a single binary",
Run: func(cmd *cobra.Command, args []string) {
if len(args) != 0 {
cmd.Help()
diff --git a/core/cmd/mkimage/main.go b/core/cmd/mkimage/main.go
index ddf8813..1238c04 100644
--- a/core/cmd/mkimage/main.go
+++ b/core/cmd/mkimage/main.go
@@ -35,6 +35,7 @@
outputPath = flag.String("out", "", "Output disk image")
initramfsPath = flag.String("initramfs", "", "External initramfs [optional]")
enrolmentCredentialsPath = flag.String("enrolment-credentials", "", "Enrolment credentials [optional]")
+ dataPartitionSizeMiB = flag.Uint64("data-partition-size", 2048, "Override the data partition size (default 2048 MiB)")
)
func mibToSectors(size uint64) uint64 {
@@ -71,7 +72,7 @@
Type: SmalltownDataPartition,
Name: "SIGNOS-DATA",
Start: mibToSectors(256),
- End: mibToSectors(2560) - 1,
+ End: mibToSectors(*dataPartitionSizeMiB+256) - 1,
},
},
}