treewide: update to Kubernetes 1.31
Overall not that bad, we got rid of some workarounds and added some new
ones. Biggest change is a significant refactor of the hyperkube package
as Kubernetes really doesn't like multiple of their top-level Cobra
commands to be instantiated. One new patch for gVisor as new fields got
added to a Linux struct which caused codegen to rename an existing one.
That patch will go away once [1] is released as this has been changed
back again.
Otherwise mostly standard rebases of patches. We currently have a
warning in kubelet as our containerd CRI does not support the
RuntimeConfig RPC, but no released version of containerd has that and
the fallback works fine for now.
[1] https://go-review.googlesource.com/c/sys/+/607876
Change-Id: I275e5fb78bc1d09c4ca0e8b5705edbaa80f30d96
Reviewed-on: https://review.monogon.dev/c/monogon/+/3355
Reviewed-by: Tim Windelschmidt <tim@monogon.tech>
Tested-by: Jenkins CI
diff --git a/metropolis/node/kubernetes/hyperkube/BUILD.bazel b/metropolis/node/kubernetes/hyperkube/BUILD.bazel
index ed27296..3c4e31b 100644
--- a/metropolis/node/kubernetes/hyperkube/BUILD.bazel
+++ b/metropolis/node/kubernetes/hyperkube/BUILD.bazel
@@ -8,9 +8,7 @@
visibility = ["//visibility:private"],
deps = [
"@com_github_spf13_cobra//:cobra",
- "@com_github_spf13_pflag//:pflag",
- "@io_k8s_component_base//cli/flag",
- "@io_k8s_component_base//logs",
+ "@io_k8s_component_base//cli",
"@io_k8s_component_base//metrics/prometheus/restclient",
"@io_k8s_component_base//metrics/prometheus/version",
"@io_k8s_kubernetes//cmd/kube-apiserver/app",
diff --git a/metropolis/node/kubernetes/hyperkube/main.go b/metropolis/node/kubernetes/hyperkube/main.go
index 5f9d443..0809fa4 100644
--- a/metropolis/node/kubernetes/hyperkube/main.go
+++ b/metropolis/node/kubernetes/hyperkube/main.go
@@ -27,19 +27,25 @@
limitations under the License.
*/
-// Adapted from https://github.com/dims/hyperkube
-
+// This is the entry point for our multicall Kubernetes binary. It can act as
+// any of the Kubernetes components we use depending on its first argument.
+// This saves us a bunch of duplicated code and thus system partition size as
+// a large amount of library code is shared between all of the Kubernetes
+// components.
+//
+// As this is not intended by the K8s developers the Cobra setup is unusual
+// in that even the command structs are only created on-demand and not
+// registered with AddCommand. This is done as Kubernetes performs one-off
+// global setup inside their NewXYZCommand functions, for example for signal
+// handling and their global registries.
package main
import (
- goflag "flag"
+ "fmt"
"os"
- "path/filepath"
"github.com/spf13/cobra"
- "github.com/spf13/pflag"
- cliflag "k8s.io/component-base/cli/flag"
- "k8s.io/component-base/logs"
+ "k8s.io/component-base/cli"
_ "k8s.io/component-base/metrics/prometheus/restclient" // for client metric registration
_ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration
kubeapiserver "k8s.io/kubernetes/cmd/kube-apiserver/app"
@@ -48,70 +54,22 @@
kubelet "k8s.io/kubernetes/cmd/kubelet/app"
)
+// Map of subcommand to Cobra command generator for all subcommands
+var subcommands = map[string]func() *cobra.Command{
+ "kube-apiserver": kubeapiserver.NewAPIServerCommand,
+ "kube-controller-manager": kubecontrollermanager.NewControllerManagerCommand,
+ "kube-scheduler": func() *cobra.Command { return kubescheduler.NewSchedulerCommand() },
+ "kubelet": kubelet.NewKubeletCommand,
+}
+
func main() {
- hyperkubeCommand, allCommandFns := NewHyperKubeCommand()
-
- // TODO: once we switch everything over to Cobra commands, we can go back
- // to calling cliflag.InitFlags() (by removing its pflag.Parse() call). For
- // now, we have to set the normalize func and add the go flag set by hand.
- pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)
- pflag.CommandLine.AddGoFlagSet(goflag.CommandLine)
- // cliflag.InitFlags()
- logs.InitLogs()
- defer logs.FlushLogs()
-
- basename := filepath.Base(os.Args[0])
- if err := commandFor(basename, hyperkubeCommand, allCommandFns).Execute(); err != nil {
- os.Exit(1)
+ if len(os.Args) < 2 || subcommands[os.Args[1]] == nil {
+ fmt.Fprintf(os.Stderr, "Unknown subcommand\n")
+ } else {
+ cmdGen := subcommands[os.Args[1]]
+ cmd := cmdGen()
+ // Strip first argument as it has already been consumed
+ cmd.SetArgs(os.Args[2:])
+ os.Exit(cli.Run(cmd))
}
}
-
-func commandFor(basename string, defaultCommand *cobra.Command, commands []func() *cobra.Command) *cobra.Command {
- for _, commandFn := range commands {
- command := commandFn()
- if command.Name() == basename {
- return command
- }
- for _, alias := range command.Aliases {
- if alias == basename {
- return command
- }
- }
- }
-
- return defaultCommand
-}
-
-// NewHyperKubeCommand is the entry point for hyperkube
-func NewHyperKubeCommand() (*cobra.Command, []func() *cobra.Command) {
- // these have to be functions since the command is polymorphic. Cobra wants
- // you to be top level command to get executed
- apiserver := func() *cobra.Command { return kubeapiserver.NewAPIServerCommand() }
- controller := func() *cobra.Command { return kubecontrollermanager.NewControllerManagerCommand() }
- scheduler := func() *cobra.Command { return kubescheduler.NewSchedulerCommand() }
- kubelet := func() *cobra.Command { return kubelet.NewKubeletCommand() }
-
- commandFns := []func() *cobra.Command{
- apiserver,
- controller,
- scheduler,
- kubelet,
- }
-
- cmd := &cobra.Command{
- Use: "kube",
- Short: "Combines all Kubernetes components in a single binary",
- Run: func(cmd *cobra.Command, args []string) {
- if len(args) != 0 {
- cmd.Help()
- os.Exit(1)
- }
- },
- }
-
- for i := range commandFns {
- cmd.AddCommand(commandFns[i]())
- }
-
- return cmd, commandFns
-}
diff --git a/metropolis/test/e2e/suites/kubernetes/BUILD.bazel b/metropolis/test/e2e/suites/kubernetes/BUILD.bazel
index 302f3ac..a2ba770 100644
--- a/metropolis/test/e2e/suites/kubernetes/BUILD.bazel
+++ b/metropolis/test/e2e/suites/kubernetes/BUILD.bazel
@@ -34,6 +34,7 @@
},
deps = [
"//metropolis/node",
+ "//metropolis/proto/common",
"//metropolis/test/launch",
"//metropolis/test/localregistry",
"//metropolis/test/util",
diff --git a/metropolis/test/e2e/suites/kubernetes/run_test.go b/metropolis/test/e2e/suites/kubernetes/run_test.go
index 3608c3c..ed08203 100644
--- a/metropolis/test/e2e/suites/kubernetes/run_test.go
+++ b/metropolis/test/e2e/suites/kubernetes/run_test.go
@@ -23,11 +23,11 @@
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
podv1 "k8s.io/kubernetes/pkg/api/v1/pod"
+ common "source.monogon.dev/metropolis/node"
+ cpb "source.monogon.dev/metropolis/proto/common"
mlaunch "source.monogon.dev/metropolis/test/launch"
"source.monogon.dev/metropolis/test/localregistry"
"source.monogon.dev/metropolis/test/util"
-
- common "source.monogon.dev/metropolis/node"
)
var (
@@ -82,6 +82,10 @@
clusterOptions := mlaunch.ClusterOptions{
NumNodes: 2,
LocalRegistry: lr,
+ InitialClusterConfiguration: &cpb.ClusterConfiguration{
+ TpmMode: cpb.ClusterConfiguration_TPM_MODE_DISABLED,
+ StorageSecurityPolicy: cpb.ClusterConfiguration_STORAGE_SECURITY_POLICY_NEEDS_INSECURE,
+ },
}
cluster, err := mlaunch.LaunchCluster(ctx, clusterOptions)
if err != nil {
diff --git a/metropolis/vm/kube/generated/clientset/versioned/typed/vm/v1alpha1/BUILD.bazel b/metropolis/vm/kube/generated/clientset/versioned/typed/vm/v1alpha1/BUILD.bazel
index 6ffcb60..c69cec5 100644
--- a/metropolis/vm/kube/generated/clientset/versioned/typed/vm/v1alpha1/BUILD.bazel
+++ b/metropolis/vm/kube/generated/clientset/versioned/typed/vm/v1alpha1/BUILD.bazel
@@ -18,6 +18,7 @@
"@io_k8s_apimachinery//pkg/apis/meta/v1:meta",
"@io_k8s_apimachinery//pkg/types",
"@io_k8s_apimachinery//pkg/watch",
+ "@io_k8s_client_go//gentype",
"@io_k8s_client_go//rest",
],
)
diff --git a/metropolis/vm/kube/generated/listers/vm/v1alpha1/BUILD.bazel b/metropolis/vm/kube/generated/listers/vm/v1alpha1/BUILD.bazel
index d9fa05a..34b5aa6 100644
--- a/metropolis/vm/kube/generated/listers/vm/v1alpha1/BUILD.bazel
+++ b/metropolis/vm/kube/generated/listers/vm/v1alpha1/BUILD.bazel
@@ -16,6 +16,7 @@
"//metropolis/vm/kube/apis/vm/v1alpha1",
"@io_k8s_apimachinery//pkg/api/errors",
"@io_k8s_apimachinery//pkg/labels",
+ "@io_k8s_client_go//listers",
"@io_k8s_client_go//tools/cache",
],
)