Add Kubernetes CTS
This adds patches and build specifications for the Kubernetes Conformance Test Suite. This involves
gating various cloud-specific tests behind the providerless flag (otherwise we'd gain a ton of additional dependencies)
and an additional 60MiB in test binary size.
Since the CTS for weird reasons requires kubectl to be available in the path we first build a kubectl go_image and then
stack the CTS on top of it. The output bundle is then preseeded for use.
Test Plan: `bazel run //core/tests/e2e/k8s_cts`
Bug: T836
X-Origin-Diff: phab/D615
GitOrigin-RevId: 7d2cd780a3ffb63b217591c5854b4aec4031d83d
diff --git a/core/tests/e2e/BUILD.bazel b/core/tests/e2e/BUILD.bazel
index 974bcdd..8e74be4 100644
--- a/core/tests/e2e/BUILD.bazel
+++ b/core/tests/e2e/BUILD.bazel
@@ -7,7 +7,7 @@
"utils.go",
],
importpath = "git.monogon.dev/source/nexantic.git/core/tests/e2e",
- visibility = ["//visibility:private"],
+ visibility = ["//core/tests:__subpackages__"],
deps = [
"//core/proto/api:go_default_library",
"@io_k8s_api//apps/v1:go_default_library",
diff --git a/core/tests/e2e/k8s_cts/BUILD.bazel b/core/tests/e2e/k8s_cts/BUILD.bazel
new file mode 100644
index 0000000..648e1c5
--- /dev/null
+++ b/core/tests/e2e/k8s_cts/BUILD.bazel
@@ -0,0 +1,55 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+load("@io_bazel_rules_docker//go:image.bzl", "go_image")
+load("@io_bazel_rules_docker//container:container.bzl", "container_image")
+
+go_image(
+ name = "kubectl",
+ binary = "@io_k8s_kubernetes//cmd/kubectl",
+ pure = "on",
+)
+
+container_image(
+ name = "kubectl_in_path",
+ base = ":kubectl",
+ env = {
+ # Don't include FHS paths since they aren't available anyways
+ "PATH": "/app/cmd/kubectl",
+ },
+)
+
+go_image(
+ name = "k8s_cts_image",
+ base = ":kubectl_in_path",
+ binary = "@io_k8s_kubernetes//test/e2e:_go_default_test-pure",
+ pure = "on",
+ visibility = ["//visibility:public"],
+)
+
+go_library(
+ name = "go_default_library",
+ srcs = ["main.go"],
+ importpath = "git.monogon.dev/source/nexantic.git/core/tests/e2e/k8s_cts",
+ visibility = ["//visibility:private"],
+ deps = [
+ "//core/internal/common:go_default_library",
+ "//core/internal/launch:go_default_library",
+ "//core/tests/e2e:go_default_library",
+ "@io_k8s_api//core/v1:go_default_library",
+ "@io_k8s_api//rbac/v1:go_default_library",
+ "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
+ ],
+)
+
+go_binary(
+ name = "k8s_cts",
+ data = [
+ "//core:image",
+ "//core:swtpm_data",
+ "//core/cmd/nanoswitch:initramfs",
+ "//core/tools/ktest:linux-testing",
+ "//third_party/edk2:firmware",
+ "@com_github_bonzini_qboot//:qboot-bin",
+ ],
+ embed = [":go_default_library"],
+ visibility = ["//visibility:public"],
+)
diff --git a/core/tests/e2e/k8s_cts/main.go b/core/tests/e2e/k8s_cts/main.go
new file mode 100644
index 0000000..412ae7c
--- /dev/null
+++ b/core/tests/e2e/k8s_cts/main.go
@@ -0,0 +1,176 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This package launches a Smalltown Cluster with two nodes and spawns in the CTS container. Then it streams its output
+// to the console. When the CTS has finished it exits with the appropriate error code.
+package main
+
+import (
+ "context"
+ "io"
+ "log"
+ "os"
+ "os/signal"
+ "strings"
+ "syscall"
+ "time"
+
+ "git.monogon.dev/source/nexantic.git/core/internal/common"
+ "git.monogon.dev/source/nexantic.git/core/tests/e2e"
+
+ corev1 "k8s.io/api/core/v1"
+ rbacv1 "k8s.io/api/rbac/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+ "git.monogon.dev/source/nexantic.git/core/internal/launch"
+)
+
+// makeCTSPodSpec generates a spec for a standalone pod running the Kubernetes CTS. It also sets the test configuration
+// for the Kubernetes E2E test suite to only run CTS tests and excludes known-broken ones.
+func makeCTSPodSpec(name string, saName string) *corev1.Pod {
+ skipRegexes := []string{
+ // hostNetworking cannot be supported since we run different network stacks for the host and containers
+ "should function for node-pod communication",
+ // gVisor misreports statfs() syscalls: https://github.com/google/gvisor/issues/3339
+ `should support \((non-)?root,`,
+ "volume on tmpfs should have the correct mode",
+ "volume on default medium should have the correct mode",
+ // gVisor doesn't support the full Linux privilege machinery including SUID and NewPrivs
+ // https://github.com/google/gvisor/issues/189#issuecomment-481064000
+ "should run the container as unprivileged when false",
+ }
+ return &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ Labels: map[string]string{
+ "name": name,
+ },
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "cts",
+ Image: "bazel/core/tests/e2e/k8s_cts:k8s_cts_image",
+ Args: []string{
+ "-cluster-ip-range=10.0.0.0/17",
+ "-dump-systemd-journal=false",
+ "-ginkgo.focus=\\[Conformance\\]",
+ "-ginkgo.skip=" + strings.Join(skipRegexes, "|"),
+ "-test.parallel=8",
+ },
+ ImagePullPolicy: corev1.PullNever,
+ },
+ },
+ Tolerations: []corev1.Toleration{{ // Tolerate all taints, otherwise the CTS likes to self-evict
+ Operator: "Exists",
+ }},
+ PriorityClassName: "system-cluster-critical", // Don't evict the CTS pod
+ RestartPolicy: corev1.RestartPolicyNever,
+ ServiceAccountName: saName,
+ },
+ }
+}
+
+func main() {
+ sigs := make(chan os.Signal, 1)
+ signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+ ctx, cancel := context.WithCancel(context.Background())
+ go func() {
+ <-sigs
+ cancel()
+ }()
+
+ debugClient, portMap, err := launch.LaunchCluster(ctx, launch.ClusterOptions{NumNodes: 2})
+ if err != nil {
+ log.Fatalf("Failed to launch cluster: %v", err)
+ }
+ log.Println("Cluster initialized")
+
+ clientSet, err := e2e.GetKubeClientSet(ctx, debugClient, portMap[common.KubernetesAPIPort])
+ if err != nil {
+ log.Fatalf("Failed to get clientSet: %v", err)
+ }
+ log.Println("Credentials available")
+
+ saName := "cts"
+ ctsSA := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: saName}}
+ for {
+ if _, err := clientSet.CoreV1().ServiceAccounts("default").Create(ctx, ctsSA, metav1.CreateOptions{}); err != nil {
+ log.Printf("Failed to create ServiceAccount: %v", err)
+ time.Sleep(1 * time.Second)
+ continue
+ }
+ break
+ }
+ ctsRoleBinding := &rbacv1.ClusterRoleBinding{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: saName,
+ },
+ Subjects: []rbacv1.Subject{
+ {
+ Namespace: "default",
+ Name: saName,
+ Kind: rbacv1.ServiceAccountKind,
+ },
+ },
+ RoleRef: rbacv1.RoleRef{
+ Kind: "ClusterRole",
+ Name: "cluster-admin",
+ },
+ }
+ podName := "cts"
+ if _, err := clientSet.RbacV1().ClusterRoleBindings().Create(ctx, ctsRoleBinding, metav1.CreateOptions{}); err != nil {
+ log.Fatalf("Failed to create ClusterRoleBinding: %v", err)
+ }
+ for {
+ if _, err := clientSet.CoreV1().Pods("default").Create(ctx, makeCTSPodSpec(podName, saName), metav1.CreateOptions{}); err != nil {
+ log.Printf("Failed to create Pod: %v", err)
+ time.Sleep(1 * time.Second)
+ continue
+ }
+ break
+ }
+ var logs io.ReadCloser
+ go func() {
+ // This loops the whole .Stream()/io.Copy process because the API sometimes returns streams that immediately return EOF
+ for {
+ logs, err = clientSet.CoreV1().Pods("default").GetLogs(podName, &corev1.PodLogOptions{Follow: true}).Stream(ctx)
+ if err == nil {
+ if _, err := io.Copy(os.Stdout, logs); err != nil {
+ log.Printf("Log pump error: %v", err)
+ }
+ logs.Close()
+ } else {
+ log.Printf("Pod logs not ready yet: %v", err)
+ }
+ time.Sleep(1 * time.Second)
+ }
+ }()
+ for {
+ time.Sleep(1 * time.Second)
+ pod, err := clientSet.CoreV1().Pods("default").Get(ctx, podName, metav1.GetOptions{})
+ if err != nil {
+ log.Printf("Failed to get CTS pod: %v", err)
+ continue
+ }
+ if pod.Status.Phase == corev1.PodSucceeded {
+ return
+ }
+ if pod.Status.Phase == corev1.PodFailed {
+ log.Fatalf("CTS failed")
+ }
+ }
+}
diff --git a/core/tests/e2e/kubernetes_helpers.go b/core/tests/e2e/kubernetes_helpers.go
index d0337e6..4f9ba81 100644
--- a/core/tests/e2e/kubernetes_helpers.go
+++ b/core/tests/e2e/kubernetes_helpers.go
@@ -33,9 +33,9 @@
apb "git.monogon.dev/source/nexantic.git/core/proto/api"
)
-// getKubeClientSet gets a Kubeconfig from the debug API and creates a K8s ClientSet using it. The identity used has
+// GetKubeClientSet gets a Kubeconfig from the debug API and creates a K8s ClientSet using it. The identity used has
// the system:masters group and thus has RBAC access to everything.
-func getKubeClientSet(ctx context.Context, client apb.NodeDebugServiceClient, port uint16) (kubernetes.Interface, error) {
+func GetKubeClientSet(ctx context.Context, client apb.NodeDebugServiceClient, port uint16) (kubernetes.Interface, error) {
var lastErr = errors.New("context canceled before any operation completed")
for {
reqT, cancel := context.WithTimeout(ctx, 5*time.Second)
diff --git a/core/tests/e2e/main_test.go b/core/tests/e2e/main_test.go
index 99cfdff..c50263c 100644
--- a/core/tests/e2e/main_test.go
+++ b/core/tests/e2e/main_test.go
@@ -73,7 +73,7 @@
// Set a global timeout to make sure this terminates
ctx, cancel := context.WithTimeout(context.Background(), globalTestTimeout)
- portMap, err := launch.ConflictFreePortMap()
+ portMap, err := launch.ConflictFreePortMap(launch.NodePorts)
if err != nil {
t.Fatalf("Failed to acquire ports for e2e test: %v", err)
}
@@ -99,7 +99,7 @@
t.Parallel()
selfCtx, cancel := context.WithTimeout(ctx, largeTestTimeout)
defer cancel()
- clientSet, err := getKubeClientSet(selfCtx, debugClient, portMap[common.KubernetesAPIPort])
+ clientSet, err := GetKubeClientSet(selfCtx, debugClient, portMap[common.KubernetesAPIPort])
if err != nil {
t.Fatal(err)
}