m/test/e2e: split out tests into subpackages
The end-to-end tests have grown large enough that they merit their own
test targets. To make this more Go-idiomatic, we split away the tests
not just int separate Bazel targets, but also Go packages.
We also add per-test resource requests for Bazel, including a new
resource kind (iops). This makes the tests more deterministic and allows
use to eg. use --runs_per_test=10 to deflake test logic without hitting
resource contention issues.
//metropolis/test/e2e/suites/core:core_test PASSED in 35.1s
Stats over 10 runs: max = 35.1s, min = 26.6s, avg = 31.9s, dev = 2.6s
//metropolis/test/e2e/suites/ha:ha_test PASSED in 114.6s
Stats over 10 runs: max = 114.6s, min = 90.1s, avg = 100.9s, dev = 7.6s
//metropolis/test/e2e/suites/ha_cold:ha_cold_test PASSED in 67.8s
Stats over 10 runs: max = 67.8s, min = 55.5s, avg = 62.0s, dev = 4.1s
//metropolis/test/e2e/suites/kubernetes:kubernetes_test PASSED in 80.9s
Stats over 10 runs: max = 80.9s, min = 58.8s, avg = 68.6s, dev = 6.0s
Change-Id: I8f31e09f599fd90c9941e2b69f36789817fa90ce
Reviewed-on: https://review.monogon.dev/c/monogon/+/3086
Reviewed-by: Jan Schär <jan@monogon.tech>
Tested-by: Jenkins CI
diff --git a/metropolis/test/e2e/suites/kubernetes/kubernetes_helpers.go b/metropolis/test/e2e/suites/kubernetes/kubernetes_helpers.go
new file mode 100644
index 0000000..60d611b
--- /dev/null
+++ b/metropolis/test/e2e/suites/kubernetes/kubernetes_helpers.go
@@ -0,0 +1,219 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kubernetes
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "io"
+ "strings"
+
+ appsv1 "k8s.io/api/apps/v1"
+ batchv1 "k8s.io/api/batch/v1"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/util/intstr"
+ "k8s.io/client-go/kubernetes"
+)
+
+// makeTestDeploymentSpec generates a Deployment spec for a single pod running
+// NGINX with a readiness probe. This allows verifying that the control plane
+// is capable of scheduling simple pods and that kubelet works, its runtime is
+// set up well enough to run a simple container and the network to the pod can
+// pass readiness probe traffic.
+func makeTestDeploymentSpec(name string) *appsv1.Deployment {
+ return &appsv1.Deployment{
+ ObjectMeta: metav1.ObjectMeta{Name: name},
+ Spec: appsv1.DeploymentSpec{
+ Selector: &metav1.LabelSelector{MatchLabels: map[string]string{
+ "name": name,
+ }},
+ Template: corev1.PodTemplateSpec{
+ ObjectMeta: metav1.ObjectMeta{
+ Labels: map[string]string{
+ "name": name,
+ },
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "test",
+ ImagePullPolicy: corev1.PullNever,
+ Image: "bazel/metropolis/test/e2e/preseedtest:preseedtest_image",
+ ReadinessProbe: &corev1.Probe{
+ ProbeHandler: corev1.ProbeHandler{
+ HTTPGet: &corev1.HTTPGetAction{Port: intstr.FromInt(80)},
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ }
+}
+
+// makeHTTPServerDeploymentSpec generates the deployment spec for the test HTTP
+// server.
+func makeHTTPServerDeploymentSpec(name string) *appsv1.Deployment {
+ oneVal := int32(1)
+ return &appsv1.Deployment{
+ ObjectMeta: metav1.ObjectMeta{Name: name},
+ Spec: appsv1.DeploymentSpec{
+ Selector: &metav1.LabelSelector{MatchLabels: map[string]string{
+ "name": name,
+ }},
+ Replicas: &oneVal,
+ Template: corev1.PodTemplateSpec{
+ ObjectMeta: metav1.ObjectMeta{
+ Labels: map[string]string{
+ "name": name,
+ },
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "test",
+ ImagePullPolicy: corev1.PullIfNotPresent,
+ Image: "test.monogon.internal/metropolis/test/e2e/httpserver/httpserver_image",
+ LivenessProbe: &corev1.Probe{
+ ProbeHandler: corev1.ProbeHandler{
+ HTTPGet: &corev1.HTTPGetAction{Port: intstr.FromInt(8080)},
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ }
+}
+
+// makeHTTPServerNodePortService generates the NodePort service spec
+// for testing the NodePort functionality.
+func makeHTTPServerNodePortService(name string) *corev1.Service {
+ return &corev1.Service{
+ ObjectMeta: metav1.ObjectMeta{Name: name},
+ Spec: corev1.ServiceSpec{
+ Type: corev1.ServiceTypeNodePort,
+ Selector: map[string]string{
+ "name": name,
+ },
+ Ports: []corev1.ServicePort{{
+ Name: name,
+ Protocol: corev1.ProtocolTCP,
+ Port: 80,
+ NodePort: 80,
+ TargetPort: intstr.FromInt(8080),
+ }},
+ },
+ }
+}
+
+// makeSelftestSpec generates a Job spec for the E2E self-test image.
+func makeSelftestSpec(name string) *batchv1.Job {
+ one := int32(1)
+ return &batchv1.Job{
+ ObjectMeta: metav1.ObjectMeta{Name: name},
+ Spec: batchv1.JobSpec{
+ BackoffLimit: &one,
+ Template: corev1.PodTemplateSpec{
+ ObjectMeta: metav1.ObjectMeta{
+ Labels: map[string]string{
+ "job-name": name,
+ },
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "test",
+ ImagePullPolicy: corev1.PullIfNotPresent,
+ Image: "test.monogon.internal/metropolis/test/e2e/selftest/selftest_image",
+ },
+ },
+ RestartPolicy: corev1.RestartPolicyOnFailure,
+ },
+ },
+ },
+ }
+}
+
+// makeTestStatefulSet generates a StatefulSet spec
+func makeTestStatefulSet(name string, volumeMode corev1.PersistentVolumeMode) *appsv1.StatefulSet {
+ return &appsv1.StatefulSet{
+ ObjectMeta: metav1.ObjectMeta{Name: name},
+ Spec: appsv1.StatefulSetSpec{
+ Selector: &metav1.LabelSelector{MatchLabels: map[string]string{
+ "name": name,
+ }},
+ VolumeClaimTemplates: []corev1.PersistentVolumeClaim{
+ {
+ ObjectMeta: metav1.ObjectMeta{Name: "www"},
+ Spec: corev1.PersistentVolumeClaimSpec{
+ AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
+ Resources: corev1.VolumeResourceRequirements{
+ Requests: map[corev1.ResourceName]resource.Quantity{corev1.ResourceStorage: resource.MustParse("50Mi")},
+ },
+ VolumeMode: &volumeMode,
+ },
+ },
+ },
+ Template: corev1.PodTemplateSpec{
+ ObjectMeta: metav1.ObjectMeta{
+ Labels: map[string]string{
+ "name": name,
+ },
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "test",
+ ImagePullPolicy: corev1.PullNever,
+ Image: "bazel/metropolis/test/e2e/preseedtest:preseedtest_image",
+ ReadinessProbe: &corev1.Probe{
+ ProbeHandler: corev1.ProbeHandler{
+ HTTPGet: &corev1.HTTPGetAction{Port: intstr.FromInt(80)},
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ }
+}
+
+func getPodLogLines(ctx context.Context, cs kubernetes.Interface, podName string, nlines int64) ([]string, error) {
+ logsR := cs.CoreV1().Pods("default").GetLogs(podName, &corev1.PodLogOptions{
+ TailLines: &nlines,
+ })
+ logs, err := logsR.Stream(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("stream failed: %w", err)
+ }
+ var buf bytes.Buffer
+ _, err = io.Copy(&buf, logs)
+ if err != nil {
+ return nil, fmt.Errorf("copy failed: %w", err)
+ }
+ lineStr := strings.Trim(buf.String(), "\n")
+ lines := strings.Split(lineStr, "\n")
+ lines = lines[len(lines)-int(nlines):]
+ return lines, nil
+}