m/test/e2e: test NodePort
This was originally written as a test for validating fixes for
the issue that NodePort was not working if any non-local pods were in
the NodePort service, even for externalTrafficPolicy: cluster services.
As it turns out CL:2795 fixed this, the changes in previous versions of
this CL broke it again. So now it just consists of the test itself,
which passes.
Change-Id: If4cf4ffc46a5456b4defa330776e043593e61b29
Reviewed-on: https://review.monogon.dev/c/monogon/+/1924
Tested-by: Jenkins CI
Reviewed-by: Tim Windelschmidt <tim@monogon.tech>
diff --git a/metropolis/test/e2e/BUILD.bazel b/metropolis/test/e2e/BUILD.bazel
index c77e334..3923c8d 100644
--- a/metropolis/test/e2e/BUILD.bazel
+++ b/metropolis/test/e2e/BUILD.bazel
@@ -21,6 +21,7 @@
name = "testimages_manifest",
images = [
"//metropolis/test/e2e/selftest:selftest_image",
+ "//metropolis/test/e2e/httpserver:httpserver_image",
"//metropolis/vm/smoketest:smoketest_image",
],
)
@@ -48,6 +49,7 @@
"//metropolis/test/util",
"@io_bazel_rules_go//go/runfiles:go_default_library",
"@io_k8s_api//core/v1:core",
+ "@io_k8s_apimachinery//pkg/api/errors",
"@io_k8s_apimachinery//pkg/api/resource",
"@io_k8s_apimachinery//pkg/apis/meta/v1:meta",
"@io_k8s_kubernetes//pkg/api/v1/pod",
diff --git a/metropolis/test/e2e/httpserver/BUILD.bazel b/metropolis/test/e2e/httpserver/BUILD.bazel
new file mode 100644
index 0000000..160ebf8
--- /dev/null
+++ b/metropolis/test/e2e/httpserver/BUILD.bazel
@@ -0,0 +1,49 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_binary")
+
+go_library(
+ name = "httpserver_lib",
+ srcs = ["main.go"],
+ importpath = "source.monogon.dev/metropolis/test/e2e/httpserver",
+ visibility = ["//visibility:private"],
+)
+
+go_binary(
+ name = "httpserver",
+ embed = [":httpserver_lib"],
+ pure = "on",
+ visibility = ["//visibility:private"],
+)
+
+platform_transition_binary(
+ name = "httpserver_transitioned",
+ binary = ":httpserver",
+ target_platform = "//build/platforms:linux_amd64_static",
+ visibility = ["//visibility:private"],
+)
+
+load("@rules_pkg//pkg:tar.bzl", "pkg_tar")
+
+pkg_tar(
+ name = "httpserver_layer",
+ srcs = [":httpserver_transitioned"],
+ visibility = ["//visibility:private"],
+)
+
+load("@rules_oci//oci:defs.bzl", "oci_image", "oci_tarball")
+
+oci_image(
+ name = "httpserver_image",
+ base = "@distroless_base",
+ entrypoint = ["/httpserver"],
+ tars = [":httpserver_layer"],
+ visibility = ["//metropolis/test/e2e:__pkg__"],
+ workdir = "/app",
+)
+
+oci_tarball(
+ name = "httpserver_tarball",
+ image = ":httpserver_image",
+ repo_tags = ["bazel/metropolis/test/e2e/httpserver:httpserver_image"],
+ visibility = ["//metropolis/test/e2e:__pkg__"],
+)
diff --git a/metropolis/test/e2e/httpserver/main.go b/metropolis/test/e2e/httpserver/main.go
new file mode 100644
index 0000000..3b593db
--- /dev/null
+++ b/metropolis/test/e2e/httpserver/main.go
@@ -0,0 +1,19 @@
+// httpserver serves a test HTTP endpoint for E2E testing.
+package main
+
+import (
+ "net/http"
+ "os"
+)
+
+func main() {
+ nodeName := os.Getenv("NODE_NAME")
+ http.ListenAndServe(":8080", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("X-Node-Name", nodeName)
+ w.Header().Set("X-Remote-IP", r.RemoteAddr)
+ w.WriteHeader(http.StatusOK)
+ // Send a big chunk to root out MTU/MSS issues.
+ testPayload := make([]byte, 2000)
+ w.Write(testPayload)
+ }))
+}
diff --git a/metropolis/test/e2e/kubernetes_helpers.go b/metropolis/test/e2e/kubernetes_helpers.go
index ce9e78f..274c5c7 100644
--- a/metropolis/test/e2e/kubernetes_helpers.go
+++ b/metropolis/test/e2e/kubernetes_helpers.go
@@ -69,6 +69,63 @@
}
}
+// makeHTTPServerDeploymentSpec generates the deployment spec for the test HTTP
+// server.
+func makeHTTPServerDeploymentSpec(name string) *appsv1.Deployment {
+ oneVal := int32(1)
+ return &appsv1.Deployment{
+ ObjectMeta: metav1.ObjectMeta{Name: name},
+ Spec: appsv1.DeploymentSpec{
+ Selector: &metav1.LabelSelector{MatchLabels: map[string]string{
+ "name": name,
+ }},
+ Replicas: &oneVal,
+ Template: corev1.PodTemplateSpec{
+ ObjectMeta: metav1.ObjectMeta{
+ Labels: map[string]string{
+ "name": name,
+ },
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "test",
+ ImagePullPolicy: corev1.PullIfNotPresent,
+ Image: "test.monogon.internal/metropolis/test/e2e/httpserver/httpserver_image",
+ LivenessProbe: &corev1.Probe{
+ ProbeHandler: corev1.ProbeHandler{
+ HTTPGet: &corev1.HTTPGetAction{Port: intstr.FromInt(8080)},
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ }
+}
+
+// makeHTTPServerNodePortService generates the NodePort service spec
+// for testing the NodePort functionality.
+func makeHTTPServerNodePortService(name string) *corev1.Service {
+ return &corev1.Service{
+ ObjectMeta: metav1.ObjectMeta{Name: name},
+ Spec: corev1.ServiceSpec{
+ Type: corev1.ServiceTypeNodePort,
+ Selector: map[string]string{
+ "name": name,
+ },
+ Ports: []corev1.ServicePort{{
+ Name: name,
+ Protocol: corev1.ProtocolTCP,
+ Port: 80,
+ NodePort: 80,
+ TargetPort: intstr.FromInt(8080),
+ }},
+ },
+ }
+}
+
// makeSelftestSpec generates a Job spec for the E2E self-test image.
func makeSelftestSpec(name string) *batchv1.Job {
one := int32(1)
diff --git a/metropolis/test/e2e/main_test.go b/metropolis/test/e2e/main_test.go
index 2409aed..8617da4 100644
--- a/metropolis/test/e2e/main_test.go
+++ b/metropolis/test/e2e/main_test.go
@@ -36,6 +36,7 @@
"github.com/bazelbuild/rules_go/go/runfiles"
"google.golang.org/grpc"
corev1 "k8s.io/api/core/v1"
+ kerrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
podv1 "k8s.io/kubernetes/pkg/api/v1/pod"
@@ -393,6 +394,48 @@
}
return fmt.Errorf("job still running")
})
+ util.TestEventual(t, "Start NodePort test setup", ctx, smallTestTimeout, func(ctx context.Context) error {
+ _, err := clientSet.AppsV1().Deployments("default").Create(ctx, makeHTTPServerDeploymentSpec("nodeport-server"), metav1.CreateOptions{})
+ if err != nil && !kerrors.IsAlreadyExists(err) {
+ return err
+ }
+ _, err = clientSet.CoreV1().Services("default").Create(ctx, makeHTTPServerNodePortService("nodeport-server"), metav1.CreateOptions{})
+ if err != nil && !kerrors.IsAlreadyExists(err) {
+ return err
+ }
+ return nil
+ })
+ util.TestEventual(t, "NodePort accessible from all nodes", ctx, smallTestTimeout, func(ctx context.Context) error {
+ nodes, err := clientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+ if err != nil {
+ return err
+ }
+ // Use a new client for each attempt
+ hc := http.Client{
+ Timeout: 2 * time.Second,
+ Transport: &http.Transport{
+ Dial: cluster.SOCKSDialer.Dial,
+ },
+ }
+ for _, n := range nodes.Items {
+ var addr string
+ for _, a := range n.Status.Addresses {
+ if a.Type == corev1.NodeInternalIP {
+ addr = a.Address
+ }
+ }
+ u := url.URL{Scheme: "http", Host: addr, Path: "/"}
+ res, err := hc.Get(u.String())
+ if err != nil {
+ return fmt.Errorf("failed getting from node %q: %w", n.Name, err)
+ }
+ if res.StatusCode != http.StatusOK {
+ return fmt.Errorf("getting from node %q: HTTP %d", n.Name, res.StatusCode)
+ }
+ t.Logf("Got response from %q", n.Name)
+ }
+ return nil
+ })
if os.Getenv("HAVE_NESTED_KVM") != "" {
util.TestEventual(t, "Pod for KVM/QEMU smoke test", ctx, smallTestTimeout, func(ctx context.Context) error {
runcRuntimeClass := "runc"