metropolis/test/e2e/main_test.go - monogon - Gitiles

 // Copyright 2020 The Monogon Project Authors.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //     http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 package e2e

 import (
 	"bytes"
 	"context"
 	"crypto/ed25519"
 	"errors"
 	"fmt"
 	"log"
 	"net"
 	"net/http"
 	_ "net/http"
 	_ "net/http/pprof"
 	"os"
 	"strings"
 	"testing"
 	"time"

 	"google.golang.org/grpc"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	podv1 "k8s.io/kubernetes/pkg/api/v1/pod"

 	common "source.monogon.dev/metropolis/node"
 	"source.monogon.dev/metropolis/node/core/rpc"
 	apb "source.monogon.dev/metropolis/proto/api"
 	"source.monogon.dev/metropolis/test/launch"
 )

 const (
 	// Timeout for the global test context.
 	//
 	// Bazel would eventually time out the test after 900s ("large") if, for
 	// some reason, the context cancellation fails to abort it.
 	globalTestTimeout = 600 * time.Second

 	// Timeouts for individual end-to-end tests of different sizes.
 	smallTestTimeout = 60 * time.Second
 	largeTestTimeout = 120 * time.Second
 )

 // TestE2E is the main E2E test entrypoint for single-node freshly-bootstrapped
 // E2E tests. It starts a full Metropolis node in bootstrap mode and then runs
 // tests against it. The actual tests it performs are located in the RunGroup
 // subtest.
 func TestE2E(t *testing.T) {
 	// Run pprof server for debugging
 	go func() {
 		addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
 		if err != nil {
 			panic(err)
 		}

 		l, err := net.ListenTCP("tcp", addr)
 		if err != nil {
 			log.Fatalf("Failed to listen on pprof port: %s", l.Addr())
 		}
 		defer l.Close()

 		log.Printf("pprof server listening on %s", l.Addr())
 		log.Printf("pprof server returned an error: %v", http.Serve(l, nil))
 	}()

 	// Set a global timeout to make sure this terminates
 	ctx, cancel := context.WithTimeout(context.Background(), globalTestTimeout)
 	defer cancel()
 	portMap, err := launch.ConflictFreePortMap(launch.NodePorts)
 	if err != nil {
 		t.Fatalf("Failed to acquire ports for e2e test: %v", err)
 	}

 	procExit := make(chan struct{})

 	go func() {
 		if err := launch.Launch(ctx, launch.Options{
 			Ports:      portMap,
 			SerialPort: os.Stdout,
 			NodeParameters: &apb.NodeParameters{
 				Cluster: &apb.NodeParameters_ClusterBootstrap_{
 					ClusterBootstrap: launch.InsecureClusterBootstrap,
 				},
 			},
 		}); err != nil {
 			panic(err)
 		}
 		close(procExit)
 	}()

 	grpcDebug, err := portMap.DialGRPC(common.DebugServicePort, grpc.WithInsecure())
 	if err != nil {
 		log.Printf("Failed to dial debug service (is it running?): %v", err)
 		return
 	}
 	debug := apb.NewNodeDebugServiceClient(grpcDebug)

 	// This exists to keep the parent around while all the children race.
 	// It currently tests both a set of OS-level conditions and Kubernetes
 	// Deployments and StatefulSets
 	t.Run("RunGroup", func(t *testing.T) {
 		t.Run("Connect to Curator", func(t *testing.T) {
 			testEventual(t, "Retrieving owner credentials succesful", ctx, 60*time.Second, func(ctx context.Context) error {
 				remote := fmt.Sprintf("localhost:%v", portMap[common.CuratorServicePort])
 				initClient, err := rpc.NewEphemeralClient(remote, launch.InsecurePrivateKey, nil)
 				if err != nil {
 					return fmt.Errorf("NewInitialClient: %w", err)
 				}

 				aaa := apb.NewAAAClient(initClient)
 				cert, err := rpc.RetrieveOwnerCertificate(ctx, aaa, launch.InsecurePrivateKey)
 				if err != nil {
 					return fmt.Errorf("RetrieveOwnerCertificate: %w", err)
 				}

 				if !bytes.Equal(cert.PrivateKey.(ed25519.PrivateKey), launch.InsecurePrivateKey) {
 					t.Fatalf("Received certificate for wrong private key")
 				}

 				// Connect to management endpoint and retrieve cluster directory.
 				authClient, err := rpc.NewAuthenticatedClient(remote, *cert, nil)
 				if err != nil {
 					return fmt.Errorf("NewAuthenticatedClient: %w", err)
 				}
 				mgmt := apb.NewManagementClient(authClient)
 				res, err := mgmt.GetClusterInfo(ctx, &apb.GetClusterInfoRequest{})
 				if err != nil {
 					return fmt.Errorf("GetClusterInfo: %w", err)
 				}

 				// Ensure the node is there with its address.
 				nodes := res.ClusterDirectory.Nodes
 				if want, got := 1, len(nodes); want != got {
 					return fmt.Errorf("wanted %d nodes in cluster directory, got %d", want, got)
 				}
 				node := nodes[0]
 				if want, got := ed25519.PublicKeySize, len(node.PublicKey); want != got {
 					return fmt.Errorf("wanted %d bytes long public key, got %d", want, got)
 				}
 				if want, got := 1, len(node.Addresses); want != got {
 					return fmt.Errorf("wanted %d node address, got %d", want, got)
 				}
 				if want, got := "10.42.0.10", node.Addresses[0].Host; want != got {
 					return fmt.Errorf("wanted status address %q, got %q", want, got)
 				}

 				return nil
 			})
 		})
 		t.Run("Get Kubernetes Debug Kubeconfig", func(t *testing.T) {
 			t.Parallel()
 			selfCtx, cancel := context.WithTimeout(ctx, largeTestTimeout)
 			defer cancel()
 			clientSet, err := GetKubeClientSet(selfCtx, debug, portMap[common.KubernetesAPIPort])
 			if err != nil {
 				t.Fatal(err)
 			}
 			testEventual(t, "Node is registered and ready", ctx, largeTestTimeout, func(ctx context.Context) error {
 				nodes, err := clientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
 				if err != nil {
 					return err
 				}
 				if len(nodes.Items) < 1 {
 					return errors.New("node not registered")
 				}
 				if len(nodes.Items) > 1 {
 					return errors.New("more than one node registered (but there is only one)")
 				}
 				node := nodes.Items[0]
 				for _, cond := range node.Status.Conditions {
 					if cond.Type != corev1.NodeReady {
 						continue
 					}
 					if cond.Status != corev1.ConditionTrue {
 						return fmt.Errorf("node not ready: %v", cond.Message)
 					}
 				}
 				return nil
 			})
 			testEventual(t, "Simple deployment", ctx, largeTestTimeout, func(ctx context.Context) error {
 				_, err := clientSet.AppsV1().Deployments("default").Create(ctx, makeTestDeploymentSpec("test-deploy-1"), metav1.CreateOptions{})
 				return err
 			})
 			testEventual(t, "Simple deployment is running", ctx, largeTestTimeout, func(ctx context.Context) error {
 				res, err := clientSet.CoreV1().Pods("default").List(ctx, metav1.ListOptions{LabelSelector: "name=test-deploy-1"})
 				if err != nil {
 					return err
 				}
 				if len(res.Items) == 0 {
 					return errors.New("pod didn't get created")
 				}
 				pod := res.Items[0]
 				if podv1.IsPodAvailable(&pod, 1, metav1.NewTime(time.Now())) {
 					return nil
 				}
 				events, err := clientSet.CoreV1().Events("default").List(ctx, metav1.ListOptions{FieldSelector: fmt.Sprintf("involvedObject.name=%s,involvedObject.namespace=default", pod.Name)})
 				if err != nil || len(events.Items) == 0 {
 					return fmt.Errorf("pod is not ready: %v", pod.Status.Phase)
 				} else {
 					return fmt.Errorf("pod is not ready: %v", events.Items[0].Message)
 				}
 			})
 			testEventual(t, "Simple deployment with runc", ctx, largeTestTimeout, func(ctx context.Context) error {
 				deployment := makeTestDeploymentSpec("test-deploy-2")
 				var runcStr = "runc"
 				deployment.Spec.Template.Spec.RuntimeClassName = &runcStr
 				_, err := clientSet.AppsV1().Deployments("default").Create(ctx, deployment, metav1.CreateOptions{})
 				return err
 			})
 			testEventual(t, "Simple deployment is running on runc", ctx, largeTestTimeout, func(ctx context.Context) error {
 				res, err := clientSet.CoreV1().Pods("default").List(ctx, metav1.ListOptions{LabelSelector: "name=test-deploy-2"})
 				if err != nil {
 					return err
 				}
 				if len(res.Items) == 0 {
 					return errors.New("pod didn't get created")
 				}
 				pod := res.Items[0]
 				if podv1.IsPodAvailable(&pod, 1, metav1.NewTime(time.Now())) {
 					return nil
 				}
 				events, err := clientSet.CoreV1().Events("default").List(ctx, metav1.ListOptions{FieldSelector: fmt.Sprintf("involvedObject.name=%s,involvedObject.namespace=default", pod.Name)})
 				if err != nil || len(events.Items) == 0 {
 					return fmt.Errorf("pod is not ready: %v", pod.Status.Phase)
 				} else {
 					var errorMsg strings.Builder
 					for _, msg := range events.Items {
 						errorMsg.WriteString(" | ")
 						errorMsg.WriteString(msg.Message)
 					}
 					return fmt.Errorf("pod is not ready: %v", errorMsg.String())
 				}
 			})
 			testEventual(t, "Simple StatefulSet with PVC", ctx, largeTestTimeout, func(ctx context.Context) error {
 				_, err := clientSet.AppsV1().StatefulSets("default").Create(ctx, makeTestStatefulSet("test-statefulset-1", corev1.PersistentVolumeFilesystem), metav1.CreateOptions{})
 				return err
 			})
 			testEventual(t, "Simple StatefulSet with PVC is running", ctx, largeTestTimeout, func(ctx context.Context) error {
 				res, err := clientSet.CoreV1().Pods("default").List(ctx, metav1.ListOptions{LabelSelector: "name=test-statefulset-1"})
 				if err != nil {
 					return err
 				}
 				if len(res.Items) == 0 {
 					return errors.New("pod didn't get created")
 				}
 				pod := res.Items[0]
 				if podv1.IsPodAvailable(&pod, 1, metav1.NewTime(time.Now())) {
 					return nil
 				}
 				events, err := clientSet.CoreV1().Events("default").List(ctx, metav1.ListOptions{FieldSelector: fmt.Sprintf("involvedObject.name=%s,involvedObject.namespace=default", pod.Name)})
 				if err != nil || len(events.Items) == 0 {
 					return fmt.Errorf("pod is not ready: %v", pod.Status.Phase)
 				} else {
 					return fmt.Errorf("pod is not ready: %v", events.Items[0].Message)
 				}
 			})
 			testEventual(t, "Simple StatefulSet with Block PVC", ctx, largeTestTimeout, func(ctx context.Context) error {
 				_, err := clientSet.AppsV1().StatefulSets("default").Create(ctx, makeTestStatefulSet("test-statefulset-2", corev1.PersistentVolumeBlock), metav1.CreateOptions{})
 				return err
 			})
 			testEventual(t, "Simple StatefulSet with Block PVC is running", ctx, largeTestTimeout, func(ctx context.Context) error {
 				res, err := clientSet.CoreV1().Pods("default").List(ctx, metav1.ListOptions{LabelSelector: "name=test-statefulset-2"})
 				if err != nil {
 					return err
 				}
 				if len(res.Items) == 0 {
 					return errors.New("pod didn't get created")
 				}
 				pod := res.Items[0]
 				if podv1.IsPodAvailable(&pod, 1, metav1.NewTime(time.Now())) {
 					return nil
 				}
 				events, err := clientSet.CoreV1().Events("default").List(ctx, metav1.ListOptions{FieldSelector: fmt.Sprintf("involvedObject.name=%s,involvedObject.namespace=default", pod.Name)})
 				if err != nil || len(events.Items) == 0 {
 					return fmt.Errorf("pod is not ready: %v", pod.Status.Phase)
 				} else {
 					return fmt.Errorf("pod is not ready: %v", events.Items[0].Message)
 				}
 			})
 			testEventual(t, "Pod with preseeded image", ctx, smallTestTimeout, func(ctx context.Context) error {
 				_, err := clientSet.CoreV1().Pods("default").Create(ctx, &corev1.Pod{
 					ObjectMeta: metav1.ObjectMeta{
 						Name: "preseed-test-1",
 					},
 					Spec: corev1.PodSpec{
 						Containers: []corev1.Container{{
 							Name:            "preseed-test-1",
 							ImagePullPolicy: corev1.PullNever,
 							Image:           "bazel/metropolis/test/e2e/preseedtest:preseedtest",
 						}},
 						RestartPolicy: corev1.RestartPolicyNever,
 					},
 				}, metav1.CreateOptions{})
 				return err
 			})
 			testEventual(t, "Pod with preseeded image is completed", ctx, largeTestTimeout, func(ctx context.Context) error {
 				pod, err := clientSet.CoreV1().Pods("default").Get(ctx, "preseed-test-1", metav1.GetOptions{})
 				if err != nil {
 					return fmt.Errorf("failed to get pod: %w", err)
 				}
 				if pod.Status.Phase == corev1.PodSucceeded {
 					return nil
 				}
 				events, err := clientSet.CoreV1().Events("default").List(ctx, metav1.ListOptions{FieldSelector: fmt.Sprintf("involvedObject.name=%s,involvedObject.namespace=default", pod.Name)})
 				if err != nil || len(events.Items) == 0 {
 					return fmt.Errorf("pod is not ready: %v", pod.Status.Phase)
 				} else {
 					return fmt.Errorf("pod is not ready: %v", events.Items[len(events.Items)-1].Message)
 				}
 			})
 			if os.Getenv("HAVE_NESTED_KVM") != "" {
 				testEventual(t, "Pod for KVM/QEMU smoke test", ctx, smallTestTimeout, func(ctx context.Context) error {
 					runcRuntimeClass := "runc"
 					_, err := clientSet.CoreV1().Pods("default").Create(ctx, &corev1.Pod{
 						ObjectMeta: metav1.ObjectMeta{
 							Name: "vm-smoketest",
 						},
 						Spec: corev1.PodSpec{
 							Containers: []corev1.Container{{
 								Name:            "vm-smoketest",
 								ImagePullPolicy: corev1.PullNever,
 								Image:           "bazel/metropolis/vm/smoketest:smoketest_container",
 								Resources: corev1.ResourceRequirements{
 									Limits: corev1.ResourceList{
 										"devices.monogon.dev/kvm": *resource.NewQuantity(1, ""),
 									},
 								},
 							}},
 							RuntimeClassName: &runcRuntimeClass,
 							RestartPolicy:    corev1.RestartPolicyNever,
 						},
 					}, metav1.CreateOptions{})
 					return err
 				})
 				testEventual(t, "KVM/QEMU smoke test completion", ctx, smallTestTimeout, func(ctx context.Context) error {
 					pod, err := clientSet.CoreV1().Pods("default").Get(ctx, "vm-smoketest", metav1.GetOptions{})
 					if err != nil {
 						return fmt.Errorf("failed to get pod: %w", err)
 					}
 					if pod.Status.Phase == corev1.PodSucceeded {
 						return nil
 					}
 					events, err := clientSet.CoreV1().Events("default").List(ctx, metav1.ListOptions{FieldSelector: fmt.Sprintf("involvedObject.name=%s,involvedObject.namespace=default", pod.Name)})
 					if err != nil || len(events.Items) == 0 {
 						return fmt.Errorf("pod is not ready: %v", pod.Status.Phase)
 					} else {
 						return fmt.Errorf("pod is not ready: %v", events.Items[len(events.Items)-1].Message)
 					}
 				})
 			}
 		})
 	})

 	// Cancel the main context and wait for our subprocesses to exit
 	// to avoid leaking them and blocking the parent.
 	cancel()
 	<-procExit
 }
	// Copyright 2020 The Monogon Project Authors.
	//
	// SPDX-License-Identifier: Apache-2.0
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	package e2e

	import (
	"bytes"
	"context"
	"crypto/ed25519"
	"errors"
	"fmt"
	"log"
	"net"
	"net/http"
	_ "net/http"
	_ "net/http/pprof"
	"os"
	"strings"
	"testing"
	"time"

	"google.golang.org/grpc"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	podv1 "k8s.io/kubernetes/pkg/api/v1/pod"

	common "source.monogon.dev/metropolis/node"
	"source.monogon.dev/metropolis/node/core/rpc"
	apb "source.monogon.dev/metropolis/proto/api"
	"source.monogon.dev/metropolis/test/launch"
	)

	const (
	// Timeout for the global test context.
	//
	// Bazel would eventually time out the test after 900s ("large") if, for
	// some reason, the context cancellation fails to abort it.
	globalTestTimeout = 600 * time.Second

	// Timeouts for individual end-to-end tests of different sizes.
	smallTestTimeout = 60 * time.Second
	largeTestTimeout = 120 * time.Second
	)

	// TestE2E is the main E2E test entrypoint for single-node freshly-bootstrapped
	// E2E tests. It starts a full Metropolis node in bootstrap mode and then runs
	// tests against it. The actual tests it performs are located in the RunGroup
	// subtest.
	func TestE2E(t *testing.T) {
	// Run pprof server for debugging
	go func() {
	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
	if err != nil {
	panic(err)
	}

	l, err := net.ListenTCP("tcp", addr)
	if err != nil {
	log.Fatalf("Failed to listen on pprof port: %s", l.Addr())
	}
	defer l.Close()

	log.Printf("pprof server listening on %s", l.Addr())
	log.Printf("pprof server returned an error: %v", http.Serve(l, nil))
	}()

	// Set a global timeout to make sure this terminates
	ctx, cancel := context.WithTimeout(context.Background(), globalTestTimeout)
	defer cancel()
	portMap, err := launch.ConflictFreePortMap(launch.NodePorts)
	if err != nil {
	t.Fatalf("Failed to acquire ports for e2e test: %v", err)
	}

	procExit := make(chan struct{})

	go func() {
	if err := launch.Launch(ctx, launch.Options{
	Ports: portMap,
	SerialPort: os.Stdout,
	NodeParameters: &apb.NodeParameters{
	Cluster: &apb.NodeParameters_ClusterBootstrap_{
	ClusterBootstrap: launch.InsecureClusterBootstrap,
	},
	},
	}); err != nil {
	panic(err)
	}
	close(procExit)
	}()

	grpcDebug, err := portMap.DialGRPC(common.DebugServicePort, grpc.WithInsecure())
	if err != nil {
	log.Printf("Failed to dial debug service (is it running?): %v", err)
	return
	}
	debug := apb.NewNodeDebugServiceClient(grpcDebug)

	// This exists to keep the parent around while all the children race.
	// It currently tests both a set of OS-level conditions and Kubernetes
	// Deployments and StatefulSets
	t.Run("RunGroup", func(t *testing.T) {
	t.Run("Connect to Curator", func(t *testing.T) {
	testEventual(t, "Retrieving owner credentials succesful", ctx, 60*time.Second, func(ctx context.Context) error {
	remote := fmt.Sprintf("localhost:%v", portMap[common.CuratorServicePort])
	initClient, err := rpc.NewEphemeralClient(remote, launch.InsecurePrivateKey, nil)
	if err != nil {
	return fmt.Errorf("NewInitialClient: %w", err)
	}

	aaa := apb.NewAAAClient(initClient)
	cert, err := rpc.RetrieveOwnerCertificate(ctx, aaa, launch.InsecurePrivateKey)
	if err != nil {
	return fmt.Errorf("RetrieveOwnerCertificate: %w", err)
	}

	if !bytes.Equal(cert.PrivateKey.(ed25519.PrivateKey), launch.InsecurePrivateKey) {
	t.Fatalf("Received certificate for wrong private key")
	}

	// Connect to management endpoint and retrieve cluster directory.
	authClient, err := rpc.NewAuthenticatedClient(remote, *cert, nil)
	if err != nil {
	return fmt.Errorf("NewAuthenticatedClient: %w", err)
	}
	mgmt := apb.NewManagementClient(authClient)
	res, err := mgmt.GetClusterInfo(ctx, &apb.GetClusterInfoRequest{})
	if err != nil {
	return fmt.Errorf("GetClusterInfo: %w", err)
	}

	// Ensure the node is there with its address.
	nodes := res.ClusterDirectory.Nodes
	if want, got := 1, len(nodes); want != got {
	return fmt.Errorf("wanted %d nodes in cluster directory, got %d", want, got)
	}
	node := nodes[0]
	if want, got := ed25519.PublicKeySize, len(node.PublicKey); want != got {
	return fmt.Errorf("wanted %d bytes long public key, got %d", want, got)
	}
	if want, got := 1, len(node.Addresses); want != got {
	return fmt.Errorf("wanted %d node address, got %d", want, got)
	}
	if want, got := "10.42.0.10", node.Addresses[0].Host; want != got {
	return fmt.Errorf("wanted status address %q, got %q", want, got)
	}

	return nil
	})
	})
	t.Run("Get Kubernetes Debug Kubeconfig", func(t *testing.T) {
	t.Parallel()
	selfCtx, cancel := context.WithTimeout(ctx, largeTestTimeout)
	defer cancel()
	clientSet, err := GetKubeClientSet(selfCtx, debug, portMap[common.KubernetesAPIPort])
	if err != nil {
	t.Fatal(err)
	}
	testEventual(t, "Node is registered and ready", ctx, largeTestTimeout, func(ctx context.Context) error {
	nodes, err := clientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
	if err != nil {
	return err
	}
	if len(nodes.Items) < 1 {
	return errors.New("node not registered")
	}
	if len(nodes.Items) > 1 {
	return errors.New("more than one node registered (but there is only one)")
	}
	node := nodes.Items[0]
	for _, cond := range node.Status.Conditions {
	if cond.Type != corev1.NodeReady {
	continue
	}
	if cond.Status != corev1.ConditionTrue {
	return fmt.Errorf("node not ready: %v", cond.Message)
	}
	}
	return nil
	})
	testEventual(t, "Simple deployment", ctx, largeTestTimeout, func(ctx context.Context) error {
	_, err := clientSet.AppsV1().Deployments("default").Create(ctx, makeTestDeploymentSpec("test-deploy-1"), metav1.CreateOptions{})
	return err
	})
	testEventual(t, "Simple deployment is running", ctx, largeTestTimeout, func(ctx context.Context) error {
	res, err := clientSet.CoreV1().Pods("default").List(ctx, metav1.ListOptions{LabelSelector: "name=test-deploy-1"})
	if err != nil {
	return err
	}
	if len(res.Items) == 0 {
	return errors.New("pod didn't get created")
	}
	pod := res.Items[0]
	if podv1.IsPodAvailable(&pod, 1, metav1.NewTime(time.Now())) {
	return nil
	}
	events, err := clientSet.CoreV1().Events("default").List(ctx, metav1.ListOptions{FieldSelector: fmt.Sprintf("involvedObject.name=%s,involvedObject.namespace=default", pod.Name)})
	if err != nil \|\| len(events.Items) == 0 {
	return fmt.Errorf("pod is not ready: %v", pod.Status.Phase)
	} else {
	return fmt.Errorf("pod is not ready: %v", events.Items[0].Message)
	}
	})
	testEventual(t, "Simple deployment with runc", ctx, largeTestTimeout, func(ctx context.Context) error {
	deployment := makeTestDeploymentSpec("test-deploy-2")
	var runcStr = "runc"
	deployment.Spec.Template.Spec.RuntimeClassName = &runcStr
	_, err := clientSet.AppsV1().Deployments("default").Create(ctx, deployment, metav1.CreateOptions{})
	return err
	})
	testEventual(t, "Simple deployment is running on runc", ctx, largeTestTimeout, func(ctx context.Context) error {
	res, err := clientSet.CoreV1().Pods("default").List(ctx, metav1.ListOptions{LabelSelector: "name=test-deploy-2"})
	if err != nil {
	return err
	}
	if len(res.Items) == 0 {
	return errors.New("pod didn't get created")
	}
	pod := res.Items[0]
	if podv1.IsPodAvailable(&pod, 1, metav1.NewTime(time.Now())) {
	return nil
	}
	events, err := clientSet.CoreV1().Events("default").List(ctx, metav1.ListOptions{FieldSelector: fmt.Sprintf("involvedObject.name=%s,involvedObject.namespace=default", pod.Name)})
	if err != nil \|\| len(events.Items) == 0 {
	return fmt.Errorf("pod is not ready: %v", pod.Status.Phase)
	} else {
	var errorMsg strings.Builder
	for _, msg := range events.Items {
	errorMsg.WriteString(" \| ")
	errorMsg.WriteString(msg.Message)
	}
	return fmt.Errorf("pod is not ready: %v", errorMsg.String())
	}
	})
	testEventual(t, "Simple StatefulSet with PVC", ctx, largeTestTimeout, func(ctx context.Context) error {
	_, err := clientSet.AppsV1().StatefulSets("default").Create(ctx, makeTestStatefulSet("test-statefulset-1", corev1.PersistentVolumeFilesystem), metav1.CreateOptions{})
	return err
	})
	testEventual(t, "Simple StatefulSet with PVC is running", ctx, largeTestTimeout, func(ctx context.Context) error {
	res, err := clientSet.CoreV1().Pods("default").List(ctx, metav1.ListOptions{LabelSelector: "name=test-statefulset-1"})
	if err != nil {
	return err
	}
	if len(res.Items) == 0 {
	return errors.New("pod didn't get created")
	}
	pod := res.Items[0]
	if podv1.IsPodAvailable(&pod, 1, metav1.NewTime(time.Now())) {
	return nil
	}
	events, err := clientSet.CoreV1().Events("default").List(ctx, metav1.ListOptions{FieldSelector: fmt.Sprintf("involvedObject.name=%s,involvedObject.namespace=default", pod.Name)})
	if err != nil \|\| len(events.Items) == 0 {
	return fmt.Errorf("pod is not ready: %v", pod.Status.Phase)
	} else {
	return fmt.Errorf("pod is not ready: %v", events.Items[0].Message)
	}
	})
	testEventual(t, "Simple StatefulSet with Block PVC", ctx, largeTestTimeout, func(ctx context.Context) error {
	_, err := clientSet.AppsV1().StatefulSets("default").Create(ctx, makeTestStatefulSet("test-statefulset-2", corev1.PersistentVolumeBlock), metav1.CreateOptions{})
	return err
	})
	testEventual(t, "Simple StatefulSet with Block PVC is running", ctx, largeTestTimeout, func(ctx context.Context) error {
	res, err := clientSet.CoreV1().Pods("default").List(ctx, metav1.ListOptions{LabelSelector: "name=test-statefulset-2"})
	if err != nil {
	return err
	}
	if len(res.Items) == 0 {
	return errors.New("pod didn't get created")
	}
	pod := res.Items[0]
	if podv1.IsPodAvailable(&pod, 1, metav1.NewTime(time.Now())) {
	return nil
	}
	events, err := clientSet.CoreV1().Events("default").List(ctx, metav1.ListOptions{FieldSelector: fmt.Sprintf("involvedObject.name=%s,involvedObject.namespace=default", pod.Name)})
	if err != nil \|\| len(events.Items) == 0 {
	return fmt.Errorf("pod is not ready: %v", pod.Status.Phase)
	} else {
	return fmt.Errorf("pod is not ready: %v", events.Items[0].Message)
	}
	})
	testEventual(t, "Pod with preseeded image", ctx, smallTestTimeout, func(ctx context.Context) error {
	_, err := clientSet.CoreV1().Pods("default").Create(ctx, &corev1.Pod{
	ObjectMeta: metav1.ObjectMeta{
	Name: "preseed-test-1",
	},
	Spec: corev1.PodSpec{
	Containers: []corev1.Container{{
	Name: "preseed-test-1",
	ImagePullPolicy: corev1.PullNever,
	Image: "bazel/metropolis/test/e2e/preseedtest:preseedtest",
	}},
	RestartPolicy: corev1.RestartPolicyNever,
	},
	}, metav1.CreateOptions{})
	return err
	})
	testEventual(t, "Pod with preseeded image is completed", ctx, largeTestTimeout, func(ctx context.Context) error {
	pod, err := clientSet.CoreV1().Pods("default").Get(ctx, "preseed-test-1", metav1.GetOptions{})
	if err != nil {
	return fmt.Errorf("failed to get pod: %w", err)
	}
	if pod.Status.Phase == corev1.PodSucceeded {
	return nil
	}
	events, err := clientSet.CoreV1().Events("default").List(ctx, metav1.ListOptions{FieldSelector: fmt.Sprintf("involvedObject.name=%s,involvedObject.namespace=default", pod.Name)})
	if err != nil \|\| len(events.Items) == 0 {
	return fmt.Errorf("pod is not ready: %v", pod.Status.Phase)
	} else {
	return fmt.Errorf("pod is not ready: %v", events.Items[len(events.Items)-1].Message)
	}
	})
	if os.Getenv("HAVE_NESTED_KVM") != "" {
	testEventual(t, "Pod for KVM/QEMU smoke test", ctx, smallTestTimeout, func(ctx context.Context) error {
	runcRuntimeClass := "runc"
	_, err := clientSet.CoreV1().Pods("default").Create(ctx, &corev1.Pod{
	ObjectMeta: metav1.ObjectMeta{
	Name: "vm-smoketest",
	},
	Spec: corev1.PodSpec{
	Containers: []corev1.Container{{
	Name: "vm-smoketest",
	ImagePullPolicy: corev1.PullNever,
	Image: "bazel/metropolis/vm/smoketest:smoketest_container",
	Resources: corev1.ResourceRequirements{
	Limits: corev1.ResourceList{
	"devices.monogon.dev/kvm": *resource.NewQuantity(1, ""),
	},
	},
	}},
	RuntimeClassName: &runcRuntimeClass,
	RestartPolicy: corev1.RestartPolicyNever,
	},
	}, metav1.CreateOptions{})
	return err
	})
	testEventual(t, "KVM/QEMU smoke test completion", ctx, smallTestTimeout, func(ctx context.Context) error {
	pod, err := clientSet.CoreV1().Pods("default").Get(ctx, "vm-smoketest", metav1.GetOptions{})
	if err != nil {
	return fmt.Errorf("failed to get pod: %w", err)
	}
	if pod.Status.Phase == corev1.PodSucceeded {
	return nil
	}
	events, err := clientSet.CoreV1().Events("default").List(ctx, metav1.ListOptions{FieldSelector: fmt.Sprintf("involvedObject.name=%s,involvedObject.namespace=default", pod.Name)})
	if err != nil \|\| len(events.Items) == 0 {
	return fmt.Errorf("pod is not ready: %v", pod.Status.Phase)
	} else {
	return fmt.Errorf("pod is not ready: %v", events.Items[len(events.Items)-1].Message)
	}
	})
	}
	})
	})

	// Cancel the main context and wait for our subprocesses to exit
	// to avoid leaking them and blocking the parent.
	cancel()
	<-procExit
	}