m/node/kubernetes: fix PV mount flags and add e2e test

Mount flags did not work because of two problems:
- The provisioner did not copy them from the StorageClass to the
  PersistentVolume.
- The CSI server used = instead of |= when adding flags, so only one of
  the flags was added or removed.

There was an existing e2e test for PVs, however this only created the
PVC/PV without even attaching it to a container. I extended this test to
attach the PV and check from inside the container that it has the
expected mount flags and quota.

The existing e2e test also created a block PV, however attaching a block
PV to a container was not tested and is apparently broken, so I removed
this test for now.

Change-Id: Ie14adfafd333eab38d2b5f1b4ce8a2aa8795eae0
Reviewed-on: https://review.monogon.dev/c/monogon/+/3613
Reviewed-by: Lorenz Brun <lorenz@monogon.tech>
Tested-by: Jenkins CI
diff --git a/metropolis/test/e2e/suites/kubernetes/run_test.go b/metropolis/test/e2e/suites/kubernetes/run_test.go
index 0d53e07..5a4be60 100644
--- a/metropolis/test/e2e/suites/kubernetes/run_test.go
+++ b/metropolis/test/e2e/suites/kubernetes/run_test.go
@@ -370,11 +370,11 @@
 			return fmt.Errorf("pod is not ready: %s", errorMsg.String())
 		}
 	})
-	util.TestEventual(t, "Simple StatefulSet with PVC", ctx, largeTestTimeout, func(ctx context.Context) error {
-		_, err := clientSet.AppsV1().StatefulSets("default").Create(ctx, makeTestStatefulSet("test-statefulset-1", corev1.PersistentVolumeFilesystem), metav1.CreateOptions{})
+	util.TestEventual(t, "StatefulSet with PersistentVolume tests", ctx, smallTestTimeout, func(ctx context.Context) error {
+		_, err := clientSet.AppsV1().StatefulSets("default").Create(ctx, makeTestStatefulSet("test-statefulset-1"), metav1.CreateOptions{})
 		return err
 	})
-	util.TestEventual(t, "Simple StatefulSet with PVC is running", ctx, largeTestTimeout, func(ctx context.Context) error {
+	util.TestEventual(t, "StatefulSet with PersistentVolume tests successful", ctx, smallTestTimeout, func(ctx context.Context) error {
 		res, err := clientSet.CoreV1().Pods("default").List(ctx, metav1.ListOptions{LabelSelector: "name=test-statefulset-1"})
 		if err != nil {
 			return err
@@ -383,38 +383,19 @@
 			return errors.New("pod didn't get created")
 		}
 		pod := res.Items[0]
-		if podv1.IsPodAvailable(&pod, 1, metav1.NewTime(time.Now())) {
-			return nil
-		}
-		events, err := clientSet.CoreV1().Events("default").List(ctx, metav1.ListOptions{FieldSelector: fmt.Sprintf("involvedObject.name=%s,involvedObject.namespace=default", pod.Name)})
-		if err != nil || len(events.Items) == 0 {
-			return fmt.Errorf("pod is not ready: %v", pod.Status.Phase)
-		} else {
-			return fmt.Errorf("pod is not ready: %v", events.Items[0].Message)
-		}
-	})
-	util.TestEventual(t, "Simple StatefulSet with Block PVC", ctx, largeTestTimeout, func(ctx context.Context) error {
-		_, err := clientSet.AppsV1().StatefulSets("default").Create(ctx, makeTestStatefulSet("test-statefulset-2", corev1.PersistentVolumeBlock), metav1.CreateOptions{})
-		return err
-	})
-	util.TestEventual(t, "Simple StatefulSet with Block PVC is running", ctx, largeTestTimeout, func(ctx context.Context) error {
-		res, err := clientSet.CoreV1().Pods("default").List(ctx, metav1.ListOptions{LabelSelector: "name=test-statefulset-2"})
+		lines, err := getPodLogLines(ctx, clientSet, pod.Name, 50)
 		if err != nil {
-			return err
+			return fmt.Errorf("could not get logs: %w", err)
 		}
-		if len(res.Items) == 0 {
-			return errors.New("pod didn't get created")
+		if len(lines) > 0 {
+			switch lines[len(lines)-1] {
+			case "[TESTS-PASSED]":
+				return nil
+			case "[TESTS-FAILED]":
+				return util.Permanent(fmt.Errorf("tests failed, log:\n  %s", strings.Join(lines, "\n  ")))
+			}
 		}
-		pod := res.Items[0]
-		if podv1.IsPodAvailable(&pod, 1, metav1.NewTime(time.Now())) {
-			return nil
-		}
-		events, err := clientSet.CoreV1().Events("default").List(ctx, metav1.ListOptions{FieldSelector: fmt.Sprintf("involvedObject.name=%s,involvedObject.namespace=default", pod.Name)})
-		if err != nil || len(events.Items) == 0 {
-			return fmt.Errorf("pod is not ready: %v", pod.Status.Phase)
-		} else {
-			return fmt.Errorf("pod is not ready: %v", events.Items[0].Message)
-		}
+		return fmt.Errorf("pod is not ready: %v, log:\n  %s", pod.Status.Phase, strings.Join(lines, "\n  "))
 	})
 	util.TestEventual(t, "In-cluster self-test job", ctx, smallTestTimeout, func(ctx context.Context) error {
 		_, err := clientSet.BatchV1().Jobs("default").Create(ctx, makeSelftestSpec("selftest"), metav1.CreateOptions{})