Implement Block PVCs in our storage backend
This implements full support for Block PVCs in our Kubernetes storage backend.
The block PVCs are backed by files made available to the pods using loop devices and
have read-only and online expansion support.
This also requires a Kubernetes patch because they call losetup if block PVCs are used
with CSI to establish a form of lock on the backing block device. This lock is not
exclusive and does absolutely nothing for our use case and could get very expensive
on dense machines so I removed it.
Test Plan: Comes with E2E tests
X-Origin-Diff: phab/D746
GitOrigin-RevId: 430d3f445286c0d3498b2153df333a19f3fcab89
diff --git a/metropolis/test/e2e/kubernetes_helpers.go b/metropolis/test/e2e/kubernetes_helpers.go
index b3dceae..71e8c3a 100644
--- a/metropolis/test/e2e/kubernetes_helpers.go
+++ b/metropolis/test/e2e/kubernetes_helpers.go
@@ -103,7 +103,7 @@
}
// makeTestStatefulSet generates a StatefulSet spec
-func makeTestStatefulSet(name string) *appsv1.StatefulSet {
+func makeTestStatefulSet(name string, volumeMode corev1.PersistentVolumeMode) *appsv1.StatefulSet {
return &appsv1.StatefulSet{
ObjectMeta: metav1.ObjectMeta{Name: name},
Spec: appsv1.StatefulSetSpec{
@@ -118,6 +118,7 @@
Resources: corev1.ResourceRequirements{
Requests: map[corev1.ResourceName]resource.Quantity{corev1.ResourceStorage: resource.MustParse("50Mi")},
},
+ VolumeMode: &volumeMode,
},
},
},
diff --git a/metropolis/test/e2e/main_test.go b/metropolis/test/e2e/main_test.go
index 46e5c72..0463fc4 100644
--- a/metropolis/test/e2e/main_test.go
+++ b/metropolis/test/e2e/main_test.go
@@ -190,7 +190,7 @@
}
})
testEventual(t, "Simple StatefulSet with PVC", ctx, largeTestTimeout, func(ctx context.Context) error {
- _, err := clientSet.AppsV1().StatefulSets("default").Create(ctx, makeTestStatefulSet("test-statefulset-1"), metav1.CreateOptions{})
+ _, err := clientSet.AppsV1().StatefulSets("default").Create(ctx, makeTestStatefulSet("test-statefulset-1", corev1.PersistentVolumeFilesystem), metav1.CreateOptions{})
return err
})
testEventual(t, "Simple StatefulSet with PVC is running", ctx, largeTestTimeout, func(ctx context.Context) error {
@@ -212,6 +212,29 @@
return fmt.Errorf("pod is not ready: %v", events.Items[0].Message)
}
})
+ testEventual(t, "Simple StatefulSet with Block PVC", ctx, largeTestTimeout, func(ctx context.Context) error {
+ _, err := clientSet.AppsV1().StatefulSets("default").Create(ctx, makeTestStatefulSet("test-statefulset-2", corev1.PersistentVolumeBlock), metav1.CreateOptions{})
+ return err
+ })
+ testEventual(t, "Simple StatefulSet with Block PVC is running", ctx, largeTestTimeout, func(ctx context.Context) error {
+ res, err := clientSet.CoreV1().Pods("default").List(ctx, metav1.ListOptions{LabelSelector: "name=test-statefulset-2"})
+ if err != nil {
+ return err
+ }
+ if len(res.Items) == 0 {
+ return errors.New("pod didn't get created")
+ }
+ pod := res.Items[0]
+ if podv1.IsPodAvailable(&pod, 1, metav1.NewTime(time.Now())) {
+ return nil
+ }
+ events, err := clientSet.CoreV1().Events("default").List(ctx, metav1.ListOptions{FieldSelector: fmt.Sprintf("involvedObject.name=%s,involvedObject.namespace=default", pod.Name)})
+ if err != nil || len(events.Items) == 0 {
+ return fmt.Errorf("pod is not ready: %v", pod.Status.Phase)
+ } else {
+ return fmt.Errorf("pod is not ready: %v", events.Items[0].Message)
+ }
+ })
testEventual(t, "Pod with preseeded image", ctx, smallTestTimeout, func(ctx context.Context) error {
_, err := clientSet.CoreV1().Pods("default").Create(ctx, &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{