m/node/kubernetes: implement storage resizing
This implements persistent volume resizing in the storage provisioner.
The logic is based on https://github.com/kubernetes-csi/external-resizer
The mutation caches are an optimization to prevent unnecessary repeated
processing, because they make the controller remember changes that it
has made itself, when the watch events for those changes have not
arrived yet.
The controller supports the RecoverVolumeExpansionFailure feature, which
allows reducing the requested size when the previous resize fails due to
insufficient space. When resize fails, it is retried with backoff.
Change-Id: I0f3d40c1a592b30d25739f5d20b529dfe25dfbe1
Reviewed-on: https://review.monogon.dev/c/monogon/+/4008
Reviewed-by: Lorenz Brun <lorenz@monogon.tech>
Tested-by: Jenkins CI
diff --git a/metropolis/test/e2e/persistentvolume/main.go b/metropolis/test/e2e/persistentvolume/main.go
index c80700a..6af1258 100644
--- a/metropolis/test/e2e/persistentvolume/main.go
+++ b/metropolis/test/e2e/persistentvolume/main.go
@@ -88,16 +88,16 @@
return nil
}
-func testPersistentVolume() error {
- if err := checkFilesystemVolume("/vol/default", 0, 1*1024*1024); err != nil {
+func testPersistentVolume(expectedBytes uint64) error {
+ if err := checkFilesystemVolume("/vol/default", 0, expectedBytes); err != nil {
return err
}
- if err := checkFilesystemVolume("/vol/readonly", unix.ST_RDONLY, 1*1024*1024); err != nil {
+ if err := checkFilesystemVolume("/vol/readonly", unix.ST_RDONLY, expectedBytes); err != nil {
return err
}
// Block volumes are not supported on gVisor.
if *runtimeClass != "gvisor" {
- if err := checkBlockVolume("/vol/block", 1*1024*1024); err != nil {
+ if err := checkBlockVolume("/vol/block", expectedBytes); err != nil {
return err
}
}
@@ -108,12 +108,26 @@
flag.Parse()
fmt.Printf("PersistentVolume tests starting on %s...\n", *runtimeClass)
- if err := testPersistentVolume(); err != nil {
+ if err := testPersistentVolume(1 * 1024 * 1024); err != nil {
fmt.Println(err.Error())
// The final log line communicates the test outcome to the e2e test.
- fmt.Println("[TESTS-FAILED]")
+ fmt.Println("[INIT-FAILED]")
} else {
- fmt.Println("[TESTS-PASSED]")
+ fmt.Println("[INIT-PASSED]")
+
+ nextLog := time.Now().Add(10 * time.Second)
+ for {
+ if err := testPersistentVolume(4 * 1024 * 1024); err != nil {
+ if time.Now().After(nextLog) {
+ fmt.Println("Waiting for resize:", err.Error())
+ nextLog = time.Now().Add(10 * time.Second)
+ }
+ time.Sleep(100 * time.Millisecond)
+ } else {
+ fmt.Println("[RESIZE-PASSED]")
+ break
+ }
+ }
}
// Sleep forever, because if the process exits, Kubernetes will restart it.