Add PV provisioner
This adds a new PV provisioner which works together with the
CSI Node driver to provide storage to workloads on Smalltown.
It talks to Kubernetes and listens for PVCs which need to be provisioned
and PVs which have been released and need to be deleted.
Is is implemented as a per-node agent where every node provisions the
volumes scheduled onto it by kube-scheduler.
Test Plan: Manually tested by running `bazel run //core/cmd/dbg -- kubectl create -f $PWD/pv-test.yml` and observing a provisioned PV that's attached to the pod. An example `test-pv.yml` is in P137.
X-Origin-Diff: phab/D482
GitOrigin-RevId: 75a871b039e71dd248f937719c471e0277887964
diff --git a/core/internal/kubernetes/reconcile.go b/core/internal/kubernetes/reconcile.go
index 092cd8e..93ee4da 100644
--- a/core/internal/kubernetes/reconcile.go
+++ b/core/internal/kubernetes/reconcile.go
@@ -26,13 +26,14 @@
"context"
"time"
+ storagev1 "k8s.io/api/storage/v1"
+
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
"k8s.io/api/policy/v1beta1"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
- "k8s.io/client-go/tools/clientcmd"
"git.monogon.dev/source/nexantic.git/core/internal/common/supervisor"
)
@@ -176,25 +177,54 @@
},
}
-type reconciler func(context.Context, *kubernetes.Clientset) error
+var reclaimPolicyDelete = corev1.PersistentVolumeReclaimDelete
+var waitForConsumerBinding = storagev1.VolumeBindingWaitForFirstConsumer
-func runReconciler(masterKubeconfig []byte) supervisor.Runnable {
+var builtinStorageClasses = []*storagev1.StorageClass{
+ {
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "local",
+ Annotations: map[string]string{
+ "storageclass.kubernetes.io/is-default-class": "true",
+ },
+ Labels: map[string]string{
+ "smalltown.com/builtin": "true",
+ },
+ },
+ AllowVolumeExpansion: True(),
+ Provisioner: csiProvisionerName,
+ ReclaimPolicy: &reclaimPolicyDelete,
+ VolumeBindingMode: &waitForConsumerBinding,
+ },
+}
+
+var builtinCSIDrivers = []*storagev1.CSIDriver{
+ {
+ ObjectMeta: metav1.ObjectMeta{
+ Name: csiProvisionerName,
+ Labels: map[string]string{
+ "smalltown.com/builtin": "true",
+ },
+ },
+ Spec: storagev1.CSIDriverSpec{
+ AttachRequired: False(),
+ PodInfoOnMount: False(),
+ VolumeLifecycleModes: []storagev1.VolumeLifecycleMode{storagev1.VolumeLifecyclePersistent},
+ },
+ },
+}
+
+type reconciler func(context.Context, kubernetes.Interface) error
+
+func runReconciler(clientSet kubernetes.Interface) supervisor.Runnable {
return func(ctx context.Context) error {
log := supervisor.Logger(ctx)
- rawClientConfig, err := clientcmd.NewClientConfigFromBytes(masterKubeconfig)
- if err != nil {
- return err
- }
-
- clientConfig, err := rawClientConfig.ClientConfig()
- clientSet, err := kubernetes.NewForConfig(clientConfig)
- if err != nil {
- return err
- }
reconcilers := map[string]reconciler{
"psps": reconcilePSPs,
"clusterroles": reconcileClusterRoles,
"clusterrolebindings": reconcileClusterRoleBindings,
+ "storageclasses": reconcileSCs,
+ "csidrivers": reconcileCSIDrivers,
}
t := time.NewTicker(10 * time.Second)
reconcile := func() {
@@ -217,7 +247,7 @@
}
}
-func reconcilePSPs(ctx context.Context, clientSet *kubernetes.Clientset) error {
+func reconcilePSPs(ctx context.Context, clientSet kubernetes.Interface) error {
pspClient := clientSet.PolicyV1beta1().PodSecurityPolicies()
availablePSPs, err := pspClient.List(ctx, metav1.ListOptions{
LabelSelector: "smalltown.com/builtin=true",
@@ -250,7 +280,7 @@
return nil
}
-func reconcileClusterRoles(ctx context.Context, clientSet *kubernetes.Clientset) error {
+func reconcileClusterRoles(ctx context.Context, clientSet kubernetes.Interface) error {
crClient := clientSet.RbacV1().ClusterRoles()
availableCRs, err := crClient.List(ctx, metav1.ListOptions{
LabelSelector: "smalltown.com/builtin=true",
@@ -283,7 +313,7 @@
return nil
}
-func reconcileClusterRoleBindings(ctx context.Context, clientset *kubernetes.Clientset) error {
+func reconcileClusterRoleBindings(ctx context.Context, clientset kubernetes.Interface) error {
crbClient := clientset.RbacV1().ClusterRoleBindings()
availableCRBs, err := crbClient.List(ctx, metav1.ListOptions{
LabelSelector: "smalltown.com/builtin=true",
@@ -315,3 +345,69 @@
}
return nil
}
+
+func reconcileSCs(ctx context.Context, clientSet kubernetes.Interface) error {
+ scsClient := clientSet.StorageV1().StorageClasses()
+ availableSCs, err := scsClient.List(ctx, metav1.ListOptions{
+ LabelSelector: "smalltown.com/builtin=true",
+ })
+ if err != nil {
+ return err
+ }
+ availableSCMap := make(map[string]struct{})
+ for _, sc := range availableSCs.Items {
+ availableSCMap[sc.Name] = struct{}{}
+ }
+ expectedSCMap := make(map[string]*storagev1.StorageClass)
+ for _, sc := range builtinStorageClasses {
+ expectedSCMap[sc.Name] = sc
+ }
+ for scName, sc := range expectedSCMap {
+ if _, ok := availableSCMap[scName]; !ok {
+ if _, err := scsClient.Create(ctx, sc, metav1.CreateOptions{}); err != nil {
+ return err
+ }
+ }
+ }
+ for pspName, _ := range availableSCMap {
+ if _, ok := expectedSCMap[pspName]; !ok {
+ if err := scsClient.Delete(ctx, pspName, metav1.DeleteOptions{}); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+func reconcileCSIDrivers(ctx context.Context, clientSet kubernetes.Interface) error {
+ drvClient := clientSet.StorageV1().CSIDrivers()
+ availableDrvs, err := drvClient.List(ctx, metav1.ListOptions{
+ LabelSelector: "smalltown.com/builtin=true",
+ })
+ if err != nil {
+ return err
+ }
+ availableDrvMap := make(map[string]struct{})
+ for _, drv := range availableDrvs.Items {
+ availableDrvMap[drv.Name] = struct{}{}
+ }
+ expectedDrvMap := make(map[string]*storagev1.CSIDriver)
+ for _, drv := range builtinCSIDrivers {
+ expectedDrvMap[drv.Name] = drv
+ }
+ for drvName, drv := range expectedDrvMap {
+ if _, ok := availableDrvMap[drvName]; !ok {
+ if _, err := drvClient.Create(ctx, drv, metav1.CreateOptions{}); err != nil {
+ return err
+ }
+ }
+ }
+ for pspName, _ := range availableDrvMap {
+ if _, ok := expectedDrvMap[pspName]; !ok {
+ if err := drvClient.Delete(ctx, pspName, metav1.DeleteOptions{}); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}