blob: 93ee4da7dc8909bf075d6eda3bff62237a1a95a3 [file] [log] [blame]
Lorenz Brun878f5f92020-05-12 16:15:39 +02001// Copyright 2020 The Monogon Project Authors.
2//
3// SPDX-License-Identifier: Apache-2.0
4//
5// Licensed under the Apache License, Version 2.0 (the "License");
6// you may not use this file except in compliance with the License.
7// You may obtain a copy of the License at
8//
9// http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing, software
12// distributed under the License is distributed on an "AS IS" BASIS,
13// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14// See the License for the specific language governing permissions and
15// limitations under the License.
16
17// The reconciler ensures that a base set of K8s resources is always available in the cluster. These are necessary to
18// ensure correct out-of-the-box functionality. All resources containing the smalltown.com/builtin=true label are assumed
19// to be managed by the reconciler.
20// It currently does not revert modifications made by admins, it is planned to create an admission plugin prohibiting
21// such modifications to resources with the smalltown.com/builtin label to deal with that problem. This would also solve a
22// potential issue where you could delete resources just by adding the smalltown.com/builtin=true label.
23package kubernetes
24
25import (
26 "context"
27 "time"
28
Lorenz Brunb15abad2020-04-16 11:17:12 +020029 storagev1 "k8s.io/api/storage/v1"
30
Lorenz Brun878f5f92020-05-12 16:15:39 +020031 "go.uber.org/zap"
32 corev1 "k8s.io/api/core/v1"
33 "k8s.io/api/policy/v1beta1"
34 rbacv1 "k8s.io/api/rbac/v1"
35 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
36 "k8s.io/client-go/kubernetes"
Lorenz Brun8e3b8fc2020-05-19 14:29:40 +020037
38 "git.monogon.dev/source/nexantic.git/core/internal/common/supervisor"
Lorenz Brun878f5f92020-05-12 16:15:39 +020039)
40
41const builtinRBACPrefix = "smalltown:"
42
43// Sad workaround for all the pointer booleans in K8s specs
44func True() *bool {
45 val := true
46 return &val
47}
48func False() *bool {
49 val := false
50 return &val
51}
52
53func rbac(name string) string {
54 return builtinRBACPrefix + name
55}
56
57// Extended from https://github.com/kubernetes/kubernetes/blob/master/cluster/gce/addons/podsecuritypolicies/unprivileged-addon.yaml
58var builtinPSPs = []*v1beta1.PodSecurityPolicy{
59 {
60 ObjectMeta: metav1.ObjectMeta{
61 Name: "default",
62 Labels: map[string]string{
63 "smalltown.com/builtin": "true",
64 },
65 Annotations: map[string]string{
66 "kubernetes.io/description": "This default PSP allows the creation of pods using features that are" +
67 " generally considered safe against any sort of escape.",
68 },
69 },
70 Spec: v1beta1.PodSecurityPolicySpec{
71 AllowPrivilegeEscalation: True(),
72 AllowedCapabilities: []corev1.Capability{ // runc's default list of allowed capabilities
73 "SETPCAP",
74 "MKNOD",
75 "AUDIT_WRITE",
76 "CHOWN",
77 "NET_RAW",
78 "DAC_OVERRIDE",
79 "FOWNER",
80 "FSETID",
81 "KILL",
82 "SETGID",
83 "SETUID",
84 "NET_BIND_SERVICE",
85 "SYS_CHROOT",
86 "SETFCAP",
87 },
88 HostNetwork: false,
89 HostIPC: false,
90 HostPID: false,
91 FSGroup: v1beta1.FSGroupStrategyOptions{
92 Rule: v1beta1.FSGroupStrategyRunAsAny,
93 },
94 RunAsUser: v1beta1.RunAsUserStrategyOptions{
95 Rule: v1beta1.RunAsUserStrategyRunAsAny,
96 },
97 SELinux: v1beta1.SELinuxStrategyOptions{
98 Rule: v1beta1.SELinuxStrategyRunAsAny,
99 },
100 SupplementalGroups: v1beta1.SupplementalGroupsStrategyOptions{
101 Rule: v1beta1.SupplementalGroupsStrategyRunAsAny,
102 },
103 Volumes: []v1beta1.FSType{ // Volumes considered safe to use
104 v1beta1.ConfigMap,
105 v1beta1.EmptyDir,
106 v1beta1.Projected,
107 v1beta1.Secret,
108 v1beta1.DownwardAPI,
109 v1beta1.PersistentVolumeClaim,
110 },
111 },
112 },
113}
114
115var builtinClusterRoles = []*rbacv1.ClusterRole{
116 {
117 ObjectMeta: metav1.ObjectMeta{
118 Name: rbac("psp-default"),
119 Annotations: map[string]string{
120 "kubernetes.io/description": "This role grants access to the \"default\" PSP.",
121 },
122 },
123 Rules: []rbacv1.PolicyRule{
124 {
125 APIGroups: []string{"policy"},
126 Resources: []string{"podsecuritypolicies"},
127 ResourceNames: []string{"default"},
128 Verbs: []string{"use"},
129 },
130 },
131 },
132}
133
134var builtinClusterRoleBindings = []*rbacv1.ClusterRoleBinding{
135 {
136 ObjectMeta: metav1.ObjectMeta{
137 Name: rbac("default-psp-for-sa"),
138 Annotations: map[string]string{
139 "kubernetes.io/description": "This binding grants every service account access to the \"default\" PSP. " +
140 "Creation of Pods is still restricted by other RBAC roles. Otherwise no pods (unprivileged or not) " +
141 "can be created.",
142 },
143 },
144 RoleRef: rbacv1.RoleRef{
145 APIGroup: rbacv1.GroupName,
146 Kind: "ClusterRole",
147 Name: rbac("psp-default"),
148 },
149 Subjects: []rbacv1.Subject{
150 {
151 APIGroup: rbacv1.GroupName,
152 Kind: "Group",
153 Name: "system:serviceaccounts",
154 },
155 },
156 },
157 {
158 ObjectMeta: metav1.ObjectMeta{
159 Name: rbac("apiserver-kubelet-client"),
160 Annotations: map[string]string{
161 "kubernetes.io/description": "This binding grants the apiserver access to the kubelets. This enables " +
162 "lots of built-in functionality like reading logs or forwarding ports via the API.",
163 },
164 },
165 RoleRef: rbacv1.RoleRef{
166 APIGroup: rbacv1.GroupName,
167 Kind: "ClusterRole",
168 Name: "system:kubelet-api-admin",
169 },
170 Subjects: []rbacv1.Subject{
171 {
172 APIGroup: rbacv1.GroupName,
173 Kind: "User",
174 Name: "smalltown:apiserver-kubelet-client",
175 },
176 },
177 },
178}
179
Lorenz Brunb15abad2020-04-16 11:17:12 +0200180var reclaimPolicyDelete = corev1.PersistentVolumeReclaimDelete
181var waitForConsumerBinding = storagev1.VolumeBindingWaitForFirstConsumer
Lorenz Brun878f5f92020-05-12 16:15:39 +0200182
Lorenz Brunb15abad2020-04-16 11:17:12 +0200183var builtinStorageClasses = []*storagev1.StorageClass{
184 {
185 ObjectMeta: metav1.ObjectMeta{
186 Name: "local",
187 Annotations: map[string]string{
188 "storageclass.kubernetes.io/is-default-class": "true",
189 },
190 Labels: map[string]string{
191 "smalltown.com/builtin": "true",
192 },
193 },
194 AllowVolumeExpansion: True(),
195 Provisioner: csiProvisionerName,
196 ReclaimPolicy: &reclaimPolicyDelete,
197 VolumeBindingMode: &waitForConsumerBinding,
198 },
199}
200
201var builtinCSIDrivers = []*storagev1.CSIDriver{
202 {
203 ObjectMeta: metav1.ObjectMeta{
204 Name: csiProvisionerName,
205 Labels: map[string]string{
206 "smalltown.com/builtin": "true",
207 },
208 },
209 Spec: storagev1.CSIDriverSpec{
210 AttachRequired: False(),
211 PodInfoOnMount: False(),
212 VolumeLifecycleModes: []storagev1.VolumeLifecycleMode{storagev1.VolumeLifecyclePersistent},
213 },
214 },
215}
216
217type reconciler func(context.Context, kubernetes.Interface) error
218
219func runReconciler(clientSet kubernetes.Interface) supervisor.Runnable {
Lorenz Brun8e3b8fc2020-05-19 14:29:40 +0200220 return func(ctx context.Context) error {
221 log := supervisor.Logger(ctx)
Lorenz Brun8e3b8fc2020-05-19 14:29:40 +0200222 reconcilers := map[string]reconciler{
223 "psps": reconcilePSPs,
224 "clusterroles": reconcileClusterRoles,
225 "clusterrolebindings": reconcileClusterRoleBindings,
Lorenz Brunb15abad2020-04-16 11:17:12 +0200226 "storageclasses": reconcileSCs,
227 "csidrivers": reconcileCSIDrivers,
Lorenz Brun8e3b8fc2020-05-19 14:29:40 +0200228 }
229 t := time.NewTicker(10 * time.Second)
230 reconcile := func() {
231 for name, reconciler := range reconcilers {
232 if err := reconciler(ctx, clientSet); err != nil {
233 log.Warn("Failed to reconcile built-in resources", zap.String("kind", name), zap.Error(err))
234 }
Lorenz Brun878f5f92020-05-12 16:15:39 +0200235 }
Lorenz Brun8e3b8fc2020-05-19 14:29:40 +0200236 }
237 supervisor.Signal(ctx, supervisor.SignalHealthy)
238 reconcile()
239 for {
240 select {
241 case <-t.C:
242 reconcile()
243 case <-ctx.Done():
244 return nil
245 }
Lorenz Brun878f5f92020-05-12 16:15:39 +0200246 }
247 }
248}
249
Lorenz Brunb15abad2020-04-16 11:17:12 +0200250func reconcilePSPs(ctx context.Context, clientSet kubernetes.Interface) error {
Lorenz Brun8e3b8fc2020-05-19 14:29:40 +0200251 pspClient := clientSet.PolicyV1beta1().PodSecurityPolicies()
Lorenz Brun878f5f92020-05-12 16:15:39 +0200252 availablePSPs, err := pspClient.List(ctx, metav1.ListOptions{
253 LabelSelector: "smalltown.com/builtin=true",
254 })
255 if err != nil {
256 return err
257 }
258 availablePSPMap := make(map[string]struct{})
259 for _, psp := range availablePSPs.Items {
260 availablePSPMap[psp.Name] = struct{}{}
261 }
262 expectedPSPMap := make(map[string]*v1beta1.PodSecurityPolicy)
263 for _, psp := range builtinPSPs {
264 expectedPSPMap[psp.Name] = psp
265 }
266 for pspName, psp := range expectedPSPMap {
267 if _, ok := availablePSPMap[pspName]; !ok {
268 if _, err := pspClient.Create(ctx, psp, metav1.CreateOptions{}); err != nil {
269 return err
270 }
271 }
272 }
273 for pspName, _ := range availablePSPMap {
274 if _, ok := expectedPSPMap[pspName]; !ok {
275 if err := pspClient.Delete(ctx, pspName, metav1.DeleteOptions{}); err != nil {
276 return err
277 }
278 }
279 }
280 return nil
281}
282
Lorenz Brunb15abad2020-04-16 11:17:12 +0200283func reconcileClusterRoles(ctx context.Context, clientSet kubernetes.Interface) error {
Lorenz Brun8e3b8fc2020-05-19 14:29:40 +0200284 crClient := clientSet.RbacV1().ClusterRoles()
Lorenz Brun878f5f92020-05-12 16:15:39 +0200285 availableCRs, err := crClient.List(ctx, metav1.ListOptions{
286 LabelSelector: "smalltown.com/builtin=true",
287 })
288 if err != nil {
289 return err
290 }
291 availableCRMap := make(map[string]struct{})
292 for _, cr := range availableCRs.Items {
293 availableCRMap[cr.Name] = struct{}{}
294 }
295 expectedCRMap := make(map[string]*rbacv1.ClusterRole)
296 for _, cr := range builtinClusterRoles {
297 expectedCRMap[cr.Name] = cr
298 }
299 for crName, psp := range expectedCRMap {
300 if _, ok := availableCRMap[crName]; !ok {
301 if _, err := crClient.Create(ctx, psp, metav1.CreateOptions{}); err != nil {
302 return err
303 }
304 }
305 }
306 for crName, _ := range availableCRMap {
307 if _, ok := expectedCRMap[crName]; !ok {
308 if err := crClient.Delete(ctx, crName, metav1.DeleteOptions{}); err != nil {
309 return err
310 }
311 }
312 }
313 return nil
314}
315
Lorenz Brunb15abad2020-04-16 11:17:12 +0200316func reconcileClusterRoleBindings(ctx context.Context, clientset kubernetes.Interface) error {
Lorenz Brun878f5f92020-05-12 16:15:39 +0200317 crbClient := clientset.RbacV1().ClusterRoleBindings()
318 availableCRBs, err := crbClient.List(ctx, metav1.ListOptions{
319 LabelSelector: "smalltown.com/builtin=true",
320 })
321 if err != nil {
322 return err
323 }
324 availableCRBMap := make(map[string]struct{})
325 for _, crb := range availableCRBs.Items {
326 availableCRBMap[crb.Name] = struct{}{}
327 }
328 expectedCRBMap := make(map[string]*rbacv1.ClusterRoleBinding)
329 for _, crb := range builtinClusterRoleBindings {
330 expectedCRBMap[crb.Name] = crb
331 }
332 for crbName, psp := range expectedCRBMap {
333 if _, ok := availableCRBMap[crbName]; !ok {
334 if _, err := crbClient.Create(ctx, psp, metav1.CreateOptions{}); err != nil {
335 return err
336 }
337 }
338 }
339 for crbName, _ := range availableCRBMap {
340 if _, ok := expectedCRBMap[crbName]; !ok {
341 if err := crbClient.Delete(ctx, crbName, metav1.DeleteOptions{}); err != nil {
342 return err
343 }
344 }
345 }
346 return nil
347}
Lorenz Brunb15abad2020-04-16 11:17:12 +0200348
349func reconcileSCs(ctx context.Context, clientSet kubernetes.Interface) error {
350 scsClient := clientSet.StorageV1().StorageClasses()
351 availableSCs, err := scsClient.List(ctx, metav1.ListOptions{
352 LabelSelector: "smalltown.com/builtin=true",
353 })
354 if err != nil {
355 return err
356 }
357 availableSCMap := make(map[string]struct{})
358 for _, sc := range availableSCs.Items {
359 availableSCMap[sc.Name] = struct{}{}
360 }
361 expectedSCMap := make(map[string]*storagev1.StorageClass)
362 for _, sc := range builtinStorageClasses {
363 expectedSCMap[sc.Name] = sc
364 }
365 for scName, sc := range expectedSCMap {
366 if _, ok := availableSCMap[scName]; !ok {
367 if _, err := scsClient.Create(ctx, sc, metav1.CreateOptions{}); err != nil {
368 return err
369 }
370 }
371 }
372 for pspName, _ := range availableSCMap {
373 if _, ok := expectedSCMap[pspName]; !ok {
374 if err := scsClient.Delete(ctx, pspName, metav1.DeleteOptions{}); err != nil {
375 return err
376 }
377 }
378 }
379 return nil
380}
381
382func reconcileCSIDrivers(ctx context.Context, clientSet kubernetes.Interface) error {
383 drvClient := clientSet.StorageV1().CSIDrivers()
384 availableDrvs, err := drvClient.List(ctx, metav1.ListOptions{
385 LabelSelector: "smalltown.com/builtin=true",
386 })
387 if err != nil {
388 return err
389 }
390 availableDrvMap := make(map[string]struct{})
391 for _, drv := range availableDrvs.Items {
392 availableDrvMap[drv.Name] = struct{}{}
393 }
394 expectedDrvMap := make(map[string]*storagev1.CSIDriver)
395 for _, drv := range builtinCSIDrivers {
396 expectedDrvMap[drv.Name] = drv
397 }
398 for drvName, drv := range expectedDrvMap {
399 if _, ok := availableDrvMap[drvName]; !ok {
400 if _, err := drvClient.Create(ctx, drv, metav1.CreateOptions{}); err != nil {
401 return err
402 }
403 }
404 }
405 for pspName, _ := range availableDrvMap {
406 if _, ok := expectedDrvMap[pspName]; !ok {
407 if err := drvClient.Delete(ctx, pspName, metav1.DeleteOptions{}); err != nil {
408 return err
409 }
410 }
411 }
412 return nil
413}