metropolis/node/kubernetes/reconciler/reconciler.go

// Copyright The Monogon Project Authors.
// SPDX-License-Identifier: Apache-2.0

// The reconciler ensures that a base set of K8s resources is always available
// in the cluster. These are necessary to ensure correct out-of-the-box
// functionality. All resources containing the
// metropolis.monogon.dev/builtin=true label are assumed to be managed by the
// reconciler.
// It currently does not revert modifications made by admins, it is  planned to
// create an admission plugin prohibiting such modifications to resources with
// the metropolis.monogon.dev/builtin label to deal with that problem. This
// would also solve a potential issue where you could delete resources just by
// adding the metropolis.monogon.dev/builtin=true label.
package reconciler

import (
	"context"
	"errors"
	"fmt"
	"strings"

	apiequality "k8s.io/apimachinery/pkg/api/equality"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	apivalidation "k8s.io/apimachinery/pkg/api/validation"
	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"

	"source.monogon.dev/osbase/supervisor"
)

const (
	// BuiltinLabelKey is used as a k8s label to mark built-in objects (ie.,
	// managed by the reconciler)
	BuiltinLabelKey = "metropolis.monogon.dev/builtin"
	// BuiltinLabelValue is used as a k8s label value, under the
	// BuiltinLabelKey key.
	BuiltinLabelValue = "true"
	// BuiltinRBACPrefix is used to prefix all built-in objects that are part
	// of the rbac/v1 API (eg.  {Cluster,}Role{Binding,} objects). This
	// corresponds to the colon-separated 'namespaces' notation used by
	// Kubernetes system (system:) objects.
	BuiltinRBACPrefix = "metropolis:"
)

// builtinLabels makes a kubernetes-compatible label dictionary (key->value)
// that is used to mark objects that are built-in into Metropolis (ie., managed
// by the reconciler). These are then subsequently retrieved by listBuiltins.
// The extra argument specifies what other labels are to be merged into the the
// labels dictionary, for convenience. If nil or empty, no extra labels will be
// applied.
func builtinLabels(extra map[string]string) map[string]string {
	l := map[string]string{
		BuiltinLabelKey: BuiltinLabelValue,
	}
	for k, v := range extra {
		l[k] = v
	}
	return l
}

// listBuiltins returns a k8s client ListOptions structure that allows to
// retrieve all objects that are built-in into Metropolis currently present in
// the API server (ie., ones that are to be managed by the reconciler). These
// are created by applying builtinLabels to their metadata labels.
var listBuiltins = meta.ListOptions{
	LabelSelector: fmt.Sprintf("%s=%s", BuiltinLabelKey, BuiltinLabelValue),
}

// builtinRBACName returns a name that is compatible with colon-delimited
// 'namespaced' objects, a la system:*.
// These names are to be used by all builtins created as part of the rbac/v1
// Kubernetes API.
func builtinRBACName(name string) string {
	return BuiltinRBACPrefix + name
}

// resource is a type of resource to be managed by the reconciler. All
// built-ins/reconciled objects must implement this interface to be managed
// correctly by the reconciler.
type resource interface {
	// List returns a list of objects currently present on the target
	// (ie. k8s API server).
	List(ctx context.Context) ([]meta.Object, error)
	// Create creates an object on the target. The el argument is
	// an object returned by the Expected() call.
	Create(ctx context.Context, el meta.Object) error
	// Update updates an existing object, by name, on the target.
	// The el argument is an object returned by the Expected() call.
	Update(ctx context.Context, el meta.Object) error
	// Delete deletes an object, by name, from the target.
	Delete(ctx context.Context, name string, opts meta.DeleteOptions) error
	// Expected returns a list of all objects expected to be present on the
	// target. Objects are identified by their name, as returned by GetName.
	Expected() []meta.Object
}

func allResources(clientSet kubernetes.Interface) map[string]resource {
	return map[string]resource{
		"clusterroles":        resourceClusterRoles{clientSet},
		"clusterrolebindings": resourceClusterRoleBindings{clientSet},
		"storageclasses":      resourceStorageClasses{clientSet},
		"csidrivers":          resourceCSIDrivers{clientSet},
		"runtimeclasses":      resourceRuntimeClasses{clientSet},
	}
}

func reconcileAll(ctx context.Context, clientSet kubernetes.Interface) error {
	resources := allResources(clientSet)
	for name, resource := range resources {
		err := reconcile(ctx, resource, name)
		if err != nil {
			return fmt.Errorf("resource %s: %w", name, err)
		}
	}
	return nil
}

func reconcile(ctx context.Context, r resource, rname string) error {
	log := supervisor.Logger(ctx)
	present, err := r.List(ctx)
	if err != nil {
		return err
	}
	presentMap := make(map[string]meta.Object)
	for _, el := range present {
		presentMap[el.GetName()] = el
	}
	expected := r.Expected()
	expectedMap := make(map[string]meta.Object)
	for _, el := range expected {
		expectedMap[el.GetName()] = el
	}
	for name, expectedEl := range expectedMap {
		if presentEl, ok := presentMap[name]; ok {
			// The object already exists. Update it if it is different than expected.

			// The server rejects updates which don't have an up to date ResourceVersion.
			expectedEl.SetResourceVersion(presentEl.GetResourceVersion())

			// Clear out fields set by the server, such that comparison succeeds if
			// there are no other changes.
			presentEl.SetUID("")
			presentEl.SetGeneration(0)
			presentEl.SetCreationTimestamp(meta.Time{})
			presentEl.SetManagedFields(nil)

			if !apiequality.Semantic.DeepEqual(presentEl, expectedEl) {
				log.Infof("Updating %s object %q", rname, name)
				if err := r.Update(ctx, expectedEl); err != nil {
					if !isImmutableError(err) {
						return err
					}
					log.Infof("Failed to update object due to immutable fields; deleting and recreating: %v", err)

					// Only delete if the ResourceVersion has not changed. If it has
					// changed, that means another reconciler was faster than us and
					// has already recreated the object.
					resourceVersion := presentEl.GetResourceVersion()
					deleteOpts := meta.DeleteOptions{
						Preconditions: &meta.Preconditions{
							ResourceVersion: &resourceVersion,
						},
					}
					// ResourceVersion must be cleared when creating.
					expectedEl.SetResourceVersion("")

					if err := r.Delete(ctx, name, deleteOpts); err != nil {
						return err
					}
					if err := r.Create(ctx, expectedEl); err != nil {
						return err
					}
				}
			}
		} else {
			log.Infof("Creating %s object %q", rname, name)
			if err := r.Create(ctx, expectedEl); err != nil {
				return err
			}
		}
	}
	for name := range presentMap {
		if _, ok := expectedMap[name]; !ok {
			log.Infof("Deleting %s object %q", rname, name)
			if err := r.Delete(ctx, name, meta.DeleteOptions{}); err != nil {
				return err
			}
		}
	}
	return nil
}

// isImmutableError returns true if err indicates that an update failed because
// of an attempt to update one or more immutable fields.
func isImmutableError(err error) bool {
	if !apierrors.IsInvalid(err) {
		return false
	}
	var status apierrors.APIStatus
	if !errors.As(err, &status) {
		return false
	}
	details := status.Status().Details
	if details == nil || len(details.Causes) == 0 {
		return false
	}
	for _, cause := range details.Causes {
		if !strings.Contains(cause.Message, apivalidation.FieldImmutableErrorMsg) {
			return false
		}
	}
	return true
}