From 9b56712203cfaf871c2887c295dde7c095253aed Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Mon, 13 Jul 2026 14:46:32 +0200 Subject: [PATCH] Migrate EvictionReconciler to SSA status updates Replace MergeFromWithOptimisticLock + retry with Status().Apply() for all eviction status updates. The evictionStatusCfg helper seeds every apply with the full set of owned fields so SSA never prunes values between reconcile cycles. As the sole owner of every condition on the Eviction CR, the reconciler seeds all three condition types on each apply. The apply configuration constructor for the cluster-scoped Eviction resource takes only a name (no namespace). Signed-off-by: Fabian Wiesel --- internal/controller/eviction/controller.go | 285 +++++++++++---------- 1 file changed, 150 insertions(+), 135 deletions(-) diff --git a/internal/controller/eviction/controller.go b/internal/controller/eviction/controller.go index 8e80c15f..133f2509 100644 --- a/internal/controller/eviction/controller.go +++ b/internal/controller/eviction/controller.go @@ -31,12 +31,13 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/util/retry" + k8sacmetav1 "k8s.io/client-go/applyconfigurations/meta/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" logger "sigs.k8s.io/controller-runtime/pkg/log" kvmv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + apiv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/applyconfigurations/api/v1" "github.com/cobaltcore-dev/openstack-hypervisor-operator/internal/global" "github.com/cobaltcore-dev/openstack-hypervisor-operator/internal/openstack" "github.com/cobaltcore-dev/openstack-hypervisor-operator/internal/utils" @@ -104,6 +105,32 @@ func isMigrationTaskState(taskState string) bool { return strings.Contains(taskState, "migrat") || strings.Contains(taskState, "resize") } +// evictionStatusCfg builds an EvictionStatusApplyConfiguration pre-populated +// with the current scalar fields and the conditions this controller owns. +// Call sites upsert the single condition they are setting via +// utils.SetApplyConfigurationStatusCondition, then apply. +func evictionStatusCfg(eviction *kvmv1.Eviction) *apiv1.EvictionStatusApplyConfiguration { + cfg := apiv1.EvictionStatus(). + WithHypervisorServiceId(eviction.Status.HypervisorServiceId). + WithOutstandingRamMb(eviction.Status.OutstandingRamMb) + cfg.OutstandingInstances = eviction.Status.OutstandingInstances + // Seed all conditions this controller owns so they are not pruned on Apply. + for _, t := range []string{kvmv1.ConditionTypeEvicting, kvmv1.ConditionTypePreflight, kvmv1.ConditionTypeMigration} { + if c := meta.FindStatusCondition(eviction.Status.Conditions, t); c != nil { + cfg.WithConditions(utils.ConditionFromStatus(*c)) + } + } + return cfg +} + +// applyStatus applies the eviction status via SSA with this controller as the +// field owner. +func (r *EvictionReconciler) applyStatus(ctx context.Context, name string, cfg *apiv1.EvictionStatusApplyConfiguration) error { + return r.Status().Apply(ctx, + apiv1.Eviction(name).WithStatus(cfg), + client.ForceOwnership, client.FieldOwner(EvictionControllerName)) +} + // +kubebuilder:rbac:groups=kvm.cloud.sap,resources=evictions,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=kvm.cloud.sap,resources=evictions/status,verbs=get;update;patch // +kubebuilder:rbac:groups=kvm.cloud.sap,resources=evictions/finalizers,verbs=update @@ -137,14 +164,14 @@ func (r *EvictionReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c statusCondition := meta.FindStatusCondition(eviction.Status.Conditions, kvmv1.ConditionTypeEvicting) if statusCondition == nil { - meta.SetStatusCondition(&eviction.Status.Conditions, metav1.Condition{ - Type: kvmv1.ConditionTypeEvicting, - Status: metav1.ConditionTrue, - Message: "Running", - Reason: kvmv1.ConditionReasonRunning, - }) - - return ctrl.Result{}, r.updateStatus(ctx, eviction) + statusCfg := evictionStatusCfg(eviction) + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, + *k8sacmetav1.Condition(). + WithType(kvmv1.ConditionTypeEvicting). + WithStatus(metav1.ConditionTrue). + WithReason(kvmv1.ConditionReasonRunning). + WithMessage("Running")) + return ctrl.Result{}, r.applyStatus(ctx, eviction.Name, statusCfg) } switch statusCondition.Status { @@ -177,36 +204,16 @@ func (r *EvictionReconciler) handleRunning(ctx context.Context, eviction *kvmv1. return r.evictNext(ctx, eviction, limit) } - meta.SetStatusCondition(&eviction.Status.Conditions, metav1.Condition{ - Type: kvmv1.ConditionTypeEvicting, - Status: metav1.ConditionFalse, - Message: "eviction completed successfully", - Reason: kvmv1.ConditionReasonSucceeded, - }) - - eviction.Status.OutstandingRamMb = 0 + statusCfg := evictionStatusCfg(eviction) + statusCfg.WithOutstandingRamMb(0) + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, + *k8sacmetav1.Condition(). + WithType(kvmv1.ConditionTypeEvicting). + WithStatus(metav1.ConditionFalse). + WithReason(kvmv1.ConditionReasonSucceeded). + WithMessage("eviction completed successfully")) logger.FromContext(ctx).Info("succeeded") - return ctrl.Result{}, r.updateStatus(ctx, eviction) -} - -func (r *EvictionReconciler) updateStatus(ctx context.Context, eviction *kvmv1.Eviction) error { - desiredStatus := eviction.Status.DeepCopy() - return retry.RetryOnConflict(utils.StatusPatchBackoff, func() error { - freshEviction := &kvmv1.Eviction{} - if err := r.Get(ctx, client.ObjectKeyFromObject(eviction), freshEviction); err != nil { - return err - } - freshBase := freshEviction.DeepCopy() - // Apply desired conditions and scalar fields onto the fresh status - for _, c := range desiredStatus.Conditions { - meta.SetStatusCondition(&freshEviction.Status.Conditions, c) - } - freshEviction.Status.OutstandingInstances = desiredStatus.OutstandingInstances - freshEviction.Status.OutstandingRamMb = desiredStatus.OutstandingRamMb - freshEviction.Status.HypervisorServiceId = desiredStatus.HypervisorServiceId - return r.Status().Patch(ctx, freshEviction, client.MergeFromWithOptions(freshBase, - client.MergeFromWithOptimisticLock{}), client.FieldOwner(EvictionControllerName)) - }) + return ctrl.Result{}, r.applyStatus(ctx, eviction.Name, statusCfg) } func (r *EvictionReconciler) handlePreflight(ctx context.Context, eviction *kvmv1.Eviction, hv *kvmv1.Hypervisor) (ctrl.Result, error) { @@ -214,14 +221,20 @@ func (r *EvictionReconciler) handlePreflight(ctx context.Context, eviction *kvmv // If the hypervisor should exist, then we need to ensure it is disabled before we start evicting if expectHypervisor && !meta.IsStatusConditionTrue(hv.Status.Conditions, kvmv1.ConditionTypeHypervisorDisabled) { - // Hypervisor is not disabled (yet?), reflect that as a failing preflight check - if meta.SetStatusCondition(&eviction.Status.Conditions, metav1.Condition{ - Type: kvmv1.ConditionTypePreflight, - Status: metav1.ConditionFalse, - Message: "hypervisor not disabled", - Reason: kvmv1.ConditionReasonFailed, - }) { - return ctrl.Result{}, r.updateStatus(ctx, eviction) + // Hypervisor is not disabled (yet?), reflect that as a failing preflight check. + // Only apply if the condition is new or changed, to avoid unnecessary API calls on repeated requeues. + existing := meta.FindStatusCondition(eviction.Status.Conditions, kvmv1.ConditionTypePreflight) + if existing == nil || existing.Status != metav1.ConditionFalse { + statusCfg := evictionStatusCfg(eviction) + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, + *k8sacmetav1.Condition(). + WithType(kvmv1.ConditionTypePreflight). + WithStatus(metav1.ConditionFalse). + WithReason(kvmv1.ConditionReasonFailed). + WithMessage("hypervisor not disabled")) + if err := r.applyStatus(ctx, eviction.Name, statusCfg); err != nil { + return ctrl.Result{}, err + } } return ctrl.Result{RequeueAfter: global.DefaultPollTime}, nil // Wait for hypervisor to be disabled } @@ -236,59 +249,62 @@ func (r *EvictionReconciler) handlePreflight(ctx context.Context, eviction *kvmv if expectHypervisor { // Abort eviction - meta.SetStatusCondition(&eviction.Status.Conditions, metav1.Condition{ - Type: kvmv1.ConditionTypeEvicting, - Status: metav1.ConditionFalse, - Message: fmt.Sprintf("failed to get hypervisor %v", err), - Reason: kvmv1.ConditionReasonFailed, - }) - return ctrl.Result{}, r.updateStatus(ctx, eviction) - } else { - // That is (likely) an eviction for a node that never registered - // so we are good to go - msg := "eviction completed successfully due to expected case of no hypervisor" - meta.SetStatusCondition(&eviction.Status.Conditions, metav1.Condition{ - Type: kvmv1.ConditionTypeEvicting, - Status: metav1.ConditionFalse, - Message: msg, - Reason: kvmv1.ConditionReasonSucceeded, - }) - eviction.Status.OutstandingRamMb = 0 - logger.FromContext(ctx).Info(msg) - return ctrl.Result{}, r.updateStatus(ctx, eviction) + statusCfg := evictionStatusCfg(eviction) + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, + *k8sacmetav1.Condition(). + WithType(kvmv1.ConditionTypeEvicting). + WithStatus(metav1.ConditionFalse). + WithReason(kvmv1.ConditionReasonFailed). + WithMessage(fmt.Sprintf("failed to get hypervisor %v", err))) + return ctrl.Result{}, r.applyStatus(ctx, eviction.Name, statusCfg) } + + // That is (likely) an eviction for a node that never registered + // so we are good to go + msg := "eviction completed successfully due to expected case of no hypervisor" + statusCfg := evictionStatusCfg(eviction) + statusCfg.WithOutstandingRamMb(0) + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, + *k8sacmetav1.Condition(). + WithType(kvmv1.ConditionTypeEvicting). + WithStatus(metav1.ConditionFalse). + WithReason(kvmv1.ConditionReasonSucceeded). + WithMessage(msg)) + logger.FromContext(ctx).Info(msg) + return ctrl.Result{}, r.applyStatus(ctx, eviction.Name, statusCfg) } + var instances []string if hypervisor.Servers != nil { - uuids := make([]string, len(*hypervisor.Servers)) + instances = make([]string, len(*hypervisor.Servers)) for i, server := range *hypervisor.Servers { - uuids[i] = server.UUID + instances[i] = server.UUID } - eviction.Status.OutstandingInstances = uuids } - // Update status - eviction.Status.HypervisorServiceId = hypervisor.ID - eviction.Status.OutstandingRamMb = int64(hypervisor.MemoryMBUsed) - meta.SetStatusCondition(&eviction.Status.Conditions, metav1.Condition{ - Type: kvmv1.ConditionTypePreflight, - Status: metav1.ConditionTrue, - Message: "Preflight checks passed, hypervisor is disabled and ready for eviction", - Reason: kvmv1.ConditionReasonSucceeded, - }) - return ctrl.Result{}, r.updateStatus(ctx, eviction) + statusCfg := evictionStatusCfg(eviction) + statusCfg.WithHypervisorServiceId(hypervisor.ID) + statusCfg.WithOutstandingRamMb(int64(hypervisor.MemoryMBUsed)) + statusCfg.OutstandingInstances = instances + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, + *k8sacmetav1.Condition(). + WithType(kvmv1.ConditionTypePreflight). + WithStatus(metav1.ConditionTrue). + WithReason(kvmv1.ConditionReasonSucceeded). + WithMessage("Preflight checks passed, hypervisor is disabled and ready for eviction")) + return ctrl.Result{}, r.applyStatus(ctx, eviction.Name, statusCfg) } // removeGone drops a specific instance from the outstanding set when it is // gone (NotFound), recording a success condition. Returns true if the error was // a NotFound and was handled, false otherwise (caller should treat the original // error as a real failure). -func (r *EvictionReconciler) removeGone(eviction *kvmv1.Eviction, uuid string, err error) bool { +func (r *EvictionReconciler) removeGone(statusCfg *apiv1.EvictionStatusApplyConfiguration, uuid string, err error) bool { if !gophercloud.ResponseCodeIs(err, http.StatusNotFound) { return false } - eviction.Status.OutstandingInstances = removeInstance(eviction.Status.OutstandingInstances, uuid) - r.setMigrationSucceeded(eviction, fmt.Sprintf("Instance %s is gone", uuid)) + statusCfg.OutstandingInstances = removeInstance(statusCfg.OutstandingInstances, uuid) + r.setMigrationSucceeded(statusCfg, fmt.Sprintf("Instance %s is gone", uuid)) return true } @@ -303,10 +319,12 @@ func (r *EvictionReconciler) evictNext(ctx context.Context, eviction *kvmv1.Evic } baseLog := logger.FromContext(ctx).WithName("Evict") - // Snapshot the current queue; we mutate eviction.Status.OutstandingInstances - // as instances complete, so iterate over a copy. + // Snapshot the current queue; we mutate statusCfg.OutstandingInstances as + // instances complete, so iterate over a copy. outstanding := append([]string(nil), eviction.Status.OutstandingInstances...) + statusCfg := evictionStatusCfg(eviction) + inFlight := 0 // migrations currently running (or terminating) started := 0 // migrations we triggered this pass var candidates []candidate @@ -321,7 +339,7 @@ func (r *EvictionReconciler) evictNext(ctx context.Context, eviction *kvmv1.Evic vm, err := servers.Get(vmCtx, r.computeClient, uuid).Extract() if err != nil { - if r.removeGone(eviction, uuid, err) { + if r.removeGone(statusCfg, uuid, err) { continue } // Transient Get error - leave the instance queued and retry soon. @@ -339,14 +357,14 @@ func (r *EvictionReconciler) evictNext(ctx context.Context, eviction *kvmv1.Evic case "ERROR": // Needs manual intervention (or another operator fixes it); // deprioritize and record the failure, but don't hold a slot. - eviction.Status.OutstandingInstances = deprioritize(eviction.Status.OutstandingInstances, uuid) + statusCfg.OutstandingInstances = deprioritize(statusCfg.OutstandingInstances, uuid) log.Info("error", "faultMessage", vm.Fault.Message) - meta.SetStatusCondition(&eviction.Status.Conditions, metav1.Condition{ - Type: kvmv1.ConditionTypeMigration, - Status: metav1.ConditionFalse, - Message: fmt.Sprintf("Migration of instance %s failed: %s", vm.ID, vm.Fault.Message), - Reason: kvmv1.ConditionReasonFailed, - }) + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, + *k8sacmetav1.Condition(). + WithType(kvmv1.ConditionTypeMigration). + WithStatus(metav1.ConditionFalse). + WithReason(kvmv1.ConditionReasonFailed). + WithMessage(fmt.Sprintf("Migration of instance %s failed: %s", vm.ID, vm.Fault.Message))) errs = append(errs, fmt.Errorf("error migrating instance %v", uuid)) continue } @@ -369,7 +387,7 @@ func (r *EvictionReconciler) evictNext(ctx context.Context, eviction *kvmv1.Evic if vm.Status == "VERIFY_RESIZE" { log.Info("confirm-resize") err := servers.ConfirmResize(vmCtx, r.computeClient, vm.ID).ExtractErr() - if err != nil && !r.removeGone(eviction, uuid, err) { + if err != nil && !r.removeGone(statusCfg, uuid, err) { // Retry confirm on the next pass. errs = append(errs, err) } @@ -380,21 +398,21 @@ func (r *EvictionReconciler) evictNext(ctx context.Context, eviction *kvmv1.Evic } log.Info("migrated") - eviction.Status.OutstandingInstances = removeInstance(eviction.Status.OutstandingInstances, uuid) - r.setMigrationSucceeded(eviction, fmt.Sprintf("Migration of instance %s finished", vm.ID)) + statusCfg.OutstandingInstances = removeInstance(statusCfg.OutstandingInstances, uuid) + r.setMigrationSucceeded(statusCfg, fmt.Sprintf("Migration of instance %s finished", vm.ID)) continue } if vm.TaskState == "deleting" { // Just wait for it to disappear; deprioritize and count as busy so // we don't spend a fresh slot on it. - eviction.Status.OutstandingInstances = deprioritize(eviction.Status.OutstandingInstances, uuid) - meta.SetStatusCondition(&eviction.Status.Conditions, metav1.Condition{ - Type: kvmv1.ConditionTypeMigration, - Status: metav1.ConditionFalse, - Message: fmt.Sprintf("Live migration of terminating instance %s skipped", vm.ID), - Reason: kvmv1.ConditionReasonFailed, - }) + statusCfg.OutstandingInstances = deprioritize(statusCfg.OutstandingInstances, uuid) + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, + *k8sacmetav1.Condition(). + WithType(kvmv1.ConditionTypeMigration). + WithStatus(metav1.ConditionFalse). + WithReason(kvmv1.ConditionReasonFailed). + WithMessage(fmt.Sprintf("Live migration of terminating instance %s skipped", vm.ID))) inFlight++ continue } @@ -424,9 +442,18 @@ func (r *EvictionReconciler) evictNext(ctx context.Context, eviction *kvmv1.Evic migErr = r.coldMigrate(migrateCtx, c.id, eviction) } if migErr != nil { - if r.removeGone(eviction, c.id, migErr) { + if r.removeGone(statusCfg, c.id, migErr) { continue } + // A migration trigger failed for a reason other than the instance + // being gone; record the failure on the MigratingInstance condition + // so it is visible even though we keep retrying. + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, + *k8sacmetav1.Condition(). + WithType(kvmv1.ConditionTypeMigration). + WithStatus(metav1.ConditionFalse). + WithReason(kvmv1.ConditionReasonFailed). + WithMessage(migErr.Error())) errs = append(errs, migErr) continue } @@ -440,7 +467,7 @@ func (r *EvictionReconciler) evictNext(ctx context.Context, eviction *kvmv1.Evic // condition and retried via RequeueAfter, so returning them as a reconcile // error would only discard our RequeueAfter (controller-runtime ignores the // result when the error is non-nil) and emit a warning. - if err := r.updateStatus(ctx, eviction); err != nil { + if err := r.applyStatus(ctx, eviction.Name, statusCfg); err != nil { return ctrl.Result{}, err } @@ -449,12 +476,12 @@ func (r *EvictionReconciler) evictNext(ctx context.Context, eviction *kvmv1.Evic } baseLog.Info("poll", "inFlight", inFlight, "started", started, - "outstanding", len(eviction.Status.OutstandingInstances), "limit", limit) + "outstanding", len(statusCfg.OutstandingInstances), "limit", limit) // Requeue while there is still work; use a short retry when nothing is // actually migrating yet (e.g. all instances errored) so we don't idle. requeue := global.DefaultPollTime - if inFlight+started == 0 && len(eviction.Status.OutstandingInstances) > 0 { + if inFlight+started == 0 && len(statusCfg.OutstandingInstances) > 0 { requeue = global.ShortRetryTime } return ctrl.Result{RequeueAfter: requeue}, nil @@ -465,20 +492,22 @@ func (r *EvictionReconciler) evictNext(ctx context.Context, eviction *kvmv1.Evic // outstanding - an earlier ERROR instance moved to the back of the queue means // the eviction is not actually clean yet. The condition is only allowed to flip // to Succeeded once the whole eviction completes (OutstandingInstances empty). -func (r *EvictionReconciler) setMigrationSucceeded(eviction *kvmv1.Eviction, msg string) { - prior := meta.FindStatusCondition(eviction.Status.Conditions, kvmv1.ConditionTypeMigration) - stickyFailure := len(eviction.Status.OutstandingInstances) > 0 && prior != nil && - prior.Status == metav1.ConditionFalse && - prior.Reason == kvmv1.ConditionReasonFailed - if stickyFailure { - return +func (r *EvictionReconciler) setMigrationSucceeded(statusCfg *apiv1.EvictionStatusApplyConfiguration, msg string) { + if len(statusCfg.OutstandingInstances) > 0 { + for _, c := range statusCfg.Conditions { + if c.Type != nil && *c.Type == kvmv1.ConditionTypeMigration && + c.Status != nil && *c.Status == metav1.ConditionFalse && + c.Reason != nil && *c.Reason == kvmv1.ConditionReasonFailed { + return + } + } } - meta.SetStatusCondition(&eviction.Status.Conditions, metav1.Condition{ - Type: kvmv1.ConditionTypeMigration, - Status: metav1.ConditionFalse, - Message: msg, - Reason: kvmv1.ConditionReasonSucceeded, - }) + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, + *k8sacmetav1.Condition(). + WithType(kvmv1.ConditionTypeMigration). + WithStatus(metav1.ConditionFalse). + WithReason(kvmv1.ConditionReasonSucceeded). + WithMessage(msg)) } func (r *EvictionReconciler) liveMigrate(ctx context.Context, uuid string, eviction *kvmv1.Eviction) error { @@ -490,14 +519,7 @@ func (r *EvictionReconciler) liveMigrate(ctx context.Context, uuid string, evict res := servers.LiveMigrate(ctx, r.computeClient, uuid, liveMigrateOpts) if res.Err != nil { - err := fmt.Errorf("failed to evict VM %s due to %w", uuid, res.Err) - meta.SetStatusCondition(&eviction.Status.Conditions, metav1.Condition{ - Type: kvmv1.ConditionTypeMigration, - Status: metav1.ConditionFalse, - Message: err.Error(), - Reason: kvmv1.ConditionReasonFailed, - }) - return err + return fmt.Errorf("failed to evict VM %s due to %w", uuid, res.Err) } log.Info("Live migrating server", "server", uuid, "source", eviction.Spec.Hypervisor, "X-Openstack-Request-Id", res.Header.Get("X-Openstack-Request-Id")) @@ -509,14 +531,7 @@ func (r *EvictionReconciler) coldMigrate(ctx context.Context, uuid string, evict res := servers.Migrate(ctx, r.computeClient, uuid) if res.Err != nil { - err := fmt.Errorf("failed to evict stopped server %s due to %w", uuid, res.Err) - meta.SetStatusCondition(&eviction.Status.Conditions, metav1.Condition{ - Type: kvmv1.ConditionTypeMigration, - Status: metav1.ConditionFalse, - Message: err.Error(), - Reason: kvmv1.ConditionReasonFailed, - }) - return err + return fmt.Errorf("failed to evict stopped server %s due to %w", uuid, res.Err) } log.Info("Cold-migrating server", "server", uuid, "source", eviction.Spec.Hypervisor, "X-Openstack-Request-Id", res.Header.Get("X-Openstack-Request-Id"))