From 2bcb77fb7a6a00396ab62e6292ff00381d642f34 Mon Sep 17 00:00:00 2001 From: juliusclausnitzer Date: Mon, 2 Nov 2026 13:16:31 +0100 Subject: [PATCH 01/13] CRD changes Signed-off-by: juliusclausnitzer --- api/v1alpha1/committed_resource_change_types.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/v1alpha1/committed_resource_change_types.go b/api/v1alpha1/committed_resource_change_types.go index 29fb90e13..754c4b868 100644 --- a/api/v1alpha1/committed_resource_change_types.go +++ b/api/v1alpha1/committed_resource_change_types.go @@ -58,6 +58,8 @@ const ( ChangeRequestReasonAccepted = "Accepted" ChangeRequestReasonRejected = "Rejected" ChangeRequestReasonSuperseded = "Superseded" + // ChangeReasonRollbackFailed is the Finished reason when rollback exhausted its timeout. + ChangeReasonRollbackFailed = "RollbackFailed" ) // CommittedResourceChangeSpec defines the desired change for a CommittedResourceChange. From 26a2f57207c3210aa3f798d7fa298aeebb8f9b6f Mon Sep 17 00:00:00 2001 From: juliusclausnitzer Date: Mon, 2 Nov 2026 13:16:35 +0100 Subject: [PATCH 02/13] reonciler Signed-off-by: juliusclausnitzer --- .../commitments/changes/reconciler.go | 118 +++++++++++++++++- 1 file changed, 112 insertions(+), 6 deletions(-) diff --git a/internal/scheduling/reservations/commitments/changes/reconciler.go b/internal/scheduling/reservations/commitments/changes/reconciler.go index 73e244a99..21ff1772a 100644 --- a/internal/scheduling/reservations/commitments/changes/reconciler.go +++ b/internal/scheduling/reservations/commitments/changes/reconciler.go @@ -34,6 +34,10 @@ type CommittedResourceChangeReconciler struct { // AwaitingSchedulingTimeout is how long to wait in AwaitingScheduling before rolling back. // Zero (default) means no timeout. AwaitingSchedulingTimeout time.Duration + // RollbackTimeout is how long to attempt snapshot restore in RollingBack/DryRunCleanup + // before giving up and terminating with RollbackFailed. + // Zero (default) means no timeout. + RollbackTimeout time.Duration } func (r *CommittedResourceChangeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -225,17 +229,119 @@ func (r *CommittedResourceChangeReconciler) reconcileAwaitingScheduling(ctx cont // On restore complete: advances to Rejected. // On permanent cleanup failure: advances to Rejected with reason RollbackFailed. func (r *CommittedResourceChangeReconciler) reconcileRollingBack(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange) (ctrl.Result, error) { - // TODO (#780): for each entry in SnapshotBefore, converge CR spec back to snapshot value - // using SSA with fieldManager=txFieldManager. - // nil snapshot entry → CR did not exist before → delete it. - // Gate terminal transition on each CR confirming its spec reverted (check observedGeneration). - return ctrl.Result{}, nil + allSettled, err := r.restoreFromSnapshot(ctx, logger, tx) + if err != nil { + return ctrl.Result{}, err + } + if !allSettled { + if r.RollbackTimeout > 0 { + progressingCond := meta.FindStatusCondition(tx.Status.Conditions, v1alpha1.ChangeConditionProgressing) + if progressingCond != nil && time.Since(progressingCond.LastTransitionTime.Time) > r.RollbackTimeout { + logger.Info("rollback timed out, terminalizing", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "timeout", r.RollbackTimeout) + return r.terminateWithRollbackFailed(ctx, tx, "timed out restoring pre-change state") + } + } + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseRejected, nil) } // reconcileDryRunCleanup restores pre-change state after a dry-run would-accept outcome. // Re-uses snapshot rollback logic; terminal phase is Accepted (not Rejected). func (r *CommittedResourceChangeReconciler) reconcileDryRunCleanup(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange) (ctrl.Result, error) { - // TODO (#780): same convergence as reconcileRollingBack; terminal → advanceTo(Accepted, nil). + allSettled, err := r.restoreFromSnapshot(ctx, logger, tx) + if err != nil { + return ctrl.Result{}, err + } + if !allSettled { + if r.RollbackTimeout > 0 { + progressingCond := meta.FindStatusCondition(tx.Status.Conditions, v1alpha1.ChangeConditionProgressing) + if progressingCond != nil && time.Since(progressingCond.LastTransitionTime.Time) > r.RollbackTimeout { + logger.Info("dry-run cleanup timed out, terminalizing", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "timeout", r.RollbackTimeout) + return r.terminateWithRollbackFailed(ctx, tx, "timed out restoring pre-change state after dry run") + } + } + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseAccepted, nil) +} + +// restoreFromSnapshot converges CommittedResources back to their pre-change state recorded in +// tx.Status.SnapshotBefore. It is idempotent and safe to call on every requeue. +// +// Pass 1 (write): for each snapshot entry, delete the CR (nil entry = CR was new) or restore +// its spec via CreateOrUpdate (non-nil entry = CR existed before). +// Pass 2 (settle): for each non-nil entry, verify the CR controller has processed the restored +// spec by checking cond.ObservedGeneration == cr.Generation on the Ready condition. +// +// Returns (true, nil) when all non-nil entries are settled. +// Returns (false, nil) when still waiting for settlement. +// Returns (false, err) on API errors that should cause a requeue. +func (r *CommittedResourceChangeReconciler) restoreFromSnapshot(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange) (bool, error) { + for uuid, snapSpec := range tx.Status.SnapshotBefore { + name := commitmentName(uuid) + if snapSpec == nil { + toDelete := &v1alpha1.CommittedResource{ObjectMeta: metav1.ObjectMeta{Name: name}} + if err := r.Delete(ctx, toDelete); client.IgnoreNotFound(err) != nil { + logger.Error(err, "rollback: failed to delete CR", "name", name, "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID) + r.markError(ctx, tx, fmt.Sprintf("rollback: failed to delete %s: %v", name, err)) + return false, err + } + continue + } + cr := &v1alpha1.CommittedResource{ObjectMeta: metav1.ObjectMeta{Name: name}} + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, cr, func() error { + cr.Spec = *snapSpec + return nil + }); err != nil { + logger.Error(err, "rollback: failed to restore CR spec", "name", name, "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID) + r.markError(ctx, tx, fmt.Sprintf("rollback: failed to restore %s: %v", name, err)) + return false, err + } + } + + for uuid, snapSpec := range tx.Status.SnapshotBefore { + if snapSpec == nil { + continue + } + name := commitmentName(uuid) + var cr v1alpha1.CommittedResource + if err := r.Get(ctx, client.ObjectKey{Name: name}, &cr); err != nil { + if client.IgnoreNotFound(err) == nil { + return false, nil + } + return false, err + } + cond := meta.FindStatusCondition(cr.Status.Conditions, v1alpha1.CommittedResourceConditionReady) + if cond == nil || cond.ObservedGeneration != cr.Generation { + return false, nil + } + } + return true, nil +} + +// terminateWithRollbackFailed stamps Finished=True/RollbackFailed and Progressing=False/RetryExhausted, +// then advances the phase to Rejected. Used when the rollback itself cannot complete within RollbackTimeout. +func (r *CommittedResourceChangeReconciler) terminateWithRollbackFailed(ctx context.Context, tx *v1alpha1.CommittedResourceChange, msg string) (ctrl.Result, error) { + tx.Status.Phase = v1alpha1.ChangePhaseRejected + tx.Status.Message = msg + meta.SetStatusCondition(&tx.Status.Conditions, metav1.Condition{ + Type: v1alpha1.ChangeConditionFinished, + Status: metav1.ConditionTrue, + Reason: v1alpha1.ChangeReasonRollbackFailed, + Message: msg, + ObservedGeneration: tx.Generation, + }) + meta.SetStatusCondition(&tx.Status.Conditions, metav1.Condition{ + Type: v1alpha1.ChangeConditionProgressing, + Status: metav1.ConditionFalse, + Reason: v1alpha1.ChangeReasonRetryExhausted, + Message: msg, + ObservedGeneration: tx.Generation, + }) + if err := r.Status().Update(ctx, tx); err != nil { + return ctrl.Result{}, err + } return ctrl.Result{}, nil } From 35d176a7f3308bda01c77b7f49703236877fd2f3 Mon Sep 17 00:00:00 2001 From: juliusclausnitzer Date: Mon, 2 Nov 2026 13:16:39 +0100 Subject: [PATCH 03/13] test Signed-off-by: juliusclausnitzer --- .../commitments/changes/reconciler_test.go | 432 ++++++++++++++++++ 1 file changed, 432 insertions(+) diff --git a/internal/scheduling/reservations/commitments/changes/reconciler_test.go b/internal/scheduling/reservations/commitments/changes/reconciler_test.go index cfbdcebc8..7da4e9257 100644 --- a/internal/scheduling/reservations/commitments/changes/reconciler_test.go +++ b/internal/scheduling/reservations/commitments/changes/reconciler_test.go @@ -1080,3 +1080,435 @@ func TestCRTReconciler_Terminal(t *testing.T) { }) } } + +// ============================================================================ +// TestCRTReconciler_RollingBack +// ============================================================================ + +// driveToRollingBack drives a CRC to RollingBack by having a CR reject placement. +func driveToRollingBack(t *testing.T, r *CommittedResourceChangeReconciler, c client.Client, txName, crName string) { + t.Helper() + driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseAwaitingScheduling) + setCRReady(t, c, crName, metav1.ConditionFalse, v1alpha1.CommittedResourceReasonRejected, "no capacity") + driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseRollingBack) +} + +func TestCRTReconciler_RollingBack(t *testing.T) { + tests := []struct { + name string + preObjects []client.Object + txSpec v1alpha1.CommittedResourceChangeSpec + reconcilerOpt func(r *CommittedResourceChangeReconciler) + afterRollingBack func(t *testing.T, c client.Client, txName string) + wantPhase v1alpha1.ChangePhase + wantFinishedReason string + wantProgressingStatus metav1.ConditionStatus + wantProgressingReason string + // crName is derived as commitment- from txSpec.NewState (single-entry tests). + // For multi-CR tests the check is done inline. + }{ + { + name: "nil snapshot: new CR is deleted and rollback completes", + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-rb1": minimalCRSpec("az-1")}, + }, + // SnapshotBefore[uuid-rb1] = nil (CR did not exist before). + // After rollback, the CR must be deleted; no settlement gate. + wantPhase: v1alpha1.ChangePhaseRejected, + wantFinishedReason: string(v1alpha1.ChangePhaseRejected), + wantProgressingStatus: metav1.ConditionFalse, + wantProgressingReason: string(v1alpha1.ChangePhaseRejected), + }, + { + name: "non-nil snapshot: existing CR spec is restored and rollback completes", + preObjects: []client.Object{ + &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-rb2"}, + Spec: *minimalCRSpec("az-old"), + }, + }, + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-rb2": minimalCRSpec("az-new")}, + }, + // SnapshotBefore[uuid-rb2] = minimalCRSpec("az-old"). + // After rollback, commitment-uuid-rb2 must have spec az-old. + // Simulate CR controller settling the restored spec. + afterRollingBack: func(t *testing.T, c client.Client, _ string) { + setCRReady(t, c, "commitment-uuid-rb2", metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted, "") + }, + wantPhase: v1alpha1.ChangePhaseRejected, + wantFinishedReason: string(v1alpha1.ChangePhaseRejected), + wantProgressingStatus: metav1.ConditionFalse, + wantProgressingReason: string(v1alpha1.ChangePhaseRejected), + }, + { + name: "CRs not yet settled: stays in RollingBack", + preObjects: []client.Object{ + &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-rb3"}, + Spec: *minimalCRSpec("az-old"), + }, + }, + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-rb3": minimalCRSpec("az-new")}, + }, + // The fake client does not increment Generation on spec updates, so we must explicitly + // stale the condition to simulate "CR controller has not yet processed the restored spec". + afterRollingBack: func(t *testing.T, c client.Client, _ string) { + setCRReadyStaleGen(t, c, "commitment-uuid-rb3") + }, + wantPhase: v1alpha1.ChangePhaseRollingBack, + wantProgressingStatus: metav1.ConditionTrue, + wantProgressingReason: "Advancing", + }, + { + name: "rollback timeout: terminates with RollbackFailed", + preObjects: []client.Object{ + &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-rb4"}, + Spec: *minimalCRSpec("az-old"), + }, + }, + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-rb4": minimalCRSpec("az-new")}, + }, + reconcilerOpt: func(r *CommittedResourceChangeReconciler) { + r.RollbackTimeout = 50 * time.Millisecond + }, + // Stale the CR condition AND backdate the Progressing timestamp to trigger the timeout. + afterRollingBack: func(t *testing.T, c client.Client, txName string) { + setCRReadyStaleGen(t, c, "commitment-uuid-rb4") + backdateProgressingCondition(t, c, txName, 1*time.Second) + }, + wantPhase: v1alpha1.ChangePhaseRejected, + wantFinishedReason: v1alpha1.ChangeReasonRollbackFailed, + wantProgressingStatus: metav1.ConditionFalse, + wantProgressingReason: v1alpha1.ChangeReasonRetryExhausted, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := testScheme(t) + txName := "tx-rb-" + tt.name[:5] + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: txName}, + Spec: tt.txSpec, + } + objs := append([]client.Object{tx}, tt.preObjects...) + c := testClient(t, scheme, objs...) + r := testReconciler(c, scheme) + if tt.reconcilerOpt != nil { + tt.reconcilerOpt(r) + } + + // Pick the first UUID for the rejection trigger (single-CR tests). + var crName string + for uuid := range tt.txSpec.NewState { + crName = commitmentName(uuid) + break + } + + driveToRollingBack(t, r, c, txName, crName) + + if tt.afterRollingBack != nil { + tt.afterRollingBack(t, c, txName) + } + + doReconcile(t, r, txName) + + tx = getTx(t, c, txName) + if tx.Status.Phase != tt.wantPhase { + t.Errorf("phase: want %q, got %q", tt.wantPhase, tx.Status.Phase) + } + if tt.wantFinishedReason != "" { + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionFinished, metav1.ConditionTrue, tt.wantFinishedReason) + } + if tt.wantProgressingStatus != "" { + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionProgressing, tt.wantProgressingStatus, tt.wantProgressingReason) + } + }) + } +} + +// TestCRTReconciler_RollingBack_MultiCR verifies that all non-nil SnapshotBefore entries must +// have settled (observedGeneration == generation) before the CRC advances to Rejected. +func TestCRTReconciler_RollingBack_MultiCR(t *testing.T) { + const txName = "tx-rb-multi" + scheme := testScheme(t) + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: txName}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{ + "uuid-m1": minimalCRSpec("az-new"), + "uuid-m2": minimalCRSpec("az-new"), + }, + }, + } + // Both CRs pre-exist so their snapshots are non-nil; rollback must restore both. + preObjects := []client.Object{ + &v1alpha1.CommittedResource{ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-m1"}, Spec: *minimalCRSpec("az-old")}, + &v1alpha1.CommittedResource{ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-m2"}, Spec: *minimalCRSpec("az-old")}, + } + objs := append([]client.Object{tx}, preObjects...) + c := testClient(t, scheme, objs...) + r := testReconciler(c, scheme) + + driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseAwaitingScheduling) + setCRReady(t, c, "commitment-uuid-m1", metav1.ConditionFalse, v1alpha1.CommittedResourceReasonRejected, "no capacity") + driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseRollingBack) + + // Stale both CRs to simulate the CR controller not having processed the restored spec yet. + // (The fake client does not increment Generation on spec writes.) + setCRReadyStaleGen(t, c, "commitment-uuid-m1") + setCRReadyStaleGen(t, c, "commitment-uuid-m2") + + // Neither CR settled — must stay in RollingBack. + doReconcile(t, r, txName) + if got := getTx(t, c, txName).Status.Phase; got != v1alpha1.ChangePhaseRollingBack { + t.Fatalf("after first rollback reconcile: want RollingBack, got %q", got) + } + + // Settle only the first CR — still waiting on the second. + setCRReady(t, c, "commitment-uuid-m1", metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted, "") + doReconcile(t, r, txName) + if got := getTx(t, c, txName).Status.Phase; got != v1alpha1.ChangePhaseRollingBack { + t.Fatalf("after one CR settled: want RollingBack, got %q", got) + } + + // Settle the second CR — now all restored. + setCRReady(t, c, "commitment-uuid-m2", metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted, "") + doReconcile(t, r, txName) + if got := getTx(t, c, txName).Status.Phase; got != v1alpha1.ChangePhaseRejected { + t.Errorf("after all CRs settled: want Rejected, got %q", got) + } +} + +// TestCRTReconciler_RollingBack_Idempotent verifies that re-running RollingBack multiple +// times after restoration is safe and produces a stable Rejected terminal state. +func TestCRTReconciler_RollingBack_Idempotent(t *testing.T) { + const txName = "tx-rb-idem" + scheme := testScheme(t) + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: txName}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-idem-rb": minimalCRSpec("az-1")}, + }, + } + c := testClient(t, scheme, tx) + r := testReconciler(c, scheme) + + // nil snapshot entry — CR was new, rollback deletes it. No settlement gate needed. + driveToRollingBack(t, r, c, txName, "commitment-uuid-idem-rb") + + reconcileN(t, r, txName, 3) + + if got := getTx(t, c, txName).Status.Phase; got != v1alpha1.ChangePhaseRejected { + t.Errorf("phase: want Rejected, got %q", got) + } +} + +// ============================================================================ +// TestCRTReconciler_DryRunCleanup +// ============================================================================ + +// driveToDryRunCleanup drives a DryRun CRC to DryRunCleanup. +func driveToDryRunCleanup(t *testing.T, r *CommittedResourceChangeReconciler, c client.Client, txName, crName string) { + t.Helper() + driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseAwaitingScheduling) + setCRReady(t, c, crName, metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted, "") + driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseDryRunCleanup) +} + +func TestCRTReconciler_DryRunCleanup(t *testing.T) { + tests := []struct { + name string + preObjects []client.Object + txSpec v1alpha1.CommittedResourceChangeSpec + reconcilerOpt func(r *CommittedResourceChangeReconciler) + afterDryRunCleanup func(t *testing.T, c client.Client, txName string) + wantPhase v1alpha1.ChangePhase + wantFinishedReason string + wantProgressingStatus metav1.ConditionStatus + wantProgressingReason string + }{ + { + name: "nil snapshot: new CR deleted and dry-run completes as Accepted", + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + DryRun: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dry1": minimalCRSpec("az-1")}, + }, + wantPhase: v1alpha1.ChangePhaseAccepted, + wantFinishedReason: string(v1alpha1.ChangePhaseAccepted), + wantProgressingStatus: metav1.ConditionFalse, + wantProgressingReason: string(v1alpha1.ChangePhaseAccepted), + }, + { + name: "non-nil snapshot: existing CR spec restored and dry-run completes as Accepted", + preObjects: []client.Object{ + &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-dry2"}, + Spec: *minimalCRSpec("az-old"), + }, + }, + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + DryRun: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dry2": minimalCRSpec("az-new")}, + }, + afterDryRunCleanup: func(t *testing.T, c client.Client, _ string) { + setCRReady(t, c, "commitment-uuid-dry2", metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted, "") + }, + wantPhase: v1alpha1.ChangePhaseAccepted, + wantFinishedReason: string(v1alpha1.ChangePhaseAccepted), + wantProgressingStatus: metav1.ConditionFalse, + wantProgressingReason: string(v1alpha1.ChangePhaseAccepted), + }, + { + name: "CRs not yet settled: stays in DryRunCleanup", + preObjects: []client.Object{ + &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-dry3"}, + Spec: *minimalCRSpec("az-old"), + }, + }, + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + DryRun: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dry3": minimalCRSpec("az-new")}, + }, + // Stale the condition to simulate "CR controller has not yet processed the restored spec". + afterDryRunCleanup: func(t *testing.T, c client.Client, _ string) { + setCRReadyStaleGen(t, c, "commitment-uuid-dry3") + }, + wantPhase: v1alpha1.ChangePhaseDryRunCleanup, + wantProgressingStatus: metav1.ConditionTrue, + wantProgressingReason: "Advancing", + }, + { + name: "dry-run cleanup timeout: terminates with RollbackFailed", + preObjects: []client.Object{ + &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-dry4"}, + Spec: *minimalCRSpec("az-old"), + }, + }, + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + DryRun: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dry4": minimalCRSpec("az-new")}, + }, + reconcilerOpt: func(r *CommittedResourceChangeReconciler) { + r.RollbackTimeout = 50 * time.Millisecond + }, + afterDryRunCleanup: func(t *testing.T, c client.Client, txName string) { + setCRReadyStaleGen(t, c, "commitment-uuid-dry4") + backdateProgressingCondition(t, c, txName, 1*time.Second) + }, + wantPhase: v1alpha1.ChangePhaseRejected, + wantFinishedReason: v1alpha1.ChangeReasonRollbackFailed, + wantProgressingStatus: metav1.ConditionFalse, + wantProgressingReason: v1alpha1.ChangeReasonRetryExhausted, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := testScheme(t) + txName := "tx-dry-" + tt.name[:5] + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: txName}, + Spec: tt.txSpec, + } + objs := append([]client.Object{tx}, tt.preObjects...) + c := testClient(t, scheme, objs...) + r := testReconciler(c, scheme) + if tt.reconcilerOpt != nil { + tt.reconcilerOpt(r) + } + + var crName string + for uuid := range tt.txSpec.NewState { + crName = commitmentName(uuid) + break + } + + driveToDryRunCleanup(t, r, c, txName, crName) + + if tt.afterDryRunCleanup != nil { + tt.afterDryRunCleanup(t, c, txName) + } + + doReconcile(t, r, txName) + + tx = getTx(t, c, txName) + if tx.Status.Phase != tt.wantPhase { + t.Errorf("phase: want %q, got %q", tt.wantPhase, tx.Status.Phase) + } + if tt.wantFinishedReason != "" { + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionFinished, metav1.ConditionTrue, tt.wantFinishedReason) + } + if tt.wantProgressingStatus != "" { + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionProgressing, tt.wantProgressingStatus, tt.wantProgressingReason) + } + }) + } +} + +// TestCRTReconciler_DryRun_WouldReject verifies that DryRun=true + rejection follows +// the RollingBack path and terminates as Rejected (not Accepted). +func TestCRTReconciler_DryRun_WouldReject(t *testing.T) { + const txName = "tx-dry-reject" + scheme := testScheme(t) + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: txName}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + DryRun: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dr": minimalCRSpec("az-1")}, + }, + } + c := testClient(t, scheme, tx) + r := testReconciler(c, scheme) + + driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseAwaitingScheduling) + // Reject placement → must go to RollingBack (not DryRunCleanup). + setCRReady(t, c, "commitment-uuid-dr", metav1.ConditionFalse, v1alpha1.CommittedResourceReasonRejected, "no capacity") + driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseRollingBack) + + // nil snapshot → delete CR → settle immediately → Rejected. + doReconcile(t, r, txName) + if got := getTx(t, c, txName).Status.Phase; got != v1alpha1.ChangePhaseRejected { + t.Errorf("DryRun would-reject: want Rejected, got %q", got) + } + assertCondition(t, getTx(t, c, txName).Status.Conditions, v1alpha1.ChangeConditionFinished, metav1.ConditionTrue, string(v1alpha1.ChangePhaseRejected)) +} From 75e3038b0f1bb8bc1244731fee34fe8d7094502c Mon Sep 17 00:00:00 2001 From: juliusclausnitzer Date: Mon, 2 Nov 2026 16:15:21 +0100 Subject: [PATCH 04/13] fix bug Signed-off-by: juliusclausnitzer --- .../committed_resource_change_types.go | 6 +++++ api/v1alpha1/zz_generated.deepcopy.go | 4 +++ ...cortex.cloud_committedresourcechanges.yaml | 7 +++++ .../commitments/changes/reconciler.go | 27 ++++++++++--------- .../commitments/changes/reconciler_test.go | 18 ++++++++++--- 5 files changed, 46 insertions(+), 16 deletions(-) diff --git a/api/v1alpha1/committed_resource_change_types.go b/api/v1alpha1/committed_resource_change_types.go index 754c4b868..6ca6bedf0 100644 --- a/api/v1alpha1/committed_resource_change_types.go +++ b/api/v1alpha1/committed_resource_change_types.go @@ -112,6 +112,12 @@ type CommittedResourceChangeStatus struct { // +kubebuilder:validation:Optional UpsertedCommitments []corev1.ObjectReference `json:"upsertedCommitments,omitempty"` + // RollbackStartedAt is the time the CRC entered RollingBack or DryRunCleanup. + // Used to measure RollbackTimeout from the correct starting point rather than relying on + // the Progressing condition's LastTransitionTime, which is only updated on Status changes. + // +kubebuilder:validation:Optional + RollbackStartedAt *metav1.Time `json:"rollbackStartedAt,omitempty"` + // Conditions holds Progressing and Finished conditions. // +kubebuilder:validation:Optional Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index a60d5a1f4..c038084e1 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -194,6 +194,10 @@ func (in *CommittedResourceChangeStatus) DeepCopyInto(out *CommittedResourceChan *out = make([]v1.ObjectReference, len(*in)) copy(*out, *in) } + if in.RollbackStartedAt != nil { + in, out := &in.RollbackStartedAt, &out.RollbackStartedAt + *out = (*in).DeepCopy() + } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]metav1.Condition, len(*in)) diff --git a/helm/library/cortex/files/crds/cortex.cloud_committedresourcechanges.yaml b/helm/library/cortex/files/crds/cortex.cloud_committedresourcechanges.yaml index ecf607f4c..410278265 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_committedresourcechanges.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_committedresourcechanges.yaml @@ -262,6 +262,13 @@ spec: - message: phase cannot transition out of a terminal state rule: oldSelf == '' || !['Accepted','Rejected','Superseded'].exists(t, t == oldSelf) || self == oldSelf + rollbackStartedAt: + description: |- + RollbackStartedAt is the time the CRC entered RollingBack or DryRunCleanup. + Used to measure RollbackTimeout from the correct starting point rather than relying on + the Progressing condition's LastTransitionTime, which is only updated on Status changes. + format: date-time + type: string snapshotBefore: additionalProperties: description: CommittedResourceSpec defines the desired state of diff --git a/internal/scheduling/reservations/commitments/changes/reconciler.go b/internal/scheduling/reservations/commitments/changes/reconciler.go index 21ff1772a..639f15b9b 100644 --- a/internal/scheduling/reservations/commitments/changes/reconciler.go +++ b/internal/scheduling/reservations/commitments/changes/reconciler.go @@ -189,6 +189,8 @@ func (r *CommittedResourceChangeReconciler) reconcileAwaitingScheduling(ctx cont logger.Info("CR rejected, rolling back", "cr", ref.Name, "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "message", cond.Message) return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseRollingBack, func(s *v1alpha1.CommittedResourceChangeStatus) { s.Message = fmt.Sprintf("CR %s rejected: %s", ref.Name, cond.Message) + now := metav1.Now() + s.RollbackStartedAt = &now }) default: // Still transitioning — wait. @@ -204,6 +206,8 @@ func (r *CommittedResourceChangeReconciler) reconcileAwaitingScheduling(ctx cont logger.Info("timed out waiting for scheduling, rolling back", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "timeout", r.AwaitingSchedulingTimeout) return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseRollingBack, func(s *v1alpha1.CommittedResourceChangeStatus) { s.Message = "timed out waiting for scheduling" + now := metav1.Now() + s.RollbackStartedAt = &now meta.SetStatusCondition(&s.Conditions, metav1.Condition{ Type: v1alpha1.ChangeConditionProgressing, Status: metav1.ConditionFalse, @@ -220,7 +224,10 @@ func (r *CommittedResourceChangeReconciler) reconcileAwaitingScheduling(ctx cont } if tx.Spec.DryRun { - return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseDryRunCleanup, nil) + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseDryRunCleanup, func(s *v1alpha1.CommittedResourceChangeStatus) { + now := metav1.Now() + s.RollbackStartedAt = &now + }) } return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseAccepted, nil) } @@ -234,12 +241,9 @@ func (r *CommittedResourceChangeReconciler) reconcileRollingBack(ctx context.Con return ctrl.Result{}, err } if !allSettled { - if r.RollbackTimeout > 0 { - progressingCond := meta.FindStatusCondition(tx.Status.Conditions, v1alpha1.ChangeConditionProgressing) - if progressingCond != nil && time.Since(progressingCond.LastTransitionTime.Time) > r.RollbackTimeout { - logger.Info("rollback timed out, terminalizing", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "timeout", r.RollbackTimeout) - return r.terminateWithRollbackFailed(ctx, tx, "timed out restoring pre-change state") - } + if r.RollbackTimeout > 0 && tx.Status.RollbackStartedAt != nil && time.Since(tx.Status.RollbackStartedAt.Time) > r.RollbackTimeout { + logger.Info("rollback timed out, terminalizing", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "timeout", r.RollbackTimeout) + return r.terminateWithRollbackFailed(ctx, tx, "timed out restoring pre-change state") } return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } @@ -254,12 +258,9 @@ func (r *CommittedResourceChangeReconciler) reconcileDryRunCleanup(ctx context.C return ctrl.Result{}, err } if !allSettled { - if r.RollbackTimeout > 0 { - progressingCond := meta.FindStatusCondition(tx.Status.Conditions, v1alpha1.ChangeConditionProgressing) - if progressingCond != nil && time.Since(progressingCond.LastTransitionTime.Time) > r.RollbackTimeout { - logger.Info("dry-run cleanup timed out, terminalizing", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "timeout", r.RollbackTimeout) - return r.terminateWithRollbackFailed(ctx, tx, "timed out restoring pre-change state after dry run") - } + if r.RollbackTimeout > 0 && tx.Status.RollbackStartedAt != nil && time.Since(tx.Status.RollbackStartedAt.Time) > r.RollbackTimeout { + logger.Info("dry-run cleanup timed out, terminalizing", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "timeout", r.RollbackTimeout) + return r.terminateWithRollbackFailed(ctx, tx, "timed out restoring pre-change state after dry run") } return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } diff --git a/internal/scheduling/reservations/commitments/changes/reconciler_test.go b/internal/scheduling/reservations/commitments/changes/reconciler_test.go index 7da4e9257..38affc535 100644 --- a/internal/scheduling/reservations/commitments/changes/reconciler_test.go +++ b/internal/scheduling/reservations/commitments/changes/reconciler_test.go @@ -158,6 +158,18 @@ func backdateProgressingCondition(t *testing.T, c client.Client, txName string, } } +// backdateRollbackStartedAt sets RollbackStartedAt to simulate a rollback that has been running +// for the given duration. Used to trigger the RollbackTimeout check in tests. +func backdateRollbackStartedAt(t *testing.T, c client.Client, txName string, age time.Duration) { + t.Helper() + tx := getTx(t, c, txName) + past := metav1.NewTime(time.Now().Add(-age)) + tx.Status.RollbackStartedAt = &past + if err := c.Status().Update(testCtx, tx); err != nil { + t.Fatalf("backdate RollbackStartedAt on %q: %v", txName, err) + } +} + // assertCondition verifies a condition in the given list by type, status, and optional reason. // Pass an empty wantReason to skip the reason check. func assertCondition(t *testing.T, conditions []metav1.Condition, condType string, wantStatus metav1.ConditionStatus, wantReason string) { @@ -1187,10 +1199,10 @@ func TestCRTReconciler_RollingBack(t *testing.T) { reconcilerOpt: func(r *CommittedResourceChangeReconciler) { r.RollbackTimeout = 50 * time.Millisecond }, - // Stale the CR condition AND backdate the Progressing timestamp to trigger the timeout. + // Stale the CR condition AND backdate RollbackStartedAt to trigger the timeout. afterRollingBack: func(t *testing.T, c client.Client, txName string) { setCRReadyStaleGen(t, c, "commitment-uuid-rb4") - backdateProgressingCondition(t, c, txName, 1*time.Second) + backdateRollbackStartedAt(t, c, txName, 1*time.Second) }, wantPhase: v1alpha1.ChangePhaseRejected, wantFinishedReason: v1alpha1.ChangeReasonRollbackFailed, @@ -1430,7 +1442,7 @@ func TestCRTReconciler_DryRunCleanup(t *testing.T) { }, afterDryRunCleanup: func(t *testing.T, c client.Client, txName string) { setCRReadyStaleGen(t, c, "commitment-uuid-dry4") - backdateProgressingCondition(t, c, txName, 1*time.Second) + backdateRollbackStartedAt(t, c, txName, 1*time.Second) }, wantPhase: v1alpha1.ChangePhaseRejected, wantFinishedReason: v1alpha1.ChangeReasonRollbackFailed, From 4241bdc7b20d32bf326851ca7ee0c295488cfb0f Mon Sep 17 00:00:00 2001 From: juliusclausnitzer Date: Mon, 2 Nov 2026 16:49:38 +0100 Subject: [PATCH 05/13] fix Signed-off-by: juliusclausnitzer --- .../reservations/commitments/changes/reconciler.go | 13 +++++++------ .../commitments/changes/reconciler_test.go | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/internal/scheduling/reservations/commitments/changes/reconciler.go b/internal/scheduling/reservations/commitments/changes/reconciler.go index 639f15b9b..7a3b8d3de 100644 --- a/internal/scheduling/reservations/commitments/changes/reconciler.go +++ b/internal/scheduling/reservations/commitments/changes/reconciler.go @@ -243,7 +243,7 @@ func (r *CommittedResourceChangeReconciler) reconcileRollingBack(ctx context.Con if !allSettled { if r.RollbackTimeout > 0 && tx.Status.RollbackStartedAt != nil && time.Since(tx.Status.RollbackStartedAt.Time) > r.RollbackTimeout { logger.Info("rollback timed out, terminalizing", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "timeout", r.RollbackTimeout) - return r.terminateWithRollbackFailed(ctx, tx, "timed out restoring pre-change state") + return r.terminateWithRollbackFailed(ctx, tx, v1alpha1.ChangePhaseRejected, "timed out restoring pre-change state") } return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } @@ -260,7 +260,7 @@ func (r *CommittedResourceChangeReconciler) reconcileDryRunCleanup(ctx context.C if !allSettled { if r.RollbackTimeout > 0 && tx.Status.RollbackStartedAt != nil && time.Since(tx.Status.RollbackStartedAt.Time) > r.RollbackTimeout { logger.Info("dry-run cleanup timed out, terminalizing", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "timeout", r.RollbackTimeout) - return r.terminateWithRollbackFailed(ctx, tx, "timed out restoring pre-change state after dry run") + return r.terminateWithRollbackFailed(ctx, tx, v1alpha1.ChangePhaseAccepted, "timed out restoring pre-change state after dry run") } return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } @@ -321,10 +321,11 @@ func (r *CommittedResourceChangeReconciler) restoreFromSnapshot(ctx context.Cont return true, nil } -// terminateWithRollbackFailed stamps Finished=True/RollbackFailed and Progressing=False/RetryExhausted, -// then advances the phase to Rejected. Used when the rollback itself cannot complete within RollbackTimeout. -func (r *CommittedResourceChangeReconciler) terminateWithRollbackFailed(ctx context.Context, tx *v1alpha1.CommittedResourceChange, msg string) (ctrl.Result, error) { - tx.Status.Phase = v1alpha1.ChangePhaseRejected +// terminateWithRollbackFailed stamps Finished=True/RollbackFailed and Progressing=False/RetryExhausted. +// phase controls the terminal FSM phase: Rejected for a failed rollback, Accepted for a DryRunCleanup +// timeout (the scheduler accepted the change; only the post-probe cleanup stalled). +func (r *CommittedResourceChangeReconciler) terminateWithRollbackFailed(ctx context.Context, tx *v1alpha1.CommittedResourceChange, phase v1alpha1.ChangePhase, msg string) (ctrl.Result, error) { + tx.Status.Phase = phase tx.Status.Message = msg meta.SetStatusCondition(&tx.Status.Conditions, metav1.Condition{ Type: v1alpha1.ChangeConditionFinished, diff --git a/internal/scheduling/reservations/commitments/changes/reconciler_test.go b/internal/scheduling/reservations/commitments/changes/reconciler_test.go index 38affc535..efbf96d71 100644 --- a/internal/scheduling/reservations/commitments/changes/reconciler_test.go +++ b/internal/scheduling/reservations/commitments/changes/reconciler_test.go @@ -1444,7 +1444,7 @@ func TestCRTReconciler_DryRunCleanup(t *testing.T) { setCRReadyStaleGen(t, c, "commitment-uuid-dry4") backdateRollbackStartedAt(t, c, txName, 1*time.Second) }, - wantPhase: v1alpha1.ChangePhaseRejected, + wantPhase: v1alpha1.ChangePhaseAccepted, wantFinishedReason: v1alpha1.ChangeReasonRollbackFailed, wantProgressingStatus: metav1.ConditionFalse, wantProgressingReason: v1alpha1.ChangeReasonRetryExhausted, From 0fe90e711efd1bfacaef52829687315dd19271b0 Mon Sep 17 00:00:00 2001 From: juliusclausnitzer Date: Wed, 4 Nov 2026 14:48:46 +0100 Subject: [PATCH 06/13] cleanup fix Signed-off-by: juliusclausnitzer --- .../reservations/commitments/changes/reconciler.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/internal/scheduling/reservations/commitments/changes/reconciler.go b/internal/scheduling/reservations/commitments/changes/reconciler.go index 7a3b8d3de..859f4cdb1 100644 --- a/internal/scheduling/reservations/commitments/changes/reconciler.go +++ b/internal/scheduling/reservations/commitments/changes/reconciler.go @@ -189,8 +189,6 @@ func (r *CommittedResourceChangeReconciler) reconcileAwaitingScheduling(ctx cont logger.Info("CR rejected, rolling back", "cr", ref.Name, "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "message", cond.Message) return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseRollingBack, func(s *v1alpha1.CommittedResourceChangeStatus) { s.Message = fmt.Sprintf("CR %s rejected: %s", ref.Name, cond.Message) - now := metav1.Now() - s.RollbackStartedAt = &now }) default: // Still transitioning — wait. @@ -206,8 +204,6 @@ func (r *CommittedResourceChangeReconciler) reconcileAwaitingScheduling(ctx cont logger.Info("timed out waiting for scheduling, rolling back", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "timeout", r.AwaitingSchedulingTimeout) return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseRollingBack, func(s *v1alpha1.CommittedResourceChangeStatus) { s.Message = "timed out waiting for scheduling" - now := metav1.Now() - s.RollbackStartedAt = &now meta.SetStatusCondition(&s.Conditions, metav1.Condition{ Type: v1alpha1.ChangeConditionProgressing, Status: metav1.ConditionFalse, @@ -224,10 +220,7 @@ func (r *CommittedResourceChangeReconciler) reconcileAwaitingScheduling(ctx cont } if tx.Spec.DryRun { - return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseDryRunCleanup, func(s *v1alpha1.CommittedResourceChangeStatus) { - now := metav1.Now() - s.RollbackStartedAt = &now - }) + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseDryRunCleanup, nil) } return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseAccepted, nil) } @@ -365,6 +358,10 @@ func (r *CommittedResourceChangeReconciler) advanceTo( if mutate != nil { mutate(&tx.Status) } + if (phase == v1alpha1.ChangePhaseRollingBack || phase == v1alpha1.ChangePhaseDryRunCleanup) && tx.Status.RollbackStartedAt == nil { + now := metav1.Now() + tx.Status.RollbackStartedAt = &now + } progressingStatus, progressingReason := metav1.ConditionTrue, "Advancing" if tx.IsTerminal() { progressingStatus, progressingReason = metav1.ConditionFalse, string(phase) From 74f6a9fe37b037e9b94f34738a58b154d1213d58 Mon Sep 17 00:00:00 2001 From: juliusclausnitzer Date: Mon, 9 Nov 2026 12:04:14 +0100 Subject: [PATCH 07/13] lint fix Signed-off-by: juliusclausnitzer --- .../nova/plugins/filters/filter_quota_enforcement_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go index 2c492bb38..d27640105 100644 --- a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go @@ -461,7 +461,9 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { // Fully used, so it provides no CR headroom itself — the request // falls through to the PAYG check where its amount is deducted. Status: v1alpha1.CommittedResourceStatus{ - UsedResources: map[string]resource.Quantity{"memory": resource.MustParse("100Gi")}, + Billing: &v1alpha1.BillingStatus{ + UsedResources: map[string]resource.Quantity{"memory": resource.MustParse("100Gi")}, + }, }, }, &v1alpha1.ProjectQuota{ @@ -500,7 +502,9 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { Amount: resource.MustParse("100Gi"), }, Status: v1alpha1.CommittedResourceStatus{ - UsedResources: map[string]resource.Quantity{"memory": resource.MustParse("100Gi")}, + Billing: &v1alpha1.BillingStatus{ + UsedResources: map[string]resource.Quantity{"memory": resource.MustParse("100Gi")}, + }, }, }, &v1alpha1.ProjectQuota{ From 9beb21d025720b7b8649b6393a8684fa5db6b379 Mon Sep 17 00:00:00 2001 From: juliusclausnitzer Date: Tue, 10 Nov 2026 16:28:12 +0100 Subject: [PATCH 08/13] get rid of aggressive dry run Signed-off-by: juliusclausnitzer --- .../committed_resource_change_types.go | 9 +---- ...cortex.cloud_committedresourcechanges.yaml | 10 +---- .../commitments/changes/reconciler.go | 39 ++++--------------- 3 files changed, 10 insertions(+), 48 deletions(-) diff --git a/api/v1alpha1/committed_resource_change_types.go b/api/v1alpha1/committed_resource_change_types.go index 6ca6bedf0..ccd394e9c 100644 --- a/api/v1alpha1/committed_resource_change_types.go +++ b/api/v1alpha1/committed_resource_change_types.go @@ -20,8 +20,6 @@ const ( ChangePhaseAwaitingScheduling ChangePhase = "AwaitingScheduling" // ChangePhaseRollingBack: a CR was rejected or timed out; restoring pre-change state. ChangePhaseRollingBack ChangePhase = "RollingBack" - // ChangePhaseDryRunCleanup: dry-run would-accept; restoring pre-change state. - ChangePhaseDryRunCleanup ChangePhase = "DryRunCleanup" // ChangePhaseAccepted: terminal; change request committed successfully. ChangePhaseAccepted ChangePhase = "Accepted" // ChangePhaseRejected: terminal; change request rolled back. @@ -83,10 +81,6 @@ type CommittedResourceChangeSpec struct { // +kubebuilder:validation:Optional AllowRejection bool `json:"allowRejection,omitempty"` - // DryRun controls whether the change request commits or only probes schedulability. - // +kubebuilder:validation:Optional - DryRun bool `json:"dryRun,omitempty"` - // NewState maps commitmentUUID to the desired CommittedResourceSpec. // A nil value means the corresponding CommittedResource should be deleted. // +kubebuilder:validation:Optional @@ -112,7 +106,7 @@ type CommittedResourceChangeStatus struct { // +kubebuilder:validation:Optional UpsertedCommitments []corev1.ObjectReference `json:"upsertedCommitments,omitempty"` - // RollbackStartedAt is the time the CRC entered RollingBack or DryRunCleanup. + // RollbackStartedAt is the time the CRC entered RollingBack. // Used to measure RollbackTimeout from the correct starting point rather than relying on // the Progressing condition's LastTransitionTime, which is only updated on Status changes. // +kubebuilder:validation:Optional @@ -133,7 +127,6 @@ type CommittedResourceChangeStatus struct { // +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase" // +kubebuilder:printcolumn:name="AZ",type="string",JSONPath=".spec.availabilityZone" // +kubebuilder:printcolumn:name="Source",type="string",JSONPath=".spec.source" -// +kubebuilder:printcolumn:name="DryRun",type="boolean",JSONPath=".spec.dryRun",priority=1 // +kubebuilder:printcolumn:name="Progressing",type="string",JSONPath=".status.conditions[?(@.type=='Progressing')].status",priority=1 // +kubebuilder:printcolumn:name="Finished",type="string",JSONPath=".status.conditions[?(@.type=='Finished')].status",priority=1 diff --git a/helm/library/cortex/files/crds/cortex.cloud_committedresourcechanges.yaml b/helm/library/cortex/files/crds/cortex.cloud_committedresourcechanges.yaml index 410278265..f75b21044 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_committedresourcechanges.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_committedresourcechanges.yaml @@ -26,10 +26,6 @@ spec: - jsonPath: .spec.source name: Source type: string - - jsonPath: .spec.dryRun - name: DryRun - priority: 1 - type: boolean - jsonPath: .status.conditions[?(@.type=='Progressing')].status name: Progressing priority: 1 @@ -78,10 +74,6 @@ spec: correlationID: description: CorrelationID is the request/correlation ID. type: string - dryRun: - description: DryRun controls whether the change request commits or - only probes schedulability. - type: boolean newState: additionalProperties: description: CommittedResourceSpec defines the desired state of @@ -264,7 +256,7 @@ spec: t == oldSelf) || self == oldSelf rollbackStartedAt: description: |- - RollbackStartedAt is the time the CRC entered RollingBack or DryRunCleanup. + RollbackStartedAt is the time the CRC entered RollingBack. Used to measure RollbackTimeout from the correct starting point rather than relying on the Progressing condition's LastTransitionTime, which is only updated on Status changes. format: date-time diff --git a/internal/scheduling/reservations/commitments/changes/reconciler.go b/internal/scheduling/reservations/commitments/changes/reconciler.go index cf278577c..9eddf84a4 100644 --- a/internal/scheduling/reservations/commitments/changes/reconciler.go +++ b/internal/scheduling/reservations/commitments/changes/reconciler.go @@ -35,7 +35,7 @@ type CommittedResourceChangeReconciler struct { // AwaitingSchedulingTimeout is how long to wait in AwaitingScheduling before rolling back. // Zero (default) means no timeout. AwaitingSchedulingTimeout time.Duration - // RollbackTimeout is how long to attempt snapshot restore in RollingBack/DryRunCleanup + // RollbackTimeout is how long to attempt snapshot restore in RollingBack // before giving up and terminating with RollbackFailed. // Zero (default) means no timeout. RollbackTimeout time.Duration @@ -66,8 +66,6 @@ func (r *CommittedResourceChangeReconciler) Reconcile(ctx context.Context, req c return r.reconcileAwaitingScheduling(ctx, logger, &tx) case v1alpha1.ChangePhaseRollingBack: return r.reconcileRollingBack(ctx, logger, &tx) - case v1alpha1.ChangePhaseDryRunCleanup: - return r.reconcileDryRunCleanup(ctx, logger, &tx) case v1alpha1.ChangePhaseAccepted, v1alpha1.ChangePhaseRejected, v1alpha1.ChangePhaseSuperseded: return ctrl.Result{}, nil default: @@ -155,7 +153,7 @@ func (r *CommittedResourceChangeReconciler) reconcileAllocating(ctx context.Cont // reconcileAwaitingScheduling reads UpsertedCommitments and waits for placement settlement. // Gates on each CR's ready-condition observedGeneration matching the generation written in Allocating. -// All CRs accepted → Accepted (or DryRunCleanup when DryRun=true). +// All CRs accepted → Accepted. // Any CR rejected → RollingBack. // Not yet settled → wait for the CR watch to re-enqueue. func (r *CommittedResourceChangeReconciler) reconcileAwaitingScheduling(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange) (ctrl.Result, error) { @@ -221,9 +219,6 @@ func (r *CommittedResourceChangeReconciler) reconcileAwaitingScheduling(ctx cont return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } - if tx.Spec.DryRun { - return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseDryRunCleanup, nil) - } return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseAccepted, nil) } @@ -238,30 +233,13 @@ func (r *CommittedResourceChangeReconciler) reconcileRollingBack(ctx context.Con if !allSettled { if r.RollbackTimeout > 0 && tx.Status.RollbackStartedAt != nil && time.Since(tx.Status.RollbackStartedAt.Time) > r.RollbackTimeout { logger.Info("rollback timed out, terminalizing", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "timeout", r.RollbackTimeout) - return r.terminateWithRollbackFailed(ctx, tx, v1alpha1.ChangePhaseRejected, "timed out restoring pre-change state") + return r.terminateWithRollbackFailed(ctx, tx, "timed out restoring pre-change state") } return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseRejected, nil) } -// reconcileDryRunCleanup restores pre-change state after a dry-run would-accept outcome. -// Re-uses snapshot rollback logic; terminal phase is Accepted (not Rejected). -func (r *CommittedResourceChangeReconciler) reconcileDryRunCleanup(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange) (ctrl.Result, error) { - allSettled, err := r.restoreFromSnapshot(ctx, logger, tx) - if err != nil { - return ctrl.Result{}, err - } - if !allSettled { - if r.RollbackTimeout > 0 && tx.Status.RollbackStartedAt != nil && time.Since(tx.Status.RollbackStartedAt.Time) > r.RollbackTimeout { - logger.Info("dry-run cleanup timed out, terminalizing", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "timeout", r.RollbackTimeout) - return r.terminateWithRollbackFailed(ctx, tx, v1alpha1.ChangePhaseAccepted, "timed out restoring pre-change state after dry run") - } - return ctrl.Result{RequeueAfter: 5 * time.Second}, nil - } - return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseAccepted, nil) -} - // restoreFromSnapshot converges CommittedResources back to their pre-change state recorded in // tx.Status.SnapshotBefore. It is idempotent and safe to call on every requeue. // @@ -316,11 +294,10 @@ func (r *CommittedResourceChangeReconciler) restoreFromSnapshot(ctx context.Cont return true, nil } -// terminateWithRollbackFailed stamps Finished=True/RollbackFailed and Progressing=False/RetryExhausted. -// phase controls the terminal FSM phase: Rejected for a failed rollback, Accepted for a DryRunCleanup -// timeout (the scheduler accepted the change; only the post-probe cleanup stalled). -func (r *CommittedResourceChangeReconciler) terminateWithRollbackFailed(ctx context.Context, tx *v1alpha1.CommittedResourceChange, phase v1alpha1.ChangePhase, msg string) (ctrl.Result, error) { - tx.Status.Phase = phase +// terminateWithRollbackFailed stamps Finished=True/RollbackFailed and Progressing=False/RetryExhausted +// and terminates the transaction as Rejected. +func (r *CommittedResourceChangeReconciler) terminateWithRollbackFailed(ctx context.Context, tx *v1alpha1.CommittedResourceChange, msg string) (ctrl.Result, error) { + tx.Status.Phase = v1alpha1.ChangePhaseRejected tx.Status.Message = msg meta.SetStatusCondition(&tx.Status.Conditions, metav1.Condition{ Type: v1alpha1.ChangeConditionFinished, @@ -360,7 +337,7 @@ func (r *CommittedResourceChangeReconciler) advanceTo( if mutate != nil { mutate(&tx.Status) } - if (phase == v1alpha1.ChangePhaseRollingBack || phase == v1alpha1.ChangePhaseDryRunCleanup) && tx.Status.RollbackStartedAt == nil { + if phase == v1alpha1.ChangePhaseRollingBack && tx.Status.RollbackStartedAt == nil { now := metav1.Now() tx.Status.RollbackStartedAt = &now } From e8efd323786a75d4baf46f307b389a861b5d3c93 Mon Sep 17 00:00:00 2001 From: juliusclausnitzer Date: Tue, 10 Nov 2026 16:28:16 +0100 Subject: [PATCH 09/13] test Signed-off-by: juliusclausnitzer --- .../commitments/changes/reconciler_test.go | 199 ------------------ 1 file changed, 199 deletions(-) diff --git a/internal/scheduling/reservations/commitments/changes/reconciler_test.go b/internal/scheduling/reservations/commitments/changes/reconciler_test.go index efbf96d71..e3cdc4dcc 100644 --- a/internal/scheduling/reservations/commitments/changes/reconciler_test.go +++ b/internal/scheduling/reservations/commitments/changes/reconciler_test.go @@ -544,18 +544,6 @@ func TestCRTReconciler_AwaitingScheduling(t *testing.T) { wantFinished: true, wantFinishedReason: string(v1alpha1.ChangePhaseAccepted), }, - { - name: "DryRun=true and all CRs accepted → DryRunCleanup", - txOpts: []func(*v1alpha1.CommittedResourceChange){ - func(tx *v1alpha1.CommittedResourceChange) { tx.Spec.DryRun = true }, - }, - setupCR: func(t *testing.T, c client.Client) { - setCRReady(t, c, crName, metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted, "") - }, - wantPhase: v1alpha1.ChangePhaseDryRunCleanup, - wantProgressingStatus: metav1.ConditionTrue, - wantProgressingReason: "Advancing", - }, { name: "CR rejected → RollingBack with rejection message", setupCR: func(t *testing.T, c client.Client) { @@ -1337,190 +1325,3 @@ func TestCRTReconciler_RollingBack_Idempotent(t *testing.T) { t.Errorf("phase: want Rejected, got %q", got) } } - -// ============================================================================ -// TestCRTReconciler_DryRunCleanup -// ============================================================================ - -// driveToDryRunCleanup drives a DryRun CRC to DryRunCleanup. -func driveToDryRunCleanup(t *testing.T, r *CommittedResourceChangeReconciler, c client.Client, txName, crName string) { - t.Helper() - driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseAwaitingScheduling) - setCRReady(t, c, crName, metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted, "") - driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseDryRunCleanup) -} - -func TestCRTReconciler_DryRunCleanup(t *testing.T) { - tests := []struct { - name string - preObjects []client.Object - txSpec v1alpha1.CommittedResourceChangeSpec - reconcilerOpt func(r *CommittedResourceChangeReconciler) - afterDryRunCleanup func(t *testing.T, c client.Client, txName string) - wantPhase v1alpha1.ChangePhase - wantFinishedReason string - wantProgressingStatus metav1.ConditionStatus - wantProgressingReason string - }{ - { - name: "nil snapshot: new CR deleted and dry-run completes as Accepted", - txSpec: v1alpha1.CommittedResourceChangeSpec{ - AvailabilityZone: "az-1", - Source: v1alpha1.TxSourceAPI, - AllowRejection: true, - DryRun: true, - NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dry1": minimalCRSpec("az-1")}, - }, - wantPhase: v1alpha1.ChangePhaseAccepted, - wantFinishedReason: string(v1alpha1.ChangePhaseAccepted), - wantProgressingStatus: metav1.ConditionFalse, - wantProgressingReason: string(v1alpha1.ChangePhaseAccepted), - }, - { - name: "non-nil snapshot: existing CR spec restored and dry-run completes as Accepted", - preObjects: []client.Object{ - &v1alpha1.CommittedResource{ - ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-dry2"}, - Spec: *minimalCRSpec("az-old"), - }, - }, - txSpec: v1alpha1.CommittedResourceChangeSpec{ - AvailabilityZone: "az-1", - Source: v1alpha1.TxSourceAPI, - AllowRejection: true, - DryRun: true, - NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dry2": minimalCRSpec("az-new")}, - }, - afterDryRunCleanup: func(t *testing.T, c client.Client, _ string) { - setCRReady(t, c, "commitment-uuid-dry2", metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted, "") - }, - wantPhase: v1alpha1.ChangePhaseAccepted, - wantFinishedReason: string(v1alpha1.ChangePhaseAccepted), - wantProgressingStatus: metav1.ConditionFalse, - wantProgressingReason: string(v1alpha1.ChangePhaseAccepted), - }, - { - name: "CRs not yet settled: stays in DryRunCleanup", - preObjects: []client.Object{ - &v1alpha1.CommittedResource{ - ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-dry3"}, - Spec: *minimalCRSpec("az-old"), - }, - }, - txSpec: v1alpha1.CommittedResourceChangeSpec{ - AvailabilityZone: "az-1", - Source: v1alpha1.TxSourceAPI, - AllowRejection: true, - DryRun: true, - NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dry3": minimalCRSpec("az-new")}, - }, - // Stale the condition to simulate "CR controller has not yet processed the restored spec". - afterDryRunCleanup: func(t *testing.T, c client.Client, _ string) { - setCRReadyStaleGen(t, c, "commitment-uuid-dry3") - }, - wantPhase: v1alpha1.ChangePhaseDryRunCleanup, - wantProgressingStatus: metav1.ConditionTrue, - wantProgressingReason: "Advancing", - }, - { - name: "dry-run cleanup timeout: terminates with RollbackFailed", - preObjects: []client.Object{ - &v1alpha1.CommittedResource{ - ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-dry4"}, - Spec: *minimalCRSpec("az-old"), - }, - }, - txSpec: v1alpha1.CommittedResourceChangeSpec{ - AvailabilityZone: "az-1", - Source: v1alpha1.TxSourceAPI, - AllowRejection: true, - DryRun: true, - NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dry4": minimalCRSpec("az-new")}, - }, - reconcilerOpt: func(r *CommittedResourceChangeReconciler) { - r.RollbackTimeout = 50 * time.Millisecond - }, - afterDryRunCleanup: func(t *testing.T, c client.Client, txName string) { - setCRReadyStaleGen(t, c, "commitment-uuid-dry4") - backdateRollbackStartedAt(t, c, txName, 1*time.Second) - }, - wantPhase: v1alpha1.ChangePhaseAccepted, - wantFinishedReason: v1alpha1.ChangeReasonRollbackFailed, - wantProgressingStatus: metav1.ConditionFalse, - wantProgressingReason: v1alpha1.ChangeReasonRetryExhausted, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - scheme := testScheme(t) - txName := "tx-dry-" + tt.name[:5] - tx := &v1alpha1.CommittedResourceChange{ - ObjectMeta: metav1.ObjectMeta{Name: txName}, - Spec: tt.txSpec, - } - objs := append([]client.Object{tx}, tt.preObjects...) - c := testClient(t, scheme, objs...) - r := testReconciler(c, scheme) - if tt.reconcilerOpt != nil { - tt.reconcilerOpt(r) - } - - var crName string - for uuid := range tt.txSpec.NewState { - crName = commitmentName(uuid) - break - } - - driveToDryRunCleanup(t, r, c, txName, crName) - - if tt.afterDryRunCleanup != nil { - tt.afterDryRunCleanup(t, c, txName) - } - - doReconcile(t, r, txName) - - tx = getTx(t, c, txName) - if tx.Status.Phase != tt.wantPhase { - t.Errorf("phase: want %q, got %q", tt.wantPhase, tx.Status.Phase) - } - if tt.wantFinishedReason != "" { - assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionFinished, metav1.ConditionTrue, tt.wantFinishedReason) - } - if tt.wantProgressingStatus != "" { - assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionProgressing, tt.wantProgressingStatus, tt.wantProgressingReason) - } - }) - } -} - -// TestCRTReconciler_DryRun_WouldReject verifies that DryRun=true + rejection follows -// the RollingBack path and terminates as Rejected (not Accepted). -func TestCRTReconciler_DryRun_WouldReject(t *testing.T) { - const txName = "tx-dry-reject" - scheme := testScheme(t) - tx := &v1alpha1.CommittedResourceChange{ - ObjectMeta: metav1.ObjectMeta{Name: txName}, - Spec: v1alpha1.CommittedResourceChangeSpec{ - AvailabilityZone: "az-1", - Source: v1alpha1.TxSourceAPI, - AllowRejection: true, - DryRun: true, - NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dr": minimalCRSpec("az-1")}, - }, - } - c := testClient(t, scheme, tx) - r := testReconciler(c, scheme) - - driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseAwaitingScheduling) - // Reject placement → must go to RollingBack (not DryRunCleanup). - setCRReady(t, c, "commitment-uuid-dr", metav1.ConditionFalse, v1alpha1.CommittedResourceReasonRejected, "no capacity") - driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseRollingBack) - - // nil snapshot → delete CR → settle immediately → Rejected. - doReconcile(t, r, txName) - if got := getTx(t, c, txName).Status.Phase; got != v1alpha1.ChangePhaseRejected { - t.Errorf("DryRun would-reject: want Rejected, got %q", got) - } - assertCondition(t, getTx(t, c, txName).Status.Conditions, v1alpha1.ChangeConditionFinished, metav1.ConditionTrue, string(v1alpha1.ChangePhaseRejected)) -} From a84baacd2e6fb2598c049945076196a5ad155f3b Mon Sep 17 00:00:00 2001 From: juliusclausnitzer Date: Tue, 10 Nov 2026 16:28:56 +0100 Subject: [PATCH 10/13] adjusted docs Signed-off-by: juliusclausnitzer --- .../03-b-committed-resource-transaction-architecture.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/03-reservations-and-inventory/03-b-committed-resource-transaction-architecture.md b/docs/03-reservations-and-inventory/03-b-committed-resource-transaction-architecture.md index 3e5827bd1..3ec471b33 100644 --- a/docs/03-reservations-and-inventory/03-b-committed-resource-transaction-architecture.md +++ b/docs/03-reservations-and-inventory/03-b-committed-resource-transaction-architecture.md @@ -76,11 +76,8 @@ stateDiagram-v2 Allocating --> Superseded : newer CRC for same AZ took over AwaitingScheduling --> Accepted : all CRs Ready (observedGeneration matches) - AwaitingScheduling --> DryRunCleanup : all CRs Ready, but DryRun=true AwaitingScheduling --> RollingBack : CR rejected or timeout - DryRunCleanup --> Accepted : pre-state restored (would-accept) - RollingBack --> Rejected : pre-state restored (RollbackFailed on permanent failure) Accepted --> [*] From d3f6d77cb6384baf9112231c5410ee241df3d113 Mon Sep 17 00:00:00 2001 From: juliusclausnitzer Date: Tue, 10 Nov 2026 17:30:33 +0100 Subject: [PATCH 11/13] Revert "get rid of aggressive dry run" Restore the full CRC dry-run + cleanup implementation. The removal was unnecessary: dry-run only ever probes net-positive new commitments under a random ID, so it cannot delete or clobber existing customer resources. This reverts the three removal commits (9beb21d0, e8efd323, a84baacd). --- .../committed_resource_change_types.go | 9 +- ...itted-resource-transaction-architecture.md | 3 + ...cortex.cloud_committedresourcechanges.yaml | 10 +- .../commitments/changes/reconciler.go | 39 +++- .../commitments/changes/reconciler_test.go | 199 ++++++++++++++++++ 5 files changed, 250 insertions(+), 10 deletions(-) diff --git a/api/v1alpha1/committed_resource_change_types.go b/api/v1alpha1/committed_resource_change_types.go index ccd394e9c..6ca6bedf0 100644 --- a/api/v1alpha1/committed_resource_change_types.go +++ b/api/v1alpha1/committed_resource_change_types.go @@ -20,6 +20,8 @@ const ( ChangePhaseAwaitingScheduling ChangePhase = "AwaitingScheduling" // ChangePhaseRollingBack: a CR was rejected or timed out; restoring pre-change state. ChangePhaseRollingBack ChangePhase = "RollingBack" + // ChangePhaseDryRunCleanup: dry-run would-accept; restoring pre-change state. + ChangePhaseDryRunCleanup ChangePhase = "DryRunCleanup" // ChangePhaseAccepted: terminal; change request committed successfully. ChangePhaseAccepted ChangePhase = "Accepted" // ChangePhaseRejected: terminal; change request rolled back. @@ -81,6 +83,10 @@ type CommittedResourceChangeSpec struct { // +kubebuilder:validation:Optional AllowRejection bool `json:"allowRejection,omitempty"` + // DryRun controls whether the change request commits or only probes schedulability. + // +kubebuilder:validation:Optional + DryRun bool `json:"dryRun,omitempty"` + // NewState maps commitmentUUID to the desired CommittedResourceSpec. // A nil value means the corresponding CommittedResource should be deleted. // +kubebuilder:validation:Optional @@ -106,7 +112,7 @@ type CommittedResourceChangeStatus struct { // +kubebuilder:validation:Optional UpsertedCommitments []corev1.ObjectReference `json:"upsertedCommitments,omitempty"` - // RollbackStartedAt is the time the CRC entered RollingBack. + // RollbackStartedAt is the time the CRC entered RollingBack or DryRunCleanup. // Used to measure RollbackTimeout from the correct starting point rather than relying on // the Progressing condition's LastTransitionTime, which is only updated on Status changes. // +kubebuilder:validation:Optional @@ -127,6 +133,7 @@ type CommittedResourceChangeStatus struct { // +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase" // +kubebuilder:printcolumn:name="AZ",type="string",JSONPath=".spec.availabilityZone" // +kubebuilder:printcolumn:name="Source",type="string",JSONPath=".spec.source" +// +kubebuilder:printcolumn:name="DryRun",type="boolean",JSONPath=".spec.dryRun",priority=1 // +kubebuilder:printcolumn:name="Progressing",type="string",JSONPath=".status.conditions[?(@.type=='Progressing')].status",priority=1 // +kubebuilder:printcolumn:name="Finished",type="string",JSONPath=".status.conditions[?(@.type=='Finished')].status",priority=1 diff --git a/docs/03-reservations-and-inventory/03-b-committed-resource-transaction-architecture.md b/docs/03-reservations-and-inventory/03-b-committed-resource-transaction-architecture.md index 3ec471b33..3e5827bd1 100644 --- a/docs/03-reservations-and-inventory/03-b-committed-resource-transaction-architecture.md +++ b/docs/03-reservations-and-inventory/03-b-committed-resource-transaction-architecture.md @@ -76,8 +76,11 @@ stateDiagram-v2 Allocating --> Superseded : newer CRC for same AZ took over AwaitingScheduling --> Accepted : all CRs Ready (observedGeneration matches) + AwaitingScheduling --> DryRunCleanup : all CRs Ready, but DryRun=true AwaitingScheduling --> RollingBack : CR rejected or timeout + DryRunCleanup --> Accepted : pre-state restored (would-accept) + RollingBack --> Rejected : pre-state restored (RollbackFailed on permanent failure) Accepted --> [*] diff --git a/helm/library/cortex/files/crds/cortex.cloud_committedresourcechanges.yaml b/helm/library/cortex/files/crds/cortex.cloud_committedresourcechanges.yaml index f75b21044..410278265 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_committedresourcechanges.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_committedresourcechanges.yaml @@ -26,6 +26,10 @@ spec: - jsonPath: .spec.source name: Source type: string + - jsonPath: .spec.dryRun + name: DryRun + priority: 1 + type: boolean - jsonPath: .status.conditions[?(@.type=='Progressing')].status name: Progressing priority: 1 @@ -74,6 +78,10 @@ spec: correlationID: description: CorrelationID is the request/correlation ID. type: string + dryRun: + description: DryRun controls whether the change request commits or + only probes schedulability. + type: boolean newState: additionalProperties: description: CommittedResourceSpec defines the desired state of @@ -256,7 +264,7 @@ spec: t == oldSelf) || self == oldSelf rollbackStartedAt: description: |- - RollbackStartedAt is the time the CRC entered RollingBack. + RollbackStartedAt is the time the CRC entered RollingBack or DryRunCleanup. Used to measure RollbackTimeout from the correct starting point rather than relying on the Progressing condition's LastTransitionTime, which is only updated on Status changes. format: date-time diff --git a/internal/scheduling/reservations/commitments/changes/reconciler.go b/internal/scheduling/reservations/commitments/changes/reconciler.go index 9eddf84a4..cf278577c 100644 --- a/internal/scheduling/reservations/commitments/changes/reconciler.go +++ b/internal/scheduling/reservations/commitments/changes/reconciler.go @@ -35,7 +35,7 @@ type CommittedResourceChangeReconciler struct { // AwaitingSchedulingTimeout is how long to wait in AwaitingScheduling before rolling back. // Zero (default) means no timeout. AwaitingSchedulingTimeout time.Duration - // RollbackTimeout is how long to attempt snapshot restore in RollingBack + // RollbackTimeout is how long to attempt snapshot restore in RollingBack/DryRunCleanup // before giving up and terminating with RollbackFailed. // Zero (default) means no timeout. RollbackTimeout time.Duration @@ -66,6 +66,8 @@ func (r *CommittedResourceChangeReconciler) Reconcile(ctx context.Context, req c return r.reconcileAwaitingScheduling(ctx, logger, &tx) case v1alpha1.ChangePhaseRollingBack: return r.reconcileRollingBack(ctx, logger, &tx) + case v1alpha1.ChangePhaseDryRunCleanup: + return r.reconcileDryRunCleanup(ctx, logger, &tx) case v1alpha1.ChangePhaseAccepted, v1alpha1.ChangePhaseRejected, v1alpha1.ChangePhaseSuperseded: return ctrl.Result{}, nil default: @@ -153,7 +155,7 @@ func (r *CommittedResourceChangeReconciler) reconcileAllocating(ctx context.Cont // reconcileAwaitingScheduling reads UpsertedCommitments and waits for placement settlement. // Gates on each CR's ready-condition observedGeneration matching the generation written in Allocating. -// All CRs accepted → Accepted. +// All CRs accepted → Accepted (or DryRunCleanup when DryRun=true). // Any CR rejected → RollingBack. // Not yet settled → wait for the CR watch to re-enqueue. func (r *CommittedResourceChangeReconciler) reconcileAwaitingScheduling(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange) (ctrl.Result, error) { @@ -219,6 +221,9 @@ func (r *CommittedResourceChangeReconciler) reconcileAwaitingScheduling(ctx cont return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } + if tx.Spec.DryRun { + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseDryRunCleanup, nil) + } return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseAccepted, nil) } @@ -233,13 +238,30 @@ func (r *CommittedResourceChangeReconciler) reconcileRollingBack(ctx context.Con if !allSettled { if r.RollbackTimeout > 0 && tx.Status.RollbackStartedAt != nil && time.Since(tx.Status.RollbackStartedAt.Time) > r.RollbackTimeout { logger.Info("rollback timed out, terminalizing", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "timeout", r.RollbackTimeout) - return r.terminateWithRollbackFailed(ctx, tx, "timed out restoring pre-change state") + return r.terminateWithRollbackFailed(ctx, tx, v1alpha1.ChangePhaseRejected, "timed out restoring pre-change state") } return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseRejected, nil) } +// reconcileDryRunCleanup restores pre-change state after a dry-run would-accept outcome. +// Re-uses snapshot rollback logic; terminal phase is Accepted (not Rejected). +func (r *CommittedResourceChangeReconciler) reconcileDryRunCleanup(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange) (ctrl.Result, error) { + allSettled, err := r.restoreFromSnapshot(ctx, logger, tx) + if err != nil { + return ctrl.Result{}, err + } + if !allSettled { + if r.RollbackTimeout > 0 && tx.Status.RollbackStartedAt != nil && time.Since(tx.Status.RollbackStartedAt.Time) > r.RollbackTimeout { + logger.Info("dry-run cleanup timed out, terminalizing", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "timeout", r.RollbackTimeout) + return r.terminateWithRollbackFailed(ctx, tx, v1alpha1.ChangePhaseAccepted, "timed out restoring pre-change state after dry run") + } + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseAccepted, nil) +} + // restoreFromSnapshot converges CommittedResources back to their pre-change state recorded in // tx.Status.SnapshotBefore. It is idempotent and safe to call on every requeue. // @@ -294,10 +316,11 @@ func (r *CommittedResourceChangeReconciler) restoreFromSnapshot(ctx context.Cont return true, nil } -// terminateWithRollbackFailed stamps Finished=True/RollbackFailed and Progressing=False/RetryExhausted -// and terminates the transaction as Rejected. -func (r *CommittedResourceChangeReconciler) terminateWithRollbackFailed(ctx context.Context, tx *v1alpha1.CommittedResourceChange, msg string) (ctrl.Result, error) { - tx.Status.Phase = v1alpha1.ChangePhaseRejected +// terminateWithRollbackFailed stamps Finished=True/RollbackFailed and Progressing=False/RetryExhausted. +// phase controls the terminal FSM phase: Rejected for a failed rollback, Accepted for a DryRunCleanup +// timeout (the scheduler accepted the change; only the post-probe cleanup stalled). +func (r *CommittedResourceChangeReconciler) terminateWithRollbackFailed(ctx context.Context, tx *v1alpha1.CommittedResourceChange, phase v1alpha1.ChangePhase, msg string) (ctrl.Result, error) { + tx.Status.Phase = phase tx.Status.Message = msg meta.SetStatusCondition(&tx.Status.Conditions, metav1.Condition{ Type: v1alpha1.ChangeConditionFinished, @@ -337,7 +360,7 @@ func (r *CommittedResourceChangeReconciler) advanceTo( if mutate != nil { mutate(&tx.Status) } - if phase == v1alpha1.ChangePhaseRollingBack && tx.Status.RollbackStartedAt == nil { + if (phase == v1alpha1.ChangePhaseRollingBack || phase == v1alpha1.ChangePhaseDryRunCleanup) && tx.Status.RollbackStartedAt == nil { now := metav1.Now() tx.Status.RollbackStartedAt = &now } diff --git a/internal/scheduling/reservations/commitments/changes/reconciler_test.go b/internal/scheduling/reservations/commitments/changes/reconciler_test.go index e3cdc4dcc..efbf96d71 100644 --- a/internal/scheduling/reservations/commitments/changes/reconciler_test.go +++ b/internal/scheduling/reservations/commitments/changes/reconciler_test.go @@ -544,6 +544,18 @@ func TestCRTReconciler_AwaitingScheduling(t *testing.T) { wantFinished: true, wantFinishedReason: string(v1alpha1.ChangePhaseAccepted), }, + { + name: "DryRun=true and all CRs accepted → DryRunCleanup", + txOpts: []func(*v1alpha1.CommittedResourceChange){ + func(tx *v1alpha1.CommittedResourceChange) { tx.Spec.DryRun = true }, + }, + setupCR: func(t *testing.T, c client.Client) { + setCRReady(t, c, crName, metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted, "") + }, + wantPhase: v1alpha1.ChangePhaseDryRunCleanup, + wantProgressingStatus: metav1.ConditionTrue, + wantProgressingReason: "Advancing", + }, { name: "CR rejected → RollingBack with rejection message", setupCR: func(t *testing.T, c client.Client) { @@ -1325,3 +1337,190 @@ func TestCRTReconciler_RollingBack_Idempotent(t *testing.T) { t.Errorf("phase: want Rejected, got %q", got) } } + +// ============================================================================ +// TestCRTReconciler_DryRunCleanup +// ============================================================================ + +// driveToDryRunCleanup drives a DryRun CRC to DryRunCleanup. +func driveToDryRunCleanup(t *testing.T, r *CommittedResourceChangeReconciler, c client.Client, txName, crName string) { + t.Helper() + driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseAwaitingScheduling) + setCRReady(t, c, crName, metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted, "") + driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseDryRunCleanup) +} + +func TestCRTReconciler_DryRunCleanup(t *testing.T) { + tests := []struct { + name string + preObjects []client.Object + txSpec v1alpha1.CommittedResourceChangeSpec + reconcilerOpt func(r *CommittedResourceChangeReconciler) + afterDryRunCleanup func(t *testing.T, c client.Client, txName string) + wantPhase v1alpha1.ChangePhase + wantFinishedReason string + wantProgressingStatus metav1.ConditionStatus + wantProgressingReason string + }{ + { + name: "nil snapshot: new CR deleted and dry-run completes as Accepted", + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + DryRun: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dry1": minimalCRSpec("az-1")}, + }, + wantPhase: v1alpha1.ChangePhaseAccepted, + wantFinishedReason: string(v1alpha1.ChangePhaseAccepted), + wantProgressingStatus: metav1.ConditionFalse, + wantProgressingReason: string(v1alpha1.ChangePhaseAccepted), + }, + { + name: "non-nil snapshot: existing CR spec restored and dry-run completes as Accepted", + preObjects: []client.Object{ + &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-dry2"}, + Spec: *minimalCRSpec("az-old"), + }, + }, + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + DryRun: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dry2": minimalCRSpec("az-new")}, + }, + afterDryRunCleanup: func(t *testing.T, c client.Client, _ string) { + setCRReady(t, c, "commitment-uuid-dry2", metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted, "") + }, + wantPhase: v1alpha1.ChangePhaseAccepted, + wantFinishedReason: string(v1alpha1.ChangePhaseAccepted), + wantProgressingStatus: metav1.ConditionFalse, + wantProgressingReason: string(v1alpha1.ChangePhaseAccepted), + }, + { + name: "CRs not yet settled: stays in DryRunCleanup", + preObjects: []client.Object{ + &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-dry3"}, + Spec: *minimalCRSpec("az-old"), + }, + }, + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + DryRun: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dry3": minimalCRSpec("az-new")}, + }, + // Stale the condition to simulate "CR controller has not yet processed the restored spec". + afterDryRunCleanup: func(t *testing.T, c client.Client, _ string) { + setCRReadyStaleGen(t, c, "commitment-uuid-dry3") + }, + wantPhase: v1alpha1.ChangePhaseDryRunCleanup, + wantProgressingStatus: metav1.ConditionTrue, + wantProgressingReason: "Advancing", + }, + { + name: "dry-run cleanup timeout: terminates with RollbackFailed", + preObjects: []client.Object{ + &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-dry4"}, + Spec: *minimalCRSpec("az-old"), + }, + }, + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + DryRun: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dry4": minimalCRSpec("az-new")}, + }, + reconcilerOpt: func(r *CommittedResourceChangeReconciler) { + r.RollbackTimeout = 50 * time.Millisecond + }, + afterDryRunCleanup: func(t *testing.T, c client.Client, txName string) { + setCRReadyStaleGen(t, c, "commitment-uuid-dry4") + backdateRollbackStartedAt(t, c, txName, 1*time.Second) + }, + wantPhase: v1alpha1.ChangePhaseAccepted, + wantFinishedReason: v1alpha1.ChangeReasonRollbackFailed, + wantProgressingStatus: metav1.ConditionFalse, + wantProgressingReason: v1alpha1.ChangeReasonRetryExhausted, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := testScheme(t) + txName := "tx-dry-" + tt.name[:5] + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: txName}, + Spec: tt.txSpec, + } + objs := append([]client.Object{tx}, tt.preObjects...) + c := testClient(t, scheme, objs...) + r := testReconciler(c, scheme) + if tt.reconcilerOpt != nil { + tt.reconcilerOpt(r) + } + + var crName string + for uuid := range tt.txSpec.NewState { + crName = commitmentName(uuid) + break + } + + driveToDryRunCleanup(t, r, c, txName, crName) + + if tt.afterDryRunCleanup != nil { + tt.afterDryRunCleanup(t, c, txName) + } + + doReconcile(t, r, txName) + + tx = getTx(t, c, txName) + if tx.Status.Phase != tt.wantPhase { + t.Errorf("phase: want %q, got %q", tt.wantPhase, tx.Status.Phase) + } + if tt.wantFinishedReason != "" { + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionFinished, metav1.ConditionTrue, tt.wantFinishedReason) + } + if tt.wantProgressingStatus != "" { + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionProgressing, tt.wantProgressingStatus, tt.wantProgressingReason) + } + }) + } +} + +// TestCRTReconciler_DryRun_WouldReject verifies that DryRun=true + rejection follows +// the RollingBack path and terminates as Rejected (not Accepted). +func TestCRTReconciler_DryRun_WouldReject(t *testing.T) { + const txName = "tx-dry-reject" + scheme := testScheme(t) + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: txName}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + DryRun: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dr": minimalCRSpec("az-1")}, + }, + } + c := testClient(t, scheme, tx) + r := testReconciler(c, scheme) + + driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseAwaitingScheduling) + // Reject placement → must go to RollingBack (not DryRunCleanup). + setCRReady(t, c, "commitment-uuid-dr", metav1.ConditionFalse, v1alpha1.CommittedResourceReasonRejected, "no capacity") + driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseRollingBack) + + // nil snapshot → delete CR → settle immediately → Rejected. + doReconcile(t, r, txName) + if got := getTx(t, c, txName).Status.Phase; got != v1alpha1.ChangePhaseRejected { + t.Errorf("DryRun would-reject: want Rejected, got %q", got) + } + assertCondition(t, getTx(t, c, txName).Status.Conditions, v1alpha1.ChangeConditionFinished, metav1.ConditionTrue, string(v1alpha1.ChangePhaseRejected)) +} From a3f2d9ab894ef0b53a16a4d207f6cddaca7e0d14 Mon Sep 17 00:00:00 2001 From: juliusclausnitzer Date: Thu, 12 Nov 2026 14:20:36 +0100 Subject: [PATCH 12/13] review fixxes Signed-off-by: juliusclausnitzer --- .../committed_resource_change_types.go | 16 +- api/v1alpha1/zz_generated.deepcopy.go | 4 - ...cortex.cloud_committedresourcechanges.yaml | 10 +- .../commitments/changes/reconciler.go | 197 +++---- .../commitments/changes/reconciler_test.go | 530 +++++++++--------- 5 files changed, 369 insertions(+), 388 deletions(-) diff --git a/api/v1alpha1/committed_resource_change_types.go b/api/v1alpha1/committed_resource_change_types.go index 6ca6bedf0..bb6cef9f1 100644 --- a/api/v1alpha1/committed_resource_change_types.go +++ b/api/v1alpha1/committed_resource_change_types.go @@ -48,6 +48,11 @@ const ( // ChangeConditionFinished is True once the change request has reached a terminal phase. ChangeConditionFinished = "Finished" + // ChangeConditionRestoring is True while the change request is restoring pre-change state + // (RollingBack or DryRunCleanup). Its LastTransitionTime marks when restore began and its + // Message carries the last restore error (e.g. a transient k8s API error). + ChangeConditionRestoring = "Restoring" + // Reasons for ChangeConditionProgressing=False (non-terminal problems). ChangeReasonStalled = "Stalled" ChangeReasonTimedOut = "TimedOut" @@ -60,6 +65,9 @@ const ( ChangeRequestReasonSuperseded = "Superseded" // ChangeReasonRollbackFailed is the Finished reason when rollback exhausted its timeout. ChangeReasonRollbackFailed = "RollbackFailed" + // ChangeReasonInvalidSpec is the Finished reason when a change request is rejected for a + // malformed spec (e.g. a dry run that would commit or mutate existing commitments). + ChangeReasonInvalidSpec = "InvalidSpec" ) // CommittedResourceChangeSpec defines the desired change for a CommittedResourceChange. @@ -112,13 +120,7 @@ type CommittedResourceChangeStatus struct { // +kubebuilder:validation:Optional UpsertedCommitments []corev1.ObjectReference `json:"upsertedCommitments,omitempty"` - // RollbackStartedAt is the time the CRC entered RollingBack or DryRunCleanup. - // Used to measure RollbackTimeout from the correct starting point rather than relying on - // the Progressing condition's LastTransitionTime, which is only updated on Status changes. - // +kubebuilder:validation:Optional - RollbackStartedAt *metav1.Time `json:"rollbackStartedAt,omitempty"` - - // Conditions holds Progressing and Finished conditions. + // Conditions holds Progressing, Finished and Restoring conditions. // +kubebuilder:validation:Optional Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 464ae3e9a..1c3957a1d 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -231,10 +231,6 @@ func (in *CommittedResourceChangeStatus) DeepCopyInto(out *CommittedResourceChan *out = make([]v1.ObjectReference, len(*in)) copy(*out, *in) } - if in.RollbackStartedAt != nil { - in, out := &in.RollbackStartedAt, &out.RollbackStartedAt - *out = (*in).DeepCopy() - } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]metav1.Condition, len(*in)) diff --git a/helm/library/cortex/files/crds/cortex.cloud_committedresourcechanges.yaml b/helm/library/cortex/files/crds/cortex.cloud_committedresourcechanges.yaml index 410278265..ef49c268f 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_committedresourcechanges.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_committedresourcechanges.yaml @@ -195,7 +195,8 @@ spec: of a CommittedResourceChange. properties: conditions: - description: Conditions holds Progressing and Finished conditions. + description: Conditions holds Progressing, Finished and Restoring + conditions. items: description: Condition contains details for one aspect of the current state of this API Resource. @@ -262,13 +263,6 @@ spec: - message: phase cannot transition out of a terminal state rule: oldSelf == '' || !['Accepted','Rejected','Superseded'].exists(t, t == oldSelf) || self == oldSelf - rollbackStartedAt: - description: |- - RollbackStartedAt is the time the CRC entered RollingBack or DryRunCleanup. - Used to measure RollbackTimeout from the correct starting point rather than relying on - the Progressing condition's LastTransitionTime, which is only updated on Status changes. - format: date-time - type: string snapshotBefore: additionalProperties: description: CommittedResourceSpec defines the desired state of diff --git a/internal/scheduling/reservations/commitments/changes/reconciler.go b/internal/scheduling/reservations/commitments/changes/reconciler.go index cf278577c..403ff687f 100644 --- a/internal/scheduling/reservations/commitments/changes/reconciler.go +++ b/internal/scheduling/reservations/commitments/changes/reconciler.go @@ -65,9 +65,9 @@ func (r *CommittedResourceChangeReconciler) Reconcile(ctx context.Context, req c case v1alpha1.ChangePhaseAwaitingScheduling: return r.reconcileAwaitingScheduling(ctx, logger, &tx) case v1alpha1.ChangePhaseRollingBack: - return r.reconcileRollingBack(ctx, logger, &tx) + return r.reconcileRestore(ctx, logger, &tx, v1alpha1.ChangePhaseRejected) case v1alpha1.ChangePhaseDryRunCleanup: - return r.reconcileDryRunCleanup(ctx, logger, &tx) + return r.reconcileRestore(ctx, logger, &tx, v1alpha1.ChangePhaseAccepted) case v1alpha1.ChangePhaseAccepted, v1alpha1.ChangePhaseRejected, v1alpha1.ChangePhaseSuperseded: return ctrl.Result{}, nil default: @@ -78,10 +78,19 @@ func (r *CommittedResourceChangeReconciler) Reconcile(ctx context.Context, req c // commitmentName returns the CommittedResource name for a given commitment UUID. func commitmentName(uuid string) string { return "commitment-" + uuid } -// reconcilePreparing snapshots pre-change CR specs and advances to Allocating. +// reconcilePreparing validates the spec, snapshots pre-change CR specs, and advances to Allocating. // Always snapshots regardless of AllowRejection — syncer-mode CRTs never roll back but the // snapshot is cheap and keeps this method free of branching. func (r *CommittedResourceChangeReconciler) reconcilePreparing(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange) (ctrl.Result, error) { + // Guard: a dry run must be rejectable, otherwise Allocating advances straight to Accepted + // and the probe would silently commit. + if tx.Spec.DryRun && !tx.Spec.AllowRejection { + logger.Info("rejecting change: dryRun requires allowRejection", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID) + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseRejected, v1alpha1.ChangeReasonInvalidSpec, func(s *v1alpha1.CommittedResourceChangeStatus) { + s.Message = "dryRun requires allowRejection" + }) + } + // Snapshot current CR specs for all keys in NewState so rollback can restore them. // nil entry = CR did not exist before = delete it on rollback. snapshot := make(map[string]*v1alpha1.CommittedResourceSpec, len(tx.Spec.NewState)) @@ -99,8 +108,22 @@ func (r *CommittedResourceChangeReconciler) reconcilePreparing(ctx context.Conte snapshot[uuid] = cr.Spec.DeepCopy() } + // Guard: a dry run must only probe new commitments. A non-nil snapshot entry means an + // existing CR would be mutated, which a dry run must never do. + if tx.Spec.DryRun { + for uuid, snapSpec := range snapshot { + if snapSpec != nil { + msg := "dryRun must not modify existing commitment " + uuid + logger.Info("rejecting change: "+msg, "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID) + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseRejected, v1alpha1.ChangeReasonInvalidSpec, func(s *v1alpha1.CommittedResourceChangeStatus) { + s.Message = msg + }) + } + } + } + logger.Info("starting transaction", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "newStateSize", len(tx.Spec.NewState), "allowRejection", tx.Spec.AllowRejection) - return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseAllocating, func(s *v1alpha1.CommittedResourceChangeStatus) { + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseAllocating, "", func(s *v1alpha1.CommittedResourceChangeStatus) { s.SnapshotBefore = snapshot }) } @@ -148,7 +171,7 @@ func (r *CommittedResourceChangeReconciler) reconcileAllocating(ctx context.Cont if !tx.Spec.AllowRejection { nextPhase = v1alpha1.ChangePhaseAccepted } - return r.advanceTo(ctx, tx, nextPhase, func(s *v1alpha1.CommittedResourceChangeStatus) { + return r.advanceTo(ctx, tx, nextPhase, "", func(s *v1alpha1.CommittedResourceChangeStatus) { s.UpsertedCommitments = upserted }) } @@ -189,7 +212,7 @@ func (r *CommittedResourceChangeReconciler) reconcileAwaitingScheduling(ctx cont // Fail-fast: start rollback immediately on first rejection rather than waiting for all CRs // to settle — this unblocks the AZ sooner and the first rejection message is sufficient for diagnosis. logger.Info("CR rejected, rolling back", "cr", ref.Name, "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "message", cond.Message) - return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseRollingBack, func(s *v1alpha1.CommittedResourceChangeStatus) { + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseRollingBack, "", func(s *v1alpha1.CommittedResourceChangeStatus) { s.Message = fmt.Sprintf("CR %s rejected: %s", ref.Name, cond.Message) }) default: @@ -204,7 +227,7 @@ func (r *CommittedResourceChangeReconciler) reconcileAwaitingScheduling(ctx cont progressingCond := meta.FindStatusCondition(tx.Status.Conditions, v1alpha1.ChangeConditionProgressing) if progressingCond != nil && time.Since(progressingCond.LastTransitionTime.Time) > r.AwaitingSchedulingTimeout { logger.Info("timed out waiting for scheduling, rolling back", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "timeout", r.AwaitingSchedulingTimeout) - return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseRollingBack, func(s *v1alpha1.CommittedResourceChangeStatus) { + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseRollingBack, "", func(s *v1alpha1.CommittedResourceChangeStatus) { s.Message = "timed out waiting for scheduling" meta.SetStatusCondition(&s.Conditions, metav1.Condition{ Type: v1alpha1.ChangeConditionProgressing, @@ -222,66 +245,61 @@ func (r *CommittedResourceChangeReconciler) reconcileAwaitingScheduling(ctx cont } if tx.Spec.DryRun { - return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseDryRunCleanup, nil) + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseDryRunCleanup, "", nil) } - return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseAccepted, nil) + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseAccepted, "", nil) } -// reconcileRollingBack converges affected CR specs back to the pre-change SnapshotBefore state. -// On restore complete: advances to Rejected. -// On permanent cleanup failure: advances to Rejected with reason RollbackFailed. -func (r *CommittedResourceChangeReconciler) reconcileRollingBack(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange) (ctrl.Result, error) { - allSettled, err := r.restoreFromSnapshot(ctx, logger, tx) - if err != nil { - return ctrl.Result{}, err - } - if !allSettled { - if r.RollbackTimeout > 0 && tx.Status.RollbackStartedAt != nil && time.Since(tx.Status.RollbackStartedAt.Time) > r.RollbackTimeout { - logger.Info("rollback timed out, terminalizing", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "timeout", r.RollbackTimeout) - return r.terminateWithRollbackFailed(ctx, tx, v1alpha1.ChangePhaseRejected, "timed out restoring pre-change state") +// reconcileRestore converges affected CR specs back to the pre-change SnapshotBefore state and +// advances to terminalPhase (Rejected for a rollback, Accepted for a dry-run cleanup). +// On a persistent restore failure past RollbackTimeout it terminates at terminalPhase with the +// Finished reason RollbackFailed; the Restoring condition's LastTransitionTime marks the start and +// its Message carries the last error. +func (r *CommittedResourceChangeReconciler) reconcileRestore(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange, terminalPhase v1alpha1.ChangePhase) (ctrl.Result, error) { + if err := r.restoreFromSnapshot(ctx, logger, tx); err != nil { + if r.RollbackTimeout > 0 { + if c := meta.FindStatusCondition(tx.Status.Conditions, v1alpha1.ChangeConditionRestoring); c != nil && time.Since(c.LastTransitionTime.Time) > r.RollbackTimeout { + logger.Info("restore timed out, terminalizing", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "timeout", r.RollbackTimeout, "terminalPhase", terminalPhase) + msg := fmt.Sprintf("timed out restoring pre-change state: %v", err) + return r.advanceTo(ctx, tx, terminalPhase, v1alpha1.ChangeReasonRollbackFailed, func(s *v1alpha1.CommittedResourceChangeStatus) { + s.Message = msg + }) + } } - return ctrl.Result{RequeueAfter: 5 * time.Second}, nil - } - return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseRejected, nil) -} - -// reconcileDryRunCleanup restores pre-change state after a dry-run would-accept outcome. -// Re-uses snapshot rollback logic; terminal phase is Accepted (not Rejected). -func (r *CommittedResourceChangeReconciler) reconcileDryRunCleanup(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange) (ctrl.Result, error) { - allSettled, err := r.restoreFromSnapshot(ctx, logger, tx) - if err != nil { - return ctrl.Result{}, err - } - if !allSettled { - if r.RollbackTimeout > 0 && tx.Status.RollbackStartedAt != nil && time.Since(tx.Status.RollbackStartedAt.Time) > r.RollbackTimeout { - logger.Info("dry-run cleanup timed out, terminalizing", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "timeout", r.RollbackTimeout) - return r.terminateWithRollbackFailed(ctx, tx, v1alpha1.ChangePhaseAccepted, "timed out restoring pre-change state after dry run") + // Record the last restore error on the Restoring condition (Status stays True so its + // LastTransitionTime remains the restore-start marker) and requeue with backoff. + meta.SetStatusCondition(&tx.Status.Conditions, metav1.Condition{ + Type: v1alpha1.ChangeConditionRestoring, + Status: metav1.ConditionTrue, + Reason: v1alpha1.ChangeReasonError, + Message: err.Error(), + ObservedGeneration: tx.Generation, + }) + if updErr := r.Status().Update(ctx, tx); updErr != nil { + return ctrl.Result{}, updErr } - return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + return ctrl.Result{}, err } - return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseAccepted, nil) + return r.advanceTo(ctx, tx, terminalPhase, "", nil) } // restoreFromSnapshot converges CommittedResources back to their pre-change state recorded in // tx.Status.SnapshotBefore. It is idempotent and safe to call on every requeue. // -// Pass 1 (write): for each snapshot entry, delete the CR (nil entry = CR was new) or restore -// its spec via CreateOrUpdate (non-nil entry = CR existed before). -// Pass 2 (settle): for each non-nil entry, verify the CR controller has processed the restored -// spec by checking cond.ObservedGeneration == cr.Generation on the Ready condition. +// For each snapshot entry it deletes the CR (nil entry = CR was new) or restores its spec via +// CreateOrUpdate (non-nil entry = CR existed before). Once the spec is written back the CR is in +// sync with the pre-change state, which is all rollback can guarantee: whether the CR controller +// then re-places reservations is outside this reconciler's control, so there is nothing to wait for. // -// Returns (true, nil) when all non-nil entries are settled. -// Returns (false, nil) when still waiting for settlement. -// Returns (false, err) on API errors that should cause a requeue. -func (r *CommittedResourceChangeReconciler) restoreFromSnapshot(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange) (bool, error) { +// Returns nil once all specs are written, or the API error (which should cause a requeue). +func (r *CommittedResourceChangeReconciler) restoreFromSnapshot(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange) error { for uuid, snapSpec := range tx.Status.SnapshotBefore { name := commitmentName(uuid) if snapSpec == nil { toDelete := &v1alpha1.CommittedResource{ObjectMeta: metav1.ObjectMeta{Name: name}} if err := r.Delete(ctx, toDelete); client.IgnoreNotFound(err) != nil { logger.Error(err, "rollback: failed to delete CR", "name", name, "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID) - r.markError(ctx, tx, fmt.Sprintf("rollback: failed to delete %s: %v", name, err)) - return false, err + return err } continue } @@ -291,61 +309,17 @@ func (r *CommittedResourceChangeReconciler) restoreFromSnapshot(ctx context.Cont return nil }); err != nil { logger.Error(err, "rollback: failed to restore CR spec", "name", name, "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID) - r.markError(ctx, tx, fmt.Sprintf("rollback: failed to restore %s: %v", name, err)) - return false, err + return err } } - - for uuid, snapSpec := range tx.Status.SnapshotBefore { - if snapSpec == nil { - continue - } - name := commitmentName(uuid) - var cr v1alpha1.CommittedResource - if err := r.Get(ctx, client.ObjectKey{Name: name}, &cr); err != nil { - if client.IgnoreNotFound(err) == nil { - return false, nil - } - return false, err - } - cond := meta.FindStatusCondition(cr.Status.Conditions, v1alpha1.CommittedResourceConditionReady) - if cond == nil || cond.ObservedGeneration != cr.Generation { - return false, nil - } - } - return true, nil -} - -// terminateWithRollbackFailed stamps Finished=True/RollbackFailed and Progressing=False/RetryExhausted. -// phase controls the terminal FSM phase: Rejected for a failed rollback, Accepted for a DryRunCleanup -// timeout (the scheduler accepted the change; only the post-probe cleanup stalled). -func (r *CommittedResourceChangeReconciler) terminateWithRollbackFailed(ctx context.Context, tx *v1alpha1.CommittedResourceChange, phase v1alpha1.ChangePhase, msg string) (ctrl.Result, error) { - tx.Status.Phase = phase - tx.Status.Message = msg - meta.SetStatusCondition(&tx.Status.Conditions, metav1.Condition{ - Type: v1alpha1.ChangeConditionFinished, - Status: metav1.ConditionTrue, - Reason: v1alpha1.ChangeReasonRollbackFailed, - Message: msg, - ObservedGeneration: tx.Generation, - }) - meta.SetStatusCondition(&tx.Status.Conditions, metav1.Condition{ - Type: v1alpha1.ChangeConditionProgressing, - Status: metav1.ConditionFalse, - Reason: v1alpha1.ChangeReasonRetryExhausted, - Message: msg, - ObservedGeneration: tx.Generation, - }) - if err := r.Status().Update(ctx, tx); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{}, nil + return nil } // advanceTo updates the transaction status to the given phase. -// mutate is called (when non-nil) to allow phase advance and other status mutations -// (e.g. UpsertedCommitments, SnapshotBefore) to be persisted atomically in one Update call. -// Progressing is set to False on terminal phases to reflect that no further work is expected. +// reason overrides the Finished/Progressing reason on terminal phases (e.g. RollbackFailed, +// InvalidSpec); when empty, string(phase) is used. mutate is called (when non-nil) to allow +// phase advance and other status mutations (e.g. UpsertedCommitments, SnapshotBefore, Message) +// to be persisted atomically in one Update call. // // Status().Update is used (not Patch) so that nil map values in SnapshotBefore and similar // fields are preserved — JSON merge patch semantics treat null map values as deletions. @@ -353,6 +327,7 @@ func (r *CommittedResourceChangeReconciler) advanceTo( ctx context.Context, tx *v1alpha1.CommittedResourceChange, phase v1alpha1.ChangePhase, + reason string, mutate func(*v1alpha1.CommittedResourceChangeStatus), ) (ctrl.Result, error) { @@ -360,22 +335,34 @@ func (r *CommittedResourceChangeReconciler) advanceTo( if mutate != nil { mutate(&tx.Status) } - if (phase == v1alpha1.ChangePhaseRollingBack || phase == v1alpha1.ChangePhaseDryRunCleanup) && tx.Status.RollbackStartedAt == nil { - now := metav1.Now() - tx.Status.RollbackStartedAt = &now + // Stamp Restoring=True on entry to a restore phase. Its Status stays True for the duration, + // so LastTransitionTime is a stable "restore started" marker used to measure RollbackTimeout. + if phase == v1alpha1.ChangePhaseRollingBack || phase == v1alpha1.ChangePhaseDryRunCleanup { + meta.SetStatusCondition(&tx.Status.Conditions, metav1.Condition{ + Type: v1alpha1.ChangeConditionRestoring, + Status: metav1.ConditionTrue, + Reason: "Restoring", + Message: tx.Status.Message, + ObservedGeneration: tx.Generation, + }) } progressingStatus, progressingReason := metav1.ConditionTrue, "Advancing" if tx.IsTerminal() { - progressingStatus, progressingReason = metav1.ConditionFalse, string(phase) + finishedReason := reason + if finishedReason == "" { + finishedReason = string(phase) + } + progressingStatus, progressingReason = metav1.ConditionFalse, finishedReason meta.SetStatusCondition(&tx.Status.Conditions, metav1.Condition{ Type: v1alpha1.ChangeConditionFinished, Status: metav1.ConditionTrue, - Reason: string(phase), + Reason: finishedReason, + Message: tx.Status.Message, ObservedGeneration: tx.Generation, }) } // Preserve a Progressing=False condition already stamped by the mutate func (e.g. TimedOut). - // Terminal phases always overwrite to ensure Progressing=False/. + // Terminal phases always overwrite to ensure Progressing=False/. existingProgressing := meta.FindStatusCondition(tx.Status.Conditions, v1alpha1.ChangeConditionProgressing) if tx.IsTerminal() || existingProgressing == nil || existingProgressing.Status != metav1.ConditionFalse { meta.SetStatusCondition(&tx.Status.Conditions, metav1.Condition{ diff --git a/internal/scheduling/reservations/commitments/changes/reconciler_test.go b/internal/scheduling/reservations/commitments/changes/reconciler_test.go index efbf96d71..5eaf3850b 100644 --- a/internal/scheduling/reservations/commitments/changes/reconciler_test.go +++ b/internal/scheduling/reservations/commitments/changes/reconciler_test.go @@ -5,10 +5,12 @@ package changes import ( "context" + "errors" "strings" "testing" "time" + apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -17,6 +19,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "github.com/cobaltcore-dev/cortex/api/v1alpha1" ) @@ -36,6 +39,22 @@ func testScheme(t *testing.T) *runtime.Scheme { return s } +// crcByUpsertedCRIndexFunc indexes CRCs by the names of the CRs they upserted (test copy of the +// production field index) so the fake client can serve crToChangeRequest lookups. +func crcByUpsertedCRIndexFunc(obj client.Object) []string { + tx, ok := obj.(*v1alpha1.CommittedResourceChange) + if !ok { + return nil + } + names := make([]string, 0, len(tx.Status.UpsertedCommitments)) + for _, ref := range tx.Status.UpsertedCommitments { + if ref.Name != "" { + names = append(names, ref.Name) + } + } + return names +} + func testClient(t *testing.T, scheme *runtime.Scheme, objects ...client.Object) client.Client { t.Helper() return fake.NewClientBuilder(). @@ -45,18 +64,49 @@ func testClient(t *testing.T, scheme *runtime.Scheme, objects ...client.Object) &v1alpha1.CommittedResourceChange{}, &v1alpha1.CommittedResource{}, ). - WithIndex(&v1alpha1.CommittedResourceChange{}, idxCRCByUpsertedCR, func(obj client.Object) []string { - tx, ok := obj.(*v1alpha1.CommittedResourceChange) - if !ok { - return nil + WithIndex(&v1alpha1.CommittedResourceChange{}, idxCRCByUpsertedCR, crcByUpsertedCRIndexFunc). + Build() +} + +// testClientFailingCRWrites builds a client that returns an error on CommittedResource +// Delete/Update/Create while *fail is true, so tests can force restoreFromSnapshot to fail. +func testClientFailingCRWrites(t *testing.T, scheme *runtime.Scheme, fail *bool, objects ...client.Object) client.Client { + t.Helper() + failCR := func(obj client.Object) error { + if *fail { + if _, ok := obj.(*v1alpha1.CommittedResource); ok { + return apierrors.NewInternalError(errors.New("injected CR write failure")) } - names := make([]string, 0, len(tx.Status.UpsertedCommitments)) - for _, ref := range tx.Status.UpsertedCommitments { - if ref.Name != "" { - names = append(names, ref.Name) + } + return nil + } + return fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objects...). + WithStatusSubresource( + &v1alpha1.CommittedResourceChange{}, + &v1alpha1.CommittedResource{}, + ). + WithIndex(&v1alpha1.CommittedResourceChange{}, idxCRCByUpsertedCR, crcByUpsertedCRIndexFunc). + WithInterceptorFuncs(interceptor.Funcs{ + Delete: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if err := failCR(obj); err != nil { + return err } - } - return names + return cl.Delete(ctx, obj, opts...) + }, + Update: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + if err := failCR(obj); err != nil { + return err + } + return cl.Update(ctx, obj, opts...) + }, + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if err := failCR(obj); err != nil { + return err + } + return cl.Create(ctx, obj, opts...) + }, }). Build() } @@ -158,15 +208,19 @@ func backdateProgressingCondition(t *testing.T, c client.Client, txName string, } } -// backdateRollbackStartedAt sets RollbackStartedAt to simulate a rollback that has been running -// for the given duration. Used to trigger the RollbackTimeout check in tests. -func backdateRollbackStartedAt(t *testing.T, c client.Client, txName string, age time.Duration) { +// backdateRestoringCondition sets the Restoring condition's LastTransitionTime to the past, +// simulating a restore that has been retrying for the given duration (to trigger RollbackTimeout). +func backdateRestoringCondition(t *testing.T, c client.Client, txName string, age time.Duration) { t.Helper() tx := getTx(t, c, txName) - past := metav1.NewTime(time.Now().Add(-age)) - tx.Status.RollbackStartedAt = &past + for i, cond := range tx.Status.Conditions { + if cond.Type == v1alpha1.ChangeConditionRestoring { + tx.Status.Conditions[i].LastTransitionTime = metav1.NewTime(time.Now().Add(-age)) + break + } + } if err := c.Status().Update(testCtx, tx); err != nil { - t.Fatalf("backdate RollbackStartedAt on %q: %v", txName, err) + t.Fatalf("backdate Restoring condition on %q: %v", txName, err) } } @@ -1107,35 +1161,27 @@ func driveToRollingBack(t *testing.T, r *CommittedResourceChangeReconciler, c cl func TestCRTReconciler_RollingBack(t *testing.T) { tests := []struct { - name string - preObjects []client.Object - txSpec v1alpha1.CommittedResourceChangeSpec - reconcilerOpt func(r *CommittedResourceChangeReconciler) - afterRollingBack func(t *testing.T, c client.Client, txName string) - wantPhase v1alpha1.ChangePhase - wantFinishedReason string - wantProgressingStatus metav1.ConditionStatus - wantProgressingReason string - // crName is derived as commitment- from txSpec.NewState (single-entry tests). - // For multi-CR tests the check is done inline. + name string + preObjects []client.Object + txSpec v1alpha1.CommittedResourceChangeSpec + // wantCRSpecAZ, when set, asserts the restored CR's spec.AvailabilityZone. + wantRestoredCR string + wantCRSpecAZ string + // wantCRAbsent, when set, asserts the CR was deleted by rollback. + wantCRAbsent string }{ { - name: "nil snapshot: new CR is deleted and rollback completes", + name: "nil snapshot: new CR is deleted and rollback completes as Rejected", txSpec: v1alpha1.CommittedResourceChangeSpec{ AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, AllowRejection: true, NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-rb1": minimalCRSpec("az-1")}, }, - // SnapshotBefore[uuid-rb1] = nil (CR did not exist before). - // After rollback, the CR must be deleted; no settlement gate. - wantPhase: v1alpha1.ChangePhaseRejected, - wantFinishedReason: string(v1alpha1.ChangePhaseRejected), - wantProgressingStatus: metav1.ConditionFalse, - wantProgressingReason: string(v1alpha1.ChangePhaseRejected), + wantCRAbsent: "commitment-uuid-rb1", }, { - name: "non-nil snapshot: existing CR spec is restored and rollback completes", + name: "non-nil snapshot: existing CR spec is restored and rollback completes as Rejected", preObjects: []client.Object{ &v1alpha1.CommittedResource{ ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-rb2"}, @@ -1148,66 +1194,8 @@ func TestCRTReconciler_RollingBack(t *testing.T) { AllowRejection: true, NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-rb2": minimalCRSpec("az-new")}, }, - // SnapshotBefore[uuid-rb2] = minimalCRSpec("az-old"). - // After rollback, commitment-uuid-rb2 must have spec az-old. - // Simulate CR controller settling the restored spec. - afterRollingBack: func(t *testing.T, c client.Client, _ string) { - setCRReady(t, c, "commitment-uuid-rb2", metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted, "") - }, - wantPhase: v1alpha1.ChangePhaseRejected, - wantFinishedReason: string(v1alpha1.ChangePhaseRejected), - wantProgressingStatus: metav1.ConditionFalse, - wantProgressingReason: string(v1alpha1.ChangePhaseRejected), - }, - { - name: "CRs not yet settled: stays in RollingBack", - preObjects: []client.Object{ - &v1alpha1.CommittedResource{ - ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-rb3"}, - Spec: *minimalCRSpec("az-old"), - }, - }, - txSpec: v1alpha1.CommittedResourceChangeSpec{ - AvailabilityZone: "az-1", - Source: v1alpha1.TxSourceAPI, - AllowRejection: true, - NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-rb3": minimalCRSpec("az-new")}, - }, - // The fake client does not increment Generation on spec updates, so we must explicitly - // stale the condition to simulate "CR controller has not yet processed the restored spec". - afterRollingBack: func(t *testing.T, c client.Client, _ string) { - setCRReadyStaleGen(t, c, "commitment-uuid-rb3") - }, - wantPhase: v1alpha1.ChangePhaseRollingBack, - wantProgressingStatus: metav1.ConditionTrue, - wantProgressingReason: "Advancing", - }, - { - name: "rollback timeout: terminates with RollbackFailed", - preObjects: []client.Object{ - &v1alpha1.CommittedResource{ - ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-rb4"}, - Spec: *minimalCRSpec("az-old"), - }, - }, - txSpec: v1alpha1.CommittedResourceChangeSpec{ - AvailabilityZone: "az-1", - Source: v1alpha1.TxSourceAPI, - AllowRejection: true, - NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-rb4": minimalCRSpec("az-new")}, - }, - reconcilerOpt: func(r *CommittedResourceChangeReconciler) { - r.RollbackTimeout = 50 * time.Millisecond - }, - // Stale the CR condition AND backdate RollbackStartedAt to trigger the timeout. - afterRollingBack: func(t *testing.T, c client.Client, txName string) { - setCRReadyStaleGen(t, c, "commitment-uuid-rb4") - backdateRollbackStartedAt(t, c, txName, 1*time.Second) - }, - wantPhase: v1alpha1.ChangePhaseRejected, - wantFinishedReason: v1alpha1.ChangeReasonRollbackFailed, - wantProgressingStatus: metav1.ConditionFalse, - wantProgressingReason: v1alpha1.ChangeReasonRetryExhausted, + wantRestoredCR: "commitment-uuid-rb2", + wantCRSpecAZ: "az-old", }, } @@ -1222,11 +1210,7 @@ func TestCRTReconciler_RollingBack(t *testing.T) { objs := append([]client.Object{tx}, tt.preObjects...) c := testClient(t, scheme, objs...) r := testReconciler(c, scheme) - if tt.reconcilerOpt != nil { - tt.reconcilerOpt(r) - } - // Pick the first UUID for the rejection trigger (single-CR tests). var crName string for uuid := range tt.txSpec.NewState { crName = commitmentName(uuid) @@ -1235,28 +1219,31 @@ func TestCRTReconciler_RollingBack(t *testing.T) { driveToRollingBack(t, r, c, txName, crName) - if tt.afterRollingBack != nil { - tt.afterRollingBack(t, c, txName) - } - + // Restore completes in a single reconcile: specs are written back, no settlement wait. doReconcile(t, r, txName) tx = getTx(t, c, txName) - if tx.Status.Phase != tt.wantPhase { - t.Errorf("phase: want %q, got %q", tt.wantPhase, tx.Status.Phase) + if tx.Status.Phase != v1alpha1.ChangePhaseRejected { + t.Errorf("phase: want Rejected, got %q", tx.Status.Phase) } - if tt.wantFinishedReason != "" { - assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionFinished, metav1.ConditionTrue, tt.wantFinishedReason) + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionFinished, metav1.ConditionTrue, string(v1alpha1.ChangePhaseRejected)) + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionProgressing, metav1.ConditionFalse, string(v1alpha1.ChangePhaseRejected)) + + if tt.wantCRAbsent != "" && crExists(t, c, tt.wantCRAbsent) { + t.Errorf("CR %q should be deleted by rollback but exists", tt.wantCRAbsent) } - if tt.wantProgressingStatus != "" { - assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionProgressing, tt.wantProgressingStatus, tt.wantProgressingReason) + if tt.wantRestoredCR != "" { + cr := getCR(t, c, tt.wantRestoredCR) + if cr.Spec.AvailabilityZone != tt.wantCRSpecAZ { + t.Errorf("restored CR %q AZ: want %q, got %q", tt.wantRestoredCR, tt.wantCRSpecAZ, cr.Spec.AvailabilityZone) + } } }) } } -// TestCRTReconciler_RollingBack_MultiCR verifies that all non-nil SnapshotBefore entries must -// have settled (observedGeneration == generation) before the CRC advances to Rejected. +// TestCRTReconciler_RollingBack_MultiCR verifies that a multi-CR reject rolls every affected CR +// back to its snapshot in a single reconcile. func TestCRTReconciler_RollingBack_MultiCR(t *testing.T) { const txName = "tx-rb-multi" scheme := testScheme(t) @@ -1272,7 +1259,7 @@ func TestCRTReconciler_RollingBack_MultiCR(t *testing.T) { }, }, } - // Both CRs pre-exist so their snapshots are non-nil; rollback must restore both. + // Both CRs pre-exist with az-old so their snapshots are non-nil; rollback must restore both. preObjects := []client.Object{ &v1alpha1.CommittedResource{ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-m1"}, Spec: *minimalCRSpec("az-old")}, &v1alpha1.CommittedResource{ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-m2"}, Spec: *minimalCRSpec("az-old")}, @@ -1285,34 +1272,21 @@ func TestCRTReconciler_RollingBack_MultiCR(t *testing.T) { setCRReady(t, c, "commitment-uuid-m1", metav1.ConditionFalse, v1alpha1.CommittedResourceReasonRejected, "no capacity") driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseRollingBack) - // Stale both CRs to simulate the CR controller not having processed the restored spec yet. - // (The fake client does not increment Generation on spec writes.) - setCRReadyStaleGen(t, c, "commitment-uuid-m1") - setCRReadyStaleGen(t, c, "commitment-uuid-m2") - - // Neither CR settled — must stay in RollingBack. doReconcile(t, r, txName) - if got := getTx(t, c, txName).Status.Phase; got != v1alpha1.ChangePhaseRollingBack { - t.Fatalf("after first rollback reconcile: want RollingBack, got %q", got) - } - // Settle only the first CR — still waiting on the second. - setCRReady(t, c, "commitment-uuid-m1", metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted, "") - doReconcile(t, r, txName) - if got := getTx(t, c, txName).Status.Phase; got != v1alpha1.ChangePhaseRollingBack { - t.Fatalf("after one CR settled: want RollingBack, got %q", got) - } - - // Settle the second CR — now all restored. - setCRReady(t, c, "commitment-uuid-m2", metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted, "") - doReconcile(t, r, txName) if got := getTx(t, c, txName).Status.Phase; got != v1alpha1.ChangePhaseRejected { - t.Errorf("after all CRs settled: want Rejected, got %q", got) + t.Fatalf("phase: want Rejected, got %q", got) + } + // Both CRs must have their az-old spec restored. + for _, name := range []string{"commitment-uuid-m1", "commitment-uuid-m2"} { + if got := getCR(t, c, name).Spec.AvailabilityZone; got != "az-old" { + t.Errorf("CR %q AZ: want az-old (restored), got %q", name, got) + } } } // TestCRTReconciler_RollingBack_Idempotent verifies that re-running RollingBack multiple -// times after restoration is safe and produces a stable Rejected terminal state. +// times is safe and produces a stable Rejected terminal state. func TestCRTReconciler_RollingBack_Idempotent(t *testing.T) { const txName = "tx-rb-idem" scheme := testScheme(t) @@ -1328,9 +1302,7 @@ func TestCRTReconciler_RollingBack_Idempotent(t *testing.T) { c := testClient(t, scheme, tx) r := testReconciler(c, scheme) - // nil snapshot entry — CR was new, rollback deletes it. No settlement gate needed. driveToRollingBack(t, r, c, txName, "commitment-uuid-idem-rb") - reconcileN(t, r, txName, 3) if got := getTx(t, c, txName).Status.Phase; got != v1alpha1.ChangePhaseRejected { @@ -1338,6 +1310,77 @@ func TestCRTReconciler_RollingBack_Idempotent(t *testing.T) { } } +// TestCRTReconciler_Restore_Timeout verifies that when restore keeps failing past RollbackTimeout, +// the CRC terminates with Finished reason RollbackFailed — at Rejected for a rollback and at +// Accepted for a dry-run cleanup (the schedulability verdict is preserved). +func TestCRTReconciler_Restore_Timeout(t *testing.T) { + tests := []struct { + name string + dryRun bool + driveToPhase v1alpha1.ChangePhase + wantPhase v1alpha1.ChangePhase + rejectCR bool // reject the CR to reach RollingBack; else accept to reach DryRunCleanup + }{ + { + name: "rollback restore times out → Rejected/RollbackFailed", + dryRun: false, + driveToPhase: v1alpha1.ChangePhaseRollingBack, + wantPhase: v1alpha1.ChangePhaseRejected, + rejectCR: true, + }, + { + name: "dry-run cleanup restore times out → Accepted/RollbackFailed", + dryRun: true, + driveToPhase: v1alpha1.ChangePhaseDryRunCleanup, + wantPhase: v1alpha1.ChangePhaseAccepted, + rejectCR: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := testScheme(t) + txName := "tx-to-" + tt.name[:5] + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: txName}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + DryRun: tt.dryRun, + // New CR (nil snapshot) → rollback deletes it; the failing client makes Delete error. + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-to": minimalCRSpec("az-1")}, + }, + } + fail := false + c := testClientFailingCRWrites(t, scheme, &fail, tx) + r := testReconciler(c, scheme) + r.RollbackTimeout = 50 * time.Millisecond + + driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseAwaitingScheduling) + if tt.rejectCR { + setCRReady(t, c, "commitment-uuid-to", metav1.ConditionFalse, v1alpha1.CommittedResourceReasonRejected, "no capacity") + } else { + setCRReady(t, c, "commitment-uuid-to", metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted, "") + } + driveToPhase(t, r, c, txName, tt.driveToPhase) + + // Now make restore fail and backdate the Restoring condition past the timeout. + fail = true + backdateRestoringCondition(t, c, txName, 1*time.Second) + + doReconcile(t, r, txName) + + tx = getTx(t, c, txName) + if tx.Status.Phase != tt.wantPhase { + t.Errorf("phase: want %q, got %q", tt.wantPhase, tx.Status.Phase) + } + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionFinished, metav1.ConditionTrue, v1alpha1.ChangeReasonRollbackFailed) + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionProgressing, metav1.ConditionFalse, v1alpha1.ChangeReasonRollbackFailed) + }) + } +} + // ============================================================================ // TestCRTReconciler_DryRunCleanup // ============================================================================ @@ -1351,82 +1394,92 @@ func driveToDryRunCleanup(t *testing.T, r *CommittedResourceChangeReconciler, c } func TestCRTReconciler_DryRunCleanup(t *testing.T) { - tests := []struct { - name string - preObjects []client.Object - txSpec v1alpha1.CommittedResourceChangeSpec - reconcilerOpt func(r *CommittedResourceChangeReconciler) - afterDryRunCleanup func(t *testing.T, c client.Client, txName string) - wantPhase v1alpha1.ChangePhase - wantFinishedReason string - wantProgressingStatus metav1.ConditionStatus - wantProgressingReason string - }{ - { - name: "nil snapshot: new CR deleted and dry-run completes as Accepted", - txSpec: v1alpha1.CommittedResourceChangeSpec{ - AvailabilityZone: "az-1", - Source: v1alpha1.TxSourceAPI, - AllowRejection: true, - DryRun: true, - NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dry1": minimalCRSpec("az-1")}, - }, - wantPhase: v1alpha1.ChangePhaseAccepted, - wantFinishedReason: string(v1alpha1.ChangePhaseAccepted), - wantProgressingStatus: metav1.ConditionFalse, - wantProgressingReason: string(v1alpha1.ChangePhaseAccepted), + // Dry-run requires a net-new commitment (nil snapshot); an existing CR is rejected by a guard, + // so every DryRunCleanup case deletes a freshly-created CR and terminates as Accepted. + const txName = "tx-dry-cleanup" + scheme := testScheme(t) + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: txName}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + DryRun: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dry1": minimalCRSpec("az-1")}, }, - { - name: "non-nil snapshot: existing CR spec restored and dry-run completes as Accepted", - preObjects: []client.Object{ - &v1alpha1.CommittedResource{ - ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-dry2"}, - Spec: *minimalCRSpec("az-old"), - }, - }, - txSpec: v1alpha1.CommittedResourceChangeSpec{ - AvailabilityZone: "az-1", - Source: v1alpha1.TxSourceAPI, - AllowRejection: true, - DryRun: true, - NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dry2": minimalCRSpec("az-new")}, - }, - afterDryRunCleanup: func(t *testing.T, c client.Client, _ string) { - setCRReady(t, c, "commitment-uuid-dry2", metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted, "") - }, - wantPhase: v1alpha1.ChangePhaseAccepted, - wantFinishedReason: string(v1alpha1.ChangePhaseAccepted), - wantProgressingStatus: metav1.ConditionFalse, - wantProgressingReason: string(v1alpha1.ChangePhaseAccepted), + } + c := testClient(t, scheme, tx) + r := testReconciler(c, scheme) + + driveToDryRunCleanup(t, r, c, txName, "commitment-uuid-dry1") + + // Restore completes in a single reconcile; the probe CR is deleted, verdict preserved as Accepted. + doReconcile(t, r, txName) + + tx = getTx(t, c, txName) + if tx.Status.Phase != v1alpha1.ChangePhaseAccepted { + t.Errorf("phase: want Accepted, got %q", tx.Status.Phase) + } + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionFinished, metav1.ConditionTrue, string(v1alpha1.ChangePhaseAccepted)) + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionProgressing, metav1.ConditionFalse, string(v1alpha1.ChangePhaseAccepted)) + if crExists(t, c, "commitment-uuid-dry1") { + t.Errorf("dry-run probe CR should be deleted on cleanup but exists") + } +} + +// TestCRTReconciler_DryRun_WouldReject verifies that DryRun=true + rejection follows the +// RollingBack path and terminates as Rejected (not Accepted). +func TestCRTReconciler_DryRun_WouldReject(t *testing.T) { + const txName = "tx-dry-reject" + scheme := testScheme(t) + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: txName}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + DryRun: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dr": minimalCRSpec("az-1")}, }, + } + c := testClient(t, scheme, tx) + r := testReconciler(c, scheme) + + driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseAwaitingScheduling) + setCRReady(t, c, "commitment-uuid-dr", metav1.ConditionFalse, v1alpha1.CommittedResourceReasonRejected, "no capacity") + driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseRollingBack) + + doReconcile(t, r, txName) + if got := getTx(t, c, txName).Status.Phase; got != v1alpha1.ChangePhaseRejected { + t.Errorf("DryRun would-reject: want Rejected, got %q", got) + } + assertCondition(t, getTx(t, c, txName).Status.Conditions, v1alpha1.ChangeConditionFinished, metav1.ConditionTrue, string(v1alpha1.ChangePhaseRejected)) +} + +// TestCRTReconciler_DryRunGuards verifies the spec guards reject invalid dry-run requests. +func TestCRTReconciler_DryRunGuards(t *testing.T) { + tests := []struct { + name string + preObjects []client.Object + txSpec v1alpha1.CommittedResourceChangeSpec + wantMessage string + }{ { - name: "CRs not yet settled: stays in DryRunCleanup", - preObjects: []client.Object{ - &v1alpha1.CommittedResource{ - ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-dry3"}, - Spec: *minimalCRSpec("az-old"), - }, - }, + name: "dryRun without allowRejection is rejected", txSpec: v1alpha1.CommittedResourceChangeSpec{ AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, - AllowRejection: true, + AllowRejection: false, DryRun: true, - NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dry3": minimalCRSpec("az-new")}, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-g1": minimalCRSpec("az-1")}, }, - // Stale the condition to simulate "CR controller has not yet processed the restored spec". - afterDryRunCleanup: func(t *testing.T, c client.Client, _ string) { - setCRReadyStaleGen(t, c, "commitment-uuid-dry3") - }, - wantPhase: v1alpha1.ChangePhaseDryRunCleanup, - wantProgressingStatus: metav1.ConditionTrue, - wantProgressingReason: "Advancing", + wantMessage: "dryRun requires allowRejection", }, { - name: "dry-run cleanup timeout: terminates with RollbackFailed", + name: "dryRun modifying an existing commitment is rejected", preObjects: []client.Object{ &v1alpha1.CommittedResource{ - ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-dry4"}, + ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-g2"}, Spec: *minimalCRSpec("az-old"), }, }, @@ -1435,26 +1488,16 @@ func TestCRTReconciler_DryRunCleanup(t *testing.T) { Source: v1alpha1.TxSourceAPI, AllowRejection: true, DryRun: true, - NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dry4": minimalCRSpec("az-new")}, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-g2": minimalCRSpec("az-new")}, }, - reconcilerOpt: func(r *CommittedResourceChangeReconciler) { - r.RollbackTimeout = 50 * time.Millisecond - }, - afterDryRunCleanup: func(t *testing.T, c client.Client, txName string) { - setCRReadyStaleGen(t, c, "commitment-uuid-dry4") - backdateRollbackStartedAt(t, c, txName, 1*time.Second) - }, - wantPhase: v1alpha1.ChangePhaseAccepted, - wantFinishedReason: v1alpha1.ChangeReasonRollbackFailed, - wantProgressingStatus: metav1.ConditionFalse, - wantProgressingReason: v1alpha1.ChangeReasonRetryExhausted, + wantMessage: "must not modify existing commitment", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { scheme := testScheme(t) - txName := "tx-dry-" + tt.name[:5] + txName := "tx-guard-" + tt.name[:5] tx := &v1alpha1.CommittedResourceChange{ ObjectMeta: metav1.ObjectMeta{Name: txName}, Spec: tt.txSpec, @@ -1462,65 +1505,24 @@ func TestCRTReconciler_DryRunCleanup(t *testing.T) { objs := append([]client.Object{tx}, tt.preObjects...) c := testClient(t, scheme, objs...) r := testReconciler(c, scheme) - if tt.reconcilerOpt != nil { - tt.reconcilerOpt(r) - } - - var crName string - for uuid := range tt.txSpec.NewState { - crName = commitmentName(uuid) - break - } - - driveToDryRunCleanup(t, r, c, txName, crName) - - if tt.afterDryRunCleanup != nil { - tt.afterDryRunCleanup(t, c, txName) - } doReconcile(t, r, txName) tx = getTx(t, c, txName) - if tx.Status.Phase != tt.wantPhase { - t.Errorf("phase: want %q, got %q", tt.wantPhase, tx.Status.Phase) + if tx.Status.Phase != v1alpha1.ChangePhaseRejected { + t.Errorf("phase: want Rejected, got %q", tx.Status.Phase) } - if tt.wantFinishedReason != "" { - assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionFinished, metav1.ConditionTrue, tt.wantFinishedReason) + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionFinished, metav1.ConditionTrue, v1alpha1.ChangeReasonInvalidSpec) + if !strings.Contains(tx.Status.Message, tt.wantMessage) { + t.Errorf("message: want substring %q in %q", tt.wantMessage, tx.Status.Message) } - if tt.wantProgressingStatus != "" { - assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionProgressing, tt.wantProgressingStatus, tt.wantProgressingReason) + // A rejected dry-run must not have created or mutated the target CR beyond the guard point. + if len(tt.preObjects) > 0 { + // existing CR must be untouched (still az-old) + if got := getCR(t, c, "commitment-uuid-g2").Spec.AvailabilityZone; got != "az-old" { + t.Errorf("existing CR must be untouched: AZ want az-old, got %q", got) + } } }) } } - -// TestCRTReconciler_DryRun_WouldReject verifies that DryRun=true + rejection follows -// the RollingBack path and terminates as Rejected (not Accepted). -func TestCRTReconciler_DryRun_WouldReject(t *testing.T) { - const txName = "tx-dry-reject" - scheme := testScheme(t) - tx := &v1alpha1.CommittedResourceChange{ - ObjectMeta: metav1.ObjectMeta{Name: txName}, - Spec: v1alpha1.CommittedResourceChangeSpec{ - AvailabilityZone: "az-1", - Source: v1alpha1.TxSourceAPI, - AllowRejection: true, - DryRun: true, - NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dr": minimalCRSpec("az-1")}, - }, - } - c := testClient(t, scheme, tx) - r := testReconciler(c, scheme) - - driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseAwaitingScheduling) - // Reject placement → must go to RollingBack (not DryRunCleanup). - setCRReady(t, c, "commitment-uuid-dr", metav1.ConditionFalse, v1alpha1.CommittedResourceReasonRejected, "no capacity") - driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseRollingBack) - - // nil snapshot → delete CR → settle immediately → Rejected. - doReconcile(t, r, txName) - if got := getTx(t, c, txName).Status.Phase; got != v1alpha1.ChangePhaseRejected { - t.Errorf("DryRun would-reject: want Rejected, got %q", got) - } - assertCondition(t, getTx(t, c, txName).Status.Conditions, v1alpha1.ChangeConditionFinished, metav1.ConditionTrue, string(v1alpha1.ChangePhaseRejected)) -} From 3d0808c513bbf4da204d308103dd9acc2ef88235 Mon Sep 17 00:00:00 2001 From: juliusclausnitzer Date: Tue, 17 Nov 2026 11:44:34 +0100 Subject: [PATCH 13/13] review fixes Signed-off-by: juliusclausnitzer --- .../commitments/changes/reconciler.go | 22 +- .../commitments/changes/reconciler_test.go | 453 ++++++++---------- 2 files changed, 202 insertions(+), 273 deletions(-) diff --git a/internal/scheduling/reservations/commitments/changes/reconciler.go b/internal/scheduling/reservations/commitments/changes/reconciler.go index 403ff687f..6ccc7f264 100644 --- a/internal/scheduling/reservations/commitments/changes/reconciler.go +++ b/internal/scheduling/reservations/commitments/changes/reconciler.go @@ -94,6 +94,7 @@ func (r *CommittedResourceChangeReconciler) reconcilePreparing(ctx context.Conte // Snapshot current CR specs for all keys in NewState so rollback can restore them. // nil entry = CR did not exist before = delete it on rollback. snapshot := make(map[string]*v1alpha1.CommittedResourceSpec, len(tx.Spec.NewState)) + var existingCommitment string for uuid := range tx.Spec.NewState { var cr v1alpha1.CommittedResource if err := r.Get(ctx, client.ObjectKey{Name: commitmentName(uuid)}, &cr); err != nil { @@ -106,20 +107,17 @@ func (r *CommittedResourceChangeReconciler) reconcilePreparing(ctx context.Conte return ctrl.Result{}, err } snapshot[uuid] = cr.Spec.DeepCopy() + existingCommitment = uuid } - // Guard: a dry run must only probe new commitments. A non-nil snapshot entry means an - // existing CR would be mutated, which a dry run must never do. - if tx.Spec.DryRun { - for uuid, snapSpec := range snapshot { - if snapSpec != nil { - msg := "dryRun must not modify existing commitment " + uuid - logger.Info("rejecting change: "+msg, "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID) - return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseRejected, v1alpha1.ChangeReasonInvalidSpec, func(s *v1alpha1.CommittedResourceChangeStatus) { - s.Message = msg - }) - } - } + // Guard: a dry run must only probe new commitments. An existing CR (non-nil snapshot entry) + // would be mutated, which a dry run must never do. + if tx.Spec.DryRun && existingCommitment != "" { + msg := "dryRun must not modify existing commitment " + existingCommitment + logger.Info("rejecting change: "+msg, "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID) + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseRejected, v1alpha1.ChangeReasonInvalidSpec, func(s *v1alpha1.CommittedResourceChangeStatus) { + s.Message = msg + }) } logger.Info("starting transaction", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "newStateSize", len(tx.Spec.NewState), "allowRejection", tx.Spec.AllowRejection) diff --git a/internal/scheduling/reservations/commitments/changes/reconciler_test.go b/internal/scheduling/reservations/commitments/changes/reconciler_test.go index 5eaf3850b..430bc4cff 100644 --- a/internal/scheduling/reservations/commitments/changes/reconciler_test.go +++ b/internal/scheduling/reservations/commitments/changes/reconciler_test.go @@ -1148,314 +1148,245 @@ func TestCRTReconciler_Terminal(t *testing.T) { } // ============================================================================ -// TestCRTReconciler_RollingBack +// TestCRTReconciler_Restore (RollingBack + DryRunCleanup share reconcileRestore) // ============================================================================ -// driveToRollingBack drives a CRC to RollingBack by having a CR reject placement. -func driveToRollingBack(t *testing.T, r *CommittedResourceChangeReconciler, c client.Client, txName, crName string) { - t.Helper() - driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseAwaitingScheduling) - setCRReady(t, c, crName, metav1.ConditionFalse, v1alpha1.CommittedResourceReasonRejected, "no capacity") - driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseRollingBack) -} +// restoreEntry selects how the harness drives a CRC into a restore phase. +type restoreEntry int + +const ( + entryReject restoreEntry = iota // reject a CR → RollingBack + entryAccept // accept all CRs (DryRun) → DryRunCleanup +) -func TestCRTReconciler_RollingBack(t *testing.T) { +// TestCRTReconciler_Restore exercises the shared restore path for both RollingBack (terminal +// Rejected) and DryRunCleanup (terminal Accepted): snapshot restore, multi-CR, re-create of a +// deleted CR, idempotency, and the RollbackTimeout → RollbackFailed path. +func TestCRTReconciler_Restore(t *testing.T) { tests := []struct { - name string - preObjects []client.Object - txSpec v1alpha1.CommittedResourceChangeSpec - // wantCRSpecAZ, when set, asserts the restored CR's spec.AvailabilityZone. - wantRestoredCR string - wantCRSpecAZ string - // wantCRAbsent, when set, asserts the CR was deleted by rollback. - wantCRAbsent string + name string + preObjects []client.Object + txSpec v1alpha1.CommittedResourceChangeSpec + entry restoreEntry + failRestore bool // inject a restore failure so RollbackTimeout fires + reconcileCount int // total reconciles in the restore phase (default 1; >1 checks idempotency) + wantPhase v1alpha1.ChangePhase + wantFinishedReason string + wantProgressingReason string + checkCRs func(t *testing.T, c client.Client) }{ { - name: "nil snapshot: new CR is deleted and rollback completes as Rejected", + name: "rollback: new CR is deleted", + entry: entryReject, txSpec: v1alpha1.CommittedResourceChangeSpec{ - AvailabilityZone: "az-1", - Source: v1alpha1.TxSourceAPI, - AllowRejection: true, - NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-rb1": minimalCRSpec("az-1")}, + AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-rb1": minimalCRSpec("az-1")}, + }, + wantPhase: v1alpha1.ChangePhaseRejected, + wantFinishedReason: string(v1alpha1.ChangePhaseRejected), + wantProgressingReason: string(v1alpha1.ChangePhaseRejected), + checkCRs: func(t *testing.T, c client.Client) { + if crExists(t, c, "commitment-uuid-rb1") { + t.Errorf("new CR should be deleted by rollback but exists") + } }, - wantCRAbsent: "commitment-uuid-rb1", }, { - name: "non-nil snapshot: existing CR spec is restored and rollback completes as Rejected", + name: "rollback: existing CR spec is restored", preObjects: []client.Object{ - &v1alpha1.CommittedResource{ - ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-rb2"}, - Spec: *minimalCRSpec("az-old"), - }, + &v1alpha1.CommittedResource{ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-rb2"}, Spec: *minimalCRSpec("az-old")}, }, + entry: entryReject, txSpec: v1alpha1.CommittedResourceChangeSpec{ - AvailabilityZone: "az-1", - Source: v1alpha1.TxSourceAPI, - AllowRejection: true, - NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-rb2": minimalCRSpec("az-new")}, + AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-rb2": minimalCRSpec("az-new")}, + }, + wantPhase: v1alpha1.ChangePhaseRejected, + wantFinishedReason: string(v1alpha1.ChangePhaseRejected), + wantProgressingReason: string(v1alpha1.ChangePhaseRejected), + checkCRs: func(t *testing.T, c client.Client) { + if got := getCR(t, c, "commitment-uuid-rb2").Spec.AvailabilityZone; got != "az-old" { + t.Errorf("restored CR AZ: want az-old, got %q", got) + } }, - wantRestoredCR: "commitment-uuid-rb2", - wantCRSpecAZ: "az-old", }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - scheme := testScheme(t) - txName := "tx-rb-" + tt.name[:5] - tx := &v1alpha1.CommittedResourceChange{ - ObjectMeta: metav1.ObjectMeta{Name: txName}, - Spec: tt.txSpec, - } - objs := append([]client.Object{tx}, tt.preObjects...) - c := testClient(t, scheme, objs...) - r := testReconciler(c, scheme) - - var crName string - for uuid := range tt.txSpec.NewState { - crName = commitmentName(uuid) - break - } - - driveToRollingBack(t, r, c, txName, crName) - - // Restore completes in a single reconcile: specs are written back, no settlement wait. - doReconcile(t, r, txName) - - tx = getTx(t, c, txName) - if tx.Status.Phase != v1alpha1.ChangePhaseRejected { - t.Errorf("phase: want Rejected, got %q", tx.Status.Phase) - } - assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionFinished, metav1.ConditionTrue, string(v1alpha1.ChangePhaseRejected)) - assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionProgressing, metav1.ConditionFalse, string(v1alpha1.ChangePhaseRejected)) - - if tt.wantCRAbsent != "" && crExists(t, c, tt.wantCRAbsent) { - t.Errorf("CR %q should be deleted by rollback but exists", tt.wantCRAbsent) - } - if tt.wantRestoredCR != "" { - cr := getCR(t, c, tt.wantRestoredCR) - if cr.Spec.AvailabilityZone != tt.wantCRSpecAZ { - t.Errorf("restored CR %q AZ: want %q, got %q", tt.wantRestoredCR, tt.wantCRSpecAZ, cr.Spec.AvailabilityZone) + { + name: "rollback: deleted CR is re-created", + preObjects: []client.Object{ + &v1alpha1.CommittedResource{ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-del"}, Spec: *minimalCRSpec("az-old")}, + }, + entry: entryReject, + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{ + "uuid-del": nil, // delete an existing CR (snapshot non-nil) + "uuid-new": minimalCRSpec("az-new"), // new CR that will be rejected + }, + }, + wantPhase: v1alpha1.ChangePhaseRejected, + wantFinishedReason: string(v1alpha1.ChangePhaseRejected), + wantProgressingReason: string(v1alpha1.ChangePhaseRejected), + checkCRs: func(t *testing.T, c client.Client) { + // The deleted CR must be re-created with its original spec. + if got := getCR(t, c, "commitment-uuid-del").Spec.AvailabilityZone; got != "az-old" { + t.Errorf("re-created CR AZ: want az-old, got %q", got) + } + // The new CR must be removed. + if crExists(t, c, "commitment-uuid-new") { + t.Errorf("new CR should be deleted by rollback but exists") } - } - }) - } -} - -// TestCRTReconciler_RollingBack_MultiCR verifies that a multi-CR reject rolls every affected CR -// back to its snapshot in a single reconcile. -func TestCRTReconciler_RollingBack_MultiCR(t *testing.T) { - const txName = "tx-rb-multi" - scheme := testScheme(t) - tx := &v1alpha1.CommittedResourceChange{ - ObjectMeta: metav1.ObjectMeta{Name: txName}, - Spec: v1alpha1.CommittedResourceChangeSpec{ - AvailabilityZone: "az-1", - Source: v1alpha1.TxSourceAPI, - AllowRejection: true, - NewState: map[string]*v1alpha1.CommittedResourceSpec{ - "uuid-m1": minimalCRSpec("az-new"), - "uuid-m2": minimalCRSpec("az-new"), }, }, - } - // Both CRs pre-exist with az-old so their snapshots are non-nil; rollback must restore both. - preObjects := []client.Object{ - &v1alpha1.CommittedResource{ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-m1"}, Spec: *minimalCRSpec("az-old")}, - &v1alpha1.CommittedResource{ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-m2"}, Spec: *minimalCRSpec("az-old")}, - } - objs := append([]client.Object{tx}, preObjects...) - c := testClient(t, scheme, objs...) - r := testReconciler(c, scheme) - - driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseAwaitingScheduling) - setCRReady(t, c, "commitment-uuid-m1", metav1.ConditionFalse, v1alpha1.CommittedResourceReasonRejected, "no capacity") - driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseRollingBack) - - doReconcile(t, r, txName) - - if got := getTx(t, c, txName).Status.Phase; got != v1alpha1.ChangePhaseRejected { - t.Fatalf("phase: want Rejected, got %q", got) - } - // Both CRs must have their az-old spec restored. - for _, name := range []string{"commitment-uuid-m1", "commitment-uuid-m2"} { - if got := getCR(t, c, name).Spec.AvailabilityZone; got != "az-old" { - t.Errorf("CR %q AZ: want az-old (restored), got %q", name, got) - } - } -} - -// TestCRTReconciler_RollingBack_Idempotent verifies that re-running RollingBack multiple -// times is safe and produces a stable Rejected terminal state. -func TestCRTReconciler_RollingBack_Idempotent(t *testing.T) { - const txName = "tx-rb-idem" - scheme := testScheme(t) - tx := &v1alpha1.CommittedResourceChange{ - ObjectMeta: metav1.ObjectMeta{Name: txName}, - Spec: v1alpha1.CommittedResourceChangeSpec{ - AvailabilityZone: "az-1", - Source: v1alpha1.TxSourceAPI, - AllowRejection: true, - NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-idem-rb": minimalCRSpec("az-1")}, + { + name: "rollback: multi-CR restores all", + preObjects: []client.Object{ + &v1alpha1.CommittedResource{ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-m1"}, Spec: *minimalCRSpec("az-old")}, + &v1alpha1.CommittedResource{ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-m2"}, Spec: *minimalCRSpec("az-old")}, + }, + entry: entryReject, + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{ + "uuid-m1": minimalCRSpec("az-new"), + "uuid-m2": minimalCRSpec("az-new"), + }, + }, + wantPhase: v1alpha1.ChangePhaseRejected, + wantFinishedReason: string(v1alpha1.ChangePhaseRejected), + wantProgressingReason: string(v1alpha1.ChangePhaseRejected), + checkCRs: func(t *testing.T, c client.Client) { + for _, name := range []string{"commitment-uuid-m1", "commitment-uuid-m2"} { + if got := getCR(t, c, name).Spec.AvailabilityZone; got != "az-old" { + t.Errorf("CR %q AZ: want az-old (restored), got %q", name, got) + } + } + }, }, - } - c := testClient(t, scheme, tx) - r := testReconciler(c, scheme) - - driveToRollingBack(t, r, c, txName, "commitment-uuid-idem-rb") - reconcileN(t, r, txName, 3) - - if got := getTx(t, c, txName).Status.Phase; got != v1alpha1.ChangePhaseRejected { - t.Errorf("phase: want Rejected, got %q", got) - } -} - -// TestCRTReconciler_Restore_Timeout verifies that when restore keeps failing past RollbackTimeout, -// the CRC terminates with Finished reason RollbackFailed — at Rejected for a rollback and at -// Accepted for a dry-run cleanup (the schedulability verdict is preserved). -func TestCRTReconciler_Restore_Timeout(t *testing.T) { - tests := []struct { - name string - dryRun bool - driveToPhase v1alpha1.ChangePhase - wantPhase v1alpha1.ChangePhase - rejectCR bool // reject the CR to reach RollingBack; else accept to reach DryRunCleanup - }{ { - name: "rollback restore times out → Rejected/RollbackFailed", - dryRun: false, - driveToPhase: v1alpha1.ChangePhaseRollingBack, - wantPhase: v1alpha1.ChangePhaseRejected, - rejectCR: true, + name: "rollback: idempotent across repeated reconciles", + entry: entryReject, + reconcileCount: 3, + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-idem-rb": minimalCRSpec("az-1")}, + }, + wantPhase: v1alpha1.ChangePhaseRejected, + wantFinishedReason: string(v1alpha1.ChangePhaseRejected), + wantProgressingReason: string(v1alpha1.ChangePhaseRejected), }, { - name: "dry-run cleanup restore times out → Accepted/RollbackFailed", - dryRun: true, - driveToPhase: v1alpha1.ChangePhaseDryRunCleanup, - wantPhase: v1alpha1.ChangePhaseAccepted, - rejectCR: false, + name: "rollback: restore keeps failing → RollbackFailed/Rejected", + entry: entryReject, + failRestore: true, + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-rbto": minimalCRSpec("az-1")}, + }, + wantPhase: v1alpha1.ChangePhaseRejected, + wantFinishedReason: v1alpha1.ChangeReasonRollbackFailed, + wantProgressingReason: v1alpha1.ChangeReasonRollbackFailed, + }, + { + name: "dry-run cleanup: probe CR deleted, verdict preserved as Accepted", + entry: entryAccept, + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, AllowRejection: true, DryRun: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dry1": minimalCRSpec("az-1")}, + }, + wantPhase: v1alpha1.ChangePhaseAccepted, + wantFinishedReason: string(v1alpha1.ChangePhaseAccepted), + wantProgressingReason: string(v1alpha1.ChangePhaseAccepted), + checkCRs: func(t *testing.T, c client.Client) { + if crExists(t, c, "commitment-uuid-dry1") { + t.Errorf("dry-run probe CR should be deleted on cleanup but exists") + } + }, + }, + { + name: "dry-run would-reject: follows RollingBack → Rejected", + entry: entryReject, + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, AllowRejection: true, DryRun: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dr": minimalCRSpec("az-1")}, + }, + wantPhase: v1alpha1.ChangePhaseRejected, + wantFinishedReason: string(v1alpha1.ChangePhaseRejected), + wantProgressingReason: string(v1alpha1.ChangePhaseRejected), + }, + { + name: "dry-run cleanup: restore keeps failing → RollbackFailed/Accepted", + entry: entryAccept, + failRestore: true, + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, AllowRejection: true, DryRun: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dryto": minimalCRSpec("az-1")}, + }, + wantPhase: v1alpha1.ChangePhaseAccepted, + wantFinishedReason: v1alpha1.ChangeReasonRollbackFailed, + wantProgressingReason: v1alpha1.ChangeReasonRollbackFailed, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { scheme := testScheme(t) - txName := "tx-to-" + tt.name[:5] + const txName = "tx-restore" tx := &v1alpha1.CommittedResourceChange{ ObjectMeta: metav1.ObjectMeta{Name: txName}, - Spec: v1alpha1.CommittedResourceChangeSpec{ - AvailabilityZone: "az-1", - Source: v1alpha1.TxSourceAPI, - AllowRejection: true, - DryRun: tt.dryRun, - // New CR (nil snapshot) → rollback deletes it; the failing client makes Delete error. - NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-to": minimalCRSpec("az-1")}, - }, + Spec: tt.txSpec, } + objs := append([]client.Object{tx}, tt.preObjects...) + fail := false - c := testClientFailingCRWrites(t, scheme, &fail, tx) + var c client.Client + if tt.failRestore { + c = testClientFailingCRWrites(t, scheme, &fail, objs...) + } else { + c = testClient(t, scheme, objs...) + } r := testReconciler(c, scheme) - r.RollbackTimeout = 50 * time.Millisecond + if tt.failRestore { + r.RollbackTimeout = 50 * time.Millisecond + } driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseAwaitingScheduling) - if tt.rejectCR { - setCRReady(t, c, "commitment-uuid-to", metav1.ConditionFalse, v1alpha1.CommittedResourceReasonRejected, "no capacity") - } else { - setCRReady(t, c, "commitment-uuid-to", metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted, "") + upserted := getTx(t, c, txName).Status.UpsertedCommitments + + switch tt.entry { + case entryReject: + setCRReady(t, c, upserted[0].Name, metav1.ConditionFalse, v1alpha1.CommittedResourceReasonRejected, "no capacity") + driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseRollingBack) + case entryAccept: + for _, ref := range upserted { + setCRReady(t, c, ref.Name, metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted, "") + } + driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseDryRunCleanup) } - driveToPhase(t, r, c, txName, tt.driveToPhase) - // Now make restore fail and backdate the Restoring condition past the timeout. - fail = true - backdateRestoringCondition(t, c, txName, 1*time.Second) + if tt.failRestore { + fail = true + backdateRestoringCondition(t, c, txName, 1*time.Second) + } - doReconcile(t, r, txName) + n := tt.reconcileCount + if n == 0 { + n = 1 + } + reconcileN(t, r, txName, n) tx = getTx(t, c, txName) if tx.Status.Phase != tt.wantPhase { t.Errorf("phase: want %q, got %q", tt.wantPhase, tx.Status.Phase) } - assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionFinished, metav1.ConditionTrue, v1alpha1.ChangeReasonRollbackFailed) - assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionProgressing, metav1.ConditionFalse, v1alpha1.ChangeReasonRollbackFailed) + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionFinished, metav1.ConditionTrue, tt.wantFinishedReason) + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionProgressing, metav1.ConditionFalse, tt.wantProgressingReason) + if tt.checkCRs != nil { + tt.checkCRs(t, c) + } }) } } -// ============================================================================ -// TestCRTReconciler_DryRunCleanup -// ============================================================================ - -// driveToDryRunCleanup drives a DryRun CRC to DryRunCleanup. -func driveToDryRunCleanup(t *testing.T, r *CommittedResourceChangeReconciler, c client.Client, txName, crName string) { - t.Helper() - driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseAwaitingScheduling) - setCRReady(t, c, crName, metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted, "") - driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseDryRunCleanup) -} - -func TestCRTReconciler_DryRunCleanup(t *testing.T) { - // Dry-run requires a net-new commitment (nil snapshot); an existing CR is rejected by a guard, - // so every DryRunCleanup case deletes a freshly-created CR and terminates as Accepted. - const txName = "tx-dry-cleanup" - scheme := testScheme(t) - tx := &v1alpha1.CommittedResourceChange{ - ObjectMeta: metav1.ObjectMeta{Name: txName}, - Spec: v1alpha1.CommittedResourceChangeSpec{ - AvailabilityZone: "az-1", - Source: v1alpha1.TxSourceAPI, - AllowRejection: true, - DryRun: true, - NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dry1": minimalCRSpec("az-1")}, - }, - } - c := testClient(t, scheme, tx) - r := testReconciler(c, scheme) - - driveToDryRunCleanup(t, r, c, txName, "commitment-uuid-dry1") - - // Restore completes in a single reconcile; the probe CR is deleted, verdict preserved as Accepted. - doReconcile(t, r, txName) - - tx = getTx(t, c, txName) - if tx.Status.Phase != v1alpha1.ChangePhaseAccepted { - t.Errorf("phase: want Accepted, got %q", tx.Status.Phase) - } - assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionFinished, metav1.ConditionTrue, string(v1alpha1.ChangePhaseAccepted)) - assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionProgressing, metav1.ConditionFalse, string(v1alpha1.ChangePhaseAccepted)) - if crExists(t, c, "commitment-uuid-dry1") { - t.Errorf("dry-run probe CR should be deleted on cleanup but exists") - } -} - -// TestCRTReconciler_DryRun_WouldReject verifies that DryRun=true + rejection follows the -// RollingBack path and terminates as Rejected (not Accepted). -func TestCRTReconciler_DryRun_WouldReject(t *testing.T) { - const txName = "tx-dry-reject" - scheme := testScheme(t) - tx := &v1alpha1.CommittedResourceChange{ - ObjectMeta: metav1.ObjectMeta{Name: txName}, - Spec: v1alpha1.CommittedResourceChangeSpec{ - AvailabilityZone: "az-1", - Source: v1alpha1.TxSourceAPI, - AllowRejection: true, - DryRun: true, - NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dr": minimalCRSpec("az-1")}, - }, - } - c := testClient(t, scheme, tx) - r := testReconciler(c, scheme) - - driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseAwaitingScheduling) - setCRReady(t, c, "commitment-uuid-dr", metav1.ConditionFalse, v1alpha1.CommittedResourceReasonRejected, "no capacity") - driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseRollingBack) - - doReconcile(t, r, txName) - if got := getTx(t, c, txName).Status.Phase; got != v1alpha1.ChangePhaseRejected { - t.Errorf("DryRun would-reject: want Rejected, got %q", got) - } - assertCondition(t, getTx(t, c, txName).Status.Conditions, v1alpha1.ChangeConditionFinished, metav1.ConditionTrue, string(v1alpha1.ChangePhaseRejected)) -} - // TestCRTReconciler_DryRunGuards verifies the spec guards reject invalid dry-run requests. func TestCRTReconciler_DryRunGuards(t *testing.T) { tests := []struct {