From 538b52ca289df4d41efcb6e1ebc99e9ea8632b47 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 17 Nov 2026 15:28:23 +0100 Subject: [PATCH 1/3] bump app version [skip ci] (#1547) bump app version [skip ci] ``` bumped cortex: sha-67c88600 -> sha-96a0f8cd noop cortex-shim (sha-01a68db0) noop cortex-postgres (sha-f69b58ee) ``` Signed-off-by: SoWieMarkus <44964229+SoWieMarkus@users.noreply.github.com> Co-authored-by: SoWieMarkus <44964229+SoWieMarkus@users.noreply.github.com> --- helm/library/cortex/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/library/cortex/Chart.yaml b/helm/library/cortex/Chart.yaml index 0fada199e..cd823dc57 100644 --- a/helm/library/cortex/Chart.yaml +++ b/helm/library/cortex/Chart.yaml @@ -3,6 +3,6 @@ name: cortex description: A Helm chart to distribute cortex. type: application version: 0.9.3 -appVersion: "sha-67c88600" +appVersion: "sha-96a0f8cd" icon: "https://example.com/icon.png" dependencies: [] From d231ff9af83cd536c0314c80cd46140eb32c3eca Mon Sep 17 00:00:00 2001 From: "cortex-ai-agents[bot]" <279748396+cortex-ai-agents[bot]@users.noreply.github.com> Date: Tue, 17 Nov 2026 15:50:37 +0100 Subject: [PATCH 2/3] Release cortex 0.9.4 (#1549) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Release prep for #1548: changelog entry and helm chart version bumps. Merge this before merging #1548. - Adds CHANGELOG.md entry for cortex 0.9.4 - Bumps version in 10 helm Chart.yaml files (bundles and libraries) ## Test plan - [ ] Verify CHANGELOG.md entry is correct and complete - [ ] Verify all helm chart versions are bumped consistently - [ ] Ensure CI passes (linting, chart validation) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: cortex-ai-agents[bot] <279748396+cortex-ai-agents[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 4.7 --- CHANGELOG.md | 17 +++++++++++++++++ helm/bundles/cortex-cinder/Chart.yaml | 8 ++++---- helm/bundles/cortex-crds/Chart.yaml | 4 ++-- helm/bundles/cortex-ironcore/Chart.yaml | 4 ++-- helm/bundles/cortex-manila/Chart.yaml | 8 ++++---- helm/bundles/cortex-nova/Chart.yaml | 8 ++++---- helm/bundles/cortex-placement-shim/Chart.yaml | 4 ++-- helm/bundles/cortex-pods/Chart.yaml | 4 ++-- helm/library/cortex-postgres/Chart.yaml | 2 +- helm/library/cortex-shim/Chart.yaml | 2 +- helm/library/cortex/Chart.yaml | 2 +- 11 files changed, 40 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 532970022..47d34441b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## cortex 0.9.4 + +### Bug Fixes + +- **Fix false host oversubscription alerts after capacity refactoring** ([#1543](https://github.com/cobaltcore-dev/cortex/pull/1543), @mblos) + After the unified capacity calculation refactoring (#1504), every host with committed-resource reservations **and** running VMs was incorrectly reported as oversubscribed. This patch corrects the false-positive alerting logic in the host oversubscription controller. + +### Helm Chart Versions + +| Chart | Version | +|---|---| +| cortex | 0.9.4 | +| cortex-postgres | 0.6.36 | +| cortex-shim | 0.1.42 | +| bundles | 0.0.118 | +| cortex-placement-shim | 0.1.42 | + ## Changelog — cortex 0.9.3 **Release PR:** [#1545](https://github.com/cobaltcore-dev/cortex/pull/1545) diff --git a/helm/bundles/cortex-cinder/Chart.yaml b/helm/bundles/cortex-cinder/Chart.yaml index 5fdb4b0d8..7d078f234 100644 --- a/helm/bundles/cortex-cinder/Chart.yaml +++ b/helm/bundles/cortex-cinder/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-cinder description: A Helm chart deploying Cortex for Cinder. type: application -version: 0.0.117 +version: 0.0.118 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.35 + version: 0.6.36 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.9.3 + version: 0.9.4 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.9.3 + version: 0.9.4 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-crds/Chart.yaml b/helm/bundles/cortex-crds/Chart.yaml index a6bbb6a10..d1984a35a 100644 --- a/helm/bundles/cortex-crds/Chart.yaml +++ b/helm/bundles/cortex-crds/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-crds description: A Helm chart deploying Cortex CRDs. type: application -version: 0.0.117 +version: 0.0.118 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.9.3 + version: 0.9.4 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/bundles/cortex-ironcore/Chart.yaml b/helm/bundles/cortex-ironcore/Chart.yaml index 14ebd4318..63e8f1611 100644 --- a/helm/bundles/cortex-ironcore/Chart.yaml +++ b/helm/bundles/cortex-ironcore/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-ironcore description: A Helm chart deploying Cortex for IronCore. type: application -version: 0.0.117 +version: 0.0.118 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.9.3 + version: 0.9.4 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/bundles/cortex-manila/Chart.yaml b/helm/bundles/cortex-manila/Chart.yaml index bcce4d43e..6812843ac 100644 --- a/helm/bundles/cortex-manila/Chart.yaml +++ b/helm/bundles/cortex-manila/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-manila description: A Helm chart deploying Cortex for Manila. type: application -version: 0.0.117 +version: 0.0.118 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.35 + version: 0.6.36 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.9.3 + version: 0.9.4 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.9.3 + version: 0.9.4 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-nova/Chart.yaml b/helm/bundles/cortex-nova/Chart.yaml index eb2b88446..3018e1dbc 100644 --- a/helm/bundles/cortex-nova/Chart.yaml +++ b/helm/bundles/cortex-nova/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-nova description: A Helm chart deploying Cortex for Nova. type: application -version: 0.0.117 +version: 0.0.118 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.35 + version: 0.6.36 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.9.3 + version: 0.9.4 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.9.3 + version: 0.9.4 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-placement-shim/Chart.yaml b/helm/bundles/cortex-placement-shim/Chart.yaml index 112753310..e70abddf2 100644 --- a/helm/bundles/cortex-placement-shim/Chart.yaml +++ b/helm/bundles/cortex-placement-shim/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-placement-shim description: A Helm chart deploying the Cortex placement shim. type: application -version: 0.1.41 +version: 0.1.42 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-shim - name: cortex-shim repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.1.41 + version: 0.1.42 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case # of issues. See: https://github.com/sapcc/helm-charts/pkgs/container/helm-charts%2Fowner-info diff --git a/helm/bundles/cortex-pods/Chart.yaml b/helm/bundles/cortex-pods/Chart.yaml index ae841da58..953cd3a00 100644 --- a/helm/bundles/cortex-pods/Chart.yaml +++ b/helm/bundles/cortex-pods/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-pods description: A Helm chart deploying Cortex for Pods. type: application -version: 0.0.117 +version: 0.0.118 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.9.3 + version: 0.9.4 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/library/cortex-postgres/Chart.yaml b/helm/library/cortex-postgres/Chart.yaml index 98c934d97..1974581c0 100644 --- a/helm/library/cortex-postgres/Chart.yaml +++ b/helm/library/cortex-postgres/Chart.yaml @@ -5,5 +5,5 @@ apiVersion: v2 name: cortex-postgres description: Postgres setup for Cortex. type: application -version: 0.6.35 +version: 0.6.36 appVersion: "sha-f69b58ee" diff --git a/helm/library/cortex-shim/Chart.yaml b/helm/library/cortex-shim/Chart.yaml index f109d06d1..ed07ec646 100644 --- a/helm/library/cortex-shim/Chart.yaml +++ b/helm/library/cortex-shim/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cortex-shim description: A Helm chart to distribute cortex shims. type: application -version: 0.1.41 +version: 0.1.42 appVersion: "sha-01a68db0" icon: "https://example.com/icon.png" dependencies: [] diff --git a/helm/library/cortex/Chart.yaml b/helm/library/cortex/Chart.yaml index cd823dc57..1d8547f8b 100644 --- a/helm/library/cortex/Chart.yaml +++ b/helm/library/cortex/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cortex description: A Helm chart to distribute cortex. type: application -version: 0.9.3 +version: 0.9.4 appVersion: "sha-96a0f8cd" icon: "https://example.com/icon.png" dependencies: [] From 6097a3172a10d0ef5ec1a58085cd03c5fe4fd75e Mon Sep 17 00:00:00 2001 From: juliusclausnitzer Date: Tue, 17 Nov 2026 15:57:15 +0100 Subject: [PATCH 3/3] feat: implement CRC rollback on failure + DryRun Cleanup (#1463) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Implements the two previously-stubbed CRC FSM phase handlers - Adds `restoreFromSnapshot`: a shared two-pass (write → settle) helper that converges affected `CommittedResource`s back to the pre-change `SnapshotBefore` state; idempotent and crash-safe - `reconcileRollingBack`: on CR rejection or scheduling timeout, restores the snapshot and terminates as `Rejected`; on permanent restore failure (timeout) terminates `Rejected` with reason `RollbackFailed` - `reconcileDryRunCleanup`: for a would-accept dry-run, restores the snapshot and terminates as `Accepted` (schedulability verdict preserved); a would-reject dry-run routes through `RollingBack` → `Rejected` - Adds `RollbackStartedAt` status timestamp to measure `RollbackTimeout` from phase entry rather than the `Progressing` condition's `LastTransitionTime` (which only updates on status flips) - Adds `RollbackTimeout` reconciler knob and `ChangeReasonRollbackFailed` --------- Signed-off-by: juliusclausnitzer --- .../committed_resource_change_types.go | 12 +- ...cortex.cloud_committedresourcechanges.yaml | 3 +- .../commitments/changes/reconciler.go | 148 +++++-- .../commitments/changes/reconciler_test.go | 397 +++++++++++++++++- 4 files changed, 519 insertions(+), 41 deletions(-) diff --git a/api/v1alpha1/committed_resource_change_types.go b/api/v1alpha1/committed_resource_change_types.go index 29fb90e13..bb6cef9f1 100644 --- a/api/v1alpha1/committed_resource_change_types.go +++ b/api/v1alpha1/committed_resource_change_types.go @@ -48,6 +48,11 @@ const ( // ChangeConditionFinished is True once the change request has reached a terminal phase. ChangeConditionFinished = "Finished" + // ChangeConditionRestoring is True while the change request is restoring pre-change state + // (RollingBack or DryRunCleanup). Its LastTransitionTime marks when restore began and its + // Message carries the last restore error (e.g. a transient k8s API error). + ChangeConditionRestoring = "Restoring" + // Reasons for ChangeConditionProgressing=False (non-terminal problems). ChangeReasonStalled = "Stalled" ChangeReasonTimedOut = "TimedOut" @@ -58,6 +63,11 @@ const ( ChangeRequestReasonAccepted = "Accepted" ChangeRequestReasonRejected = "Rejected" ChangeRequestReasonSuperseded = "Superseded" + // ChangeReasonRollbackFailed is the Finished reason when rollback exhausted its timeout. + ChangeReasonRollbackFailed = "RollbackFailed" + // ChangeReasonInvalidSpec is the Finished reason when a change request is rejected for a + // malformed spec (e.g. a dry run that would commit or mutate existing commitments). + ChangeReasonInvalidSpec = "InvalidSpec" ) // CommittedResourceChangeSpec defines the desired change for a CommittedResourceChange. @@ -110,7 +120,7 @@ type CommittedResourceChangeStatus struct { // +kubebuilder:validation:Optional UpsertedCommitments []corev1.ObjectReference `json:"upsertedCommitments,omitempty"` - // Conditions holds Progressing and Finished conditions. + // Conditions holds Progressing, Finished and Restoring conditions. // +kubebuilder:validation:Optional Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` diff --git a/helm/library/cortex/files/crds/cortex.cloud_committedresourcechanges.yaml b/helm/library/cortex/files/crds/cortex.cloud_committedresourcechanges.yaml index ecf607f4c..ef49c268f 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_committedresourcechanges.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_committedresourcechanges.yaml @@ -195,7 +195,8 @@ spec: of a CommittedResourceChange. properties: conditions: - description: Conditions holds Progressing and Finished conditions. + description: Conditions holds Progressing, Finished and Restoring + conditions. items: description: Condition contains details for one aspect of the current state of this API Resource. diff --git a/internal/scheduling/reservations/commitments/changes/reconciler.go b/internal/scheduling/reservations/commitments/changes/reconciler.go index d07284f73..bafa610b4 100644 --- a/internal/scheduling/reservations/commitments/changes/reconciler.go +++ b/internal/scheduling/reservations/commitments/changes/reconciler.go @@ -35,6 +35,10 @@ type CommittedResourceChangeReconciler struct { // AwaitingSchedulingTimeout is how long to wait in AwaitingScheduling before rolling back. // Zero (default) means no timeout. AwaitingSchedulingTimeout time.Duration + // RollbackTimeout is how long to attempt snapshot restore in RollingBack/DryRunCleanup + // before giving up and terminating with RollbackFailed. + // Zero (default) means no timeout. + RollbackTimeout time.Duration } func (r *CommittedResourceChangeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -61,9 +65,9 @@ func (r *CommittedResourceChangeReconciler) Reconcile(ctx context.Context, req c case v1alpha1.ChangePhaseAwaitingScheduling: return r.reconcileAwaitingScheduling(ctx, logger, &tx) case v1alpha1.ChangePhaseRollingBack: - return r.reconcileRollingBack(ctx, logger, &tx) + return r.reconcileRestore(ctx, logger, &tx, v1alpha1.ChangePhaseRejected) case v1alpha1.ChangePhaseDryRunCleanup: - return r.reconcileDryRunCleanup(ctx, logger, &tx) + return r.reconcileRestore(ctx, logger, &tx, v1alpha1.ChangePhaseAccepted) case v1alpha1.ChangePhaseAccepted, v1alpha1.ChangePhaseRejected, v1alpha1.ChangePhaseSuperseded: return ctrl.Result{}, nil default: @@ -74,13 +78,23 @@ func (r *CommittedResourceChangeReconciler) Reconcile(ctx context.Context, req c // commitmentName returns the CommittedResource name for a given commitment UUID. func commitmentName(uuid string) string { return "commitment-" + uuid } -// reconcilePreparing snapshots pre-change CR specs and advances to Allocating. +// reconcilePreparing validates the spec, snapshots pre-change CR specs, and advances to Allocating. // Always snapshots regardless of AllowRejection — syncer-mode CRTs never roll back but the // snapshot is cheap and keeps this method free of branching. func (r *CommittedResourceChangeReconciler) reconcilePreparing(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange) (ctrl.Result, error) { + // Guard: a dry run must be rejectable, otherwise Allocating advances straight to Accepted + // and the probe would silently commit. + if tx.Spec.DryRun && !tx.Spec.AllowRejection { + logger.Info("rejecting change: dryRun requires allowRejection", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID) + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseRejected, v1alpha1.ChangeReasonInvalidSpec, func(s *v1alpha1.CommittedResourceChangeStatus) { + s.Message = "dryRun requires allowRejection" + }) + } + // Snapshot current CR specs for all keys in NewState so rollback can restore them. // nil entry = CR did not exist before = delete it on rollback. snapshot := make(map[string]*v1alpha1.CommittedResourceSpec, len(tx.Spec.NewState)) + var existingCommitment string for uuid := range tx.Spec.NewState { var cr v1alpha1.CommittedResource if err := r.Get(ctx, client.ObjectKey{Name: commitmentName(uuid)}, &cr); err != nil { @@ -93,10 +107,21 @@ func (r *CommittedResourceChangeReconciler) reconcilePreparing(ctx context.Conte return ctrl.Result{}, err } snapshot[uuid] = cr.Spec.DeepCopy() + existingCommitment = uuid + } + + // Guard: a dry run must only probe new commitments. An existing CR (non-nil snapshot entry) + // would be mutated, which a dry run must never do. + if tx.Spec.DryRun && existingCommitment != "" { + msg := "dryRun must not modify existing commitment " + existingCommitment + logger.Info("rejecting change: "+msg, "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID) + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseRejected, v1alpha1.ChangeReasonInvalidSpec, func(s *v1alpha1.CommittedResourceChangeStatus) { + s.Message = msg + }) } logger.Info("starting transaction", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "newStateSize", len(tx.Spec.NewState), "allowRejection", tx.Spec.AllowRejection) - return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseAllocating, func(s *v1alpha1.CommittedResourceChangeStatus) { + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseAllocating, "", func(s *v1alpha1.CommittedResourceChangeStatus) { s.SnapshotBefore = snapshot }) } @@ -144,7 +169,7 @@ func (r *CommittedResourceChangeReconciler) reconcileAllocating(ctx context.Cont if !tx.Spec.AllowRejection { nextPhase = v1alpha1.ChangePhaseAccepted } - return r.advanceTo(ctx, tx, nextPhase, func(s *v1alpha1.CommittedResourceChangeStatus) { + return r.advanceTo(ctx, tx, nextPhase, "", func(s *v1alpha1.CommittedResourceChangeStatus) { s.UpsertedCommitments = upserted }) } @@ -185,7 +210,7 @@ func (r *CommittedResourceChangeReconciler) reconcileAwaitingScheduling(ctx cont // Fail-fast: start rollback immediately on first rejection rather than waiting for all CRs // to settle — this unblocks the AZ sooner and the first rejection message is sufficient for diagnosis. logger.Info("CR rejected, rolling back", "cr", ref.Name, "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "message", cond.Message) - return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseRollingBack, func(s *v1alpha1.CommittedResourceChangeStatus) { + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseRollingBack, "", func(s *v1alpha1.CommittedResourceChangeStatus) { s.Message = fmt.Sprintf("CR %s rejected: %s", ref.Name, cond.Message) }) default: @@ -200,7 +225,7 @@ func (r *CommittedResourceChangeReconciler) reconcileAwaitingScheduling(ctx cont progressingCond := meta.FindStatusCondition(tx.Status.Conditions, v1alpha1.ChangeConditionProgressing) if progressingCond != nil && time.Since(progressingCond.LastTransitionTime.Time) > r.AwaitingSchedulingTimeout { logger.Info("timed out waiting for scheduling, rolling back", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "timeout", r.AwaitingSchedulingTimeout) - return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseRollingBack, func(s *v1alpha1.CommittedResourceChangeStatus) { + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseRollingBack, "", func(s *v1alpha1.CommittedResourceChangeStatus) { s.Message = "timed out waiting for scheduling" meta.SetStatusCondition(&s.Conditions, metav1.Condition{ Type: v1alpha1.ChangeConditionProgressing, @@ -218,33 +243,81 @@ func (r *CommittedResourceChangeReconciler) reconcileAwaitingScheduling(ctx cont } if tx.Spec.DryRun { - return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseDryRunCleanup, nil) + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseDryRunCleanup, "", nil) } - return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseAccepted, nil) + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseAccepted, "", nil) } -// reconcileRollingBack converges affected CR specs back to the pre-change SnapshotBefore state. -// On restore complete: advances to Rejected. -// On permanent cleanup failure: advances to Rejected with reason RollbackFailed. -func (r *CommittedResourceChangeReconciler) reconcileRollingBack(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange) (ctrl.Result, error) { - // TODO (#780): for each entry in SnapshotBefore, converge CR spec back to snapshot value - // using SSA with fieldManager=txFieldManager. - // nil snapshot entry → CR did not exist before → delete it. - // Gate terminal transition on each CR confirming its spec reverted (check observedGeneration). - return ctrl.Result{}, nil +// reconcileRestore converges affected CR specs back to the pre-change SnapshotBefore state and +// advances to terminalPhase (Rejected for a rollback, Accepted for a dry-run cleanup). +// On a persistent restore failure past RollbackTimeout it terminates at terminalPhase with the +// Finished reason RollbackFailed; the Restoring condition's LastTransitionTime marks the start and +// its Message carries the last error. +func (r *CommittedResourceChangeReconciler) reconcileRestore(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange, terminalPhase v1alpha1.ChangePhase) (ctrl.Result, error) { + if err := r.restoreFromSnapshot(ctx, logger, tx); err != nil { + if r.RollbackTimeout > 0 { + if c := meta.FindStatusCondition(tx.Status.Conditions, v1alpha1.ChangeConditionRestoring); c != nil && time.Since(c.LastTransitionTime.Time) > r.RollbackTimeout { + logger.Info("restore timed out, terminalizing", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "timeout", r.RollbackTimeout, "terminalPhase", terminalPhase) + msg := fmt.Sprintf("timed out restoring pre-change state: %v", err) + return r.advanceTo(ctx, tx, terminalPhase, v1alpha1.ChangeReasonRollbackFailed, func(s *v1alpha1.CommittedResourceChangeStatus) { + s.Message = msg + }) + } + } + // Record the last restore error on the Restoring condition (Status stays True so its + // LastTransitionTime remains the restore-start marker) and requeue with backoff. + meta.SetStatusCondition(&tx.Status.Conditions, metav1.Condition{ + Type: v1alpha1.ChangeConditionRestoring, + Status: metav1.ConditionTrue, + Reason: v1alpha1.ChangeReasonError, + Message: err.Error(), + ObservedGeneration: tx.Generation, + }) + if updErr := r.Status().Update(ctx, tx); updErr != nil { + return ctrl.Result{}, updErr + } + return ctrl.Result{}, err + } + return r.advanceTo(ctx, tx, terminalPhase, "", nil) } -// reconcileDryRunCleanup restores pre-change state after a dry-run would-accept outcome. -// Re-uses snapshot rollback logic; terminal phase is Accepted (not Rejected). -func (r *CommittedResourceChangeReconciler) reconcileDryRunCleanup(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange) (ctrl.Result, error) { - // TODO (#780): same convergence as reconcileRollingBack; terminal → advanceTo(Accepted, nil). - return ctrl.Result{}, nil +// restoreFromSnapshot converges CommittedResources back to their pre-change state recorded in +// tx.Status.SnapshotBefore. It is idempotent and safe to call on every requeue. +// +// For each snapshot entry it deletes the CR (nil entry = CR was new) or restores its spec via +// CreateOrUpdate (non-nil entry = CR existed before). Once the spec is written back the CR is in +// sync with the pre-change state, which is all rollback can guarantee: whether the CR controller +// then re-places reservations is outside this reconciler's control, so there is nothing to wait for. +// +// Returns nil once all specs are written, or the API error (which should cause a requeue). +func (r *CommittedResourceChangeReconciler) restoreFromSnapshot(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange) error { + for uuid, snapSpec := range tx.Status.SnapshotBefore { + name := commitmentName(uuid) + if snapSpec == nil { + toDelete := &v1alpha1.CommittedResource{ObjectMeta: metav1.ObjectMeta{Name: name}} + if err := r.Delete(ctx, toDelete); client.IgnoreNotFound(err) != nil { + logger.Error(err, "rollback: failed to delete CR", "name", name, "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID) + return err + } + continue + } + cr := &v1alpha1.CommittedResource{ObjectMeta: metav1.ObjectMeta{Name: name}} + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, cr, func() error { + cr.Spec = *snapSpec + return nil + }); err != nil { + logger.Error(err, "rollback: failed to restore CR spec", "name", name, "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID) + return err + } + } + return nil } // advanceTo updates the transaction status to the given phase. -// mutate is called (when non-nil) to allow phase advance and other status mutations -// (e.g. UpsertedCommitments, SnapshotBefore) to be persisted atomically in one Update call. -// Progressing is set to False on terminal phases to reflect that no further work is expected. +// reason overrides the Finished/Progressing reason on terminal phases (e.g. RollbackFailed, +// InvalidSpec); when empty, string(phase) is used. mutate is called (when non-nil) to allow +// phase advance and other status mutations (e.g. UpsertedCommitments, SnapshotBefore, Message) +// to be persisted atomically in one Update call. // // Status().Update is used (not Patch) so that nil map values in SnapshotBefore and similar // fields are preserved — JSON merge patch semantics treat null map values as deletions. @@ -252,6 +325,7 @@ func (r *CommittedResourceChangeReconciler) advanceTo( ctx context.Context, tx *v1alpha1.CommittedResourceChange, phase v1alpha1.ChangePhase, + reason string, mutate func(*v1alpha1.CommittedResourceChangeStatus), ) (ctrl.Result, error) { @@ -259,18 +333,34 @@ func (r *CommittedResourceChangeReconciler) advanceTo( if mutate != nil { mutate(&tx.Status) } + // Stamp Restoring=True on entry to a restore phase. Its Status stays True for the duration, + // so LastTransitionTime is a stable "restore started" marker used to measure RollbackTimeout. + if phase == v1alpha1.ChangePhaseRollingBack || phase == v1alpha1.ChangePhaseDryRunCleanup { + meta.SetStatusCondition(&tx.Status.Conditions, metav1.Condition{ + Type: v1alpha1.ChangeConditionRestoring, + Status: metav1.ConditionTrue, + Reason: "Restoring", + Message: tx.Status.Message, + ObservedGeneration: tx.Generation, + }) + } progressingStatus, progressingReason := metav1.ConditionTrue, "Advancing" if tx.IsTerminal() { - progressingStatus, progressingReason = metav1.ConditionFalse, string(phase) + finishedReason := reason + if finishedReason == "" { + finishedReason = string(phase) + } + progressingStatus, progressingReason = metav1.ConditionFalse, finishedReason meta.SetStatusCondition(&tx.Status.Conditions, metav1.Condition{ Type: v1alpha1.ChangeConditionFinished, Status: metav1.ConditionTrue, - Reason: string(phase), + Reason: finishedReason, + Message: tx.Status.Message, ObservedGeneration: tx.Generation, }) } // Preserve a Progressing=False condition already stamped by the mutate func (e.g. TimedOut). - // Terminal phases always overwrite to ensure Progressing=False/. + // Terminal phases always overwrite to ensure Progressing=False/. existingProgressing := meta.FindStatusCondition(tx.Status.Conditions, v1alpha1.ChangeConditionProgressing) if tx.IsTerminal() || existingProgressing == nil || existingProgressing.Status != metav1.ConditionFalse { meta.SetStatusCondition(&tx.Status.Conditions, metav1.Condition{ diff --git a/internal/scheduling/reservations/commitments/changes/reconciler_test.go b/internal/scheduling/reservations/commitments/changes/reconciler_test.go index cfbdcebc8..430bc4cff 100644 --- a/internal/scheduling/reservations/commitments/changes/reconciler_test.go +++ b/internal/scheduling/reservations/commitments/changes/reconciler_test.go @@ -5,10 +5,12 @@ package changes import ( "context" + "errors" "strings" "testing" "time" + apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -17,6 +19,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "github.com/cobaltcore-dev/cortex/api/v1alpha1" ) @@ -36,6 +39,22 @@ func testScheme(t *testing.T) *runtime.Scheme { return s } +// crcByUpsertedCRIndexFunc indexes CRCs by the names of the CRs they upserted (test copy of the +// production field index) so the fake client can serve crToChangeRequest lookups. +func crcByUpsertedCRIndexFunc(obj client.Object) []string { + tx, ok := obj.(*v1alpha1.CommittedResourceChange) + if !ok { + return nil + } + names := make([]string, 0, len(tx.Status.UpsertedCommitments)) + for _, ref := range tx.Status.UpsertedCommitments { + if ref.Name != "" { + names = append(names, ref.Name) + } + } + return names +} + func testClient(t *testing.T, scheme *runtime.Scheme, objects ...client.Object) client.Client { t.Helper() return fake.NewClientBuilder(). @@ -45,18 +64,49 @@ func testClient(t *testing.T, scheme *runtime.Scheme, objects ...client.Object) &v1alpha1.CommittedResourceChange{}, &v1alpha1.CommittedResource{}, ). - WithIndex(&v1alpha1.CommittedResourceChange{}, idxCRCByUpsertedCR, func(obj client.Object) []string { - tx, ok := obj.(*v1alpha1.CommittedResourceChange) - if !ok { - return nil + WithIndex(&v1alpha1.CommittedResourceChange{}, idxCRCByUpsertedCR, crcByUpsertedCRIndexFunc). + Build() +} + +// testClientFailingCRWrites builds a client that returns an error on CommittedResource +// Delete/Update/Create while *fail is true, so tests can force restoreFromSnapshot to fail. +func testClientFailingCRWrites(t *testing.T, scheme *runtime.Scheme, fail *bool, objects ...client.Object) client.Client { + t.Helper() + failCR := func(obj client.Object) error { + if *fail { + if _, ok := obj.(*v1alpha1.CommittedResource); ok { + return apierrors.NewInternalError(errors.New("injected CR write failure")) } - names := make([]string, 0, len(tx.Status.UpsertedCommitments)) - for _, ref := range tx.Status.UpsertedCommitments { - if ref.Name != "" { - names = append(names, ref.Name) + } + return nil + } + return fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objects...). + WithStatusSubresource( + &v1alpha1.CommittedResourceChange{}, + &v1alpha1.CommittedResource{}, + ). + WithIndex(&v1alpha1.CommittedResourceChange{}, idxCRCByUpsertedCR, crcByUpsertedCRIndexFunc). + WithInterceptorFuncs(interceptor.Funcs{ + Delete: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if err := failCR(obj); err != nil { + return err } - } - return names + return cl.Delete(ctx, obj, opts...) + }, + Update: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + if err := failCR(obj); err != nil { + return err + } + return cl.Update(ctx, obj, opts...) + }, + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if err := failCR(obj); err != nil { + return err + } + return cl.Create(ctx, obj, opts...) + }, }). Build() } @@ -158,6 +208,22 @@ func backdateProgressingCondition(t *testing.T, c client.Client, txName string, } } +// backdateRestoringCondition sets the Restoring condition's LastTransitionTime to the past, +// simulating a restore that has been retrying for the given duration (to trigger RollbackTimeout). +func backdateRestoringCondition(t *testing.T, c client.Client, txName string, age time.Duration) { + t.Helper() + tx := getTx(t, c, txName) + for i, cond := range tx.Status.Conditions { + if cond.Type == v1alpha1.ChangeConditionRestoring { + tx.Status.Conditions[i].LastTransitionTime = metav1.NewTime(time.Now().Add(-age)) + break + } + } + if err := c.Status().Update(testCtx, tx); err != nil { + t.Fatalf("backdate Restoring condition on %q: %v", txName, err) + } +} + // assertCondition verifies a condition in the given list by type, status, and optional reason. // Pass an empty wantReason to skip the reason check. func assertCondition(t *testing.T, conditions []metav1.Condition, condType string, wantStatus metav1.ConditionStatus, wantReason string) { @@ -1080,3 +1146,314 @@ func TestCRTReconciler_Terminal(t *testing.T) { }) } } + +// ============================================================================ +// TestCRTReconciler_Restore (RollingBack + DryRunCleanup share reconcileRestore) +// ============================================================================ + +// restoreEntry selects how the harness drives a CRC into a restore phase. +type restoreEntry int + +const ( + entryReject restoreEntry = iota // reject a CR → RollingBack + entryAccept // accept all CRs (DryRun) → DryRunCleanup +) + +// TestCRTReconciler_Restore exercises the shared restore path for both RollingBack (terminal +// Rejected) and DryRunCleanup (terminal Accepted): snapshot restore, multi-CR, re-create of a +// deleted CR, idempotency, and the RollbackTimeout → RollbackFailed path. +func TestCRTReconciler_Restore(t *testing.T) { + tests := []struct { + name string + preObjects []client.Object + txSpec v1alpha1.CommittedResourceChangeSpec + entry restoreEntry + failRestore bool // inject a restore failure so RollbackTimeout fires + reconcileCount int // total reconciles in the restore phase (default 1; >1 checks idempotency) + wantPhase v1alpha1.ChangePhase + wantFinishedReason string + wantProgressingReason string + checkCRs func(t *testing.T, c client.Client) + }{ + { + name: "rollback: new CR is deleted", + entry: entryReject, + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-rb1": minimalCRSpec("az-1")}, + }, + wantPhase: v1alpha1.ChangePhaseRejected, + wantFinishedReason: string(v1alpha1.ChangePhaseRejected), + wantProgressingReason: string(v1alpha1.ChangePhaseRejected), + checkCRs: func(t *testing.T, c client.Client) { + if crExists(t, c, "commitment-uuid-rb1") { + t.Errorf("new CR should be deleted by rollback but exists") + } + }, + }, + { + name: "rollback: existing CR spec is restored", + preObjects: []client.Object{ + &v1alpha1.CommittedResource{ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-rb2"}, Spec: *minimalCRSpec("az-old")}, + }, + entry: entryReject, + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-rb2": minimalCRSpec("az-new")}, + }, + wantPhase: v1alpha1.ChangePhaseRejected, + wantFinishedReason: string(v1alpha1.ChangePhaseRejected), + wantProgressingReason: string(v1alpha1.ChangePhaseRejected), + checkCRs: func(t *testing.T, c client.Client) { + if got := getCR(t, c, "commitment-uuid-rb2").Spec.AvailabilityZone; got != "az-old" { + t.Errorf("restored CR AZ: want az-old, got %q", got) + } + }, + }, + { + name: "rollback: deleted CR is re-created", + preObjects: []client.Object{ + &v1alpha1.CommittedResource{ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-del"}, Spec: *minimalCRSpec("az-old")}, + }, + entry: entryReject, + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{ + "uuid-del": nil, // delete an existing CR (snapshot non-nil) + "uuid-new": minimalCRSpec("az-new"), // new CR that will be rejected + }, + }, + wantPhase: v1alpha1.ChangePhaseRejected, + wantFinishedReason: string(v1alpha1.ChangePhaseRejected), + wantProgressingReason: string(v1alpha1.ChangePhaseRejected), + checkCRs: func(t *testing.T, c client.Client) { + // The deleted CR must be re-created with its original spec. + if got := getCR(t, c, "commitment-uuid-del").Spec.AvailabilityZone; got != "az-old" { + t.Errorf("re-created CR AZ: want az-old, got %q", got) + } + // The new CR must be removed. + if crExists(t, c, "commitment-uuid-new") { + t.Errorf("new CR should be deleted by rollback but exists") + } + }, + }, + { + name: "rollback: multi-CR restores all", + preObjects: []client.Object{ + &v1alpha1.CommittedResource{ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-m1"}, Spec: *minimalCRSpec("az-old")}, + &v1alpha1.CommittedResource{ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-m2"}, Spec: *minimalCRSpec("az-old")}, + }, + entry: entryReject, + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{ + "uuid-m1": minimalCRSpec("az-new"), + "uuid-m2": minimalCRSpec("az-new"), + }, + }, + wantPhase: v1alpha1.ChangePhaseRejected, + wantFinishedReason: string(v1alpha1.ChangePhaseRejected), + wantProgressingReason: string(v1alpha1.ChangePhaseRejected), + checkCRs: func(t *testing.T, c client.Client) { + for _, name := range []string{"commitment-uuid-m1", "commitment-uuid-m2"} { + if got := getCR(t, c, name).Spec.AvailabilityZone; got != "az-old" { + t.Errorf("CR %q AZ: want az-old (restored), got %q", name, got) + } + } + }, + }, + { + name: "rollback: idempotent across repeated reconciles", + entry: entryReject, + reconcileCount: 3, + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-idem-rb": minimalCRSpec("az-1")}, + }, + wantPhase: v1alpha1.ChangePhaseRejected, + wantFinishedReason: string(v1alpha1.ChangePhaseRejected), + wantProgressingReason: string(v1alpha1.ChangePhaseRejected), + }, + { + name: "rollback: restore keeps failing → RollbackFailed/Rejected", + entry: entryReject, + failRestore: true, + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-rbto": minimalCRSpec("az-1")}, + }, + wantPhase: v1alpha1.ChangePhaseRejected, + wantFinishedReason: v1alpha1.ChangeReasonRollbackFailed, + wantProgressingReason: v1alpha1.ChangeReasonRollbackFailed, + }, + { + name: "dry-run cleanup: probe CR deleted, verdict preserved as Accepted", + entry: entryAccept, + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, AllowRejection: true, DryRun: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dry1": minimalCRSpec("az-1")}, + }, + wantPhase: v1alpha1.ChangePhaseAccepted, + wantFinishedReason: string(v1alpha1.ChangePhaseAccepted), + wantProgressingReason: string(v1alpha1.ChangePhaseAccepted), + checkCRs: func(t *testing.T, c client.Client) { + if crExists(t, c, "commitment-uuid-dry1") { + t.Errorf("dry-run probe CR should be deleted on cleanup but exists") + } + }, + }, + { + name: "dry-run would-reject: follows RollingBack → Rejected", + entry: entryReject, + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, AllowRejection: true, DryRun: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dr": minimalCRSpec("az-1")}, + }, + wantPhase: v1alpha1.ChangePhaseRejected, + wantFinishedReason: string(v1alpha1.ChangePhaseRejected), + wantProgressingReason: string(v1alpha1.ChangePhaseRejected), + }, + { + name: "dry-run cleanup: restore keeps failing → RollbackFailed/Accepted", + entry: entryAccept, + failRestore: true, + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, AllowRejection: true, DryRun: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-dryto": minimalCRSpec("az-1")}, + }, + wantPhase: v1alpha1.ChangePhaseAccepted, + wantFinishedReason: v1alpha1.ChangeReasonRollbackFailed, + wantProgressingReason: v1alpha1.ChangeReasonRollbackFailed, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := testScheme(t) + const txName = "tx-restore" + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: txName}, + Spec: tt.txSpec, + } + objs := append([]client.Object{tx}, tt.preObjects...) + + fail := false + var c client.Client + if tt.failRestore { + c = testClientFailingCRWrites(t, scheme, &fail, objs...) + } else { + c = testClient(t, scheme, objs...) + } + r := testReconciler(c, scheme) + if tt.failRestore { + r.RollbackTimeout = 50 * time.Millisecond + } + + driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseAwaitingScheduling) + upserted := getTx(t, c, txName).Status.UpsertedCommitments + + switch tt.entry { + case entryReject: + setCRReady(t, c, upserted[0].Name, metav1.ConditionFalse, v1alpha1.CommittedResourceReasonRejected, "no capacity") + driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseRollingBack) + case entryAccept: + for _, ref := range upserted { + setCRReady(t, c, ref.Name, metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted, "") + } + driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseDryRunCleanup) + } + + if tt.failRestore { + fail = true + backdateRestoringCondition(t, c, txName, 1*time.Second) + } + + n := tt.reconcileCount + if n == 0 { + n = 1 + } + reconcileN(t, r, txName, n) + + tx = getTx(t, c, txName) + if tx.Status.Phase != tt.wantPhase { + t.Errorf("phase: want %q, got %q", tt.wantPhase, tx.Status.Phase) + } + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionFinished, metav1.ConditionTrue, tt.wantFinishedReason) + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionProgressing, metav1.ConditionFalse, tt.wantProgressingReason) + if tt.checkCRs != nil { + tt.checkCRs(t, c) + } + }) + } +} + +// TestCRTReconciler_DryRunGuards verifies the spec guards reject invalid dry-run requests. +func TestCRTReconciler_DryRunGuards(t *testing.T) { + tests := []struct { + name string + preObjects []client.Object + txSpec v1alpha1.CommittedResourceChangeSpec + wantMessage string + }{ + { + name: "dryRun without allowRejection is rejected", + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: false, + DryRun: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-g1": minimalCRSpec("az-1")}, + }, + wantMessage: "dryRun requires allowRejection", + }, + { + name: "dryRun modifying an existing commitment is rejected", + preObjects: []client.Object{ + &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-g2"}, + Spec: *minimalCRSpec("az-old"), + }, + }, + txSpec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + DryRun: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-g2": minimalCRSpec("az-new")}, + }, + wantMessage: "must not modify existing commitment", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := testScheme(t) + txName := "tx-guard-" + tt.name[:5] + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: txName}, + Spec: tt.txSpec, + } + objs := append([]client.Object{tx}, tt.preObjects...) + c := testClient(t, scheme, objs...) + r := testReconciler(c, scheme) + + doReconcile(t, r, txName) + + tx = getTx(t, c, txName) + if tx.Status.Phase != v1alpha1.ChangePhaseRejected { + t.Errorf("phase: want Rejected, got %q", tx.Status.Phase) + } + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionFinished, metav1.ConditionTrue, v1alpha1.ChangeReasonInvalidSpec) + if !strings.Contains(tx.Status.Message, tt.wantMessage) { + t.Errorf("message: want substring %q in %q", tt.wantMessage, tx.Status.Message) + } + // A rejected dry-run must not have created or mutated the target CR beyond the guard point. + if len(tt.preObjects) > 0 { + // existing CR must be untouched (still az-old) + if got := getCR(t, c, "commitment-uuid-g2").Spec.AvailabilityZone; got != "az-old" { + t.Errorf("existing CR must be untouched: AZ want az-old, got %q", got) + } + } + }) + } +}