diff --git a/api/v1alpha1/committed_resource_change_types.go b/api/v1alpha1/committed_resource_change_types.go new file mode 100644 index 000000000..29fb90e13 --- /dev/null +++ b/api/v1alpha1/committed_resource_change_types.go @@ -0,0 +1,164 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ChangePhase represents the FSM phase of a CommittedResourceChange. +type ChangePhase string + +const ( + // ChangePhasePreparing: change request created; sanity checks and optional snapshot in progress. + ChangePhasePreparing ChangePhase = "Preparing" + // ChangePhaseAllocating: CR specs are being written. + ChangePhaseAllocating ChangePhase = "Allocating" + // ChangePhaseAwaitingScheduling: CR specs written; waiting for placement success. + ChangePhaseAwaitingScheduling ChangePhase = "AwaitingScheduling" + // ChangePhaseRollingBack: a CR was rejected or timed out; restoring pre-change state. + ChangePhaseRollingBack ChangePhase = "RollingBack" + // ChangePhaseDryRunCleanup: dry-run would-accept; restoring pre-change state. + ChangePhaseDryRunCleanup ChangePhase = "DryRunCleanup" + // ChangePhaseAccepted: terminal; change request committed successfully. + ChangePhaseAccepted ChangePhase = "Accepted" + // ChangePhaseRejected: terminal; change request rolled back. + ChangePhaseRejected ChangePhase = "Rejected" + // ChangePhaseSuperseded: terminal; abandoned because a newer change request for the same AZ took over. + ChangePhaseSuperseded ChangePhase = "Superseded" +) + +// TxSource identifies the originator of a CommittedResourceChange. +type TxSource string + +const ( + // TxSourceAPI: change request created by the change-commitments HTTP API. + TxSourceAPI TxSource = "api" + // TxSourceSyncer: change request created by the Limes syncer. + TxSourceSyncer TxSource = "syncer" +) + +const ( + // ChangeConditionProgressing is True while the change request is advancing normally. + // False + reason indicates a problem while still non-terminal. + ChangeConditionProgressing = "Progressing" + + // ChangeConditionFinished is True once the change request has reached a terminal phase. + ChangeConditionFinished = "Finished" + + // Reasons for ChangeConditionProgressing=False (non-terminal problems). + ChangeReasonStalled = "Stalled" + ChangeReasonTimedOut = "TimedOut" + ChangeReasonError = "Error" + ChangeReasonRetryExhausted = "RetryExhausted" + + // Reasons for ChangeConditionFinished=True. + ChangeRequestReasonAccepted = "Accepted" + ChangeRequestReasonRejected = "Rejected" + ChangeRequestReasonSuperseded = "Superseded" +) + +// CommittedResourceChangeSpec defines the desired change for a CommittedResourceChange. +type CommittedResourceChangeSpec struct { + // CorrelationID is the request/correlation ID. + // +kubebuilder:validation:Optional + CorrelationID string `json:"correlationID,omitempty"` + + // AvailabilityZone is the availability zone this change request is scoped to. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + AvailabilityZone string `json:"availabilityZone"` + + // Source identifies who created this change request. + // +kubebuilder:validation:Enum=api;syncer + // +kubebuilder:validation:Required + Source TxSource `json:"source"` + + // AllowRejection controls whether the change request may be rejected and rolled back + // on placement failure. When false, the reconciler retries until Accepted or Superseded. + // +kubebuilder:validation:Optional + AllowRejection bool `json:"allowRejection,omitempty"` + + // DryRun controls whether the change request commits or only probes schedulability. + // +kubebuilder:validation:Optional + DryRun bool `json:"dryRun,omitempty"` + + // NewState maps commitmentUUID to the desired CommittedResourceSpec. + // A nil value means the corresponding CommittedResource should be deleted. + // +kubebuilder:validation:Optional + NewState map[string]*CommittedResourceSpec `json:"newState,omitempty"` +} + +// CommittedResourceChangeStatus defines the observed state of a CommittedResourceChange. +type CommittedResourceChangeStatus struct { + // Phase is the current FSM phase of the change request. + // +kubebuilder:validation:Optional + // +kubebuilder:validation:XValidation:rule="oldSelf == '' || !['Accepted','Rejected','Superseded'].exists(t, t == oldSelf) || self == oldSelf",message="phase cannot transition out of a terminal state" + Phase ChangePhase `json:"phase,omitempty"` + + // SnapshotBefore records the CommittedResource spec state before this change request. + // Key is commitmentUUID; nil value means the CR did not exist before (will be deleted on rollback). + // Populated in Preparing when AllowRejection=true. + // +kubebuilder:validation:Optional + SnapshotBefore map[string]*CommittedResourceSpec `json:"snapshotBefore,omitempty"` + + // UpsertedCommitments holds references to CommittedResource CRDs written by this change request. + // Watched in AwaitingScheduling to gate on placement success. + // These are plain object references, not owner references. + // +kubebuilder:validation:Optional + UpsertedCommitments []corev1.ObjectReference `json:"upsertedCommitments,omitempty"` + + // Conditions holds Progressing and Finished conditions. + // +kubebuilder:validation:Optional + Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` + + // Message is a human-readable description of the current state or terminal outcome. + // +kubebuilder:validation:Optional + Message string `json:"message,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster,shortName=crc +// +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase" +// +kubebuilder:printcolumn:name="AZ",type="string",JSONPath=".spec.availabilityZone" +// +kubebuilder:printcolumn:name="Source",type="string",JSONPath=".spec.source" +// +kubebuilder:printcolumn:name="DryRun",type="boolean",JSONPath=".spec.dryRun",priority=1 +// +kubebuilder:printcolumn:name="Progressing",type="string",JSONPath=".status.conditions[?(@.type=='Progressing')].status",priority=1 +// +kubebuilder:printcolumn:name="Finished",type="string",JSONPath=".status.conditions[?(@.type=='Finished')].status",priority=1 + +// CommittedResourceChange is the Schema for the committedresourcechanges API +type CommittedResourceChange struct { + metav1.TypeMeta `json:",inline"` + + // +optional + metav1.ObjectMeta `json:"metadata,omitempty,omitzero"` + + // +required + Spec CommittedResourceChangeSpec `json:"spec"` + + // +optional + Status CommittedResourceChangeStatus `json:"status,omitempty,omitzero"` +} + +// IsTerminal reports whether the change request has reached a terminal phase. +func (tx *CommittedResourceChange) IsTerminal() bool { + return tx.Status.Phase == ChangePhaseAccepted || + tx.Status.Phase == ChangePhaseRejected || + tx.Status.Phase == ChangePhaseSuperseded +} + +// +kubebuilder:object:root=true + +// CommittedResourceChangeList contains a list of CommittedResourceChange +type CommittedResourceChangeList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CommittedResourceChange `json:"items"` +} + +func init() { + SchemeBuilder.Register(&CommittedResourceChange{}, &CommittedResourceChangeList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index c4f4115ff..a60d5a1f4 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -9,9 +9,9 @@ package v1alpha1 import ( apiv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" - corev1 "k8s.io/api/core/v1" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -80,6 +80,139 @@ func (in *CommittedResourceAllocation) DeepCopy() *CommittedResourceAllocation { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CommittedResourceChange) DeepCopyInto(out *CommittedResourceChange) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CommittedResourceChange. +func (in *CommittedResourceChange) DeepCopy() *CommittedResourceChange { + if in == nil { + return nil + } + out := new(CommittedResourceChange) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CommittedResourceChange) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CommittedResourceChangeList) DeepCopyInto(out *CommittedResourceChangeList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CommittedResourceChange, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CommittedResourceChangeList. +func (in *CommittedResourceChangeList) DeepCopy() *CommittedResourceChangeList { + if in == nil { + return nil + } + out := new(CommittedResourceChangeList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CommittedResourceChangeList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CommittedResourceChangeSpec) DeepCopyInto(out *CommittedResourceChangeSpec) { + *out = *in + if in.NewState != nil { + in, out := &in.NewState, &out.NewState + *out = make(map[string]*CommittedResourceSpec, len(*in)) + for key, val := range *in { + var outVal *CommittedResourceSpec + if val == nil { + (*out)[key] = nil + } else { + inVal := (*in)[key] + in, out := &inVal, &outVal + *out = new(CommittedResourceSpec) + (*in).DeepCopyInto(*out) + } + (*out)[key] = outVal + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CommittedResourceChangeSpec. +func (in *CommittedResourceChangeSpec) DeepCopy() *CommittedResourceChangeSpec { + if in == nil { + return nil + } + out := new(CommittedResourceChangeSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CommittedResourceChangeStatus) DeepCopyInto(out *CommittedResourceChangeStatus) { + *out = *in + if in.SnapshotBefore != nil { + in, out := &in.SnapshotBefore, &out.SnapshotBefore + *out = make(map[string]*CommittedResourceSpec, len(*in)) + for key, val := range *in { + var outVal *CommittedResourceSpec + if val == nil { + (*out)[key] = nil + } else { + inVal := (*in)[key] + in, out := &inVal, &outVal + *out = new(CommittedResourceSpec) + (*in).DeepCopyInto(*out) + } + (*out)[key] = outVal + } + } + if in.UpsertedCommitments != nil { + in, out := &in.UpsertedCommitments, &out.UpsertedCommitments + *out = make([]v1.ObjectReference, len(*in)) + copy(*out, *in) + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CommittedResourceChangeStatus. +func (in *CommittedResourceChangeStatus) DeepCopy() *CommittedResourceChangeStatus { + if in == nil { + return nil + } + out := new(CommittedResourceChangeStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CommittedResourceList) DeepCopyInto(out *CommittedResourceList) { *out = *in @@ -223,7 +356,7 @@ func (in *CommittedResourceStatus) DeepCopyInto(out *CommittedResourceStatus) { } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make([]v1.Condition, len(*in)) + *out = make([]metav1.Condition, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -307,7 +440,7 @@ func (in *DatasourceSpec) DeepCopyInto(out *DatasourceSpec) { out.DatabaseSecretRef = in.DatabaseSecretRef if in.SSOSecretRef != nil { in, out := &in.SSOSecretRef, &out.SSOSecretRef - *out = new(corev1.SecretReference) + *out = new(v1.SecretReference) **out = **in } } @@ -334,7 +467,7 @@ func (in *DatasourceStatus) DeepCopyInto(out *DatasourceStatus) { in.NextSyncTime.DeepCopyInto(&out.NextSyncTime) if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make([]v1.Condition, len(*in)) + *out = make([]metav1.Condition, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -484,12 +617,12 @@ func (in *DecisionSpec) DeepCopyInto(out *DecisionSpec) { } if in.MachineRef != nil { in, out := &in.MachineRef, &out.MachineRef - *out = new(corev1.ObjectReference) + *out = new(v1.ObjectReference) **out = **in } if in.PodRef != nil { in, out := &in.PodRef, &out.PodRef - *out = new(corev1.ObjectReference) + *out = new(v1.ObjectReference) **out = **in } } @@ -514,10 +647,10 @@ func (in *DecisionStatus) DeepCopyInto(out *DecisionStatus) { } if in.History != nil { in, out := &in.History, &out.History - *out = new([]corev1.ObjectReference) + *out = new([]v1.ObjectReference) if **in != nil { in, out := *in, *out - *out = make([]corev1.ObjectReference, len(*in)) + *out = make([]v1.ObjectReference, len(*in)) copy(*out, *in) } } @@ -528,7 +661,7 @@ func (in *DecisionStatus) DeepCopyInto(out *DecisionStatus) { } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make([]v1.Condition, len(*in)) + *out = make([]metav1.Condition, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -624,7 +757,7 @@ func (in *DeschedulingStatus) DeepCopyInto(out *DeschedulingStatus) { *out = *in if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make([]v1.Condition, len(*in)) + *out = make([]metav1.Condition, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -914,7 +1047,7 @@ func (in *FlavorGroupCapacityStatus) DeepCopyInto(out *FlavorGroupCapacityStatus in.LastReconcileAt.DeepCopyInto(&out.LastReconcileAt) if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make([]v1.Condition, len(*in)) + *out = make([]metav1.Condition, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -1008,12 +1141,12 @@ func (in *KPIDependenciesSpec) DeepCopyInto(out *KPIDependenciesSpec) { *out = *in if in.Datasources != nil { in, out := &in.Datasources, &out.Datasources - *out = make([]corev1.ObjectReference, len(*in)) + *out = make([]v1.ObjectReference, len(*in)) copy(*out, *in) } if in.Knowledges != nil { in, out := &in.Knowledges, &out.Knowledges - *out = make([]corev1.ObjectReference, len(*in)) + *out = make([]v1.ObjectReference, len(*in)) copy(*out, *in) } } @@ -1082,7 +1215,7 @@ func (in *KPIStatus) DeepCopyInto(out *KPIStatus) { *out = *in if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make([]v1.Condition, len(*in)) + *out = make([]metav1.Condition, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -1131,12 +1264,12 @@ func (in *KnowledgeDependenciesSpec) DeepCopyInto(out *KnowledgeDependenciesSpec *out = *in if in.Datasources != nil { in, out := &in.Datasources, &out.Datasources - *out = make([]corev1.ObjectReference, len(*in)) + *out = make([]v1.ObjectReference, len(*in)) copy(*out, *in) } if in.Knowledges != nil { in, out := &in.Knowledges, &out.Knowledges - *out = make([]corev1.ObjectReference, len(*in)) + *out = make([]v1.ObjectReference, len(*in)) copy(*out, *in) } } @@ -1225,7 +1358,7 @@ func (in *KnowledgeStatus) DeepCopyInto(out *KnowledgeStatus) { in.Raw.DeepCopyInto(&out.Raw) if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make([]v1.Condition, len(*in)) + *out = make([]metav1.Condition, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -1491,7 +1624,7 @@ func (in *PipelineStatus) DeepCopyInto(out *PipelineStatus) { *out = *in if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make([]v1.Condition, len(*in)) + *out = make([]metav1.Condition, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -1631,7 +1764,7 @@ func (in *ProjectQuotaStatus) DeepCopyInto(out *ProjectQuotaStatus) { } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make([]v1.Condition, len(*in)) + *out = make([]metav1.Condition, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -1781,7 +1914,7 @@ func (in *ReservationStatus) DeepCopyInto(out *ReservationStatus) { *out = *in if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make([]v1.Condition, len(*in)) + *out = make([]metav1.Condition, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } diff --git a/docs/03-reservations-and-inventory/03-b-committed-resource-transaction-architecture.md b/docs/03-reservations-and-inventory/03-b-committed-resource-transaction-architecture.md new file mode 100644 index 000000000..c35c5a6ae --- /dev/null +++ b/docs/03-reservations-and-inventory/03-b-committed-resource-transaction-architecture.md @@ -0,0 +1,125 @@ + + +# Committed-resource transaction architecture (target) + +## Components + +| Component | Kind | Owned by | +|-----------|------|----------| +| `CommittedResourceChange` (CRC) | CRD | Spec: created by API handler or syncer; Status: CRC reconciler | +| `CommittedResource` (CR) | CRD | Spec: CRC reconciler (sole writer); Status: CR controller | +| `Reservation` | CRD | Spec: CR controller (sole writer); Status: Reservation controller | +| Change-commitments API | HTTP handler | Creates CRC with `source=api` | +| Limes syncer | Background poller | Creates CRC with `source=syncer` on each diff | +| CRC reconciler | Controller | Drives CRC FSM; writes CR specs; rolls back on failure | +| CR controller | Controller | Derives Reservation slots from CR spec; reports CR readiness | +| Reservation controller | Controller | Attempts host placement; reports slot readiness | + +## Ownership + +```mermaid +flowchart TD + API["API handler\nsource=api"] -->|creates| CRC["CommittedResourceChange"] + Syncer["Limes syncer\nsource=syncer"] -->|creates on diff| CRC + + CRC -->|"CRC reconciler — writes spec"| CR["CommittedResource"] + CR -->|"CR controller — writes spec"| Res["Reservation"] + Res -->|"Reservation controller — tracks placement"| VM["VM / scheduler\nlive-migration"] + + CR -.->|"CR controller reports readiness\nCRC reconciler gates on this"| CRC +``` + +No controller writes its own spec. Spec flows down; status flows up. + +## Component interaction + +The three reconcilers operate independently and communicate only through CRD status conditions. + +```mermaid +sequenceDiagram + participant W as API handler or Syncer + participant CRC as CRC reconciler + participant CRCD as CommittedResource CRD + participant CRCtl as CR controller + participant ResCRD as Reservation CRD + participant ResCtl as Reservation controller + + W->>CRCD: create CommittedResourceChange + CRC->>CRCD: write CR.spec (Allocating phase) + Note over CRC,CRCD: CRC reconciler is sole writer of CR.spec + CRCtl->>ResCRD: reconcile Reservation slots from CR.spec + Note over CRCtl,ResCRD: CR controller is sole writer of Reservation.spec + ResCtl->>ResCRD: attempt placement, write Reservation.status + Note over ResCtl: reacts to VM moves, live-migration, in-flight actions + CRCtl->>CRCD: aggregate slot readiness, write CR.status + CRC->>CRCD: read CR.status.Ready + observedGeneration, advance to Accepted or RollingBack +``` + +## CRC state machine + +```mermaid +stateDiagram-v2 + [*] --> Preparing : CRC created by API handler or syncer + + Preparing --> Allocating : snapshot taken + + Allocating --> AwaitingScheduling : API mode — CR specs written + Allocating --> Accepted : Syncer mode — apply-and-done + Allocating --> Superseded : newer CRC for same AZ took over + + AwaitingScheduling --> Accepted : all CRs Ready (observedGeneration matches) + AwaitingScheduling --> DryRunCleanup : all CRs Ready, but DryRun=true + AwaitingScheduling --> RollingBack : CR rejected or timeout + + DryRunCleanup --> Accepted : pre-state restored (would-accept) + + RollingBack --> Rejected : pre-state restored (RollbackFailed on permanent failure) + + Accepted --> [*] + Rejected --> [*] + Superseded --> [*] +``` + +Terminal phases (`Accepted`, `Rejected`, `Superseded`) set `Finished=True`. Non-terminal problems set +`Progressing=False` with a reason (`Stalled`, `TimedOut`, `Error`, `RetryExhausted`) without advancing +the phase. A stuck non-terminal CRC triggers an alert; after a threshold it is forcibly moved to +`Superseded` so its AZ is unblocked and the syncer repairs state on the next poll. + +## Request flows + +### API-mode flow + +1. **API handler** receives a change-commitments request from Limes; creates a CRC (`source=api`, `allowRejection=true`, optional `dryRun=true`). +2. **CRC reconciler — Preparing**: validates the request; snapshots current CR specs as the rollback journal. +3. **CRC reconciler — Allocating**: writes the desired CR specs atomically; records which CRs were touched. +4. **CRC reconciler — AwaitingScheduling**: watches the touched CRs; waits until each reports its ready condition against the generation just written. +5. **CR controller**: for each CR, derives and reconciles Reservation slots; reports `Ready` once all slots are placed. +6. **Reservation controller**: places each slot on a host; reports slot readiness. +7. **CRC reconciler**: all CRs ready → `Accepted`; any CR rejected or timeout → rolls all CR specs back to the snapshot → `Rejected`. +8. **API handler** polls the CRC by name; maps terminal phase to HTTP response. + +### Syncer-mode flow + +1. **Limes syncer** polls Limes per project; diffs desired state against current CRs; on any delta creates a CRC (`source=syncer`, `allowRejection=false`). +2. **CRC reconciler — Preparing**: no snapshot (non-rejectable; syncer repairs on next poll if anything drifts). +3. **CRC reconciler — Allocating**: recency check — skips changes for any project another CRC touched in the same AZ in the last ~10 minutes (prevents clobbering an in-flight API change); writes CR specs; advances directly to `Accepted`. +4. **CR controller**: reconciles Reservation slots async; CRC is already terminal. +5. Slots converge in the background; the next syncer poll repairs any remaining drift. + +## Key invariants + +- **Single writer per spec layer.** The CRC reconciler is the only component that writes `CommittedResource.spec`. The CR controller is the only component that writes `Reservation.spec`. No controller writes its own spec. +- **AwaitingScheduling gates on generation.** The CRC reconciler advances from `AwaitingScheduling` only when a CR's ready condition carries an `observedGeneration` that matches the generation the CRC reconciler wrote — not an older reconcile cycle's result. +- **Rollback is durable.** The pre-change snapshot is stored in `CRC.status` before any CR spec is touched. A process restart resumes from the persisted phase and snapshot. +- **Syncer-mode never rolls back.** Syncer CRTs are non-rejectable: on any error they retry until `Accepted` or are superseded by a newer CRC. The syncer repairs drift on its next poll. +- **At most one active CRC per AZ.** The CRC reconciler serialises with `MaxConcurrentReconciles=1` plus an in-memory per-AZ active-transaction map (rebuilt from CRDs on startup). Younger CRTs requeue until the active one reaches a terminal phase. +- **Syncer recency guard prevents clobbering.** A syncer CRC skips changes to any project that an API CRC touched in the same AZ within the last ~10 minutes — the API change is authoritative; the syncer will reconcile remaining drift on its next poll. +- **Terminal CRTs are retained for auditability.** `Accepted` CRTs are kept for ~1 hour; `Rejected` CRTs for ~5 days. Non-terminal CRCs (including stuck `RollingBack`) are never GC'd — deleting one loses the rollback journal and the AZ active-marker. + +--- + +[Prev: CR lifecycle internals](03-a-cr-lifecycle-internals.md) · [Next: Failover reservations for KVM HA »](03-failover-reservations.md) diff --git a/helm/bundles/cortex-nova/templates/kpis_kvm.yaml b/helm/bundles/cortex-nova/templates/kpis_kvm.yaml index 7fc4f0223..51d1a6df4 100644 --- a/helm/bundles/cortex-nova/templates/kpis_kvm.yaml +++ b/helm/bundles/cortex-nova/templates/kpis_kvm.yaml @@ -52,6 +52,7 @@ spec: gvks: - cortex.cloud/v1alpha1/ReservationList - cortex.cloud/v1alpha1/CommittedResourceList + - cortex.cloud/v1alpha1/CommittedResourceChangeList - kvm.cloud.sap/v1/HypervisorList - cobaltcore.cloud/v1alpha1/VirtualMachineList description: | diff --git a/helm/bundles/cortex-nova/values.yaml b/helm/bundles/cortex-nova/values.yaml index 28b643566..5a05833e7 100644 --- a/helm/bundles/cortex-nova/values.yaml +++ b/helm/bundles/cortex-nova/values.yaml @@ -137,6 +137,8 @@ cortex: &cortex - cortex.cloud/v1alpha1/ReservationList - cortex.cloud/v1alpha1/CommittedResource - cortex.cloud/v1alpha1/CommittedResourceList + - cortex.cloud/v1alpha1/CommittedResourceChange + - cortex.cloud/v1alpha1/CommittedResourceChangeList - cortex.cloud/v1alpha1/ProjectQuota - cortex.cloud/v1alpha1/ProjectQuotaList - cortex.cloud/v1alpha1/FlavorGroupCapacity @@ -159,6 +161,7 @@ cortex: &cortex gvks: - cortex.cloud/v1alpha1/Reservation - cortex.cloud/v1alpha1/CommittedResource + - cortex.cloud/v1alpha1/CommittedResourceChange monitoring: labels: &cortexMonitoringLabels github_org: cobaltcore-dev diff --git a/helm/library/cortex/files/crds/cortex.cloud_committedresourcechanges.yaml b/helm/library/cortex/files/crds/cortex.cloud_committedresourcechanges.yaml new file mode 100644 index 000000000..ecf607f4c --- /dev/null +++ b/helm/library/cortex/files/crds/cortex.cloud_committedresourcechanges.yaml @@ -0,0 +1,421 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.23.0 + name: committedresourcechanges.cortex.cloud +spec: + group: cortex.cloud + names: + kind: CommittedResourceChange + listKind: CommittedResourceChangeList + plural: committedresourcechanges + shortNames: + - crc + singular: committedresourcechange + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .spec.availabilityZone + name: AZ + type: string + - jsonPath: .spec.source + name: Source + type: string + - jsonPath: .spec.dryRun + name: DryRun + priority: 1 + type: boolean + - jsonPath: .status.conditions[?(@.type=='Progressing')].status + name: Progressing + priority: 1 + type: string + - jsonPath: .status.conditions[?(@.type=='Finished')].status + name: Finished + priority: 1 + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: CommittedResourceChange is the Schema for the committedresourcechanges + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CommittedResourceChangeSpec defines the desired change for + a CommittedResourceChange. + properties: + allowRejection: + description: |- + AllowRejection controls whether the change request may be rejected and rolled back + on placement failure. When false, the reconciler retries until Accepted or Superseded. + type: boolean + availabilityZone: + description: AvailabilityZone is the availability zone this change + request is scoped to. + minLength: 1 + type: string + correlationID: + description: CorrelationID is the request/correlation ID. + type: string + dryRun: + description: DryRun controls whether the change request commits or + only probes schedulability. + type: boolean + newState: + additionalProperties: + description: CommittedResourceSpec defines the desired state of + CommittedResource, + properties: + allowRejection: + description: |- + AllowRejection controls what the CommittedResource controller does when placement fails + for a guaranteed or confirmed commitment. + true — controller may reject: on failure, child Reservations are rolled back and the CR + is marked Rejected. Use this when the caller is making a first-time placement + decision and a "no" answer is acceptable (e.g. the change-commitments API). + false — controller must retry: on failure, existing child Reservations are kept and the + CR is set to Reserving so the controller retries later. Use this when the caller + is restoring already-committed state that Cortex must honour (e.g. the syncer). + Only meaningful for state=guaranteed or state=confirmed; ignored for all other states. + type: boolean + amount: + anyOf: + - type: integer + - type: string + description: |- + Amount is the total committed quantity. + memory: MiB expressed in K8s binary SI notation (e.g. "1280Gi", "640Mi"). + cores: integer core count (e.g. "40"). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + availabilityZone: + description: AvailabilityZone specifies the availability zone + for this commitment. + type: string + commitmentUUID: + description: UUID of the commitment this resource corresponds + to. + type: string + confirmedAt: + description: ConfirmedAt is when the commitment was confirmed. + format: date-time + type: string + domainID: + description: DomainID of the OpenStack domain this commitment + belongs to. + type: string + endTime: + description: EndTime is when Reservation slots expire. Nil for + unbounded commitments with no expiry. + format: date-time + type: string + flavorGroupName: + description: FlavorGroupName identifies the flavor group this + commitment targets, e.g. "kvm_v2_hana_s". + type: string + projectID: + description: ProjectID of the OpenStack project this commitment + belongs to. + type: string + resourceType: + description: 'ResourceType identifies the kind of resource committed: + memory drives Reservation slots; cores uses an arithmetic + check only.' + enum: + - memory + - cores + type: string + schedulingDomain: + description: SchedulingDomain specifies the scheduling domain + for this committed resource (e.g., "nova", "ironcore"). + type: string + startTime: + description: |- + StartTime is the activation time for Reservation slots. + Nil for guaranteed commitments (slots are active from creation); set to ConfirmedAt for confirmed ones. + format: date-time + type: string + state: + description: State is the lifecycle state of the commitment. + enum: + - planned + - pending + - guaranteed + - confirmed + - superseded + - expired + type: string + required: + - amount + - availabilityZone + - commitmentUUID + - domainID + - flavorGroupName + - projectID + - resourceType + - state + type: object + description: |- + NewState maps commitmentUUID to the desired CommittedResourceSpec. + A nil value means the corresponding CommittedResource should be deleted. + type: object + source: + description: Source identifies who created this change request. + enum: + - api + - syncer + type: string + required: + - availabilityZone + - source + type: object + status: + description: CommittedResourceChangeStatus defines the observed state + of a CommittedResourceChange. + properties: + conditions: + description: Conditions holds Progressing and Finished conditions. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + message: + description: Message is a human-readable description of the current + state or terminal outcome. + type: string + phase: + description: Phase is the current FSM phase of the change request. + type: string + x-kubernetes-validations: + - message: phase cannot transition out of a terminal state + rule: oldSelf == '' || !['Accepted','Rejected','Superseded'].exists(t, + t == oldSelf) || self == oldSelf + snapshotBefore: + additionalProperties: + description: CommittedResourceSpec defines the desired state of + CommittedResource, + properties: + allowRejection: + description: |- + AllowRejection controls what the CommittedResource controller does when placement fails + for a guaranteed or confirmed commitment. + true — controller may reject: on failure, child Reservations are rolled back and the CR + is marked Rejected. Use this when the caller is making a first-time placement + decision and a "no" answer is acceptable (e.g. the change-commitments API). + false — controller must retry: on failure, existing child Reservations are kept and the + CR is set to Reserving so the controller retries later. Use this when the caller + is restoring already-committed state that Cortex must honour (e.g. the syncer). + Only meaningful for state=guaranteed or state=confirmed; ignored for all other states. + type: boolean + amount: + anyOf: + - type: integer + - type: string + description: |- + Amount is the total committed quantity. + memory: MiB expressed in K8s binary SI notation (e.g. "1280Gi", "640Mi"). + cores: integer core count (e.g. "40"). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + availabilityZone: + description: AvailabilityZone specifies the availability zone + for this commitment. + type: string + commitmentUUID: + description: UUID of the commitment this resource corresponds + to. + type: string + confirmedAt: + description: ConfirmedAt is when the commitment was confirmed. + format: date-time + type: string + domainID: + description: DomainID of the OpenStack domain this commitment + belongs to. + type: string + endTime: + description: EndTime is when Reservation slots expire. Nil for + unbounded commitments with no expiry. + format: date-time + type: string + flavorGroupName: + description: FlavorGroupName identifies the flavor group this + commitment targets, e.g. "kvm_v2_hana_s". + type: string + projectID: + description: ProjectID of the OpenStack project this commitment + belongs to. + type: string + resourceType: + description: 'ResourceType identifies the kind of resource committed: + memory drives Reservation slots; cores uses an arithmetic + check only.' + enum: + - memory + - cores + type: string + schedulingDomain: + description: SchedulingDomain specifies the scheduling domain + for this committed resource (e.g., "nova", "ironcore"). + type: string + startTime: + description: |- + StartTime is the activation time for Reservation slots. + Nil for guaranteed commitments (slots are active from creation); set to ConfirmedAt for confirmed ones. + format: date-time + type: string + state: + description: State is the lifecycle state of the commitment. + enum: + - planned + - pending + - guaranteed + - confirmed + - superseded + - expired + type: string + required: + - amount + - availabilityZone + - commitmentUUID + - domainID + - flavorGroupName + - projectID + - resourceType + - state + type: object + description: |- + SnapshotBefore records the CommittedResource spec state before this change request. + Key is commitmentUUID; nil value means the CR did not exist before (will be deleted on rollback). + Populated in Preparing when AllowRejection=true. + type: object + upsertedCommitments: + description: |- + UpsertedCommitments holds references to CommittedResource CRDs written by this change request. + Watched in AwaitingScheduling to gate on placement success. + These are plain object references, not owner references. + items: + description: ObjectReference contains enough information to let + you inspect or modify the referred object. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + type: array + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/internal/scheduling/reservations/commitments/changes/field_index.go b/internal/scheduling/reservations/commitments/changes/field_index.go new file mode 100644 index 000000000..0d3a96fa0 --- /dev/null +++ b/internal/scheduling/reservations/commitments/changes/field_index.go @@ -0,0 +1,39 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package changes + +import ( + "context" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/pkg/multicluster" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// idxCRCByUpsertedCR indexes CommittedResourceChanges by each CR name in UpsertedCommitments. +// Used by crToChangeRequest to map a CR watch event back to its owning change request. +const idxCRCByUpsertedCR = "status.upsertedCommitments.name" + +// indexCRCByUpsertedCR registers the field index on all clusters that serve CRC objects. +// Called once from SetupWithManager; no deduplication guard needed. +func indexCRCByUpsertedCR(ctx context.Context, mcl *multicluster.Client) error { + return mcl.IndexField(ctx, + &v1alpha1.CommittedResourceChange{}, + &v1alpha1.CommittedResourceChangeList{}, + idxCRCByUpsertedCR, + func(obj client.Object) []string { + tx, ok := obj.(*v1alpha1.CommittedResourceChange) + if !ok { + return nil + } + names := make([]string, 0, len(tx.Status.UpsertedCommitments)) + for _, ref := range tx.Status.UpsertedCommitments { + if ref.Name != "" { + names = append(names, ref.Name) + } + } + return names + }, + ) +} diff --git a/internal/scheduling/reservations/commitments/changes/reconciler.go b/internal/scheduling/reservations/commitments/changes/reconciler.go new file mode 100644 index 000000000..73e244a99 --- /dev/null +++ b/internal/scheduling/reservations/commitments/changes/reconciler.go @@ -0,0 +1,342 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package changes + +import ( + "context" + "fmt" + "time" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/commitments" + "github.com/cobaltcore-dev/cortex/pkg/multicluster" +) + +// CommittedResourceChangeReconciler reconciles CommittedResourceChange CRDs. +// It is the sole writer of CommittedResource.Spec; it reads CommittedResource.Status +// to drive its own FSM. It never writes CommittedResource.Status. +// +// TODO (#783): add perAZActive map[string]types.UID + mu sync.Mutex for per-AZ serialization. +type CommittedResourceChangeReconciler struct { + client.Client + Scheme *runtime.Scheme + // AwaitingSchedulingTimeout is how long to wait in AwaitingScheduling before rolling back. + // Zero (default) means no timeout. + AwaitingSchedulingTimeout time.Duration +} + +func (r *CommittedResourceChangeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + ctx = commitments.WithNewGlobalRequestID(ctx) + logger := commitments.LoggerFromContext(ctx).WithValues("committedResourceChangeRequest", req.Name) + + var tx v1alpha1.CommittedResourceChange + if err := r.Get(ctx, req.NamespacedName, &tx); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + if !tx.DeletionTimestamp.IsZero() { + return ctrl.Result{}, nil + } + + logger.V(1).Info("reconcile entry", "rv", tx.ResourceVersion, "gen", tx.Generation, "phase", tx.Status.Phase) + + switch tx.Status.Phase { + case "", v1alpha1.ChangePhasePreparing: + return r.reconcilePreparing(ctx, logger, &tx) + case v1alpha1.ChangePhaseAllocating: + return r.reconcileAllocating(ctx, logger, &tx) + case v1alpha1.ChangePhaseAwaitingScheduling: + return r.reconcileAwaitingScheduling(ctx, logger, &tx) + case v1alpha1.ChangePhaseRollingBack: + return r.reconcileRollingBack(ctx, logger, &tx) + case v1alpha1.ChangePhaseDryRunCleanup: + return r.reconcileDryRunCleanup(ctx, logger, &tx) + case v1alpha1.ChangePhaseAccepted, v1alpha1.ChangePhaseRejected, v1alpha1.ChangePhaseSuperseded: + return ctrl.Result{}, nil + default: + return ctrl.Result{}, fmt.Errorf("unknown phase %q", tx.Status.Phase) + } +} + +// commitmentName returns the CommittedResource name for a given commitment UUID. +func commitmentName(uuid string) string { return "commitment-" + uuid } + +// reconcilePreparing snapshots pre-change CR specs and advances to Allocating. +// Always snapshots regardless of AllowRejection — syncer-mode CRTs never roll back but the +// snapshot is cheap and keeps this method free of branching. +func (r *CommittedResourceChangeReconciler) reconcilePreparing(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange) (ctrl.Result, error) { + // Snapshot current CR specs for all keys in NewState so rollback can restore them. + // nil entry = CR did not exist before = delete it on rollback. + snapshot := make(map[string]*v1alpha1.CommittedResourceSpec, len(tx.Spec.NewState)) + for uuid := range tx.Spec.NewState { + var cr v1alpha1.CommittedResource + if err := r.Get(ctx, client.ObjectKey{Name: commitmentName(uuid)}, &cr); err != nil { + if client.IgnoreNotFound(err) == nil { + snapshot[uuid] = nil + continue + } + logger.Error(err, "failed to snapshot CR", "commitmentUUID", uuid, "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID) + r.markError(ctx, tx, fmt.Sprintf("failed to snapshot commitment-%s: %v", uuid, err)) + return ctrl.Result{}, err + } + snapshot[uuid] = cr.Spec.DeepCopy() + } + + logger.Info("starting transaction", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "newStateSize", len(tx.Spec.NewState), "allowRejection", tx.Spec.AllowRejection) + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseAllocating, func(s *v1alpha1.CommittedResourceChangeStatus) { + s.SnapshotBefore = snapshot + }) +} + +// reconcileAllocating writes CommittedResource specs from NewState and records upserted refs. +// API-mode: advances to AwaitingScheduling. +// Syncer-mode: advances directly to Accepted (apply-and-done, no scheduling gate). +// +// UpsertedCommitments and the phase advance must happen in one patch (via the mutate function) +// so that a crash between CR spec writes and status persistence does not leave AwaitingScheduling +// with an empty UpsertedCommitments list — which would incorrectly advance to Accepted. +func (r *CommittedResourceChangeReconciler) reconcileAllocating(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange) (ctrl.Result, error) { + var upserted []corev1.ObjectReference + + for uuid, desiredSpec := range tx.Spec.NewState { + name := commitmentName(uuid) + if desiredSpec == nil { + toDelete := &v1alpha1.CommittedResource{ObjectMeta: metav1.ObjectMeta{Name: name}} + if err := r.Delete(ctx, toDelete); client.IgnoreNotFound(err) != nil { + logger.Error(err, "failed to delete CR", "commitmentUUID", uuid, "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID) + r.markError(ctx, tx, fmt.Sprintf("failed to delete commitment-%s: %v", uuid, err)) + return ctrl.Result{}, err + } + continue + } + + cr := &v1alpha1.CommittedResource{ObjectMeta: metav1.ObjectMeta{Name: name}} + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, cr, func() error { + cr.Spec = *desiredSpec + return nil + }); err != nil { + logger.Error(err, "failed to upsert CR spec", "commitmentUUID", uuid, "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID) + r.markError(ctx, tx, fmt.Sprintf("failed to upsert commitment-%s: %v", uuid, err)) + return ctrl.Result{}, err + } + upserted = append(upserted, corev1.ObjectReference{ + APIVersion: v1alpha1.GroupVersion.String(), + Kind: "CommittedResource", + Name: name, + }) + } + + // TODO (#782): syncer-mode recency check before advancing + nextPhase := v1alpha1.ChangePhaseAwaitingScheduling + if !tx.Spec.AllowRejection { + nextPhase = v1alpha1.ChangePhaseAccepted + } + return r.advanceTo(ctx, tx, nextPhase, func(s *v1alpha1.CommittedResourceChangeStatus) { + s.UpsertedCommitments = upserted + }) +} + +// reconcileAwaitingScheduling reads UpsertedCommitments and waits for placement settlement. +// Gates on each CR's ready-condition observedGeneration matching the generation written in Allocating. +// All CRs accepted → Accepted (or DryRunCleanup when DryRun=true). +// Any CR rejected → RollingBack. +// Not yet settled → wait for the CR watch to re-enqueue. +func (r *CommittedResourceChangeReconciler) reconcileAwaitingScheduling(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange) (ctrl.Result, error) { + allAccepted := true + acceptedCount := 0 + for _, ref := range tx.Status.UpsertedCommitments { + var cr v1alpha1.CommittedResource + if err := r.Get(ctx, client.ObjectKey{Name: ref.Name}, &cr); err != nil { + if client.IgnoreNotFound(err) == nil { + // CR briefly unavailable — wait for the next watch event. + allAccepted = false + continue + } + logger.Error(err, "failed to get upserted CR", "cr", ref.Name, "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID) + r.markError(ctx, tx, fmt.Sprintf("failed to get %s: %v", ref.Name, err)) + return ctrl.Result{}, err + } + + cond := meta.FindStatusCondition(cr.Status.Conditions, v1alpha1.CommittedResourceConditionReady) + if cond == nil || cond.ObservedGeneration != cr.Generation { + // CR controller hasn't caught up to the spec we wrote — wait. + allAccepted = false + continue + } + + switch { + case cond.Status == metav1.ConditionTrue && cond.Reason == v1alpha1.CommittedResourceReasonAccepted: + // CR accepted — placement succeeded; keep iterating. + acceptedCount++ + case cond.Status == metav1.ConditionFalse && cond.Reason == v1alpha1.CommittedResourceReasonRejected: + // Fail-fast: start rollback immediately on first rejection rather than waiting for all CRs + // to settle — this unblocks the AZ sooner and the first rejection message is sufficient for diagnosis. + logger.Info("CR rejected, rolling back", "cr", ref.Name, "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "message", cond.Message) + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseRollingBack, func(s *v1alpha1.CommittedResourceChangeStatus) { + s.Message = fmt.Sprintf("CR %s rejected: %s", ref.Name, cond.Message) + }) + default: + // Still transitioning — wait. + allAccepted = false + } + } + + if !allAccepted { + logger.V(1).Info("awaiting scheduling", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "ready", acceptedCount, "total", len(tx.Status.UpsertedCommitments)) + if r.AwaitingSchedulingTimeout > 0 { + progressingCond := meta.FindStatusCondition(tx.Status.Conditions, v1alpha1.ChangeConditionProgressing) + if progressingCond != nil && time.Since(progressingCond.LastTransitionTime.Time) > r.AwaitingSchedulingTimeout { + logger.Info("timed out waiting for scheduling, rolling back", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "timeout", r.AwaitingSchedulingTimeout) + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseRollingBack, func(s *v1alpha1.CommittedResourceChangeStatus) { + s.Message = "timed out waiting for scheduling" + meta.SetStatusCondition(&s.Conditions, metav1.Condition{ + Type: v1alpha1.ChangeConditionProgressing, + Status: metav1.ConditionFalse, + Reason: v1alpha1.ChangeReasonTimedOut, + Message: "timed out waiting for scheduling", + ObservedGeneration: tx.Generation, + }) + }) + } + } + // Primary requeue is event-driven via the CR watch (crToChangeRequest). + // RequeueAfter is a safety-net for missed watch events (reconnects, startup races). + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + + if tx.Spec.DryRun { + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseDryRunCleanup, nil) + } + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseAccepted, nil) +} + +// reconcileRollingBack converges affected CR specs back to the pre-change SnapshotBefore state. +// On restore complete: advances to Rejected. +// On permanent cleanup failure: advances to Rejected with reason RollbackFailed. +func (r *CommittedResourceChangeReconciler) reconcileRollingBack(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange) (ctrl.Result, error) { + // TODO (#780): for each entry in SnapshotBefore, converge CR spec back to snapshot value + // using SSA with fieldManager=txFieldManager. + // nil snapshot entry → CR did not exist before → delete it. + // Gate terminal transition on each CR confirming its spec reverted (check observedGeneration). + return ctrl.Result{}, nil +} + +// reconcileDryRunCleanup restores pre-change state after a dry-run would-accept outcome. +// Re-uses snapshot rollback logic; terminal phase is Accepted (not Rejected). +func (r *CommittedResourceChangeReconciler) reconcileDryRunCleanup(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange) (ctrl.Result, error) { + // TODO (#780): same convergence as reconcileRollingBack; terminal → advanceTo(Accepted, nil). + return ctrl.Result{}, nil +} + +// advanceTo updates the transaction status to the given phase. +// mutate is called (when non-nil) to allow phase advance and other status mutations +// (e.g. UpsertedCommitments, SnapshotBefore) to be persisted atomically in one Update call. +// Progressing is set to False on terminal phases to reflect that no further work is expected. +// +// Status().Update is used (not Patch) so that nil map values in SnapshotBefore and similar +// fields are preserved — JSON merge patch semantics treat null map values as deletions. +func (r *CommittedResourceChangeReconciler) advanceTo( + ctx context.Context, + tx *v1alpha1.CommittedResourceChange, + phase v1alpha1.ChangePhase, + mutate func(*v1alpha1.CommittedResourceChangeStatus), +) (ctrl.Result, error) { + + tx.Status.Phase = phase + if mutate != nil { + mutate(&tx.Status) + } + progressingStatus, progressingReason := metav1.ConditionTrue, "Advancing" + if tx.IsTerminal() { + progressingStatus, progressingReason = metav1.ConditionFalse, string(phase) + meta.SetStatusCondition(&tx.Status.Conditions, metav1.Condition{ + Type: v1alpha1.ChangeConditionFinished, + Status: metav1.ConditionTrue, + Reason: string(phase), + ObservedGeneration: tx.Generation, + }) + } + // Preserve a Progressing=False condition already stamped by the mutate func (e.g. TimedOut). + // Terminal phases always overwrite to ensure Progressing=False/. + existingProgressing := meta.FindStatusCondition(tx.Status.Conditions, v1alpha1.ChangeConditionProgressing) + if tx.IsTerminal() || existingProgressing == nil || existingProgressing.Status != metav1.ConditionFalse { + meta.SetStatusCondition(&tx.Status.Conditions, metav1.Condition{ + Type: v1alpha1.ChangeConditionProgressing, + Status: progressingStatus, + Reason: progressingReason, + ObservedGeneration: tx.Generation, + }) + } + if err := r.Status().Update(ctx, tx); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil +} + +// markError stamps Progressing=False/Error on the transaction as a best-effort status update. +// The update error is logged but not returned — callers return the original error for requeue. +func (r *CommittedResourceChangeReconciler) markError(ctx context.Context, tx *v1alpha1.CommittedResourceChange, msg string) { + meta.SetStatusCondition(&tx.Status.Conditions, metav1.Condition{ + Type: v1alpha1.ChangeConditionProgressing, + Status: metav1.ConditionFalse, + Reason: v1alpha1.ChangeReasonError, + Message: msg, + ObservedGeneration: tx.Generation, + }) + if err := r.Status().Update(ctx, tx); err != nil { + ctrl.LoggerFrom(ctx).Error(err, "failed to mark error condition") + } +} + +// SetupWithManager registers the controller and configures watches. +// TODO (#783): set MaxConcurrentReconciles: 1 via WithOptions(controller.Options{MaxConcurrentReconciles: 1}). +// TODO (#783): add predicate on WatchesMulticluster() to re-trigger same-AZ non-terminal siblings on terminal transition. +func (r *CommittedResourceChangeReconciler) SetupWithManager(mgr ctrl.Manager, mcl *multicluster.Client) error { + ctx := context.Background() + if err := indexCRCByUpsertedCR(ctx, mcl); err != nil { + return fmt.Errorf("failed to set up CRC-by-upserted-CR field index: %w", err) + } + bldr := multicluster.BuildController(mcl, mgr) + var err error + bldr, err = bldr.WatchesMulticluster( + &v1alpha1.CommittedResourceChange{}, + &handler.EnqueueRequestForObject{}, + ) + if err != nil { + return err + } + // Re-enqueues this transaction when a CR it upserted settles placement. + bldr, err = bldr.WatchesMulticluster( + &v1alpha1.CommittedResource{}, + handler.EnqueueRequestsFromMapFunc(r.crToChangeRequest), + ) + if err != nil { + return err + } + return bldr.Named("committed-resource-change-request").Complete(r) +} + +// crToChangeRequest maps a CommittedResource watch event back to the transaction that upserted it. +func (r *CommittedResourceChangeReconciler) crToChangeRequest(ctx context.Context, obj client.Object) []ctrl.Request { + var txList v1alpha1.CommittedResourceChangeList + if err := r.List(ctx, &txList, client.MatchingFields{idxCRCByUpsertedCR: obj.GetName()}); err != nil { + commitments.LoggerFromContext(ctx).Error(err, "failed to list CRCs by upserted CR", "cr", obj.GetName()) + return nil + } + reqs := make([]ctrl.Request, 0, len(txList.Items)) + for i := range txList.Items { + reqs = append(reqs, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(&txList.Items[i])}) + } + return reqs +} diff --git a/internal/scheduling/reservations/commitments/changes/reconciler_test.go b/internal/scheduling/reservations/commitments/changes/reconciler_test.go new file mode 100644 index 000000000..cfbdcebc8 --- /dev/null +++ b/internal/scheduling/reservations/commitments/changes/reconciler_test.go @@ -0,0 +1,1082 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package changes + +import ( + "context" + "strings" + "testing" + "time" + + apimeta "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" +) + +// ============================================================================ +// Helpers +// ============================================================================ + +var testCtx = context.Background() + +func testScheme(t *testing.T) *runtime.Scheme { + t.Helper() + s := runtime.NewScheme() + if err := v1alpha1.AddToScheme(s); err != nil { + t.Fatalf("add scheme: %v", err) + } + return s +} + +func testClient(t *testing.T, scheme *runtime.Scheme, objects ...client.Object) client.Client { + t.Helper() + return fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objects...). + WithStatusSubresource( + &v1alpha1.CommittedResourceChange{}, + &v1alpha1.CommittedResource{}, + ). + WithIndex(&v1alpha1.CommittedResourceChange{}, idxCRCByUpsertedCR, func(obj client.Object) []string { + tx, ok := obj.(*v1alpha1.CommittedResourceChange) + if !ok { + return nil + } + names := make([]string, 0, len(tx.Status.UpsertedCommitments)) + for _, ref := range tx.Status.UpsertedCommitments { + if ref.Name != "" { + names = append(names, ref.Name) + } + } + return names + }). + Build() +} + +func testReconciler(c client.Client, scheme *runtime.Scheme) *CommittedResourceChangeReconciler { + return &CommittedResourceChangeReconciler{Client: c, Scheme: scheme} +} + +func doReconcile(t *testing.T, r *CommittedResourceChangeReconciler, name string) ctrl.Result { + t.Helper() + res, err := r.Reconcile(testCtx, ctrl.Request{NamespacedName: types.NamespacedName{Name: name}}) + if err != nil { + t.Fatalf("Reconcile(%q): %v", name, err) + } + return res +} + +func getTx(t *testing.T, c client.Client, name string) *v1alpha1.CommittedResourceChange { + t.Helper() + var tx v1alpha1.CommittedResourceChange + if err := c.Get(testCtx, types.NamespacedName{Name: name}, &tx); err != nil { + t.Fatalf("get CRC %q: %v", name, err) + } + return &tx +} + +func getCR(t *testing.T, c client.Client, name string) *v1alpha1.CommittedResource { + t.Helper() + var cr v1alpha1.CommittedResource + if err := c.Get(testCtx, types.NamespacedName{Name: name}, &cr); err != nil { + t.Fatalf("get CR %q: %v", name, err) + } + return &cr +} + +// driveToPhase calls Reconcile until the CRT reaches target, up to 5 attempts. +func driveToPhase(t *testing.T, r *CommittedResourceChangeReconciler, c client.Client, txName string, target v1alpha1.ChangePhase) { + t.Helper() + const maxSteps = 5 + for range maxSteps { + if getTx(t, c, txName).Status.Phase == target { + return + } + doReconcile(t, r, txName) + } + if got := getTx(t, c, txName).Status.Phase; got != target { + t.Fatalf("driveToPhase: wanted %q, got %q after %d steps", target, got, maxSteps) + } +} + +// setCRReady simulates the CR controller writing a Ready condition using the CR's current Generation. +// + +func setCRReady(t *testing.T, c client.Client, crName string, status metav1.ConditionStatus, reason, message string) { + t.Helper() + cr := getCR(t, c, crName) + apimeta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ + Type: v1alpha1.CommittedResourceConditionReady, + Status: status, + Reason: reason, + Message: message, + ObservedGeneration: cr.Generation, + LastTransitionTime: metav1.Now(), + }) + if err := c.Status().Update(testCtx, cr); err != nil { + t.Fatalf("update CR %q status: %v", crName, err) + } +} + +// setCRReadyStaleGen sets Ready=True/Accepted but with a stale ObservedGeneration (generation-1), +// so the reconciler's generation gate treats the condition as not yet caught up. +func setCRReadyStaleGen(t *testing.T, c client.Client, crName string) { + t.Helper() + cr := getCR(t, c, crName) + apimeta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ + Type: v1alpha1.CommittedResourceConditionReady, + Status: metav1.ConditionTrue, + Reason: v1alpha1.CommittedResourceReasonAccepted, + ObservedGeneration: cr.Generation - 1, + LastTransitionTime: metav1.Now(), + }) + if err := c.Status().Update(testCtx, cr); err != nil { + t.Fatalf("update CR %q status (stale gen): %v", crName, err) + } +} + +// backdateProgressingCondition sets the Progressing condition's LastTransitionTime to the past. +func backdateProgressingCondition(t *testing.T, c client.Client, txName string, age time.Duration) { + t.Helper() + tx := getTx(t, c, txName) + for i, cond := range tx.Status.Conditions { + if cond.Type == v1alpha1.ChangeConditionProgressing { + tx.Status.Conditions[i].LastTransitionTime = metav1.NewTime(time.Now().Add(-age)) + break + } + } + if err := c.Status().Update(testCtx, tx); err != nil { + t.Fatalf("backdate Progressing condition on %q: %v", txName, err) + } +} + +// assertCondition verifies a condition in the given list by type, status, and optional reason. +// Pass an empty wantReason to skip the reason check. +func assertCondition(t *testing.T, conditions []metav1.Condition, condType string, wantStatus metav1.ConditionStatus, wantReason string) { + t.Helper() + cond := apimeta.FindStatusCondition(conditions, condType) + if cond == nil { + t.Errorf("condition %q not found in %v", condType, conditions) + return + } + if cond.Status != wantStatus { + t.Errorf("condition %q status: want %v, got %v", condType, wantStatus, cond.Status) + } + if wantReason != "" && cond.Reason != wantReason { + t.Errorf("condition %q reason: want %q, got %q", condType, wantReason, cond.Reason) + } +} + +// minimalCRSpec returns a minimal valid CommittedResourceSpec for testing. +func minimalCRSpec(az string) *v1alpha1.CommittedResourceSpec { + return &v1alpha1.CommittedResourceSpec{ + CommitmentUUID: "test-uuid", + FlavorGroupName: "kvm_v2", + ResourceType: v1alpha1.CommittedResourceTypeMemory, + Amount: resource.MustParse("1Gi"), + AvailabilityZone: az, + ProjectID: "project-1", + DomainID: "domain-1", + State: v1alpha1.CommitmentStatusConfirmed, + } +} + +// ============================================================================ +// TestCRTReconciler_Preparing +// ============================================================================ + +func TestCRTReconciler_Preparing(t *testing.T) { + tests := []struct { + name string + preObjects []client.Object + tx *v1alpha1.CommittedResourceChange + wantPhase v1alpha1.ChangePhase + wantSnapshotNilFor []string // UUIDs that must have nil snapshot entry (CR did not exist) + wantSnapshotSetFor []string // UUIDs that must have non-nil snapshot entry (CR existed) + wantProgressingStatus metav1.ConditionStatus + wantProgressingReason string + }{ + { + name: "no pre-existing CR: snapshot entry is nil, phase advances to Allocating", + tx: &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-prep-1"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-a": minimalCRSpec("az-1")}, + }, + }, + wantPhase: v1alpha1.ChangePhaseAllocating, + wantSnapshotNilFor: []string{"uuid-a"}, + wantProgressingStatus: metav1.ConditionTrue, + wantProgressingReason: "Advancing", + }, + { + name: "pre-existing CR spec is captured in SnapshotBefore", + preObjects: []client.Object{ + &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-b"}, + Spec: *minimalCRSpec("az-1"), + }, + }, + tx: &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-prep-2"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-b": minimalCRSpec("az-1")}, + }, + }, + wantPhase: v1alpha1.ChangePhaseAllocating, + wantSnapshotSetFor: []string{"uuid-b"}, + wantProgressingStatus: metav1.ConditionTrue, + wantProgressingReason: "Advancing", + }, + { + name: "Preparing idempotent: running reconcile twice does not error", + tx: &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-prep-3"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-c": minimalCRSpec("az-1")}, + }, + }, + wantPhase: v1alpha1.ChangePhaseAllocating, + wantProgressingStatus: metav1.ConditionTrue, + wantProgressingReason: "Advancing", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := testScheme(t) + objs := append([]client.Object{tt.tx}, tt.preObjects...) + c := testClient(t, scheme, objs...) + r := testReconciler(c, scheme) + + doReconcile(t, r, tt.tx.Name) + if tt.name == "Preparing idempotent: running reconcile twice does not error" { + // Reset to Preparing to verify a second run is also error-free. + stored := getTx(t, c, tt.tx.Name) + stored.Status.Phase = "" + if err := c.Status().Update(testCtx, stored); err != nil { + t.Fatalf("reset phase: %v", err) + } + doReconcile(t, r, tt.tx.Name) + } + + tx := getTx(t, c, tt.tx.Name) + if tx.Status.Phase != tt.wantPhase { + t.Errorf("phase: want %q, got %q", tt.wantPhase, tx.Status.Phase) + } + if tt.wantProgressingStatus != "" { + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionProgressing, tt.wantProgressingStatus, tt.wantProgressingReason) + } + for _, uuid := range tt.wantSnapshotNilFor { + snap, ok := tx.Status.SnapshotBefore[uuid] + if !ok { + t.Errorf("SnapshotBefore: missing key %q", uuid) + continue + } + if snap != nil { + t.Errorf("SnapshotBefore[%q]: want nil, got non-nil", uuid) + } + } + for _, uuid := range tt.wantSnapshotSetFor { + snap, ok := tx.Status.SnapshotBefore[uuid] + if !ok { + t.Errorf("SnapshotBefore: missing key %q", uuid) + continue + } + if snap == nil { + t.Errorf("SnapshotBefore[%q]: want non-nil snapshot, got nil", uuid) + } + } + }) + } +} + +// ============================================================================ +// TestCRTReconciler_Allocating +// ============================================================================ + +func TestCRTReconciler_Allocating(t *testing.T) { + tests := []struct { + name string + preObjects []client.Object + tx *v1alpha1.CommittedResourceChange + wantPhase v1alpha1.ChangePhase + wantCRExists []string + wantCRAbsent []string + wantUpsertedCommitments int + wantProgressingStatus metav1.ConditionStatus + wantProgressingReason string + wantFinished bool + wantFinishedReason string + }{ + { + name: "API-mode: creates CR and advances to AwaitingScheduling", + tx: &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-alloc-1"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-1": minimalCRSpec("az-1")}, + }, + }, + wantPhase: v1alpha1.ChangePhaseAwaitingScheduling, + wantCRExists: []string{"commitment-uuid-1"}, + wantUpsertedCommitments: 1, + wantProgressingStatus: metav1.ConditionTrue, + wantProgressingReason: "Advancing", + }, + { + name: "syncer-mode: applies CR and advances directly to Accepted (no scheduling gate)", + tx: &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-alloc-2"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceSyncer, + AllowRejection: false, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-2": minimalCRSpec("az-1")}, + }, + }, + wantPhase: v1alpha1.ChangePhaseAccepted, + wantCRExists: []string{"commitment-uuid-2"}, + wantUpsertedCommitments: 1, + wantProgressingStatus: metav1.ConditionFalse, + wantProgressingReason: string(v1alpha1.ChangePhaseAccepted), + wantFinished: true, + wantFinishedReason: string(v1alpha1.ChangePhaseAccepted), + }, + { + name: "nil spec deletes CR and it is absent from UpsertedCommitments", + preObjects: []client.Object{ + &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-3"}, + Spec: *minimalCRSpec("az-1"), + }, + }, + tx: &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-alloc-3"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-3": nil}, + }, + }, + wantPhase: v1alpha1.ChangePhaseAwaitingScheduling, + wantCRAbsent: []string{"commitment-uuid-3"}, + wantUpsertedCommitments: 0, + wantProgressingStatus: metav1.ConditionTrue, + wantProgressingReason: "Advancing", + }, + { + name: "pre-existing CR with different spec: spec is overwritten by CreateOrUpdate", + preObjects: []client.Object{ + &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-4"}, + Spec: *minimalCRSpec("az-old"), + }, + }, + tx: &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-alloc-4"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-4": minimalCRSpec("az-new")}, + }, + }, + wantPhase: v1alpha1.ChangePhaseAwaitingScheduling, + wantCRExists: []string{"commitment-uuid-4"}, + wantUpsertedCommitments: 1, + wantProgressingStatus: metav1.ConditionTrue, + wantProgressingReason: "Advancing", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := testScheme(t) + objs := append([]client.Object{tt.tx}, tt.preObjects...) + c := testClient(t, scheme, objs...) + r := testReconciler(c, scheme) + + driveToPhase(t, r, c, tt.tx.Name, tt.wantPhase) + + tx := getTx(t, c, tt.tx.Name) + if tx.Status.Phase != tt.wantPhase { + t.Errorf("phase: want %q, got %q", tt.wantPhase, tx.Status.Phase) + } + if got := len(tx.Status.UpsertedCommitments); got != tt.wantUpsertedCommitments { + t.Errorf("UpsertedCommitments len: want %d, got %d", tt.wantUpsertedCommitments, got) + } + if tt.wantProgressingStatus != "" { + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionProgressing, tt.wantProgressingStatus, tt.wantProgressingReason) + } + if tt.wantFinished { + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionFinished, metav1.ConditionTrue, tt.wantFinishedReason) + } + for _, name := range tt.wantCRExists { + getCR(t, c, name) // fatals if absent + } + for _, name := range tt.wantCRAbsent { + var cr v1alpha1.CommittedResource + if err := c.Get(testCtx, types.NamespacedName{Name: name}, &cr); err == nil { + t.Errorf("CR %q should be absent but exists", name) + } + } + }) + } +} + +// TestCRTReconciler_Allocating_Idempotent verifies that re-running Allocating with the same +// desired spec is idempotent: no error, phase advances correctly again. +func TestCRTReconciler_Allocating_Idempotent(t *testing.T) { + scheme := testScheme(t) + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-idem"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-idem": minimalCRSpec("az-1")}, + }, + } + c := testClient(t, scheme, tx) + r := testReconciler(c, scheme) + + driveToPhase(t, r, c, tx.Name, v1alpha1.ChangePhaseAwaitingScheduling) + + // Reset to Allocating to re-run it. + stored := getTx(t, c, tx.Name) + stored.Status.Phase = v1alpha1.ChangePhaseAllocating + if err := c.Status().Update(testCtx, stored); err != nil { + t.Fatalf("reset phase to Allocating: %v", err) + } + + doReconcile(t, r, tx.Name) // second run of Allocating + + final := getTx(t, c, tx.Name) + if final.Status.Phase != v1alpha1.ChangePhaseAwaitingScheduling { + t.Errorf("idempotent Allocating: want AwaitingScheduling, got %q", final.Status.Phase) + } +} + +// ============================================================================ +// TestCRTReconciler_AwaitingScheduling +// ============================================================================ + +func TestCRTReconciler_AwaitingScheduling(t *testing.T) { + const txName = "tx-await" + const crName = "commitment-uuid-await" + + // setup drives a CRT to AwaitingScheduling and returns the client and reconciler. + setup := func(t *testing.T, txOpts ...func(*v1alpha1.CommittedResourceChange)) (client.Client, *CommittedResourceChangeReconciler) { + t.Helper() + scheme := testScheme(t) + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: txName}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-await": minimalCRSpec("az-1")}, + }, + } + for _, opt := range txOpts { + opt(tx) + } + c := testClient(t, scheme, tx) + r := testReconciler(c, scheme) + driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseAwaitingScheduling) + return c, r + } + + tests := []struct { + name string + txOpts []func(*v1alpha1.CommittedResourceChange) + setupCR func(t *testing.T, c client.Client) + setupTx func(t *testing.T, c client.Client) + reconcilerOpt func(r *CommittedResourceChangeReconciler) + wantPhase v1alpha1.ChangePhase + wantMessage string // substring that must appear in tx.Status.Message; empty = no check + wantProgressingStatus metav1.ConditionStatus + wantProgressingReason string + wantFinished bool + wantFinishedReason string + }{ + { + name: "all CRs accepted → Accepted", + setupCR: func(t *testing.T, c client.Client) { + setCRReady(t, c, crName, metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted, "") + }, + wantPhase: v1alpha1.ChangePhaseAccepted, + wantProgressingStatus: metav1.ConditionFalse, + wantProgressingReason: string(v1alpha1.ChangePhaseAccepted), + wantFinished: true, + wantFinishedReason: string(v1alpha1.ChangePhaseAccepted), + }, + { + name: "DryRun=true and all CRs accepted → DryRunCleanup", + txOpts: []func(*v1alpha1.CommittedResourceChange){ + func(tx *v1alpha1.CommittedResourceChange) { tx.Spec.DryRun = true }, + }, + setupCR: func(t *testing.T, c client.Client) { + setCRReady(t, c, crName, metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted, "") + }, + wantPhase: v1alpha1.ChangePhaseDryRunCleanup, + wantProgressingStatus: metav1.ConditionTrue, + wantProgressingReason: "Advancing", + }, + { + name: "CR rejected → RollingBack with rejection message", + setupCR: func(t *testing.T, c client.Client) { + setCRReady(t, c, crName, metav1.ConditionFalse, v1alpha1.CommittedResourceReasonRejected, "no capacity on host") + }, + wantPhase: v1alpha1.ChangePhaseRollingBack, + wantMessage: "no capacity on host", + wantProgressingStatus: metav1.ConditionTrue, + wantProgressingReason: "Advancing", + }, + { + name: "stale observedGeneration → stays in AwaitingScheduling", + setupCR: func(t *testing.T, c client.Client) { + setCRReadyStaleGen(t, c, crName) + }, + wantPhase: v1alpha1.ChangePhaseAwaitingScheduling, + wantProgressingStatus: metav1.ConditionTrue, + wantProgressingReason: "Advancing", + }, + { + name: "ConditionTrue with non-Accepted reason → stays in AwaitingScheduling", + setupCR: func(t *testing.T, c client.Client) { + setCRReady(t, c, crName, metav1.ConditionTrue, v1alpha1.CommittedResourceReasonReserving, "still placing") + }, + wantPhase: v1alpha1.ChangePhaseAwaitingScheduling, + wantProgressingStatus: metav1.ConditionTrue, + wantProgressingReason: "Advancing", + }, + { + name: "timeout exceeded → RollingBack with timed out message", + reconcilerOpt: func(r *CommittedResourceChangeReconciler) { + r.AwaitingSchedulingTimeout = 50 * time.Millisecond + }, + setupTx: func(t *testing.T, c client.Client) { + backdateProgressingCondition(t, c, txName, 1*time.Second) + }, + wantPhase: v1alpha1.ChangePhaseRollingBack, + wantMessage: "timed out", + wantProgressingStatus: metav1.ConditionFalse, + wantProgressingReason: v1alpha1.ChangeReasonTimedOut, + }, + { + name: "timeout=0 means no timeout: CRT stays in AwaitingScheduling indefinitely", + // No reconcilerOpt, so AwaitingSchedulingTimeout=0 (disabled). + // CR ready condition is never set → should stay in AwaitingScheduling. + wantPhase: v1alpha1.ChangePhaseAwaitingScheduling, + wantProgressingStatus: metav1.ConditionTrue, + wantProgressingReason: "Advancing", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c, r := setup(t, tt.txOpts...) + + if tt.reconcilerOpt != nil { + tt.reconcilerOpt(r) + } + if tt.setupCR != nil { + tt.setupCR(t, c) + } + if tt.setupTx != nil { + tt.setupTx(t, c) + } + + doReconcile(t, r, txName) + + tx := getTx(t, c, txName) + if tx.Status.Phase != tt.wantPhase { + t.Errorf("phase: want %q, got %q", tt.wantPhase, tx.Status.Phase) + } + if tt.wantMessage != "" && !strings.Contains(tx.Status.Message, tt.wantMessage) { + t.Errorf("message: want substring %q in %q", tt.wantMessage, tx.Status.Message) + } + if tt.wantProgressingStatus != "" { + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionProgressing, tt.wantProgressingStatus, tt.wantProgressingReason) + } + if tt.wantFinished { + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionFinished, metav1.ConditionTrue, tt.wantFinishedReason) + } + }) + } +} + +// setPhase directly sets a CRT's phase via Status().Update for test setup. +func setPhase(t *testing.T, c client.Client, txName string, phase v1alpha1.ChangePhase) { + t.Helper() + tx := getTx(t, c, txName) + tx.Status.Phase = phase + if err := c.Status().Update(testCtx, tx); err != nil { + t.Fatalf("setPhase %q on %q: %v", phase, txName, err) + } +} + +// reconcileN calls doReconcile exactly n times without inspecting the phase between calls. +func reconcileN(t *testing.T, r *CommittedResourceChangeReconciler, name string, n int) { + t.Helper() + for range n { + doReconcile(t, r, name) + } +} + +// ============================================================================ +// TestCRTReconciler_Idempotency +// ============================================================================ + +// TestCRTReconciler_Idempotency verifies that phase handlers are re-entrant: running the same +// handler multiple times (simulating crash-then-requeue) produces the same end state. +// The fake client never returns transient errors; these tests confirm re-entrancy — the property +// that makes controller-runtime's error-then-requeue pattern safe. +func TestCRTReconciler_Idempotency(t *testing.T) { + tests := []struct { + name string + run func(t *testing.T) + }{ + { + name: "Preparing: snapshot is consistent across re-runs", + run: func(t *testing.T) { + scheme := testScheme(t) + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-idem-1"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-idem-1": minimalCRSpec("az-1")}, + }, + } + c := testClient(t, scheme, tx) + r := testReconciler(c, scheme) + + doReconcile(t, r, tx.Name) // Preparing → Allocating + snap1 := getTx(t, c, tx.Name).Status.SnapshotBefore + + setPhase(t, c, tx.Name, "") // reset to Preparing + doReconcile(t, r, tx.Name) // Preparing → Allocating (re-run) + snap2 := getTx(t, c, tx.Name).Status.SnapshotBefore + + for i, snap := range []map[string]*v1alpha1.CommittedResourceSpec{snap1, snap2} { + v, ok := snap["uuid-idem-1"] + if !ok { + t.Errorf("run %d: snapshot missing uuid-idem-1", i+1) + } else if v != nil { + t.Errorf("run %d: snapshot[uuid-idem-1]: want nil, got non-nil", i+1) + } + } + }, + }, + { + name: "Allocating: UpsertedCommitments and CR spec stable across re-runs", + run: func(t *testing.T) { + scheme := testScheme(t) + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-idem-2"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-idem-2": minimalCRSpec("az-1")}, + }, + } + c := testClient(t, scheme, tx) + r := testReconciler(c, scheme) + + driveToPhase(t, r, c, tx.Name, v1alpha1.ChangePhaseAwaitingScheduling) + upserted1 := getTx(t, c, tx.Name).Status.UpsertedCommitments + + setPhase(t, c, tx.Name, v1alpha1.ChangePhaseAllocating) // reset to Allocating + doReconcile(t, r, tx.Name) // Allocating → AwaitingScheduling (re-run) + upserted2 := getTx(t, c, tx.Name).Status.UpsertedCommitments + + if len(upserted1) != 1 || len(upserted2) != 1 { + t.Errorf("UpsertedCommitments: want len 1 on each run, got %d, %d", len(upserted1), len(upserted2)) + } + if len(upserted1) > 0 && len(upserted2) > 0 && upserted1[0].Name != upserted2[0].Name { + t.Errorf("UpsertedCommitments name changed: %q → %q", upserted1[0].Name, upserted2[0].Name) + } + getCR(t, c, "commitment-uuid-idem-2") // CR must still exist + }, + }, + { + name: "AwaitingScheduling: multiple calls without CR ready stay waiting", + run: func(t *testing.T) { + scheme := testScheme(t) + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-idem-3"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-idem-3": minimalCRSpec("az-1")}, + }, + } + c := testClient(t, scheme, tx) + r := testReconciler(c, scheme) + + driveToPhase(t, r, c, tx.Name, v1alpha1.ChangePhaseAwaitingScheduling) + reconcileN(t, r, tx.Name, 3) // CR never set ready — must stay in AwaitingScheduling + + if got := getTx(t, c, tx.Name).Status.Phase; got != v1alpha1.ChangePhaseAwaitingScheduling { + t.Errorf("phase: want AwaitingScheduling, got %q", got) + } + }, + }, + { + name: "AwaitingScheduling → Accepted stays Accepted on repeated reconciles", + run: func(t *testing.T) { + scheme := testScheme(t) + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-idem-4"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-idem-4": minimalCRSpec("az-1")}, + }, + } + c := testClient(t, scheme, tx) + r := testReconciler(c, scheme) + + driveToPhase(t, r, c, tx.Name, v1alpha1.ChangePhaseAwaitingScheduling) + setCRReady(t, c, "commitment-uuid-idem-4", metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted, "") + reconcileN(t, r, tx.Name, 3) // first call → Accepted; subsequent calls → terminal no-op + + if got := getTx(t, c, tx.Name).Status.Phase; got != v1alpha1.ChangePhaseAccepted { + t.Errorf("phase: want Accepted, got %q", got) + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, tt.run) + } +} + +// ============================================================================ +// TestCRTReconciler_APIScenarios +// ============================================================================ + +// crExists returns true if a CommittedResource with the given name exists. +func crExists(t *testing.T, c client.Client, name string) bool { + t.Helper() + var cr v1alpha1.CommittedResource + err := c.Get(testCtx, types.NamespacedName{Name: name}, &cr) + if err == nil { + return true + } + if client.IgnoreNotFound(err) == nil { + return false + } + t.Fatalf("crExists(%q): %v", name, err) + return false +} + +// TestCRTReconciler_APIScenarios covers realistic multi-CR change patterns as seen from the +// CR acceptance API: mixed creates, updates, and deletes in a single transaction. +func TestCRTReconciler_APIScenarios(t *testing.T) { + tests := []struct { + name string + // pre-existing CRs before the transaction is created + preObjects []client.Object + // the CRT to create + tx *v1alpha1.CommittedResourceChange + // expected phase after driving the FSM to a stable point + wantPhase v1alpha1.ChangePhase + // CR names that must exist after Allocating + wantCRExists []string + // CR names that must be absent after Allocating + wantCRAbsent []string + // exact UpsertedCommitments count (nils are not upserted) + wantUpsertedCount int + // UUIDs that must have nil snapshot (CR did not exist before) + wantSnapshotNilFor []string + // UUIDs that must have non-nil snapshot (CR existed before) + wantSnapshotSetFor []string + }{ + { + name: "multiple new CRs: all created, all upserted", + tx: &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-api-1"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{ + "uuid-a1": minimalCRSpec("az-1"), + "uuid-a2": minimalCRSpec("az-1"), + "uuid-a3": minimalCRSpec("az-1"), + }, + }, + }, + wantPhase: v1alpha1.ChangePhaseAwaitingScheduling, + wantCRExists: []string{"commitment-uuid-a1", "commitment-uuid-a2", "commitment-uuid-a3"}, + wantUpsertedCount: 3, + wantSnapshotNilFor: []string{"uuid-a1", "uuid-a2", "uuid-a3"}, + }, + { + name: "one new CR + one existing CR updated: both upserted, snapshot captures existing", + preObjects: []client.Object{ + &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-b2"}, + Spec: *minimalCRSpec("az-old"), + }, + }, + tx: &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-api-2"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{ + "uuid-b1": minimalCRSpec("az-1"), // new + "uuid-b2": minimalCRSpec("az-new"), // existing, spec updated + }, + }, + }, + wantPhase: v1alpha1.ChangePhaseAwaitingScheduling, + wantCRExists: []string{"commitment-uuid-b1", "commitment-uuid-b2"}, + wantUpsertedCount: 2, + wantSnapshotNilFor: []string{"uuid-b1"}, + wantSnapshotSetFor: []string{"uuid-b2"}, + }, + { + name: "one CR deleted + one CR updated: deleted absent, only updated upserted", + preObjects: []client.Object{ + &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-c1"}, + Spec: *minimalCRSpec("az-1"), + }, + &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-c2"}, + Spec: *minimalCRSpec("az-1"), + }, + }, + tx: &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-api-3"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{ + "uuid-c1": nil, // delete + "uuid-c2": minimalCRSpec("az-1"), // update + }, + }, + }, + wantPhase: v1alpha1.ChangePhaseAwaitingScheduling, + wantCRExists: []string{"commitment-uuid-c2"}, + wantCRAbsent: []string{"commitment-uuid-c1"}, + wantUpsertedCount: 1, + wantSnapshotSetFor: []string{"uuid-c1", "uuid-c2"}, + }, + { + name: "empty NewState: no CRs touched, advances all the way to Accepted", + tx: &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-api-4"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{}, + }, + }, + // AwaitingScheduling with empty UpsertedCommitments → allAccepted=true immediately → Accepted + wantPhase: v1alpha1.ChangePhaseAccepted, + wantUpsertedCount: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := testScheme(t) + objs := append([]client.Object{tt.tx}, tt.preObjects...) + c := testClient(t, scheme, objs...) + r := testReconciler(c, scheme) + + driveToPhase(t, r, c, tt.tx.Name, tt.wantPhase) + + tx := getTx(t, c, tt.tx.Name) + if tx.Status.Phase != tt.wantPhase { + t.Errorf("phase: want %q, got %q", tt.wantPhase, tx.Status.Phase) + } + if got := len(tx.Status.UpsertedCommitments); got != tt.wantUpsertedCount { + t.Errorf("UpsertedCommitments count: want %d, got %d", tt.wantUpsertedCount, got) + } + for _, name := range tt.wantCRExists { + if !crExists(t, c, name) { + t.Errorf("CR %q should exist but is absent", name) + } + } + for _, name := range tt.wantCRAbsent { + if crExists(t, c, name) { + t.Errorf("CR %q should be absent but exists", name) + } + } + for _, uuid := range tt.wantSnapshotNilFor { + snap, ok := tx.Status.SnapshotBefore[uuid] + if !ok { + t.Errorf("SnapshotBefore: missing key %q", uuid) + } else if snap != nil { + t.Errorf("SnapshotBefore[%q]: want nil, got non-nil", uuid) + } + } + for _, uuid := range tt.wantSnapshotSetFor { + snap, ok := tx.Status.SnapshotBefore[uuid] + if !ok { + t.Errorf("SnapshotBefore: missing key %q", uuid) + } else if snap == nil { + t.Errorf("SnapshotBefore[%q]: want non-nil, got nil", uuid) + } + } + }) + } +} + +// ============================================================================ +// TestCRTReconciler_EdgeCases +// ============================================================================ + +// TestCRTReconciler_EdgeCases covers paths that don't fit the main FSM tables. +func TestCRTReconciler_EdgeCases(t *testing.T) { + t.Run("DeletionTimestamp: reconcile is a no-op", func(t *testing.T) { + scheme := testScheme(t) + now := metav1.Now() + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tx-del", + DeletionTimestamp: &now, + // Finalizers required for the fake client to allow a non-zero DeletionTimestamp. + Finalizers: []string{"test.io/hold"}, + }, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + }, + } + c := testClient(t, scheme, tx) + r := testReconciler(c, scheme) + + doReconcile(t, r, tx.Name) // must not error, must not advance phase + + got := getTx(t, c, tx.Name).Status.Phase + if got != "" { + t.Errorf("phase: want empty (no-op), got %q", got) + } + }) + + t.Run("unknown phase returns error", func(t *testing.T) { + scheme := testScheme(t) + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-unknown"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + }, + } + c := testClient(t, scheme, tx) + r := testReconciler(c, scheme) + + // Seed an unknown phase. + tx.Status.Phase = "Bogus" + if err := c.Status().Update(testCtx, tx); err != nil { + t.Fatalf("seed phase: %v", err) + } + + _, err := r.Reconcile(testCtx, ctrl.Request{NamespacedName: types.NamespacedName{Name: tx.Name}}) + if err == nil { + t.Error("expected error for unknown phase, got nil") + } + }) + + t.Run("crToTransaction maps CR name to owning CRT via field index", func(t *testing.T) { + scheme := testScheme(t) + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-map"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-map": minimalCRSpec("az-1")}, + }, + } + c := testClient(t, scheme, tx) + r := testReconciler(c, scheme) + + // Drive to AwaitingScheduling so UpsertedCommitments is populated. + driveToPhase(t, r, c, tx.Name, v1alpha1.ChangePhaseAwaitingScheduling) + + // crToTransaction should return a request for the owning CRT. + cr := getCR(t, c, "commitment-uuid-map") + reqs := r.crToChangeRequest(testCtx, cr) + if len(reqs) != 1 { + t.Fatalf("crToTransaction: want 1 request, got %d", len(reqs)) + } + if reqs[0].Name != tx.Name { + t.Errorf("crToTransaction: want request for %q, got %q", tx.Name, reqs[0].Name) + } + }) +} + +// ============================================================================ +// TestCRTReconciler_Terminal +// ============================================================================ + +func TestCRTReconciler_Terminal(t *testing.T) { + tests := []struct { + name string + phase v1alpha1.ChangePhase + }{ + {"Accepted is a no-op", v1alpha1.ChangePhaseAccepted}, + {"Rejected is a no-op", v1alpha1.ChangePhaseRejected}, + {"Superseded is a no-op", v1alpha1.ChangePhaseSuperseded}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := testScheme(t) + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-term"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + }, + } + c := testClient(t, scheme, tx) + r := testReconciler(c, scheme) + + // Seed the terminal phase directly via Status().Update. + tx.Status.Phase = tt.phase + if err := c.Status().Update(testCtx, tx); err != nil { + t.Fatalf("seed phase %q: %v", tt.phase, err) + } + + doReconcile(t, r, tx.Name) // must not error + + // Phase must remain unchanged. + got := getTx(t, c, tx.Name).Status.Phase + if got != tt.phase { + t.Errorf("terminal phase changed: want %q, got %q", tt.phase, got) + } + }) + } +}