diff --git a/Tiltfile b/Tiltfile index 0c912021b..47b4dd672 100644 --- a/Tiltfile +++ b/Tiltfile @@ -205,6 +205,9 @@ for (bundle_chart_path, bundle_chart_name) in bundle_charts: local('sh helm/sync.sh ' + dep_chart_path) pkg_out = str(local('helm package ' + dep_chart_path)).strip() gen_tgz = os.path.basename(pkg_out) + # Always remove stale old versions of this chart (e.g. cortex-0.5.1.tgz when current is 0.9.0). + # Uses [0-9] prefix so cortex-[0-9]*.tgz does not accidentally match cortex-postgres-*.tgz. + local('find ' + bundle_chart_path + '/charts/ -maxdepth 1 -name "' + dep_chart_name + '-[0-9]*.tgz" ! -name "' + gen_tgz + '" -delete') # If the file isn't there yet, copy it over. if not os.path.exists(bundle_chart_path + '/charts/' + gen_tgz): print('Adding ' + dep_chart_name + ' to ' + bundle_chart_name) diff --git a/internal/knowledge/extractor/plugins/compute/flavor_groups.go b/internal/knowledge/extractor/plugins/compute/flavor_groups.go index fdebd4404..5b9578cae 100644 --- a/internal/knowledge/extractor/plugins/compute/flavor_groups.go +++ b/internal/knowledge/extractor/plugins/compute/flavor_groups.go @@ -180,13 +180,14 @@ func (e *FlavorGroupExtractor) Extract() ([]plugins.Feature, error) { continue } - // Sort flavors by size descending (largest first), tie break by name for consistent ordering + // Sort flavors by memory descending (largest first); tie-break by fewest vCPUs to + // minimize CPU blocking when multiple flavors share the same memory size. sort.Slice(flavors, func(i, j int) bool { if flavors[i].MemoryMB != flavors[j].MemoryMB { return flavors[i].MemoryMB > flavors[j].MemoryMB } if flavors[i].VCPUs != flavors[j].VCPUs { - return flavors[i].VCPUs > flavors[j].VCPUs + return flavors[i].VCPUs < flavors[j].VCPUs } return flavors[i].Name < flavors[j].Name }) diff --git a/internal/knowledge/extractor/plugins/compute/flavor_groups_test.go b/internal/knowledge/extractor/plugins/compute/flavor_groups_test.go index 9b2b4dabd..97d304118 100644 --- a/internal/knowledge/extractor/plugins/compute/flavor_groups_test.go +++ b/internal/knowledge/extractor/plugins/compute/flavor_groups_test.go @@ -92,7 +92,7 @@ func TestFlavorGroupExtractor_Extract(t *testing.T) { Ephemeral: 5, ExtraSpecs: `{"capabilities:hypervisor_type":"CH","quota:hw_version":"ch"}`, }, - // Corner case: Same memory as gp_c8_m32_v2 but MORE VCPUs (should come first) + // Corner case: Same memory as gp_c8_m32_v2 but MORE VCPUs (should come last, after fewer-VCPUs flavor) &nova.Flavor{ ID: "8", Name: "gp_c12_m32_v2", @@ -178,15 +178,17 @@ func TestFlavorGroupExtractor_Extract(t *testing.T) { t.Errorf("expected largest flavor to have quota:hw_version=v2, got %s", v2Group.LargestFlavor.ExtraSpecs["quota:hw_version"]) } - // Verify smallest flavor in v2 group should be gp_c4_m16_ch is NOT in v2, so it's gp_c8_m32_v2 (lowest memory among v2 flavors) - if v2Group.SmallestFlavor.Name != "gp_c8_m32_v2" { - t.Errorf("expected smallest flavor to be gp_c8_m32_v2, got %s", v2Group.SmallestFlavor.Name) + // Verify smallest flavor in v2 group: at 32768 MB we have gp_c8_m32_v2 (8 VCPUs), + // gp_c12_m32_alt (12 VCPUs), and gp_c12_m32_v2 (12 VCPUs). With fewest-VCPUs-first + // tiebreak, gp_c8_m32_v2 sorts first and gp_c12_m32_v2 sorts last → smallest. + if v2Group.SmallestFlavor.Name != "gp_c12_m32_v2" { + t.Errorf("expected smallest flavor to be gp_c12_m32_v2, got %s", v2Group.SmallestFlavor.Name) } if v2Group.SmallestFlavor.MemoryMB != 32768 { t.Errorf("expected smallest flavor memory to be 32768 MB, got %d", v2Group.SmallestFlavor.MemoryMB) } - if v2Group.SmallestFlavor.VCPUs != 8 { - t.Errorf("expected smallest flavor VCPUs to be 8, got %d", v2Group.SmallestFlavor.VCPUs) + if v2Group.SmallestFlavor.VCPUs != 12 { + t.Errorf("expected smallest flavor VCPUs to be 12, got %d", v2Group.SmallestFlavor.VCPUs) } // Verify Cloud-Hypervisor group @@ -224,8 +226,8 @@ func TestFlavorGroupExtractor_Extract(t *testing.T) { fg.Name, current.Name, current.MemoryMB, next.Name, next.MemoryMB) } - // Secondary sort: if memory equal, VCPUs descending - if current.MemoryMB == next.MemoryMB && current.VCPUs < next.VCPUs { + // Secondary sort: if memory equal, VCPUs ascending (fewest VCPUs first) + if current.MemoryMB == next.MemoryMB && current.VCPUs > next.VCPUs { t.Errorf("Flavors in group %s with equal memory not sorted by VCPUs: %s (%d VCPUs) should come after %s (%d VCPUs)", fg.Name, current.Name, current.VCPUs, next.Name, next.VCPUs) } diff --git a/internal/scheduling/reservations/commitments/api/change_commitments.go b/internal/scheduling/reservations/commitments/api/change_commitments.go index ab6314851..c3c59edee 100644 --- a/internal/scheduling/reservations/commitments/api/change_commitments.go +++ b/internal/scheduling/reservations/commitments/api/change_commitments.go @@ -115,6 +115,8 @@ type crSnapshot struct { // HandleChangeCommitments implements POST /commitments/v1/change-commitments from the Limes LIQUID API. // It writes CommittedResource CRDs (one per commitment) and polls their status conditions until // the controller confirms or rejects each one. On any failure the whole batch is rolled back. +// TODO (#786): replace direct CR writes + changeMutex with a CommittedResourceChange (source=api); +// poll CRC terminal phase and map Accepted/Rejected/Superseded to HTTP response. Flag-gated switchover. // // See: https://github.com/sapcc/go-api-declarations/blob/main/liquid/commitment.go // See: https://pkg.go.dev/github.com/sapcc/go-api-declarations/liquid @@ -228,6 +230,8 @@ func (api *HTTPAPI) HandleChangeCommitments(w http.ResponseWriter, r *http.Reque } } +// TODO (#788): once #786 switchover is stable, remove: changeMutex, crSnapshot type, reverse-order +// rollbackCR, performDryRun probe CRs, and this direct-write path entirely. func (api *HTTPAPI) processCommitmentChanges(ctx context.Context, w http.ResponseWriter, logger logr.Logger, req liquid.CommitmentChangeRequest, resp *liquid.CommitmentChangeResponse) error { flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, api.client, nil) if err != nil { diff --git a/internal/scheduling/reservations/commitments/changes/reconciler.go b/internal/scheduling/reservations/commitments/changes/reconciler.go index ec2be9b82..d07284f73 100644 --- a/internal/scheduling/reservations/commitments/changes/reconciler.go +++ b/internal/scheduling/reservations/commitments/changes/reconciler.go @@ -301,6 +301,10 @@ func (r *CommittedResourceChangeReconciler) markError(ctx context.Context, tx *v } } +// TODO (#781): add a TTL/GC controller that deletes terminal CommittedResourceChanges after their +// retention window (Accepted ~1h, Rejected ~5d). Non-terminal CRCs must never be GC'd — deleting +// one loses the rollback journal and the #783 AZ active-marker. + // SetupWithManager registers the controller and configures watches. // TODO (#783): set MaxConcurrentReconciles: 1 via WithOptions(controller.Options{MaxConcurrentReconciles: 1}). // TODO (#783): add predicate on WatchesMulticluster() to re-trigger same-AZ non-terminal siblings on terminal transition. diff --git a/internal/scheduling/reservations/commitments/committed_resource_controller.go b/internal/scheduling/reservations/commitments/committed_resource_controller.go index a4e55ad04..06a9a65ec 100644 --- a/internal/scheduling/reservations/commitments/committed_resource_controller.go +++ b/internal/scheduling/reservations/commitments/committed_resource_controller.go @@ -7,7 +7,6 @@ import ( "context" "errors" "fmt" - "strconv" "time" "k8s.io/apimachinery/pkg/api/equality" @@ -126,12 +125,7 @@ func (r *CommittedResourceController) reconcilePending(ctx context.Context, logg // error would cause controller-runtime to ignore RequeueAfter and use exponential backoff. return ctrl.Result{RequeueAfter: delay}, nil } - // alreadyReserving is true when the CR condition already shows Reserving for this generation, - // meaning at least one reconcile cycle (~1s) has elapsed since we first entered the waiting state. - // We only nudge stale Reservation slots after that delay to give the normal echo path a chance - // to complete first, keeping logs quiet in the happy path. - alreadyReserving := isAlreadyReserving(cr) - allReady, anyFailed, readySlots, failReason, firstSlotReason, failedSlots, err := r.checkChildReservationStatus(ctx, cr, result.TotalSlots, alreadyReserving) + allReady, anyFailed, readySlots, failReason, firstSlotReason, failedSlots, err := r.checkChildReservationStatus(ctx, cr, result.TotalSlots) if err != nil { return ctrl.Result{}, err } @@ -158,14 +152,8 @@ func (r *CommittedResourceController) reconcilePending(ctx context.Context, logg // Reset the retry timer: applyReservationState just succeeded, so the watch suppression // gate should not fire while we wait for slots to become ready. // Fallback requeue: in case the watch event is missed, - // re-check after 1s rather than waiting for the full watch timeout. - // Use "still waiting" message after the first cycle so the condition is a visible hint - // that a nudge has been (or is about to be) sent to stale slots. - msg := "waiting for reservation placement" - if alreadyReserving { - msg = "still waiting for reservation placement" - } - if err := r.patchNotReady(ctx, cr, v1alpha1.CommittedResourceReasonReserving, msg, true); err != nil { + // re-check after 3s rather than waiting for the full watch timeout. + if err := r.patchNotReady(ctx, cr, v1alpha1.CommittedResourceReasonReserving, "waiting for reservation placement", true); err != nil { return ctrl.Result{}, err } return ctrl.Result{RequeueAfter: 3 * time.Second}, nil @@ -189,6 +177,8 @@ func (r *CommittedResourceController) reconcilePending(ctx context.Context, logg return ctrl.Result{}, nil } +// TODO (#790): after reservation allocation changes (#787/#790), react here with a global view: +// resize/merge/recreate reservation slots and remap PAYG VMs onto available capacity. func (r *CommittedResourceController) reconcileCommitted(ctx context.Context, logger logr.Logger, cr *v1alpha1.CommittedResource) (ctrl.Result, error) { logger.V(1).Info("reconciling committed resource", "generation", cr.Generation, @@ -203,46 +193,7 @@ func (r *CommittedResourceController) reconcileCommitted(ctx context.Context, lo // Without this guard the controller oscillates: apply bad spec → rollback → // Reservation watch re-enqueues → apply bad spec again → loop. if isRejectedForGeneration(cr) { - // Recovery: if the current spec matches the accepted spec, the rejection was - // spurious (e.g. transient host failure). Check if slots have since recovered. - // Only safe when specs match — if they differ, the rejection was genuine (bad new spec) - // and calling setAccepted would re-expose the current bad spec on the next reconcile. - if cr.Status.AcceptedSpec != nil && equality.Semantic.DeepEqual(cr.Spec, *cr.Status.AcceptedSpec) { - result, applyErr := r.applyReservationState(ctx, logger, cr) - if applyErr != nil { - logger.V(1).Info("spurious-rejection recovery: apply failed, will retry", "err", applyErr) - } else { - allReady, anyFailed, _, _, _, _, checkErr := r.checkChildReservationStatus(ctx, cr, result.TotalSlots, false) - if checkErr == nil && allReady && !anyFailed { - logger.Info("recovered from spurious rejection: all slots ready", "generation", cr.Generation) - initialGen := cr.Generation - if err := r.setAccepted(ctx, cr); err != nil { - return ctrl.Result{}, err - } - var latest v1alpha1.CommittedResource - if err := r.Get(ctx, client.ObjectKeyFromObject(cr), &latest); err != nil { - logger.V(1).Info("re-Get after accept failed, requeueing", "err", err) - return ctrl.Result{RequeueAfter: 1 * time.Second}, nil - } - if latest.Generation > initialGen { - logger.V(1).Info("generation advanced during reconcile, requeueing", - "processedGen", initialGen, - "latestGen", latest.Generation, - ) - return ctrl.Result{RequeueAfter: 1 * time.Second}, nil - } - return ctrl.Result{}, nil - } - if checkErr != nil { - logger.V(1).Info("spurious-rejection recovery: slot check failed, will retry", "err", checkErr) - } - } - // Slots exist but are not all ready yet. No need to re-apply the same accepted spec; - // requeue and wait for the reservation controller to complete placement. - return ctrl.Result{RequeueAfter: r.retryDelay(cr)}, nil - } - logger.V(1).Info("spec already rejected for current generation, maintaining rollback state", "generation", cr.Generation) - return ctrl.Result{}, r.rollbackToAccepted(ctx, logger, cr) + return r.handleRejectedGeneration(ctx, logger, cr) } // Spec errors are permanent regardless of AllowRejection — a bad spec won't fix itself. if _, err := FromCommittedResource(*cr); err != nil { @@ -267,8 +218,7 @@ func (r *CommittedResourceController) reconcileCommitted(ctx context.Context, lo // error would cause controller-runtime to ignore RequeueAfter and use exponential backoff. return ctrl.Result{RequeueAfter: delay}, nil } - alreadyReserving := isAlreadyReserving(cr) - allReady, anyFailed, readySlots, failReason, firstSlotReason, failedSlots, err := r.checkChildReservationStatus(ctx, cr, result.TotalSlots, alreadyReserving) + allReady, anyFailed, readySlots, failReason, firstSlotReason, failedSlots, err := r.checkChildReservationStatus(ctx, cr, result.TotalSlots) if err != nil { return ctrl.Result{}, err } @@ -289,17 +239,8 @@ func (r *CommittedResourceController) reconcileCommitted(ctx context.Context, lo } if !allReady { // Reservation controller hasn't processed all slots yet; Reservation watch will re-enqueue. - // Reset the retry timer: applyReservationState just succeeded, so the watch suppression - // gate should not fire while we wait for slots to become ready. - // Fallback requeue: in case the watch event is missed, - // re-check after 1s rather than waiting for the full watch timeout. - // Use "still waiting" message after the first cycle so the condition is a visible hint - // that a nudge has been (or is about to be) sent to stale slots. - msg := "waiting for reservation placement" - if alreadyReserving { - msg = "still waiting for reservation placement" - } - if err := r.patchNotReady(ctx, cr, v1alpha1.CommittedResourceReasonReserving, msg, true); err != nil { + // Fallback requeue: in case the watch event is missed, re-check after 3s. + if err := r.patchNotReady(ctx, cr, v1alpha1.CommittedResourceReasonReserving, "waiting for reservation placement", true); err != nil { return ctrl.Result{}, err } return ctrl.Result{RequeueAfter: 3 * time.Second}, nil @@ -326,10 +267,58 @@ func (r *CommittedResourceController) reconcileCommitted(ctx context.Context, lo return ctrl.Result{}, nil } +// handleRejectedGeneration handles the case where isRejectedForGeneration(cr) is true. +// If the current spec matches AcceptedSpec, attempts spurious-rejection recovery: applies the +// spec and checks child slots. On success, calls setAccepted. On failure or if slots are +// not yet ready, requeues. If specs differ, the rejection is genuine: maintains rollback state. +func (r *CommittedResourceController) handleRejectedGeneration(ctx context.Context, logger logr.Logger, cr *v1alpha1.CommittedResource) (ctrl.Result, error) { + // Recovery: if the current spec matches the accepted spec, the rejection was + // spurious (e.g. transient host failure). Check if slots have since recovered. + // Only safe when specs match — if they differ, the rejection was genuine (bad new spec) + // and calling setAccepted would re-expose the current bad spec on the next reconcile. + if cr.Status.AcceptedSpec != nil && equality.Semantic.DeepEqual(cr.Spec, *cr.Status.AcceptedSpec) { + result, applyErr := r.applyReservationState(ctx, logger, cr) + if applyErr != nil { + logger.V(1).Info("spurious-rejection recovery: apply failed, will retry", "err", applyErr) + } else { + allReady, anyFailed, _, _, _, _, checkErr := r.checkChildReservationStatus(ctx, cr, result.TotalSlots) + if checkErr == nil && allReady && !anyFailed { + logger.Info("recovered from spurious rejection: all slots ready", "generation", cr.Generation) + initialGen := cr.Generation + if err := r.setAccepted(ctx, cr); err != nil { + return ctrl.Result{}, err + } + var latest v1alpha1.CommittedResource + if err := r.Get(ctx, client.ObjectKeyFromObject(cr), &latest); err != nil { + logger.V(1).Info("re-Get after accept failed, requeueing", "err", err) + return ctrl.Result{RequeueAfter: 1 * time.Second}, nil + } + if latest.Generation > initialGen { + logger.V(1).Info("generation advanced during reconcile, requeueing", + "processedGen", initialGen, + "latestGen", latest.Generation, + ) + return ctrl.Result{RequeueAfter: 1 * time.Second}, nil + } + return ctrl.Result{}, nil + } + if checkErr != nil { + logger.V(1).Info("spurious-rejection recovery: slot check failed, will retry", "err", checkErr) + } + } + // Slots exist but are not all ready yet. No need to re-apply the same accepted spec; + // requeue and wait for the reservation controller to complete placement. + return ctrl.Result{RequeueAfter: r.retryDelay(cr)}, nil + } + logger.V(1).Info("spec already rejected for current generation, maintaining rollback state", "generation", cr.Generation) + return ctrl.Result{}, r.rollbackToAccepted(ctx, logger, cr) +} + // reconcileCoresHeadroom handles acceptance of CPU core commitments. // No Reservation CRDs are created; instead it reads the flavor group's total CPU capacity // from the FlavorGroupCapacity CRD and sums already-accepted CPU CRs for the same // (flavorGroup, AZ) to check whether sufficient headroom exists. +// TODO (#789) unify with main reservation path once resource type abstraction is in place func (r *CommittedResourceController) reconcileCoresHeadroom(ctx context.Context, logger logr.Logger, cr *v1alpha1.CommittedResource) (ctrl.Result, error) { if isRejectedForGeneration(cr) { logger.V(1).Info("spec already rejected for current generation", "generation", cr.Generation) @@ -482,7 +471,9 @@ func (r *CommittedResourceController) applyReservationState(ctx context.Context, // Returns allReady=false, anyFailed=false when some children have no condition yet (placement pending). // failReason includes a slot count when more than one slot failed. // failedSlots lists the names of all failing reservation slots for log context. -func (r *CommittedResourceController) checkChildReservationStatus(ctx context.Context, cr *v1alpha1.CommittedResource, expectedSlots int, shouldNudge bool) (allReady, anyFailed bool, readySlots int, failReason, firstFailMessage string, failedSlots []string, err error) { +// TODO (#789) the CRC reconciler (#779) gates AwaitingScheduling→Accepted on +// cond.ObservedGeneration == res.Generation; ensure this check stays consistent with that gate. +func (r *CommittedResourceController) checkChildReservationStatus(ctx context.Context, cr *v1alpha1.CommittedResource, expectedSlots int) (allReady, anyFailed bool, readySlots int, failReason, firstFailMessage string, failedSlots []string, err error) { var list v1alpha1.ReservationList if err := r.List(ctx, &list, client.MatchingLabels{v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelCommittedResource}, @@ -517,30 +508,8 @@ func (r *CommittedResourceController) checkChildReservationStatus(ctx context.Co ) if cond == nil || cond.ObservedGeneration != res.Generation { // Reservation controller hasn't processed the current generation yet. - // Poke the Reservation to generate a fresh watch event so the - // Reservation controller wakes up. - // Use gen+ms as the annotation value so each nudge attempt is a distinct string: - // a same-value patch is a no-op on the API server (no watch event generated), and - // in a double-stale scenario the first nudge may itself be processed with a stale - // object, so periodic retries (every 3s via the fallback requeue) are needed. - // shouldNudge is false on the first cycle (CR condition not yet set for this - // generation) to give the normal path ~3s to complete before intervening. - if shouldNudge { - nudgeVal := strconv.FormatInt(cr.Generation, 10) + "-" + strconv.FormatInt(time.Now().UnixMilli(), 10) - ctrl.LoggerFrom(ctx).V(1).Info("nudging stale reservation: condition ObservedGeneration behind reservation generation", - "slot", res.Name, - "resGen", res.Generation, - "condObsGen", condObsGen, - ) - old := res.DeepCopy() - if res.Annotations == nil { - res.Annotations = map[string]string{} - } - res.Annotations["cortex.cloud/echo-nudge"] = nudgeVal - if pErr := r.Patch(ctx, &res, client.MergeFrom(old)); client.IgnoreNotFound(pErr) != nil { - ctrl.LoggerFrom(ctx).V(1).Info("failed to nudge stale reservation", "slot", res.Name, "err", pErr) - } - } + // The Reservation watch in SetupWithManager re-enqueues this CR when the + // reservation status changes; the RequeueAfter is the fallback. allReady = false continue } @@ -635,6 +604,7 @@ func DeleteChildReservations(ctx context.Context, k8sClient client.Client, cr *v // AcceptedSpec is a full snapshot of the spec at the last successful reconcile, so rollback always // targets the correct AZ, amount, project, domain — even when the current spec has been mutated. // If AcceptedSpec is nil (CR was never accepted), all child Reservations are deleted. +// TODO (#789) remove once status.acceptedSpec is dropped (after #785+#786 land; spec written only by committed CRCs) func (r *CommittedResourceController) rollbackToAccepted(ctx context.Context, logger logr.Logger, cr *v1alpha1.CommittedResource) error { if cr.Status.AcceptedSpec == nil { return r.deleteChildReservations(ctx, cr) @@ -680,18 +650,6 @@ func isRejectedForGeneration(cr *v1alpha1.CommittedResource) bool { cond.ObservedGeneration == cr.Generation } -// isAlreadyReserving returns true when the CR condition already shows Reserving for this -// generation, meaning at least one reconcile cycle (~1s) has elapsed since we first entered -// the waiting-for-slots state. Used to gate the echo-nudge so the first reconcile cycle is -// always silent, giving the normal Reservation echo path a chance to complete first. -func isAlreadyReserving(cr *v1alpha1.CommittedResource) bool { - cond := meta.FindStatusCondition(cr.Status.Conditions, v1alpha1.CommittedResourceConditionReady) - return cond != nil && - cond.Status == metav1.ConditionFalse && - cond.Reason == v1alpha1.CommittedResourceReasonReserving && - cond.ObservedGeneration == cr.Generation -} - // retryDelay computes an exponential backoff interval for the AllowRejection=false retry paths. // The exponent is derived from the time already spent in Reserving state: each doubling period // advances one step, giving the same base→2*base→4*base→… sequence as a counter would, but diff --git a/internal/scheduling/reservations/commitments/committed_resource_controller_test.go b/internal/scheduling/reservations/commitments/committed_resource_controller_test.go index 589f13b85..2c6f30c0e 100644 --- a/internal/scheduling/reservations/commitments/committed_resource_controller_test.go +++ b/internal/scheduling/reservations/commitments/committed_resource_controller_test.go @@ -1152,10 +1152,9 @@ func TestCommittedResourceController_BadSpec(t *testing.T) { // Tests: checkChildReservationStatus generation guard // ============================================================================ -// TestCommittedResourceController_NudgeDelay verifies that the echo-nudge is not sent on -// the first reconcile cycle (when the condition is not yet Reserving for this generation), -// but is sent on the second cycle — and that the condition message reflects the cycle. -func TestCommittedResourceController_NudgeDelay(t *testing.T) { +// TestCommittedResourceController_ReservingMessage verifies that successive reconcile cycles +// both show "waiting for reservation placement" (no echo-nudge annotation is ever written). +func TestCommittedResourceController_ReservingMessage(t *testing.T) { scheme := newCRTestScheme(t) cr := newTestCommittedResource("test-cr", v1alpha1.CommitmentStatusPending) cr.Spec.AllowRejection = false @@ -1170,7 +1169,7 @@ func TestCommittedResourceController_NudgeDelay(t *testing.T) { } return updated } - getChildAnnotation := func() string { + hasNudgeAnnotation := func() bool { t.Helper() var list v1alpha1.ReservationList if err := k8sClient.List(context.Background(), &list, client.MatchingLabels{ @@ -1181,42 +1180,27 @@ func TestCommittedResourceController_NudgeDelay(t *testing.T) { for _, res := range list.Items { if res.Spec.CommittedResourceReservation != nil && res.Spec.CommittedResourceReservation.CommitmentUUID == cr.Spec.CommitmentUUID { - return res.Annotations["cortex.cloud/echo-nudge"] + _, ok := res.Annotations["cortex.cloud/echo-nudge"] + return ok } } - return "" + return false } - // Cycle 1: no Reserving condition exists yet → isAlreadyReserving=false → no nudge, - // condition message = "waiting for reservation placement". - if _, err := controller.Reconcile(context.Background(), reconcileReq(cr.Name)); err != nil { - t.Fatalf("reconcile 1: %v", err) - } - cond1 := meta.FindStatusCondition(getCR().Status.Conditions, v1alpha1.CommittedResourceConditionReady) - if cond1 == nil || cond1.Reason != v1alpha1.CommittedResourceReasonReserving { - t.Fatalf("cycle 1: expected Reserving condition, got %+v", cond1) - } - if cond1.Message != "waiting for reservation placement" { - t.Errorf("cycle 1: message want %q, got %q", "waiting for reservation placement", cond1.Message) - } - if got := getChildAnnotation(); got != "" { - t.Errorf("cycle 1: echo-nudge annotation should not be set, got %q", got) - } - - // Cycle 2: Reserving condition is now set for this generation → isAlreadyReserving=true → - // nudge fires, condition message = "still waiting for reservation placement". - if _, err := controller.Reconcile(context.Background(), reconcileReq(cr.Name)); err != nil { - t.Fatalf("reconcile 2: %v", err) - } - cond2 := meta.FindStatusCondition(getCR().Status.Conditions, v1alpha1.CommittedResourceConditionReady) - if cond2 == nil || cond2.Reason != v1alpha1.CommittedResourceReasonReserving { - t.Fatalf("cycle 2: expected Reserving condition, got %+v", cond2) - } - if cond2.Message != "still waiting for reservation placement" { - t.Errorf("cycle 2: message want %q, got %q", "still waiting for reservation placement", cond2.Message) - } - if got := getChildAnnotation(); got == "" { - t.Errorf("cycle 2: echo-nudge annotation should be set") + for cycle := 1; cycle <= 2; cycle++ { + if _, err := controller.Reconcile(context.Background(), reconcileReq(cr.Name)); err != nil { + t.Fatalf("reconcile %d: %v", cycle, err) + } + cond := meta.FindStatusCondition(getCR().Status.Conditions, v1alpha1.CommittedResourceConditionReady) + if cond == nil || cond.Reason != v1alpha1.CommittedResourceReasonReserving { + t.Fatalf("cycle %d: expected Reserving condition, got %+v", cycle, cond) + } + if cond.Message != "waiting for reservation placement" { + t.Errorf("cycle %d: message want %q, got %q", cycle, "waiting for reservation placement", cond.Message) + } + if hasNudgeAnnotation() { + t.Errorf("cycle %d: echo-nudge annotation must not be set", cycle) + } } } @@ -1228,42 +1212,26 @@ func TestCheckChildReservationStatus_GenerationGuard(t *testing.T) { obsGen int64 condStatus metav1.ConditionStatus // "" = no condition set condMessage string - shouldNudge bool wantAllReady bool wantAnyFailed bool wantReadySlots int wantReason string wantFailedSlots []string - wantNudge bool }{ { name: "Ready=False at stale generation: treated as pending", obsGen: 1, condStatus: metav1.ConditionFalse, condMessage: "no hosts available", - shouldNudge: true, - wantAllReady: false, - wantAnyFailed: false, - wantReadySlots: 0, - wantNudge: true, - }, - { - name: "Ready=False at stale generation: no nudge on first cycle", - obsGen: 1, - condStatus: metav1.ConditionFalse, - condMessage: "no hosts available", - shouldNudge: false, wantAllReady: false, wantAnyFailed: false, wantReadySlots: 0, - wantNudge: false, }, { name: "Ready=False at current generation: is a current failure", obsGen: 2, condStatus: metav1.ConditionFalse, condMessage: "no hosts available", - shouldNudge: true, wantAllReady: false, wantAnyFailed: true, wantReadySlots: 0, @@ -1274,29 +1242,24 @@ func TestCheckChildReservationStatus_GenerationGuard(t *testing.T) { name: "Ready=True at current generation: allReady", obsGen: 2, condStatus: metav1.ConditionTrue, - shouldNudge: true, wantAllReady: true, wantReadySlots: 1, }, { - name: "no condition yet at current generation: still pending", + name: "no condition yet: still pending", obsGen: 2, condStatus: "", // no condition - shouldNudge: true, wantAllReady: false, wantAnyFailed: false, wantReadySlots: 0, - wantNudge: true, }, { - name: "parentGen written to spec but echo not yet in status: nudge annotation set", - obsGen: 1, // obs behind cr.Generation=2; spec.parentGeneration already set to cr.Generation + name: "condition ObservedGeneration behind reservation generation: still pending", + obsGen: 1, // obs behind cr.Generation=2 condStatus: metav1.ConditionTrue, - shouldNudge: true, wantAllReady: false, wantAnyFailed: false, wantReadySlots: 0, - wantNudge: true, }, } @@ -1338,7 +1301,7 @@ func TestCheckChildReservationStatus_GenerationGuard(t *testing.T) { } controller := &CommittedResourceController{Client: k8sClient, Scheme: scheme} - allReady, anyFailed, readySlots, reason, _, failedSlots, err := controller.checkChildReservationStatus(context.Background(), cr, 1, tt.shouldNudge) + allReady, anyFailed, readySlots, reason, _, failedSlots, err := controller.checkChildReservationStatus(context.Background(), cr, 1) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -1357,14 +1320,6 @@ func TestCheckChildReservationStatus_GenerationGuard(t *testing.T) { if !slices.Equal(failedSlots, tt.wantFailedSlots) { t.Errorf("failedSlots: want %v, got %v", tt.wantFailedSlots, failedSlots) } - var updated v1alpha1.Reservation - if err := k8sClient.Get(context.Background(), client.ObjectKey{Name: child.Name}, &updated); err != nil { - t.Fatalf("get updated reservation: %v", err) - } - _, hasNudge := updated.Annotations["cortex.cloud/echo-nudge"] - if hasNudge != tt.wantNudge { - t.Errorf("echo-nudge annotation: want %v, got %v", tt.wantNudge, hasNudge) - } }) } } diff --git a/internal/scheduling/reservations/commitments/payg_candidates.go b/internal/scheduling/reservations/commitments/payg_candidates.go index 952fe3cab..49a96b607 100644 --- a/internal/scheduling/reservations/commitments/payg_candidates.go +++ b/internal/scheduling/reservations/commitments/payg_candidates.go @@ -84,6 +84,8 @@ func ScanAZForPaygCandidates( // buildAllocatedVMSet lists CR Reservations in az and returns the set of VM UUIDs present // in any Status.Allocation (confirmed placements). Filtering by AZ keeps the set small — // only slots relevant to the current scan are included. One cache scan shared across all HV filters. +// TODO (#790): replace full List scan with the idxReservationByAllocationVMUUID field index +// (field_index.go) for O(1) reverse lookup instead of scanning all reservations. func buildAllocatedVMSet(ctx context.Context, k8sClient client.Client, az string) (map[string]struct{}, error) { var resList v1alpha1.ReservationList if err := k8sClient.List(ctx, &resList, diff --git a/internal/scheduling/reservations/commitments/reservation_controller.go b/internal/scheduling/reservations/commitments/reservation_controller.go index 6364ec5d8..6c31b0725 100644 --- a/internal/scheduling/reservations/commitments/reservation_controller.go +++ b/internal/scheduling/reservations/commitments/reservation_controller.go @@ -66,10 +66,8 @@ type CommitmentReservationController struct { // move the current state of the cluster closer to the desired state. // Note: This controller only handles commitment reservations, as filtered by the predicate. func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - // Fetch the reservation object first to check for creator request ID. var res v1alpha1.Reservation if err := r.Get(ctx, req.NamespacedName, &res); err != nil { - // Ignore not-found errors, since they can't be fixed by an immediate requeue return ctrl.Result{}, client.IgnoreNotFound(err) } @@ -94,95 +92,46 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr }) patch := client.MergeFrom(old) if err := r.Status().Patch(ctx, &res, patch); err != nil { - // Ignore not-found errors during background deletion if client.IgnoreNotFound(err) != nil { logger.Error(err, "failed to patch reservation status") return ctrl.Result{}, err } - // Object was deleted, no need to continue return ctrl.Result{}, nil } - return ctrl.Result{}, nil // Don't need to requeue. + return ctrl.Result{}, nil } // Spec.TargetHost was cleared (e.g. oversubscription eviction) while the slot was confirmed — // revoke status so the slot re-enters the placement flow on the next reconcile. if res.Status.Host != "" && res.Spec.TargetHost == "" { - old := res.DeepCopy() - res.Status.Host = "" - if res.Status.CommittedResourceReservation != nil { - res.Status.CommittedResourceReservation.Allocation = nil - } - meta.SetStatusCondition(&res.Status.Conditions, metav1.Condition{ - Type: v1alpha1.ReservationConditionReady, - Status: metav1.ConditionFalse, - Reason: "PlacementRevoked", - Message: "target host was cleared; pending re-placement", - ObservedGeneration: res.Generation, - }) - if err := r.Status().Patch(ctx, &res, client.MergeFrom(old)); client.IgnoreNotFound(err) != nil { - return ctrl.Result{}, err - } - logger.Info("revoked ready status after placement eviction, slot re-enters placement flow", - "component", "oversubscription-check") - return ctrl.Result{}, nil + return r.reconcileEvicted(ctx, logger, &res) } - // Slot is placed and confirmed — verify allocations and keep ObservedGeneration current. + // Slot is placed and confirmed. + // If the spec changed since last confirmed placement (ObservedGeneration behind), clear + // TargetHost to trigger re-placement via reconcileEvicted → reconcileUnplaced on the next + // two cycles. This handles both benign changes (same host re-confirmed, new ObservedGeneration + // stamped) and significant changes (resources/flavorGroup — new host may be selected). if res.Spec.TargetHost != "" && res.Status.Host == res.Spec.TargetHost { - // Deprecated: remove when Spec.CommittedResourceReservation.Allocations is removed from the schema. - // One-time upgrade migration: blind-scheduler slots populated Spec.Allocations on the old schema. - // Seed AnnotationCandidateAllocation from the lexically-first Spec entry so the existing - // annotation recovery path re-confirms the VM on the next pass. - if res.Spec.CommittedResourceReservation != nil && - res.Annotations[v1alpha1.AnnotationCandidateAllocation] == "" && - len(res.Spec.CommittedResourceReservation.Allocations) > 0 { //nolint:staticcheck - keys := make([]string, 0, len(res.Spec.CommittedResourceReservation.Allocations)) //nolint:staticcheck - for k := range res.Spec.CommittedResourceReservation.Allocations { //nolint:staticcheck - keys = append(keys, k) - } - sort.Strings(keys) - vmUUID := keys[0] - logger.Info("migrating first Spec allocation to annotation (one-time upgrade path)", - "vm", vmUUID, "reservation", res.Name) - old := res.DeepCopy() - if res.Annotations == nil { - res.Annotations = make(map[string]string) - } - res.Annotations[v1alpha1.AnnotationCandidateAllocation] = vmUUID - if err := r.Patch(ctx, &res, client.MergeFrom(old)); client.IgnoreNotFound(err) != nil { - return ctrl.Result{}, err - } - } - - logger.V(1).Info("reservation is active, verifying allocations") - if err := r.reconcileAllocations(ctx, &res); err != nil { - logger.Error(err, "failed to reconcile allocations") - return ctrl.Result{}, err - } - // Refresh ObservedGeneration if the spec changed since the last reconcile - // (e.g. endTime extension or metadata update via syncReservationMetadata). cond := meta.FindStatusCondition(res.Status.Conditions, v1alpha1.ReservationConditionReady) - if cond == nil || cond.ObservedGeneration < res.Generation { + if cond != nil && cond.ObservedGeneration < res.Generation { + logger.Info("spec changed since last placement, clearing target host to trigger re-placement", + "host", res.Spec.TargetHost, + "lastObservedGen", cond.ObservedGeneration, + "currentGen", res.Generation, + ) old := res.DeepCopy() - meta.SetStatusCondition(&res.Status.Conditions, metav1.Condition{ - Type: v1alpha1.ReservationConditionReady, - Status: metav1.ConditionTrue, - Reason: "ReservationActive", - Message: "reservation is successfully scheduled", - ObservedGeneration: res.Generation, - }) - if err := r.Status().Patch(ctx, &res, client.MergeFrom(old)); client.IgnoreNotFound(err) != nil { + res.Spec.TargetHost = "" + if err := r.Patch(ctx, &res, client.MergeFrom(old)); client.IgnoreNotFound(err) != nil { return ctrl.Result{}, err } + return ctrl.Result{}, nil } - return ctrl.Result{RequeueAfter: r.Conf.RequeueIntervalActive.Duration}, nil + return r.reconcilePlaced(ctx, logger, &res) } - // Sync Spec values to Status fields for non-pre-allocated reservations - // This ensures the observed state reflects the desired state from Spec - // When TargetHost is set in Spec but not synced to Status, this means - // the scheduler found a host and we need to mark the reservation as ready. + // Spec.TargetHost was written by the scheduler but not yet reflected in Status. + // Sync Status.Host once and mark ready; the next reconcile runs allocation verification. if res.Spec.TargetHost != "" && res.Status.Host != res.Spec.TargetHost { old := res.DeepCopy() res.Status.Host = res.Spec.TargetHost @@ -195,19 +144,109 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr }) patch := client.MergeFrom(old) if err := r.Status().Patch(ctx, &res, patch); err != nil { - // Ignore not-found errors during background deletion if client.IgnoreNotFound(err) != nil { logger.Error(err, "failed to sync spec to status") return ctrl.Result{}, err } - // Object was deleted, no need to continue return ctrl.Result{}, nil } logger.V(1).Info("synced spec to status and marked ready", "host", res.Status.Host) - // Return and let next reconcile handle allocation verification return ctrl.Result{}, nil } + // No target host yet — run the scheduler to find a placement. + return r.reconcileUnplaced(ctx, logger, &res) +} + +// reconcileEvicted handles the case where Spec.TargetHost was cleared (e.g. oversubscription eviction) +// while the slot was already placed. Clears Status.Host and allocations so the slot re-enters +// the normal placement flow on the next reconcile. +func (r *CommitmentReservationController) reconcileEvicted(ctx context.Context, logger logr.Logger, res *v1alpha1.Reservation) (ctrl.Result, error) { + old := res.DeepCopy() + res.Status.Host = "" + if res.Status.CommittedResourceReservation != nil { + res.Status.CommittedResourceReservation.Allocation = nil + } + meta.SetStatusCondition(&res.Status.Conditions, metav1.Condition{ + Type: v1alpha1.ReservationConditionReady, + Status: metav1.ConditionFalse, + Reason: "PlacementRevoked", + Message: "target host was cleared; pending re-placement", + ObservedGeneration: res.Generation, + }) + if err := r.Status().Patch(ctx, res, client.MergeFrom(old)); client.IgnoreNotFound(err) != nil { + return ctrl.Result{}, err + } + logger.Info("revoked ready status after placement eviction, slot re-enters placement flow", + "component", "oversubscription-check") + return ctrl.Result{}, nil +} + +// reconcilePlaced handles an active reservation: verifies that allocated VMs are still on the +// expected host and keeps ObservedGeneration current after benign spec updates (e.g. endTime extension). +// TODO (#787) watch VM CRD for allocation changes once VM CRD is ready — today this relies on +// the periodic HV-CRD watch + RequeueIntervalActive instead. +func (r *CommitmentReservationController) reconcilePlaced(ctx context.Context, logger logr.Logger, res *v1alpha1.Reservation) (ctrl.Result, error) { + // Deprecated: remove when Spec.CommittedResourceReservation.Allocations is removed from the schema. + // One-time upgrade migration: blind-scheduler slots populated Spec.Allocations on the old schema. + // Seed AnnotationCandidateAllocation from the lexically-first Spec entry so the existing + // annotation recovery path re-confirms the VM on the next pass. + if res.Spec.CommittedResourceReservation != nil && + res.Annotations[v1alpha1.AnnotationCandidateAllocation] == "" && + len(res.Spec.CommittedResourceReservation.Allocations) > 0 { //nolint:staticcheck + keys := make([]string, 0, len(res.Spec.CommittedResourceReservation.Allocations)) //nolint:staticcheck + for k := range res.Spec.CommittedResourceReservation.Allocations { //nolint:staticcheck + keys = append(keys, k) + } + sort.Strings(keys) + vmUUID := keys[0] + logger.Info("migrating first Spec allocation to annotation (one-time upgrade path)", + "vm", vmUUID, "reservation", res.Name) + old := res.DeepCopy() + if res.Annotations == nil { + res.Annotations = make(map[string]string) + } + res.Annotations[v1alpha1.AnnotationCandidateAllocation] = vmUUID + if err := r.Patch(ctx, res, client.MergeFrom(old)); client.IgnoreNotFound(err) != nil { + return ctrl.Result{}, err + } + } + + logger.V(1).Info("reservation is active, verifying allocations") + if err := r.reconcileAllocations(ctx, res); err != nil { + logger.Error(err, "failed to reconcile allocations") + return ctrl.Result{}, err + } + + // Refresh ObservedGeneration if the spec changed since the last reconcile + // (e.g. endTime extension or metadata update via syncReservationMetadata). + cond := meta.FindStatusCondition(res.Status.Conditions, v1alpha1.ReservationConditionReady) + if cond == nil || cond.ObservedGeneration < res.Generation { + old := res.DeepCopy() + meta.SetStatusCondition(&res.Status.Conditions, metav1.Condition{ + Type: v1alpha1.ReservationConditionReady, + Status: metav1.ConditionTrue, + Reason: "ReservationActive", + Message: "reservation is successfully scheduled", + ObservedGeneration: res.Generation, + }) + if err := r.Status().Patch(ctx, res, client.MergeFrom(old)); client.IgnoreNotFound(err) != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{RequeueAfter: r.Conf.RequeueIntervalActive.Duration}, nil +} + +// reconcileUnplaced runs the in-process scheduler to find a host for an unplaced reservation slot, +// then writes the selected host to Spec.TargetHost. The next reconcile cycle detects +// Spec.TargetHost != Status.Host and syncs Status in the Reconcile dispatcher. +// TODO (#778) replace HV-CRD host scoring with VM CRD data when ready +func (r *CommitmentReservationController) reconcileUnplaced(ctx context.Context, logger logr.Logger, res *v1alpha1.Reservation) (ctrl.Result, error) { + resourceName := "" + if res.Spec.CommittedResourceReservation != nil { + resourceName = res.Spec.CommittedResourceReservation.ResourceName + } + // Get project ID from CommittedResourceReservation spec if available. projectID := "" if res.Spec.CommittedResourceReservation != nil { @@ -280,7 +319,7 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr ObservedGeneration: res.Generation, }) patch := client.MergeFrom(old) - if err := r.Status().Patch(ctx, &res, patch); err != nil { + if err := r.Status().Patch(ctx, res, patch); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } return ctrl.Result{RequeueAfter: r.Conf.RequeueIntervalRetry.Duration}, nil @@ -357,13 +396,11 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr ObservedGeneration: res.Generation, }) patch := client.MergeFrom(old) - if err := r.Status().Patch(ctx, &res, patch); err != nil { - // Ignore not-found errors during background deletion + if err := r.Status().Patch(ctx, res, patch); err != nil { if client.IgnoreNotFound(err) != nil { logger.Error(err, "failed to patch reservation status") return ctrl.Result{}, err } - // Object was deleted, no need to continue return ctrl.Result{}, nil } return ctrl.Result{}, fmt.Errorf("no hosts found for reservation %s (flavor %s)", res.Name, resourceName) @@ -377,18 +414,14 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr old := res.DeepCopy() res.Spec.TargetHost = host - if err := r.Patch(ctx, &res, client.MergeFrom(old)); err != nil { - // Ignore not-found errors during background deletion + if err := r.Patch(ctx, res, client.MergeFrom(old)); err != nil { if client.IgnoreNotFound(err) != nil { logger.Error(err, "failed to patch reservation spec") return ctrl.Result{}, err } - // Object was deleted, no need to continue return ctrl.Result{}, nil } - // The Spec patch will trigger a re-reconcile, which will sync Status in the - // "Sync Spec values to Status" section above return ctrl.Result{}, nil } @@ -397,6 +430,8 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr // host have their partial allocation recomputed; VMs that have disappeared are removed. // 2. Promote a PAYG candidate VM (AnnotationCandidateAllocation) into // status.allocation when it is confirmed active on the expected host. +// +// TODO (#778) replace HV-CRD VMSource with VM CRD once available func (r *CommitmentReservationController) reconcileAllocations(ctx context.Context, res *v1alpha1.Reservation) error { logger := LoggerFromContext(ctx) @@ -494,7 +529,7 @@ func (r *CommitmentReservationController) reconcileAllocations(ctx context.Conte continue } newStatusAllocations[vmUUID] = v1alpha1.CommittedResourceAllocation{ - // TODO replace with VM CRD ref once the VM CRD is strictly considered as being the mandatory gate + // TODO (#778) replace with VM CRD ref once VM CRD is the mandatory source of truth ObjectRef: corev1.ObjectReference{Name: vmUUID}, Resources: newAlloc, } @@ -807,6 +842,7 @@ func (r *CommitmentReservationController) SetupWithManager(mgr ctrl.Manager, mcl // assigned to that host. This replaces periodic polling for the established-VM // verification path — changes are detected in seconds rather than up to // RequeueIntervalActive. RequeueIntervalActive remains as a safety-net fallback. + // TODO (#787) add VM CRD watch for allocation changes once VM CRD is ready bldr, err = bldr.WatchesMulticluster( &hv1.Hypervisor{}, handler.EnqueueRequestsFromMapFunc(r.hypervisorToReservations), diff --git a/internal/scheduling/reservations/commitments/reservation_manager_test.go b/internal/scheduling/reservations/commitments/reservation_manager_test.go index 84b798cc4..51c241b0a 100644 --- a/internal/scheduling/reservations/commitments/reservation_manager_test.go +++ b/internal/scheduling/reservations/commitments/reservation_manager_test.go @@ -1177,9 +1177,10 @@ func variableRatioFlavorGroup() compute.FlavorGroupFeature { } } -func TestNewReservation_VariableRatioGroup_SelectsLargestByMemory(t *testing.T) { +func TestNewReservation_VariableRatioGroup_SelectsLargestByMemoryFewestCPU(t *testing.T) { // For GP (variable CPU:RAM ratio) groups, flavor selection is driven by memory - // descending, not by ratio. The largest flavor fitting the delta is always chosen. + // descending. Among flavors at the same memory level, the one with fewest vCPUs + // is preferred to minimize CPU blocking on the host. manager := &ReservationManager{} fg := variableRatioFlavorGroup() @@ -1208,7 +1209,7 @@ func TestNewReservation_VariableRatioGroup_SelectsLargestByMemory(t *testing.T) wantCores: 8, }, { - name: "delta equals c8_m16: picks c8_m16 (more vCPUs than c4_m8 at same memory)", + name: "delta equals c8_m16: picks c8_m16 (only 16 GiB flavor)", deltaMemoryMB: 16384, wantFlavor: "c8_m16", wantCores: 8, @@ -1245,6 +1246,39 @@ func TestNewReservation_VariableRatioGroup_SelectsLargestByMemory(t *testing.T) } } +// sameMemotyFlavorGroup returns a flavor group where two flavors share the same memory +// but differ in vCPU count, to exercise the fewest-vCPU tiebreak in selectFlavor. +func sameMemoryFlavorGroup() compute.FlavorGroupFeature { + minRatio := uint64(2048) // MiB/vCPU + maxRatio := uint64(4096) // MiB/vCPU + return compute.FlavorGroupFeature{ + Name: "gp-same-mem", + Flavors: []compute.FlavorInGroup{ + {Name: "c4_m16", VCPUs: 4, MemoryMB: 16384, DiskGB: 50}, // fewest vCPUs at 16 GiB + {Name: "c8_m16", VCPUs: 8, MemoryMB: 16384, DiskGB: 50}, // more vCPUs at 16 GiB + {Name: "c4_m8", VCPUs: 4, MemoryMB: 8192, DiskGB: 25}, + }, + SmallestFlavor: compute.FlavorInGroup{Name: "c4_m8", VCPUs: 4, MemoryMB: 8192, DiskGB: 25}, + LargestFlavor: compute.FlavorInGroup{Name: "c4_m16", VCPUs: 4, MemoryMB: 16384, DiskGB: 50}, + RamCoreRatioMin: &minRatio, + RamCoreRatioMax: &maxRatio, + } +} + +func TestSelectFlavor_SameMemory_PrefersFewestCPU(t *testing.T) { + // When two flavors share the same memory, selectFlavor must pick the one with + // fewer vCPUs to avoid over-blocking CPU capacity on the host. + fg := sameMemoryFlavorGroup() + + flavorInGroup, _ := selectFlavor(int64(16384)*1024*1024, fg) + if got := flavorInGroup.Name; got != "c4_m16" { + t.Errorf("want c4_m16 (fewest vCPUs at 16 GiB), got %s", got) + } + if got := flavorInGroup.VCPUs; got != 4 { + t.Errorf("want 4 vCPUs, got %d", got) + } +} + // ============================================================================ // Tests: selectFlavor // ============================================================================ diff --git a/internal/scheduling/reservations/commitments/syncer.go b/internal/scheduling/reservations/commitments/syncer.go index 1071b0bd5..b13339cd6 100644 --- a/internal/scheduling/reservations/commitments/syncer.go +++ b/internal/scheduling/reservations/commitments/syncer.go @@ -230,9 +230,10 @@ func (s *Syncer) getCommitmentStates(ctx context.Context, log logr.Logger, flavo } // SyncReservations fetches commitments from Limes and synchronizes Reservation CRDs. +// TODO (#785): route writes through CommittedResourceChange (source=syncer) instead of writing +// CR.spec directly. The per-project recency check (skip projects touched in the last ~10 min by +// an API CRC) moves to the CRC reconciler (#782). func (s *Syncer) SyncReservations(ctx context.Context) error { - // TODO handle concurrency with change API: consider creation time of reservations and status ready - // Create context with request ID for this sync execution runID := fmt.Sprintf("sync-%d", time.Now().Unix()) ctx = reservations.WithGlobalRequestID(ctx, runID) @@ -488,6 +489,10 @@ func (s *Syncer) SyncReservations(ctx context.Context) error { // time and can settle on hosts whose labels no longer match; pruning them lets the reconciler // re-create fresh slots. A zero max age or per-run cap disables pruning. Returns the number of // Reservation CRDs deleted. +// TODO (#790): this only deletes stale slots one by one. The target behaviour (#790) is for the +// CR controller to take a global view across all slots of a CR: collect unused/empty capacity, +// compute an optimal new slot layout, and recreate at the best size. That logic belongs in the +// CR controller (committed_resource_controller.go), not the syncer. func (s *Syncer) pruneAgedReservations(ctx context.Context, logger logr.Logger) (int, error) { maxAge := s.resourceConfig.StaleReservationMaxAge.Duration maxDeletes := s.resourceConfig.MaxReservationDeletesPerRun