From a78ba48d8ccdeedec0a2df5584c4ce0676a5646e Mon Sep 17 00:00:00 2001 From: juliusclausnitzer Date: Mon, 28 Sep 2026 13:02:04 +0200 Subject: [PATCH 1/4] main fix Signed-off-by: juliusclausnitzer --- api/v1alpha1/flavor_group_capacity_types.go | 12 +++ api/v1alpha1/zz_generated.deepcopy.go | 7 ++ .../cortex.cloud_flavorgroupcapacities.yaml | 19 ++++ .../reservations/capacity/controller.go | 21 +++++ .../reservations/capacity/controller_test.go | 59 ++++++++++++ .../commitments/api/report_capacity_test.go | 90 +++++++++++++++++++ .../reservations/commitments/capacity.go | 36 +++++--- 7 files changed, 234 insertions(+), 10 deletions(-) diff --git a/api/v1alpha1/flavor_group_capacity_types.go b/api/v1alpha1/flavor_group_capacity_types.go index 3da5090af..fc54b0cd5 100644 --- a/api/v1alpha1/flavor_group_capacity_types.go +++ b/api/v1alpha1/flavor_group_capacity_types.go @@ -99,6 +99,18 @@ type FlavorGroupCapacityStatus struct { // +kubebuilder:validation:Optional ExclusivelyFreeSlots int64 `json:"exclusivelyFreeSlots,omitempty"` + // ExclusivelyReservedCapacity is the reservation-blocked memory and CPU (failover slots and + // unfilled committed-resource slots) on the hosts exclusively assigned to this group by the + // round-robin split. These slots are subtracted from both the placeable probe and the split, + // so they are neither running nor free; they must be added back to report installed capacity. + // +kubebuilder:validation:Optional + ExclusivelyReservedCapacity map[string]resource.Quantity `json:"exclusivelyReservedCapacity,omitempty"` + + // ExclusivelyReservedSlots is the number of smallest-flavor VM slots represented by + // ExclusivelyReservedCapacity. + // +kubebuilder:validation:Optional + ExclusivelyReservedSlots int64 `json:"exclusivelyReservedSlots,omitempty"` + // RunningInstances is the number of VMs running in this (flavor group × AZ) whose // flavor belongs to this group. // +kubebuilder:validation:Optional diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index dadbbb074..336538f12 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -882,6 +882,13 @@ func (in *FlavorGroupCapacityStatus) DeepCopyInto(out *FlavorGroupCapacityStatus (*out)[key] = val.DeepCopy() } } + if in.ExclusivelyReservedCapacity != nil { + in, out := &in.ExclusivelyReservedCapacity, &out.ExclusivelyReservedCapacity + *out = make(map[string]resource.Quantity, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } if in.RunningResources != nil { in, out := &in.RunningResources, &out.RunningResources *out = make(map[string]resource.Quantity, len(*in)) diff --git a/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml b/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml index f5203abc2..8236967e1 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml @@ -193,6 +193,25 @@ spec: ExclusivelyFreeCapacity, this is not constrained by slot packing or CPU/memory ratio. Safe to sum across groups — no host is counted twice. type: object + exclusivelyReservedCapacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + ExclusivelyReservedCapacity is the reservation-blocked memory and CPU (failover slots and + unfilled committed-resource slots) on the hosts exclusively assigned to this group by the + round-robin split. These slots are subtracted from both the placeable probe and the split, + so they are neither running nor free; they must be added back to report installed capacity. + type: object + exclusivelyReservedSlots: + description: |- + ExclusivelyReservedSlots is the number of smallest-flavor VM slots represented by + ExclusivelyReservedCapacity. + format: int64 + type: integer flavors: description: Flavors holds per-flavor capacity data for all flavors in the group. diff --git a/internal/scheduling/reservations/capacity/controller.go b/internal/scheduling/reservations/capacity/controller.go index 80a702edc..6ec6250d3 100644 --- a/internal/scheduling/reservations/capacity/controller.go +++ b/internal/scheduling/reservations/capacity/controller.go @@ -578,6 +578,23 @@ func (c *Reconciler) reconcileAZ( rawExclusiveByGroup[gName] = raw } + // Reservation blocks (failover + unfilled committed slots) attributed to each group over its + // exclusively-assigned hosts. These slots were subtracted from the placeable probe and the + // split, so they are neither running nor free; the report adds them back as installed capacity. + reservedByGroup := make(map[string]map[string]int64, len(exclusiveHosts)) + for gName, hostList := range exclusiveHosts { + reserved := make(map[string]int64, 2) + for _, hostName := range hostList { + blocked, ok := blockedByReservations[hostName] + if !ok { + continue + } + reserved[ResourceMemory] += blocked[ResourceMemory] + reserved[ResourceCores] += blocked[ResourceCores] + } + reservedByGroup[gName] = reserved + } + for _, r := range results { if !r.allFresh { if err := c.markCRDNotReady(ctx, r.groupName, az); err != nil { @@ -592,6 +609,7 @@ func (c *Reconciler) reconcileAZ( freeResources[r.groupName], exclusiveResources[r.groupName], rawExclusiveByGroup[r.groupName], + reservedByGroup[r.groupName], ); err != nil { logger.Error(err, "failed to write FlavorGroupCapacity CRD", "flavorGroup", r.groupName, "az", az) @@ -631,6 +649,7 @@ func (c *Reconciler) writeCRD( freeRes map[string]int64, exclusiveRes map[string]int64, exclusivelyRawRes map[string]int64, + reservedRes map[string]int64, ) error { crdName := crdNameFor(groupName, az) @@ -676,11 +695,13 @@ func (c *Reconciler) writeCRD( existing.Status.FreeCapacity = resMapToQuantity(freeRes) existing.Status.ExclusivelyFreeCapacity = resMapToQuantity(exclusiveRes) existing.Status.ExclusivelyRawCapacity = resMapToQuantity(exclusivelyRawRes) + existing.Status.ExclusivelyReservedCapacity = resMapToQuantity(reservedRes) var exclusivelyFreeSlots int64 if flavorMemBytes := int64(groupData.SmallestFlavor.MemoryMB) * 1024 * 1024; flavorMemBytes > 0 { //nolint:gosec flavorVCPUs := int64(groupData.SmallestFlavor.VCPUs) //nolint:gosec exclusivelyFreeSlots = flavorSlots(exclusiveRes, flavorMemBytes, flavorVCPUs) existing.Status.ExclusivelyFreeSlots = exclusivelyFreeSlots + existing.Status.ExclusivelyReservedSlots = flavorSlots(reservedRes, flavorMemBytes, flavorVCPUs) } existing.Status.LastReconcileAt = metav1.Now() diff --git a/internal/scheduling/reservations/capacity/controller_test.go b/internal/scheduling/reservations/capacity/controller_test.go index 8a80967b5..4cce05b1b 100644 --- a/internal/scheduling/reservations/capacity/controller_test.go +++ b/internal/scheduling/reservations/capacity/controller_test.go @@ -266,6 +266,65 @@ func TestReconcileAZ_CreatesCRD(t *testing.T) { } } +func TestReconcileAZ_AttributesReservedCapacity(t *testing.T) { + const ( + groupName = "hana-v2" + az = "qa-de-1a" + memMB = 4096 // flavor memory (1 slot) + flavorMem = int64(memMB) * 1024 * 1024 + flavorCPUs = 2 + ) + + scheme := newTestScheme(t) + // Host holds 2 flavor slots worth of memory; 1 slot is reserved (blocked), 1 remains free. + hv := newHypervisor("host-1", az, 2*flavorMem) + knowledge := newFlavorGroupKnowledge(t, groupName, memMB) + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(knowledge, hv). + WithStatusSubresource(&v1alpha1.FlavorGroupCapacity{}, &v1alpha1.Knowledge{}). + Build() + + schedulerServer := newMockSchedulerServer(t, []string{"host-1"}) + defer schedulerServer.Close() + + ctrl := newController(t, fakeClient, Config{ + SchedulerURL: schedulerServer.URL, + TotalPipeline: "kvm-report-capacity", + PlaceablePipeline: "kvm-general-purpose", + }) + + smallFlavor := compute.FlavorInGroup{Name: groupName + "-small", MemoryMB: memMB, VCPUs: flavorCPUs} + groupData := compute.FlavorGroupFeature{SmallestFlavor: smallFlavor, Flavors: []compute.FlavorInGroup{smallFlavor}} + hvByName := map[string]hv1.Hypervisor{"host-1": *hv} + + // One flavor slot on host-1 is blocked by reservations (e.g. a failover slot). + blockedByReservations := map[string]map[string]int64{ + "host-1": {ResourceMemory: flavorMem, ResourceCores: flavorCPUs}, + } + + ctrl.reconcileAZ(context.Background(), az, + map[string]compute.FlavorGroupFeature{groupName: groupData}, + hvByName, blockedByReservations, map[vmUsageKey]vmUsage{}) + + var crd v1alpha1.FlavorGroupCapacity + if err := fakeClient.Get(context.Background(), types.NamespacedName{Name: crdNameFor(groupName, az)}, &crd); err != nil { + t.Fatalf("failed to get CRD: %v", err) + } + reservedMem := crd.Status.ExclusivelyReservedCapacity[string(v1alpha1.CommittedResourceTypeMemory)] + if reservedMem.Value() != flavorMem { + t.Errorf("ExclusivelyReservedCapacity[memory] = %d, want %d (one blocked slot on assigned host)", reservedMem.Value(), flavorMem) + } + reservedCores := crd.Status.ExclusivelyReservedCapacity[string(v1alpha1.CommittedResourceTypeCores)] + if reservedCores.Value() != flavorCPUs { + t.Errorf("ExclusivelyReservedCapacity[cores] = %d, want %d", reservedCores.Value(), flavorCPUs) + } + if crd.Status.ExclusivelyReservedSlots != 1 { + t.Errorf("ExclusivelyReservedSlots = %d, want 1", crd.Status.ExclusivelyReservedSlots) + } +} + func TestReconcileAZ_SkipsCRDWriteOnSchedulerError(t *testing.T) { const ( groupName = "hana-v2" diff --git a/internal/scheduling/reservations/commitments/api/report_capacity_test.go b/internal/scheduling/reservations/commitments/api/report_capacity_test.go index c67b735ac..55fb2101c 100644 --- a/internal/scheduling/reservations/commitments/api/report_capacity_test.go +++ b/internal/scheduling/reservations/commitments/api/report_capacity_test.go @@ -421,6 +421,96 @@ func TestCapacityCalculator_VariableRatio_PrefersRawCapacity(t *testing.T) { } } +// TestCapacityCalculator_ReservedSlots verifies that reserved-but-empty slots (failover + +// unfilled committed reservations, stored in ExclusivelyReservedCapacity/Slots) are added back +// into reported capacity for a fixed-ratio group, and counted as usage so availability stays +// equal to the truly-free amount. +func TestCapacityCalculator_ReservedSlots(t *testing.T) { + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + const flavorMemBytes = 32752 * 1024 * 1024 + + // running=200, exclusively free=800 slots, reserved=100 slots. + // capacity = 200 + 800 + 100 = 1100; usage = 200 + 100 = 300; availability = 800. + crd := createTestFlavorGroupCapacity(200, 800*flavorMemBytes, true) + crd.Status.ExclusivelyReservedSlots = 100 + crd.Status.ExclusivelyReservedCapacity = map[string]resource.Quantity{ + string(v1alpha1.CommittedResourceTypeMemory): *resource.NewQuantity(100*flavorMemBytes, resource.BinarySI), + string(v1alpha1.CommittedResourceTypeCores): *resource.NewQuantity(100*8, resource.DecimalSI), + } + calc := commitments.NewCapacityCalculator( + fake.NewClientBuilder().WithScheme(scheme). + WithObjects(createTestFlavorGroupKnowledge(t), crd).WithStatusSubresource(crd).Build(), + defaultCapacityConfig, + ) + report, err := calc.CalculateCapacity(context.Background(), + liquid.ServiceCapacityRequest{AllAZs: []liquid.AvailabilityZone{"az-one"}}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + for _, res := range []string{"hw_version_test-group_ram", "hw_version_test-group_instances"} { + az := report.Resources[liquid.ResourceName(res)].PerAZ["az-one"] + if az.Capacity != 1100 { + t.Errorf("%s: capacity = %d, want 1100 (running 200 + free 800 + reserved 100)", res, az.Capacity) + } + if usage := az.Usage.UnwrapOr(0); usage != 300 { + t.Errorf("%s: usage = %d, want 300 (running 200 + reserved 100)", res, usage) + } + if avail := az.Capacity - az.Usage.UnwrapOr(0); avail != 800 { + t.Errorf("%s: availability = %d, want 800 (free slots)", res, avail) + } + } + cores := report.Resources["hw_version_test-group_cores"].PerAZ["az-one"] + if cores.Capacity != 800 { + t.Errorf("cores capacity = %d, want 800 (reserved cores added back)", cores.Capacity) + } + if usage := cores.Usage.UnwrapOr(0); usage != 800 { + t.Errorf("cores usage = %d, want 800 (reserved cores)", usage) + } +} + +// TestCapacityCalculator_VariableRatio_RawIgnoresReserved verifies that for a variable-ratio +// group the RAM raw path already includes reserved hardware, so ExclusivelyReservedCapacity is +// NOT added again (no double count). +func TestCapacityCalculator_VariableRatio_RawIgnoresReserved(t *testing.T) { + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + const ( + ramUnitGiB = 2 + flavorMemMiB = 2048 + flavorMemBytes = int64(flavorMemMiB) * 1024 * 1024 + rawMemBytes = 20 * 1024 * 1024 * 1024 // 20 GiB → 10 declared units + ) + knowledge := createVariableRatioFlavorGroupKnowledge(t, flavorMemMiB) + crd := createFlavorGroupCapacityWithResources(3, 5*flavorMemBytes, 3*flavorMemBytes, 3*8, rawMemBytes) + // Reserved is set but must be ignored on the raw RAM path. + crd.Status.ExclusivelyReservedCapacity = map[string]resource.Quantity{ + string(v1alpha1.CommittedResourceTypeMemory): *resource.NewQuantity(4*1024*1024*1024, resource.BinarySI), + } + crd.Status.ExclusivelyReservedSlots = 2 + cfg := commitments.APIConfig{ + FlavorGroupResourceConfig: map[string]commitments.FlavorGroupResourcesConfig{ + "*": {RAM: commitments.RAMResourceTypeConfig{HasCapacity: true, RAMUnitGiB: ramUnitGiB}}, + }, + } + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme).WithObjects(knowledge, crd).WithStatusSubresource(crd).Build() + report, err := commitments.NewCapacityCalculator(fakeClient, cfg).CalculateCapacity( + context.Background(), liquid.ServiceCapacityRequest{AllAZs: []liquid.AvailabilityZone{"az-one"}}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + az := report.Resources["hw_version_test-group_ram"].PerAZ["az-one"] + if az.Capacity != 10 { + t.Errorf("RAM capacity = %d, want 10 (raw hardware only, reserved not double counted)", az.Capacity) + } +} + func verifyPerAZMatchesRequest(t *testing.T, res *liquid.ResourceCapacityReport, requestedAZs []liquid.AvailabilityZone) { t.Helper() if res == nil { diff --git a/internal/scheduling/reservations/commitments/capacity.go b/internal/scheduling/reservations/commitments/capacity.go index 4cb62fdbe..ed92ec0cc 100644 --- a/internal/scheduling/reservations/commitments/capacity.go +++ b/internal/scheduling/reservations/commitments/capacity.go @@ -35,8 +35,9 @@ func NewCapacityCalculator(client client.Client, conf APIConfig) *CapacityCalcul // CalculateCapacity computes per-AZ capacity for all flavor groups. // For each flavor group, three resources are reported: _ram, _cores, _instances. // All values are read from FlavorGroupCapacity CRDs pre-computed by the capacity controller: -// - Capacity: RunningInstances + ExclusivelyFreeCapacity converted to slots. -// - Usage: RunningInstances / RunningResources. +// - Capacity: RunningInstances + ExclusivelyFreeCapacity + ExclusivelyReservedCapacity, in slots. +// - Usage: RunningInstances / RunningResources + ExclusivelyReservedCapacity (so availability +// stays equal to the truly-free amount and reserved capacity is not advertised as available). func (c *CapacityCalculator) CalculateCapacity(ctx context.Context, req liquid.ServiceCapacityRequest) (liquid.ServiceCapacityReport, error) { knowledge := &reservations.FlavorGroupKnowledgeClient{Client: c.client} flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, nil) @@ -95,9 +96,22 @@ func (c *CapacityCalculator) CalculateCapacity(ctx context.Context, req liquid.S // ExclusivelyFreeSlots is pre-computed by the controller using min(memSlots, cpuSlots). exclusiveFreeSlots := uint64(crd.Status.ExclusivelyFreeSlots) //nolint:gosec - // Capacity = running + exclusively free, all derived from CRD bytes. + // Reservation-blocked slots (failover + unfilled committed) on this group's assigned + // hosts. They are subtracted from both the placeable probe and the split, so they are + // neither running nor free. Add them back so reported capacity reflects installed hardware. + reservedSlots := uint64(crd.Status.ExclusivelyReservedSlots) //nolint:gosec + reservedCores := int64(0) + if qty, ok := crd.Status.ExclusivelyReservedCapacity[string(v1alpha1.CommittedResourceTypeCores)]; ok { + reservedCores = qty.Value() + } + reservedMemBytes := int64(0) + if qty, ok := crd.Status.ExclusivelyReservedCapacity[string(v1alpha1.CommittedResourceTypeMemory)]; ok { + reservedMemBytes = qty.Value() + } + + // Capacity = running + exclusively free + reserved, all derived from CRD bytes. runningInstances := uint64(crd.Status.RunningInstances) //nolint:gosec - instancesCapacity := runningInstances + exclusiveFreeSlots + instancesCapacity := runningInstances + exclusiveFreeSlots + reservedSlots // RAM capacity: running bytes + exclusively free bytes → declared units. // Fixed-ratio groups report in slots (1 unit = 1 instance). @@ -120,7 +134,7 @@ func (c *CapacityCalculator) CalculateCapacity(ctx context.Context, req liquid.S if qty, ok := crd.Status.ExclusivelyRawCapacity[string(v1alpha1.CommittedResourceTypeMemory)]; ok && qty.Value() > 0 { ramCapacity = uint64(qty.Value()) / uint64(ramUnitBytes) //nolint:gosec } else { - ramCapacity = uint64(runningMemBytes+freeMemBytes) / uint64(ramUnitBytes) + ramCapacity = uint64(runningMemBytes+freeMemBytes+reservedMemBytes) / uint64(ramUnitBytes) } } @@ -134,19 +148,21 @@ func (c *CapacityCalculator) CalculateCapacity(ctx context.Context, req liquid.S if qty, ok := crd.Status.ExclusivelyFreeCapacity[string(v1alpha1.CommittedResourceTypeCores)]; ok { freeCoresCount = qty.Value() } - coresCapacity = uint64(runningCoresCount + freeCoresCount) + coresCapacity = uint64(runningCoresCount + freeCoresCount + reservedCores) ramEntry := &liquid.AZResourceCapacityReport{Capacity: ramCapacity} coresEntry := &liquid.AZResourceCapacityReport{Capacity: coresCapacity} instancesEntry := &liquid.AZResourceCapacityReport{Capacity: instancesCapacity} - // Usage from actual running VMs — only when CRD data is fresh. + // Usage from actual running VMs plus reserved-but-empty slots — only when CRD data is + // fresh. Reserved slots are counted as usage so that capacity − usage stays equal to + // the truly-free amount and reserved capacity is never advertised as available. if apimeta.IsStatusConditionTrue(crd.Status.Conditions, v1alpha1.FlavorGroupCapacityConditionReady) { - instancesEntry.Usage = Some[uint64](runningInstances) - coresEntry.Usage = Some[uint64](uint64(runningCoresCount)) + instancesEntry.Usage = Some[uint64](runningInstances + reservedSlots) + coresEntry.Usage = Some[uint64](uint64(runningCoresCount + reservedCores)) if groupData.HasFixedRamCoreRatio() { - ramEntry.Usage = Some[uint64](runningInstances) + ramEntry.Usage = Some[uint64](runningInstances + reservedSlots) } else if ramUnitBytes > 0 { runningMemBytes := int64(0) if qty, ok := crd.Status.RunningResources[string(v1alpha1.CommittedResourceTypeMemory)]; ok { From 9fc4e6ccaca5f7856a42be1bdca3d6b098d0fa2a Mon Sep 17 00:00:00 2001 From: juliusclausnitzer Date: Mon, 28 Sep 2026 14:54:54 +0200 Subject: [PATCH 2/4] per reservation mapping Signed-off-by: juliusclausnitzer --- api/v1alpha1/flavor_group_capacity_types.go | 8 +- .../cortex.cloud_flavorgroupcapacities.yaml | 8 +- .../reservations/capacity/controller.go | 136 +++++++++++++--- .../reservations/capacity/controller_test.go | 146 ++++++++++++++++-- .../reservations/commitments/capacity.go | 7 +- 5 files changed, 263 insertions(+), 42 deletions(-) diff --git a/api/v1alpha1/flavor_group_capacity_types.go b/api/v1alpha1/flavor_group_capacity_types.go index fc54b0cd5..b4403e836 100644 --- a/api/v1alpha1/flavor_group_capacity_types.go +++ b/api/v1alpha1/flavor_group_capacity_types.go @@ -100,9 +100,11 @@ type FlavorGroupCapacityStatus struct { ExclusivelyFreeSlots int64 `json:"exclusivelyFreeSlots,omitempty"` // ExclusivelyReservedCapacity is the reservation-blocked memory and CPU (failover slots and - // unfilled committed-resource slots) on the hosts exclusively assigned to this group by the - // round-robin split. These slots are subtracted from both the placeable probe and the split, - // so they are neither running nor free; they must be added back to report installed capacity. + // unfilled committed-resource slots) attributed to this group by each reservation's own + // ResourceGroup. These slots are subtracted from both the placeable probe and the split, so + // they are neither running nor free; they must be added back to report installed capacity. + // Because every reservation belongs to exactly one group, the sum across groups never + // exceeds installed capacity. // +kubebuilder:validation:Optional ExclusivelyReservedCapacity map[string]resource.Quantity `json:"exclusivelyReservedCapacity,omitempty"` diff --git a/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml b/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml index 8236967e1..b5c89078d 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml @@ -202,9 +202,11 @@ spec: x-kubernetes-int-or-string: true description: |- ExclusivelyReservedCapacity is the reservation-blocked memory and CPU (failover slots and - unfilled committed-resource slots) on the hosts exclusively assigned to this group by the - round-robin split. These slots are subtracted from both the placeable probe and the split, - so they are neither running nor free; they must be added back to report installed capacity. + unfilled committed-resource slots) attributed to this group by each reservation's own + ResourceGroup. These slots are subtracted from both the placeable probe and the split, so + they are neither running nor free; they must be added back to report installed capacity. + Because every reservation belongs to exactly one group, the sum across groups never + exceeds installed capacity. type: object exclusivelyReservedSlots: description: |- diff --git a/internal/scheduling/reservations/capacity/controller.go b/internal/scheduling/reservations/capacity/controller.go index 6ec6250d3..190e9a99f 100644 --- a/internal/scheduling/reservations/capacity/controller.go +++ b/internal/scheduling/reservations/capacity/controller.go @@ -244,10 +244,16 @@ func (c *Reconciler) reconcileAll(ctx context.Context) error { blockedByReservations = map[string]map[string]int64{} } + reservedByGroupAZ, err := c.reservedResourcesByGroupAZ(ctx, flavorGroups, hvByName) + if err != nil { + logger.Error(err, "failed to attribute reserved capacity to flavor groups, reserved capacity will be omitted") + reservedByGroupAZ = map[vmUsageKey]map[string]int64{} + } + usageByKey := c.computeVMUsage(ctx, flavorGroups, hvList.Items) for _, az := range azs { - c.reconcileAZ(ctx, az, flavorGroups, hvByName, blockedByReservations, usageByKey) + c.reconcileAZ(ctx, az, flavorGroups, hvByName, blockedByReservations, reservedByGroupAZ, usageByKey) } logger.Info("capacity reconcile cycle completed", @@ -505,6 +511,7 @@ func (c *Reconciler) reconcileAZ( flavorGroups map[string]compute.FlavorGroupFeature, hvByName map[string]hv1.Hypervisor, blockedByReservations map[string]map[string]int64, + reservedByGroupAZ map[vmUsageKey]map[string]int64, usageByKey map[vmUsageKey]vmUsage, ) { @@ -578,23 +585,6 @@ func (c *Reconciler) reconcileAZ( rawExclusiveByGroup[gName] = raw } - // Reservation blocks (failover + unfilled committed slots) attributed to each group over its - // exclusively-assigned hosts. These slots were subtracted from the placeable probe and the - // split, so they are neither running nor free; the report adds them back as installed capacity. - reservedByGroup := make(map[string]map[string]int64, len(exclusiveHosts)) - for gName, hostList := range exclusiveHosts { - reserved := make(map[string]int64, 2) - for _, hostName := range hostList { - blocked, ok := blockedByReservations[hostName] - if !ok { - continue - } - reserved[ResourceMemory] += blocked[ResourceMemory] - reserved[ResourceCores] += blocked[ResourceCores] - } - reservedByGroup[gName] = reserved - } - for _, r := range results { if !r.allFresh { if err := c.markCRDNotReady(ctx, r.groupName, az); err != nil { @@ -609,7 +599,7 @@ func (c *Reconciler) reconcileAZ( freeResources[r.groupName], exclusiveResources[r.groupName], rawExclusiveByGroup[r.groupName], - reservedByGroup[r.groupName], + reservedByGroupAZ[vmUsageKey{r.groupName, az}], ); err != nil { logger.Error(err, "failed to write FlavorGroupCapacity CRD", "flavorGroup", r.groupName, "az", az) @@ -881,6 +871,114 @@ func (c *Reconciler) blockedResourcesByHost(ctx context.Context) (map[string]map return blocked, nil } +// reservedResourcesByGroupAZ attributes reservation-blocked capacity (failover slots and +// unfilled committed-resource slots) to the flavor group that owns each reservation, keyed by +// (group, AZ). Attribution uses each reservation's own ResourceGroup rather than the host it +// sits on, so a host that is fully packed by empty reservations — and therefore assigned to no +// group by the round-robin split — still contributes its reserved capacity to the right group. +// Only placed reservations (with a target/status host that is a known hypervisor) are counted, +// each exactly once; pending reservations are picked up on a later reconcile once placed. +func (c *Reconciler) reservedResourcesByGroupAZ( + ctx context.Context, + flavorGroups map[string]compute.FlavorGroupFeature, + hvByName map[string]hv1.Hypervisor, +) (map[vmUsageKey]map[string]int64, error) { + + logger := LoggerFromContext(ctx) + var list v1alpha1.ReservationList + if err := c.client.List(ctx, &list); err != nil { + return nil, fmt.Errorf("failed to list reservations: %w", err) + } + + flavorToGroup := make(map[string]string) + for groupName, gd := range flavorGroups { + for _, f := range gd.Flavors { + flavorToGroup[f.Name] = groupName + } + } + + reserved := make(map[vmUsageKey]map[string]int64) + for i := range list.Items { + res := &list.Items[i] + // Only committed and failover reservations represent capacity that is reserved but + // empty; in-flight reservations are VMs in buildup and are counted as running instead. + if res.Spec.Type != v1alpha1.ReservationTypeCommittedResource && res.Spec.Type != v1alpha1.ReservationTypeFailover { + continue + } + host := res.Status.Host + if host == "" { + host = res.Spec.TargetHost + } + if host == "" { + continue // pending; reported on a later cycle once placed + } + hv, ok := hvByName[host] + if !ok { + continue // host is not a known hypervisor in this cluster + } + az := hv.Labels["topology.kubernetes.io/zone"] + if az == "" { + continue + } + group, ok := resolveReservationGroup(res, flavorGroups, flavorToGroup) + if !ok { + logger.Info("reservation could not be attributed to a flavor group; its reserved capacity is not reported", + "reservation", res.Name, "type", res.Spec.Type, "az", az) + continue + } + blockedRes := reservations.UnusedReservationCapacity(res, false) + key := vmUsageKey{group: group, az: az} + if reserved[key] == nil { + reserved[key] = make(map[string]int64) + } + if qty, ok := blockedRes[hv1.ResourceMemory]; ok { + reserved[key][ResourceMemory] += qty.Value() + } + if qty, ok := blockedRes[hv1.ResourceCPU]; ok { + reserved[key][ResourceCores] += qty.Value() + } + } + return reserved, nil +} + +// resolveReservationGroup determines the flavor group a reservation belongs to. Committed +// reservations carry the group directly. Failover reservations carry either the group name or — +// when useFlavorGroupResources is disabled — the flavor name, which is resolved to its group. +// Returns false if no group can be determined. +func resolveReservationGroup( + res *v1alpha1.Reservation, + flavorGroups map[string]compute.FlavorGroupFeature, + flavorToGroup map[string]string, +) (string, bool) { + + switch res.Spec.Type { + case v1alpha1.ReservationTypeCommittedResource: + if res.Spec.CommittedResourceReservation == nil { + return "", false + } + if g := res.Spec.CommittedResourceReservation.ResourceGroup; g != "" { + if _, ok := flavorGroups[g]; ok { + return g, true + } + } + return "", false + case v1alpha1.ReservationTypeFailover: + if res.Spec.FailoverReservation == nil { + return "", false + } + g := res.Spec.FailoverReservation.ResourceGroup + if _, ok := flavorGroups[g]; ok { + return g, true + } + if group, ok := flavorToGroup[g]; ok { + return group, true + } + return "", false + default: + return "", false + } +} + // sumCommittedCapacity sums active CommittedResource amounts (memory type, guaranteed/confirmed) // for the given (flavorGroup, az) pair. Returns the total in smallest-flavor slots. func (c *Reconciler) sumCommittedCapacity(ctx context.Context, groupName, az string, smallestFlavorBytes int64) (int64, error) { diff --git a/internal/scheduling/reservations/capacity/controller_test.go b/internal/scheduling/reservations/capacity/controller_test.go index 4cce05b1b..6370ad593 100644 --- a/internal/scheduling/reservations/capacity/controller_test.go +++ b/internal/scheduling/reservations/capacity/controller_test.go @@ -225,7 +225,7 @@ func TestReconcileAZ_CreatesCRD(t *testing.T) { ctrl.reconcileAZ(context.Background(), az, map[string]compute.FlavorGroupFeature{groupName: groupData}, - hvByName, map[string]map[string]int64{}, map[vmUsageKey]vmUsage{}) + hvByName, map[string]map[string]int64{}, nil, map[vmUsageKey]vmUsage{}) var crd v1alpha1.FlavorGroupCapacity if err := fakeClient.Get(context.Background(), types.NamespacedName{Name: crdNameFor(groupName, az)}, &crd); err != nil { @@ -266,7 +266,7 @@ func TestReconcileAZ_CreatesCRD(t *testing.T) { } } -func TestReconcileAZ_AttributesReservedCapacity(t *testing.T) { +func TestReconcileAZ_WritesReservedCapacity(t *testing.T) { const ( groupName = "hana-v2" az = "qa-de-1a" @@ -276,7 +276,7 @@ func TestReconcileAZ_AttributesReservedCapacity(t *testing.T) { ) scheme := newTestScheme(t) - // Host holds 2 flavor slots worth of memory; 1 slot is reserved (blocked), 1 remains free. + // Host holds 2 flavor slots worth of memory; 1 slot is reserved, 1 remains free. hv := newHypervisor("host-1", az, 2*flavorMem) knowledge := newFlavorGroupKnowledge(t, groupName, memMB) @@ -299,14 +299,15 @@ func TestReconcileAZ_AttributesReservedCapacity(t *testing.T) { groupData := compute.FlavorGroupFeature{SmallestFlavor: smallFlavor, Flavors: []compute.FlavorInGroup{smallFlavor}} hvByName := map[string]hv1.Hypervisor{"host-1": *hv} - // One flavor slot on host-1 is blocked by reservations (e.g. a failover slot). - blockedByReservations := map[string]map[string]int64{ - "host-1": {ResourceMemory: flavorMem, ResourceCores: flavorCPUs}, + // One flavor slot is reserved for this (group, AZ) — attributed upstream by + // reservedResourcesByGroupAZ and passed in here; reconcileAZ must persist it to the CRD. + reservedByGroupAZ := map[vmUsageKey]map[string]int64{ + {group: groupName, az: az}: {ResourceMemory: flavorMem, ResourceCores: flavorCPUs}, } ctrl.reconcileAZ(context.Background(), az, map[string]compute.FlavorGroupFeature{groupName: groupData}, - hvByName, blockedByReservations, map[vmUsageKey]vmUsage{}) + hvByName, map[string]map[string]int64{}, reservedByGroupAZ, map[vmUsageKey]vmUsage{}) var crd v1alpha1.FlavorGroupCapacity if err := fakeClient.Get(context.Background(), types.NamespacedName{Name: crdNameFor(groupName, az)}, &crd); err != nil { @@ -314,7 +315,7 @@ func TestReconcileAZ_AttributesReservedCapacity(t *testing.T) { } reservedMem := crd.Status.ExclusivelyReservedCapacity[string(v1alpha1.CommittedResourceTypeMemory)] if reservedMem.Value() != flavorMem { - t.Errorf("ExclusivelyReservedCapacity[memory] = %d, want %d (one blocked slot on assigned host)", reservedMem.Value(), flavorMem) + t.Errorf("ExclusivelyReservedCapacity[memory] = %d, want %d", reservedMem.Value(), flavorMem) } reservedCores := crd.Status.ExclusivelyReservedCapacity[string(v1alpha1.CommittedResourceTypeCores)] if reservedCores.Value() != flavorCPUs { @@ -325,6 +326,123 @@ func TestReconcileAZ_AttributesReservedCapacity(t *testing.T) { } } +// TestReservedResourcesByGroupAZ verifies that reservation-blocked capacity is attributed to a +// flavor group by each reservation's own ResourceGroup — independent of the round-robin split — +// so a host fully packed by empty reservations still contributes, and a host shared by two groups +// splits its reservations correctly. It also covers failover ResourceGroup carrying a flavor name +// and the skipping of pending and in-flight reservations. +func TestReservedResourcesByGroupAZ(t *testing.T) { + const az = "qa-de-1a" + const flavorMem = int64(4096) * 1024 * 1024 + const flavorCPUs = 2 + + scheme := newTestScheme(t) + // host-1 is shared by group-a and group-b reservations; host-2 is fully packed by a single + // failover reservation (zero free slots — it would drop out of the split's exclusiveHosts). + hv1a := newHypervisor("host-1", az, 2*flavorMem) + hv2 := newHypervisor("host-2", az, flavorMem) + + resources := map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: *resource.NewQuantity(flavorMem, resource.BinarySI), + hv1.ResourceCPU: *resource.NewQuantity(flavorCPUs, resource.DecimalSI), + } + + // committed reservation on host-1 → group-a (carries the group directly). + committedA := &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: "committed-a"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + AvailabilityZone: az, + TargetHost: "host-1", + Resources: resources, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ResourceGroup: "group-a"}, + }, + Status: v1alpha1.ReservationStatus{Host: "host-1"}, + } + // failover reservation on the same host-1 → group-b (carries the group name). + failoverB := &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: "failover-b"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeFailover, + AvailabilityZone: az, + TargetHost: "host-1", + Resources: resources, + FailoverReservation: &v1alpha1.FailoverReservationSpec{ResourceGroup: "group-b"}, + }, + Status: v1alpha1.ReservationStatus{Host: "host-1"}, + } + // failover reservation on the fully-packed host-2 → group-a, resolved from the flavor NAME. + failoverAByFlavor := &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: "failover-a-flavor"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeFailover, + AvailabilityZone: az, + TargetHost: "host-2", + Resources: resources, + FailoverReservation: &v1alpha1.FailoverReservationSpec{ResourceGroup: "group-a-small"}, + }, + Status: v1alpha1.ReservationStatus{Host: "host-2"}, + } + // pending reservation (no placed host) → skipped, reported on a later cycle. + pending := &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: "pending"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + AvailabilityZone: az, + Resources: resources, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ResourceGroup: "group-a"}, + }, + } + // in-flight reservation → skipped (counted as running, not reserved). + inflight := &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: "inflight"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeInFlight, + AvailabilityZone: az, + TargetHost: "host-1", + Resources: resources, + }, + Status: v1alpha1.ReservationStatus{Host: "host-1"}, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(committedA, failoverB, failoverAByFlavor, pending, inflight). + Build() + + c := newController(t, fakeClient, Config{}) + + flavorA := compute.FlavorInGroup{Name: "group-a-small", MemoryMB: 4096, VCPUs: flavorCPUs} + flavorB := compute.FlavorInGroup{Name: "group-b-small", MemoryMB: 4096, VCPUs: flavorCPUs} + flavorGroups := map[string]compute.FlavorGroupFeature{ + "group-a": {Name: "group-a", SmallestFlavor: flavorA, Flavors: []compute.FlavorInGroup{flavorA}}, + "group-b": {Name: "group-b", SmallestFlavor: flavorB, Flavors: []compute.FlavorInGroup{flavorB}}, + } + hvByName := map[string]hv1.Hypervisor{"host-1": *hv1a, "host-2": *hv2} + + reserved, err := c.reservedResourcesByGroupAZ(context.Background(), flavorGroups, hvByName) + if err != nil { + t.Fatalf("reservedResourcesByGroupAZ failed: %v", err) + } + + // group-a: committed on host-1 + failover on fully-packed host-2 = 2 slots. + a := reserved[vmUsageKey{group: "group-a", az: az}] + if a[ResourceMemory] != 2*flavorMem { + t.Errorf("group-a reserved memory = %d, want %d", a[ResourceMemory], 2*flavorMem) + } + if a[ResourceCores] != 2*flavorCPUs { + t.Errorf("group-a reserved cores = %d, want %d", a[ResourceCores], 2*flavorCPUs) + } + // group-b: single failover on the shared host-1 = 1 slot. + b := reserved[vmUsageKey{group: "group-b", az: az}] + if b[ResourceMemory] != flavorMem { + t.Errorf("group-b reserved memory = %d, want %d", b[ResourceMemory], flavorMem) + } + if b[ResourceCores] != flavorCPUs { + t.Errorf("group-b reserved cores = %d, want %d", b[ResourceCores], flavorCPUs) + } +} + func TestReconcileAZ_SkipsCRDWriteOnSchedulerError(t *testing.T) { const ( groupName = "hana-v2" @@ -361,7 +479,7 @@ func TestReconcileAZ_SkipsCRDWriteOnSchedulerError(t *testing.T) { ctrl.reconcileAZ(context.Background(), az, map[string]compute.FlavorGroupFeature{groupName: groupData}, - map[string]hv1.Hypervisor{}, map[string]map[string]int64{}, map[vmUsageKey]vmUsage{}) + map[string]hv1.Hypervisor{}, map[string]map[string]int64{}, nil, map[vmUsageKey]vmUsage{}) // Stale probes → CRD must NOT be written; last good state is preserved. var list v1alpha1.FlavorGroupCapacityList @@ -433,7 +551,7 @@ func TestReconcileAZ_MarksExistingCRDNotReadyOnSchedulerError(t *testing.T) { ctrl.reconcileAZ(context.Background(), az, map[string]compute.FlavorGroupFeature{groupName: groupData}, - hvByName, map[string]map[string]int64{}, map[vmUsageKey]vmUsage{}) + hvByName, map[string]map[string]int64{}, nil, map[vmUsageKey]vmUsage{}) var crd v1alpha1.FlavorGroupCapacity if err := fakeClient.Get(context.Background(), types.NamespacedName{Name: crdName}, &crd); err != nil { @@ -498,9 +616,9 @@ func TestReconcileAZ_IdempotentUpdate(t *testing.T) { groups := map[string]compute.FlavorGroupFeature{groupName: groupData} // First call - ctrl.reconcileAZ(context.Background(), az, groups, hvByName, map[string]map[string]int64{}, map[vmUsageKey]vmUsage{}) + ctrl.reconcileAZ(context.Background(), az, groups, hvByName, map[string]map[string]int64{}, nil, map[vmUsageKey]vmUsage{}) // Second call — should not error on the already-existing CRD. - ctrl.reconcileAZ(context.Background(), az, groups, hvByName, map[string]map[string]int64{}, map[vmUsageKey]vmUsage{}) + ctrl.reconcileAZ(context.Background(), az, groups, hvByName, map[string]map[string]int64{}, nil, map[vmUsageKey]vmUsage{}) var crd v1alpha1.FlavorGroupCapacity if err := fakeClient.Get(context.Background(), types.NamespacedName{Name: crdName}, &crd); err != nil { @@ -728,7 +846,7 @@ func TestReconcileAZ_ZeroMemoryFlavorSkipped(t *testing.T) { // reconcileAZ logs and skips groups with zero memory; it does not return an error. c.reconcileAZ(context.Background(), "az-a", map[string]compute.FlavorGroupFeature{"hana-v2": groupData}, - nil, nil, nil) + nil, nil, nil, nil) // No CRD should have been created. var list v1alpha1.FlavorGroupCapacityList @@ -1048,7 +1166,7 @@ func TestComputeVMUsage_ZerosOutWhenAllVMsRemoved(t *testing.T) { } // Now run reconcileAZ to verify the CRD gets zeroed out. - ctrl.reconcileAZ(context.Background(), az, groups, hvByName, map[string]map[string]int64{}, usageByKey) + ctrl.reconcileAZ(context.Background(), az, groups, hvByName, map[string]map[string]int64{}, nil, usageByKey) var crd v1alpha1.FlavorGroupCapacity if err := fakeClient.Get(context.Background(), types.NamespacedName{Name: crdName}, &crd); err != nil { diff --git a/internal/scheduling/reservations/commitments/capacity.go b/internal/scheduling/reservations/commitments/capacity.go index ed92ec0cc..ac44ffd72 100644 --- a/internal/scheduling/reservations/commitments/capacity.go +++ b/internal/scheduling/reservations/commitments/capacity.go @@ -96,9 +96,10 @@ func (c *CapacityCalculator) CalculateCapacity(ctx context.Context, req liquid.S // ExclusivelyFreeSlots is pre-computed by the controller using min(memSlots, cpuSlots). exclusiveFreeSlots := uint64(crd.Status.ExclusivelyFreeSlots) //nolint:gosec - // Reservation-blocked slots (failover + unfilled committed) on this group's assigned - // hosts. They are subtracted from both the placeable probe and the split, so they are - // neither running nor free. Add them back so reported capacity reflects installed hardware. + // Reservation-blocked slots (failover + unfilled committed) attributed to this group + // by each reservation's own ResourceGroup. They are subtracted from both the placeable + // probe and the split, so they are neither running nor free. Add them back so reported + // capacity reflects installed hardware. reservedSlots := uint64(crd.Status.ExclusivelyReservedSlots) //nolint:gosec reservedCores := int64(0) if qty, ok := crd.Status.ExclusivelyReservedCapacity[string(v1alpha1.CommittedResourceTypeCores)]; ok { From fa5ec3e083c321ea20f7eefce2820a27da5b98cd Mon Sep 17 00:00:00 2001 From: juliusclausnitzer Date: Mon, 28 Sep 2026 15:10:04 +0200 Subject: [PATCH 3/4] shorten comments Signed-off-by: juliusclausnitzer --- api/v1alpha1/flavor_group_capacity_types.go | 10 ++++------ .../cortex.cloud_flavorgroupcapacities.yaml | 10 ++++------ .../reservations/capacity/controller.go | 18 +++++++----------- .../reservations/capacity/controller_test.go | 9 ++++----- .../reservations/commitments/capacity.go | 6 ++---- 5 files changed, 21 insertions(+), 32 deletions(-) diff --git a/api/v1alpha1/flavor_group_capacity_types.go b/api/v1alpha1/flavor_group_capacity_types.go index b4403e836..77f30a5cf 100644 --- a/api/v1alpha1/flavor_group_capacity_types.go +++ b/api/v1alpha1/flavor_group_capacity_types.go @@ -99,12 +99,10 @@ type FlavorGroupCapacityStatus struct { // +kubebuilder:validation:Optional ExclusivelyFreeSlots int64 `json:"exclusivelyFreeSlots,omitempty"` - // ExclusivelyReservedCapacity is the reservation-blocked memory and CPU (failover slots and - // unfilled committed-resource slots) attributed to this group by each reservation's own - // ResourceGroup. These slots are subtracted from both the placeable probe and the split, so - // they are neither running nor free; they must be added back to report installed capacity. - // Because every reservation belongs to exactly one group, the sum across groups never - // exceeds installed capacity. + // ExclusivelyReservedCapacity is the reservation-blocked memory and CPU (failover + unfilled + // committed slots) attributed to this group by each reservation's own ResourceGroup. These + // slots are neither running nor free, so they are added back to report installed capacity; + // each reservation belongs to one group, so the sum across groups never exceeds capacity. // +kubebuilder:validation:Optional ExclusivelyReservedCapacity map[string]resource.Quantity `json:"exclusivelyReservedCapacity,omitempty"` diff --git a/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml b/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml index b5c89078d..ee3a6c8b2 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml @@ -201,12 +201,10 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: |- - ExclusivelyReservedCapacity is the reservation-blocked memory and CPU (failover slots and - unfilled committed-resource slots) attributed to this group by each reservation's own - ResourceGroup. These slots are subtracted from both the placeable probe and the split, so - they are neither running nor free; they must be added back to report installed capacity. - Because every reservation belongs to exactly one group, the sum across groups never - exceeds installed capacity. + ExclusivelyReservedCapacity is the reservation-blocked memory and CPU (failover + unfilled + committed slots) attributed to this group by each reservation's own ResourceGroup. These + slots are neither running nor free, so they are added back to report installed capacity; + each reservation belongs to one group, so the sum across groups never exceeds capacity. type: object exclusivelyReservedSlots: description: |- diff --git a/internal/scheduling/reservations/capacity/controller.go b/internal/scheduling/reservations/capacity/controller.go index 190e9a99f..aff1d6b22 100644 --- a/internal/scheduling/reservations/capacity/controller.go +++ b/internal/scheduling/reservations/capacity/controller.go @@ -871,13 +871,10 @@ func (c *Reconciler) blockedResourcesByHost(ctx context.Context) (map[string]map return blocked, nil } -// reservedResourcesByGroupAZ attributes reservation-blocked capacity (failover slots and -// unfilled committed-resource slots) to the flavor group that owns each reservation, keyed by -// (group, AZ). Attribution uses each reservation's own ResourceGroup rather than the host it -// sits on, so a host that is fully packed by empty reservations — and therefore assigned to no -// group by the round-robin split — still contributes its reserved capacity to the right group. -// Only placed reservations (with a target/status host that is a known hypervisor) are counted, -// each exactly once; pending reservations are picked up on a later reconcile once placed. +// reservedResourcesByGroupAZ sums reservation-blocked capacity (failover + unfilled committed +// slots) per (group, AZ), attributing each reservation by its own ResourceGroup rather than by +// host — so a host fully packed by empty reservations still counts. Only placed reservations are +// included, each once. func (c *Reconciler) reservedResourcesByGroupAZ( ctx context.Context, flavorGroups map[string]compute.FlavorGroupFeature, @@ -941,10 +938,9 @@ func (c *Reconciler) reservedResourcesByGroupAZ( return reserved, nil } -// resolveReservationGroup determines the flavor group a reservation belongs to. Committed -// reservations carry the group directly. Failover reservations carry either the group name or — -// when useFlavorGroupResources is disabled — the flavor name, which is resolved to its group. -// Returns false if no group can be determined. +// resolveReservationGroup returns the flavor group a reservation belongs to. Committed reservations +// carry the group directly; failover reservations carry the group name, or the flavor name (resolved +// to its group) when useFlavorGroupResources is disabled. Returns false if none can be determined. func resolveReservationGroup( res *v1alpha1.Reservation, flavorGroups map[string]compute.FlavorGroupFeature, diff --git a/internal/scheduling/reservations/capacity/controller_test.go b/internal/scheduling/reservations/capacity/controller_test.go index 6370ad593..b3711bc6d 100644 --- a/internal/scheduling/reservations/capacity/controller_test.go +++ b/internal/scheduling/reservations/capacity/controller_test.go @@ -326,11 +326,10 @@ func TestReconcileAZ_WritesReservedCapacity(t *testing.T) { } } -// TestReservedResourcesByGroupAZ verifies that reservation-blocked capacity is attributed to a -// flavor group by each reservation's own ResourceGroup — independent of the round-robin split — -// so a host fully packed by empty reservations still contributes, and a host shared by two groups -// splits its reservations correctly. It also covers failover ResourceGroup carrying a flavor name -// and the skipping of pending and in-flight reservations. +// TestReservedResourcesByGroupAZ verifies reservations are attributed by their own ResourceGroup +// (independent of the split): a host fully packed by empty reservations still counts, a shared host +// splits across two groups, failover ResourceGroup may hold a flavor name, and pending/in-flight +// reservations are skipped. func TestReservedResourcesByGroupAZ(t *testing.T) { const az = "qa-de-1a" const flavorMem = int64(4096) * 1024 * 1024 diff --git a/internal/scheduling/reservations/commitments/capacity.go b/internal/scheduling/reservations/commitments/capacity.go index ac44ffd72..d1f39e503 100644 --- a/internal/scheduling/reservations/commitments/capacity.go +++ b/internal/scheduling/reservations/commitments/capacity.go @@ -96,10 +96,8 @@ func (c *CapacityCalculator) CalculateCapacity(ctx context.Context, req liquid.S // ExclusivelyFreeSlots is pre-computed by the controller using min(memSlots, cpuSlots). exclusiveFreeSlots := uint64(crd.Status.ExclusivelyFreeSlots) //nolint:gosec - // Reservation-blocked slots (failover + unfilled committed) attributed to this group - // by each reservation's own ResourceGroup. They are subtracted from both the placeable - // probe and the split, so they are neither running nor free. Add them back so reported - // capacity reflects installed hardware. + // Reservation-blocked slots (failover + unfilled committed), neither running nor free. + // Add them back so reported capacity reflects installed hardware. reservedSlots := uint64(crd.Status.ExclusivelyReservedSlots) //nolint:gosec reservedCores := int64(0) if qty, ok := crd.Status.ExclusivelyReservedCapacity[string(v1alpha1.CommittedResourceTypeCores)]; ok { From b0329406a224f520034f41e9da4fe0e1b08d3a6e Mon Sep 17 00:00:00 2001 From: juliusclausnitzer Date: Mon, 28 Sep 2026 18:34:16 +0200 Subject: [PATCH 4/4] Apply suggestion from @mblos Co-authored-by: Marcel <156897072+mblos@users.noreply.github.com> Signed-off-by: juliusclausnitzer --- internal/scheduling/reservations/commitments/capacity.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/scheduling/reservations/commitments/capacity.go b/internal/scheduling/reservations/commitments/capacity.go index d1f39e503..91987d888 100644 --- a/internal/scheduling/reservations/commitments/capacity.go +++ b/internal/scheduling/reservations/commitments/capacity.go @@ -167,7 +167,7 @@ func (c *CapacityCalculator) CalculateCapacity(ctx context.Context, req liquid.S if qty, ok := crd.Status.RunningResources[string(v1alpha1.CommittedResourceTypeMemory)]; ok { runningMemBytes = qty.Value() } - ramEntry.Usage = Some[uint64](uint64(runningMemBytes) / uint64(ramUnitBytes)) + ramEntry.Usage = Some[uint64](uint64(runningMemBytes+reservedMemBytes) / uint64(ramUnitBytes)) } }