diff --git a/api/v1alpha1/flavor_group_capacity_types.go b/api/v1alpha1/flavor_group_capacity_types.go index 3da5090af..77f30a5cf 100644 --- a/api/v1alpha1/flavor_group_capacity_types.go +++ b/api/v1alpha1/flavor_group_capacity_types.go @@ -99,6 +99,18 @@ type FlavorGroupCapacityStatus struct { // +kubebuilder:validation:Optional ExclusivelyFreeSlots int64 `json:"exclusivelyFreeSlots,omitempty"` + // ExclusivelyReservedCapacity is the reservation-blocked memory and CPU (failover + unfilled + // committed slots) attributed to this group by each reservation's own ResourceGroup. These + // slots are neither running nor free, so they are added back to report installed capacity; + // each reservation belongs to one group, so the sum across groups never exceeds capacity. + // +kubebuilder:validation:Optional + ExclusivelyReservedCapacity map[string]resource.Quantity `json:"exclusivelyReservedCapacity,omitempty"` + + // ExclusivelyReservedSlots is the number of smallest-flavor VM slots represented by + // ExclusivelyReservedCapacity. + // +kubebuilder:validation:Optional + ExclusivelyReservedSlots int64 `json:"exclusivelyReservedSlots,omitempty"` + // RunningInstances is the number of VMs running in this (flavor group × AZ) whose // flavor belongs to this group. // +kubebuilder:validation:Optional diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index dadbbb074..336538f12 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -882,6 +882,13 @@ func (in *FlavorGroupCapacityStatus) DeepCopyInto(out *FlavorGroupCapacityStatus (*out)[key] = val.DeepCopy() } } + if in.ExclusivelyReservedCapacity != nil { + in, out := &in.ExclusivelyReservedCapacity, &out.ExclusivelyReservedCapacity + *out = make(map[string]resource.Quantity, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } if in.RunningResources != nil { in, out := &in.RunningResources, &out.RunningResources *out = make(map[string]resource.Quantity, len(*in)) diff --git a/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml b/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml index f5203abc2..ee3a6c8b2 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml @@ -193,6 +193,25 @@ spec: ExclusivelyFreeCapacity, this is not constrained by slot packing or CPU/memory ratio. Safe to sum across groups — no host is counted twice. type: object + exclusivelyReservedCapacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + ExclusivelyReservedCapacity is the reservation-blocked memory and CPU (failover + unfilled + committed slots) attributed to this group by each reservation's own ResourceGroup. These + slots are neither running nor free, so they are added back to report installed capacity; + each reservation belongs to one group, so the sum across groups never exceeds capacity. + type: object + exclusivelyReservedSlots: + description: |- + ExclusivelyReservedSlots is the number of smallest-flavor VM slots represented by + ExclusivelyReservedCapacity. + format: int64 + type: integer flavors: description: Flavors holds per-flavor capacity data for all flavors in the group. diff --git a/internal/scheduling/reservations/capacity/controller.go b/internal/scheduling/reservations/capacity/controller.go index 80a702edc..aff1d6b22 100644 --- a/internal/scheduling/reservations/capacity/controller.go +++ b/internal/scheduling/reservations/capacity/controller.go @@ -244,10 +244,16 @@ func (c *Reconciler) reconcileAll(ctx context.Context) error { blockedByReservations = map[string]map[string]int64{} } + reservedByGroupAZ, err := c.reservedResourcesByGroupAZ(ctx, flavorGroups, hvByName) + if err != nil { + logger.Error(err, "failed to attribute reserved capacity to flavor groups, reserved capacity will be omitted") + reservedByGroupAZ = map[vmUsageKey]map[string]int64{} + } + usageByKey := c.computeVMUsage(ctx, flavorGroups, hvList.Items) for _, az := range azs { - c.reconcileAZ(ctx, az, flavorGroups, hvByName, blockedByReservations, usageByKey) + c.reconcileAZ(ctx, az, flavorGroups, hvByName, blockedByReservations, reservedByGroupAZ, usageByKey) } logger.Info("capacity reconcile cycle completed", @@ -505,6 +511,7 @@ func (c *Reconciler) reconcileAZ( flavorGroups map[string]compute.FlavorGroupFeature, hvByName map[string]hv1.Hypervisor, blockedByReservations map[string]map[string]int64, + reservedByGroupAZ map[vmUsageKey]map[string]int64, usageByKey map[vmUsageKey]vmUsage, ) { @@ -592,6 +599,7 @@ func (c *Reconciler) reconcileAZ( freeResources[r.groupName], exclusiveResources[r.groupName], rawExclusiveByGroup[r.groupName], + reservedByGroupAZ[vmUsageKey{r.groupName, az}], ); err != nil { logger.Error(err, "failed to write FlavorGroupCapacity CRD", "flavorGroup", r.groupName, "az", az) @@ -631,6 +639,7 @@ func (c *Reconciler) writeCRD( freeRes map[string]int64, exclusiveRes map[string]int64, exclusivelyRawRes map[string]int64, + reservedRes map[string]int64, ) error { crdName := crdNameFor(groupName, az) @@ -676,11 +685,13 @@ func (c *Reconciler) writeCRD( existing.Status.FreeCapacity = resMapToQuantity(freeRes) existing.Status.ExclusivelyFreeCapacity = resMapToQuantity(exclusiveRes) existing.Status.ExclusivelyRawCapacity = resMapToQuantity(exclusivelyRawRes) + existing.Status.ExclusivelyReservedCapacity = resMapToQuantity(reservedRes) var exclusivelyFreeSlots int64 if flavorMemBytes := int64(groupData.SmallestFlavor.MemoryMB) * 1024 * 1024; flavorMemBytes > 0 { //nolint:gosec flavorVCPUs := int64(groupData.SmallestFlavor.VCPUs) //nolint:gosec exclusivelyFreeSlots = flavorSlots(exclusiveRes, flavorMemBytes, flavorVCPUs) existing.Status.ExclusivelyFreeSlots = exclusivelyFreeSlots + existing.Status.ExclusivelyReservedSlots = flavorSlots(reservedRes, flavorMemBytes, flavorVCPUs) } existing.Status.LastReconcileAt = metav1.Now() @@ -860,6 +871,110 @@ func (c *Reconciler) blockedResourcesByHost(ctx context.Context) (map[string]map return blocked, nil } +// reservedResourcesByGroupAZ sums reservation-blocked capacity (failover + unfilled committed +// slots) per (group, AZ), attributing each reservation by its own ResourceGroup rather than by +// host — so a host fully packed by empty reservations still counts. Only placed reservations are +// included, each once. +func (c *Reconciler) reservedResourcesByGroupAZ( + ctx context.Context, + flavorGroups map[string]compute.FlavorGroupFeature, + hvByName map[string]hv1.Hypervisor, +) (map[vmUsageKey]map[string]int64, error) { + + logger := LoggerFromContext(ctx) + var list v1alpha1.ReservationList + if err := c.client.List(ctx, &list); err != nil { + return nil, fmt.Errorf("failed to list reservations: %w", err) + } + + flavorToGroup := make(map[string]string) + for groupName, gd := range flavorGroups { + for _, f := range gd.Flavors { + flavorToGroup[f.Name] = groupName + } + } + + reserved := make(map[vmUsageKey]map[string]int64) + for i := range list.Items { + res := &list.Items[i] + // Only committed and failover reservations represent capacity that is reserved but + // empty; in-flight reservations are VMs in buildup and are counted as running instead. + if res.Spec.Type != v1alpha1.ReservationTypeCommittedResource && res.Spec.Type != v1alpha1.ReservationTypeFailover { + continue + } + host := res.Status.Host + if host == "" { + host = res.Spec.TargetHost + } + if host == "" { + continue // pending; reported on a later cycle once placed + } + hv, ok := hvByName[host] + if !ok { + continue // host is not a known hypervisor in this cluster + } + az := hv.Labels["topology.kubernetes.io/zone"] + if az == "" { + continue + } + group, ok := resolveReservationGroup(res, flavorGroups, flavorToGroup) + if !ok { + logger.Info("reservation could not be attributed to a flavor group; its reserved capacity is not reported", + "reservation", res.Name, "type", res.Spec.Type, "az", az) + continue + } + blockedRes := reservations.UnusedReservationCapacity(res, false) + key := vmUsageKey{group: group, az: az} + if reserved[key] == nil { + reserved[key] = make(map[string]int64) + } + if qty, ok := blockedRes[hv1.ResourceMemory]; ok { + reserved[key][ResourceMemory] += qty.Value() + } + if qty, ok := blockedRes[hv1.ResourceCPU]; ok { + reserved[key][ResourceCores] += qty.Value() + } + } + return reserved, nil +} + +// resolveReservationGroup returns the flavor group a reservation belongs to. Committed reservations +// carry the group directly; failover reservations carry the group name, or the flavor name (resolved +// to its group) when useFlavorGroupResources is disabled. Returns false if none can be determined. +func resolveReservationGroup( + res *v1alpha1.Reservation, + flavorGroups map[string]compute.FlavorGroupFeature, + flavorToGroup map[string]string, +) (string, bool) { + + switch res.Spec.Type { + case v1alpha1.ReservationTypeCommittedResource: + if res.Spec.CommittedResourceReservation == nil { + return "", false + } + if g := res.Spec.CommittedResourceReservation.ResourceGroup; g != "" { + if _, ok := flavorGroups[g]; ok { + return g, true + } + } + return "", false + case v1alpha1.ReservationTypeFailover: + if res.Spec.FailoverReservation == nil { + return "", false + } + g := res.Spec.FailoverReservation.ResourceGroup + if _, ok := flavorGroups[g]; ok { + return g, true + } + if group, ok := flavorToGroup[g]; ok { + return group, true + } + return "", false + default: + return "", false + } +} + // sumCommittedCapacity sums active CommittedResource amounts (memory type, guaranteed/confirmed) // for the given (flavorGroup, az) pair. Returns the total in smallest-flavor slots. func (c *Reconciler) sumCommittedCapacity(ctx context.Context, groupName, az string, smallestFlavorBytes int64) (int64, error) { diff --git a/internal/scheduling/reservations/capacity/controller_test.go b/internal/scheduling/reservations/capacity/controller_test.go index 8a80967b5..b3711bc6d 100644 --- a/internal/scheduling/reservations/capacity/controller_test.go +++ b/internal/scheduling/reservations/capacity/controller_test.go @@ -225,7 +225,7 @@ func TestReconcileAZ_CreatesCRD(t *testing.T) { ctrl.reconcileAZ(context.Background(), az, map[string]compute.FlavorGroupFeature{groupName: groupData}, - hvByName, map[string]map[string]int64{}, map[vmUsageKey]vmUsage{}) + hvByName, map[string]map[string]int64{}, nil, map[vmUsageKey]vmUsage{}) var crd v1alpha1.FlavorGroupCapacity if err := fakeClient.Get(context.Background(), types.NamespacedName{Name: crdNameFor(groupName, az)}, &crd); err != nil { @@ -266,6 +266,182 @@ func TestReconcileAZ_CreatesCRD(t *testing.T) { } } +func TestReconcileAZ_WritesReservedCapacity(t *testing.T) { + const ( + groupName = "hana-v2" + az = "qa-de-1a" + memMB = 4096 // flavor memory (1 slot) + flavorMem = int64(memMB) * 1024 * 1024 + flavorCPUs = 2 + ) + + scheme := newTestScheme(t) + // Host holds 2 flavor slots worth of memory; 1 slot is reserved, 1 remains free. + hv := newHypervisor("host-1", az, 2*flavorMem) + knowledge := newFlavorGroupKnowledge(t, groupName, memMB) + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(knowledge, hv). + WithStatusSubresource(&v1alpha1.FlavorGroupCapacity{}, &v1alpha1.Knowledge{}). + Build() + + schedulerServer := newMockSchedulerServer(t, []string{"host-1"}) + defer schedulerServer.Close() + + ctrl := newController(t, fakeClient, Config{ + SchedulerURL: schedulerServer.URL, + TotalPipeline: "kvm-report-capacity", + PlaceablePipeline: "kvm-general-purpose", + }) + + smallFlavor := compute.FlavorInGroup{Name: groupName + "-small", MemoryMB: memMB, VCPUs: flavorCPUs} + groupData := compute.FlavorGroupFeature{SmallestFlavor: smallFlavor, Flavors: []compute.FlavorInGroup{smallFlavor}} + hvByName := map[string]hv1.Hypervisor{"host-1": *hv} + + // One flavor slot is reserved for this (group, AZ) — attributed upstream by + // reservedResourcesByGroupAZ and passed in here; reconcileAZ must persist it to the CRD. + reservedByGroupAZ := map[vmUsageKey]map[string]int64{ + {group: groupName, az: az}: {ResourceMemory: flavorMem, ResourceCores: flavorCPUs}, + } + + ctrl.reconcileAZ(context.Background(), az, + map[string]compute.FlavorGroupFeature{groupName: groupData}, + hvByName, map[string]map[string]int64{}, reservedByGroupAZ, map[vmUsageKey]vmUsage{}) + + var crd v1alpha1.FlavorGroupCapacity + if err := fakeClient.Get(context.Background(), types.NamespacedName{Name: crdNameFor(groupName, az)}, &crd); err != nil { + t.Fatalf("failed to get CRD: %v", err) + } + reservedMem := crd.Status.ExclusivelyReservedCapacity[string(v1alpha1.CommittedResourceTypeMemory)] + if reservedMem.Value() != flavorMem { + t.Errorf("ExclusivelyReservedCapacity[memory] = %d, want %d", reservedMem.Value(), flavorMem) + } + reservedCores := crd.Status.ExclusivelyReservedCapacity[string(v1alpha1.CommittedResourceTypeCores)] + if reservedCores.Value() != flavorCPUs { + t.Errorf("ExclusivelyReservedCapacity[cores] = %d, want %d", reservedCores.Value(), flavorCPUs) + } + if crd.Status.ExclusivelyReservedSlots != 1 { + t.Errorf("ExclusivelyReservedSlots = %d, want 1", crd.Status.ExclusivelyReservedSlots) + } +} + +// TestReservedResourcesByGroupAZ verifies reservations are attributed by their own ResourceGroup +// (independent of the split): a host fully packed by empty reservations still counts, a shared host +// splits across two groups, failover ResourceGroup may hold a flavor name, and pending/in-flight +// reservations are skipped. +func TestReservedResourcesByGroupAZ(t *testing.T) { + const az = "qa-de-1a" + const flavorMem = int64(4096) * 1024 * 1024 + const flavorCPUs = 2 + + scheme := newTestScheme(t) + // host-1 is shared by group-a and group-b reservations; host-2 is fully packed by a single + // failover reservation (zero free slots — it would drop out of the split's exclusiveHosts). + hv1a := newHypervisor("host-1", az, 2*flavorMem) + hv2 := newHypervisor("host-2", az, flavorMem) + + resources := map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: *resource.NewQuantity(flavorMem, resource.BinarySI), + hv1.ResourceCPU: *resource.NewQuantity(flavorCPUs, resource.DecimalSI), + } + + // committed reservation on host-1 → group-a (carries the group directly). + committedA := &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: "committed-a"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + AvailabilityZone: az, + TargetHost: "host-1", + Resources: resources, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ResourceGroup: "group-a"}, + }, + Status: v1alpha1.ReservationStatus{Host: "host-1"}, + } + // failover reservation on the same host-1 → group-b (carries the group name). + failoverB := &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: "failover-b"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeFailover, + AvailabilityZone: az, + TargetHost: "host-1", + Resources: resources, + FailoverReservation: &v1alpha1.FailoverReservationSpec{ResourceGroup: "group-b"}, + }, + Status: v1alpha1.ReservationStatus{Host: "host-1"}, + } + // failover reservation on the fully-packed host-2 → group-a, resolved from the flavor NAME. + failoverAByFlavor := &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: "failover-a-flavor"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeFailover, + AvailabilityZone: az, + TargetHost: "host-2", + Resources: resources, + FailoverReservation: &v1alpha1.FailoverReservationSpec{ResourceGroup: "group-a-small"}, + }, + Status: v1alpha1.ReservationStatus{Host: "host-2"}, + } + // pending reservation (no placed host) → skipped, reported on a later cycle. + pending := &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: "pending"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + AvailabilityZone: az, + Resources: resources, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ResourceGroup: "group-a"}, + }, + } + // in-flight reservation → skipped (counted as running, not reserved). + inflight := &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: "inflight"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeInFlight, + AvailabilityZone: az, + TargetHost: "host-1", + Resources: resources, + }, + Status: v1alpha1.ReservationStatus{Host: "host-1"}, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(committedA, failoverB, failoverAByFlavor, pending, inflight). + Build() + + c := newController(t, fakeClient, Config{}) + + flavorA := compute.FlavorInGroup{Name: "group-a-small", MemoryMB: 4096, VCPUs: flavorCPUs} + flavorB := compute.FlavorInGroup{Name: "group-b-small", MemoryMB: 4096, VCPUs: flavorCPUs} + flavorGroups := map[string]compute.FlavorGroupFeature{ + "group-a": {Name: "group-a", SmallestFlavor: flavorA, Flavors: []compute.FlavorInGroup{flavorA}}, + "group-b": {Name: "group-b", SmallestFlavor: flavorB, Flavors: []compute.FlavorInGroup{flavorB}}, + } + hvByName := map[string]hv1.Hypervisor{"host-1": *hv1a, "host-2": *hv2} + + reserved, err := c.reservedResourcesByGroupAZ(context.Background(), flavorGroups, hvByName) + if err != nil { + t.Fatalf("reservedResourcesByGroupAZ failed: %v", err) + } + + // group-a: committed on host-1 + failover on fully-packed host-2 = 2 slots. + a := reserved[vmUsageKey{group: "group-a", az: az}] + if a[ResourceMemory] != 2*flavorMem { + t.Errorf("group-a reserved memory = %d, want %d", a[ResourceMemory], 2*flavorMem) + } + if a[ResourceCores] != 2*flavorCPUs { + t.Errorf("group-a reserved cores = %d, want %d", a[ResourceCores], 2*flavorCPUs) + } + // group-b: single failover on the shared host-1 = 1 slot. + b := reserved[vmUsageKey{group: "group-b", az: az}] + if b[ResourceMemory] != flavorMem { + t.Errorf("group-b reserved memory = %d, want %d", b[ResourceMemory], flavorMem) + } + if b[ResourceCores] != flavorCPUs { + t.Errorf("group-b reserved cores = %d, want %d", b[ResourceCores], flavorCPUs) + } +} + func TestReconcileAZ_SkipsCRDWriteOnSchedulerError(t *testing.T) { const ( groupName = "hana-v2" @@ -302,7 +478,7 @@ func TestReconcileAZ_SkipsCRDWriteOnSchedulerError(t *testing.T) { ctrl.reconcileAZ(context.Background(), az, map[string]compute.FlavorGroupFeature{groupName: groupData}, - map[string]hv1.Hypervisor{}, map[string]map[string]int64{}, map[vmUsageKey]vmUsage{}) + map[string]hv1.Hypervisor{}, map[string]map[string]int64{}, nil, map[vmUsageKey]vmUsage{}) // Stale probes → CRD must NOT be written; last good state is preserved. var list v1alpha1.FlavorGroupCapacityList @@ -374,7 +550,7 @@ func TestReconcileAZ_MarksExistingCRDNotReadyOnSchedulerError(t *testing.T) { ctrl.reconcileAZ(context.Background(), az, map[string]compute.FlavorGroupFeature{groupName: groupData}, - hvByName, map[string]map[string]int64{}, map[vmUsageKey]vmUsage{}) + hvByName, map[string]map[string]int64{}, nil, map[vmUsageKey]vmUsage{}) var crd v1alpha1.FlavorGroupCapacity if err := fakeClient.Get(context.Background(), types.NamespacedName{Name: crdName}, &crd); err != nil { @@ -439,9 +615,9 @@ func TestReconcileAZ_IdempotentUpdate(t *testing.T) { groups := map[string]compute.FlavorGroupFeature{groupName: groupData} // First call - ctrl.reconcileAZ(context.Background(), az, groups, hvByName, map[string]map[string]int64{}, map[vmUsageKey]vmUsage{}) + ctrl.reconcileAZ(context.Background(), az, groups, hvByName, map[string]map[string]int64{}, nil, map[vmUsageKey]vmUsage{}) // Second call — should not error on the already-existing CRD. - ctrl.reconcileAZ(context.Background(), az, groups, hvByName, map[string]map[string]int64{}, map[vmUsageKey]vmUsage{}) + ctrl.reconcileAZ(context.Background(), az, groups, hvByName, map[string]map[string]int64{}, nil, map[vmUsageKey]vmUsage{}) var crd v1alpha1.FlavorGroupCapacity if err := fakeClient.Get(context.Background(), types.NamespacedName{Name: crdName}, &crd); err != nil { @@ -669,7 +845,7 @@ func TestReconcileAZ_ZeroMemoryFlavorSkipped(t *testing.T) { // reconcileAZ logs and skips groups with zero memory; it does not return an error. c.reconcileAZ(context.Background(), "az-a", map[string]compute.FlavorGroupFeature{"hana-v2": groupData}, - nil, nil, nil) + nil, nil, nil, nil) // No CRD should have been created. var list v1alpha1.FlavorGroupCapacityList @@ -989,7 +1165,7 @@ func TestComputeVMUsage_ZerosOutWhenAllVMsRemoved(t *testing.T) { } // Now run reconcileAZ to verify the CRD gets zeroed out. - ctrl.reconcileAZ(context.Background(), az, groups, hvByName, map[string]map[string]int64{}, usageByKey) + ctrl.reconcileAZ(context.Background(), az, groups, hvByName, map[string]map[string]int64{}, nil, usageByKey) var crd v1alpha1.FlavorGroupCapacity if err := fakeClient.Get(context.Background(), types.NamespacedName{Name: crdName}, &crd); err != nil { diff --git a/internal/scheduling/reservations/commitments/api/report_capacity_test.go b/internal/scheduling/reservations/commitments/api/report_capacity_test.go index c67b735ac..55fb2101c 100644 --- a/internal/scheduling/reservations/commitments/api/report_capacity_test.go +++ b/internal/scheduling/reservations/commitments/api/report_capacity_test.go @@ -421,6 +421,96 @@ func TestCapacityCalculator_VariableRatio_PrefersRawCapacity(t *testing.T) { } } +// TestCapacityCalculator_ReservedSlots verifies that reserved-but-empty slots (failover + +// unfilled committed reservations, stored in ExclusivelyReservedCapacity/Slots) are added back +// into reported capacity for a fixed-ratio group, and counted as usage so availability stays +// equal to the truly-free amount. +func TestCapacityCalculator_ReservedSlots(t *testing.T) { + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + const flavorMemBytes = 32752 * 1024 * 1024 + + // running=200, exclusively free=800 slots, reserved=100 slots. + // capacity = 200 + 800 + 100 = 1100; usage = 200 + 100 = 300; availability = 800. + crd := createTestFlavorGroupCapacity(200, 800*flavorMemBytes, true) + crd.Status.ExclusivelyReservedSlots = 100 + crd.Status.ExclusivelyReservedCapacity = map[string]resource.Quantity{ + string(v1alpha1.CommittedResourceTypeMemory): *resource.NewQuantity(100*flavorMemBytes, resource.BinarySI), + string(v1alpha1.CommittedResourceTypeCores): *resource.NewQuantity(100*8, resource.DecimalSI), + } + calc := commitments.NewCapacityCalculator( + fake.NewClientBuilder().WithScheme(scheme). + WithObjects(createTestFlavorGroupKnowledge(t), crd).WithStatusSubresource(crd).Build(), + defaultCapacityConfig, + ) + report, err := calc.CalculateCapacity(context.Background(), + liquid.ServiceCapacityRequest{AllAZs: []liquid.AvailabilityZone{"az-one"}}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + for _, res := range []string{"hw_version_test-group_ram", "hw_version_test-group_instances"} { + az := report.Resources[liquid.ResourceName(res)].PerAZ["az-one"] + if az.Capacity != 1100 { + t.Errorf("%s: capacity = %d, want 1100 (running 200 + free 800 + reserved 100)", res, az.Capacity) + } + if usage := az.Usage.UnwrapOr(0); usage != 300 { + t.Errorf("%s: usage = %d, want 300 (running 200 + reserved 100)", res, usage) + } + if avail := az.Capacity - az.Usage.UnwrapOr(0); avail != 800 { + t.Errorf("%s: availability = %d, want 800 (free slots)", res, avail) + } + } + cores := report.Resources["hw_version_test-group_cores"].PerAZ["az-one"] + if cores.Capacity != 800 { + t.Errorf("cores capacity = %d, want 800 (reserved cores added back)", cores.Capacity) + } + if usage := cores.Usage.UnwrapOr(0); usage != 800 { + t.Errorf("cores usage = %d, want 800 (reserved cores)", usage) + } +} + +// TestCapacityCalculator_VariableRatio_RawIgnoresReserved verifies that for a variable-ratio +// group the RAM raw path already includes reserved hardware, so ExclusivelyReservedCapacity is +// NOT added again (no double count). +func TestCapacityCalculator_VariableRatio_RawIgnoresReserved(t *testing.T) { + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + const ( + ramUnitGiB = 2 + flavorMemMiB = 2048 + flavorMemBytes = int64(flavorMemMiB) * 1024 * 1024 + rawMemBytes = 20 * 1024 * 1024 * 1024 // 20 GiB → 10 declared units + ) + knowledge := createVariableRatioFlavorGroupKnowledge(t, flavorMemMiB) + crd := createFlavorGroupCapacityWithResources(3, 5*flavorMemBytes, 3*flavorMemBytes, 3*8, rawMemBytes) + // Reserved is set but must be ignored on the raw RAM path. + crd.Status.ExclusivelyReservedCapacity = map[string]resource.Quantity{ + string(v1alpha1.CommittedResourceTypeMemory): *resource.NewQuantity(4*1024*1024*1024, resource.BinarySI), + } + crd.Status.ExclusivelyReservedSlots = 2 + cfg := commitments.APIConfig{ + FlavorGroupResourceConfig: map[string]commitments.FlavorGroupResourcesConfig{ + "*": {RAM: commitments.RAMResourceTypeConfig{HasCapacity: true, RAMUnitGiB: ramUnitGiB}}, + }, + } + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme).WithObjects(knowledge, crd).WithStatusSubresource(crd).Build() + report, err := commitments.NewCapacityCalculator(fakeClient, cfg).CalculateCapacity( + context.Background(), liquid.ServiceCapacityRequest{AllAZs: []liquid.AvailabilityZone{"az-one"}}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + az := report.Resources["hw_version_test-group_ram"].PerAZ["az-one"] + if az.Capacity != 10 { + t.Errorf("RAM capacity = %d, want 10 (raw hardware only, reserved not double counted)", az.Capacity) + } +} + func verifyPerAZMatchesRequest(t *testing.T, res *liquid.ResourceCapacityReport, requestedAZs []liquid.AvailabilityZone) { t.Helper() if res == nil { diff --git a/internal/scheduling/reservations/commitments/capacity.go b/internal/scheduling/reservations/commitments/capacity.go index 4cb62fdbe..91987d888 100644 --- a/internal/scheduling/reservations/commitments/capacity.go +++ b/internal/scheduling/reservations/commitments/capacity.go @@ -35,8 +35,9 @@ func NewCapacityCalculator(client client.Client, conf APIConfig) *CapacityCalcul // CalculateCapacity computes per-AZ capacity for all flavor groups. // For each flavor group, three resources are reported: _ram, _cores, _instances. // All values are read from FlavorGroupCapacity CRDs pre-computed by the capacity controller: -// - Capacity: RunningInstances + ExclusivelyFreeCapacity converted to slots. -// - Usage: RunningInstances / RunningResources. +// - Capacity: RunningInstances + ExclusivelyFreeCapacity + ExclusivelyReservedCapacity, in slots. +// - Usage: RunningInstances / RunningResources + ExclusivelyReservedCapacity (so availability +// stays equal to the truly-free amount and reserved capacity is not advertised as available). func (c *CapacityCalculator) CalculateCapacity(ctx context.Context, req liquid.ServiceCapacityRequest) (liquid.ServiceCapacityReport, error) { knowledge := &reservations.FlavorGroupKnowledgeClient{Client: c.client} flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, nil) @@ -95,9 +96,21 @@ func (c *CapacityCalculator) CalculateCapacity(ctx context.Context, req liquid.S // ExclusivelyFreeSlots is pre-computed by the controller using min(memSlots, cpuSlots). exclusiveFreeSlots := uint64(crd.Status.ExclusivelyFreeSlots) //nolint:gosec - // Capacity = running + exclusively free, all derived from CRD bytes. + // Reservation-blocked slots (failover + unfilled committed), neither running nor free. + // Add them back so reported capacity reflects installed hardware. + reservedSlots := uint64(crd.Status.ExclusivelyReservedSlots) //nolint:gosec + reservedCores := int64(0) + if qty, ok := crd.Status.ExclusivelyReservedCapacity[string(v1alpha1.CommittedResourceTypeCores)]; ok { + reservedCores = qty.Value() + } + reservedMemBytes := int64(0) + if qty, ok := crd.Status.ExclusivelyReservedCapacity[string(v1alpha1.CommittedResourceTypeMemory)]; ok { + reservedMemBytes = qty.Value() + } + + // Capacity = running + exclusively free + reserved, all derived from CRD bytes. runningInstances := uint64(crd.Status.RunningInstances) //nolint:gosec - instancesCapacity := runningInstances + exclusiveFreeSlots + instancesCapacity := runningInstances + exclusiveFreeSlots + reservedSlots // RAM capacity: running bytes + exclusively free bytes → declared units. // Fixed-ratio groups report in slots (1 unit = 1 instance). @@ -120,7 +133,7 @@ func (c *CapacityCalculator) CalculateCapacity(ctx context.Context, req liquid.S if qty, ok := crd.Status.ExclusivelyRawCapacity[string(v1alpha1.CommittedResourceTypeMemory)]; ok && qty.Value() > 0 { ramCapacity = uint64(qty.Value()) / uint64(ramUnitBytes) //nolint:gosec } else { - ramCapacity = uint64(runningMemBytes+freeMemBytes) / uint64(ramUnitBytes) + ramCapacity = uint64(runningMemBytes+freeMemBytes+reservedMemBytes) / uint64(ramUnitBytes) } } @@ -134,25 +147,27 @@ func (c *CapacityCalculator) CalculateCapacity(ctx context.Context, req liquid.S if qty, ok := crd.Status.ExclusivelyFreeCapacity[string(v1alpha1.CommittedResourceTypeCores)]; ok { freeCoresCount = qty.Value() } - coresCapacity = uint64(runningCoresCount + freeCoresCount) + coresCapacity = uint64(runningCoresCount + freeCoresCount + reservedCores) ramEntry := &liquid.AZResourceCapacityReport{Capacity: ramCapacity} coresEntry := &liquid.AZResourceCapacityReport{Capacity: coresCapacity} instancesEntry := &liquid.AZResourceCapacityReport{Capacity: instancesCapacity} - // Usage from actual running VMs — only when CRD data is fresh. + // Usage from actual running VMs plus reserved-but-empty slots — only when CRD data is + // fresh. Reserved slots are counted as usage so that capacity − usage stays equal to + // the truly-free amount and reserved capacity is never advertised as available. if apimeta.IsStatusConditionTrue(crd.Status.Conditions, v1alpha1.FlavorGroupCapacityConditionReady) { - instancesEntry.Usage = Some[uint64](runningInstances) - coresEntry.Usage = Some[uint64](uint64(runningCoresCount)) + instancesEntry.Usage = Some[uint64](runningInstances + reservedSlots) + coresEntry.Usage = Some[uint64](uint64(runningCoresCount + reservedCores)) if groupData.HasFixedRamCoreRatio() { - ramEntry.Usage = Some[uint64](runningInstances) + ramEntry.Usage = Some[uint64](runningInstances + reservedSlots) } else if ramUnitBytes > 0 { runningMemBytes := int64(0) if qty, ok := crd.Status.RunningResources[string(v1alpha1.CommittedResourceTypeMemory)]; ok { runningMemBytes = qty.Value() } - ramEntry.Usage = Some[uint64](uint64(runningMemBytes) / uint64(ramUnitBytes)) + ramEntry.Usage = Some[uint64](uint64(runningMemBytes+reservedMemBytes) / uint64(ramUnitBytes)) } }