Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions api/v1alpha1/flavor_group_capacity_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,18 @@ type FlavorGroupCapacityStatus struct {
// +kubebuilder:validation:Optional
ExclusivelyFreeSlots int64 `json:"exclusivelyFreeSlots,omitempty"`

// ExclusivelyReservedCapacity is the reservation-blocked memory and CPU (failover + unfilled
// committed slots) attributed to this group by each reservation's own ResourceGroup. These
// slots are neither running nor free, so they are added back to report installed capacity;
// each reservation belongs to one group, so the sum across groups never exceeds capacity.
// +kubebuilder:validation:Optional
ExclusivelyReservedCapacity map[string]resource.Quantity `json:"exclusivelyReservedCapacity,omitempty"`

// ExclusivelyReservedSlots is the number of smallest-flavor VM slots represented by
// ExclusivelyReservedCapacity.
// +kubebuilder:validation:Optional
ExclusivelyReservedSlots int64 `json:"exclusivelyReservedSlots,omitempty"`

// RunningInstances is the number of VMs running in this (flavor group × AZ) whose
// flavor belongs to this group.
// +kubebuilder:validation:Optional
Expand Down
7 changes: 7 additions & 0 deletions api/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,25 @@ spec:
ExclusivelyFreeCapacity, this is not constrained by slot packing or CPU/memory ratio.
Safe to sum across groups — no host is counted twice.
type: object
exclusivelyReservedCapacity:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: |-
ExclusivelyReservedCapacity is the reservation-blocked memory and CPU (failover + unfilled
committed slots) attributed to this group by each reservation's own ResourceGroup. These
slots are neither running nor free, so they are added back to report installed capacity;
each reservation belongs to one group, so the sum across groups never exceeds capacity.
type: object
exclusivelyReservedSlots:
description: |-
ExclusivelyReservedSlots is the number of smallest-flavor VM slots represented by
ExclusivelyReservedCapacity.
format: int64
type: integer
flavors:
description: Flavors holds per-flavor capacity data for all flavors
in the group.
Expand Down
117 changes: 116 additions & 1 deletion internal/scheduling/reservations/capacity/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -244,10 +244,16 @@ func (c *Reconciler) reconcileAll(ctx context.Context) error {
blockedByReservations = map[string]map[string]int64{}
}

reservedByGroupAZ, err := c.reservedResourcesByGroupAZ(ctx, flavorGroups, hvByName)
if err != nil {
logger.Error(err, "failed to attribute reserved capacity to flavor groups, reserved capacity will be omitted")
reservedByGroupAZ = map[vmUsageKey]map[string]int64{}
}

usageByKey := c.computeVMUsage(ctx, flavorGroups, hvList.Items)

for _, az := range azs {
c.reconcileAZ(ctx, az, flavorGroups, hvByName, blockedByReservations, usageByKey)
c.reconcileAZ(ctx, az, flavorGroups, hvByName, blockedByReservations, reservedByGroupAZ, usageByKey)
}

logger.Info("capacity reconcile cycle completed",
Expand Down Expand Up @@ -505,6 +511,7 @@ func (c *Reconciler) reconcileAZ(
flavorGroups map[string]compute.FlavorGroupFeature,
hvByName map[string]hv1.Hypervisor,
blockedByReservations map[string]map[string]int64,
reservedByGroupAZ map[vmUsageKey]map[string]int64,
usageByKey map[vmUsageKey]vmUsage,
) {

Expand Down Expand Up @@ -592,6 +599,7 @@ func (c *Reconciler) reconcileAZ(
freeResources[r.groupName],
exclusiveResources[r.groupName],
rawExclusiveByGroup[r.groupName],
reservedByGroupAZ[vmUsageKey{r.groupName, az}],
); err != nil {
logger.Error(err, "failed to write FlavorGroupCapacity CRD",
"flavorGroup", r.groupName, "az", az)
Expand Down Expand Up @@ -631,6 +639,7 @@ func (c *Reconciler) writeCRD(
freeRes map[string]int64,
exclusiveRes map[string]int64,
exclusivelyRawRes map[string]int64,
reservedRes map[string]int64,
) error {

crdName := crdNameFor(groupName, az)
Expand Down Expand Up @@ -676,11 +685,13 @@ func (c *Reconciler) writeCRD(
existing.Status.FreeCapacity = resMapToQuantity(freeRes)
existing.Status.ExclusivelyFreeCapacity = resMapToQuantity(exclusiveRes)
existing.Status.ExclusivelyRawCapacity = resMapToQuantity(exclusivelyRawRes)
existing.Status.ExclusivelyReservedCapacity = resMapToQuantity(reservedRes)
var exclusivelyFreeSlots int64
if flavorMemBytes := int64(groupData.SmallestFlavor.MemoryMB) * 1024 * 1024; flavorMemBytes > 0 { //nolint:gosec
flavorVCPUs := int64(groupData.SmallestFlavor.VCPUs) //nolint:gosec
exclusivelyFreeSlots = flavorSlots(exclusiveRes, flavorMemBytes, flavorVCPUs)
existing.Status.ExclusivelyFreeSlots = exclusivelyFreeSlots
existing.Status.ExclusivelyReservedSlots = flavorSlots(reservedRes, flavorMemBytes, flavorVCPUs)
}
existing.Status.LastReconcileAt = metav1.Now()

Expand Down Expand Up @@ -860,6 +871,110 @@ func (c *Reconciler) blockedResourcesByHost(ctx context.Context) (map[string]map
return blocked, nil
}

// reservedResourcesByGroupAZ sums reservation-blocked capacity (failover + unfilled committed
// slots) per (group, AZ), attributing each reservation by its own ResourceGroup rather than by
// host — so a host fully packed by empty reservations still counts. Only placed reservations are
// included, each once.
func (c *Reconciler) reservedResourcesByGroupAZ(
ctx context.Context,
flavorGroups map[string]compute.FlavorGroupFeature,
hvByName map[string]hv1.Hypervisor,
) (map[vmUsageKey]map[string]int64, error) {

logger := LoggerFromContext(ctx)
var list v1alpha1.ReservationList
if err := c.client.List(ctx, &list); err != nil {
return nil, fmt.Errorf("failed to list reservations: %w", err)
}

flavorToGroup := make(map[string]string)
for groupName, gd := range flavorGroups {
for _, f := range gd.Flavors {
flavorToGroup[f.Name] = groupName
}
}

reserved := make(map[vmUsageKey]map[string]int64)
for i := range list.Items {
res := &list.Items[i]
// Only committed and failover reservations represent capacity that is reserved but
// empty; in-flight reservations are VMs in buildup and are counted as running instead.
if res.Spec.Type != v1alpha1.ReservationTypeCommittedResource && res.Spec.Type != v1alpha1.ReservationTypeFailover {
continue
}
host := res.Status.Host
if host == "" {
host = res.Spec.TargetHost
}
if host == "" {
continue // pending; reported on a later cycle once placed
}
hv, ok := hvByName[host]
if !ok {
continue // host is not a known hypervisor in this cluster
}
az := hv.Labels["topology.kubernetes.io/zone"]
if az == "" {
continue
}
group, ok := resolveReservationGroup(res, flavorGroups, flavorToGroup)
if !ok {
logger.Info("reservation could not be attributed to a flavor group; its reserved capacity is not reported",
"reservation", res.Name, "type", res.Spec.Type, "az", az)
continue
}
blockedRes := reservations.UnusedReservationCapacity(res, false)
key := vmUsageKey{group: group, az: az}
if reserved[key] == nil {
reserved[key] = make(map[string]int64)
}
if qty, ok := blockedRes[hv1.ResourceMemory]; ok {
reserved[key][ResourceMemory] += qty.Value()
}
if qty, ok := blockedRes[hv1.ResourceCPU]; ok {
reserved[key][ResourceCores] += qty.Value()
}
}
return reserved, nil
}

// resolveReservationGroup returns the flavor group a reservation belongs to. Committed reservations
// carry the group directly; failover reservations carry the group name, or the flavor name (resolved
// to its group) when useFlavorGroupResources is disabled. Returns false if none can be determined.
func resolveReservationGroup(
res *v1alpha1.Reservation,
flavorGroups map[string]compute.FlavorGroupFeature,
flavorToGroup map[string]string,
) (string, bool) {

switch res.Spec.Type {
case v1alpha1.ReservationTypeCommittedResource:
if res.Spec.CommittedResourceReservation == nil {
return "", false
}
if g := res.Spec.CommittedResourceReservation.ResourceGroup; g != "" {
if _, ok := flavorGroups[g]; ok {
return g, true
}
}
return "", false
case v1alpha1.ReservationTypeFailover:
if res.Spec.FailoverReservation == nil {
return "", false
}
g := res.Spec.FailoverReservation.ResourceGroup
if _, ok := flavorGroups[g]; ok {
return g, true
}
if group, ok := flavorToGroup[g]; ok {
return group, true
}
return "", false
default:
return "", false
}
}

// sumCommittedCapacity sums active CommittedResource amounts (memory type, guaranteed/confirmed)
// for the given (flavorGroup, az) pair. Returns the total in smallest-flavor slots.
func (c *Reconciler) sumCommittedCapacity(ctx context.Context, groupName, az string, smallestFlavorBytes int64) (int64, error) {
Expand Down
Loading
Loading