From bd52afcc7eb171cb7c357cc960c0ff00cdc199f3 Mon Sep 17 00:00:00 2001 From: Philipp Matthes Date: Fri, 6 Nov 2026 09:54:49 +0100 Subject: [PATCH] Fix incorrect per-NUMA-cell allocation reporting addAllocationCapacity keyed per-cell memory and CPU accounting by the domain's guest NUMA cell id (memnode cellid / cpu numa cell id), which is a per-VM virtual index and almost always 0 for single-vNUMA VMs. As a result, nearly all allocation was attributed to host cell 0 regardless of which host physical NUMA node the VMs were actually pinned to; only VMs spanning multiple vNUMA nodes showed multi-cell allocation. Key the accounting by the host physical NUMA node from the memnode nodeset instead, and resolve each guest CPU NUMA cell to its host node via the same guest-cell-to-host-node mapping so CPU and memory are attributed consistently. Add parseSingleNodeset to parse the nodeset and reject multi-node sets that cannot be attributed to a single cell. Co-authored-by: jeoff.zheng@sap.com Signed-off-by: Philipp Matthes --- internal/libvirt/libvirt.go | 50 ++++++++++--- internal/libvirt/libvirt_test.go | 124 +++++++++++++++++++++++++++++-- internal/libvirt/utils.go | 26 +++++++ internal/libvirt/utils_test.go | 34 +++++++++ 4 files changed, 218 insertions(+), 16 deletions(-) diff --git a/internal/libvirt/libvirt.go b/internal/libvirt/libvirt.go index 063f224..b4d4aa2 100644 --- a/internal/libvirt/libvirt.go +++ b/internal/libvirt/libvirt.go @@ -518,13 +518,35 @@ func (l *LibVirt) addAllocationCapacity(old v1.Hypervisor) (v1.Hypervisor, error ) totalCpuAlloc.Add(cpuAlloc) - // Add memory allocation to the cells this domain is using. + // Map each of the domain's guest NUMA cell ids to the host physical + // NUMA node it is pinned to. The + // entries carry this binding: `cellid` is the guest's virtual NUMA + // cell index (0 for a single-vNUMA VM), while `nodeset` is the host + // physical NUMA node the cell is placed on. We must account per host + // node (the key used by cellsById, which comes from the host + // capabilities), not per guest cell id, otherwise every single-vNUMA + // VM would be attributed to cell 0 regardless of where it actually + // runs. + hostNodeByCellID := make(map[uint64]uint64, len(domInfo.NumaTune.MemNodes)) for _, memoryNode := range domInfo.NumaTune.MemNodes { - cell, ok := cellsById[memoryNode.CellID] + hostNode, err := parseSingleNodeset(memoryNode.Nodeset) + if err != nil { + return old, fmt.Errorf( + "domain %s memnode cellid %d has unexpected nodeset %q: %w", + domInfo.Name, memoryNode.CellID, memoryNode.Nodeset, err, + ) + } + hostNodeByCellID[memoryNode.CellID] = hostNode + } + + // Add memory allocation to the host cells this domain is using. + for _, memoryNode := range domInfo.NumaTune.MemNodes { + hostNode := hostNodeByCellID[memoryNode.CellID] + cell, ok := cellsById[hostNode] if !ok { return old, fmt.Errorf( - "domain %s uses unknown memory cell %d", - domInfo.Name, memoryNode.CellID, + "domain %s uses unknown host memory cell %d", + domInfo.Name, hostNode, ) } memAllocCell := cell.Allocation[v1.ResourceMemory] @@ -535,21 +557,31 @@ func (l *LibVirt) addAllocationCapacity(old v1.Hypervisor) (v1.Hypervisor, error NewQuantity(memAlloc.Value()/nCells, resource.BinarySI) memAllocCell.Add(memAllocPerCell) cell.Allocation[v1.ResourceMemory] = memAllocCell - cellsById[memoryNode.CellID] = cell + cellsById[hostNode] = cell } - // Add cpu allocation to the cells this domain is using. + // Add cpu allocation to the host cells this domain is using. The guest + // CPU NUMA cells () share the same cell ids as + // the memnode entries above, so we resolve each to its host node via + // the same mapping. if domInfo.CPU.Numa == nil { return old, fmt.Errorf("missing cpu numa for dom %s", domInfo.Name) } for _, cpuCell := range domInfo.CPU.Numa.Cells { - cell, ok := cellsById[cpuCell.ID] + hostNode, ok := hostNodeByCellID[cpuCell.ID] if !ok { return old, fmt.Errorf( - "domain %s uses unknown cpu cell %d", + "domain %s cpu cell %d has no memnode mapping to a host node", domInfo.Name, cpuCell.ID, ) } + cell, ok := cellsById[hostNode] + if !ok { + return old, fmt.Errorf( + "domain %s uses unknown host cpu cell %d", + domInfo.Name, hostNode, + ) + } cpuAllocCell := cell.Allocation[v1.ResourceCPU] // If a domain is using multiple cpu cells, assume the distribution // across cells is even. @@ -558,7 +590,7 @@ func (l *LibVirt) addAllocationCapacity(old v1.Hypervisor) (v1.Hypervisor, error NewQuantity(cpuAlloc.Value()/nCells, resource.DecimalSI) cpuAllocCell.Add(cpuAllocPerCell) cell.Allocation[v1.ResourceCPU] = cpuAllocCell - cellsById[cpuCell.ID] = cell + cellsById[hostNode] = cell } } cellsAsSlice := []v1.Cell{} diff --git a/internal/libvirt/libvirt_test.go b/internal/libvirt/libvirt_test.go index 4ff05ab..d5905d2 100644 --- a/internal/libvirt/libvirt_test.go +++ b/internal/libvirt/libvirt_test.go @@ -476,8 +476,13 @@ func TestAddDomainCapabilities_UnsupportedFiltered(t *testing.T) { func TestAddAllocationCapacity_InstanceSpreadAcrossMultipleNumaCells(t *testing.T) { // Test scenario: An instance with 128 GiB memory and 16 CPUs is spread - // across NUMA cells 1 and 2. Each cell should report 64 GiB memory + // across host NUMA cells 1 and 2. Each cell should report 64 GiB memory // allocation and 8 CPU allocation. + // + // The instance's guest NUMA cell ids (0 and 1) deliberately differ from + // the host physical NUMA nodes it is pinned to (1 and 2). Allocation must + // be attributed to the host nodes from the memnode nodeset, not to the + // guest cell ids, which are a per-VM virtual index. caps := capabilities.Capabilities{ Host: capabilities.CapabilitiesHost{ Topology: capabilities.CapabilitiesHostTopology{ @@ -574,19 +579,19 @@ func TestAddAllocationCapacity_InstanceSpreadAcrossMultipleNumaCells(t *testing. {VCPU: 15, CPUSet: "135"}, }, }, - // Instance uses NUMA cells 1 and 2 for memory + // Guest NUMA cells 0 and 1 are pinned to host nodes 1 and 2. NumaTune: &dominfo.DomainNumaTune{ MemNodes: []dominfo.DomainNumaMemNode{ - {CellID: 1, Mode: "strict", Nodeset: "1"}, - {CellID: 2, Mode: "strict", Nodeset: "2"}, + {CellID: 0, Mode: "strict", Nodeset: "1"}, + {CellID: 1, Mode: "strict", Nodeset: "2"}, }, }, - // Instance uses NUMA cells 1 and 2 for CPUs + // Guest CPU NUMA cells share the same ids as the memnodes above. CPU: &dominfo.DomainCPU{ Numa: &dominfo.DomainCPUNuma{ Cells: []dominfo.DomainCPUNumaCell{ - {ID: 1, CPUs: "0-7", Memory: 64, Unit: "GiB"}, - {ID: 2, CPUs: "8-15", Memory: 64, Unit: "GiB"}, + {ID: 0, CPUs: "0-7", Memory: 64, Unit: "GiB"}, + {ID: 1, CPUs: "8-15", Memory: 64, Unit: "GiB"}, }, }, }, @@ -815,6 +820,111 @@ func TestAddAllocationCapacity_Success(t *testing.T) { } } +func TestAddAllocationCapacity_SingleVNumaInstancePinnedToNonZeroHostCell(t *testing.T) { + // Regression test: a single-vNUMA instance (guest cell id 0) pinned to + // host NUMA node 1 must have its allocation attributed to host cell 1, not + // host cell 0. Keying by the guest cell id instead of the memnode nodeset + // would pile every such instance onto cell 0 regardless of placement. + caps := capabilities.Capabilities{ + Host: capabilities.CapabilitiesHost{ + Topology: capabilities.CapabilitiesHostTopology{ + CellSpec: capabilities.CapabilitiesHostTopologyCells{ + Num: 2, + Cells: []capabilities.CapabilitiesHostTopologyCell{ + { + ID: 0, + Memory: capabilities.CapabilitiesHostTopologyCellMemory{ + Unit: "GiB", Value: 64, + }, + Pages: []capabilities.CapabilitiesHostTopologyCellPages{ + {Unit: "KiB", Size: 4, Value: 1000000}, + {Unit: "KiB", Size: 2048, Value: 32768}, + }, + CPUs: capabilities.CapabilitiesHostTopologyCellCPUs{Num: 16}, + }, + { + ID: 1, + Memory: capabilities.CapabilitiesHostTopologyCellMemory{ + Unit: "GiB", Value: 64, + }, + Pages: []capabilities.CapabilitiesHostTopologyCellPages{ + {Unit: "KiB", Size: 4, Value: 1000000}, + {Unit: "KiB", Size: 2048, Value: 32768}, + }, + CPUs: capabilities.CapabilitiesHostTopologyCellCPUs{Num: 16}, + }, + }, + }, + }, + }, + } + + domInfos := []dominfo.DomainInfo{ + { + Name: "single-vnuma-on-cell-1", + Memory: &dominfo.DomainMemory{Unit: "GiB", Value: 8}, + CPUTune: &dominfo.DomainCPUTune{ + VCPUPins: []dominfo.DomainVCPUPin{ + {VCPU: 0, CPUSet: "8"}, + {VCPU: 1, CPUSet: "9"}, + }, + }, + // Guest cell 0 is pinned to host node 1. + NumaTune: &dominfo.DomainNumaTune{ + MemNodes: []dominfo.DomainNumaMemNode{ + {CellID: 0, Mode: "strict", Nodeset: "1"}, + }, + }, + CPU: &dominfo.DomainCPU{ + Numa: &dominfo.DomainCPUNuma{ + Cells: []dominfo.DomainCPUNumaCell{ + {ID: 0, CPUs: "0-1", Memory: 8, Unit: "GiB"}, + }, + }, + }, + }, + } + + l := &LibVirt{ + capabilitiesClient: &mockCapabilitiesClient{caps: caps}, + domainInfoClient: &mockDomInfoClient{infos: domInfos}, + } + + result, err := l.addAllocationCapacity(v1.Hypervisor{}) + if err != nil { + t.Fatalf("addAllocationCapacity() returned unexpected error: %v", err) + } + + cellsByID := make(map[uint64]v1.Cell) + for _, cell := range result.Status.Cells { + cellsByID[cell.CellID] = cell + } + + // Host cell 0 must be untouched. + cell0Mem := cellsByID[0].Allocation["memory"] + if !cell0Mem.IsZero() { + t.Errorf("Cell 0: expected zero memory allocation, got %s", cell0Mem.String()) + } + cell0Cpu := cellsByID[0].Allocation["cpu"] + if !cell0Cpu.IsZero() { + t.Errorf("Cell 0: expected zero CPU allocation, got %s", cell0Cpu.String()) + } + + // Host cell 1 must carry the whole allocation. + expectedMem := resource.NewQuantity(8*1024*1024*1024, resource.BinarySI) + cell1Mem := cellsByID[1].Allocation["memory"] + if !cell1Mem.Equal(*expectedMem) { + t.Errorf("Cell 1: expected memory allocation %s, got %s", + expectedMem.String(), cell1Mem.String()) + } + expectedCpu := resource.NewQuantity(2, resource.DecimalSI) + cell1Cpu := cellsByID[1].Allocation["cpu"] + if !cell1Cpu.Equal(*expectedCpu) { + t.Errorf("Cell 1: expected CPU allocation %s, got %s", + expectedCpu.String(), cell1Cpu.String()) + } +} + func TestAddAllocationCapacity_FallsBackToPhysicalMemoryWithoutHugepages(t *testing.T) { // When libvirt reports no hugepages for a cell, the memory capacity should // fall back to the cell's total physical memory instead of reporting zero. diff --git a/internal/libvirt/utils.go b/internal/libvirt/utils.go index baac145..abbb623 100644 --- a/internal/libvirt/utils.go +++ b/internal/libvirt/utils.go @@ -19,8 +19,11 @@ package libvirt import ( "encoding/hex" + "errors" "fmt" "math" + "strconv" + "strings" "k8s.io/apimachinery/pkg/api/resource" @@ -135,3 +138,26 @@ func HugepageMemoryToResource(pages []capabilities.CapabilitiesHostTopologyCellP } return *total, nil } + +// parseSingleNodeset parses a libvirt NUMA nodeset attribute that is expected +// to reference exactly one host physical NUMA node, returning its id. +// +// For the per-vNUMA memory binding Nova emits +// (), each guest cell is pinned to a single +// host node, so the nodeset is a plain node id such as "1". A nodeset that is +// empty, a range, or a comma-separated list cannot be attributed to a single +// host cell and is rejected so the allocation is never silently misreported. +func parseSingleNodeset(nodeset string) (uint64, error) { + trimmed := strings.TrimSpace(nodeset) + if trimmed == "" { + return 0, errors.New("empty nodeset") + } + if strings.ContainsAny(trimmed, ",-") { + return 0, fmt.Errorf("nodeset %q references more than one host node", nodeset) + } + node, err := strconv.ParseUint(trimmed, 10, 64) + if err != nil { + return 0, fmt.Errorf("invalid nodeset %q: %w", nodeset, err) + } + return node, nil +} diff --git a/internal/libvirt/utils_test.go b/internal/libvirt/utils_test.go index 2c15c18..9d19be3 100644 --- a/internal/libvirt/utils_test.go +++ b/internal/libvirt/utils_test.go @@ -506,3 +506,37 @@ func TestMemoryToResourceNegativeValues(t *testing.T) { }) } } + +func TestParseSingleNodeset(t *testing.T) { + tests := []struct { + name string + nodeset string + expected uint64 + wantErr bool + }{ + {name: "single node", nodeset: "1", expected: 1}, + {name: "node zero", nodeset: "0", expected: 0}, + {name: "whitespace trimmed", nodeset: " 2 ", expected: 2}, + {name: "empty", nodeset: "", wantErr: true}, + {name: "range rejected", nodeset: "0-1", wantErr: true}, + {name: "list rejected", nodeset: "0,1", wantErr: true}, + {name: "non-numeric rejected", nodeset: "x", wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseSingleNodeset(tt.nodeset) + if tt.wantErr { + if err == nil { + t.Fatalf("expected error for nodeset %q, got none", tt.nodeset) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tt.expected { + t.Errorf("expected %d, got %d", tt.expected, got) + } + }) + } +}