diff --git a/internal/libvirt/libvirt.go b/internal/libvirt/libvirt.go index 063f224..b4d4aa2 100644 --- a/internal/libvirt/libvirt.go +++ b/internal/libvirt/libvirt.go @@ -518,13 +518,35 @@ func (l *LibVirt) addAllocationCapacity(old v1.Hypervisor) (v1.Hypervisor, error ) totalCpuAlloc.Add(cpuAlloc) - // Add memory allocation to the cells this domain is using. + // Map each of the domain's guest NUMA cell ids to the host physical + // NUMA node it is pinned to. The + // entries carry this binding: `cellid` is the guest's virtual NUMA + // cell index (0 for a single-vNUMA VM), while `nodeset` is the host + // physical NUMA node the cell is placed on. We must account per host + // node (the key used by cellsById, which comes from the host + // capabilities), not per guest cell id, otherwise every single-vNUMA + // VM would be attributed to cell 0 regardless of where it actually + // runs. + hostNodeByCellID := make(map[uint64]uint64, len(domInfo.NumaTune.MemNodes)) for _, memoryNode := range domInfo.NumaTune.MemNodes { - cell, ok := cellsById[memoryNode.CellID] + hostNode, err := parseSingleNodeset(memoryNode.Nodeset) + if err != nil { + return old, fmt.Errorf( + "domain %s memnode cellid %d has unexpected nodeset %q: %w", + domInfo.Name, memoryNode.CellID, memoryNode.Nodeset, err, + ) + } + hostNodeByCellID[memoryNode.CellID] = hostNode + } + + // Add memory allocation to the host cells this domain is using. + for _, memoryNode := range domInfo.NumaTune.MemNodes { + hostNode := hostNodeByCellID[memoryNode.CellID] + cell, ok := cellsById[hostNode] if !ok { return old, fmt.Errorf( - "domain %s uses unknown memory cell %d", - domInfo.Name, memoryNode.CellID, + "domain %s uses unknown host memory cell %d", + domInfo.Name, hostNode, ) } memAllocCell := cell.Allocation[v1.ResourceMemory] @@ -535,21 +557,31 @@ func (l *LibVirt) addAllocationCapacity(old v1.Hypervisor) (v1.Hypervisor, error NewQuantity(memAlloc.Value()/nCells, resource.BinarySI) memAllocCell.Add(memAllocPerCell) cell.Allocation[v1.ResourceMemory] = memAllocCell - cellsById[memoryNode.CellID] = cell + cellsById[hostNode] = cell } - // Add cpu allocation to the cells this domain is using. + // Add cpu allocation to the host cells this domain is using. The guest + // CPU NUMA cells () share the same cell ids as + // the memnode entries above, so we resolve each to its host node via + // the same mapping. if domInfo.CPU.Numa == nil { return old, fmt.Errorf("missing cpu numa for dom %s", domInfo.Name) } for _, cpuCell := range domInfo.CPU.Numa.Cells { - cell, ok := cellsById[cpuCell.ID] + hostNode, ok := hostNodeByCellID[cpuCell.ID] if !ok { return old, fmt.Errorf( - "domain %s uses unknown cpu cell %d", + "domain %s cpu cell %d has no memnode mapping to a host node", domInfo.Name, cpuCell.ID, ) } + cell, ok := cellsById[hostNode] + if !ok { + return old, fmt.Errorf( + "domain %s uses unknown host cpu cell %d", + domInfo.Name, hostNode, + ) + } cpuAllocCell := cell.Allocation[v1.ResourceCPU] // If a domain is using multiple cpu cells, assume the distribution // across cells is even. @@ -558,7 +590,7 @@ func (l *LibVirt) addAllocationCapacity(old v1.Hypervisor) (v1.Hypervisor, error NewQuantity(cpuAlloc.Value()/nCells, resource.DecimalSI) cpuAllocCell.Add(cpuAllocPerCell) cell.Allocation[v1.ResourceCPU] = cpuAllocCell - cellsById[cpuCell.ID] = cell + cellsById[hostNode] = cell } } cellsAsSlice := []v1.Cell{} diff --git a/internal/libvirt/libvirt_test.go b/internal/libvirt/libvirt_test.go index 4ff05ab..d5905d2 100644 --- a/internal/libvirt/libvirt_test.go +++ b/internal/libvirt/libvirt_test.go @@ -476,8 +476,13 @@ func TestAddDomainCapabilities_UnsupportedFiltered(t *testing.T) { func TestAddAllocationCapacity_InstanceSpreadAcrossMultipleNumaCells(t *testing.T) { // Test scenario: An instance with 128 GiB memory and 16 CPUs is spread - // across NUMA cells 1 and 2. Each cell should report 64 GiB memory + // across host NUMA cells 1 and 2. Each cell should report 64 GiB memory // allocation and 8 CPU allocation. + // + // The instance's guest NUMA cell ids (0 and 1) deliberately differ from + // the host physical NUMA nodes it is pinned to (1 and 2). Allocation must + // be attributed to the host nodes from the memnode nodeset, not to the + // guest cell ids, which are a per-VM virtual index. caps := capabilities.Capabilities{ Host: capabilities.CapabilitiesHost{ Topology: capabilities.CapabilitiesHostTopology{ @@ -574,19 +579,19 @@ func TestAddAllocationCapacity_InstanceSpreadAcrossMultipleNumaCells(t *testing. {VCPU: 15, CPUSet: "135"}, }, }, - // Instance uses NUMA cells 1 and 2 for memory + // Guest NUMA cells 0 and 1 are pinned to host nodes 1 and 2. NumaTune: &dominfo.DomainNumaTune{ MemNodes: []dominfo.DomainNumaMemNode{ - {CellID: 1, Mode: "strict", Nodeset: "1"}, - {CellID: 2, Mode: "strict", Nodeset: "2"}, + {CellID: 0, Mode: "strict", Nodeset: "1"}, + {CellID: 1, Mode: "strict", Nodeset: "2"}, }, }, - // Instance uses NUMA cells 1 and 2 for CPUs + // Guest CPU NUMA cells share the same ids as the memnodes above. CPU: &dominfo.DomainCPU{ Numa: &dominfo.DomainCPUNuma{ Cells: []dominfo.DomainCPUNumaCell{ - {ID: 1, CPUs: "0-7", Memory: 64, Unit: "GiB"}, - {ID: 2, CPUs: "8-15", Memory: 64, Unit: "GiB"}, + {ID: 0, CPUs: "0-7", Memory: 64, Unit: "GiB"}, + {ID: 1, CPUs: "8-15", Memory: 64, Unit: "GiB"}, }, }, }, @@ -815,6 +820,111 @@ func TestAddAllocationCapacity_Success(t *testing.T) { } } +func TestAddAllocationCapacity_SingleVNumaInstancePinnedToNonZeroHostCell(t *testing.T) { + // Regression test: a single-vNUMA instance (guest cell id 0) pinned to + // host NUMA node 1 must have its allocation attributed to host cell 1, not + // host cell 0. Keying by the guest cell id instead of the memnode nodeset + // would pile every such instance onto cell 0 regardless of placement. + caps := capabilities.Capabilities{ + Host: capabilities.CapabilitiesHost{ + Topology: capabilities.CapabilitiesHostTopology{ + CellSpec: capabilities.CapabilitiesHostTopologyCells{ + Num: 2, + Cells: []capabilities.CapabilitiesHostTopologyCell{ + { + ID: 0, + Memory: capabilities.CapabilitiesHostTopologyCellMemory{ + Unit: "GiB", Value: 64, + }, + Pages: []capabilities.CapabilitiesHostTopologyCellPages{ + {Unit: "KiB", Size: 4, Value: 1000000}, + {Unit: "KiB", Size: 2048, Value: 32768}, + }, + CPUs: capabilities.CapabilitiesHostTopologyCellCPUs{Num: 16}, + }, + { + ID: 1, + Memory: capabilities.CapabilitiesHostTopologyCellMemory{ + Unit: "GiB", Value: 64, + }, + Pages: []capabilities.CapabilitiesHostTopologyCellPages{ + {Unit: "KiB", Size: 4, Value: 1000000}, + {Unit: "KiB", Size: 2048, Value: 32768}, + }, + CPUs: capabilities.CapabilitiesHostTopologyCellCPUs{Num: 16}, + }, + }, + }, + }, + }, + } + + domInfos := []dominfo.DomainInfo{ + { + Name: "single-vnuma-on-cell-1", + Memory: &dominfo.DomainMemory{Unit: "GiB", Value: 8}, + CPUTune: &dominfo.DomainCPUTune{ + VCPUPins: []dominfo.DomainVCPUPin{ + {VCPU: 0, CPUSet: "8"}, + {VCPU: 1, CPUSet: "9"}, + }, + }, + // Guest cell 0 is pinned to host node 1. + NumaTune: &dominfo.DomainNumaTune{ + MemNodes: []dominfo.DomainNumaMemNode{ + {CellID: 0, Mode: "strict", Nodeset: "1"}, + }, + }, + CPU: &dominfo.DomainCPU{ + Numa: &dominfo.DomainCPUNuma{ + Cells: []dominfo.DomainCPUNumaCell{ + {ID: 0, CPUs: "0-1", Memory: 8, Unit: "GiB"}, + }, + }, + }, + }, + } + + l := &LibVirt{ + capabilitiesClient: &mockCapabilitiesClient{caps: caps}, + domainInfoClient: &mockDomInfoClient{infos: domInfos}, + } + + result, err := l.addAllocationCapacity(v1.Hypervisor{}) + if err != nil { + t.Fatalf("addAllocationCapacity() returned unexpected error: %v", err) + } + + cellsByID := make(map[uint64]v1.Cell) + for _, cell := range result.Status.Cells { + cellsByID[cell.CellID] = cell + } + + // Host cell 0 must be untouched. + cell0Mem := cellsByID[0].Allocation["memory"] + if !cell0Mem.IsZero() { + t.Errorf("Cell 0: expected zero memory allocation, got %s", cell0Mem.String()) + } + cell0Cpu := cellsByID[0].Allocation["cpu"] + if !cell0Cpu.IsZero() { + t.Errorf("Cell 0: expected zero CPU allocation, got %s", cell0Cpu.String()) + } + + // Host cell 1 must carry the whole allocation. + expectedMem := resource.NewQuantity(8*1024*1024*1024, resource.BinarySI) + cell1Mem := cellsByID[1].Allocation["memory"] + if !cell1Mem.Equal(*expectedMem) { + t.Errorf("Cell 1: expected memory allocation %s, got %s", + expectedMem.String(), cell1Mem.String()) + } + expectedCpu := resource.NewQuantity(2, resource.DecimalSI) + cell1Cpu := cellsByID[1].Allocation["cpu"] + if !cell1Cpu.Equal(*expectedCpu) { + t.Errorf("Cell 1: expected CPU allocation %s, got %s", + expectedCpu.String(), cell1Cpu.String()) + } +} + func TestAddAllocationCapacity_FallsBackToPhysicalMemoryWithoutHugepages(t *testing.T) { // When libvirt reports no hugepages for a cell, the memory capacity should // fall back to the cell's total physical memory instead of reporting zero. diff --git a/internal/libvirt/utils.go b/internal/libvirt/utils.go index baac145..abbb623 100644 --- a/internal/libvirt/utils.go +++ b/internal/libvirt/utils.go @@ -19,8 +19,11 @@ package libvirt import ( "encoding/hex" + "errors" "fmt" "math" + "strconv" + "strings" "k8s.io/apimachinery/pkg/api/resource" @@ -135,3 +138,26 @@ func HugepageMemoryToResource(pages []capabilities.CapabilitiesHostTopologyCellP } return *total, nil } + +// parseSingleNodeset parses a libvirt NUMA nodeset attribute that is expected +// to reference exactly one host physical NUMA node, returning its id. +// +// For the per-vNUMA memory binding Nova emits +// (), each guest cell is pinned to a single +// host node, so the nodeset is a plain node id such as "1". A nodeset that is +// empty, a range, or a comma-separated list cannot be attributed to a single +// host cell and is rejected so the allocation is never silently misreported. +func parseSingleNodeset(nodeset string) (uint64, error) { + trimmed := strings.TrimSpace(nodeset) + if trimmed == "" { + return 0, errors.New("empty nodeset") + } + if strings.ContainsAny(trimmed, ",-") { + return 0, fmt.Errorf("nodeset %q references more than one host node", nodeset) + } + node, err := strconv.ParseUint(trimmed, 10, 64) + if err != nil { + return 0, fmt.Errorf("invalid nodeset %q: %w", nodeset, err) + } + return node, nil +} diff --git a/internal/libvirt/utils_test.go b/internal/libvirt/utils_test.go index 2c15c18..9d19be3 100644 --- a/internal/libvirt/utils_test.go +++ b/internal/libvirt/utils_test.go @@ -506,3 +506,37 @@ func TestMemoryToResourceNegativeValues(t *testing.T) { }) } } + +func TestParseSingleNodeset(t *testing.T) { + tests := []struct { + name string + nodeset string + expected uint64 + wantErr bool + }{ + {name: "single node", nodeset: "1", expected: 1}, + {name: "node zero", nodeset: "0", expected: 0}, + {name: "whitespace trimmed", nodeset: " 2 ", expected: 2}, + {name: "empty", nodeset: "", wantErr: true}, + {name: "range rejected", nodeset: "0-1", wantErr: true}, + {name: "list rejected", nodeset: "0,1", wantErr: true}, + {name: "non-numeric rejected", nodeset: "x", wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseSingleNodeset(tt.nodeset) + if tt.wantErr { + if err == nil { + t.Fatalf("expected error for nodeset %q, got none", tt.nodeset) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tt.expected { + t.Errorf("expected %d, got %d", tt.expected, got) + } + }) + } +}