From 015815d2dd4f204bf9b9afdbf5c5e26c50f528b1 Mon Sep 17 00:00:00 2001
From: Philipp Matthes
Date: Tue, 17 Nov 2026 13:02:13 +0100
Subject: [PATCH 1/2] Adapt to upstream vm crd refactoring
Bump vm-operator and adapt to the refactored VirtualMachine CRD: the
Scheduled condition and Spec.TargetHost/LabelTargetHost are gone, the
status conditions were redesigned, and a cortex subspec now carries the
scheduling result (host candidates, intent, observed generation, last
transition time).
- vm_handler: write only the cortex subspec/labels, preserve the
out-of-band Ownership.Domain, stamp ObservedGeneration/LastTransitionTime
correctly, and stop writing operator-owned conditions.
- inflight vm owner handler: release in-flight reservations on a fresh
settled (status=False) Migrating/Resizing condition instead of removed
terminal reasons; cover evacuation via the Migrating condition.
- failover partition graph KPI: skip deleted / to-be-deleted VMs when
counting VMs per host.
- update tests, CRD pin in Tiltfile, RBAC, and docs.
Signed-off-by: Philipp Matthes
---
Tiltfile | 2 +-
.../02-nova-smart-scheduling.md | 5 +-
.../01-multicluster-client.md | 4 +-
go.mod | 2 +-
go.sum | 2 +
helm/bundles/cortex-nova/templates/rbac.yaml | 1 -
.../compute/failover_partition_graph.go | 9 +-
internal/scheduling/nova/vm_handler.go | 81 ++++---------
internal/scheduling/nova/vm_handler_test.go | 111 ++++++++++++++----
.../reservations/inflight/controller_test.go | 70 +++++------
.../reservations/inflight/vm_owner_handler.go | 77 ++++++------
.../inflight/vm_owner_handler_test.go | 57 +++++----
12 files changed, 232 insertions(+), 189 deletions(-)
diff --git a/Tiltfile b/Tiltfile
index 47b4dd672..d0ebeb509 100644
--- a/Tiltfile
+++ b/Tiltfile
@@ -135,7 +135,7 @@ local('kubectl wait --namespace cert-manager --for=condition=available deploymen
# Pinned SHAs must match the versions in go.mod.
url = 'https://raw.githubusercontent.com/cobaltcore-dev/openstack-hypervisor-operator/df757ad112ea027f47c198266cb3a323d52ee2fe/charts/openstack-hypervisor-operator/crds/kvm.cloud.sap_hypervisors.yaml'
local('curl -L ' + url + ' | kubectl apply -f -')
-url = 'https://raw.githubusercontent.com/cobaltcore-dev/vm-operator/66e5e459efeb01bfc599a1cda5921528ab47be19/dist/files/crds/cobaltcore.cloud_virtualmachines.yaml'
+url = 'https://raw.githubusercontent.com/cobaltcore-dev/vm-operator/f859ba2eb3bf/dist/files/crds/cobaltcore.cloud_virtualmachines.yaml'
local('curl -L ' + url + ' | kubectl apply -f -')
########### Cortex Manager & CRDs
diff --git a/docs/02-external-scheduler-api/02-nova-smart-scheduling.md b/docs/02-external-scheduler-api/02-nova-smart-scheduling.md
index adb5f5ec9..dbf142d53 100644
--- a/docs/02-external-scheduler-api/02-nova-smart-scheduling.md
+++ b/docs/02-external-scheduler-api/02-nova-smart-scheduling.md
@@ -161,7 +161,10 @@ HOST:.spec.cortex.hostCandidates[0].name
`.spec.cortex.explanation` carries a human-readable summary of why the chosen host won,
`.spec.cortex.intent` records the scheduling intent (e.g. initial placement or evacuation), and
`.spec.cortex.hostCandidates` lists the candidate hosts in ranked order; `-o yaml` on a single
-VM shows the full detail. The detector pipeline emits **`Descheduling`** resources for VMs it
+VM shows the full detail. Cortex only ever writes this `spec.cortex` subspec (plus the immutable
+spec and labels on create); the VM's `status.conditions` are owned exclusively by the vm-operator
+and reflect its own observations (Running, Migrating, Resizing, …), not the scheduling decision.
+The detector pipeline emits **`Descheduling`** resources for VMs it
wants to move:
```bash
diff --git a/docs/06-cortex-library/01-multicluster-client.md b/docs/06-cortex-library/01-multicluster-client.md
index 9765efe53..66f123f1b 100644
--- a/docs/06-cortex-library/01-multicluster-client.md
+++ b/docs/06-cortex-library/01-multicluster-client.md
@@ -630,8 +630,8 @@ kubectl --context kind-cortex-remote-az-b get virtualmachines 2>/dev/null || ech
```
You should see the `VirtualMachine` named `cortex-test-instance-001` only in `kind-cortex-remote-az-b`.
-Describe it to see the full scheduling result — the target host under `spec.targetHost`, the ranked
-`spec.cortex.hostCandidates`, and `spec.cortex.explanation`, the human-readable summary of why that host
+Describe it to see the full scheduling result — the ranked `spec.cortex.hostCandidates` (the first entry is
+the selected host), and `spec.cortex.explanation`, the human-readable summary of why that host
won:
```bash
diff --git a/go.mod b/go.mod
index b874eb5b8..199de503f 100644
--- a/go.mod
+++ b/go.mod
@@ -4,7 +4,7 @@ go 1.27
require (
github.com/cobaltcore-dev/openstack-hypervisor-operator v1.4.0
- github.com/cobaltcore-dev/vm-operator v0.2.10
+ github.com/cobaltcore-dev/vm-operator v0.2.11-0.20261117084340-f859ba2eb3bf
github.com/go-gorp/gorp v2.2.0+incompatible
github.com/go-logr/logr v1.4.4
github.com/google/uuid v1.6.0
diff --git a/go.sum b/go.sum
index c3e037bbf..e8aa5bfef 100644
--- a/go.sum
+++ b/go.sum
@@ -26,6 +26,8 @@ github.com/cobaltcore-dev/openstack-hypervisor-operator v1.4.0 h1:lAb+qBhyfgf77Y
github.com/cobaltcore-dev/openstack-hypervisor-operator v1.4.0/go.mod h1:VIbkpZpnTeLQrEesadTeANk0MB0t+enfIGfJIX/a2vY=
github.com/cobaltcore-dev/vm-operator v0.2.10 h1:W/cNZBDva2pq9kn4QYTcfn4VPxyNEoYqxJBU9hc8Ulg=
github.com/cobaltcore-dev/vm-operator v0.2.10/go.mod h1:0Lanmhb7onfOIHX+4h1lGLC7vga/nNYabJ6a/y/APk4=
+github.com/cobaltcore-dev/vm-operator v0.2.11-0.20261117084340-f859ba2eb3bf h1:12SpyudWEoEB8QZ4OGS/R6iYYoY+BwjlDUc8Xf6WFVk=
+github.com/cobaltcore-dev/vm-operator v0.2.11-0.20261117084340-f859ba2eb3bf/go.mod h1:0Lanmhb7onfOIHX+4h1lGLC7vga/nNYabJ6a/y/APk4=
github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4=
github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
diff --git a/helm/bundles/cortex-nova/templates/rbac.yaml b/helm/bundles/cortex-nova/templates/rbac.yaml
index 590b12094..ee8c4c94a 100644
--- a/helm/bundles/cortex-nova/templates/rbac.yaml
+++ b/helm/bundles/cortex-nova/templates/rbac.yaml
@@ -55,7 +55,6 @@ rules:
- cobaltcore.cloud
resources:
- virtualmachines
- - virtualmachines/status
verbs:
- create
- get
diff --git a/internal/knowledge/kpis/plugins/compute/failover_partition_graph.go b/internal/knowledge/kpis/plugins/compute/failover_partition_graph.go
index 6ab253e17..04951692e 100644
--- a/internal/knowledge/kpis/plugins/compute/failover_partition_graph.go
+++ b/internal/knowledge/kpis/plugins/compute/failover_partition_graph.go
@@ -17,6 +17,7 @@ import (
vmv1alpha1 "github.com/cobaltcore-dev/vm-operator/api/v1alpha1"
"github.com/prometheus/client_golang/prometheus"
corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/api/resource"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -120,7 +121,9 @@ func (k *FailoverPartitionGraphKPI) Collect(ch chan<- prometheus.Metric) {
}
// Count the vms running on each host so the node series can expose it. A vm
// Nova hasn't placed yet has no host and is skipped (it also nil-panics on
- // Status.Nova otherwise).
+ // Status.Nova otherwise). The vm-operator retains the last-known Nova fields
+ // after a server disappears, so skip vms confirmed deleted or scheduled for
+ // deletion to avoid counting ghosts against a host.
vmList := &vmv1alpha1.VirtualMachineList{}
if err := k.Client.List(ctx, vmList); err != nil {
failoverPartitionGraphKPILogger.Error(err, "Failed to list virtual machines")
@@ -131,6 +134,10 @@ func (k *FailoverPartitionGraphKPI) Collect(ch chan<- prometheus.Metric) {
if vm.Status.Nova == nil || vm.Status.Nova.Host == "" {
continue
}
+ if vm.Status.DeletionTime != nil ||
+ meta.IsStatusConditionTrue(vm.Status.Conditions, vmv1alpha1.ConditionTypeDeleted) {
+ continue
+ }
vmsByHost[vm.Status.Nova.Host]++
}
for _, hv := range hypervisorList.Items {
diff --git a/internal/scheduling/nova/vm_handler.go b/internal/scheduling/nova/vm_handler.go
index bbc22d635..3b48bacff 100644
--- a/internal/scheduling/nova/vm_handler.go
+++ b/internal/scheduling/nova/vm_handler.go
@@ -18,7 +18,6 @@ import (
vmv1alpha1 "github.com/cobaltcore-dev/vm-operator/api/v1alpha1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
- "k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
@@ -36,9 +35,10 @@ type VirtualMachineHandler struct {
// Upsert creates or patches the VirtualMachine resource described by the
// scheduling result and request. Region and AZ are immutable on the CRD and only
-// written on create; mutable fields (target host, cortex spec, labels) are
-// patched on updates. A human-readable explanation of the result and the intent
-// are recorded on the VM's cortex spec.
+// written on create; mutable fields (cortex spec, labels) are patched on
+// updates. A human-readable explanation of the result and the intent are
+// recorded on the VM's cortex spec. Conditions are owned by the vm-operator and
+// never written here.
func (h *VirtualMachineHandler) Upsert(ctx context.Context, result compute.PlacementResult, request api.ExternalSchedulerRequest, intent v1alpha1.SchedulingIntent) error {
log := ctrl.LoggerFrom(ctx)
@@ -75,20 +75,14 @@ func (h *VirtualMachineHandler) Upsert(ctx context.Context, result compute.Place
vm.ObjectMeta = metav1.ObjectMeta{
Name: request.Spec.Data.InstanceUUID,
Labels: map[string]string{
- vmv1alpha1.LabelRegion: targetHvRegion,
- vmv1alpha1.LabelAZ: targetHvAZ,
- vmv1alpha1.LabelTargetHost: targetHv.Name,
+ vmv1alpha1.LabelRegion: targetHvRegion,
+ vmv1alpha1.LabelAZ: targetHvAZ,
},
}
// Add hierarchical location information to the virtual machine spec.
vm.Spec.Region = targetHvRegion
vm.Spec.AZ = targetHvAZ
- vm.Spec.TargetHost = &vmv1alpha1.HostRef{
- APIVersion: hv1.GroupVersion.String(),
- Kind: "Hypervisor",
- Name: targetHost,
- }
// Derive the hypervisor type from the flavor extra spec hypervisor type.
if hvType, ok := request.Spec.Data.Flavor.
@@ -157,31 +151,6 @@ func (h *VirtualMachineHandler) Upsert(ctx context.Context, result compute.Place
vm.Spec.Cortex.Explanation = explanation
vm.Spec.Cortex.Intent = string(intent)
- // Write the ConditionTypeScheduled condition to indicate that the VM has
- // been successfully scheduled. For that we need to extract the scheduling
- // intent from request and verify that the operation
- var reason string
- switch intent {
- case api.LiveMigrationIntent:
- reason = vmv1alpha1.ConditionReasonScheduledLiveMigration
- case api.RebuildIntent:
- reason = vmv1alpha1.ConditionReasonScheduledRebuild
- case api.ResizeIntent:
- reason = vmv1alpha1.ConditionReasonScheduledResize
- case api.EvacuateIntent:
- reason = vmv1alpha1.ConditionReasonScheduledEvacuation
- case api.CreateIntent:
- reason = vmv1alpha1.ConditionReasonScheduledInitialPlacement
- default:
- reason = "Unknown intent: " + string(intent)
- }
- meta.SetStatusCondition(&vm.Status.Conditions, metav1.Condition{
- Type: vmv1alpha1.ConditionTypeScheduled,
- Status: metav1.ConditionTrue,
- Reason: reason,
- Message: "VM has been successfully scheduled by Cortex",
- })
-
// If the vm resource doesn't exist yet, create it.
vmInCluster := &vmv1alpha1.VirtualMachine{}
err := h.Get(ctx, types.NamespacedName{
@@ -189,17 +158,14 @@ func (h *VirtualMachineHandler) Upsert(ctx context.Context, result compute.Place
Name: vm.Name,
}, vmInCluster)
if err != nil && apierrors.IsNotFound(err) {
- // Create ignores the status subresource and writes the server response
- // (with an empty status) back into vm, so snapshot the conditions and
- // restore them before persisting the status.
- conditions := vm.Status.Conditions
+ // First time we see this VM: this intent is its first transition. We
+ // cannot report an observed generation yet (nothing is persisted), so we
+ // leave ObservedGeneration unset. Conditions are owned by the vm-operator
+ // and never written here, so the status subresource is left untouched.
+ vm.Spec.Cortex.LastTransitionTime = new(metav1.Now())
if err := h.Create(ctx, vm); err != nil {
return fmt.Errorf("failed to create VM: %w", err)
}
- vm.Status.Conditions = conditions
- if err := h.Status().Update(ctx, vm); err != nil {
- return fmt.Errorf("failed to update VM status: %w", err)
- }
log.Info("created virtual machine resource", "instanceUUID", vm.Name, "targetHost", targetHost, "intent", intent)
return nil
} else if err != nil {
@@ -207,11 +173,10 @@ func (h *VirtualMachineHandler) Upsert(ctx context.Context, result compute.Place
}
// The vm already exists, so update the existing resource with the new
- // object meta, spec, and status. For map data like the labels, we preserve
+ // object meta and spec. For map data like the labels, we preserve
// existing map entries that are not inside the new object, and only
- // store/override the entries that are present in the new object. For status
- // conditions, we only update the conditions that are present in the new
- // object. In contrast, for the host candidates we completely replace the
+ // store/override the entries that are present in the new object.
+ // In contrast, for the host candidates we completely replace the
// existing list with the new list from the object. For the remaining spec
// fields, we follow the same approach as for labels: preserve existing
// entries that are not present in the new object, and only store/override
@@ -222,28 +187,32 @@ func (h *VirtualMachineHandler) Upsert(ctx context.Context, result compute.Place
maps.Copy(vmInCluster.Labels, vm.Labels)
// Region and AZ are immutable and are only ever written on create, so we
// leave vmInCluster.Spec.Region and vmInCluster.Spec.AZ untouched.
- vmInCluster.Spec.TargetHost = vm.Spec.TargetHost
vmInCluster.Spec.HypervisorType = vm.Spec.HypervisorType
if vmInCluster.Spec.Resources == nil {
vmInCluster.Spec.Resources = map[vmv1alpha1.ResourceName]resource.Quantity{}
}
maps.Copy(vmInCluster.Spec.Resources, vm.Spec.Resources)
+ // Nova does not expose the project's domain; it is populated out of band
+ // (via keystone) and must not be clobbered by a scheduling write.
+ domain := vmInCluster.Spec.Nova.Ownership.Domain
vmInCluster.Spec.Nova = vm.Spec.Nova
+ vmInCluster.Spec.Nova.Ownership.Domain = domain
if vmInCluster.Spec.Cortex == nil {
vmInCluster.Spec.Cortex = &vmv1alpha1.CortexSpec{}
}
vmInCluster.Spec.Cortex.HostCandidates = vm.Spec.Cortex.HostCandidates
vmInCluster.Spec.Cortex.Explanation = vm.Spec.Cortex.Explanation
+ // LastTransitionTime marks a change of intent; only bump it when the intent
+ // actually transitioned, otherwise carry the existing timestamp forward.
+ if vmInCluster.Spec.Cortex.Intent != vm.Spec.Cortex.Intent {
+ vmInCluster.Spec.Cortex.LastTransitionTime = new(metav1.Now())
+ }
vmInCluster.Spec.Cortex.Intent = vm.Spec.Cortex.Intent
+ // Record the generation we observed on the resource we just read.
+ vmInCluster.Spec.Cortex.ObservedGeneration = vmInCluster.Generation
if err := h.Update(ctx, vmInCluster); err != nil {
return fmt.Errorf("failed to update VM: %w", err)
}
- for _, cond := range vm.Status.Conditions {
- meta.SetStatusCondition(&vmInCluster.Status.Conditions, cond)
- }
- if err := h.Status().Update(ctx, vmInCluster); err != nil {
- return fmt.Errorf("failed to update VM status: %w", err)
- }
log.Info("updated virtual machine resource", "instanceUUID", vm.Name, "targetHost", targetHost, "intent", intent)
return nil
}
diff --git a/internal/scheduling/nova/vm_handler_test.go b/internal/scheduling/nova/vm_handler_test.go
index 6c81f6f60..c1bd27b7e 100644
--- a/internal/scheduling/nova/vm_handler_test.go
+++ b/internal/scheduling/nova/vm_handler_test.go
@@ -6,6 +6,7 @@ package nova
import (
"context"
"testing"
+ "time"
api "github.com/cobaltcore-dev/cortex/api/external/nova"
"github.com/cobaltcore-dev/cortex/internal/scheduling/compute"
@@ -57,17 +58,18 @@ func vmTestRequest() api.ExternalSchedulerRequest {
}
func vmTestResult(targetHost *string) compute.PlacementResult {
- return compute.PlacementResult{TargetHost: targetHost}
+ res := compute.PlacementResult{TargetHost: targetHost}
+ if targetHost != nil {
+ res.OrderedHosts = []string{*targetHost}
+ }
+ return res
}
-func vmTestHypervisor(region, az string) *hv1.Hypervisor {
- labels := map[string]string{}
+func vmTestHypervisor(region string) *hv1.Hypervisor {
+ labels := map[string]string{corev1.LabelTopologyZone: "az-1"}
if region != "" {
labels[corev1.LabelTopologyRegion] = region
}
- if az != "" {
- labels[corev1.LabelTopologyZone] = az
- }
return &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "compute-1", Labels: labels}}
}
@@ -76,7 +78,7 @@ func TestVirtualMachineHandlerCreate(t *testing.T) {
fakeClient := fake.NewClientBuilder().
WithScheme(scheme).
WithStatusSubresource(&vmopv1alpha1.VirtualMachine{}).
- WithObjects(vmTestHypervisor("region-1", "az-1")).
+ WithObjects(vmTestHypervisor("region-1")).
Build()
h := &VirtualMachineHandler{Client: fakeClient}
@@ -92,8 +94,8 @@ func TestVirtualMachineHandlerCreate(t *testing.T) {
if vm.Spec.Region != "region-1" || vm.Spec.AZ != "az-1" {
t.Errorf("region/az = %q/%q, want region-1/az-1", vm.Spec.Region, vm.Spec.AZ)
}
- if vm.Spec.TargetHost == nil || vm.Spec.TargetHost.Name != "compute-1" {
- t.Errorf("target host ref = %+v, want name compute-1", vm.Spec.TargetHost)
+ if len(vm.Spec.Cortex.HostCandidates) != 1 || vm.Spec.Cortex.HostCandidates[0].Name != "compute-1" {
+ t.Errorf("host candidates = %+v, want name compute-1", vm.Spec.Cortex.HostCandidates)
}
if cpu := vm.Spec.Resources[vmopv1alpha1.ResourceCPU]; cpu.Value() != 4 {
t.Errorf("cpu = %d, want 4", cpu.Value())
@@ -104,8 +106,20 @@ func TestVirtualMachineHandlerCreate(t *testing.T) {
if vm.Spec.Nova.Ownership.Project != "project-1" || vm.Spec.Nova.Flavor.Name != "m1.large" {
t.Errorf("nova spec = %+v, want project-1 / m1.large", vm.Spec.Nova)
}
- if vm.Labels[vmopv1alpha1.LabelTargetHost] != "compute-1" {
- t.Errorf("target host label = %q, want compute-1", vm.Labels[vmopv1alpha1.LabelTargetHost])
+ // Intent is the VM's first transition, so a timestamp is stamped, but no
+ // generation has been observed yet.
+ if vm.Spec.Cortex.Intent != string(api.CreateIntent) {
+ t.Errorf("intent = %q, want %q", vm.Spec.Cortex.Intent, api.CreateIntent)
+ }
+ if vm.Spec.Cortex.LastTransitionTime == nil {
+ t.Error("expected LastTransitionTime to be stamped on create")
+ }
+ if vm.Spec.Cortex.ObservedGeneration != 0 {
+ t.Errorf("observedGeneration = %d, want 0 on create", vm.Spec.Cortex.ObservedGeneration)
+ }
+ // Conditions are owned by the vm-operator and must not be written here.
+ if len(vm.Status.Conditions) != 0 {
+ t.Errorf("expected no conditions written, got %+v", vm.Status.Conditions)
}
}
@@ -113,19 +127,23 @@ func TestVirtualMachineHandlerUpdate(t *testing.T) {
scheme := vmTestScheme(t)
existing := &vmopv1alpha1.VirtualMachine{
ObjectMeta: metav1.ObjectMeta{
- Name: "vm-uuid-1",
- Labels: map[string]string{vmopv1alpha1.LabelTargetHost: "old-host"},
+ Name: "vm-uuid-1",
},
Spec: vmopv1alpha1.VMSpec{
- Region: "region-1",
- AZ: "az-1",
- TargetHost: &corev1.ObjectReference{Name: "hv-old"},
+ Region: "region-1",
+ AZ: "az-1",
+ Nova: vmopv1alpha1.NovaSpec{
+ // Domain is populated out of band (keystone) and must survive a
+ // scheduling update.
+ Ownership: vmopv1alpha1.Ownership{Domain: "domain-1"},
+ },
+ Cortex: &vmopv1alpha1.CortexSpec{Intent: string(api.CreateIntent)},
},
}
fakeClient := fake.NewClientBuilder().
WithScheme(scheme).
WithStatusSubresource(&vmopv1alpha1.VirtualMachine{}).
- WithObjects(vmTestHypervisor("region-1", "az-1"), existing).
+ WithObjects(vmTestHypervisor("region-1"), existing).
Build()
h := &VirtualMachineHandler{Client: fakeClient}
@@ -138,16 +156,63 @@ func TestVirtualMachineHandlerUpdate(t *testing.T) {
if err := fakeClient.Get(context.Background(), client.ObjectKey{Name: "vm-uuid-1"}, vm); err != nil {
t.Fatalf("get vm: %v", err)
}
- if vm.Spec.TargetHost == nil || vm.Spec.TargetHost.Name != "compute-1" {
- t.Errorf("target host ref = %+v, want name compute-1", vm.Spec.TargetHost)
- }
- if vm.Labels[vmopv1alpha1.LabelTargetHost] != "compute-1" {
- t.Errorf("target host label = %q, want compute-1", vm.Labels[vmopv1alpha1.LabelTargetHost])
+ if len(vm.Spec.Cortex.HostCandidates) != 1 || vm.Spec.Cortex.HostCandidates[0].Name != "compute-1" {
+ t.Errorf("host candidates = %+v, want name compute-1", vm.Spec.Cortex.HostCandidates)
}
// Immutable fields must remain unchanged.
if vm.Spec.Region != "region-1" || vm.Spec.AZ != "az-1" {
t.Errorf("immutable region/az changed: %q/%q", vm.Spec.Region, vm.Spec.AZ)
}
+ // The out-of-band domain must be preserved across the Nova spec overwrite.
+ if vm.Spec.Nova.Ownership.Domain != "domain-1" {
+ t.Errorf("domain = %q, want domain-1 preserved", vm.Spec.Nova.Ownership.Domain)
+ }
+}
+
+func TestVirtualMachineHandlerUpdateBumpsOnIntentChange(t *testing.T) {
+ scheme := vmTestScheme(t)
+ old := metav1.NewTime(metav1.Now().Add(-time.Hour).Truncate(time.Second))
+ existing := &vmopv1alpha1.VirtualMachine{
+ ObjectMeta: metav1.ObjectMeta{Name: "vm-uuid-1"},
+ Spec: vmopv1alpha1.VMSpec{
+ Region: "region-1",
+ AZ: "az-1",
+ Cortex: &vmopv1alpha1.CortexSpec{
+ Intent: string(api.CreateIntent),
+ LastTransitionTime: &old,
+ },
+ },
+ }
+ fakeClient := fake.NewClientBuilder().
+ WithScheme(scheme).
+ WithStatusSubresource(&vmopv1alpha1.VirtualMachine{}).
+ WithObjects(vmTestHypervisor("region-1"), existing).
+ Build()
+ h := &VirtualMachineHandler{Client: fakeClient}
+
+ targetHost := "compute-1"
+ // Same intent: the timestamp must be carried forward unchanged.
+ if err := h.Upsert(context.Background(), vmTestResult(&targetHost), vmTestRequest(), api.CreateIntent); err != nil {
+ t.Fatalf("Upsert same intent: %v", err)
+ }
+ vm := &vmopv1alpha1.VirtualMachine{}
+ if err := fakeClient.Get(context.Background(), client.ObjectKey{Name: "vm-uuid-1"}, vm); err != nil {
+ t.Fatalf("get vm: %v", err)
+ }
+ if !vm.Spec.Cortex.LastTransitionTime.Equal(&old) {
+ t.Errorf("LastTransitionTime = %v, want preserved %v on unchanged intent", vm.Spec.Cortex.LastTransitionTime, old)
+ }
+
+ // Changed intent: the timestamp must be bumped.
+ if err := h.Upsert(context.Background(), vmTestResult(&targetHost), vmTestRequest(), api.LiveMigrationIntent); err != nil {
+ t.Fatalf("Upsert new intent: %v", err)
+ }
+ if err := fakeClient.Get(context.Background(), client.ObjectKey{Name: "vm-uuid-1"}, vm); err != nil {
+ t.Fatalf("get vm: %v", err)
+ }
+ if vm.Spec.Cortex.LastTransitionTime.Equal(&old) {
+ t.Error("expected LastTransitionTime to be bumped on intent change")
+ }
}
func TestVirtualMachineHandlerNoTargetHost(t *testing.T) {
@@ -171,7 +236,7 @@ func TestVirtualMachineHandlerMissingRegion(t *testing.T) {
scheme := vmTestScheme(t)
fakeClient := fake.NewClientBuilder().
WithScheme(scheme).
- WithObjects(vmTestHypervisor("", "az-1")). // no region label
+ WithObjects(vmTestHypervisor("")). // no region label
Build()
h := &VirtualMachineHandler{Client: fakeClient}
diff --git a/internal/scheduling/reservations/inflight/controller_test.go b/internal/scheduling/reservations/inflight/controller_test.go
index a185aa6f8..709a85ef0 100644
--- a/internal/scheduling/reservations/inflight/controller_test.go
+++ b/internal/scheduling/reservations/inflight/controller_test.go
@@ -247,7 +247,7 @@ func TestReconcile(t *testing.T) {
name: "non-create: live migration ongoing is kept",
objects: []client.Object{
newInFlightReservation("res-1", "vm-uuid-1", "host-1", inFlightOpts{intent: novaapi.LiveMigrationIntent, resources: cpu4}),
- newVM("vm-uuid-1", "host-1", cond(vmv1alpha1.ConditionTypeMigrating, metav1.ConditionTrue, vmv1alpha1.ConditionReasonMigratingOngoing)),
+ newVM("vm-uuid-1", "host-1", cond(vmv1alpha1.ConditionTypeMigrating, metav1.ConditionTrue, vmv1alpha1.ConditionReasonMigratingLiveInProgress)),
},
wantRequeue: requeueInterval,
wantKept: []string{"res-1"},
@@ -261,7 +261,7 @@ func TestReconcile(t *testing.T) {
objects: []client.Object{
newInFlightReservation("a", "vm-uuid-1", "host-1", createA),
newInFlightReservation("b", "vm-uuid-1", "host-2", createA),
- newVM("vm-uuid-1", "host-1", cond(vmv1alpha1.ConditionTypeRunning, metav1.ConditionTrue, "Running")),
+ newVM("vm-uuid-1", "host-1", cond(vmv1alpha1.ConditionTypeRunning, metav1.ConditionTrue, vmv1alpha1.ConditionReasonRunningOK)),
},
reconcile: "a",
wantDeleted: []string{"a", "b"},
@@ -296,79 +296,71 @@ func TestReconcile(t *testing.T) {
wantDeleted: []string{"res-1"},
},
{
- // UC2: an ongoing resize keeps the hold and surfaces the Resizing reason.
- name: "resize: ongoing requeues",
+ // UC2: an in-progress resize keeps the hold and surfaces the Resizing reason.
+ name: "resize: in progress requeues",
objects: []client.Object{
newInFlightReservation("res-1", "vm-uuid-1", "host-1", inFlightOpts{intent: novaapi.ResizeIntent}),
- newVM("vm-uuid-1", "host-1", cond(vmv1alpha1.ConditionTypeResizing, metav1.ConditionTrue, vmv1alpha1.ConditionReasonResizingOngoing)),
+ newVM("vm-uuid-1", "host-1", cond(vmv1alpha1.ConditionTypeResizing, metav1.ConditionTrue, vmv1alpha1.ConditionReasonResizingInProgress)),
},
wantRequeue: requeueInterval,
wantKept: []string{"res-1"},
wantReadyStatus: metav1.ConditionUnknown, wantReadyReason: "Resizing",
},
{
- // UC2: a terminal Resizing reason that transitioned before the hold was
- // created is stale (previous operation) and must not release it.
- name: "resize: stale terminal condition is kept",
+ // UC2: a settled Resizing condition that transitioned before the hold
+ // was created is stale (idle VM / previous operation) and must not
+ // release it.
+ name: "resize: stale settled condition is kept",
objects: []client.Object{
newInFlightReservation("res-1", "vm-uuid-1", "host-1", inFlightOpts{intent: novaapi.ResizeIntent, createdAt: &resCreated}),
- vmWithCond("vm-uuid-1", vmv1alpha1.ConditionTypeResizing, vmv1alpha1.ConditionReasonResizingCompleted, stalePast),
+ vmWithCond("vm-uuid-1", vmv1alpha1.ConditionTypeResizing, vmv1alpha1.ConditionReasonResizingInProgress, stalePast),
},
wantRequeue: requeueInterval,
wantKept: []string{"res-1"},
wantReadyStatus: metav1.ConditionUnknown, wantReadyReason: "Resizing",
},
{
- // UC3: an ongoing live migration keeps both holds.
- name: "live migration: ongoing requeues",
+ // UC3: an in-progress live migration keeps both holds.
+ name: "live migration: in progress requeues",
objects: []client.Object{
newInFlightReservation("res-1", "vm-uuid-1", "host-1", inFlightOpts{intent: novaapi.LiveMigrationIntent}),
- newVM("vm-uuid-1", "host-1", cond(vmv1alpha1.ConditionTypeMigrating, metav1.ConditionTrue, vmv1alpha1.ConditionReasonMigratingOngoing)),
+ newVM("vm-uuid-1", "host-1", cond(vmv1alpha1.ConditionTypeMigrating, metav1.ConditionTrue, vmv1alpha1.ConditionReasonMigratingLiveInProgress)),
},
wantRequeue: requeueInterval,
wantKept: []string{"res-1"},
wantReadyStatus: metav1.ConditionUnknown, wantReadyReason: "Migrating",
},
{
- // UC3: a fresh terminal migration releases both target and source holds
+ // UC3: a fresh settled migration releases both target and source holds
// (the whole operation group) through Reconcile.
- name: "live migration: fresh terminal releases target and source group",
+ name: "live migration: fresh settled releases target and source group",
objects: []client.Object{
newInFlightReservation("target", "vm-uuid-1", "host-2", inFlightOpts{operationID: "op-m", intent: novaapi.LiveMigrationIntent, createdAt: &resCreated}),
newInFlightReservation("source", "vm-uuid-1", "host-1", inFlightOpts{operationID: "op-m", intent: novaapi.LiveMigrationIntent, createdAt: &resCreated}),
- vmWithCond("vm-uuid-1", vmv1alpha1.ConditionTypeMigrating, vmv1alpha1.ConditionReasonMigratingCompleted, freshFuture),
+ vmWithCond("vm-uuid-1", vmv1alpha1.ConditionTypeMigrating, vmv1alpha1.ConditionReasonMigratingLiveInProgress, freshFuture),
},
reconcile: "source",
wantDeleted: []string{"target", "source"},
},
}
- // Every terminal resize reason, when fresh, releases the hold through Reconcile.
- for _, r := range []string{
- vmv1alpha1.ConditionReasonResizingCompleted,
- vmv1alpha1.ConditionReasonResizingReverted,
- vmv1alpha1.ConditionReasonResizingFailed,
- } {
- tests = append(tests, testCase{
- name: "resize: fresh terminal " + r + " deletes reservation",
- objects: []client.Object{
- newInFlightReservation("res-1", "vm-uuid-1", "host-1", inFlightOpts{intent: novaapi.ResizeIntent, createdAt: &resCreated}),
- vmWithCond("vm-uuid-1", vmv1alpha1.ConditionTypeResizing, r, freshFuture),
- },
- wantDeleted: []string{"res-1"},
- })
- }
- // Every terminal live-migration reason, when fresh, releases the hold.
- for _, r := range []string{
- vmv1alpha1.ConditionReasonMigratingCompleted,
- vmv1alpha1.ConditionReasonMigratingRolledBack,
- vmv1alpha1.ConditionReasonMigratingAborting,
- vmv1alpha1.ConditionReasonMigratingFailed,
+ // A fresh settled (status=False) operation condition releases the hold
+ // through Reconcile, for every operation that maps onto the Resizing or
+ // Migrating condition.
+ for _, op := range []struct {
+ name string
+ intent v1alpha1.SchedulingIntent
+ condType string
+ reason string
+ }{
+ {"resize", novaapi.ResizeIntent, vmv1alpha1.ConditionTypeResizing, vmv1alpha1.ConditionReasonResizingInProgress},
+ {"live migration", novaapi.LiveMigrationIntent, vmv1alpha1.ConditionTypeMigrating, vmv1alpha1.ConditionReasonMigratingLiveInProgress},
+ {"evacuate", novaapi.EvacuateIntent, vmv1alpha1.ConditionTypeMigrating, vmv1alpha1.ConditionReasonMigratingColdInProgress},
} {
tests = append(tests, testCase{
- name: "live migration: fresh terminal " + r + " deletes reservation",
+ name: op.name + ": fresh settled condition deletes reservation",
objects: []client.Object{
- newInFlightReservation("res-1", "vm-uuid-1", "host-1", inFlightOpts{intent: novaapi.LiveMigrationIntent, createdAt: &resCreated}),
- vmWithCond("vm-uuid-1", vmv1alpha1.ConditionTypeMigrating, r, freshFuture),
+ newInFlightReservation("res-1", "vm-uuid-1", "host-1", inFlightOpts{intent: op.intent, createdAt: &resCreated}),
+ vmWithCond("vm-uuid-1", op.condType, op.reason, freshFuture),
},
wantDeleted: []string{"res-1"},
})
diff --git a/internal/scheduling/reservations/inflight/vm_owner_handler.go b/internal/scheduling/reservations/inflight/vm_owner_handler.go
index 036990315..4cd53b9ff 100644
--- a/internal/scheduling/reservations/inflight/vm_owner_handler.go
+++ b/internal/scheduling/reservations/inflight/vm_owner_handler.go
@@ -5,7 +5,6 @@ package inflight
import (
"context"
- "slices"
"time"
novaapi "github.com/cobaltcore-dev/cortex/api/external/nova"
@@ -252,11 +251,6 @@ func (h *VirtualMachineHandler) terminalForIntent(res *v1alpha1.Reservation, vm
// live (host assigned and the domain running / Nova ACTIVE), or on an
// explicit Nova failure (Nova ERROR). While the VM is still building,
// keep blocking all candidates.
- //
- // Scheduled=False is intentionally NOT used as a release trigger: it is
- // not written on failure today, and it reflects the latest scheduling
- // result for the VM rather than this operation, so it cannot be
- // correlated to an OperationID.
placed := vm.Status.Host != nil && vm.Status.Host.Name != ""
running := meta.IsStatusConditionTrue(vm.Status.Conditions, vmv1alpha1.ConditionTypeRunning)
novaActive := vm.Status.Nova != nil && vm.Status.Nova.Status == novaStatusActive
@@ -269,7 +263,7 @@ func (h *VirtualMachineHandler) terminalForIntent(res *v1alpha1.Reservation, vm
}
// We have at least some VM-CRD signal but it is not yet decisive; keep
// waiting for a confirmed running/ACTIVE (or ERROR) signal.
- if placed || meta.FindStatusCondition(vm.Status.Conditions, vmv1alpha1.ConditionTypeScheduled) != nil || vm.Status.Nova != nil {
+ if placed || vm.Status.Nova != nil {
return TerminalResult{RequeueAfter: requeueInterval, Reason: reasonAwaitingPlacement,
Message: "VM not yet confirmed running on a host"}, true
}
@@ -292,60 +286,59 @@ func (h *VirtualMachineHandler) terminalForIntent(res *v1alpha1.Reservation, vm
Message: "unmanaged-source resize not yet confirmed running on target"}, true
}
// KVM-source resize: release when the resize is no longer in progress.
- return terminalByReason(res.CreationTimestamp, vm.Status.Conditions, vmv1alpha1.ConditionTypeResizing,
- reasonResizing, "resize in progress",
- vmv1alpha1.ConditionReasonResizingCompleted,
- vmv1alpha1.ConditionReasonResizingReverted,
- vmv1alpha1.ConditionReasonResizingFailed)
+ return terminalBySettledStatus(res.CreationTimestamp, vm.Status.Conditions, vmv1alpha1.ConditionTypeResizing,
+ reasonResizing, "resize in progress")
- case novaapi.LiveMigrationIntent:
- // UC3 live migration: release both the forward (target) and rollback
- // (source) reservations once the migration settles, including an abort.
- return terminalByReason(res.CreationTimestamp, vm.Status.Conditions, vmv1alpha1.ConditionTypeMigrating,
- reasonMigrating, "live migration in progress",
- vmv1alpha1.ConditionReasonMigratingCompleted,
- vmv1alpha1.ConditionReasonMigratingRolledBack,
- vmv1alpha1.ConditionReasonMigratingAborting,
- vmv1alpha1.ConditionReasonMigratingFailed)
+ case novaapi.LiveMigrationIntent, novaapi.EvacuateIntent:
+ // UC3 live migration and evacuation: the vm-operator's Migrating
+ // condition now covers both live and cold migrations (an evacuation is a
+ // cold migration). Release both the forward (target) and rollback
+ // (source) reservations once the migration is no longer in progress.
+ return terminalBySettledStatus(res.CreationTimestamp, vm.Status.Conditions, vmv1alpha1.ConditionTypeMigrating,
+ reasonMigrating, "migration in progress")
default:
- // Other intents (rebuild, evacuate, ...) have no dedicated condition
- // yet; keep the hold pending a future condition (the optional TTL
- // backstop only acts if an EndTime is set).
+ // Other intents (e.g. rebuild) have no dedicated condition yet; keep the
+ // hold pending a future condition (the optional TTL backstop only acts if
+ // an EndTime is set).
return TerminalResult{}, false
}
}
-// terminalByReason returns (result, ok) for a condition whose terminal reasons
-// are listed. ok=false when the condition is absent (caller keeps waiting).
+// terminalBySettledStatus returns (result, ok) for an operation-progress
+// condition (Migrating, Resizing) under the vm-operator's current semantics:
//
-// A terminal reason is only trusted when it is *fresh*: the condition's
-// LastTransitionTime must be at or after createdAt (the reservation's creation
-// time). Nova serializes its own operations per VM, so when this VM-owned
-// reservation was created no other Nova operation on the VM was in flight; a
-// terminal condition that transitioned *before* this reservation existed
-// therefore describes a previous operation and must not release the freshly
-// created holds. In that stale case we keep waiting for the vm-operator to
-// transition the condition for the current operation.
+// - status=True: the operation is in progress — keep waiting.
+// - status=Unknown: authoritative datasources disagree — keep waiting.
+// - status=False: the operation is not in progress. This status also covers a
+// VM that was never migrated/resized, so it is only trusted as a completion
+// signal when it is *fresh*: the condition's LastTransitionTime must be at or
+// after createdAt (the reservation's creation time). Nova serializes its own
+// operations per VM, so when this VM-owned reservation was created no other
+// Nova operation on the VM was in flight; a False that transitioned *before*
+// this reservation existed therefore describes an idle VM (or a previous
+// operation) and must not release the freshly created holds.
//
-// When Done is false the result carries the waiting reason/message.
-func terminalByReason(createdAt metav1.Time, conditions []metav1.Condition, condType, waitReason, waitMessage string, terminalReasons ...string) (TerminalResult, bool) {
+// ok=false when the condition is absent, so the caller keeps waiting for the
+// vm-operator to record the operation. When Done is false the result carries the
+// waiting reason/message.
+func terminalBySettledStatus(createdAt metav1.Time, conditions []metav1.Condition, condType, waitReason, waitMessage string) (TerminalResult, bool) {
cond := meta.FindStatusCondition(conditions, condType)
if cond == nil {
return TerminalResult{}, false
}
- if slices.Contains(terminalReasons, cond.Reason) && !conditionIsStale(cond, createdAt) {
+ if cond.Status == metav1.ConditionFalse && !conditionIsStale(cond, createdAt) {
return TerminalResult{Done: true}, true
}
return TerminalResult{RequeueAfter: requeueInterval, Reason: waitReason, Message: waitMessage}, true
}
// conditionIsStale reports whether cond transitioned before the reservation was
-// created, i.e. the condition reflects a previous operation rather than the one
-// this reservation belongs to. A zero LastTransitionTime is treated as stale:
-// without a trustworthy transition time we cannot attribute the terminal signal
-// to the current operation, so we conservatively keep waiting rather than risk
-// releasing holds early.
+// created, i.e. the condition reflects a previous operation (or a long-idle VM)
+// rather than the one this reservation belongs to. A zero LastTransitionTime is
+// treated as stale: without a trustworthy transition time we cannot attribute
+// the signal to the current operation, so we conservatively keep waiting rather
+// than risk releasing holds early.
func conditionIsStale(cond *metav1.Condition, createdAt metav1.Time) bool {
if cond.LastTransitionTime.IsZero() {
return true
diff --git a/internal/scheduling/reservations/inflight/vm_owner_handler_test.go b/internal/scheduling/reservations/inflight/vm_owner_handler_test.go
index 23d7d679e..473374f97 100644
--- a/internal/scheduling/reservations/inflight/vm_owner_handler_test.go
+++ b/internal/scheduling/reservations/inflight/vm_owner_handler_test.go
@@ -128,19 +128,26 @@ func TestVMHandler_TerminalState(t *testing.T) {
tests := []tc{
// UC1 initial placement.
{name: "create: building -> wait", intent: novaapi.CreateIntent,
- vm: newVM("vm-1", "host-9", cond(vmv1alpha1.ConditionTypeScheduled, metav1.ConditionUnknown, "InitialPlacement"))},
+ vm: newVM("vm-1", "host-9")},
{name: "create: Running=True -> done", intent: novaapi.CreateIntent, wantDone: true,
- vm: newVM("vm-1", "host-9", cond(vmv1alpha1.ConditionTypeRunning, metav1.ConditionTrue, "Running"))},
+ vm: newVM("vm-1", "host-9", cond(vmv1alpha1.ConditionTypeRunning, metav1.ConditionTrue, vmv1alpha1.ConditionReasonRunningOK))},
{name: "create: nova ACTIVE -> done", intent: novaapi.CreateIntent, wantDone: true, vm: withNovaStatus(newVM("vm-1", "host-9"), "ACTIVE")},
{name: "create: no live signal -> wait", intent: novaapi.CreateIntent, vm: withNovaStatus(newVM("vm-1", "host-9"), "BUILD")},
- {name: "create: scheduled false alone -> wait", intent: novaapi.CreateIntent,
- vm: newVM("vm-1", "", cond(vmv1alpha1.ConditionTypeScheduled, metav1.ConditionFalse, "NoHostFound"))},
+ {name: "create: placed but no live signal -> wait", intent: novaapi.CreateIntent,
+ vm: newVM("vm-1", "host-9", cond(vmv1alpha1.ConditionTypeRunning, metav1.ConditionFalse, vmv1alpha1.ConditionReasonRunningShutoff))},
{name: "create: nova error -> done", intent: novaapi.CreateIntent, wantDone: true, vm: withNovaStatus(newVM("vm-1", ""), "ERROR")},
- // UC2 resize / UC3 live migration ongoing.
- {name: "resize: ongoing -> wait", intent: novaapi.ResizeIntent,
- vm: newVM("vm-1", "host-1", cond(vmv1alpha1.ConditionTypeResizing, metav1.ConditionTrue, vmv1alpha1.ConditionReasonResizingOngoing))},
- {name: "live migration: ongoing -> wait", intent: novaapi.LiveMigrationIntent,
- vm: newVM("vm-1", "host-1", cond(vmv1alpha1.ConditionTypeMigrating, metav1.ConditionTrue, vmv1alpha1.ConditionReasonMigratingOngoing))},
+ // UC2 resize / UC3 live migration in progress (status=True).
+ {name: "resize: in progress -> wait", intent: novaapi.ResizeIntent,
+ vm: newVM("vm-1", "host-1", cond(vmv1alpha1.ConditionTypeResizing, metav1.ConditionTrue, vmv1alpha1.ConditionReasonResizingInProgress))},
+ {name: "live migration: in progress -> wait", intent: novaapi.LiveMigrationIntent,
+ vm: newVM("vm-1", "host-1", cond(vmv1alpha1.ConditionTypeMigrating, metav1.ConditionTrue, vmv1alpha1.ConditionReasonMigratingLiveInProgress))},
+ {name: "evacuate: in progress -> wait", intent: novaapi.EvacuateIntent,
+ vm: newVM("vm-1", "host-1", cond(vmv1alpha1.ConditionTypeMigrating, metav1.ConditionTrue, vmv1alpha1.ConditionReasonMigratingColdInProgress))},
+ // Unknown status reflects conflicting observations -> keep waiting.
+ {name: "resize: unknown -> wait", intent: novaapi.ResizeIntent,
+ vm: newVM("vm-1", "host-1", cond(vmv1alpha1.ConditionTypeResizing, metav1.ConditionUnknown, ""))},
+ {name: "live migration: unknown -> wait", intent: novaapi.LiveMigrationIntent,
+ vm: newVM("vm-1", "host-1", cond(vmv1alpha1.ConditionTypeMigrating, metav1.ConditionUnknown, ""))},
// No VM CRD -> no terminal signal, keep waiting for the VM CRD
// (regardless of intent).
{name: "no VM CRD: create -> wait", intent: novaapi.CreateIntent},
@@ -153,7 +160,7 @@ func TestVMHandler_TerminalState(t *testing.T) {
unmanaged := map[string]string{LabelInFlightSource: LabelInFlightSourceUnmanaged}
tests = append(tests,
tc{name: "unmanaged resize: landed and running -> done", intent: novaapi.ResizeIntent, labels: unmanaged, wantDone: true,
- vm: newVM("vm-1", "host-1", cond(vmv1alpha1.ConditionTypeRunning, metav1.ConditionTrue, "Running"))},
+ vm: newVM("vm-1", "host-1", cond(vmv1alpha1.ConditionTypeRunning, metav1.ConditionTrue, vmv1alpha1.ConditionReasonRunningOK))},
tc{name: "unmanaged resize: landed and nova ACTIVE -> done", intent: novaapi.ResizeIntent, labels: unmanaged, wantDone: true,
vm: withNovaStatus(newVM("vm-1", "host-1"), "ACTIVE")},
tc{name: "unmanaged resize: placed but not running -> wait", intent: novaapi.ResizeIntent, labels: unmanaged,
@@ -161,24 +168,30 @@ func TestVMHandler_TerminalState(t *testing.T) {
tc{name: "unmanaged resize: nova ERROR -> wait", intent: novaapi.ResizeIntent, labels: unmanaged,
vm: withNovaStatus(newVM("vm-1", "host-1"), "ERROR")},
)
- for _, r := range []string{vmv1alpha1.ConditionReasonResizingCompleted, vmv1alpha1.ConditionReasonResizingReverted, vmv1alpha1.ConditionReasonResizingFailed} {
- tests = append(tests, tc{name: "resize: terminal " + r, intent: novaapi.ResizeIntent, wantDone: true,
- vm: newVM("vm-1", "host-1", cond(vmv1alpha1.ConditionTypeResizing, metav1.ConditionFalse, r))})
- }
- for _, r := range []string{vmv1alpha1.ConditionReasonMigratingCompleted, vmv1alpha1.ConditionReasonMigratingRolledBack,
- vmv1alpha1.ConditionReasonMigratingAborting, vmv1alpha1.ConditionReasonMigratingFailed} {
- tests = append(tests, tc{name: "live migration: terminal " + r, intent: novaapi.LiveMigrationIntent, wantDone: true,
- vm: newVM("vm-1", "host-1", cond(vmv1alpha1.ConditionTypeMigrating, metav1.ConditionFalse, r))})
+ // A freshly settled operation condition (status=False transitioned at/after
+ // the reservation was created) releases the holds.
+ for _, op := range []struct {
+ name string
+ intent v1alpha1.SchedulingIntent
+ condType, reason string
+ }{
+ {"resize", novaapi.ResizeIntent, vmv1alpha1.ConditionTypeResizing, vmv1alpha1.ConditionReasonResizingInProgress},
+ {"live migration", novaapi.LiveMigrationIntent, vmv1alpha1.ConditionTypeMigrating, vmv1alpha1.ConditionReasonMigratingLiveInProgress},
+ {"evacuate", novaapi.EvacuateIntent, vmv1alpha1.ConditionTypeMigrating, vmv1alpha1.ConditionReasonMigratingColdInProgress},
+ } {
+ tests = append(tests, tc{name: op.name + ": settled -> done", intent: op.intent, wantDone: true,
+ vm: newVM("vm-1", "host-1", cond(op.condType, metav1.ConditionFalse, op.reason))})
}
- // A terminal condition left over from a previous operation (transition before
- // the reservation was created, or unknown) must not release fresh holds.
+ // A settled (status=False) condition left over from a previous operation
+ // (transition before the reservation was created, or with no transition
+ // time) describes an idle VM and must not release fresh holds.
for _, op := range []struct {
name string
intent v1alpha1.SchedulingIntent
condType, reason string
}{
- {"resize", novaapi.ResizeIntent, vmv1alpha1.ConditionTypeResizing, vmv1alpha1.ConditionReasonResizingCompleted},
- {"live_migration", novaapi.LiveMigrationIntent, vmv1alpha1.ConditionTypeMigrating, vmv1alpha1.ConditionReasonMigratingCompleted},
+ {"resize", novaapi.ResizeIntent, vmv1alpha1.ConditionTypeResizing, vmv1alpha1.ConditionReasonResizingInProgress},
+ {"live_migration", novaapi.LiveMigrationIntent, vmv1alpha1.ConditionTypeMigrating, vmv1alpha1.ConditionReasonMigratingLiveInProgress},
} {
tests = append(tests,
tc{name: "stale " + op.name + " -> wait", intent: op.intent, created: &created, vm: withCond(op.condType, op.reason, stale)},
From 5cfd549fcd34f601f6c203d25f635ae9073861a4 Mon Sep 17 00:00:00 2001
From: Philipp Matthes
Date: Tue, 17 Nov 2026 13:46:50 +0100
Subject: [PATCH 2/2] Adopt renamed cortex subspec fields and scheduling error
Bump vm-operator to pick up the renamed CortexSpec fields (all now
prefixed with "scheduling"/"scheduled") and the new SchedulingError:
- HostCandidates -> SchedulingCandidates
- Explanation -> SchedulingExplanation
- Intent -> SchedulingIntent
- LastTransitionTime -> SchedulingTime
- ObservedGeneration -> ScheduledGeneration
- new SchedulingError (empty on success; cleared on a successful update)
Update the vm_handler, its tests, the CRD pin in the Tiltfile, and the
docs to match.
Signed-off-by: Philipp Matthes
---
Tiltfile | 2 +-
.../02-nova-smart-scheduling.md | 13 +++---
.../01-multicluster-client.md | 4 +-
go.mod | 2 +-
go.sum | 2 +
internal/scheduling/nova/vm_handler.go | 33 ++++++++-------
internal/scheduling/nova/vm_handler_test.go | 40 ++++++++++---------
7 files changed, 53 insertions(+), 43 deletions(-)
diff --git a/Tiltfile b/Tiltfile
index d0ebeb509..b11622ad7 100644
--- a/Tiltfile
+++ b/Tiltfile
@@ -135,7 +135,7 @@ local('kubectl wait --namespace cert-manager --for=condition=available deploymen
# Pinned SHAs must match the versions in go.mod.
url = 'https://raw.githubusercontent.com/cobaltcore-dev/openstack-hypervisor-operator/df757ad112ea027f47c198266cb3a323d52ee2fe/charts/openstack-hypervisor-operator/crds/kvm.cloud.sap_hypervisors.yaml'
local('curl -L ' + url + ' | kubectl apply -f -')
-url = 'https://raw.githubusercontent.com/cobaltcore-dev/vm-operator/f859ba2eb3bf/dist/files/crds/cobaltcore.cloud_virtualmachines.yaml'
+url = 'https://raw.githubusercontent.com/cobaltcore-dev/vm-operator/4876b6fc19aa/dist/files/crds/cobaltcore.cloud_virtualmachines.yaml'
local('curl -L ' + url + ' | kubectl apply -f -')
########### Cortex Manager & CRDs
diff --git a/docs/02-external-scheduler-api/02-nova-smart-scheduling.md b/docs/02-external-scheduler-api/02-nova-smart-scheduling.md
index dbf142d53..c36e203c7 100644
--- a/docs/02-external-scheduler-api/02-nova-smart-scheduling.md
+++ b/docs/02-external-scheduler-api/02-nova-smart-scheduling.md
@@ -153,14 +153,15 @@ resource (the vm-operator CRD, cluster-scoped) so operators can inspect it direc
```bash
kubectl get virtualmachines -o custom-columns=\
NAME:.metadata.name,\
-EXPLANATION:.spec.cortex.explanation,\
-INTENT:.spec.cortex.intent,\
-HOST:.spec.cortex.hostCandidates[0].name
+EXPLANATION:.spec.cortex.schedulingExplanation,\
+INTENT:.spec.cortex.schedulingIntent,\
+HOST:.spec.cortex.schedulingCandidates[0].name
```
-`.spec.cortex.explanation` carries a human-readable summary of why the chosen host won,
-`.spec.cortex.intent` records the scheduling intent (e.g. initial placement or evacuation), and
-`.spec.cortex.hostCandidates` lists the candidate hosts in ranked order; `-o yaml` on a single
+`.spec.cortex.schedulingExplanation` carries a human-readable summary of why the chosen host won,
+`.spec.cortex.schedulingIntent` records the scheduling intent (e.g. initial placement or evacuation), and
+`.spec.cortex.schedulingCandidates` lists the candidate hosts in ranked order (`.spec.cortex.schedulingError`
+is set instead when scheduling failed); `-o yaml` on a single
VM shows the full detail. Cortex only ever writes this `spec.cortex` subspec (plus the immutable
spec and labels on create); the VM's `status.conditions` are owned exclusively by the vm-operator
and reflect its own observations (Running, Migrating, Resizing, …), not the scheduling decision.
diff --git a/docs/06-cortex-library/01-multicluster-client.md b/docs/06-cortex-library/01-multicluster-client.md
index 66f123f1b..cb24cffa1 100644
--- a/docs/06-cortex-library/01-multicluster-client.md
+++ b/docs/06-cortex-library/01-multicluster-client.md
@@ -630,8 +630,8 @@ kubectl --context kind-cortex-remote-az-b get virtualmachines 2>/dev/null || ech
```
You should see the `VirtualMachine` named `cortex-test-instance-001` only in `kind-cortex-remote-az-b`.
-Describe it to see the full scheduling result — the ranked `spec.cortex.hostCandidates` (the first entry is
-the selected host), and `spec.cortex.explanation`, the human-readable summary of why that host
+Describe it to see the full scheduling result — the ranked `spec.cortex.schedulingCandidates` (the first entry is
+the selected host), and `spec.cortex.schedulingExplanation`, the human-readable summary of why that host
won:
```bash
diff --git a/go.mod b/go.mod
index 199de503f..c9478f6b5 100644
--- a/go.mod
+++ b/go.mod
@@ -4,7 +4,7 @@ go 1.27
require (
github.com/cobaltcore-dev/openstack-hypervisor-operator v1.4.0
- github.com/cobaltcore-dev/vm-operator v0.2.11-0.20261117084340-f859ba2eb3bf
+ github.com/cobaltcore-dev/vm-operator v0.2.11-0.20261117124215-4876b6fc19aa
github.com/go-gorp/gorp v2.2.0+incompatible
github.com/go-logr/logr v1.4.4
github.com/google/uuid v1.6.0
diff --git a/go.sum b/go.sum
index e8aa5bfef..efec11424 100644
--- a/go.sum
+++ b/go.sum
@@ -28,6 +28,8 @@ github.com/cobaltcore-dev/vm-operator v0.2.10 h1:W/cNZBDva2pq9kn4QYTcfn4VPxyNEoY
github.com/cobaltcore-dev/vm-operator v0.2.10/go.mod h1:0Lanmhb7onfOIHX+4h1lGLC7vga/nNYabJ6a/y/APk4=
github.com/cobaltcore-dev/vm-operator v0.2.11-0.20261117084340-f859ba2eb3bf h1:12SpyudWEoEB8QZ4OGS/R6iYYoY+BwjlDUc8Xf6WFVk=
github.com/cobaltcore-dev/vm-operator v0.2.11-0.20261117084340-f859ba2eb3bf/go.mod h1:0Lanmhb7onfOIHX+4h1lGLC7vga/nNYabJ6a/y/APk4=
+github.com/cobaltcore-dev/vm-operator v0.2.11-0.20261117124215-4876b6fc19aa h1:GoxVGt2o0jbvQtlRfEs9ipVm4KKB+k6FehP0hAS5v+U=
+github.com/cobaltcore-dev/vm-operator v0.2.11-0.20261117124215-4876b6fc19aa/go.mod h1:0Lanmhb7onfOIHX+4h1lGLC7vga/nNYabJ6a/y/APk4=
github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4=
github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
diff --git a/internal/scheduling/nova/vm_handler.go b/internal/scheduling/nova/vm_handler.go
index 3b48bacff..d5134f5a7 100644
--- a/internal/scheduling/nova/vm_handler.go
+++ b/internal/scheduling/nova/vm_handler.go
@@ -139,17 +139,18 @@ func (h *VirtualMachineHandler) Upsert(ctx context.Context, result compute.Place
// Write the cortex spec (our scheduling result). The intent was resolved by
// the API layer (with a graceful fallback to CreateIntent for initial
- // placements) and passed in.
+ // placements) and passed in. We only reach this point with a target host, so
+ // scheduling succeeded and SchedulingError stays empty.
vm.Spec.Cortex = &vmv1alpha1.CortexSpec{}
for _, host := range result.OrderedHosts {
- vm.Spec.Cortex.HostCandidates = append(vm.Spec.Cortex.HostCandidates, vmv1alpha1.HostRef{
+ vm.Spec.Cortex.SchedulingCandidates = append(vm.Spec.Cortex.SchedulingCandidates, vmv1alpha1.HostRef{
APIVersion: hv1.GroupVersion.String(),
Kind: "Hypervisor",
Name: host,
})
}
- vm.Spec.Cortex.Explanation = explanation
- vm.Spec.Cortex.Intent = string(intent)
+ vm.Spec.Cortex.SchedulingExplanation = explanation
+ vm.Spec.Cortex.SchedulingIntent = string(intent)
// If the vm resource doesn't exist yet, create it.
vmInCluster := &vmv1alpha1.VirtualMachine{}
@@ -159,10 +160,10 @@ func (h *VirtualMachineHandler) Upsert(ctx context.Context, result compute.Place
}, vmInCluster)
if err != nil && apierrors.IsNotFound(err) {
// First time we see this VM: this intent is its first transition. We
- // cannot report an observed generation yet (nothing is persisted), so we
- // leave ObservedGeneration unset. Conditions are owned by the vm-operator
+ // cannot report a scheduled generation yet (nothing is persisted), so we
+ // leave ScheduledGeneration unset. Conditions are owned by the vm-operator
// and never written here, so the status subresource is left untouched.
- vm.Spec.Cortex.LastTransitionTime = new(metav1.Now())
+ vm.Spec.Cortex.SchedulingTime = new(metav1.Now())
if err := h.Create(ctx, vm); err != nil {
return fmt.Errorf("failed to create VM: %w", err)
}
@@ -176,7 +177,7 @@ func (h *VirtualMachineHandler) Upsert(ctx context.Context, result compute.Place
// object meta and spec. For map data like the labels, we preserve
// existing map entries that are not inside the new object, and only
// store/override the entries that are present in the new object.
- // In contrast, for the host candidates we completely replace the
+ // In contrast, for the scheduling candidates we completely replace the
// existing list with the new list from the object. For the remaining spec
// fields, we follow the same approach as for labels: preserve existing
// entries that are not present in the new object, and only store/override
@@ -200,16 +201,18 @@ func (h *VirtualMachineHandler) Upsert(ctx context.Context, result compute.Place
if vmInCluster.Spec.Cortex == nil {
vmInCluster.Spec.Cortex = &vmv1alpha1.CortexSpec{}
}
- vmInCluster.Spec.Cortex.HostCandidates = vm.Spec.Cortex.HostCandidates
- vmInCluster.Spec.Cortex.Explanation = vm.Spec.Cortex.Explanation
- // LastTransitionTime marks a change of intent; only bump it when the intent
+ vmInCluster.Spec.Cortex.SchedulingCandidates = vm.Spec.Cortex.SchedulingCandidates
+ vmInCluster.Spec.Cortex.SchedulingExplanation = vm.Spec.Cortex.SchedulingExplanation
+ // This scheduling succeeded, so clear any error left from a previous attempt.
+ vmInCluster.Spec.Cortex.SchedulingError = ""
+ // SchedulingTime marks a change of intent; only bump it when the intent
// actually transitioned, otherwise carry the existing timestamp forward.
- if vmInCluster.Spec.Cortex.Intent != vm.Spec.Cortex.Intent {
- vmInCluster.Spec.Cortex.LastTransitionTime = new(metav1.Now())
+ if vmInCluster.Spec.Cortex.SchedulingIntent != vm.Spec.Cortex.SchedulingIntent {
+ vmInCluster.Spec.Cortex.SchedulingTime = new(metav1.Now())
}
- vmInCluster.Spec.Cortex.Intent = vm.Spec.Cortex.Intent
+ vmInCluster.Spec.Cortex.SchedulingIntent = vm.Spec.Cortex.SchedulingIntent
// Record the generation we observed on the resource we just read.
- vmInCluster.Spec.Cortex.ObservedGeneration = vmInCluster.Generation
+ vmInCluster.Spec.Cortex.ScheduledGeneration = vmInCluster.Generation
if err := h.Update(ctx, vmInCluster); err != nil {
return fmt.Errorf("failed to update VM: %w", err)
}
diff --git a/internal/scheduling/nova/vm_handler_test.go b/internal/scheduling/nova/vm_handler_test.go
index c1bd27b7e..d03147e07 100644
--- a/internal/scheduling/nova/vm_handler_test.go
+++ b/internal/scheduling/nova/vm_handler_test.go
@@ -94,8 +94,8 @@ func TestVirtualMachineHandlerCreate(t *testing.T) {
if vm.Spec.Region != "region-1" || vm.Spec.AZ != "az-1" {
t.Errorf("region/az = %q/%q, want region-1/az-1", vm.Spec.Region, vm.Spec.AZ)
}
- if len(vm.Spec.Cortex.HostCandidates) != 1 || vm.Spec.Cortex.HostCandidates[0].Name != "compute-1" {
- t.Errorf("host candidates = %+v, want name compute-1", vm.Spec.Cortex.HostCandidates)
+ if len(vm.Spec.Cortex.SchedulingCandidates) != 1 || vm.Spec.Cortex.SchedulingCandidates[0].Name != "compute-1" {
+ t.Errorf("scheduling candidates = %+v, want name compute-1", vm.Spec.Cortex.SchedulingCandidates)
}
if cpu := vm.Spec.Resources[vmopv1alpha1.ResourceCPU]; cpu.Value() != 4 {
t.Errorf("cpu = %d, want 4", cpu.Value())
@@ -107,15 +107,19 @@ func TestVirtualMachineHandlerCreate(t *testing.T) {
t.Errorf("nova spec = %+v, want project-1 / m1.large", vm.Spec.Nova)
}
// Intent is the VM's first transition, so a timestamp is stamped, but no
- // generation has been observed yet.
- if vm.Spec.Cortex.Intent != string(api.CreateIntent) {
- t.Errorf("intent = %q, want %q", vm.Spec.Cortex.Intent, api.CreateIntent)
+ // generation has been scheduled yet.
+ if vm.Spec.Cortex.SchedulingIntent != string(api.CreateIntent) {
+ t.Errorf("intent = %q, want %q", vm.Spec.Cortex.SchedulingIntent, api.CreateIntent)
}
- if vm.Spec.Cortex.LastTransitionTime == nil {
- t.Error("expected LastTransitionTime to be stamped on create")
+ if vm.Spec.Cortex.SchedulingTime == nil {
+ t.Error("expected SchedulingTime to be stamped on create")
}
- if vm.Spec.Cortex.ObservedGeneration != 0 {
- t.Errorf("observedGeneration = %d, want 0 on create", vm.Spec.Cortex.ObservedGeneration)
+ if vm.Spec.Cortex.ScheduledGeneration != 0 {
+ t.Errorf("scheduledGeneration = %d, want 0 on create", vm.Spec.Cortex.ScheduledGeneration)
+ }
+ // A successful scheduling records no error.
+ if vm.Spec.Cortex.SchedulingError != "" {
+ t.Errorf("schedulingError = %q, want empty on success", vm.Spec.Cortex.SchedulingError)
}
// Conditions are owned by the vm-operator and must not be written here.
if len(vm.Status.Conditions) != 0 {
@@ -137,7 +141,7 @@ func TestVirtualMachineHandlerUpdate(t *testing.T) {
// scheduling update.
Ownership: vmopv1alpha1.Ownership{Domain: "domain-1"},
},
- Cortex: &vmopv1alpha1.CortexSpec{Intent: string(api.CreateIntent)},
+ Cortex: &vmopv1alpha1.CortexSpec{SchedulingIntent: string(api.CreateIntent)},
},
}
fakeClient := fake.NewClientBuilder().
@@ -156,8 +160,8 @@ func TestVirtualMachineHandlerUpdate(t *testing.T) {
if err := fakeClient.Get(context.Background(), client.ObjectKey{Name: "vm-uuid-1"}, vm); err != nil {
t.Fatalf("get vm: %v", err)
}
- if len(vm.Spec.Cortex.HostCandidates) != 1 || vm.Spec.Cortex.HostCandidates[0].Name != "compute-1" {
- t.Errorf("host candidates = %+v, want name compute-1", vm.Spec.Cortex.HostCandidates)
+ if len(vm.Spec.Cortex.SchedulingCandidates) != 1 || vm.Spec.Cortex.SchedulingCandidates[0].Name != "compute-1" {
+ t.Errorf("scheduling candidates = %+v, want name compute-1", vm.Spec.Cortex.SchedulingCandidates)
}
// Immutable fields must remain unchanged.
if vm.Spec.Region != "region-1" || vm.Spec.AZ != "az-1" {
@@ -178,8 +182,8 @@ func TestVirtualMachineHandlerUpdateBumpsOnIntentChange(t *testing.T) {
Region: "region-1",
AZ: "az-1",
Cortex: &vmopv1alpha1.CortexSpec{
- Intent: string(api.CreateIntent),
- LastTransitionTime: &old,
+ SchedulingIntent: string(api.CreateIntent),
+ SchedulingTime: &old,
},
},
}
@@ -199,8 +203,8 @@ func TestVirtualMachineHandlerUpdateBumpsOnIntentChange(t *testing.T) {
if err := fakeClient.Get(context.Background(), client.ObjectKey{Name: "vm-uuid-1"}, vm); err != nil {
t.Fatalf("get vm: %v", err)
}
- if !vm.Spec.Cortex.LastTransitionTime.Equal(&old) {
- t.Errorf("LastTransitionTime = %v, want preserved %v on unchanged intent", vm.Spec.Cortex.LastTransitionTime, old)
+ if !vm.Spec.Cortex.SchedulingTime.Equal(&old) {
+ t.Errorf("SchedulingTime = %v, want preserved %v on unchanged intent", vm.Spec.Cortex.SchedulingTime, old)
}
// Changed intent: the timestamp must be bumped.
@@ -210,8 +214,8 @@ func TestVirtualMachineHandlerUpdateBumpsOnIntentChange(t *testing.T) {
if err := fakeClient.Get(context.Background(), client.ObjectKey{Name: "vm-uuid-1"}, vm); err != nil {
t.Fatalf("get vm: %v", err)
}
- if vm.Spec.Cortex.LastTransitionTime.Equal(&old) {
- t.Error("expected LastTransitionTime to be bumped on intent change")
+ if vm.Spec.Cortex.SchedulingTime.Equal(&old) {
+ t.Error("expected SchedulingTime to be bumped on intent change")
}
}