diff --git a/Tiltfile b/Tiltfile index 47b4dd672..b11622ad7 100644 --- a/Tiltfile +++ b/Tiltfile @@ -135,7 +135,7 @@ local('kubectl wait --namespace cert-manager --for=condition=available deploymen # Pinned SHAs must match the versions in go.mod. url = 'https://raw.githubusercontent.com/cobaltcore-dev/openstack-hypervisor-operator/df757ad112ea027f47c198266cb3a323d52ee2fe/charts/openstack-hypervisor-operator/crds/kvm.cloud.sap_hypervisors.yaml' local('curl -L ' + url + ' | kubectl apply -f -') -url = 'https://raw.githubusercontent.com/cobaltcore-dev/vm-operator/66e5e459efeb01bfc599a1cda5921528ab47be19/dist/files/crds/cobaltcore.cloud_virtualmachines.yaml' +url = 'https://raw.githubusercontent.com/cobaltcore-dev/vm-operator/4876b6fc19aa/dist/files/crds/cobaltcore.cloud_virtualmachines.yaml' local('curl -L ' + url + ' | kubectl apply -f -') ########### Cortex Manager & CRDs diff --git a/docs/02-external-scheduler-api/02-nova-smart-scheduling.md b/docs/02-external-scheduler-api/02-nova-smart-scheduling.md index adb5f5ec9..c36e203c7 100644 --- a/docs/02-external-scheduler-api/02-nova-smart-scheduling.md +++ b/docs/02-external-scheduler-api/02-nova-smart-scheduling.md @@ -153,15 +153,19 @@ resource (the vm-operator CRD, cluster-scoped) so operators can inspect it direc ```bash kubectl get virtualmachines -o custom-columns=\ NAME:.metadata.name,\ -EXPLANATION:.spec.cortex.explanation,\ -INTENT:.spec.cortex.intent,\ -HOST:.spec.cortex.hostCandidates[0].name +EXPLANATION:.spec.cortex.schedulingExplanation,\ +INTENT:.spec.cortex.schedulingIntent,\ +HOST:.spec.cortex.schedulingCandidates[0].name ``` -`.spec.cortex.explanation` carries a human-readable summary of why the chosen host won, -`.spec.cortex.intent` records the scheduling intent (e.g. initial placement or evacuation), and -`.spec.cortex.hostCandidates` lists the candidate hosts in ranked order; `-o yaml` on a single -VM shows the full detail. The detector pipeline emits **`Descheduling`** resources for VMs it +`.spec.cortex.schedulingExplanation` carries a human-readable summary of why the chosen host won, +`.spec.cortex.schedulingIntent` records the scheduling intent (e.g. initial placement or evacuation), and +`.spec.cortex.schedulingCandidates` lists the candidate hosts in ranked order (`.spec.cortex.schedulingError` +is set instead when scheduling failed); `-o yaml` on a single +VM shows the full detail. Cortex only ever writes this `spec.cortex` subspec (plus the immutable +spec and labels on create); the VM's `status.conditions` are owned exclusively by the vm-operator +and reflect its own observations (Running, Migrating, Resizing, …), not the scheduling decision. +The detector pipeline emits **`Descheduling`** resources for VMs it wants to move: ```bash diff --git a/docs/06-cortex-library/01-multicluster-client.md b/docs/06-cortex-library/01-multicluster-client.md index 9765efe53..cb24cffa1 100644 --- a/docs/06-cortex-library/01-multicluster-client.md +++ b/docs/06-cortex-library/01-multicluster-client.md @@ -630,8 +630,8 @@ kubectl --context kind-cortex-remote-az-b get virtualmachines 2>/dev/null || ech ``` You should see the `VirtualMachine` named `cortex-test-instance-001` only in `kind-cortex-remote-az-b`. -Describe it to see the full scheduling result — the target host under `spec.targetHost`, the ranked -`spec.cortex.hostCandidates`, and `spec.cortex.explanation`, the human-readable summary of why that host +Describe it to see the full scheduling result — the ranked `spec.cortex.schedulingCandidates` (the first entry is +the selected host), and `spec.cortex.schedulingExplanation`, the human-readable summary of why that host won: ```bash diff --git a/go.mod b/go.mod index b874eb5b8..c9478f6b5 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.27 require ( github.com/cobaltcore-dev/openstack-hypervisor-operator v1.4.0 - github.com/cobaltcore-dev/vm-operator v0.2.10 + github.com/cobaltcore-dev/vm-operator v0.2.11-0.20261117124215-4876b6fc19aa github.com/go-gorp/gorp v2.2.0+incompatible github.com/go-logr/logr v1.4.4 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index c3e037bbf..efec11424 100644 --- a/go.sum +++ b/go.sum @@ -26,6 +26,10 @@ github.com/cobaltcore-dev/openstack-hypervisor-operator v1.4.0 h1:lAb+qBhyfgf77Y github.com/cobaltcore-dev/openstack-hypervisor-operator v1.4.0/go.mod h1:VIbkpZpnTeLQrEesadTeANk0MB0t+enfIGfJIX/a2vY= github.com/cobaltcore-dev/vm-operator v0.2.10 h1:W/cNZBDva2pq9kn4QYTcfn4VPxyNEoYqxJBU9hc8Ulg= github.com/cobaltcore-dev/vm-operator v0.2.10/go.mod h1:0Lanmhb7onfOIHX+4h1lGLC7vga/nNYabJ6a/y/APk4= +github.com/cobaltcore-dev/vm-operator v0.2.11-0.20261117084340-f859ba2eb3bf h1:12SpyudWEoEB8QZ4OGS/R6iYYoY+BwjlDUc8Xf6WFVk= +github.com/cobaltcore-dev/vm-operator v0.2.11-0.20261117084340-f859ba2eb3bf/go.mod h1:0Lanmhb7onfOIHX+4h1lGLC7vga/nNYabJ6a/y/APk4= +github.com/cobaltcore-dev/vm-operator v0.2.11-0.20261117124215-4876b6fc19aa h1:GoxVGt2o0jbvQtlRfEs9ipVm4KKB+k6FehP0hAS5v+U= +github.com/cobaltcore-dev/vm-operator v0.2.11-0.20261117124215-4876b6fc19aa/go.mod h1:0Lanmhb7onfOIHX+4h1lGLC7vga/nNYabJ6a/y/APk4= github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= diff --git a/helm/bundles/cortex-nova/templates/rbac.yaml b/helm/bundles/cortex-nova/templates/rbac.yaml index 590b12094..ee8c4c94a 100644 --- a/helm/bundles/cortex-nova/templates/rbac.yaml +++ b/helm/bundles/cortex-nova/templates/rbac.yaml @@ -55,7 +55,6 @@ rules: - cobaltcore.cloud resources: - virtualmachines - - virtualmachines/status verbs: - create - get diff --git a/internal/knowledge/kpis/plugins/compute/failover_partition_graph.go b/internal/knowledge/kpis/plugins/compute/failover_partition_graph.go index 6ab253e17..04951692e 100644 --- a/internal/knowledge/kpis/plugins/compute/failover_partition_graph.go +++ b/internal/knowledge/kpis/plugins/compute/failover_partition_graph.go @@ -17,6 +17,7 @@ import ( vmv1alpha1 "github.com/cobaltcore-dev/vm-operator/api/v1alpha1" "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -120,7 +121,9 @@ func (k *FailoverPartitionGraphKPI) Collect(ch chan<- prometheus.Metric) { } // Count the vms running on each host so the node series can expose it. A vm // Nova hasn't placed yet has no host and is skipped (it also nil-panics on - // Status.Nova otherwise). + // Status.Nova otherwise). The vm-operator retains the last-known Nova fields + // after a server disappears, so skip vms confirmed deleted or scheduled for + // deletion to avoid counting ghosts against a host. vmList := &vmv1alpha1.VirtualMachineList{} if err := k.Client.List(ctx, vmList); err != nil { failoverPartitionGraphKPILogger.Error(err, "Failed to list virtual machines") @@ -131,6 +134,10 @@ func (k *FailoverPartitionGraphKPI) Collect(ch chan<- prometheus.Metric) { if vm.Status.Nova == nil || vm.Status.Nova.Host == "" { continue } + if vm.Status.DeletionTime != nil || + meta.IsStatusConditionTrue(vm.Status.Conditions, vmv1alpha1.ConditionTypeDeleted) { + continue + } vmsByHost[vm.Status.Nova.Host]++ } for _, hv := range hypervisorList.Items { diff --git a/internal/scheduling/nova/vm_handler.go b/internal/scheduling/nova/vm_handler.go index bbc22d635..d5134f5a7 100644 --- a/internal/scheduling/nova/vm_handler.go +++ b/internal/scheduling/nova/vm_handler.go @@ -18,7 +18,6 @@ import ( vmv1alpha1 "github.com/cobaltcore-dev/vm-operator/api/v1alpha1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -36,9 +35,10 @@ type VirtualMachineHandler struct { // Upsert creates or patches the VirtualMachine resource described by the // scheduling result and request. Region and AZ are immutable on the CRD and only -// written on create; mutable fields (target host, cortex spec, labels) are -// patched on updates. A human-readable explanation of the result and the intent -// are recorded on the VM's cortex spec. +// written on create; mutable fields (cortex spec, labels) are patched on +// updates. A human-readable explanation of the result and the intent are +// recorded on the VM's cortex spec. Conditions are owned by the vm-operator and +// never written here. func (h *VirtualMachineHandler) Upsert(ctx context.Context, result compute.PlacementResult, request api.ExternalSchedulerRequest, intent v1alpha1.SchedulingIntent) error { log := ctrl.LoggerFrom(ctx) @@ -75,20 +75,14 @@ func (h *VirtualMachineHandler) Upsert(ctx context.Context, result compute.Place vm.ObjectMeta = metav1.ObjectMeta{ Name: request.Spec.Data.InstanceUUID, Labels: map[string]string{ - vmv1alpha1.LabelRegion: targetHvRegion, - vmv1alpha1.LabelAZ: targetHvAZ, - vmv1alpha1.LabelTargetHost: targetHv.Name, + vmv1alpha1.LabelRegion: targetHvRegion, + vmv1alpha1.LabelAZ: targetHvAZ, }, } // Add hierarchical location information to the virtual machine spec. vm.Spec.Region = targetHvRegion vm.Spec.AZ = targetHvAZ - vm.Spec.TargetHost = &vmv1alpha1.HostRef{ - APIVersion: hv1.GroupVersion.String(), - Kind: "Hypervisor", - Name: targetHost, - } // Derive the hypervisor type from the flavor extra spec hypervisor type. if hvType, ok := request.Spec.Data.Flavor. @@ -145,42 +139,18 @@ func (h *VirtualMachineHandler) Upsert(ctx context.Context, result compute.Place // Write the cortex spec (our scheduling result). The intent was resolved by // the API layer (with a graceful fallback to CreateIntent for initial - // placements) and passed in. + // placements) and passed in. We only reach this point with a target host, so + // scheduling succeeded and SchedulingError stays empty. vm.Spec.Cortex = &vmv1alpha1.CortexSpec{} for _, host := range result.OrderedHosts { - vm.Spec.Cortex.HostCandidates = append(vm.Spec.Cortex.HostCandidates, vmv1alpha1.HostRef{ + vm.Spec.Cortex.SchedulingCandidates = append(vm.Spec.Cortex.SchedulingCandidates, vmv1alpha1.HostRef{ APIVersion: hv1.GroupVersion.String(), Kind: "Hypervisor", Name: host, }) } - vm.Spec.Cortex.Explanation = explanation - vm.Spec.Cortex.Intent = string(intent) - - // Write the ConditionTypeScheduled condition to indicate that the VM has - // been successfully scheduled. For that we need to extract the scheduling - // intent from request and verify that the operation - var reason string - switch intent { - case api.LiveMigrationIntent: - reason = vmv1alpha1.ConditionReasonScheduledLiveMigration - case api.RebuildIntent: - reason = vmv1alpha1.ConditionReasonScheduledRebuild - case api.ResizeIntent: - reason = vmv1alpha1.ConditionReasonScheduledResize - case api.EvacuateIntent: - reason = vmv1alpha1.ConditionReasonScheduledEvacuation - case api.CreateIntent: - reason = vmv1alpha1.ConditionReasonScheduledInitialPlacement - default: - reason = "Unknown intent: " + string(intent) - } - meta.SetStatusCondition(&vm.Status.Conditions, metav1.Condition{ - Type: vmv1alpha1.ConditionTypeScheduled, - Status: metav1.ConditionTrue, - Reason: reason, - Message: "VM has been successfully scheduled by Cortex", - }) + vm.Spec.Cortex.SchedulingExplanation = explanation + vm.Spec.Cortex.SchedulingIntent = string(intent) // If the vm resource doesn't exist yet, create it. vmInCluster := &vmv1alpha1.VirtualMachine{} @@ -189,17 +159,14 @@ func (h *VirtualMachineHandler) Upsert(ctx context.Context, result compute.Place Name: vm.Name, }, vmInCluster) if err != nil && apierrors.IsNotFound(err) { - // Create ignores the status subresource and writes the server response - // (with an empty status) back into vm, so snapshot the conditions and - // restore them before persisting the status. - conditions := vm.Status.Conditions + // First time we see this VM: this intent is its first transition. We + // cannot report a scheduled generation yet (nothing is persisted), so we + // leave ScheduledGeneration unset. Conditions are owned by the vm-operator + // and never written here, so the status subresource is left untouched. + vm.Spec.Cortex.SchedulingTime = new(metav1.Now()) if err := h.Create(ctx, vm); err != nil { return fmt.Errorf("failed to create VM: %w", err) } - vm.Status.Conditions = conditions - if err := h.Status().Update(ctx, vm); err != nil { - return fmt.Errorf("failed to update VM status: %w", err) - } log.Info("created virtual machine resource", "instanceUUID", vm.Name, "targetHost", targetHost, "intent", intent) return nil } else if err != nil { @@ -207,11 +174,10 @@ func (h *VirtualMachineHandler) Upsert(ctx context.Context, result compute.Place } // The vm already exists, so update the existing resource with the new - // object meta, spec, and status. For map data like the labels, we preserve + // object meta and spec. For map data like the labels, we preserve // existing map entries that are not inside the new object, and only - // store/override the entries that are present in the new object. For status - // conditions, we only update the conditions that are present in the new - // object. In contrast, for the host candidates we completely replace the + // store/override the entries that are present in the new object. + // In contrast, for the scheduling candidates we completely replace the // existing list with the new list from the object. For the remaining spec // fields, we follow the same approach as for labels: preserve existing // entries that are not present in the new object, and only store/override @@ -222,28 +188,34 @@ func (h *VirtualMachineHandler) Upsert(ctx context.Context, result compute.Place maps.Copy(vmInCluster.Labels, vm.Labels) // Region and AZ are immutable and are only ever written on create, so we // leave vmInCluster.Spec.Region and vmInCluster.Spec.AZ untouched. - vmInCluster.Spec.TargetHost = vm.Spec.TargetHost vmInCluster.Spec.HypervisorType = vm.Spec.HypervisorType if vmInCluster.Spec.Resources == nil { vmInCluster.Spec.Resources = map[vmv1alpha1.ResourceName]resource.Quantity{} } maps.Copy(vmInCluster.Spec.Resources, vm.Spec.Resources) + // Nova does not expose the project's domain; it is populated out of band + // (via keystone) and must not be clobbered by a scheduling write. + domain := vmInCluster.Spec.Nova.Ownership.Domain vmInCluster.Spec.Nova = vm.Spec.Nova + vmInCluster.Spec.Nova.Ownership.Domain = domain if vmInCluster.Spec.Cortex == nil { vmInCluster.Spec.Cortex = &vmv1alpha1.CortexSpec{} } - vmInCluster.Spec.Cortex.HostCandidates = vm.Spec.Cortex.HostCandidates - vmInCluster.Spec.Cortex.Explanation = vm.Spec.Cortex.Explanation - vmInCluster.Spec.Cortex.Intent = vm.Spec.Cortex.Intent + vmInCluster.Spec.Cortex.SchedulingCandidates = vm.Spec.Cortex.SchedulingCandidates + vmInCluster.Spec.Cortex.SchedulingExplanation = vm.Spec.Cortex.SchedulingExplanation + // This scheduling succeeded, so clear any error left from a previous attempt. + vmInCluster.Spec.Cortex.SchedulingError = "" + // SchedulingTime marks a change of intent; only bump it when the intent + // actually transitioned, otherwise carry the existing timestamp forward. + if vmInCluster.Spec.Cortex.SchedulingIntent != vm.Spec.Cortex.SchedulingIntent { + vmInCluster.Spec.Cortex.SchedulingTime = new(metav1.Now()) + } + vmInCluster.Spec.Cortex.SchedulingIntent = vm.Spec.Cortex.SchedulingIntent + // Record the generation we observed on the resource we just read. + vmInCluster.Spec.Cortex.ScheduledGeneration = vmInCluster.Generation if err := h.Update(ctx, vmInCluster); err != nil { return fmt.Errorf("failed to update VM: %w", err) } - for _, cond := range vm.Status.Conditions { - meta.SetStatusCondition(&vmInCluster.Status.Conditions, cond) - } - if err := h.Status().Update(ctx, vmInCluster); err != nil { - return fmt.Errorf("failed to update VM status: %w", err) - } log.Info("updated virtual machine resource", "instanceUUID", vm.Name, "targetHost", targetHost, "intent", intent) return nil } diff --git a/internal/scheduling/nova/vm_handler_test.go b/internal/scheduling/nova/vm_handler_test.go index 6c81f6f60..d03147e07 100644 --- a/internal/scheduling/nova/vm_handler_test.go +++ b/internal/scheduling/nova/vm_handler_test.go @@ -6,6 +6,7 @@ package nova import ( "context" "testing" + "time" api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/internal/scheduling/compute" @@ -57,17 +58,18 @@ func vmTestRequest() api.ExternalSchedulerRequest { } func vmTestResult(targetHost *string) compute.PlacementResult { - return compute.PlacementResult{TargetHost: targetHost} + res := compute.PlacementResult{TargetHost: targetHost} + if targetHost != nil { + res.OrderedHosts = []string{*targetHost} + } + return res } -func vmTestHypervisor(region, az string) *hv1.Hypervisor { - labels := map[string]string{} +func vmTestHypervisor(region string) *hv1.Hypervisor { + labels := map[string]string{corev1.LabelTopologyZone: "az-1"} if region != "" { labels[corev1.LabelTopologyRegion] = region } - if az != "" { - labels[corev1.LabelTopologyZone] = az - } return &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "compute-1", Labels: labels}} } @@ -76,7 +78,7 @@ func TestVirtualMachineHandlerCreate(t *testing.T) { fakeClient := fake.NewClientBuilder(). WithScheme(scheme). WithStatusSubresource(&vmopv1alpha1.VirtualMachine{}). - WithObjects(vmTestHypervisor("region-1", "az-1")). + WithObjects(vmTestHypervisor("region-1")). Build() h := &VirtualMachineHandler{Client: fakeClient} @@ -92,8 +94,8 @@ func TestVirtualMachineHandlerCreate(t *testing.T) { if vm.Spec.Region != "region-1" || vm.Spec.AZ != "az-1" { t.Errorf("region/az = %q/%q, want region-1/az-1", vm.Spec.Region, vm.Spec.AZ) } - if vm.Spec.TargetHost == nil || vm.Spec.TargetHost.Name != "compute-1" { - t.Errorf("target host ref = %+v, want name compute-1", vm.Spec.TargetHost) + if len(vm.Spec.Cortex.SchedulingCandidates) != 1 || vm.Spec.Cortex.SchedulingCandidates[0].Name != "compute-1" { + t.Errorf("scheduling candidates = %+v, want name compute-1", vm.Spec.Cortex.SchedulingCandidates) } if cpu := vm.Spec.Resources[vmopv1alpha1.ResourceCPU]; cpu.Value() != 4 { t.Errorf("cpu = %d, want 4", cpu.Value()) @@ -104,8 +106,24 @@ func TestVirtualMachineHandlerCreate(t *testing.T) { if vm.Spec.Nova.Ownership.Project != "project-1" || vm.Spec.Nova.Flavor.Name != "m1.large" { t.Errorf("nova spec = %+v, want project-1 / m1.large", vm.Spec.Nova) } - if vm.Labels[vmopv1alpha1.LabelTargetHost] != "compute-1" { - t.Errorf("target host label = %q, want compute-1", vm.Labels[vmopv1alpha1.LabelTargetHost]) + // Intent is the VM's first transition, so a timestamp is stamped, but no + // generation has been scheduled yet. + if vm.Spec.Cortex.SchedulingIntent != string(api.CreateIntent) { + t.Errorf("intent = %q, want %q", vm.Spec.Cortex.SchedulingIntent, api.CreateIntent) + } + if vm.Spec.Cortex.SchedulingTime == nil { + t.Error("expected SchedulingTime to be stamped on create") + } + if vm.Spec.Cortex.ScheduledGeneration != 0 { + t.Errorf("scheduledGeneration = %d, want 0 on create", vm.Spec.Cortex.ScheduledGeneration) + } + // A successful scheduling records no error. + if vm.Spec.Cortex.SchedulingError != "" { + t.Errorf("schedulingError = %q, want empty on success", vm.Spec.Cortex.SchedulingError) + } + // Conditions are owned by the vm-operator and must not be written here. + if len(vm.Status.Conditions) != 0 { + t.Errorf("expected no conditions written, got %+v", vm.Status.Conditions) } } @@ -113,19 +131,23 @@ func TestVirtualMachineHandlerUpdate(t *testing.T) { scheme := vmTestScheme(t) existing := &vmopv1alpha1.VirtualMachine{ ObjectMeta: metav1.ObjectMeta{ - Name: "vm-uuid-1", - Labels: map[string]string{vmopv1alpha1.LabelTargetHost: "old-host"}, + Name: "vm-uuid-1", }, Spec: vmopv1alpha1.VMSpec{ - Region: "region-1", - AZ: "az-1", - TargetHost: &corev1.ObjectReference{Name: "hv-old"}, + Region: "region-1", + AZ: "az-1", + Nova: vmopv1alpha1.NovaSpec{ + // Domain is populated out of band (keystone) and must survive a + // scheduling update. + Ownership: vmopv1alpha1.Ownership{Domain: "domain-1"}, + }, + Cortex: &vmopv1alpha1.CortexSpec{SchedulingIntent: string(api.CreateIntent)}, }, } fakeClient := fake.NewClientBuilder(). WithScheme(scheme). WithStatusSubresource(&vmopv1alpha1.VirtualMachine{}). - WithObjects(vmTestHypervisor("region-1", "az-1"), existing). + WithObjects(vmTestHypervisor("region-1"), existing). Build() h := &VirtualMachineHandler{Client: fakeClient} @@ -138,16 +160,63 @@ func TestVirtualMachineHandlerUpdate(t *testing.T) { if err := fakeClient.Get(context.Background(), client.ObjectKey{Name: "vm-uuid-1"}, vm); err != nil { t.Fatalf("get vm: %v", err) } - if vm.Spec.TargetHost == nil || vm.Spec.TargetHost.Name != "compute-1" { - t.Errorf("target host ref = %+v, want name compute-1", vm.Spec.TargetHost) - } - if vm.Labels[vmopv1alpha1.LabelTargetHost] != "compute-1" { - t.Errorf("target host label = %q, want compute-1", vm.Labels[vmopv1alpha1.LabelTargetHost]) + if len(vm.Spec.Cortex.SchedulingCandidates) != 1 || vm.Spec.Cortex.SchedulingCandidates[0].Name != "compute-1" { + t.Errorf("scheduling candidates = %+v, want name compute-1", vm.Spec.Cortex.SchedulingCandidates) } // Immutable fields must remain unchanged. if vm.Spec.Region != "region-1" || vm.Spec.AZ != "az-1" { t.Errorf("immutable region/az changed: %q/%q", vm.Spec.Region, vm.Spec.AZ) } + // The out-of-band domain must be preserved across the Nova spec overwrite. + if vm.Spec.Nova.Ownership.Domain != "domain-1" { + t.Errorf("domain = %q, want domain-1 preserved", vm.Spec.Nova.Ownership.Domain) + } +} + +func TestVirtualMachineHandlerUpdateBumpsOnIntentChange(t *testing.T) { + scheme := vmTestScheme(t) + old := metav1.NewTime(metav1.Now().Add(-time.Hour).Truncate(time.Second)) + existing := &vmopv1alpha1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{Name: "vm-uuid-1"}, + Spec: vmopv1alpha1.VMSpec{ + Region: "region-1", + AZ: "az-1", + Cortex: &vmopv1alpha1.CortexSpec{ + SchedulingIntent: string(api.CreateIntent), + SchedulingTime: &old, + }, + }, + } + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&vmopv1alpha1.VirtualMachine{}). + WithObjects(vmTestHypervisor("region-1"), existing). + Build() + h := &VirtualMachineHandler{Client: fakeClient} + + targetHost := "compute-1" + // Same intent: the timestamp must be carried forward unchanged. + if err := h.Upsert(context.Background(), vmTestResult(&targetHost), vmTestRequest(), api.CreateIntent); err != nil { + t.Fatalf("Upsert same intent: %v", err) + } + vm := &vmopv1alpha1.VirtualMachine{} + if err := fakeClient.Get(context.Background(), client.ObjectKey{Name: "vm-uuid-1"}, vm); err != nil { + t.Fatalf("get vm: %v", err) + } + if !vm.Spec.Cortex.SchedulingTime.Equal(&old) { + t.Errorf("SchedulingTime = %v, want preserved %v on unchanged intent", vm.Spec.Cortex.SchedulingTime, old) + } + + // Changed intent: the timestamp must be bumped. + if err := h.Upsert(context.Background(), vmTestResult(&targetHost), vmTestRequest(), api.LiveMigrationIntent); err != nil { + t.Fatalf("Upsert new intent: %v", err) + } + if err := fakeClient.Get(context.Background(), client.ObjectKey{Name: "vm-uuid-1"}, vm); err != nil { + t.Fatalf("get vm: %v", err) + } + if vm.Spec.Cortex.SchedulingTime.Equal(&old) { + t.Error("expected SchedulingTime to be bumped on intent change") + } } func TestVirtualMachineHandlerNoTargetHost(t *testing.T) { @@ -171,7 +240,7 @@ func TestVirtualMachineHandlerMissingRegion(t *testing.T) { scheme := vmTestScheme(t) fakeClient := fake.NewClientBuilder(). WithScheme(scheme). - WithObjects(vmTestHypervisor("", "az-1")). // no region label + WithObjects(vmTestHypervisor("")). // no region label Build() h := &VirtualMachineHandler{Client: fakeClient} diff --git a/internal/scheduling/reservations/inflight/controller_test.go b/internal/scheduling/reservations/inflight/controller_test.go index a185aa6f8..709a85ef0 100644 --- a/internal/scheduling/reservations/inflight/controller_test.go +++ b/internal/scheduling/reservations/inflight/controller_test.go @@ -247,7 +247,7 @@ func TestReconcile(t *testing.T) { name: "non-create: live migration ongoing is kept", objects: []client.Object{ newInFlightReservation("res-1", "vm-uuid-1", "host-1", inFlightOpts{intent: novaapi.LiveMigrationIntent, resources: cpu4}), - newVM("vm-uuid-1", "host-1", cond(vmv1alpha1.ConditionTypeMigrating, metav1.ConditionTrue, vmv1alpha1.ConditionReasonMigratingOngoing)), + newVM("vm-uuid-1", "host-1", cond(vmv1alpha1.ConditionTypeMigrating, metav1.ConditionTrue, vmv1alpha1.ConditionReasonMigratingLiveInProgress)), }, wantRequeue: requeueInterval, wantKept: []string{"res-1"}, @@ -261,7 +261,7 @@ func TestReconcile(t *testing.T) { objects: []client.Object{ newInFlightReservation("a", "vm-uuid-1", "host-1", createA), newInFlightReservation("b", "vm-uuid-1", "host-2", createA), - newVM("vm-uuid-1", "host-1", cond(vmv1alpha1.ConditionTypeRunning, metav1.ConditionTrue, "Running")), + newVM("vm-uuid-1", "host-1", cond(vmv1alpha1.ConditionTypeRunning, metav1.ConditionTrue, vmv1alpha1.ConditionReasonRunningOK)), }, reconcile: "a", wantDeleted: []string{"a", "b"}, @@ -296,79 +296,71 @@ func TestReconcile(t *testing.T) { wantDeleted: []string{"res-1"}, }, { - // UC2: an ongoing resize keeps the hold and surfaces the Resizing reason. - name: "resize: ongoing requeues", + // UC2: an in-progress resize keeps the hold and surfaces the Resizing reason. + name: "resize: in progress requeues", objects: []client.Object{ newInFlightReservation("res-1", "vm-uuid-1", "host-1", inFlightOpts{intent: novaapi.ResizeIntent}), - newVM("vm-uuid-1", "host-1", cond(vmv1alpha1.ConditionTypeResizing, metav1.ConditionTrue, vmv1alpha1.ConditionReasonResizingOngoing)), + newVM("vm-uuid-1", "host-1", cond(vmv1alpha1.ConditionTypeResizing, metav1.ConditionTrue, vmv1alpha1.ConditionReasonResizingInProgress)), }, wantRequeue: requeueInterval, wantKept: []string{"res-1"}, wantReadyStatus: metav1.ConditionUnknown, wantReadyReason: "Resizing", }, { - // UC2: a terminal Resizing reason that transitioned before the hold was - // created is stale (previous operation) and must not release it. - name: "resize: stale terminal condition is kept", + // UC2: a settled Resizing condition that transitioned before the hold + // was created is stale (idle VM / previous operation) and must not + // release it. + name: "resize: stale settled condition is kept", objects: []client.Object{ newInFlightReservation("res-1", "vm-uuid-1", "host-1", inFlightOpts{intent: novaapi.ResizeIntent, createdAt: &resCreated}), - vmWithCond("vm-uuid-1", vmv1alpha1.ConditionTypeResizing, vmv1alpha1.ConditionReasonResizingCompleted, stalePast), + vmWithCond("vm-uuid-1", vmv1alpha1.ConditionTypeResizing, vmv1alpha1.ConditionReasonResizingInProgress, stalePast), }, wantRequeue: requeueInterval, wantKept: []string{"res-1"}, wantReadyStatus: metav1.ConditionUnknown, wantReadyReason: "Resizing", }, { - // UC3: an ongoing live migration keeps both holds. - name: "live migration: ongoing requeues", + // UC3: an in-progress live migration keeps both holds. + name: "live migration: in progress requeues", objects: []client.Object{ newInFlightReservation("res-1", "vm-uuid-1", "host-1", inFlightOpts{intent: novaapi.LiveMigrationIntent}), - newVM("vm-uuid-1", "host-1", cond(vmv1alpha1.ConditionTypeMigrating, metav1.ConditionTrue, vmv1alpha1.ConditionReasonMigratingOngoing)), + newVM("vm-uuid-1", "host-1", cond(vmv1alpha1.ConditionTypeMigrating, metav1.ConditionTrue, vmv1alpha1.ConditionReasonMigratingLiveInProgress)), }, wantRequeue: requeueInterval, wantKept: []string{"res-1"}, wantReadyStatus: metav1.ConditionUnknown, wantReadyReason: "Migrating", }, { - // UC3: a fresh terminal migration releases both target and source holds + // UC3: a fresh settled migration releases both target and source holds // (the whole operation group) through Reconcile. - name: "live migration: fresh terminal releases target and source group", + name: "live migration: fresh settled releases target and source group", objects: []client.Object{ newInFlightReservation("target", "vm-uuid-1", "host-2", inFlightOpts{operationID: "op-m", intent: novaapi.LiveMigrationIntent, createdAt: &resCreated}), newInFlightReservation("source", "vm-uuid-1", "host-1", inFlightOpts{operationID: "op-m", intent: novaapi.LiveMigrationIntent, createdAt: &resCreated}), - vmWithCond("vm-uuid-1", vmv1alpha1.ConditionTypeMigrating, vmv1alpha1.ConditionReasonMigratingCompleted, freshFuture), + vmWithCond("vm-uuid-1", vmv1alpha1.ConditionTypeMigrating, vmv1alpha1.ConditionReasonMigratingLiveInProgress, freshFuture), }, reconcile: "source", wantDeleted: []string{"target", "source"}, }, } - // Every terminal resize reason, when fresh, releases the hold through Reconcile. - for _, r := range []string{ - vmv1alpha1.ConditionReasonResizingCompleted, - vmv1alpha1.ConditionReasonResizingReverted, - vmv1alpha1.ConditionReasonResizingFailed, - } { - tests = append(tests, testCase{ - name: "resize: fresh terminal " + r + " deletes reservation", - objects: []client.Object{ - newInFlightReservation("res-1", "vm-uuid-1", "host-1", inFlightOpts{intent: novaapi.ResizeIntent, createdAt: &resCreated}), - vmWithCond("vm-uuid-1", vmv1alpha1.ConditionTypeResizing, r, freshFuture), - }, - wantDeleted: []string{"res-1"}, - }) - } - // Every terminal live-migration reason, when fresh, releases the hold. - for _, r := range []string{ - vmv1alpha1.ConditionReasonMigratingCompleted, - vmv1alpha1.ConditionReasonMigratingRolledBack, - vmv1alpha1.ConditionReasonMigratingAborting, - vmv1alpha1.ConditionReasonMigratingFailed, + // A fresh settled (status=False) operation condition releases the hold + // through Reconcile, for every operation that maps onto the Resizing or + // Migrating condition. + for _, op := range []struct { + name string + intent v1alpha1.SchedulingIntent + condType string + reason string + }{ + {"resize", novaapi.ResizeIntent, vmv1alpha1.ConditionTypeResizing, vmv1alpha1.ConditionReasonResizingInProgress}, + {"live migration", novaapi.LiveMigrationIntent, vmv1alpha1.ConditionTypeMigrating, vmv1alpha1.ConditionReasonMigratingLiveInProgress}, + {"evacuate", novaapi.EvacuateIntent, vmv1alpha1.ConditionTypeMigrating, vmv1alpha1.ConditionReasonMigratingColdInProgress}, } { tests = append(tests, testCase{ - name: "live migration: fresh terminal " + r + " deletes reservation", + name: op.name + ": fresh settled condition deletes reservation", objects: []client.Object{ - newInFlightReservation("res-1", "vm-uuid-1", "host-1", inFlightOpts{intent: novaapi.LiveMigrationIntent, createdAt: &resCreated}), - vmWithCond("vm-uuid-1", vmv1alpha1.ConditionTypeMigrating, r, freshFuture), + newInFlightReservation("res-1", "vm-uuid-1", "host-1", inFlightOpts{intent: op.intent, createdAt: &resCreated}), + vmWithCond("vm-uuid-1", op.condType, op.reason, freshFuture), }, wantDeleted: []string{"res-1"}, }) diff --git a/internal/scheduling/reservations/inflight/vm_owner_handler.go b/internal/scheduling/reservations/inflight/vm_owner_handler.go index 036990315..4cd53b9ff 100644 --- a/internal/scheduling/reservations/inflight/vm_owner_handler.go +++ b/internal/scheduling/reservations/inflight/vm_owner_handler.go @@ -5,7 +5,6 @@ package inflight import ( "context" - "slices" "time" novaapi "github.com/cobaltcore-dev/cortex/api/external/nova" @@ -252,11 +251,6 @@ func (h *VirtualMachineHandler) terminalForIntent(res *v1alpha1.Reservation, vm // live (host assigned and the domain running / Nova ACTIVE), or on an // explicit Nova failure (Nova ERROR). While the VM is still building, // keep blocking all candidates. - // - // Scheduled=False is intentionally NOT used as a release trigger: it is - // not written on failure today, and it reflects the latest scheduling - // result for the VM rather than this operation, so it cannot be - // correlated to an OperationID. placed := vm.Status.Host != nil && vm.Status.Host.Name != "" running := meta.IsStatusConditionTrue(vm.Status.Conditions, vmv1alpha1.ConditionTypeRunning) novaActive := vm.Status.Nova != nil && vm.Status.Nova.Status == novaStatusActive @@ -269,7 +263,7 @@ func (h *VirtualMachineHandler) terminalForIntent(res *v1alpha1.Reservation, vm } // We have at least some VM-CRD signal but it is not yet decisive; keep // waiting for a confirmed running/ACTIVE (or ERROR) signal. - if placed || meta.FindStatusCondition(vm.Status.Conditions, vmv1alpha1.ConditionTypeScheduled) != nil || vm.Status.Nova != nil { + if placed || vm.Status.Nova != nil { return TerminalResult{RequeueAfter: requeueInterval, Reason: reasonAwaitingPlacement, Message: "VM not yet confirmed running on a host"}, true } @@ -292,60 +286,59 @@ func (h *VirtualMachineHandler) terminalForIntent(res *v1alpha1.Reservation, vm Message: "unmanaged-source resize not yet confirmed running on target"}, true } // KVM-source resize: release when the resize is no longer in progress. - return terminalByReason(res.CreationTimestamp, vm.Status.Conditions, vmv1alpha1.ConditionTypeResizing, - reasonResizing, "resize in progress", - vmv1alpha1.ConditionReasonResizingCompleted, - vmv1alpha1.ConditionReasonResizingReverted, - vmv1alpha1.ConditionReasonResizingFailed) + return terminalBySettledStatus(res.CreationTimestamp, vm.Status.Conditions, vmv1alpha1.ConditionTypeResizing, + reasonResizing, "resize in progress") - case novaapi.LiveMigrationIntent: - // UC3 live migration: release both the forward (target) and rollback - // (source) reservations once the migration settles, including an abort. - return terminalByReason(res.CreationTimestamp, vm.Status.Conditions, vmv1alpha1.ConditionTypeMigrating, - reasonMigrating, "live migration in progress", - vmv1alpha1.ConditionReasonMigratingCompleted, - vmv1alpha1.ConditionReasonMigratingRolledBack, - vmv1alpha1.ConditionReasonMigratingAborting, - vmv1alpha1.ConditionReasonMigratingFailed) + case novaapi.LiveMigrationIntent, novaapi.EvacuateIntent: + // UC3 live migration and evacuation: the vm-operator's Migrating + // condition now covers both live and cold migrations (an evacuation is a + // cold migration). Release both the forward (target) and rollback + // (source) reservations once the migration is no longer in progress. + return terminalBySettledStatus(res.CreationTimestamp, vm.Status.Conditions, vmv1alpha1.ConditionTypeMigrating, + reasonMigrating, "migration in progress") default: - // Other intents (rebuild, evacuate, ...) have no dedicated condition - // yet; keep the hold pending a future condition (the optional TTL - // backstop only acts if an EndTime is set). + // Other intents (e.g. rebuild) have no dedicated condition yet; keep the + // hold pending a future condition (the optional TTL backstop only acts if + // an EndTime is set). return TerminalResult{}, false } } -// terminalByReason returns (result, ok) for a condition whose terminal reasons -// are listed. ok=false when the condition is absent (caller keeps waiting). +// terminalBySettledStatus returns (result, ok) for an operation-progress +// condition (Migrating, Resizing) under the vm-operator's current semantics: // -// A terminal reason is only trusted when it is *fresh*: the condition's -// LastTransitionTime must be at or after createdAt (the reservation's creation -// time). Nova serializes its own operations per VM, so when this VM-owned -// reservation was created no other Nova operation on the VM was in flight; a -// terminal condition that transitioned *before* this reservation existed -// therefore describes a previous operation and must not release the freshly -// created holds. In that stale case we keep waiting for the vm-operator to -// transition the condition for the current operation. +// - status=True: the operation is in progress — keep waiting. +// - status=Unknown: authoritative datasources disagree — keep waiting. +// - status=False: the operation is not in progress. This status also covers a +// VM that was never migrated/resized, so it is only trusted as a completion +// signal when it is *fresh*: the condition's LastTransitionTime must be at or +// after createdAt (the reservation's creation time). Nova serializes its own +// operations per VM, so when this VM-owned reservation was created no other +// Nova operation on the VM was in flight; a False that transitioned *before* +// this reservation existed therefore describes an idle VM (or a previous +// operation) and must not release the freshly created holds. // -// When Done is false the result carries the waiting reason/message. -func terminalByReason(createdAt metav1.Time, conditions []metav1.Condition, condType, waitReason, waitMessage string, terminalReasons ...string) (TerminalResult, bool) { +// ok=false when the condition is absent, so the caller keeps waiting for the +// vm-operator to record the operation. When Done is false the result carries the +// waiting reason/message. +func terminalBySettledStatus(createdAt metav1.Time, conditions []metav1.Condition, condType, waitReason, waitMessage string) (TerminalResult, bool) { cond := meta.FindStatusCondition(conditions, condType) if cond == nil { return TerminalResult{}, false } - if slices.Contains(terminalReasons, cond.Reason) && !conditionIsStale(cond, createdAt) { + if cond.Status == metav1.ConditionFalse && !conditionIsStale(cond, createdAt) { return TerminalResult{Done: true}, true } return TerminalResult{RequeueAfter: requeueInterval, Reason: waitReason, Message: waitMessage}, true } // conditionIsStale reports whether cond transitioned before the reservation was -// created, i.e. the condition reflects a previous operation rather than the one -// this reservation belongs to. A zero LastTransitionTime is treated as stale: -// without a trustworthy transition time we cannot attribute the terminal signal -// to the current operation, so we conservatively keep waiting rather than risk -// releasing holds early. +// created, i.e. the condition reflects a previous operation (or a long-idle VM) +// rather than the one this reservation belongs to. A zero LastTransitionTime is +// treated as stale: without a trustworthy transition time we cannot attribute +// the signal to the current operation, so we conservatively keep waiting rather +// than risk releasing holds early. func conditionIsStale(cond *metav1.Condition, createdAt metav1.Time) bool { if cond.LastTransitionTime.IsZero() { return true diff --git a/internal/scheduling/reservations/inflight/vm_owner_handler_test.go b/internal/scheduling/reservations/inflight/vm_owner_handler_test.go index 23d7d679e..473374f97 100644 --- a/internal/scheduling/reservations/inflight/vm_owner_handler_test.go +++ b/internal/scheduling/reservations/inflight/vm_owner_handler_test.go @@ -128,19 +128,26 @@ func TestVMHandler_TerminalState(t *testing.T) { tests := []tc{ // UC1 initial placement. {name: "create: building -> wait", intent: novaapi.CreateIntent, - vm: newVM("vm-1", "host-9", cond(vmv1alpha1.ConditionTypeScheduled, metav1.ConditionUnknown, "InitialPlacement"))}, + vm: newVM("vm-1", "host-9")}, {name: "create: Running=True -> done", intent: novaapi.CreateIntent, wantDone: true, - vm: newVM("vm-1", "host-9", cond(vmv1alpha1.ConditionTypeRunning, metav1.ConditionTrue, "Running"))}, + vm: newVM("vm-1", "host-9", cond(vmv1alpha1.ConditionTypeRunning, metav1.ConditionTrue, vmv1alpha1.ConditionReasonRunningOK))}, {name: "create: nova ACTIVE -> done", intent: novaapi.CreateIntent, wantDone: true, vm: withNovaStatus(newVM("vm-1", "host-9"), "ACTIVE")}, {name: "create: no live signal -> wait", intent: novaapi.CreateIntent, vm: withNovaStatus(newVM("vm-1", "host-9"), "BUILD")}, - {name: "create: scheduled false alone -> wait", intent: novaapi.CreateIntent, - vm: newVM("vm-1", "", cond(vmv1alpha1.ConditionTypeScheduled, metav1.ConditionFalse, "NoHostFound"))}, + {name: "create: placed but no live signal -> wait", intent: novaapi.CreateIntent, + vm: newVM("vm-1", "host-9", cond(vmv1alpha1.ConditionTypeRunning, metav1.ConditionFalse, vmv1alpha1.ConditionReasonRunningShutoff))}, {name: "create: nova error -> done", intent: novaapi.CreateIntent, wantDone: true, vm: withNovaStatus(newVM("vm-1", ""), "ERROR")}, - // UC2 resize / UC3 live migration ongoing. - {name: "resize: ongoing -> wait", intent: novaapi.ResizeIntent, - vm: newVM("vm-1", "host-1", cond(vmv1alpha1.ConditionTypeResizing, metav1.ConditionTrue, vmv1alpha1.ConditionReasonResizingOngoing))}, - {name: "live migration: ongoing -> wait", intent: novaapi.LiveMigrationIntent, - vm: newVM("vm-1", "host-1", cond(vmv1alpha1.ConditionTypeMigrating, metav1.ConditionTrue, vmv1alpha1.ConditionReasonMigratingOngoing))}, + // UC2 resize / UC3 live migration in progress (status=True). + {name: "resize: in progress -> wait", intent: novaapi.ResizeIntent, + vm: newVM("vm-1", "host-1", cond(vmv1alpha1.ConditionTypeResizing, metav1.ConditionTrue, vmv1alpha1.ConditionReasonResizingInProgress))}, + {name: "live migration: in progress -> wait", intent: novaapi.LiveMigrationIntent, + vm: newVM("vm-1", "host-1", cond(vmv1alpha1.ConditionTypeMigrating, metav1.ConditionTrue, vmv1alpha1.ConditionReasonMigratingLiveInProgress))}, + {name: "evacuate: in progress -> wait", intent: novaapi.EvacuateIntent, + vm: newVM("vm-1", "host-1", cond(vmv1alpha1.ConditionTypeMigrating, metav1.ConditionTrue, vmv1alpha1.ConditionReasonMigratingColdInProgress))}, + // Unknown status reflects conflicting observations -> keep waiting. + {name: "resize: unknown -> wait", intent: novaapi.ResizeIntent, + vm: newVM("vm-1", "host-1", cond(vmv1alpha1.ConditionTypeResizing, metav1.ConditionUnknown, ""))}, + {name: "live migration: unknown -> wait", intent: novaapi.LiveMigrationIntent, + vm: newVM("vm-1", "host-1", cond(vmv1alpha1.ConditionTypeMigrating, metav1.ConditionUnknown, ""))}, // No VM CRD -> no terminal signal, keep waiting for the VM CRD // (regardless of intent). {name: "no VM CRD: create -> wait", intent: novaapi.CreateIntent}, @@ -153,7 +160,7 @@ func TestVMHandler_TerminalState(t *testing.T) { unmanaged := map[string]string{LabelInFlightSource: LabelInFlightSourceUnmanaged} tests = append(tests, tc{name: "unmanaged resize: landed and running -> done", intent: novaapi.ResizeIntent, labels: unmanaged, wantDone: true, - vm: newVM("vm-1", "host-1", cond(vmv1alpha1.ConditionTypeRunning, metav1.ConditionTrue, "Running"))}, + vm: newVM("vm-1", "host-1", cond(vmv1alpha1.ConditionTypeRunning, metav1.ConditionTrue, vmv1alpha1.ConditionReasonRunningOK))}, tc{name: "unmanaged resize: landed and nova ACTIVE -> done", intent: novaapi.ResizeIntent, labels: unmanaged, wantDone: true, vm: withNovaStatus(newVM("vm-1", "host-1"), "ACTIVE")}, tc{name: "unmanaged resize: placed but not running -> wait", intent: novaapi.ResizeIntent, labels: unmanaged, @@ -161,24 +168,30 @@ func TestVMHandler_TerminalState(t *testing.T) { tc{name: "unmanaged resize: nova ERROR -> wait", intent: novaapi.ResizeIntent, labels: unmanaged, vm: withNovaStatus(newVM("vm-1", "host-1"), "ERROR")}, ) - for _, r := range []string{vmv1alpha1.ConditionReasonResizingCompleted, vmv1alpha1.ConditionReasonResizingReverted, vmv1alpha1.ConditionReasonResizingFailed} { - tests = append(tests, tc{name: "resize: terminal " + r, intent: novaapi.ResizeIntent, wantDone: true, - vm: newVM("vm-1", "host-1", cond(vmv1alpha1.ConditionTypeResizing, metav1.ConditionFalse, r))}) - } - for _, r := range []string{vmv1alpha1.ConditionReasonMigratingCompleted, vmv1alpha1.ConditionReasonMigratingRolledBack, - vmv1alpha1.ConditionReasonMigratingAborting, vmv1alpha1.ConditionReasonMigratingFailed} { - tests = append(tests, tc{name: "live migration: terminal " + r, intent: novaapi.LiveMigrationIntent, wantDone: true, - vm: newVM("vm-1", "host-1", cond(vmv1alpha1.ConditionTypeMigrating, metav1.ConditionFalse, r))}) + // A freshly settled operation condition (status=False transitioned at/after + // the reservation was created) releases the holds. + for _, op := range []struct { + name string + intent v1alpha1.SchedulingIntent + condType, reason string + }{ + {"resize", novaapi.ResizeIntent, vmv1alpha1.ConditionTypeResizing, vmv1alpha1.ConditionReasonResizingInProgress}, + {"live migration", novaapi.LiveMigrationIntent, vmv1alpha1.ConditionTypeMigrating, vmv1alpha1.ConditionReasonMigratingLiveInProgress}, + {"evacuate", novaapi.EvacuateIntent, vmv1alpha1.ConditionTypeMigrating, vmv1alpha1.ConditionReasonMigratingColdInProgress}, + } { + tests = append(tests, tc{name: op.name + ": settled -> done", intent: op.intent, wantDone: true, + vm: newVM("vm-1", "host-1", cond(op.condType, metav1.ConditionFalse, op.reason))}) } - // A terminal condition left over from a previous operation (transition before - // the reservation was created, or unknown) must not release fresh holds. + // A settled (status=False) condition left over from a previous operation + // (transition before the reservation was created, or with no transition + // time) describes an idle VM and must not release fresh holds. for _, op := range []struct { name string intent v1alpha1.SchedulingIntent condType, reason string }{ - {"resize", novaapi.ResizeIntent, vmv1alpha1.ConditionTypeResizing, vmv1alpha1.ConditionReasonResizingCompleted}, - {"live_migration", novaapi.LiveMigrationIntent, vmv1alpha1.ConditionTypeMigrating, vmv1alpha1.ConditionReasonMigratingCompleted}, + {"resize", novaapi.ResizeIntent, vmv1alpha1.ConditionTypeResizing, vmv1alpha1.ConditionReasonResizingInProgress}, + {"live_migration", novaapi.LiveMigrationIntent, vmv1alpha1.ConditionTypeMigrating, vmv1alpha1.ConditionReasonMigratingLiveInProgress}, } { tests = append(tests, tc{name: "stale " + op.name + " -> wait", intent: op.intent, created: &created, vm: withCond(op.condType, op.reason, stale)},