From 22e8032d68e5bdc7bfc5d8254aaccf7c4f0be3fa Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Wed, 11 Nov 2026 14:23:58 +0100 Subject: [PATCH] Block eviction completion while Placement usage remains The hypervisor server list can be empty before the source agent has finished a migration. Check the host's Placement usage and keep Evicting=True while any usage remains. Signed-off-by: Fabian Wiesel --- internal/controller/eviction/controller.go | 33 +++++++- .../controller/eviction/controller_test.go | 82 +++++++++++++++++-- internal/openstack/placement.go | 23 ++++++ internal/openstack/placement_test.go | 32 ++++++++ 4 files changed, 163 insertions(+), 7 deletions(-) diff --git a/internal/controller/eviction/controller.go b/internal/controller/eviction/controller.go index 8e80c15f..0c934a7f 100644 --- a/internal/controller/eviction/controller.go +++ b/internal/controller/eviction/controller.go @@ -45,8 +45,9 @@ import ( // EvictionReconciler reconciles a Eviction object type EvictionReconciler struct { client.Client - Scheme *runtime.Scheme - computeClient *gophercloud.ServiceClient + Scheme *runtime.Scheme + computeClient *gophercloud.ServiceClient + placementClient *gophercloud.ServiceClient } const ( @@ -177,6 +178,29 @@ func (r *EvictionReconciler) handleRunning(ctx context.Context, eviction *kvmv1. return r.evictNext(ctx, eviction, limit) } + inUse, err := openstack.ResourceProviderInUse(ctx, r.placementClient, hypervisor.Status.HypervisorID) + if err != nil { + meta.SetStatusCondition(&eviction.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeEvicting, + Status: metav1.ConditionTrue, + Message: fmt.Sprintf("waiting for host allocations: %v", err), + Reason: kvmv1.ConditionReasonRunning, + }) + if updateErr := r.updateStatus(ctx, eviction); updateErr != nil { + return ctrl.Result{}, errors.Join(err, updateErr) + } + return ctrl.Result{}, err + } + if inUse { + meta.SetStatusCondition(&eviction.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeEvicting, + Status: metav1.ConditionTrue, + Message: "waiting for host allocations to drain", + Reason: kvmv1.ConditionReasonRunning, + }) + return ctrl.Result{RequeueAfter: global.DefaultPollTime}, r.updateStatus(ctx, eviction) + } + meta.SetStatusCondition(&eviction.Status.Conditions, metav1.Condition{ Type: kvmv1.ConditionTypeEvicting, Status: metav1.ConditionFalse, @@ -542,5 +566,10 @@ func (r *EvictionReconciler) SetupWithManager(mgr ctrl.Manager) error { } r.computeClient.Microversion = "2.90" // Xena (or later) + if r.placementClient, err = openstack.GetServiceClient(ctx, "placement", nil); err != nil { + return err + } + r.placementClient.Microversion = "1.39" // yoga, or later + return r.registerWithManager(mgr) } diff --git a/internal/controller/eviction/controller_test.go b/internal/controller/eviction/controller_test.go index 4be3a0ef..0d50fba8 100644 --- a/internal/controller/eviction/controller_test.go +++ b/internal/controller/eviction/controller_test.go @@ -20,6 +20,7 @@ package eviction import ( "fmt" "net/http" + "time" "github.com/gophercloud/gophercloud/v2/testhelper" "github.com/gophercloud/gophercloud/v2/testhelper/client" @@ -143,8 +144,9 @@ var _ = Describe("Eviction Controller", func() { Name: resourceName, Namespace: namespaceName, } - reconcileRequest = ctrl.Request{NamespacedName: typeNamespacedName} - fakeServer testhelper.FakeServer + reconcileRequest = ctrl.Request{NamespacedName: typeNamespacedName} + fakeServer testhelper.FakeServer + placementUsagesBody string ) BeforeEach(func(ctx SpecContext) { @@ -157,11 +159,23 @@ var _ = Describe("Eviction Controller", func() { Fail("Unhandled request to fake server: " + r.Method + " " + r.URL.Path) }) + By("Installing the placement usages mock") + fakeServer.Mux.HandleFunc("GET /resource_providers/{provider_id}/usages", func(w http.ResponseWriter, r *http.Request) { + Expect(r.PathValue("provider_id")).To(Equal(hypervisorId)) + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, placementUsagesBody) + }) + + By("Setting default placement usages") + placementUsagesBody = `{"usages": {}}` + By("Creating the EvictionReconciler") evictionReconciler = &EvictionReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), - computeClient: client.ServiceClient(fakeServer), + Client: k8sClient, + Scheme: k8sClient.Scheme(), + computeClient: client.ServiceClient(fakeServer), + placementClient: client.ServiceClient(fakeServer), } }) @@ -395,6 +409,64 @@ var _ = Describe("Eviction Controller", func() { )) }) }) + + Context("When placement still has usage after the instances left", func() { + BeforeEach(func(ctx SpecContext) { + By("Mocking hypervisor API to return enabled status") + fakeServer.Mux.HandleFunc("GET /os-hypervisors/{hypervisor_id}", func(w http.ResponseWriter, r *http.Request) { + rHypervisorId := r.PathValue("hypervisor_id") + Expect(rHypervisorId).To(Equal(hypervisorId)) + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err := fmt.Fprintf(w, hypervisorTpl, "null", "enabled") + Expect(err).To(Succeed()) + }) + + By("Mocking service update API") + fakeServer.Mux.HandleFunc("PUT /os-services/{service_id}", func(w http.ResponseWriter, r *http.Request) { + rServiceId := r.PathValue("service_id") + Expect(rServiceId).To(Equal(serviceId)) + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err := fmt.Fprintf(w, `{"service": {"id": "%v", "status": "disabled"}}`, serviceId) + Expect(err).To(Succeed()) + }) + + By("Seeding placement with remaining usages on the host") + placementUsagesBody = `{"usages": {"VCPU": 2}}` + }) + + It("keeps the eviction running until placement is clear", func(ctx SpecContext) { + By("Reconciling until placement blocks completion") + Eventually(func(g Gomega) { + _, err := evictionReconciler.Reconcile(ctx, reconcileRequest) + g.Expect(err).NotTo(HaveOccurred()) + + resource := &kvmv1.Eviction{} + g.Expect(k8sClient.Get(ctx, typeNamespacedName, resource)).To(Succeed()) + condition := meta.FindStatusCondition(resource.Status.Conditions, kvmv1.ConditionTypeEvicting) + g.Expect(condition).NotTo(BeNil()) + g.Expect(condition.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(condition.Message).To(ContainSubstring("waiting for host allocations to drain")) + }).WithTimeout(10 * time.Second).Should(Succeed()) + + By("Clearing placement usage") + placementUsagesBody = `{"usages": {}}` + + By("Reconciling until the eviction completes") + Eventually(func(g Gomega) { + _, err := evictionReconciler.Reconcile(ctx, reconcileRequest) + g.Expect(err).NotTo(HaveOccurred()) + + resource := &kvmv1.Eviction{} + g.Expect(k8sClient.Get(ctx, typeNamespacedName, resource)).To(Succeed()) + condition := meta.FindStatusCondition(resource.Status.Conditions, kvmv1.ConditionTypeEvicting) + g.Expect(condition).NotTo(BeNil()) + g.Expect(condition.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(condition.Reason).To(Equal(kvmv1.ConditionReasonSucceeded)) + }).WithTimeout(10 * time.Second).Should(Succeed()) + }) + }) }) Context("Failure Modes", func() { diff --git a/internal/openstack/placement.go b/internal/openstack/placement.go index c73bae69..cf5975b2 100644 --- a/internal/openstack/placement.go +++ b/internal/openstack/placement.go @@ -21,6 +21,7 @@ import ( "context" "errors" "fmt" + "net/http" "github.com/gophercloud/gophercloud/v2" "github.com/gophercloud/gophercloud/v2/openstack/placement/v1/resourceproviders" @@ -123,6 +124,28 @@ func deleteConsumerAllocations(ctx context.Context, client *gophercloud.ServiceC return } +// ResourceProviderInUse reports whether the resource provider has any +// non-zero usage. A missing provider is treated as not in use. +func ResourceProviderInUse(ctx context.Context, client *gophercloud.ServiceClient, providerID string) (bool, error) { + if providerID == "" { + return false, nil + } + + usage, err := resourceproviders.GetUsages(ctx, client, providerID).Extract() + if err != nil { + if gophercloud.ResponseCodeIs(err, http.StatusNotFound) { + return false, nil + } + return false, fmt.Errorf("getting usages for resource provider %s: %w", providerID, err) + } + for _, amount := range usage.Usages { + if amount > 0 { + return true, nil + } + } + return false, nil +} + // Remove all empty Allocations for a certain provider, and if it is empty, delete it func CleanupResourceProvider(ctx context.Context, client *gophercloud.ServiceClient, provider *resourceproviders.ResourceProvider) error { if provider == nil { diff --git a/internal/openstack/placement_test.go b/internal/openstack/placement_test.go index ab836358..024953a5 100644 --- a/internal/openstack/placement_test.go +++ b/internal/openstack/placement_test.go @@ -364,4 +364,36 @@ var _ = Describe("Placement API", func() { }) }) }) + + Describe("ResourceProviderInUse", func() { + const ( + providerID = "test-provider-uuid" + usagesPattern = "/resource_providers/{provider_id}/usages" + ) + + DescribeTable("reports whether the provider has usage", + func(status int, body string, expected bool, expectErr bool) { + fakeServer.Mux.HandleFunc(usagesPattern, func(w http.ResponseWriter, r *http.Request) { + Expect(r.Method).To(Equal(http.MethodGet)) + Expect(r.PathValue("provider_id")).To(Equal(providerID)) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + fmt.Fprint(w, body) + }) + + inUse, err := ResourceProviderInUse(ctx, client.ServiceClient(fakeServer), providerID) + if expectErr { + Expect(err).To(HaveOccurred()) + return + } + Expect(err).NotTo(HaveOccurred()) + Expect(inUse).To(Equal(expected)) + }, + Entry("no usages", http.StatusOK, `{"usages": {}}`, false, false), + Entry("zero usage", http.StatusOK, `{"usages": {"VCPU": 0}}`, false, false), + Entry("positive usage", http.StatusOK, `{"usages": {"VCPU": 2}}`, true, false), + Entry("provider not found", http.StatusNotFound, `{"error": "Not Found"}`, false, false), + Entry("usage query failure", http.StatusInternalServerError, `{"error": "Internal Server Error"}`, false, true), + ) + }) })