From 94385725f6e67d9fa12fd23337d292e2d96a83a3 Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Tue, 17 Nov 2026 14:39:55 +0100 Subject: [PATCH 1/2] Propagate OpenStack errors during offboarding When an OpenStack API call failed during offboarding, setOffboardingCondition recorded the error in the status condition and returned a nil error. Once a subsequent reconcile produced the same condition message, the status update became a no-op, emitted no watch event, and offboarding stopped retrying. Return the OpenStack error so controller-runtime retries failed calls with backoff. Signed-off-by: Fabian Wiesel --- internal/controller/offboarding_controller.go | 24 +++++++------- .../controller/offboarding_controller_test.go | 33 ++++++++++++------- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/internal/controller/offboarding_controller.go b/internal/controller/offboarding_controller.go index 776aa932..03e8aff7 100644 --- a/internal/controller/offboarding_controller.go +++ b/internal/controller/offboarding_controller.go @@ -75,7 +75,7 @@ func (r *HypervisorOffboardingReconciler) Reconcile(ctx context.Context, req ctr offboardedCondition := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeOffboarded) if offboardedCondition == nil { // Start offboarding - return r.setOffboardingCondition(ctx, hv, "Hypervisor is being offboarded, removing host from nova") + return ctrl.Result{}, r.setOffboardingCondition(ctx, hv, "Hypervisor is being offboarded, removing host from nova") } if offboardedCondition.Status == metav1.ConditionTrue { @@ -104,7 +104,7 @@ func (r *HypervisorOffboardingReconciler) Reconcile(ctx context.Context, req ctr if cond := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeAgentPodsEvicted); cond != nil && cond.Message != "" { msg = "Waiting for agent pods to be evicted: " + cond.Message } - return r.setOffboardingCondition(ctx, hv, msg) + return ctrl.Result{}, r.setOffboardingCondition(ctx, hv, msg) } hypervisor, err := openstack.GetHypervisorByName(ctx, r.computeClient, hostname, true) @@ -113,7 +113,7 @@ func (r *HypervisorOffboardingReconciler) Reconcile(ctx context.Context, req ctr // We are (hopefully) done return ctrl.Result{}, r.markOffboarded(ctx, hv) } - return r.setOffboardingCondition(ctx, hv, fmt.Sprintf("cannot get hypervisor by name %s due to %s", hostname, err)) + return ctrl.Result{}, errors.Join(err, r.setOffboardingCondition(ctx, hv, fmt.Sprintf("cannot get hypervisor by name %s due to %s", hostname, err))) } // TODO: remove since RunningVMs is only available until micro-version 2.87, and also is updated asynchronously @@ -121,14 +121,14 @@ func (r *HypervisorOffboardingReconciler) Reconcile(ctx context.Context, req ctr if hypervisor.RunningVMs > 0 { // Still running VMs, cannot delete the service msg := fmt.Sprintf("Hypervisor is being offboarded, but still has %d running VMs", hypervisor.RunningVMs) - return r.setOffboardingCondition(ctx, hv, msg) + return ctrl.Result{}, r.setOffboardingCondition(ctx, hv, msg) } if hypervisor.Servers != nil && len(*hypervisor.Servers) > 0 { // Still VMs assigned to the host, cannot delete the service msg := fmt.Sprintf("Hypervisor is being offboarded, but still has %d assigned VMs, "+ "check with `openstack server list --all-projects --host %s`", len(*hypervisor.Servers), hostname) - return r.setOffboardingCondition(ctx, hv, msg) + return ctrl.Result{}, r.setOffboardingCondition(ctx, hv, msg) } // Wait for aggregates controller to remove from all aggregates @@ -138,22 +138,22 @@ func (r *HypervisorOffboardingReconciler) Reconcile(ctx context.Context, req ctr aggregateNames[i] = agg.Name } msg := fmt.Sprintf("Waiting for aggregates to be removed, current: %v", aggregateNames) - return r.setOffboardingCondition(ctx, hv, msg) + return ctrl.Result{}, r.setOffboardingCondition(ctx, hv, msg) } err = services.Delete(ctx, r.computeClient, hypervisor.Service.ID).ExtractErr() if err != nil && !gophercloud.ResponseCodeIs(err, http.StatusNotFound) { msg := fmt.Sprintf("cannot delete service %s due to %v", hypervisor.Service.ID, err) - return r.setOffboardingCondition(ctx, hv, msg) + return ctrl.Result{}, errors.Join(err, r.setOffboardingCondition(ctx, hv, msg)) } rp, err := resourceproviders.Get(ctx, r.placementClient, hypervisor.ID).Extract() if err != nil && !gophercloud.ResponseCodeIs(err, http.StatusNotFound) { - return r.setOffboardingCondition(ctx, hv, fmt.Sprintf("cannot get resource provider: %v", err)) + return ctrl.Result{}, errors.Join(err, r.setOffboardingCondition(ctx, hv, fmt.Sprintf("cannot get resource provider: %v", err))) } if err = openstack.CleanupResourceProvider(ctx, r.placementClient, rp); err != nil { - return r.setOffboardingCondition(ctx, hv, fmt.Sprintf("cannot clean up resource provider: %v", err)) + return ctrl.Result{}, errors.Join(err, r.setOffboardingCondition(ctx, hv, fmt.Sprintf("cannot clean up resource provider: %v", err))) } return ctrl.Result{}, r.markOffboarded(ctx, hv) @@ -170,16 +170,16 @@ func (r *HypervisorOffboardingReconciler) applyStatus(ctx context.Context, hv *k k8sclient.ForceOwnership, k8sclient.FieldOwner(OffboardingControllerName)) } -func (r *HypervisorOffboardingReconciler) setOffboardingCondition(ctx context.Context, hv *kvmv1.Hypervisor, message string) (ctrl.Result, error) { +func (r *HypervisorOffboardingReconciler) setOffboardingCondition(ctx context.Context, hv *kvmv1.Hypervisor, message string) error { err := r.applyStatus(ctx, hv, k8sacmetav1.Condition(). WithType(kvmv1.ConditionTypeOffboarded). WithStatus(metav1.ConditionFalse). WithReason("Offboarding"). WithMessage(message)) if err != nil { - return ctrl.Result{}, fmt.Errorf("cannot update hypervisor status due to %w", err) + return fmt.Errorf("cannot update hypervisor status due to %w", err) } - return ctrl.Result{}, nil + return nil } func (r *HypervisorOffboardingReconciler) markOffboarded(ctx context.Context, hv *kvmv1.Hypervisor) error { diff --git a/internal/controller/offboarding_controller_test.go b/internal/controller/offboarding_controller_test.go index b3b4465a..9e2dbd55 100644 --- a/internal/controller/offboarding_controller_test.go +++ b/internal/controller/offboarding_controller_test.go @@ -343,8 +343,9 @@ var _ = Describe("Offboarding Controller", func() { }) It("should set offboarding condition with error", func(ctx SpecContext) { - _, err := offboardingReconciler.Reconcile(ctx, reconcileReq) - Expect(err).NotTo(HaveOccurred()) + result, err := offboardingReconciler.Reconcile(ctx, reconcileReq) + Expect(err).To(HaveOccurred()) + Expect(result).To(Equal(ctrl.Result{})) sharedOffboardingErrorCheck(ctx, "") }) }) @@ -375,8 +376,9 @@ var _ = Describe("Offboarding Controller", func() { }) It("should set offboarding condition about running VMs", func(ctx SpecContext) { - _, err := offboardingReconciler.Reconcile(ctx, reconcileReq) + result, err := offboardingReconciler.Reconcile(ctx, reconcileReq) Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(ctrl.Result{})) sharedOffboardingErrorCheck(ctx, "still has 2 running VMs") }) }) @@ -413,8 +415,11 @@ var _ = Describe("Offboarding Controller", func() { }) It("should wait for aggregates to be cleared", func(ctx SpecContext) { - _, err := offboardingReconciler.Reconcile(ctx, reconcileReq) + result, err := offboardingReconciler.Reconcile(ctx, reconcileReq) Expect(err).NotTo(HaveOccurred()) + // Aggregates are cleared by the aggregates controller, whose changes + // are Kubernetes-watched, so no polling requeue is required. + Expect(result).To(Equal(ctrl.Result{})) sharedOffboardingErrorCheck(ctx, "Waiting for aggregates to be removed") }) }) @@ -457,8 +462,11 @@ var _ = Describe("Offboarding Controller", func() { }) It("should wait for the agent pods to be evicted and not delete the compute service", func(ctx SpecContext) { - _, err := offboardingReconciler.Reconcile(ctx, reconcileReq) + result, err := offboardingReconciler.Reconcile(ctx, reconcileReq) Expect(err).NotTo(HaveOccurred()) + // Eviction progress is Kubernetes-watched via the AgentPodsEvicted + // condition, so no polling requeue is required. + Expect(result).To(Equal(ctrl.Result{})) sharedOffboardingErrorCheck(ctx, "Waiting for agent pods to be evicted") Expect(getHypervisorsCalled).To(Equal(0), "external hypervisor lookup must be short-circuited by the AgentPodsEvicted gate") @@ -493,8 +501,9 @@ var _ = Describe("Offboarding Controller", func() { }) It("should set offboarding condition with error", func(ctx SpecContext) { - _, err := offboardingReconciler.Reconcile(ctx, reconcileReq) - Expect(err).NotTo(HaveOccurred()) + result, err := offboardingReconciler.Reconcile(ctx, reconcileReq) + Expect(err).To(HaveOccurred()) + Expect(result).To(Equal(ctrl.Result{})) sharedOffboardingErrorCheck(ctx, "cannot delete service") }) }) @@ -531,8 +540,9 @@ var _ = Describe("Offboarding Controller", func() { }) It("should set offboarding condition with error", func(ctx SpecContext) { - _, err := offboardingReconciler.Reconcile(ctx, reconcileReq) - Expect(err).NotTo(HaveOccurred()) + result, err := offboardingReconciler.Reconcile(ctx, reconcileReq) + Expect(err).To(HaveOccurred()) + Expect(result).To(Equal(ctrl.Result{})) sharedOffboardingErrorCheck(ctx, "cannot get resource provider") }) }) @@ -581,8 +591,9 @@ var _ = Describe("Offboarding Controller", func() { }) It("should set offboarding condition with error", func(ctx SpecContext) { - _, err := offboardingReconciler.Reconcile(ctx, reconcileReq) - Expect(err).NotTo(HaveOccurred()) + result, err := offboardingReconciler.Reconcile(ctx, reconcileReq) + Expect(err).To(HaveOccurred()) + Expect(result).To(Equal(ctrl.Result{})) sharedOffboardingErrorCheck(ctx, "cannot clean up resource provider") }) }) From 9c910959e3c260d5a551ccc268e375708b8554df Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Tue, 17 Nov 2026 14:42:11 +0100 Subject: [PATCH 2/2] Clean up resource provider even after compute service deletion Deleting the compute service also removes the hypervisor record in Nova. If resource provider cleanup fails on the same reconcile, the next reconcile receives ErrNoHypervisor and previously marked the hypervisor offboarded without cleaning up Placement. Fall back to Status.HypervisorID so the resource provider is still cleaned up, and skip cleanup when Placement already returns NotFound. Signed-off-by: Fabian Wiesel --- internal/controller/offboarding_controller.go | 32 ++- .../controller/offboarding_controller_test.go | 195 ++++++++++++++++++ 2 files changed, 220 insertions(+), 7 deletions(-) diff --git a/internal/controller/offboarding_controller.go b/internal/controller/offboarding_controller.go index 03e8aff7..5606ca72 100644 --- a/internal/controller/offboarding_controller.go +++ b/internal/controller/offboarding_controller.go @@ -110,7 +110,11 @@ func (r *HypervisorOffboardingReconciler) Reconcile(ctx context.Context, req ctr hypervisor, err := openstack.GetHypervisorByName(ctx, r.computeClient, hostname, true) if err != nil { if errors.Is(err, openstack.ErrNoHypervisor) { - // We are (hopefully) done + // Deleting the compute service also removed its hypervisor record, + // so only the resource provider may remain to be cleaned up. + if err := r.cleanupResourceProvider(ctx, hv.Status.HypervisorID); err != nil { + return ctrl.Result{}, errors.Join(err, r.setOffboardingCondition(ctx, hv, err.Error())) + } return ctrl.Result{}, r.markOffboarded(ctx, hv) } return ctrl.Result{}, errors.Join(err, r.setOffboardingCondition(ctx, hv, fmt.Sprintf("cannot get hypervisor by name %s due to %s", hostname, err))) @@ -147,16 +151,30 @@ func (r *HypervisorOffboardingReconciler) Reconcile(ctx context.Context, req ctr return ctrl.Result{}, errors.Join(err, r.setOffboardingCondition(ctx, hv, msg)) } - rp, err := resourceproviders.Get(ctx, r.placementClient, hypervisor.ID).Extract() - if err != nil && !gophercloud.ResponseCodeIs(err, http.StatusNotFound) { - return ctrl.Result{}, errors.Join(err, r.setOffboardingCondition(ctx, hv, fmt.Sprintf("cannot get resource provider: %v", err))) + if err := r.cleanupResourceProvider(ctx, hypervisor.ID); err != nil { + return ctrl.Result{}, errors.Join(err, r.setOffboardingCondition(ctx, hv, err.Error())) } + return ctrl.Result{}, r.markOffboarded(ctx, hv) +} + +// cleanupResourceProvider removes the Placement resource provider belonging to +// the given hypervisor. A missing resource provider is not an error. +func (r *HypervisorOffboardingReconciler) cleanupResourceProvider(ctx context.Context, hypervisorID string) error { + if hypervisorID == "" { + return nil + } + rp, err := resourceproviders.Get(ctx, r.placementClient, hypervisorID).Extract() + if gophercloud.ResponseCodeIs(err, http.StatusNotFound) { + return nil + } + if err != nil { + return fmt.Errorf("cannot get resource provider: %w", err) + } if err = openstack.CleanupResourceProvider(ctx, r.placementClient, rp); err != nil { - return ctrl.Result{}, errors.Join(err, r.setOffboardingCondition(ctx, hv, fmt.Sprintf("cannot clean up resource provider: %v", err))) + return fmt.Errorf("cannot clean up resource provider: %w", err) } - - return ctrl.Result{}, r.markOffboarded(ctx, hv) + return nil } func (r *HypervisorOffboardingReconciler) applyStatus(ctx context.Context, hv *kvmv1.Hypervisor, cond *k8sacmetav1.ConditionApplyConfiguration) error { diff --git a/internal/controller/offboarding_controller_test.go b/internal/controller/offboarding_controller_test.go index 9e2dbd55..ac12d09e 100644 --- a/internal/controller/offboarding_controller_test.go +++ b/internal/controller/offboarding_controller_test.go @@ -597,5 +597,200 @@ var _ = Describe("Offboarding Controller", func() { sharedOffboardingErrorCheck(ctx, "cannot clean up resource provider") }) }) + + Context("When the compute service was already deleted in a previous reconcile", func() { + // hypervisorID matches the ID in the HypervisorWithServers const + const hypervisorID = "c48f6247-abe4-4a24-824e-ea39e108874f" + + BeforeEach(func(ctx SpecContext) { + By("Setting the IDs stored by a previous reconcile") + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, resourceName, hypervisor)).To(Succeed()) + hypervisor.Status.ServiceID = serviceId + hypervisor.Status.HypervisorID = hypervisorID + meta.SetStatusCondition(&hypervisor.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeEvicting, + Status: metav1.ConditionFalse, + Reason: "dontcare", + Message: "dontcare", + }) + Expect(k8sClient.Status().Update(ctx, hypervisor)).To(Succeed()) + }) + + Context("When the resource provider delete fails once and then succeeds", func() { + var ( + listHypervisorsCalls int + deleteRPCalled int + ) + + BeforeEach(func() { + listHypervisorsCalls = 0 + deleteRPCalled = 0 + + // First list call returns the hypervisor, later ones an + // empty list, as Nova does after the service was deleted. + fakeServer.Mux.HandleFunc("GET /os-hypervisors/detail", func(w http.ResponseWriter, r *http.Request) { + listHypervisorsCalls++ + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + if listHypervisorsCalls == 1 { + _, err := fmt.Fprintf(w, HypervisorWithServers, serviceId, "", hypervisorName) + Expect(err).NotTo(HaveOccurred()) + } else { + _, err := fmt.Fprint(w, `{"hypervisors": []}`) + Expect(err).NotTo(HaveOccurred()) + } + }) + + fakeServer.Mux.HandleFunc("GET /resource_providers/"+hypervisorID, func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err := fmt.Fprint(w, `{"uuid": "rp-uuid", "name": "hv-test"}`) + Expect(err).NotTo(HaveOccurred()) + }) + + fakeServer.Mux.HandleFunc("GET /resource_providers/rp-uuid/allocations", func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err := fmt.Fprint(w, `{"allocations": {}}`) + Expect(err).NotTo(HaveOccurred()) + }) + + fakeServer.Mux.HandleFunc("DELETE /resource_providers/rp-uuid", func(w http.ResponseWriter, r *http.Request) { + deleteRPCalled++ + if deleteRPCalled == 1 { + w.WriteHeader(http.StatusInternalServerError) + _, err := fmt.Fprint(w, `{"error": "Internal Server Error"}`) + Expect(err).NotTo(HaveOccurred()) + } else { + w.WriteHeader(http.StatusAccepted) + } + }) + + fakeServer.Mux.HandleFunc("DELETE /os-services/"+serviceId, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + }) + }) + + It("should clean up the resource provider and only then mark the hypervisor offboarded", func(ctx SpecContext) { + By("Reconciling while the hypervisor still exists") + result, err := offboardingReconciler.Reconcile(ctx, reconcileReq) + Expect(err).To(HaveOccurred()) + Expect(result).To(Equal(ctrl.Result{})) + sharedOffboardingErrorCheck(ctx, "cannot clean up resource provider") + Expect(deleteRPCalled).To(Equal(1)) + + By("Reconciling after the compute service was deleted") + result, err = offboardingReconciler.Reconcile(ctx, reconcileReq) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(ctrl.Result{})) + // The hypervisor no longer exists in Nova, but the resource + // provider must still be deleted before offboarding is done. + Expect(deleteRPCalled).To(Equal(2)) + + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, resourceName, hypervisor)).To(Succeed()) + Expect(hypervisor.Status.Conditions).To(ContainElement( + SatisfyAll( + HaveField("Type", kvmv1.ConditionTypeOffboarded), + HaveField("Status", metav1.ConditionTrue), + HaveField("Reason", "Offboarded"), + HaveField("Message", "Offboarding successful"), + ), + )) + }) + }) + + Context("When the resource provider delete keeps failing", func() { + var deleteRPCalled int + + BeforeEach(func() { + deleteRPCalled = 0 + + fakeServer.Mux.HandleFunc("GET /os-hypervisors/detail", func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err := fmt.Fprint(w, `{"hypervisors": []}`) + Expect(err).NotTo(HaveOccurred()) + }) + + fakeServer.Mux.HandleFunc("GET /resource_providers/"+hypervisorID, func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err := fmt.Fprint(w, `{"uuid": "rp-uuid", "name": "hv-test"}`) + Expect(err).NotTo(HaveOccurred()) + }) + + fakeServer.Mux.HandleFunc("GET /resource_providers/rp-uuid/allocations", func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err := fmt.Fprint(w, `{"allocations": {}}`) + Expect(err).NotTo(HaveOccurred()) + }) + + fakeServer.Mux.HandleFunc("DELETE /resource_providers/rp-uuid", func(w http.ResponseWriter, r *http.Request) { + deleteRPCalled++ + w.WriteHeader(http.StatusInternalServerError) + _, err := fmt.Fprint(w, `{"error": "Internal Server Error"}`) + Expect(err).NotTo(HaveOccurred()) + }) + }) + + It("should return an error and not mark the hypervisor offboarded", func(ctx SpecContext) { + _, err := offboardingReconciler.Reconcile(ctx, reconcileReq) + Expect(err).To(HaveOccurred()) + + By("Reconciling again while the resource provider delete still fails") + result, err := offboardingReconciler.Reconcile(ctx, reconcileReq) + Expect(err).To(HaveOccurred()) + Expect(result).To(Equal(ctrl.Result{})) + sharedOffboardingErrorCheck(ctx, "cannot clean up resource provider") + Expect(deleteRPCalled).To(Equal(2)) + }) + }) + + Context("When the resource provider is already gone", func() { + var deleteRPCalled int + + BeforeEach(func() { + deleteRPCalled = 0 + + fakeServer.Mux.HandleFunc("GET /os-hypervisors/detail", func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err := fmt.Fprint(w, `{"hypervisors": []}`) + Expect(err).NotTo(HaveOccurred()) + }) + + fakeServer.Mux.HandleFunc("GET /resource_providers/"+hypervisorID, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + _, err := fmt.Fprint(w, `{"error": "Not Found"}`) + Expect(err).NotTo(HaveOccurred()) + }) + + fakeServer.Mux.HandleFunc("DELETE /resource_providers/rp-uuid", func(w http.ResponseWriter, r *http.Request) { + deleteRPCalled++ + w.WriteHeader(http.StatusAccepted) + }) + }) + + It("should mark the hypervisor offboarded without deleting anything", func(ctx SpecContext) { + result, err := offboardingReconciler.Reconcile(ctx, reconcileReq) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(ctrl.Result{})) + Expect(deleteRPCalled).To(BeZero()) + + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, resourceName, hypervisor)).To(Succeed()) + Expect(hypervisor.Status.Conditions).To(ContainElement( + SatisfyAll( + HaveField("Type", kvmv1.ConditionTypeOffboarded), + HaveField("Status", metav1.ConditionTrue), + HaveField("Reason", "Offboarded"), + ), + )) + }) + }) + }) }) })