diff --git a/internal/controller/offboarding_controller.go b/internal/controller/offboarding_controller.go index 776aa932..5606ca72 100644 --- a/internal/controller/offboarding_controller.go +++ b/internal/controller/offboarding_controller.go @@ -75,7 +75,7 @@ func (r *HypervisorOffboardingReconciler) Reconcile(ctx context.Context, req ctr offboardedCondition := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeOffboarded) if offboardedCondition == nil { // Start offboarding - return r.setOffboardingCondition(ctx, hv, "Hypervisor is being offboarded, removing host from nova") + return ctrl.Result{}, r.setOffboardingCondition(ctx, hv, "Hypervisor is being offboarded, removing host from nova") } if offboardedCondition.Status == metav1.ConditionTrue { @@ -104,16 +104,20 @@ func (r *HypervisorOffboardingReconciler) Reconcile(ctx context.Context, req ctr if cond := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeAgentPodsEvicted); cond != nil && cond.Message != "" { msg = "Waiting for agent pods to be evicted: " + cond.Message } - return r.setOffboardingCondition(ctx, hv, msg) + return ctrl.Result{}, r.setOffboardingCondition(ctx, hv, msg) } hypervisor, err := openstack.GetHypervisorByName(ctx, r.computeClient, hostname, true) if err != nil { if errors.Is(err, openstack.ErrNoHypervisor) { - // We are (hopefully) done + // Deleting the compute service also removed its hypervisor record, + // so only the resource provider may remain to be cleaned up. + if err := r.cleanupResourceProvider(ctx, hv.Status.HypervisorID); err != nil { + return ctrl.Result{}, errors.Join(err, r.setOffboardingCondition(ctx, hv, err.Error())) + } return ctrl.Result{}, r.markOffboarded(ctx, hv) } - return r.setOffboardingCondition(ctx, hv, fmt.Sprintf("cannot get hypervisor by name %s due to %s", hostname, err)) + return ctrl.Result{}, errors.Join(err, r.setOffboardingCondition(ctx, hv, fmt.Sprintf("cannot get hypervisor by name %s due to %s", hostname, err))) } // TODO: remove since RunningVMs is only available until micro-version 2.87, and also is updated asynchronously @@ -121,14 +125,14 @@ func (r *HypervisorOffboardingReconciler) Reconcile(ctx context.Context, req ctr if hypervisor.RunningVMs > 0 { // Still running VMs, cannot delete the service msg := fmt.Sprintf("Hypervisor is being offboarded, but still has %d running VMs", hypervisor.RunningVMs) - return r.setOffboardingCondition(ctx, hv, msg) + return ctrl.Result{}, r.setOffboardingCondition(ctx, hv, msg) } if hypervisor.Servers != nil && len(*hypervisor.Servers) > 0 { // Still VMs assigned to the host, cannot delete the service msg := fmt.Sprintf("Hypervisor is being offboarded, but still has %d assigned VMs, "+ "check with `openstack server list --all-projects --host %s`", len(*hypervisor.Servers), hostname) - return r.setOffboardingCondition(ctx, hv, msg) + return ctrl.Result{}, r.setOffboardingCondition(ctx, hv, msg) } // Wait for aggregates controller to remove from all aggregates @@ -138,25 +142,39 @@ func (r *HypervisorOffboardingReconciler) Reconcile(ctx context.Context, req ctr aggregateNames[i] = agg.Name } msg := fmt.Sprintf("Waiting for aggregates to be removed, current: %v", aggregateNames) - return r.setOffboardingCondition(ctx, hv, msg) + return ctrl.Result{}, r.setOffboardingCondition(ctx, hv, msg) } err = services.Delete(ctx, r.computeClient, hypervisor.Service.ID).ExtractErr() if err != nil && !gophercloud.ResponseCodeIs(err, http.StatusNotFound) { msg := fmt.Sprintf("cannot delete service %s due to %v", hypervisor.Service.ID, err) - return r.setOffboardingCondition(ctx, hv, msg) + return ctrl.Result{}, errors.Join(err, r.setOffboardingCondition(ctx, hv, msg)) } - rp, err := resourceproviders.Get(ctx, r.placementClient, hypervisor.ID).Extract() - if err != nil && !gophercloud.ResponseCodeIs(err, http.StatusNotFound) { - return r.setOffboardingCondition(ctx, hv, fmt.Sprintf("cannot get resource provider: %v", err)) + if err := r.cleanupResourceProvider(ctx, hypervisor.ID); err != nil { + return ctrl.Result{}, errors.Join(err, r.setOffboardingCondition(ctx, hv, err.Error())) } + return ctrl.Result{}, r.markOffboarded(ctx, hv) +} + +// cleanupResourceProvider removes the Placement resource provider belonging to +// the given hypervisor. A missing resource provider is not an error. +func (r *HypervisorOffboardingReconciler) cleanupResourceProvider(ctx context.Context, hypervisorID string) error { + if hypervisorID == "" { + return nil + } + rp, err := resourceproviders.Get(ctx, r.placementClient, hypervisorID).Extract() + if gophercloud.ResponseCodeIs(err, http.StatusNotFound) { + return nil + } + if err != nil { + return fmt.Errorf("cannot get resource provider: %w", err) + } if err = openstack.CleanupResourceProvider(ctx, r.placementClient, rp); err != nil { - return r.setOffboardingCondition(ctx, hv, fmt.Sprintf("cannot clean up resource provider: %v", err)) + return fmt.Errorf("cannot clean up resource provider: %w", err) } - - return ctrl.Result{}, r.markOffboarded(ctx, hv) + return nil } func (r *HypervisorOffboardingReconciler) applyStatus(ctx context.Context, hv *kvmv1.Hypervisor, cond *k8sacmetav1.ConditionApplyConfiguration) error { @@ -170,16 +188,16 @@ func (r *HypervisorOffboardingReconciler) applyStatus(ctx context.Context, hv *k k8sclient.ForceOwnership, k8sclient.FieldOwner(OffboardingControllerName)) } -func (r *HypervisorOffboardingReconciler) setOffboardingCondition(ctx context.Context, hv *kvmv1.Hypervisor, message string) (ctrl.Result, error) { +func (r *HypervisorOffboardingReconciler) setOffboardingCondition(ctx context.Context, hv *kvmv1.Hypervisor, message string) error { err := r.applyStatus(ctx, hv, k8sacmetav1.Condition(). WithType(kvmv1.ConditionTypeOffboarded). WithStatus(metav1.ConditionFalse). WithReason("Offboarding"). WithMessage(message)) if err != nil { - return ctrl.Result{}, fmt.Errorf("cannot update hypervisor status due to %w", err) + return fmt.Errorf("cannot update hypervisor status due to %w", err) } - return ctrl.Result{}, nil + return nil } func (r *HypervisorOffboardingReconciler) markOffboarded(ctx context.Context, hv *kvmv1.Hypervisor) error { diff --git a/internal/controller/offboarding_controller_test.go b/internal/controller/offboarding_controller_test.go index b3b4465a..ac12d09e 100644 --- a/internal/controller/offboarding_controller_test.go +++ b/internal/controller/offboarding_controller_test.go @@ -343,8 +343,9 @@ var _ = Describe("Offboarding Controller", func() { }) It("should set offboarding condition with error", func(ctx SpecContext) { - _, err := offboardingReconciler.Reconcile(ctx, reconcileReq) - Expect(err).NotTo(HaveOccurred()) + result, err := offboardingReconciler.Reconcile(ctx, reconcileReq) + Expect(err).To(HaveOccurred()) + Expect(result).To(Equal(ctrl.Result{})) sharedOffboardingErrorCheck(ctx, "") }) }) @@ -375,8 +376,9 @@ var _ = Describe("Offboarding Controller", func() { }) It("should set offboarding condition about running VMs", func(ctx SpecContext) { - _, err := offboardingReconciler.Reconcile(ctx, reconcileReq) + result, err := offboardingReconciler.Reconcile(ctx, reconcileReq) Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(ctrl.Result{})) sharedOffboardingErrorCheck(ctx, "still has 2 running VMs") }) }) @@ -413,8 +415,11 @@ var _ = Describe("Offboarding Controller", func() { }) It("should wait for aggregates to be cleared", func(ctx SpecContext) { - _, err := offboardingReconciler.Reconcile(ctx, reconcileReq) + result, err := offboardingReconciler.Reconcile(ctx, reconcileReq) Expect(err).NotTo(HaveOccurred()) + // Aggregates are cleared by the aggregates controller, whose changes + // are Kubernetes-watched, so no polling requeue is required. + Expect(result).To(Equal(ctrl.Result{})) sharedOffboardingErrorCheck(ctx, "Waiting for aggregates to be removed") }) }) @@ -457,8 +462,11 @@ var _ = Describe("Offboarding Controller", func() { }) It("should wait for the agent pods to be evicted and not delete the compute service", func(ctx SpecContext) { - _, err := offboardingReconciler.Reconcile(ctx, reconcileReq) + result, err := offboardingReconciler.Reconcile(ctx, reconcileReq) Expect(err).NotTo(HaveOccurred()) + // Eviction progress is Kubernetes-watched via the AgentPodsEvicted + // condition, so no polling requeue is required. + Expect(result).To(Equal(ctrl.Result{})) sharedOffboardingErrorCheck(ctx, "Waiting for agent pods to be evicted") Expect(getHypervisorsCalled).To(Equal(0), "external hypervisor lookup must be short-circuited by the AgentPodsEvicted gate") @@ -493,8 +501,9 @@ var _ = Describe("Offboarding Controller", func() { }) It("should set offboarding condition with error", func(ctx SpecContext) { - _, err := offboardingReconciler.Reconcile(ctx, reconcileReq) - Expect(err).NotTo(HaveOccurred()) + result, err := offboardingReconciler.Reconcile(ctx, reconcileReq) + Expect(err).To(HaveOccurred()) + Expect(result).To(Equal(ctrl.Result{})) sharedOffboardingErrorCheck(ctx, "cannot delete service") }) }) @@ -531,8 +540,9 @@ var _ = Describe("Offboarding Controller", func() { }) It("should set offboarding condition with error", func(ctx SpecContext) { - _, err := offboardingReconciler.Reconcile(ctx, reconcileReq) - Expect(err).NotTo(HaveOccurred()) + result, err := offboardingReconciler.Reconcile(ctx, reconcileReq) + Expect(err).To(HaveOccurred()) + Expect(result).To(Equal(ctrl.Result{})) sharedOffboardingErrorCheck(ctx, "cannot get resource provider") }) }) @@ -581,10 +591,206 @@ var _ = Describe("Offboarding Controller", func() { }) It("should set offboarding condition with error", func(ctx SpecContext) { - _, err := offboardingReconciler.Reconcile(ctx, reconcileReq) - Expect(err).NotTo(HaveOccurred()) + result, err := offboardingReconciler.Reconcile(ctx, reconcileReq) + Expect(err).To(HaveOccurred()) + Expect(result).To(Equal(ctrl.Result{})) sharedOffboardingErrorCheck(ctx, "cannot clean up resource provider") }) }) + + Context("When the compute service was already deleted in a previous reconcile", func() { + // hypervisorID matches the ID in the HypervisorWithServers const + const hypervisorID = "c48f6247-abe4-4a24-824e-ea39e108874f" + + BeforeEach(func(ctx SpecContext) { + By("Setting the IDs stored by a previous reconcile") + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, resourceName, hypervisor)).To(Succeed()) + hypervisor.Status.ServiceID = serviceId + hypervisor.Status.HypervisorID = hypervisorID + meta.SetStatusCondition(&hypervisor.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeEvicting, + Status: metav1.ConditionFalse, + Reason: "dontcare", + Message: "dontcare", + }) + Expect(k8sClient.Status().Update(ctx, hypervisor)).To(Succeed()) + }) + + Context("When the resource provider delete fails once and then succeeds", func() { + var ( + listHypervisorsCalls int + deleteRPCalled int + ) + + BeforeEach(func() { + listHypervisorsCalls = 0 + deleteRPCalled = 0 + + // First list call returns the hypervisor, later ones an + // empty list, as Nova does after the service was deleted. + fakeServer.Mux.HandleFunc("GET /os-hypervisors/detail", func(w http.ResponseWriter, r *http.Request) { + listHypervisorsCalls++ + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + if listHypervisorsCalls == 1 { + _, err := fmt.Fprintf(w, HypervisorWithServers, serviceId, "", hypervisorName) + Expect(err).NotTo(HaveOccurred()) + } else { + _, err := fmt.Fprint(w, `{"hypervisors": []}`) + Expect(err).NotTo(HaveOccurred()) + } + }) + + fakeServer.Mux.HandleFunc("GET /resource_providers/"+hypervisorID, func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err := fmt.Fprint(w, `{"uuid": "rp-uuid", "name": "hv-test"}`) + Expect(err).NotTo(HaveOccurred()) + }) + + fakeServer.Mux.HandleFunc("GET /resource_providers/rp-uuid/allocations", func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err := fmt.Fprint(w, `{"allocations": {}}`) + Expect(err).NotTo(HaveOccurred()) + }) + + fakeServer.Mux.HandleFunc("DELETE /resource_providers/rp-uuid", func(w http.ResponseWriter, r *http.Request) { + deleteRPCalled++ + if deleteRPCalled == 1 { + w.WriteHeader(http.StatusInternalServerError) + _, err := fmt.Fprint(w, `{"error": "Internal Server Error"}`) + Expect(err).NotTo(HaveOccurred()) + } else { + w.WriteHeader(http.StatusAccepted) + } + }) + + fakeServer.Mux.HandleFunc("DELETE /os-services/"+serviceId, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + }) + }) + + It("should clean up the resource provider and only then mark the hypervisor offboarded", func(ctx SpecContext) { + By("Reconciling while the hypervisor still exists") + result, err := offboardingReconciler.Reconcile(ctx, reconcileReq) + Expect(err).To(HaveOccurred()) + Expect(result).To(Equal(ctrl.Result{})) + sharedOffboardingErrorCheck(ctx, "cannot clean up resource provider") + Expect(deleteRPCalled).To(Equal(1)) + + By("Reconciling after the compute service was deleted") + result, err = offboardingReconciler.Reconcile(ctx, reconcileReq) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(ctrl.Result{})) + // The hypervisor no longer exists in Nova, but the resource + // provider must still be deleted before offboarding is done. + Expect(deleteRPCalled).To(Equal(2)) + + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, resourceName, hypervisor)).To(Succeed()) + Expect(hypervisor.Status.Conditions).To(ContainElement( + SatisfyAll( + HaveField("Type", kvmv1.ConditionTypeOffboarded), + HaveField("Status", metav1.ConditionTrue), + HaveField("Reason", "Offboarded"), + HaveField("Message", "Offboarding successful"), + ), + )) + }) + }) + + Context("When the resource provider delete keeps failing", func() { + var deleteRPCalled int + + BeforeEach(func() { + deleteRPCalled = 0 + + fakeServer.Mux.HandleFunc("GET /os-hypervisors/detail", func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err := fmt.Fprint(w, `{"hypervisors": []}`) + Expect(err).NotTo(HaveOccurred()) + }) + + fakeServer.Mux.HandleFunc("GET /resource_providers/"+hypervisorID, func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err := fmt.Fprint(w, `{"uuid": "rp-uuid", "name": "hv-test"}`) + Expect(err).NotTo(HaveOccurred()) + }) + + fakeServer.Mux.HandleFunc("GET /resource_providers/rp-uuid/allocations", func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err := fmt.Fprint(w, `{"allocations": {}}`) + Expect(err).NotTo(HaveOccurred()) + }) + + fakeServer.Mux.HandleFunc("DELETE /resource_providers/rp-uuid", func(w http.ResponseWriter, r *http.Request) { + deleteRPCalled++ + w.WriteHeader(http.StatusInternalServerError) + _, err := fmt.Fprint(w, `{"error": "Internal Server Error"}`) + Expect(err).NotTo(HaveOccurred()) + }) + }) + + It("should return an error and not mark the hypervisor offboarded", func(ctx SpecContext) { + _, err := offboardingReconciler.Reconcile(ctx, reconcileReq) + Expect(err).To(HaveOccurred()) + + By("Reconciling again while the resource provider delete still fails") + result, err := offboardingReconciler.Reconcile(ctx, reconcileReq) + Expect(err).To(HaveOccurred()) + Expect(result).To(Equal(ctrl.Result{})) + sharedOffboardingErrorCheck(ctx, "cannot clean up resource provider") + Expect(deleteRPCalled).To(Equal(2)) + }) + }) + + Context("When the resource provider is already gone", func() { + var deleteRPCalled int + + BeforeEach(func() { + deleteRPCalled = 0 + + fakeServer.Mux.HandleFunc("GET /os-hypervisors/detail", func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err := fmt.Fprint(w, `{"hypervisors": []}`) + Expect(err).NotTo(HaveOccurred()) + }) + + fakeServer.Mux.HandleFunc("GET /resource_providers/"+hypervisorID, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + _, err := fmt.Fprint(w, `{"error": "Not Found"}`) + Expect(err).NotTo(HaveOccurred()) + }) + + fakeServer.Mux.HandleFunc("DELETE /resource_providers/rp-uuid", func(w http.ResponseWriter, r *http.Request) { + deleteRPCalled++ + w.WriteHeader(http.StatusAccepted) + }) + }) + + It("should mark the hypervisor offboarded without deleting anything", func(ctx SpecContext) { + result, err := offboardingReconciler.Reconcile(ctx, reconcileReq) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(ctrl.Result{})) + Expect(deleteRPCalled).To(BeZero()) + + hypervisor := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, resourceName, hypervisor)).To(Succeed()) + Expect(hypervisor.Status.Conditions).To(ContainElement( + SatisfyAll( + HaveField("Type", kvmv1.ConditionTypeOffboarded), + HaveField("Status", metav1.ConditionTrue), + HaveField("Reason", "Offboarded"), + ), + )) + }) + }) + }) }) })