Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 35 additions & 17 deletions internal/controller/offboarding_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ func (r *HypervisorOffboardingReconciler) Reconcile(ctx context.Context, req ctr
offboardedCondition := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeOffboarded)
if offboardedCondition == nil {
// Start offboarding
return r.setOffboardingCondition(ctx, hv, "Hypervisor is being offboarded, removing host from nova")
return ctrl.Result{}, r.setOffboardingCondition(ctx, hv, "Hypervisor is being offboarded, removing host from nova")
}

if offboardedCondition.Status == metav1.ConditionTrue {
Expand Down Expand Up @@ -104,31 +104,35 @@ func (r *HypervisorOffboardingReconciler) Reconcile(ctx context.Context, req ctr
if cond := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeAgentPodsEvicted); cond != nil && cond.Message != "" {
msg = "Waiting for agent pods to be evicted: " + cond.Message
}
return r.setOffboardingCondition(ctx, hv, msg)
return ctrl.Result{}, r.setOffboardingCondition(ctx, hv, msg)
}

hypervisor, err := openstack.GetHypervisorByName(ctx, r.computeClient, hostname, true)
if err != nil {
if errors.Is(err, openstack.ErrNoHypervisor) {
// We are (hopefully) done
// Deleting the compute service also removed its hypervisor record,
// so only the resource provider may remain to be cleaned up.
if err := r.cleanupResourceProvider(ctx, hv.Status.HypervisorID); err != nil {
return ctrl.Result{}, errors.Join(err, r.setOffboardingCondition(ctx, hv, err.Error()))
}
return ctrl.Result{}, r.markOffboarded(ctx, hv)
}
return r.setOffboardingCondition(ctx, hv, fmt.Sprintf("cannot get hypervisor by name %s due to %s", hostname, err))
return ctrl.Result{}, errors.Join(err, r.setOffboardingCondition(ctx, hv, fmt.Sprintf("cannot get hypervisor by name %s due to %s", hostname, err)))
}

// TODO: remove since RunningVMs is only available until micro-version 2.87, and also is updated asynchronously
// so it might be not accurate
if hypervisor.RunningVMs > 0 {
// Still running VMs, cannot delete the service
msg := fmt.Sprintf("Hypervisor is being offboarded, but still has %d running VMs", hypervisor.RunningVMs)
return r.setOffboardingCondition(ctx, hv, msg)
return ctrl.Result{}, r.setOffboardingCondition(ctx, hv, msg)
}

if hypervisor.Servers != nil && len(*hypervisor.Servers) > 0 {
// Still VMs assigned to the host, cannot delete the service
msg := fmt.Sprintf("Hypervisor is being offboarded, but still has %d assigned VMs, "+
"check with `openstack server list --all-projects --host %s`", len(*hypervisor.Servers), hostname)
return r.setOffboardingCondition(ctx, hv, msg)
return ctrl.Result{}, r.setOffboardingCondition(ctx, hv, msg)
}

// Wait for aggregates controller to remove from all aggregates
Expand All @@ -138,25 +142,39 @@ func (r *HypervisorOffboardingReconciler) Reconcile(ctx context.Context, req ctr
aggregateNames[i] = agg.Name
}
msg := fmt.Sprintf("Waiting for aggregates to be removed, current: %v", aggregateNames)
return r.setOffboardingCondition(ctx, hv, msg)
return ctrl.Result{}, r.setOffboardingCondition(ctx, hv, msg)
}

err = services.Delete(ctx, r.computeClient, hypervisor.Service.ID).ExtractErr()
if err != nil && !gophercloud.ResponseCodeIs(err, http.StatusNotFound) {
msg := fmt.Sprintf("cannot delete service %s due to %v", hypervisor.Service.ID, err)
return r.setOffboardingCondition(ctx, hv, msg)
return ctrl.Result{}, errors.Join(err, r.setOffboardingCondition(ctx, hv, msg))
}

rp, err := resourceproviders.Get(ctx, r.placementClient, hypervisor.ID).Extract()
if err != nil && !gophercloud.ResponseCodeIs(err, http.StatusNotFound) {
return r.setOffboardingCondition(ctx, hv, fmt.Sprintf("cannot get resource provider: %v", err))
if err := r.cleanupResourceProvider(ctx, hypervisor.ID); err != nil {
return ctrl.Result{}, errors.Join(err, r.setOffboardingCondition(ctx, hv, err.Error()))
}

return ctrl.Result{}, r.markOffboarded(ctx, hv)
}

// cleanupResourceProvider removes the Placement resource provider belonging to
// the given hypervisor. A missing resource provider is not an error.
func (r *HypervisorOffboardingReconciler) cleanupResourceProvider(ctx context.Context, hypervisorID string) error {
if hypervisorID == "" {
return nil
}
rp, err := resourceproviders.Get(ctx, r.placementClient, hypervisorID).Extract()
if gophercloud.ResponseCodeIs(err, http.StatusNotFound) {
return nil
}
if err != nil {
return fmt.Errorf("cannot get resource provider: %w", err)
}
if err = openstack.CleanupResourceProvider(ctx, r.placementClient, rp); err != nil {
return r.setOffboardingCondition(ctx, hv, fmt.Sprintf("cannot clean up resource provider: %v", err))
return fmt.Errorf("cannot clean up resource provider: %w", err)
}

return ctrl.Result{}, r.markOffboarded(ctx, hv)
return nil
}

func (r *HypervisorOffboardingReconciler) applyStatus(ctx context.Context, hv *kvmv1.Hypervisor, cond *k8sacmetav1.ConditionApplyConfiguration) error {
Expand All @@ -170,16 +188,16 @@ func (r *HypervisorOffboardingReconciler) applyStatus(ctx context.Context, hv *k
k8sclient.ForceOwnership, k8sclient.FieldOwner(OffboardingControllerName))
}

func (r *HypervisorOffboardingReconciler) setOffboardingCondition(ctx context.Context, hv *kvmv1.Hypervisor, message string) (ctrl.Result, error) {
func (r *HypervisorOffboardingReconciler) setOffboardingCondition(ctx context.Context, hv *kvmv1.Hypervisor, message string) error {
err := r.applyStatus(ctx, hv, k8sacmetav1.Condition().
WithType(kvmv1.ConditionTypeOffboarded).
WithStatus(metav1.ConditionFalse).
WithReason("Offboarding").
WithMessage(message))
if err != nil {
return ctrl.Result{}, fmt.Errorf("cannot update hypervisor status due to %w", err)
return fmt.Errorf("cannot update hypervisor status due to %w", err)
}
return ctrl.Result{}, nil
return nil
}

func (r *HypervisorOffboardingReconciler) markOffboarded(ctx context.Context, hv *kvmv1.Hypervisor) error {
Expand Down
228 changes: 217 additions & 11 deletions internal/controller/offboarding_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -343,8 +343,9 @@ var _ = Describe("Offboarding Controller", func() {
})

It("should set offboarding condition with error", func(ctx SpecContext) {
_, err := offboardingReconciler.Reconcile(ctx, reconcileReq)
Expect(err).NotTo(HaveOccurred())
result, err := offboardingReconciler.Reconcile(ctx, reconcileReq)
Expect(err).To(HaveOccurred())
Expect(result).To(Equal(ctrl.Result{}))
sharedOffboardingErrorCheck(ctx, "")
})
})
Expand Down Expand Up @@ -375,8 +376,9 @@ var _ = Describe("Offboarding Controller", func() {
})

It("should set offboarding condition about running VMs", func(ctx SpecContext) {
_, err := offboardingReconciler.Reconcile(ctx, reconcileReq)
result, err := offboardingReconciler.Reconcile(ctx, reconcileReq)
Expect(err).NotTo(HaveOccurred())
Expect(result).To(Equal(ctrl.Result{}))
sharedOffboardingErrorCheck(ctx, "still has 2 running VMs")
})
})
Expand Down Expand Up @@ -413,8 +415,11 @@ var _ = Describe("Offboarding Controller", func() {
})

It("should wait for aggregates to be cleared", func(ctx SpecContext) {
_, err := offboardingReconciler.Reconcile(ctx, reconcileReq)
result, err := offboardingReconciler.Reconcile(ctx, reconcileReq)
Expect(err).NotTo(HaveOccurred())
// Aggregates are cleared by the aggregates controller, whose changes
// are Kubernetes-watched, so no polling requeue is required.
Expect(result).To(Equal(ctrl.Result{}))
sharedOffboardingErrorCheck(ctx, "Waiting for aggregates to be removed")
})
})
Expand Down Expand Up @@ -457,8 +462,11 @@ var _ = Describe("Offboarding Controller", func() {
})

It("should wait for the agent pods to be evicted and not delete the compute service", func(ctx SpecContext) {
_, err := offboardingReconciler.Reconcile(ctx, reconcileReq)
result, err := offboardingReconciler.Reconcile(ctx, reconcileReq)
Expect(err).NotTo(HaveOccurred())
// Eviction progress is Kubernetes-watched via the AgentPodsEvicted
// condition, so no polling requeue is required.
Expect(result).To(Equal(ctrl.Result{}))
sharedOffboardingErrorCheck(ctx, "Waiting for agent pods to be evicted")
Expect(getHypervisorsCalled).To(Equal(0),
"external hypervisor lookup must be short-circuited by the AgentPodsEvicted gate")
Expand Down Expand Up @@ -493,8 +501,9 @@ var _ = Describe("Offboarding Controller", func() {
})

It("should set offboarding condition with error", func(ctx SpecContext) {
_, err := offboardingReconciler.Reconcile(ctx, reconcileReq)
Expect(err).NotTo(HaveOccurred())
result, err := offboardingReconciler.Reconcile(ctx, reconcileReq)
Expect(err).To(HaveOccurred())
Expect(result).To(Equal(ctrl.Result{}))
sharedOffboardingErrorCheck(ctx, "cannot delete service")
})
})
Expand Down Expand Up @@ -531,8 +540,9 @@ var _ = Describe("Offboarding Controller", func() {
})

It("should set offboarding condition with error", func(ctx SpecContext) {
_, err := offboardingReconciler.Reconcile(ctx, reconcileReq)
Expect(err).NotTo(HaveOccurred())
result, err := offboardingReconciler.Reconcile(ctx, reconcileReq)
Expect(err).To(HaveOccurred())
Expect(result).To(Equal(ctrl.Result{}))
sharedOffboardingErrorCheck(ctx, "cannot get resource provider")
})
})
Expand Down Expand Up @@ -581,10 +591,206 @@ var _ = Describe("Offboarding Controller", func() {
})

It("should set offboarding condition with error", func(ctx SpecContext) {
_, err := offboardingReconciler.Reconcile(ctx, reconcileReq)
Expect(err).NotTo(HaveOccurred())
result, err := offboardingReconciler.Reconcile(ctx, reconcileReq)
Expect(err).To(HaveOccurred())
Expect(result).To(Equal(ctrl.Result{}))
sharedOffboardingErrorCheck(ctx, "cannot clean up resource provider")
})
})

Context("When the compute service was already deleted in a previous reconcile", func() {
// hypervisorID matches the ID in the HypervisorWithServers const
const hypervisorID = "c48f6247-abe4-4a24-824e-ea39e108874f"

BeforeEach(func(ctx SpecContext) {
By("Setting the IDs stored by a previous reconcile")
hypervisor := &kvmv1.Hypervisor{}
Expect(k8sClient.Get(ctx, resourceName, hypervisor)).To(Succeed())
hypervisor.Status.ServiceID = serviceId
hypervisor.Status.HypervisorID = hypervisorID
meta.SetStatusCondition(&hypervisor.Status.Conditions, metav1.Condition{
Type: kvmv1.ConditionTypeEvicting,
Status: metav1.ConditionFalse,
Reason: "dontcare",
Message: "dontcare",
})
Expect(k8sClient.Status().Update(ctx, hypervisor)).To(Succeed())
})

Context("When the resource provider delete fails once and then succeeds", func() {
var (
listHypervisorsCalls int
deleteRPCalled int
)

BeforeEach(func() {
listHypervisorsCalls = 0
deleteRPCalled = 0

// First list call returns the hypervisor, later ones an
// empty list, as Nova does after the service was deleted.
fakeServer.Mux.HandleFunc("GET /os-hypervisors/detail", func(w http.ResponseWriter, r *http.Request) {
listHypervisorsCalls++
w.Header().Add("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
if listHypervisorsCalls == 1 {
_, err := fmt.Fprintf(w, HypervisorWithServers, serviceId, "", hypervisorName)
Expect(err).NotTo(HaveOccurred())
} else {
_, err := fmt.Fprint(w, `{"hypervisors": []}`)
Expect(err).NotTo(HaveOccurred())
}
})

fakeServer.Mux.HandleFunc("GET /resource_providers/"+hypervisorID, func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_, err := fmt.Fprint(w, `{"uuid": "rp-uuid", "name": "hv-test"}`)
Expect(err).NotTo(HaveOccurred())
})

fakeServer.Mux.HandleFunc("GET /resource_providers/rp-uuid/allocations", func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_, err := fmt.Fprint(w, `{"allocations": {}}`)
Expect(err).NotTo(HaveOccurred())
})

fakeServer.Mux.HandleFunc("DELETE /resource_providers/rp-uuid", func(w http.ResponseWriter, r *http.Request) {
deleteRPCalled++
if deleteRPCalled == 1 {
w.WriteHeader(http.StatusInternalServerError)
_, err := fmt.Fprint(w, `{"error": "Internal Server Error"}`)
Expect(err).NotTo(HaveOccurred())
} else {
w.WriteHeader(http.StatusAccepted)
}
})

fakeServer.Mux.HandleFunc("DELETE /os-services/"+serviceId, func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent)
})
})

It("should clean up the resource provider and only then mark the hypervisor offboarded", func(ctx SpecContext) {
By("Reconciling while the hypervisor still exists")
result, err := offboardingReconciler.Reconcile(ctx, reconcileReq)
Expect(err).To(HaveOccurred())
Expect(result).To(Equal(ctrl.Result{}))
sharedOffboardingErrorCheck(ctx, "cannot clean up resource provider")
Expect(deleteRPCalled).To(Equal(1))

By("Reconciling after the compute service was deleted")
result, err = offboardingReconciler.Reconcile(ctx, reconcileReq)
Expect(err).NotTo(HaveOccurred())
Expect(result).To(Equal(ctrl.Result{}))
// The hypervisor no longer exists in Nova, but the resource
// provider must still be deleted before offboarding is done.
Expect(deleteRPCalled).To(Equal(2))

hypervisor := &kvmv1.Hypervisor{}
Expect(k8sClient.Get(ctx, resourceName, hypervisor)).To(Succeed())
Expect(hypervisor.Status.Conditions).To(ContainElement(
SatisfyAll(
HaveField("Type", kvmv1.ConditionTypeOffboarded),
HaveField("Status", metav1.ConditionTrue),
HaveField("Reason", "Offboarded"),
HaveField("Message", "Offboarding successful"),
),
))
})
})

Context("When the resource provider delete keeps failing", func() {
var deleteRPCalled int

BeforeEach(func() {
deleteRPCalled = 0

fakeServer.Mux.HandleFunc("GET /os-hypervisors/detail", func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_, err := fmt.Fprint(w, `{"hypervisors": []}`)
Expect(err).NotTo(HaveOccurred())
})

fakeServer.Mux.HandleFunc("GET /resource_providers/"+hypervisorID, func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_, err := fmt.Fprint(w, `{"uuid": "rp-uuid", "name": "hv-test"}`)
Expect(err).NotTo(HaveOccurred())
})

fakeServer.Mux.HandleFunc("GET /resource_providers/rp-uuid/allocations", func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_, err := fmt.Fprint(w, `{"allocations": {}}`)
Expect(err).NotTo(HaveOccurred())
})

fakeServer.Mux.HandleFunc("DELETE /resource_providers/rp-uuid", func(w http.ResponseWriter, r *http.Request) {
deleteRPCalled++
w.WriteHeader(http.StatusInternalServerError)
_, err := fmt.Fprint(w, `{"error": "Internal Server Error"}`)
Expect(err).NotTo(HaveOccurred())
})
})

It("should return an error and not mark the hypervisor offboarded", func(ctx SpecContext) {
_, err := offboardingReconciler.Reconcile(ctx, reconcileReq)
Expect(err).To(HaveOccurred())

By("Reconciling again while the resource provider delete still fails")
result, err := offboardingReconciler.Reconcile(ctx, reconcileReq)
Expect(err).To(HaveOccurred())
Expect(result).To(Equal(ctrl.Result{}))
sharedOffboardingErrorCheck(ctx, "cannot clean up resource provider")
Expect(deleteRPCalled).To(Equal(2))
})
})

Context("When the resource provider is already gone", func() {
var deleteRPCalled int

BeforeEach(func() {
deleteRPCalled = 0

fakeServer.Mux.HandleFunc("GET /os-hypervisors/detail", func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_, err := fmt.Fprint(w, `{"hypervisors": []}`)
Expect(err).NotTo(HaveOccurred())
})

fakeServer.Mux.HandleFunc("GET /resource_providers/"+hypervisorID, func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNotFound)
_, err := fmt.Fprint(w, `{"error": "Not Found"}`)
Expect(err).NotTo(HaveOccurred())
})

fakeServer.Mux.HandleFunc("DELETE /resource_providers/rp-uuid", func(w http.ResponseWriter, r *http.Request) {
deleteRPCalled++
w.WriteHeader(http.StatusAccepted)
})
})

It("should mark the hypervisor offboarded without deleting anything", func(ctx SpecContext) {
result, err := offboardingReconciler.Reconcile(ctx, reconcileReq)
Expect(err).NotTo(HaveOccurred())
Expect(result).To(Equal(ctrl.Result{}))
Expect(deleteRPCalled).To(BeZero())

hypervisor := &kvmv1.Hypervisor{}
Expect(k8sClient.Get(ctx, resourceName, hypervisor)).To(Succeed())
Expect(hypervisor.Status.Conditions).To(ContainElement(
SatisfyAll(
HaveField("Type", kvmv1.ConditionTypeOffboarded),
HaveField("Status", metav1.ConditionTrue),
HaveField("Reason", "Offboarded"),
),
))
})
})
})
})
})
Loading