From 92d15276d5e0f647919a3f655b864716ff09c911 Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Mon, 16 Nov 2026 11:12:18 +0100 Subject: [PATCH] Onboarding: wait for a live compute service and recover from a stale identity Onboarding looked up the hypervisor/service pair only once, while the fields were empty, and adopted any record matching the hostname, including the leftover of a dead agent. After a node replacement the stored service ID can therefore point at an orphan: the enable step runs against the wrong service and the smoke test loops on 'No valid host' until someone clears status.serviceId by hand. Onboarding now adopts only a record whose service is up, and waits for one to register otherwise. When the stored service vanishes (the enable step returns 404) or the smoke test cannot be scheduled, the stored identity is cleared and the next resolve starts over with the current service. Signed-off-by: Fabian Wiesel --- internal/controller/onboarding_controller.go | 66 +++++++- .../controller/onboarding_controller_test.go | 157 +++++++++++++++++- 2 files changed, 213 insertions(+), 10 deletions(-) diff --git a/internal/controller/onboarding_controller.go b/internal/controller/onboarding_controller.go index aa923e90..2fd333d3 100644 --- a/internal/controller/onboarding_controller.go +++ b/internal/controller/onboarding_controller.go @@ -52,7 +52,10 @@ import ( "github.com/cobaltcore-dev/openstack-hypervisor-operator/internal/utils" ) -var errRequeue = errors.New("requeue requested") +var ( + errRequeue = errors.New("requeue requested") + errNoLiveService = errors.New("no compute service is up for the hostname") +) const ( defaultWaitTime = 1 * time.Minute @@ -132,6 +135,9 @@ func (r *OnboardingController) Reconcile(ctx context.Context, req ctrl.Request) if errors.Is(err, errRequeue) { return ctrl.Result{RequeueAfter: r.getRequeueInterval()}, nil } + if errors.Is(err, errNoLiveService) { + return r.waitForNovaService(hv, statusCfg, apply) + } return ctrl.Result{}, err } statusCfg.WithHypervisorID(hypervisorID).WithServiceID(serviceID) @@ -158,7 +164,11 @@ func (r *OnboardingController) Reconcile(ctx context.Context, req ctrl.Request) } switch status.Reason { case kvmv1.ConditionReasonInitial: - return ctrl.Result{}, r.initialOnboarding(ctx, hv, statusCfg, apply) + err := r.initialOnboarding(ctx, hv, statusCfg, apply) + if errors.Is(err, errRequeue) { + return ctrl.Result{RequeueAfter: r.getRequeueInterval()}, nil + } + return ctrl.Result{}, err case kvmv1.ConditionReasonTesting: if hv.Spec.SkipTests { return r.completeOnboarding(ctx, computeHost, hv, statusCfg, apply) @@ -237,6 +247,13 @@ func (r *OnboardingController) initialOnboarding(ctx context.Context, hv *kvmv1. ForcedDown: &falseVal, } if result := services.Update(ctx, r.computeClient, hv.Status.ServiceID, opts); result.Err != nil { + if gophercloud.ResponseCodeIs(result.Err, http.StatusNotFound) { + r.resetNovaIdentity(statusCfg, "nova-compute service vanished; re-resolving identity") + if err := apply(); err != nil { + return err + } + return errRequeue + } return result.Err } @@ -277,6 +294,15 @@ func (r *OnboardingController) smokeTest(ctx context.Context, hv *kvmv1.Hypervis return ctrl.Result{RequeueAfter: r.getRequeueInterval()}, nil } + if strings.Contains(strings.ToLower(server.Fault.Message), "no valid host") { + if err = servers.Delete(ctx, r.testComputeClient, id).ExtractErr(); err != nil && + !gophercloud.ResponseCodeIs(err, http.StatusNotFound) { + log.Error(err, "failed to delete test instance", "id", id) + } + r.resetNovaIdentity(statusCfg, "test instance cannot be scheduled; re-resolving identity") + return ctrl.Result{RequeueAfter: r.getRequeueInterval()}, apply() + } + // Set condition back to testing if err = setTestingCondition("Server ended up in error state: " + server.Fault.Message); err != nil { return ctrl.Result{}, err @@ -483,18 +509,42 @@ func (r *OnboardingController) lookupNovaProperties(ctx context.Context, hv *kvm return "", "", err } - if len(hs) < 1 { - return "", "", errRequeue - } - + // Records without a live service are leftovers of a dead or replaced agent. for _, h := range hs { short, _, _ := strings.Cut(h.HypervisorHostname, ".") - if short == shortHypervisorAddress { + if short == shortHypervisorAddress && h.State == "up" { return h.ID, h.Service.ID, nil } } - return "", "", fmt.Errorf("could not find exact match for %v", shortHypervisorAddress) + return "", "", errNoLiveService +} + +func (r *OnboardingController) waitForNovaService(hv *kvmv1.Hypervisor, statusCfg *apiv1.HypervisorStatusApplyConfiguration, apply func() error) (ctrl.Result, error) { + message := "Waiting for the nova-compute service to report up" + status := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeOnboarding) + if status == nil || status.Message != message { + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, + *k8sacmetav1.Condition(). + WithType(kvmv1.ConditionTypeOnboarding). + WithStatus(metav1.ConditionTrue). + WithReason(kvmv1.ConditionReasonInitial). + WithMessage(message)) + if err := apply(); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{RequeueAfter: r.getRequeueInterval()}, nil +} + +func (r *OnboardingController) resetNovaIdentity(statusCfg *apiv1.HypervisorStatusApplyConfiguration, message string) { + statusCfg.WithHypervisorID("").WithServiceID("") + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, + *k8sacmetav1.Condition(). + WithType(kvmv1.ConditionTypeOnboarding). + WithStatus(metav1.ConditionTrue). + WithReason(kvmv1.ConditionReasonInitial). + WithMessage(message)) } func (r *OnboardingController) createOrGetTestServer(ctx context.Context, zone, computeHost string, nodeUid types.UID) (*servers.Server, error) { diff --git a/internal/controller/onboarding_controller_test.go b/internal/controller/onboarding_controller_test.go index 13349dae..20d7b770 100644 --- a/internal/controller/onboarding_controller_test.go +++ b/internal/controller/onboarding_controller_test.go @@ -21,6 +21,7 @@ import ( "fmt" "net/http" "os" + "strings" "time" "github.com/gophercloud/gophercloud/v2/testhelper" @@ -84,11 +85,23 @@ const ( }` ) +// hypervisorRecordJSON builds a GET /os-hypervisors/detail record. +func hypervisorRecordJSON(hypervisorID, serviceID, state, status string) string { + return fmt.Sprintf(`{"service": {"id": "%s"}, "hypervisor_hostname": %q, "id": "%s", "state": "%s", "status": "%s", "hypervisor_version": 2002000}`, + serviceID, hypervisorName, hypervisorID, state, status) +} + +// hypervisorListJSON wraps records into a GET /os-hypervisors/detail response. +func hypervisorListJSON(records ...string) string { + return `{"hypervisors": [` + strings.Join(records, ",") + `]}` +} + +const hypervisorName = "test-host" + var _ = Describe("Onboarding Controller", func() { const ( region = "test-region" availabilityZone = "test-az" - hypervisorName = "test-host" serviceId = "service-id" hypervisorId = "c48f6247-abe4-4a24-824e-ea39e108874f" @@ -312,11 +325,15 @@ var _ = Describe("Onboarding Controller", func() { }) Context("initial setup of a new hypervisor", func() { + var hypervisorListBody string + BeforeEach(func() { + hypervisorListBody = fmt.Sprintf(HypervisorWithServers, serviceId, "", hypervisorName) fakeServer.Mux.HandleFunc("GET /os-hypervisors/detail", func(w http.ResponseWriter, r *http.Request) { w.Header().Add("Content-Type", "application/json") w.WriteHeader(http.StatusOK) - Expect(fmt.Fprintf(w, HypervisorWithServers, serviceId, "", hypervisorName)).ToNot(BeNil()) + _, err := fmt.Fprint(w, hypervisorListBody) + Expect(err).NotTo(HaveOccurred()) }) fakeServer.Mux.HandleFunc("PUT /os-services/service-id", func(w http.ResponseWriter, r *http.Request) { @@ -406,6 +423,77 @@ var _ = Describe("Onboarding Controller", func() { )) }) }) + + When("no compute service is up for the hostname yet", func() { + BeforeEach(func() { + hypervisorListBody = hypervisorListJSON(hypervisorRecordJSON("hv-old", "service-old", "down", "enabled")) + }) + + It("waits for a live service instead of adopting the stale record", func(ctx SpecContext) { + result, err := onboardingReconciler.Reconcile(ctx, reconcileReq) + Expect(err).NotTo(HaveOccurred()) + Expect(result.RequeueAfter).To(BeNumerically(">", 0)) + + hv := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, namespacedName, hv)).To(Succeed()) + Expect(hv.Status.ServiceID).To(BeEmpty()) + Expect(hv.Status.HypervisorID).To(BeEmpty()) + condition := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeOnboarding) + Expect(condition).NotTo(BeNil()) + Expect(condition.Reason).To(Equal(kvmv1.ConditionReasonInitial)) + Expect(condition.Message).To(ContainSubstring("nova-compute service to report up")) + }) + }) + + When("the stored compute service vanished", func() { + var enableOldCalls, enableNewCalls int + + BeforeEach(func(ctx SpecContext) { + hv := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, namespacedName, hv)).To(Succeed()) + hv.Status.HypervisorID = "hv-old" + hv.Status.ServiceID = "service-old" + meta.SetStatusCondition(&hv.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeOnboarding, + Status: metav1.ConditionTrue, + Reason: kvmv1.ConditionReasonInitial, + }) + hv.Status.Aggregates = []kvmv1.Aggregate{ + {Name: availabilityZone, UUID: "uuid-" + availabilityZone}, + {Name: testAggregateName, UUID: "uuid-" + testAggregateName}, + } + Expect(k8sClient.Status().Update(ctx, hv)).To(Succeed()) + + hypervisorListBody = hypervisorListJSON(hypervisorRecordJSON("hv-new", "service-new", "up", "disabled")) + + fakeServer.Mux.HandleFunc("PUT /os-services/service-old", func(w http.ResponseWriter, r *http.Request) { + enableOldCalls++ + w.WriteHeader(http.StatusNotFound) + }) + fakeServer.Mux.HandleFunc("PUT /os-services/service-new", func(w http.ResponseWriter, r *http.Request) { + enableNewCalls++ + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err := fmt.Fprint(w, `{"service": {"id": "service-new", "status": "enabled"}}`) + Expect(err).NotTo(HaveOccurred()) + }) + }) + + It("re-resolves the identity and enables the replacement service", func(ctx SpecContext) { + result, err := onboardingReconciler.Reconcile(ctx, reconcileReq) + Expect(err).NotTo(HaveOccurred()) + Expect(result.RequeueAfter).To(BeNumerically(">", 0)) + + hv := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, namespacedName, hv)).To(Succeed()) + Expect(hv.Status.ServiceID).To(BeEmpty()) + + // The next reconciles adopt the replacement service and enable it. + Expect(reconcileLoop(ctx, 2)).To(Succeed()) + Expect(enableOldCalls).To(Equal(1)) + Expect(enableNewCalls).To(Equal(1)) + }) + }) }) Context("running tests after initial setup", func() { @@ -512,6 +600,16 @@ var _ = Describe("Onboarding Controller", func() { }) }) + When("the identity is already stored", func() { + It("does not query Nova for it", func(ctx SpecContext) { + // Without aggregates onboarding requeues before any Nova call; + // an identity lookup would hit an unhandled route and fail. + result, err := onboardingReconciler.Reconcile(ctx, reconcileReq) + Expect(err).NotTo(HaveOccurred()) + Expect(result.RequeueAfter).To(BeNumerically(">", 0)) + }) + }) + When("SkipTests is set to true", func() { BeforeEach(func(ctx SpecContext) { hv := &kvmv1.Hypervisor{} @@ -732,6 +830,61 @@ var _ = Describe("Onboarding Controller", func() { }) }) + When("the smoke test cannot be scheduled", func() { + var serverDeleted bool + + BeforeEach(func(ctx SpecContext) { + By("Overriding HV status to Testing state") + hv := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, namespacedName, hv)).To(Succeed()) + meta.SetStatusCondition(&hv.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeOnboarding, + Status: metav1.ConditionTrue, + Reason: kvmv1.ConditionReasonTesting, + }) + Expect(k8sClient.Status().Update(ctx, hv)).To(Succeed()) + + serverName := fmt.Sprintf("%s-%s-%s", testPrefixName, hypervisorName, string(hv.UID)) + + // GET /servers/detail returns the test server in ERROR state. + serverDetailHandler = func(w http.ResponseWriter, _ *http.Request) { + _, err := fmt.Fprintf(w, + `{"servers": [{"id": "server-id", "name": %q, "status": "ERROR"}], "servers_links": []}`, + serverName) + Expect(err).NotTo(HaveOccurred()) + } + + fakeServer.Mux.HandleFunc("GET /servers/server-id", func(w http.ResponseWriter, _ *http.Request) { + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err := fmt.Fprintf(w, + `{"server": {"id": "server-id", "name": %q, "status": "ERROR", "fault": {"message": "No valid host was found. There are not enough hosts available.", "code": 500}}}`, + serverName) + Expect(err).NotTo(HaveOccurred()) + }) + + serverDeleteHandler = func(w http.ResponseWriter, _ *http.Request) { + serverDeleted = true + w.WriteHeader(http.StatusAccepted) + } + }) + + It("clears the identity to re-resolve it", func(ctx SpecContext) { + result, err := onboardingReconciler.Reconcile(ctx, reconcileReq) + Expect(err).NotTo(HaveOccurred()) + Expect(result.RequeueAfter).To(BeNumerically(">", 0)) + Expect(serverDeleted).To(BeTrue()) + + hv := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, namespacedName, hv)).To(Succeed()) + Expect(hv.Status.ServiceID).To(BeEmpty()) + condition := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeOnboarding) + Expect(condition).NotTo(BeNil()) + Expect(condition.Reason).To(Equal(kvmv1.ConditionReasonInitial)) + Expect(condition.Message).To(ContainSubstring("cannot be scheduled")) + }) + }) + When("the test server ended up in an error state", func() { var ( serverDeleteCalls int