diff --git a/internal/controller/onboarding_controller.go b/internal/controller/onboarding_controller.go index aa923e90..2fd333d3 100644 --- a/internal/controller/onboarding_controller.go +++ b/internal/controller/onboarding_controller.go @@ -52,7 +52,10 @@ import ( "github.com/cobaltcore-dev/openstack-hypervisor-operator/internal/utils" ) -var errRequeue = errors.New("requeue requested") +var ( + errRequeue = errors.New("requeue requested") + errNoLiveService = errors.New("no compute service is up for the hostname") +) const ( defaultWaitTime = 1 * time.Minute @@ -132,6 +135,9 @@ func (r *OnboardingController) Reconcile(ctx context.Context, req ctrl.Request) if errors.Is(err, errRequeue) { return ctrl.Result{RequeueAfter: r.getRequeueInterval()}, nil } + if errors.Is(err, errNoLiveService) { + return r.waitForNovaService(hv, statusCfg, apply) + } return ctrl.Result{}, err } statusCfg.WithHypervisorID(hypervisorID).WithServiceID(serviceID) @@ -158,7 +164,11 @@ func (r *OnboardingController) Reconcile(ctx context.Context, req ctrl.Request) } switch status.Reason { case kvmv1.ConditionReasonInitial: - return ctrl.Result{}, r.initialOnboarding(ctx, hv, statusCfg, apply) + err := r.initialOnboarding(ctx, hv, statusCfg, apply) + if errors.Is(err, errRequeue) { + return ctrl.Result{RequeueAfter: r.getRequeueInterval()}, nil + } + return ctrl.Result{}, err case kvmv1.ConditionReasonTesting: if hv.Spec.SkipTests { return r.completeOnboarding(ctx, computeHost, hv, statusCfg, apply) @@ -237,6 +247,13 @@ func (r *OnboardingController) initialOnboarding(ctx context.Context, hv *kvmv1. ForcedDown: &falseVal, } if result := services.Update(ctx, r.computeClient, hv.Status.ServiceID, opts); result.Err != nil { + if gophercloud.ResponseCodeIs(result.Err, http.StatusNotFound) { + r.resetNovaIdentity(statusCfg, "nova-compute service vanished; re-resolving identity") + if err := apply(); err != nil { + return err + } + return errRequeue + } return result.Err } @@ -277,6 +294,15 @@ func (r *OnboardingController) smokeTest(ctx context.Context, hv *kvmv1.Hypervis return ctrl.Result{RequeueAfter: r.getRequeueInterval()}, nil } + if strings.Contains(strings.ToLower(server.Fault.Message), "no valid host") { + if err = servers.Delete(ctx, r.testComputeClient, id).ExtractErr(); err != nil && + !gophercloud.ResponseCodeIs(err, http.StatusNotFound) { + log.Error(err, "failed to delete test instance", "id", id) + } + r.resetNovaIdentity(statusCfg, "test instance cannot be scheduled; re-resolving identity") + return ctrl.Result{RequeueAfter: r.getRequeueInterval()}, apply() + } + // Set condition back to testing if err = setTestingCondition("Server ended up in error state: " + server.Fault.Message); err != nil { return ctrl.Result{}, err @@ -483,18 +509,42 @@ func (r *OnboardingController) lookupNovaProperties(ctx context.Context, hv *kvm return "", "", err } - if len(hs) < 1 { - return "", "", errRequeue - } - + // Records without a live service are leftovers of a dead or replaced agent. for _, h := range hs { short, _, _ := strings.Cut(h.HypervisorHostname, ".") - if short == shortHypervisorAddress { + if short == shortHypervisorAddress && h.State == "up" { return h.ID, h.Service.ID, nil } } - return "", "", fmt.Errorf("could not find exact match for %v", shortHypervisorAddress) + return "", "", errNoLiveService +} + +func (r *OnboardingController) waitForNovaService(hv *kvmv1.Hypervisor, statusCfg *apiv1.HypervisorStatusApplyConfiguration, apply func() error) (ctrl.Result, error) { + message := "Waiting for the nova-compute service to report up" + status := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeOnboarding) + if status == nil || status.Message != message { + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, + *k8sacmetav1.Condition(). + WithType(kvmv1.ConditionTypeOnboarding). + WithStatus(metav1.ConditionTrue). + WithReason(kvmv1.ConditionReasonInitial). + WithMessage(message)) + if err := apply(); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{RequeueAfter: r.getRequeueInterval()}, nil +} + +func (r *OnboardingController) resetNovaIdentity(statusCfg *apiv1.HypervisorStatusApplyConfiguration, message string) { + statusCfg.WithHypervisorID("").WithServiceID("") + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, + *k8sacmetav1.Condition(). + WithType(kvmv1.ConditionTypeOnboarding). + WithStatus(metav1.ConditionTrue). + WithReason(kvmv1.ConditionReasonInitial). + WithMessage(message)) } func (r *OnboardingController) createOrGetTestServer(ctx context.Context, zone, computeHost string, nodeUid types.UID) (*servers.Server, error) { diff --git a/internal/controller/onboarding_controller_test.go b/internal/controller/onboarding_controller_test.go index 13349dae..20d7b770 100644 --- a/internal/controller/onboarding_controller_test.go +++ b/internal/controller/onboarding_controller_test.go @@ -21,6 +21,7 @@ import ( "fmt" "net/http" "os" + "strings" "time" "github.com/gophercloud/gophercloud/v2/testhelper" @@ -84,11 +85,23 @@ const ( }` ) +// hypervisorRecordJSON builds a GET /os-hypervisors/detail record. +func hypervisorRecordJSON(hypervisorID, serviceID, state, status string) string { + return fmt.Sprintf(`{"service": {"id": "%s"}, "hypervisor_hostname": %q, "id": "%s", "state": "%s", "status": "%s", "hypervisor_version": 2002000}`, + serviceID, hypervisorName, hypervisorID, state, status) +} + +// hypervisorListJSON wraps records into a GET /os-hypervisors/detail response. +func hypervisorListJSON(records ...string) string { + return `{"hypervisors": [` + strings.Join(records, ",") + `]}` +} + +const hypervisorName = "test-host" + var _ = Describe("Onboarding Controller", func() { const ( region = "test-region" availabilityZone = "test-az" - hypervisorName = "test-host" serviceId = "service-id" hypervisorId = "c48f6247-abe4-4a24-824e-ea39e108874f" @@ -312,11 +325,15 @@ var _ = Describe("Onboarding Controller", func() { }) Context("initial setup of a new hypervisor", func() { + var hypervisorListBody string + BeforeEach(func() { + hypervisorListBody = fmt.Sprintf(HypervisorWithServers, serviceId, "", hypervisorName) fakeServer.Mux.HandleFunc("GET /os-hypervisors/detail", func(w http.ResponseWriter, r *http.Request) { w.Header().Add("Content-Type", "application/json") w.WriteHeader(http.StatusOK) - Expect(fmt.Fprintf(w, HypervisorWithServers, serviceId, "", hypervisorName)).ToNot(BeNil()) + _, err := fmt.Fprint(w, hypervisorListBody) + Expect(err).NotTo(HaveOccurred()) }) fakeServer.Mux.HandleFunc("PUT /os-services/service-id", func(w http.ResponseWriter, r *http.Request) { @@ -406,6 +423,77 @@ var _ = Describe("Onboarding Controller", func() { )) }) }) + + When("no compute service is up for the hostname yet", func() { + BeforeEach(func() { + hypervisorListBody = hypervisorListJSON(hypervisorRecordJSON("hv-old", "service-old", "down", "enabled")) + }) + + It("waits for a live service instead of adopting the stale record", func(ctx SpecContext) { + result, err := onboardingReconciler.Reconcile(ctx, reconcileReq) + Expect(err).NotTo(HaveOccurred()) + Expect(result.RequeueAfter).To(BeNumerically(">", 0)) + + hv := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, namespacedName, hv)).To(Succeed()) + Expect(hv.Status.ServiceID).To(BeEmpty()) + Expect(hv.Status.HypervisorID).To(BeEmpty()) + condition := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeOnboarding) + Expect(condition).NotTo(BeNil()) + Expect(condition.Reason).To(Equal(kvmv1.ConditionReasonInitial)) + Expect(condition.Message).To(ContainSubstring("nova-compute service to report up")) + }) + }) + + When("the stored compute service vanished", func() { + var enableOldCalls, enableNewCalls int + + BeforeEach(func(ctx SpecContext) { + hv := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, namespacedName, hv)).To(Succeed()) + hv.Status.HypervisorID = "hv-old" + hv.Status.ServiceID = "service-old" + meta.SetStatusCondition(&hv.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeOnboarding, + Status: metav1.ConditionTrue, + Reason: kvmv1.ConditionReasonInitial, + }) + hv.Status.Aggregates = []kvmv1.Aggregate{ + {Name: availabilityZone, UUID: "uuid-" + availabilityZone}, + {Name: testAggregateName, UUID: "uuid-" + testAggregateName}, + } + Expect(k8sClient.Status().Update(ctx, hv)).To(Succeed()) + + hypervisorListBody = hypervisorListJSON(hypervisorRecordJSON("hv-new", "service-new", "up", "disabled")) + + fakeServer.Mux.HandleFunc("PUT /os-services/service-old", func(w http.ResponseWriter, r *http.Request) { + enableOldCalls++ + w.WriteHeader(http.StatusNotFound) + }) + fakeServer.Mux.HandleFunc("PUT /os-services/service-new", func(w http.ResponseWriter, r *http.Request) { + enableNewCalls++ + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err := fmt.Fprint(w, `{"service": {"id": "service-new", "status": "enabled"}}`) + Expect(err).NotTo(HaveOccurred()) + }) + }) + + It("re-resolves the identity and enables the replacement service", func(ctx SpecContext) { + result, err := onboardingReconciler.Reconcile(ctx, reconcileReq) + Expect(err).NotTo(HaveOccurred()) + Expect(result.RequeueAfter).To(BeNumerically(">", 0)) + + hv := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, namespacedName, hv)).To(Succeed()) + Expect(hv.Status.ServiceID).To(BeEmpty()) + + // The next reconciles adopt the replacement service and enable it. + Expect(reconcileLoop(ctx, 2)).To(Succeed()) + Expect(enableOldCalls).To(Equal(1)) + Expect(enableNewCalls).To(Equal(1)) + }) + }) }) Context("running tests after initial setup", func() { @@ -512,6 +600,16 @@ var _ = Describe("Onboarding Controller", func() { }) }) + When("the identity is already stored", func() { + It("does not query Nova for it", func(ctx SpecContext) { + // Without aggregates onboarding requeues before any Nova call; + // an identity lookup would hit an unhandled route and fail. + result, err := onboardingReconciler.Reconcile(ctx, reconcileReq) + Expect(err).NotTo(HaveOccurred()) + Expect(result.RequeueAfter).To(BeNumerically(">", 0)) + }) + }) + When("SkipTests is set to true", func() { BeforeEach(func(ctx SpecContext) { hv := &kvmv1.Hypervisor{} @@ -732,6 +830,61 @@ var _ = Describe("Onboarding Controller", func() { }) }) + When("the smoke test cannot be scheduled", func() { + var serverDeleted bool + + BeforeEach(func(ctx SpecContext) { + By("Overriding HV status to Testing state") + hv := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, namespacedName, hv)).To(Succeed()) + meta.SetStatusCondition(&hv.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeOnboarding, + Status: metav1.ConditionTrue, + Reason: kvmv1.ConditionReasonTesting, + }) + Expect(k8sClient.Status().Update(ctx, hv)).To(Succeed()) + + serverName := fmt.Sprintf("%s-%s-%s", testPrefixName, hypervisorName, string(hv.UID)) + + // GET /servers/detail returns the test server in ERROR state. + serverDetailHandler = func(w http.ResponseWriter, _ *http.Request) { + _, err := fmt.Fprintf(w, + `{"servers": [{"id": "server-id", "name": %q, "status": "ERROR"}], "servers_links": []}`, + serverName) + Expect(err).NotTo(HaveOccurred()) + } + + fakeServer.Mux.HandleFunc("GET /servers/server-id", func(w http.ResponseWriter, _ *http.Request) { + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err := fmt.Fprintf(w, + `{"server": {"id": "server-id", "name": %q, "status": "ERROR", "fault": {"message": "No valid host was found. There are not enough hosts available.", "code": 500}}}`, + serverName) + Expect(err).NotTo(HaveOccurred()) + }) + + serverDeleteHandler = func(w http.ResponseWriter, _ *http.Request) { + serverDeleted = true + w.WriteHeader(http.StatusAccepted) + } + }) + + It("clears the identity to re-resolve it", func(ctx SpecContext) { + result, err := onboardingReconciler.Reconcile(ctx, reconcileReq) + Expect(err).NotTo(HaveOccurred()) + Expect(result.RequeueAfter).To(BeNumerically(">", 0)) + Expect(serverDeleted).To(BeTrue()) + + hv := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, namespacedName, hv)).To(Succeed()) + Expect(hv.Status.ServiceID).To(BeEmpty()) + condition := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeOnboarding) + Expect(condition).NotTo(BeNil()) + Expect(condition.Reason).To(Equal(kvmv1.ConditionReasonInitial)) + Expect(condition.Message).To(ContainSubstring("cannot be scheduled")) + }) + }) + When("the test server ended up in an error state", func() { var ( serverDeleteCalls int