Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 58 additions & 8 deletions internal/controller/onboarding_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,10 @@ import (
"github.com/cobaltcore-dev/openstack-hypervisor-operator/internal/utils"
)

var errRequeue = errors.New("requeue requested")
var (
errRequeue = errors.New("requeue requested")
errNoLiveService = errors.New("no compute service is up for the hostname")
)

const (
defaultWaitTime = 1 * time.Minute
Expand Down Expand Up @@ -132,6 +135,9 @@ func (r *OnboardingController) Reconcile(ctx context.Context, req ctrl.Request)
if errors.Is(err, errRequeue) {
return ctrl.Result{RequeueAfter: r.getRequeueInterval()}, nil
}
if errors.Is(err, errNoLiveService) {
return r.waitForNovaService(hv, statusCfg, apply)
}
return ctrl.Result{}, err
}
statusCfg.WithHypervisorID(hypervisorID).WithServiceID(serviceID)
Expand All @@ -158,7 +164,11 @@ func (r *OnboardingController) Reconcile(ctx context.Context, req ctrl.Request)
}
switch status.Reason {
case kvmv1.ConditionReasonInitial:
return ctrl.Result{}, r.initialOnboarding(ctx, hv, statusCfg, apply)
err := r.initialOnboarding(ctx, hv, statusCfg, apply)
if errors.Is(err, errRequeue) {
return ctrl.Result{RequeueAfter: r.getRequeueInterval()}, nil
}
return ctrl.Result{}, err
case kvmv1.ConditionReasonTesting:
if hv.Spec.SkipTests {
return r.completeOnboarding(ctx, computeHost, hv, statusCfg, apply)
Expand Down Expand Up @@ -237,6 +247,13 @@ func (r *OnboardingController) initialOnboarding(ctx context.Context, hv *kvmv1.
ForcedDown: &falseVal,
}
if result := services.Update(ctx, r.computeClient, hv.Status.ServiceID, opts); result.Err != nil {
if gophercloud.ResponseCodeIs(result.Err, http.StatusNotFound) {
r.resetNovaIdentity(statusCfg, "nova-compute service vanished; re-resolving identity")
if err := apply(); err != nil {
return err
}
return errRequeue
}
return result.Err
}

Expand Down Expand Up @@ -277,6 +294,15 @@ func (r *OnboardingController) smokeTest(ctx context.Context, hv *kvmv1.Hypervis
return ctrl.Result{RequeueAfter: r.getRequeueInterval()}, nil
}

if strings.Contains(strings.ToLower(server.Fault.Message), "no valid host") {
if err = servers.Delete(ctx, r.testComputeClient, id).ExtractErr(); err != nil &&
!gophercloud.ResponseCodeIs(err, http.StatusNotFound) {
log.Error(err, "failed to delete test instance", "id", id)
}
r.resetNovaIdentity(statusCfg, "test instance cannot be scheduled; re-resolving identity")
return ctrl.Result{RequeueAfter: r.getRequeueInterval()}, apply()
}

// Set condition back to testing
if err = setTestingCondition("Server ended up in error state: " + server.Fault.Message); err != nil {
return ctrl.Result{}, err
Expand Down Expand Up @@ -483,18 +509,42 @@ func (r *OnboardingController) lookupNovaProperties(ctx context.Context, hv *kvm
return "", "", err
}

if len(hs) < 1 {
return "", "", errRequeue
}

// Records without a live service are leftovers of a dead or replaced agent.
for _, h := range hs {
short, _, _ := strings.Cut(h.HypervisorHostname, ".")
if short == shortHypervisorAddress {
if short == shortHypervisorAddress && h.State == "up" {
return h.ID, h.Service.ID, nil
}
}

return "", "", fmt.Errorf("could not find exact match for %v", shortHypervisorAddress)
return "", "", errNoLiveService
}

func (r *OnboardingController) waitForNovaService(hv *kvmv1.Hypervisor, statusCfg *apiv1.HypervisorStatusApplyConfiguration, apply func() error) (ctrl.Result, error) {
message := "Waiting for the nova-compute service to report up"
status := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeOnboarding)
if status == nil || status.Message != message {
utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions,
*k8sacmetav1.Condition().
WithType(kvmv1.ConditionTypeOnboarding).
WithStatus(metav1.ConditionTrue).
WithReason(kvmv1.ConditionReasonInitial).
WithMessage(message))
if err := apply(); err != nil {
return ctrl.Result{}, err
}
}
return ctrl.Result{RequeueAfter: r.getRequeueInterval()}, nil
}

func (r *OnboardingController) resetNovaIdentity(statusCfg *apiv1.HypervisorStatusApplyConfiguration, message string) {
statusCfg.WithHypervisorID("").WithServiceID("")
utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions,
*k8sacmetav1.Condition().
WithType(kvmv1.ConditionTypeOnboarding).
WithStatus(metav1.ConditionTrue).
WithReason(kvmv1.ConditionReasonInitial).
WithMessage(message))
}

func (r *OnboardingController) createOrGetTestServer(ctx context.Context, zone, computeHost string, nodeUid types.UID) (*servers.Server, error) {
Expand Down
157 changes: 155 additions & 2 deletions internal/controller/onboarding_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"fmt"
"net/http"
"os"
"strings"
"time"

"github.com/gophercloud/gophercloud/v2/testhelper"
Expand Down Expand Up @@ -84,11 +85,23 @@ const (
}`
)

// hypervisorRecordJSON builds a GET /os-hypervisors/detail record.
func hypervisorRecordJSON(hypervisorID, serviceID, state, status string) string {
return fmt.Sprintf(`{"service": {"id": "%s"}, "hypervisor_hostname": %q, "id": "%s", "state": "%s", "status": "%s", "hypervisor_version": 2002000}`,
serviceID, hypervisorName, hypervisorID, state, status)
}

// hypervisorListJSON wraps records into a GET /os-hypervisors/detail response.
func hypervisorListJSON(records ...string) string {
return `{"hypervisors": [` + strings.Join(records, ",") + `]}`
}

const hypervisorName = "test-host"

var _ = Describe("Onboarding Controller", func() {
const (
region = "test-region"
availabilityZone = "test-az"
hypervisorName = "test-host"
serviceId = "service-id"
hypervisorId = "c48f6247-abe4-4a24-824e-ea39e108874f"

Expand Down Expand Up @@ -312,11 +325,15 @@ var _ = Describe("Onboarding Controller", func() {
})

Context("initial setup of a new hypervisor", func() {
var hypervisorListBody string

BeforeEach(func() {
hypervisorListBody = fmt.Sprintf(HypervisorWithServers, serviceId, "", hypervisorName)
fakeServer.Mux.HandleFunc("GET /os-hypervisors/detail", func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
Expect(fmt.Fprintf(w, HypervisorWithServers, serviceId, "", hypervisorName)).ToNot(BeNil())
_, err := fmt.Fprint(w, hypervisorListBody)
Expect(err).NotTo(HaveOccurred())
})

fakeServer.Mux.HandleFunc("PUT /os-services/service-id", func(w http.ResponseWriter, r *http.Request) {
Expand Down Expand Up @@ -406,6 +423,77 @@ var _ = Describe("Onboarding Controller", func() {
))
})
})

When("no compute service is up for the hostname yet", func() {
BeforeEach(func() {
hypervisorListBody = hypervisorListJSON(hypervisorRecordJSON("hv-old", "service-old", "down", "enabled"))
})

It("waits for a live service instead of adopting the stale record", func(ctx SpecContext) {
result, err := onboardingReconciler.Reconcile(ctx, reconcileReq)
Expect(err).NotTo(HaveOccurred())
Expect(result.RequeueAfter).To(BeNumerically(">", 0))

hv := &kvmv1.Hypervisor{}
Expect(k8sClient.Get(ctx, namespacedName, hv)).To(Succeed())
Expect(hv.Status.ServiceID).To(BeEmpty())
Expect(hv.Status.HypervisorID).To(BeEmpty())
condition := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeOnboarding)
Expect(condition).NotTo(BeNil())
Expect(condition.Reason).To(Equal(kvmv1.ConditionReasonInitial))
Expect(condition.Message).To(ContainSubstring("nova-compute service to report up"))
})
})

When("the stored compute service vanished", func() {
var enableOldCalls, enableNewCalls int

BeforeEach(func(ctx SpecContext) {
hv := &kvmv1.Hypervisor{}
Expect(k8sClient.Get(ctx, namespacedName, hv)).To(Succeed())
hv.Status.HypervisorID = "hv-old"
hv.Status.ServiceID = "service-old"
meta.SetStatusCondition(&hv.Status.Conditions, metav1.Condition{
Type: kvmv1.ConditionTypeOnboarding,
Status: metav1.ConditionTrue,
Reason: kvmv1.ConditionReasonInitial,
})
hv.Status.Aggregates = []kvmv1.Aggregate{
{Name: availabilityZone, UUID: "uuid-" + availabilityZone},
{Name: testAggregateName, UUID: "uuid-" + testAggregateName},
}
Expect(k8sClient.Status().Update(ctx, hv)).To(Succeed())

hypervisorListBody = hypervisorListJSON(hypervisorRecordJSON("hv-new", "service-new", "up", "disabled"))

fakeServer.Mux.HandleFunc("PUT /os-services/service-old", func(w http.ResponseWriter, r *http.Request) {
enableOldCalls++
w.WriteHeader(http.StatusNotFound)
})
fakeServer.Mux.HandleFunc("PUT /os-services/service-new", func(w http.ResponseWriter, r *http.Request) {
enableNewCalls++
w.Header().Add("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_, err := fmt.Fprint(w, `{"service": {"id": "service-new", "status": "enabled"}}`)
Expect(err).NotTo(HaveOccurred())
})
})

It("re-resolves the identity and enables the replacement service", func(ctx SpecContext) {
result, err := onboardingReconciler.Reconcile(ctx, reconcileReq)
Expect(err).NotTo(HaveOccurred())
Expect(result.RequeueAfter).To(BeNumerically(">", 0))

hv := &kvmv1.Hypervisor{}
Expect(k8sClient.Get(ctx, namespacedName, hv)).To(Succeed())
Expect(hv.Status.ServiceID).To(BeEmpty())

// The next reconciles adopt the replacement service and enable it.
Expect(reconcileLoop(ctx, 2)).To(Succeed())
Expect(enableOldCalls).To(Equal(1))
Expect(enableNewCalls).To(Equal(1))
})
})
})

Context("running tests after initial setup", func() {
Expand Down Expand Up @@ -512,6 +600,16 @@ var _ = Describe("Onboarding Controller", func() {
})
})

When("the identity is already stored", func() {
It("does not query Nova for it", func(ctx SpecContext) {
// Without aggregates onboarding requeues before any Nova call;
// an identity lookup would hit an unhandled route and fail.
result, err := onboardingReconciler.Reconcile(ctx, reconcileReq)
Expect(err).NotTo(HaveOccurred())
Expect(result.RequeueAfter).To(BeNumerically(">", 0))
})
})

When("SkipTests is set to true", func() {
BeforeEach(func(ctx SpecContext) {
hv := &kvmv1.Hypervisor{}
Expand Down Expand Up @@ -732,6 +830,61 @@ var _ = Describe("Onboarding Controller", func() {
})
})

When("the smoke test cannot be scheduled", func() {
var serverDeleted bool

BeforeEach(func(ctx SpecContext) {
By("Overriding HV status to Testing state")
hv := &kvmv1.Hypervisor{}
Expect(k8sClient.Get(ctx, namespacedName, hv)).To(Succeed())
meta.SetStatusCondition(&hv.Status.Conditions, metav1.Condition{
Type: kvmv1.ConditionTypeOnboarding,
Status: metav1.ConditionTrue,
Reason: kvmv1.ConditionReasonTesting,
})
Expect(k8sClient.Status().Update(ctx, hv)).To(Succeed())

serverName := fmt.Sprintf("%s-%s-%s", testPrefixName, hypervisorName, string(hv.UID))

// GET /servers/detail returns the test server in ERROR state.
serverDetailHandler = func(w http.ResponseWriter, _ *http.Request) {
_, err := fmt.Fprintf(w,
`{"servers": [{"id": "server-id", "name": %q, "status": "ERROR"}], "servers_links": []}`,
serverName)
Expect(err).NotTo(HaveOccurred())
}

fakeServer.Mux.HandleFunc("GET /servers/server-id", func(w http.ResponseWriter, _ *http.Request) {
w.Header().Add("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_, err := fmt.Fprintf(w,
`{"server": {"id": "server-id", "name": %q, "status": "ERROR", "fault": {"message": "No valid host was found. There are not enough hosts available.", "code": 500}}}`,
serverName)
Expect(err).NotTo(HaveOccurred())
})

serverDeleteHandler = func(w http.ResponseWriter, _ *http.Request) {
serverDeleted = true
w.WriteHeader(http.StatusAccepted)
}
})

It("clears the identity to re-resolve it", func(ctx SpecContext) {
result, err := onboardingReconciler.Reconcile(ctx, reconcileReq)
Expect(err).NotTo(HaveOccurred())
Expect(result.RequeueAfter).To(BeNumerically(">", 0))
Expect(serverDeleted).To(BeTrue())

hv := &kvmv1.Hypervisor{}
Expect(k8sClient.Get(ctx, namespacedName, hv)).To(Succeed())
Expect(hv.Status.ServiceID).To(BeEmpty())
condition := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeOnboarding)
Expect(condition).NotTo(BeNil())
Expect(condition.Reason).To(Equal(kvmv1.ConditionReasonInitial))
Expect(condition.Message).To(ContainSubstring("cannot be scheduled"))
})
})

When("the test server ended up in an error state", func() {
var (
serverDeleteCalls int
Expand Down
Loading