From 073f51ab4c1bb082d06459e615df96641eeb302c Mon Sep 17 00:00:00 2001 From: Fabian Wiesel Date: Tue, 10 Mar 2026 17:16:15 +0100 Subject: [PATCH] Move Hypervisor disabling to own controller Signed-off-by: Fabian Wiesel --- api/v1/hypervisor_types.go | 3 +- .../crds/kvm.cloud.sap_hypervisors.yaml | 1 + cmd/main.go | 8 + .../hypervisor_compute_service_controller.go | 197 ++++++++++ ...ervisor_compute_service_controller_test.go | 359 ++++++++++++++++++ .../hypervisor_maintenance_controller.go | 90 +---- .../hypervisor_maintenance_controller_test.go | 138 ------- internal/controller/onboarding_controller.go | 5 + .../controller/onboarding_controller_test.go | 20 + 9 files changed, 594 insertions(+), 227 deletions(-) create mode 100644 internal/controller/hypervisor_compute_service_controller.go create mode 100644 internal/controller/hypervisor_compute_service_controller_test.go diff --git a/api/v1/hypervisor_types.go b/api/v1/hypervisor_types.go index 4af9043a..aa5195f4 100644 --- a/api/v1/hypervisor_types.go +++ b/api/v1/hypervisor_types.go @@ -180,7 +180,7 @@ type HypervisorSpec struct { InstallCertificate bool `json:"installCertificate"` // +kubebuilder:optional - // +kubebuilder:validation:Enum:="";manual;auto;ha;termination + // +kubebuilder:validation:Enum:="";manual;auto;ha;no-schedule;termination // Maintenance indicates whether the hypervisor is in maintenance mode. Maintenance string `json:"maintenance,omitempty"` @@ -213,6 +213,7 @@ const ( MaintenanceManual = "manual" // manual maintenance mode by external user MaintenanceAuto = "auto" // automatic maintenance mode MaintenanceHA = "ha" // high availability maintenance mode + MaintenanceNoSchedule = "no-schedule" // compute service disabled, no eviction MaintenanceTermination = "termination" // internal use only, when node is terminating state ) diff --git a/charts/openstack-hypervisor-operator/crds/kvm.cloud.sap_hypervisors.yaml b/charts/openstack-hypervisor-operator/crds/kvm.cloud.sap_hypervisors.yaml index 1e10c618..bc54c73f 100644 --- a/charts/openstack-hypervisor-operator/crds/kvm.cloud.sap_hypervisors.yaml +++ b/charts/openstack-hypervisor-operator/crds/kvm.cloud.sap_hypervisors.yaml @@ -222,6 +222,7 @@ spec: - manual - auto - ha + - no-schedule - termination type: string maintenanceReason: diff --git a/cmd/main.go b/cmd/main.go index 5e4bf315..3757afc8 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -354,6 +354,14 @@ func main() { os.Exit(1) } + if err = (&controller.HypervisorComputeServiceController{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", controller.HypervisorComputeServiceControllerName) + os.Exit(1) + } + if err = (&controller.OnboardingController{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), diff --git a/internal/controller/hypervisor_compute_service_controller.go b/internal/controller/hypervisor_compute_service_controller.go new file mode 100644 index 00000000..2dbf0bc4 --- /dev/null +++ b/internal/controller/hypervisor_compute_service_controller.go @@ -0,0 +1,197 @@ +/* +SPDX-FileCopyrightText: Copyright 2024 SAP SE or an SAP affiliate company and cobaltcore-dev contributors +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "errors" + "fmt" + + "github.com/gophercloud/gophercloud/v2" + "github.com/gophercloud/gophercloud/v2/openstack/compute/v2/services" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + k8sacmetav1 "k8s.io/client-go/applyconfigurations/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" + logger "sigs.k8s.io/controller-runtime/pkg/log" + + kvmv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + apiv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/applyconfigurations/api/v1" + "github.com/cobaltcore-dev/openstack-hypervisor-operator/internal/openstack" + "github.com/cobaltcore-dev/openstack-hypervisor-operator/internal/utils" +) + +const ( + HypervisorComputeServiceControllerName = "HypervisorComputeService" +) + +type HypervisorComputeServiceController struct { + k8sclient.Client + Scheme *runtime.Scheme + computeClient *gophercloud.ServiceClient +} + +// +kubebuilder:rbac:groups=kvm.cloud.sap,resources=hypervisors,verbs=get;list;watch;update;patch +// +kubebuilder:rbac:groups=kvm.cloud.sap,resources=hypervisors/status,verbs=get;list;watch;update;patch +func (r *HypervisorComputeServiceController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := logger.FromContext(ctx).WithName(req.Name) + ctx = logger.IntoContext(ctx, log) + + hv := &kvmv1.Hypervisor{} + if err := r.Get(ctx, req.NamespacedName, hv); err != nil { + // ignore not found errors, could be deleted + return ctrl.Result{}, k8sclient.IgnoreNotFound(err) + } + + // We can only control the compute service if there is a serviceID + serviceId := hv.Status.ServiceID + if serviceId == "" { + return ctrl.Result{}, nil // Service not yet registered, nothing to do + } + + // If onboarding hasn't started yet, no compute service exists + onboardingCondition := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeOnboarding) + if onboardingCondition == nil { + return ctrl.Result{}, nil // Onboarding not started, so no compute service + } + + // Determine if compute service should be disabled and the reason + onboardingInProgress := onboardingCondition.Status == metav1.ConditionTrue && + (onboardingCondition.Reason == kvmv1.ConditionReasonTesting || + onboardingCondition.Reason == kvmv1.ConditionReasonHandover) + maintenanceDisabling := hv.Spec.Maintenance == kvmv1.MaintenanceNoSchedule || + hv.Spec.Maintenance == kvmv1.MaintenanceManual || + hv.Spec.Maintenance == kvmv1.MaintenanceAuto || + hv.Spec.Maintenance == kvmv1.MaintenanceTermination + aborted := onboardingCondition.Status == metav1.ConditionFalse && + onboardingCondition.Reason == kvmv1.ConditionReasonAborted + + // During onboarding (testing/handover), always enable the compute service + // Otherwise, disable if maintenance mode requires it or onboarding was aborted + shouldDisable := !onboardingInProgress && (maintenanceDisabling || aborted) + + // Build the desired condition. This controller only owns + // ConditionTypeHypervisorDisabled; ConditionTypeReady is owned by the + // ready.Controller. + var desired metav1.Condition + var reason string + var message string + if shouldDisable { + // Determine the reason based on why compute service is being disabled + switch { + case aborted: + reason = kvmv1.ConditionReasonAborted + message = "Compute service disabled due to aborted onboarding" + case hv.Spec.Maintenance == kvmv1.MaintenanceNoSchedule: + reason = kvmv1.ConditionReasonSucceeded + message = "Compute service disabled per spec (no-schedule)" + case hv.Spec.Maintenance == kvmv1.MaintenanceTermination: + reason = kvmv1.ConditionReasonTerminating + message = "Compute service disabled due to termination" + default: + reason = kvmv1.ConditionReasonSucceeded + message = fmt.Sprintf("Compute service disabled per spec (maintenance: %s)", hv.Spec.Maintenance) + } + desired = metav1.Condition{ + Type: kvmv1.ConditionTypeHypervisorDisabled, + Status: metav1.ConditionTrue, + Message: message, + Reason: reason, + } + } else { + reason = kvmv1.ConditionReasonSucceeded + message = "Compute service is enabled" + desired = metav1.Condition{ + Type: kvmv1.ConditionTypeHypervisorDisabled, + Status: metav1.ConditionFalse, + Message: message, + Reason: reason, + } + } + + // Desired state already achieved + if existing := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeHypervisorDisabled); existing != nil && + existing.Status == desired.Status && existing.Reason == desired.Reason && existing.Message == desired.Message { + return ctrl.Result{}, nil + } + + statusCfg := apiv1.HypervisorStatus() + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, *utils.ConditionFromStatus(desired)) + + fail := func(cause error) (ctrl.Result, error) { + utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, + *k8sacmetav1.Condition(). + WithType(kvmv1.ConditionTypeHypervisorDisabled). + WithStatus(metav1.ConditionUnknown). + WithReason(kvmv1.ConditionReasonFailed). + WithMessage(cause.Error())) + applyErr := r.applyStatus(ctx, hv.Name, statusCfg) + return ctrl.Result{}, errors.Join(cause, applyErr) + } + + if shouldDisable { + log.Info("Disabling compute service", "id", serviceId, "reason", reason) + opts := services.UpdateOpts{ + Status: services.ServiceDisabled, + DisabledReason: message, + } + if _, err := services.Update(ctx, r.computeClient, serviceId, opts).Extract(); err != nil { + return fail(fmt.Errorf("failed to disable compute service due to %w", err)) + } + } else { + log.Info("Enabling compute service", "id", serviceId) + // The service may be forced down previously due to an HA event, + // so we need to ensure it not only enabled, but also not forced to be down. + falseVal := false + opts := openstack.UpdateServiceOpts{ + Status: services.ServiceEnabled, + ForcedDown: &falseVal, + } + if _, err := services.Update(ctx, r.computeClient, serviceId, opts).Extract(); err != nil { + return fail(fmt.Errorf("failed to enable compute service due to %w", err)) + } + } + + return ctrl.Result{}, r.applyStatus(ctx, hv.Name, statusCfg) +} + +// applyStatus applies the hypervisor status via SSA with this controller as the +// field owner. +func (r *HypervisorComputeServiceController) applyStatus(ctx context.Context, name string, cfg *apiv1.HypervisorStatusApplyConfiguration) error { + return r.Status().Apply(ctx, + apiv1.Hypervisor(name).WithStatus(cfg), + k8sclient.ForceOwnership, k8sclient.FieldOwner(HypervisorComputeServiceControllerName)) +} + +// SetupWithManager sets up the controller with the Manager. +func (r *HypervisorComputeServiceController) SetupWithManager(mgr ctrl.Manager) error { + ctx := context.Background() + + var err error + if r.computeClient, err = openstack.GetServiceClient(ctx, "compute", nil); err != nil { + return err + } + r.computeClient.Microversion = "2.90" // Xena (or later) + + return ctrl.NewControllerManagedBy(mgr). + Named(HypervisorComputeServiceControllerName). + For(&kvmv1.Hypervisor{}). // trigger the r.Reconcile whenever a hypervisor is created/updated/deleted. + Complete(r) +} diff --git a/internal/controller/hypervisor_compute_service_controller_test.go b/internal/controller/hypervisor_compute_service_controller_test.go new file mode 100644 index 00000000..5eef096b --- /dev/null +++ b/internal/controller/hypervisor_compute_service_controller_test.go @@ -0,0 +1,359 @@ +/* +SPDX-FileCopyrightText: Copyright 2024 SAP SE or an SAP affiliate company and cobaltcore-dev contributors +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "net/http" + + "github.com/gophercloud/gophercloud/v2/testhelper" + "github.com/gophercloud/gophercloud/v2/testhelper/client" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + + kvmv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" +) + +var _ = Describe("HypervisorComputeServiceController", Label("test"), func() { + const ( + EOF = "EOF" + hostName = "test-hostname" + region = "region" + zone = "zone" + serviceId = "test-service-id" + ) + var ( + controller *HypervisorComputeServiceController + fakeServer testhelper.FakeServer + hypervisorName = types.NamespacedName{Name: "test-node"} + numApiRequest = 0 + ) + + mockApi := func(expectedStatus string, expectedReason string) { + fakeServer.Mux.HandleFunc(fmt.Sprintf("/os-services/%v", serviceId), func(w http.ResponseWriter, r *http.Request) { + numApiRequest++ + Expect(r.Method).To(Equal("PUT")) + Expect(r.Header.Get("Content-Type")).To(Equal("application/json")) + body := make([]byte, r.ContentLength) + _, err := r.Body.Read(body) + Expect(err == nil || err.Error() == EOF).To(BeTrue()) + + bodyStr := string(body) + Expect(bodyStr).To(ContainSubstring(fmt.Sprintf(`"status":"%s"`, expectedStatus))) + if expectedReason != "" { + Expect(bodyStr).To(ContainSubstring(expectedReason)) + } + + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err = fmt.Fprintf(w, `{"service": {"id": "%v", "status": "%s"}}`, serviceId, expectedStatus) + Expect(err).NotTo(HaveOccurred()) + }) + } + + mockApiEnable := func() { + mockApi("enabled", "") + } + + mockApiDisable := func(reason string) { + mockApi("disabled", reason) + } + + BeforeEach(func(ctx SpecContext) { + fakeServer = testhelper.SetupHTTP() + numApiRequest = 0 + DeferCleanup(fakeServer.Teardown) + + controller = &HypervisorComputeServiceController{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + computeClient: client.ServiceClient(fakeServer), + } + + By("creating the hypervisor resource") + hypervisor := &kvmv1.Hypervisor{ + Name: hypervisorName.Name, + Labels: map[string]string{ + corev1.LabelHostname: hostName, + corev1.LabelTopologyRegion: region, + corev1.LabelTopologyZone: zone, + }, + Spec: kvmv1.HypervisorSpec{ + LifecycleEnabled: true, + }, + } + Expect(k8sClient.Create(ctx, hypervisor)).To(Succeed()) + DeferCleanup(func(ctx SpecContext) { + Expect(k8sClient.Delete(ctx, hypervisor)).To(Succeed()) + }) + + By("setting service ID and onboarding to completed successfully") + hypervisor.Status.ServiceID = serviceId + meta.SetStatusCondition(&hypervisor.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeOnboarding, + Status: metav1.ConditionFalse, + Reason: kvmv1.ConditionReasonSucceeded, + Message: "Onboarding succeeded", + }) + Expect(k8sClient.Status().Update(ctx, hypervisor)).To(Succeed()) + }) + + Context("Happy path", func() { + JustBeforeEach(func(ctx context.Context) { + By("reconciling twice") + for range 2 { + _, err := controller.Reconcile(ctx, ctrl.Request{NamespacedName: hypervisorName}) + Expect(err).To(Succeed()) + } + }) + + Context("with maintenance=no-schedule", func() { + BeforeEach(func(ctx SpecContext) { + hv := &kvmv1.Hypervisor{} + By("setting hv.Spec.Maintenance=no-schedule") + Expect(k8sClient.Get(ctx, hypervisorName, hv)).To(Succeed()) + hv.Spec.Maintenance = kvmv1.MaintenanceNoSchedule + Expect(k8sClient.Update(ctx, hv)).To(Succeed()) + + mockApiDisable("no-schedule") + }) + + It("should have called the api server once", func() { + Expect(numApiRequest).To(Equal(1)) + }) + + It("should reflect the disabled state in the status", func(ctx SpecContext) { + hv := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hv)).To(Succeed()) + + condition := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeHypervisorDisabled) + Expect(condition).NotTo(BeNil()) + Expect(condition.Status).To(Equal(metav1.ConditionTrue)) + Expect(condition.Reason).To(Equal(kvmv1.ConditionReasonSucceeded)) + Expect(condition.Message).To(ContainSubstring("no-schedule")) + }) + }) + + Context("with maintenance=manual", func() { + BeforeEach(func(ctx SpecContext) { + hv := &kvmv1.Hypervisor{} + By("setting hv.Spec.Maintenance=manual") + Expect(k8sClient.Get(ctx, hypervisorName, hv)).To(Succeed()) + hv.Spec.Maintenance = kvmv1.MaintenanceManual + hv.Spec.MaintenanceReason = "testing" + Expect(k8sClient.Update(ctx, hv)).To(Succeed()) + + mockApiDisable("manual") + }) + + It("should have called the api server once", func() { + Expect(numApiRequest).To(Equal(1)) + }) + + It("should reflect the disabled state in the status", func(ctx SpecContext) { + hv := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hv)).To(Succeed()) + + condition := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeHypervisorDisabled) + Expect(condition).NotTo(BeNil()) + Expect(condition.Status).To(Equal(metav1.ConditionTrue)) + Expect(condition.Reason).To(Equal(kvmv1.ConditionReasonSucceeded)) + }) + }) + + Context("with onboarding aborted", func() { + BeforeEach(func(ctx SpecContext) { + hv := &kvmv1.Hypervisor{} + By("setting onboarding to aborted") + Expect(k8sClient.Get(ctx, hypervisorName, hv)).To(Succeed()) + meta.SetStatusCondition(&hv.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeOnboarding, + Status: metav1.ConditionFalse, + Reason: kvmv1.ConditionReasonAborted, + Message: "Onboarding aborted", + }) + Expect(k8sClient.Status().Update(ctx, hv)).To(Succeed()) + + mockApiDisable("aborted") + }) + + It("should have called the api server once", func() { + Expect(numApiRequest).To(Equal(1)) + }) + + It("should reflect the disabled state in the status", func(ctx SpecContext) { + hv := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hv)).To(Succeed()) + + condition := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeHypervisorDisabled) + Expect(condition).NotTo(BeNil()) + Expect(condition.Status).To(Equal(metav1.ConditionTrue)) + Expect(condition.Reason).To(Equal(kvmv1.ConditionReasonAborted)) + }) + }) + + Context("with onboarding in testing phase", func() { + BeforeEach(func(ctx SpecContext) { + hv := &kvmv1.Hypervisor{} + By("setting onboarding to testing (in progress)") + Expect(k8sClient.Get(ctx, hypervisorName, hv)).To(Succeed()) + meta.SetStatusCondition(&hv.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeOnboarding, + Status: metav1.ConditionTrue, + Reason: kvmv1.ConditionReasonTesting, + Message: "Testing in progress", + }) + Expect(k8sClient.Status().Update(ctx, hv)).To(Succeed()) + + mockApiEnable() + }) + + It("should have called the api server once", func() { + Expect(numApiRequest).To(Equal(1)) + }) + + It("should reflect the enabled state in the status", func(ctx SpecContext) { + hv := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hv)).To(Succeed()) + + condition := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeHypervisorDisabled) + Expect(condition).NotTo(BeNil()) + Expect(condition.Status).To(Equal(metav1.ConditionFalse)) + Expect(condition.Reason).To(Equal(kvmv1.ConditionReasonSucceeded)) + }) + }) + + Context("with onboarding in handover phase", func() { + BeforeEach(func(ctx SpecContext) { + hv := &kvmv1.Hypervisor{} + By("setting onboarding to handover (ready for completion)") + Expect(k8sClient.Get(ctx, hypervisorName, hv)).To(Succeed()) + meta.SetStatusCondition(&hv.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeOnboarding, + Status: metav1.ConditionTrue, + Reason: kvmv1.ConditionReasonHandover, + Message: "Handover in progress", + }) + Expect(k8sClient.Status().Update(ctx, hv)).To(Succeed()) + + mockApiEnable() + }) + + It("should have called the api server once", func() { + Expect(numApiRequest).To(Equal(1)) + }) + + It("should reflect the enabled state in the status", func(ctx SpecContext) { + hv := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hv)).To(Succeed()) + + condition := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeHypervisorDisabled) + Expect(condition).NotTo(BeNil()) + Expect(condition.Status).To(Equal(metav1.ConditionFalse)) + Expect(condition.Reason).To(Equal(kvmv1.ConditionReasonSucceeded)) + }) + }) + + Context("with compute service enabled (default)", func() { + BeforeEach(func() { + mockApiEnable() + }) + + It("should have called the api server once", func() { + Expect(numApiRequest).To(Equal(1)) + }) + + It("should reflect the enabled state in the status", func(ctx SpecContext) { + hv := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hv)).To(Succeed()) + + condition := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeHypervisorDisabled) + Expect(condition).NotTo(BeNil()) + Expect(condition.Status).To(Equal(metav1.ConditionFalse)) + Expect(condition.Reason).To(Equal(kvmv1.ConditionReasonSucceeded)) + Expect(condition.Message).To(Equal("Compute service is enabled")) + }) + }) + }) + + Context("Failure modes", func() { + JustBeforeEach(func(ctx context.Context) { + By("reconciling once (expecting error)") + //nolint:errcheck // Ignoring error - failure tests expect errors + controller.Reconcile(ctx, ctrl.Request{NamespacedName: hypervisorName}) + }) + + Context("when enabling compute service fails", func() { + BeforeEach(func(ctx SpecContext) { + // Mock API to return error + fakeServer.Mux.HandleFunc(fmt.Sprintf("/os-services/%v", serviceId), func(w http.ResponseWriter, r *http.Request) { + numApiRequest++ + w.WriteHeader(http.StatusInternalServerError) + _, err := fmt.Fprint(w, `{"error": "internal server error"}`) + Expect(err).NotTo(HaveOccurred()) + }) + }) + + It("should reflect the error in status with Unknown condition", func(ctx SpecContext) { + hv := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hv)).To(Succeed()) + + condition := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeHypervisorDisabled) + Expect(condition).NotTo(BeNil()) + Expect(condition.Status).To(Equal(metav1.ConditionUnknown)) + Expect(condition.Reason).To(Equal(kvmv1.ConditionReasonFailed)) + Expect(numApiRequest).To(BeNumerically(">", 0)) + }) + }) + + Context("when disabling compute service fails", func() { + BeforeEach(func(ctx SpecContext) { + hv := &kvmv1.Hypervisor{} + By("setting hv.Spec.Maintenance=no-schedule") + Expect(k8sClient.Get(ctx, hypervisorName, hv)).To(Succeed()) + hv.Spec.Maintenance = kvmv1.MaintenanceNoSchedule + Expect(k8sClient.Update(ctx, hv)).To(Succeed()) + + // Mock API to return error + fakeServer.Mux.HandleFunc(fmt.Sprintf("/os-services/%v", serviceId), func(w http.ResponseWriter, r *http.Request) { + numApiRequest++ + w.WriteHeader(http.StatusInternalServerError) + _, err := fmt.Fprint(w, `{"error": "internal server error"}`) + Expect(err).NotTo(HaveOccurred()) + }) + }) + + It("should reflect the error in status with Unknown condition", func(ctx SpecContext) { + hv := &kvmv1.Hypervisor{} + Expect(k8sClient.Get(ctx, hypervisorName, hv)).To(Succeed()) + + condition := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeHypervisorDisabled) + Expect(condition).NotTo(BeNil()) + Expect(condition.Status).To(Equal(metav1.ConditionUnknown)) + Expect(condition.Reason).To(Equal(kvmv1.ConditionReasonFailed)) + Expect(numApiRequest).To(BeNumerically(">", 0)) + }) + }) + }) +}) diff --git a/internal/controller/hypervisor_maintenance_controller.go b/internal/controller/hypervisor_maintenance_controller.go index fee246db..587def6e 100644 --- a/internal/controller/hypervisor_maintenance_controller.go +++ b/internal/controller/hypervisor_maintenance_controller.go @@ -17,8 +17,7 @@ limitations under the License. package controller -// This controller only takes care of enabling or disabling the compute -// service depending on the hypervisor spec Maintenance field +// This controller takes care of triggering evictions based on the maintenance field import ( "context" @@ -35,7 +34,6 @@ import ( logger "sigs.k8s.io/controller-runtime/pkg/log" "github.com/gophercloud/gophercloud/v2" - "github.com/gophercloud/gophercloud/v2/openstack/compute/v2/services" kvmv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" apiv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/applyconfigurations/api/v1" @@ -75,16 +73,6 @@ func (hec *HypervisorMaintenanceController) Reconcile(ctx context.Context, req c // Retain fields owned by this controller so SSA does not prune them on Apply. statusCfg := apiv1.HypervisorStatus().WithEvicted(hv.Status.Evicted) - retainStatusCondition(statusCfg, hv.Status.Conditions, kvmv1.ConditionTypeHypervisorDisabled) - - if err := hec.reconcileComputeService(ctx, hv, statusCfg); err != nil { - retainStatusCondition(statusCfg, hv.Status.Conditions, kvmv1.ConditionTypeEvicting) - retainStatusCondition(statusCfg, hv.Status.Conditions, kvmv1.ConditionTypeIncomingMigrationsSettled) - applyErr := hec.Status().Apply(ctx, - apiv1.Hypervisor(hv.Name).WithStatus(statusCfg), - k8sclient.ForceOwnership, k8sclient.FieldOwner(HypervisorMaintenanceControllerName)) - return ctrl.Result{}, errors.Join(err, applyErr) - } requeue, err := hec.reconcileEviction(ctx, hv, statusCfg) applyErr := hec.Status().Apply(ctx, @@ -110,80 +98,6 @@ func retainStatusCondition(statusCfg *apiv1.HypervisorStatusApplyConfiguration, utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, *utils.ConditionFromStatus(*condition)) } -// reconcileComputeService enables/disables the nova-compute service based on -// hv.Spec.Maintenance and sets the HypervisorDisabled condition on statusCfg. -func (hec *HypervisorMaintenanceController) reconcileComputeService(ctx context.Context, hv *kvmv1.Hypervisor, statusCfg *apiv1.HypervisorStatusApplyConfiguration) error { - log := logger.FromContext(ctx) - serviceId := hv.Status.ServiceID - - // We can only do something here, if there is a service to begin with. - // The onboarding should take care of that - if serviceId == "" { - return nil - } - - switch hv.Spec.Maintenance { - case kvmv1.MaintenanceUnset: - existing := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeHypervisorDisabled) - if existing == nil || existing.Status != metav1.ConditionFalse { - // We need to enable the host as per spec. - // Also clear forced_down in case a previous HA event set it. - falseVal := false - enableService := openstack.UpdateServiceOpts{ - Status: services.ServiceEnabled, - ForcedDown: &falseVal, - } - log.Info("Enabling hypervisor", "id", serviceId) - if _, err := services.Update(ctx, hec.computeClient, serviceId, enableService).Extract(); err != nil { - err = fmt.Errorf("failed to enable hypervisor due to %w", err) - utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, - *k8sacmetav1.Condition(). - WithType(kvmv1.ConditionTypeHypervisorDisabled). - WithStatus(metav1.ConditionUnknown). - WithMessage(err.Error()). - WithReason(kvmv1.ConditionReasonFailed)) - return err - } - } - utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, - *k8sacmetav1.Condition(). - WithType(kvmv1.ConditionTypeHypervisorDisabled). - WithStatus(metav1.ConditionFalse). - WithMessage("Hypervisor is enabled"). - WithReason(kvmv1.ConditionReasonSucceeded)) - - case kvmv1.MaintenanceManual, kvmv1.MaintenanceAuto, kvmv1.MaintenanceTermination: - // Disable the compute service. - existing := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeHypervisorDisabled) - if existing == nil || existing.Status != metav1.ConditionTrue { - disableService := services.UpdateOpts{ - Status: services.ServiceDisabled, - DisabledReason: "Hypervisor CRD: spec.maintenance=" + hv.Spec.Maintenance, - } - // We need to disable the host as per spec - log.Info("Disabling hypervisor", "id", serviceId) - if _, err := services.Update(ctx, hec.computeClient, serviceId, disableService).Extract(); err != nil { - err = fmt.Errorf("failed to disable hypervisor due to %w", err) - utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, - *k8sacmetav1.Condition(). - WithType(kvmv1.ConditionTypeHypervisorDisabled). - WithStatus(metav1.ConditionUnknown). - WithMessage(err.Error()). - WithReason(kvmv1.ConditionReasonFailed)) - return err - } - } - utils.SetApplyConfigurationStatusCondition(&statusCfg.Conditions, - *k8sacmetav1.Condition(). - WithType(kvmv1.ConditionTypeHypervisorDisabled). - WithStatus(metav1.ConditionTrue). - WithMessage("Hypervisor is disabled"). - WithReason(kvmv1.ConditionReasonSucceeded)) - } - - return nil -} - // reconcileEviction creates/deletes the Eviction CR and sets the ConditionTypeEvicting // and ConditionTypeIncomingMigrationsSettled conditions plus the Evicted scalar on // statusCfg. When eviction should be removed, the condition entries are filtered out @@ -194,7 +108,7 @@ func (hec *HypervisorMaintenanceController) reconcileEviction(ctx context.Contex } switch hv.Spec.Maintenance { - case kvmv1.MaintenanceUnset: + case kvmv1.MaintenanceUnset, kvmv1.MaintenanceNoSchedule: // Avoid deleting the eviction over and over. if !hv.Status.Evicted && meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeEvicting) == nil && diff --git a/internal/controller/hypervisor_maintenance_controller_test.go b/internal/controller/hypervisor_maintenance_controller_test.go index 75e97c3d..5c797264 100644 --- a/internal/controller/hypervisor_maintenance_controller_test.go +++ b/internal/controller/hypervisor_maintenance_controller_test.go @@ -179,144 +179,6 @@ var _ = Describe("HypervisorMaintenanceController", func() { Expect(k8sClient.Status().Update(ctx, hypervisor)).To(Succeed()) }) - Describe("Enabling or Disabling the Nova Service", func() { - Context("when enabling the service fails", func() { - BeforeEach(func(ctx SpecContext) { - expectReconcileErr = true - hypervisor := &kvmv1.Hypervisor{} - Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) - meta.SetStatusCondition(&hypervisor.Status.Conditions, metav1.Condition{ - Type: kvmv1.ConditionTypeHypervisorDisabled, - Status: metav1.ConditionTrue, - Reason: kvmv1.ConditionReasonSucceeded, - Message: "Hypervisor is disabled", - }) - Expect(k8sClient.Status().Update(ctx, hypervisor)).To(Succeed()) - - fakeServer.Mux.HandleFunc("PUT /os-services/1234", func(w http.ResponseWriter, _ *http.Request) { - http.Error(w, "nova unavailable", http.StatusServiceUnavailable) - }) - }) - - It("reports the failed transition", func(ctx SpecContext) { - Expect(reconcileErr).To(MatchError(ContainSubstring("failed to enable hypervisor"))) - updated := &kvmv1.Hypervisor{} - Expect(k8sClient.Get(ctx, hypervisorName, updated)).To(Succeed()) - condition := meta.FindStatusCondition(updated.Status.Conditions, kvmv1.ConditionTypeHypervisorDisabled) - Expect(condition).NotTo(BeNil()) - Expect(condition.Status).To(Equal(metav1.ConditionUnknown)) - Expect(condition.Reason).To(Equal(kvmv1.ConditionReasonFailed)) - Expect(condition.Message).To(ContainSubstring("failed to enable hypervisor")) - }) - }) - - Context("when disabling the service fails", func() { - BeforeEach(func(ctx SpecContext) { - expectReconcileErr = true - hypervisor := &kvmv1.Hypervisor{} - Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) - meta.SetStatusCondition(&hypervisor.Status.Conditions, metav1.Condition{ - Type: kvmv1.ConditionTypeEvicting, - Status: metav1.ConditionTrue, - Reason: kvmv1.ConditionReasonRunning, - Message: "Evicting", - }) - meta.SetStatusCondition(&hypervisor.Status.Conditions, metav1.Condition{ - Type: kvmv1.ConditionTypeIncomingMigrationsSettled, - Status: metav1.ConditionFalse, - Reason: kvmv1.ConditionReasonWaiting, - Message: "Waiting for incoming migrations", - }) - Expect(k8sClient.Status().Update(ctx, hypervisor)).To(Succeed()) - Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) - hypervisor.Spec.Maintenance = kvmv1.MaintenanceAuto - Expect(k8sClient.Update(ctx, hypervisor)).To(Succeed()) - - fakeServer.Mux.HandleFunc("PUT /os-services/1234", func(w http.ResponseWriter, _ *http.Request) { - http.Error(w, "nova unavailable", http.StatusServiceUnavailable) - }) - }) - - It("reports the failure without starting eviction", func(ctx SpecContext) { - Expect(reconcileErr).To(MatchError(ContainSubstring("failed to disable hypervisor"))) - updated := &kvmv1.Hypervisor{} - Expect(k8sClient.Get(ctx, hypervisorName, updated)).To(Succeed()) - condition := meta.FindStatusCondition(updated.Status.Conditions, kvmv1.ConditionTypeHypervisorDisabled) - Expect(condition).NotTo(BeNil()) - Expect(condition.Status).To(Equal(metav1.ConditionUnknown)) - Expect(condition.Reason).To(Equal(kvmv1.ConditionReasonFailed)) - Expect(condition.Message).To(ContainSubstring("failed to disable hypervisor")) - evictingCondition := meta.FindStatusCondition(updated.Status.Conditions, kvmv1.ConditionTypeEvicting) - Expect(evictingCondition).NotTo(BeNil()) - Expect(evictingCondition.Status).To(Equal(metav1.ConditionTrue)) - migrationsCondition := meta.FindStatusCondition(updated.Status.Conditions, kvmv1.ConditionTypeIncomingMigrationsSettled) - Expect(migrationsCondition).NotTo(BeNil()) - Expect(migrationsCondition.Status).To(Equal(metav1.ConditionFalse)) - - eviction := &kvmv1.Eviction{} - err := k8sClient.Get(ctx, hypervisorName, eviction) - Expect(err).To(HaveOccurred()) - Expect(k8sclient.IgnoreNotFound(err)).To(Succeed()) - }) - }) - - Context("Spec.Maintenance=\"\"", func() { - BeforeEach(func(ctx SpecContext) { - hypervisor := &kvmv1.Hypervisor{} - Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) - hypervisor.Spec.Maintenance = "" - Expect(k8sClient.Update(ctx, hypervisor)).To(Succeed()) - expectedBody := `{"status": "enabled", "forced_down": false}` - mockServiceUpdate(expectedBody) - }) - - It("should set the ConditionTypeHypervisorDisabled to false", func(ctx SpecContext) { - updated := &kvmv1.Hypervisor{} - Expect(k8sClient.Get(ctx, hypervisorName, updated)).To(Succeed()) - Expect(meta.IsStatusConditionFalse(updated.Status.Conditions, kvmv1.ConditionTypeHypervisorDisabled)).To(BeTrue()) - }) - }) // Spec.Maintenance="" - }) - - for _, mode := range []string{"auto", "manual"} { - Context(fmt.Sprintf("Spec.Maintenance=\"%v\"", mode), func() { - BeforeEach(func(ctx SpecContext) { - hypervisor := &kvmv1.Hypervisor{} - Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) - hypervisor.Spec.Maintenance = mode - if mode == "manual" { - hypervisor.Spec.MaintenanceReason = "Test maintenance reason" - } - Expect(k8sClient.Update(ctx, hypervisor)).To(Succeed()) - expectedBody := fmt.Sprintf(`{"disabled_reason": "Hypervisor CRD: spec.maintenance=%v", "status": "disabled"}`, mode) - mockServiceUpdate(expectedBody) - }) - - It("should set the ConditionTypeHypervisorDisabled to true", func(ctx SpecContext) { - updated := &kvmv1.Hypervisor{} - Expect(k8sClient.Get(ctx, hypervisorName, updated)).To(Succeed()) - Expect(meta.IsStatusConditionTrue(updated.Status.Conditions, kvmv1.ConditionTypeHypervisorDisabled)).To(BeTrue()) - }) - }) // Spec.Maintenance="" - } - - Context("Spec.Maintenance=\"ha\"", func() { - BeforeEach(func(ctx SpecContext) { - hypervisor := &kvmv1.Hypervisor{} - Expect(k8sClient.Get(ctx, hypervisorName, hypervisor)).To(Succeed()) - hypervisor.Spec.Maintenance = kvmv1.MaintenanceHA - Expect(k8sClient.Update(ctx, hypervisor)).To(Succeed()) - }) - - It("should not call the Nova API (kvm-ha-service owns enable/disable for ha mode)", func(ctx SpecContext) { - // The controller takes no action for ha mode; kvm-ha-service handles it. - // Verify the hypervisor is still accessible and no condition is set by this controller. - updated := &kvmv1.Hypervisor{} - Expect(k8sClient.Get(ctx, hypervisorName, updated)).To(Succeed()) - Expect(meta.IsStatusConditionTrue(updated.Status.Conditions, kvmv1.ConditionTypeHypervisorDisabled)).To(BeFalse()) - }) - }) // Spec.Maintenance="ha" - Describe("Eviction reconciliation", func() { Context("Spec.Maintenance=\"\"", func() { BeforeEach(func(ctx SpecContext) { diff --git a/internal/controller/onboarding_controller.go b/internal/controller/onboarding_controller.go index aa923e90..f94ebe79 100644 --- a/internal/controller/onboarding_controller.go +++ b/internal/controller/onboarding_controller.go @@ -345,6 +345,11 @@ func (r *OnboardingController) completeOnboarding(ctx context.Context, host stri return ctrl.Result{RequeueAfter: r.getRequeueInterval()}, nil } + // Wait for HypervisorComputeService controller to enable the compute service + if !meta.IsStatusConditionFalse(hv.Status.Conditions, kvmv1.ConditionTypeHypervisorDisabled) { + return ctrl.Result{}, nil + } + // Wait for HypervisorInstanceHa controller to enable HA if hv.Spec.HighAvailability && !meta.IsStatusConditionTrue(hv.Status.Conditions, kvmv1.ConditionTypeHaEnabled) { return ctrl.Result{}, nil diff --git a/internal/controller/onboarding_controller_test.go b/internal/controller/onboarding_controller_test.go index 13349dae..7e2fc010 100644 --- a/internal/controller/onboarding_controller_test.go +++ b/internal/controller/onboarding_controller_test.go @@ -544,6 +544,16 @@ var _ = Describe("Onboarding Controller", func() { }) Expect(k8sClient.Status().Update(ctx, hv)).To(Succeed()) + By("Simulating HypervisorComputeService controller enabling compute service") + Expect(k8sClient.Get(ctx, namespacedName, hv)).To(Succeed()) + meta.SetStatusCondition(&hv.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeHypervisorDisabled, + Status: metav1.ConditionFalse, + Reason: kvmv1.ConditionReasonSucceeded, + Message: "Compute service is enabled", + }) + Expect(k8sClient.Status().Update(ctx, hv)).To(Succeed()) + By("Reconciling again to call completeOnboarding and set RemovingTestAggregate") err = reconcileLoop(ctx, 1) Expect(err).NotTo(HaveOccurred()) @@ -623,6 +633,16 @@ var _ = Describe("Onboarding Controller", func() { }) Expect(k8sClient.Status().Update(ctx, hv)).To(Succeed()) + By("Simulating HypervisorComputeService controller enabling compute service") + Expect(k8sClient.Get(ctx, namespacedName, hv)).To(Succeed()) + meta.SetStatusCondition(&hv.Status.Conditions, metav1.Condition{ + Type: kvmv1.ConditionTypeHypervisorDisabled, + Status: metav1.ConditionFalse, + Reason: kvmv1.ConditionReasonSucceeded, + Message: "Compute service is enabled", + }) + Expect(k8sClient.Status().Update(ctx, hv)).To(Succeed()) + By("Simulating aggregates controller setting condition after removing test aggregate") Expect(k8sClient.Get(ctx, namespacedName, hv)).To(Succeed()) hv.Status.Aggregates = []kvmv1.Aggregate{