Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 31 additions & 2 deletions internal/controller/eviction/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,9 @@ import (
// EvictionReconciler reconciles a Eviction object
type EvictionReconciler struct {
client.Client
Scheme *runtime.Scheme
computeClient *gophercloud.ServiceClient
Scheme *runtime.Scheme
computeClient *gophercloud.ServiceClient
placementClient *gophercloud.ServiceClient
}

const (
Expand Down Expand Up @@ -177,6 +178,29 @@ func (r *EvictionReconciler) handleRunning(ctx context.Context, eviction *kvmv1.
return r.evictNext(ctx, eviction, limit)
}

inUse, err := openstack.ResourceProviderInUse(ctx, r.placementClient, hypervisor.Status.HypervisorID)
if err != nil {
meta.SetStatusCondition(&eviction.Status.Conditions, metav1.Condition{
Type: kvmv1.ConditionTypeEvicting,
Status: metav1.ConditionTrue,
Message: fmt.Sprintf("waiting for host allocations: %v", err),
Reason: kvmv1.ConditionReasonRunning,
})
if updateErr := r.updateStatus(ctx, eviction); updateErr != nil {
return ctrl.Result{}, errors.Join(err, updateErr)
}
return ctrl.Result{}, err
}
if inUse {
meta.SetStatusCondition(&eviction.Status.Conditions, metav1.Condition{
Type: kvmv1.ConditionTypeEvicting,
Status: metav1.ConditionTrue,
Message: "waiting for host allocations to drain",
Reason: kvmv1.ConditionReasonRunning,
})
return ctrl.Result{RequeueAfter: global.DefaultPollTime}, r.updateStatus(ctx, eviction)
}

meta.SetStatusCondition(&eviction.Status.Conditions, metav1.Condition{
Type: kvmv1.ConditionTypeEvicting,
Status: metav1.ConditionFalse,
Expand Down Expand Up @@ -542,5 +566,10 @@ func (r *EvictionReconciler) SetupWithManager(mgr ctrl.Manager) error {
}
r.computeClient.Microversion = "2.90" // Xena (or later)

if r.placementClient, err = openstack.GetServiceClient(ctx, "placement", nil); err != nil {
return err
}
r.placementClient.Microversion = "1.39" // yoga, or later

return r.registerWithManager(mgr)
}
82 changes: 77 additions & 5 deletions internal/controller/eviction/controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ package eviction
import (
"fmt"
"net/http"
"time"

"github.com/gophercloud/gophercloud/v2/testhelper"
"github.com/gophercloud/gophercloud/v2/testhelper/client"
Expand Down Expand Up @@ -143,8 +144,9 @@ var _ = Describe("Eviction Controller", func() {
Name: resourceName,
Namespace: namespaceName,
}
reconcileRequest = ctrl.Request{NamespacedName: typeNamespacedName}
fakeServer testhelper.FakeServer
reconcileRequest = ctrl.Request{NamespacedName: typeNamespacedName}
fakeServer testhelper.FakeServer
placementUsagesBody string
)

BeforeEach(func(ctx SpecContext) {
Expand All @@ -157,11 +159,23 @@ var _ = Describe("Eviction Controller", func() {
Fail("Unhandled request to fake server: " + r.Method + " " + r.URL.Path)
})

By("Installing the placement usages mock")
fakeServer.Mux.HandleFunc("GET /resource_providers/{provider_id}/usages", func(w http.ResponseWriter, r *http.Request) {
Expect(r.PathValue("provider_id")).To(Equal(hypervisorId))
w.Header().Add("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, placementUsagesBody)
})

By("Setting default placement usages")
placementUsagesBody = `{"usages": {}}`

By("Creating the EvictionReconciler")
evictionReconciler = &EvictionReconciler{
Client: k8sClient,
Scheme: k8sClient.Scheme(),
computeClient: client.ServiceClient(fakeServer),
Client: k8sClient,
Scheme: k8sClient.Scheme(),
computeClient: client.ServiceClient(fakeServer),
placementClient: client.ServiceClient(fakeServer),
}
})

Expand Down Expand Up @@ -395,6 +409,64 @@ var _ = Describe("Eviction Controller", func() {
))
})
})

Context("When placement still has usage after the instances left", func() {
BeforeEach(func(ctx SpecContext) {
By("Mocking hypervisor API to return enabled status")
fakeServer.Mux.HandleFunc("GET /os-hypervisors/{hypervisor_id}", func(w http.ResponseWriter, r *http.Request) {
rHypervisorId := r.PathValue("hypervisor_id")
Expect(rHypervisorId).To(Equal(hypervisorId))
w.Header().Add("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_, err := fmt.Fprintf(w, hypervisorTpl, "null", "enabled")
Expect(err).To(Succeed())
})

By("Mocking service update API")
fakeServer.Mux.HandleFunc("PUT /os-services/{service_id}", func(w http.ResponseWriter, r *http.Request) {
rServiceId := r.PathValue("service_id")
Expect(rServiceId).To(Equal(serviceId))
w.Header().Add("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_, err := fmt.Fprintf(w, `{"service": {"id": "%v", "status": "disabled"}}`, serviceId)
Expect(err).To(Succeed())
})

By("Seeding placement with remaining usages on the host")
placementUsagesBody = `{"usages": {"VCPU": 2}}`
})

It("keeps the eviction running until placement is clear", func(ctx SpecContext) {
By("Reconciling until placement blocks completion")
Eventually(func(g Gomega) {
_, err := evictionReconciler.Reconcile(ctx, reconcileRequest)
g.Expect(err).NotTo(HaveOccurred())

resource := &kvmv1.Eviction{}
g.Expect(k8sClient.Get(ctx, typeNamespacedName, resource)).To(Succeed())
condition := meta.FindStatusCondition(resource.Status.Conditions, kvmv1.ConditionTypeEvicting)
g.Expect(condition).NotTo(BeNil())
g.Expect(condition.Status).To(Equal(metav1.ConditionTrue))
g.Expect(condition.Message).To(ContainSubstring("waiting for host allocations to drain"))
}).WithTimeout(10 * time.Second).Should(Succeed())

By("Clearing placement usage")
placementUsagesBody = `{"usages": {}}`

By("Reconciling until the eviction completes")
Eventually(func(g Gomega) {
_, err := evictionReconciler.Reconcile(ctx, reconcileRequest)
g.Expect(err).NotTo(HaveOccurred())

resource := &kvmv1.Eviction{}
g.Expect(k8sClient.Get(ctx, typeNamespacedName, resource)).To(Succeed())
condition := meta.FindStatusCondition(resource.Status.Conditions, kvmv1.ConditionTypeEvicting)
g.Expect(condition).NotTo(BeNil())
g.Expect(condition.Status).To(Equal(metav1.ConditionFalse))
g.Expect(condition.Reason).To(Equal(kvmv1.ConditionReasonSucceeded))
}).WithTimeout(10 * time.Second).Should(Succeed())
})
})
})

Context("Failure Modes", func() {
Expand Down
23 changes: 23 additions & 0 deletions internal/openstack/placement.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"context"
"errors"
"fmt"
"net/http"

"github.com/gophercloud/gophercloud/v2"
"github.com/gophercloud/gophercloud/v2/openstack/placement/v1/resourceproviders"
Expand Down Expand Up @@ -123,6 +124,28 @@ func deleteConsumerAllocations(ctx context.Context, client *gophercloud.ServiceC
return
}

// ResourceProviderInUse reports whether the resource provider has any
// non-zero usage. A missing provider is treated as not in use.
func ResourceProviderInUse(ctx context.Context, client *gophercloud.ServiceClient, providerID string) (bool, error) {
if providerID == "" {
return false, nil
}

usage, err := resourceproviders.GetUsages(ctx, client, providerID).Extract()
if err != nil {
if gophercloud.ResponseCodeIs(err, http.StatusNotFound) {
return false, nil
}
return false, fmt.Errorf("getting usages for resource provider %s: %w", providerID, err)
}
for _, amount := range usage.Usages {
if amount > 0 {
return true, nil
}
}
return false, nil
}

// Remove all empty Allocations for a certain provider, and if it is empty, delete it
func CleanupResourceProvider(ctx context.Context, client *gophercloud.ServiceClient, provider *resourceproviders.ResourceProvider) error {
if provider == nil {
Expand Down
32 changes: 32 additions & 0 deletions internal/openstack/placement_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -364,4 +364,36 @@ var _ = Describe("Placement API", func() {
})
})
})

Describe("ResourceProviderInUse", func() {
const (
providerID = "test-provider-uuid"
usagesPattern = "/resource_providers/{provider_id}/usages"
)

DescribeTable("reports whether the provider has usage",
func(status int, body string, expected bool, expectErr bool) {
fakeServer.Mux.HandleFunc(usagesPattern, func(w http.ResponseWriter, r *http.Request) {
Expect(r.Method).To(Equal(http.MethodGet))
Expect(r.PathValue("provider_id")).To(Equal(providerID))
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
fmt.Fprint(w, body)
})

inUse, err := ResourceProviderInUse(ctx, client.ServiceClient(fakeServer), providerID)
if expectErr {
Expect(err).To(HaveOccurred())
return
}
Expect(err).NotTo(HaveOccurred())
Expect(inUse).To(Equal(expected))
},
Entry("no usages", http.StatusOK, `{"usages": {}}`, false, false),
Entry("zero usage", http.StatusOK, `{"usages": {"VCPU": 0}}`, false, false),
Entry("positive usage", http.StatusOK, `{"usages": {"VCPU": 2}}`, true, false),
Entry("provider not found", http.StatusNotFound, `{"error": "Not Found"}`, false, false),
Entry("usage query failure", http.StatusInternalServerError, `{"error": "Internal Server Error"}`, false, true),
)
})
})
Loading