diff --git a/.github/workflows/cortex-agent-release.yaml b/.github/workflows/cortex-agent-release.yaml index 4587e28ef..2a03cbeb7 100644 --- a/.github/workflows/cortex-agent-release.yaml +++ b/.github/workflows/cortex-agent-release.yaml @@ -38,6 +38,16 @@ on: CORTEX_AI_AGENTS_CLIENT_PKEY: required: true +# Serialize release runs per PR, but never cancel one in flight: the release +# agent is a stateful multi-step mutator that opens a prep PR partway through, +# so cancelling it would leave an orphaned prep PR and a half-prepared release. +# A second event for the same PR therefore queues behind the running one; the +# /release command is idempotent, so the queued run converges rather than +# duplicating. Mirrors the serialize-don't-cancel policy of update-appversion. +concurrency: + group: cortex-agent-release-${{ inputs.pr_number }} + cancel-in-progress: false + jobs: release: runs-on: ubuntu-latest diff --git a/.github/workflows/cortex-agent-review.yaml b/.github/workflows/cortex-agent-review.yaml index 56ee9bc4a..71d47bac1 100644 --- a/.github/workflows/cortex-agent-review.yaml +++ b/.github/workflows/cortex-agent-review.yaml @@ -37,6 +37,17 @@ on: CORTEX_AI_AGENTS_CLIENT_PKEY: required: true +# Serialize review runs per PR so two reviews never run at once. A new push +# (synchronize) or review comment queues behind the in-flight review rather +# than cancelling it: cancelling marks the superseded run (and the parent +# "Cortex Agents" run it was called from) as cancelled, which GitHub surfaces +# as a red ✗ check on the PR. Serializing without cancelling avoids that, and +# the queued run reviews the latest state once it starts. Mirrors the +# serialize-don't-cancel policy of the release and update-appversion workflows. +concurrency: + group: cortex-agent-review-${{ github.event.pull_request.number }} + cancel-in-progress: false + jobs: review: runs-on: ubuntu-latest diff --git a/.github/workflows/cortex-agents-entry.yaml b/.github/workflows/cortex-agents-entry.yaml index e652116f2..b651475b0 100644 --- a/.github/workflows/cortex-agents-entry.yaml +++ b/.github/workflows/cortex-agents-entry.yaml @@ -267,7 +267,14 @@ jobs: release-gate: needs: config - if: needs.config.outputs.release_active == 'true' && github.event_name == 'pull_request_target' + # Only react to the PR being opened or reopened, never to synchronize: the + # release agent is a one-shot per PR that itself pushes release-prep commits, + # and reacting to those pushes retriggered a second, redundant release run + # (often after the PR had already merged/closed). + if: >- + needs.config.outputs.release_active == 'true' && + github.event_name == 'pull_request_target' && + (github.event.action == 'opened' || github.event.action == 'reopened') runs-on: ubuntu-latest permissions: contents: read diff --git a/.github/workflows/cortex-agents-hub.yaml b/.github/workflows/cortex-agents-hub.yaml index fc9ca1a7d..ce318519f 100644 --- a/.github/workflows/cortex-agents-hub.yaml +++ b/.github/workflows/cortex-agents-hub.yaml @@ -8,7 +8,7 @@ name: Cortex Agents on: pull_request_target: - types: [opened, synchronize] + types: [opened, synchronize, reopened] pull_request: types: [opened, synchronize, reopened] issue_comment: diff --git a/CHANGELOG.md b/CHANGELOG.md index d11094fef..b5b8bfc50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,71 @@ # Changelog +## Release PR [#1458](https://github.com/cobaltcore-dev/cortex/pull/1458) — 2026-11-02 + +### Bumped Versions + +| Package | Previous | New | +|---|---|---| +| cortex | 0.7.2 | 0.7.3 | +| cortex-postgres | 0.6.29 | 0.6.30 | +| cortex-shim | 0.1.34 | 0.1.35 | +| bundles | 0.0.109 | 0.0.110 | +| cortex-placement-shim | 0.1.34 | 0.1.35 | + +### ⚠️ Deprecation + +- **Slot-based failover reservations deprecated:** The older slot-based failover reservation controller is now deprecated in favor of the new whole-host failover reservation controller (see Features below). The slot-based controller is retained only for migration. New deployments should enable the failover host reservations controller. + +### Features + +- **Failover host controller** ([#1417](https://github.com/cobaltcore-dev/cortex/pull/1417)) — A brand-new failover host controller replaces the slot-based failover system with whole-host reservations. Instead of reserving scattered capacity slots, Cortex now keeps a fraction of entire hosts empty so that when a host dies its full workload can be gang-evacuated onto a single reserved host in one move. Hosts are automatically grouped into **failover partitions** — sets of interchangeable hosts discovered from the scheduler's own placement decisions (per flavor), so partitions always reflect live filters, weights, and constraints without manual configuration. Includes a new `failover_partition_graph` KPI plugin with full test coverage, and extensive documentation (`docs/03-reservations-and-inventory/04-failover-host-reservations.md`). Six new alerts added: `CortexNovaFailoverHostCalculationStalled`, `CortexNovaFailoverHostCalculationAborting`, `CortexNovaFailoverHostReconcileErrors`, `CortexNovaFailoverHostSchedulerErrors`, and supporting metrics (`cortex_failover_host_calculation_runs_total`, `cortex_failover_host_calculation_aborts_total`, `cortex_failover_host_reservations_errors_total`, `cortex_failover_host_scheduler_requests_total`). + +- **`CommittedResourceChange` CRD & reconciler** ([#1432](https://github.com/cobaltcore-dev/cortex/pull/1432), [#1446](https://github.com/cobaltcore-dev/cortex/pull/1446)) — Introduces the `CommittedResourceChange` (CRC) custom resource definition and a reconciler happy path, enabling a transactional architecture for committed-resource lifecycle management. The API handler or Limes syncer creates a CRC; the CRC reconciler drives its FSM, writes `CommittedResource` specs, and rolls back on failure — no controller writes its own spec (spec flows down, status flows up). Includes new CRD types (`committed_resource_change_types.go`), a Helm CRD manifest (`cortex.cloud_committedresourcechanges.yaml`), RBAC cluster-role updates for `committedresourcechanges`, and architecture documentation (`03-b-committed-resource-transaction-architecture.md`). + +- **`index_zz` self-registration pattern** ([#1450](https://github.com/cobaltcore-dev/cortex/pull/1450)) — Replaces hand-maintained plugin registration maps with a self-registering `zz_index.go` pattern across all cortex plugin families (datasources, extractors, KPIs). The old `supported_syncers.go`, `supported_extractors.go`, and `supported_kpis.go` files have been removed. Adding a new plugin no longer requires editing a central index file. + +- **Reservations monitoring moved to KPI layer** ([#1445](https://github.com/cobaltcore-dev/cortex/pull/1445)) — Reservation state monitoring has been lifted out of the quota-enforcement scheduling filter and into the KPI subsystem (`reservation_state.go`), decoupling observability from the scheduling hot path. The old `filter_quota_enforcement_metrics.go` (111 lines) was removed. + +- **Pipeline monitor refactor** — Replaced the singleton pipeline monitor with an event-based reporting mechanism, improving observability and reducing coupling. + +### Bug Fixes + +- **Log metrics now include info level** ([#1433](https://github.com/cobaltcore-dev/cortex/pull/1433)) — Ensures informational log entries are captured in metrics; previously only warn/error levels were counted. + +- **Avoid race conditions between agent workflows** ([#1449](https://github.com/cobaltcore-dev/cortex/pull/1449)) — Serialized the review workflow to prevent concurrent agent runs from colliding. + +### Chores & Refactoring + +- **Enhanced `make` command-line output** ([#1447](https://github.com/cobaltcore-dev/cortex/pull/1447)) — Cleaner, more readable build output. +- **Removed redundant lint phase from `make` target** ([#1451](https://github.com/cobaltcore-dev/cortex/pull/1451)) — Streamlines the build by eliminating a duplicate lint step. +- **Enforce safe context in Tiltfile** ([#1448](https://github.com/cobaltcore-dev/cortex/pull/1448)) — Guards local development against accidentally running against the wrong cluster. Configure allowed contexts via the `TILT_SAFE_CONTEXTS` environment variable. + +### Documentation + +- Updated documentation for make targets, phase runner, tool versions ([#1453](https://github.com/cobaltcore-dev/cortex/pull/1453)), Tiltfile safe-context guard, and the new plugin self-registration pattern. + +### Dependency Updates + +| Dependency | Old | New | Notes | +|---|---|---|---| +| `kube-prometheus-stack` | v92.14.0 | **v92.17.0** | via v92.14.1, v92.16.0, v92.16.1 | +| `golang.org/x/net` | — | **v0.62.0** | 🔒 **Security patch** | +| `debian:trixie-slim` | — | Latest digest | 2 digest updates | +| `github.com/sapcc/go-bits` | — | Latest digest | 2 digest updates | +| External dependencies | — | **v0.37.2** | | + +### ⚠️ Operational Notes + +1. **CRD installation ordering:** The new `CommittedResourceChange` CRD must be installed before the manager is upgraded. Helm handles this via the `crds/` folder, but verify ordering in CI/CD pipelines. +2. **RBAC changes:** The manager's cluster role now includes `committedresourcechanges`. Ensure RBAC manifests are applied before the new controller starts. +3. **Security dependency — prioritize rollout:** `golang.org/x/net v0.62.0` patches a security vulnerability. This release should be prioritized accordingly. +4. **Metrics relocation — update dashboards/alerts:** Reservation state metrics have moved from the scheduling filter (`filter_quota_enforcement_metrics`) to the KPI subsystem (`reservation_state`). Dashboards or alerts referencing the old metric names must be updated. New alert rules have been added in `helm/bundles/cortex-nova/templates/alerts.yaml`. +5. **Failover host alerts:** Six new Prometheus alert rules have been added for the failover host controller. Review thresholds and routing before rollout. + +**Contributors:** @umswmayj, @PhilippMatthes, @SoWieMarkus, @mblos, @renovate[bot], @cortex-ai-agents[bot] + +**PR:** [#1458](https://github.com/cobaltcore-dev/cortex/pull/1458) | **Commits:** 27 | **Files changed:** 100 | **+2,827 / −464** + ## Release PR [#1427](https://github.com/cobaltcore-dev/cortex/pull/1427) — 2026-10-26 ### Bumped Versions diff --git a/Makefile b/Makefile index b6684d28a..b86e7d0f8 100644 --- a/Makefile +++ b/Makefile @@ -5,8 +5,27 @@ else GOBIN=$(shell go env GOBIN) endif +# Phases run by `make all`, in order. Each is a normal target below. +PHASES ?= crds deepcopy lint-fix format test + .PHONY: all -all: crds deepcopy lint-fix format lint test +all: ## Run all phases (crds, deepcopy, lint-fix, format, test), quietly unless one fails. + @rc=0; \ + for phase in $(PHASES); do \ + printf '\033[1;34m▶ %-10s\033[0m ' "$$phase"; \ + log="$$(mktemp)"; \ + if $(MAKE) --no-print-directory "$$phase" >"$$log" 2>&1; then \ + printf '\033[1;32mok\033[0m\n'; \ + else \ + printf '\033[1;31mFAILED\033[0m\n'; \ + cat "$$log"; \ + rc=1; \ + rm -f "$$log"; \ + break; \ + fi; \ + rm -f "$$log"; \ + done; \ + exit $$rc .PHONY: help help: ## Display this help. diff --git a/Tiltfile b/Tiltfile index f72add3e1..0c912021b 100644 --- a/Tiltfile +++ b/Tiltfile @@ -6,6 +6,49 @@ analytics_settings(False) +# Guard against deploying to a shared/prod cluster by accident. Both `tilt up` +# and `tilt down` evaluate this Tiltfile, so failing here aborts either one +# before we touch the cluster (the kubectl calls below would otherwise hit it). +# +# The current kubectl context must be in the safe list, otherwise we bail out. +# The defaults cover the common local clusters; extend the list for your own +# throwaway clusters via TILT_SAFE_CONTEXTS (comma-separated), e.g. +# TILT_SAFE_CONTEXTS="my-dev-cluster,another-sandbox" tilt up +DEFAULT_SAFE_CONTEXTS = [ + 'kind', + 'kind-kind', + 'docker-desktop', + 'minikube', + 'rancher-desktop', + 'orbstack', + 'k3d', + 'colima', +] + +safe_contexts = list(DEFAULT_SAFE_CONTEXTS) +for ctx in os.getenv('TILT_SAFE_CONTEXTS', '').split(','): + ctx = ctx.strip() + if ctx: + safe_contexts.append(ctx) + +current_context = k8s_context() +if current_context not in safe_contexts: + fail( + ('Refusing to run against kubectl context "{}": it is not in the safe ' + + 'list ({}). This guards against accidentally deploying to a shared or ' + + 'production cluster. Switch to a local/throwaway context, or add this ' + + 'one via TILT_SAFE_CONTEXTS="{}".').format( + current_context, + ', '.join(safe_contexts), + current_context, + ) + ) + +# Tell Tilt the current context is allowed (we have already validated it above), +# so its own built-in remote-cluster guard does not fire for e.g. k3d/colima +# contexts added through TILT_SAFE_CONTEXTS. +allow_k8s_contexts(current_context) + # Use the ACTIVE_DEPLOYMENTS env var to select which Cortex bundles to deploy. ACTIVE_DEPLOYMENTS_ENV = os.getenv( 'ACTIVE_DEPLOYMENTS', diff --git a/api/v1alpha1/committed_resource_change_types.go b/api/v1alpha1/committed_resource_change_types.go new file mode 100644 index 000000000..29fb90e13 --- /dev/null +++ b/api/v1alpha1/committed_resource_change_types.go @@ -0,0 +1,164 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ChangePhase represents the FSM phase of a CommittedResourceChange. +type ChangePhase string + +const ( + // ChangePhasePreparing: change request created; sanity checks and optional snapshot in progress. + ChangePhasePreparing ChangePhase = "Preparing" + // ChangePhaseAllocating: CR specs are being written. + ChangePhaseAllocating ChangePhase = "Allocating" + // ChangePhaseAwaitingScheduling: CR specs written; waiting for placement success. + ChangePhaseAwaitingScheduling ChangePhase = "AwaitingScheduling" + // ChangePhaseRollingBack: a CR was rejected or timed out; restoring pre-change state. + ChangePhaseRollingBack ChangePhase = "RollingBack" + // ChangePhaseDryRunCleanup: dry-run would-accept; restoring pre-change state. + ChangePhaseDryRunCleanup ChangePhase = "DryRunCleanup" + // ChangePhaseAccepted: terminal; change request committed successfully. + ChangePhaseAccepted ChangePhase = "Accepted" + // ChangePhaseRejected: terminal; change request rolled back. + ChangePhaseRejected ChangePhase = "Rejected" + // ChangePhaseSuperseded: terminal; abandoned because a newer change request for the same AZ took over. + ChangePhaseSuperseded ChangePhase = "Superseded" +) + +// TxSource identifies the originator of a CommittedResourceChange. +type TxSource string + +const ( + // TxSourceAPI: change request created by the change-commitments HTTP API. + TxSourceAPI TxSource = "api" + // TxSourceSyncer: change request created by the Limes syncer. + TxSourceSyncer TxSource = "syncer" +) + +const ( + // ChangeConditionProgressing is True while the change request is advancing normally. + // False + reason indicates a problem while still non-terminal. + ChangeConditionProgressing = "Progressing" + + // ChangeConditionFinished is True once the change request has reached a terminal phase. + ChangeConditionFinished = "Finished" + + // Reasons for ChangeConditionProgressing=False (non-terminal problems). + ChangeReasonStalled = "Stalled" + ChangeReasonTimedOut = "TimedOut" + ChangeReasonError = "Error" + ChangeReasonRetryExhausted = "RetryExhausted" + + // Reasons for ChangeConditionFinished=True. + ChangeRequestReasonAccepted = "Accepted" + ChangeRequestReasonRejected = "Rejected" + ChangeRequestReasonSuperseded = "Superseded" +) + +// CommittedResourceChangeSpec defines the desired change for a CommittedResourceChange. +type CommittedResourceChangeSpec struct { + // CorrelationID is the request/correlation ID. + // +kubebuilder:validation:Optional + CorrelationID string `json:"correlationID,omitempty"` + + // AvailabilityZone is the availability zone this change request is scoped to. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + AvailabilityZone string `json:"availabilityZone"` + + // Source identifies who created this change request. + // +kubebuilder:validation:Enum=api;syncer + // +kubebuilder:validation:Required + Source TxSource `json:"source"` + + // AllowRejection controls whether the change request may be rejected and rolled back + // on placement failure. When false, the reconciler retries until Accepted or Superseded. + // +kubebuilder:validation:Optional + AllowRejection bool `json:"allowRejection,omitempty"` + + // DryRun controls whether the change request commits or only probes schedulability. + // +kubebuilder:validation:Optional + DryRun bool `json:"dryRun,omitempty"` + + // NewState maps commitmentUUID to the desired CommittedResourceSpec. + // A nil value means the corresponding CommittedResource should be deleted. + // +kubebuilder:validation:Optional + NewState map[string]*CommittedResourceSpec `json:"newState,omitempty"` +} + +// CommittedResourceChangeStatus defines the observed state of a CommittedResourceChange. +type CommittedResourceChangeStatus struct { + // Phase is the current FSM phase of the change request. + // +kubebuilder:validation:Optional + // +kubebuilder:validation:XValidation:rule="oldSelf == '' || !['Accepted','Rejected','Superseded'].exists(t, t == oldSelf) || self == oldSelf",message="phase cannot transition out of a terminal state" + Phase ChangePhase `json:"phase,omitempty"` + + // SnapshotBefore records the CommittedResource spec state before this change request. + // Key is commitmentUUID; nil value means the CR did not exist before (will be deleted on rollback). + // Populated in Preparing when AllowRejection=true. + // +kubebuilder:validation:Optional + SnapshotBefore map[string]*CommittedResourceSpec `json:"snapshotBefore,omitempty"` + + // UpsertedCommitments holds references to CommittedResource CRDs written by this change request. + // Watched in AwaitingScheduling to gate on placement success. + // These are plain object references, not owner references. + // +kubebuilder:validation:Optional + UpsertedCommitments []corev1.ObjectReference `json:"upsertedCommitments,omitempty"` + + // Conditions holds Progressing and Finished conditions. + // +kubebuilder:validation:Optional + Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` + + // Message is a human-readable description of the current state or terminal outcome. + // +kubebuilder:validation:Optional + Message string `json:"message,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster,shortName=crc +// +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase" +// +kubebuilder:printcolumn:name="AZ",type="string",JSONPath=".spec.availabilityZone" +// +kubebuilder:printcolumn:name="Source",type="string",JSONPath=".spec.source" +// +kubebuilder:printcolumn:name="DryRun",type="boolean",JSONPath=".spec.dryRun",priority=1 +// +kubebuilder:printcolumn:name="Progressing",type="string",JSONPath=".status.conditions[?(@.type=='Progressing')].status",priority=1 +// +kubebuilder:printcolumn:name="Finished",type="string",JSONPath=".status.conditions[?(@.type=='Finished')].status",priority=1 + +// CommittedResourceChange is the Schema for the committedresourcechanges API +type CommittedResourceChange struct { + metav1.TypeMeta `json:",inline"` + + // +optional + metav1.ObjectMeta `json:"metadata,omitempty,omitzero"` + + // +required + Spec CommittedResourceChangeSpec `json:"spec"` + + // +optional + Status CommittedResourceChangeStatus `json:"status,omitempty,omitzero"` +} + +// IsTerminal reports whether the change request has reached a terminal phase. +func (tx *CommittedResourceChange) IsTerminal() bool { + return tx.Status.Phase == ChangePhaseAccepted || + tx.Status.Phase == ChangePhaseRejected || + tx.Status.Phase == ChangePhaseSuperseded +} + +// +kubebuilder:object:root=true + +// CommittedResourceChangeList contains a list of CommittedResourceChange +type CommittedResourceChangeList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CommittedResourceChange `json:"items"` +} + +func init() { + SchemeBuilder.Register(&CommittedResourceChange{}, &CommittedResourceChangeList{}) +} diff --git a/api/v1alpha1/reservation_types.go b/api/v1alpha1/reservation_types.go index 2b9e25470..34ee24f10 100644 --- a/api/v1alpha1/reservation_types.go +++ b/api/v1alpha1/reservation_types.go @@ -179,6 +179,17 @@ type FailoverHostReservationSpec struct { // See Spec.TargetHost for the target host of the failover reservation. // +kubebuilder:validation:Optional SourceHosts []string `json:"sourceHosts,omitempty"` + + // PartitionDescription is a short, human-readable label describing the + // failover partition this reservation belongs to: what distinguishes its + // hosts from other partitions (e.g. "spec.aggregates=[hana]") and which + // flavors the whole partition can run (e.g. "flavors: hana.4xl, hana.8xl + // (+3 more)"). It carries no scheduling semantics and exists purely to make a + // reservation understandable at a glance — in particular, which workloads a + // reserved host stands ready to absorb. See describePartition in the + // failoverhost controller. + // +kubebuilder:validation:Optional + PartitionDescription string `json:"partitionDescription,omitempty"` } // ReservationSpec defines the desired state of Reservation. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index c4f4115ff..a60d5a1f4 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -9,9 +9,9 @@ package v1alpha1 import ( apiv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" - corev1 "k8s.io/api/core/v1" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -80,6 +80,139 @@ func (in *CommittedResourceAllocation) DeepCopy() *CommittedResourceAllocation { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CommittedResourceChange) DeepCopyInto(out *CommittedResourceChange) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CommittedResourceChange. +func (in *CommittedResourceChange) DeepCopy() *CommittedResourceChange { + if in == nil { + return nil + } + out := new(CommittedResourceChange) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CommittedResourceChange) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CommittedResourceChangeList) DeepCopyInto(out *CommittedResourceChangeList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CommittedResourceChange, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CommittedResourceChangeList. +func (in *CommittedResourceChangeList) DeepCopy() *CommittedResourceChangeList { + if in == nil { + return nil + } + out := new(CommittedResourceChangeList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CommittedResourceChangeList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CommittedResourceChangeSpec) DeepCopyInto(out *CommittedResourceChangeSpec) { + *out = *in + if in.NewState != nil { + in, out := &in.NewState, &out.NewState + *out = make(map[string]*CommittedResourceSpec, len(*in)) + for key, val := range *in { + var outVal *CommittedResourceSpec + if val == nil { + (*out)[key] = nil + } else { + inVal := (*in)[key] + in, out := &inVal, &outVal + *out = new(CommittedResourceSpec) + (*in).DeepCopyInto(*out) + } + (*out)[key] = outVal + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CommittedResourceChangeSpec. +func (in *CommittedResourceChangeSpec) DeepCopy() *CommittedResourceChangeSpec { + if in == nil { + return nil + } + out := new(CommittedResourceChangeSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CommittedResourceChangeStatus) DeepCopyInto(out *CommittedResourceChangeStatus) { + *out = *in + if in.SnapshotBefore != nil { + in, out := &in.SnapshotBefore, &out.SnapshotBefore + *out = make(map[string]*CommittedResourceSpec, len(*in)) + for key, val := range *in { + var outVal *CommittedResourceSpec + if val == nil { + (*out)[key] = nil + } else { + inVal := (*in)[key] + in, out := &inVal, &outVal + *out = new(CommittedResourceSpec) + (*in).DeepCopyInto(*out) + } + (*out)[key] = outVal + } + } + if in.UpsertedCommitments != nil { + in, out := &in.UpsertedCommitments, &out.UpsertedCommitments + *out = make([]v1.ObjectReference, len(*in)) + copy(*out, *in) + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CommittedResourceChangeStatus. +func (in *CommittedResourceChangeStatus) DeepCopy() *CommittedResourceChangeStatus { + if in == nil { + return nil + } + out := new(CommittedResourceChangeStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CommittedResourceList) DeepCopyInto(out *CommittedResourceList) { *out = *in @@ -223,7 +356,7 @@ func (in *CommittedResourceStatus) DeepCopyInto(out *CommittedResourceStatus) { } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make([]v1.Condition, len(*in)) + *out = make([]metav1.Condition, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -307,7 +440,7 @@ func (in *DatasourceSpec) DeepCopyInto(out *DatasourceSpec) { out.DatabaseSecretRef = in.DatabaseSecretRef if in.SSOSecretRef != nil { in, out := &in.SSOSecretRef, &out.SSOSecretRef - *out = new(corev1.SecretReference) + *out = new(v1.SecretReference) **out = **in } } @@ -334,7 +467,7 @@ func (in *DatasourceStatus) DeepCopyInto(out *DatasourceStatus) { in.NextSyncTime.DeepCopyInto(&out.NextSyncTime) if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make([]v1.Condition, len(*in)) + *out = make([]metav1.Condition, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -484,12 +617,12 @@ func (in *DecisionSpec) DeepCopyInto(out *DecisionSpec) { } if in.MachineRef != nil { in, out := &in.MachineRef, &out.MachineRef - *out = new(corev1.ObjectReference) + *out = new(v1.ObjectReference) **out = **in } if in.PodRef != nil { in, out := &in.PodRef, &out.PodRef - *out = new(corev1.ObjectReference) + *out = new(v1.ObjectReference) **out = **in } } @@ -514,10 +647,10 @@ func (in *DecisionStatus) DeepCopyInto(out *DecisionStatus) { } if in.History != nil { in, out := &in.History, &out.History - *out = new([]corev1.ObjectReference) + *out = new([]v1.ObjectReference) if **in != nil { in, out := *in, *out - *out = make([]corev1.ObjectReference, len(*in)) + *out = make([]v1.ObjectReference, len(*in)) copy(*out, *in) } } @@ -528,7 +661,7 @@ func (in *DecisionStatus) DeepCopyInto(out *DecisionStatus) { } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make([]v1.Condition, len(*in)) + *out = make([]metav1.Condition, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -624,7 +757,7 @@ func (in *DeschedulingStatus) DeepCopyInto(out *DeschedulingStatus) { *out = *in if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make([]v1.Condition, len(*in)) + *out = make([]metav1.Condition, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -914,7 +1047,7 @@ func (in *FlavorGroupCapacityStatus) DeepCopyInto(out *FlavorGroupCapacityStatus in.LastReconcileAt.DeepCopyInto(&out.LastReconcileAt) if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make([]v1.Condition, len(*in)) + *out = make([]metav1.Condition, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -1008,12 +1141,12 @@ func (in *KPIDependenciesSpec) DeepCopyInto(out *KPIDependenciesSpec) { *out = *in if in.Datasources != nil { in, out := &in.Datasources, &out.Datasources - *out = make([]corev1.ObjectReference, len(*in)) + *out = make([]v1.ObjectReference, len(*in)) copy(*out, *in) } if in.Knowledges != nil { in, out := &in.Knowledges, &out.Knowledges - *out = make([]corev1.ObjectReference, len(*in)) + *out = make([]v1.ObjectReference, len(*in)) copy(*out, *in) } } @@ -1082,7 +1215,7 @@ func (in *KPIStatus) DeepCopyInto(out *KPIStatus) { *out = *in if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make([]v1.Condition, len(*in)) + *out = make([]metav1.Condition, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -1131,12 +1264,12 @@ func (in *KnowledgeDependenciesSpec) DeepCopyInto(out *KnowledgeDependenciesSpec *out = *in if in.Datasources != nil { in, out := &in.Datasources, &out.Datasources - *out = make([]corev1.ObjectReference, len(*in)) + *out = make([]v1.ObjectReference, len(*in)) copy(*out, *in) } if in.Knowledges != nil { in, out := &in.Knowledges, &out.Knowledges - *out = make([]corev1.ObjectReference, len(*in)) + *out = make([]v1.ObjectReference, len(*in)) copy(*out, *in) } } @@ -1225,7 +1358,7 @@ func (in *KnowledgeStatus) DeepCopyInto(out *KnowledgeStatus) { in.Raw.DeepCopyInto(&out.Raw) if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make([]v1.Condition, len(*in)) + *out = make([]metav1.Condition, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -1491,7 +1624,7 @@ func (in *PipelineStatus) DeepCopyInto(out *PipelineStatus) { *out = *in if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make([]v1.Condition, len(*in)) + *out = make([]metav1.Condition, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -1631,7 +1764,7 @@ func (in *ProjectQuotaStatus) DeepCopyInto(out *ProjectQuotaStatus) { } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make([]v1.Condition, len(*in)) + *out = make([]metav1.Condition, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -1781,7 +1914,7 @@ func (in *ReservationStatus) DeepCopyInto(out *ReservationStatus) { *out = *in if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make([]v1.Condition, len(*in)) + *out = make([]metav1.Condition, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 9dcd8f85d..9ba24d770 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -39,6 +39,13 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources" "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack" + // Register all supported OpenStack syncers through their func init(). + _ "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/cinder" + _ "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/identity" + _ "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/limes" + _ "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/manila" + _ "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/nova" + _ "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/placement" "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/prometheus" "github.com/cobaltcore-dev/cortex/internal/knowledge/db" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor" @@ -52,13 +59,13 @@ import ( "github.com/cobaltcore-dev/cortex/internal/scheduling/manila" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/crs" - novafilters "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/plugins/filters" "github.com/cobaltcore-dev/cortex/internal/scheduling/pods" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/capacity" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/commitments" commitmentsapi "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/commitments/api" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/failover" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/failoverhost" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/inflight" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/quota" "github.com/cobaltcore-dev/cortex/pkg/cache" @@ -397,12 +404,6 @@ func main() { detectorPipelineMonitor := schedulinglib.NewDetectorPipelineMonitor() metrics.Registry.MustRegister(&detectorPipelineMonitor) - // Filter-specific metrics that don't fit the generic per-step monitor (e.g. - // custom labels). Register them globally so they're available wherever the - // filter runs. - novafilters.QuotaEnforcementMetricsSingleton = novafilters.NewQuotaEnforcementMetrics() - metrics.Registry.MustRegister(novafilters.QuotaEnforcementMetricsSingleton) - // Initialize commitments API for LIQUID interface (Postgres-backed usage reporting). commitmentsConfig := conf.GetConfigOrDie[commitments.Config]() var commitmentsVMSource reservations.VMSource @@ -602,8 +603,6 @@ func main() { if slices.Contains(mainConfig.EnabledControllers, "committed-resource-reservations-controller") { setupLog.Info("enabling controller", "controller", "committed-resource-reservations-controller") - monitor := reservations.NewMonitor(multiclusterClient) - metrics.Registry.MustRegister(&monitor) if err := (&commitments.CommitmentReservationController{ Client: multiclusterClient, @@ -791,6 +790,30 @@ func main() { "maxVMsToProcess", failoverConfig.MaxVMsToProcess, "vmSelectionRotationInterval", failoverConfig.VMSelectionRotationInterval) } + if slices.Contains(mainConfig.EnabledControllers, "failover-host-reservations-controller") { + setupLog.Info("enabling controller", "controller", "failover-host-reservations-controller") + failoverHostConfig := conf.GetConfigOrDie[failoverhost.Config]().Failover.Hosts + failoverHostMonitor := failoverhost.NewMonitor() + if err := metrics.Registry.Register(failoverHostMonitor); err != nil { + setupLog.Error(err, "failed to register failover host monitor metrics, continuing without metrics") + failoverHostMonitor = nil + } + controller := &failoverhost.Controller{ + Client: multiclusterClient, + Config: failoverHostConfig, + Monitor: failoverHostMonitor, + } + if err := controller.SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", + "controller", "failover-host-reservations-controller") + os.Exit(1) + } + setupLog.Info("failover-host-reservations-controller registered", + "ratio", failoverHostConfig.Ratio, + "minimum", failoverHostConfig.Minimum, + "generationInterval", failoverHostConfig.GenerationInterval, + "externalSchedulerURL", failoverHostConfig.ExternalSchedulerURL) + } if slices.Contains(mainConfig.EnabledControllers, "capacity-controller") { setupLog.Info("enabling controller", "controller", "capacity-controller") capacityConfig := conf.GetConfigOrDie[capacity.Config]() diff --git a/docs/01-getting-started/02-architecture-at-a-glance.md b/docs/01-getting-started/02-architecture-at-a-glance.md index a2fc77c1b..7979199f3 100644 --- a/docs/01-getting-started/02-architecture-at-a-glance.md +++ b/docs/01-getting-started/02-architecture-at-a-glance.md @@ -146,14 +146,16 @@ controllers; a `cortex-cinder` bundle enables the Cinder pipeline controller ins ### The plugin model -Pipeline steps, knowledge extractors, and datasources are all *named* — a resource references a step -by name, and the binary resolves the name to code. There are three registration styles: - -- **Scheduling plugins** (filters, weighers, detectors) **self-register** via `init()` into - package-level `Index` maps of factory functions. Importing the package registers the step. -- **Knowledge extractors and KPIs** are registered in **hand-maintained static maps** — a step is - available only if it has an entry. -- **Datasources** are dispatched by a **typed switch / map** keyed on the datasource kind. +Pipeline steps, knowledge extractors, KPIs, and datasources are all *named* — a resource references a +plugin by name, and the binary resolves the name to code. All categories use the same registration +pattern: each plugin file has a `func init()` that registers a factory function into a package-level +`Index` map declared in a `zz_index.go` file. Importing the package is enough — no central list to +edit. The `Index` maps live in: + +- **Scheduling plugins** — `internal/scheduling//plugins/filters/`, `.../weighers/`, `.../detectors/` +- **Knowledge extractors** — `internal/knowledge/extractor/plugins/zz_index.go` +- **KPIs** — `internal/knowledge/kpis/plugins/zz_index.go` +- **Datasources** — e.g. `internal/knowledge/datasources/plugins/openstack/zz_index.go` A per-domain admission webhook validates a `Pipeline` when it is applied: it **rejects** a step whose parameters are invalid or a step of the wrong kind for the pipeline type, but an **unknown step name is diff --git a/docs/01-getting-started/05-make-targets.md b/docs/01-getting-started/05-make-targets.md index ed44cd1ef..599f6704f 100644 --- a/docs/01-getting-started/05-make-targets.md +++ b/docs/01-getting-started/05-make-targets.md @@ -14,25 +14,29 @@ committed. ## The default target -Running `make` with no argument runs the full local loop: +Running `make` with no argument runs a phase runner that iterates over `PHASES` in order: ```make -all: crds deepcopy lint-fix format lint test +PHASES ?= crds deepcopy lint-fix format test ``` -In order, that regenerates CRDs and deepcopy code, applies lint autofixes, formats, lints, and runs -the tests. It is the "did I break anything" command — run it before opening a pull request. +For each phase the runner prints a concise `▶ phase ok` or `▶ phase FAILED` status line, capturing +the full output per phase and only showing it when a phase fails. It stops on the first failure. The +separate `lint` target is not in the default list because `lint-fix` already runs `golangci-lint` +with `--fix`, which reports and fixes in one pass. + +This is the "did I break anything" command — run it before opening a pull request. ## The targets, one by one | Target | What it does | |---|---| -| `all` | The full loop above. The default. | +| `all` | Phase runner that executes `PHASES` in order, printing concise status per phase and stopping on the first failure. | | `generate` | `deepcopy` + `crds` — regenerate everything derived from the Go types. | | `crds` | Regenerate CRD manifests from the `api/v1alpha1` types (via `controller-gen`). | | `deepcopy` | Regenerate the `zz_generated.deepcopy.go` files (via `controller-gen`). | | `format` | `gofmt -w .` — rewrite Go files in place. | -| `lint` | `golangci-lint run` — report issues without changing files. | +| `lint` | `golangci-lint run` — report issues without changing files. Not part of the default `make all` run. | | `lint-fix` | `golangci-lint run --fix` — apply the autofixable subset. | | `test` | `go test ./...`. | | `testsum` | The tests via `gotestsum` for a nicer, grouped summary. | @@ -45,8 +49,8 @@ same on your machine and in CI: | Tool | Pinned version | |---|---| -| `controller-gen` | v0.22.0 | -| `golangci-lint` | v2.13.2 | +| `controller-gen` | v0.23.0 | +| `golangci-lint` | v2.15.0 | | `gotestsum` | v1.13.0 | `./bin` is git-ignored; deleting it just triggers a re-install on the next `make`. diff --git a/docs/01-getting-started/08-local-development-with-tilt.md b/docs/01-getting-started/08-local-development-with-tilt.md index 944d64b41..295279987 100644 --- a/docs/01-getting-started/08-local-development-with-tilt.md +++ b/docs/01-getting-started/08-local-development-with-tilt.md @@ -66,7 +66,30 @@ loop, narrow it with `ACTIVE_DEPLOYMENTS`: export ACTIVE_DEPLOYMENTS=nova ``` -## Step 4 — Start Tilt +## Step 4 — Safe-context guard + +Before deploying, the Tiltfile validates the active `kubectl` context against a list of known-local +context names. If the context is not on the list, `tilt up` (and `tilt down`) will refuse to proceed, +protecting shared or production clusters from accidental deployments. + +The built-in safe list is: + +`kind`, `kind-kind`, `docker-desktop`, `minikube`, `rancher-desktop`, `orbstack`, `k3d`, `colima` + +Because the cluster created above has the context `kind-cortex-dev`, you must add it via the +`TILT_SAFE_CONTEXTS` environment variable (comma-separated): + +```bash +export TILT_SAFE_CONTEXTS=kind-cortex-dev +``` + +You can add more than one context at a time: + +```bash +export TILT_SAFE_CONTEXTS="kind-cortex-dev,my-sandbox" +``` + +## Step 5 — Start Tilt ```bash tilt up @@ -89,7 +112,7 @@ NAME READY STATUS RESTAR cortex-nova-scheduling-controller-manager-... 1/1 Running 0 1m ``` -## Step 5 — Make a change and watch it reload +## Step 6 — Make a change and watch it reload Edit any Go source under the manager's packages — for example, add a log line to a reconciler — and save. Tilt detects the change, rebuilds the image, and redeploys automatically. In the Tilt UI the @@ -101,7 +124,7 @@ kubectl logs deploy/cortex-nova-scheduling-controller-manager --follow This is the core inner loop: edit, save, and let Tilt reconcile the running deployment. -## Step 6 — Clean up +## Step 7 — Clean up Stop Tilt with `Ctrl-C`, then remove the cluster: diff --git a/docs/02-external-scheduler-api/01-the-scheduling-engine.md b/docs/02-external-scheduler-api/01-the-scheduling-engine.md index 4c1f37f2b..e7d21fb54 100644 --- a/docs/02-external-scheduler-api/01-the-scheduling-engine.md +++ b/docs/02-external-scheduler-api/01-the-scheduling-engine.md @@ -134,7 +134,7 @@ warning and then ignored** at run time, so a pipeline can safely reference a ste add without breaking the rollout. What the webhook *does* reject is a known step whose parameters are invalid, or a step of the wrong kind for the pipeline `type` (a detector in a filter-weigher pipeline, or a filter/weigher in a detector pipeline), or an unknown `type`. The next section shows both outcomes in -practice. (Knowledge extractors and KPIs use hand-maintained maps instead — see +practice. (Knowledge extractors, KPIs, and datasources use the same `init()` self-registration pattern — see [Architecture at a glance](../01-getting-started/02-architecture-at-a-glance.md).) ## Inspecting and editing a pipeline diff --git a/docs/02-external-scheduler-api/02-nova-smart-scheduling.md b/docs/02-external-scheduler-api/02-nova-smart-scheduling.md index b1540706f..3d7a84958 100644 --- a/docs/02-external-scheduler-api/02-nova-smart-scheduling.md +++ b/docs/02-external-scheduler-api/02-nova-smart-scheduling.md @@ -122,7 +122,7 @@ pay-as-you-go placement and is not slotted. This is how a committed reservation > confirmed location — and several Nova requests can be in flight at once. How Cortex reserves for its > candidates before the next request scores, and how it reconciles against what Nova actually did, is the > subject of -> [Concurrency and in-flight reservations](../03-reservations-and-inventory/04-concurrency-and-in-flight-reservations.md). +> [Concurrency and in-flight reservations](../03-reservations-and-inventory/05-concurrency-and-in-flight-reservations.md). ## Closing the loop: descheduling diff --git a/docs/02-external-scheduler-api/07-extending-cortex.md b/docs/02-external-scheduler-api/07-extending-cortex.md index c9e84c558..36bafb81e 100644 --- a/docs/02-external-scheduler-api/07-extending-cortex.md +++ b/docs/02-external-scheduler-api/07-extending-cortex.md @@ -19,14 +19,15 @@ implementation in code, then reference it by name from a custom resource. The mo ## Choose the right registration style -Cortex uses three registration mechanisms depending on the plugin category: +All plugin categories use the same `init()` self-registration pattern, differing only in which +package-level `Index` map they register into: | Plugin | Registration | Referenced by | |---|---|---| | Filter / weigher / detector | Self-register via `init()` into a package-level `Index` map of factory functions (e.g. `internal/scheduling/nova/plugins/filters/`) | `Pipeline` CRD step name | -| Knowledge extractor | Entry in the hand-maintained `supportedExtractors` map (`internal/knowledge/extractor/supported_extractors.go`) | `Knowledge` CRD | -| KPI | Entry in the hand-maintained `supportedKPIs` map (`internal/knowledge/kpis/supported_kpis.go`) | `KPI` CRD | -| Datasource | Typed switch / map on datasource kind | `Datasource` CRD | +| Knowledge extractor | Self-register via `init()` into the `Index` map in `internal/knowledge/extractor/plugins/zz_index.go` | `Knowledge` CRD | +| KPI | Self-register via `init()` into the `Index` map in `internal/knowledge/kpis/plugins/zz_index.go` | `KPI` CRD | +| Datasource | Self-register via `init()` into the `Index` map in the datasource plugin package (e.g. `internal/knowledge/datasources/plugins/openstack/zz_index.go`) | `Datasource` CRD | ## Add a scheduling step (filter / weigher / detector) @@ -66,9 +67,17 @@ result, is the signal your step was picked up. See ## Add a knowledge extractor or KPI -1. Implement the extractor/KPI. -2. Add its name to the hand-maintained static map (unlike scheduling steps, these are **not** - self-registering — a missing map entry means the name is unavailable). +1. Implement the extractor/KPI in the appropriate plugin subdirectory. +2. Add a `func init()` at the bottom of the file that registers a factory function into the + package-level `Index` map (declared in `zz_index.go`) — for example: + + ```go + func init() { + plugins.Index["my_extractor"] = func() plugins.FeatureExtractor { return &MyExtractor{} } + } + ``` + + Because registration happens in `init()`, importing the package is enough — no central list to edit. 3. Reference it from a `Knowledge` or `KPI` resource. Verify with the feature/KPI metrics: @@ -80,11 +89,12 @@ cortex_kpi_state{name="my-kpi"} 1 ## Add a datasource kind -Datasources are dispatched by a typed switch/map on kind. Add the new kind to that dispatch and -implement its ingestion, then reference it from a `Datasource` resource. The datasource types are the -`DatasourceType` / `OpenStackDatasourceType` constants in -`api/v1alpha1/datasource_types.go`, and the dispatch is -`internal/knowledge/datasources/plugins/openstack/supported_syncers.go`. +Datasources self-register via `init()` into the `Index` map in their plugin package. Add the new kind +by implementing its ingestion and registering a factory function in `init()`, then reference it from a +`Datasource` resource. The datasource types are the `DatasourceType` / `OpenStackDatasourceType` +constants in `api/v1alpha1/datasource_types.go`, and the `Index` maps are in +`internal/knowledge/datasources/plugins/openstack/zz_index.go` and +`internal/knowledge/datasources/plugins/prometheus/zz_index.go`. ## Build and test locally diff --git a/docs/03-reservations-and-inventory/01-reservations-overview.md b/docs/03-reservations-and-inventory/01-reservations-overview.md index d79376f9b..93b3b13ad 100644 --- a/docs/03-reservations-and-inventory/01-reservations-overview.md +++ b/docs/03-reservations-and-inventory/01-reservations-overview.md @@ -46,7 +46,9 @@ are checked arithmetically against headroom. This is the subject of These reserve **headroom for evacuation**: if a host fails, its workloads need somewhere to land. Cortex pre-clears space to absorb up to `m` simultaneous host failures, shared across the protected workloads rather than duplicated per workload. This is the subject of -[Failover reservations for KVM HA](03-failover-reservations.md). +[Failover reservations for KVM HA](03-failover-reservations.md) — now **deprecated** in favour of the +whole-host approach in [Failover host reservations](04-failover-host-reservations.md), which keeps entire +hosts empty as gang-evacuation targets. ### In-flight reservations @@ -59,7 +61,7 @@ in-flight reservation blocks that capacity from "decided" until "running", then > The reasoning here is pessimistic by necessity: because concurrent requests race, Cortex assumes a slot > it just recommended is already taken until the placement is realized or abandoned. In-flight > reservations are the persistence layer behind the scheduler's concurrency handling — -> [Concurrency and in-flight reservations](04-concurrency-and-in-flight-reservations.md) explains the +> [Concurrency and in-flight reservations](05-concurrency-and-in-flight-reservations.md) explains the > races they defend against and how the scheduler serializes around them. ## Inspecting reservations diff --git a/docs/03-reservations-and-inventory/03-b-committed-resource-transaction-architecture.md b/docs/03-reservations-and-inventory/03-b-committed-resource-transaction-architecture.md new file mode 100644 index 000000000..c35c5a6ae --- /dev/null +++ b/docs/03-reservations-and-inventory/03-b-committed-resource-transaction-architecture.md @@ -0,0 +1,125 @@ + + +# Committed-resource transaction architecture (target) + +## Components + +| Component | Kind | Owned by | +|-----------|------|----------| +| `CommittedResourceChange` (CRC) | CRD | Spec: created by API handler or syncer; Status: CRC reconciler | +| `CommittedResource` (CR) | CRD | Spec: CRC reconciler (sole writer); Status: CR controller | +| `Reservation` | CRD | Spec: CR controller (sole writer); Status: Reservation controller | +| Change-commitments API | HTTP handler | Creates CRC with `source=api` | +| Limes syncer | Background poller | Creates CRC with `source=syncer` on each diff | +| CRC reconciler | Controller | Drives CRC FSM; writes CR specs; rolls back on failure | +| CR controller | Controller | Derives Reservation slots from CR spec; reports CR readiness | +| Reservation controller | Controller | Attempts host placement; reports slot readiness | + +## Ownership + +```mermaid +flowchart TD + API["API handler\nsource=api"] -->|creates| CRC["CommittedResourceChange"] + Syncer["Limes syncer\nsource=syncer"] -->|creates on diff| CRC + + CRC -->|"CRC reconciler — writes spec"| CR["CommittedResource"] + CR -->|"CR controller — writes spec"| Res["Reservation"] + Res -->|"Reservation controller — tracks placement"| VM["VM / scheduler\nlive-migration"] + + CR -.->|"CR controller reports readiness\nCRC reconciler gates on this"| CRC +``` + +No controller writes its own spec. Spec flows down; status flows up. + +## Component interaction + +The three reconcilers operate independently and communicate only through CRD status conditions. + +```mermaid +sequenceDiagram + participant W as API handler or Syncer + participant CRC as CRC reconciler + participant CRCD as CommittedResource CRD + participant CRCtl as CR controller + participant ResCRD as Reservation CRD + participant ResCtl as Reservation controller + + W->>CRCD: create CommittedResourceChange + CRC->>CRCD: write CR.spec (Allocating phase) + Note over CRC,CRCD: CRC reconciler is sole writer of CR.spec + CRCtl->>ResCRD: reconcile Reservation slots from CR.spec + Note over CRCtl,ResCRD: CR controller is sole writer of Reservation.spec + ResCtl->>ResCRD: attempt placement, write Reservation.status + Note over ResCtl: reacts to VM moves, live-migration, in-flight actions + CRCtl->>CRCD: aggregate slot readiness, write CR.status + CRC->>CRCD: read CR.status.Ready + observedGeneration, advance to Accepted or RollingBack +``` + +## CRC state machine + +```mermaid +stateDiagram-v2 + [*] --> Preparing : CRC created by API handler or syncer + + Preparing --> Allocating : snapshot taken + + Allocating --> AwaitingScheduling : API mode — CR specs written + Allocating --> Accepted : Syncer mode — apply-and-done + Allocating --> Superseded : newer CRC for same AZ took over + + AwaitingScheduling --> Accepted : all CRs Ready (observedGeneration matches) + AwaitingScheduling --> DryRunCleanup : all CRs Ready, but DryRun=true + AwaitingScheduling --> RollingBack : CR rejected or timeout + + DryRunCleanup --> Accepted : pre-state restored (would-accept) + + RollingBack --> Rejected : pre-state restored (RollbackFailed on permanent failure) + + Accepted --> [*] + Rejected --> [*] + Superseded --> [*] +``` + +Terminal phases (`Accepted`, `Rejected`, `Superseded`) set `Finished=True`. Non-terminal problems set +`Progressing=False` with a reason (`Stalled`, `TimedOut`, `Error`, `RetryExhausted`) without advancing +the phase. A stuck non-terminal CRC triggers an alert; after a threshold it is forcibly moved to +`Superseded` so its AZ is unblocked and the syncer repairs state on the next poll. + +## Request flows + +### API-mode flow + +1. **API handler** receives a change-commitments request from Limes; creates a CRC (`source=api`, `allowRejection=true`, optional `dryRun=true`). +2. **CRC reconciler — Preparing**: validates the request; snapshots current CR specs as the rollback journal. +3. **CRC reconciler — Allocating**: writes the desired CR specs atomically; records which CRs were touched. +4. **CRC reconciler — AwaitingScheduling**: watches the touched CRs; waits until each reports its ready condition against the generation just written. +5. **CR controller**: for each CR, derives and reconciles Reservation slots; reports `Ready` once all slots are placed. +6. **Reservation controller**: places each slot on a host; reports slot readiness. +7. **CRC reconciler**: all CRs ready → `Accepted`; any CR rejected or timeout → rolls all CR specs back to the snapshot → `Rejected`. +8. **API handler** polls the CRC by name; maps terminal phase to HTTP response. + +### Syncer-mode flow + +1. **Limes syncer** polls Limes per project; diffs desired state against current CRs; on any delta creates a CRC (`source=syncer`, `allowRejection=false`). +2. **CRC reconciler — Preparing**: no snapshot (non-rejectable; syncer repairs on next poll if anything drifts). +3. **CRC reconciler — Allocating**: recency check — skips changes for any project another CRC touched in the same AZ in the last ~10 minutes (prevents clobbering an in-flight API change); writes CR specs; advances directly to `Accepted`. +4. **CR controller**: reconciles Reservation slots async; CRC is already terminal. +5. Slots converge in the background; the next syncer poll repairs any remaining drift. + +## Key invariants + +- **Single writer per spec layer.** The CRC reconciler is the only component that writes `CommittedResource.spec`. The CR controller is the only component that writes `Reservation.spec`. No controller writes its own spec. +- **AwaitingScheduling gates on generation.** The CRC reconciler advances from `AwaitingScheduling` only when a CR's ready condition carries an `observedGeneration` that matches the generation the CRC reconciler wrote — not an older reconcile cycle's result. +- **Rollback is durable.** The pre-change snapshot is stored in `CRC.status` before any CR spec is touched. A process restart resumes from the persisted phase and snapshot. +- **Syncer-mode never rolls back.** Syncer CRTs are non-rejectable: on any error they retry until `Accepted` or are superseded by a newer CRC. The syncer repairs drift on its next poll. +- **At most one active CRC per AZ.** The CRC reconciler serialises with `MaxConcurrentReconciles=1` plus an in-memory per-AZ active-transaction map (rebuilt from CRDs on startup). Younger CRTs requeue until the active one reaches a terminal phase. +- **Syncer recency guard prevents clobbering.** A syncer CRC skips changes to any project that an API CRC touched in the same AZ within the last ~10 minutes — the API change is authoritative; the syncer will reconcile remaining drift on its next poll. +- **Terminal CRTs are retained for auditability.** `Accepted` CRTs are kept for ~1 hour; `Rejected` CRTs for ~5 days. Non-terminal CRCs (including stuck `RollingBack`) are never GC'd — deleting one loses the rollback journal and the AZ active-marker. + +--- + +[Prev: CR lifecycle internals](03-a-cr-lifecycle-internals.md) · [Next: Failover reservations for KVM HA »](03-failover-reservations.md) diff --git a/docs/03-reservations-and-inventory/03-failover-reservations.md b/docs/03-reservations-and-inventory/03-failover-reservations.md index d9e4a6bb1..a2d232d49 100644 --- a/docs/03-reservations-and-inventory/03-failover-reservations.md +++ b/docs/03-reservations-and-inventory/03-failover-reservations.md @@ -6,6 +6,16 @@ # Failover reservations for KVM HA +> [!WARNING] +> **Deprecated.** The slot-based failover reservation system described on this page (the +> `failover-reservations-controller`, reserving a shared `FailoverReservation` pool) is being **replaced by the +> failover *host* system** — see [Failover host reservations](04-failover-host-reservations.md). Rather than +> reserving scattered headroom slots, the new system keeps whole hosts empty so a failed host's entire workload +> gang-evacuates onto one reserved host. The `FailoverReservation` type is marked deprecated in +> `api/v1alpha1/reservation_types.go` and will be removed; new deployments should enable the +> `failover-host-reservations-controller` instead. This page is retained to document the existing system during +> migration. + Where committed resources reserve space a customer paid for, **failover reservations** reserve space *nobody is using yet* — headroom kept clear so that when a host fails, the workloads on it have somewhere to be evacuated. This page explains the two design choices that make failover headroom affordable: it is @@ -94,4 +104,4 @@ same slot. ## Next -[Prev: Committed-resource reservations](02-committed-resource-reservations.md) · [Next: Concurrency and in-flight reservations »](04-concurrency-and-in-flight-reservations.md) +[Prev: Committed-resource reservations](02-committed-resource-reservations.md) · [Next: Failover host reservations »](04-failover-host-reservations.md) diff --git a/docs/03-reservations-and-inventory/04-failover-host-reservations.md b/docs/03-reservations-and-inventory/04-failover-host-reservations.md new file mode 100644 index 000000000..757ab26db --- /dev/null +++ b/docs/03-reservations-and-inventory/04-failover-host-reservations.md @@ -0,0 +1,528 @@ + + +# Failover host reservations + +The previous page reserved **headroom** — a shared pool of free capacity spread across the fleet, sized to +absorb `m` host failures. This page describes a different, coarser-grained mechanism that is replacing it: +instead of reserving scattered slots, Cortex keeps a fraction of **whole hosts** empty, so that when a host +dies its *entire* workload can be gang-evacuated onto one reserved host in a single move (*gang-evacuated*: +all the host's VMs moved together, in one placement decision, rather than one at a time). + +> [!IMPORTANT] +> Failover host reservations **supersede** the slot-based failover reservations described on the +> [previous page](03-failover-reservations.md). The older system is deprecated and will be removed. New +> deployments should enable the failover-host reservations controller; the old slot-based controller is +> retained only for migration. The key reason for the switch: whole-host reservations trade a little more +> idle capacity for a pre-validated, gang-evacuable landing zone, so no placement has to be re-solved under +> pressure the moment a host dies. + +## Partitions: hosts are not interchangeable + +Hosts cannot fail over onto *any* other host. A HANA host, a general-purpose host, and an external-customer +host each carry scheduling constraints — flavors, image properties, aggregates, availability zone — that only +matching hosts satisfy. A failover target is only useful if the dead host's VMs would actually be *accepted* +there. + +Cortex therefore groups hosts into **failover partitions**: sets of hosts within which every host can be +evacuated onto every other host. Crucially, these groups are **not configured by hand**. They are *discovered* +from the scheduler's own placement decisions, so the grouping always reflects the live filters, weights and +constraints — a partition can never contain a target the scheduler would in fact reject. + +One could imagine solving this differently: hand-craft the partitions from the hypervisors directly — write +rules that read each host's flavors, image properties, aggregates and availability zone and group the +"compatible" ones together. But those are exactly the constraints the VM scheduling pipeline *already* +encodes. Re-expressing them here would mean re-implementing a second, parallel copy of the placement logic, +which then has to be kept in lock-step with the real one forever; the moment a filter changes in the pipeline +and not in the failover rules, the partitions quietly diverge from what the scheduler would actually accept, +and a "failover target" may reject the very workload it was reserved for. + +So Cortex takes a **self-partitioning approach** instead. Rather than restating the rules, it asks the real +scheduler — per **flavor** — which hosts would accept that flavor, and lets the partitions *fall out* of those +answers (the next section works through how). The compatibility rules are never duplicated: whatever the +pipeline does, the partitions inherit, automatically and for free, including every future change to the +filters and weighers. + +The probes are seeded from the **flavor catalogue**, not from the VMs that happen to be running. Cortex takes +every flavor in the `flavor-groups` Knowledge CRD whose hypervisor type it supports — the +`capabilities:hypervisor_type` extra-spec is `qemu` (the Nova label for KVM) or `ch` (Cloud Hypervisor); +flavors with any other, or no, hypervisor type +are skipped — crosses each with every availability zone in the fleet, and asks the scheduler which hosts would +accept it. These `(flavor, availability zone)` permutations are the **seeds** of the partitioning. +Seeding from flavors rather than running VMs closes two blind spots the VM-based approach had: + +- **A group of hosts running no VMs is still discovered.** Flavor seeds exist whether or not anything is + currently scheduled, so a partition is found up front rather than only once a VM happens to land on one of its + hosts. +- **One "exotic" VM can no longer veto a host.** Because seeds carry no host affiliation, a single VM whose + scheduling constraints no peer satisfies can never keep its host out of the partition it matches under every + other flavor. + +Everything on this page is a continuous background process: Cortex periodically re-derives the partitions and +the set of reserved hosts from the current state of the fleet, and nudges the actual reservations toward that +freshly computed target. Nothing here is a one-time setup step; the picture re-settles every few minutes as +the fleet changes. + +## How a reservation is kept up to date + +The mechanism has two cooperating halves: + +- A **planner** runs on a fixed interval (default every `5m`). It looks at the current fleet, works out which + hosts *should* be reserved as failover targets, and publishes that as the desired picture. This is the + expensive part — it talks to the scheduler — so it runs on a slow cadence, never on every event. +- A **reconciler** continuously nudges reality toward that desired picture: it creates reservations that + should exist, removes ones that no longer should, and updates ones that have drifted. It is cheap, and it + does nothing clever — it just closes the gap. + +```mermaid +flowchart LR + PLAN["Planner (periodic)
decides which hosts to reserve"] -->|"publishes desired picture"| STORE[("desired picture")] + STORE -.->|"read"| RECON["Reconciler (continuous)
makes reality match"] + RECON -->|"create / update / remove"| REAL[("actual reservations")] +``` + +Because the planner republishes the **whole** desired picture each pass (rather than editing it in place), the +reconciler gets pruning for free: any reservation that exists in the fleet but is absent from the latest +picture is simply one the planner no longer wants, and is removed. + +A corollary matters for safety: the planner only ever publishes a picture it computed from a *complete* +snapshot. If anything goes wrong while gathering the inputs — the fleet snapshot can't be collected, or a +scheduler probe fails — the whole pass is aborted and the previously published picture is left untouched; the +planner simply retries on the next cycle. The reconciler therefore never acts on a half-gathered view, and +reservations are never recomputed from partial data. + +```mermaid +stateDiagram-v2 + direction LR + [*] --> compare + compare --> create: wanted, missing + compare --> remove: present, no longer wanted + compare --> update: present but drifted + compare --> noop: already matches + create --> [*] + remove --> [*] + update --> [*] + noop --> [*] +``` + +## How the planner decides: building the failover graph + +The planner's job — "which hosts should be reserved?" — comes down to building a graph of who can rescue whom, +grouping that graph into partitions, and then reserving a fraction of each partition. + +The important thing to understand is that the graph is **narrowed in several steps**, and each step throws out +candidates for a *different* reason. Keeping those reasons straight is the whole algorithm, because the single +most common question — *"why does this host have no failover target?"* — has a different answer at each step: + +1. **Discover who can rescue whom** (Step 1 below). For every flavor seed, ask the scheduler which hosts would + accept it, then pair up the hosts that co-accept a seed and keep the pair only if the target is big enough + for the whole source host — yielding one-way rescue links `source → target`. *A target drops out if it is + smaller than the source.* +2. **Group into partitions** (Step 2). Keep only links that work *both* ways, then cluster those mutual pairs + into groups where everyone can rescue everyone. *A one-way link leaves no trace in the partitions.* +3. **Make the partitions disjoint** (Step 3). A host can land in two overlapping groups; assign it to one. + *A group left with a single host is then discarded.* +4. **Pick the failover hosts** (Step 4). Reserve a fraction of each partition as empty landing zones. + +Each emphasised clause is an edge case, worked through below. A +[troubleshooting table](#why-does-this-host-have-no-failover-target) at the end maps each symptom back to the +step that caused it. + +### Why Bron-Kerbosch for the partitioning + +Step 2 reduces to a classic graph question: in the undirected graph of mutual rescue pairs, a failover +partition is a set of hosts that are *all pairwise* connected — a **maximal clique**. Cortex enumerates those +cliques with the **Bron-Kerbosch** algorithm, and the choice is deliberate: + +- **The problem is literally maximal-clique enumeration, not something weaker.** "Every host can evacuate onto + every other host" is the definition of a clique. Cheaper alternatives solve a *different* problem and would + silently admit invalid partitions: connected components or a union-find grouping would lump together hosts + linked only transitively (`a`↔`b`, `b`↔`c` but not `a`↔`c`), producing a "partition" in which `a` cannot + actually rescue `c`. A greedy "grow a group while everyone stays connected" heuristic avoids that but picks + an arbitrary clique and misses overlapping ones, so some hosts would lose a partition they genuinely belong + to. +- **We need *all maximal* cliques, including overlapping ones.** A host can legitimately belong to more than + one partition (it is Step 3's own job to resolve that overlap afterwards). Bron-Kerbosch is the standard, + well-understood algorithm that enumerates *exactly* the maximal cliques — no duplicates, nothing missed — + which is what makes the downstream overlap-resolution well-defined. +- **The fleet is partitioned by construction, so the worst case never bites.** Maximal-clique enumeration is + exponential in the general case, which sounds alarming. But the mutual-rescue graph is not a general graph: + it is already almost-disconnected into compatibility islands (HANA, general-purpose, external-customer, …), + and cliques only form *within* an island. Each clique-search therefore runs over a small, dense subgraph, + not the whole fleet, which keeps it comfortably fast in practice. + +With that settled, the four steps in detail: + +### Step 1 — Discover who can rescue whom + +Cortex finds, for every host, the hosts that host's *entire* workload could be gang-evacuated onto. The result +is a set of directed **rescue links**: + +> A link `source → target` means *"`source`'s whole workload would be accepted on `target`, and `target` is big +> enough to hold it at once."* + +Direction matters: `source → target` does **not** imply `target → source`. The target might simply be too +small to take the source's workload even when the reverse would fit. + +Rather than reimplementing placement logic, Cortex **asks the real scheduler** — once per **flavor seed** — +which hosts would accept that flavor in that availability zone. This guarantees the rescue links obey exactly +the same rules as ordinary placement. The hosts that *co-accept* a seed are mutually compatible under that +flavor, so Cortex pairs them up: for every ordered pair `(a, b)` of hosts that both accept a seed it emits the +link `a → b`, subject to the whole-host fit check below. + +```mermaid +flowchart TB + SEED["seed: flavor F in az1"] --> SCHED["scheduler accepts: {A, B, C}"] + SCHED --> PAIR["pair up co-accepting hosts:
A↔B, A↔C, B↔C"] + PAIR --> FIT{"for each pair (a, b):
is b big enough for all of a?"} + FIT -->|yes| EDGE["rescue link a → b"] + FIT -->|no| DROP["discard that direction"] +``` + +Because the seeds come from the flavor catalogue rather than the running VMs, two failure modes of the old +VM-based discovery simply disappear: + +- **A group of hosts running no VMs is still discovered.** A flavor seed exists regardless of what is currently + scheduled, so such a group produces rescue links and forms its partition up front — no VM has to land on it + first. +- **One "exotic" VM can no longer veto a host.** Discovery no longer intersects the answers for a *host's own* + VMs, so a single VM whose constraints no peer satisfies can never empty that intersection and keep its host + out of the partition it matches under every other flavor. + +Three things keep this probing faithful and safe: + +- **Each probe mirrors a real evacuation** — it carries the flavor, its availability zone and a KVM image + hypervisor type, so exactly the same filters apply as in a genuine move, and tells the scheduler to treat it + as an **evacuation** (a cold migration). The choice of *evacuation* over *live migration* is deliberate. A + failover ultimately *evacuates* a dead host's workload — a cold migration, which is the **weaker** check: it + does not verify the CPU architecture, CPU modes, hypervisor features and emulated devices that live migration + does. A failover target is still reserved **empty**, so before its reservation goes active Cortex must drain + whatever runs on it; that drain uses live migration. But rather than prove per host that every peer is + live-migratable at partition time, Cortex assumes a partition holds enough peers to absorb that drain and + leaves arranging it to the draining code. Probing with the evacuation (cold-migration) requirement is what + lets a partition form even across hosts that are not pairwise live-migratable. +- **Candidates are judged as if empty.** The probe tells the scheduler to measure each candidate against its + *total* capacity, ignoring whatever VMs currently run there. This is deliberate: a failover target is reserved + **empty**, so the right question is "could this host hold the workload once cleared?", not "does it have room + right now?". Were the probe to measure live free capacity instead, a busy host would accept no one, gain no + inbound rescue link, form no mutual edge, and stay orphaned — even though reserving it would drain it. Whether + two hosts are *compatible* is thus independent of how full they happen to be; which host is cheapest to + actually empty is decided later, in Step 4. +- **The probing never changes anything.** It is strictly read-only, so asking "where could this flavor go?" for + every seed never disturbs live scheduling. It also tells the scheduler to *ignore* existing failover host + reservations — otherwise a host already reserved for failover would look occupied and vanish from its own + partition, fragmenting the grouping. + +A hard **whole-host fit** check discards any target that is smaller than its source (memory first, then CPU). +An evacuation moves *all* of the source's workload onto the target at once, so the target must be at least as +large as the source. This check compares *total* capacities, independent of the empty-host probing above, so an +undersized target is dropped whether or not it currently runs anything. Equal-sized co-accepting hosts +therefore produce links in *both* directions, which Step 2 turns into a partition; an undersized host only ever +appears as a source, never as a landing-zone target for a larger peer. + +Two outcomes of a seed lookup must not be confused, because they are handled oppositely — and the distinction +is exactly what upholds the "consistent picture or nothing" rule: + +- **The scheduler accepts the seed on fewer than two hosts** (one host, or none — a short candidate set, no + error). This is a valid answer, not a failure. A seed accepted on fewer than two hosts can form no pair, so it + simply contributes no link, and the pass continues normally. +- **The lookup itself fails** (the scheduler is unreachable, times out, or returns a non-200) — or the seed set + itself cannot be gathered, because the `flavor-groups` Knowledge CRD is missing or not yet ready. This is an + infrastructure failure, not an answer — on + a partial view Cortex cannot assert anything about the failover graph, so it **aborts the entire pass** and + leaves the last published picture untouched, retrying next cycle. Swallowing the error and skipping just that + seed would collapse rescue links and prune reservations that are in fact still valid, so a single transient + blip must never be allowed to erase coverage. + +### Step 2 — Group hosts into partitions + +A **failover partition is a set of hosts in which every host can be evacuated onto every other host.** For that +to hold, rescue must work in *both* directions between any two members: `a` can rescue `b` **and** `b` can +rescue `a`. So Step 2 first keeps only the **mutual** links — the one-way links from Step 1 are discarded here. + +**One-way links are dropped.** This is why a link can exist in Step 1 yet leave no trace in the partitions: + +The reason mutuality is required — not just a convenient graph property — is that a failover must be +*reversible*. A VM evacuated onto a target host should not be **locked in** there once the original host +recovers. If rescue only worked one way (`source → target` but not back), the evacuated workload could land on +the target and then have nowhere to return to, stranding it. Insisting the link work in *both* directions +guarantees the partition can be restored: after the evacuation, the VMs can fail back off the target just as +they failed onto it, so the partition returns to a fully failover-able state rather than degrading with each +event. + +```mermaid +flowchart LR + subgraph dir["rescue links from Step 1"] + direction LR + a1((a)) --> b1((b)) + b1 --> a1 + b1 --> c1((c)) + c1 -. "no c→b
(c too small for b)" .-> b1 + end + dir ==> mut + subgraph mut["only mutual pairs survive"] + direction LR + a2((a)) --- b2((b)) + b2 -. "dropped:
works one way only" .- c2((c)) + end + style c2 fill:#ffebee,stroke:#c62828 +``` + +Within the mutual graph, a partition is a group where **everyone can rescue everyone**. Finding the largest +such groups is a classic graph problem (finding *cliques*), and that is exactly what Cortex does — no hand +tuning, the groups fall straight out of the mutual links. + +**Why groups, and why they can overlap.** Consider three hosts where `a`↔`b` and `b`↔`c` are mutual but `a` +and `c` cannot rescue each other. Then `{a, b, c}` is *not* a valid partition — `a` and `c` don't fit the "all +rescue all" rule. Instead there are two valid groups, `{a, b}` and `{b, c}`, and `b` sits in both. That overlap +is expected here; Step 3 resolves it. + +**Hosts that accept no seed get folded in.** Because discovery now probes flavor seeds rather than running VMs, +a host with no VMs on it is probed exactly like any other and, as long as it *accepts some seed*, forms rescue +links and joins a clique directly — no special handling needed. The one case left over is a host that accepts +**no seed at all** (for example an unschedulable host that every filter rejects): it has no rescue links of its +own and would otherwise be a group of one. Such a host is **recruited** into a partition as long as **every** +member of that partition can rescue onto it *and* it is the same effective size as those members. The +reachability check is one-directional on purpose: the host would be reserved empty, so it is assumed able to +evacuate anywhere; only the inbound direction (`member → host`) is verified. But the host also becomes a +*source* of the partition's eventual failover target, so it must itself be rescuable by its new peers. Having no +rescue links to prove this, Cortex falls back to the same whole-host fit rule used in Step 1 and requires equal +capacity (memory then CPU, rounded to GiB): a host larger than the partition could never be evacuated onto a +smaller reserved target, so it is **not** recruited. If even one member can't reach it, or the sizes differ, the +host stays on its own: + +Flavor seeding also removes a blind spot the old VM-based discovery had: **a partition no longer needs a running +VM to be discovered.** Rescue links used to be derived *from* running VMs, so a group of hosts with no VMs at +all produced no links, formed no clique, and stayed invisible — a HANA partition with no HANA VM running simply +did not exist until one landed. With flavor seeds the HANA flavor is probed whether or not anything runs, so the +partition is found up front and a failover host can be reserved before the first VM ever arrives. + +```mermaid +flowchart TB + subgraph ok["host E (accepts no seed) is recruited"] + direction LR + a3((a)) -->|reaches| e1((E)) + b3((b)) -->|reaches| e1 + okr["every member of {a, b} reaches E
⇒ partition becomes {a, b, E}"] + end + subgraph no["host F stays alone"] + direction LR + a4((a)) -->|reaches| f1((F)) + b4((b)) -. "b does NOT reach F" .-> f1 + nor["not every member reaches F
⇒ F is left on its own"] + end + style nor fill:#ffebee,stroke:#c62828 +``` + +The whole grouping is computed deterministically: given the same fleet, the same partitions come out every +time, so reservations don't flap between passes for no reason. + +### Step 3 — Make the partitions disjoint + +Step 2 can leave a host in **two** partitions at once (like `b` above). But a host can only be reserved for one +partition — reserving it for two would produce conflicting records about who it's meant to rescue, and one would +silently overwrite the other. So before planning, Cortex forces the partitions apart: each shared host is kept +in exactly **one** partition — the **smallest** one it belongs to (ties broken deterministically by the +partition's first host name). + +Keeping the shared host in the *smallest* partition is what preserves coverage for as many partitions as +possible. A small partition only survives if it keeps its shared hosts — losing one can drop it below the +two-host minimum and collapse it entirely. A large partition, by contrast, usually stays well above that +minimum even after giving a host up. So assigning the shared host to the smaller group lets *both* groups +survive, whereas assigning it to the larger one would often strand the smaller group: + +```mermaid +flowchart TB + subgraph before["overlapping groups from Step 2"] + p1["{a, b, c, d} (4 hosts)"] + p2["{c, e} (2 hosts)"] + end + before ==>|"shared host c stays with the SMALLER group"| after + subgraph after["disjoint partitions — both survive"] + q1["{a, b, d} ✓ shrank by one, still ≥ 2"] + q2["{c, e} ✓ keeps c, stays a valid pair"] + end + style q1 fill:#e8f5e9,stroke:#2e7d32 + style q2 fill:#e8f5e9,stroke:#2e7d32 +``` + +Had `c` instead been kept with the larger `{a, b, c, d}`, the pair `{c, e}` would have collapsed to the lone +host `{e}` and been discarded (a lone host has no peer to fail over from), leaving `e` unprotected this pass +for no reason other than how the overlap was resolved. Preferring the smaller partition removes that failure +mode wherever the larger partition can spare the host — which, being larger, it usually can. + +**Edge case — coverage can still be lost when neither partition can spare the host.** The smallest-first rule +does not make the problem vanish entirely: if a host is shared between two partitions that are *both* size two, +whichever one loses it collapses to a lone host and is discarded. There is no assignment that saves both — the +host is the only peer each has — so one partition is unavoidably stranded, and the deterministic tiebreak +decides which. This is now the only way the disjoint step costs a host its failover target. + +### Step 4 — Pick the failover hosts in each partition + +For each partition, Cortex decides how many hosts to reserve as failover targets. It deliberately does **not** +pick a fixed number: too few leaves no room to evacuate into, too many wastes capacity. The count scales with +the partition size, under three rules: + +- **A ratio of the partition** (default `10%`), rounded **down**. +- **A floor** (default `1`), so even a small partition keeps at least one target. +- **Never all of them** — at least one host must be left to actually run workload, so a partition is never + fully reserved. And a partition of fewer than two hosts reserves nothing at all: there would be no peer for a + reserved host to rescue. + +Worked through for the defaults, the interplay is easier to see than to state: + +| partition size | 10 %, rounded down | after the floor of 1 | after "leave ≥ 1 running" | **reserved** | +|---:|---:|---:|---:|---:| +| 1 | — | — | — | **0** (no peer) | +| 2 | 0 | 1 | 1 | **1** | +| 10 | 1 | 1 | 9 | **1** | +| 15 | 1 | 1 | 14 | **1** | +| 25 | 2 | 2 | 24 | **2** | + +The floor dominates small partitions; the ratio only starts to matter once a partition is large enough that +10 % of it exceeds 1 (here, around 20 hosts). + +**Which** hosts become the targets comes down to the *order* in which the partition's hosts are considered. +Cortex builds one ordered slice of candidates and then simply takes the first `reserved`-many from it. The +order is what encodes the policy — and it is built to **retain existing reservations first and only create new +ones when the count demands it**: + +``` +candidate order ──────────────────────────────────────────────────────────────▶ +┌───────────────────────────────┬───────────────────────────────────────────────┐ +│ already-reserved hosts │ not-yet-reserved hosts │ +│ (this partition), by name │ by ascending workload (cheapest to empty) │ +├───────────────────────────────┼───────────────────────────────────────────────┤ +│ R1 R2 R3 … │ N1 N2 N3 N4 … │ +└───────────────────────────────┴───────────────────────────────────────────────┘ + └──────────── take the first `reserved` of these ────────────┘ + kept reservations are consumed before any new one is created +``` + +Reading the slice left to right: + +- **Already-reserved hosts come first, so they stay reserved.** A host being prepared as a failover target may + have VMs mid-way through being moved off it; flipping the designation each pass would throw that work away. + Because they sit at the front, the planner only reaches a *new* host once every existing reservation has + already been re-chosen. If there are *more* already-reserved hosts than the count needs, the surplus falls + past the cut-off and is released back into normal service; if there are fewer, the remainder is filled from + the new hosts. +- **New targets, when needed, are the cheapest to empty.** The not-yet-reserved hosts are ordered by ascending + workload, so the first new host the planner reaches is the one whose VMs are cheapest to migrate away. + +So in the common steady state where the right number of hosts is already reserved, the cut-off lands exactly +at the boundary between the two groups: every existing reservation is retained and no new one is created. New +reservations only appear when the count grows or an existing target leaves the partition. + +Each chosen host is then reserved in full, recording the other partition members as the hosts it stands ready +to rescue. + +Each reservation also carries a human-readable **partition description** (stored in the reservation's +`partitionDescription` field) made of two parts: what distinguishes the partition's hosts from other partitions +(e.g. `spec.aggregates=[hana]`), and **which flavors the partition can run** — the flavors every host in the +partition was seen to accept, so a listed flavor is guaranteed to fit anywhere in it. A long flavor list is +capped (e.g. `flavors: hana.4xl, hana.8xl (+3 more)`) to keep the field compact; the full set is logged each +pass. The description carries no scheduling semantics — it exists purely so an operator can tell, at a glance, +what a reserved host is standing by to absorb. + +**A reserved host is listed as a source of its sibling reservations — and that is intentional.** When a +partition reserves more than one failover host, each reservation records *every* other partition member as a +source host, including the partition's *other* reserved hosts. At first glance this looks wrong: why list a +host that is itself being kept empty as a source to rescue? The reason is what happens after a failover +completes. Once a reserved host has absorbed an evacuation and is converted back into a normal host, it +instantly needs a failover target of its own — and because it was already recorded as a source on its sibling +reservations, that target is already in place. No existing reservation has to be rewritten to account for the +newly-returned host; the planner only needs to create *additional* reservations to rebuild the failover +headroom the partition just spent. Leaving the reserved hosts in each other's source lists is therefore a +deliberate shortcut: it keeps the post-failover reservations consistent without an extra adjustment pass, so +recovery reduces to "create what's now missing" rather than "re-derive who rescues whom". + +## The reservation lifecycle + +A failover host reservation blocks its target host from receiving ordinary placements, so the host stays empty +and available as a landing zone. Over its life a reservation moves through these stages: + +```mermaid +stateDiagram-v2 + [*] --> Designated: host chosen as a failover target + Designated --> Active: all VMs moved away, host now empty + Designated --> Failed: draining fails or takes too long + Failed --> Designated: cooldown elapses, eligible again + Active --> InUse: a gang evacuation lands here + InUse --> Used: evacuation complete → host returns to service + Active --> Revoked: no longer part of any partition + Designated --> Revoked: no longer part of any partition + Used --> [*] + Revoked --> [*] +``` + +- **Designated** — the host has been chosen as a target but may still have VMs that need draining off it. No + *new* VMs are placed here. It stays in this stage until the host is empty. +- **Active** — the host is empty and ready to absorb an evacuation. +- **Failed** — draining failed repeatedly or took too long; the host is held back from being re-chosen for a + cooldown period. +- **In Use** — a gang evacuation onto this host is underway (and may absorb further failures until it + completes). +- **Revoked** — the host no longer belongs to any partition (for example it became unschedulable), so the + reservation is dropped. +- **Used** — the failover event finished; the host becomes a regular host again. + +> [!NOTE] +> Treat this diagram as the *intended progression*, not a strict state machine. The stages are tracked +> independently rather than as a single current-state field, so the arrows show the normal path rather than the +> only possible transitions. + +## Configuration + +Enable the controller and configure it under the nested `failover.hosts` key: + +```yaml +enabledControllers: + - failover-host-reservations-controller +# ... +failover: + hosts: + ratio: 0.1 # fraction of each partition to reserve (0, 1] + minimum: 1 # floor per partition, > 0 + generationInterval: 5m # how often to recompute, ≥ 1m + externalSchedulerURL: "http://localhost:8080/scheduler/nova/external" +``` + +The flavors the partitioning is seeded from (see [Step 1](#step-1--discover-who-can-rescue-whom)) are read +from the `flavor-groups` Knowledge CRD, which Cortex already extracts into the cluster — so the controller +needs no OpenStack credentials of its own to enumerate them. + +> [!NOTE] +> The settings live under `failover.hosts` as a deliberate example of grouping a controller's config under its +> own key rather than adding more flat top-level keys. This is independent of the (deprecated) slot-based +> failover controller configured alongside it. Bad values for `ratio`, `minimum`, or the interval are rejected +> at startup. + +Like the other reservation kinds, failover host reservations make a host look fuller to the pipeline than its +raw capacity suggests — here by reserving the host **entirely**. The next page turns from the *kinds* of +reserved capacity to the concurrency problem all of them share: how the scheduler keeps parallel requests from +double-spending the same slot. + +## Why does this host have no failover target? + +Because the graph is narrowed in several steps, a host can end up unprotected for quite different reasons. This +table maps the symptom back to the step that caused it: + +| Symptom | Why | Step | +|---|---|---| +| a host accepts no seed, gets no rescue links | every filter rejects it (e.g. it is unschedulable) | 1 · Discover rescue links | +| a target was discarded | it's smaller than the source host | 1 · Discover rescue links | +| a link doesn't lead to a partition | it only works one way, not both | 2 · Group into partitions | +| a host that accepts no seed is left on its own | not *every* member of a partition can reach it | 2 · Group into partitions | +| a host loses its partition | it was shared between two size-two partitions, and the one that lost it collapsed | 3 · Make partitions disjoint | +| a partition reserves nothing | it has fewer than two hosts | 4 · Pick failover hosts | + +The controller logs each of these decisions as it makes them, so a host's fate through the steps can be traced +pass by pass. + +## Next + +[Prev: Failover reservations (deprecated)](03-failover-reservations.md) · [Next: Concurrency and in-flight reservations »](05-concurrency-and-in-flight-reservations.md) diff --git a/docs/03-reservations-and-inventory/04-concurrency-and-in-flight-reservations.md b/docs/03-reservations-and-inventory/05-concurrency-and-in-flight-reservations.md similarity index 98% rename from docs/03-reservations-and-inventory/04-concurrency-and-in-flight-reservations.md rename to docs/03-reservations-and-inventory/05-concurrency-and-in-flight-reservations.md index f71147d49..9f54deeec 100644 --- a/docs/03-reservations-and-inventory/04-concurrency-and-in-flight-reservations.md +++ b/docs/03-reservations-and-inventory/05-concurrency-and-in-flight-reservations.md @@ -133,4 +133,4 @@ Placement API shim. ## Next -[Prev: Failover reservations](03-failover-reservations.md) · [Next: The Placement API shim »](05-placement-api-shim.md) +[Prev: Failover host reservations](04-failover-host-reservations.md) · [Next: The Placement API shim »](06-placement-api-shim.md) diff --git a/docs/03-reservations-and-inventory/05-placement-api-shim.md b/docs/03-reservations-and-inventory/06-placement-api-shim.md similarity index 99% rename from docs/03-reservations-and-inventory/05-placement-api-shim.md rename to docs/03-reservations-and-inventory/06-placement-api-shim.md index e8bde2417..2fd8d38ff 100644 --- a/docs/03-reservations-and-inventory/05-placement-api-shim.md +++ b/docs/03-reservations-and-inventory/06-placement-api-shim.md @@ -195,4 +195,4 @@ facts: the knowledge database. ## Next -[Prev: Concurrency and in-flight reservations](04-concurrency-and-in-flight-reservations.md) · [Next: Chapter 4 — The knowledge database »](../04-knowledge-database/readme.md) +[Prev: Concurrency and in-flight reservations](05-concurrency-and-in-flight-reservations.md) · [Next: Chapter 4 — The knowledge database »](../04-knowledge-database/readme.md) diff --git a/docs/03-reservations-and-inventory/readme.md b/docs/03-reservations-and-inventory/readme.md index 87724bb24..b1069b479 100644 --- a/docs/03-reservations-and-inventory/readme.md +++ b/docs/03-reservations-and-inventory/readme.md @@ -13,8 +13,9 @@ Cortex intends to become the authority for inventory itself. The chapter opens with an overview of the three reservation kinds, then goes deep on committed resources (customer commitments synced from Limes, plus the quota and capacity controllers that -support them) and failover reservations (headroom for evacuating workloads when a host fails). It -then covers the concurrency problem all reservations share — keeping parallel scheduling requests from +support them) and failover reservations (headroom for evacuating workloads when a host fails) — both +the deprecated slot-based system and the whole-host system that replaces it. It then covers the +concurrency problem all reservations share — keeping parallel scheduling requests from double-spending the same slot — and closes with the **Placement API shim**, the OpenStack-Placement-compatible front through which Cortex will eventually serve inventory for the hosts it understands. @@ -23,9 +24,10 @@ it understands. 1. [Reservations overview](01-reservations-overview.md) — committed, failover, and in-flight reservations. 2. [Committed-resource reservations and Limes quota/capacity](02-committed-resource-reservations.md) — commitments, `FlavorGroupCapacity`, `ProjectQuota`, and the LIQUID API. -3. [Failover reservations for KVM HA](03-failover-reservations.md) — shared, best-effort evacuation headroom. -4. [Concurrency and in-flight reservations](04-concurrency-and-in-flight-reservations.md) — how parallel requests avoid double-spending capacity. -5. [The Placement API shim](05-placement-api-shim.md) — passthrough today, KVM-backed inventory planned. +3. [Failover reservations for KVM HA](03-failover-reservations.md) — shared, best-effort evacuation headroom (**deprecated**). +4. [Failover host reservations](04-failover-host-reservations.md) — whole-host evacuation targets grouped into discovered failover partitions. +5. [Concurrency and in-flight reservations](05-concurrency-and-in-flight-reservations.md) — how parallel requests avoid double-spending capacity. +6. [The Placement API shim](06-placement-api-shim.md) — passthrough today, KVM-backed inventory planned. ## Next diff --git a/docs/04-knowledge-database/03-feature-extraction.md b/docs/04-knowledge-database/03-feature-extraction.md index b8ce805db..e9e6c960f 100644 --- a/docs/04-knowledge-database/03-feature-extraction.md +++ b/docs/04-knowledge-database/03-feature-extraction.md @@ -20,8 +20,8 @@ Postgres database and the `Knowledge` spec, and `Extract() ([]Feature, error)` r compute extractors pair a `.go` file with a `.sql` file — the base helper runs the SQL query against the datasource database and boxes the rows into typed features. -Extractors are dispatched by a hand-maintained map, `supportedExtractors`, in -`internal/knowledge/extractor/supported_extractors.go` — the same static-registration pattern the KPIs use. +Extractors self-register via `func init()` into the package-level `Index` map declared in +`internal/knowledge/extractor/plugins/zz_index.go` — the same self-registration pattern the KPIs use. There are two plugin families: - **compute** (`plugins/compute/`) — the bulk of them: `host_utilization_extractor` (joins a hypervisor's @@ -32,8 +32,9 @@ There are two plugin families: - **storage** (`plugins/storage/`) — currently `netapp_storage_pool_cpu_usage_extractor`, the feature behind Manila's one weigher. -To add an extractor, implement the interface and register it in `supportedExtractors` under a new name — the -process in [Extend Cortex](../02-external-scheduler-api/07-extending-cortex.md). +To add an extractor, implement the interface and add a `func init()` that registers a factory function +into the `Index` map under a new name — the process in +[Extend Cortex](../02-external-scheduler-api/07-extending-cortex.md). ## Declaring inputs and where features live diff --git a/docs/04-knowledge-database/04-kpis-and-metrics-api.md b/docs/04-knowledge-database/04-kpis-and-metrics-api.md index d60a86d72..4b01a84d8 100644 --- a/docs/04-knowledge-database/04-kpis-and-metrics-api.md +++ b/docs/04-knowledge-database/04-kpis-and-metrics-api.md @@ -25,8 +25,9 @@ it when the resource goes away. > endpoint** — the same endpoint that carries the sync and reconcile metrics. Scrape that, and the KPI > gauges are alongside everything else. -Like extractors, KPIs are dispatched by a hand-maintained map — `supportedKPIs` in -`internal/knowledge/kpis/supported_kpis.go`. Registering a new KPI means adding it there; see +Like extractors, KPIs self-register via `func init()` into the package-level `Index` map declared in +`internal/knowledge/kpis/plugins/zz_index.go`. Registering a new KPI means adding an `init()` that +inserts a factory function into that map; see [Extend Cortex](../02-external-scheduler-api/07-extending-cortex.md). ## Four plugin families diff --git a/docs/04-knowledge-database/05-infrastructure-dashboard.md b/docs/04-knowledge-database/05-infrastructure-dashboard.md index 8cef52c1d..582a34d21 100644 --- a/docs/04-knowledge-database/05-infrastructure-dashboard.md +++ b/docs/04-knowledge-database/05-infrastructure-dashboard.md @@ -38,7 +38,7 @@ The dashboard definitions are development/operations tooling, deliberately kept `cortex-compute-kvm-details.json` (the compute KVM boards), `cortex-compute-vmware-overview.json` and `cortex-compute-vmware-details.json` (the compute VMware boards), `cortex-global-compute-capacity.json`, `cortex-placement-shim-status.json` (the - [shim](../03-reservations-and-inventory/05-placement-api-shim.md) status board), + [shim](../03-reservations-and-inventory/06-placement-api-shim.md) status board), `cortex-project-flavor-usage.json`, `cortex-status-global.json`, `cortex-status.json`, and `project.json`. diff --git a/docs/06-cortex-library/09-supervisor.md b/docs/06-cortex-library/09-supervisor.md index 55d9654e3..09af07255 100644 --- a/docs/06-cortex-library/09-supervisor.md +++ b/docs/06-cortex-library/09-supervisor.md @@ -6,7 +6,7 @@ # The `supervisor` package -The [Placement API shim](../03-reservations-and-inventory/05-placement-api-shim.md) sits on a hot path: +The [Placement API shim](../03-reservations-and-inventory/06-placement-api-shim.md) sits on a hot path: schedulers hit it constantly, and it forwards most of that traffic to upstream OpenStack Placement. That request path must stay up *even when the shim's Kubernetes controller-manager cannot* — for instance when it loses connectivity to a local or remote apiserver. `pkg/shim/supervisor` is what decouples the two, so @@ -56,7 +56,7 @@ So a lost apiserver connection becomes a *degraded* state (the manager loops, re > by the pod going unready. The package is deliberately shim-agnostic so future shims can reuse it; today the -[Placement API shim](../03-reservations-and-inventory/05-placement-api-shim.md) is its user, enabled by +[Placement API shim](../03-reservations-and-inventory/06-placement-api-shim.md) is its user, enabled by `--self-heal` (on by default). It is the resilience half of the shim's design: the shim page explains *what* the shim serves, this package explains *why a control-plane hiccup does not stop it serving*. That closes the tour of `pkg/` — and the book. From here, the code and its generated CRD/godoc reference are diff --git a/docs/readme.md b/docs/readme.md index 510e91707..b2d959ed0 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -55,8 +55,8 @@ and `cmd/shim`, and the Helm defaults under `helm/`. 1. [Reservations overview](03-reservations-and-inventory/01-reservations-overview.md) 2. [Committed-resource reservations and Limes quota/capacity](03-reservations-and-inventory/02-committed-resource-reservations.md) 3. [Failover reservations for KVM HA](03-reservations-and-inventory/03-failover-reservations.md) -4. [Concurrency and in-flight reservations](03-reservations-and-inventory/04-concurrency-and-in-flight-reservations.md) -5. [The Placement API shim](03-reservations-and-inventory/05-placement-api-shim.md) +4. [Concurrency and in-flight reservations](03-reservations-and-inventory/05-concurrency-and-in-flight-reservations.md) +5. [The Placement API shim](03-reservations-and-inventory/06-placement-api-shim.md) ### [4 — The knowledge database](04-knowledge-database/readme.md) @@ -91,7 +91,7 @@ and `cmd/shim`, and the Helm defaults under `helm/`. chapters for your domain, and keep [Monitoring Cortex](04-knowledge-database/06-monitoring.md) nearby. - **Integrators** wire Cortex into an existing scheduler — start with [The scheduling engine](02-external-scheduler-api/01-the-scheduling-engine.md) and the - [Placement API shim](03-reservations-and-inventory/05-placement-api-shim.md). + [Placement API shim](03-reservations-and-inventory/06-placement-api-shim.md). - **Developers** extend Cortex — read Chapter 1, then [The Cortex library](06-cortex-library/readme.md) and [Extend Cortex](02-external-scheduler-api/07-extending-cortex.md). diff --git a/go.mod b/go.mod index e8643f3be..1430500a3 100644 --- a/go.mod +++ b/go.mod @@ -16,15 +16,15 @@ require ( github.com/prometheus/client_golang v1.25.0 github.com/prometheus/client_model v0.6.3 github.com/sapcc/go-api-declarations v1.25.1 - github.com/sapcc/go-bits v0.0.0-20261023092235-568ad9d4c1f2 + github.com/sapcc/go-bits v0.0.0-20261029170309-6e36605dd3a1 go.uber.org/zap v1.28.0 go.xyrillian.de/gg v1.19.0 golang.org/x/sync v0.24.0 golang.org/x/term v0.47.0 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.37.1 - k8s.io/apimachinery v0.37.1 - k8s.io/client-go v0.37.1 + k8s.io/api v0.37.2 + k8s.io/apimachinery v0.37.2 + k8s.io/client-go v0.37.2 sigs.k8s.io/controller-runtime v0.25.2 ) @@ -72,6 +72,7 @@ require ( github.com/gorilla/mux v1.8.1 // indirect github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect + github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -83,10 +84,11 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/opencontainers/runc v1.3.6 // indirect + github.com/pandatix/go-abnf v0.5.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/poy/onpar v0.3.5 // indirect - github.com/prometheus/common v0.72.1 // indirect + github.com/prometheus/common v0.73.0 // indirect github.com/prometheus/procfs v0.22.0 // indirect github.com/sirupsen/logrus v1.9.4 // indirect github.com/spf13/cobra v1.10.2 // indirect @@ -107,7 +109,7 @@ require ( go.yaml.in/yaml/v3 v3.0.5 // indirect go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect - golang.org/x/net v0.61.0 // indirect + golang.org/x/net v0.62.0 // indirect golang.org/x/oauth2 v0.37.0 // indirect golang.org/x/sys v0.49.0 // indirect golang.org/x/text v0.43.0 // indirect @@ -125,7 +127,7 @@ require ( k8s.io/component-base v0.37.0 // indirect k8s.io/klog/v2 v2.140.0 // indirect k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad // indirect - k8s.io/streaming v0.37.1 // indirect + k8s.io/streaming v0.37.2 // indirect k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.36.0 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect diff --git a/go.sum b/go.sum index ec102d933..863f1172e 100644 --- a/go.sum +++ b/go.sum @@ -121,6 +121,8 @@ github.com/gotestyourself/gotestyourself v2.2.0+incompatible h1:AQwinXlbQR2HvPjQ github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -167,6 +169,8 @@ github.com/opencontainers/runc v1.3.6 h1:SLGIymCtsk80iNPWgbc8dtjI30r+5mTVV+4dN8/ github.com/opencontainers/runc v1.3.6/go.mod h1:o1wyv76EDlTkcf0KTFgN8bMWLPvgF/HfX709lDv+rr4= github.com/ory/dockertest v3.3.5+incompatible h1:iLLK6SQwIhcbrG783Dghaaa3WPzGc+4Emza6EbVUUGA= github.com/ory/dockertest v3.3.5+incompatible/go.mod h1:1vX4m9wsvi00u5bseYwXaSnhNrne+V0E6LAcBILJdPs= +github.com/pandatix/go-abnf v0.5.1 h1:iyLvSu2Aq12iE0r4n948Qw21BHDuckLq/8WIXAAF+HI= +github.com/pandatix/go-abnf v0.5.1/go.mod h1:VBmbtpFgQNEaQpyAvmi+Ecd9wAYJKzA3TmiB3L93sVg= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -178,8 +182,8 @@ github.com/prometheus/client_golang v1.25.0 h1:cDtW1TcFsMi1RJc8khwCKB4VhzwV04aKq github.com/prometheus/client_golang v1.25.0/go.mod h1:j2X/4/kaQHLPnmyEU8zjmkndo1T317VpCIQEml8LTKk= github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo= github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM= -github.com/prometheus/common v0.72.1 h1:ZAiQed/nk8IIcsdzb2TlJu23NEUDyDLQ9TXTN2WRK6Y= -github.com/prometheus/common v0.72.1/go.mod h1:1IoBY4IRaJnhs2ESjNzjilP1cJ/P8pimGoomq7zRXrA= +github.com/prometheus/common v0.73.0 h1:NGIOmVi8jrgiiPQgXxD6rP0FfxFsuZxnMQkel0qBy/g= +github.com/prometheus/common v0.73.0/go.mod h1:Nt+CsVkf/mj4wDAmtSwq1HTPohHSX6fMr6+/f9B5UqM= github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0oics= github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= @@ -187,8 +191,8 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sapcc/go-api-declarations v1.25.1 h1:Lrbf8TZY/ARg0J9IrlrObqnXjrzSzmYKQd/L/a71Lsg= github.com/sapcc/go-api-declarations v1.25.1/go.mod h1:bJkK8wphzlmUmN5lxBv8BbfxUXMOkW+UiC10xBHXsFs= -github.com/sapcc/go-bits v0.0.0-20261023092235-568ad9d4c1f2 h1:wbji0B3aQIzAWXx9qZfx3YNGttRffUvNN02TWpO4H8o= -github.com/sapcc/go-bits v0.0.0-20261023092235-568ad9d4c1f2/go.mod h1:ZiQj8yOMniyYxmkMPJin1p/juJtM0UVXweOZRw/9oQs= +github.com/sapcc/go-bits v0.0.0-20261029170309-6e36605dd3a1 h1:bgfB935Ox8K6Fa75T/VlSIAQ0M4v9mKdbHWBYRrRKPE= +github.com/sapcc/go-bits v0.0.0-20261029170309-6e36605dd3a1/go.mod h1:qV1JUJLrG2keKgLcPfI407VwjrYjCm6C9bwx6jX5C1g= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= @@ -245,8 +249,8 @@ golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJk golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= -golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw= -golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= @@ -285,24 +289,24 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -k8s.io/api v0.37.1 h1:l6N77U7tjwB5L056bgrBTJIEdevac/naBZ3iSvDNfpM= -k8s.io/api v0.37.1/go.mod h1:zSlbB1YpJ1YQlFVQy20UYll81UJSJJUMLhkhvg6Z78M= +k8s.io/api v0.37.2 h1:PnxqnUFy93jr2GcpWDkKOm8wfy25kT9wa572MuQS8oc= +k8s.io/api v0.37.2/go.mod h1:+NVRLft99kx+Jg9v34GmHw867Dapim89TSyCsn2hK+M= k8s.io/apiextensions-apiserver v0.37.0 h1:zRMQ3+/LIE5oZ0tVvXwYHC+dIkSP5cjNWju7AZU1LOI= k8s.io/apiextensions-apiserver v0.37.0/go.mod h1:HU0PfSBwchHL5iDau6jjt9zU6ryWkDDlaVUiq91NK80= -k8s.io/apimachinery v0.37.1 h1:hGCYyvKHCwtwMitj2vU4vYx0Z16N9GyZk9BBnz0wDAE= -k8s.io/apimachinery v0.37.1/go.mod h1:jF84AyUi/IRIXRot5f+lm6MpxoWI+F1XgjaMmwCdTFw= +k8s.io/apimachinery v0.37.2 h1:G4t9RGoPDbQML38hz/kQQiltttgd3egg2ODCR8aIc2o= +k8s.io/apimachinery v0.37.2/go.mod h1:qaAvSNvgU2x07wRM95MSHSJTPQ/ob/LeNAGbeDgqQJg= k8s.io/apiserver v0.37.0 h1:TXg7OxsOWrAH8J4Zi/gBAZuMw1Dfdd+6cca2h4qjRqo= k8s.io/apiserver v0.37.0/go.mod h1:OddHDF4gy9qyIb8o/3+qaeP6S0vEObWLgOygVqXksv0= -k8s.io/client-go v0.37.1 h1:QTv/5ha4jAHtW9qxxVBkQVFBRDb4jHfFopQqqMdc+wM= -k8s.io/client-go v0.37.1/go.mod h1:dnAPtTnCNY38Ho04D2KdY1F4IKausa9UbqaAZKl60SY= +k8s.io/client-go v0.37.2 h1:sgFtyRZukGkOrsu+QgS7LYnbFFJ5ut3MchdBNduD//E= +k8s.io/client-go v0.37.2/go.mod h1:bo3LFC8YbCtKFPX21Yj8puMsoJLAv/+SvuOODma0s54= k8s.io/component-base v0.37.0 h1:3SdSa4+itMdFTDFTeR8CxKGmSTSMXFlKL4ky8OqjguM= k8s.io/component-base v0.37.0/go.mod h1:LjOebp4R9y6LODWZQv102ZQxGheLcDO2ZJLAw6bbh4I= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad h1:oXImqH8mQNk7PmvzKhmN3ddJoY6OnyM225MXwGHPm0A= k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad/go.mod h1:0/mqHCVhlumdJ3BhCfnjSZQE037nAhNodh1/hK0T8/I= -k8s.io/streaming v0.37.1 h1:TpzVfQeFuVndn2g9mFqxy1UcUYPwDzqjUmwR/IzJCWc= -k8s.io/streaming v0.37.1/go.mod h1:APlJR26ZWRcVy5bIEj0QRrKUXROtBHPcxl2NT7EAzPU= +k8s.io/streaming v0.37.2 h1:xmzp9nbqIbjvsRcqrBYa+K5XPQ6Jy/Dm3zZ8mN916AI= +k8s.io/streaming v0.37.2/go.mod h1:APlJR26ZWRcVy5bIEj0QRrKUXROtBHPcxl2NT7EAzPU= k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 h1:jVkFFVfXdXP74B/zbO3hM3hpSFD0xvhQ5U686DPurkE= k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3/go.mod h1:M2s5JB1lIYP3jzZdorPLHXIPJzt9vv2muW5a6L9DtNM= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.36.0 h1:/YpDJ4vReG7ZmzSpBGxduXgywWkJU9zHubgJG03MT+Y= diff --git a/helm/bundles/cortex-cinder/Chart.yaml b/helm/bundles/cortex-cinder/Chart.yaml index b78ae3a86..a903223b3 100644 --- a/helm/bundles/cortex-cinder/Chart.yaml +++ b/helm/bundles/cortex-cinder/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-cinder description: A Helm chart deploying Cortex for Cinder. type: application -version: 0.0.109 +version: 0.0.110 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.29 + version: 0.6.30 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.7.2 + version: 0.7.3 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.7.2 + version: 0.7.3 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-crds/Chart.yaml b/helm/bundles/cortex-crds/Chart.yaml index 54f807f46..18f44b407 100644 --- a/helm/bundles/cortex-crds/Chart.yaml +++ b/helm/bundles/cortex-crds/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-crds description: A Helm chart deploying Cortex CRDs. type: application -version: 0.0.109 +version: 0.0.110 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.7.2 + version: 0.7.3 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/bundles/cortex-ironcore/Chart.yaml b/helm/bundles/cortex-ironcore/Chart.yaml index ff3435d3e..1848560dd 100644 --- a/helm/bundles/cortex-ironcore/Chart.yaml +++ b/helm/bundles/cortex-ironcore/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-ironcore description: A Helm chart deploying Cortex for IronCore. type: application -version: 0.0.109 +version: 0.0.110 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.7.2 + version: 0.7.3 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/bundles/cortex-manila/Chart.yaml b/helm/bundles/cortex-manila/Chart.yaml index 61aa37d44..0f2f3df64 100644 --- a/helm/bundles/cortex-manila/Chart.yaml +++ b/helm/bundles/cortex-manila/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-manila description: A Helm chart deploying Cortex for Manila. type: application -version: 0.0.109 +version: 0.0.110 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.29 + version: 0.6.30 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.7.2 + version: 0.7.3 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.7.2 + version: 0.7.3 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-nova/Chart.yaml b/helm/bundles/cortex-nova/Chart.yaml index 0ed98c9d1..938021e99 100644 --- a/helm/bundles/cortex-nova/Chart.yaml +++ b/helm/bundles/cortex-nova/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-nova description: A Helm chart deploying Cortex for Nova. type: application -version: 0.0.109 +version: 0.0.110 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.29 + version: 0.6.30 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.7.2 + version: 0.7.3 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.7.2 + version: 0.7.3 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-nova/templates/alerts.yaml b/helm/bundles/cortex-nova/templates/alerts.yaml index 1a4377d1d..bb82e889a 100644 --- a/helm/bundles/cortex-nova/templates/alerts.yaml +++ b/helm/bundles/cortex-nova/templates/alerts.yaml @@ -1029,5 +1029,109 @@ spec: controller logs (filter on "periodic reconciliation completed" and "processed VM failover reservations") to identify the affected VMs and the reason their reservations cannot be created. + - alert: CortexNovaFailoverHostCalculationStalled + # The failover host controller recomputes its desired reservations on a + # periodic calculation loop. runs_total only increments when a pass + # completes, so increase()==0 over a window several times the loop interval + # means the loop is wedged, panicking, or every pass is aborting before it + # completes — in all cases the failover host reservations are no longer + # being maintained and will silently drift from the cluster. A pod that is + # down is covered by CortexNovaSchedulingDown, so require the series to + # still be exported (the metric is present but flat). + expr: | + increase(cortex_failover_host_calculation_runs_total{service="cortex-nova-metrics"}[30m]) == 0 + for: 30m + labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex_status?var-service=cortex-nova-metrics" +{{- end }} + context: failover-hosts + service: cortex + severity: warning + support_group: workload-management + playbook: "" + annotations: + summary: "Failover host calculation loop is not completing passes" + description: > + The failover host controller has not completed a calculation-loop pass in + the last 30 minutes. Its failover host reservations are no longer being + recomputed and will drift from the actual cluster state. Check the + failover-host-controller logs for a stuck loop or repeated aborts. + - alert: CortexNovaFailoverHostCalculationAborting + # Each calculation pass aborts (without touching the desired state) when the + # cluster snapshot is inconsistent (reason="collect"), the Nova flavor seed + # set cannot be gathered (reason="seeding"), or the external scheduler lookup + # fails (reason="pathfinding"). A sustained abort rate means + # no pass is producing a fresh desired state, so reservations go stale. + expr: rate(cortex_failover_host_calculation_aborts_total{service="cortex-nova-metrics"}[15m]) > 0 + for: 30m + labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex_status?var-service=cortex-nova-metrics" +{{- end }} + context: failover-hosts + service: cortex + severity: warning + support_group: workload-management + playbook: "" + annotations: + summary: "Failover host calculation loop keeps aborting (reason=`{{ "{{" }} $labels.reason {{ "}}" }}`)" + description: > + The failover host calculation loop has been aborting before producing a + desired state for over 30 minutes (reason=`{{ "{{" }} $labels.reason {{ "}}" }}`). + reason="collect" means the cluster snapshot was inconsistent (e.g. a VM on an + unknown hypervisor, or a hypervisor without an availability zone); + reason="seeding" means the Nova flavor catalogue could not be listed; + reason="pathfinding" means the external scheduler lookup failed. Failover host + reservations are not being refreshed while this persists. + - alert: CortexNovaFailoverHostReconcileErrors + # The reconciler failed to create, update or delete failover host + # reservations. A sustained error rate means the desired reservations + # computed by the loop cannot be applied to the cluster. + expr: rate(cortex_failover_host_reservations_errors_total{service="cortex-nova-metrics"}[15m]) > 0 + for: 30m + labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex_status?var-service=cortex-nova-metrics" +{{- end }} + context: failover-hosts + service: cortex + severity: warning + support_group: workload-management + playbook: "" + annotations: + summary: "Failover host reservations cannot be reconciled (op=`{{ "{{" }} $labels.op {{ "}}" }}`)" + description: > + The failover host reconciler has been failing to {{ "{{" }} $labels.op {{ "}}" }} + reservations for over 30 minutes. The desired failover host reservations computed + by the calculation loop are not being applied, so actual reservations drift from + the desired set. Check the failover-host-controller-reconcile logs and the + Reservation objects for the affected hosts. + - alert: CortexNovaFailoverHostSchedulerErrors + # The path finder asks the external scheduler, per flavor seed, which hosts + # would accept that flavor. A high fraction of failed requests (transport + # errors or non-2xx) makes every calculation pass abort, so this is the root + # cause behind a persistent CortexNovaFailoverHostCalculationAborting with + # reason="pathfinding". + expr: | + (sum(rate(cortex_failover_host_scheduler_requests_total{service="cortex-nova-metrics",status!~"2.."}[15m]))) + / (sum(rate(cortex_failover_host_scheduler_requests_total{service="cortex-nova-metrics"}[15m]))) > 0.1 + for: 15m + labels: +{{- if .Values.alerts.persesDashboardBaseURL }} + persesDashboard: "{{ .Values.alerts.persesDashboardBaseURL }}/cortex_status?var-service=cortex-nova-metrics" +{{- end }} + context: failover-hosts + service: cortex + severity: warning + support_group: workload-management + playbook: "" + annotations: + summary: "Failover host path finder scheduler error rate >10%" + description: > + More than 10% of the external scheduler requests issued by the failover host + path finder are failing (transport errors or non-2xx responses). The path finder + cannot determine valid evacuation targets, so failover partitions and reservations + cannot be recomputed. Check that the external scheduler is reachable and healthy. {{- end }} {{- end }} diff --git a/helm/bundles/cortex-nova/templates/kpis_kvm.yaml b/helm/bundles/cortex-nova/templates/kpis_kvm.yaml index 7fc4f0223..b4cd7c507 100644 --- a/helm/bundles/cortex-nova/templates/kpis_kvm.yaml +++ b/helm/bundles/cortex-nova/templates/kpis_kvm.yaml @@ -15,6 +15,20 @@ spec: --- apiVersion: cortex.cloud/v1alpha1 kind: KPI +metadata: + name: failover-partition-graph +spec: + schedulingDomain: nova + impl: failover_partition_graph_kpi + description: | + This KPI exposes the failover partition graph derived from the hypervisors + and failover host reservations: nodes (cortex_failover_partition_graph_node) + are the hypervisors (value 1 when the hypervisor has a failover path, 0 when + it has none), edges (cortex_failover_partition_graph_edge) are the + source->target evacuation paths within each partition. +--- +apiVersion: cortex.cloud/v1alpha1 +kind: KPI metadata: name: kvm-project-utilization spec: @@ -52,6 +66,7 @@ spec: gvks: - cortex.cloud/v1alpha1/ReservationList - cortex.cloud/v1alpha1/CommittedResourceList + - cortex.cloud/v1alpha1/CommittedResourceChangeList - kvm.cloud.sap/v1/HypervisorList - cobaltcore.cloud/v1alpha1/VirtualMachineList description: | diff --git a/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml b/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml index a8278993f..817e75ab6 100644 --- a/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml +++ b/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml @@ -80,6 +80,10 @@ spec: This step will filter out hosts that do not have enough available capacity to host the requested flavor. Reservation unlocking behavior is controlled by the call-time Options.LockReservations flag, not by a step param. + params: + # FailoverHostReservation is a new feature and is not yet counted as + # reserved capacity during scheduling. + - {key: ignoredReservationTypes, stringListValue: ["FailoverHostReservation"]} - name: filter_allowed_projects description: | This step filters hosts based on allowed projects defined in the @@ -112,8 +116,8 @@ spec: Rejects all hosts if neither tier has headroom. Runs in shadow mode by default: the filter logs and emits the - cortex_nova_filter_quota_enforcement_decisions_total metric for - would-be rejects but never actually removes hosts. To globally reject + cortex_filter_weigher_pipeline_step_events_total{event="quota_enforcement_decision"} + metric for would-be rejects but never actually removes hosts. To globally reject requests without headroom, add `params: [{key: enforce, boolValue: true}]` to this step. The shadow default is intentional so that newly-rolled-out releases never silently start rejecting requests. @@ -250,6 +254,10 @@ spec: This step will filter out hosts that do not have enough available capacity to host the requested flavor. Reservation unlocking behavior is controlled by the call-time Options.LockReservations flag, not by a step param. + params: + # FailoverHostReservation is a new feature and is not yet counted as + # reserved capacity during scheduling. + - {key: ignoredReservationTypes, stringListValue: ["FailoverHostReservation"]} - name: filter_allowed_projects description: | This step filters hosts based on allowed projects defined in the @@ -282,8 +290,8 @@ spec: Rejects all hosts if neither tier has headroom. Runs in shadow mode by default: the filter logs and emits the - cortex_nova_filter_quota_enforcement_decisions_total metric for - would-be rejects but never actually removes hosts. To globally reject + cortex_filter_weigher_pipeline_step_events_total{event="quota_enforcement_decision"} + metric for would-be rejects but never actually removes hosts. To globally reject requests without headroom, add `params: [{key: enforce, boolValue: true}]` to this step. The shadow default is intentional so that newly-rolled-out releases never silently start rejecting requests. @@ -370,6 +378,9 @@ spec: current reservations residing on this host from the available capacity. params: - {key: lockReserved, boolValue: true} + # FailoverHostReservation is a new feature and is not yet counted as + # reserved capacity during scheduling. + - {key: ignoredReservationTypes, stringListValue: ["FailoverHostReservation"]} - name: filter_has_requested_traits description: | This step filters hosts that do not have the requested traits given by the @@ -567,6 +578,10 @@ spec: to host the requested flavor. Reservations are locked via the call-time Options.LockReservations flag (set by the ack caller) to prevent false-positive unlocking of CR slots during evacuation validation. + params: + # FailoverHostReservation is a new feature and is not yet counted as + # reserved capacity during scheduling. + - {key: ignoredReservationTypes, stringListValue: ["FailoverHostReservation"]} - name: filter_has_requested_traits description: | This step filters hosts that do not have the requested traits given by the @@ -637,7 +652,7 @@ spec: empty datacenter scenario. params: - {key: ignoreAllocations, boolValue: true} - - {key: ignoredReservationTypes, stringListValue: ["CommittedResourceReservation", "FailoverReservation"]} + - {key: ignoredReservationTypes, stringListValue: ["CommittedResourceReservation", "FailoverReservation", "FailoverHostReservation"]} - name: filter_has_requested_traits description: | Ensures hosts have the hardware traits required by the flavor. diff --git a/helm/bundles/cortex-nova/values.yaml b/helm/bundles/cortex-nova/values.yaml index 28b643566..755bb8f42 100644 --- a/helm/bundles/cortex-nova/values.yaml +++ b/helm/bundles/cortex-nova/values.yaml @@ -137,6 +137,8 @@ cortex: &cortex - cortex.cloud/v1alpha1/ReservationList - cortex.cloud/v1alpha1/CommittedResource - cortex.cloud/v1alpha1/CommittedResourceList + - cortex.cloud/v1alpha1/CommittedResourceChange + - cortex.cloud/v1alpha1/CommittedResourceChangeList - cortex.cloud/v1alpha1/ProjectQuota - cortex.cloud/v1alpha1/ProjectQuotaList - cortex.cloud/v1alpha1/FlavorGroupCapacity @@ -159,6 +161,7 @@ cortex: &cortex gvks: - cortex.cloud/v1alpha1/Reservation - cortex.cloud/v1alpha1/CommittedResource + - cortex.cloud/v1alpha1/CommittedResourceChange monitoring: labels: &cortexMonitoringLabels github_org: cobaltcore-dev @@ -185,6 +188,7 @@ cortex-scheduling-controllers: - hypervisor-overcommit-controller - committed-resource-reservations-controller - failover-reservations-controller + - failover-host-reservations-controller - quota-controller - capacity-controller enabledTasks: @@ -358,6 +362,41 @@ cortex-scheduling-controllers: # A zero value disables pruning. maxReservationDeletesPerRun: 20 + # --- Failover host reservations controller --- + # Keeps whole compute hosts reserved as gang-evacuation targets: it groups + # hosts into "failover partitions" (sets of hosts that can all be evacuated + # onto one another, discovered from the scheduler) and keeps a share of each + # partition empty and reserved, so when a host fails its entire workload can + # be moved onto a reserved host in one step. Partitions are discovered by + # probing the scheduler with flavor "seeds" (every Nova flavor crossed with + # every availability zone), not from the VMs currently running, so a + # partition is found even when its hosts are idle. This is independent of the + # flavor-based failover controller configured above (schedulerURL, + # reconcileInterval, flavorFailoverRequirements, ...). + # + # Nested under `failover.hosts` as a deliberate role model for grouping a + # controller's settings under its own key, rather than adding yet more flat + # top-level keys. Other controller configs should be refactored toward this. + failover: + hosts: + # Fraction of each partition's hosts to keep reserved as failover + # targets, rounded down (e.g. 0.1 of a 10-host partition reserves 1 + # host). Must be within (0, 1]. Combined with `minimum` below, which + # sets a floor so even small partitions keep some headroom. + ratio: 0.1 + # Minimum number of failover hosts to reserve per partition, applied even + # when `ratio` would round down to fewer. Must be greater than 0. The + # controller never reserves so many that a partition is left with no host + # to run workload (it always keeps at least one source host). + minimum: 1 + # How often the failover partitions are recalculated and the reservations + # updated to match. Must be at least 1m. + generationInterval: 5m + # URL of the nova external scheduler API, used to discover which hosts a + # given host's workload could be evacuated onto. Typically the same + # scheduler endpoint the flavor-based failover controller uses. + externalSchedulerURL: "http://localhost:8080/scheduler/nova/external" + cortex-knowledge-controllers: <<: *cortex namePrefix: cortex-nova-knowledge diff --git a/helm/bundles/cortex-placement-shim/Chart.yaml b/helm/bundles/cortex-placement-shim/Chart.yaml index b9b3f783c..10cfd29ac 100644 --- a/helm/bundles/cortex-placement-shim/Chart.yaml +++ b/helm/bundles/cortex-placement-shim/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-placement-shim description: A Helm chart deploying the Cortex placement shim. type: application -version: 0.1.34 +version: 0.1.35 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-shim - name: cortex-shim repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.1.34 + version: 0.1.35 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case # of issues. See: https://github.com/sapcc/helm-charts/pkgs/container/helm-charts%2Fowner-info diff --git a/helm/bundles/cortex-pods/Chart.yaml b/helm/bundles/cortex-pods/Chart.yaml index 4f565f388..707912dd2 100644 --- a/helm/bundles/cortex-pods/Chart.yaml +++ b/helm/bundles/cortex-pods/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-pods description: A Helm chart deploying Cortex for Pods. type: application -version: 0.0.109 +version: 0.0.110 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.7.2 + version: 0.7.3 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/dev/cortex-prometheus-operator/Chart.yaml b/helm/dev/cortex-prometheus-operator/Chart.yaml index 650a960aa..01a302f80 100644 --- a/helm/dev/cortex-prometheus-operator/Chart.yaml +++ b/helm/dev/cortex-prometheus-operator/Chart.yaml @@ -10,4 +10,4 @@ dependencies: # CRDs of the prometheus operator, such as PrometheusRule, ServiceMonitor, etc. - name: kube-prometheus-stack repository: oci://ghcr.io/prometheus-community/charts - version: 92.13.1 + version: 92.17.0 diff --git a/helm/library/cortex-postgres/Chart.yaml b/helm/library/cortex-postgres/Chart.yaml index ccccc54d2..15082460c 100644 --- a/helm/library/cortex-postgres/Chart.yaml +++ b/helm/library/cortex-postgres/Chart.yaml @@ -5,5 +5,5 @@ apiVersion: v2 name: cortex-postgres description: Postgres setup for Cortex. type: application -version: 0.6.29 -appVersion: "sha-4f5a623f" +version: 0.6.30 +appVersion: "sha-7f53e4df" diff --git a/helm/library/cortex-shim/Chart.yaml b/helm/library/cortex-shim/Chart.yaml index e18aa113e..c0e7bbb03 100644 --- a/helm/library/cortex-shim/Chart.yaml +++ b/helm/library/cortex-shim/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cortex-shim description: A Helm chart to distribute cortex shims. type: application -version: 0.1.34 -appVersion: "sha-eac4ca69" +version: 0.1.35 +appVersion: "sha-63f36907" icon: "https://example.com/icon.png" dependencies: [] diff --git a/helm/library/cortex/Chart.yaml b/helm/library/cortex/Chart.yaml index ddf9b080f..fa068a3db 100644 --- a/helm/library/cortex/Chart.yaml +++ b/helm/library/cortex/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cortex description: A Helm chart to distribute cortex. type: application -version: 0.7.2 -appVersion: "sha-ece8785b" +version: 0.7.3 +appVersion: "sha-b74c4f13" icon: "https://example.com/icon.png" dependencies: [] diff --git a/helm/library/cortex/files/crds/cortex.cloud_committedresourcechanges.yaml b/helm/library/cortex/files/crds/cortex.cloud_committedresourcechanges.yaml new file mode 100644 index 000000000..ecf607f4c --- /dev/null +++ b/helm/library/cortex/files/crds/cortex.cloud_committedresourcechanges.yaml @@ -0,0 +1,421 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.23.0 + name: committedresourcechanges.cortex.cloud +spec: + group: cortex.cloud + names: + kind: CommittedResourceChange + listKind: CommittedResourceChangeList + plural: committedresourcechanges + shortNames: + - crc + singular: committedresourcechange + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .spec.availabilityZone + name: AZ + type: string + - jsonPath: .spec.source + name: Source + type: string + - jsonPath: .spec.dryRun + name: DryRun + priority: 1 + type: boolean + - jsonPath: .status.conditions[?(@.type=='Progressing')].status + name: Progressing + priority: 1 + type: string + - jsonPath: .status.conditions[?(@.type=='Finished')].status + name: Finished + priority: 1 + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: CommittedResourceChange is the Schema for the committedresourcechanges + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CommittedResourceChangeSpec defines the desired change for + a CommittedResourceChange. + properties: + allowRejection: + description: |- + AllowRejection controls whether the change request may be rejected and rolled back + on placement failure. When false, the reconciler retries until Accepted or Superseded. + type: boolean + availabilityZone: + description: AvailabilityZone is the availability zone this change + request is scoped to. + minLength: 1 + type: string + correlationID: + description: CorrelationID is the request/correlation ID. + type: string + dryRun: + description: DryRun controls whether the change request commits or + only probes schedulability. + type: boolean + newState: + additionalProperties: + description: CommittedResourceSpec defines the desired state of + CommittedResource, + properties: + allowRejection: + description: |- + AllowRejection controls what the CommittedResource controller does when placement fails + for a guaranteed or confirmed commitment. + true — controller may reject: on failure, child Reservations are rolled back and the CR + is marked Rejected. Use this when the caller is making a first-time placement + decision and a "no" answer is acceptable (e.g. the change-commitments API). + false — controller must retry: on failure, existing child Reservations are kept and the + CR is set to Reserving so the controller retries later. Use this when the caller + is restoring already-committed state that Cortex must honour (e.g. the syncer). + Only meaningful for state=guaranteed or state=confirmed; ignored for all other states. + type: boolean + amount: + anyOf: + - type: integer + - type: string + description: |- + Amount is the total committed quantity. + memory: MiB expressed in K8s binary SI notation (e.g. "1280Gi", "640Mi"). + cores: integer core count (e.g. "40"). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + availabilityZone: + description: AvailabilityZone specifies the availability zone + for this commitment. + type: string + commitmentUUID: + description: UUID of the commitment this resource corresponds + to. + type: string + confirmedAt: + description: ConfirmedAt is when the commitment was confirmed. + format: date-time + type: string + domainID: + description: DomainID of the OpenStack domain this commitment + belongs to. + type: string + endTime: + description: EndTime is when Reservation slots expire. Nil for + unbounded commitments with no expiry. + format: date-time + type: string + flavorGroupName: + description: FlavorGroupName identifies the flavor group this + commitment targets, e.g. "kvm_v2_hana_s". + type: string + projectID: + description: ProjectID of the OpenStack project this commitment + belongs to. + type: string + resourceType: + description: 'ResourceType identifies the kind of resource committed: + memory drives Reservation slots; cores uses an arithmetic + check only.' + enum: + - memory + - cores + type: string + schedulingDomain: + description: SchedulingDomain specifies the scheduling domain + for this committed resource (e.g., "nova", "ironcore"). + type: string + startTime: + description: |- + StartTime is the activation time for Reservation slots. + Nil for guaranteed commitments (slots are active from creation); set to ConfirmedAt for confirmed ones. + format: date-time + type: string + state: + description: State is the lifecycle state of the commitment. + enum: + - planned + - pending + - guaranteed + - confirmed + - superseded + - expired + type: string + required: + - amount + - availabilityZone + - commitmentUUID + - domainID + - flavorGroupName + - projectID + - resourceType + - state + type: object + description: |- + NewState maps commitmentUUID to the desired CommittedResourceSpec. + A nil value means the corresponding CommittedResource should be deleted. + type: object + source: + description: Source identifies who created this change request. + enum: + - api + - syncer + type: string + required: + - availabilityZone + - source + type: object + status: + description: CommittedResourceChangeStatus defines the observed state + of a CommittedResourceChange. + properties: + conditions: + description: Conditions holds Progressing and Finished conditions. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + message: + description: Message is a human-readable description of the current + state or terminal outcome. + type: string + phase: + description: Phase is the current FSM phase of the change request. + type: string + x-kubernetes-validations: + - message: phase cannot transition out of a terminal state + rule: oldSelf == '' || !['Accepted','Rejected','Superseded'].exists(t, + t == oldSelf) || self == oldSelf + snapshotBefore: + additionalProperties: + description: CommittedResourceSpec defines the desired state of + CommittedResource, + properties: + allowRejection: + description: |- + AllowRejection controls what the CommittedResource controller does when placement fails + for a guaranteed or confirmed commitment. + true — controller may reject: on failure, child Reservations are rolled back and the CR + is marked Rejected. Use this when the caller is making a first-time placement + decision and a "no" answer is acceptable (e.g. the change-commitments API). + false — controller must retry: on failure, existing child Reservations are kept and the + CR is set to Reserving so the controller retries later. Use this when the caller + is restoring already-committed state that Cortex must honour (e.g. the syncer). + Only meaningful for state=guaranteed or state=confirmed; ignored for all other states. + type: boolean + amount: + anyOf: + - type: integer + - type: string + description: |- + Amount is the total committed quantity. + memory: MiB expressed in K8s binary SI notation (e.g. "1280Gi", "640Mi"). + cores: integer core count (e.g. "40"). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + availabilityZone: + description: AvailabilityZone specifies the availability zone + for this commitment. + type: string + commitmentUUID: + description: UUID of the commitment this resource corresponds + to. + type: string + confirmedAt: + description: ConfirmedAt is when the commitment was confirmed. + format: date-time + type: string + domainID: + description: DomainID of the OpenStack domain this commitment + belongs to. + type: string + endTime: + description: EndTime is when Reservation slots expire. Nil for + unbounded commitments with no expiry. + format: date-time + type: string + flavorGroupName: + description: FlavorGroupName identifies the flavor group this + commitment targets, e.g. "kvm_v2_hana_s". + type: string + projectID: + description: ProjectID of the OpenStack project this commitment + belongs to. + type: string + resourceType: + description: 'ResourceType identifies the kind of resource committed: + memory drives Reservation slots; cores uses an arithmetic + check only.' + enum: + - memory + - cores + type: string + schedulingDomain: + description: SchedulingDomain specifies the scheduling domain + for this committed resource (e.g., "nova", "ironcore"). + type: string + startTime: + description: |- + StartTime is the activation time for Reservation slots. + Nil for guaranteed commitments (slots are active from creation); set to ConfirmedAt for confirmed ones. + format: date-time + type: string + state: + description: State is the lifecycle state of the commitment. + enum: + - planned + - pending + - guaranteed + - confirmed + - superseded + - expired + type: string + required: + - amount + - availabilityZone + - commitmentUUID + - domainID + - flavorGroupName + - projectID + - resourceType + - state + type: object + description: |- + SnapshotBefore records the CommittedResource spec state before this change request. + Key is commitmentUUID; nil value means the CR did not exist before (will be deleted on rollback). + Populated in Preparing when AllowRejection=true. + type: object + upsertedCommitments: + description: |- + UpsertedCommitments holds references to CommittedResource CRDs written by this change request. + Watched in AwaitingScheduling to gate on placement success. + These are plain object references, not owner references. + items: + description: ObjectReference contains enough information to let + you inspect or modify the referred object. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + type: array + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/helm/library/cortex/files/crds/cortex.cloud_reservations.yaml b/helm/library/cortex/files/crds/cortex.cloud_reservations.yaml index a9bdceebf..29152f333 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_reservations.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_reservations.yaml @@ -172,6 +172,17 @@ spec: FailoverHostReservation contains fields specific to failover host reservations. Only used when Type is FailoverHostReservation. properties: + partitionDescription: + description: |- + PartitionDescription is a short, human-readable label describing the + failover partition this reservation belongs to: what distinguishes its + hosts from other partitions (e.g. "spec.aggregates=[hana]") and which + flavors the whole partition can run (e.g. "flavors: hana.4xl, hana.8xl + (+3 more)"). It carries no scheduling semantics and exists purely to make a + reservation understandable at a glance — in particular, which workloads a + reserved host stands ready to absorb. See describePartition in the + failoverhost controller. + type: string sourceHosts: description: |- SourceHosts is a list of hosts inside the same failover partition from diff --git a/helm/library/cortex/templates/rbac/role.yaml b/helm/library/cortex/templates/rbac/role.yaml index 661a3be71..54991d0b4 100644 --- a/helm/library/cortex/templates/rbac/role.yaml +++ b/helm/library/cortex/templates/rbac/role.yaml @@ -14,6 +14,7 @@ rules: - datasources - reservations - committedresources + - committedresourcechanges - projectquotas - flavorgroupcapacities - decisions @@ -36,6 +37,7 @@ rules: - datasources/finalizers - reservations/finalizers - committedresources/finalizers + - committedresourcechanges/finalizers - projectquotas/finalizers - flavorgroupcapacities/finalizers - decisions/finalizers @@ -52,6 +54,7 @@ rules: - datasources/status - reservations/status - committedresources/status + - committedresourcechanges/status - projectquotas/status - flavorgroupcapacities/status - decisions/status diff --git a/internal/knowledge/datasources/plugins/openstack/cinder/cinder_sync.go b/internal/knowledge/datasources/plugins/openstack/cinder/cinder_sync.go index 7f5d4f0e3..7f2db669c 100644 --- a/internal/knowledge/datasources/plugins/openstack/cinder/cinder_sync.go +++ b/internal/knowledge/datasources/plugins/openstack/cinder/cinder_sync.go @@ -8,7 +8,9 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources" + "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack" "github.com/cobaltcore-dev/cortex/internal/knowledge/db" + "github.com/cobaltcore-dev/cortex/pkg/keystone" "github.com/go-gorp/gorp" ) @@ -67,3 +69,20 @@ func (s *CinderSyncer) SyncAllStoragePools(ctx context.Context) (int64, error) { } return int64(len(pools)), nil } + +func init() { + openstack.Index[v1alpha1.OpenStackDatasourceTypeCinder] = func( + datasource v1alpha1.Datasource, + authenticatedDB *db.DB, + authenticatedKeystone keystone.KeystoneClient, + monitor datasources.Monitor, + ) openstack.Syncer { + + return &CinderSyncer{ + DB: *authenticatedDB, + Mon: monitor, + Conf: datasource.Spec.OpenStack.Cinder, + API: NewCinderAPI(monitor, authenticatedKeystone, datasource.Spec.OpenStack.Cinder), + } + } +} diff --git a/internal/knowledge/datasources/plugins/openstack/controller.go b/internal/knowledge/datasources/plugins/openstack/controller.go index c16d8e1e9..03cb0f0c0 100644 --- a/internal/knowledge/datasources/plugins/openstack/controller.go +++ b/internal/knowledge/datasources/plugins/openstack/controller.go @@ -169,14 +169,9 @@ func (r *OpenStackDatasourceReconciler) Reconcile(ctx context.Context, req ctrl. } log.V(1).Info("Looking for supported syncer of datasource") - syncer, err := getSupportedSyncer( - *datasource, - authenticatedDB, - authenticatedKeystone, - r.Monitor, - ) - if err != nil { - log.Error(err, "failed to get supported syncer for datasource", "type", datasource.Spec.OpenStack.Type) + newSyncer, ok := Index[datasource.Spec.OpenStack.Type] + if !ok { + log.Error(nil, "failed to get supported syncer for datasource", "type", datasource.Spec.OpenStack.Type) old := datasource.DeepCopy() meta.SetStatusCondition(&datasource.Status.Conditions, metav1.Condition{ Type: v1alpha1.DatasourceConditionReady, @@ -191,6 +186,7 @@ func (r *OpenStackDatasourceReconciler) Reconcile(ctx context.Context, req ctrl. } return ctrl.Result{}, nil } + syncer := newSyncer(*datasource, authenticatedDB, authenticatedKeystone, r.Monitor) // Initialize the syncer before syncing. log.V(1).Info("Initializing syncer for datasource") diff --git a/internal/knowledge/datasources/plugins/openstack/identity/identity_sync.go b/internal/knowledge/datasources/plugins/openstack/identity/identity_sync.go index bbf915146..140ffd2b6 100644 --- a/internal/knowledge/datasources/plugins/openstack/identity/identity_sync.go +++ b/internal/knowledge/datasources/plugins/openstack/identity/identity_sync.go @@ -8,7 +8,9 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources" + "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack" "github.com/cobaltcore-dev/cortex/internal/knowledge/db" + "github.com/cobaltcore-dev/cortex/pkg/keystone" "github.com/go-gorp/gorp" ) @@ -90,3 +92,20 @@ func (s *IdentitySyncer) SyncProjects(ctx context.Context) (int64, error) { } return int64(len(projects)), nil } + +func init() { + openstack.Index[v1alpha1.OpenStackDatasourceTypeIdentity] = func( + datasource v1alpha1.Datasource, + authenticatedDB *db.DB, + authenticatedKeystone keystone.KeystoneClient, + monitor datasources.Monitor, + ) openstack.Syncer { + + return &IdentitySyncer{ + DB: *authenticatedDB, + Mon: monitor, + Conf: datasource.Spec.OpenStack.Identity, + API: NewIdentityAPI(monitor, authenticatedKeystone, datasource.Spec.OpenStack.Identity), + } + } +} diff --git a/internal/knowledge/datasources/plugins/openstack/limes/limes_sync.go b/internal/knowledge/datasources/plugins/openstack/limes/limes_sync.go index 75b5b654f..263a82f5b 100644 --- a/internal/knowledge/datasources/plugins/openstack/limes/limes_sync.go +++ b/internal/knowledge/datasources/plugins/openstack/limes/limes_sync.go @@ -8,8 +8,10 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources" + "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack" "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/identity" "github.com/cobaltcore-dev/cortex/internal/knowledge/db" + "github.com/cobaltcore-dev/cortex/pkg/keystone" "github.com/go-gorp/gorp" ) @@ -76,3 +78,20 @@ func (s *LimesSyncer) SyncCommitments(ctx context.Context) (int64, error) { } return int64(len(commitments)), nil } + +func init() { + openstack.Index[v1alpha1.OpenStackDatasourceTypeLimes] = func( + datasource v1alpha1.Datasource, + authenticatedDB *db.DB, + authenticatedKeystone keystone.KeystoneClient, + monitor datasources.Monitor, + ) openstack.Syncer { + + return &LimesSyncer{ + DB: *authenticatedDB, + Mon: monitor, + Conf: datasource.Spec.OpenStack.Limes, + API: NewLimesAPI(monitor, authenticatedKeystone, datasource.Spec.OpenStack.Limes), + } + } +} diff --git a/internal/knowledge/datasources/plugins/openstack/manila/manila_sync.go b/internal/knowledge/datasources/plugins/openstack/manila/manila_sync.go index c7816526c..92d191af3 100644 --- a/internal/knowledge/datasources/plugins/openstack/manila/manila_sync.go +++ b/internal/knowledge/datasources/plugins/openstack/manila/manila_sync.go @@ -8,7 +8,9 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources" + "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack" "github.com/cobaltcore-dev/cortex/internal/knowledge/db" + "github.com/cobaltcore-dev/cortex/pkg/keystone" "github.com/go-gorp/gorp" ) @@ -68,3 +70,20 @@ func (s *ManilaSyncer) SyncAllStoragePools(ctx context.Context) (int64, error) { } return int64(len(pools)), nil } + +func init() { + openstack.Index[v1alpha1.OpenStackDatasourceTypeManila] = func( + datasource v1alpha1.Datasource, + authenticatedDB *db.DB, + authenticatedKeystone keystone.KeystoneClient, + monitor datasources.Monitor, + ) openstack.Syncer { + + return &ManilaSyncer{ + DB: *authenticatedDB, + Mon: monitor, + Conf: datasource.Spec.OpenStack.Manila, + API: NewManilaAPI(monitor, authenticatedKeystone, datasource.Spec.OpenStack.Manila), + } + } +} diff --git a/internal/knowledge/datasources/plugins/openstack/nova/nova_sync.go b/internal/knowledge/datasources/plugins/openstack/nova/nova_sync.go index a47449c27..5a9c4e52a 100644 --- a/internal/knowledge/datasources/plugins/openstack/nova/nova_sync.go +++ b/internal/knowledge/datasources/plugins/openstack/nova/nova_sync.go @@ -10,7 +10,9 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources" + "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack" "github.com/cobaltcore-dev/cortex/internal/knowledge/db" + "github.com/cobaltcore-dev/cortex/pkg/keystone" "github.com/go-gorp/gorp" ) @@ -219,3 +221,20 @@ func (s *NovaSyncer) SyncAllAggregates(ctx context.Context) (int64, error) { } return int64(len(allAggregates)), nil } + +func init() { + openstack.Index[v1alpha1.OpenStackDatasourceTypeNova] = func( + datasource v1alpha1.Datasource, + authenticatedDB *db.DB, + authenticatedKeystone keystone.KeystoneClient, + monitor datasources.Monitor, + ) openstack.Syncer { + + return &NovaSyncer{ + DB: *authenticatedDB, + Mon: monitor, + Conf: datasource.Spec.OpenStack.Nova, + API: NewNovaAPI(monitor, authenticatedKeystone, datasource.Spec.OpenStack.Nova), + } + } +} diff --git a/internal/knowledge/datasources/plugins/openstack/placement/placement_sync.go b/internal/knowledge/datasources/plugins/openstack/placement/placement_sync.go index 2ee89e464..c6d8076e7 100644 --- a/internal/knowledge/datasources/plugins/openstack/placement/placement_sync.go +++ b/internal/knowledge/datasources/plugins/openstack/placement/placement_sync.go @@ -8,7 +8,9 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources" + "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack" "github.com/cobaltcore-dev/cortex/internal/knowledge/db" + "github.com/cobaltcore-dev/cortex/pkg/keystone" "github.com/go-gorp/gorp" ) @@ -136,3 +138,20 @@ func (s *PlacementSyncer) SyncInventoryUsages(ctx context.Context) (int64, error } return int64(len(inventoryUsages)), err } + +func init() { + openstack.Index[v1alpha1.OpenStackDatasourceTypePlacement] = func( + datasource v1alpha1.Datasource, + authenticatedDB *db.DB, + authenticatedKeystone keystone.KeystoneClient, + monitor datasources.Monitor, + ) openstack.Syncer { + + return &PlacementSyncer{ + DB: *authenticatedDB, + Mon: monitor, + Conf: datasource.Spec.OpenStack.Placement, + API: NewPlacementAPI(monitor, authenticatedKeystone, datasource.Spec.OpenStack.Placement), + } + } +} diff --git a/internal/knowledge/datasources/plugins/openstack/supported_syncers.go b/internal/knowledge/datasources/plugins/openstack/supported_syncers.go deleted file mode 100644 index 31d3cfd57..000000000 --- a/internal/knowledge/datasources/plugins/openstack/supported_syncers.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package openstack - -import ( - "errors" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources" - "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/cinder" - "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/identity" - "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/limes" - "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/manila" - "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/nova" - "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/plugins/openstack/placement" - "github.com/cobaltcore-dev/cortex/internal/knowledge/db" - "github.com/cobaltcore-dev/cortex/pkg/keystone" -) - -// Get the supported syncer for the given datasource. -func getSupportedSyncer( - datasource v1alpha1.Datasource, - authenticatedDB *db.DB, - authenticatedKeystone keystone.KeystoneClient, - monitor datasources.Monitor, -) (Syncer, error) { - - switch datasource.Spec.OpenStack.Type { - case v1alpha1.OpenStackDatasourceTypeNova: - return &nova.NovaSyncer{ - DB: *authenticatedDB, - Mon: monitor, - Conf: datasource.Spec.OpenStack.Nova, - API: nova.NewNovaAPI(monitor, authenticatedKeystone, datasource.Spec.OpenStack.Nova), - }, nil - case v1alpha1.OpenStackDatasourceTypeManila: - return &manila.ManilaSyncer{ - DB: *authenticatedDB, - Mon: monitor, - Conf: datasource.Spec.OpenStack.Manila, - API: manila.NewManilaAPI(monitor, authenticatedKeystone, datasource.Spec.OpenStack.Manila), - }, nil - case v1alpha1.OpenStackDatasourceTypePlacement: - return &placement.PlacementSyncer{ - DB: *authenticatedDB, - Mon: monitor, - Conf: datasource.Spec.OpenStack.Placement, - API: placement.NewPlacementAPI(monitor, authenticatedKeystone, datasource.Spec.OpenStack.Placement), - }, nil - case v1alpha1.OpenStackDatasourceTypeIdentity: - return &identity.IdentitySyncer{ - DB: *authenticatedDB, - Mon: monitor, - Conf: datasource.Spec.OpenStack.Identity, - API: identity.NewIdentityAPI(monitor, authenticatedKeystone, datasource.Spec.OpenStack.Identity), - }, nil - case v1alpha1.OpenStackDatasourceTypeLimes: - return &limes.LimesSyncer{ - DB: *authenticatedDB, - Mon: monitor, - Conf: datasource.Spec.OpenStack.Limes, - API: limes.NewLimesAPI(monitor, authenticatedKeystone, datasource.Spec.OpenStack.Limes), - }, nil - case v1alpha1.OpenStackDatasourceTypeCinder: - return &cinder.CinderSyncer{ - DB: *authenticatedDB, - Mon: monitor, - Conf: datasource.Spec.OpenStack.Cinder, - API: cinder.NewCinderAPI(monitor, authenticatedKeystone, datasource.Spec.OpenStack.Cinder), - }, nil - default: - return nil, errors.New("unsupported openstack datasource type") - } -} diff --git a/internal/knowledge/datasources/plugins/openstack/zz_index.go b/internal/knowledge/datasources/plugins/openstack/zz_index.go new file mode 100644 index 000000000..2fb841c66 --- /dev/null +++ b/internal/knowledge/datasources/plugins/openstack/zz_index.go @@ -0,0 +1,20 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package openstack + +import ( + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources" + "github.com/cobaltcore-dev/cortex/internal/knowledge/db" + "github.com/cobaltcore-dev/cortex/pkg/keystone" +) + +// Index of all supported OpenStack syncers, keyed by datasource type. +// Each syncer registers itself here through a func init() in its own package. +var Index = map[v1alpha1.OpenStackDatasourceType]func( + datasource v1alpha1.Datasource, + authenticatedDB *db.DB, + authenticatedKeystone keystone.KeystoneClient, + monitor datasources.Monitor, +) Syncer{} diff --git a/internal/knowledge/datasources/plugins/prometheus/controller.go b/internal/knowledge/datasources/plugins/prometheus/controller.go index 4983c89ba..fb9b1d8cb 100644 --- a/internal/knowledge/datasources/plugins/prometheus/controller.go +++ b/internal/knowledge/datasources/plugins/prometheus/controller.go @@ -79,7 +79,7 @@ func (r *PrometheusDatasourceReconciler) Reconcile(ctx context.Context, req ctrl log.V(1).Info("first reconcile this process lifetime, forcing sync despite timestamp", "name", datasource.Name) } - newSyncerFunc, ok := supportedMetricSyncers[datasource.Spec.Prometheus.Type] + newSyncerFunc, ok := Index[datasource.Spec.Prometheus.Type] if !ok { log.V(1).Info("skipping datasource, unsupported metric type", "metricType", datasource.Spec.Prometheus.Type) old := datasource.DeepCopy() diff --git a/internal/knowledge/datasources/plugins/prometheus/supported_syncers.go b/internal/knowledge/datasources/plugins/prometheus/supported_syncers.go deleted file mode 100644 index c15b23a47..000000000 --- a/internal/knowledge/datasources/plugins/prometheus/supported_syncers.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package prometheus - -import ( - "net/http" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources" - "github.com/cobaltcore-dev/cortex/internal/knowledge/db" -) - -// All supported Prometheus metric syncers. -var supportedMetricSyncers = map[string]func( - ds v1alpha1.Datasource, - authenticatedDB *db.DB, - authenticatedHTTP *http.Client, - prometheusURL string, - monitor datasources.Monitor, -) typedSyncer{ - "vrops_host_metric": newTypedSyncer[VROpsHostMetric], - "vrops_vm_metric": newTypedSyncer[VROpsVMMetric], - "node_exporter_metric": newTypedSyncer[NodeExporterMetric], - "netapp_aggregate_labels_metric": newTypedSyncer[NetAppAggregateLabelsMetric], - "netapp_node_metric": newTypedSyncer[NetAppNodeMetric], - "netapp_volume_aggregate_labels_metric": newTypedSyncer[NetAppVolumeAggrLabelsMetric], - "kvm_libvirt_domain_metric": newTypedSyncer[KVMDomainMetric], -} diff --git a/internal/knowledge/datasources/plugins/prometheus/types.go b/internal/knowledge/datasources/plugins/prometheus/types.go index 9fa3b78e9..5cfe9fe0e 100644 --- a/internal/knowledge/datasources/plugins/prometheus/types.go +++ b/internal/knowledge/datasources/plugins/prometheus/types.go @@ -292,3 +292,13 @@ func (m NetAppVolumeAggrLabelsMetric) With(n string, t time.Time, v float64) Pro m.Value = v return m } + +func init() { + Index["vrops_host_metric"] = newTypedSyncer[VROpsHostMetric] + Index["vrops_vm_metric"] = newTypedSyncer[VROpsVMMetric] + Index["node_exporter_metric"] = newTypedSyncer[NodeExporterMetric] + Index["netapp_aggregate_labels_metric"] = newTypedSyncer[NetAppAggregateLabelsMetric] + Index["netapp_node_metric"] = newTypedSyncer[NetAppNodeMetric] + Index["netapp_volume_aggregate_labels_metric"] = newTypedSyncer[NetAppVolumeAggrLabelsMetric] + Index["kvm_libvirt_domain_metric"] = newTypedSyncer[KVMDomainMetric] +} diff --git a/internal/knowledge/datasources/plugins/prometheus/zz_index.go b/internal/knowledge/datasources/plugins/prometheus/zz_index.go new file mode 100644 index 000000000..6c17c8c3d --- /dev/null +++ b/internal/knowledge/datasources/plugins/prometheus/zz_index.go @@ -0,0 +1,22 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package prometheus + +import ( + "net/http" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/knowledge/datasources" + "github.com/cobaltcore-dev/cortex/internal/knowledge/db" +) + +// Index of all supported Prometheus metric syncers, keyed by metric type. +// Each metric registers itself here through a func init() in types.go. +var Index = map[string]func( + ds v1alpha1.Datasource, + authenticatedDB *db.DB, + authenticatedHTTP *http.Client, + prometheusURL string, + monitor datasources.Monitor, +) typedSyncer{} diff --git a/internal/knowledge/extractor/controller.go b/internal/knowledge/extractor/controller.go index 7bb87c33e..f77528eae 100644 --- a/internal/knowledge/extractor/controller.go +++ b/internal/knowledge/extractor/controller.go @@ -12,6 +12,10 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/db" + "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins" + // Register all supported feature extractors through their func init(). + _ "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + _ "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/storage" "github.com/cobaltcore-dev/cortex/pkg/multicluster" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -59,7 +63,7 @@ func (r *KnowledgeReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ctrl.Result{RequeueAfter: waitFor}, nil } - extractor, ok := supportedExtractors[knowledge.Spec.Extractor.Name] + newExtractor, ok := plugins.Index[knowledge.Spec.Extractor.Name] if !ok { log.V(1).Info("skipping knowledge extraction, unsupported extractor", "name", knowledge.Spec.Extractor.Name) old := knowledge.DeepCopy() @@ -145,6 +149,7 @@ func (r *KnowledgeReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } // Initialize and run the extractor. + extractor := newExtractor() wrapped := monitorFeatureExtractor(knowledge.Spec.Extractor.Name, extractor, r.Monitor) if err := wrapped.Init(authenticatedDatasourceDB, r.Client, knowledge.Spec); err != nil { log.Error(err, "failed to initialize feature extractor", "name", knowledge.Spec) diff --git a/internal/knowledge/extractor/plugins/compute/flavor_groups.go b/internal/knowledge/extractor/plugins/compute/flavor_groups.go index 19e729b8f..fdebd4404 100644 --- a/internal/knowledge/extractor/plugins/compute/flavor_groups.go +++ b/internal/knowledge/extractor/plugins/compute/flavor_groups.go @@ -254,3 +254,9 @@ func (e *FlavorGroupExtractor) Extract() ([]plugins.Feature, error) { return e.Extracted(features) } + +func init() { + plugins.Index["flavor_groups"] = func() plugins.FeatureExtractor { + return &FlavorGroupExtractor{} + } +} diff --git a/internal/knowledge/extractor/plugins/compute/host_az.go b/internal/knowledge/extractor/plugins/compute/host_az.go index 8e690b1c9..e80287afd 100644 --- a/internal/knowledge/extractor/plugins/compute/host_az.go +++ b/internal/knowledge/extractor/plugins/compute/host_az.go @@ -31,3 +31,9 @@ var hostAZQuery string func (e *HostAZExtractor) Extract() ([]plugins.Feature, error) { return e.ExtractSQL(hostAZQuery) } + +func init() { + plugins.Index["host_az_extractor"] = func() plugins.FeatureExtractor { + return &HostAZExtractor{} + } +} diff --git a/internal/knowledge/extractor/plugins/compute/host_capabilities.go b/internal/knowledge/extractor/plugins/compute/host_capabilities.go index 7ebf05555..94abc4323 100644 --- a/internal/knowledge/extractor/plugins/compute/host_capabilities.go +++ b/internal/knowledge/extractor/plugins/compute/host_capabilities.go @@ -35,3 +35,9 @@ var hostCapabilitiesQuery string func (e *HostCapabilitiesExtractor) Extract() ([]plugins.Feature, error) { return e.ExtractSQL(hostCapabilitiesQuery) } + +func init() { + plugins.Index["host_capabilities_extractor"] = func() plugins.FeatureExtractor { + return &HostCapabilitiesExtractor{} + } +} diff --git a/internal/knowledge/extractor/plugins/compute/host_pinned_projects.go b/internal/knowledge/extractor/plugins/compute/host_pinned_projects.go index 9427ea65a..871b3dd61 100644 --- a/internal/knowledge/extractor/plugins/compute/host_pinned_projects.go +++ b/internal/knowledge/extractor/plugins/compute/host_pinned_projects.go @@ -45,3 +45,9 @@ var hostPinnedProjectsQuery string func (e *HostPinnedProjectsExtractor) Extract() ([]plugins.Feature, error) { return e.ExtractSQL(hostPinnedProjectsQuery) } + +func init() { + plugins.Index["host_pinned_projects_extractor"] = func() plugins.FeatureExtractor { + return &HostPinnedProjectsExtractor{} + } +} diff --git a/internal/knowledge/extractor/plugins/compute/host_utilization.go b/internal/knowledge/extractor/plugins/compute/host_utilization.go index 73f7c1ac2..04561bc88 100644 --- a/internal/knowledge/extractor/plugins/compute/host_utilization.go +++ b/internal/knowledge/extractor/plugins/compute/host_utilization.go @@ -47,3 +47,9 @@ var hostUtilizationQuery string func (e *HostUtilizationExtractor) Extract() ([]plugins.Feature, error) { return e.ExtractSQL(hostUtilizationQuery) } + +func init() { + plugins.Index["host_utilization_extractor"] = func() plugins.FeatureExtractor { + return &HostUtilizationExtractor{} + } +} diff --git a/internal/knowledge/extractor/plugins/compute/libvirt_domain_cpu_steal_pct.go b/internal/knowledge/extractor/plugins/compute/libvirt_domain_cpu_steal_pct.go index 9898c591f..88b0c812c 100644 --- a/internal/knowledge/extractor/plugins/compute/libvirt_domain_cpu_steal_pct.go +++ b/internal/knowledge/extractor/plugins/compute/libvirt_domain_cpu_steal_pct.go @@ -35,3 +35,9 @@ var libvirtDomainCPUStealPctSQL string func (e *LibvirtDomainCPUStealPctExtractor) Extract() ([]plugins.Feature, error) { return e.ExtractSQL(libvirtDomainCPUStealPctSQL) } + +func init() { + plugins.Index["kvm_libvirt_domain_cpu_steal_pct_extractor"] = func() plugins.FeatureExtractor { + return &LibvirtDomainCPUStealPctExtractor{} + } +} diff --git a/internal/knowledge/extractor/plugins/compute/vm_host_residency.go b/internal/knowledge/extractor/plugins/compute/vm_host_residency.go index 7f15c7894..ff75fca1a 100644 --- a/internal/knowledge/extractor/plugins/compute/vm_host_residency.go +++ b/internal/knowledge/extractor/plugins/compute/vm_host_residency.go @@ -91,3 +91,9 @@ func (e *VMHostResidencyExtractor) Extract() ([]plugins.Feature, error) { } return e.Extracted(features) } + +func init() { + plugins.Index["vm_host_residency_extractor"] = func() plugins.FeatureExtractor { + return &VMHostResidencyExtractor{} + } +} diff --git a/internal/knowledge/extractor/plugins/compute/vm_life_span.go b/internal/knowledge/extractor/plugins/compute/vm_life_span.go index 981d75f3a..5016ad6a0 100644 --- a/internal/knowledge/extractor/plugins/compute/vm_life_span.go +++ b/internal/knowledge/extractor/plugins/compute/vm_life_span.go @@ -109,3 +109,9 @@ func (e *VMLifeSpanHistogramExtractor) Extract() ([]plugins.Feature, error) { deletedVMLifeSpansBuckets = append(deletedVMLifeSpansBuckets, runningVMLifeSpansBuckets...) return e.Extracted(deletedVMLifeSpansBuckets) } + +func init() { + plugins.Index["vm_life_span_histogram_extractor"] = func() plugins.FeatureExtractor { + return &VMLifeSpanHistogramExtractor{} + } +} diff --git a/internal/knowledge/extractor/plugins/compute/vmware_host_details.go b/internal/knowledge/extractor/plugins/compute/vmware_host_details.go index c87a8aa93..ea168d089 100644 --- a/internal/knowledge/extractor/plugins/compute/vmware_host_details.go +++ b/internal/knowledge/extractor/plugins/compute/vmware_host_details.go @@ -136,3 +136,9 @@ func (e *VMwareHostDetailsExtractor) Extract() ([]plugins.Feature, error) { } return e.Extracted(hostDetails) } + +func init() { + plugins.Index["vmware_host_details_extractor"] = func() plugins.FeatureExtractor { + return &VMwareHostDetailsExtractor{} + } +} diff --git a/internal/knowledge/extractor/plugins/compute/vrops_hostsystem_contention_long_term.go b/internal/knowledge/extractor/plugins/compute/vrops_hostsystem_contention_long_term.go index ada48460a..0ac8b1b3e 100644 --- a/internal/knowledge/extractor/plugins/compute/vrops_hostsystem_contention_long_term.go +++ b/internal/knowledge/extractor/plugins/compute/vrops_hostsystem_contention_long_term.go @@ -105,3 +105,9 @@ func (e *VROpsHostsystemContentionLongTermExtractor) Extract() ([]plugins.Featur return e.Extracted(features) } + +func init() { + plugins.Index["vrops_hostsystem_contention_long_term_extractor"] = func() plugins.FeatureExtractor { + return &VROpsHostsystemContentionLongTermExtractor{} + } +} diff --git a/internal/knowledge/extractor/plugins/compute/vrops_hostsystem_contention_short_term.go b/internal/knowledge/extractor/plugins/compute/vrops_hostsystem_contention_short_term.go index 80f17e483..6fa656c71 100644 --- a/internal/knowledge/extractor/plugins/compute/vrops_hostsystem_contention_short_term.go +++ b/internal/knowledge/extractor/plugins/compute/vrops_hostsystem_contention_short_term.go @@ -105,3 +105,9 @@ func (e *VROpsHostsystemContentionShortTermExtractor) Extract() ([]plugins.Featu return e.Extracted(features) } + +func init() { + plugins.Index["vrops_hostsystem_contention_short_term_extractor"] = func() plugins.FeatureExtractor { + return &VROpsHostsystemContentionShortTermExtractor{} + } +} diff --git a/internal/knowledge/extractor/plugins/compute/vrops_hostsystem_resolver.go b/internal/knowledge/extractor/plugins/compute/vrops_hostsystem_resolver.go index ada567854..2c945f6e5 100644 --- a/internal/knowledge/extractor/plugins/compute/vrops_hostsystem_resolver.go +++ b/internal/knowledge/extractor/plugins/compute/vrops_hostsystem_resolver.go @@ -33,3 +33,9 @@ var vropsHostsystemSQL string func (e *VROpsHostsystemResolver) Extract() ([]plugins.Feature, error) { return e.ExtractSQL(vropsHostsystemSQL) } + +func init() { + plugins.Index["vrops_hostsystem_resolver"] = func() plugins.FeatureExtractor { + return &VROpsHostsystemResolver{} + } +} diff --git a/internal/knowledge/extractor/plugins/compute/vrops_project_noisiness.go b/internal/knowledge/extractor/plugins/compute/vrops_project_noisiness.go index 638c76e26..aa6ba9c02 100644 --- a/internal/knowledge/extractor/plugins/compute/vrops_project_noisiness.go +++ b/internal/knowledge/extractor/plugins/compute/vrops_project_noisiness.go @@ -33,3 +33,9 @@ var vropsProjectNoisinessSQL string func (e *VROpsProjectNoisinessExtractor) Extract() ([]plugins.Feature, error) { return e.ExtractSQL(vropsProjectNoisinessSQL) } + +func init() { + plugins.Index["vrops_project_noisiness_extractor"] = func() plugins.FeatureExtractor { + return &VROpsProjectNoisinessExtractor{} + } +} diff --git a/internal/knowledge/extractor/plugins/storage/storage_pool_cpu_usage.go b/internal/knowledge/extractor/plugins/storage/storage_pool_cpu_usage.go index 326034692..cefdef05b 100644 --- a/internal/knowledge/extractor/plugins/storage/storage_pool_cpu_usage.go +++ b/internal/knowledge/extractor/plugins/storage/storage_pool_cpu_usage.go @@ -35,3 +35,9 @@ var storagePoolCPUUsageQuery string func (e *StoragePoolCPUUsageExtractor) Extract() ([]plugins.Feature, error) { return e.ExtractSQL(storagePoolCPUUsageQuery) } + +func init() { + plugins.Index["netapp_storage_pool_cpu_usage_extractor"] = func() plugins.FeatureExtractor { + return &StoragePoolCPUUsageExtractor{} + } +} diff --git a/internal/knowledge/extractor/plugins/zz_index.go b/internal/knowledge/extractor/plugins/zz_index.go new file mode 100644 index 000000000..8d5f8304d --- /dev/null +++ b/internal/knowledge/extractor/plugins/zz_index.go @@ -0,0 +1,8 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package plugins + +// Index of all supported feature extractors, keyed by name. +// Each extractor registers itself here through a func init() in its own file. +var Index = map[string]func() FeatureExtractor{} diff --git a/internal/knowledge/extractor/supported_extractors.go b/internal/knowledge/extractor/supported_extractors.go deleted file mode 100644 index 4cfcd006a..000000000 --- a/internal/knowledge/extractor/supported_extractors.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package extractor - -import ( - "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins" - "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" - "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/storage" -) - -// All supported feature extractors. -var supportedExtractors = map[string]plugins.FeatureExtractor{ - "vrops_hostsystem_resolver": &compute.VROpsHostsystemResolver{}, - "vrops_project_noisiness_extractor": &compute.VROpsProjectNoisinessExtractor{}, - "vrops_hostsystem_contention_long_term_extractor": &compute.VROpsHostsystemContentionLongTermExtractor{}, - "vrops_hostsystem_contention_short_term_extractor": &compute.VROpsHostsystemContentionShortTermExtractor{}, - "kvm_libvirt_domain_cpu_steal_pct_extractor": &compute.LibvirtDomainCPUStealPctExtractor{}, - "host_utilization_extractor": &compute.HostUtilizationExtractor{}, - "host_capabilities_extractor": &compute.HostCapabilitiesExtractor{}, - "vm_host_residency_extractor": &compute.VMHostResidencyExtractor{}, - "vm_life_span_histogram_extractor": &compute.VMLifeSpanHistogramExtractor{}, - "host_az_extractor": &compute.HostAZExtractor{}, - "host_pinned_projects_extractor": &compute.HostPinnedProjectsExtractor{}, - "vmware_host_details_extractor": &compute.VMwareHostDetailsExtractor{}, - "flavor_groups": &compute.FlavorGroupExtractor{}, - - "netapp_storage_pool_cpu_usage_extractor": &storage.StoragePoolCPUUsageExtractor{}, -} diff --git a/internal/knowledge/kpis/controller.go b/internal/knowledge/kpis/controller.go index 507c26733..422ecc0c0 100644 --- a/internal/knowledge/kpis/controller.go +++ b/internal/knowledge/kpis/controller.go @@ -11,6 +11,11 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/db" "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins" + // Register all supported kpis through their func init(). + _ "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins/compute" + _ "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins/deployment" + _ "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins/infrastructure" + _ "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins/storage" "github.com/cobaltcore-dev/cortex/pkg/conf" "github.com/cobaltcore-dev/cortex/pkg/multicluster" corev1 "k8s.io/api/core/v1" @@ -42,8 +47,9 @@ type Controller struct { // Configuration for the controller. Config ControllerConfig - // The supported kpis to manage. - supportedKPIs map[string]plugins.KPI + // The supported kpis to manage, keyed by name. Each value constructs a + // fresh kpi instance. + supportedKPIs map[string]func() plugins.KPI // Registered kpis by name. registeredKPIsByResourceName map[string]plugins.KPI } @@ -242,12 +248,11 @@ func (c *Controller) handleKPIChange(ctx context.Context, obj *v1alpha1.KPI) err // initialize and register it now. if dependenciesReadyTotal == dependenciesTotal && !registered { log.Info("kpi: registering new kpi", "name", obj.Name) - var ok bool - registeredKPI, ok = c.supportedKPIs[obj.Spec.Impl] + newKPI, ok := c.supportedKPIs[obj.Spec.Impl] if !ok { return fmt.Errorf("kpi %s not supported", obj.Name) } - registeredKPI = &kpilogger{kpi: registeredKPI} + registeredKPI = &kpilogger{kpi: newKPI()} // Get joint database connection for all dependencies. jointDB, err := c.getJointDB(ctx, obj.Spec.Dependencies.Datasources) if err != nil { @@ -437,7 +442,7 @@ func (c *Controller) handleKnowledgeDeleted( func (c *Controller) SetupWithManager(mgr manager.Manager, mcl *multicluster.Client) error { if c.supportedKPIs == nil { // This can be overridden for tests. - c.supportedKPIs = supportedKPIs + c.supportedKPIs = plugins.Index } if err := mgr.Add(manager.RunnableFunc(c.InitAllKPIs)); err != nil { return err diff --git a/internal/knowledge/kpis/controller_test.go b/internal/knowledge/kpis/controller_test.go index 0aeb96485..dd4079328 100644 --- a/internal/knowledge/kpis/controller_test.go +++ b/internal/knowledge/kpis/controller_test.go @@ -220,13 +220,12 @@ func (mc *mockController) handleKPIChange(ctx context.Context, obj *v1alpha1.KPI // initialize and register it now. if dependenciesReadyTotal == dependenciesTotal && !registered { log.Info("kpi: registering new kpi", "name", obj.Name) - var ok bool - registeredKPI, ok = mc.supportedKPIs[obj.Spec.Impl] + newKPI, ok := mc.supportedKPIs[obj.Spec.Impl] if !ok { return fmt.Errorf("kpi %s not supported", obj.Name) } // Create a wrapper that logs operations (equivalent to kpilogger) - registeredKPI = &mockKPILogger{kpi: registeredKPI} + registeredKPI = &mockKPILogger{kpi: newKPI()} // Get joint database connection for all dependencies using mock method. jointDB, err := mc.getJointDB(ctx, obj.Spec.Dependencies.Datasources, @@ -465,8 +464,8 @@ func TestController_Reconcile(t *testing.T) { baseController := Controller{ Client: fakeClient, Config: ControllerConfig{SchedulingDomain: "test-operator"}, - supportedKPIs: map[string]plugins.KPI{ - "test_kpi": mockKPIInstance, + supportedKPIs: map[string]func() plugins.KPI{ + "test_kpi": func() plugins.KPI { return mockKPIInstance }, }, registeredKPIsByResourceName: make(map[string]plugins.KPI), } @@ -545,8 +544,8 @@ func TestController_Reconcile_KPIDeleted(t *testing.T) { controller := &Controller{ Client: fakeClient, Config: ControllerConfig{SchedulingDomain: "test-operator"}, - supportedKPIs: map[string]plugins.KPI{ - "test_kpi": mockKPIInstance, + supportedKPIs: map[string]func() plugins.KPI{ + "test_kpi": func() plugins.KPI { return mockKPIInstance }, }, registeredKPIsByResourceName: map[string]plugins.KPI{ "deleted-kpi": mockKPIInstance, @@ -758,8 +757,8 @@ func TestController_handleKPIChange(t *testing.T) { mockKPIInstance := &mockKPI{name: "test_kpi", initError: tt.mockInitError} baseController := Controller{ Client: fakeClient, - supportedKPIs: map[string]plugins.KPI{ - "test_kpi": mockKPIInstance, + supportedKPIs: map[string]func() plugins.KPI{ + "test_kpi": func() plugins.KPI { return mockKPIInstance }, }, registeredKPIsByResourceName: make(map[string]plugins.KPI), } @@ -970,8 +969,8 @@ func TestController_InitAllKPIs(t *testing.T) { baseController := Controller{ Client: fakeClient, Config: ControllerConfig{SchedulingDomain: "test-operator"}, - supportedKPIs: map[string]plugins.KPI{ - "test_kpi": mockKPIInstance, + supportedKPIs: map[string]func() plugins.KPI{ + "test_kpi": func() plugins.KPI { return mockKPIInstance }, }, } diff --git a/internal/knowledge/kpis/plugins/compute/failover_partition_graph.go b/internal/knowledge/kpis/plugins/compute/failover_partition_graph.go new file mode 100644 index 000000000..6ab253e17 --- /dev/null +++ b/internal/knowledge/kpis/plugins/compute/failover_partition_graph.go @@ -0,0 +1,156 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package compute + +import ( + "context" + "maps" + "strconv" + "time" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/knowledge/db" + "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins" + "github.com/cobaltcore-dev/cortex/pkg/conf" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + vmv1alpha1 "github.com/cobaltcore-dev/vm-operator/api/v1alpha1" + "github.com/prometheus/client_golang/prometheus" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var failoverPartitionGraphKPILogger = ctrl.Log.WithName("failover-partition-graph-kpi") + +// KPI exposing the failover partition graph derived from the hypervisors and +// failover host reservations present in the cluster. The graph has two series: +// +// - cortex_failover_partition_graph_node{host,az,memory,cpu,vms} — one per +// hypervisor, whether or not it participates in any failover reservation, +// carrying the hypervisor's availability zone, total capacity and the number +// of vms currently running on it. The value is 1 when the hypervisor has at +// least one failover path (it is the source or target of an edge) and 0 when +// it has none. +// - cortex_failover_partition_graph_edge{source,target} 1 — one per directed +// evacuation path: every SourceHost of a failover host reservation may fail +// over onto the reservation's TargetHost. +type FailoverPartitionGraphKPI struct { + // Common base for all KPIs that provides standard functionality. + plugins.BaseKPI[struct{}] + + node *prometheus.Desc + edge *prometheus.Desc +} + +func (FailoverPartitionGraphKPI) GetName() string { return "failover_partition_graph_kpi" } + +// Initialize the KPI. +func (k *FailoverPartitionGraphKPI) Init(db *db.DB, client client.Client, opts conf.RawOpts) error { + if err := k.BaseKPI.Init(db, client, opts); err != nil { + return err + } + k.node = prometheus.NewDesc( + "cortex_failover_partition_graph_node", + "A hypervisor in the failover partition graph", + []string{"host", "az", "memory", "cpu", "vms"}, + nil, + ) + k.edge = prometheus.NewDesc( + "cortex_failover_partition_graph_edge", + "A directed evacuation path (source can fail over onto target) in the failover partition graph", + []string{"source", "target"}, + nil, + ) + return nil +} + +// Conform to the prometheus collector interface by providing the descriptors. +func (k *FailoverPartitionGraphKPI) Describe(ch chan<- *prometheus.Desc) { + ch <- k.node + ch <- k.edge +} + +// Collect the failover partition graph metrics: one node per hypervisor and one +// edge per failover evacuation path. +func (k *FailoverPartitionGraphKPI) Collect(ch chan<- prometheus.Metric) { + // Bound the list calls so a slow API server can't hang the Prometheus scrape + // indefinitely; if they fail we log so the disappearance of the metrics is + // not silent. + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + reservationList := &v1alpha1.ReservationList{} + if err := k.Client.List(ctx, reservationList); err != nil { + failoverPartitionGraphKPILogger.Error(err, "Failed to list reservations") + return + } + // A (source,target) pair could in principle repeat across reservations, and + // Prometheus rejects duplicate series within a single scrape, so collapse the + // edges into a set first. Track which hosts take part in any path so nodes + // without a failover path can be reported with value 0. + type edge struct{ source, target string } + emittedEdges := map[edge]struct{}{} + hasPath := map[string]struct{}{} + for _, r := range reservationList.Items { + if r.Spec.Type != v1alpha1.ReservationTypeFailoverHost { + continue + } + target := r.Spec.TargetHost + if target == "" || r.Spec.FailoverHostReservation == nil { + continue + } + for _, source := range r.Spec.FailoverHostReservation.SourceHosts { + hasPath[source] = struct{}{} + hasPath[target] = struct{}{} + e := edge{source: source, target: target} + if _, ok := emittedEdges[e]; ok { + continue + } + emittedEdges[e] = struct{}{} + ch <- prometheus.MustNewConstMetric( + k.edge, prometheus.GaugeValue, 1, source, target, + ) + } + } + hypervisorList := &hv1.HypervisorList{} + if err := k.Client.List(ctx, hypervisorList); err != nil { + failoverPartitionGraphKPILogger.Error(err, "Failed to list hypervisors") + return + } + // Count the vms running on each host so the node series can expose it. A vm + // Nova hasn't placed yet has no host and is skipped (it also nil-panics on + // Status.Nova otherwise). + vmList := &vmv1alpha1.VirtualMachineList{} + if err := k.Client.List(ctx, vmList); err != nil { + failoverPartitionGraphKPILogger.Error(err, "Failed to list virtual machines") + return + } + vmsByHost := map[string]int{} + for _, vm := range vmList.Items { + if vm.Status.Nova == nil || vm.Status.Nova.Host == "" { + continue + } + vmsByHost[vm.Status.Nova.Host]++ + } + for _, hv := range hypervisorList.Items { + capacity := map[hv1.ResourceName]resource.Quantity{} + maps.Copy(capacity, hv.Status.Capacity) // base capacity + maps.Copy(capacity, hv.Status.EffectiveCapacity) + memory := capacity[hv1.ResourceMemory] + cpu := capacity[hv1.ResourceCPU] + value := 0.0 + if _, ok := hasPath[hv.Name]; ok { + value = 1 + } + ch <- prometheus.MustNewConstMetric( + k.node, prometheus.GaugeValue, value, + hv.Name, hv.Labels[corev1.LabelTopologyZone], + memory.String(), cpu.String(), strconv.Itoa(vmsByHost[hv.Name]), + ) + } +} + +func init() { + plugins.Index["failover_partition_graph_kpi"] = func() plugins.KPI { return &FailoverPartitionGraphKPI{} } +} diff --git a/internal/knowledge/kpis/plugins/compute/failover_partition_graph_test.go b/internal/knowledge/kpis/plugins/compute/failover_partition_graph_test.go new file mode 100644 index 000000000..f95592bb9 --- /dev/null +++ b/internal/knowledge/kpis/plugins/compute/failover_partition_graph_test.go @@ -0,0 +1,200 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package compute + +import ( + "testing" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/pkg/conf" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + vmv1alpha1 "github.com/cobaltcore-dev/vm-operator/api/v1alpha1" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func failoverHostReservation(name, target string, sources ...string) v1alpha1.Reservation { + return v1alpha1.Reservation{ + ObjectMeta: v1.ObjectMeta{Name: name}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeFailoverHost, + SchedulingDomain: "nova", + TargetHost: target, + FailoverHostReservation: &v1alpha1.FailoverHostReservationSpec{ + SourceHosts: sources, + }, + }, + } +} + +func hypervisor(name, az string) hv1.Hypervisor { + return hv1.Hypervisor{ + ObjectMeta: v1.ObjectMeta{Name: name, Labels: map[string]string{corev1.LabelTopologyZone: az}}, + Status: hv1.HypervisorStatus{ + Capacity: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: resource.MustParse("256Gi"), + hv1.ResourceCPU: resource.MustParse("64"), + }, + }, + } +} + +// virtualMachine builds a vm Nova reports as placed on the given host. +func virtualMachine(name, host string) vmv1alpha1.VirtualMachine { + return vmv1alpha1.VirtualMachine{ + ObjectMeta: v1.ObjectMeta{Name: name}, + Status: vmv1alpha1.VMStatus{Nova: &vmv1alpha1.NovaStatus{Host: host}}, + } +} + +func TestFailoverPartitionGraphKPI_GetName(t *testing.T) { + kpi := &FailoverPartitionGraphKPI{} + if name := kpi.GetName(); name != "failover_partition_graph_kpi" { + t.Errorf("expected name failover_partition_graph_kpi, got %q", name) + } +} + +func TestFailoverPartitionGraphKPI_Describe(t *testing.T) { + kpi := &FailoverPartitionGraphKPI{} + if err := kpi.Init(nil, nil, conf.NewRawOpts("{}")); err != nil { + t.Fatalf("expected no error, got %v", err) + } + ch := make(chan *prometheus.Desc, 2) + kpi.Describe(ch) + close(ch) + count := 0 + for range ch { + count++ + } + if count != 2 { + t.Errorf("expected 2 descriptors, got %d", count) + } +} + +func TestFailoverPartitionGraphKPI_Collect(t *testing.T) { + scheme, err := v1alpha1.SchemeBuilder.Build() + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if err := hv1.AddToScheme(scheme); err != nil { + t.Fatalf("expected no error, got %v", err) + } + if err := vmv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("expected no error, got %v", err) + } + + // Two failover-host reservations sharing source host h1 (so the edge for h1 + // is emitted to both targets), plus a non-failover reservation that must be + // ignored. + resA := failoverHostReservation("failover-host-t1", "t1", "h1", "h2") + resB := failoverHostReservation("failover-host-t2", "t2", "h1", "h3") + other := v1alpha1.Reservation{ + ObjectMeta: v1.ObjectMeta{Name: "r-other"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeInFlight, + SchedulingDomain: "nova", + TargetHost: "ignored", + }, + } + // The hypervisors backing the graph. h-idle takes part in no reservation, so + // its node is reported with value 0. + t1 := hypervisor("t1", "az-a") + t2 := hypervisor("t2", "az-b") + h1 := hypervisor("h1", "az-a") + h2 := hypervisor("h2", "az-a") + h3 := hypervisor("h3", "az-b") + idle := hypervisor("h-idle", "az-a") + + // Two vms on h1 and one on t2; every other host (incl. idle) has none, and an + // unplaced vm must not be counted. + vm1 := virtualMachine("vm1", "h1") + vm2 := virtualMachine("vm2", "h1") + vm3 := virtualMachine("vm3", "t2") + vmUnplaced := virtualMachine("vm-unplaced", "") + + client := fake.NewClientBuilder().WithScheme(scheme). + WithObjects(&resA, &resB, &other, &t1, &t2, &h1, &h2, &h3, &idle, + &vm1, &vm2, &vm3, &vmUnplaced).Build() + + kpi := &FailoverPartitionGraphKPI{} + if err := kpi.Init(nil, client, conf.NewRawOpts("{}")); err != nil { + t.Fatalf("expected no error, got %v", err) + } + + ch := make(chan prometheus.Metric, 32) + kpi.Collect(ch) + close(ch) + + var nodes, edges int + nodeAZ := map[string]string{} + nodeValue := map[string]float64{} + nodeVMs := map[string]string{} + edgeSet := map[string]bool{} + for m := range ch { + var metric dto.Metric + if err := m.Write(&metric); err != nil { + t.Fatalf("failed to write metric: %v", err) + } + labels := map[string]string{} + for _, l := range metric.Label { + labels[l.GetName()] = l.GetValue() + } + switch m.Desc().String() { + case kpi.node.String(): + nodes++ + nodeAZ[labels["host"]] = labels["az"] + nodeValue[labels["host"]] = metric.GetGauge().GetValue() + nodeVMs[labels["host"]] = labels["vms"] + if labels["memory"] != "256Gi" || labels["cpu"] != "64" { + t.Errorf("node %s: unexpected resources memory=%q cpu=%q", labels["host"], labels["memory"], labels["cpu"]) + } + case kpi.edge.String(): + edges++ + edgeSet[labels["source"]+"->"+labels["target"]] = true + default: + t.Errorf("unexpected metric descriptor %s", m.Desc().String()) + } + } + + // Nodes: one per hypervisor, including the idle one. + if nodes != 6 { + t.Errorf("expected 6 node metrics, got %d", nodes) + } + wantAZ := map[string]string{"t1": "az-a", "t2": "az-b", "h1": "az-a", "h2": "az-a", "h3": "az-b", "h-idle": "az-a"} + for host, az := range wantAZ { + if nodeAZ[host] != az { + t.Errorf("node %s: expected az %q, got %q", host, az, nodeAZ[host]) + } + } + // Every hypervisor with a failover path reports 1; the idle one reports 0. + for _, host := range []string{"t1", "t2", "h1", "h2", "h3"} { + if nodeValue[host] != 1 { + t.Errorf("node %s: expected value 1, got %v", host, nodeValue[host]) + } + } + if nodeValue["h-idle"] != 0 { + t.Errorf("node h-idle: expected value 0, got %v", nodeValue["h-idle"]) + } + // The vms label counts placed vms per host: h1 has two, t2 has one, the rest + // zero. The unplaced vm is not counted anywhere. + wantVMs := map[string]string{"h1": "2", "t2": "1", "t1": "0", "h2": "0", "h3": "0", "h-idle": "0"} + for host, want := range wantVMs { + if nodeVMs[host] != want { + t.Errorf("node %s: expected vms %q, got %q", host, want, nodeVMs[host]) + } + } + // Edges: h1->t1, h2->t1, h1->t2, h3->t2. + if edges != 4 { + t.Errorf("expected 4 edge metrics, got %d", edges) + } + for _, want := range []string{"h1->t1", "h2->t1", "h1->t2", "h3->t2"} { + if !edgeSet[want] { + t.Errorf("missing edge %s, got %v", want, edgeSet) + } + } +} diff --git a/internal/knowledge/kpis/plugins/compute/host_contention.go b/internal/knowledge/kpis/plugins/compute/host_contention.go index ae633bbac..2bc732db6 100644 --- a/internal/knowledge/kpis/plugins/compute/host_contention.go +++ b/internal/knowledge/kpis/plugins/compute/host_contention.go @@ -91,3 +91,7 @@ func (k *VMwareHostContentionKPI) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstHistogram(k.hostCPUContentionAvg, counts[key], sums[key], hist) } } + +func init() { + plugins.Index["vmware_host_contention_kpi"] = func() plugins.KPI { return &VMwareHostContentionKPI{} } +} diff --git a/internal/knowledge/kpis/plugins/compute/project_noisiness.go b/internal/knowledge/kpis/plugins/compute/project_noisiness.go index d58708966..5386c7281 100644 --- a/internal/knowledge/kpis/plugins/compute/project_noisiness.go +++ b/internal/knowledge/kpis/plugins/compute/project_noisiness.go @@ -71,3 +71,7 @@ func (k *VMwareProjectNoisinessKPI) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstHistogram(k.projectNoisinessDesc, counts[key], sums[key], hist) } } + +func init() { + plugins.Index["vmware_project_noisiness_kpi"] = func() plugins.KPI { return &VMwareProjectNoisinessKPI{} } +} diff --git a/internal/knowledge/kpis/plugins/compute/vm_commitments.go b/internal/knowledge/kpis/plugins/compute/vm_commitments.go index d84868298..d0ca26cbf 100644 --- a/internal/knowledge/kpis/plugins/compute/vm_commitments.go +++ b/internal/knowledge/kpis/plugins/compute/vm_commitments.go @@ -213,3 +213,7 @@ func (k *VMCommitmentsKPI) Collect(ch chan<- prometheus.Metric) { ) } } + +func init() { + plugins.Index["vm_commitments_kpi"] = func() plugins.KPI { return &VMCommitmentsKPI{} } +} diff --git a/internal/knowledge/kpis/plugins/compute/vm_faults.go b/internal/knowledge/kpis/plugins/compute/vm_faults.go index daa0a73c7..82308ac9d 100644 --- a/internal/knowledge/kpis/plugins/compute/vm_faults.go +++ b/internal/knowledge/kpis/plugins/compute/vm_faults.go @@ -149,3 +149,7 @@ func (k *VMFaultsKPI) Collect(ch chan<- prometheus.Metric) { } vmFaultsKPIlogger.V(1).Info("collected metrics", "nMetrics", len(counts)) } + +func init() { + plugins.Index["vm_faults_kpi"] = func() plugins.KPI { return &VMFaultsKPI{} } +} diff --git a/internal/knowledge/kpis/plugins/compute/vm_life_span.go b/internal/knowledge/kpis/plugins/compute/vm_life_span.go index 3524140ba..897b6fcc8 100644 --- a/internal/knowledge/kpis/plugins/compute/vm_life_span.go +++ b/internal/knowledge/kpis/plugins/compute/vm_life_span.go @@ -104,3 +104,7 @@ func (k *VMLifeSpanKPI) collectVMBuckets(ch chan<- prometheus.Metric, deleted bo ) } } + +func init() { + plugins.Index["vm_life_span_kpi"] = func() plugins.KPI { return &VMLifeSpanKPI{} } +} diff --git a/internal/knowledge/kpis/plugins/compute/vm_migration_statistics.go b/internal/knowledge/kpis/plugins/compute/vm_migration_statistics.go index ac2e66257..a4bfdf431 100644 --- a/internal/knowledge/kpis/plugins/compute/vm_migration_statistics.go +++ b/internal/knowledge/kpis/plugins/compute/vm_migration_statistics.go @@ -88,3 +88,7 @@ func (k *VMMigrationStatisticsKPI) Collect(ch chan<- prometheus.Metric) { ) } } + +func init() { + plugins.Index["vm_migration_statistics_kpi"] = func() plugins.KPI { return &VMMigrationStatisticsKPI{} } +} diff --git a/internal/knowledge/kpis/plugins/deployment/datasource_state.go b/internal/knowledge/kpis/plugins/deployment/datasource_state.go index b878f4d8c..330392535 100644 --- a/internal/knowledge/kpis/plugins/deployment/datasource_state.go +++ b/internal/knowledge/kpis/plugins/deployment/datasource_state.go @@ -123,3 +123,7 @@ func (k *DatasourceStateKPI) Collect(ch chan<- prometheus.Metric) { } datasourceStateKPILogger.V(1).Info("Collected datasource state metrics", "count", len(datasources)) } + +func init() { + plugins.Index["datasource_state_kpi"] = func() plugins.KPI { return &DatasourceStateKPI{} } +} diff --git a/internal/knowledge/kpis/plugins/deployment/decision_state.go b/internal/knowledge/kpis/plugins/deployment/decision_state.go index f11e1f9a4..1d07ac7f9 100644 --- a/internal/knowledge/kpis/plugins/deployment/decision_state.go +++ b/internal/knowledge/kpis/plugins/deployment/decision_state.go @@ -91,3 +91,7 @@ func (k *DecisionStateKPI) Collect(ch chan<- prometheus.Metric) { string(k.Options.DecisionSchedulingDomain), "success", ) } + +func init() { + plugins.Index["decision_state_kpi"] = func() plugins.KPI { return &DecisionStateKPI{} } +} diff --git a/internal/knowledge/kpis/plugins/deployment/knowledge_state.go b/internal/knowledge/kpis/plugins/deployment/knowledge_state.go index c21ec72f7..353bc1ea3 100644 --- a/internal/knowledge/kpis/plugins/deployment/knowledge_state.go +++ b/internal/knowledge/kpis/plugins/deployment/knowledge_state.go @@ -77,3 +77,7 @@ func (k *KnowledgeStateKPI) Collect(ch chan<- prometheus.Metric) { ) } } + +func init() { + plugins.Index["knowledge_state_kpi"] = func() plugins.KPI { return &KnowledgeStateKPI{} } +} diff --git a/internal/knowledge/kpis/plugins/deployment/kpi_state.go b/internal/knowledge/kpis/plugins/deployment/kpi_state.go index 5dc114ed8..bb0fe603c 100644 --- a/internal/knowledge/kpis/plugins/deployment/kpi_state.go +++ b/internal/knowledge/kpis/plugins/deployment/kpi_state.go @@ -77,3 +77,7 @@ func (k *KPIStateKPI) Collect(ch chan<- prometheus.Metric) { ) } } + +func init() { + plugins.Index["kpi_state_kpi"] = func() plugins.KPI { return &KPIStateKPI{} } +} diff --git a/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi.go b/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi.go index 0419f27dd..d31a66e13 100644 --- a/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi.go +++ b/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi.go @@ -193,3 +193,7 @@ func labelValueForSnakeKey(snakeKey string, labels map[string]string) string { } return "" } + +func init() { + plugins.Index["multicluster_object_count_kpi"] = func() plugins.KPI { return &MulticlusterObjectCountKPI{} } +} diff --git a/internal/knowledge/kpis/plugins/deployment/pipeline_state.go b/internal/knowledge/kpis/plugins/deployment/pipeline_state.go index 9a9203911..d9a042a6a 100644 --- a/internal/knowledge/kpis/plugins/deployment/pipeline_state.go +++ b/internal/knowledge/kpis/plugins/deployment/pipeline_state.go @@ -77,3 +77,7 @@ func (k *PipelineStateKPI) Collect(ch chan<- prometheus.Metric) { ) } } + +func init() { + plugins.Index["pipeline_state_kpi"] = func() plugins.KPI { return &PipelineStateKPI{} } +} diff --git a/internal/knowledge/kpis/plugins/deployment/reservation_state.go b/internal/knowledge/kpis/plugins/deployment/reservation_state.go index ba6963d53..1f1eefe99 100644 --- a/internal/knowledge/kpis/plugins/deployment/reservation_state.go +++ b/internal/knowledge/kpis/plugins/deployment/reservation_state.go @@ -5,6 +5,7 @@ package deployment import ( "context" + "strings" "time" "github.com/cobaltcore-dev/cortex/api/v1alpha1" @@ -33,12 +34,18 @@ type ReservationStateKPIOpts struct { // carries the condition's Reason so that alerts can target specific failure // modes (e.g. reason="InstanceNotFound" for in-flight reservations whose VM // has not spawned on any hypervisor yet). +// +// All metrics are filtered to the configured ReservationSchedulingDomain. type ReservationStateKPI struct { // Common base for all KPIs that provides standard functionality. plugins.BaseKPI[ReservationStateKPIOpts] // Prometheus descriptor for the reservation state metric. counter *prometheus.Desc + // Prometheus descriptor for the reservation count metric. + reservationCount *prometheus.Desc + // Prometheus descriptor for the reserved resources metric. + allocatedResources *prometheus.Desc } func (ReservationStateKPI) GetName() string { return "reservation_state_kpi" } @@ -54,11 +61,27 @@ func (k *ReservationStateKPI) Init(db *db.DB, client client.Client, opts conf.Ra []string{"domain", "type", "state", "reason"}, nil, ) + k.reservationCount = prometheus.NewDesc( + "cortex_reservation_count", + "Number of reservations by type, readiness and error status.", + []string{"domain", "type", "status_ready", "status_error"}, + nil, + ) + k.allocatedResources = prometheus.NewDesc( + "cortex_reservation_allocated_resources", + "Resource units allocated across reservations, by type, host and resource type.", + []string{"domain", "type", "status_ready", "status_error", "host", "resource"}, + nil, + ) return nil } -// Conform to the prometheus collector interface by providing the descriptor. -func (k *ReservationStateKPI) Describe(ch chan<- *prometheus.Desc) { ch <- k.counter } +// Conform to the prometheus collector interface by providing the descriptors. +func (k *ReservationStateKPI) Describe(ch chan<- *prometheus.Desc) { + ch <- k.counter + ch <- k.reservationCount + ch <- k.allocatedResources +} // Collect the reservation state metrics. func (k *ReservationStateKPI) Collect(ch chan<- prometheus.Metric) { @@ -74,44 +97,92 @@ func (k *ReservationStateKPI) Collect(ch chan<- prometheus.Metric) { reservationStateKPILogger.Error(err, "Failed to list reservations") return } - // Aggregate counts by (type, state, reason) so that we emit one time - // series per bucket rather than one per reservation. This keeps metric - // cardinality bounded regardless of how many reservations exist. - type bucket struct { + domain := string(k.Options.ReservationSchedulingDomain) + + // Aggregate counts by (type, state, reason) for cortex_reservation_state, by + // (type, ready, error) for cortex_reservation_count, and by (type, ready, error, + // host, resource) for cortex_reservation_allocated_resources. Aggregating keeps + // metric cardinality bounded regardless of how many reservations exist. + type stateBucket struct { reservationType string state string reason string } - counts := map[bucket]float64{} + type countBucket struct { + reservationType string + statusReady string + statusError string + } + type resourceBucket struct { + reservationType string + statusReady string + statusError string + host string + resource string + } + stateCounts := map[stateBucket]float64{} + reservationCounts := map[countBucket]float64{} + allocatedResources := map[resourceBucket]uint64{} for _, r := range reservationList.Items { if r.Spec.SchedulingDomain != k.Options.ReservationSchedulingDomain { continue } + reservationType := string(r.Spec.Type) + + // Condition-derived labels. "state" mirrors the ready condition for + // cortex_reservation_state; "statusReady"/"statusError" carry the raw + // condition status and message for the ported count/resource metrics. state := "unknown" reason := "" + statusReady := "Unknown" + statusError := "" cond := meta.FindStatusCondition(r.Status.Conditions, v1alpha1.ReservationConditionReady) if cond != nil { reason = cond.Reason + statusReady = string(cond.Status) switch cond.Status { case metav1.ConditionTrue: state = "ready" case metav1.ConditionFalse: state = "error" + statusError = strings.ReplaceAll(cond.Message, ",", ";") default: state = "unknown" } } - counts[bucket{ - reservationType: string(r.Spec.Type), - state: state, - reason: reason, - }]++ + + stateCounts[stateBucket{reservationType: reservationType, state: state, reason: reason}]++ + reservationCounts[countBucket{reservationType: reservationType, statusReady: statusReady, statusError: statusError}]++ + for resourceName, resourceQuantity := range r.Spec.Resources { + allocatedResources[resourceBucket{ + reservationType: reservationType, + statusReady: statusReady, + statusError: statusError, + host: r.Status.Host, + resource: string(resourceName), + }] += resourceQuantity.AsDec().UnscaledBig().Uint64() + } } - for b, v := range counts { + for b, v := range stateCounts { ch <- prometheus.MustNewConstMetric( k.counter, prometheus.GaugeValue, v, - string(k.Options.ReservationSchedulingDomain), - b.reservationType, b.state, b.reason, + domain, b.reservationType, b.state, b.reason, + ) + } + for b, v := range reservationCounts { + ch <- prometheus.MustNewConstMetric( + k.reservationCount, prometheus.GaugeValue, v, + domain, b.reservationType, b.statusReady, b.statusError, ) } + for b, v := range allocatedResources { + ch <- prometheus.MustNewConstMetric( + k.allocatedResources, prometheus.GaugeValue, float64(v), + domain, b.reservationType, b.statusReady, b.statusError, b.host, b.resource, + ) + } +} + +func init() { + plugins.Index["reservation_state_kpi"] = func() plugins.KPI { return &ReservationStateKPI{} } } diff --git a/internal/knowledge/kpis/plugins/deployment/reservation_state_test.go b/internal/knowledge/kpis/plugins/deployment/reservation_state_test.go index 19f543ad9..0b7770178 100644 --- a/internal/knowledge/kpis/plugins/deployment/reservation_state_test.go +++ b/internal/knowledge/kpis/plugins/deployment/reservation_state_test.go @@ -4,12 +4,15 @@ package deployment import ( + "strings" "testing" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/pkg/conf" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" + "k8s.io/apimachinery/pkg/api/resource" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -34,15 +37,15 @@ func TestReservationStateKPI_Describe(t *testing.T) { if err := kpi.Init(nil, nil, conf.NewRawOpts(`{"reservationSchedulingDomain": "nova"}`)); err != nil { t.Fatalf("expected no error, got %v", err) } - ch := make(chan *prometheus.Desc, 1) + ch := make(chan *prometheus.Desc, 3) kpi.Describe(ch) close(ch) descCount := 0 for range ch { descCount++ } - if descCount != 1 { - t.Errorf("expected 1 descriptor, got %d", descCount) + if descCount != 3 { + t.Errorf("expected 3 descriptors, got %d", descCount) } } @@ -293,9 +296,15 @@ func TestReservationStateKPI_Collect(t *testing.T) { kpi.Collect(ch) close(ch) + // Count only cortex_reservation_state series; the expectedCount values + // describe that metric's (type,state,reason) buckets. The ported + // cortex_reservation_count / _allocated_resources series are asserted + // separately in TestReservationStateKPI_CollectPortedMetrics. metricsCount := 0 - for range ch { - metricsCount++ + for m := range ch { + if strings.Contains(m.Desc().String(), "cortex_reservation_state") { + metricsCount++ + } } if metricsCount != tt.expectedCount { t.Errorf("%s: expected %d metrics, got %d", tt.description, tt.expectedCount, metricsCount) @@ -346,6 +355,11 @@ func TestReservationStateKPI_CollectLabels(t *testing.T) { found := false for m := range ch { + // Scope to cortex_reservation_state; cortex_reservation_count carries a + // type label too but no state/reason. + if !strings.Contains(m.Desc().String(), "cortex_reservation_state") { + continue + } var metric dto.Metric if err := m.Write(&metric); err != nil { t.Fatalf("failed to write metric: %v", err) @@ -375,3 +389,99 @@ func TestReservationStateKPI_CollectLabels(t *testing.T) { t.Fatal("expected an in-flight reservation metric to be collected") } } + +// TestReservationStateKPI_CollectPortedMetrics verifies the two metrics ported +// from the former reservations monitor: cortex_reservation_count (by type) and +// cortex_reservation_allocated_resources (by type, host, resource). Both are +// domain-filtered like the rest of the KPI. +func TestReservationStateKPI_CollectPortedMetrics(t *testing.T) { + scheme, err := v1alpha1.SchemeBuilder.Build() + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + // One ready committed-resource reservation on host-1 with resources, plus a + // reservation in a different domain that must be excluded from every metric. + res := &v1alpha1.Reservation{ + ObjectMeta: v1.ObjectMeta{Name: "r-cr"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + SchedulingDomain: "nova", + Resources: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: *resource.NewQuantity(16, resource.DecimalSI), + hv1.ResourceCPU: *resource.NewQuantity(4, resource.DecimalSI), + }, + }, + Status: v1alpha1.ReservationStatus{ + Host: "host-1", + Conditions: []v1.Condition{ + {Type: v1alpha1.ReservationConditionReady, Status: v1.ConditionTrue, Reason: "ReservationReady"}, + }, + }, + } + resOtherDomain := &v1alpha1.Reservation{ + ObjectMeta: v1.ObjectMeta{Name: "r-other"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + SchedulingDomain: "ironcore", + Resources: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: *resource.NewQuantity(99, resource.DecimalSI), + }, + }, + Status: v1alpha1.ReservationStatus{Host: "host-9"}, + } + + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(res, resOtherDomain).Build() + + kpi := &ReservationStateKPI{} + if err := kpi.Init(nil, client, conf.NewRawOpts(`{"reservationSchedulingDomain": "nova"}`)); err != nil { + t.Fatalf("expected no error, got %v", err) + } + ch := make(chan prometheus.Metric, 16) + kpi.Collect(ch) + close(ch) + + var countSeries, resourceSeries int + resourceByName := map[string]float64{} + for m := range ch { + desc := m.Desc().String() + var metric dto.Metric + if err := m.Write(&metric); err != nil { + t.Fatalf("failed to write metric: %v", err) + } + labels := map[string]string{} + for _, l := range metric.Label { + labels[l.GetName()] = l.GetValue() + } + // No series from the excluded ironcore reservation should appear. + if labels["host"] == "host-9" || labels["domain"] == "ironcore" { + t.Errorf("cross-domain reservation leaked into metric: %v", labels) + } + switch { + case strings.Contains(desc, "cortex_reservation_count"): + countSeries++ + if labels["type"] != string(v1alpha1.ReservationTypeCommittedResource) { + t.Errorf("expected type label on count metric, got %q", labels["type"]) + } + case strings.Contains(desc, "cortex_reservation_allocated_resources"): + resourceSeries++ + if labels["host"] != "host-1" { + t.Errorf("expected host=host-1, got %q", labels["host"]) + } + resourceByName[labels["resource"]] = metric.Gauge.GetValue() + } + } + + if countSeries != 1 { + t.Errorf("expected 1 count series, got %d", countSeries) + } + if resourceSeries != 2 { + t.Errorf("expected 2 resource series (memory, cpu), got %d", resourceSeries) + } + if got := resourceByName[string(hv1.ResourceMemory)]; got != 16 { + t.Errorf("expected memory=16, got %v", got) + } + if got := resourceByName[string(hv1.ResourceCPU)]; got != 4 { + t.Errorf("expected cpu=4, got %v", got) + } +} diff --git a/internal/knowledge/kpis/plugins/infrastructure/kvm_hana_stacking.go b/internal/knowledge/kpis/plugins/infrastructure/kvm_hana_stacking.go index 480685697..03d051b77 100644 --- a/internal/knowledge/kpis/plugins/infrastructure/kvm_hana_stacking.go +++ b/internal/knowledge/kpis/plugins/infrastructure/kvm_hana_stacking.go @@ -117,3 +117,7 @@ func (k *KVMHanaStackingKPI) queryHanaStacking() ([]kvmHanaStackingRow, error) { } return rows, nil } + +func init() { + plugins.Index["kvm_hana_stacking_kpi"] = func() plugins.KPI { return &KVMHanaStackingKPI{} } +} diff --git a/internal/knowledge/kpis/plugins/infrastructure/kvm_host_capacity.go b/internal/knowledge/kpis/plugins/infrastructure/kvm_host_capacity.go index 670f0bfb6..da7c2b824 100644 --- a/internal/knowledge/kpis/plugins/infrastructure/kvm_host_capacity.go +++ b/internal/knowledge/kpis/plugins/infrastructure/kvm_host_capacity.go @@ -215,3 +215,7 @@ func (k *KVMHostCapacityKPI) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric(k.capacityPerHost, prometheus.GaugeValue, paygRAM.AsApproximateFloat64(), append(labels, "ram", "available")...) } } + +func init() { + plugins.Index["kvm_host_capacity_kpi"] = func() plugins.KPI { return &KVMHostCapacityKPI{} } +} diff --git a/internal/knowledge/kpis/plugins/infrastructure/kvm_project_utilization.go b/internal/knowledge/kpis/plugins/infrastructure/kvm_project_utilization.go index 605dcc03d..7310b3971 100644 --- a/internal/knowledge/kpis/plugins/infrastructure/kvm_project_utilization.go +++ b/internal/knowledge/kpis/plugins/infrastructure/kvm_project_utilization.go @@ -201,3 +201,7 @@ func (k *KVMProjectUtilizationKPI) queryProjectInstanceCount() ([]kvmProjectInst } return usages, nil } + +func init() { + plugins.Index["kvm_project_utilization_kpi"] = func() plugins.KPI { return &KVMProjectUtilizationKPI{} } +} diff --git a/internal/knowledge/kpis/plugins/infrastructure/vmware_host_capacity.go b/internal/knowledge/kpis/plugins/infrastructure/vmware_host_capacity.go index a98437408..7cd5114e9 100644 --- a/internal/knowledge/kpis/plugins/infrastructure/vmware_host_capacity.go +++ b/internal/knowledge/kpis/plugins/infrastructure/vmware_host_capacity.go @@ -113,3 +113,7 @@ func (k *VMwareHostCapacityKPI) getHostUtilizations() (map[string]compute.HostUt } return m, nil } + +func init() { + plugins.Index["vmware_host_capacity_kpi"] = func() plugins.KPI { return &VMwareHostCapacityKPI{} } +} diff --git a/internal/knowledge/kpis/plugins/infrastructure/vmware_project_commitments.go b/internal/knowledge/kpis/plugins/infrastructure/vmware_project_commitments.go index d93c8a586..78fbc05d0 100644 --- a/internal/knowledge/kpis/plugins/infrastructure/vmware_project_commitments.go +++ b/internal/knowledge/kpis/plugins/infrastructure/vmware_project_commitments.go @@ -301,3 +301,7 @@ func (k *VMwareProjectCommitmentsKPI) getProjectsWithDomains() (map[string]proje } return projectMap, nil } + +func init() { + plugins.Index["vmware_project_commitments_kpi"] = func() plugins.KPI { return &VMwareProjectCommitmentsKPI{} } +} diff --git a/internal/knowledge/kpis/plugins/infrastructure/vmware_project_utilization.go b/internal/knowledge/kpis/plugins/infrastructure/vmware_project_utilization.go index 68cd213cf..18c09a116 100644 --- a/internal/knowledge/kpis/plugins/infrastructure/vmware_project_utilization.go +++ b/internal/knowledge/kpis/plugins/infrastructure/vmware_project_utilization.go @@ -210,3 +210,7 @@ func (k *VMwareProjectUtilizationKPI) queryProjectInstanceCount() ([]vmwareProje } return usages, nil } + +func init() { + plugins.Index["vmware_project_utilization_kpi"] = func() plugins.KPI { return &VMwareProjectUtilizationKPI{} } +} diff --git a/internal/knowledge/kpis/plugins/storage/storage_pool_cpu.go b/internal/knowledge/kpis/plugins/storage/storage_pool_cpu.go index 9502349fa..30dd527b9 100644 --- a/internal/knowledge/kpis/plugins/storage/storage_pool_cpu.go +++ b/internal/knowledge/kpis/plugins/storage/storage_pool_cpu.go @@ -91,3 +91,7 @@ func (k *NetAppStoragePoolCPUUsageKPI) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstHistogram(k.storagePoolCPUUsageAvg, counts[key], sums[key], hist) } } + +func init() { + plugins.Index["netapp_storage_pool_cpu_usage_kpi"] = func() plugins.KPI { return &NetAppStoragePoolCPUUsageKPI{} } +} diff --git a/internal/knowledge/kpis/plugins/zz_index.go b/internal/knowledge/kpis/plugins/zz_index.go new file mode 100644 index 000000000..d003390ff --- /dev/null +++ b/internal/knowledge/kpis/plugins/zz_index.go @@ -0,0 +1,8 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package plugins + +// Index of all supported kpis, keyed by name. +// Each kpi registers itself here through a func init() in its own file. +var Index = map[string]func() KPI{} diff --git a/internal/knowledge/kpis/supported_kpis.go b/internal/knowledge/kpis/supported_kpis.go deleted file mode 100644 index 617d58008..000000000 --- a/internal/knowledge/kpis/supported_kpis.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package kpis - -import ( - "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins" - "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins/compute" - "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins/deployment" - "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins/infrastructure" - "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins/storage" -) - -// Configuration of supported kpis. -var supportedKPIs = map[string]plugins.KPI{ - "vmware_host_contention_kpi": &compute.VMwareHostContentionKPI{}, - "vmware_project_noisiness_kpi": &compute.VMwareProjectNoisinessKPI{}, - "vm_migration_statistics_kpi": &compute.VMMigrationStatisticsKPI{}, - "vm_life_span_kpi": &compute.VMLifeSpanKPI{}, - "vm_commitments_kpi": &compute.VMCommitmentsKPI{}, - "vm_faults_kpi": &compute.VMFaultsKPI{}, - - "kvm_host_capacity_kpi": &infrastructure.KVMHostCapacityKPI{}, - "kvm_project_utilization_kpi": &infrastructure.KVMProjectUtilizationKPI{}, - "kvm_hana_stacking_kpi": &infrastructure.KVMHanaStackingKPI{}, - "vmware_project_utilization_kpi": &infrastructure.VMwareProjectUtilizationKPI{}, - "vmware_project_commitments_kpi": &infrastructure.VMwareProjectCommitmentsKPI{}, - "vmware_host_capacity_kpi": &infrastructure.VMwareHostCapacityKPI{}, - - "netapp_storage_pool_cpu_usage_kpi": &storage.NetAppStoragePoolCPUUsageKPI{}, - - "datasource_state_kpi": &deployment.DatasourceStateKPI{}, - "knowledge_state_kpi": &deployment.KnowledgeStateKPI{}, - "decision_state_kpi": &deployment.DecisionStateKPI{}, - "kpi_state_kpi": &deployment.KPIStateKPI{}, - "pipeline_state_kpi": &deployment.PipelineStateKPI{}, - "reservation_state_kpi": &deployment.ReservationStateKPI{}, - - "multicluster_object_count_kpi": &deployment.MulticlusterObjectCountKPI{}, -} diff --git a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go index 86d56e405..09bc67fa5 100644 --- a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go +++ b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go @@ -24,7 +24,8 @@ type FilterQuotaEnforcementOpts struct { // metrics for would-be rejects, but never actually removes hosts from the // result. Set to true to globally reject requests without headroom. The // shadow default is intentional so that operators can observe - // cortex_nova_filter_quota_enforcement_decisions_total before opting in to + // cortex_filter_weigher_pipeline_step_events_total{event="quota_enforcement_decision"} + // before opting in to // enforcement, and so that newly-rolled-out pipelines never silently start // rejecting requests. Enforce bool `json:"enforce,omitempty"` @@ -63,15 +64,34 @@ func (FilterQuotaEnforcementOpts) Validate() error { return nil } // - Enforce=true: on a no-headroom decision the filter clears all host // activations to globally reject the request. // -// Every accept/reject/skip outcome is recorded as a Prometheus counter: -// cortex_nova_filter_quota_enforcement_decisions_total{mode,decision,resource, -// availability_zone,flavor_group}. +// Every accept/reject/skip outcome is reported as a named pipeline event +// ("quota_enforcement_decision") and exported by the pipeline step monitor as +// cortex_filter_weigher_pipeline_step_events_total{pipeline,step,event,mode, +// decision,resource,availability_zone,flavor_group}. // // Disabled by default; activated by adding "filter_quota_enforcement" to the pipeline config. type FilterQuotaEnforcement struct { lib.BaseFilter[api.ExternalSchedulerRequest, FilterQuotaEnforcementOpts] } +// recordDecision appends an accept/reject/skip outcome as a named pipeline +// event. The pipeline step monitor exports these as +// cortex_filter_weigher_pipeline_step_events_total, labeled by pipeline, step, +// event and the labels attached here, so operators can observe quota decisions +// without a global metrics singleton. +func recordDecision(result *lib.FilterWeigherPipelineStepResult, mode, decision, resource, az, flavorGroup string) { + result.Events = append(result.Events, lib.FilterWeigherPipelineStepEvent{ + Name: "quota_enforcement_decision", + Labels: map[string]string{ + "mode": mode, + "decision": decision, + "resource": resource, + "availability_zone": az, + "flavor_group": flavorGroup, + }, + }) +} + func (s *FilterQuotaEnforcement) Run(traceLog logr.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) @@ -90,19 +110,19 @@ func (s *FilterQuotaEnforcement) Run(traceLog logr.Logger, request api.ExternalS switch intent { case api.EvacuateIntent, api.LiveMigrationIntent: traceLog.V(1).Info("skipping quota enforcement for non-consuming intent", "intent", intent) - QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped_intent_migration", "", "", "") + recordDecision(result, mode, "accept_skipped_intent_migration", "", "", "") return result, nil case api.ReserveForFailoverIntent, api.ReuseFailoverReservationIntent: traceLog.V(1).Info("skipping quota enforcement for failover reservation intent") - QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped_intent_internal", "", "", "") + recordDecision(result, mode, "accept_skipped_intent_internal", "", "", "") return result, nil case api.ReserveForCommittedResourceIntent: traceLog.V(1).Info("skipping quota enforcement for committed resource reservation intent") - QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped_intent_internal", "", "", "") + recordDecision(result, mode, "accept_skipped_intent_internal", "", "", "") return result, nil case api.CapacityProbeIntent: traceLog.V(1).Info("skipping quota enforcement for capacity probe intent") - QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped", "", "", "") + recordDecision(result, mode, "accept_skipped", "", "", "") return result, nil } } @@ -116,17 +136,17 @@ func (s *FilterQuotaEnforcement) Run(traceLog logr.Logger, request api.ExternalS if projectID == "" { traceLog.V(1).Info("no project ID in request, skipping quota enforcement") - QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped_missing_data", "", az, hwVersion) + recordDecision(result, mode, "accept_skipped_missing_data", "", az, hwVersion) return result, nil } if az == "" { traceLog.V(1).Info("no availability zone in request, skipping quota enforcement") - QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped_missing_data", "", "", hwVersion) + recordDecision(result, mode, "accept_skipped_missing_data", "", "", hwVersion) return result, nil } if hwVersion == "" { traceLog.V(1).Info("no hw_version in flavor extra specs, skipping quota enforcement") - QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped_missing_data", "", az, "") + recordDecision(result, mode, "accept_skipped_missing_data", "", az, "") return result, nil } @@ -231,7 +251,7 @@ func (s *FilterQuotaEnforcement) Run(traceLog logr.Logger, request api.ExternalS if crHasHeadroom { traceLog.V(1).Info("quota enforcement ACCEPT: CR headroom available") - QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_cr", "ram", az, hwVersion) + recordDecision(result, mode, "accept_cr", "ram", az, hwVersion) return result, nil } @@ -244,7 +264,7 @@ func (s *FilterQuotaEnforcement) Run(traceLog logr.Logger, request api.ExternalS if apierrors.IsNotFound(err) { traceLog.V(1).Info("no ProjectQuota CRD found for project+AZ, skipping enforcement", "projectID", projectID, "az", az) - QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_no_quota", "", az, hwVersion) + recordDecision(result, mode, "accept_no_quota", "", az, hwVersion) return result, nil } traceLog.Error(err, "failed to get ProjectQuota", "name", pqName) @@ -293,7 +313,7 @@ func (s *FilterQuotaEnforcement) Run(traceLog logr.Logger, request api.ExternalS ) if headroom < check.request { - QuotaEnforcementMetricsSingleton.RecordDecision(mode, "reject", check.label, az, hwVersion) + recordDecision(result, mode, "reject", check.label, az, hwVersion) if !s.Options.Enforce { traceLog.V(1).Info("quota enforcement SHADOW: would reject but enforce=false", "projectID", projectID, @@ -321,7 +341,7 @@ func (s *FilterQuotaEnforcement) Run(traceLog logr.Logger, request api.ExternalS } traceLog.V(1).Info("quota enforcement ACCEPT: PAYG headroom available") - QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_payg", "", az, hwVersion) + recordDecision(result, mode, "accept_payg", "", az, hwVersion) return result, nil } diff --git a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_metrics.go b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_metrics.go deleted file mode 100644 index 3586fae75..000000000 --- a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_metrics.go +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package filters - -import ( - "sync" - - "github.com/prometheus/client_golang/prometheus" - logf "sigs.k8s.io/controller-runtime/pkg/log" -) - -// log is the package-level logger for non-trace (global) log messages emitted -// by the nova filters package. -var log = logf.Log.WithName("filters") - -// QuotaEnforcementMetrics holds Prometheus metrics for the quota-enforcement -// nova filter. The filter runs in either "enforce" mode (rejects when no -// headroom) or "shadow" mode (logs/counts what it would have rejected without -// removing hosts). The decisions counter exposes the same outcomes in both -// modes so operators can compare a shadow rollout against the existing -// generic step metrics (e.g. cortex_filter_weigher_pipeline_step_removed_hosts). -type QuotaEnforcementMetrics struct { - // Decisions is the counter of every accept/reject/skip outcome the filter - // produces. Labels: - // - mode: "enforce" | "shadow" - // - decision: "accept_cr" | "accept_payg" | "accept_no_quota" | - // "accept_skipped_intent_migration" | - // "accept_skipped_intent_internal" | - // "accept_skipped_missing_data" | - // "reject" - // - resource: "ram" | "cores" | "instances" | "" (empty when - // the decision is not driven by a single resource) - // - availability_zone: AZ string or "" if unknown at decision time - // - flavor_group: hw_version string or "" if unknown at decision time - Decisions *prometheus.CounterVec -} - -// NewQuotaEnforcementMetrics constructs the metrics struct. The returned -// *QuotaEnforcementMetrics implements prometheus.Collector, so the caller is -// expected to register it with a registry (typically metrics.Registry from -// cmd/manager). This matches the Monitor pattern used elsewhere in cortex -// (e.g. db.Monitor, LogMetricsMonitor, PipelineMonitor). -func NewQuotaEnforcementMetrics() *QuotaEnforcementMetrics { - return &QuotaEnforcementMetrics{ - Decisions: prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cortex_nova_filter_quota_enforcement_decisions_total", - Help: "Decisions made by the FilterQuotaEnforcement nova filter, " + - "labeled by enforcement mode, decision outcome, the resource " + - "that drove a reject (if any), availability zone, and flavor group.", - }, - []string{"mode", "decision", "resource", "availability_zone", "flavor_group"}, - ), - } -} - -// Describe implements prometheus.Collector by delegating to the underlying -// counter vec. Nil-safe to mirror RecordDecision's defensive guard: a stray -// MustRegister on an uninitialized struct must not crash the manager. -func (m *QuotaEnforcementMetrics) Describe(ch chan<- *prometheus.Desc) { - if m == nil || m.Decisions == nil { - return - } - m.Decisions.Describe(ch) -} - -// Collect implements prometheus.Collector by delegating to the underlying -// counter vec. Nil-safe — Collect runs on every Prometheus scrape, so a nil -// receiver must not panic the metrics endpoint. -func (m *QuotaEnforcementMetrics) Collect(ch chan<- prometheus.Metric) { - if m == nil || m.Decisions == nil { - return - } - m.Decisions.Collect(ch) -} - -// recordDecisionNilOnce ensures the "metrics not initialized" warning is -// emitted at most once per process to keep logs clean while still surfacing -// misconfigurations. A pointer is used so tests can swap in a freshly armed -// once without copying a sync.Once value (which is forbidden after first use). -var recordDecisionNilOnce = &sync.Once{} - -// RecordDecision is a small helper that nil-guards the singleton and increments -// the decisions counter. Filter Run() uses it directly. If the receiver is nil -// (i.e. QuotaEnforcementMetricsSingleton was never initialized — typically a -// missing wiring step in cmd/manager) we log a warn-level message exactly once -// so the misconfiguration is observable both in logs and via -// cortex_log_messages_total{level="warn"}. -func (m *QuotaEnforcementMetrics) RecordDecision(mode, decision, resource, az, flavorGroup string) { - if m == nil || m.Decisions == nil { - recordDecisionNilOnce.Do(func() { - log.V(1).Info("QuotaEnforcementMetrics is nil; decision metric not recorded "+ - "(is QuotaEnforcementMetricsSingleton initialized in cmd/manager?)", - "mode", mode, - "decision", decision, - "resource", resource, - "availability_zone", az, - "flavor_group", flavorGroup, - ) - }) - return - } - m.Decisions.WithLabelValues(mode, decision, resource, az, flavorGroup).Inc() -} - -// QuotaEnforcementMetricsSingleton is set from cmd/manager/main.go during -// initialization. The filter's Run method reads it via RecordDecision, which -// nil-guards itself, so unit tests that construct a bare FilterQuotaEnforcement -// without setting this singleton stay safe. -var QuotaEnforcementMetricsSingleton *QuotaEnforcementMetrics diff --git a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go index ec1695651..2543c5e20 100644 --- a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go @@ -4,23 +4,18 @@ package filters import ( - "bytes" - "strings" - "sync" + "maps" "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" "github.com/go-logr/logr" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/testutil" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func makeQuotaEnforcementRequest(projectID, az, hwVersion string, memoryMB, numInstances uint64, hints map[string]any) api.ExternalSchedulerRequest { @@ -51,18 +46,6 @@ func makeQuotaEnforcementRequest(projectID, az, hwVersion string, memoryMB, numI } } -// installFreshMetrics swaps in a fresh QuotaEnforcementMetrics for the duration -// of a test and restores the previous singleton afterwards. Each case starts -// from a known-zero state and is fully isolated. -func installFreshMetrics(t *testing.T) *QuotaEnforcementMetrics { - t.Helper() - prev := QuotaEnforcementMetricsSingleton - m := NewQuotaEnforcementMetrics() - QuotaEnforcementMetricsSingleton = m - t.Cleanup(func() { QuotaEnforcementMetricsSingleton = prev }) - return m -} - func TestFilterQuotaEnforcement_Run(t *testing.T) { scheme := runtime.NewScheme() if err := v1alpha1.AddToScheme(scheme); err != nil { @@ -915,8 +898,6 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - m := installFreshMetrics(t) - fakeClient := fake.NewClientBuilder(). WithScheme(scheme). WithObjects(tt.objects...). @@ -943,22 +924,40 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { } } - // Metric assertions: exactly one increment on the expected label set - // and exactly one series in the vector (no stray increments). - got := testutil.ToFloat64(m.Decisions.WithLabelValues( - tt.expectMode, tt.expectDecision, tt.expectResource, tt.expectAZ, tt.expectFG, - )) - if got != 1 { - t.Errorf("expected 1 increment for mode=%q decision=%q resource=%q az=%q flavor_group=%q; got %v", - tt.expectMode, tt.expectDecision, tt.expectResource, tt.expectAZ, tt.expectFG, got) - } - if n := testutil.CollectAndCount(m.Decisions); n != 1 { - t.Errorf("expected exactly 1 metric series in vector, got %d", n) - } + // Event assertions: exactly one quota_enforcement_decision event with + // the expected label set (no stray events). + assertDecisionEvent(t, result.Events, + tt.expectMode, tt.expectDecision, tt.expectResource, tt.expectAZ, tt.expectFG) }) } } +// assertDecisionEvent checks that events contains exactly one +// quota_enforcement_decision event carrying the expected labels. +func assertDecisionEvent(t *testing.T, events []lib.FilterWeigherPipelineStepEvent, mode, decision, resource, az, flavorGroup string) { + t.Helper() + want := map[string]string{ + "mode": mode, + "decision": decision, + "resource": resource, + "availability_zone": az, + "flavor_group": flavorGroup, + } + var matches int + for _, ev := range events { + if ev.Name != "quota_enforcement_decision" { + continue + } + matches++ + if !maps.Equal(ev.Labels, want) { + t.Errorf("event labels = %v, want %v", ev.Labels, want) + } + } + if matches != 1 { + t.Errorf("expected exactly 1 quota_enforcement_decision event, got %d (events: %v)", matches, events) + } +} + // TestFilterQuotaEnforcement_DefaultIsShadow guards the operational invariant // that a freshly-constructed FilterQuotaEnforcement (zero-value Options) runs // in shadow mode. The pipeline yaml intentionally does not set the `enforce` @@ -985,8 +984,6 @@ func TestFilterQuotaEnforcement_DefaultIsShadow(t *testing.T) { }, } - m := installFreshMetrics(t) - fakeClient := fake.NewClientBuilder(). WithScheme(scheme). WithObjects(objects...). @@ -1012,12 +1009,8 @@ func TestFilterQuotaEnforcement_DefaultIsShadow(t *testing.T) { len(request.Hosts), len(result.Activations)) } - // Decision metric must record mode=shadow with a reject decision. - if got := testutil.ToFloat64(m.Decisions.WithLabelValues( - "shadow", "reject", "ram", "az-1", "hana_v2", - )); got != 1 { - t.Errorf("expected exactly 1 increment on shadow-reject metric, got %v", got) - } + // Decision event must record mode=shadow with a reject decision. + assertDecisionEvent(t, result.Events, "shadow", "reject", "ram", "az-1", "hana_v2") } func TestQuantityToGiB(t *testing.T) { @@ -1042,52 +1035,6 @@ func TestQuantityToGiB(t *testing.T) { } } -func TestNewQuotaEnforcementMetrics(t *testing.T) { - m := NewQuotaEnforcementMetrics() - if m == nil || m.Decisions == nil { - t.Fatal("expected non-nil metrics with non-nil Decisions vec") - } - // The constructor no longer registers, so the caller is responsible for - // registering with a registry. Verify *QuotaEnforcementMetrics implements - // prometheus.Collector by registering it. - reg := prometheus.NewRegistry() - reg.MustRegister(m) - // Increment on a label set; verify it lands. - m.RecordDecision("shadow", "reject", "ram", "az-1", "hana_v2") - got := testutil.ToFloat64(m.Decisions.WithLabelValues("shadow", "reject", "ram", "az-1", "hana_v2")) - if got != 1 { - t.Errorf("expected 1 after RecordDecision, got %v", got) - } - // Re-registering the same collector must fail (proves it was registered). - if err := reg.Register(m); err == nil { - t.Error("expected error re-registering already-registered metric") - } -} - -func TestQuotaEnforcementMetrics_RecordDecision_NilWarns(t *testing.T) { - // Capture log output to confirm the nil-receiver path warns exactly once, - // even when called many times. We freshly arm the package-level sync.Once - // so this test deterministically covers the warn path regardless of test - // ordering. - origOnce := recordDecisionNilOnce - recordDecisionNilOnce = &sync.Once{} - t.Cleanup(func() { recordDecisionNilOnce = origOnce }) - - var buf bytes.Buffer - logf.SetLogger(zap.New(zap.WriteTo(&buf), zap.UseDevMode(true))) - - var nilMetrics *QuotaEnforcementMetrics - // Must not panic, and must warn exactly once across multiple calls. - nilMetrics.RecordDecision("enforce", "reject", "ram", "az-1", "hana_v2") - nilMetrics.RecordDecision("enforce", "reject", "ram", "az-1", "hana_v2") - - const msg = "QuotaEnforcementMetrics is nil" - if got := strings.Count(buf.String(), msg); got != 1 { - t.Errorf("expected warn message %q to appear exactly once, got %d; output: %q", - msg, got, buf.String()) - } -} - func TestFilterQuotaEnforcement_SkipsForNonPlacementIntent(t *testing.T) { scheme := runtime.NewScheme() if err := v1alpha1.AddToScheme(scheme); err != nil { diff --git a/internal/scheduling/reservations/capacity_accounting.go b/internal/scheduling/reservations/capacity_accounting.go index 72cd19f1f..f73debff0 100644 --- a/internal/scheduling/reservations/capacity_accounting.go +++ b/internal/scheduling/reservations/capacity_accounting.go @@ -129,7 +129,16 @@ func HostFreeCapacity(hostReservations []v1alpha1.Reservation, hv hv1.Hypervisor // ignoreAllocations is true or when mid-migration (TargetHost != Status.Host). // // FailoverReservations: always block the full Spec.Resources. +// +// FailoverHostReservations: block nothing. FailoverHostReservation is a new feature +// that reserves an entire host, and is intentionally not yet counted as blocked capacity +// by any consumer of this function (capacity filter, capacity controller, oversubscription +// controller). Returning an empty block here keeps all of them consistent in one place. func UnusedReservationCapacity(res *v1alpha1.Reservation, ignoreAllocations bool) map[hv1.ResourceName]resource.Quantity { + if res.Spec.Type == v1alpha1.ReservationTypeFailoverHost { + // TODO: When failover reservations are in use, account for them. + return nil + } if res.Spec.Type == v1alpha1.ReservationTypeCommittedResource && !ignoreAllocations && res.Spec.TargetHost == res.Status.Host && diff --git a/internal/scheduling/reservations/capacity_accounting_test.go b/internal/scheduling/reservations/capacity_accounting_test.go index d86e1091d..140b58f8f 100644 --- a/internal/scheduling/reservations/capacity_accounting_test.go +++ b/internal/scheduling/reservations/capacity_accounting_test.go @@ -39,6 +39,16 @@ func TestUnusedReservationCapacity(t *testing.T) { }, wantMemoryBytes: 480 * 1024 * 1024 * 1024, }, + { + name: "failover host: nothing blocked (not yet counted as reserved)", + res: &v1alpha1.Reservation{ + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeFailoverHost, + Resources: map[hv1.ResourceName]resource.Quantity{hv1.ResourceMemory: gib(480)}, + }, + }, + wantMemoryBytes: 0, + }, { name: "CR no allocations: full slot blocked", res: &v1alpha1.Reservation{ diff --git a/internal/scheduling/reservations/commitments/changes/field_index.go b/internal/scheduling/reservations/commitments/changes/field_index.go new file mode 100644 index 000000000..0d3a96fa0 --- /dev/null +++ b/internal/scheduling/reservations/commitments/changes/field_index.go @@ -0,0 +1,39 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package changes + +import ( + "context" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/pkg/multicluster" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// idxCRCByUpsertedCR indexes CommittedResourceChanges by each CR name in UpsertedCommitments. +// Used by crToChangeRequest to map a CR watch event back to its owning change request. +const idxCRCByUpsertedCR = "status.upsertedCommitments.name" + +// indexCRCByUpsertedCR registers the field index on all clusters that serve CRC objects. +// Called once from SetupWithManager; no deduplication guard needed. +func indexCRCByUpsertedCR(ctx context.Context, mcl *multicluster.Client) error { + return mcl.IndexField(ctx, + &v1alpha1.CommittedResourceChange{}, + &v1alpha1.CommittedResourceChangeList{}, + idxCRCByUpsertedCR, + func(obj client.Object) []string { + tx, ok := obj.(*v1alpha1.CommittedResourceChange) + if !ok { + return nil + } + names := make([]string, 0, len(tx.Status.UpsertedCommitments)) + for _, ref := range tx.Status.UpsertedCommitments { + if ref.Name != "" { + names = append(names, ref.Name) + } + } + return names + }, + ) +} diff --git a/internal/scheduling/reservations/commitments/changes/reconciler.go b/internal/scheduling/reservations/commitments/changes/reconciler.go new file mode 100644 index 000000000..73e244a99 --- /dev/null +++ b/internal/scheduling/reservations/commitments/changes/reconciler.go @@ -0,0 +1,342 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package changes + +import ( + "context" + "fmt" + "time" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/commitments" + "github.com/cobaltcore-dev/cortex/pkg/multicluster" +) + +// CommittedResourceChangeReconciler reconciles CommittedResourceChange CRDs. +// It is the sole writer of CommittedResource.Spec; it reads CommittedResource.Status +// to drive its own FSM. It never writes CommittedResource.Status. +// +// TODO (#783): add perAZActive map[string]types.UID + mu sync.Mutex for per-AZ serialization. +type CommittedResourceChangeReconciler struct { + client.Client + Scheme *runtime.Scheme + // AwaitingSchedulingTimeout is how long to wait in AwaitingScheduling before rolling back. + // Zero (default) means no timeout. + AwaitingSchedulingTimeout time.Duration +} + +func (r *CommittedResourceChangeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + ctx = commitments.WithNewGlobalRequestID(ctx) + logger := commitments.LoggerFromContext(ctx).WithValues("committedResourceChangeRequest", req.Name) + + var tx v1alpha1.CommittedResourceChange + if err := r.Get(ctx, req.NamespacedName, &tx); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + if !tx.DeletionTimestamp.IsZero() { + return ctrl.Result{}, nil + } + + logger.V(1).Info("reconcile entry", "rv", tx.ResourceVersion, "gen", tx.Generation, "phase", tx.Status.Phase) + + switch tx.Status.Phase { + case "", v1alpha1.ChangePhasePreparing: + return r.reconcilePreparing(ctx, logger, &tx) + case v1alpha1.ChangePhaseAllocating: + return r.reconcileAllocating(ctx, logger, &tx) + case v1alpha1.ChangePhaseAwaitingScheduling: + return r.reconcileAwaitingScheduling(ctx, logger, &tx) + case v1alpha1.ChangePhaseRollingBack: + return r.reconcileRollingBack(ctx, logger, &tx) + case v1alpha1.ChangePhaseDryRunCleanup: + return r.reconcileDryRunCleanup(ctx, logger, &tx) + case v1alpha1.ChangePhaseAccepted, v1alpha1.ChangePhaseRejected, v1alpha1.ChangePhaseSuperseded: + return ctrl.Result{}, nil + default: + return ctrl.Result{}, fmt.Errorf("unknown phase %q", tx.Status.Phase) + } +} + +// commitmentName returns the CommittedResource name for a given commitment UUID. +func commitmentName(uuid string) string { return "commitment-" + uuid } + +// reconcilePreparing snapshots pre-change CR specs and advances to Allocating. +// Always snapshots regardless of AllowRejection — syncer-mode CRTs never roll back but the +// snapshot is cheap and keeps this method free of branching. +func (r *CommittedResourceChangeReconciler) reconcilePreparing(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange) (ctrl.Result, error) { + // Snapshot current CR specs for all keys in NewState so rollback can restore them. + // nil entry = CR did not exist before = delete it on rollback. + snapshot := make(map[string]*v1alpha1.CommittedResourceSpec, len(tx.Spec.NewState)) + for uuid := range tx.Spec.NewState { + var cr v1alpha1.CommittedResource + if err := r.Get(ctx, client.ObjectKey{Name: commitmentName(uuid)}, &cr); err != nil { + if client.IgnoreNotFound(err) == nil { + snapshot[uuid] = nil + continue + } + logger.Error(err, "failed to snapshot CR", "commitmentUUID", uuid, "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID) + r.markError(ctx, tx, fmt.Sprintf("failed to snapshot commitment-%s: %v", uuid, err)) + return ctrl.Result{}, err + } + snapshot[uuid] = cr.Spec.DeepCopy() + } + + logger.Info("starting transaction", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "newStateSize", len(tx.Spec.NewState), "allowRejection", tx.Spec.AllowRejection) + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseAllocating, func(s *v1alpha1.CommittedResourceChangeStatus) { + s.SnapshotBefore = snapshot + }) +} + +// reconcileAllocating writes CommittedResource specs from NewState and records upserted refs. +// API-mode: advances to AwaitingScheduling. +// Syncer-mode: advances directly to Accepted (apply-and-done, no scheduling gate). +// +// UpsertedCommitments and the phase advance must happen in one patch (via the mutate function) +// so that a crash between CR spec writes and status persistence does not leave AwaitingScheduling +// with an empty UpsertedCommitments list — which would incorrectly advance to Accepted. +func (r *CommittedResourceChangeReconciler) reconcileAllocating(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange) (ctrl.Result, error) { + var upserted []corev1.ObjectReference + + for uuid, desiredSpec := range tx.Spec.NewState { + name := commitmentName(uuid) + if desiredSpec == nil { + toDelete := &v1alpha1.CommittedResource{ObjectMeta: metav1.ObjectMeta{Name: name}} + if err := r.Delete(ctx, toDelete); client.IgnoreNotFound(err) != nil { + logger.Error(err, "failed to delete CR", "commitmentUUID", uuid, "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID) + r.markError(ctx, tx, fmt.Sprintf("failed to delete commitment-%s: %v", uuid, err)) + return ctrl.Result{}, err + } + continue + } + + cr := &v1alpha1.CommittedResource{ObjectMeta: metav1.ObjectMeta{Name: name}} + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, cr, func() error { + cr.Spec = *desiredSpec + return nil + }); err != nil { + logger.Error(err, "failed to upsert CR spec", "commitmentUUID", uuid, "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID) + r.markError(ctx, tx, fmt.Sprintf("failed to upsert commitment-%s: %v", uuid, err)) + return ctrl.Result{}, err + } + upserted = append(upserted, corev1.ObjectReference{ + APIVersion: v1alpha1.GroupVersion.String(), + Kind: "CommittedResource", + Name: name, + }) + } + + // TODO (#782): syncer-mode recency check before advancing + nextPhase := v1alpha1.ChangePhaseAwaitingScheduling + if !tx.Spec.AllowRejection { + nextPhase = v1alpha1.ChangePhaseAccepted + } + return r.advanceTo(ctx, tx, nextPhase, func(s *v1alpha1.CommittedResourceChangeStatus) { + s.UpsertedCommitments = upserted + }) +} + +// reconcileAwaitingScheduling reads UpsertedCommitments and waits for placement settlement. +// Gates on each CR's ready-condition observedGeneration matching the generation written in Allocating. +// All CRs accepted → Accepted (or DryRunCleanup when DryRun=true). +// Any CR rejected → RollingBack. +// Not yet settled → wait for the CR watch to re-enqueue. +func (r *CommittedResourceChangeReconciler) reconcileAwaitingScheduling(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange) (ctrl.Result, error) { + allAccepted := true + acceptedCount := 0 + for _, ref := range tx.Status.UpsertedCommitments { + var cr v1alpha1.CommittedResource + if err := r.Get(ctx, client.ObjectKey{Name: ref.Name}, &cr); err != nil { + if client.IgnoreNotFound(err) == nil { + // CR briefly unavailable — wait for the next watch event. + allAccepted = false + continue + } + logger.Error(err, "failed to get upserted CR", "cr", ref.Name, "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID) + r.markError(ctx, tx, fmt.Sprintf("failed to get %s: %v", ref.Name, err)) + return ctrl.Result{}, err + } + + cond := meta.FindStatusCondition(cr.Status.Conditions, v1alpha1.CommittedResourceConditionReady) + if cond == nil || cond.ObservedGeneration != cr.Generation { + // CR controller hasn't caught up to the spec we wrote — wait. + allAccepted = false + continue + } + + switch { + case cond.Status == metav1.ConditionTrue && cond.Reason == v1alpha1.CommittedResourceReasonAccepted: + // CR accepted — placement succeeded; keep iterating. + acceptedCount++ + case cond.Status == metav1.ConditionFalse && cond.Reason == v1alpha1.CommittedResourceReasonRejected: + // Fail-fast: start rollback immediately on first rejection rather than waiting for all CRs + // to settle — this unblocks the AZ sooner and the first rejection message is sufficient for diagnosis. + logger.Info("CR rejected, rolling back", "cr", ref.Name, "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "message", cond.Message) + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseRollingBack, func(s *v1alpha1.CommittedResourceChangeStatus) { + s.Message = fmt.Sprintf("CR %s rejected: %s", ref.Name, cond.Message) + }) + default: + // Still transitioning — wait. + allAccepted = false + } + } + + if !allAccepted { + logger.V(1).Info("awaiting scheduling", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "ready", acceptedCount, "total", len(tx.Status.UpsertedCommitments)) + if r.AwaitingSchedulingTimeout > 0 { + progressingCond := meta.FindStatusCondition(tx.Status.Conditions, v1alpha1.ChangeConditionProgressing) + if progressingCond != nil && time.Since(progressingCond.LastTransitionTime.Time) > r.AwaitingSchedulingTimeout { + logger.Info("timed out waiting for scheduling, rolling back", "az", tx.Spec.AvailabilityZone, "correlationID", tx.Spec.CorrelationID, "timeout", r.AwaitingSchedulingTimeout) + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseRollingBack, func(s *v1alpha1.CommittedResourceChangeStatus) { + s.Message = "timed out waiting for scheduling" + meta.SetStatusCondition(&s.Conditions, metav1.Condition{ + Type: v1alpha1.ChangeConditionProgressing, + Status: metav1.ConditionFalse, + Reason: v1alpha1.ChangeReasonTimedOut, + Message: "timed out waiting for scheduling", + ObservedGeneration: tx.Generation, + }) + }) + } + } + // Primary requeue is event-driven via the CR watch (crToChangeRequest). + // RequeueAfter is a safety-net for missed watch events (reconnects, startup races). + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + + if tx.Spec.DryRun { + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseDryRunCleanup, nil) + } + return r.advanceTo(ctx, tx, v1alpha1.ChangePhaseAccepted, nil) +} + +// reconcileRollingBack converges affected CR specs back to the pre-change SnapshotBefore state. +// On restore complete: advances to Rejected. +// On permanent cleanup failure: advances to Rejected with reason RollbackFailed. +func (r *CommittedResourceChangeReconciler) reconcileRollingBack(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange) (ctrl.Result, error) { + // TODO (#780): for each entry in SnapshotBefore, converge CR spec back to snapshot value + // using SSA with fieldManager=txFieldManager. + // nil snapshot entry → CR did not exist before → delete it. + // Gate terminal transition on each CR confirming its spec reverted (check observedGeneration). + return ctrl.Result{}, nil +} + +// reconcileDryRunCleanup restores pre-change state after a dry-run would-accept outcome. +// Re-uses snapshot rollback logic; terminal phase is Accepted (not Rejected). +func (r *CommittedResourceChangeReconciler) reconcileDryRunCleanup(ctx context.Context, logger logr.Logger, tx *v1alpha1.CommittedResourceChange) (ctrl.Result, error) { + // TODO (#780): same convergence as reconcileRollingBack; terminal → advanceTo(Accepted, nil). + return ctrl.Result{}, nil +} + +// advanceTo updates the transaction status to the given phase. +// mutate is called (when non-nil) to allow phase advance and other status mutations +// (e.g. UpsertedCommitments, SnapshotBefore) to be persisted atomically in one Update call. +// Progressing is set to False on terminal phases to reflect that no further work is expected. +// +// Status().Update is used (not Patch) so that nil map values in SnapshotBefore and similar +// fields are preserved — JSON merge patch semantics treat null map values as deletions. +func (r *CommittedResourceChangeReconciler) advanceTo( + ctx context.Context, + tx *v1alpha1.CommittedResourceChange, + phase v1alpha1.ChangePhase, + mutate func(*v1alpha1.CommittedResourceChangeStatus), +) (ctrl.Result, error) { + + tx.Status.Phase = phase + if mutate != nil { + mutate(&tx.Status) + } + progressingStatus, progressingReason := metav1.ConditionTrue, "Advancing" + if tx.IsTerminal() { + progressingStatus, progressingReason = metav1.ConditionFalse, string(phase) + meta.SetStatusCondition(&tx.Status.Conditions, metav1.Condition{ + Type: v1alpha1.ChangeConditionFinished, + Status: metav1.ConditionTrue, + Reason: string(phase), + ObservedGeneration: tx.Generation, + }) + } + // Preserve a Progressing=False condition already stamped by the mutate func (e.g. TimedOut). + // Terminal phases always overwrite to ensure Progressing=False/. + existingProgressing := meta.FindStatusCondition(tx.Status.Conditions, v1alpha1.ChangeConditionProgressing) + if tx.IsTerminal() || existingProgressing == nil || existingProgressing.Status != metav1.ConditionFalse { + meta.SetStatusCondition(&tx.Status.Conditions, metav1.Condition{ + Type: v1alpha1.ChangeConditionProgressing, + Status: progressingStatus, + Reason: progressingReason, + ObservedGeneration: tx.Generation, + }) + } + if err := r.Status().Update(ctx, tx); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil +} + +// markError stamps Progressing=False/Error on the transaction as a best-effort status update. +// The update error is logged but not returned — callers return the original error for requeue. +func (r *CommittedResourceChangeReconciler) markError(ctx context.Context, tx *v1alpha1.CommittedResourceChange, msg string) { + meta.SetStatusCondition(&tx.Status.Conditions, metav1.Condition{ + Type: v1alpha1.ChangeConditionProgressing, + Status: metav1.ConditionFalse, + Reason: v1alpha1.ChangeReasonError, + Message: msg, + ObservedGeneration: tx.Generation, + }) + if err := r.Status().Update(ctx, tx); err != nil { + ctrl.LoggerFrom(ctx).Error(err, "failed to mark error condition") + } +} + +// SetupWithManager registers the controller and configures watches. +// TODO (#783): set MaxConcurrentReconciles: 1 via WithOptions(controller.Options{MaxConcurrentReconciles: 1}). +// TODO (#783): add predicate on WatchesMulticluster() to re-trigger same-AZ non-terminal siblings on terminal transition. +func (r *CommittedResourceChangeReconciler) SetupWithManager(mgr ctrl.Manager, mcl *multicluster.Client) error { + ctx := context.Background() + if err := indexCRCByUpsertedCR(ctx, mcl); err != nil { + return fmt.Errorf("failed to set up CRC-by-upserted-CR field index: %w", err) + } + bldr := multicluster.BuildController(mcl, mgr) + var err error + bldr, err = bldr.WatchesMulticluster( + &v1alpha1.CommittedResourceChange{}, + &handler.EnqueueRequestForObject{}, + ) + if err != nil { + return err + } + // Re-enqueues this transaction when a CR it upserted settles placement. + bldr, err = bldr.WatchesMulticluster( + &v1alpha1.CommittedResource{}, + handler.EnqueueRequestsFromMapFunc(r.crToChangeRequest), + ) + if err != nil { + return err + } + return bldr.Named("committed-resource-change-request").Complete(r) +} + +// crToChangeRequest maps a CommittedResource watch event back to the transaction that upserted it. +func (r *CommittedResourceChangeReconciler) crToChangeRequest(ctx context.Context, obj client.Object) []ctrl.Request { + var txList v1alpha1.CommittedResourceChangeList + if err := r.List(ctx, &txList, client.MatchingFields{idxCRCByUpsertedCR: obj.GetName()}); err != nil { + commitments.LoggerFromContext(ctx).Error(err, "failed to list CRCs by upserted CR", "cr", obj.GetName()) + return nil + } + reqs := make([]ctrl.Request, 0, len(txList.Items)) + for i := range txList.Items { + reqs = append(reqs, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(&txList.Items[i])}) + } + return reqs +} diff --git a/internal/scheduling/reservations/commitments/changes/reconciler_test.go b/internal/scheduling/reservations/commitments/changes/reconciler_test.go new file mode 100644 index 000000000..cfbdcebc8 --- /dev/null +++ b/internal/scheduling/reservations/commitments/changes/reconciler_test.go @@ -0,0 +1,1082 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package changes + +import ( + "context" + "strings" + "testing" + "time" + + apimeta "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" +) + +// ============================================================================ +// Helpers +// ============================================================================ + +var testCtx = context.Background() + +func testScheme(t *testing.T) *runtime.Scheme { + t.Helper() + s := runtime.NewScheme() + if err := v1alpha1.AddToScheme(s); err != nil { + t.Fatalf("add scheme: %v", err) + } + return s +} + +func testClient(t *testing.T, scheme *runtime.Scheme, objects ...client.Object) client.Client { + t.Helper() + return fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objects...). + WithStatusSubresource( + &v1alpha1.CommittedResourceChange{}, + &v1alpha1.CommittedResource{}, + ). + WithIndex(&v1alpha1.CommittedResourceChange{}, idxCRCByUpsertedCR, func(obj client.Object) []string { + tx, ok := obj.(*v1alpha1.CommittedResourceChange) + if !ok { + return nil + } + names := make([]string, 0, len(tx.Status.UpsertedCommitments)) + for _, ref := range tx.Status.UpsertedCommitments { + if ref.Name != "" { + names = append(names, ref.Name) + } + } + return names + }). + Build() +} + +func testReconciler(c client.Client, scheme *runtime.Scheme) *CommittedResourceChangeReconciler { + return &CommittedResourceChangeReconciler{Client: c, Scheme: scheme} +} + +func doReconcile(t *testing.T, r *CommittedResourceChangeReconciler, name string) ctrl.Result { + t.Helper() + res, err := r.Reconcile(testCtx, ctrl.Request{NamespacedName: types.NamespacedName{Name: name}}) + if err != nil { + t.Fatalf("Reconcile(%q): %v", name, err) + } + return res +} + +func getTx(t *testing.T, c client.Client, name string) *v1alpha1.CommittedResourceChange { + t.Helper() + var tx v1alpha1.CommittedResourceChange + if err := c.Get(testCtx, types.NamespacedName{Name: name}, &tx); err != nil { + t.Fatalf("get CRC %q: %v", name, err) + } + return &tx +} + +func getCR(t *testing.T, c client.Client, name string) *v1alpha1.CommittedResource { + t.Helper() + var cr v1alpha1.CommittedResource + if err := c.Get(testCtx, types.NamespacedName{Name: name}, &cr); err != nil { + t.Fatalf("get CR %q: %v", name, err) + } + return &cr +} + +// driveToPhase calls Reconcile until the CRT reaches target, up to 5 attempts. +func driveToPhase(t *testing.T, r *CommittedResourceChangeReconciler, c client.Client, txName string, target v1alpha1.ChangePhase) { + t.Helper() + const maxSteps = 5 + for range maxSteps { + if getTx(t, c, txName).Status.Phase == target { + return + } + doReconcile(t, r, txName) + } + if got := getTx(t, c, txName).Status.Phase; got != target { + t.Fatalf("driveToPhase: wanted %q, got %q after %d steps", target, got, maxSteps) + } +} + +// setCRReady simulates the CR controller writing a Ready condition using the CR's current Generation. +// + +func setCRReady(t *testing.T, c client.Client, crName string, status metav1.ConditionStatus, reason, message string) { + t.Helper() + cr := getCR(t, c, crName) + apimeta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ + Type: v1alpha1.CommittedResourceConditionReady, + Status: status, + Reason: reason, + Message: message, + ObservedGeneration: cr.Generation, + LastTransitionTime: metav1.Now(), + }) + if err := c.Status().Update(testCtx, cr); err != nil { + t.Fatalf("update CR %q status: %v", crName, err) + } +} + +// setCRReadyStaleGen sets Ready=True/Accepted but with a stale ObservedGeneration (generation-1), +// so the reconciler's generation gate treats the condition as not yet caught up. +func setCRReadyStaleGen(t *testing.T, c client.Client, crName string) { + t.Helper() + cr := getCR(t, c, crName) + apimeta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ + Type: v1alpha1.CommittedResourceConditionReady, + Status: metav1.ConditionTrue, + Reason: v1alpha1.CommittedResourceReasonAccepted, + ObservedGeneration: cr.Generation - 1, + LastTransitionTime: metav1.Now(), + }) + if err := c.Status().Update(testCtx, cr); err != nil { + t.Fatalf("update CR %q status (stale gen): %v", crName, err) + } +} + +// backdateProgressingCondition sets the Progressing condition's LastTransitionTime to the past. +func backdateProgressingCondition(t *testing.T, c client.Client, txName string, age time.Duration) { + t.Helper() + tx := getTx(t, c, txName) + for i, cond := range tx.Status.Conditions { + if cond.Type == v1alpha1.ChangeConditionProgressing { + tx.Status.Conditions[i].LastTransitionTime = metav1.NewTime(time.Now().Add(-age)) + break + } + } + if err := c.Status().Update(testCtx, tx); err != nil { + t.Fatalf("backdate Progressing condition on %q: %v", txName, err) + } +} + +// assertCondition verifies a condition in the given list by type, status, and optional reason. +// Pass an empty wantReason to skip the reason check. +func assertCondition(t *testing.T, conditions []metav1.Condition, condType string, wantStatus metav1.ConditionStatus, wantReason string) { + t.Helper() + cond := apimeta.FindStatusCondition(conditions, condType) + if cond == nil { + t.Errorf("condition %q not found in %v", condType, conditions) + return + } + if cond.Status != wantStatus { + t.Errorf("condition %q status: want %v, got %v", condType, wantStatus, cond.Status) + } + if wantReason != "" && cond.Reason != wantReason { + t.Errorf("condition %q reason: want %q, got %q", condType, wantReason, cond.Reason) + } +} + +// minimalCRSpec returns a minimal valid CommittedResourceSpec for testing. +func minimalCRSpec(az string) *v1alpha1.CommittedResourceSpec { + return &v1alpha1.CommittedResourceSpec{ + CommitmentUUID: "test-uuid", + FlavorGroupName: "kvm_v2", + ResourceType: v1alpha1.CommittedResourceTypeMemory, + Amount: resource.MustParse("1Gi"), + AvailabilityZone: az, + ProjectID: "project-1", + DomainID: "domain-1", + State: v1alpha1.CommitmentStatusConfirmed, + } +} + +// ============================================================================ +// TestCRTReconciler_Preparing +// ============================================================================ + +func TestCRTReconciler_Preparing(t *testing.T) { + tests := []struct { + name string + preObjects []client.Object + tx *v1alpha1.CommittedResourceChange + wantPhase v1alpha1.ChangePhase + wantSnapshotNilFor []string // UUIDs that must have nil snapshot entry (CR did not exist) + wantSnapshotSetFor []string // UUIDs that must have non-nil snapshot entry (CR existed) + wantProgressingStatus metav1.ConditionStatus + wantProgressingReason string + }{ + { + name: "no pre-existing CR: snapshot entry is nil, phase advances to Allocating", + tx: &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-prep-1"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-a": minimalCRSpec("az-1")}, + }, + }, + wantPhase: v1alpha1.ChangePhaseAllocating, + wantSnapshotNilFor: []string{"uuid-a"}, + wantProgressingStatus: metav1.ConditionTrue, + wantProgressingReason: "Advancing", + }, + { + name: "pre-existing CR spec is captured in SnapshotBefore", + preObjects: []client.Object{ + &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-b"}, + Spec: *minimalCRSpec("az-1"), + }, + }, + tx: &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-prep-2"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-b": minimalCRSpec("az-1")}, + }, + }, + wantPhase: v1alpha1.ChangePhaseAllocating, + wantSnapshotSetFor: []string{"uuid-b"}, + wantProgressingStatus: metav1.ConditionTrue, + wantProgressingReason: "Advancing", + }, + { + name: "Preparing idempotent: running reconcile twice does not error", + tx: &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-prep-3"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-c": minimalCRSpec("az-1")}, + }, + }, + wantPhase: v1alpha1.ChangePhaseAllocating, + wantProgressingStatus: metav1.ConditionTrue, + wantProgressingReason: "Advancing", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := testScheme(t) + objs := append([]client.Object{tt.tx}, tt.preObjects...) + c := testClient(t, scheme, objs...) + r := testReconciler(c, scheme) + + doReconcile(t, r, tt.tx.Name) + if tt.name == "Preparing idempotent: running reconcile twice does not error" { + // Reset to Preparing to verify a second run is also error-free. + stored := getTx(t, c, tt.tx.Name) + stored.Status.Phase = "" + if err := c.Status().Update(testCtx, stored); err != nil { + t.Fatalf("reset phase: %v", err) + } + doReconcile(t, r, tt.tx.Name) + } + + tx := getTx(t, c, tt.tx.Name) + if tx.Status.Phase != tt.wantPhase { + t.Errorf("phase: want %q, got %q", tt.wantPhase, tx.Status.Phase) + } + if tt.wantProgressingStatus != "" { + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionProgressing, tt.wantProgressingStatus, tt.wantProgressingReason) + } + for _, uuid := range tt.wantSnapshotNilFor { + snap, ok := tx.Status.SnapshotBefore[uuid] + if !ok { + t.Errorf("SnapshotBefore: missing key %q", uuid) + continue + } + if snap != nil { + t.Errorf("SnapshotBefore[%q]: want nil, got non-nil", uuid) + } + } + for _, uuid := range tt.wantSnapshotSetFor { + snap, ok := tx.Status.SnapshotBefore[uuid] + if !ok { + t.Errorf("SnapshotBefore: missing key %q", uuid) + continue + } + if snap == nil { + t.Errorf("SnapshotBefore[%q]: want non-nil snapshot, got nil", uuid) + } + } + }) + } +} + +// ============================================================================ +// TestCRTReconciler_Allocating +// ============================================================================ + +func TestCRTReconciler_Allocating(t *testing.T) { + tests := []struct { + name string + preObjects []client.Object + tx *v1alpha1.CommittedResourceChange + wantPhase v1alpha1.ChangePhase + wantCRExists []string + wantCRAbsent []string + wantUpsertedCommitments int + wantProgressingStatus metav1.ConditionStatus + wantProgressingReason string + wantFinished bool + wantFinishedReason string + }{ + { + name: "API-mode: creates CR and advances to AwaitingScheduling", + tx: &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-alloc-1"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-1": minimalCRSpec("az-1")}, + }, + }, + wantPhase: v1alpha1.ChangePhaseAwaitingScheduling, + wantCRExists: []string{"commitment-uuid-1"}, + wantUpsertedCommitments: 1, + wantProgressingStatus: metav1.ConditionTrue, + wantProgressingReason: "Advancing", + }, + { + name: "syncer-mode: applies CR and advances directly to Accepted (no scheduling gate)", + tx: &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-alloc-2"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceSyncer, + AllowRejection: false, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-2": minimalCRSpec("az-1")}, + }, + }, + wantPhase: v1alpha1.ChangePhaseAccepted, + wantCRExists: []string{"commitment-uuid-2"}, + wantUpsertedCommitments: 1, + wantProgressingStatus: metav1.ConditionFalse, + wantProgressingReason: string(v1alpha1.ChangePhaseAccepted), + wantFinished: true, + wantFinishedReason: string(v1alpha1.ChangePhaseAccepted), + }, + { + name: "nil spec deletes CR and it is absent from UpsertedCommitments", + preObjects: []client.Object{ + &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-3"}, + Spec: *minimalCRSpec("az-1"), + }, + }, + tx: &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-alloc-3"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-3": nil}, + }, + }, + wantPhase: v1alpha1.ChangePhaseAwaitingScheduling, + wantCRAbsent: []string{"commitment-uuid-3"}, + wantUpsertedCommitments: 0, + wantProgressingStatus: metav1.ConditionTrue, + wantProgressingReason: "Advancing", + }, + { + name: "pre-existing CR with different spec: spec is overwritten by CreateOrUpdate", + preObjects: []client.Object{ + &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-4"}, + Spec: *minimalCRSpec("az-old"), + }, + }, + tx: &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-alloc-4"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-4": minimalCRSpec("az-new")}, + }, + }, + wantPhase: v1alpha1.ChangePhaseAwaitingScheduling, + wantCRExists: []string{"commitment-uuid-4"}, + wantUpsertedCommitments: 1, + wantProgressingStatus: metav1.ConditionTrue, + wantProgressingReason: "Advancing", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := testScheme(t) + objs := append([]client.Object{tt.tx}, tt.preObjects...) + c := testClient(t, scheme, objs...) + r := testReconciler(c, scheme) + + driveToPhase(t, r, c, tt.tx.Name, tt.wantPhase) + + tx := getTx(t, c, tt.tx.Name) + if tx.Status.Phase != tt.wantPhase { + t.Errorf("phase: want %q, got %q", tt.wantPhase, tx.Status.Phase) + } + if got := len(tx.Status.UpsertedCommitments); got != tt.wantUpsertedCommitments { + t.Errorf("UpsertedCommitments len: want %d, got %d", tt.wantUpsertedCommitments, got) + } + if tt.wantProgressingStatus != "" { + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionProgressing, tt.wantProgressingStatus, tt.wantProgressingReason) + } + if tt.wantFinished { + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionFinished, metav1.ConditionTrue, tt.wantFinishedReason) + } + for _, name := range tt.wantCRExists { + getCR(t, c, name) // fatals if absent + } + for _, name := range tt.wantCRAbsent { + var cr v1alpha1.CommittedResource + if err := c.Get(testCtx, types.NamespacedName{Name: name}, &cr); err == nil { + t.Errorf("CR %q should be absent but exists", name) + } + } + }) + } +} + +// TestCRTReconciler_Allocating_Idempotent verifies that re-running Allocating with the same +// desired spec is idempotent: no error, phase advances correctly again. +func TestCRTReconciler_Allocating_Idempotent(t *testing.T) { + scheme := testScheme(t) + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-idem"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-idem": minimalCRSpec("az-1")}, + }, + } + c := testClient(t, scheme, tx) + r := testReconciler(c, scheme) + + driveToPhase(t, r, c, tx.Name, v1alpha1.ChangePhaseAwaitingScheduling) + + // Reset to Allocating to re-run it. + stored := getTx(t, c, tx.Name) + stored.Status.Phase = v1alpha1.ChangePhaseAllocating + if err := c.Status().Update(testCtx, stored); err != nil { + t.Fatalf("reset phase to Allocating: %v", err) + } + + doReconcile(t, r, tx.Name) // second run of Allocating + + final := getTx(t, c, tx.Name) + if final.Status.Phase != v1alpha1.ChangePhaseAwaitingScheduling { + t.Errorf("idempotent Allocating: want AwaitingScheduling, got %q", final.Status.Phase) + } +} + +// ============================================================================ +// TestCRTReconciler_AwaitingScheduling +// ============================================================================ + +func TestCRTReconciler_AwaitingScheduling(t *testing.T) { + const txName = "tx-await" + const crName = "commitment-uuid-await" + + // setup drives a CRT to AwaitingScheduling and returns the client and reconciler. + setup := func(t *testing.T, txOpts ...func(*v1alpha1.CommittedResourceChange)) (client.Client, *CommittedResourceChangeReconciler) { + t.Helper() + scheme := testScheme(t) + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: txName}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-await": minimalCRSpec("az-1")}, + }, + } + for _, opt := range txOpts { + opt(tx) + } + c := testClient(t, scheme, tx) + r := testReconciler(c, scheme) + driveToPhase(t, r, c, txName, v1alpha1.ChangePhaseAwaitingScheduling) + return c, r + } + + tests := []struct { + name string + txOpts []func(*v1alpha1.CommittedResourceChange) + setupCR func(t *testing.T, c client.Client) + setupTx func(t *testing.T, c client.Client) + reconcilerOpt func(r *CommittedResourceChangeReconciler) + wantPhase v1alpha1.ChangePhase + wantMessage string // substring that must appear in tx.Status.Message; empty = no check + wantProgressingStatus metav1.ConditionStatus + wantProgressingReason string + wantFinished bool + wantFinishedReason string + }{ + { + name: "all CRs accepted → Accepted", + setupCR: func(t *testing.T, c client.Client) { + setCRReady(t, c, crName, metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted, "") + }, + wantPhase: v1alpha1.ChangePhaseAccepted, + wantProgressingStatus: metav1.ConditionFalse, + wantProgressingReason: string(v1alpha1.ChangePhaseAccepted), + wantFinished: true, + wantFinishedReason: string(v1alpha1.ChangePhaseAccepted), + }, + { + name: "DryRun=true and all CRs accepted → DryRunCleanup", + txOpts: []func(*v1alpha1.CommittedResourceChange){ + func(tx *v1alpha1.CommittedResourceChange) { tx.Spec.DryRun = true }, + }, + setupCR: func(t *testing.T, c client.Client) { + setCRReady(t, c, crName, metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted, "") + }, + wantPhase: v1alpha1.ChangePhaseDryRunCleanup, + wantProgressingStatus: metav1.ConditionTrue, + wantProgressingReason: "Advancing", + }, + { + name: "CR rejected → RollingBack with rejection message", + setupCR: func(t *testing.T, c client.Client) { + setCRReady(t, c, crName, metav1.ConditionFalse, v1alpha1.CommittedResourceReasonRejected, "no capacity on host") + }, + wantPhase: v1alpha1.ChangePhaseRollingBack, + wantMessage: "no capacity on host", + wantProgressingStatus: metav1.ConditionTrue, + wantProgressingReason: "Advancing", + }, + { + name: "stale observedGeneration → stays in AwaitingScheduling", + setupCR: func(t *testing.T, c client.Client) { + setCRReadyStaleGen(t, c, crName) + }, + wantPhase: v1alpha1.ChangePhaseAwaitingScheduling, + wantProgressingStatus: metav1.ConditionTrue, + wantProgressingReason: "Advancing", + }, + { + name: "ConditionTrue with non-Accepted reason → stays in AwaitingScheduling", + setupCR: func(t *testing.T, c client.Client) { + setCRReady(t, c, crName, metav1.ConditionTrue, v1alpha1.CommittedResourceReasonReserving, "still placing") + }, + wantPhase: v1alpha1.ChangePhaseAwaitingScheduling, + wantProgressingStatus: metav1.ConditionTrue, + wantProgressingReason: "Advancing", + }, + { + name: "timeout exceeded → RollingBack with timed out message", + reconcilerOpt: func(r *CommittedResourceChangeReconciler) { + r.AwaitingSchedulingTimeout = 50 * time.Millisecond + }, + setupTx: func(t *testing.T, c client.Client) { + backdateProgressingCondition(t, c, txName, 1*time.Second) + }, + wantPhase: v1alpha1.ChangePhaseRollingBack, + wantMessage: "timed out", + wantProgressingStatus: metav1.ConditionFalse, + wantProgressingReason: v1alpha1.ChangeReasonTimedOut, + }, + { + name: "timeout=0 means no timeout: CRT stays in AwaitingScheduling indefinitely", + // No reconcilerOpt, so AwaitingSchedulingTimeout=0 (disabled). + // CR ready condition is never set → should stay in AwaitingScheduling. + wantPhase: v1alpha1.ChangePhaseAwaitingScheduling, + wantProgressingStatus: metav1.ConditionTrue, + wantProgressingReason: "Advancing", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c, r := setup(t, tt.txOpts...) + + if tt.reconcilerOpt != nil { + tt.reconcilerOpt(r) + } + if tt.setupCR != nil { + tt.setupCR(t, c) + } + if tt.setupTx != nil { + tt.setupTx(t, c) + } + + doReconcile(t, r, txName) + + tx := getTx(t, c, txName) + if tx.Status.Phase != tt.wantPhase { + t.Errorf("phase: want %q, got %q", tt.wantPhase, tx.Status.Phase) + } + if tt.wantMessage != "" && !strings.Contains(tx.Status.Message, tt.wantMessage) { + t.Errorf("message: want substring %q in %q", tt.wantMessage, tx.Status.Message) + } + if tt.wantProgressingStatus != "" { + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionProgressing, tt.wantProgressingStatus, tt.wantProgressingReason) + } + if tt.wantFinished { + assertCondition(t, tx.Status.Conditions, v1alpha1.ChangeConditionFinished, metav1.ConditionTrue, tt.wantFinishedReason) + } + }) + } +} + +// setPhase directly sets a CRT's phase via Status().Update for test setup. +func setPhase(t *testing.T, c client.Client, txName string, phase v1alpha1.ChangePhase) { + t.Helper() + tx := getTx(t, c, txName) + tx.Status.Phase = phase + if err := c.Status().Update(testCtx, tx); err != nil { + t.Fatalf("setPhase %q on %q: %v", phase, txName, err) + } +} + +// reconcileN calls doReconcile exactly n times without inspecting the phase between calls. +func reconcileN(t *testing.T, r *CommittedResourceChangeReconciler, name string, n int) { + t.Helper() + for range n { + doReconcile(t, r, name) + } +} + +// ============================================================================ +// TestCRTReconciler_Idempotency +// ============================================================================ + +// TestCRTReconciler_Idempotency verifies that phase handlers are re-entrant: running the same +// handler multiple times (simulating crash-then-requeue) produces the same end state. +// The fake client never returns transient errors; these tests confirm re-entrancy — the property +// that makes controller-runtime's error-then-requeue pattern safe. +func TestCRTReconciler_Idempotency(t *testing.T) { + tests := []struct { + name string + run func(t *testing.T) + }{ + { + name: "Preparing: snapshot is consistent across re-runs", + run: func(t *testing.T) { + scheme := testScheme(t) + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-idem-1"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-idem-1": minimalCRSpec("az-1")}, + }, + } + c := testClient(t, scheme, tx) + r := testReconciler(c, scheme) + + doReconcile(t, r, tx.Name) // Preparing → Allocating + snap1 := getTx(t, c, tx.Name).Status.SnapshotBefore + + setPhase(t, c, tx.Name, "") // reset to Preparing + doReconcile(t, r, tx.Name) // Preparing → Allocating (re-run) + snap2 := getTx(t, c, tx.Name).Status.SnapshotBefore + + for i, snap := range []map[string]*v1alpha1.CommittedResourceSpec{snap1, snap2} { + v, ok := snap["uuid-idem-1"] + if !ok { + t.Errorf("run %d: snapshot missing uuid-idem-1", i+1) + } else if v != nil { + t.Errorf("run %d: snapshot[uuid-idem-1]: want nil, got non-nil", i+1) + } + } + }, + }, + { + name: "Allocating: UpsertedCommitments and CR spec stable across re-runs", + run: func(t *testing.T) { + scheme := testScheme(t) + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-idem-2"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-idem-2": minimalCRSpec("az-1")}, + }, + } + c := testClient(t, scheme, tx) + r := testReconciler(c, scheme) + + driveToPhase(t, r, c, tx.Name, v1alpha1.ChangePhaseAwaitingScheduling) + upserted1 := getTx(t, c, tx.Name).Status.UpsertedCommitments + + setPhase(t, c, tx.Name, v1alpha1.ChangePhaseAllocating) // reset to Allocating + doReconcile(t, r, tx.Name) // Allocating → AwaitingScheduling (re-run) + upserted2 := getTx(t, c, tx.Name).Status.UpsertedCommitments + + if len(upserted1) != 1 || len(upserted2) != 1 { + t.Errorf("UpsertedCommitments: want len 1 on each run, got %d, %d", len(upserted1), len(upserted2)) + } + if len(upserted1) > 0 && len(upserted2) > 0 && upserted1[0].Name != upserted2[0].Name { + t.Errorf("UpsertedCommitments name changed: %q → %q", upserted1[0].Name, upserted2[0].Name) + } + getCR(t, c, "commitment-uuid-idem-2") // CR must still exist + }, + }, + { + name: "AwaitingScheduling: multiple calls without CR ready stay waiting", + run: func(t *testing.T) { + scheme := testScheme(t) + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-idem-3"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-idem-3": minimalCRSpec("az-1")}, + }, + } + c := testClient(t, scheme, tx) + r := testReconciler(c, scheme) + + driveToPhase(t, r, c, tx.Name, v1alpha1.ChangePhaseAwaitingScheduling) + reconcileN(t, r, tx.Name, 3) // CR never set ready — must stay in AwaitingScheduling + + if got := getTx(t, c, tx.Name).Status.Phase; got != v1alpha1.ChangePhaseAwaitingScheduling { + t.Errorf("phase: want AwaitingScheduling, got %q", got) + } + }, + }, + { + name: "AwaitingScheduling → Accepted stays Accepted on repeated reconciles", + run: func(t *testing.T) { + scheme := testScheme(t) + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-idem-4"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", Source: v1alpha1.TxSourceAPI, AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-idem-4": minimalCRSpec("az-1")}, + }, + } + c := testClient(t, scheme, tx) + r := testReconciler(c, scheme) + + driveToPhase(t, r, c, tx.Name, v1alpha1.ChangePhaseAwaitingScheduling) + setCRReady(t, c, "commitment-uuid-idem-4", metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted, "") + reconcileN(t, r, tx.Name, 3) // first call → Accepted; subsequent calls → terminal no-op + + if got := getTx(t, c, tx.Name).Status.Phase; got != v1alpha1.ChangePhaseAccepted { + t.Errorf("phase: want Accepted, got %q", got) + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, tt.run) + } +} + +// ============================================================================ +// TestCRTReconciler_APIScenarios +// ============================================================================ + +// crExists returns true if a CommittedResource with the given name exists. +func crExists(t *testing.T, c client.Client, name string) bool { + t.Helper() + var cr v1alpha1.CommittedResource + err := c.Get(testCtx, types.NamespacedName{Name: name}, &cr) + if err == nil { + return true + } + if client.IgnoreNotFound(err) == nil { + return false + } + t.Fatalf("crExists(%q): %v", name, err) + return false +} + +// TestCRTReconciler_APIScenarios covers realistic multi-CR change patterns as seen from the +// CR acceptance API: mixed creates, updates, and deletes in a single transaction. +func TestCRTReconciler_APIScenarios(t *testing.T) { + tests := []struct { + name string + // pre-existing CRs before the transaction is created + preObjects []client.Object + // the CRT to create + tx *v1alpha1.CommittedResourceChange + // expected phase after driving the FSM to a stable point + wantPhase v1alpha1.ChangePhase + // CR names that must exist after Allocating + wantCRExists []string + // CR names that must be absent after Allocating + wantCRAbsent []string + // exact UpsertedCommitments count (nils are not upserted) + wantUpsertedCount int + // UUIDs that must have nil snapshot (CR did not exist before) + wantSnapshotNilFor []string + // UUIDs that must have non-nil snapshot (CR existed before) + wantSnapshotSetFor []string + }{ + { + name: "multiple new CRs: all created, all upserted", + tx: &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-api-1"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{ + "uuid-a1": minimalCRSpec("az-1"), + "uuid-a2": minimalCRSpec("az-1"), + "uuid-a3": minimalCRSpec("az-1"), + }, + }, + }, + wantPhase: v1alpha1.ChangePhaseAwaitingScheduling, + wantCRExists: []string{"commitment-uuid-a1", "commitment-uuid-a2", "commitment-uuid-a3"}, + wantUpsertedCount: 3, + wantSnapshotNilFor: []string{"uuid-a1", "uuid-a2", "uuid-a3"}, + }, + { + name: "one new CR + one existing CR updated: both upserted, snapshot captures existing", + preObjects: []client.Object{ + &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-b2"}, + Spec: *minimalCRSpec("az-old"), + }, + }, + tx: &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-api-2"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{ + "uuid-b1": minimalCRSpec("az-1"), // new + "uuid-b2": minimalCRSpec("az-new"), // existing, spec updated + }, + }, + }, + wantPhase: v1alpha1.ChangePhaseAwaitingScheduling, + wantCRExists: []string{"commitment-uuid-b1", "commitment-uuid-b2"}, + wantUpsertedCount: 2, + wantSnapshotNilFor: []string{"uuid-b1"}, + wantSnapshotSetFor: []string{"uuid-b2"}, + }, + { + name: "one CR deleted + one CR updated: deleted absent, only updated upserted", + preObjects: []client.Object{ + &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-c1"}, + Spec: *minimalCRSpec("az-1"), + }, + &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "commitment-uuid-c2"}, + Spec: *minimalCRSpec("az-1"), + }, + }, + tx: &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-api-3"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{ + "uuid-c1": nil, // delete + "uuid-c2": minimalCRSpec("az-1"), // update + }, + }, + }, + wantPhase: v1alpha1.ChangePhaseAwaitingScheduling, + wantCRExists: []string{"commitment-uuid-c2"}, + wantCRAbsent: []string{"commitment-uuid-c1"}, + wantUpsertedCount: 1, + wantSnapshotSetFor: []string{"uuid-c1", "uuid-c2"}, + }, + { + name: "empty NewState: no CRs touched, advances all the way to Accepted", + tx: &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-api-4"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{}, + }, + }, + // AwaitingScheduling with empty UpsertedCommitments → allAccepted=true immediately → Accepted + wantPhase: v1alpha1.ChangePhaseAccepted, + wantUpsertedCount: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := testScheme(t) + objs := append([]client.Object{tt.tx}, tt.preObjects...) + c := testClient(t, scheme, objs...) + r := testReconciler(c, scheme) + + driveToPhase(t, r, c, tt.tx.Name, tt.wantPhase) + + tx := getTx(t, c, tt.tx.Name) + if tx.Status.Phase != tt.wantPhase { + t.Errorf("phase: want %q, got %q", tt.wantPhase, tx.Status.Phase) + } + if got := len(tx.Status.UpsertedCommitments); got != tt.wantUpsertedCount { + t.Errorf("UpsertedCommitments count: want %d, got %d", tt.wantUpsertedCount, got) + } + for _, name := range tt.wantCRExists { + if !crExists(t, c, name) { + t.Errorf("CR %q should exist but is absent", name) + } + } + for _, name := range tt.wantCRAbsent { + if crExists(t, c, name) { + t.Errorf("CR %q should be absent but exists", name) + } + } + for _, uuid := range tt.wantSnapshotNilFor { + snap, ok := tx.Status.SnapshotBefore[uuid] + if !ok { + t.Errorf("SnapshotBefore: missing key %q", uuid) + } else if snap != nil { + t.Errorf("SnapshotBefore[%q]: want nil, got non-nil", uuid) + } + } + for _, uuid := range tt.wantSnapshotSetFor { + snap, ok := tx.Status.SnapshotBefore[uuid] + if !ok { + t.Errorf("SnapshotBefore: missing key %q", uuid) + } else if snap == nil { + t.Errorf("SnapshotBefore[%q]: want non-nil, got nil", uuid) + } + } + }) + } +} + +// ============================================================================ +// TestCRTReconciler_EdgeCases +// ============================================================================ + +// TestCRTReconciler_EdgeCases covers paths that don't fit the main FSM tables. +func TestCRTReconciler_EdgeCases(t *testing.T) { + t.Run("DeletionTimestamp: reconcile is a no-op", func(t *testing.T) { + scheme := testScheme(t) + now := metav1.Now() + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tx-del", + DeletionTimestamp: &now, + // Finalizers required for the fake client to allow a non-zero DeletionTimestamp. + Finalizers: []string{"test.io/hold"}, + }, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + }, + } + c := testClient(t, scheme, tx) + r := testReconciler(c, scheme) + + doReconcile(t, r, tx.Name) // must not error, must not advance phase + + got := getTx(t, c, tx.Name).Status.Phase + if got != "" { + t.Errorf("phase: want empty (no-op), got %q", got) + } + }) + + t.Run("unknown phase returns error", func(t *testing.T) { + scheme := testScheme(t) + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-unknown"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + }, + } + c := testClient(t, scheme, tx) + r := testReconciler(c, scheme) + + // Seed an unknown phase. + tx.Status.Phase = "Bogus" + if err := c.Status().Update(testCtx, tx); err != nil { + t.Fatalf("seed phase: %v", err) + } + + _, err := r.Reconcile(testCtx, ctrl.Request{NamespacedName: types.NamespacedName{Name: tx.Name}}) + if err == nil { + t.Error("expected error for unknown phase, got nil") + } + }) + + t.Run("crToTransaction maps CR name to owning CRT via field index", func(t *testing.T) { + scheme := testScheme(t) + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-map"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + AllowRejection: true, + NewState: map[string]*v1alpha1.CommittedResourceSpec{"uuid-map": minimalCRSpec("az-1")}, + }, + } + c := testClient(t, scheme, tx) + r := testReconciler(c, scheme) + + // Drive to AwaitingScheduling so UpsertedCommitments is populated. + driveToPhase(t, r, c, tx.Name, v1alpha1.ChangePhaseAwaitingScheduling) + + // crToTransaction should return a request for the owning CRT. + cr := getCR(t, c, "commitment-uuid-map") + reqs := r.crToChangeRequest(testCtx, cr) + if len(reqs) != 1 { + t.Fatalf("crToTransaction: want 1 request, got %d", len(reqs)) + } + if reqs[0].Name != tx.Name { + t.Errorf("crToTransaction: want request for %q, got %q", tx.Name, reqs[0].Name) + } + }) +} + +// ============================================================================ +// TestCRTReconciler_Terminal +// ============================================================================ + +func TestCRTReconciler_Terminal(t *testing.T) { + tests := []struct { + name string + phase v1alpha1.ChangePhase + }{ + {"Accepted is a no-op", v1alpha1.ChangePhaseAccepted}, + {"Rejected is a no-op", v1alpha1.ChangePhaseRejected}, + {"Superseded is a no-op", v1alpha1.ChangePhaseSuperseded}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := testScheme(t) + tx := &v1alpha1.CommittedResourceChange{ + ObjectMeta: metav1.ObjectMeta{Name: "tx-term"}, + Spec: v1alpha1.CommittedResourceChangeSpec{ + AvailabilityZone: "az-1", + Source: v1alpha1.TxSourceAPI, + }, + } + c := testClient(t, scheme, tx) + r := testReconciler(c, scheme) + + // Seed the terminal phase directly via Status().Update. + tx.Status.Phase = tt.phase + if err := c.Status().Update(testCtx, tx); err != nil { + t.Fatalf("seed phase %q: %v", tt.phase, err) + } + + doReconcile(t, r, tx.Name) // must not error + + // Phase must remain unchanged. + got := getTx(t, c, tx.Name).Status.Phase + if got != tt.phase { + t.Errorf("terminal phase changed: want %q, got %q", tt.phase, got) + } + }) + } +} diff --git a/internal/scheduling/reservations/commitments/host_oversubscription_controller_test.go b/internal/scheduling/reservations/commitments/host_oversubscription_controller_test.go index bb62fc0c6..2bceb0fe9 100644 --- a/internal/scheduling/reservations/commitments/host_oversubscription_controller_test.go +++ b/internal/scheduling/reservations/commitments/host_oversubscription_controller_test.go @@ -80,6 +80,26 @@ func TestComputeViolations(t *testing.T) { wantViolation: true, wantExcessGiB: 256, }, + { + name: "failover host reservation does not count toward oversubscription", + hv: hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: "host-1"}, + Status: hv1.HypervisorStatus{ + EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: testGiB(512), + }, + }, + }, + slots: []v1alpha1.Reservation{{ + Spec: v1alpha1.ReservationSpec{ + // Full-host block, but FailoverHostReservation is not yet counted. + Type: v1alpha1.ReservationTypeFailoverHost, + TargetHost: "host-1", + Resources: map[hv1.ResourceName]resource.Quantity{hv1.ResourceMemory: testGiB(512)}, + }, + Status: v1alpha1.ReservationStatus{Host: "host-1"}, + }}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/internal/scheduling/reservations/failoverhost/controller.go b/internal/scheduling/reservations/failoverhost/controller.go new file mode 100644 index 000000000..8f001304f --- /dev/null +++ b/internal/scheduling/reservations/failoverhost/controller.go @@ -0,0 +1,416 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +// Package failoverhost keeps a fleet of OpenStack Nova compute hosts ready to +// absorb the workload of a failed host. +// +// # What a failover host reservation is +// +// When a hypervisor fails, Nova has to evacuate every virtual machine running +// on it onto other hosts. For that to succeed quickly and predictably there +// must be somewhere to put those vms. This package keeps a fraction of every +// group of interchangeable hosts empty and reserved, so a whole host's workload +// can be gang-evacuated onto a reserved "failover host" in one move. A failover +// host reservation (v1alpha1.Reservation of type FailoverHostReservation) +// blocks a target host from receiving ordinary placements so it stays available +// for this purpose. Its full lifecycle (Designated -> Active -> In Use, plus the +// Failed and Revoked/Used terminal states) is documented on the +// ReservationTypeFailoverHost constant in the v1alpha1 package. +// +// # Partitions +// +// Hosts are not interchangeable across the whole cloud: a HANA host, a +// general-purpose host and an external-customer host can generally only ever +// fail over within their own kind, because their workloads carry scheduling +// constraints (flavors, image properties, aggregates) that only matching hosts +// satisfy. We therefore group hosts into failover partitions: maximal sets of +// hosts within which every host can be evacuated onto every other host. We do +// not configure these groups by hand; we discover them from the scheduler's own +// placement decisions, so the grouping always reflects the live constraints. +// +// # The two halves of the controller +// +// The controller has a producer and a consumer that communicate through the +// desiredReservations map: +// +// - A periodic calculation loop (Start -> loop, both in controller.go) +// recomputes the desired set of failover host reservations from the current +// cluster state on a fixed interval, and stores it as desiredReservations. +// - An event-driven reconciler (Reconcile, below) brings the actual +// reservations in the cluster in line with that desired set, creating, +// updating and pruning Reservation objects. +// +// The loop never writes Reservation objects itself; it only recomputes the +// desired state and nudges the reconciler. Keeping the expensive, external-call +// heavy computation out of the reconcile path means reconciles stay cheap and +// idempotent. +// +// # The calculation +// +// The calculation half is a ReservationGenerator (generation.go): loop calls +// Generate each pass, swaps its result in as the desired state, and wakes the +// reconciler. The production generator runs a four-stage pipeline (Seeder -> +// PathFinder -> PartitionBuilder -> ReservationPlanner); see pipelineGenerator +// in generation.go for the end-to-end flow. The interface lets the whole +// generation strategy be swapped without touching the controller's plumbing. +package failoverhost + +import ( + "context" + "errors" + "maps" + "net/http" + "sync" + "time" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/source" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +// Config is the top-level config section this controller reads from the shared +// cortex conf.json. The controller's own settings are nested under failover.hosts +// so they sit alongside the (flavor-based) failover controller's settings without +// colliding with the flat, top-level keys read from the same file. +type Config struct { + Failover struct { + Hosts ControllerConfig `json:"hosts"` + } `json:"failover"` +} + +// ControllerConfig contains configuration for failover host reservations. +type ControllerConfig struct { + // Ratio is a global ratio that determines how many failover hosts inside + // a partition are created. For example, a ratio of 0.1 means that + // if a partition consists of 10 hosts, 1 failover host will be created. + // The number of resulting failover hosts in a partition will be rounded + // down. This means if there are not sufficient hosts inside a partition, + // it is possible that no failover hosts will be created. + Ratio float64 `json:"ratio"` + + // Minimum is the minimum number of failover hosts that should be created + // in a partition, regardless of the ratio. This ensures that even small + // partitions have at least a certain number of failover hosts. + Minimum int `json:"minimum"` + + // GenerationInterval is the interval within which failover partitions are + // recalculated and failover host reservations are updated accordingly. + GenerationInterval metav1.Duration `json:"generationInterval"` + + // ExternalSchedulerURL is the URL of the external scheduler service that + // will be used to request failover host placements. + ExternalSchedulerURL string `json:"externalSchedulerURL"` +} + +// Controller keeps the cluster's failover host reservations in sync with the +// current cluster state. It has two cooperating halves (see the package doc for +// the bigger picture): +// +// - a periodic calculation loop (Start -> loop) that recomputes the desired +// reservations and stores them in desiredReservations, and +// - the Reconcile method that drives the actual Reservation objects toward +// that desired set. +// +// The loop is the single writer of desiredReservations and Reconcile the single +// reader, both synchronised by desiredReservationsMu. +type Controller struct { + client.Client // k8s client + + Config ControllerConfig + + // Monitor records Prometheus metrics for the calculation loop and the + // reconciler. It may be nil (e.g. in tests); every call site nil-guards it. + Monitor *Monitor + + // desiredReservations is the desired state computed by the latest loop + // pass: the failover host reservations that should exist right now, keyed + // by their object reference (name only; reservations are cluster-scoped). + // Reconcile compares the actual cluster against this map. It is replaced + // wholesale each pass rather than mutated in place. + desiredReservations map[corev1.ObjectReference]v1alpha1.Reservation + + // desiredReservationsMu guards desiredReservations against concurrent access + // by the calculation loop (writer) and Reconcile (reader). + desiredReservationsMu sync.Mutex + + // reconcileCh carries a raw wake-up signal from the loop to the reconciler. + // After a pass the loop sends one event per reservation that is either + // desired or still present, so the reconciler re-examines each against the + // freshly computed desired state. The event payload only identifies the + // reservation; the reconciler reads desiredReservations itself. + reconcileCh chan event.GenericEvent + + // generator computes the desired reservations each pass (the swappable + // producer half of the controller). It is defaulted to the production + // pipelineGenerator in SetupWithManager but can be set beforehand to + // substitute a fake in tests. See generation.go. + generator ReservationGenerator +} + +// Reconcile drives a single failover host reservation toward the desired state +// computed by the calculation loop. It is triggered by the loop (via +// reconcileCh) for every reservation that is either desired or still present, so +// between them the two cover create, update, delete and no-op: +// +// - desired, not present -> create it. +// - not desired, present -> prune it (it no longer matches any partition). +// - desired and present -> update it if the spec/labels drifted. +// - neither desired nor present, or already in sync -> nothing to do. +// +// It is level-triggered and idempotent: it reads the current actual and desired +// state on every call and makes at most one write, so redundant events and +// retries are harmless. +func (c *Controller) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := logf.FromContext(ctx).WithName("failoverhost-controller-reconcile"). + WithValues("reservationName", req.Name) + + // Fetch the actual reservation from the cluster, if it exists. + actual := &v1alpha1.Reservation{} + actualExists := true + if err := c.Get(ctx, req.NamespacedName, actual); err != nil { + if !apierrors.IsNotFound(err) { + log.Error(err, "failed to get reservation") + return ctrl.Result{}, err + } + actualExists = false + } + + // Fetch the desired reservation produced by the partitioning loop. + c.desiredReservationsMu.Lock() + desired, desiredExists := c.desiredReservations[corev1.ObjectReference{Name: req.Name}] + c.desiredReservationsMu.Unlock() + + // Before acting on a desired reservation, confirm the hypervisor it targets + // still exists and still has the capacity the reservation reserves (a failover + // host reservation blocks a host's full capacity, so a drift means we would + // under- or over-reserve). The reservation's resources are the host's capacity + // overlaid with its effective capacity (see the planner), so rebuild that same + // view and compare it to the reservation. A missing hypervisor or a mismatch + // returns an error so the reconcile is retried with exponential backoff, giving + // the hypervisor CRD and the calculation loop time to settle. + if desiredExists { + hv := &hv1.Hypervisor{} + if err := c.Get(ctx, client.ObjectKey{Name: desired.Spec.TargetHost}, hv); err != nil { + log.Error(err, "failed to get hypervisor for reservation", "host", desired.Spec.TargetHost) + c.Monitor.RecordReservationError("hypervisor") + return ctrl.Result{}, err + } + capacity := map[hv1.ResourceName]resource.Quantity{} + maps.Copy(capacity, hv.Status.Capacity) + maps.Copy(capacity, hv.Status.EffectiveCapacity) + // equality.Semantic compares resource.Quantity by value, unlike + // reflect.DeepEqual which trips over its unexported cached state. + if !equality.Semantic.DeepEqual(capacity, desired.Spec.Resources) { + log.Error(nil, "hypervisor capacity does not match reservation, retrying", + "host", desired.Spec.TargetHost, "hypervisorCapacity", capacity, "reservationResources", desired.Spec.Resources) + c.Monitor.RecordReservationError("hypervisor") + return ctrl.Result{}, errors.New("hypervisor capacity does not match reservation") + } + } + + switch { + case !desiredExists && !actualExists: + // Nothing to do, both the desired and actual state agree it is gone. + log.V(1).Info("reservation neither desired nor present, nothing to do") + return ctrl.Result{}, nil + + case !desiredExists && actualExists: + // The reservation is no longer part of any failover partition, prune it. + if err := c.Delete(ctx, actual); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + log.Error(err, "failed to delete stale reservation") + c.Monitor.RecordReservationError("delete") + return ctrl.Result{}, err + } + log.Info("deleted stale failover host reservation", "host", actual.Spec.TargetHost) + c.Monitor.RecordReservationDeleted() + return ctrl.Result{}, nil + + case desiredExists && !actualExists: + // The reservation is desired but not yet present, create it. + toCreate := desired.DeepCopy() + if err := c.Create(ctx, toCreate); err != nil { + if apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, nil + } + log.Error(err, "failed to create failover host reservation") + c.Monitor.RecordReservationError("create") + return ctrl.Result{}, err + } + log.Info("created failover host reservation", "host", desired.Spec.TargetHost) + c.Monitor.RecordReservationCreated() + return ctrl.Result{}, nil + + default: + // Both exist. Update the actual spec if it drifted from the desired one. + // equality.Semantic handles resource.Quantity correctly, unlike + // reflect.DeepEqual which trips over its unexported cached state. + if equality.Semantic.DeepEqual(actual.Spec, desired.Spec) && + maps.Equal(actual.Labels, desired.Labels) { + log.V(1).Info("reservation already matches desired state") + return ctrl.Result{}, nil + } + actual.Spec = desired.Spec + // Replace labels outright rather than merging, so a label that is no + // longer desired is removed — a merge would keep it and make every pass + // re-detect drift and rewrite the object forever. + actual.Labels = maps.Clone(desired.Labels) + if err := c.Update(ctx, actual); err != nil { + log.Error(err, "failed to update failover host reservation") + c.Monitor.RecordReservationError("update") + return ctrl.Result{}, err + } + log.Info("updated failover host reservation", "host", desired.Spec.TargetHost) + c.Monitor.RecordReservationUpdated() + return ctrl.Result{}, nil + } +} + +// Start runs the periodic calculation loop until ctx is cancelled. +// +// It implements manager.Runnable so the manager owns its lifecycle (it is +// registered via mgr.Add in SetupWithManager). Start first validates the config +// — a misconfigured ratio, minimum or interval is a programming/deployment error +// that should fail fast rather than silently produce no reservations — then runs +// one pass immediately (so reservations exist without waiting a full interval) +// and re-runs loop on every tick. On shutdown it closes reconcileCh so the +// reconciler's event source drains cleanly. +func (c *Controller) Start(ctx context.Context) error { + log := logf.FromContext(ctx).WithName("failoverhost-controller-startup") + log.Info("starting failover host controller") + + // Validate the config + interval := c.Config.GenerationInterval.Duration + if interval <= 0 || interval < time.Minute { + return errors.New("generation interval too low or not set") + } + if c.Config.Ratio <= 0 || c.Config.Ratio > 1 { + return errors.New("failover ratio must be between 0 and 1") + } + if c.Config.Minimum <= 0 { + return errors.New("minimum must be greater than 0") + } + + ticker := time.NewTicker(interval) + defer ticker.Stop() + defer close(c.reconcileCh) + // Run the routine once before entering the ticker loop. + c.loop(ctx) + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + c.loop(ctx) + } + } +} + +// SetupWithManager wires both halves of the controller into the manager: +// +// - It registers the Controller as a Runnable (mgr.Add) so the manager starts +// and stops the periodic calculation loop (Start) alongside the controller. +// - It registers the reconciler with a channel event source, so the loop can +// enqueue reconcile requests by sending on reconcileCh. +// +// It also defaults the generator to the production pipelineGenerator. Tests may +// set it beforehand to substitute a fake (e.g. a path finder pointed at a stub +// scheduler); it is only defaulted here when nil. +func (c *Controller) SetupWithManager(mgr ctrl.Manager) error { + if mgr == nil { + return errors.New("manager cannot be nil") + } + // Default the generator. Tests may set it before calling Setup to substitute + // a fake; here we wire the production pipeline stages. + if c.generator == nil { + c.generator = &pipelineGenerator{ + Client: c.Client, + Monitor: c.Monitor, + seeder: &seeder{k8s: c.Client}, + pathFinder: &evacuationPathFinder{ + // Timeout so a hung scheduler can't stall the calculation loop. + httpClient: &http.Client{Timeout: 30 * time.Second}, + schedulerURL: c.Config.ExternalSchedulerURL, + monitor: c.Monitor, + }, + partitionBuilder: cliquePartitionBuilder{}, + reservationPlanner: ratioReservationPlanner{ratio: c.Config.Ratio, minimum: c.Config.Minimum}, + } + } + + c.reconcileCh = make(chan event.GenericEvent) + src := source.Channel(c.reconcileCh, &handler.EnqueueRequestForObject{}) + + // Register the loop so the manager starts and stops it alongside the controller. + if err := mgr.Add(c); err != nil { + return err + } + + return ctrl.NewControllerManagedBy(mgr). + Named("failoverhost-controller"). + WatchesRawSource(src). + Complete(c) +} + +// loop runs one calculation pass: it asks the generator for the desired failover +// host reservations, swaps them in as the new desired state, and wakes the +// reconciler. The heavy lifting (snapshotting the cluster and running the +// pipeline) lives in the generator; see generation.go. +// +// The desired map is swapped in wholesale under the lock Reconcile reads it with, +// so Reconcile never sees a half-built state. Replacing (rather than mutating) +// the map is what lets Reconcile prune reservations that are no longer planned: +// anything absent from the new map but still present in the cluster is detected +// as stale. +// +// A generator that returns ok=false aborts the pass without touching the desired +// state: a partial recalculation is worse than a slightly stale one, as it could +// drop reservations that are still valid. +func (c *Controller) loop(ctx context.Context) { + desired, topo, ok := c.generator.Generate(ctx) + if !ok { + return // the generator already logged the reason and recorded the abort. + } + c.desiredReservationsMu.Lock() + c.desiredReservations = desired + c.desiredReservationsMu.Unlock() + c.publish(ctx, topo, desired) +} + +// publish wakes the reconciler for every reservation that is either desired now +// or already present in the cluster. The union matters: desired-but-absent +// reservations need creating, and present-but-no-longer-desired reservations +// need pruning, so both sets must be re-examined. Sending is a raw signal — the +// reconciler re-reads the desired state itself — so the event payload only needs +// to identify the reservation. Sends block on reconcileCh; a cancelled ctx (the +// manager shutting down) unblocks and returns. +func (c *Controller) publish(ctx context.Context, topo topology, desired map[corev1.ObjectReference]v1alpha1.Reservation) { + toReconcile := make(map[corev1.ObjectReference]v1alpha1.Reservation) + for k, v := range topo.existing { + toReconcile[k] = v + } + for k, v := range desired { + toReconcile[k] = v + } + for _, reservation := range toReconcile { + evt := event.GenericEvent{Object: &reservation} + select { + case c.reconcileCh <- evt: + case <-ctx.Done(): + return + } + } +} diff --git a/internal/scheduling/reservations/failoverhost/controller_test.go b/internal/scheduling/reservations/failoverhost/controller_test.go new file mode 100644 index 000000000..fd7ee3c2f --- /dev/null +++ b/internal/scheduling/reservations/failoverhost/controller_test.go @@ -0,0 +1,266 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package failoverhost + +import ( + "context" + "testing" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + vmv1alpha1 "github.com/cobaltcore-dev/vm-operator/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/event" +) + +// newTestClient builds a fake controller-runtime client with the schemes the +// failover host controller needs (reservations, hypervisors, virtual machines) +// and the Reservation status subresource registered. Shared by the Reconcile +// tests here and the loop test in loop_test.go. +func newTestClient(t *testing.T, objs ...client.Object) client.Client { + t.Helper() + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("add v1alpha1 scheme: %v", err) + } + if err := hv1.AddToScheme(scheme); err != nil { + t.Fatalf("add hv1 scheme: %v", err) + } + if err := vmv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("add vmv1alpha1 scheme: %v", err) + } + return fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objs...). + WithStatusSubresource(&v1alpha1.Reservation{}). + Build() +} + +// failoverReservation builds a minimal failover host reservation targeting the +// given host, matching the shape the planner produces. Its resources match the +// capacity buildHV(host, 64, 32) gives the hypervisor, so the controller's +// hypervisor-size check passes. +func failoverReservation(host string) v1alpha1.Reservation { + return v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "failover-host-" + host, + Labels: map[string]string{v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelFailoverHost}, + }, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeFailoverHost, + TargetHost: host, + Resources: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: *resource.NewQuantity(64*1024*1024*1024, resource.BinarySI), + hv1.ResourceCPU: *resource.NewQuantity(32, resource.DecimalSI), + }, + }, + } +} + +func reconcileReq(name string) ctrl.Request { + return ctrl.Request{NamespacedName: types.NamespacedName{Name: name}} +} + +// Reconcile covers the four reconcile branches: desired-but-absent (create), +// present-but-undesired (delete), present-and-drifted (update), and in-sync +// (no-op). Each branch is driven by seeding desiredReservations and the fake +// cluster independently, then asserting the resulting cluster state. + +func TestReconcileCreatesDesiredReservation(t *testing.T) { + c := &Controller{Client: newTestClient(t, buildHV("hv1", 64, 32))} + desired := failoverReservation("hv1") + c.desiredReservations = map[corev1.ObjectReference]v1alpha1.Reservation{ + {Name: desired.Name}: desired, + } + req := reconcileReq(desired.Name) + if _, err := c.Reconcile(context.Background(), req); err != nil { + t.Fatalf("reconcile: %v", err) + } + got := &v1alpha1.Reservation{} + if err := c.Get(context.Background(), req.NamespacedName, got); err != nil { + t.Fatalf("expected reservation to be created: %v", err) + } + if got.Spec.TargetHost != "hv1" { + t.Errorf("target host = %q, want hv1", got.Spec.TargetHost) + } +} + +func TestReconcileDeletesStaleReservation(t *testing.T) { + stale := failoverReservation("hvX") + c := &Controller{Client: newTestClient(t, &stale)} + c.desiredReservations = map[corev1.ObjectReference]v1alpha1.Reservation{} + req := reconcileReq(stale.Name) + if _, err := c.Reconcile(context.Background(), req); err != nil { + t.Fatalf("reconcile: %v", err) + } + got := &v1alpha1.Reservation{} + if err := c.Get(context.Background(), req.NamespacedName, got); !apierrors.IsNotFound(err) { + t.Errorf("expected reservation to be deleted, got err=%v", err) + } +} + +func TestReconcileUpdatesDriftedReservation(t *testing.T) { + actual := failoverReservation("hv1") + actual.Spec.AvailabilityZone = "az-old" + c := &Controller{Client: newTestClient(t, &actual, buildHV("hv1", 64, 32))} + desired := failoverReservation("hv1") + desired.Spec.AvailabilityZone = "az-new" + c.desiredReservations = map[corev1.ObjectReference]v1alpha1.Reservation{ + {Name: desired.Name}: desired, + } + req := reconcileReq(desired.Name) + if _, err := c.Reconcile(context.Background(), req); err != nil { + t.Fatalf("reconcile: %v", err) + } + got := &v1alpha1.Reservation{} + if err := c.Get(context.Background(), req.NamespacedName, got); err != nil { + t.Fatalf("get: %v", err) + } + if got.Spec.AvailabilityZone != "az-new" { + t.Errorf("availability zone = %q, want az-new", got.Spec.AvailabilityZone) + } +} + +func TestReconcileNoopWhenInSync(t *testing.T) { + actual := failoverReservation("hv1") + c := &Controller{Client: newTestClient(t, &actual, buildHV("hv1", 64, 32))} + desired := failoverReservation("hv1") + c.desiredReservations = map[corev1.ObjectReference]v1alpha1.Reservation{ + {Name: desired.Name}: desired, + } + req := reconcileReq(desired.Name) + before := &v1alpha1.Reservation{} + if err := c.Get(context.Background(), req.NamespacedName, before); err != nil { + t.Fatalf("get: %v", err) + } + if _, err := c.Reconcile(context.Background(), req); err != nil { + t.Fatalf("reconcile: %v", err) + } + after := &v1alpha1.Reservation{} + if err := c.Get(context.Background(), req.NamespacedName, after); err != nil { + t.Fatalf("get: %v", err) + } + // An update bumps the resourceVersion, so an unchanged one proves the no-op. + if before.ResourceVersion != after.ResourceVersion { + t.Errorf("reservation was rewritten: %q -> %q", before.ResourceVersion, after.ResourceVersion) + } +} + +func TestReconcileErrorsWhenHypervisorMissing(t *testing.T) { + c := &Controller{Client: newTestClient(t)} + desired := failoverReservation("hv1") + c.desiredReservations = map[corev1.ObjectReference]v1alpha1.Reservation{ + {Name: desired.Name}: desired, + } + if _, err := c.Reconcile(context.Background(), reconcileReq(desired.Name)); err == nil { + t.Fatal("expected an error to trigger backoff when the hypervisor is missing") + } +} + +func TestReconcileErrorsWhenHypervisorSizeMismatches(t *testing.T) { + // The hypervisor reports 32Gi but the reservation reserves 64Gi. + c := &Controller{Client: newTestClient(t, buildHV("hv1", 32, 32))} + desired := failoverReservation("hv1") + c.desiredReservations = map[corev1.ObjectReference]v1alpha1.Reservation{ + {Name: desired.Name}: desired, + } + if _, err := c.Reconcile(context.Background(), reconcileReq(desired.Name)); err == nil { + t.Fatal("expected an error to trigger backoff on a memory mismatch") + } +} + +func TestReconcileErrorsWhenHypervisorCPUMismatches(t *testing.T) { + // Memory matches but the hypervisor reports 16 CPUs against the reserved 32. + c := &Controller{Client: newTestClient(t, buildHV("hv1", 64, 16))} + desired := failoverReservation("hv1") + c.desiredReservations = map[corev1.ObjectReference]v1alpha1.Reservation{ + {Name: desired.Name}: desired, + } + if _, err := c.Reconcile(context.Background(), reconcileReq(desired.Name)); err == nil { + t.Fatal("expected an error to trigger backoff on a cpu mismatch") + } +} + +// buildHV builds a hypervisor in az1 with an availability-zone label and the +// given memory (GiB) and cpu capacity, so collect accepts it and the planner can +// order hosts by cost. It is shared by the generator tests (generation_test.go) +// and the path finder tests (pathfinding_test.go). +func buildHV(name string, memGi, cpu int64) *hv1.Hypervisor { + return &hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{corev1.LabelTopologyZone: "az1"}, + }, + Status: hv1.HypervisorStatus{ + Capacity: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: *resource.NewQuantity(memGi*1024*1024*1024, resource.BinarySI), + hv1.ResourceCPU: *resource.NewQuantity(cpu, resource.DecimalSI), + }, + }, + } +} + +// fakeGenerator is a stub ReservationGenerator returning fixed output, so the +// controller's loop plumbing can be tested without the real pipeline. +type fakeGenerator struct { + desired map[corev1.ObjectReference]v1alpha1.Reservation + topo topology + ok bool +} + +func (f fakeGenerator) Generate(context.Context) (map[corev1.ObjectReference]v1alpha1.Reservation, topology, bool) { + return f.desired, f.topo, f.ok +} + +// TestLoopPublishesGeneratorOutput checks the controller half: loop swaps the +// generator's desired reservations in as the new desired state. +func TestLoopPublishesGeneratorOutput(t *testing.T) { + desired := failoverReservation("hvA") + c := &Controller{ + Client: newTestClient(t), + reconcileCh: make(chan event.GenericEvent, 16), + generator: fakeGenerator{ + desired: map[corev1.ObjectReference]v1alpha1.Reservation{{Name: desired.Name}: desired}, + ok: true, + }, + } + c.loop(context.Background()) + + c.desiredReservationsMu.Lock() + defer c.desiredReservationsMu.Unlock() + if _, ok := c.desiredReservations[corev1.ObjectReference{Name: desired.Name}]; !ok { + t.Fatalf("expected generator output to become the desired state") + } +} + +// TestLoopKeepsDesiredStateWhenGeneratorAborts checks that an aborting generator +// (ok=false) leaves the previously published desired state untouched, so a +// partial recalculation never drops still-valid reservations. +func TestLoopKeepsDesiredStateWhenGeneratorAborts(t *testing.T) { + kept := failoverReservation("hvKept") + c := &Controller{ + Client: newTestClient(t), + reconcileCh: make(chan event.GenericEvent, 16), + desiredReservations: map[corev1.ObjectReference]v1alpha1.Reservation{{Name: kept.Name}: kept}, + generator: fakeGenerator{ok: false}, + } + c.loop(context.Background()) + + c.desiredReservationsMu.Lock() + defer c.desiredReservationsMu.Unlock() + if len(c.desiredReservations) != 1 { + t.Fatalf("desired reservations = %d, want the previous 1 kept", len(c.desiredReservations)) + } + if _, ok := c.desiredReservations[corev1.ObjectReference{Name: kept.Name}]; !ok { + t.Errorf("expected the previously published reservation to survive an aborted pass") + } +} diff --git a/internal/scheduling/reservations/failoverhost/generation.go b/internal/scheduling/reservations/failoverhost/generation.go new file mode 100644 index 000000000..fde3c7462 --- /dev/null +++ b/internal/scheduling/reservations/failoverhost/generation.go @@ -0,0 +1,255 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package failoverhost + +import ( + "context" + "sort" + "time" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +// ReservationGenerator computes the full desired set of failover host +// reservations from the current cluster state. It is the swappable "producer" +// half of the controller: the controller's loop calls Generate each pass, swaps +// the result in as the desired state, and wakes the reconciler. Replacing this +// interface replaces the entire generation strategy without touching the +// controller's plumbing (ticker, lock/swap, publish). +type ReservationGenerator interface { + // Generate snapshots the cluster and returns the desired failover host + // reservations keyed by their object reference (name only; reservations are + // cluster-scoped), together with the topology snapshot it reasoned over — the + // controller needs topo.existing to wake the reconciler for every reservation + // that may need pruning. ok=false means the pass should be abandoned without + // touching the published desired state; the generator has already logged the + // reason and recorded the abort metric. + Generate(ctx context.Context) (desired map[corev1.ObjectReference]v1alpha1.Reservation, topo topology, ok bool) +} + +// topology is the host layout a single calculation pass reasons over: every +// known hypervisor and the failover host reservations that exist right now. It +// is a consistent snapshot built once per pass by collect, so every pipeline +// stage reasons over the same view of the cluster. +type topology struct { + // hypervisors indexes every known hypervisor by its name. + hypervisors map[string]hv1.Hypervisor + // existing holds the failover host reservations currently in the cluster, + // keyed by their object reference. The planner prefers to keep these stable. + existing map[corev1.ObjectReference]v1alpha1.Reservation +} + +// pipelineGenerator is the production ReservationGenerator. It derives the +// desired reservations by running a four-stage pipeline, each stage a small +// interface so it can be understood, tested and swapped in isolation: +// +// 1. Seeder (seeding.go) enumerates the hypothetical placements to probe with: +// every flavor crossed with every availability zone in the fleet. These +// flavor "seeds" replace the running vms the pipeline used to derive paths +// from, so a partition is found even when its hosts run no vms and no single +// exotic vm can veto a host from its partition. +// 2. PathFinder (pathfinding.go) probes the scheduler per seed and turns the +// answers into the directed edges of a failover graph: source -> target means +// "a host's whole workload can be evacuated onto target, and target is big +// enough to hold it". +// 3. PartitionBuilder (partitioning.go) clusters those edges into partitions +// (maximal sets of mutually-evacuable hosts). +// 4. ReservationPlanner (planning.go) decides, per partition, which hosts +// become failover targets, honouring the configured ratio and minimum. +type pipelineGenerator struct { + client.Client // k8s client + + // Monitor records the calculation-side Prometheus metrics (per-stage aborts + // and the per-pass summary). It may be nil (e.g. in tests); every call site + // nil-guards it. + Monitor *Monitor + + // The pipeline stages run by Generate. They are defaulted to the production + // implementations in SetupWithManager but can be set beforehand to substitute + // fakes in tests. + seeder Seeder + pathFinder PathFinder + partitionBuilder PartitionBuilder + reservationPlanner ReservationPlanner +} + +// Generate runs one calculation pass of the failover host pipeline: +// +// 1. collect — snapshot the cluster into a consistent topology. +// 2. Seeds — enumerate the flavor × availability-zone permutations to probe. +// 3. FindPaths — probe the scheduler per seed and find, for each host, the hosts +// its whole workload could be gang-evacuated onto (the directed edges of the +// failover graph). +// 4. BuildPartitions — cluster those edges into failover partitions: maximal +// sets of hosts that can all fail over onto one another. These cliques can +// overlap, so Generate then assigns each shared host to exactly one partition +// (the smallest) to keep the planner input disjoint. +// 5. PlanReservations — per partition, pick which hosts become failover +// targets, honouring the configured ratio/minimum and keeping existing +// reservations stable. +// +// Any error or inconsistency aborts the whole pass (ok=false) without producing +// a partial result: a partial recalculation is worse than a slightly stale one, +// as it could drop reservations that are still valid. +func (g *pipelineGenerator) Generate(ctx context.Context) (map[corev1.ObjectReference]v1alpha1.Reservation, topology, bool) { + log := logf.FromContext(ctx).WithName("failoverhost-controller-calculation-loop") + log.Info("recalculating failover host reservations") + + start := time.Now() + + topo, ok := g.collect(ctx) + if !ok { + g.Monitor.RecordCalculationAbort("collect") + return nil, topology{}, false // collect already logged the reason. + } + + // Enumerate the seeds (flavor × availability-zone permutations) to probe + // with. A seeding failure (Nova unreachable, auth expired) is an + // infrastructure error on a partial view, so abort the pass and keep the last + // published picture rather than recomputing from a shrunken seed set. + seeds, err := g.seeder.Seeds(ctx, topo) + if err != nil { + log.Error(err, "failed to generate failover seeds") + g.Monitor.RecordCalculationAbort("seeding") + return nil, topology{}, false + } + + paths, err := g.pathFinder.FindPaths(ctx, topo, seeds) + if err != nil { + log.Error(err, "failed to find failover paths") + g.Monitor.RecordCalculationAbort("pathfinding") + return nil, topology{}, false + } + + // Cluster the failover edges into partitions. See partitioning.go for why a + // partition is a maximal clique of mutually-evacuable hosts, and how empty + // hosts are recruited into the partition that can reach them. + partitions := g.partitionBuilder.BuildPartitions(paths) + log.V(1).Info("clustered failover paths into partitions", + "paths", len(paths), "partitions", len(partitions)) + + // Maximal cliques overlap: a host with a mutual edge into two otherwise + // disjoint cliques (a—b, b—c but not a—c) appears in both. The planner keys + // reservations by target host name ("failover-host-"+host), so a host picked + // as a failover target by two partitions would yield two reservations with + // the same name but conflicting SourceHosts, and whichever is written last to + // the desired map wins — silently dropping the other partition's coverage. To + // keep the planner input disjoint we assign each shared host to exactly one + // partition: the smallest it belongs to. Assigning to the smallest keeps the + // most partitions alive — a small partition only survives if it retains its + // shared hosts, whereas a large one usually stays above the two-host minimum + // even after giving one up — so failover space reaches as many partitions as + // possible. Ties are broken by the partition's first host name so the choice + // is deterministic. Partitions are already sorted by their first member, so a + // stable sort by ascending size keeps that tiebreak order. + sort.SliceStable(partitions, func(i, j int) bool { + return len(partitions[i].Hosts) < len(partitions[j].Hosts) + }) + assigned := map[string]bool{} + deduped := make([]Partition, 0, len(partitions)) + for _, partition := range partitions { + kept := make([]hv1.Hypervisor, 0, len(partition.Hosts)) + for _, hv := range partition.Hosts { + if assigned[hv.Name] { + continue + } + assigned[hv.Name] = true + kept = append(kept, hv) + } + // Drop a partition emptied out by the dedupe, and one reduced below two + // hosts: a lone host offers no failover peer (see PlanReservations). + if len(kept) >= 2 { + // Keep the partition's supported flavors as computed over its full + // membership. Dropping a shared host can only relax the all-hosts + // intersection, never tighten it, so every listed flavor still runs on + // every remaining host — the set is at worst conservative. + deduped = append(deduped, Partition{Hosts: kept, Flavors: partition.Flavors}) + } + } + // Restore the deterministic ordering the builder produced (by first host + // name), which the size sort above disturbed. + sort.SliceStable(deduped, func(i, j int) bool { + return deduped[i].Hosts[0].Name < deduped[j].Hosts[0].Name + }) + partitions = deduped + labels := characterizePartitions(partitions) + for i, partition := range partitions { + // Describe what sets this partition apart from the others: the shared + // fields no other partition shares, plus the flavors it can run. See + // characterizePartitions and Partition.Flavors. + log.V(1).Info("characterized failover partition", + "partition", i, "partitionSize", len(partition.Hosts), + "distinctBy", labels[i], "flavors", partition.Flavors) + } + + // Plan the desired reservations per partition and collect them into one map. + desired := make(map[corev1.ObjectReference]v1alpha1.Reservation) + for i, partition := range partitions { + description := describePartition(labels[i], partition.Flavors) + planned := g.reservationPlanner.PlanReservations(topo, partition.Hosts, description) + for _, reservation := range planned { + desired[corev1.ObjectReference{Name: reservation.Name}] = reservation + } + log.V(1).Info("planned failover hosts for partition", + "partitionSize", len(partition.Hosts), "planned", len(planned)) + } + log.Info("generated desired failover host reservations", + "partitions", len(partitions), "desiredReservations", len(desired), + "existingReservations", len(topo.existing)) + g.Monitor.RecordCalculation(calculationSummary{ + duration: time.Since(start), + partitions: len(partitions), + desiredReservations: len(desired), + existingReservations: len(topo.existing), + }) + + return desired, topo, true +} + +// collect snapshots the current cluster into a topology. It returns ok=false if +// the snapshot is inconsistent (e.g. a hypervisor lacks an availability zone), +// in which case the pass must abort rather than compute on a partial view. +func (g *pipelineGenerator) collect(ctx context.Context) (topology, bool) { + log := logf.FromContext(ctx).WithName("failoverhost-controller-calculation-loop") + + hvs := &hv1.HypervisorList{} + if err := g.List(ctx, hvs); err != nil { + log.Error(err, "failed to list hypervisors") + return topology{}, false + } + hvsByName := make(map[string]hv1.Hypervisor) + for i := range hvs.Items { + hvsByName[hvs.Items[i].Name] = hvs.Items[i] + } + log.V(1).Info("listed cluster state", "hypervisors", len(hvsByName)) + + // Every hypervisor must carry an availability zone: the seeder permutes + // flavors across these AZs, and the planner stamps each reservation with its + // host's AZ. A hypervisor missing its AZ label is a torn snapshot, so abort + // rather than seed or plan from an inconsistent set of hosts. + for name, hv := range hvsByName { + if _, ok := hv.Labels[corev1.LabelTopologyZone]; !ok { + log.Error(nil, "hypervisor has no availability zone, aborting", "host", name) + return topology{}, false + } + } + + allReservations := &v1alpha1.ReservationList{} + if err := g.List(ctx, allReservations); err != nil { + log.Error(err, "failed to list existing failover host reservations") + return topology{}, false + } + existing := make(map[corev1.ObjectReference]v1alpha1.Reservation) + for _, reservation := range allReservations.Items { + if reservation.Spec.Type == v1alpha1.ReservationTypeFailoverHost { + existing[corev1.ObjectReference{Name: reservation.Name}] = reservation + } + } + + return topology{hypervisors: hvsByName, existing: existing}, true +} diff --git a/internal/scheduling/reservations/failoverhost/generation_test.go b/internal/scheduling/reservations/failoverhost/generation_test.go new file mode 100644 index 000000000..074ea1a98 --- /dev/null +++ b/internal/scheduling/reservations/failoverhost/generation_test.go @@ -0,0 +1,192 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package failoverhost + +import ( + "context" + "testing" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + corev1 "k8s.io/api/core/v1" +) + +// fakeSeeder returns a fixed set of seeds, so generator tests drive the pipeline +// without reading the flavor-groups Knowledge CRD. +type fakeSeeder struct{ seeds []Seed } + +func (f fakeSeeder) Seeds(context.Context, topology) ([]Seed, error) { return f.seeds, nil } + +// fakePartitionBuilder returns a fixed set of partitions regardless of input, so +// a test can drive Generate with overlapping cliques the real builder would +// produce. +type fakePartitionBuilder struct{ partitions []Partition } + +func (f fakePartitionBuilder) BuildPartitions([]Path) []Partition { + return f.partitions +} + +// TestGenerateProducesReservations exercises the whole pipeline end-to-end +// against a fake cluster and a stub external scheduler: collect -> Seeds -> +// FindPaths (via HTTP) -> BuildPartitions -> PlanReservations. It asserts the +// generator turns a pair of mutually-evacuable hosts into a desired failover +// host reservation. +func TestGenerateProducesReservations(t *testing.T) { + // Two equally-sized hosts. A single flavor seed is accepted on both, so the + // path finder discovers the mutual edges hvA<->hvB and they form one partition. + hvA := buildHV("hvA", 64, 32) + hvB := buildHV("hvB", 64, 32) + + finder, server := stubScheduler(t, map[string][]string{ + "flavor": {"hvA", "hvB"}, + }, nil) + defer server.Close() + + g := &pipelineGenerator{ + Client: newTestClient(t, hvA, hvB), + seeder: fakeSeeder{seeds: []Seed{seed("flavor")}}, + pathFinder: finder, + partitionBuilder: cliquePartitionBuilder{}, + reservationPlanner: ratioReservationPlanner{ratio: 0.5, minimum: 1}, + } + desired, _, ok := g.Generate(context.Background()) + if !ok { + t.Fatalf("Generate returned ok=false") + } + if len(desired) == 0 { + t.Fatalf("expected at least one desired reservation, got none") + } + for _, r := range desired { + if r.Spec.Type != v1alpha1.ReservationTypeFailoverHost { + t.Errorf("unexpected reservation type %q", r.Spec.Type) + } + if r.Spec.AvailabilityZone != "az1" { + t.Errorf("availability zone = %q, want az1", r.Spec.AvailabilityZone) + } + } +} + +// TestGenerateDeduplicatesSharedHost checks that when overlapping maximal +// cliques share a host (hvB in both {hvA,hvB} and {hvB,hvC}), Generate assigns +// that host to exactly one partition before planning, so planning never sees hvB +// in two partitions and cannot emit two reservations with the same name. The two +// partitions are equal size, so the deterministic tiebreak keeps hvB in the one +// with the smaller first host name ({hvA,hvB}); the other shrinks to the single +// host hvC, too small to reserve. The surviving {hvA,hvB} partition reserves one +// target (hvA, the first equal-cost host), whose only source host is hvB — proof +// hvB was planned once, as a source, not as a conflicting second target. +func TestGenerateDeduplicatesSharedHost(t *testing.T) { + hvA := buildHV("hvA", 64, 32) + hvB := buildHV("hvB", 64, 32) + hvC := buildHV("hvC", 64, 32) + + // The path finder's edges are irrelevant here: the fake builder ignores them + // and hands Generate the overlapping cliques directly. The stub still needs an + // answer so FindPaths succeeds; one seed accepted on all three suffices. + finder, server := stubScheduler(t, map[string][]string{ + "flavor": {"hvA", "hvB", "hvC"}, + }, nil) + defer server.Close() + + g := &pipelineGenerator{ + Client: newTestClient(t, hvA, hvB, hvC), + seeder: fakeSeeder{seeds: []Seed{seed("flavor")}}, + pathFinder: finder, + partitionBuilder: fakePartitionBuilder{partitions: []Partition{ + {Hosts: []hv1.Hypervisor{*hvA, *hvB}}, {Hosts: []hv1.Hypervisor{*hvB, *hvC}}, + }}, + reservationPlanner: ratioReservationPlanner{ratio: 0.5, minimum: 1}, + } + desired, _, ok := g.Generate(context.Background()) + if !ok { + t.Fatalf("Generate returned ok=false") + } + // Exactly one reservation survives: {hvA,hvB} reserves hvA, whose source host + // is the deduped hvB. {hvB,hvC} lost hvB and shrank to lone hvC, reserving + // nothing. + if len(desired) != 1 { + t.Fatalf("desired reservations = %d, want 1", len(desired)) + } + aRes, ok := desired[corev1.ObjectReference{Name: "failover-host-hvA"}] + if !ok { + t.Fatalf("expected a reservation targeting hvA") + } + if got := aRes.Spec.FailoverHostReservation.SourceHosts; len(got) != 1 || got[0] != "hvB" { + t.Errorf("hvA source hosts = %v, want [hvB]", got) + } +} + +// TestGenerateSharedHostKeptInSmallestPartition checks that when a shared host +// sits in partitions of different sizes, Generate assigns it to the smallest +// one, keeping the most partitions alive. Here hvC is shared between +// {hvA,hvB,hvC} (size 3) and {hvC,hvD} (size 2). Keeping hvC in the smaller +// {hvC,hvD} lets that partition survive, while {hvA,hvB,hvC} merely shrinks to +// {hvA,hvB} and stays above the two-host minimum — so both partitions retain a +// failover target. Keeping hvC in the larger partition instead would collapse +// {hvC,hvD} to a lone host and leave hvD unprotected. +func TestGenerateSharedHostKeptInSmallestPartition(t *testing.T) { + hvA := buildHV("hvA", 64, 32) + hvB := buildHV("hvB", 64, 32) + hvC := buildHV("hvC", 64, 32) + hvD := buildHV("hvD", 64, 32) + + finder, server := stubScheduler(t, map[string][]string{ + "flavor": {"hvA", "hvB", "hvC", "hvD"}, + }, nil) + defer server.Close() + + g := &pipelineGenerator{ + Client: newTestClient(t, hvA, hvB, hvC, hvD), + seeder: fakeSeeder{seeds: []Seed{seed("flavor")}}, + pathFinder: finder, + partitionBuilder: fakePartitionBuilder{partitions: []Partition{ + {Hosts: []hv1.Hypervisor{*hvA, *hvB, *hvC}}, {Hosts: []hv1.Hypervisor{*hvC, *hvD}}, + }}, + reservationPlanner: ratioReservationPlanner{ratio: 0.5, minimum: 1}, + } + desired, _, ok := g.Generate(context.Background()) + if !ok { + t.Fatalf("Generate returned ok=false") + } + // Both partitions survive: {hvC,hvD} keeps the shared hvC and {hvA,hvB,hvC} + // shrinks to {hvA,hvB}. Each reserves one target, so the shared host did not + // cost either partition its coverage. + if len(desired) != 2 { + t.Fatalf("desired reservations = %d, want 2", len(desired)) + } + // {hvC,hvD} must still exist, proving hvC stayed in the smaller partition. + // Whichever of hvC/hvD is reserved, its sole source host is the other. + cdReserved := false + for name, r := range desired { + sources := r.Spec.FailoverHostReservation.SourceHosts + if name.Name == "failover-host-hvC" && len(sources) == 1 && sources[0] == "hvD" { + cdReserved = true + } + if name.Name == "failover-host-hvD" && len(sources) == 1 && sources[0] == "hvC" { + cdReserved = true + } + } + if !cdReserved { + t.Errorf("expected a {hvC,hvD} reservation, got %v", desired) + } +} + +// TestGenerateAbortsOnMissingAvailabilityZone checks that collect rejects a torn +// snapshot: a hypervisor without an availability-zone label makes Generate +// return ok=false so the pass is abandoned rather than seeding from an +// inconsistent host set. +func TestGenerateAbortsOnMissingAvailabilityZone(t *testing.T) { + hvA := buildHV("hvA", 64, 32) + hvA.Labels = nil // strip the availability-zone label + + g := &pipelineGenerator{ + Client: newTestClient(t, hvA), + seeder: fakeSeeder{seeds: []Seed{seed("flavor")}}, + partitionBuilder: cliquePartitionBuilder{}, + reservationPlanner: ratioReservationPlanner{ratio: 0.5, minimum: 1}, + } + if _, _, ok := g.Generate(context.Background()); ok { + t.Fatalf("expected Generate to abort (ok=false) on a hypervisor with no AZ") + } +} diff --git a/internal/scheduling/reservations/failoverhost/integration_test.go b/internal/scheduling/reservations/failoverhost/integration_test.go new file mode 100644 index 000000000..013d8e269 --- /dev/null +++ b/internal/scheduling/reservations/failoverhost/integration_test.go @@ -0,0 +1,520 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package failoverhost + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "slices" + "sync" + "testing" + + novaapi "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" + "github.com/cobaltcore-dev/cortex/internal/scheduling/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/plugins/filters" + "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/plugins/weighers" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// ============================================================================ +// Integration Tests +// +// These exercise the whole failover-host generation pipeline (seeder -> +// path finder -> clique builder -> planner) against a *real* Nova +// filter/weigher pipeline served over HTTP, rather than a stubbed scheduler. +// The invariant under test is the ADR 0011 whole-host guarantee: a reserved +// failover target must be able to absorb the entire workload of every host it +// is recorded as rescuing. Since eligibility is static (ADR A5) that reduces +// to: the target must accept every protected flavor its source hosts accept. +// ============================================================================ + +// IntegrationTestCase defines one failover-host generation scenario. +type IntegrationTestCase struct { + Name string + Hypervisors []*hv1.Hypervisor + Flavors []compute.FlavorInGroup + + // Ratio/Minimum configure the planner. Ratio defaults to 1.0 and Minimum to + // 1 when left zero, so a scenario reserves as many targets as the partition + // allows unless it says otherwise. + Ratio float64 + Minimum int + + // ExpectReservations asserts at least one reservation is produced (most + // scenarios want this; a scenario proving nothing is reserved sets it false). + ExpectReservations bool + + // ForbiddenTargets names hosts that must never be reserved as a failover + // target, e.g. a host too small to absorb a larger peer's whole workload. + ForbiddenTargets []string + + // ExpectReservationCount, when non-zero, asserts the exact number of + // reservations produced. Used by scenarios that must reserve more than one + // target so the sibling-source wiring can be checked. + ExpectReservationCount int + + // ExpectSiblingSources asserts every reservation lists all the other + // reservations' target hosts among its source hosts — the post-failover + // shortcut a multi-target partition relies on. + ExpectSiblingSources bool +} + +func TestIntegration(t *testing.T) { + const traitX = "CUSTOM_TRAIT_X" + testCases := []IntegrationTestCase{ + // ===================================================================== + // Whole-host evacuability + // ===================================================================== + { + // The general flavor makes all four hosts mutually evacuable, so a + // naive clustering would lump the two plain "basic" hosts into the same + // partition as the two trait-X hosts and could reserve a basic host as + // their failover target. But a CUSTOM_TRAIT_X VM on a special host could + // never restart on a basic host: the real scheduler rejects the trait-X + // flavor there. The two special hosts on their own form a valid trait-X + // partition (and the two basic hosts a valid general one), so the + // coverage is genuinely achievable — the algorithm just must not merge + // the incompatible hosts together. A single shared flavor is enough to + // forge a rescue link, so this is the regression guard for the + // same-flavor-set check that keeps the clique step from merging hosts + // that do not accept exactly the same flavors. + Name: "trait-incompatible host is not reserved as a failover target", + Hypervisors: []*hv1.Hypervisor{ + hvWithTraits("basic-1"), + hvWithTraits("basic-2"), + hvWithTraits("special-1", traitX), + hvWithTraits("special-2", traitX), + }, + Flavors: []compute.FlavorInGroup{ + flavor("general", ""), + flavor("traitx", traitX), + }, + // Production defaults: floor(4 × 0.1) = 0, raised to the minimum of 1, + // so exactly one host is reserved. It is the cheapest/first by name, + // "basic-1", which cannot run traitx — so even a single reservation + // violates the whole-host guarantee. + Ratio: 0.1, + Minimum: 1, + ExpectReservations: true, + }, + // ===================================================================== + // Baseline: a homogeneous partition is safe + // ===================================================================== + { + // Three identical hosts accept the one flavor, so any of them can + // rescue any other: whichever becomes the target trivially satisfies + // the whole-host invariant. Guards against the check above being + // vacuously true. + Name: "homogeneous partition reserves a compatible target", + Hypervisors: []*hv1.Hypervisor{ + hvWithTraits("hv-1"), + hvWithTraits("hv-2"), + hvWithTraits("hv-3"), + }, + Flavors: []compute.FlavorInGroup{flavor("general", "")}, + ExpectReservations: true, + }, + // ===================================================================== + // Whole-host capacity fit + // ===================================================================== + { + // Two large hosts and one small host all accept the one general flavor, + // so under AssumeEmptyHosts the real capacity filter admits all three and + // they co-accept the seed. But an evacuation moves a whole host's + // workload at once, so a large host's VMs could never fit on the small + // one: the path finder's whole-host fit check must drop every + // large→small edge, leaving "small" only ever a source, never a + // mutual-edge target. The small host must therefore never be reserved — + // even though it happily accepts the flavor — or a large peer's workload + // would have nowhere to land. + Name: "undersized host is never reserved as a failover target", + Hypervisors: []*hv1.Hypervisor{ + buildHV("large-1", 128, 64), + buildHV("large-2", 128, 64), + buildHV("small-1", 16, 8), + }, + Flavors: []compute.FlavorInGroup{flavor("general", "")}, + ExpectReservations: true, + ForbiddenTargets: []string{"small-1"}, + }, + // ===================================================================== + // Availability-zone isolation + // ===================================================================== + { + // Two hosts in az1 and two in az2 all accept the one flavor, but the real + // AZ filter only ever admits hosts of the seed's own zone, so a seed in + // az1 co-accepts only the az1 hosts and vice versa. The two zones must + // therefore form two separate partitions: no reservation may record a + // cross-AZ source, because a VM in az1 could not be evacuated onto an az2 + // host. Each zone reserves its own target, so the fleet yields exactly + // two reservations — a wrong merge into one four-host partition would, at + // the default ratio of 1.0, cap at three instead. + Name: "hosts in different AZs are not merged into one partition", + Hypervisors: []*hv1.Hypervisor{ + hvPtrInAZ("az1-1", "az1"), + hvPtrInAZ("az1-2", "az1"), + hvPtrInAZ("az2-1", "az2"), + hvPtrInAZ("az2-2", "az2"), + }, + Flavors: []compute.FlavorInGroup{flavor("general", "")}, + ExpectReservations: true, + ExpectReservationCount: 2, + }, + // ===================================================================== + // Multi-target partition: siblings list each other as sources + // ===================================================================== + { + // A ratio of 1.0 would reserve the whole partition, but the never-reserve- + // all rule caps it at len(partition)-1, so a three-host partition reserves + // two targets. Each reservation must then list the *other* reserved host + // among its sources: once a reserved host absorbs an evacuation and + // returns to service it instantly needs a target of its own, which is why + // the planner keeps the sibling reserved hosts in each other's source + // lists. Whole-host evacuability still holds since the hosts are identical. + Name: "multi-target partition records sibling reserved hosts as sources", + Hypervisors: []*hv1.Hypervisor{ + hvWithTraits("hv-1"), + hvWithTraits("hv-2"), + hvWithTraits("hv-3"), + }, + Flavors: []compute.FlavorInGroup{flavor("general", "")}, + Ratio: 1.0, + Minimum: 1, + ExpectReservations: true, + ExpectReservationCount: 2, + ExpectSiblingSources: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + env := newIntegrationTestEnv(t, tc.Hypervisors, tc.Flavors) + defer env.Close() + + ratio, minimum := tc.Ratio, tc.Minimum + if ratio == 0 { + ratio = 1.0 + } + if minimum == 0 { + minimum = 1 + } + + desired := env.Generate(t, ratio, minimum) + if tc.ExpectReservations && len(desired) == 0 { + t.Fatal("expected at least one failover host reservation, got none") + } + if tc.ExpectReservationCount != 0 && len(desired) != tc.ExpectReservationCount { + t.Fatalf("expected %d reservations, got %d", tc.ExpectReservationCount, len(desired)) + } + for _, res := range desired { + if slices.Contains(tc.ForbiddenTargets, res.Spec.TargetHost) { + t.Errorf("host %q was reserved as a failover target but must never be", res.Spec.TargetHost) + } + } + if tc.ExpectSiblingSources { + env.VerifySiblingSources(t, desired) + } + + env.VerifyWholeHostEvacuable(t, desired) + }) + } +} + +// ============================================================================ +// Test Environment +// ============================================================================ + +// integrationTestEnv wires a real Nova filter/weigher pipeline (served over +// HTTP) on top of a fake k8s client holding the fleet and the flavor-groups +// Knowledge CRD, so the generation pipeline can be driven end to end. +type integrationTestEnv struct { + k8sClient client.Client + server *httptest.Server + schedulerURL string + flavors []compute.FlavorInGroup + hypervisors map[string]hv1.Hypervisor +} + +func (env *integrationTestEnv) Close() { env.server.Close() } + +// newIntegrationTestEnv builds the fake cluster and the real scheduler pipeline. +func newIntegrationTestEnv(t *testing.T, hypervisors []*hv1.Hypervisor, flavors []compute.FlavorInGroup) *integrationTestEnv { + t.Helper() + + objects := make([]client.Object, 0, len(hypervisors)+1) + hvsByName := make(map[string]hv1.Hypervisor, len(hypervisors)) + for _, hv := range hypervisors { + objects = append(objects, hv) + hvsByName[hv.Name] = *hv + } + objects = append(objects, flavorGroupsKnowledge(t, true, flavors...)) + + k8sClient := newTestClient(t, objects...) + + controller := &nova.FilterWeigherPipelineController{ + BasePipelineController: lib.BasePipelineController[lib.FilterWeigherPipeline[novaapi.ExternalSchedulerRequest]]{ + Client: k8sClient, + Pipelines: make(map[string]lib.FilterWeigherPipeline[novaapi.ExternalSchedulerRequest]), + PipelineConfigs: make(map[string]v1alpha1.Pipeline), + }, + Monitor: sharedPipelineMonitor(), + } + // Mirror the production KVM pipeline for the dimensions that shape a failover + // partition: AZ, trait compatibility and whole-host capacity. + pipeline := v1alpha1.Pipeline{ + ObjectMeta: metav1.ObjectMeta{Name: evacProbePipeline}, + Spec: v1alpha1.PipelineSpec{ + Type: v1alpha1.PipelineTypeFilterWeigher, + Filters: []v1alpha1.FilterSpec{ + {Name: "filter_correct_az"}, + {Name: "filter_has_requested_traits"}, + {Name: "filter_has_enough_capacity"}, + }, + }, + } + result := lib.InitNewFilterWeigherPipeline( + context.Background(), k8sClient, pipeline.Name, + filters.Index, pipeline.Spec.Filters, + weighers.Index, pipeline.Spec.Weighers, + controller.Monitor, + ) + if len(result.FilterErrors) > 0 || len(result.WeigherErrors) > 0 { + t.Fatalf("init pipeline: filters=%v weighers=%v", result.FilterErrors, result.WeigherErrors) + } + controller.Pipelines[pipeline.Name] = result.Pipeline + controller.PipelineConfigs[pipeline.Name] = pipeline + + api := &integrationHTTPAPI{delegate: controller} + mux := http.NewServeMux() + mux.HandleFunc("/scheduler/nova/external", api.handle) + server := httptest.NewServer(mux) + + return &integrationTestEnv{ + k8sClient: k8sClient, + server: server, + schedulerURL: server.URL + "/scheduler/nova/external", + flavors: flavors, + hypervisors: hvsByName, + } +} + +// ============================================================================ +// Environment Helper Methods +// ============================================================================ + +// Generate runs the real generation pipeline (real seeder, HTTP path finder, +// clique builder and planner) and returns the desired reservations. +func (env *integrationTestEnv) Generate(t *testing.T, ratio float64, minimum int) map[corev1.ObjectReference]v1alpha1.Reservation { + t.Helper() + g := &pipelineGenerator{ + Client: env.k8sClient, + seeder: &seeder{k8s: env.k8sClient}, + pathFinder: &evacuationPathFinder{httpClient: http.DefaultClient, schedulerURL: env.schedulerURL}, + partitionBuilder: cliquePartitionBuilder{}, + reservationPlanner: ratioReservationPlanner{ratio: ratio, minimum: minimum}, + } + desired, _, ok := g.Generate(context.Background()) + if !ok { + t.Fatal("Generate returned ok=false") + } + return desired +} + +// AcceptsByHost returns, per host, the set of flavors the real scheduler accepts +// on it — the ground truth the reservations must respect. It reuses the path +// finder's own evacuation-intent empty-host probe so the answer matches exactly +// what the generation pipeline sees. It probes every availability zone present +// in the fleet (the AZ filter only ever admits hosts of the seed's own zone), so +// the ground truth covers multi-AZ fleets too. +func (env *integrationTestEnv) AcceptsByHost(t *testing.T) map[string]map[string]bool { + t.Helper() + zones := map[string]bool{} + for _, hv := range env.hypervisors { + zones[hv.Labels[corev1.LabelTopologyZone]] = true + } + finder := &evacuationPathFinder{httpClient: http.DefaultClient, schedulerURL: env.schedulerURL} + accepts := map[string]map[string]bool{} + for _, flavor := range env.flavors { + for zone := range zones { + accepting, err := finder.fetchSeedCandidates(context.Background(), Seed{ + Flavor: Flavor{ + Name: flavor.Name, + MemoryMB: flavor.MemoryMB, + VCPUs: flavor.VCPUs, + ExtraSpecs: flavor.ExtraSpecs, + }, + AvailabilityZone: zone, + }, env.hypervisors) + if err != nil { + t.Fatalf("probe accepting for %q in %q: %v", flavor.Name, zone, err) + } + for _, hv := range accepting { + if accepts[hv.Name] == nil { + accepts[hv.Name] = map[string]bool{} + } + accepts[hv.Name][flavor.Name] = true + } + } + } + return accepts +} + +// ============================================================================ +// Verification Helpers +// ============================================================================ + +// VerifyWholeHostEvacuable asserts the ADR 0011 invariant: for every reservation, +// the target host must accept every protected flavor each of its recorded source +// hosts accepts. Otherwise that source's VMs of the uncovered flavor would have +// nowhere to restart on the reserved target. +func (env *integrationTestEnv) VerifyWholeHostEvacuable(t *testing.T, desired map[corev1.ObjectReference]v1alpha1.Reservation) { + t.Helper() + accepts := env.AcceptsByHost(t) + for _, res := range desired { + target := res.Spec.TargetHost + if res.Spec.FailoverHostReservation == nil { + t.Fatalf("reservation %s has no FailoverHostReservation spec", res.Name) + } + for _, source := range res.Spec.FailoverHostReservation.SourceHosts { + for flavor := range accepts[source] { + if !accepts[target][flavor] { + t.Errorf("reservation %s: target %q is recorded as a failover host for source %q, "+ + "but cannot run flavor %q that %q accepts — the source's %q VMs could not be evacuated onto it", + res.Name, target, source, flavor, source, flavor) + } + } + } + } +} + +// VerifySiblingSources asserts that in a partition reserving more than one +// failover host, every reservation lists each *other* reserved host among its +// source hosts. The planner keeps the reserved hosts in each other's source +// lists on purpose: once a reserved host absorbs an evacuation and returns to +// service it instantly needs a failover target of its own, and leaving it a +// recorded source means that target is already in place. +func (env *integrationTestEnv) VerifySiblingSources(t *testing.T, desired map[corev1.ObjectReference]v1alpha1.Reservation) { + t.Helper() + targets := make([]string, 0, len(desired)) + for _, res := range desired { + targets = append(targets, res.Spec.TargetHost) + } + for _, res := range desired { + if res.Spec.FailoverHostReservation == nil { + t.Fatalf("reservation %s has no FailoverHostReservation spec", res.Name) + } + for _, sibling := range targets { + if sibling == res.Spec.TargetHost { + continue + } + if !slices.Contains(res.Spec.FailoverHostReservation.SourceHosts, sibling) { + t.Errorf("reservation for target %q does not list sibling reserved host %q as a source", + res.Spec.TargetHost, sibling) + } + } + } +} + +// evacProbePipeline is the name of the real Nova filter/weigher pipeline the +// probe exercises. +const evacProbePipeline = "kvm-general-purpose-load-balancing" + +// integrationHTTPAPI is a thin, deterministic HTTP front-end for the real Nova +// filter/weigher pipeline — the same shape the failover package's integration +// tests use. It does NOT shuffle, so the probe answers are stable. +type integrationHTTPAPI struct { + delegate nova.HTTPAPIDelegate +} + +func (a *integrationHTTPAPI) handle(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + var reqData novaapi.ExternalSchedulerRequest + if err := json.NewDecoder(r.Body).Decode(&reqData); err != nil { + http.Error(w, "decode", http.StatusBadRequest) + return + } + rawBytes, err := json.Marshal(reqData) + if err != nil { + http.Error(w, "marshal", http.StatusInternalServerError) + return + } + decision := &v1alpha1.Decision{ + TypeMeta: metav1.TypeMeta{Kind: "Decision", APIVersion: "cortex.cloud/v1alpha1"}, + ObjectMeta: metav1.ObjectMeta{GenerateName: "nova-"}, + Spec: v1alpha1.DecisionSpec{ + SchedulingDomain: v1alpha1.SchedulingDomainNova, + PipelineRef: corev1.ObjectReference{Name: evacProbePipeline}, + ResourceID: reqData.Spec.Data.InstanceUUID, + NovaRaw: &runtime.RawExtension{Raw: rawBytes}, + }, + } + if err := a.delegate.ProcessNewDecisionFromAPI(r.Context(), decision); err != nil { + http.Error(w, "process", http.StatusInternalServerError) + return + } + if decision.Status.Result == nil { + http.Error(w, "no result", http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(novaapi.ExternalSchedulerResponse{Hosts: decision.Status.Result.OrderedHosts}); err != nil { + http.Error(w, "encode", http.StatusInternalServerError) + } +} + +// ============================================================================ +// Object Creation Functions +// ============================================================================ + +// hvWithTraits builds a fixed-size hypervisor in az1 with the given traits. The +// size is uniform across the fleet so every host is a valid gang-fit target for +// every other, isolating the trait (not capacity) as the compatibility axis. +func hvWithTraits(name string, traits ...string) *hv1.Hypervisor { + h := buildHV(name, 64, 32) + h.Status.Traits = traits + return h +} + +// hvPtrInAZ builds a fixed-size hypervisor (matching hvWithTraits) in the given +// availability zone, so a scenario can exercise the AZ filter rather than only +// az1 (buildHV, which it reuses, hardcodes az1). It is distinct from seeding +// test's hvInAZ, which returns a value and sets no capacity. +func hvPtrInAZ(name, az string) *hv1.Hypervisor { + h := buildHV(name, 64, 32) + h.Labels[corev1.LabelTopologyZone] = az + return h +} + +// flavor builds a generic flavor that requires the given trait (or none). Its +// hypervisor type is set to a supported value ("ch") only so the seeder keeps +// it; the specific type is irrelevant to what this test exercises. +func flavor(name, requiredTrait string) compute.FlavorInGroup { + extra := map[string]string{"capabilities:hypervisor_type": "ch"} + if requiredTrait != "" { + extra["trait:"+requiredTrait] = "required" + } + return compute.FlavorInGroup{Name: name, VCPUs: 1, MemoryMB: 1024, ExtraSpecs: extra} +} + +// sharedPipelineMonitor guards the one-time Prometheus collector registration +// the pipeline monitor performs, so multiple environments can be built in one +// test run without a duplicate-registration panic. +var ( + pipelineMonitorOnce sync.Once + pipelineMonitor lib.FilterWeigherPipelineMonitor +) + +func sharedPipelineMonitor() lib.FilterWeigherPipelineMonitor { + pipelineMonitorOnce.Do(func() { pipelineMonitor = lib.NewPipelineMonitor() }) + return pipelineMonitor +} diff --git a/internal/scheduling/reservations/failoverhost/monitor.go b/internal/scheduling/reservations/failoverhost/monitor.go new file mode 100644 index 000000000..099bec328 --- /dev/null +++ b/internal/scheduling/reservations/failoverhost/monitor.go @@ -0,0 +1,207 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package failoverhost + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +// Monitor provides Prometheus metrics for the failover host controller. It +// covers two activities: the periodic calculation loop that recomputes the +// desired reservations, and the reconciler that drives the actual Reservation +// objects toward that desired state. +// +// Unlike the sibling failover controller, the calculation loop here is +// cluster-wide (one pass covers every partition and availability zone at once, +// and a partition clique is not guaranteed to be single-AZ), so the loop-level +// metrics carry no availability-zone label. The scheduler-request and reconcile +// metrics do carry a small label each where it is actionable. +type Monitor struct { + calculationRuns prometheus.Counter + calculationDuration prometheus.Histogram + calculationAborts *prometheus.CounterVec + partitions prometheus.Gauge + desiredReservations prometheus.Gauge + existingReservations prometheus.Gauge + + schedulerRequests *prometheus.CounterVec + schedulerRequestDuration prometheus.Histogram + + reservationsCreated prometheus.Counter + reservationsUpdated prometheus.Counter + reservationsDeleted prometheus.Counter + reservationsErrors *prometheus.CounterVec +} + +// calculationSummary holds the statistics of a single calculation-loop pass, +// filled across the pass and recorded once at the end via RecordCalculation. +type calculationSummary struct { + duration time.Duration + partitions int + desiredReservations int + existingReservations int +} + +// NewMonitor creates a new monitor with Prometheus metrics. +func NewMonitor() *Monitor { + m := &Monitor{ + calculationRuns: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "cortex_failover_host_calculation_runs_total", + Help: "Total number of completed failover host calculation-loop passes since pod restart", + }), + calculationDuration: prometheus.NewHistogram(prometheus.HistogramOpts{ + Name: "cortex_failover_host_calculation_duration_seconds", + Help: "Duration of failover host calculation-loop passes", + Buckets: []float64{0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60}, + }), + calculationAborts: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "cortex_failover_host_calculation_aborts_total", + Help: "Total number of failover host calculation-loop passes aborted before producing a desired state, by reason", + }, []string{"reason"}), + partitions: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "cortex_failover_host_partitions", + Help: "Number of failover partitions from the last calculation-loop pass", + }), + desiredReservations: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "cortex_failover_host_desired_reservations", + Help: "Number of desired failover host reservations from the last calculation-loop pass", + }), + existingReservations: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "cortex_failover_host_existing_reservations", + Help: "Number of failover host reservations present in the cluster at the last calculation-loop pass", + }), + schedulerRequests: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "cortex_failover_host_scheduler_requests_total", + Help: "Total number of external scheduler requests issued by the failover host path finder, by status (HTTP status code or 'error' on transport failure)", + }, []string{"status"}), + schedulerRequestDuration: prometheus.NewHistogram(prometheus.HistogramOpts{ + Name: "cortex_failover_host_scheduler_request_duration_seconds", + Help: "Duration of external scheduler requests issued by the failover host path finder", + Buckets: []float64{0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30}, + }), + reservationsCreated: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "cortex_failover_host_reservations_created_total", + Help: "Total number of failover host reservations created since pod restart", + }), + reservationsUpdated: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "cortex_failover_host_reservations_updated_total", + Help: "Total number of failover host reservations updated since pod restart", + }), + reservationsDeleted: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "cortex_failover_host_reservations_deleted_total", + Help: "Total number of stale failover host reservations deleted since pod restart", + }), + reservationsErrors: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "cortex_failover_host_reservations_errors_total", + Help: "Total number of failed failover host reservation writes since pod restart, by operation", + }, []string{"op"}), + } + // Pre-initialize the labelled series so they appear at zero before the first + // event, which lets rate()/increase() alerts behave from pod startup. + m.calculationAborts.WithLabelValues("collect") + m.calculationAborts.WithLabelValues("seeding") + m.calculationAborts.WithLabelValues("pathfinding") + m.reservationsErrors.WithLabelValues("create") + m.reservationsErrors.WithLabelValues("update") + m.reservationsErrors.WithLabelValues("delete") + return m +} + +// RecordCalculation records the metrics of a single completed calculation pass. +func (m *Monitor) RecordCalculation(summary calculationSummary) { + if m == nil { + return + } + m.calculationRuns.Inc() + m.calculationDuration.Observe(summary.duration.Seconds()) + m.partitions.Set(float64(summary.partitions)) + m.desiredReservations.Set(float64(summary.desiredReservations)) + m.existingReservations.Set(float64(summary.existingReservations)) +} + +// RecordCalculationAbort records a calculation pass that aborted before +// producing a desired state. reason is "collect", "seeding" or "pathfinding". +func (m *Monitor) RecordCalculationAbort(reason string) { + if m == nil { + return + } + m.calculationAborts.WithLabelValues(reason).Inc() +} + +// RecordSchedulerRequest records one external scheduler request. status is the +// HTTP status code as a string, or "error" on a transport-level failure. +func (m *Monitor) RecordSchedulerRequest(status string, duration time.Duration) { + if m == nil { + return + } + m.schedulerRequests.WithLabelValues(status).Inc() + m.schedulerRequestDuration.Observe(duration.Seconds()) +} + +// RecordReservationCreated records a successful reservation create. +func (m *Monitor) RecordReservationCreated() { + if m == nil { + return + } + m.reservationsCreated.Inc() +} + +// RecordReservationUpdated records a successful reservation update. +func (m *Monitor) RecordReservationUpdated() { + if m == nil { + return + } + m.reservationsUpdated.Inc() +} + +// RecordReservationDeleted records a successful stale-reservation delete. +func (m *Monitor) RecordReservationDeleted() { + if m == nil { + return + } + m.reservationsDeleted.Inc() +} + +// RecordReservationError records a failed reservation write. op is "create", +// "update" or "delete". +func (m *Monitor) RecordReservationError(op string) { + if m == nil { + return + } + m.reservationsErrors.WithLabelValues(op).Inc() +} + +// Describe implements prometheus.Collector. +func (m *Monitor) Describe(ch chan<- *prometheus.Desc) { + m.calculationRuns.Describe(ch) + m.calculationDuration.Describe(ch) + m.calculationAborts.Describe(ch) + m.partitions.Describe(ch) + m.desiredReservations.Describe(ch) + m.existingReservations.Describe(ch) + m.schedulerRequests.Describe(ch) + m.schedulerRequestDuration.Describe(ch) + m.reservationsCreated.Describe(ch) + m.reservationsUpdated.Describe(ch) + m.reservationsDeleted.Describe(ch) + m.reservationsErrors.Describe(ch) +} + +// Collect implements prometheus.Collector. +func (m *Monitor) Collect(ch chan<- prometheus.Metric) { + m.calculationRuns.Collect(ch) + m.calculationDuration.Collect(ch) + m.calculationAborts.Collect(ch) + m.partitions.Collect(ch) + m.desiredReservations.Collect(ch) + m.existingReservations.Collect(ch) + m.schedulerRequests.Collect(ch) + m.schedulerRequestDuration.Collect(ch) + m.reservationsCreated.Collect(ch) + m.reservationsUpdated.Collect(ch) + m.reservationsDeleted.Collect(ch) + m.reservationsErrors.Collect(ch) +} diff --git a/internal/scheduling/reservations/failoverhost/monitor_test.go b/internal/scheduling/reservations/failoverhost/monitor_test.go new file mode 100644 index 000000000..3d109bf87 --- /dev/null +++ b/internal/scheduling/reservations/failoverhost/monitor_test.go @@ -0,0 +1,90 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package failoverhost + +import ( + "strings" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestMonitor_RecordCalculation(t *testing.T) { + m := NewMonitor() + m.RecordCalculation(calculationSummary{ + duration: 500 * time.Millisecond, + partitions: 3, + desiredReservations: 4, + existingReservations: 5, + }) + + if got := testutil.ToFloat64(m.calculationRuns); got != 1 { + t.Errorf("calculationRuns = %v, want 1", got) + } + if got := testutil.ToFloat64(m.partitions); got != 3 { + t.Errorf("partitions = %v, want 3", got) + } + if got := testutil.ToFloat64(m.desiredReservations); got != 4 { + t.Errorf("desiredReservations = %v, want 4", got) + } + if got := testutil.ToFloat64(m.existingReservations); got != 5 { + t.Errorf("existingReservations = %v, want 5", got) + } +} + +func TestMonitor_Counters(t *testing.T) { + m := NewMonitor() + m.RecordCalculationAbort("collect") + m.RecordCalculationAbort("pathfinding") + m.RecordCalculationAbort("pathfinding") + m.RecordReservationCreated() + m.RecordReservationUpdated() + m.RecordReservationDeleted() + m.RecordReservationError("create") + m.RecordSchedulerRequest("200", 50*time.Millisecond) + m.RecordSchedulerRequest("error", 100*time.Millisecond) + + expected := ` +# HELP cortex_failover_host_calculation_aborts_total Total number of failover host calculation-loop passes aborted before producing a desired state, by reason +# TYPE cortex_failover_host_calculation_aborts_total counter +cortex_failover_host_calculation_aborts_total{reason="collect"} 1 +cortex_failover_host_calculation_aborts_total{reason="pathfinding"} 2 +cortex_failover_host_calculation_aborts_total{reason="seeding"} 0 +` + if err := testutil.CollectAndCompare(m.calculationAborts, strings.NewReader(expected)); err != nil { + t.Error(err) + } + + if got := testutil.ToFloat64(m.reservationsCreated); got != 1 { + t.Errorf("reservationsCreated = %v, want 1", got) + } + if got := testutil.ToFloat64(m.reservationsUpdated); got != 1 { + t.Errorf("reservationsUpdated = %v, want 1", got) + } + if got := testutil.ToFloat64(m.reservationsDeleted); got != 1 { + t.Errorf("reservationsDeleted = %v, want 1", got) + } + if got := testutil.ToFloat64(m.reservationsErrors.WithLabelValues("create")); got != 1 { + t.Errorf("reservationsErrors{create} = %v, want 1", got) + } + if got := testutil.ToFloat64(m.schedulerRequests.WithLabelValues("200")); got != 1 { + t.Errorf("schedulerRequests{200} = %v, want 1", got) + } + if got := testutil.ToFloat64(m.schedulerRequests.WithLabelValues("error")); got != 1 { + t.Errorf("schedulerRequests{error} = %v, want 1", got) + } +} + +// A nil monitor must be safe to call so controllers and tests can run without one. +func TestMonitor_NilSafe(t *testing.T) { + var m *Monitor + m.RecordCalculation(calculationSummary{}) + m.RecordCalculationAbort("collect") + m.RecordSchedulerRequest("200", time.Second) + m.RecordReservationCreated() + m.RecordReservationUpdated() + m.RecordReservationDeleted() + m.RecordReservationError("create") +} diff --git a/internal/scheduling/reservations/failoverhost/partitioning.go b/internal/scheduling/reservations/failoverhost/partitioning.go new file mode 100644 index 000000000..3ce215ef5 --- /dev/null +++ b/internal/scheduling/reservations/failoverhost/partitioning.go @@ -0,0 +1,428 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package failoverhost + +import ( + "fmt" + "maps" + "slices" + "sort" + "strconv" + "strings" + + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +// Partition is one failover partition: a set of hypervisors in which every host +// can be evacuated onto every other host, plus the flavors that whole set can +// run. Flavors answers "which flavors can be spawned anywhere in this partition": +// a flavor is listed only if *every* host in the partition accepts it, so a +// listed flavor is guaranteed to run on all of them. Flavors is sorted and may be +// empty — e.g. when the members' accept sets are disjoint, so no single flavor is +// proven to fit everywhere in the partition. +type Partition struct { + // Hosts are the partition's members, sorted by name. + Hosts []hv1.Hypervisor + // Flavors are the flavor names accepted on every host of the partition, + // sorted. + Flavors []string +} + +// PartitionBuilder is the third pipeline stage. It clusters the directed +// failover edges found by the PathFinder into failover partitions. +// +// A failover partition is a maximal set of hypervisors in which every host can +// be evacuated onto every other host. For example, HANA hosts, general-purpose +// hosts and external-customer hosts typically each form their own partition, +// because their workloads can only ever fail over within the group. +type PartitionBuilder interface { + // BuildPartitions groups the failover Paths into partitions, each carrying the + // flavors its whole host set can run. The output is deterministic regardless + // of edge order. + BuildPartitions(paths []Path) []Partition +} + +// cliquePartitionBuilder builds partitions by finding maximal cliques in the +// undirected graph of mutual failover edges: an undirected edge a—b exists only +// if both a→b and b→a are valid paths, since failover requires that the hosts +// can evacuate onto each other. A failover partition is exactly a maximal clique +// in that graph — a largest-possible set of hosts all pairwise mutually +// evacuable. +// +// Empty hosts (hosts without any outgoing edge, e.g. because they host no vms) +// fall out as single-element cliques. They are recruited into an existing +// partition whose members all have an outgoing edge toward the empty host, as +// we then assume the reverse direction is also possible. +type cliquePartitionBuilder struct{} + +// BuildPartitions implements PartitionBuilder. It works in five steps: +// +// 1. Build the directed adjacency from the edges, record which flavors each host +// accepts (it is an endpoint of an edge carrying that flavor), and note which +// hosts have no outgoing edge at all (the "empty" hosts, handled specially in +// step 3). +// 2. Derive the undirected "mutual" adjacency (a—b iff both a→b and b→a *and* +// a and b accept the same flavors), and run Bron-Kerbosch over it to +// enumerate every maximal clique. Each clique is one partition. The +// same-flavors requirement enforces the ADR 0011 whole-host guarantee: any +// clique member may become the failover target with the rest its sources, so +// the target must run every flavor any source accepts; a single co-accepted +// flavor is enough to forge a mutual edge but not to prove this. Bron-Kerbosch +// recursively grows a candidate clique R by trying each still-eligible vertex +// in P, intersecting P and the already-seen set X with that vertex's +// neighbours; a clique is maximal when both P and X are empty. Non-empty hosts +// with no mutual edge are emitted as their own single-host partitions. +// 3. Recruit each empty host into the first partition all of whose members can +// reach it (member→empty for every member). An empty host that no partition +// fully reaches stays on its own. +// 4. Translate host names back to Hypervisor values and sort, so the result is +// deterministic despite the map iteration used throughout. +// 5. Compute each partition's supported flavors: the flavors accepted on *every* +// one of its hosts (the intersection of the per-host accept sets from step 1). +func (cliquePartitionBuilder) BuildPartitions(paths []Path) []Partition { + // Collect all hosts and the directed adjacency, keyed by host name. Also record + // which flavors each host accepts: a host is an endpoint of an edge exactly + // when it co-accepted that edge's flavors, so both endpoints accept them. + hvsByName := map[string]hv1.Hypervisor{} + outgoing := map[string]map[string]bool{} + acceptedBy := map[string]map[string]bool{} + accept := func(host string, flavors []string) { + if acceptedBy[host] == nil { + acceptedBy[host] = map[string]bool{} + } + for _, flavor := range flavors { + acceptedBy[host][flavor] = true + } + } + for _, path := range paths { + src, dst := path.Source, path.Target + hvsByName[src.Name] = src + hvsByName[dst.Name] = dst + if outgoing[src.Name] == nil { + outgoing[src.Name] = map[string]bool{} + } + outgoing[src.Name][dst.Name] = true + accept(src.Name, path.Flavors) + accept(dst.Name, path.Flavors) + } + + // Build the undirected adjacency from mutual edges only. Hosts without any + // outgoing edge are empty hosts and are recruited into partitions later. + emptyHosts := []string{} + for name := range hvsByName { + if len(outgoing[name]) == 0 { + emptyHosts = append(emptyHosts, name) + } + } + mutual := map[string]map[string]bool{} + for a, dsts := range outgoing { + for b := range dsts { + // b must also reach a (the edge is mutual), and the two must accept the + // same flavors. The clique both hosts land in may designate either of + // them the failover target, with the other becoming its source; a target + // must run every flavor its source accepts (the ADR 0011 whole-host + // guarantee), so hosts with differing accept sets must never be merged. + // A single co-accepted flavor is enough to forge a mutual edge, but it is + // not enough to prove whole-host evacuability — e.g. two hosts sharing a + // general flavor where only one also runs a trait-gated flavor. + if outgoing[b][a] && sameFlavors(acceptedBy[a], acceptedBy[b]) { + if mutual[a] == nil { + mutual[a] = map[string]bool{} + } + mutual[a][b] = true + } + } + } + + // Find all maximal cliques in the undirected mutual graph using the + // Bron-Kerbosch algorithm. These are our failover partitions. + cliques := [][]string{} + all := map[string]bool{} + for a := range mutual { + all[a] = true + } + var bronKerbosch func(r, p, x map[string]bool) + bronKerbosch = func(r, p, x map[string]bool) { + if len(p) == 0 && len(x) == 0 { + if len(r) == 0 { + return // Don't emit the empty clique when there are no mutual edges. + } + clique := make([]string, 0, len(r)) + for name := range r { + clique = append(clique, name) + } + cliques = append(cliques, clique) + return + } + // Iterate over a copy of p, as p is mutated inside the loop. + candidates := make([]string, 0, len(p)) + for v := range p { + candidates = append(candidates, v) + } + for _, v := range candidates { + if !p[v] { + continue + } + nr := map[string]bool{v: true} + maps.Copy(nr, r) + np, nx := map[string]bool{}, map[string]bool{} + for n := range p { + if mutual[v][n] { + np[n] = true + } + } + for n := range x { + if mutual[v][n] { + nx[n] = true + } + } + bronKerbosch(nr, np, nx) + delete(p, v) + x[v] = true + } + } + bronKerbosch(map[string]bool{}, all, map[string]bool{}) + + // Non-empty hosts that share no mutual edge with any other host are not part + // of any clique. They form their own single-element partition. + covered := map[string]bool{} + for _, clique := range cliques { + for _, name := range clique { + covered[name] = true + } + } + for name := range outgoing { + if !covered[name] { + cliques = append(cliques, []string{name}) + } + } + + // Sort before recruitment so the result is deterministic: Bron-Kerbosch and + // the loops above both iterate maps, so clique contents and order are random + // per run. We sort each clique's members, then order the cliques, so an empty + // host reachable by several cliques is always recruited into the same one. + for _, clique := range cliques { + slices.Sort(clique) + } + sort.SliceStable(cliques, func(i, j int) bool { + return slices.Compare(cliques[i], cliques[j]) < 0 + }) + slices.Sort(emptyHosts) + + // effectiveCapacity returns a host's gang-fit capacity: its base capacity with + // effectiveCapacity overlaid on top (fallback = capacity), matching how the + // PathFinder sizes hosts. Copied into a fresh map so we never mutate Status. + effectiveCapacity := func(name string) map[hv1.ResourceName]resource.Quantity { + capa := map[hv1.ResourceName]resource.Quantity{} + maps.Copy(capa, hvsByName[name].Status.Capacity) + maps.Copy(capa, hvsByName[name].Status.EffectiveCapacity) + return capa + } + for _, empty := range emptyHosts { + emptyCapa := effectiveCapacity(empty) + recruited := false + for i, clique := range cliques { + allReach := len(clique) > 0 + for _, member := range clique { + // Every member must reach the empty host (member can evacuate onto + // it), the empty host must be the same effective size as the member, + // and the two must accept the same flavors. The empty host has no vms, + // so Step 1 never produced an empty->member edge to prove it can rescue + // its peers; we fall back to the same gang-fit rule and require equal + // capacity. A larger empty host would be recorded as a source of a + // smaller reserved target it could never actually evacuate onto. The + // flavor check mirrors the mutual-edge rule: a recruited host may become + // the partition's target, so it must run every flavor its peers accept. + if !outgoing[member][empty] || + cmpResources(emptyCapa, effectiveCapacity(member)) != 0 || + !sameFlavors(acceptedBy[empty], acceptedBy[member]) { + allReach = false + break + } + } + if allReach { + cliques[i] = append(clique, empty) + recruited = true + break + } + } + if !recruited { + cliques = append(cliques, []string{empty}) + } + } + + // Translate the clique host names back into hypervisors and compute each + // partition's supported flavors: those accepted on *every* host, so a listed + // flavor is guaranteed to run anywhere in the partition. Sort for deterministic + // output, as the clustering relies on map iteration. + partitions := make([]Partition, 0, len(cliques)) + for _, clique := range cliques { + slices.Sort(clique) + hosts := make([]hv1.Hypervisor, 0, len(clique)) + for _, name := range clique { + hosts = append(hosts, hvsByName[name]) + } + partitions = append(partitions, Partition{ + Hosts: hosts, + Flavors: sharedFlavors(clique, acceptedBy), + }) + } + sort.SliceStable(partitions, func(i, j int) bool { + return partitions[i].Hosts[0].Name < partitions[j].Hosts[0].Name + }) + return partitions +} + +// sameFlavors reports whether two hosts accept exactly the same set of flavors. +// It is the flavor-compatibility test for merging hosts into a partition: since +// any member can become the partition's failover target and the rest its +// sources, the target must run every flavor any source accepts, so all members +// must share one accept set. An empty host's accept set comes from its inbound +// edges (see acceptedBy), so a recruited host is compared the same way. +func sameFlavors(a, b map[string]bool) bool { + if len(a) != len(b) { + return false + } + for flavor := range a { + if !b[flavor] { + return false + } + } + return true +} + +// sharedFlavors returns the flavors accepted on *every* host in the clique, +// sorted. A host accepts a flavor when it is an endpoint of an edge the path +// finder tagged with that flavor (both endpoints co-accepted it), so even a +// recruited host counts through its inbound edges. The result is empty when the +// members' accept sets don't intersect. +func sharedFlavors(clique []string, acceptedBy map[string]map[string]bool) []string { + if len(clique) == 0 { + return nil + } + shared := []string{} + for flavor := range acceptedBy[clique[0]] { + onAll := true + for _, host := range clique[1:] { + if !acceptedBy[host][flavor] { + onAll = false + break + } + } + if onAll { + shared = append(shared, flavor) + } + } + slices.Sort(shared) + return shared +} + +// characterizePartitions returns a short, human-readable label per partition +// describing what sets it apart from the others, e.g. "spec.aggregates=[hana]". +// The result is index-aligned with partitions. +// +// It compares partitions on just the three dimensions that actually shape a +// failover partition — aggregates, traits and (rounded) capacity — rather than +// reflecting over every field of the hypervisor. For each dimension we take the +// value shared by all of a partition's hosts (dropping it if they disagree), then +// keep only the *distinguishing* ones: a dimension is dropped if its shared value +// is identical across every partition, as such a fleet-wide constant says nothing +// about what makes a partition special. The survivors are rendered as a compact, +// sorted "key=value key=value" string for at-a-glance logging. Flavor support is +// reported separately (see Partition.Flavors), so it is not repeated here. +func characterizePartitions(partitions []Partition) []string { + // fields returns the three comparable dimensions of a hypervisor, each as a + // stable string: its aggregates, its traits, and its capacity rounded to GiB + // (so tiny per-node memory discrepancies don't split an otherwise-shared + // value). Capacity reuses hvCapacity and the GiB rounding cmpResources applies. + fields := func(hv hv1.Hypervisor) map[string]string { + aggregates := slices.Clone(hv.Spec.Aggregates) + slices.Sort(aggregates) + traits := slices.Clone(hv.Status.Traits) + slices.Sort(traits) + capa := hvCapacity(hv) + const gib = 1 << 30 + memGi := capa[hv1.ResourceMemory] + cpu := capa[hv1.ResourceCPU] + return map[string]string{ + "spec.aggregates": fmt.Sprintf("%v", aggregates), + "status.traits": fmt.Sprintf("%v", traits), + "status.capacity.memGi": strconv.FormatInt(memGi.Value()/gib, 10), + "status.capacity.cpu": cpu.String(), + } + } + // shared[i] holds the dimensions common to every hypervisor in partition i. + shared := make([]map[string]string, len(partitions)) + for i, partition := range partitions { + if len(partition.Hosts) == 0 { + shared[i] = map[string]string{} + continue + } + common := fields(partition.Hosts[0]) + for _, hv := range partition.Hosts[1:] { + f := fields(hv) + for key, want := range common { + if f[key] != want { + delete(common, key) + } + } + } + shared[i] = common + } + // Count how many partitions carry each (key, value). A dimension common to all + // of them is a fleet-wide constant and tells us nothing distinguishing. + seen := map[string]int{} + for _, common := range shared { + for key, val := range common { + seen[key+"="+val]++ + } + } + // Render the distinguishing dimensions of each partition as a sorted label. A + // dimension is distinguishing unless every partition shares it (a fleet-wide + // constant); with a single partition there is nothing to contrast against, so + // all its shared dimensions survive. + labels := make([]string, len(partitions)) + for i, common := range shared { + parts := make([]string, 0, len(common)) + for key, val := range common { + pair := key + "=" + val + if len(partitions) == 1 || seen[pair] < len(partitions) { + parts = append(parts, pair) + } + } + slices.Sort(parts) + labels[i] = strings.Join(parts, " ") + } + return labels +} + +// maxDescribedFlavors caps how many flavor names describePartition spells out, so +// a partition that runs hundreds of flavors doesn't bloat the reservation's +// PartitionDescription field. The rest are summarized as a "(+k more)" count; the +// full set is still logged per pass. +const maxDescribedFlavors = 5 + +// describePartition renders the human-readable PartitionDescription stored on each +// reservation: the distinguishing label (see characterizePartitions) followed by +// the flavors the partition can run (see Partition.Flavors), e.g. +// "spec.aggregates=[hana] | flavors: hana.4xl, hana.8xl (+3 more)". flavors is +// assumed sorted. Either half is omitted when empty, so a partition with no +// distinguishing label still reports its flavors and vice versa. +func describePartition(label string, flavors []string) string { + parts := []string{} + if label != "" { + parts = append(parts, label) + } + if len(flavors) > 0 { + shown := flavors + suffix := "" + if len(flavors) > maxDescribedFlavors { + shown = flavors[:maxDescribedFlavors] + suffix = fmt.Sprintf(" (+%d more)", len(flavors)-maxDescribedFlavors) + } + parts = append(parts, "flavors: "+strings.Join(shown, ", ")+suffix) + } + return strings.Join(parts, " | ") +} diff --git a/internal/scheduling/reservations/failoverhost/partitioning_test.go b/internal/scheduling/reservations/failoverhost/partitioning_test.go new file mode 100644 index 000000000..063ee4390 --- /dev/null +++ b/internal/scheduling/reservations/failoverhost/partitioning_test.go @@ -0,0 +1,331 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package failoverhost + +import ( + "slices" + "strings" + "testing" + + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// hv builds a minimal hypervisor identified only by its name. +func hv(name string) hv1.Hypervisor { + return hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: name}} +} + +// hvWithMemory builds a hypervisor with the given name and memory capacity in GiB. +func hvWithMemory(name string, memGiB int64) hv1.Hypervisor { + h := hv(name) + h.Status.Capacity = map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: *resource.NewQuantity(memGiB*(1<<30), resource.BinarySI), + } + return h +} + +// edgesHV turns directed hypervisor pairs into failover paths, preserving the +// hypervisors' capacity (unlike edges, which rebuilds bare hosts from names). No +// supporting flavors are attached; flavor propagation is covered separately. +func edgesHV(pairs ...[2]hv1.Hypervisor) []Path { + paths := make([]Path, 0, len(pairs)) + for _, p := range pairs { + paths = append(paths, Path{Source: p[0], Target: p[1]}) + } + return paths +} + +// edges turns a list of directed name pairs into failover paths. +func edges(pairs ...[2]string) []Path { + paths := make([]Path, 0, len(pairs)) + for _, p := range pairs { + paths = append(paths, Path{Source: hv(p[0]), Target: hv(p[1])}) + } + return paths +} + +// partitionNames extracts the sorted host names of each partition. +func partitionNames(partitions []Partition) [][]string { + out := make([][]string, 0, len(partitions)) + for _, partition := range partitions { + names := make([]string, 0, len(partition.Hosts)) + for _, hv := range partition.Hosts { + names = append(names, hv.Name) + } + slices.Sort(names) + out = append(out, names) + } + return out +} + +func TestBuildPartitions(t *testing.T) { + tests := []struct { + name string + paths []Path + want [][]string + }{ + { + name: "empty input", + paths: nil, + want: [][]string{}, + }, + { + name: "single mutual edge forms a partition", + paths: edges([2]string{"a", "b"}, [2]string{"b", "a"}), + want: [][]string{{"a", "b"}}, + }, + { + name: "one-directional edge is not a partition", + paths: edges([2]string{"a", "b"}), + // b is empty (no outgoing edge) and a can reach b, so a-b form one. + want: [][]string{{"a", "b"}}, + }, + { + name: "fully connected triangle", + paths: edges( + [2]string{"a", "b"}, [2]string{"b", "a"}, + [2]string{"b", "c"}, [2]string{"c", "b"}, + [2]string{"a", "c"}, [2]string{"c", "a"}, + ), + want: [][]string{{"a", "b", "c"}}, + }, + { + name: "two separate partitions", + paths: edges( + [2]string{"a", "b"}, [2]string{"b", "a"}, + [2]string{"c", "d"}, [2]string{"d", "c"}, + ), + want: [][]string{{"a", "b"}, {"c", "d"}}, + }, + { + name: "missing mutual edge splits the clique", + paths: edges( + [2]string{"a", "b"}, [2]string{"b", "a"}, + [2]string{"b", "c"}, [2]string{"c", "b"}, + // a and c are not mutually reachable. + ), + // a-b and b-c are the two maximal cliques. + want: [][]string{{"a", "b"}, {"b", "c"}}, + }, + { + name: "empty host recruited into partition reaching it", + paths: edges( + [2]string{"a", "b"}, [2]string{"b", "a"}, + // e has no outgoing edges but both a and b reach it. + [2]string{"a", "e"}, [2]string{"b", "e"}, + ), + want: [][]string{{"a", "b", "e"}}, + }, + { + name: "empty host not reachable by all stays alone", + paths: edges( + [2]string{"a", "b"}, [2]string{"b", "a"}, + // only a reaches e, so e cannot be recruited. + [2]string{"a", "e"}, + ), + want: [][]string{{"a", "b"}, {"e"}}, + }, + { + name: "larger empty host is not recruited", + paths: edgesHV( + [2]hv1.Hypervisor{hvWithMemory("a", 1), hvWithMemory("b", 1)}, + [2]hv1.Hypervisor{hvWithMemory("b", 1), hvWithMemory("a", 1)}, + // a and b reach the empty host e, but e is larger than they are, + // so they could never evacuate onto it and it stays alone. + [2]hv1.Hypervisor{hvWithMemory("a", 1), hvWithMemory("e", 4)}, + [2]hv1.Hypervisor{hvWithMemory("b", 1), hvWithMemory("e", 4)}, + ), + want: [][]string{{"a", "b"}, {"e"}}, + }, + { + name: "equal-sized empty host is recruited", + paths: edgesHV( + [2]hv1.Hypervisor{hvWithMemory("a", 4), hvWithMemory("b", 4)}, + [2]hv1.Hypervisor{hvWithMemory("b", 4), hvWithMemory("a", 4)}, + [2]hv1.Hypervisor{hvWithMemory("a", 4), hvWithMemory("e", 4)}, + [2]hv1.Hypervisor{hvWithMemory("b", 4), hvWithMemory("e", 4)}, + ), + want: [][]string{{"a", "b", "e"}}, + }, + } + + builder := cliquePartitionBuilder{} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := builder.BuildPartitions(tt.paths) + gotNames := partitionNames(got) + if !slices.EqualFunc(gotNames, tt.want, slices.Equal) { + t.Errorf("got %v, want %v", gotNames, tt.want) + } + }) + } +} + +// hvWithAggregates builds a hypervisor with the given name and aggregates. +func hvWithAggregates(name string, aggregates ...string) hv1.Hypervisor { + h := hv(name) + h.Spec.Aggregates = aggregates + return h +} + +// parts wraps plain host slices into flavorless Partitions, for the +// characterize tests that only exercise the host dimensions. +func parts(hostSets ...[]hv1.Hypervisor) []Partition { + out := make([]Partition, 0, len(hostSets)) + for _, hosts := range hostSets { + out = append(out, Partition{Hosts: hosts}) + } + return out +} + +func TestCharacterizePartitions(t *testing.T) { + t.Run("no partitions", func(t *testing.T) { + if got := characterizePartitions(nil); len(got) != 0 { + t.Errorf("got %v, want empty", got) + } + }) + + t.Run("distinguishing aggregate is reported per partition", func(t *testing.T) { + labels := characterizePartitions(parts( + []hv1.Hypervisor{hvWithAggregates("a", "hana"), hvWithAggregates("b", "hana")}, + []hv1.Hypervisor{hvWithAggregates("c", "general"), hvWithAggregates("d", "general")}, + )) + want := []string{"spec.aggregates=[hana]", "spec.aggregates=[general]"} + if !slices.Equal(labels, want) { + t.Errorf("got %v, want %v", labels, want) + } + }) + + t.Run("field shared across all partitions is dropped", func(t *testing.T) { + // Both partitions share the same aggregate, so it distinguishes neither. + labels := characterizePartitions(parts( + []hv1.Hypervisor{hvWithAggregates("a", "hana")}, + []hv1.Hypervisor{hvWithAggregates("b", "hana")}, + )) + for i, label := range labels { + if strings.Contains(label, "aggregates") { + t.Errorf("partition %d label %q should not mention the shared aggregate", i, label) + } + } + }) + + t.Run("single partition keeps its distinguishing fields", func(t *testing.T) { + // With only one partition nothing is fleet-wide, so shared fields survive. + labels := characterizePartitions(parts( + []hv1.Hypervisor{hvWithAggregates("a", "hana")}, + )) + if len(labels) != 1 || !strings.Contains(labels[0], "spec.aggregates=[hana]") { + t.Errorf("got %v, want a label containing spec.aggregates=[hana]", labels) + } + }) + + t.Run("empty partition yields an empty label", func(t *testing.T) { + labels := characterizePartitions(parts([]hv1.Hypervisor{})) + if len(labels) != 1 || labels[0] != "" { + t.Errorf("got %v, want one empty label", labels) + } + }) +} + +// pathWithFlavors builds a directed edge between bare hosts carrying the given +// supporting flavors. +func pathWithFlavors(src, dst string, flavors ...string) Path { + return Path{Source: hv(src), Target: hv(dst), Flavors: flavors} +} + +func TestBuildPartitionsFlavors(t *testing.T) { + t.Run("hosts with differing accept sets split into separate partitions", func(t *testing.T) { + // a and b accept {flavorAll, flavorAB}; c accepts only flavorAll. A single + // shared flavor would once have merged all three, but c could never be a + // failover target for a or b's flavorAB vms, so the builder must not merge + // it in. a-b form one partition running both flavors; c, with no mutual peer + // of a matching accept set, falls out as its own single-host partition. + paths := []Path{ + pathWithFlavors("a", "b", "flavorAll", "flavorAB"), + pathWithFlavors("b", "a", "flavorAll", "flavorAB"), + pathWithFlavors("b", "c", "flavorAll"), + pathWithFlavors("c", "b", "flavorAll"), + pathWithFlavors("a", "c", "flavorAll"), + pathWithFlavors("c", "a", "flavorAll"), + } + got := cliquePartitionBuilder{}.BuildPartitions(paths) + wantNames := [][]string{{"a", "b"}, {"c"}} + if !slices.EqualFunc(partitionNames(got), wantNames, slices.Equal) { + t.Fatalf("partitions = %v, want %v", partitionNames(got), wantNames) + } + if want := []string{"flavorAB", "flavorAll"}; !slices.Equal(got[0].Flavors, want) { + t.Errorf("partition a-b flavors = %v, want %v", got[0].Flavors, want) + } + }) + + t.Run("recruited host contributes its inbound-edge flavors", func(t *testing.T) { + // e has no outgoing edge (so the builder treats it as an empty host to + // recruit), but it is the target of a<->b's probes for flavorX, which by the + // path finder's semantics means e co-accepted flavorX too. Since a and b also + // accept flavorX, it survives the all-hosts intersection and is listed. + paths := []Path{ + {Source: hvWithMemory("a", 4), Target: hvWithMemory("b", 4), Flavors: []string{"flavorX"}}, + {Source: hvWithMemory("b", 4), Target: hvWithMemory("a", 4), Flavors: []string{"flavorX"}}, + {Source: hvWithMemory("a", 4), Target: hvWithMemory("e", 4), Flavors: []string{"flavorX"}}, + {Source: hvWithMemory("b", 4), Target: hvWithMemory("e", 4), Flavors: []string{"flavorX"}}, + } + got := cliquePartitionBuilder{}.BuildPartitions(paths) + if len(got) != 1 || len(got[0].Hosts) != 3 { + t.Fatalf("got %d partitions %v, want one of 3 hosts", len(got), partitionNames(got)) + } + if want := []string{"flavorX"}; !slices.Equal(got[0].Flavors, want) { + t.Errorf("flavors = %v, want %v", got[0].Flavors, want) + } + }) + + t.Run("empty host with a differing accept set is not recruited", func(t *testing.T) { + // a and b accept {flavorX, flavorAB}; the empty host e is only ever the + // target of flavorX probes, so it accepts {flavorX}. Recruiting e would let + // it become a failover target that cannot run a or b's flavorAB vms, so the + // builder must leave it out. a-b keep both flavors; e stays on its own. + paths := []Path{ + {Source: hvWithMemory("a", 4), Target: hvWithMemory("b", 4), Flavors: []string{"flavorX", "flavorAB"}}, + {Source: hvWithMemory("b", 4), Target: hvWithMemory("a", 4), Flavors: []string{"flavorX", "flavorAB"}}, + {Source: hvWithMemory("a", 4), Target: hvWithMemory("e", 4), Flavors: []string{"flavorX"}}, + {Source: hvWithMemory("b", 4), Target: hvWithMemory("e", 4), Flavors: []string{"flavorX"}}, + } + got := cliquePartitionBuilder{}.BuildPartitions(paths) + wantNames := [][]string{{"a", "b"}, {"e"}} + if !slices.EqualFunc(partitionNames(got), wantNames, slices.Equal) { + t.Fatalf("partitions = %v, want %v", partitionNames(got), wantNames) + } + if want := []string{"flavorAB", "flavorX"}; !slices.Equal(got[0].Flavors, want) { + t.Errorf("partition a-b flavors = %v, want %v", got[0].Flavors, want) + } + }) +} + +func TestDescribePartition(t *testing.T) { + tests := []struct { + name string + label string + flavors []string + want string + }{ + {"label and flavors", "spec.aggregates=[hana]", []string{"a", "b"}, "spec.aggregates=[hana] | flavors: a, b"}, + {"label only", "spec.aggregates=[hana]", nil, "spec.aggregates=[hana]"}, + {"flavors only", "", []string{"a", "b"}, "flavors: a, b"}, + {"neither", "", nil, ""}, + { + name: "flavor list is capped", + label: "", + flavors: []string{"f1", "f2", "f3", "f4", "f5", "f6", "f7"}, + want: "flavors: f1, f2, f3, f4, f5 (+2 more)", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := describePartition(tt.label, tt.flavors); got != tt.want { + t.Errorf("describePartition(%q, %v) = %q, want %q", tt.label, tt.flavors, got, tt.want) + } + }) + } +} diff --git a/internal/scheduling/reservations/failoverhost/pathfinding.go b/internal/scheduling/reservations/failoverhost/pathfinding.go new file mode 100644 index 000000000..5ca983e97 --- /dev/null +++ b/internal/scheduling/reservations/failoverhost/pathfinding.go @@ -0,0 +1,361 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package failoverhost + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "maps" + "net/http" + "slices" + "sort" + "strconv" + "time" + + api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/scheduling" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "k8s.io/apimachinery/pkg/api/resource" + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +// Path is one directed edge of the failover graph: source's whole workload can +// be gang-evacuated onto target. It also carries the flavors whose seed proved +// the edge — i.e. the flavors source and target co-accept — so downstream stages +// can report which flavors a partition can actually run. Flavors is sorted and +// deduplicated. +type Path struct { + // Source and Target are the endpoints: source can be evacuated onto target. + Source, Target hv1.Hypervisor + // Flavors are the names of the flavor seeds both endpoints co-accepted, which + // is what produced this edge. Sorted and deduplicated. + Flavors []string +} + +// PathFinder is the second pipeline stage. It discovers the directed edges of the +// failover graph: for every host it reports the hosts that host's workload could +// be gang-evacuated onto. An edge source->target means "a host's whole workload +// can be evacuated onto target, and target is large enough to hold it". +// +// Edges are no longer derived from the vms that happen to run on a host. Instead +// we probe with flavor seeds (see Seeder): for every (flavor, availability zone) +// permutation we ask the scheduler which hosts would accept it, and treat the +// hosts that co-accept a seed as mutually compatible under that flavor. This +// escapes two blind spots of vm-seeding — partitions whose hosts run no vms, and +// a single exotic vm vetoing a host from its partition — both documented on Seed. +// +// We probe with evacuation intent: a failover ultimately cold-migrates +// (evacuates) a dead host's workload, and evacuation is the weaker check — it +// does not verify the CPU/feature compatibility that live migration does. We +// deliberately do not probe live-migratability here even though a freshly +// reserved host is drained by live migration before its reservation goes active: +// we assume a partition holds enough peers to absorb that drain, which later +// draining code arranges, rather than proving it per host at partition time. +// +// The direction matters. source->target does not imply target->source: target +// might be too small for source's workload even if the reverse fits. The +// PartitionBuilder later keeps only the mutual edges, so this stage is free to +// report one-directional possibilities. +type PathFinder interface { + // FindPaths returns the failover edges as Paths, discovered by probing the + // scheduler with the given seeds; each Path also records the flavors its + // endpoints co-accepted. A seed the scheduler accepts on fewer than two hosts + // yields no edge and is not an error. Any lookup error, by contrast, is an + // infrastructure failure (scheduler unreachable/timed out/non-200) that + // invalidates the pass, so it is returned to abort it rather than silently + // collapsing the failover graph. + FindPaths(ctx context.Context, topo topology, seeds []Seed) ([]Path, error) +} + +// evacuationPathFinder finds failover paths by asking the external cortex +// scheduler, per seed, which hosts would accept that flavor, then pairing up the +// co-accepting hosts subject to gang-level fit. +// +// Asking the real scheduler (rather than reimplementing placement here) is +// deliberate: it means failover paths are computed with exactly the same +// filters, weighers and constraints that govern ordinary placement, so a +// partition can never contain a target the scheduler would actually reject. +type evacuationPathFinder struct { + // httpClient issues the scheduler requests. Give it a timeout so a hung + // scheduler cannot stall the whole calculation loop. Separately injected so + // tests can point it at a stub server. + httpClient *http.Client + // schedulerURL is the external cortex scheduler endpoint to POST to. + schedulerURL string + // monitor records scheduler request metrics. May be nil (e.g. in tests). + monitor *Monitor +} + +// FindPaths implements PathFinder. +// +// For each seed we ask the scheduler which hosts would accept that flavor in +// that availability zone. Every pair of hosts that co-accept a seed is mutually +// compatible under that flavor, so for each such pair (a, b) we emit the directed +// edge a->b whenever b is at least as large as a. The direction encodes the hard, +// gang-level fit constraint: a drain/evacuation moves a whole host's workload +// onto the target at once, so the target must be at least as large as the source. +// Equal-sized co-accepting hosts therefore yield edges both ways, which the +// PartitionBuilder turns into a mutual edge and hence a clique; an undersized +// target only ever appears as a source, never a target, so it is never chosen as +// a landing zone for a larger peer. +// +// Edges are deduplicated: the same pair can be co-accepted by many seeds (e.g. +// every flavor a pair of identical hosts both run), but the graph only needs each +// directed edge once. Capacity is compared on fresh copies of the hypervisors' +// capacity maps so we never mutate the shared Status.Capacity. +func (f *evacuationPathFinder) FindPaths(ctx context.Context, topo topology, seeds []Seed) ([]Path, error) { + log := logf.FromContext(ctx) + + // edges[src][dst] records the flavor seeds that produced a discovered directed + // edge, deduplicating across seeds. We materialize it into the []Path slice at + // the end, carrying each edge's supporting flavors. + edges := map[string]map[string]map[string]bool{} + for _, seed := range seeds { + accepting, err := f.fetchSeedCandidates(ctx, seed, topo.hypervisors) + if err != nil { + // A lookup error is an infrastructure failure (the scheduler is + // unreachable, timed out, or returned a non-200) — not an answer that + // this seed has no candidates, which comes back as a short slice with a + // nil error. We cannot assert anything about the failover graph on a + // partial view, so abort the whole pass. The caller leaves the desired + // state untouched and retries next cycle; swallowing the error here + // would instead collapse every host's paths and prune all still-valid + // reservations. + return nil, fmt.Errorf( + "failed to fetch candidates for flavor %q in az %q: %w", + seed.Flavor.Name, seed.AvailabilityZone, err) + } + // Fewer than two accepting hosts can form no pair, so contribute no edge. + if len(accepting) < 2 { + continue + } + acceptingNames := make([]string, len(accepting)) + for i, hv := range accepting { + acceptingNames[i] = hv.Name + } + log.V(1).Info("fetched candidates for seed", + "flavor", seed.Flavor.Name, "az", seed.AvailabilityZone, "hosts", acceptingNames) + // Pair up every co-accepting host, emitting source->target when the target + // is at least as large as the source (the gang-fit constraint). Record the + // seed's flavor on the edge so downstream stages know which flavors the pair + // co-accepts. + for _, src := range accepting { + for _, dst := range accepting { + if src.Name == dst.Name { + continue + } + if cmpResources(hvCapacity(src), hvCapacity(dst)) > 0 { + continue // Target smaller than source: it cannot hold the gang. + } + if edges[src.Name] == nil { + edges[src.Name] = map[string]map[string]bool{} + } + if edges[src.Name][dst.Name] == nil { + edges[src.Name][dst.Name] = map[string]bool{} + } + edges[src.Name][dst.Name][seed.Flavor.Name] = true + } + } + } + + // Materialize the deduplicated edges into Paths, sorted so the output is + // deterministic despite the map iteration above. Each edge's flavor set is + // flattened into a sorted slice. + paths := []Path{} + srcNames := make([]string, 0, len(edges)) + for src := range edges { + srcNames = append(srcNames, src) + } + sort.Strings(srcNames) + for _, src := range srcNames { + dstNames := make([]string, 0, len(edges[src])) + for dst := range edges[src] { + dstNames = append(dstNames, dst) + } + sort.Strings(dstNames) + for _, dst := range dstNames { + flavors := make([]string, 0, len(edges[src][dst])) + for flavor := range edges[src][dst] { + flavors = append(flavors, flavor) + } + sort.Strings(flavors) + paths = append(paths, Path{ + Source: topo.hypervisors[src], + Target: topo.hypervisors[dst], + Flavors: flavors, + }) + log.V(1).Info("added failover path", + "sourceHost", src, "targetHost", dst, "flavors", flavors) + } + } + return paths, nil +} + +// hvCapacity returns a hypervisor's gang-fit capacity: its base capacity with +// effectiveCapacity overlaid on top. Copied into a fresh map so we never mutate +// the shared Status. +func hvCapacity(hv hv1.Hypervisor) map[hv1.ResourceName]resource.Quantity { + capa := map[hv1.ResourceName]resource.Quantity{} + maps.Copy(capa, hv.Status.Capacity) + maps.Copy(capa, hv.Status.EffectiveCapacity) + return capa +} + +// fetchSeedCandidates asks the external cortex scheduler which of the known +// hypervisors would accept the seed's flavor in its availability zone, and +// returns that subset. +// +// The request is shaped to mirror a real evacuation as closely as possible: it +// carries the seed's availability zone, flavor and a KVM image hypervisor type so +// the same filters apply, and offers every known host as a candidate (Hosts) so +// the scheduler has a host set to filter over. Four options matter: +// +// - ReadOnly, so probing placements never mutate shared scheduling state. +// - SkipInflight and SkipCommittedResourceTracking, which ReadOnly runs must +// set (scheduling.Options.Validate rejects a ReadOnly run otherwise, and the +// scheduler calls Validate) so this probe neither creates in-flight +// reservations nor writes CR allocations. +// - IgnoredReservationTypes = FailoverHost, so existing failover host +// reservations don't block the very hosts we are reasoning about — otherwise +// a reserved host would be hidden from its own partition and the clustering +// would fragment. +// - AssumeEmptyHosts, so candidates are judged by their total capacity rather +// than their current free capacity. A failover target is reserved *empty*, so +// what matters for partitioning is whether a host could hold the workload once +// cleared, not whether it has room right now. Without this a busy host never +// accepts anyone, gets no inbound links, forms no mutual edge, and stays +// orphaned — even though reserving it would drain it. Compatibility (flavors, +// traits, AZ) is thus occupancy-independent; which host is actually cheapest +// to empty is left to the planner's candidate ordering. The hard whole-host +// fit check in FindPaths still compares *total* capacities, so an undersized +// target is dropped regardless of this option. +// +// We probe with evacuation intent (the "evacuate" _nova_check_type, which maps to +// EvacuateIntent in GetIntent) and set no source_host: a failover evacuates (cold +// migrates) the workload, and evacuation does not run the live-migratability +// feature check, so there is no source host to anchor it to. This is weaker than +// the old live-migration probe, which is intentional — see the PathFinder doc. +func (f *evacuationPathFinder) fetchSeedCandidates(ctx context.Context, seed Seed, hvsByName map[string]hv1.Hypervisor) ([]hv1.Hypervisor, error) { + log := logf.FromContext(ctx) + + // Build valid image properties. We set the image hypervisor type explicitly so + // the filter that checks flavor/image compatibility can evaluate it. Since we + // only handle KVM virtual machines, we set it to KVM. + imageProps := map[string]any{ + "img_hv_type": string(api.NovaImageMetaHVTypeKVM), + } + // Offer every known host as a candidate. The scheduler filters/weighs over + // exactly this host set — it does not gather the cluster itself — so leaving + // Hosts empty would make it return no candidates. Each host also needs a + // weight entry, which the scheduler requires to accompany the host set. + hosts := make([]api.ExternalSchedulerHost, 0, len(hvsByName)) + weights := make(map[string]float64, len(hvsByName)) + for name := range hvsByName { + hosts = append(hosts, api.ExternalSchedulerHost{ComputeHost: name}) + weights[name] = 0 + } + requestObj := api.ExternalSchedulerRequest{ + Hosts: hosts, + Weights: weights, + Options: scheduling.Options{ + // Read-only probe: don't write any resources. ReadOnly additionally + // requires SkipInflight and SkipCommittedResourceTracking, else the + // scheduler rejects the request in Options.Validate. + ReadOnly: true, + SkipInflight: true, + SkipCommittedResourceTracking: true, + // Judge candidates by total capacity, not current free capacity: a + // failover target is reserved empty, so a busy host that would fit the + // workload once cleared must still be a valid candidate. Without this, + // occupied hosts never accept anyone and stay orphaned. + AssumeEmptyHosts: true, + // Ignore existing failover host reservations, otherwise they would + // block these hosts, separate them from their failover partition and + // break the clustering. + IgnoredReservationTypes: []v1alpha1.ReservationType{ + v1alpha1.ReservationTypeFailoverHost, + }, + }, + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ + NumInstances: 1, + AvailabilityZone: seed.AvailabilityZone, + // Probe with evacuation intent. A failover cold-migrates (evacuates) + // the workload, and evacuation does not run the live-migratability + // feature check, so no source_host is needed. The check type must be + // the literal "evacuate" nova uses, which maps to EvacuateIntent in + // GetIntent. See the PathFinder doc for why this is deliberately + // weaker than a live-migration probe. + SchedulerHints: map[string]any{ + "_nova_check_type": "evacuate", + }, + Flavor: api.NovaObject[api.NovaFlavor]{ + Data: api.NovaFlavor{ + Name: seed.Flavor.Name, + ExtraSpecs: seed.Flavor.ExtraSpecs, + MemoryMB: seed.Flavor.MemoryMB, + VCPUs: seed.Flavor.VCPUs, + }, + }, + Image: api.NovaObject[api.NovaImageMeta]{ + Data: api.NovaImageMeta{ + Properties: api.NovaObject[map[string]any]{ + Data: imageProps, + }, + }, + }, + }, + }, + } + log.V(1).Info("external scheduler request prepared", "request", requestObj) + + reqBody, err := json.Marshal(requestObj) + if err != nil { + log.Error(err, "failed to marshal external scheduler request") + return nil, fmt.Errorf("failed to marshal external scheduler request: %w", err) + } + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, + f.schedulerURL, bytes.NewReader(reqBody)) + if err != nil { + log.Error(err, "failed to create HTTP request") + return nil, fmt.Errorf("failed to create HTTP request: %w", err) + } + httpReq.Header.Set("Content-Type", "application/json") + + startTime := time.Now() + response, err := f.httpClient.Do(httpReq) + duration := time.Since(startTime) + if err != nil { + f.monitor.RecordSchedulerRequest("error", duration) + log.Error(err, "failed to send external scheduler request") + return nil, fmt.Errorf("failed to send external scheduler request: %w", err) + } + defer response.Body.Close() + f.monitor.RecordSchedulerRequest(strconv.Itoa(response.StatusCode), duration) + log.V(1).Info("external scheduler request sent", + "duration", duration, "status", response.StatusCode) + + if response.StatusCode != http.StatusOK { + log.Error(nil, "external scheduler request failed", "status", response.StatusCode) + return nil, fmt.Errorf("external scheduler request failed with status: %d", response.StatusCode) + } + + var responseObj api.ExternalSchedulerResponse + if err := json.NewDecoder(response.Body).Decode(&responseObj); err != nil { + log.Error(err, "failed to decode external scheduler response") + return nil, fmt.Errorf("failed to decode external scheduler response: %w", err) + } + log.V(1).Info("external scheduler response received", "response", responseObj) + + candidates := []hv1.Hypervisor{} + for _, hv := range hvsByName { + if slices.Contains(responseObj.Hosts, hv.Name) { + candidates = append(candidates, hv) + } + } + return candidates, nil +} diff --git a/internal/scheduling/reservations/failoverhost/pathfinding_test.go b/internal/scheduling/reservations/failoverhost/pathfinding_test.go new file mode 100644 index 000000000..e489e0ba4 --- /dev/null +++ b/internal/scheduling/reservations/failoverhost/pathfinding_test.go @@ -0,0 +1,216 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package failoverhost + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "slices" + "testing" + + api "github.com/cobaltcore-dev/cortex/api/external/nova" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" +) + +// stubScheduler returns a path finder wired to an httptest server whose +// per-flavor answers come from candidatesByFlavor (keyed by the seed's flavor +// name); a flavor in errFlavors gets an HTTP 500. The handler asserts the request +// carries valid options, a non-empty host set, a kvm img_hv_type, and evacuation +// intent with no source_host — mirroring what the new seeded probe sends, so a +// regression that drops any of them is caught here. The caller closes the server. +func stubScheduler(t *testing.T, candidatesByFlavor map[string][]string, errFlavors map[string]bool) (*evacuationPathFinder, *httptest.Server) { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req api.ExternalSchedulerRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Errorf("decode request: %v", err) + } + if err := req.Options.Validate(); err != nil { + t.Errorf("invalid scheduling options: %v", err) + } + if !req.Options.AssumeEmptyHosts { + // A failover target is reserved empty, so candidates must be judged by + // total capacity. Dropping this would orphan every busy host. + t.Errorf("request for flavor %q must set AssumeEmptyHosts", req.Spec.Data.Flavor.Data.Name) + } + if len(req.Hosts) == 0 { + t.Errorf("request for flavor %q has no candidate hosts", req.Spec.Data.Flavor.Data.Name) + } + if got := req.Spec.Data.Image.Data.Properties.Data["img_hv_type"]; got != string(api.NovaImageMetaHVTypeKVM) { + // The hypervisor-type filter needs img_hv_type to judge flavor/image + // compatibility; dropping it would make that filter misbehave. + t.Errorf("request for flavor %q must set img_hv_type=%q, got %q", req.Spec.Data.Flavor.Data.Name, api.NovaImageMetaHVTypeKVM, got) + } + // The probe uses evacuation intent (cold migration) and sets no source_host: + // a failover evacuates rather than live-migrates, so the live-migratability + // feature check is deliberately skipped. Catch a regression to the stricter + // live-migration probe here. + if intent, err := req.GetIntent(); err != nil || intent != api.EvacuateIntent { + t.Errorf("request for flavor %q must use evacuate intent, got %v (err=%v)", req.Spec.Data.Flavor.Data.Name, intent, err) + } + if _, ok := req.Spec.Data.SchedulerHints["source_host"]; ok { + t.Errorf("request for flavor %q must not set source_host", req.Spec.Data.Flavor.Data.Name) + } + flavor := req.Spec.Data.Flavor.Data.Name + if errFlavors[flavor] { + w.WriteHeader(http.StatusInternalServerError) + return + } + if err := json.NewEncoder(w).Encode(api.ExternalSchedulerResponse{Hosts: candidatesByFlavor[flavor]}); err != nil { + t.Errorf("encode response: %v", err) + } + })) + finder := &evacuationPathFinder{httpClient: server.Client(), schedulerURL: server.URL} + return finder, server +} + +// pathNames collapses failover edges into sorted "source->target" strings, for +// order-independent comparison. +func pathNames(paths []Path) []string { + out := make([]string, 0, len(paths)) + for _, p := range paths { + out = append(out, p.Source.Name+"->"+p.Target.Name) + } + slices.Sort(out) + return out +} + +// hostTopo builds a topology of the named hosts (each sized memGi GiB), so +// FindPaths can be exercised without a k8s client. Hosts carry az1 so a seed in +// az1 lines up with them. +func hostTopo(hostMemGi map[string]int64) topology { + hypervisors := map[string]hv1.Hypervisor{} + for name, memGi := range hostMemGi { + hypervisors[name] = *buildHV(name, memGi, 1) + } + return topology{hypervisors: hypervisors} +} + +// seed is a tiny constructor for a flavor seed named name in az1. +func seed(name string) Seed { + return Seed{Flavor: Flavor{Name: name}, AvailabilityZone: "az1"} +} + +func TestFindPathsErrorsOnLookupFailure(t *testing.T) { + // A seed's lookup fails with a scheduler error. That's an infrastructure + // failure, not an answer that the seed has no candidates, so FindPaths must + // return an error to abort the pass rather than silently dropping paths — + // otherwise the caller would publish an empty desired state and prune valid + // reservations. + finder, server := stubScheduler(t, + map[string][]string{"good": {"a", "b"}}, + map[string]bool{"bad": true}, + ) + defer server.Close() + + // "bad" is probed and errors; the pass must abort regardless of "good". + topo := hostTopo(map[string]int64{"a": 32, "b": 32}) + if _, err := finder.FindPaths(context.Background(), topo, []Seed{seed("bad"), seed("good")}); err == nil { + t.Fatal("FindPaths: expected error on scheduler lookup failure, got nil") + } +} + +func TestFindPathsPairsCoAcceptingHosts(t *testing.T) { + // A single seed accepted on a, b and c makes all three mutually compatible. + // Equal-sized hosts yield edges both ways between every pair. + finder, server := stubScheduler(t, map[string][]string{ + "flavor": {"a", "b", "c"}, + }, nil) + defer server.Close() + + topo := hostTopo(map[string]int64{"a": 32, "b": 32, "c": 32}) + got, err := finder.FindPaths(context.Background(), topo, []Seed{seed("flavor")}) + if err != nil { + t.Fatalf("FindPaths: %v", err) + } + want := []string{"a->b", "a->c", "b->a", "b->c", "c->a", "c->b"} + if !slices.Equal(pathNames(got), want) { + t.Errorf("paths = %v, want %v", pathNames(got), want) + } +} + +func TestFindPathsDropsTargetSmallerThanSource(t *testing.T) { + // A seed is accepted on "big" (64Gi), "small" (16Gi) and "equal" (64Gi). The + // gang-fit constraint keeps an edge only toward a target at least as large as + // the source, so no edge points at "small", and big<->equal are mutual. + finder, server := stubScheduler(t, map[string][]string{ + "flavor": {"big", "small", "equal"}, + }, nil) + defer server.Close() + + topo := hostTopo(map[string]int64{"big": 64, "small": 16, "equal": 64}) + got, err := finder.FindPaths(context.Background(), topo, []Seed{seed("flavor")}) + if err != nil { + t.Fatalf("FindPaths: %v", err) + } + // small can reach the larger big/equal (and vice versa is dropped); big and + // equal are mutual. Nothing points at small. + want := []string{"big->equal", "equal->big", "small->big", "small->equal"} + if !slices.Equal(pathNames(got), want) { + t.Errorf("paths = %v, want %v", pathNames(got), want) + } +} + +func TestFindPathsDeduplicatesAcrossSeeds(t *testing.T) { + // Two seeds both accepted on the same pair of equal hosts. Each directed edge + // must appear once, not once per seed. + finder, server := stubScheduler(t, map[string][]string{ + "flavorA": {"hvA", "hvB"}, + "flavorB": {"hvA", "hvB"}, + }, nil) + defer server.Close() + + topo := hostTopo(map[string]int64{"hvA": 64, "hvB": 64}) + got, err := finder.FindPaths(context.Background(), topo, []Seed{seed("flavorA"), seed("flavorB")}) + if err != nil { + t.Fatalf("FindPaths: %v", err) + } + if want := []string{"hvA->hvB", "hvB->hvA"}; !slices.Equal(pathNames(got), want) { + t.Errorf("paths = %v, want %v", pathNames(got), want) + } +} + +func TestFindPathsSingleCandidateYieldsNoEdge(t *testing.T) { + // A seed accepted on only one host can form no pair, so contributes no edge. + finder, server := stubScheduler(t, map[string][]string{ + "flavor": {"only"}, + }, nil) + defer server.Close() + + topo := hostTopo(map[string]int64{"only": 64, "other": 64}) + got, err := finder.FindPaths(context.Background(), topo, []Seed{seed("flavor")}) + if err != nil { + t.Fatalf("FindPaths: %v", err) + } + if len(got) != 0 { + t.Errorf("paths = %v, want none", pathNames(got)) + } +} + +func TestFindPathsRecordsSupportingFlavors(t *testing.T) { + // flavorA is accepted on both hosts, flavorB only on hvA. The edge hvA<->hvB is + // produced solely by flavorA, so each directed edge must carry exactly + // ["flavorA"] — flavorB co-accepts on no pair and leaves no trace. + finder, server := stubScheduler(t, map[string][]string{ + "flavorA": {"hvA", "hvB"}, + "flavorB": {"hvA"}, + }, nil) + defer server.Close() + + topo := hostTopo(map[string]int64{"hvA": 64, "hvB": 64}) + got, err := finder.FindPaths(context.Background(), topo, []Seed{seed("flavorA"), seed("flavorB")}) + if err != nil { + t.Fatalf("FindPaths: %v", err) + } + for _, p := range got { + if want := []string{"flavorA"}; !slices.Equal(p.Flavors, want) { + t.Errorf("edge %s->%s flavors = %v, want %v", p.Source.Name, p.Target.Name, p.Flavors, want) + } + } + if want := []string{"hvA->hvB", "hvB->hvA"}; !slices.Equal(pathNames(got), want) { + t.Errorf("paths = %v, want %v", pathNames(got), want) + } +} diff --git a/internal/scheduling/reservations/failoverhost/planning.go b/internal/scheduling/reservations/failoverhost/planning.go new file mode 100644 index 000000000..8fc8485ea --- /dev/null +++ b/internal/scheduling/reservations/failoverhost/planning.go @@ -0,0 +1,199 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package failoverhost + +import ( + "maps" + "slices" + "sort" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ReservationPlanner is the fourth pipeline stage. For a single failover +// partition it decides which hosts should become failover targets, i.e. be kept +// empty so another host in the partition can gang-evacuate onto them. +// +// We do not reserve a fixed number per partition. Reserving too few leaves no +// landing space when a host fails; reserving too many wastes capacity. So the +// count scales with the partition: a configured ratio of the hosts, floored at a +// configured minimum so even small partitions keep some headroom. +type ReservationPlanner interface { + // PlanReservations returns the desired failover host reservations for the + // given partition. The topology supplies the reservations that already + // exist, which the planner prefers to keep so target hosts stay stable + // across passes rather than flapping from one host to another. description + // is a human-readable label of what distinguishes the partition's hosts, + // stored verbatim on each reservation. See characterizePartitions. + PlanReservations(topo topology, partition []hv1.Hypervisor, description string) []v1alpha1.Reservation +} + +// ratioReservationPlanner reserves a ratio of each partition's hosts as failover +// targets, never fewer than a configured minimum. It prefers to keep existing +// reservations and, when it must create new ones, picks the hosts with the +// least allocated capacity so that the eventual evacuation migrates as few +// resources as possible. +type ratioReservationPlanner struct { + // ratio is the fraction of a partition's hosts to reserve as failover + // targets, rounded down (e.g. 0.1 of a 10-host partition reserves 1). + ratio float64 + // minimum is the floor on reservations per partition, applied even when the + // ratio would round down to fewer. + minimum int +} + +// PlanReservations implements ReservationPlanner. +// +// It builds a freshly computed reservation for every host in the partition and +// returns the first desiredCount of them, where desiredCount = +// max(len(partition)*ratio, minimum), ordered [already-reserved..., cheapest-new...]: +// +// - Hosts that already hold a reservation for this partition come first, so a +// host that is already a failover target stays one. This stability matters: +// a designated failover host may have vms being live-migrated off it, and +// flipping the designation to a different host each pass would waste that +// work. If there are more such hosts than desiredCount, the surplus is left +// out of the result and the reconciler prunes it, freeing the host back into +// normal service. +// - Not-yet-reserved hosts come next, ordered by ascending allocated capacity, +// so when we must designate a fresh target we pick the host whose vms are +// cheapest to migrate away. +// +// Crucially the spec is recomputed for every host, including already-reserved +// ones, rather than carried over from topo.existing. A kept reservation's +// SourceHosts, Resources and AvailabilityZone therefore always track the current +// partition: if the partition's membership or a host's capacity changed, the +// recomputed spec drifts from the stored one and the reconciler updates it. +func (p ratioReservationPlanner) PlanReservations(topo topology, partition []hv1.Hypervisor, description string) []v1alpha1.Reservation { + // A partition of fewer than two hosts offers no failover: there is no peer + // for a reserved host to receive an evacuation from. Reserving the lone host + // would only block it from normal placement for no benefit, so skip it. + if len(partition) < 2 { + return nil + } + + // Record which hosts already hold a failover reservation, so we can order + // their recomputed reservations first and keep target hosts stable. We use + // the spec's target host, not Status.Host: a freshly created reservation the + // scheduler hasn't placed yet has an empty Status.Host. + hostsReserved := []string{} + for _, reservation := range topo.existing { + hostsReserved = append(hostsReserved, reservation.Spec.TargetHost) + } + + // Build a freshly computed reservation for every host in the partition and + // split them into already-reserved and fresh candidates. + existing := []v1alpha1.Reservation{} + fresh := []v1alpha1.Reservation{} + for _, hv := range partition { + sourceHosts := []string{} // Other hosts in the same partition. + for _, otherHv := range partition { + if otherHv.Name != hv.Name { + sourceHosts = append(sourceHosts, otherHv.Name) + } + } + reservedResources := map[hv1.ResourceName]resource.Quantity{} + maps.Copy(reservedResources, hv.Status.Capacity) // base capacity + maps.Copy(reservedResources, hv.Status.EffectiveCapacity) + reservation := v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "failover-host-" + hv.Name, + Labels: map[string]string{ + v1alpha1.LabelReservationType: v1alpha1.ReservationTypeLabelFailoverHost, + }, + }, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeFailoverHost, + SchedulingDomain: v1alpha1.SchedulingDomainNova, + AvailabilityZone: hv.Labels[corev1.LabelTopologyZone], + Resources: reservedResources, // Allocate full host. + TargetHost: hv.Name, + FailoverHostReservation: &v1alpha1.FailoverHostReservationSpec{ + SourceHosts: sourceHosts, + PartitionDescription: description, + }, + }, + } + if slices.Contains(hostsReserved, hv.Name) { + existing = append(existing, reservation) + } else { + fresh = append(fresh, reservation) + } + } + // topo.existing is a map, so sort for a stable order. Otherwise, when a + // partition holds more already-reserved hosts than we need, the truncation + // below would keep a different subset each pass and flap the target hosts. + sort.SliceStable(existing, func(i, j int) bool { + return existing[i].Spec.TargetHost < existing[j].Spec.TargetHost + }) + // Order new candidates by ascending allocated capacity, so that if we have to + // create reservations we pick the hosts that are cheapest to evacuate onto. + sort.SliceStable(fresh, func(i, j int) bool { + return cmpResources(fresh[i].Spec.Resources, fresh[j].Spec.Resources) < 0 + }) + + // Candidate order is [already-reserved..., cheapest-new...]. Reserve a ratio + // of the partition, never fewer than the minimum, but never so many that no + // host is left to run workload — always keep at least one source host. + available := make([]v1alpha1.Reservation, 0, len(existing)+len(fresh)) + available = append(available, existing...) + available = append(available, fresh...) + desiredCount := int(float64(len(partition)) * p.ratio) + if desiredCount < p.minimum { + desiredCount = p.minimum + } + if maxReservable := len(partition) - 1; desiredCount > maxReservable { + desiredCount = maxReservable + } + if len(available) > desiredCount { + available = available[:desiredCount] + } + return available +} + +// cmpResources compares the resources of hypervisor a with the resources +// of hypervisor b. If the resources of a are greater then b, it returns 1. +// If the resources of a are less than b, it returns -1. +// If the resources are equal, it returns 0. +func cmpResources(a, b map[hv1.ResourceName]resource.Quantity) int { + // Consider memory first, then consider cpu if there isn't already + // a decisive comparison can be made based on memory. + memA, ok := a[hv1.ResourceMemory] + if !ok { + memA = *resource.NewQuantity(0, resource.BinarySI) + } + memB, ok := b[hv1.ResourceMemory] + if !ok { + memB = *resource.NewQuantity(0, resource.BinarySI) + } + // Round to GiB to absorb small per-node discrepancies in reported memory + // capacity (Linux frees varying amounts during boot, so identical hardware + // may show slightly different effective capacities). + const gib = 1 << 30 + memA = *resource.NewQuantity(memA.Value()/gib*gib, resource.BinarySI) + memB = *resource.NewQuantity(memB.Value()/gib*gib, resource.BinarySI) + if memA.Cmp(memB) > 0 { + return 1 + } else if memA.Cmp(memB) < 0 { + return -1 + } + cpuA, ok := a[hv1.ResourceCPU] + if !ok { + cpuA = *resource.NewQuantity(0, resource.DecimalSI) + } + cpuB, ok := b[hv1.ResourceCPU] + if !ok { + cpuB = *resource.NewQuantity(0, resource.DecimalSI) + } + if cpuA.Cmp(cpuB) > 0 { + return 1 + } else if cpuA.Cmp(cpuB) < 0 { + return -1 + } + return 0 +} diff --git a/internal/scheduling/reservations/failoverhost/planning_test.go b/internal/scheduling/reservations/failoverhost/planning_test.go new file mode 100644 index 000000000..419039a08 --- /dev/null +++ b/internal/scheduling/reservations/failoverhost/planning_test.go @@ -0,0 +1,224 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package failoverhost + +import ( + "slices" + "testing" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// res builds a resource map from an optional memory and cpu quantity string. +func res(mem, cpu string) map[hv1.ResourceName]resource.Quantity { + out := map[hv1.ResourceName]resource.Quantity{} + if mem != "" { + out[hv1.ResourceMemory] = resource.MustParse(mem) + } + if cpu != "" { + out[hv1.ResourceCPU] = resource.MustParse(cpu) + } + return out +} + +func TestCmpResources(t *testing.T) { + tests := []struct { + name string + a, b map[hv1.ResourceName]resource.Quantity + want int + }{ + {"equal", res("8Gi", "4"), res("8Gi", "4"), 0}, + {"memory greater dominates", res("16Gi", "1"), res("8Gi", "99"), 1}, + {"memory less dominates", res("4Gi", "99"), res("8Gi", "1"), -1}, + {"cpu breaks memory tie greater", res("8Gi", "8"), res("8Gi", "4"), 1}, + {"cpu breaks memory tie less", res("8Gi", "2"), res("8Gi", "4"), -1}, + {"missing memory treated as zero", res("", "4"), res("8Gi", "4"), -1}, + {"both empty are equal", res("", ""), res("", ""), 0}, + {"missing cpu treated as zero", res("8Gi", ""), res("8Gi", "4"), -1}, + // Sub-GiB discrepancies (e.g. from different amounts of memory freed during + // Linux boot) must not be treated as different capacities. + {"sub-gib difference treated as equal", res("4227288964Ki", "4"), res("4227172876Ki", "4"), 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := cmpResources(tt.a, tt.b); got != tt.want { + t.Errorf("cmpResources() = %d, want %d", got, tt.want) + } + }) + } +} + +// partitionOf builds a partition of hosts, each with the given memory capacity +// in GiB so the planner can order them by allocation cost. +func partitionOf(memByHost map[string]int64) []hv1.Hypervisor { + partition := make([]hv1.Hypervisor, 0, len(memByHost)) + for name, memGi := range memByHost { + partition = append(partition, hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{corev1.LabelTopologyZone: "az1"}, + }, + Status: hv1.HypervisorStatus{ + Capacity: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: *resource.NewQuantity(memGi*1024*1024*1024, resource.BinarySI), + }, + }, + }) + } + slices.SortFunc(partition, func(a, b hv1.Hypervisor) int { return cmpStrings(a.Name, b.Name) }) + return partition +} + +func cmpStrings(a, b string) int { + switch { + case a < b: + return -1 + case a > b: + return 1 + default: + return 0 + } +} + +func TestPlanReservationsRatioAndMinimum(t *testing.T) { + partition := partitionOf(map[string]int64{"hv1": 64, "hv2": 32, "hv3": 16, "hv4": 48}) + empty := topology{existing: map[corev1.ObjectReference]v1alpha1.Reservation{}} + + tests := []struct { + name string + ratio float64 + minimum int + want int // number of planned reservations + }{ + {"ratio rounds down", 0.5, 1, 2}, // 4 * 0.5 = 2 + {"minimum floors ratio", 0.1, 2, 2}, // 4 * 0.1 = 0, floored to 2 + {"ratio above minimum wins", 0.75, 1, 3}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + planner := ratioReservationPlanner{ratio: tt.ratio, minimum: tt.minimum} + got := planner.PlanReservations(empty, partition, "") + if len(got) != tt.want { + t.Fatalf("planned %d reservations, want %d", len(got), tt.want) + } + for _, r := range got { + if r.Spec.Type != v1alpha1.ReservationTypeFailoverHost { + t.Errorf("unexpected reservation type %q", r.Spec.Type) + } + if r.Spec.AvailabilityZone != "az1" { + t.Errorf("availability zone = %q, want az1", r.Spec.AvailabilityZone) + } + } + }) + } +} + +func TestPlanReservationsPrefersSmallestHosts(t *testing.T) { + // With no existing reservations, the planner creates new ones for the hosts + // that are cheapest to evacuate, i.e. the smallest capacity first. + partition := partitionOf(map[string]int64{"big": 64, "small": 16, "medium": 32}) + empty := topology{existing: map[corev1.ObjectReference]v1alpha1.Reservation{}} + planner := ratioReservationPlanner{ratio: 0.34, minimum: 1} // 3 * 0.34 = 1 + + got := planner.PlanReservations(empty, partition, "") + if len(got) != 1 { + t.Fatalf("planned %d reservations, want 1", len(got)) + } + if got[0].Spec.TargetHost != "small" { + t.Errorf("target host = %q, want smallest host 'small'", got[0].Spec.TargetHost) + } +} + +func TestPlanReservationsKeepsExistingStable(t *testing.T) { + partition := partitionOf(map[string]int64{"big": 64, "small": 16}) + existing := v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: "failover-host-big"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeFailoverHost, + TargetHost: "big", // already reserved, even though it's the bigger host + Resources: res("64Gi", ""), + }, + } + topo := topology{existing: map[corev1.ObjectReference]v1alpha1.Reservation{ + {Name: existing.Name}: existing, + }} + planner := ratioReservationPlanner{ratio: 0.5, minimum: 1} // reserve 1 of 2 + + got := planner.PlanReservations(topo, partition, "") + if len(got) != 1 { + t.Fatalf("planned %d reservations, want 1", len(got)) + } + // The existing reservation wins over creating a new one on the smaller host, + // so target hosts don't flip between passes. + if got[0].Spec.TargetHost != "big" { + t.Errorf("target host = %q, want existing 'big'", got[0].Spec.TargetHost) + } +} + +func TestPlanReservationsRecomputesExistingSpec(t *testing.T) { + // A host that is already reserved keeps its designation, but its spec is + // recomputed from the current partition rather than carried over stale. Here + // the partition grew (hv3 joined), so the kept reservation's SourceHosts must + // now include hv3 and its stale Resources must be refreshed. + partition := partitionOf(map[string]int64{"hv1": 64, "hv2": 32, "hv3": 16}) + existing := v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: "failover-host-hv1"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeFailoverHost, + TargetHost: "hv1", + Resources: res("8Gi", ""), // stale capacity + FailoverHostReservation: &v1alpha1.FailoverHostReservationSpec{SourceHosts: []string{"hv2"}}, + }, + } + topo := topology{existing: map[corev1.ObjectReference]v1alpha1.Reservation{ + {Name: existing.Name}: existing, + }} + planner := ratioReservationPlanner{ratio: 0.34, minimum: 1} // 3 * 0.34 = 1 + + got := planner.PlanReservations(topo, partition, "spec.aggregates=[hana]") + if len(got) != 1 { + t.Fatalf("planned %d reservations, want 1", len(got)) + } + r := got[0] + if r.Spec.TargetHost != "hv1" { + t.Fatalf("target host = %q, want kept 'hv1'", r.Spec.TargetHost) + } + wantSources := []string{"hv2", "hv3"} + if r.Spec.FailoverHostReservation == nil || + !slices.Equal(r.Spec.FailoverHostReservation.SourceHosts, wantSources) { + t.Errorf("source hosts = %v, want %v", r.Spec.FailoverHostReservation, wantSources) + } + if got := r.Spec.FailoverHostReservation.PartitionDescription; got != "spec.aggregates=[hana]" { + t.Errorf("partition description = %q, want %q", got, "spec.aggregates=[hana]") + } + wantMem := int64(64) * 1024 * 1024 * 1024 + if mem := r.Spec.Resources[hv1.ResourceMemory]; mem.Value() != wantMem { + t.Errorf("memory = %d, want refreshed %d", mem.Value(), wantMem) + } +} + +func TestPlanReservationsSkipsSingleHostPartition(t *testing.T) { + // even with minimum >= 1 — reserving it would just waste the host. + partition := partitionOf(map[string]int64{"solo": 32}) + empty := topology{existing: map[corev1.ObjectReference]v1alpha1.Reservation{}} + planner := ratioReservationPlanner{ratio: 1, minimum: 1} + if got := planner.PlanReservations(empty, partition, ""); len(got) != 0 { + t.Errorf("planned %d reservations for a 1-host partition, want 0", len(got)) + } +} + +func TestPlanReservationsLeavesOneSourceHost(t *testing.T) { + // With minimum larger than the partition, we must still leave at least one + // host unreserved to run workload; cap desiredCount at len(partition)-1. + partition := partitionOf(map[string]int64{"hv1": 32, "hv2": 32, "hv3": 32}) + empty := topology{existing: map[corev1.ObjectReference]v1alpha1.Reservation{}} + planner := ratioReservationPlanner{ratio: 1, minimum: 99} + if got := planner.PlanReservations(empty, partition, ""); len(got) != len(partition)-1 { + t.Errorf("planned %d reservations, want %d (one source host kept)", len(got), len(partition)-1) + } +} diff --git a/internal/scheduling/reservations/failoverhost/seeding.go b/internal/scheduling/reservations/failoverhost/seeding.go new file mode 100644 index 000000000..1af4c98fc --- /dev/null +++ b/internal/scheduling/reservations/failoverhost/seeding.go @@ -0,0 +1,175 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package failoverhost + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +// Flavor is the slice of a flavor the seeder permutes into a scheduler probe: +// its name, size, and extra specs (which carry the hypervisor type, traits and +// other constraints the pipeline filters on). It deliberately mirrors only the +// fields fed into api.NovaFlavor, so the rest of the flavor-group shape never +// leaks into the pathfinder. +type Flavor struct { + // Name is the flavor name, forwarded as the probe's flavor name so + // flavor-name-sensitive filters behave as in a real placement. + Name string + // MemoryMB and VCPUs size the probe. RAM is in MiB (despite the "MB" label), + // matching what the scheduler expects on NovaFlavor.MemoryMB. + MemoryMB uint64 + VCPUs uint64 + // ExtraSpecs carries the flavor's extra specs (hypervisor type, custom + // traits, hardware requirements) so the same filters apply as for a real vm. + ExtraSpecs map[string]string +} + +// Seed is a single hypothetical placement the PathFinder probes the scheduler +// with: a flavor in a given availability zone. It replaces the running vm the +// old pathfinder seeded from. +// +// Seeding from flavors rather than vms fixes two blind spots of the vm-based +// approach: +// +// - A group of hosts that happen to run no vms produced no probe at all, so +// its partition was never discovered — even though it is perfectly capable +// of absorbing a failover once a vm lands there. A flavor seed exists +// whether or not anything is currently scheduled, so the partition is found +// up front. +// - A single "exotic" vm on a host (one whose scheduling constraints no peer +// satisfies) emptied that host's candidate intersection and kept it out of +// the partition it otherwise belongs to. Flavor seeds carry no host +// affiliation, so one odd flavor can never veto a host from a partition it +// matches under every other flavor. +type Seed struct { + // Flavor is the flavor to probe placement for. + Flavor Flavor + // AvailabilityZone is the AZ to probe in, drawn from the hypervisor CRDs. + AvailabilityZone string +} + +// Seeder produces the seeds a single calculation pass probes with. It is the +// first input to the pipeline, replacing the per-host vm enumeration. +type Seeder interface { + // Seeds returns every (flavor, availability zone) permutation to probe, + // restricted to flavors whose hypervisor type this controller supports (see + // the hvType filter in seeder.Seeds). The topology supplies the availability + // zones (the distinct topology-zone labels of the known hypervisors); the + // flavors come from the flavor-groups Knowledge CRD. Any error aborts the + // pass, exactly like a failed scheduler probe, so partitions are never + // recomputed from a shrunken seed set. + Seeds(ctx context.Context, topo topology) ([]Seed, error) +} + +// seeder builds seeds by crossing the flavor catalogue with the availability +// zones present in the fleet. The flavors come from the flavor-groups Knowledge +// CRD (already extracted into the cluster, so no extra OpenStack round-trip is +// needed); the AZs come from the hypervisor CRDs. +type seeder struct { + // k8s reads the flavor-groups Knowledge CRD. + k8s client.Client +} + +// Seeds implements Seeder. It reads the flavors from the flavor-groups Knowledge +// CRD, collects the distinct availability zones from the hypervisor CRDs, and +// returns every (flavor, AZ) pair. The output is sorted (by AZ then flavor name) +// so a pass is deterministic despite the map iteration over hypervisors. +func (s *seeder) Seeds(ctx context.Context, topo topology) ([]Seed, error) { + log := logf.FromContext(ctx) + + flavors, err := s.listFlavors(ctx) + if err != nil { + return nil, err + } + + // Collect the distinct availability zones from the hypervisor CRDs. collect + // already guarantees every hypervisor carries a topology-zone label; we read + // from every known hypervisor (including empty ones) so a partition is seeded + // in each AZ it spans. + azSet := map[string]bool{} + for _, hv := range topo.hypervisors { + if az, ok := hv.Labels[corev1.LabelTopologyZone]; ok && az != "" { + azSet[az] = true + } + } + azs := make([]string, 0, len(azSet)) + for az := range azSet { + azs = append(azs, az) + } + sort.Strings(azs) + // Sort flavors by name so the nested loop below emits seeds in a stable order + // (ascending az, then ascending flavor name) despite the map iteration above. + sort.SliceStable(flavors, func(i, j int) bool { return flavors[i].Name < flavors[j].Name }) + + seeds := make([]Seed, 0, len(flavors)*len(azs)) + for _, az := range azs { + for _, flavor := range flavors { + // Only seed with flavors whose capabilities:hypervisor_type extra-spec + // names a hypervisor this controller supports: "qemu" (the Nova label + // for KVM guests, which the probe mirrors with an img_hv_type=KVM image, + // see fetchSeedCandidates) and "ch" (Cloud Hypervisor). A flavor with no + // hypervisor type, or one we don't support, is skipped so it never + // contributes a probe — and hence never shapes a partition — for a guest + // kind the failover machinery doesn't handle. + hvType, ok := flavor.ExtraSpecs["capabilities:hypervisor_type"] + if !ok { + log.V(1).Info("skipping flavor with no hypervisor type", + "flavor", flavor.Name) + continue + } + if !strings.EqualFold(hvType, "qemu") && !strings.EqualFold(hvType, "ch") { + log.V(1).Info("skipping flavor with unsupported hypervisor type", + "flavor", flavor.Name, "hvType", hvType) + continue + } + seeds = append(seeds, Seed{Flavor: flavor, AvailabilityZone: az}) + } + } + log.V(1).Info("generated failover seeds", + "flavors", len(flavors), "availabilityZones", len(azs), "seeds", len(seeds)) + return seeds, nil +} + +// listFlavors reads every flavor from the flavor-groups Knowledge CRD. The CRD +// is cluster-scoped and must be ready; otherwise the pass aborts rather than +// seeding from a partial catalogue. Flavors from every group are flattened into +// a single list — the groups themselves carry no meaning for seeding, only their +// member flavors do. +func (s *seeder) listFlavors(ctx context.Context) ([]Flavor, error) { + knowledge := &v1alpha1.Knowledge{} + if err := s.k8s.Get(ctx, types.NamespacedName{Name: "flavor-groups"}, knowledge); err != nil { + return nil, fmt.Errorf("failed to get flavor groups knowledge: %w", err) + } + if !meta.IsStatusConditionTrue(knowledge.Status.Conditions, v1alpha1.KnowledgeConditionReady) { + return nil, errors.New("flavor groups knowledge is not ready") + } + groups, err := v1alpha1.UnboxFeatureList[compute.FlavorGroupFeature](knowledge.Status.Raw) + if err != nil { + return nil, fmt.Errorf("failed to unbox flavor group features: %w", err) + } + var flavors []Flavor + for _, group := range groups { + for _, f := range group.Flavors { + flavors = append(flavors, Flavor{ + Name: f.Name, + MemoryMB: f.MemoryMB, + VCPUs: f.VCPUs, + ExtraSpecs: f.ExtraSpecs, + }) + } + } + return flavors, nil +} diff --git a/internal/scheduling/reservations/failoverhost/seeding_test.go b/internal/scheduling/reservations/failoverhost/seeding_test.go new file mode 100644 index 000000000..62069b210 --- /dev/null +++ b/internal/scheduling/reservations/failoverhost/seeding_test.go @@ -0,0 +1,136 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package failoverhost + +import ( + "context" + "testing" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// flavorGroupsKnowledge builds a ready flavor-groups Knowledge CRD whose groups +// carry the given flavors. The grouping is irrelevant for seeding, so the test +// flavors are all placed in a single group. +func flavorGroupsKnowledge(t *testing.T, ready bool, flavors ...compute.FlavorInGroup) *v1alpha1.Knowledge { + t.Helper() + raw, err := v1alpha1.BoxFeatureList([]compute.FlavorGroupFeature{{ + Name: "group", + Flavors: flavors, + }}) + if err != nil { + t.Fatalf("box features: %v", err) + } + k := &v1alpha1.Knowledge{ + ObjectMeta: metav1.ObjectMeta{Name: "flavor-groups"}, + Status: v1alpha1.KnowledgeStatus{Raw: raw}, + } + if ready { + k.Status.Conditions = []metav1.Condition{{ + Type: v1alpha1.KnowledgeConditionReady, + Status: metav1.ConditionTrue, + }} + } + return k +} + +// hvInAZ builds a hypervisor labelled with the given availability zone. +func hvInAZ(name, az string) hv1.Hypervisor { + return hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{corev1.LabelTopologyZone: az}, + }} +} + +// qemuFlavor builds a flavor with a supported (qemu) hypervisor type so it +// passes the hypervisor-type filter in Seeds. +func qemuFlavor(name string) compute.FlavorInGroup { + return compute.FlavorInGroup{Name: name, ExtraSpecs: map[string]string{"capabilities:hypervisor_type": "qemu"}} +} + +func TestSeedsCrossesFlavorsWithAZs(t *testing.T) { + // Two flavors and two distinct AZs across three hosts (az1 appears twice, so + // it must be deduplicated) yield 2×2 = 4 seeds, sorted by az then flavor name. + k8s := newTestClient(t, flavorGroupsKnowledge(t, true, qemuFlavor("large"), qemuFlavor("small"))) + s := &seeder{k8s: k8s} + topo := topology{hypervisors: map[string]hv1.Hypervisor{ + "a": hvInAZ("a", "az1"), + "b": hvInAZ("b", "az1"), + "c": hvInAZ("c", "az2"), + }} + + seeds, err := s.Seeds(context.Background(), topo) + if err != nil { + t.Fatalf("Seeds: %v", err) + } + got := make([]string, len(seeds)) + for i, sd := range seeds { + got[i] = sd.AvailabilityZone + "/" + sd.Flavor.Name + } + want := []string{"az1/large", "az1/small", "az2/large", "az2/small"} + if len(got) != len(want) { + t.Fatalf("seeds = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("seed[%d] = %q, want %q", i, got[i], want[i]) + } + } +} + +func TestSeedsFiltersUnsupportedHypervisorTypes(t *testing.T) { + // Only flavors whose hypervisor type is qemu or ch are seeded; others (and + // flavors with no hypervisor type at all) are skipped. + k8s := newTestClient(t, flavorGroupsKnowledge(t, true, + qemuFlavor("qemu-flavor"), + compute.FlavorInGroup{Name: "ch-flavor", ExtraSpecs: map[string]string{"capabilities:hypervisor_type": "CH"}}, + compute.FlavorInGroup{Name: "vmware-flavor", ExtraSpecs: map[string]string{"capabilities:hypervisor_type": "vmware"}}, + compute.FlavorInGroup{Name: "no-type-flavor"}, + )) + s := &seeder{k8s: k8s} + topo := topology{hypervisors: map[string]hv1.Hypervisor{"a": hvInAZ("a", "az1")}} + + seeds, err := s.Seeds(context.Background(), topo) + if err != nil { + t.Fatalf("Seeds: %v", err) + } + got := make([]string, len(seeds)) + for i, sd := range seeds { + got[i] = sd.Flavor.Name + } + want := []string{"ch-flavor", "qemu-flavor"} + if len(got) != len(want) { + t.Fatalf("seeds = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("seed[%d] = %q, want %q", i, got[i], want[i]) + } + } +} + +func TestSeedsFailsWhenKnowledgeNotReady(t *testing.T) { + // A flavor-groups CRD that has not reached Ready aborts the pass rather than + // seeding from a partial (or empty) catalogue. + k8s := newTestClient(t, flavorGroupsKnowledge(t, false, qemuFlavor("f"))) + s := &seeder{k8s: k8s} + topo := topology{hypervisors: map[string]hv1.Hypervisor{"a": hvInAZ("a", "az1")}} + if _, err := s.Seeds(context.Background(), topo); err == nil { + t.Fatal("Seeds: expected not-ready error, got nil") + } +} + +func TestSeedsFailsWhenKnowledgeMissing(t *testing.T) { + // No flavor-groups CRD at all is an error, not an empty seed set. + k8s := newTestClient(t) + s := &seeder{k8s: k8s} + topo := topology{hypervisors: map[string]hv1.Hypervisor{"a": hvInAZ("a", "az1")}} + if _, err := s.Seeds(context.Background(), topo); err == nil { + t.Fatal("Seeds: expected missing-knowledge error, got nil") + } +} diff --git a/internal/scheduling/reservations/monitor.go b/internal/scheduling/reservations/monitor.go deleted file mode 100644 index 6cefa301b..000000000 --- a/internal/scheduling/reservations/monitor.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package reservations - -import ( - "context" - "strings" - "time" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" - "github.com/prometheus/client_golang/prometheus" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -var ( - monitorLog = ctrl.Log.WithName("monitor") -) - -// Monitor for reservations metrics. -type Monitor struct { - // Client for the kubernetes API. - client.Client - - // Metrics - numberOfReservations *prometheus.GaugeVec - reservedResources *prometheus.GaugeVec -} - -func NewMonitor(k8sClient client.Client) Monitor { - return Monitor{ - Client: k8sClient, - numberOfReservations: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "cortex_reservations", - Help: "Number of reservations by readiness and error status.", - }, []string{"status_ready", "status_error"}), - reservedResources: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "cortex_reservations_allocated_resources", - Help: "Resource units allocated across active reservations, by host and resource type.", - }, []string{"status_ready", "status_error", "host", "resource"}), - } -} - -// Describe the metrics for Prometheus. -func (m *Monitor) Describe(ch chan<- *prometheus.Desc) { - m.numberOfReservations.Describe(ch) - m.reservedResources.Describe(ch) -} - -// Collect the metrics on-demand and send them to Prometheus. -func (m *Monitor) Collect(ch chan<- prometheus.Metric) { - // Fetch all reservations from kubernetes. - var reservations v1alpha1.ReservationList - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - if err := m.List( - ctx, - &reservations, - ); err != nil { - monitorLog.Error(err, "failed to list reservations") - return - } - - // Reset all gauges so deleted reservations don't linger. - m.numberOfReservations.Reset() - m.reservedResources.Reset() - - countByLabels := map[string]uint64{} - for _, reservation := range reservations.Items { - readyCondition := meta.FindStatusCondition(reservation.Status.Conditions, v1alpha1.ReservationConditionReady) - readyStatus := "Unknown" - errorMsg := "" - if readyCondition != nil { - readyStatus = string(readyCondition.Status) - if readyCondition.Status == metav1.ConditionFalse { - errorMsg = readyCondition.Message - } - } - key := readyStatus + - "," + strings.ReplaceAll(errorMsg, ",", ";") - countByLabels[key]++ - } - for key, count := range countByLabels { - labelValues := strings.Split(key, ",") - m.numberOfReservations.WithLabelValues(labelValues...).Set(float64(count)) - } - m.numberOfReservations.Collect(ch) - - resourcesByLabels := map[string]map[string]uint64{} - for _, reservation := range reservations.Items { - host := reservation.Status.Host - readyCondition := meta.FindStatusCondition(reservation.Status.Conditions, v1alpha1.ReservationConditionReady) - readyStatus := "Unknown" - errorMsg := "" - if readyCondition != nil { - readyStatus = string(readyCondition.Status) - if readyCondition.Status == metav1.ConditionFalse { - errorMsg = readyCondition.Message - } - } - key := readyStatus + - "," + strings.ReplaceAll(errorMsg, ",", ";") + - "," + host - if _, ok := resourcesByLabels[key]; !ok { - resourcesByLabels[key] = map[string]uint64{} - } - for resourceName, resourceQuantity := range reservation.Spec.Resources { - resourcesByLabels[key][string(resourceName)] += resourceQuantity.AsDec().UnscaledBig().Uint64() - } - } - for key, resources := range resourcesByLabels { - labelValues := strings.Split(key, ",") - for resource, value := range resources { - m.reservedResources. - WithLabelValues(append(labelValues, resource)...). - Set(float64(value)) - } - } - m.reservedResources.Collect(ch) - monitorLog.Info("collected reservation metrics", "reservations", len(reservations.Items)) -} diff --git a/internal/scheduling/reservations/monitor_test.go b/internal/scheduling/reservations/monitor_test.go deleted file mode 100644 index cc479955a..000000000 --- a/internal/scheduling/reservations/monitor_test.go +++ /dev/null @@ -1,428 +0,0 @@ -// Copyright SAP SE -// SPDX-License-Identifier: Apache-2.0 - -package reservations - -import ( - "testing" - - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - - "github.com/cobaltcore-dev/cortex/api/v1alpha1" - hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" - "github.com/prometheus/client_golang/prometheus" - dto "github.com/prometheus/client_model/go" -) - -func TestMonitor_Init(t *testing.T) { - monitor := NewMonitor(nil) - - if monitor.numberOfReservations == nil { - t.Error("numberOfReservations metric should be initialized") - } - - if monitor.reservedResources == nil { - t.Error("reservedResources metric should be initialized") - } -} - -func TestMonitor_Describe(t *testing.T) { - monitor := NewMonitor(nil) - - ch := make(chan *prometheus.Desc, 10) - go func() { - monitor.Describe(ch) - close(ch) - }() - - // Count the number of descriptions - count := 0 - for range ch { - count++ - } - - // Should have descriptions for both metrics - if count != 2 { - t.Errorf("Expected 2 metric descriptions, got %d", count) - } -} - -func TestMonitor_Collect_EmptyList(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add scheme: %v", err) - } - - k8sClient := fake.NewClientBuilder(). - WithScheme(scheme). - Build() - - monitor := NewMonitor(k8sClient) - - ch := make(chan prometheus.Metric, 10) - go func() { - monitor.Collect(ch) - close(ch) - }() - - // Count the metrics - count := 0 - for range ch { - count++ - } - - // Should have at least the base metrics even with empty list - if count < 0 { - t.Errorf("Expected at least 0 metrics, got %d", count) - } -} - -func TestMonitor_Collect_WithReservations(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add scheme: %v", err) - } - - // Create test reservations - reservations := []v1alpha1.Reservation{ - { - ObjectMeta: ctrl.ObjectMeta{ - Name: "test-reservation-1", - }, - Spec: v1alpha1.ReservationSpec{ - Type: v1alpha1.ReservationTypeCommittedResource, - CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ - ResourceName: "test-flavor", - }, - Resources: map[hv1.ResourceName]resource.Quantity{ - hv1.ResourceMemory: resource.MustParse("1Gi"), - hv1.ResourceCPU: resource.MustParse("2"), - }, - }, - Status: v1alpha1.ReservationStatus{ - Conditions: []metav1.Condition{ - { - Type: v1alpha1.ReservationConditionReady, - Status: metav1.ConditionTrue, - Reason: "ReservationActive", - }, - }, - Host: "test-host-1", - }, - }, - { - ObjectMeta: ctrl.ObjectMeta{ - Name: "test-reservation-2", - }, - Spec: v1alpha1.ReservationSpec{ - Type: v1alpha1.ReservationTypeCommittedResource, - CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ - ResourceName: "test-flavor", - }, - Resources: map[hv1.ResourceName]resource.Quantity{ - hv1.ResourceMemory: resource.MustParse("2Gi"), - hv1.ResourceCPU: resource.MustParse("4"), - }, - }, - Status: v1alpha1.ReservationStatus{ - Conditions: []metav1.Condition{ - { - Type: v1alpha1.ReservationConditionReady, - Status: metav1.ConditionFalse, - Reason: "SomeError", - Message: "Failed due to some error", - }, - }, - }, - }, - { - ObjectMeta: ctrl.ObjectMeta{ - Name: "test-reservation-3", - }, - Spec: v1alpha1.ReservationSpec{ - Type: v1alpha1.ReservationTypeCommittedResource, - CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ - ResourceName: "test-flavor", - }, - Resources: map[hv1.ResourceName]resource.Quantity{ - hv1.ResourceMemory: resource.MustParse("4Gi"), - hv1.ResourceCPU: resource.MustParse("4"), - }, - }, - Status: v1alpha1.ReservationStatus{ - Conditions: []metav1.Condition{ - { - Type: v1alpha1.ReservationConditionReady, - Status: metav1.ConditionTrue, - Reason: "ReservationActive", - }, - }, - }, - }, - } - - // Convert to client.Object slice - objects := make([]client.Object, len(reservations)) - for i := range reservations { - objects[i] = &reservations[i] - } - - k8sClient := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(objects...). - Build() - - monitor := NewMonitor(k8sClient) - - ch := make(chan prometheus.Metric, 100) - go func() { - monitor.Collect(ch) - close(ch) - }() - - // Collect all metrics - metrics := []prometheus.Metric{} - for metric := range ch { - metrics = append(metrics, metric) - } - - if len(metrics) == 0 { - t.Error("Expected some metrics to be collected") - } - - // Verify that we have metrics for different ready states - foundReadyTrue := false - foundReadyFalse := false - - for _, metric := range metrics { - var m dto.Metric - if err := metric.Write(&m); err != nil { - continue - } - - // Check if this is a numberOfReservations metric - if m.GetGauge() != nil { - labels := make(map[string]string) - for _, label := range m.GetLabel() { - labels[label.GetName()] = label.GetValue() - } - - if labels["status_ready"] == "True" { - foundReadyTrue = true - } - if labels["status_ready"] == "False" { - foundReadyFalse = true - } - } - } - - if !foundReadyTrue { - t.Error("Expected to find Ready=True reservation metric") - } - if !foundReadyFalse { - t.Error("Expected to find Ready=False reservation metric") - } -} - -func TestMonitor_Collect_ResourceMetrics(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add scheme: %v", err) - } - - // Create test reservation with specific resource values - reservation := &v1alpha1.Reservation{ - ObjectMeta: ctrl.ObjectMeta{ - Name: "test-reservation", - }, - Spec: v1alpha1.ReservationSpec{ - Type: v1alpha1.ReservationTypeCommittedResource, - CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ - ResourceName: "test-flavor", - }, - Resources: map[hv1.ResourceName]resource.Quantity{ - hv1.ResourceMemory: resource.MustParse("1000Mi"), - hv1.ResourceCPU: resource.MustParse("2"), - }, - }, - Status: v1alpha1.ReservationStatus{ - Conditions: []metav1.Condition{ - { - Type: v1alpha1.ReservationConditionReady, - Status: metav1.ConditionTrue, - Reason: "ReservationActive", - }, - }, - }, - } - - k8sClient := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(reservation). - Build() - - monitor := NewMonitor(k8sClient) - - ch := make(chan prometheus.Metric, 100) - go func() { - monitor.Collect(ch) - close(ch) - }() - - // Collect all metrics - metrics := []prometheus.Metric{} - for metric := range ch { - metrics = append(metrics, metric) - } - - // Look for resource metrics - foundCPU := false - foundMemory := false - - for _, metric := range metrics { - var m dto.Metric - if err := metric.Write(&m); err != nil { - continue - } - - if m.GetGauge() != nil { - labels := make(map[string]string) - for _, label := range m.GetLabel() { - labels[label.GetName()] = label.GetValue() - } - - if labels["resource"] == "cpu" { - foundCPU = true - // CPU resource is stored as actual CPU count (not milli-CPUs in this context) - expectedCPU := float64(2) // 2 CPUs - if m.GetGauge().GetValue() != expectedCPU { - t.Errorf("Expected CPU value %f, got %f", expectedCPU, m.GetGauge().GetValue()) - } - } - if labels["resource"] == "memory" { - foundMemory = true - // Memory: 1000Mi = 1000 * 1024 * 1024 bytes = 1048576000 bytes - expectedMemory := float64(1048576000) // 1000Mi in bytes - if m.GetGauge().GetValue() != expectedMemory { - t.Errorf("Expected memory value %f, got %f", expectedMemory, m.GetGauge().GetValue()) - } - } - } - } - - if !foundCPU { - t.Error("Expected to find CPU resource metric") - } - if !foundMemory { - t.Error("Expected to find memory resource metric") - } -} - -func TestMonitor_Collect_ErrorHandling(t *testing.T) { - // Test with a client that will fail to list reservations - scheme := runtime.NewScheme() - // Don't add the scheme, which should cause the List operation to fail - - k8sClient := fake.NewClientBuilder(). - WithScheme(scheme). - Build() - - monitor := NewMonitor(k8sClient) - - ch := make(chan prometheus.Metric, 10) - go func() { - monitor.Collect(ch) - close(ch) - }() - - // Should not panic and should handle the error gracefully - count := 0 - for range ch { - count++ - } - - // Should not collect any metrics due to the error - if count != 0 { - t.Errorf("Expected 0 metrics due to error, got %d", count) - } -} - -func TestMonitor_Collect_LabelSanitization(t *testing.T) { - scheme := runtime.NewScheme() - if err := v1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("Failed to add scheme: %v", err) - } - - // Create test reservation with error containing commas - reservation := &v1alpha1.Reservation{ - ObjectMeta: ctrl.ObjectMeta{ - Name: "test-reservation", - }, - Spec: v1alpha1.ReservationSpec{ - Type: v1alpha1.ReservationTypeCommittedResource, - CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ - ResourceName: "test-flavor", - }, - Resources: map[hv1.ResourceName]resource.Quantity{ - hv1.ResourceMemory: resource.MustParse("1Gi"), - hv1.ResourceCPU: resource.MustParse("2"), - }, - }, - Status: v1alpha1.ReservationStatus{ - Conditions: []metav1.Condition{ - { - Type: v1alpha1.ReservationConditionReady, - Status: metav1.ConditionFalse, - Reason: "SomeError", - Message: "error with, commas, in it", - }, - }, - }, - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(reservation). - Build() - - monitor := NewMonitor(client) - - ch := make(chan prometheus.Metric, 100) - go func() { - monitor.Collect(ch) - close(ch) - }() - - // Collect all metrics - metrics := []prometheus.Metric{} - for metric := range ch { - metrics = append(metrics, metric) - } - - // Verify that commas in error messages are replaced with semicolons - foundSanitizedError := false - for _, metric := range metrics { - var m dto.Metric - if err := metric.Write(&m); err != nil { - continue - } - - if m.GetGauge() != nil { - for _, label := range m.GetLabel() { - if label.GetName() == "status_error" && label.GetValue() == "error with; commas; in it" { - foundSanitizedError = true - break - } - } - } - } - - if !foundSanitizedError { - t.Error("Expected to find sanitized error label with semicolons instead of commas") - } -} diff --git a/pkg/monitoring/log_metrics.go b/pkg/monitoring/log_metrics.go index 0137343e6..30c216ce3 100644 --- a/pkg/monitoring/log_metrics.go +++ b/pkg/monitoring/log_metrics.go @@ -23,7 +23,7 @@ func NewLogMetricsMonitor() LogMetricsMonitor { counter: prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: "cortex", Name: "log_messages_total", - Help: "Total number of log messages emitted at warn or error level.", + Help: "Total number of log messages emitted at info, warn or error level.", }, []string{"level", "file"}), } } @@ -58,10 +58,13 @@ func shortFilePath(file string) string { // increment the log-message counter for warn and error entries. func WrapCoreWithLogMetrics(monitor *LogMetricsMonitor, core zapcore.Core) zapcore.Core { return zapcore.RegisterHooks(core, func(e zapcore.Entry) error { - if e.Level >= zapcore.WarnLevel { - level := "warn" - if e.Level >= zapcore.ErrorLevel { + if e.Level >= zapcore.InfoLevel { + level := "info" + switch { + case e.Level >= zapcore.ErrorLevel: level = "error" + case e.Level >= zapcore.WarnLevel: + level = "warn" } file := "unknown" if e.Caller.Defined { diff --git a/pkg/monitoring/log_metrics_test.go b/pkg/monitoring/log_metrics_test.go index d9c7541f7..2f85523ab 100644 --- a/pkg/monitoring/log_metrics_test.go +++ b/pkg/monitoring/log_metrics_test.go @@ -111,6 +111,7 @@ func TestWrapCoreWithLogMetrics(t *testing.T) { tests := []struct { name string entries []entry + wantInfo float64 wantWarn float64 wantError float64 wantUnknown float64 @@ -144,11 +145,12 @@ func TestWrapCoreWithLogMetrics(t *testing.T) { wantError: 2, }, { - name: "debug and info are not counted", + name: "info is counted, debug is not", entries: []entry{ {zapcore.DebugLevel, true}, {zapcore.InfoLevel, true}, }, + wantInfo: 1, wantWarn: 0, wantError: 0, }, @@ -172,6 +174,7 @@ func TestWrapCoreWithLogMetrics(t *testing.T) { {zapcore.ErrorLevel, true}, {zapcore.ErrorLevel, true}, }, + wantInfo: 1, wantWarn: 2, wantError: 2, wantUnknown: 1, @@ -212,6 +215,9 @@ func TestWrapCoreWithLogMetrics(t *testing.T) { } } counts := gatherCounts(t, &monitor) + if got := sumLevel(counts, "info"); got != tt.wantInfo { + t.Errorf("info count: got %v, want %v", got, tt.wantInfo) + } if got := sumLevel(counts, "warn"); got != tt.wantWarn { t.Errorf("warn count: got %v, want %v", got, tt.wantWarn) } @@ -223,8 +229,8 @@ func TestWrapCoreWithLogMetrics(t *testing.T) { t.Errorf("warn[unknown] count: got %v, want %v", got, tt.wantUnknown) } } - if len(counts["debug"]) > 0 || len(counts["info"]) > 0 { - t.Errorf("debug/info should not be counted, got debug=%v info=%v", counts["debug"], counts["info"]) + if len(counts["debug"]) > 0 { + t.Errorf("debug should not be counted, got debug=%v", counts["debug"]) } }) } diff --git a/postgres/Dockerfile b/postgres/Dockerfile index 93cf6e257..4e9880a27 100644 --- a/postgres/Dockerfile +++ b/postgres/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:trixie-slim@sha256:62bd423aeee912639ca8ee681c2b762cb3e25137e0a92b20be455898a9f034ad +FROM debian:trixie-slim@sha256:f6b08b09a05725f9431502d3b5714f5c4576e82c1a84fa1debe0c24a7e0c3ec4 # explicitly set user/group IDs RUN set -eux; \ diff --git a/tools/mirror/apply-with-status.sh b/tools/mirror/apply-with-status.sh new file mode 100755 index 000000000..7e417f365 --- /dev/null +++ b/tools/mirror/apply-with-status.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# +# apply-with-status.sh — apply dumped Kubernetes resources into a cluster, +# status subresource included. +# +# WHAT IT DOES +# `kubectl apply` never restores the `.status` of an object: status lives on a +# separate subresource that the API server owns. When you want to replicate a +# snapshot of resources (e.g. captured with `kubectl get ... -o yaml`) into +# another cluster *with their status intact*, a plain apply is not enough. +# +# This script does it in two passes: +# 1. Sanitize and server-side apply every item, creating the objects. +# 2. Patch each object's `.status` onto the freshly created resource via the +# status subresource. +# +# Before applying, it strips metadata that binds objects to their origin +# cluster. The important one is `ownerReferences`: those point at owner +# objects (e.g. Nodes) that don't exist in the target cluster, so Kubernetes' +# garbage collector would delete the resources moments after creation. The +# other fields (uid, resourceVersion, generation, ...) are server-managed and +# would be rejected or cause conflicts if carried over. +# +# INPUT +# Each file in FILES is expected to be a Kubernetes List (the shape produced +# by `kubectl get -o yaml`), i.e. an object with an `.items[]` array. +# Capture such a snapshot from the source cluster like: +# +# kubectl get hypervisors -o yaml > hvs.yaml.out +# kubectl get virtualmachines -o yaml > vms.yaml.out +# +# USAGE +# Point your kubeconfig/context at the *target* cluster, then run from the +# directory holding the snapshot files: +# +# ./apply-with-status.sh +# +# By default the standard kubectl config (the KUBECONFIG env var, else +# ~/.kube/config) and its current context are used. Override them for a +# single run via environment variables: +# +# MIRROR_KUBECONFIG=target.yaml MIRROR_CONTEXT=kind-cluster-a ./apply-with-status.sh +# +# Adjust the FILES array below to match the snapshots you want to replay. +# +# REQUIREMENTS +# bash, kubectl, and yq (https://github.com/mikefarah/yq). +# +set -euo pipefail + +# Snapshot files to replay, relative to the current directory. Each must be a +# Kubernetes List (`.items[]`) as produced by `kubectl get ... -o yaml`. +FILES=(hvs.yaml.out vms.yaml.out) + +# Seconds to wait between the apply pass and the status-patch pass, giving the +# API server a moment to persist all created objects. +SETTLE_SECONDS=3 + +# Optional kubeconfig path and context for the target cluster. When empty, +# kubectl falls back to its default config (KUBECONFIG env var, else +# ~/.kube/config) and current context. +MIRROR_KUBECONFIG="${MIRROR_KUBECONFIG:-}" +MIRROR_CONTEXT="${MIRROR_CONTEXT:-}" + +# Assemble the kubectl flags for the chosen config/context. Empty when neither +# is set, so kubectl uses its own defaults. +kubectl_args=() +if [ -n "$MIRROR_KUBECONFIG" ]; then + kubectl_args+=(--kubeconfig "$MIRROR_KUBECONFIG") +fi +if [ -n "$MIRROR_CONTEXT" ]; then + kubectl_args+=(--context "$MIRROR_CONTEXT") +fi + +for bin in kubectl yq; do + if ! command -v "$bin" >/dev/null 2>&1; then + echo "error: required command '$bin' not found in PATH" >&2 + exit 1 + fi +done + +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +# Strip fields that bind the objects to the source cluster. ownerReferences are +# the important one: they point at owner objects that don't exist here, so the +# garbage collector deletes the resources right after creation. The rest are +# server-managed and would conflict with the target cluster's own bookkeeping. +sanitize='del(.metadata.ownerReferences, .metadata.uid, .metadata.resourceVersion, + .metadata.generation, .metadata.creationTimestamp, + .metadata.managedFields, .metadata.selfLink)' + +# Pass 1: create every object (spec only; status is ignored by apply). +for f in "${FILES[@]}"; do + clean="$tmp/$(basename "$f").clean" + yq "(.items[]) |= ($sanitize)" "$f" > "$clean" + echo ">>> applying specs from $f" + kubectl ${kubectl_args[@]+"${kubectl_args[@]}"} apply --server-side --force-conflicts -f "$clean" +done + +# Give the API server a moment to persist all created objects. +sleep "$SETTLE_SECONDS" + +# Pass 2: patch each object's status onto the now-existing resource. +for f in "${FILES[@]}"; do + n=$(yq '.items | length' "$f") + echo ">>> patching status for $n items from $f" + for i in $(seq 0 $((n - 1))); do + obj="$tmp/obj.yaml" + st="$tmp/status.yaml" + yq ".items[$i]" "$f" > "$obj" + kind=$(yq '.kind' "$obj") + name=$(yq '.metadata.name' "$obj") + # Skip items that carry no status — nothing to patch. + if [ "$(yq '.status // "null"' "$obj")" = "null" ]; then + continue + fi + yq '{"status": .status}' "$obj" > "$st" + kubectl ${kubectl_args[@]+"${kubectl_args[@]}"} patch "$kind" "$name" --subresource=status --type=merge --patch-file="$st" + done +done