From 95a1085b7fdacfb4ec4f0fc6feae2b7551945ea6 Mon Sep 17 00:00:00 2001 From: rldyourmnd Date: Thu, 27 Aug 2026 20:17:49 +0500 Subject: [PATCH] feat(observability): budget classified host signals --- config/observability-dashboards.yaml | 31 ++++++++++++++++++ config/observability-rules.yaml | 32 +++++++++++++++++++ .../dashboards_test.go | 2 +- .../openobserve_test.go | 2 +- internal/observabilityrules/rules_test.go | 8 +++-- 5 files changed, 70 insertions(+), 5 deletions(-) diff --git a/config/observability-dashboards.yaml b/config/observability-dashboards.yaml index ab6a1b8..893343b 100644 --- a/config/observability-dashboards.yaml +++ b/config/observability-dashboards.yaml @@ -95,6 +95,37 @@ dashboards: query: max(gha_diagnostic_storage_oldest_object_age_seconds) unit: seconds description: Age of the oldest retained diagnostic object. + - id: host_signals + title: Classified host signals + refresh_seconds: 60 + default_range: 24h + owner: fleet-performance + runbook: https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md + panels: + - id: all_host_signals + title: Host signals by class and member + kind: timeseries + query: sum by (host_name, signal_class) (max_over_time(gha_fleet_host_signal_events{aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]) - min_over_time(gha_fleet_host_signal_events{aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h])) + unit: count + description: OTEL-classified host events observed in each rolling hour without retaining duplicate raw logs. + - id: audit_suppression + title: Audit callback suppressions + kind: timeseries + query: sum by (host_name) (max_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]) - min_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h])) + unit: count + description: Kernel notices that audit callbacks were suppressed during high event volume. + - id: kernel_workqueue_hog + title: Kernel workqueue hog events + kind: timeseries + query: sum by (host_name) (max_over_time(gha_fleet_host_signal_events{signal_class="kernel_workqueue_hog",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[24h]) - min_over_time(gha_fleet_host_signal_events{signal_class="kernel_workqueue_hog",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[24h])) + unit: count + description: Kernel workqueue CPU-hog signals that may precede host latency or pressure. + - id: ufw_blocks + title: UFW block events + kind: timeseries + query: sum by (host_name) (max_over_time(gha_fleet_host_signal_events{signal_class="ufw_block",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]) - min_over_time(gha_fleet_host_signal_events{signal_class="ufw_block",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h])) + unit: count + description: Firewall block volume retained as an OTEL metric rather than repeated kernel log records. - id: lifecycle_latency title: Job lifecycle latency refresh_seconds: 30 diff --git a/config/observability-rules.yaml b/config/observability-rules.yaml index ba477ea..39d959e 100644 --- a/config/observability-rules.yaml +++ b/config/observability-rules.yaml @@ -2,6 +2,22 @@ schema_version: 2 backend: openobserve organization: default rules: + - id: audit_suppression_burst + severity: ticket + query_language: promql + stream_name: gha_fleet_host_signal_events + expression: max(max_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[15m]) - min_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[15m])) + operator: ">" + threshold: 30 + evaluation_seconds: 300 + hold_seconds: 600 + destination_ref: fleet_oncall + enabled: false + owner: fleet-performance + runbook: https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md + summary: Kernel audit delivery is repeatedly suppressing callbacks across the fleet. + action: Correlate the affected hosts with AppArmor and workload churn, then remove the noisy rule source without weakening audit coverage. + recovery: No member records more than thirty new suppression notices in a rolling fifteen-minute window for ten minutes. - id: collector_queue_near_full severity: page query_language: promql @@ -162,6 +178,22 @@ rules: summary: Unreclaimable kernel slab exceeds two GiB on a fleet host. action: Preserve slab, audit and AppArmor evidence; close member admission and roll to the fixed kernel after jobs drain. recovery: Every fleet host remains below two GiB unreclaimable slab after workload churn. + - id: kernel_workqueue_hog + severity: page + query_language: promql + stream_name: gha_fleet_host_signal_events + expression: sum(max_over_time(gha_fleet_host_signal_events{signal_class="kernel_workqueue_hog",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[15m])) + operator: ">" + threshold: 0 + evaluation_seconds: 60 + hold_seconds: 60 + destination_ref: fleet_oncall + enabled: false + owner: fleet-operations + runbook: https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md + summary: A fleet kernel workqueue reported a CPU hog event. + action: Preserve the kernel message and host pressure state, then close admission on the affected member if latency or PSI is elevated. + recovery: No workqueue-hog event is present in the rolling fifteen-minute observation window. - id: lifecycle_assigned_stall severity: page query_language: promql diff --git a/internal/observabilitydashboards/dashboards_test.go b/internal/observabilitydashboards/dashboards_test.go index 43bd75a..8e4e187 100644 --- a/internal/observabilitydashboards/dashboards_test.go +++ b/internal/observabilitydashboards/dashboards_test.go @@ -21,7 +21,7 @@ func TestPublishedDashboardBundleIsValidAndRenderable(t *testing.T) { if err != nil { t.Fatal(err) } - if len(bundle.Dashboards) != 7 { + if len(bundle.Dashboards) != 8 { t.Fatalf("dashboards=%d", len(bundle.Dashboards)) } rendered, err := Render(bundle) diff --git a/internal/observabilitydashboards/openobserve_test.go b/internal/observabilitydashboards/openobserve_test.go index acb9d0e..93cd756 100644 --- a/internal/observabilitydashboards/openobserve_test.go +++ b/internal/observabilitydashboards/openobserve_test.go @@ -14,7 +14,7 @@ func TestRenderOpenObserveV8IsDeterministicAndManaged(t *testing.T) { if err != nil { t.Fatal(err) } - if len(dashboards) != 7 { + if len(dashboards) != 8 { t.Fatalf("dashboards=%d", len(dashboards)) } for _, dashboard := range dashboards { diff --git a/internal/observabilityrules/rules_test.go b/internal/observabilityrules/rules_test.go index 80de00f..c8b0960 100644 --- a/internal/observabilityrules/rules_test.go +++ b/internal/observabilityrules/rules_test.go @@ -10,8 +10,8 @@ func TestRepositoryBundleIsValid(t *testing.T) { if err != nil { t.Fatal(err) } - if len(bundle.Rules) != 19 { - t.Fatalf("rules = %d, want 19", len(bundle.Rules)) + if len(bundle.Rules) != 21 { + t.Fatalf("rules = %d, want 21", len(bundle.Rules)) } } @@ -30,6 +30,8 @@ func TestRepositoryRulesUseCurrentMetricSemantics(t *testing.T) { "compute_pressure_state_stale": "gha_fleet_pressure_observer_up", "compute_root_disk_low": "system_filesystem_usage", "kernel_slab_unreclaimable": `state="slab_unreclaimable"`, + "audit_suppression_burst": `signal_class="audit_suppressed"`, + "kernel_workqueue_hog": `signal_class="kernel_workqueue_hog"`, } seen := make(map[string]bool, len(wanted)) for _, rule := range bundle.Rules { @@ -76,7 +78,7 @@ func TestRulesRejectUnsafeOrUnactionableChanges(t *testing.T) { } for name, mutate := range map[string]func(*Rule){ "unknown severity": func(r *Rule) { r.Severity = "noise" }, - "slow page": func(r *Rule) { r.HoldSecs = 3600 }, + "slow page": func(r *Rule) { r.Severity, r.HoldSecs = "page", 3600 }, "fast ticket": func(r *Rule) { r.Severity, r.HoldSecs = "ticket", 60 }, "private runbook": func(r *Rule) { r.Runbook = "https://example.invalid/private" }, "unknown operator": func(r *Rule) { r.Operator = "contains" },