feat(observability): budget classified host signals

config/observability-dashboards.yaml

@@ -95,6 +95,37 @@ dashboards:
         query: max(gha_diagnostic_storage_oldest_object_age_seconds)
         unit: seconds
         description: Age of the oldest retained diagnostic object.
+  - id: host_signals
+    title: Classified host signals
+    refresh_seconds: 60
+    default_range: 24h
+    owner: fleet-performance
+    runbook: https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md
+    panels:
+      - id: all_host_signals
+        title: Host signals by class and member
+        kind: timeseries
+        query: sum by (host_name, signal_class) (max_over_time(gha_fleet_host_signal_events{aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]) - min_over_time(gha_fleet_host_signal_events{aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]))
+        unit: count
+        description: OTEL-classified host events observed in each rolling hour without retaining duplicate raw logs.
+      - id: audit_suppression
+        title: Audit callback suppressions
+        kind: timeseries
+        query: sum by (host_name) (max_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]) - min_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]))
+        unit: count
+        description: Kernel notices that audit callbacks were suppressed during high event volume.
+      - id: kernel_workqueue_hog
+        title: Kernel workqueue hog events
+        kind: timeseries
+        query: sum by (host_name) (max_over_time(gha_fleet_host_signal_events{signal_class="kernel_workqueue_hog",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[24h]) - min_over_time(gha_fleet_host_signal_events{signal_class="kernel_workqueue_hog",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[24h]))
+        unit: count
+        description: Kernel workqueue CPU-hog signals that may precede host latency or pressure.
+      - id: ufw_blocks
+        title: UFW block events
+        kind: timeseries
+        query: sum by (host_name) (max_over_time(gha_fleet_host_signal_events{signal_class="ufw_block",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]) - min_over_time(gha_fleet_host_signal_events{signal_class="ufw_block",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[1h]))
+        unit: count
+        description: Firewall block volume retained as an OTEL metric rather than repeated kernel log records.
   - id: lifecycle_latency
     title: Job lifecycle latency
     refresh_seconds: 30

config/observability-rules.yaml

@@ -2,6 +2,22 @@ schema_version: 2
 backend: openobserve
 organization: default
 rules:
+  - id: audit_suppression_burst
+    severity: ticket
+    query_language: promql
+    stream_name: gha_fleet_host_signal_events
+    expression: max(max_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[15m]) - min_over_time(gha_fleet_host_signal_events{signal_class="audit_suppressed",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[15m]))
+    operator: ">"
+    threshold: 30
+    evaluation_seconds: 300
+    hold_seconds: 600
+    destination_ref: fleet_oncall
+    enabled: false
+    owner: fleet-performance
+    runbook: https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md
+    summary: Kernel audit delivery is repeatedly suppressing callbacks across the fleet.
+    action: Correlate the affected hosts with AppArmor and workload churn, then remove the noisy rule source without weakening audit coverage.
+    recovery: No member records more than thirty new suppression notices in a rolling fifteen-minute window for ten minutes.
   - id: collector_queue_near_full
     severity: page
     query_language: promql
@@ -162,6 +178,22 @@ rules:
     summary: Unreclaimable kernel slab exceeds two GiB on a fleet host.
     action: Preserve slab, audit and AppArmor evidence; close member admission and roll to the fixed kernel after jobs drain.
     recovery: Every fleet host remains below two GiB unreclaimable slab after workload churn.
+  - id: kernel_workqueue_hog
+    severity: page
+    query_language: promql
+    stream_name: gha_fleet_host_signal_events
+    expression: sum(max_over_time(gha_fleet_host_signal_events{signal_class="kernel_workqueue_hog",aggregation_temporality="AGGREGATION_TEMPORALITY_CUMULATIVE"}[15m]))
+    operator: ">"
+    threshold: 0
+    evaluation_seconds: 60
+    hold_seconds: 60
+    destination_ref: fleet_oncall
+    enabled: false
+    owner: fleet-operations
+    runbook: https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md
+    summary: A fleet kernel workqueue reported a CPU hog event.
+    action: Preserve the kernel message and host pressure state, then close admission on the affected member if latency or PSI is elevated.
+    recovery: No workqueue-hog event is present in the rolling fifteen-minute observation window.
   - id: lifecycle_assigned_stall
     severity: page
     query_language: promql

internal/observabilitydashboards/dashboards_test.go

@@ -21,7 +21,7 @@ func TestPublishedDashboardBundleIsValidAndRenderable(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if len(bundle.Dashboards) != 7 {
+	if len(bundle.Dashboards) != 8 {
 		t.Fatalf("dashboards=%d", len(bundle.Dashboards))
 	}
 	rendered, err := Render(bundle)

internal/observabilitydashboards/openobserve_test.go

@@ -14,7 +14,7 @@ func TestRenderOpenObserveV8IsDeterministicAndManaged(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if len(dashboards) != 7 {
+	if len(dashboards) != 8 {
 		t.Fatalf("dashboards=%d", len(dashboards))
 	}
 	for _, dashboard := range dashboards {

internal/observabilityrules/rules_test.go

@@ -10,8 +10,8 @@ func TestRepositoryBundleIsValid(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if len(bundle.Rules) != 19 {
-		t.Fatalf("rules = %d, want 19", len(bundle.Rules))
+	if len(bundle.Rules) != 21 {
+		t.Fatalf("rules = %d, want 21", len(bundle.Rules))
 	}
 }
 
@@ -30,6 +30,8 @@ func TestRepositoryRulesUseCurrentMetricSemantics(t *testing.T) {
 		"compute_pressure_state_stale":      "gha_fleet_pressure_observer_up",
 		"compute_root_disk_low":             "system_filesystem_usage",
 		"kernel_slab_unreclaimable":         `state="slab_unreclaimable"`,
+		"audit_suppression_burst":           `signal_class="audit_suppressed"`,
+		"kernel_workqueue_hog":              `signal_class="kernel_workqueue_hog"`,
 	}
 	seen := make(map[string]bool, len(wanted))
 	for _, rule := range bundle.Rules {
@@ -76,7 +78,7 @@ func TestRulesRejectUnsafeOrUnactionableChanges(t *testing.T) {
 	}
 	for name, mutate := range map[string]func(*Rule){
 		"unknown severity": func(r *Rule) { r.Severity = "noise" },
-		"slow page":        func(r *Rule) { r.HoldSecs = 3600 },
+		"slow page":        func(r *Rule) { r.Severity, r.HoldSecs = "page", 3600 },
 		"fast ticket":      func(r *Rule) { r.Severity, r.HoldSecs = "ticket", 60 },
 		"private runbook":  func(r *Rule) { r.Runbook = "https://example.invalid/private" },
 		"unknown operator": func(r *Rule) { r.Operator = "contains" },