From 705d3b208e73000177983e88db6fd1e4df767e77 Mon Sep 17 00:00:00 2001 From: rldyourmnd Date: Thu, 27 Aug 2026 20:28:19 +0500 Subject: [PATCH] feat(observability): expose host compliance state --- config/observability-dashboards.yaml | 31 ++++++ config/observability-rules.yaml | 64 +++++++++++ internal/fleetobserve/http.go | 6 +- .../dashboards_test.go | 2 +- .../openobserve_test.go | 2 +- internal/observabilityrules/rules_test.go | 8 +- internal/pressureobserve/compliance.go | 104 ++++++++++++++++++ internal/pressureobserve/compliance_test.go | 56 ++++++++++ internal/pressureobserve/observe.go | 4 +- 9 files changed, 271 insertions(+), 6 deletions(-) create mode 100644 internal/pressureobserve/compliance.go create mode 100644 internal/pressureobserve/compliance_test.go diff --git a/config/observability-dashboards.yaml b/config/observability-dashboards.yaml index 893343b..78dd59e 100644 --- a/config/observability-dashboards.yaml +++ b/config/observability-dashboards.yaml @@ -95,6 +95,37 @@ dashboards: query: max(gha_diagnostic_storage_oldest_object_age_seconds) unit: seconds description: Age of the oldest retained diagnostic object. + - id: host_compliance + title: Fleet host compliance + refresh_seconds: 60 + default_range: 24h + owner: fleet-operations + runbook: https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md + panels: + - id: compliance_coverage + title: Complete compliance observers + kind: stat + query: count(gha_fleet_host_compliance_observer_up == 1) + unit: count + description: Number of fleet hosts with readable kernel, vulnerability and package inventory state. + - id: maintenance_pending + title: Updates and reboot state + kind: timeseries + query: max by (host_name) (gha_fleet_host_standard_updates_available or gha_fleet_host_reboot_required) + unit: count + description: Standard package updates or a pending maintenance reboot reported by each host. + - id: running_kernel + title: Running kernel identity + kind: stat + query: max by (host_name, release) (gha_fleet_host_kernel_info) + unit: state + description: Exact running kernel release on every fleet host. + - id: srso_status + title: SRSO hardware status + kind: stat + query: max by (host_name, status) (gha_fleet_host_srso_status) + unit: state + description: Kernel-reported speculative return stack overflow status, including hardware microcode boundaries. - id: host_signals title: Classified host signals refresh_seconds: 60 diff --git a/config/observability-rules.yaml b/config/observability-rules.yaml index 39d959e..df30489 100644 --- a/config/observability-rules.yaml +++ b/config/observability-rules.yaml @@ -146,6 +146,22 @@ rules: summary: GitHub run or repository identity has not converged within two minutes. action: Correlate the exact queue UUID with JobAssigned, workflow run, provider lease and runner identity without recovering a transient pre-assignment gap. recovery: Both age-qualified correlation metrics remain zero while raw transition counters may continue to change. + - id: host_compliance_observer_missing + severity: page + query_language: promql + stream_name: gha_fleet_host_compliance_observer_up + expression: count(gha_fleet_host_compliance_observer_up == 1) + operator: "<" + threshold: 5 + evaluation_seconds: 60 + hold_seconds: 180 + destination_ref: fleet_oncall + enabled: false + owner: fleet-operations + runbook: https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md + summary: Kernel or package compliance telemetry is incomplete on a fleet host. + action: Restore the local observer or its read-only host inputs before declaring package and reboot state current. + recovery: Exactly five fleet hosts continuously export complete compliance state. - id: host_oom_detected severity: page query_language: promql @@ -162,6 +178,54 @@ rules: summary: A fleet host recorded a new OOM kill. action: Close admission on the affected member and preserve workload and pressure evidence. recovery: No new OOM delta occurs and memory pressure returns inside the promotion envelope. + - id: host_package_inventory_stale + severity: ticket + query_language: promql + stream_name: gha_fleet_host_package_inventory_age_seconds + expression: max(gha_fleet_host_package_inventory_age_seconds) + operator: ">" + threshold: 129600 + evaluation_seconds: 300 + hold_seconds: 3600 + destination_ref: fleet_oncall + enabled: false + owner: fleet-operations + runbook: https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md + summary: A fleet host package inventory has not refreshed for more than thirty-six hours. + action: Restore the Ubuntu metadata refresh path and preserve apt output before changing package state. + recovery: Every fleet host package inventory remains younger than thirty-six hours for one hour. + - id: host_reboot_required + severity: ticket + query_language: promql + stream_name: gha_fleet_host_reboot_required + expression: max(gha_fleet_host_reboot_required) + operator: ">" + threshold: 0 + evaluation_seconds: 300 + hold_seconds: 600 + destination_ref: fleet_oncall + enabled: false + owner: fleet-operations + runbook: https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md + summary: A fleet host still requires a reboot after package maintenance. + action: Drain jobs naturally and use the rolling reboot procedure while preserving cluster quorum. + recovery: No fleet host reports a pending reboot for ten minutes. + - id: host_standard_updates_available + severity: ticket + query_language: promql + stream_name: gha_fleet_host_standard_updates_available + expression: max(gha_fleet_host_standard_updates_available) + operator: ">" + threshold: 0 + evaluation_seconds: 300 + hold_seconds: 3600 + destination_ref: fleet_oncall + enabled: false + owner: fleet-operations + runbook: https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md + summary: Standard Ubuntu package updates remain available on a fleet host. + action: Apply the normal rolling package-maintenance procedure and prove service and runner recovery after each member. + recovery: Every fleet host reports zero standard updates for one hour. - id: kernel_slab_unreclaimable severity: ticket query_language: promql diff --git a/internal/fleetobserve/http.go b/internal/fleetobserve/http.go index cb0b19e..0244be2 100644 --- a/internal/fleetobserve/http.go +++ b/internal/fleetobserve/http.go @@ -5,6 +5,8 @@ import ( "net/http" "sync" "time" + + "github.com/NDDev-OpenNetwork/github-actions/internal/pressureobserve" ) type State struct { @@ -30,6 +32,7 @@ type Handler struct { State *State MaxStaleness time.Duration Now func() time.Time + HostRoot string } func (h Handler) ServeHTTP(writer http.ResponseWriter, request *http.Request) { @@ -56,7 +59,8 @@ func (h Handler) ServeHTTP(writer http.ResponseWriter, request *http.Request) { switch request.URL.Path { case "/metrics": writer.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") - _, _ = writer.Write([]byte(RenderPrometheus(snapshot, now, h.MaxStaleness))) + metrics := RenderPrometheus(snapshot, now, h.MaxStaleness) + pressureobserve.RenderCompliance(pressureobserve.CollectCompliance(h.HostRoot, now)) + _, _ = writer.Write([]byte(metrics)) case "/snapshot": writer.Header().Set("Content-Type", "application/json") writeHTTPJSON(writer, http.StatusOK, snapshot) diff --git a/internal/observabilitydashboards/dashboards_test.go b/internal/observabilitydashboards/dashboards_test.go index 8e4e187..3c20a01 100644 --- a/internal/observabilitydashboards/dashboards_test.go +++ b/internal/observabilitydashboards/dashboards_test.go @@ -21,7 +21,7 @@ func TestPublishedDashboardBundleIsValidAndRenderable(t *testing.T) { if err != nil { t.Fatal(err) } - if len(bundle.Dashboards) != 8 { + if len(bundle.Dashboards) != 9 { t.Fatalf("dashboards=%d", len(bundle.Dashboards)) } rendered, err := Render(bundle) diff --git a/internal/observabilitydashboards/openobserve_test.go b/internal/observabilitydashboards/openobserve_test.go index 93cd756..a4259eb 100644 --- a/internal/observabilitydashboards/openobserve_test.go +++ b/internal/observabilitydashboards/openobserve_test.go @@ -14,7 +14,7 @@ func TestRenderOpenObserveV8IsDeterministicAndManaged(t *testing.T) { if err != nil { t.Fatal(err) } - if len(dashboards) != 8 { + if len(dashboards) != 9 { t.Fatalf("dashboards=%d", len(dashboards)) } for _, dashboard := range dashboards { diff --git a/internal/observabilityrules/rules_test.go b/internal/observabilityrules/rules_test.go index c8b0960..8f1e462 100644 --- a/internal/observabilityrules/rules_test.go +++ b/internal/observabilityrules/rules_test.go @@ -10,8 +10,8 @@ func TestRepositoryBundleIsValid(t *testing.T) { if err != nil { t.Fatal(err) } - if len(bundle.Rules) != 21 { - t.Fatalf("rules = %d, want 21", len(bundle.Rules)) + if len(bundle.Rules) != 25 { + t.Fatalf("rules = %d, want 25", len(bundle.Rules)) } } @@ -32,6 +32,10 @@ func TestRepositoryRulesUseCurrentMetricSemantics(t *testing.T) { "kernel_slab_unreclaimable": `state="slab_unreclaimable"`, "audit_suppression_burst": `signal_class="audit_suppressed"`, "kernel_workqueue_hog": `signal_class="kernel_workqueue_hog"`, + "host_compliance_observer_missing": "gha_fleet_host_compliance_observer_up", + "host_package_inventory_stale": "gha_fleet_host_package_inventory_age_seconds", + "host_reboot_required": "gha_fleet_host_reboot_required", + "host_standard_updates_available": "gha_fleet_host_standard_updates_available", } seen := make(map[string]bool, len(wanted)) for _, rule := range bundle.Rules { diff --git a/internal/pressureobserve/compliance.go b/internal/pressureobserve/compliance.go new file mode 100644 index 0000000..9f49cfc --- /dev/null +++ b/internal/pressureobserve/compliance.go @@ -0,0 +1,104 @@ +package pressureobserve + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" +) + +var ( + standardUpdatesPattern = regexp.MustCompile(`(?m)([0-9]+) updates? can be applied immediately\.`) + esmUpdatesPattern = regexp.MustCompile(`(?m)([0-9]+) additional security updates? can be applied with ESM Apps\.`) +) + +type Compliance struct { + Complete bool + RebootRequired bool + KernelRelease string + SRSOStatus string + StandardUpdatesAvailable int + ESMSecurityUpdatesAvailable int + PackageInventoryAgeSeconds float64 +} + +func CollectCompliance(root string, now time.Time) Compliance { + if root == "" { + root = "/" + } + result := Compliance{PackageInventoryAgeSeconds: -1, SRSOStatus: "unknown"} + result.RebootRequired = regularFileExists(filepath.Join(root, "var", "run", "reboot-required")) + + kernel, kernelErr := os.ReadFile(filepath.Join(root, "proc", "sys", "kernel", "osrelease")) + if kernelErr == nil { + result.KernelRelease = strings.TrimSpace(string(kernel)) + } + srso, srsoErr := os.ReadFile(filepath.Join(root, "sys", "devices", "system", "cpu", "vulnerabilities", "spec_rstack_overflow")) + if srsoErr == nil { + result.SRSOStatus = classifySRSO(string(srso)) + } + + updatesPath := filepath.Join(root, "var", "lib", "update-notifier", "updates-available") + updates, updatesErr := os.ReadFile(updatesPath) + if updatesErr == nil { + result.StandardUpdatesAvailable = firstCount(standardUpdatesPattern, string(updates)) + result.ESMSecurityUpdatesAvailable = firstCount(esmUpdatesPattern, string(updates)) + if info, err := os.Stat(updatesPath); err == nil && !info.ModTime().After(now) { + result.PackageInventoryAgeSeconds = now.Sub(info.ModTime()).Seconds() + } + } + result.Complete = kernelErr == nil && result.KernelRelease != "" && srsoErr == nil && updatesErr == nil && result.PackageInventoryAgeSeconds >= 0 + return result +} + +func RenderCompliance(state Compliance) string { + var output strings.Builder + gauge := func(name, help string, value float64) { + fmt.Fprintf(&output, "# HELP %s %s\n# TYPE %s gauge\n%s %s\n", name, help, name, name, strconv.FormatFloat(value, 'f', -1, 64)) + } + gauge("gha_fleet_host_compliance_observer_up", "Whether kernel and package compliance state was read completely.", boolFloat(state.Complete)) + gauge("gha_fleet_host_reboot_required", "Whether the host requires a reboot to finish package maintenance.", boolFloat(state.RebootRequired)) + gauge("gha_fleet_host_standard_updates_available", "Standard Ubuntu package updates currently available.", float64(state.StandardUpdatesAvailable)) + gauge("gha_fleet_host_esm_security_updates_available", "Additional security updates available only through Ubuntu ESM Apps.", float64(state.ESMSecurityUpdatesAvailable)) + gauge("gha_fleet_host_package_inventory_age_seconds", "Age of the update-notifier package inventory, or -1 when unavailable.", state.PackageInventoryAgeSeconds) + fmt.Fprintf(&output, "# HELP gha_fleet_host_kernel_info Running host kernel identity.\n# TYPE gha_fleet_host_kernel_info gauge\ngha_fleet_host_kernel_info{release=%q} 1\n", escapeLabel(state.KernelRelease)) + fmt.Fprintf(&output, "# HELP gha_fleet_host_srso_status Speculative return stack overflow status reported by the running kernel.\n# TYPE gha_fleet_host_srso_status gauge\ngha_fleet_host_srso_status{status=%q} 1\n", escapeLabel(state.SRSOStatus)) + return output.String() +} + +func classifySRSO(value string) string { + normalized := strings.ToLower(strings.TrimSpace(value)) + switch { + case strings.Contains(normalized, "not affected"): + return "not_affected" + case strings.Contains(normalized, "vulnerable"): + return "vulnerable" + case strings.Contains(normalized, "mitigation"): + return "mitigated" + default: + return "unknown" + } +} + +func firstCount(pattern *regexp.Regexp, value string) int { + match := pattern.FindStringSubmatch(value) + if len(match) != 2 { + return 0 + } + count, _ := strconv.Atoi(match[1]) + return count +} + +func regularFileExists(path string) bool { + info, err := os.Stat(path) + return err == nil && info.Mode().IsRegular() +} + +func escapeLabel(value string) string { + value = strings.ReplaceAll(value, `\`, `\\`) + value = strings.ReplaceAll(value, "\n", `\n`) + return strings.ReplaceAll(value, `"`, `\"`) +} diff --git a/internal/pressureobserve/compliance_test.go b/internal/pressureobserve/compliance_test.go new file mode 100644 index 0000000..487753f --- /dev/null +++ b/internal/pressureobserve/compliance_test.go @@ -0,0 +1,56 @@ +package pressureobserve + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestCollectAndRenderCompliance(t *testing.T) { + root := t.TempDir() + write := func(path, value string) { + t.Helper() + full := filepath.Join(root, path) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(value), 0o644); err != nil { + t.Fatal(err) + } + } + write("proc/sys/kernel/osrelease", "6.8.0-138-generic\n") + write("sys/devices/system/cpu/vulnerabilities/spec_rstack_overflow", "Vulnerable: Safe RET, no microcode\n") + write("var/lib/update-notifier/updates-available", "0 updates can be applied immediately.\n3 additional security updates can be applied with ESM Apps.\n") + write("var/run/reboot-required", "*** System restart required ***\n") + now := time.Now().UTC() + state := CollectCompliance(root, now) + if !state.Complete || !state.RebootRequired || state.KernelRelease != "6.8.0-138-generic" || state.SRSOStatus != "vulnerable" || state.StandardUpdatesAvailable != 0 || state.ESMSecurityUpdatesAvailable != 3 { + t.Fatalf("state=%#v", state) + } + metrics := RenderCompliance(state) + for _, wanted := range []string{ + "gha_fleet_host_compliance_observer_up 1\n", + "gha_fleet_host_reboot_required 1\n", + "gha_fleet_host_standard_updates_available 0\n", + "gha_fleet_host_esm_security_updates_available 3\n", + `gha_fleet_host_kernel_info{release="6.8.0-138-generic"} 1`, + `gha_fleet_host_srso_status{status="vulnerable"} 1`, + } { + if !strings.Contains(metrics, wanted) { + t.Fatalf("metrics missing %q\n%s", wanted, metrics) + } + } +} + +func TestComplianceFailsObservableWhenInputsAreMissing(t *testing.T) { + state := CollectCompliance(t.TempDir(), time.Now().UTC()) + if state.Complete || state.PackageInventoryAgeSeconds != -1 || state.SRSOStatus != "unknown" { + t.Fatalf("state=%#v", state) + } + metrics := RenderCompliance(state) + if !strings.Contains(metrics, "gha_fleet_host_compliance_observer_up 0\n") { + t.Fatalf("missing incomplete metric\n%s", metrics) + } +} diff --git a/internal/pressureobserve/observe.go b/internal/pressureobserve/observe.go index 525117c..eae8e80 100644 --- a/internal/pressureobserve/observe.go +++ b/internal/pressureobserve/observe.go @@ -16,6 +16,7 @@ const DefaultMaxStaleness = 90 * time.Second type Handler struct { StatePath string + HostRoot string MaxStaleness time.Duration Now func() time.Time } @@ -44,7 +45,8 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { _, _ = fmt.Fprintf(w, "{\"healthy\":%t,\"fresh\":%t}\n", fresh, fresh) case "/metrics": w.Header().Set("Content-Type", "text/plain; version=0.0.4") - _, _ = w.Write([]byte(Render(state, now, h.maxStaleness(), err))) + metrics := Render(state, now, h.maxStaleness(), err) + RenderCompliance(CollectCompliance(h.HostRoot, now)) + _, _ = w.Write([]byte(metrics)) default: w.WriteHeader(http.StatusNotFound) }