feat(observability): expose host compliance state

config/observability-dashboards.yaml

@@ -95,6 +95,37 @@ dashboards:
         query: max(gha_diagnostic_storage_oldest_object_age_seconds)
         unit: seconds
         description: Age of the oldest retained diagnostic object.
+  - id: host_compliance
+    title: Fleet host compliance
+    refresh_seconds: 60
+    default_range: 24h
+    owner: fleet-operations
+    runbook: https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md
+    panels:
+      - id: compliance_coverage
+        title: Complete compliance observers
+        kind: stat
+        query: count(gha_fleet_host_compliance_observer_up == 1)
+        unit: count
+        description: Number of fleet hosts with readable kernel, vulnerability and package inventory state.
+      - id: maintenance_pending
+        title: Updates and reboot state
+        kind: timeseries
+        query: max by (host_name) (gha_fleet_host_standard_updates_available or gha_fleet_host_reboot_required)
+        unit: count
+        description: Standard package updates or a pending maintenance reboot reported by each host.
+      - id: running_kernel
+        title: Running kernel identity
+        kind: stat
+        query: max by (host_name, release) (gha_fleet_host_kernel_info)
+        unit: state
+        description: Exact running kernel release on every fleet host.
+      - id: srso_status
+        title: SRSO hardware status
+        kind: stat
+        query: max by (host_name, status) (gha_fleet_host_srso_status)
+        unit: state
+        description: Kernel-reported speculative return stack overflow status, including hardware microcode boundaries.
   - id: host_signals
     title: Classified host signals
     refresh_seconds: 60

config/observability-rules.yaml

@@ -146,6 +146,22 @@ rules:
     summary: GitHub run or repository identity has not converged within two minutes.
     action: Correlate the exact queue UUID with JobAssigned, workflow run, provider lease and runner identity without recovering a transient pre-assignment gap.
     recovery: Both age-qualified correlation metrics remain zero while raw transition counters may continue to change.
+  - id: host_compliance_observer_missing
+    severity: page
+    query_language: promql
+    stream_name: gha_fleet_host_compliance_observer_up
+    expression: count(gha_fleet_host_compliance_observer_up == 1)
+    operator: "<"
+    threshold: 5
+    evaluation_seconds: 60
+    hold_seconds: 180
+    destination_ref: fleet_oncall
+    enabled: false
+    owner: fleet-operations
+    runbook: https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md
+    summary: Kernel or package compliance telemetry is incomplete on a fleet host.
+    action: Restore the local observer or its read-only host inputs before declaring package and reboot state current.
+    recovery: Exactly five fleet hosts continuously export complete compliance state.
   - id: host_oom_detected
     severity: page
     query_language: promql
@@ -162,6 +178,54 @@ rules:
     summary: A fleet host recorded a new OOM kill.
     action: Close admission on the affected member and preserve workload and pressure evidence.
     recovery: No new OOM delta occurs and memory pressure returns inside the promotion envelope.
+  - id: host_package_inventory_stale
+    severity: ticket
+    query_language: promql
+    stream_name: gha_fleet_host_package_inventory_age_seconds
+    expression: max(gha_fleet_host_package_inventory_age_seconds)
+    operator: ">"
+    threshold: 129600
+    evaluation_seconds: 300
+    hold_seconds: 3600
+    destination_ref: fleet_oncall
+    enabled: false
+    owner: fleet-operations
+    runbook: https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md
+    summary: A fleet host package inventory has not refreshed for more than thirty-six hours.
+    action: Restore the Ubuntu metadata refresh path and preserve apt output before changing package state.
+    recovery: Every fleet host package inventory remains younger than thirty-six hours for one hour.
+  - id: host_reboot_required
+    severity: ticket
+    query_language: promql
+    stream_name: gha_fleet_host_reboot_required
+    expression: max(gha_fleet_host_reboot_required)
+    operator: ">"
+    threshold: 0
+    evaluation_seconds: 300
+    hold_seconds: 600
+    destination_ref: fleet_oncall
+    enabled: false
+    owner: fleet-operations
+    runbook: https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md
+    summary: A fleet host still requires a reboot after package maintenance.
+    action: Drain jobs naturally and use the rolling reboot procedure while preserving cluster quorum.
+    recovery: No fleet host reports a pending reboot for ten minutes.
+  - id: host_standard_updates_available
+    severity: ticket
+    query_language: promql
+    stream_name: gha_fleet_host_standard_updates_available
+    expression: max(gha_fleet_host_standard_updates_available)
+    operator: ">"
+    threshold: 0
+    evaluation_seconds: 300
+    hold_seconds: 3600
+    destination_ref: fleet_oncall
+    enabled: false
+    owner: fleet-operations
+    runbook: https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md
+    summary: Standard Ubuntu package updates remain available on a fleet host.
+    action: Apply the normal rolling package-maintenance procedure and prove service and runner recovery after each member.
+    recovery: Every fleet host reports zero standard updates for one hour.
   - id: kernel_slab_unreclaimable
     severity: ticket
     query_language: promql

internal/fleetobserve/http.go

@@ -5,6 +5,8 @@ import (
 	"net/http"
 	"sync"
 	"time"
+
+	"github.com/NDDev-OpenNetwork/github-actions/internal/pressureobserve"
 )
 
 type State struct {
@@ -30,6 +32,7 @@ type Handler struct {
 	State        *State
 	MaxStaleness time.Duration
 	Now          func() time.Time
+	HostRoot     string
 }
 
 func (h Handler) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
@@ -56,7 +59,8 @@ func (h Handler) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
 	switch request.URL.Path {
 	case "/metrics":
 		writer.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
-		_, _ = writer.Write([]byte(RenderPrometheus(snapshot, now, h.MaxStaleness)))
+		metrics := RenderPrometheus(snapshot, now, h.MaxStaleness) + pressureobserve.RenderCompliance(pressureobserve.CollectCompliance(h.HostRoot, now))
+		_, _ = writer.Write([]byte(metrics))
 	case "/snapshot":
 		writer.Header().Set("Content-Type", "application/json")
 		writeHTTPJSON(writer, http.StatusOK, snapshot)

internal/observabilitydashboards/dashboards_test.go

@@ -21,7 +21,7 @@ func TestPublishedDashboardBundleIsValidAndRenderable(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if len(bundle.Dashboards) != 8 {
+	if len(bundle.Dashboards) != 9 {
 		t.Fatalf("dashboards=%d", len(bundle.Dashboards))
 	}
 	rendered, err := Render(bundle)

internal/observabilitydashboards/openobserve_test.go

@@ -14,7 +14,7 @@ func TestRenderOpenObserveV8IsDeterministicAndManaged(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if len(dashboards) != 8 {
+	if len(dashboards) != 9 {
 		t.Fatalf("dashboards=%d", len(dashboards))
 	}
 	for _, dashboard := range dashboards {

internal/observabilityrules/rules_test.go

@@ -10,8 +10,8 @@ func TestRepositoryBundleIsValid(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if len(bundle.Rules) != 21 {
-		t.Fatalf("rules = %d, want 21", len(bundle.Rules))
+	if len(bundle.Rules) != 25 {
+		t.Fatalf("rules = %d, want 25", len(bundle.Rules))
 	}
 }
 
@@ -32,6 +32,10 @@ func TestRepositoryRulesUseCurrentMetricSemantics(t *testing.T) {
 		"kernel_slab_unreclaimable":         `state="slab_unreclaimable"`,
 		"audit_suppression_burst":           `signal_class="audit_suppressed"`,
 		"kernel_workqueue_hog":              `signal_class="kernel_workqueue_hog"`,
+		"host_compliance_observer_missing":  "gha_fleet_host_compliance_observer_up",
+		"host_package_inventory_stale":      "gha_fleet_host_package_inventory_age_seconds",
+		"host_reboot_required":              "gha_fleet_host_reboot_required",
+		"host_standard_updates_available":   "gha_fleet_host_standard_updates_available",
 	}
 	seen := make(map[string]bool, len(wanted))
 	for _, rule := range bundle.Rules {

internal/pressureobserve/compliance.go

@@ -0,0 +1,104 @@
+package pressureobserve
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+var (
+	standardUpdatesPattern = regexp.MustCompile(`(?m)([0-9]+) updates? can be applied immediately\.`)
+	esmUpdatesPattern      = regexp.MustCompile(`(?m)([0-9]+) additional security updates? can be applied with ESM Apps\.`)
+)
+
+type Compliance struct {
+	Complete                    bool
+	RebootRequired              bool
+	KernelRelease               string
+	SRSOStatus                  string
+	StandardUpdatesAvailable    int
+	ESMSecurityUpdatesAvailable int
+	PackageInventoryAgeSeconds  float64
+}
+
+func CollectCompliance(root string, now time.Time) Compliance {
+	if root == "" {
+		root = "/"
+	}
+	result := Compliance{PackageInventoryAgeSeconds: -1, SRSOStatus: "unknown"}
+	result.RebootRequired = regularFileExists(filepath.Join(root, "var", "run", "reboot-required"))
+
+	kernel, kernelErr := os.ReadFile(filepath.Join(root, "proc", "sys", "kernel", "osrelease"))
+	if kernelErr == nil {
+		result.KernelRelease = strings.TrimSpace(string(kernel))
+	}
+	srso, srsoErr := os.ReadFile(filepath.Join(root, "sys", "devices", "system", "cpu", "vulnerabilities", "spec_rstack_overflow"))
+	if srsoErr == nil {
+		result.SRSOStatus = classifySRSO(string(srso))
+	}
+
+	updatesPath := filepath.Join(root, "var", "lib", "update-notifier", "updates-available")
+	updates, updatesErr := os.ReadFile(updatesPath)
+	if updatesErr == nil {
+		result.StandardUpdatesAvailable = firstCount(standardUpdatesPattern, string(updates))
+		result.ESMSecurityUpdatesAvailable = firstCount(esmUpdatesPattern, string(updates))
+		if info, err := os.Stat(updatesPath); err == nil && !info.ModTime().After(now) {
+			result.PackageInventoryAgeSeconds = now.Sub(info.ModTime()).Seconds()
+		}
+	}
+	result.Complete = kernelErr == nil && result.KernelRelease != "" && srsoErr == nil && updatesErr == nil && result.PackageInventoryAgeSeconds >= 0
+	return result
+}
+
+func RenderCompliance(state Compliance) string {
+	var output strings.Builder
+	gauge := func(name, help string, value float64) {
+		fmt.Fprintf(&output, "# HELP %s %s\n# TYPE %s gauge\n%s %s\n", name, help, name, name, strconv.FormatFloat(value, 'f', -1, 64))
+	}
+	gauge("gha_fleet_host_compliance_observer_up", "Whether kernel and package compliance state was read completely.", boolFloat(state.Complete))
+	gauge("gha_fleet_host_reboot_required", "Whether the host requires a reboot to finish package maintenance.", boolFloat(state.RebootRequired))
+	gauge("gha_fleet_host_standard_updates_available", "Standard Ubuntu package updates currently available.", float64(state.StandardUpdatesAvailable))
+	gauge("gha_fleet_host_esm_security_updates_available", "Additional security updates available only through Ubuntu ESM Apps.", float64(state.ESMSecurityUpdatesAvailable))
+	gauge("gha_fleet_host_package_inventory_age_seconds", "Age of the update-notifier package inventory, or -1 when unavailable.", state.PackageInventoryAgeSeconds)
+	fmt.Fprintf(&output, "# HELP gha_fleet_host_kernel_info Running host kernel identity.\n# TYPE gha_fleet_host_kernel_info gauge\ngha_fleet_host_kernel_info{release=%q} 1\n", escapeLabel(state.KernelRelease))
+	fmt.Fprintf(&output, "# HELP gha_fleet_host_srso_status Speculative return stack overflow status reported by the running kernel.\n# TYPE gha_fleet_host_srso_status gauge\ngha_fleet_host_srso_status{status=%q} 1\n", escapeLabel(state.SRSOStatus))
+	return output.String()
+}
+
+func classifySRSO(value string) string {
+	normalized := strings.ToLower(strings.TrimSpace(value))
+	switch {
+	case strings.Contains(normalized, "not affected"):
+		return "not_affected"
+	case strings.Contains(normalized, "vulnerable"):
+		return "vulnerable"
+	case strings.Contains(normalized, "mitigation"):
+		return "mitigated"
+	default:
+		return "unknown"
+	}
+}
+
+func firstCount(pattern *regexp.Regexp, value string) int {
+	match := pattern.FindStringSubmatch(value)
+	if len(match) != 2 {
+		return 0
+	}
+	count, _ := strconv.Atoi(match[1])
+	return count
+}
+
+func regularFileExists(path string) bool {
+	info, err := os.Stat(path)
+	return err == nil && info.Mode().IsRegular()
+}
+
+func escapeLabel(value string) string {
+	value = strings.ReplaceAll(value, `\`, `\\`)
+	value = strings.ReplaceAll(value, "\n", `\n`)
+	return strings.ReplaceAll(value, `"`, `\"`)
+}

internal/pressureobserve/compliance_test.go

@@ -0,0 +1,56 @@
+package pressureobserve
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestCollectAndRenderCompliance(t *testing.T) {
+	root := t.TempDir()
+	write := func(path, value string) {
+		t.Helper()
+		full := filepath.Join(root, path)
+		if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(full, []byte(value), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	write("proc/sys/kernel/osrelease", "6.8.0-138-generic\n")
+	write("sys/devices/system/cpu/vulnerabilities/spec_rstack_overflow", "Vulnerable: Safe RET, no microcode\n")
+	write("var/lib/update-notifier/updates-available", "0 updates can be applied immediately.\n3 additional security updates can be applied with ESM Apps.\n")
+	write("var/run/reboot-required", "*** System restart required ***\n")
+	now := time.Now().UTC()
+	state := CollectCompliance(root, now)
+	if !state.Complete || !state.RebootRequired || state.KernelRelease != "6.8.0-138-generic" || state.SRSOStatus != "vulnerable" || state.StandardUpdatesAvailable != 0 || state.ESMSecurityUpdatesAvailable != 3 {
+		t.Fatalf("state=%#v", state)
+	}
+	metrics := RenderCompliance(state)
+	for _, wanted := range []string{
+		"gha_fleet_host_compliance_observer_up 1\n",
+		"gha_fleet_host_reboot_required 1\n",
+		"gha_fleet_host_standard_updates_available 0\n",
+		"gha_fleet_host_esm_security_updates_available 3\n",
+		`gha_fleet_host_kernel_info{release="6.8.0-138-generic"} 1`,
+		`gha_fleet_host_srso_status{status="vulnerable"} 1`,
+	} {
+		if !strings.Contains(metrics, wanted) {
+			t.Fatalf("metrics missing %q\n%s", wanted, metrics)
+		}
+	}
+}
+
+func TestComplianceFailsObservableWhenInputsAreMissing(t *testing.T) {
+	state := CollectCompliance(t.TempDir(), time.Now().UTC())
+	if state.Complete || state.PackageInventoryAgeSeconds != -1 || state.SRSOStatus != "unknown" {
+		t.Fatalf("state=%#v", state)
+	}
+	metrics := RenderCompliance(state)
+	if !strings.Contains(metrics, "gha_fleet_host_compliance_observer_up 0\n") {
+		t.Fatalf("missing incomplete metric\n%s", metrics)
+	}
+}

internal/pressureobserve/observe.go

@@ -16,6 +16,7 @@ const DefaultMaxStaleness = 90 * time.Second
 
 type Handler struct {
 	StatePath    string
+	HostRoot     string
 	MaxStaleness time.Duration
 	Now          func() time.Time
 }
@@ -44,7 +45,8 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		_, _ = fmt.Fprintf(w, "{\"healthy\":%t,\"fresh\":%t}\n", fresh, fresh)
 	case "/metrics":
 		w.Header().Set("Content-Type", "text/plain; version=0.0.4")
-		_, _ = w.Write([]byte(Render(state, now, h.maxStaleness(), err)))
+		metrics := Render(state, now, h.maxStaleness(), err) + RenderCompliance(CollectCompliance(h.HostRoot, now))
+		_, _ = w.Write([]byte(metrics))
 	default:
 		w.WriteHeader(http.StatusNotFound)
 	}