ROX-34459: add alert for container restart over 48h period

resources/prometheus/prometheus-rules.yaml

@@ -1059,3 +1059,23 @@ spec:
             summary: ACS CS data plane cluster lost over 50% of secured clusters over all tenants
             description: ACS CS data plane cluster lost over 50% of secured clusters over all tenants
             sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-cloud-service/runbooks/-/blob/master/sops/dp-060-secured-cluster-drop-alert.md"
+    - name: rhacs-pod-restarts
+      rules:
+        - alert: RHACSHighRestartRateOverExtendedPeriod
+          expr: |
+            rate(kube_pod_container_status_restarts_total{namespace=~"rhacs-.*"}[48h]) * 86400 >= 3
+          labels:
+            severity: warning
+          annotations:
+            summary: "Container `{{ $labels.pod }}/{{ $labels.container }}` in namespace `{{ $labels.namespace }}` is restarting frequently."
+            description: "Container `{{ $labels.pod }}/{{ $labels.container }}` in namespace `{{ $labels.namespace }}` has restarted at a rate of {{ $value | humanize }} times per day averaged over the last 48 hours."
+            sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-managed-service-runbooks/blob/master/sops/dp-003-rhacs-instance-unavailable.md"
+        - alert: RHACSCriticalRestartRateOverExtendedPeriod
+          expr: |
+            rate(kube_pod_container_status_restarts_total{namespace=~"rhacs-.*"}[48h]) * 86400 > 5
+          labels:
+            severity: critical
+          annotations:
+            summary: "Container `{{ $labels.pod }}/{{ $labels.container }}` in namespace `{{ $labels.namespace }}` is restarting very frequently."
+            description: "Container `{{ $labels.pod }}/{{ $labels.container }}` in namespace `{{ $labels.namespace }}` has restarted at a rate of {{ $value | humanize }} times per day averaged over the last 48 hours."
+            sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-managed-service-runbooks/blob/master/sops/dp-003-rhacs-instance-unavailable.md"

resources/prometheus/unit_tests/RHACSHighRestartRateOverExtendedPeriod.yaml

@@ -0,0 +1,74 @@
+rule_files:
+  - /tmp/prometheus-rules-test.yaml
+
+evaluation_interval: 1m
+
+tests:
+  # ~2 restarts/day: neither alert should fire.
+  - interval: 1h
+    input_series:
+      - series: kube_pod_container_status_restarts_total{namespace="rhacs-1234", pod="central-abc-123", container="central"}
+        values: "0+0.083333x60"
+    alert_rule_test:
+      - eval_time: 50h
+        alertname: RHACSHighRestartRateOverExtendedPeriod
+        exp_alerts: []
+      - eval_time: 50h
+        alertname: RHACSCriticalRestartRateOverExtendedPeriod
+        exp_alerts: []
+
+  # ~4 restarts/day: warning fires, critical does not.
+  - interval: 1h
+    input_series:
+      - series: kube_pod_container_status_restarts_total{namespace="rhacs-1234", pod="central-abc-123", container="central"}
+        values: "0+0.166667x60"
+    alert_rule_test:
+      - eval_time: 50h
+        alertname: RHACSHighRestartRateOverExtendedPeriod
+        exp_alerts:
+          - exp_labels:
+              alertname: RHACSHighRestartRateOverExtendedPeriod
+              container: central
+              namespace: rhacs-1234
+              pod: central-abc-123
+              severity: warning
+            exp_annotations:
+              summary: "Container `central-abc-123/central` in namespace `rhacs-1234` is restarting frequently."
+              description: "Container `central-abc-123/central` in namespace `rhacs-1234` has restarted at a rate of 4 times per day averaged over the last 48 hours."
+              sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-managed-service-runbooks/blob/master/sops/dp-003-rhacs-instance-unavailable.md"
+      - eval_time: 50h
+        alertname: RHACSCriticalRestartRateOverExtendedPeriod
+        exp_alerts: []
+
+  # ~6 restarts/day: both alerts fire.
+  - interval: 1h
+    input_series:
+      - series: kube_pod_container_status_restarts_total{namespace="rhacs-1234", pod="central-abc-123", container="central"}
+        values: "0+0.25x60"
+    alert_rule_test:
+      - eval_time: 50h
+        alertname: RHACSHighRestartRateOverExtendedPeriod
+        exp_alerts:
+          - exp_labels:
+              alertname: RHACSHighRestartRateOverExtendedPeriod
+              container: central
+              namespace: rhacs-1234
+              pod: central-abc-123
+              severity: warning
+            exp_annotations:
+              summary: "Container `central-abc-123/central` in namespace `rhacs-1234` is restarting frequently."
+              description: "Container `central-abc-123/central` in namespace `rhacs-1234` has restarted at a rate of 6 times per day averaged over the last 48 hours."
+              sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-managed-service-runbooks/blob/master/sops/dp-003-rhacs-instance-unavailable.md"
+      - eval_time: 50h
+        alertname: RHACSCriticalRestartRateOverExtendedPeriod
+        exp_alerts:
+          - exp_labels:
+              alertname: RHACSCriticalRestartRateOverExtendedPeriod
+              container: central
+              namespace: rhacs-1234
+              pod: central-abc-123
+              severity: critical
+            exp_annotations:
+              summary: "Container `central-abc-123/central` in namespace `rhacs-1234` is restarting very frequently."
+              description: "Container `central-abc-123/central` in namespace `rhacs-1234` has restarted at a rate of 6 times per day averaged over the last 48 hours."
+              sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-managed-service-runbooks/blob/master/sops/dp-003-rhacs-instance-unavailable.md"