diff --git a/resources/prometheus/prometheus-rules.yaml b/resources/prometheus/prometheus-rules.yaml index 6576b45..c6f862e 100644 --- a/resources/prometheus/prometheus-rules.yaml +++ b/resources/prometheus/prometheus-rules.yaml @@ -1059,3 +1059,23 @@ spec: summary: ACS CS data plane cluster lost over 50% of secured clusters over all tenants description: ACS CS data plane cluster lost over 50% of secured clusters over all tenants sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-cloud-service/runbooks/-/blob/master/sops/dp-060-secured-cluster-drop-alert.md" + - name: rhacs-pod-restarts + rules: + - alert: RHACSHighRestartRateOverExtendedPeriod + expr: | + rate(kube_pod_container_status_restarts_total{namespace=~"rhacs-.*"}[48h]) * 86400 >= 3 + labels: + severity: warning + annotations: + summary: "Container `{{ $labels.pod }}/{{ $labels.container }}` in namespace `{{ $labels.namespace }}` is restarting frequently." + description: "Container `{{ $labels.pod }}/{{ $labels.container }}` in namespace `{{ $labels.namespace }}` has restarted at a rate of {{ $value | humanize }} times per day averaged over the last 48 hours." + sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-managed-service-runbooks/blob/master/sops/dp-003-rhacs-instance-unavailable.md" + - alert: RHACSCriticalRestartRateOverExtendedPeriod + expr: | + rate(kube_pod_container_status_restarts_total{namespace=~"rhacs-.*"}[48h]) * 86400 > 5 + labels: + severity: critical + annotations: + summary: "Container `{{ $labels.pod }}/{{ $labels.container }}` in namespace `{{ $labels.namespace }}` is restarting very frequently." + description: "Container `{{ $labels.pod }}/{{ $labels.container }}` in namespace `{{ $labels.namespace }}` has restarted at a rate of {{ $value | humanize }} times per day averaged over the last 48 hours." + sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-managed-service-runbooks/blob/master/sops/dp-003-rhacs-instance-unavailable.md" diff --git a/resources/prometheus/unit_tests/RHACSHighRestartRateOverExtendedPeriod.yaml b/resources/prometheus/unit_tests/RHACSHighRestartRateOverExtendedPeriod.yaml new file mode 100644 index 0000000..ccd0d5e --- /dev/null +++ b/resources/prometheus/unit_tests/RHACSHighRestartRateOverExtendedPeriod.yaml @@ -0,0 +1,74 @@ +rule_files: + - /tmp/prometheus-rules-test.yaml + +evaluation_interval: 1m + +tests: + # ~2 restarts/day: neither alert should fire. + - interval: 1h + input_series: + - series: kube_pod_container_status_restarts_total{namespace="rhacs-1234", pod="central-abc-123", container="central"} + values: "0+0.083333x60" + alert_rule_test: + - eval_time: 50h + alertname: RHACSHighRestartRateOverExtendedPeriod + exp_alerts: [] + - eval_time: 50h + alertname: RHACSCriticalRestartRateOverExtendedPeriod + exp_alerts: [] + + # ~4 restarts/day: warning fires, critical does not. + - interval: 1h + input_series: + - series: kube_pod_container_status_restarts_total{namespace="rhacs-1234", pod="central-abc-123", container="central"} + values: "0+0.166667x60" + alert_rule_test: + - eval_time: 50h + alertname: RHACSHighRestartRateOverExtendedPeriod + exp_alerts: + - exp_labels: + alertname: RHACSHighRestartRateOverExtendedPeriod + container: central + namespace: rhacs-1234 + pod: central-abc-123 + severity: warning + exp_annotations: + summary: "Container `central-abc-123/central` in namespace `rhacs-1234` is restarting frequently." + description: "Container `central-abc-123/central` in namespace `rhacs-1234` has restarted at a rate of 4 times per day averaged over the last 48 hours." + sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-managed-service-runbooks/blob/master/sops/dp-003-rhacs-instance-unavailable.md" + - eval_time: 50h + alertname: RHACSCriticalRestartRateOverExtendedPeriod + exp_alerts: [] + + # ~6 restarts/day: both alerts fire. + - interval: 1h + input_series: + - series: kube_pod_container_status_restarts_total{namespace="rhacs-1234", pod="central-abc-123", container="central"} + values: "0+0.25x60" + alert_rule_test: + - eval_time: 50h + alertname: RHACSHighRestartRateOverExtendedPeriod + exp_alerts: + - exp_labels: + alertname: RHACSHighRestartRateOverExtendedPeriod + container: central + namespace: rhacs-1234 + pod: central-abc-123 + severity: warning + exp_annotations: + summary: "Container `central-abc-123/central` in namespace `rhacs-1234` is restarting frequently." + description: "Container `central-abc-123/central` in namespace `rhacs-1234` has restarted at a rate of 6 times per day averaged over the last 48 hours." + sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-managed-service-runbooks/blob/master/sops/dp-003-rhacs-instance-unavailable.md" + - eval_time: 50h + alertname: RHACSCriticalRestartRateOverExtendedPeriod + exp_alerts: + - exp_labels: + alertname: RHACSCriticalRestartRateOverExtendedPeriod + container: central + namespace: rhacs-1234 + pod: central-abc-123 + severity: critical + exp_annotations: + summary: "Container `central-abc-123/central` in namespace `rhacs-1234` is restarting very frequently." + description: "Container `central-abc-123/central` in namespace `rhacs-1234` has restarted at a rate of 6 times per day averaged over the last 48 hours." + sop_url: "https://gitlab.cee.redhat.com/stackrox/acs-managed-service-runbooks/blob/master/sops/dp-003-rhacs-instance-unavailable.md"