fix(alerts): encode sustained PromQL holds

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## Unreleased
 
+- Encode sustained PromQL holds with range subqueries and keep OpenObserve's
+  trigger threshold at one series; v0.92 defines that threshold as coverage,
+  not as a consecutive-evaluation counter.
 - Added provider derivative `v0.1.5-nddev.82`: bulk Incus inventory no longer
   requests recursive runtime statistics from concurrently deleting workers.
 - Use recursion-one Incus inventory for bulk admission, compatibility and GARM

internal/observabilityrules/openobserve.go

@@ -78,20 +78,27 @@ func RenderOpenObserve(bundle Bundle, destination string, enable bool) (OpenObse
 		}
 		frequencyMinutes := (rule.EvaluationSecs + 59) / 60
 		periodMinutes := max((rule.HoldSecs+59)/60, frequencyMinutes)
+		promQL, err := sustainedPromQL(rule)
+		if err != nil {
+			return OpenObserveBundle{}, err
+		}
 		result.Alerts = append(result.Alerts, OpenObserveAlert{
 			Name:       rule.ID,
 			OrgID:      bundle.Organization,
 			StreamType: "metrics",
 			StreamName: rule.StreamName,
 			QueryCondition: OpenObserveQuery{
 				Type:   "promql",
-				PromQL: rule.Expression,
+				PromQL: promQL,
 				PromQLCondition: OpenObserveValueCondition{
 					Column: "value", Operator: rule.Operator, Value: rule.Threshold,
 				},
 			},
 			TriggerCondition: OpenObserveTrigger{
-				Period: periodMinutes, Operator: ">=", Threshold: rule.RequiredEvaluations(),
+				// OpenObserve defines threshold as a PromQL series-coverage gate,
+				// not a consecutive-evaluation counter. Sustained time semantics
+				// live in the range-subquery expression above.
+				Period: periodMinutes, Operator: ">=", Threshold: 1,
 				Frequency: frequencyMinutes, FrequencyType: "minutes", Silence: silence,
 				Timezone: "UTC", AlignTime: true,
 			},
@@ -109,3 +116,32 @@ func RenderOpenObserve(bundle Bundle, destination string, enable bool) (OpenObse
 	sort.Slice(result.Alerts, func(i, j int) bool { return result.Alerts[i].Name < result.Alerts[j].Name })
 	return result, nil
 }
+
+func sustainedPromQL(rule Rule) (string, error) {
+	if rule.HoldSecs <= rule.EvaluationSecs {
+		return rule.Expression, nil
+	}
+	function := ""
+	switch rule.Operator {
+	case ">", ">=":
+		function = "min_over_time"
+	case "<", "<=":
+		function = "max_over_time"
+	default:
+		return "", fmt.Errorf("rule %q cannot express sustained operator %q", rule.ID, rule.Operator)
+	}
+	return fmt.Sprintf(
+		"%s((%s)[%s:%s])",
+		function,
+		rule.Expression,
+		promQLDuration(rule.HoldSecs),
+		promQLDuration(rule.EvaluationSecs),
+	), nil
+}
+
+func promQLDuration(seconds int) string {
+	if seconds%60 == 0 {
+		return fmt.Sprintf("%dm", seconds/60)
+	}
+	return fmt.Sprintf("%ds", seconds)
+}

internal/observabilityrules/rules_test.go

@@ -53,8 +53,15 @@ func TestDiagnosticExporterPageRequiresSustainedFailure(t *testing.T) {
 	}
 	for _, rule := range bundle.Rules {
 		if rule.ID == "diagnostic_export_failure" {
-			if rule.HoldSecs != 180 || rule.RequiredEvaluations() < 3 {
-				t.Fatalf("diagnostic failure hold = %d, evaluations = %d", rule.HoldSecs, rule.RequiredEvaluations())
+			rendered, err := RenderOpenObserve(bundle, "fleet_oncall", true)
+			if err != nil {
+				t.Fatal(err)
+			}
+			for _, alert := range rendered.Alerts {
+				if alert.Name == rule.ID && (rule.HoldSecs != 180 || alert.TriggerCondition.Threshold != 1 ||
+					!strings.HasPrefix(alert.QueryCondition.PromQL, "min_over_time(")) {
+					t.Fatalf("diagnostic rule=%#v alert=%#v", rule, alert)
+				}
 			}
 			return
 		}
@@ -99,13 +106,16 @@ func TestRenderOpenObserveSeparatesExpressionAndThreshold(t *testing.T) {
 	}
 	for index, alert := range rendered.Alerts {
 		rule := bundle.Rules[index]
-		if alert.Name != rule.ID || alert.StreamName != rule.StreamName || alert.QueryCondition.PromQL != rule.Expression ||
+		if alert.Name != rule.ID || alert.StreamName != rule.StreamName ||
 			alert.QueryCondition.PromQLCondition.Operator != rule.Operator ||
 			alert.QueryCondition.PromQLCondition.Value != rule.Threshold || !alert.Enabled {
 			t.Fatalf("alert %s does not preserve rule semantics: %#v", rule.ID, alert)
 		}
-		if alert.TriggerCondition.Threshold != rule.RequiredEvaluations() {
-			t.Fatalf("alert %s evaluations = %d, want %d", rule.ID, alert.TriggerCondition.Threshold, rule.RequiredEvaluations())
+		if alert.TriggerCondition.Threshold != 1 {
+			t.Fatalf("alert %s coverage threshold = %d, want 1", rule.ID, alert.TriggerCondition.Threshold)
+		}
+		if rule.HoldSecs > rule.EvaluationSecs && !strings.Contains(alert.QueryCondition.PromQL, "_over_time(") {
+			t.Fatalf("alert %s lacks sustained range query: %q", rule.ID, alert.QueryCondition.PromQL)
 		}
 	}
 }
@@ -118,11 +128,11 @@ func TestRenderOpenObserveConvertsSecondsToMinuteSchedule(t *testing.T) {
 		holdSecs         int
 		frequencyMinutes int
 		periodMinutes    int
-		evaluations      int
+		coverage         int
 	}{
-		{name: "thirty seconds", severity: "page", evaluationSecs: 30, holdSecs: 120, frequencyMinutes: 1, periodMinutes: 2, evaluations: 2},
-		{name: "sixty seconds", severity: "page", evaluationSecs: 60, holdSecs: 60, frequencyMinutes: 1, periodMinutes: 1, evaluations: 1},
-		{name: "five minutes", severity: "ticket", evaluationSecs: 300, holdSecs: 900, frequencyMinutes: 5, periodMinutes: 15, evaluations: 3},
+		{name: "thirty seconds", severity: "page", evaluationSecs: 30, holdSecs: 120, frequencyMinutes: 1, periodMinutes: 2, coverage: 1},
+		{name: "sixty seconds", severity: "page", evaluationSecs: 60, holdSecs: 60, frequencyMinutes: 1, periodMinutes: 1, coverage: 1},
+		{name: "five minutes", severity: "ticket", evaluationSecs: 300, holdSecs: 900, frequencyMinutes: 5, periodMinutes: 15, coverage: 1},
 	} {
 		t.Run(test.name, func(t *testing.T) {
 			bundle := Bundle{SchemaVersion: SchemaVersion, Backend: "openobserve", Organization: "default", Rules: []Rule{{
@@ -138,7 +148,7 @@ func TestRenderOpenObserveConvertsSecondsToMinuteSchedule(t *testing.T) {
 			}
 			trigger := rendered.Alerts[0].TriggerCondition
 			if trigger.FrequencyType != "minutes" || trigger.Frequency != test.frequencyMinutes ||
-				trigger.Period != test.periodMinutes || trigger.Threshold != test.evaluations {
+				trigger.Period != test.periodMinutes || trigger.Threshold != test.coverage {
 				t.Fatalf("trigger = %#v", trigger)
 			}
 		})