From 4b322db3d5c2dd1171180b5ab38c44851cb5e6fc Mon Sep 17 00:00:00 2001 From: rldyourmnd Date: Thu, 27 Aug 2026 19:45:04 +0500 Subject: [PATCH] fix(alerts): encode sustained PromQL holds --- CHANGELOG.md | 3 ++ internal/observabilityrules/openobserve.go | 40 ++++++++++++++++++++-- internal/observabilityrules/rules_test.go | 30 ++++++++++------ 3 files changed, 61 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3194fdb..22bc58a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased +- Encode sustained PromQL holds with range subqueries and keep OpenObserve's + trigger threshold at one series; v0.92 defines that threshold as coverage, + not as a consecutive-evaluation counter. - Added provider derivative `v0.1.5-nddev.82`: bulk Incus inventory no longer requests recursive runtime statistics from concurrently deleting workers. - Use recursion-one Incus inventory for bulk admission, compatibility and GARM diff --git a/internal/observabilityrules/openobserve.go b/internal/observabilityrules/openobserve.go index 02aaf45..1a27763 100644 --- a/internal/observabilityrules/openobserve.go +++ b/internal/observabilityrules/openobserve.go @@ -78,6 +78,10 @@ func RenderOpenObserve(bundle Bundle, destination string, enable bool) (OpenObse } frequencyMinutes := (rule.EvaluationSecs + 59) / 60 periodMinutes := max((rule.HoldSecs+59)/60, frequencyMinutes) + promQL, err := sustainedPromQL(rule) + if err != nil { + return OpenObserveBundle{}, err + } result.Alerts = append(result.Alerts, OpenObserveAlert{ Name: rule.ID, OrgID: bundle.Organization, @@ -85,13 +89,16 @@ func RenderOpenObserve(bundle Bundle, destination string, enable bool) (OpenObse StreamName: rule.StreamName, QueryCondition: OpenObserveQuery{ Type: "promql", - PromQL: rule.Expression, + PromQL: promQL, PromQLCondition: OpenObserveValueCondition{ Column: "value", Operator: rule.Operator, Value: rule.Threshold, }, }, TriggerCondition: OpenObserveTrigger{ - Period: periodMinutes, Operator: ">=", Threshold: rule.RequiredEvaluations(), + // OpenObserve defines threshold as a PromQL series-coverage gate, + // not a consecutive-evaluation counter. Sustained time semantics + // live in the range-subquery expression above. + Period: periodMinutes, Operator: ">=", Threshold: 1, Frequency: frequencyMinutes, FrequencyType: "minutes", Silence: silence, Timezone: "UTC", AlignTime: true, }, @@ -109,3 +116,32 @@ func RenderOpenObserve(bundle Bundle, destination string, enable bool) (OpenObse sort.Slice(result.Alerts, func(i, j int) bool { return result.Alerts[i].Name < result.Alerts[j].Name }) return result, nil } + +func sustainedPromQL(rule Rule) (string, error) { + if rule.HoldSecs <= rule.EvaluationSecs { + return rule.Expression, nil + } + function := "" + switch rule.Operator { + case ">", ">=": + function = "min_over_time" + case "<", "<=": + function = "max_over_time" + default: + return "", fmt.Errorf("rule %q cannot express sustained operator %q", rule.ID, rule.Operator) + } + return fmt.Sprintf( + "%s((%s)[%s:%s])", + function, + rule.Expression, + promQLDuration(rule.HoldSecs), + promQLDuration(rule.EvaluationSecs), + ), nil +} + +func promQLDuration(seconds int) string { + if seconds%60 == 0 { + return fmt.Sprintf("%dm", seconds/60) + } + return fmt.Sprintf("%ds", seconds) +} diff --git a/internal/observabilityrules/rules_test.go b/internal/observabilityrules/rules_test.go index bf5bee4..80de00f 100644 --- a/internal/observabilityrules/rules_test.go +++ b/internal/observabilityrules/rules_test.go @@ -53,8 +53,15 @@ func TestDiagnosticExporterPageRequiresSustainedFailure(t *testing.T) { } for _, rule := range bundle.Rules { if rule.ID == "diagnostic_export_failure" { - if rule.HoldSecs != 180 || rule.RequiredEvaluations() < 3 { - t.Fatalf("diagnostic failure hold = %d, evaluations = %d", rule.HoldSecs, rule.RequiredEvaluations()) + rendered, err := RenderOpenObserve(bundle, "fleet_oncall", true) + if err != nil { + t.Fatal(err) + } + for _, alert := range rendered.Alerts { + if alert.Name == rule.ID && (rule.HoldSecs != 180 || alert.TriggerCondition.Threshold != 1 || + !strings.HasPrefix(alert.QueryCondition.PromQL, "min_over_time(")) { + t.Fatalf("diagnostic rule=%#v alert=%#v", rule, alert) + } } return } @@ -99,13 +106,16 @@ func TestRenderOpenObserveSeparatesExpressionAndThreshold(t *testing.T) { } for index, alert := range rendered.Alerts { rule := bundle.Rules[index] - if alert.Name != rule.ID || alert.StreamName != rule.StreamName || alert.QueryCondition.PromQL != rule.Expression || + if alert.Name != rule.ID || alert.StreamName != rule.StreamName || alert.QueryCondition.PromQLCondition.Operator != rule.Operator || alert.QueryCondition.PromQLCondition.Value != rule.Threshold || !alert.Enabled { t.Fatalf("alert %s does not preserve rule semantics: %#v", rule.ID, alert) } - if alert.TriggerCondition.Threshold != rule.RequiredEvaluations() { - t.Fatalf("alert %s evaluations = %d, want %d", rule.ID, alert.TriggerCondition.Threshold, rule.RequiredEvaluations()) + if alert.TriggerCondition.Threshold != 1 { + t.Fatalf("alert %s coverage threshold = %d, want 1", rule.ID, alert.TriggerCondition.Threshold) + } + if rule.HoldSecs > rule.EvaluationSecs && !strings.Contains(alert.QueryCondition.PromQL, "_over_time(") { + t.Fatalf("alert %s lacks sustained range query: %q", rule.ID, alert.QueryCondition.PromQL) } } } @@ -118,11 +128,11 @@ func TestRenderOpenObserveConvertsSecondsToMinuteSchedule(t *testing.T) { holdSecs int frequencyMinutes int periodMinutes int - evaluations int + coverage int }{ - {name: "thirty seconds", severity: "page", evaluationSecs: 30, holdSecs: 120, frequencyMinutes: 1, periodMinutes: 2, evaluations: 2}, - {name: "sixty seconds", severity: "page", evaluationSecs: 60, holdSecs: 60, frequencyMinutes: 1, periodMinutes: 1, evaluations: 1}, - {name: "five minutes", severity: "ticket", evaluationSecs: 300, holdSecs: 900, frequencyMinutes: 5, periodMinutes: 15, evaluations: 3}, + {name: "thirty seconds", severity: "page", evaluationSecs: 30, holdSecs: 120, frequencyMinutes: 1, periodMinutes: 2, coverage: 1}, + {name: "sixty seconds", severity: "page", evaluationSecs: 60, holdSecs: 60, frequencyMinutes: 1, periodMinutes: 1, coverage: 1}, + {name: "five minutes", severity: "ticket", evaluationSecs: 300, holdSecs: 900, frequencyMinutes: 5, periodMinutes: 15, coverage: 1}, } { t.Run(test.name, func(t *testing.T) { bundle := Bundle{SchemaVersion: SchemaVersion, Backend: "openobserve", Organization: "default", Rules: []Rule{{ @@ -138,7 +148,7 @@ func TestRenderOpenObserveConvertsSecondsToMinuteSchedule(t *testing.T) { } trigger := rendered.Alerts[0].TriggerCondition if trigger.FrequencyType != "minutes" || trigger.Frequency != test.frequencyMinutes || - trigger.Period != test.periodMinutes || trigger.Threshold != test.evaluations { + trigger.Period != test.periodMinutes || trigger.Threshold != test.coverage { t.Fatalf("trigger = %#v", trigger) } })