From 7bd6d605bb29637c871ba536cc468238dd2e420a Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Wed, 20 May 2026 14:30:15 -0600 Subject: [PATCH 01/22] initial commit --- packages/hdx-eval/.gitignore | 11 + packages/hdx-eval/MCP_IMPROVEMENTS.md | 138 ++ packages/hdx-eval/README.md | 60 + packages/hdx-eval/ablation/REPORT.md | 59 + packages/hdx-eval/ablation/manifest.tsv | 2 + packages/hdx-eval/eslint.config.mjs | 60 + packages/hdx-eval/eval.config.example.json | 29 + packages/hdx-eval/jest.config.js | 15 + packages/hdx-eval/package.json | 42 + packages/hdx-eval/scripts/ablation-report.ts | 266 ++++ packages/hdx-eval/scripts/ablation.sh | 179 +++ .../scripts/compare-prompt-variants.sh | 125 ++ packages/hdx-eval/scripts/fast-eval.sh | 125 ++ .../hdx-eval/scripts/viewer/public/app.js | 685 ++++++++++ .../hdx-eval/scripts/viewer/public/index.html | 65 + .../hdx-eval/scripts/viewer/public/style.css | 461 +++++++ packages/hdx-eval/scripts/viewer/server.js | 183 +++ .../hdx-eval/src/__tests__/aggregate.test.ts | 179 +++ packages/hdx-eval/src/__tests__/blind.test.ts | 50 + .../src/__tests__/error-root-cause.test.ts | 228 ++++ packages/hdx-eval/src/__tests__/grade.test.ts | 169 +++ .../src/__tests__/judgePrompt.test.ts | 62 + .../src/__tests__/latency-spike.test.ts | 200 +++ .../hdx-eval/src/__tests__/markdown.test.ts | 113 ++ .../hdx-eval/src/__tests__/mcpConfig.test.ts | 99 ++ .../src/__tests__/noisy-signals.test.ts | 273 ++++ .../src/__tests__/programmatic.test.ts | 140 ++ .../hdx-eval/src/__tests__/runRun.test.ts | 138 ++ .../hdx-eval/src/__tests__/seeded.test.ts | 66 + .../__tests__/segmented-regression.test.ts | 157 +++ .../__tests__/service-health-check.test.ts | 208 +++ .../src/__tests__/streamParser.test.ts | 127 ++ .../src/__tests__/systemPrompt.test.ts | 36 + packages/hdx-eval/src/cli.ts | 670 ++++++++++ packages/hdx-eval/src/clickhouse/client.ts | 35 + packages/hdx-eval/src/clickhouse/insert.ts | 106 ++ packages/hdx-eval/src/clickhouse/schema.ts | 116 ++ packages/hdx-eval/src/generators/logs.ts | 30 + packages/hdx-eval/src/generators/templates.ts | 1191 +++++++++++++++++ packages/hdx-eval/src/generators/traces.ts | 50 + packages/hdx-eval/src/generators/types.ts | 39 + packages/hdx-eval/src/grading/blind.ts | 24 + packages/hdx-eval/src/grading/grade.ts | 261 ++++ packages/hdx-eval/src/grading/judge.ts | 259 ++++ packages/hdx-eval/src/grading/judgePrompt.ts | 77 ++ packages/hdx-eval/src/grading/programmatic.ts | 42 + packages/hdx-eval/src/grading/rubric.ts | 77 ++ packages/hdx-eval/src/grading/types.ts | 100 ++ packages/hdx-eval/src/harness/claudeSpawn.ts | 146 ++ packages/hdx-eval/src/harness/mcpConfig.ts | 59 + packages/hdx-eval/src/harness/runRun.ts | 233 ++++ packages/hdx-eval/src/harness/settingsFile.ts | 51 + packages/hdx-eval/src/harness/streamParser.ts | 156 +++ packages/hdx-eval/src/harness/systemPrompt.ts | 139 ++ packages/hdx-eval/src/harness/types.ts | 51 + packages/hdx-eval/src/hyperdx/api.ts | 197 +++ packages/hdx-eval/src/hyperdx/config.ts | 103 ++ packages/hdx-eval/src/hyperdx/setup.ts | 297 ++++ packages/hdx-eval/src/index.ts | 10 + packages/hdx-eval/src/reports/aggregate.ts | 201 +++ packages/hdx-eval/src/reports/markdown.ts | 244 ++++ packages/hdx-eval/src/reports/store.ts | 59 + packages/hdx-eval/src/rng/seeded.ts | 57 + .../scenarios/error-root-cause/generate.ts | 725 ++++++++++ .../error-root-cause/ground-truth.json | 202 +++ packages/hdx-eval/src/scenarios/index.ts | 26 + .../src/scenarios/latency-spike/generate.ts | 559 ++++++++ .../scenarios/latency-spike/ground-truth.json | 162 +++ .../src/scenarios/noisy-signals/generate.ts | 719 ++++++++++ .../scenarios/noisy-signals/ground-truth.json | 276 ++++ .../hdx-eval/src/scenarios/seedScenario.ts | 48 + .../segmented-regression/generate.ts | 297 ++++ .../segmented-regression/ground-truth.json | 138 ++ .../service-health-check/generate.ts | 428 ++++++ .../service-health-check/ground-truth.json | 183 +++ packages/hdx-eval/src/scenarios/types.ts | 49 + packages/hdx-eval/tsconfig.build.json | 4 + packages/hdx-eval/tsconfig.json | 23 + yarn.lock | 62 + 79 files changed, 13731 insertions(+) create mode 100644 packages/hdx-eval/.gitignore create mode 100644 packages/hdx-eval/MCP_IMPROVEMENTS.md create mode 100644 packages/hdx-eval/README.md create mode 100644 packages/hdx-eval/ablation/REPORT.md create mode 100644 packages/hdx-eval/ablation/manifest.tsv create mode 100644 packages/hdx-eval/eslint.config.mjs create mode 100644 packages/hdx-eval/eval.config.example.json create mode 100644 packages/hdx-eval/jest.config.js create mode 100644 packages/hdx-eval/package.json create mode 100644 packages/hdx-eval/scripts/ablation-report.ts create mode 100755 packages/hdx-eval/scripts/ablation.sh create mode 100755 packages/hdx-eval/scripts/compare-prompt-variants.sh create mode 100755 packages/hdx-eval/scripts/fast-eval.sh create mode 100644 packages/hdx-eval/scripts/viewer/public/app.js create mode 100644 packages/hdx-eval/scripts/viewer/public/index.html create mode 100644 packages/hdx-eval/scripts/viewer/public/style.css create mode 100644 packages/hdx-eval/scripts/viewer/server.js create mode 100644 packages/hdx-eval/src/__tests__/aggregate.test.ts create mode 100644 packages/hdx-eval/src/__tests__/blind.test.ts create mode 100644 packages/hdx-eval/src/__tests__/error-root-cause.test.ts create mode 100644 packages/hdx-eval/src/__tests__/grade.test.ts create mode 100644 packages/hdx-eval/src/__tests__/judgePrompt.test.ts create mode 100644 packages/hdx-eval/src/__tests__/latency-spike.test.ts create mode 100644 packages/hdx-eval/src/__tests__/markdown.test.ts create mode 100644 packages/hdx-eval/src/__tests__/mcpConfig.test.ts create mode 100644 packages/hdx-eval/src/__tests__/noisy-signals.test.ts create mode 100644 packages/hdx-eval/src/__tests__/programmatic.test.ts create mode 100644 packages/hdx-eval/src/__tests__/runRun.test.ts create mode 100644 packages/hdx-eval/src/__tests__/seeded.test.ts create mode 100644 packages/hdx-eval/src/__tests__/segmented-regression.test.ts create mode 100644 packages/hdx-eval/src/__tests__/service-health-check.test.ts create mode 100644 packages/hdx-eval/src/__tests__/streamParser.test.ts create mode 100644 packages/hdx-eval/src/__tests__/systemPrompt.test.ts create mode 100644 packages/hdx-eval/src/cli.ts create mode 100644 packages/hdx-eval/src/clickhouse/client.ts create mode 100644 packages/hdx-eval/src/clickhouse/insert.ts create mode 100644 packages/hdx-eval/src/clickhouse/schema.ts create mode 100644 packages/hdx-eval/src/generators/logs.ts create mode 100644 packages/hdx-eval/src/generators/templates.ts create mode 100644 packages/hdx-eval/src/generators/traces.ts create mode 100644 packages/hdx-eval/src/generators/types.ts create mode 100644 packages/hdx-eval/src/grading/blind.ts create mode 100644 packages/hdx-eval/src/grading/grade.ts create mode 100644 packages/hdx-eval/src/grading/judge.ts create mode 100644 packages/hdx-eval/src/grading/judgePrompt.ts create mode 100644 packages/hdx-eval/src/grading/programmatic.ts create mode 100644 packages/hdx-eval/src/grading/rubric.ts create mode 100644 packages/hdx-eval/src/grading/types.ts create mode 100644 packages/hdx-eval/src/harness/claudeSpawn.ts create mode 100644 packages/hdx-eval/src/harness/mcpConfig.ts create mode 100644 packages/hdx-eval/src/harness/runRun.ts create mode 100644 packages/hdx-eval/src/harness/settingsFile.ts create mode 100644 packages/hdx-eval/src/harness/streamParser.ts create mode 100644 packages/hdx-eval/src/harness/systemPrompt.ts create mode 100644 packages/hdx-eval/src/harness/types.ts create mode 100644 packages/hdx-eval/src/hyperdx/api.ts create mode 100644 packages/hdx-eval/src/hyperdx/config.ts create mode 100644 packages/hdx-eval/src/hyperdx/setup.ts create mode 100644 packages/hdx-eval/src/index.ts create mode 100644 packages/hdx-eval/src/reports/aggregate.ts create mode 100644 packages/hdx-eval/src/reports/markdown.ts create mode 100644 packages/hdx-eval/src/reports/store.ts create mode 100644 packages/hdx-eval/src/rng/seeded.ts create mode 100644 packages/hdx-eval/src/scenarios/error-root-cause/generate.ts create mode 100644 packages/hdx-eval/src/scenarios/error-root-cause/ground-truth.json create mode 100644 packages/hdx-eval/src/scenarios/index.ts create mode 100644 packages/hdx-eval/src/scenarios/latency-spike/generate.ts create mode 100644 packages/hdx-eval/src/scenarios/latency-spike/ground-truth.json create mode 100644 packages/hdx-eval/src/scenarios/noisy-signals/generate.ts create mode 100644 packages/hdx-eval/src/scenarios/noisy-signals/ground-truth.json create mode 100644 packages/hdx-eval/src/scenarios/seedScenario.ts create mode 100644 packages/hdx-eval/src/scenarios/segmented-regression/generate.ts create mode 100644 packages/hdx-eval/src/scenarios/segmented-regression/ground-truth.json create mode 100644 packages/hdx-eval/src/scenarios/service-health-check/generate.ts create mode 100644 packages/hdx-eval/src/scenarios/service-health-check/ground-truth.json create mode 100644 packages/hdx-eval/src/scenarios/types.ts create mode 100644 packages/hdx-eval/tsconfig.build.json create mode 100644 packages/hdx-eval/tsconfig.json diff --git a/packages/hdx-eval/.gitignore b/packages/hdx-eval/.gitignore new file mode 100644 index 0000000000..9a31e16621 --- /dev/null +++ b/packages/hdx-eval/.gitignore @@ -0,0 +1,11 @@ +runs/ +eval.config.json + +# Ablation scratch / run artifacts +ablation/_*.pgid +ablation/_*.log +ablation/autonomous-*.md +ablation/autonomous-loop*-*.md +ablation/MAIN_VS_*.md +ablation/PROMPT_VARIANT*.md +ablation/prompt-variant-*/ diff --git a/packages/hdx-eval/MCP_IMPROVEMENTS.md b/packages/hdx-eval/MCP_IMPROVEMENTS.md new file mode 100644 index 0000000000..527a606153 --- /dev/null +++ b/packages/hdx-eval/MCP_IMPROVEMENTS.md @@ -0,0 +1,138 @@ +# HyperDX MCP Server — Next Improvements + +Based on the n=1 rerun trajectories (`runs/2026-05-10T04-*`) after adding +`hyperdx_trace` and `hyperdx_log_patterns`, plus the schema audit (per-select +conditional `where`, derived-expression `groupBy`). + +## High leverage + +### 1. `hyperdx_segment_compare` — the missing primitive + +**Where it shows up:** the 25-call ClickHouse `max_turns` failure on +latency-spike. About 10 of those 25 calls were variations of "is the slowness +concentrated on `tenant.tier`? on `feature_flag`? on `http.route`? on +`backend`? on `service.version`?" — manual iteration through candidate segment +columns. HDX wins this scenario partly because it finds the segment by +accident, partly because it bails to `hyperdx_trace`. Neither MCP makes the +actual question first-class. + +**Shape:** + +```ts +hyperdx_segment_compare({ + source, metric, spanFilter, timeWindow, + segmentColumns?: string[] // omit → use mapAttributeLowCardinalityValues +}) +``` + +Returns each candidate segment ranked by metric spread (max/min ratio across +values) with a top-2 callout. `hyperdx_list_sources` already exposes +`mapAttributeLowCardinalityValues` — this tool just leverages that to +pre-compute the obvious comparisons. + +### 2. `hyperdx_anomaly_window` — when did the change happen + +ClickHouse burned 4 consecutive calls (~#7–#10 of the latency-spike run) doing +minute-bucketed p99 queries to find the inflection. HDX did the same in +calls #5 + #7. Both arms reach for this and both grind through it. + +**Shape:** + +```ts +hyperdx_anomaly_window({ source, metric, spanFilter, timeWindow, granularity?: 'auto' }) +``` + +Returns the bucketed timeseries plus a detected onset timestamp and magnitude. +Replaces the "GROUP BY toStartOfMinute, eyeball the result" pattern. + +### 3. `hyperdx_log_patterns` should support `compareWindow` + +**Where it shows up:** noisy-signals is solved with one call now, but for +incident response the natural follow-up is "which patterns *spiked*?" The +agent currently has to call `hyperdx_log_patterns` twice and diff manually. + +**Shape addition:** + +```ts +hyperdx_log_patterns({ + ..., + compareWindow?: { startTime, endTime } +}) +``` + +Returns each cluster with `count_now`, `count_baseline`, `delta_pct`. Cheap to +implement (run the sample twice). + +## Medium leverage + +### 4. `hyperdx_trace` should auto-include correlated logs + +The HDX agent's call #11 walked a trace and got 16 spans. To answer "what error +message" it would normally need a second log query joining on `TraceId`. Trace +sources have a configured `logSourceId` already (`source.logSourceId` in the +schema). The trace tool can pull correlated log lines and inline them at the +right span depth. + +### 5. Single combined "first-look" tool + +```ts +hyperdx_overview({ + source, focus: 'service' | 'span' | 'pattern', value, timeWindow +}) +``` + +Returns: + +- Timeseries of the focus's primary metric +- Top 5 error templates +- Top 3 differentiating segments +- Top 5 slowest `TraceId`s + +**One call** that replaces the agent's ~5-call exploration phase. This is the +biggest tool-call savings — both agents spend 4–6 calls warming up. + +### 6. Default a `limit` on builder `table` queries + +Looking at HDX's `groupBy: "SpanAttributes['error.type'], +SpanAttributes['decline.reason'], ServiceName"` — that returns every +combination, which can be hundreds of rows. None of the runs hit this in a bad +way, but it's a footgun. Default to `limit: 50` with `orderBy` first column +desc unless explicitly set, and surface "result truncated" in the response. + +## Description / discoverability + +### 7. ToolSearch doesn't pre-load `hyperdx_trace` / `hyperdx_log_patterns` + +In the error-root-cause run, the HDX agent did +`ToolSearch select:list_sources,query` first (call #1), then **a second** +`ToolSearch select:hyperdx_trace` later (call #10). That's a wasted turn. +**The fix isn't on the MCP server** — it's on the harness side or in the +system prompt. But: bundling the description so an agent loading +`hyperdx_query` *also* sees a pointer to `hyperdx_trace` / `hyperdx_log_patterns` +mentioned in the *first* response would help. Right now the cross-references +are in the description only. + +### 8. `pickBy` description for `hyperdx_trace` slightly off + +Doc says "earliest trace whose root span has `STATUS_CODE_ERROR`" but the +implementation actually filters by *any* span with that status (since the +picker groups by `TraceId`). Either tighten the implementation or fix the doc. + +## What to skip + +- **Don't add a `hyperdx_explain` ("what changed at time T") tool** — too + generic, hard to make accurate, and `segment_compare` + `anomaly_window` + together cover most of the value. +- **Don't split `hyperdx_query` into per-displayType tools** — same reasoning + as before; one entry point is good. +- **Don't add an `attribute_keys` tool** — `list_sources` already returns + `mapAttributeKeys` and `mapAttributeLowCardinalityValues`, which is exactly + the right shape. + +## Honest caveat on the n=1 result + +The latency-spike flip (HDX 43% → 82%, CH 60% → 26%) looks dramatic but n=1 +is high variance and CH hitting `max_turns` is partly bad luck. The real test +is whether at n≥3 the HDX advantage holds *without* CH happening to time out. +Worth running n=3 on latency-spike specifically to confirm before claiming the +new tools deliver that magnitude of improvement. diff --git a/packages/hdx-eval/README.md b/packages/hdx-eval/README.md new file mode 100644 index 0000000000..57d9b801aa --- /dev/null +++ b/packages/hdx-eval/README.md @@ -0,0 +1,60 @@ +# @hyperdx/hdx-eval + +Synthetic telemetry data generators for the HyperDX observability eval +framework. Phase 1 deliverable: deterministic, planted-anomaly traces and +logs across three investigation scenarios. + +## Scenarios + +- `error-root-cause` — checkout 5xx spike caused by a `payment-service` → + DB connection timeout, propagated up through `checkout-api`. +- `latency-spike` — `api-server` p99 jump on `GET /api/orders/search` for + enterprise tenants (slow `database.query` child span). +- `noisy-signals` — log volume dominated by 3 patterns: `inventory-service` + heartbeats, `notification-service` cache-hit chatter, `worker` + caught-exception stack traces. + +Each scenario writes into its own ClickHouse tables in the `default` +database: `eval__otel_traces` and `eval__otel_logs`. +Schema mirrors HyperDX's `otel_traces` / `otel_logs` exactly via +`CREATE TABLE ... AS otel_traces`. + +## Usage + +Pre-req: `yarn dev` running so `default.otel_traces` and +`default.otel_logs` exist (created by the OTel collector migrations). + +```bash +# List scenarios + their tables +yarn workspace @hyperdx/hdx-eval dev list + +# Seed a scenario (default seed=42, now=Date.now()) +yarn workspace @hyperdx/hdx-eval dev seed error-root-cause +yarn workspace @hyperdx/hdx-eval dev seed latency-spike --seed 7 +yarn workspace @hyperdx/hdx-eval dev seed noisy-signals \ + --now 2026-05-08T15:00:00Z + +# Drop a scenario's tables +yarn workspace @hyperdx/hdx-eval dev drop error-root-cause +``` + +Connection defaults to `http://localhost:${HDX_DEV_CH_HTTP_PORT}` (set by +`scripts/dev-env.sh`); override with `--ch-url`. + +After seeding, manually create a HyperDX Source pointing at the eval +tables (one-time per scenario) so the HyperDX MCP server can query them. + +## Determinism + +All randomness flows through a `mulberry32(seed)` PRNG. Same `(seed, now)` +produces byte-identical row content. This is what makes side-by-side +comparison of MCP A vs MCP B meaningful. + +## Tests + +```bash +yarn workspace @hyperdx/hdx-eval ci:unit +``` + +Unit tests verify (a) PRNG determinism and (b) each scenario's planted +anomaly invariants hold in the generated rows (no ClickHouse needed). diff --git a/packages/hdx-eval/ablation/REPORT.md b/packages/hdx-eval/ablation/REPORT.md new file mode 100644 index 0000000000..1b51689b8e --- /dev/null +++ b/packages/hdx-eval/ablation/REPORT.md @@ -0,0 +1,59 @@ +# Ablation Report — Split Tools / TOON Output + +Generated: 2026-05-10T11:32:58.357Z + +Each cell: HDX-only, n=see below; metric is the HyperDX MCP arm. + +## Top-line — Combined Score + +| Scenario | Baseline | Split | TOON | Both | Δ split | Δ toon | Δ both | +|---|---:|---:|---:|---:|---:|---:|---:| +| error-root-cause | 97% (n=2) | 98% (n=2) | 100% (n=2) | 97% (n=2) | +1pp | +3pp | +0pp | +| latency-spike | 38% (n=2) | 58% (n=2) | 33% (n=2) | 62% (n=2) | +21pp | -5pp | +24pp | +| noisy-signals | 82% (n=2) | 90% (n=1) | 82% (n=2) | 78% (n=2) | +7pp | -1pp | -5pp | + +## error-root-cause + +| Metric | Baseline | Split | TOON | Both | +|---|---:|---:|---:|---:| +| Combined score | 97% | 98% | 100% | 97% | +| Programmatic score | 100% | 100% | 100% | 100% | +| Judge weighted | 95% | 96% | 100% | 95% | +| Tool calls (mean) | 18.0 | 18.5 | 26.0 | 25.0 | +| Output tokens (mean) | 8943 | 7748 | 6879 | 9177 | +| Wall clock s (mean) | 158.8 | 141.6 | 152.2 | 209.3 | +| N runs | 2 | 2 | 2 | 2 | + +## latency-spike + +| Metric | Baseline | Split | TOON | Both | +|---|---:|---:|---:|---:| +| Combined score | 38% | 58% | 33% | 62% | +| Programmatic score | 70% | 80% | 63% | 77% | +| Judge weighted | 16% | 44% | 13% | 53% | +| Tool calls (mean) | 14.0 | 17.0 | 16.0 | 14.5 | +| Output tokens (mean) | 8985 | 8704 | 7617 | 4861 | +| Wall clock s (mean) | 165.2 | 163.0 | 152.3 | 125.7 | +| N runs | 2 | 2 | 2 | 2 | + +## noisy-signals + +| Metric | Baseline | Split | TOON | Both | +|---|---:|---:|---:|---:| +| Combined score | 82% | 90% | 82% | 78% | +| Programmatic score | 97% | 100% | 92% | 89% | +| Judge weighted | 73% | 83% | 75% | 70% | +| Tool calls (mean) | 8.5 | 9.0 | 14.5 | 16.0 | +| Output tokens (mean) | 5338 | 4671 | 6177 | 3621 | +| Wall clock s (mean) | 105.6 | 89.6 | 147.6 | 149.0 | +| N runs | 2 | 1 | 2 | 2 | + +## Attribution + +Per-scenario combined-score deltas vs baseline (positive = better): + +- **error-root-cause**: split=1pp, toon=3pp, both=0pp +- **latency-spike**: split=21pp, toon=-5pp, both=24pp +- **noisy-signals**: split=7pp, toon=-1pp, both=-5pp + +*N≤2 per cell — single-digit deltas are not significant. Treat any signal of magnitude < 10pp as noise.* diff --git a/packages/hdx-eval/ablation/manifest.tsv b/packages/hdx-eval/ablation/manifest.tsv new file mode 100644 index 0000000000..11073b0f3d --- /dev/null +++ b/packages/hdx-eval/ablation/manifest.tsv @@ -0,0 +1,2 @@ +variant scenario batchDir startedAt +split-fix error-root-cause runs/2026-05-11T04-23-56-949Z 2026-05-11T04:10:34Z diff --git a/packages/hdx-eval/eslint.config.mjs b/packages/hdx-eval/eslint.config.mjs new file mode 100644 index 0000000000..c04226006b --- /dev/null +++ b/packages/hdx-eval/eslint.config.mjs @@ -0,0 +1,60 @@ +import js from '@eslint/js'; +import tseslint from 'typescript-eslint'; +import prettierConfig from 'eslint-config-prettier'; +import prettierPlugin from 'eslint-plugin-prettier'; +import simpleImportSort from 'eslint-plugin-simple-import-sort'; + +export default [ + js.configs.recommended, + ...tseslint.configs.recommended, + prettierConfig, + { + ignores: [ + 'node_modules/**', + 'dist/**', + 'coverage/**', + '**/*.config.mjs', + '**/*.config.js', + 'jest.config.js', + 'scripts/viewer/**', + ], + }, + { + files: ['src/**/*.ts'], + plugins: { + '@typescript-eslint': tseslint.plugin, + 'simple-import-sort': simpleImportSort, + prettier: prettierPlugin, + }, + rules: { + '@typescript-eslint/no-explicit-any': 'off', + '@typescript-eslint/no-unused-vars': [ + 'warn', + { + argsIgnorePattern: '^_', + varsIgnorePattern: '^_', + }, + ], + 'prettier/prettier': 'error', + 'simple-import-sort/imports': 'error', + 'simple-import-sort/exports': 'error', + }, + languageOptions: { + parser: tseslint.parser, + parserOptions: { + ecmaVersion: 'latest', + sourceType: 'module', + project: './tsconfig.json', + tsconfigRootDir: import.meta.dirname, + }, + globals: { + console: 'readonly', + process: 'readonly', + setTimeout: 'readonly', + clearTimeout: 'readonly', + Buffer: 'readonly', + URL: 'readonly', + }, + }, + }, +]; diff --git a/packages/hdx-eval/eval.config.example.json b/packages/hdx-eval/eval.config.example.json new file mode 100644 index 0000000000..cf5949288a --- /dev/null +++ b/packages/hdx-eval/eval.config.example.json @@ -0,0 +1,29 @@ +{ + "_comment": "This file is generated by `hdx-eval setup-hyperdx`. Don't edit by hand.", + "hyperdx": { + "apiUrl": "http://localhost:30196", + "mcpUrl": "http://localhost:30196/mcp", + "accessKey": "uuid-from-/me-endpoint", + "connectionId": "mongo-objectid", + "scenarios": { + "error-root-cause": { + "tracesSourceId": "mongo-objectid", + "logsSourceId": "mongo-objectid" + }, + "latency-spike": { + "tracesSourceId": "mongo-objectid", + "logsSourceId": "mongo-objectid" + }, + "noisy-signals": { + "tracesSourceId": "mongo-objectid", + "logsSourceId": "mongo-objectid" + } + } + }, + "clickhouse": { + "host": "localhost", + "port": "30596", + "user": "default", + "password": "" + } +} diff --git a/packages/hdx-eval/jest.config.js b/packages/hdx-eval/jest.config.js new file mode 100644 index 0000000000..4bdc2f8362 --- /dev/null +++ b/packages/hdx-eval/jest.config.js @@ -0,0 +1,15 @@ +const { createJsWithTsPreset } = require('ts-jest'); + +const tsJestTransformCfg = createJsWithTsPreset(); + +/** @type {import("jest").Config} **/ +module.exports = { + ...tsJestTransformCfg, + testEnvironment: 'node', + rootDir: './src', + testMatch: ['**/__tests__/*.test.ts'], + testTimeout: 15000, + moduleNameMapper: { + '@/(.*)$': '/$1', + }, +}; diff --git a/packages/hdx-eval/package.json b/packages/hdx-eval/package.json new file mode 100644 index 0000000000..9dcd1c9327 --- /dev/null +++ b/packages/hdx-eval/package.json @@ -0,0 +1,42 @@ +{ + "name": "@hyperdx/hdx-eval", + "version": "0.1.0", + "license": "MIT", + "private": true, + "engines": { + "node": ">=22.16.0" + }, + "bin": "./dist/cli.js", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist/*" + ], + "dependencies": { + "@anthropic-ai/sdk": "^0.92.0", + "@clickhouse/client": "^1.12.1", + "@hyperdx/common-utils": "^0.19.0", + "commander": "^12.1.0", + "ms": "^2.1.3" + }, + "devDependencies": { + "@types/jest": "^29.5.14", + "@types/ms": "^0.7.31", + "@types/node": "^22.15.18", + "jest": "^30.2.0", + "rimraf": "^4.4.1", + "ts-jest": "^29.4.5", + "tsx": "^4.19.0", + "typescript": "^5.9.3" + }, + "scripts": { + "build": "rimraf ./dist && tsc -p tsconfig.build.json", + "dev": "NODE_OPTIONS=--max-old-space-size=8192 tsx src/cli.ts", + "lint": "npx eslint --quiet . --ext .ts", + "lint:fix": "npx eslint . --ext .ts --fix", + "ci:lint": "yarn lint && yarn tsc --noEmit", + "ci:unit": "jest --ci --forceExit", + "dev:unit": "jest --watch", + "viewer": "node scripts/viewer/server.js" + } +} diff --git a/packages/hdx-eval/scripts/ablation-report.ts b/packages/hdx-eval/scripts/ablation-report.ts new file mode 100644 index 0000000000..c74ca6055b --- /dev/null +++ b/packages/hdx-eval/scripts/ablation-report.ts @@ -0,0 +1,266 @@ +/** + * Read packages/hdx-eval/ablation/manifest.tsv (written by ablation.sh), + * grade every batch listed there, and emit a comparison markdown that + * attributes performance changes to each variant (split / toon / both) + * relative to the baseline. + * + * Usage: + * yarn workspace @hyperdx/hdx-eval dev scripts/ablation-report.ts + */ + +import { existsSync, readFileSync, writeFileSync } from 'fs'; +import { resolve } from 'path'; + +import { gradeBatch, resolveBatchDir } from '../src/grading/grade'; +import type { BatchSummary } from '../src/reports/aggregate'; +import { writeBatchSummary } from '../src/reports/store'; + +type ManifestRow = { + variant: string; + scenario: string; + batchDir: string; + startedAt: string; +}; + +const PKG_ROOT = resolve(__dirname, '..'); +const MANIFEST_PATH = resolve(PKG_ROOT, 'ablation', 'manifest.tsv'); +const REPORT_PATH = resolve(PKG_ROOT, 'ablation', 'REPORT.md'); + +const VARIANT_ORDER = ['baseline', 'split', 'toon', 'both']; +const VARIANT_LABEL: Record = { + baseline: 'Baseline', + split: 'Split', + toon: 'TOON', + both: 'Both', +}; + +function readManifest(): ManifestRow[] { + const raw = readFileSync(MANIFEST_PATH, 'utf8'); + const lines = raw.trim().split('\n'); + const out: ManifestRow[] = []; + for (const line of lines.slice(1)) { + const parts = line.split('\t'); + if (parts.length < 4) continue; + if (!parts[2] || parts[2] === '') continue; // missing batch dir + out.push({ + variant: parts[0], + scenario: parts[1], + batchDir: parts[2], + startedAt: parts[3], + }); + } + return out; +} + +async function main() { + if (!existsSync(MANIFEST_PATH)) { + console.error(`Manifest not found: ${MANIFEST_PATH}`); + process.exit(1); + } + + const rows = readManifest(); + console.log(`Manifest has ${rows.length} batch entries.`); + + // ── Grade every batch (idempotent — skips runs that already have grade.json) ── + for (const row of rows) { + const dir = resolveBatchDir(row.batchDir); + if (!existsSync(dir)) { + console.warn( + ` skip ${row.variant}/${row.scenario}: batch dir missing (${dir})`, + ); + continue; + } + console.log( + ` grading ${row.variant}/${row.scenario} (${row.batchDir})...`, + ); + try { + await gradeBatch(dir, { + judgeModel: 'claude-opus-4-7', + rerunJudge: false, + skipJudge: false, + }); + } catch (e) { + console.error( + ` grade failed: ${e instanceof Error ? e.message : String(e)}`, + ); + } + } + + // ── Render each batch's _summary.{md,json} ── + for (const row of rows) { + const dir = resolveBatchDir(row.batchDir); + if (!existsSync(dir)) continue; + try { + writeBatchSummary(dir, resolve(dir, '_summary.md')); + } catch (e) { + console.error(` summary failed for ${row.batchDir}: ${e}`); + } + } + + // ── Load per-cell summaries (HDX side only — split/toon/both don't change CH) ── + type Cell = { + variant: string; + scenario: string; + n: number; + combined: number; + programmatic: number; + judge: number; + toolCalls: number; + outputTokens: number; + wallClockS: number; + perCheck: Record; + }; + const cells: Cell[] = []; + for (const row of rows) { + const dir = resolveBatchDir(row.batchDir); + const summaryPath = resolve(dir, '_summary.json'); + if (!existsSync(summaryPath)) { + console.warn(` no _summary.json in ${row.batchDir}`); + continue; + } + const summary = JSON.parse( + readFileSync(summaryPath, 'utf8'), + ) as BatchSummary; + for (const sc of summary.scenarios) { + const hdx = sc.cells.hyperdx; + if (!hdx) continue; + cells.push({ + variant: row.variant, + scenario: sc.scenario, + n: hdx.n, + combined: hdx.combinedScore.mean, + programmatic: hdx.programmatic.mean, + judge: hdx.judge.weightedMean, + toolCalls: hdx.toolCalls.mean, + outputTokens: hdx.tokens.output, + wallClockS: hdx.durationMs.mean / 1000, + perCheck: hdx.programmatic.perCheck, + }); + } + } + + // ── Build report ── + const scenarios = Array.from(new Set(cells.map(c => c.scenario))).sort(); + const lines: string[] = []; + lines.push('# Ablation Report — Split Tools / TOON Output'); + lines.push(''); + lines.push(`Generated: ${new Date().toISOString()}`); + lines.push(''); + lines.push( + `Each cell: HDX-only, n=${rows[0]?.batchDir ? 'see below' : '?'}; metric is the HyperDX MCP arm.`, + ); + lines.push(''); + + // Top-line: combined score per (variant, scenario) + lines.push('## Top-line — Combined Score'); + lines.push(''); + lines.push( + '| Scenario | ' + + VARIANT_ORDER.map(v => VARIANT_LABEL[v]).join(' | ') + + ' | Δ split | Δ toon | Δ both |', + ); + lines.push( + '|---|' + VARIANT_ORDER.map(() => '---:').join('|') + '|---:|---:|---:|', + ); + for (const sc of scenarios) { + const byVariant: Record = {}; + for (const c of cells) { + if (c.scenario === sc) byVariant[c.variant] = c; + } + const baseline = byVariant.baseline; + const cellStr = (c?: Cell) => + c ? `${(c.combined * 100).toFixed(0)}% (n=${c.n})` : '—'; + const deltaStr = (c?: Cell) => { + if (!c || !baseline) return '—'; + const d = (c.combined - baseline.combined) * 100; + const sign = d >= 0 ? '+' : ''; + return `${sign}${d.toFixed(0)}pp`; + }; + lines.push( + '| ' + + sc + + ' | ' + + VARIANT_ORDER.map(v => cellStr(byVariant[v])).join(' | ') + + ' | ' + + deltaStr(byVariant.split) + + ' | ' + + deltaStr(byVariant.toon) + + ' | ' + + deltaStr(byVariant.both) + + ' |', + ); + } + lines.push(''); + + // Per-scenario detail tables + for (const sc of scenarios) { + lines.push(`## ${sc}`); + lines.push(''); + lines.push('| Metric | Baseline | Split | TOON | Both |'); + lines.push('|---|---:|---:|---:|---:|'); + const byVariant: Record = {}; + for (const c of cells) { + if (c.scenario === sc) byVariant[c.variant] = c; + } + const row = (label: string, fn: (c: Cell) => string): string => + '| ' + + label + + ' | ' + + VARIANT_ORDER.map(v => { + const c = byVariant[v]; + return c ? fn(c) : '—'; + }).join(' | ') + + ' |'; + lines.push(row('Combined score', c => `${(c.combined * 100).toFixed(0)}%`)); + lines.push( + row('Programmatic score', c => `${(c.programmatic * 100).toFixed(0)}%`), + ); + lines.push(row('Judge weighted', c => `${(c.judge * 100).toFixed(0)}%`)); + lines.push(row('Tool calls (mean)', c => c.toolCalls.toFixed(1))); + lines.push( + row('Output tokens (mean)', c => Math.round(c.outputTokens).toString()), + ); + lines.push(row('Wall clock s (mean)', c => c.wallClockS.toFixed(1))); + lines.push(row('N runs', c => String(c.n))); + lines.push(''); + } + + // Summary commentary + lines.push('## Attribution'); + lines.push(''); + lines.push( + 'Per-scenario combined-score deltas vs baseline (positive = better):', + ); + lines.push(''); + for (const sc of scenarios) { + const byVariant: Record = {}; + for (const c of cells) { + if (c.scenario === sc) byVariant[c.variant] = c; + } + const b = byVariant.baseline; + const s = byVariant.split; + const t = byVariant.toon; + const both = byVariant.both; + if (!b) continue; + const delta = (c?: Cell) => + c ? `${((c.combined - b.combined) * 100).toFixed(0)}pp` : '—'; + lines.push( + `- **${sc}**: split=${delta(s)}, toon=${delta(t)}, both=${delta(both)}`, + ); + } + lines.push(''); + lines.push( + '*N≤2 per cell — single-digit deltas are not significant. Treat any signal of magnitude < 10pp as noise.*', + ); + + const out = lines.join('\n') + '\n'; + writeFileSync(REPORT_PATH, out); + console.log(`\nWrote ${REPORT_PATH}`); + console.log('\n--- preview ---\n'); + console.log(out.slice(0, 4000)); +} + +main().catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/packages/hdx-eval/scripts/ablation.sh b/packages/hdx-eval/scripts/ablation.sh new file mode 100755 index 0000000000..4c7d155a5a --- /dev/null +++ b/packages/hdx-eval/scripts/ablation.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# Ablation runner for the MCP query-tool experiments. +# +# Variants: +# baseline — current MCP (hyperdx_query, JSON) +# split — HDX_MCP_SPLIT_QUERY=1 (hyperdx_timeseries / table / search / sql) +# toon — HDX_MCP_OUTPUT=toon (TOON output, ClickHouse metadata stripped) +# both — split + toon +# +# Per variant: kill the running API process, restart yarn app:dev with the +# variant env vars, wait for /health, run hdx-eval (HDX MCP only) on every +# scenario at n=$RUNS, and stash the resulting batch dirs under +# ablation//. +# +# Reads ANTHROPIC_API_KEY from ~/.config/hdx-eval-anthropic-key. + +set -uo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +cd "$REPO_ROOT" + +VARIANTS="${VARIANTS:-baseline split toon both}" +SCENARIOS="${SCENARIOS:-error-root-cause latency-spike noisy-signals}" +RUNS="${RUNS:-2}" + +OUT_DIR="$REPO_ROOT/packages/hdx-eval/ablation" +mkdir -p "$OUT_DIR" +MANIFEST="$OUT_DIR/manifest.tsv" +echo -e "variant\tscenario\tbatchDir\tstartedAt" > "$MANIFEST" + +source ~/.nvm/nvm.sh > /dev/null 2>&1 || true +nvm use 22.21.1 > /dev/null 2>&1 || true +export PATH="/usr/local/bin:$HOME/.local/bin:$PATH" + +if [[ ! -f "$HOME/.config/hdx-eval-anthropic-key" ]]; then + echo "Missing ~/.config/hdx-eval-anthropic-key — aborting" >&2 + exit 1 +fi +export ANTHROPIC_API_KEY=$(cat "$HOME/.config/hdx-eval-anthropic-key") + +# ─── dev-env (port slot, etc) ────────────────────────────────────────────── +source ./scripts/dev-env.sh > /dev/null + +# ─── ensure docker stack is up (idempotent) ──────────────────────────────── +# We bypass `yarn dev` because that script runs `docker compose down` when +# yarn app:dev exits, which would tear down ClickHouse between variants. +# Bring docker up once here; leave it running across variants. +ensure_docker() { + echo "[$(date +%H:%M:%S)] Ensuring docker stack is up (project=$HDX_DEV_PROJECT)..." + sg docker -c "docker compose -p '$HDX_DEV_PROJECT' -f docker-compose.dev.yml up -d" \ + > "$OUT_DIR/_docker.log" 2>&1 +} +ensure_docker + +# ─── env-vars per variant ────────────────────────────────────────────────── +variant_env() { + case "$1" in + baseline) echo "" ;; + split) echo "HDX_MCP_SPLIT_QUERY=1" ;; + toon) echo "HDX_MCP_OUTPUT=toon" ;; + both) echo "HDX_MCP_SPLIT_QUERY=1 HDX_MCP_OUTPUT=toon" ;; + *) echo "unknown variant: $1" >&2; exit 1 ;; + esac +} + +# ─── kill the API process group ──────────────────────────────────────────── +# We start the API in its own process group via setsid so we can kill the +# whole tree (yarn → nodemon → ts-node) with `kill -- -`. Falling back +# to pattern-pkill in case the pgid isn't tracked (first invocation). +API_PGID_FILE="$OUT_DIR/_api.pgid" + +kill_api() { + echo "[$(date +%H:%M:%S)] Killing existing API processes..." + if [[ -f "$API_PGID_FILE" ]]; then + local pgid + pgid="$(cat "$API_PGID_FILE")" + if [[ -n "$pgid" ]] && kill -0 "-$pgid" 2>/dev/null; then + kill -TERM -- "-$pgid" 2>/dev/null || true + sleep 3 + kill -KILL -- "-$pgid" 2>/dev/null || true + fi + rm -f "$API_PGID_FILE" + fi + # Best-effort cleanup of anything that survived. + pkill -9 -f 'nodemon.*src/index' >/dev/null 2>&1 || true + pkill -9 -f 'ts-node.*src/index' >/dev/null 2>&1 || true + # Wait until the API port is actually free before returning. + for _ in {1..15}; do + if ! ss -tln 2>/dev/null | grep -q ":${HYPERDX_API_PORT}\b"; then + break + fi + sleep 1 + done + ensure_docker +} + +# ─── start ONLY the @hyperdx/api workspace dev process ───────────────────── +# We bypass yarn app:dev (which also brings up Next.js, the alerts-task, and +# a common-utils nodemon — none of which the eval needs). Just +# `yarn workspace @hyperdx/api dev` is enough to host /mcp. +# +# Started via `setsid` so the process gets its own group; pgid is recorded so +# kill_api can take down the whole tree at once. +start_api() { + local env_extra="$1" + local label="$2" + local logfile="$OUT_DIR/$label.api.log" + echo "[$(date +%H:%M:%S)] Starting API with: $env_extra" + setsid bash -c "source ~/.nvm/nvm.sh && nvm use 22.21.1 > /dev/null && source ./scripts/dev-env.sh && export $env_extra && yarn workspace @hyperdx/api dev" \ + > "$logfile" 2>&1 & + local pid=$! + echo "$pid" > "$API_PGID_FILE" # setsid makes pid == pgid + echo " pgid=$pid log=$logfile" +} + +wait_for_api() { + echo -n " Waiting for API" + local deadline=$(( $(date +%s) + 240 )) + while (( $(date +%s) < deadline )); do + if curl -fsS "http://localhost:${HYPERDX_API_PORT}/health" > /dev/null 2>&1; then + echo " — ready" + return 0 + fi + echo -n "." + sleep 3 + done + echo " — TIMEOUT" + return 1 +} + +# ─── per-scenario eval invocation ────────────────────────────────────────── +run_variant() { + local variant="$1" + local env_extra + env_extra="$(variant_env "$variant")" + local started + started="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + + echo + echo "==========================================================" + echo " VARIANT: $variant ($started)" + echo "==========================================================" + + kill_api + start_api "$env_extra" "$variant" + if ! wait_for_api; then + echo "API failed to start for variant=$variant; skipping" >&2 + return 1 + fi + + for scenario in $SCENARIOS; do + echo "[$(date +%H:%M:%S)] $variant / $scenario — running n=$RUNS..." + local logfile="$OUT_DIR/$variant.$scenario.run.log" + yarn workspace @hyperdx/hdx-eval dev run "$scenario" \ + --runs "$RUNS" \ + --mcp hyperdx \ + > "$logfile" 2>&1 + local rc=$? + local batch_dir + batch_dir="$(grep -oE 'runs/[0-9TZ:.-]+' "$logfile" | head -1)" + echo -e "${variant}\t${scenario}\t${batch_dir}\t${started}" >> "$MANIFEST" + echo " -> rc=$rc batch=$batch_dir" + done +} + +# ─── main ────────────────────────────────────────────────────────────────── +echo "Repo: $REPO_ROOT" +echo "Variants: $VARIANTS" +echo "Scenarios: $SCENARIOS" +echo "Runs/cell: $RUNS" +echo "Manifest: $MANIFEST" + +for variant in $VARIANTS; do + run_variant "$variant" +done + +echo +echo "All variants done at $(date +%H:%M:%S). Manifest:" +cat "$MANIFEST" diff --git a/packages/hdx-eval/scripts/compare-prompt-variants.sh b/packages/hdx-eval/scripts/compare-prompt-variants.sh new file mode 100755 index 0000000000..e4b916aedd --- /dev/null +++ b/packages/hdx-eval/scripts/compare-prompt-variants.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# compare-prompt-variants.sh — A/B the baseline vs hypothesis system prompt +# on a fixed set of scenarios at a shared anchor. +# +# Runs N×scenarios×{baseline,hypothesis} agents against the already-seeded +# tables (no reseed), then grades the resulting batches. +# +# Usage: +# ./packages/hdx-eval/scripts/compare-prompt-variants.sh +# [SCENARIOS="latency-spike segmented-regression error-root-cause"] +# [RUNS=3] [MCP=both] [PER_SCENARIO_CONCURRENCY=10] +# [ANCHOR_TIME=2026-05-11T04:49:00Z] +# [VARIANTS="baseline hypothesis"] +# +# Defaults tuned for fast iteration: n=3 per cell, 10 agents in parallel per +# (scenario, variant), grading parallelized across all batches. Override any +# of those env vars to revert to a slower / more thorough sweep. + +set -uo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +cd "$REPO_ROOT" + +source ~/.nvm/nvm.sh > /dev/null 2>&1 || true +nvm use 22.21.1 > /dev/null 2>&1 || true +export PATH="/usr/local/bin:$HOME/.local/bin:$PATH" + +if [[ ! -f "$HOME/.config/hdx-eval-anthropic-key" ]]; then + echo "Missing ~/.config/hdx-eval-anthropic-key — aborting" >&2 + exit 1 +fi +export ANTHROPIC_API_KEY=$(cat "$HOME/.config/hdx-eval-anthropic-key") + +source ./scripts/dev-env.sh > /dev/null + +SCENARIOS="${SCENARIOS:-latency-spike segmented-regression error-root-cause}" +RUNS="${RUNS:-3}" +MCP="${MCP:-both}" +MODEL="${MODEL:-claude-sonnet-4-6}" +PER_SCENARIO_CONCURRENCY="${PER_SCENARIO_CONCURRENCY:-5}" +ANCHOR_TIME="${ANCHOR_TIME:-2026-05-11T04:49:00Z}" +VARIANTS="${VARIANTS:-baseline hypothesis}" + +OUT_DIR="$REPO_ROOT/packages/hdx-eval/ablation" +mkdir -p "$OUT_DIR" +LOGDIR="$OUT_DIR/prompt-variant-$(date -u +%Y%m%dT%H%M%SZ)" +mkdir -p "$LOGDIR" + +echo "==========================================================" +echo " compare-prompt-variants" +echo " scenarios : $SCENARIOS" +echo " variants : $VARIANTS" +echo " runs per cell : $RUNS" +echo " mcp : $MCP" +echo " per-scenario conc : $PER_SCENARIO_CONCURRENCY" +echo " anchor time : $ANCHOR_TIME" +echo " log dir : $LOGDIR" +echo "==========================================================" + +run_one_variant() { + local variant="$1" + echo + echo "[$(date +%H:%M:%S)] === variant=$variant ===" + pids=() + for scenario in $SCENARIOS; do + ( + log="$LOGDIR/run.$variant.$scenario.log" + echo " [$(date +%H:%M:%S)] start $variant/$scenario → $log" + yarn workspace @hyperdx/hdx-eval dev run "$scenario" \ + --runs "$RUNS" --mcp "$MCP" --no-reseed \ + --anchor-time "$ANCHOR_TIME" \ + --concurrency "$PER_SCENARIO_CONCURRENCY" \ + --prompt-variant "$variant" \ + --model "$MODEL" \ + --ch-url "http://localhost:${HDX_DEV_CH_HTTP_PORT}" \ + > "$log" 2>&1 + rc=$? + batch=$(grep -oE 'runs/[^ ]+' "$log" | head -1) + echo " [$(date +%H:%M:%S)] done $variant/$scenario exit=$rc batch=$batch" + if [[ -n "$batch" ]]; then + echo "$variant $scenario $batch" >> "$LOGDIR/batches.tsv" + fi + ) & + pids+=($!) + done + for pid in "${pids[@]}"; do + wait "$pid" || true + done +} + +for v in $VARIANTS; do + run_one_variant "$v" +done + +echo +echo "[$(date +%H:%M:%S)] All runs done. Batches:" +cat "$LOGDIR/batches.tsv" 2>/dev/null || echo " (no batches recorded)" + +# ─── Grade each batch (in parallel) ──────────────────────────────────────── +# Judge calls are independent per (run, scenario, mcp); fan them out instead +# of grinding through batches serially. With one batch per (variant, scenario) +# this is up to 6 concurrent grade invocations on the default 3-scenario +# × 2-variant sweep. +echo +echo "[$(date +%H:%M:%S)] Grading all batches (parallel)..." +grade_pids=() +while read -r variant scenario batch; do + [[ -z "$batch" ]] && continue + ( + log="$LOGDIR/grade.$variant.$scenario.log" + echo " [$(date +%H:%M:%S)] start grade $variant/$scenario → $log" + yarn workspace @hyperdx/hdx-eval dev grade "$batch" > "$log" 2>&1 + rc=$? + echo " [$(date +%H:%M:%S)] done grade $variant/$scenario exit=$rc" + ) & + grade_pids+=($!) +done < "$LOGDIR/batches.tsv" +for pid in "${grade_pids[@]}"; do + wait "$pid" || true +done + +echo +echo "[$(date +%H:%M:%S)] ALL DONE" +echo "Logs: $LOGDIR" +echo "Batches: $LOGDIR/batches.tsv" diff --git a/packages/hdx-eval/scripts/fast-eval.sh b/packages/hdx-eval/scripts/fast-eval.sh new file mode 100755 index 0000000000..9ca5d0592c --- /dev/null +++ b/packages/hdx-eval/scripts/fast-eval.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# fast-eval.sh — bulk seed all scenarios at a shared anchor, then run the +# eval in parallel across scenarios. +# +# Speedup vs the legacy sequential per-scenario re-seed: +# - seed once, share across all (mcp, run) cells (-50 min) +# - run scenarios concurrently, agents in parallel (variable) +# +# Usage: +# ./packages/hdx-eval/scripts/fast-eval.sh +# [SCENARIOS="error-root-cause latency-spike ..."] +# [RUNS=2] [PER_SCENARIO_CONCURRENCY=2] [SCENARIO_CONCURRENCY=2] +# +# Reads ANTHROPIC_API_KEY from ~/.config/hdx-eval-anthropic-key. +# Assumes the dev API + ClickHouse are already up at the slot's ports. + +set -uo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +cd "$REPO_ROOT" + +source ~/.nvm/nvm.sh > /dev/null 2>&1 || true +nvm use 22.21.1 > /dev/null 2>&1 || true +export PATH="/usr/local/bin:$HOME/.local/bin:$PATH" + +if [[ ! -f "$HOME/.config/hdx-eval-anthropic-key" ]]; then + echo "Missing ~/.config/hdx-eval-anthropic-key — aborting" >&2 + exit 1 +fi +export ANTHROPIC_API_KEY=$(cat "$HOME/.config/hdx-eval-anthropic-key") + +source ./scripts/dev-env.sh > /dev/null + +SCENARIOS="${SCENARIOS:-error-root-cause latency-spike noisy-signals segmented-regression}" +RUNS="${RUNS:-2}" +PER_SCENARIO_CONCURRENCY="${PER_SCENARIO_CONCURRENCY:-2}" +SCENARIO_CONCURRENCY="${SCENARIO_CONCURRENCY:-2}" +# Anchor at "now" rounded to the nearest minute. All scenarios will seed at +# this same timestamp and the agent's system prompt will use it as "now". +ANCHOR_TIME="${ANCHOR_TIME:-$(date -u -d "$(date -u +%Y-%m-%dT%H:%M:00)" +%Y-%m-%dT%H:%M:%SZ)}" + +OUT_DIR="$REPO_ROOT/packages/hdx-eval/ablation" +mkdir -p "$OUT_DIR" +LOGDIR="$OUT_DIR/fast-eval-$(date -u +%Y%m%dT%H%M%SZ)" +mkdir -p "$LOGDIR" + +echo "==========================================================" +echo " fast-eval" +echo " scenarios : $SCENARIOS" +echo " runs per cell : $RUNS" +echo " per-scenario conc : $PER_SCENARIO_CONCURRENCY" +echo " scenario conc : $SCENARIO_CONCURRENCY" +echo " anchor time : $ANCHOR_TIME" +echo " log dir : $LOGDIR" +echo "==========================================================" + +# ─── Phase 1: seed all scenarios at the shared anchor ───────────────────── +echo +echo "[$(date +%H:%M:%S)] Phase 1: seeding all scenarios at $ANCHOR_TIME (parallel)" +seed_pids=() +for scenario in $SCENARIOS; do + ( + log="$LOGDIR/seed.$scenario.log" + echo " [$(date +%H:%M:%S)] start seed $scenario → $log" + yarn workspace @hyperdx/hdx-eval dev seed "$scenario" \ + --seed 42 --now "$ANCHOR_TIME" \ + --ch-url "http://localhost:${HDX_DEV_CH_HTTP_PORT}" \ + > "$log" 2>&1 + rc=$? + echo " [$(date +%H:%M:%S)] done seed $scenario exit=$rc" + ) & + seed_pids+=($!) +done +seed_failed=0 +for pid in "${seed_pids[@]}"; do + wait "$pid" || seed_failed=1 +done +if (( seed_failed )); then + echo "One or more seeds failed — see $LOGDIR/seed.*.log" >&2 + exit 1 +fi + +# ─── Phase 2: run scenarios in parallel, each cell pool also parallel ───── +echo +echo "[$(date +%H:%M:%S)] Phase 2: running scenarios (concurrency=$SCENARIO_CONCURRENCY × $PER_SCENARIO_CONCURRENCY agents/scenario)" + +# Pool of scenario-runners. Limit to SCENARIO_CONCURRENCY in flight. +running=0 +run_pids=() +run_scens=() +for scenario in $SCENARIOS; do + while (( running >= SCENARIO_CONCURRENCY )); do + # Wait for any one to finish. + for idx in "${!run_pids[@]}"; do + pid="${run_pids[$idx]}" + if ! kill -0 "$pid" 2>/dev/null; then + wait "$pid" || true + unset 'run_pids[idx]' + unset 'run_scens[idx]' + running=$((running - 1)) + fi + done + sleep 1 + done + ( + log="$LOGDIR/run.$scenario.log" + echo " [$(date +%H:%M:%S)] start run $scenario → $log" + yarn workspace @hyperdx/hdx-eval dev run "$scenario" \ + --runs "$RUNS" --mcp both --no-reseed \ + --anchor-time "$ANCHOR_TIME" \ + --concurrency "$PER_SCENARIO_CONCURRENCY" \ + --ch-url "http://localhost:${HDX_DEV_CH_HTTP_PORT}" \ + > "$log" 2>&1 + rc=$? + batch=$(grep -oE 'runs/[^ ]+' "$log" | head -1) + echo " [$(date +%H:%M:%S)] done run $scenario exit=$rc batch=$batch" + ) & + run_pids+=($!) + run_scens+=("$scenario") + running=$((running + 1)) +done +wait +echo +echo "[$(date +%H:%M:%S)] ALL DONE" +echo "Logs: $LOGDIR" diff --git a/packages/hdx-eval/scripts/viewer/public/app.js b/packages/hdx-eval/scripts/viewer/public/app.js new file mode 100644 index 0000000000..a1dce8c5bf --- /dev/null +++ b/packages/hdx-eval/scripts/viewer/public/app.js @@ -0,0 +1,685 @@ +(() => { + const $ = (sel) => document.querySelector(sel); + const el = (tag, attrs = {}, ...children) => { + const n = document.createElement(tag); + for (const [k, v] of Object.entries(attrs)) { + if (k === 'class') n.className = v; + else if (k === 'html') n.innerHTML = v; + else if (k.startsWith('on')) n.addEventListener(k.slice(2), v); + else if (v === true) n.setAttribute(k, ''); + else if (v != null && v !== false) n.setAttribute(k, v); + } + for (const c of children.flat()) { + if (c == null || c === false) continue; + n.appendChild(typeof c === 'string' ? document.createTextNode(c) : c); + } + return n; + }; + + const state = { + batches: [], + batch: null, + cells: [], + selectedRun: null, // { scenario, mcp, idx } + runData: null, // { trajectory, grade } + activeTab: 'trajectory', + }; + + // ----- formatting helpers ----- + + const fmtMs = (ms) => { + if (ms == null) return '—'; + if (ms < 1000) return `${ms}ms`; + if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s`; + return `${(ms / 60_000).toFixed(1)}m`; + }; + + const fmtScore = (s) => (s == null ? '—' : s.toFixed(2)); + + const scoreClass = (s, max = 1) => { + if (s == null) return ''; + const n = s / max; + if (n <= 0.01) return 'bad'; + if (n < 0.5) return 'warn'; + return 'good'; + }; + + const criterionClass = (s) => { + if (s == null) return ''; + if (s === 0) return 'score-0'; + if (s <= 1) return 'score-low'; + if (s <= 2) return 'score-mid'; + return 'score-high'; + }; + + const stringifyMaybe = (v) => { + if (v == null) return ''; + if (typeof v === 'string') return v; + try { + return JSON.stringify(v, null, 2); + } catch { + return String(v); + } + }; + + // Tool results come back in a few shapes: + // - already an object/array (from toolCalls[].input) + // - a JSON-encoded string + // - newline-delimited JSON (ToolSearch output) + // - an array of {type:"text", text:"..."} content blocks (raw tool_result) + // Render them as pretty-printed JSON where possible, with text blocks unwrapped. + const tryParseJson = (s) => { + if (typeof s !== 'string') return undefined; + const t = s.trim(); + if (!t || (t[0] !== '{' && t[0] !== '[' && t[0] !== '"')) return undefined; + try { + return JSON.parse(t); + } catch { + return undefined; + } + }; + + const prettyJson = (v) => { + try { + return JSON.stringify(v, null, 2); + } catch { + return String(v); + } + }; + + const formatToolPayload = (v) => { + if (v == null) return ''; + + // Object / array: pretty-print directly, but unwrap content-block arrays. + if (typeof v === 'object') { + if ( + Array.isArray(v) && + v.length > 0 && + v.every( + (b) => b && typeof b === 'object' && b.type === 'text' && 'text' in b, + ) + ) { + return v + .map((b) => { + const inner = tryParseJson(b.text); + return inner !== undefined ? prettyJson(inner) : b.text; + }) + .join('\n\n'); + } + return prettyJson(v); + } + + if (typeof v !== 'string') return String(v); + + const trimmed = v.trim(); + if (!trimmed) return v; + + // Whole-string JSON. + const parsed = tryParseJson(trimmed); + if (parsed !== undefined) return prettyJson(parsed); + + // Newline-delimited JSON (e.g. ToolSearch). + const lines = trimmed.split('\n'); + if (lines.length > 1) { + const objs = []; + let allParsed = true; + for (const line of lines) { + const p = tryParseJson(line); + if (p === undefined) { + allParsed = false; + break; + } + objs.push(p); + } + if (allParsed) return objs.map(prettyJson).join('\n'); + } + + // wrapper with an embedded JSON preview. + const previewMatch = v.match( + /^([\s\S]*?Preview \(first [^)]+\):\n)([\s\S]*?)(\n\.\.\.\n<\/persisted-output>)$/, + ); + if (previewMatch) { + const [, header, body, footer] = previewMatch; + const previewParsed = tryParseJson(body); + if (previewParsed !== undefined) { + return header + prettyJson(previewParsed) + footer; + } + } + + return v; + }; + + // ----- data loading ----- + + async function fetchJson(url) { + const r = await fetch(url); + if (!r.ok) throw new Error(`${r.status} ${r.statusText}`); + return r.json(); + } + + async function loadBatches() { + const data = await fetchJson('/api/batches'); + state.batches = data.batches; + renderBatchPicker(); + if (state.batches.length) await loadBatch(state.batches[0].name); + } + + async function loadBatch(batchName) { + const data = await fetchJson(`/api/batches/${encodeURIComponent(batchName)}`); + state.batch = data.batch; + state.cells = data.cells || []; + state.summary = data.summary; + renderBatchMeta(); + renderNav(); + } + + async function loadRun(scenario, mcp, idx) { + state.selectedRun = { scenario, mcp, idx }; + const u = `/api/batches/${encodeURIComponent(state.batch)}/runs/${encodeURIComponent(scenario)}/${encodeURIComponent(mcp)}/${idx}`; + state.runData = await fetchJson(u); + renderNav(); + renderRun(); + } + + // ----- rendering: header ----- + + function renderBatchPicker() { + const sel = $('#batch-select'); + sel.innerHTML = ''; + for (const b of state.batches) { + sel.appendChild(el('option', { value: b.name }, b.name)); + } + sel.onchange = () => loadBatch(sel.value); + } + + function renderBatchMeta() { + const sc = state.summary?.scenarios?.length ?? state.cells.length; + const cells = state.cells.length; + $('#batch-meta').textContent = `${sc} scenario(s) · ${cells} cell(s)`; + } + + // ----- rendering: sidebar nav ----- + + function renderNav() { + const root = $('#nav'); + root.innerHTML = ''; + + const byScenario = new Map(); + for (const c of state.cells) { + if (!byScenario.has(c.scenario)) byScenario.set(c.scenario, []); + byScenario.get(c.scenario).push(c); + } + + for (const [scenario, cells] of byScenario) { + const sNode = el( + 'div', + { class: 'nav-scenario' }, + el('div', { class: 'head' }, scenario), + ); + for (const c of cells) { + const mNode = el( + 'div', + { class: 'nav-mcp' }, + el('div', { class: 'head' }, c.mcp), + ); + for (const r of c.runs) { + const active = + state.selectedRun && + state.selectedRun.scenario === scenario && + state.selectedRun.mcp === c.mcp && + state.selectedRun.idx === r.idx; + const chip = el( + 'span', + { class: `score-chip ${scoreClass(r.combinedScore)}` }, + fmtScore(r.combinedScore), + ); + const errBadge = + r.toolErrors > 0 + ? el('span', { class: 'tag error' }, `${r.toolErrors}!`) + : null; + const termBadge = + r.termination && r.termination !== 'final_answer' + ? el('span', { class: 'tag' }, r.termination) + : null; + const node = el( + 'div', + { + class: `nav-run ${active ? 'active' : ''}`, + onclick: () => loadRun(scenario, c.mcp, r.idx), + }, + el( + 'span', + {}, + `#${r.idx}`, + ' ', + el('span', { class: 'muted small' }, `${r.toolCalls ?? '?'} calls`), + ), + el('span', {}, termBadge, ' ', errBadge, ' ', chip), + ); + mNode.appendChild(node); + } + sNode.appendChild(mNode); + } + root.appendChild(sNode); + } + } + + // ----- rendering: run ----- + + function renderRun() { + $('#empty').hidden = true; + $('#run').hidden = false; + + const { trajectory: t, grade } = state.runData; + + // header strip + const hdr = $('#run-header'); + hdr.innerHTML = ''; + const fields = [ + ['scenario', t.scenario], + ['mcp', t.mcp], + ['run', `#${t.runIndex} · seed ${t.seed ?? '—'}`], + ['model', t.model || '—'], + ['termination', t.termination || '—'], + ['duration', fmtMs(t.durationMs)], + ['tool calls', String(t.toolCalls?.length ?? 0)], + [ + 'tool errors', + grade?.toolErrors + ? `${grade.toolErrors.errors}/${grade.toolErrors.total}` + : '—', + ], + [ + 'combined', + grade?.combinedScore != null ? grade.combinedScore.toFixed(3) : '—', + ], + ]; + for (const [k, v] of fields) { + hdr.appendChild( + el( + 'div', + {}, + el('div', { class: 'label' }, k), + el('div', { class: 'value' }, String(v)), + ), + ); + } + + renderTrajectory(); + renderAnswer(); + renderGrading(); + renderRaw(); + + activateTab(state.activeTab); + } + + function activateTab(name) { + state.activeTab = name; + for (const t of document.querySelectorAll('.tab')) { + t.classList.toggle('active', t.dataset.tab === name); + } + for (const p of document.querySelectorAll('.tab-pane')) { + p.classList.toggle('active', p.id === `tab-${name}`); + } + } + + // Build a unified trajectory from trajectory.messages, mixing thinking text, + // assistant text, and tool calls (with their results inline). Falls back to + // toolCalls[] when messages aren't structured the expected way. + function buildEvents(t) { + const events = []; + const toolResults = new Map(); // tool_use_id -> content + + if (Array.isArray(t.messages)) { + // First pass: collect tool_result blocks (sent in user messages) + for (const msg of t.messages) { + if (msg.type === 'user' && msg.message?.content) { + const content = msg.message.content; + if (Array.isArray(content)) { + for (const b of content) { + if (b.type === 'tool_result') { + toolResults.set(b.tool_use_id, { + content: b.content, + isError: !!b.is_error, + }); + } + } + } + } + } + // Second pass: emit thinking/text/tool_use in assistant order + for (const msg of t.messages) { + if (msg.type !== 'assistant') continue; + const content = msg.message?.content; + if (!Array.isArray(content)) continue; + for (const b of content) { + if (b.type === 'thinking') { + events.push({ kind: 'thinking', text: b.thinking }); + } else if (b.type === 'text') { + events.push({ kind: 'text', text: b.text }); + } else if (b.type === 'tool_use') { + const result = toolResults.get(b.id); + events.push({ + kind: 'tool', + name: b.name, + input: b.input, + output: result?.content, + isError: result?.isError || false, + id: b.id, + }); + } + } + } + } + + // Fallback: messages didn't give us tool calls — use toolCalls array. + const toolEventCount = events.filter((e) => e.kind === 'tool').length; + if (toolEventCount === 0 && Array.isArray(t.toolCalls)) { + for (const c of t.toolCalls) { + events.push({ + kind: 'tool', + name: c.name, + input: c.input, + output: c.output, + isError: c.isError || false, + }); + } + } + return events; + } + + function renderTrajectory() { + const t = state.runData.trajectory; + const events = buildEvents(t); + const list = $('#trajectory'); + + const hideToolSearch = $('#hide-toolsearch').checked; + const errorsOnly = $('#errors-only').checked; + const filter = $('#traj-filter').value.trim().toLowerCase(); + + list.innerHTML = ''; + let toolIdx = 0; + for (const e of events) { + let visible = true; + if (e.kind === 'tool') { + toolIdx++; + if (hideToolSearch && e.name === 'ToolSearch') visible = false; + if (errorsOnly && !e.isError) visible = false; + if (filter) { + const hay = ( + e.name + + '\n' + + formatToolPayload(e.input) + + '\n' + + formatToolPayload(e.output) + ).toLowerCase(); + if (!hay.includes(filter)) visible = false; + } + } else { + if (errorsOnly) visible = false; + if (filter && !e.text?.toLowerCase().includes(filter)) visible = false; + } + if (!visible) continue; + + if (e.kind === 'thinking') { + list.appendChild( + el('li', {}, el('div', { class: 'thinking' }, e.text)), + ); + } else if (e.kind === 'text') { + list.appendChild( + el('li', {}, el('div', { class: 'assistant-text' }, e.text)), + ); + } else if (e.kind === 'tool') { + list.appendChild(renderCall(toolIdx, e)); + } + } + } + + function renderCall(idx, c) { + const inputStr = formatToolPayload(c.input); + const outputStr = formatToolPayload(c.output); + const summary = + inputStr.length > 200 ? inputStr.slice(0, 200) + '…' : inputStr; + + const body = el( + 'div', + { class: 'body' }, + el('h4', {}, 'Input'), + el('pre', { class: 'code' }, inputStr || '(none)'), + el('h4', {}, c.isError ? 'Output (ERROR)' : 'Output'), + el( + 'pre', + { class: 'code' + (c.isError ? ' error' : '') }, + outputStr || '(none)', + ), + ); + + const li = el( + 'li', + { class: 'call' + (c.isError ? ' error' : '') }, + el( + 'div', + { + class: 'head', + onclick: (ev) => { + ev.currentTarget.parentElement.classList.toggle('open'); + }, + }, + el('span', { class: 'idx' }, `#${idx}`), + el('span', { class: 'name' + (c.isError ? ' error' : '') }, c.name), + el('span', { class: 'summary' }, summary.replace(/\s+/g, ' ')), + c.isError ? el('span', { class: 'tag error' }, 'ERROR') : null, + ), + body, + ); + return li; + } + + function renderAnswer() { + const t = state.runData.trajectory; + const pane = $('#tab-answer'); + pane.innerHTML = ''; + pane.appendChild( + el( + 'div', + { class: 'grade-section' }, + el('h3', {}, 'Agent Prompt'), + el('pre', {}, t.agentPrompt || '(none)'), + ), + ); + pane.appendChild( + el( + 'div', + { class: 'grade-section' }, + el('h3', {}, 'Final Answer'), + el('pre', {}, t.finalAnswer || '(no final answer)'), + ), + ); + if (t.systemPromptAppend) { + pane.appendChild( + el( + 'div', + { class: 'grade-section' }, + el('h3', {}, 'System Prompt Append'), + el('pre', {}, t.systemPromptAppend), + ), + ); + } + } + + function renderGrading() { + const g = state.runData.grade; + const pane = $('#tab-grading'); + pane.innerHTML = ''; + + if (!g) { + pane.appendChild(el('div', { class: 'muted' }, 'No grade file.')); + return; + } + + // Combined + pane.appendChild( + el( + 'div', + { class: 'grade-section' }, + el( + 'h3', + {}, + 'Combined Score', + el('span', { class: 'score' }, fmtScore(g.combinedScore)), + ), + el( + 'div', + { class: 'muted' }, + `Programmatic ${fmtScore(g.programmatic?.score)} · Judge ${fmtScore(g.judge?.weightedScore)} · Tool error penalty ${fmtScore(g.toolErrors?.penalty)}`, + ), + ), + ); + + // Programmatic checks + if (g.programmatic?.hits) { + const checks = el('div', { class: 'grade-section' }); + checks.appendChild( + el( + 'h3', + {}, + 'Programmatic checks', + el( + 'span', + { class: 'score' }, + fmtScore(g.programmatic.score), + ), + ), + ); + for (const h of g.programmatic.hits) { + const good = !!h.satisfied; + checks.appendChild( + el( + 'div', + { class: 'check' + (h.negative ? ' negative' : '') }, + el( + 'div', + { class: 'marker ' + (good ? 'good' : 'bad') }, + good ? '✓' : '✗', + ), + el('div', { class: 'id' }, h.id), + el( + 'div', + { class: 'weight' }, + `w${h.weight}${h.matched != null ? ` · matched=${h.matched}` : ''}`, + ), + ), + ); + } + checks.appendChild( + el( + 'div', + { class: 'muted small', style: 'margin-top:0.6rem; font-size:11px' }, + 'Negative checks score points when NOT matched — they punish false attribution to distractors.', + ), + ); + pane.appendChild(checks); + } + + // Judge scores + if (g.judge?.scores) { + const judge = el('div', { class: 'grade-section' }); + judge.appendChild( + el( + 'h3', + {}, + `LLM Judge (${g.judge.model || 'model?'})`, + el('span', { class: 'score' }, fmtScore(g.judge.weightedScore)), + ), + ); + for (const [name, val] of Object.entries(g.judge.scores)) { + judge.appendChild( + el( + 'div', + { class: 'criterion' }, + el( + 'div', + { class: 'criterion-head' }, + el('div', { class: 'criterion-name' }, name), + el( + 'div', + { class: 'criterion-score ' + criterionClass(val.score) }, + `${val.score} / 4`, + ), + ), + el('div', { class: 'rationale' }, val.rationale || ''), + ), + ); + } + pane.appendChild(judge); + } + + // Tool errors detail + if (g.toolErrors) { + const te = el('div', { class: 'grade-section' }); + te.appendChild( + el( + 'h3', + {}, + 'Tool errors', + el( + 'span', + { class: 'score' }, + `${g.toolErrors.errors}/${g.toolErrors.total} (penalty ${fmtScore(g.toolErrors.penalty)})`, + ), + ), + ); + if (g.toolErrors.samples?.length) { + te.appendChild( + el( + 'pre', + { class: 'code' }, + stringifyMaybe(g.toolErrors.samples), + ), + ); + } else { + te.appendChild(el('div', { class: 'muted' }, 'No error samples.')); + } + pane.appendChild(te); + } + } + + function renderRaw() { + const pane = $('#tab-raw'); + pane.innerHTML = ''; + pane.appendChild( + el( + 'div', + { class: 'grade-section' }, + el('h3', {}, 'trajectory.json'), + el('pre', {}, stringifyMaybe(state.runData.trajectory)), + ), + ); + pane.appendChild( + el( + 'div', + { class: 'grade-section' }, + el('h3', {}, 'grade.json'), + el('pre', {}, stringifyMaybe(state.runData.grade)), + ), + ); + } + + // ----- wiring ----- + + for (const btn of document.querySelectorAll('.tab')) { + btn.addEventListener('click', () => activateTab(btn.dataset.tab)); + } + for (const id of ['hide-toolsearch', 'errors-only']) { + $('#' + id).addEventListener('change', () => { + if (state.runData) renderTrajectory(); + }); + } + $('#traj-filter').addEventListener('input', () => { + if (state.runData) renderTrajectory(); + }); + + loadBatches().catch((e) => { + $('#empty').textContent = `Error loading batches: ${e.message}`; + }); +})(); diff --git a/packages/hdx-eval/scripts/viewer/public/index.html b/packages/hdx-eval/scripts/viewer/public/index.html new file mode 100644 index 0000000000..53f583e6e2 --- /dev/null +++ b/packages/hdx-eval/scripts/viewer/public/index.html @@ -0,0 +1,65 @@ + + + + + + HDX Eval Viewer + + + +
+
hdx-eval viewer
+
+ + + +
+
+ +
+ + +
+
+ Select a run from the sidebar. +
+ +
+
+ + + + diff --git a/packages/hdx-eval/scripts/viewer/public/style.css b/packages/hdx-eval/scripts/viewer/public/style.css new file mode 100644 index 0000000000..cb79aff72b --- /dev/null +++ b/packages/hdx-eval/scripts/viewer/public/style.css @@ -0,0 +1,461 @@ +:root { + --bg: #0f1419; + --panel: #161c23; + --panel-2: #1d242d; + --border: #2a323d; + --text: #d9e1ea; + --muted: #7a8896; + --accent: #7cc4ff; + --good: #6cd97e; + --bad: #ff6b6b; + --warn: #ffc857; + --code-bg: #0b0f14; +} + +* { + box-sizing: border-box; +} + +html, +body { + margin: 0; + padding: 0; + background: var(--bg); + color: var(--text); + font: 13px/1.45 -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, + sans-serif; + height: 100%; +} + +body { + display: flex; + flex-direction: column; +} + +header { + display: flex; + align-items: center; + gap: 1.2rem; + padding: 0.6rem 1rem; + background: var(--panel); + border-bottom: 1px solid var(--border); +} + +.title { + font-weight: 600; + letter-spacing: 0.02em; +} + +.batch-picker { + display: flex; + align-items: center; + gap: 0.5rem; +} + +select, +input[type="search"] { + background: var(--panel-2); + color: var(--text); + border: 1px solid var(--border); + border-radius: 4px; + padding: 0.25rem 0.5rem; + font: inherit; +} + +select { + min-width: 280px; +} + +main { + display: flex; + flex: 1; + min-height: 0; +} + +#sidebar { + width: 280px; + border-right: 1px solid var(--border); + background: var(--panel); + padding: 0.6rem; + overflow-y: auto; +} + +#sidebar .small { + margin-bottom: 0.4rem; + text-transform: uppercase; + letter-spacing: 0.06em; + font-size: 11px; +} + +#content { + flex: 1; + min-width: 0; + display: flex; + flex-direction: column; + overflow: hidden; +} + +.muted { + color: var(--muted); +} + +.center { + margin: auto; + padding: 2rem; +} + +#run { + display: flex; + flex-direction: column; + flex: 1; + min-height: 0; +} + +#run-header { + padding: 0.6rem 1rem; + border-bottom: 1px solid var(--border); + background: var(--panel); + display: grid; + grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); + gap: 0.5rem 1rem; +} + +#run-header .label { + color: var(--muted); + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.06em; +} + +#run-header .value { + font-weight: 500; +} + +.tabs { + display: flex; + gap: 0; + border-bottom: 1px solid var(--border); + background: var(--panel); +} + +.tab { + background: transparent; + color: var(--muted); + border: 0; + border-bottom: 2px solid transparent; + padding: 0.5rem 1rem; + cursor: pointer; + font: inherit; +} + +.tab.active { + color: var(--text); + border-bottom-color: var(--accent); +} + +.tab-pane { + display: none; + flex: 1; + overflow: auto; + padding: 0.8rem 1rem; +} + +.tab-pane.active { + display: block; +} + +/* sidebar nav */ +.nav-scenario { + margin-top: 0.4rem; +} +.nav-scenario > .head { + font-weight: 600; + padding: 0.25rem 0; + color: var(--text); +} +.nav-mcp { + margin-left: 0.6rem; +} +.nav-mcp > .head { + color: var(--accent); + padding: 0.2rem 0; +} +.nav-run { + margin-left: 1.4rem; + display: flex; + justify-content: space-between; + align-items: center; + gap: 0.4rem; + padding: 0.2rem 0.4rem; + border-radius: 3px; + cursor: pointer; + user-select: none; +} +.nav-run:hover { + background: var(--panel-2); +} +.nav-run.active { + background: var(--panel-2); + outline: 1px solid var(--accent); +} +.score-chip { + font-variant-numeric: tabular-nums; + padding: 0 0.4rem; + border-radius: 10px; + background: var(--panel-2); + border: 1px solid var(--border); + font-size: 11px; +} +.score-chip.good { + color: var(--good); + border-color: var(--good); +} +.score-chip.bad { + color: var(--bad); + border-color: var(--bad); +} +.score-chip.warn { + color: var(--warn); + border-color: var(--warn); +} + +/* trajectory */ +#traj-controls { + display: flex; + gap: 1rem; + align-items: center; + margin-bottom: 0.6rem; + font-size: 12px; +} +#traj-controls input[type="search"] { + flex: 1; + max-width: 320px; +} + +#trajectory { + list-style: none; + padding: 0; + margin: 0; +} + +.call { + background: var(--panel); + border: 1px solid var(--border); + border-radius: 4px; + margin-bottom: 0.4rem; +} + +.call.error { + border-color: var(--bad); +} + +.call > .head { + display: flex; + align-items: center; + gap: 0.6rem; + padding: 0.4rem 0.6rem; + cursor: pointer; + user-select: none; +} + +.call > .head .idx { + color: var(--muted); + font-variant-numeric: tabular-nums; + min-width: 2.5em; +} + +.call > .head .name { + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, monospace; + font-size: 12px; +} + +.call > .head .name.error { + color: var(--bad); +} + +.call > .head .summary { + color: var(--muted); + font-size: 12px; + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, monospace; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + flex: 1; +} + +.call > .body { + display: none; + padding: 0 0.6rem 0.6rem 0.6rem; + border-top: 1px solid var(--border); +} + +.call.open > .body { + display: block; +} + +.call h4 { + margin: 0.6rem 0 0.2rem; + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.06em; + color: var(--muted); + font-weight: 600; +} + +pre.code { + background: var(--code-bg); + border: 1px solid var(--border); + border-radius: 3px; + padding: 0.5rem 0.7rem; + margin: 0; + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, monospace; + font-size: 12px; + white-space: pre-wrap; + word-break: break-word; + max-height: 400px; + overflow: auto; +} + +pre.code.error { + border-color: var(--bad); +} + +.thinking { + background: var(--panel-2); + border-left: 3px solid var(--accent); + padding: 0.5rem 0.8rem; + margin: 0.4rem 0; + font-style: italic; + color: var(--muted); + white-space: pre-wrap; +} + +.assistant-text { + background: var(--panel-2); + border-left: 3px solid var(--good); + padding: 0.5rem 0.8rem; + margin: 0.4rem 0; + white-space: pre-wrap; +} + +/* answer/grading */ +#tab-answer pre { + background: var(--code-bg); + padding: 1rem; + border-radius: 4px; + white-space: pre-wrap; + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, monospace; + font-size: 12.5px; +} + +.grade-section { + background: var(--panel); + border: 1px solid var(--border); + border-radius: 4px; + padding: 0.8rem 1rem; + margin-bottom: 0.8rem; +} +.grade-section h3 { + margin: 0 0 0.5rem 0; + font-size: 13px; + display: flex; + justify-content: space-between; + align-items: baseline; +} +.grade-section h3 .score { + font-variant-numeric: tabular-nums; + font-weight: 600; +} + +.check { + display: grid; + grid-template-columns: 1.4rem 1fr auto; + gap: 0.4rem; + padding: 0.25rem 0; + border-top: 1px solid var(--border); + align-items: center; +} +.check:first-of-type { + border-top: 0; +} +.check .marker.good { + color: var(--good); +} +.check .marker.bad { + color: var(--bad); +} +.check .id { + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, monospace; + font-size: 12px; +} +.check .weight { + color: var(--muted); + font-size: 11px; +} +.check.negative .id::after { + content: " (negative)"; + color: var(--warn); + font-size: 10px; +} + +.criterion { + border-top: 1px solid var(--border); + padding: 0.5rem 0; +} +.criterion:first-of-type { + border-top: 0; +} +.criterion-head { + display: flex; + justify-content: space-between; + align-items: baseline; +} +.criterion-name { + font-weight: 600; +} +.criterion-score { + font-variant-numeric: tabular-nums; +} +.criterion .rationale { + color: var(--muted); + margin-top: 0.2rem; + font-size: 12.5px; +} + +/* color the criterion score */ +.score-0 { + color: var(--bad); +} +.score-low { + color: var(--warn); +} +.score-mid { + color: var(--accent); +} +.score-high { + color: var(--good); +} + +#tab-raw pre { + background: var(--code-bg); + padding: 1rem; + border-radius: 4px; + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, monospace; + font-size: 11.5px; + white-space: pre-wrap; + word-break: break-word; +} + +.tag { + display: inline-block; + background: var(--panel-2); + border: 1px solid var(--border); + border-radius: 3px; + padding: 0 0.4rem; + font-size: 11px; + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, monospace; +} +.tag.error { + color: var(--bad); + border-color: var(--bad); +} diff --git a/packages/hdx-eval/scripts/viewer/server.js b/packages/hdx-eval/scripts/viewer/server.js new file mode 100644 index 0000000000..2395006efd --- /dev/null +++ b/packages/hdx-eval/scripts/viewer/server.js @@ -0,0 +1,183 @@ +#!/usr/bin/env node +// HDX Eval Run Viewer — tiny stdlib HTTP server. +// Browses runs/ produced by hdx-eval: batches → scenarios → cells → run files, +// serving trajectory + grading JSON to a single-page UI. + +const http = require('node:http'); +const fs = require('node:fs'); +const path = require('node:path'); +const url = require('node:url'); + +const RUNS_DIR = path.resolve(__dirname, '..', '..', 'runs'); +const PUBLIC_DIR = path.join(__dirname, 'public'); +const PORT = Number(process.env.PORT) || 5176; + +const MIME = { + '.html': 'text/html; charset=utf-8', + '.js': 'application/javascript; charset=utf-8', + '.css': 'text/css; charset=utf-8', + '.json': 'application/json; charset=utf-8', + '.svg': 'image/svg+xml', +}; + +function sendJson(res, status, body) { + const buf = Buffer.from(JSON.stringify(body)); + res.writeHead(status, { + 'content-type': 'application/json; charset=utf-8', + 'content-length': buf.length, + 'cache-control': 'no-store', + }); + res.end(buf); +} + +function sendText(res, status, text, type = 'text/plain; charset=utf-8') { + const buf = Buffer.from(text); + res.writeHead(status, { 'content-type': type, 'content-length': buf.length }); + res.end(buf); +} + +function safeJoin(root, ...parts) { + const p = path.resolve(root, ...parts); + if (!p.startsWith(path.resolve(root))) throw new Error('path escapes root'); + return p; +} + +function readJsonSafe(p) { + try { + return JSON.parse(fs.readFileSync(p, 'utf8')); + } catch { + return null; + } +} + +function listBatches() { + if (!fs.existsSync(RUNS_DIR)) return []; + return fs + .readdirSync(RUNS_DIR, { withFileTypes: true }) + .filter((d) => d.isDirectory()) + .map((d) => d.name) + .sort() + .reverse(); +} + +function listCells(batch) { + const batchDir = safeJoin(RUNS_DIR, batch); + if (!fs.existsSync(batchDir)) return null; + const scenarios = fs + .readdirSync(batchDir, { withFileTypes: true }) + .filter((d) => d.isDirectory()) + .map((d) => d.name); + const out = []; + for (const scenario of scenarios) { + const scenarioDir = path.join(batchDir, scenario); + const mcps = fs + .readdirSync(scenarioDir, { withFileTypes: true }) + .filter((d) => d.isDirectory()) + .map((d) => d.name); + for (const mcp of mcps) { + const cellDir = path.join(scenarioDir, mcp); + const files = fs.readdirSync(cellDir); + const runIdxs = new Set(); + for (const f of files) { + const m = /^(\d+)\.json$/.exec(f); + if (m) runIdxs.add(Number(m[1])); + } + const runs = [...runIdxs].sort((a, b) => a - b).map((i) => { + const grade = readJsonSafe(path.join(cellDir, `${i}.grade.json`)); + const traj = readJsonSafe(path.join(cellDir, `${i}.json`)); + return { + idx: i, + combinedScore: grade?.combinedScore ?? null, + programmaticScore: grade?.programmatic?.score ?? null, + judgeScore: grade?.judge?.weightedScore ?? null, + termination: traj?.termination ?? null, + durationMs: traj?.durationMs ?? null, + toolCalls: traj?.toolCalls?.length ?? null, + toolErrors: grade?.toolErrors?.errors ?? null, + }; + }); + out.push({ scenario, mcp, runs }); + } + } + return out; +} + +const ROUTES = [ + [ + /^\/api\/batches$/, + (_m, _q, res) => { + const batches = listBatches().map((name) => { + const summary = readJsonSafe(path.join(RUNS_DIR, name, '_summary.json')); + return { + name, + generatedAt: summary?.generatedAt ?? null, + scenarios: summary?.scenarios?.map((s) => s.scenario) ?? null, + }; + }); + sendJson(res, 200, { batches }); + }, + ], + [ + /^\/api\/batches\/([^/]+)$/, + (m, _q, res) => { + const batch = decodeURIComponent(m[1]); + const summary = readJsonSafe(path.join(RUNS_DIR, batch, '_summary.json')); + const cells = listCells(batch); + if (!cells) return sendJson(res, 404, { error: 'batch not found' }); + sendJson(res, 200, { batch, summary, cells }); + }, + ], + [ + /^\/api\/batches\/([^/]+)\/runs\/([^/]+)\/([^/]+)\/(\d+)$/, + (m, _q, res) => { + const [, batch, scenario, mcp, idx] = m.map((x, i) => + i === 0 ? x : decodeURIComponent(x), + ); + const base = safeJoin(RUNS_DIR, batch, scenario, mcp); + const trajectory = readJsonSafe(path.join(base, `${idx}.json`)); + const grade = readJsonSafe(path.join(base, `${idx}.grade.json`)); + if (!trajectory) return sendJson(res, 404, { error: 'run not found' }); + sendJson(res, 200, { trajectory, grade }); + }, + ], +]; + +function handleApi(req, res) { + const parsed = url.parse(req.url, true); + for (const [re, fn] of ROUTES) { + const m = re.exec(parsed.pathname); + if (m) return fn(m, parsed.query, res); + } + sendJson(res, 404, { error: 'not found' }); +} + +function handleStatic(req, res) { + let pathname = url.parse(req.url).pathname; + if (pathname === '/') pathname = '/index.html'; + let filePath; + try { + filePath = safeJoin(PUBLIC_DIR, '.' + pathname); + } catch { + return sendText(res, 400, 'bad path'); + } + fs.readFile(filePath, (err, data) => { + if (err) return sendText(res, 404, 'not found'); + const ext = path.extname(filePath).toLowerCase(); + res.writeHead(200, { + 'content-type': MIME[ext] || 'application/octet-stream', + 'content-length': data.length, + 'cache-control': 'no-store', + }); + res.end(data); + }); +} + +const server = http.createServer((req, res) => { + if (req.url.startsWith('/api/')) return handleApi(req, res); + handleStatic(req, res); +}); + +server.listen(PORT, () => { + console.log(`hdx-eval viewer: http://localhost:${PORT}`); + console.log(` runs dir: ${RUNS_DIR}`); +}); diff --git a/packages/hdx-eval/src/__tests__/aggregate.test.ts b/packages/hdx-eval/src/__tests__/aggregate.test.ts new file mode 100644 index 0000000000..180a6238b6 --- /dev/null +++ b/packages/hdx-eval/src/__tests__/aggregate.test.ts @@ -0,0 +1,179 @@ +import type { GradeRecord } from '../grading/types'; +import type { RunRecord } from '../harness/types'; +import { buildAggregate, type GradedRunPair } from '../reports/aggregate'; + +function pair(args: { + scenario: string; + mcp: 'hyperdx' | 'clickhouse'; + i: number; + programmaticScore: number; + judgeScore: number; + toolCalls: number; + outputTokens: number; + durationMs: number; +}): GradedRunPair { + const run: RunRecord = { + schemaVersion: 1, + runId: `${args.scenario}-${args.mcp}-${args.i}`, + scenario: args.scenario, + mcp: args.mcp, + model: 'claude-sonnet-4-6', + runIndex: args.i, + seed: 42, + startedAt: '2026-05-09T07:50:00.000Z', + endedAt: '2026-05-09T07:51:00.000Z', + durationMs: args.durationMs, + agentPrompt: 'p', + systemPromptAppend: 's', + termination: 'final_answer', + exitCode: 0, + tools: [], + toolCalls: Array.from({ length: args.toolCalls }, (_, k) => ({ + name: `t${k}`, + input: null, + output: null, + isError: false, + startedAt: '', + endedAt: null, + durationMs: null, + })), + messages: [], + finalAnswer: 'a', + tokens: { + input: 100, + output: args.outputTokens, + cacheCreation: 0, + cacheRead: 0, + }, + totalCostUsd: 0.01, + stderr: '', + }; + const grade: GradeRecord = { + schemaVersion: 2, + runId: run.runId, + scenario: run.scenario, + mcp: run.mcp, + programmatic: { + hits: [ + { + id: 'check-a', + weight: 1, + matched: args.programmaticScore >= 0.5, + satisfied: args.programmaticScore >= 0.5, + }, + { + id: 'check-b', + weight: 1, + matched: args.programmaticScore >= 1, + satisfied: args.programmaticScore >= 1, + }, + ], + score: args.programmaticScore, + }, + judge: { + model: 'claude-opus-4-7', + scores: { + correctness: { score: Math.round(args.judgeScore * 5), rationale: '' }, + }, + weightedScore: args.judgeScore, + rawResponse: '', + durationMs: 1000, + tokens: { input: 0, output: 0, cacheCreation: 0, cacheRead: 0 }, + }, + toolErrors: { + total: args.toolCalls, + errors: 0, + rate: 0, + penalty: 0, + samples: [], + }, + combinedScore: 0.4 * args.programmaticScore + 0.6 * args.judgeScore, + gradedAt: '', + judgeModel: 'claude-opus-4-7', + }; + return { run, grade }; +} + +describe('buildAggregate', () => { + it('groups by scenario and computes per-cell means', () => { + const pairs: GradedRunPair[] = [ + pair({ + scenario: 'error-root-cause', + mcp: 'hyperdx', + i: 0, + programmaticScore: 1, + judgeScore: 0.9, + toolCalls: 13, + outputTokens: 4000, + durationMs: 60_000, + }), + pair({ + scenario: 'error-root-cause', + mcp: 'hyperdx', + i: 1, + programmaticScore: 0.8, + judgeScore: 0.8, + toolCalls: 11, + outputTokens: 3500, + durationMs: 50_000, + }), + pair({ + scenario: 'error-root-cause', + mcp: 'clickhouse', + i: 0, + programmaticScore: 1, + judgeScore: 1, + toolCalls: 7, + outputTokens: 1500, + durationMs: 30_000, + }), + ]; + const summary = buildAggregate({ batchDir: '/tmp/x', pairs }); + expect(summary.scenarios).toHaveLength(1); + const sc = summary.scenarios[0]; + const h = sc.cells.hyperdx!; + const c = sc.cells.clickhouse!; + expect(h.n).toBe(2); + expect(c.n).toBe(1); + expect(h.toolCalls.mean).toBeCloseTo(12, 5); + expect(c.toolCalls.mean).toBeCloseTo(7, 5); + expect(h.programmatic.mean).toBeCloseTo(0.9, 5); + expect(h.judge.weightedMean).toBeCloseTo(0.85, 5); + // combinedScore is mean of per-run combined values, not combined of means. + // Run 0: 0.4*1 + 0.6*0.9 = 0.94. Run 1: 0.4*0.8 + 0.6*0.8 = 0.8. Mean = 0.87. + expect(h.combinedScore.mean).toBeCloseTo(0.87, 5); + expect(sc.delta.toolCalls).toBeCloseTo(5, 5); + // HDX combined 0.87 vs CH 1.0 → -0.13 + expect(sc.delta.combinedScore!).toBeCloseTo(-0.13, 5); + }); + + it('produces an ordered, multi-scenario summary', () => { + const pairs: GradedRunPair[] = [ + pair({ + scenario: 'noisy-signals', + mcp: 'hyperdx', + i: 0, + programmaticScore: 1, + judgeScore: 1, + toolCalls: 9, + outputTokens: 3200, + durationMs: 50_000, + }), + pair({ + scenario: 'error-root-cause', + mcp: 'hyperdx', + i: 0, + programmaticScore: 1, + judgeScore: 1, + toolCalls: 13, + outputTokens: 4000, + durationMs: 60_000, + }), + ]; + const summary = buildAggregate({ batchDir: '/tmp/x', pairs }); + expect(summary.scenarios.map(s => s.scenario)).toEqual([ + 'error-root-cause', + 'noisy-signals', + ]); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/blind.test.ts b/packages/hdx-eval/src/__tests__/blind.test.ts new file mode 100644 index 0000000000..e4eb7d812d --- /dev/null +++ b/packages/hdx-eval/src/__tests__/blind.test.ts @@ -0,0 +1,50 @@ +import { blindAnswer } from '../grading/blind'; + +describe('blindAnswer', () => { + it('redacts mcp__hyperdx__ tool prefix', () => { + const out = blindAnswer('I called mcp__hyperdx__hyperdx_query to find...'); + expect(out).not.toContain('mcp__hyperdx__'); + expect(out).toContain('mcp__redacted__'); + }); + + it('redacts mcp__clickhouse__ tool prefix', () => { + const out = blindAnswer('Used mcp__clickhouse__run_query to look up...'); + expect(out).not.toContain('mcp__clickhouse__'); + expect(out).toContain('mcp__redacted__'); + }); + + it('replaces specific HyperDX tool names with a generic placeholder', () => { + const out = blindAnswer( + 'I issued hyperdx_query and used hyperdx_list_sources to discover sources.', + ); + expect(out).not.toMatch(/hyperdx_query|hyperdx_list_sources/); + expect(out).toContain('mcp_query'); + }); + + it('replaces specific ClickHouse tool names with a generic placeholder', () => { + const out = blindAnswer( + 'Ran run_query, list_tables, and list_databases to introspect.', + ); + expect(out).not.toMatch(/run_query|list_tables|list_databases/); + expect(out).toContain('mcp_query'); + }); + + it('redacts brand mentions HyperDX MCP and ClickHouse MCP', () => { + const a = blindAnswer('Using HyperDX MCP I confirmed...'); + const b = blindAnswer('Using ClickHouse MCP I confirmed...'); + expect(a).toContain('MCP A'); + expect(b).toContain('MCP B'); + expect(a).not.toContain('HyperDX MCP'); + expect(b).not.toContain('ClickHouse MCP'); + }); + + it('preserves data references that are part of the answer', () => { + const answer = + 'Root cause: payment-service db.payment.connect timeout. ' + + 'Affected users hit checkout-api 5xx.'; + const out = blindAnswer(answer); + expect(out).toContain('payment-service'); + expect(out).toContain('db.payment.connect'); + expect(out).toContain('checkout-api'); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/error-root-cause.test.ts b/packages/hdx-eval/src/__tests__/error-root-cause.test.ts new file mode 100644 index 0000000000..2f107fb29b --- /dev/null +++ b/packages/hdx-eval/src/__tests__/error-root-cause.test.ts @@ -0,0 +1,228 @@ +import { mulberry32 } from '../rng/seeded'; +import { errorRootCauseScenario } from '../scenarios/error-root-cause/generate'; +import { collectScenario } from '../scenarios/types'; + +const NOW_MS = Date.parse('2026-05-08T12:00:00.000Z'); +// Tests run at 1% volume (full v5 ≈ 6M base traces / 12M logs). +// Pattern shares scale linearly; planted anomaly + distractors stay fixed. +const TEST_VOLUME_FACTOR = 0.01; + +function run(seed: number) { + return collectScenario( + errorRootCauseScenario.generate({ + rng: mulberry32(seed), + nowMs: NOW_MS, + volumeFactor: TEST_VOLUME_FACTOR, + }), + ); +} + +describe('error-root-cause scenario', () => { + // Generate once per run() to avoid 5x cost across tests. + const result = run(42); + + it('produces high-volume background + distractors + anomaly at 1%', () => { + // Full v5: 6M base × ~1.6 spans + distractors → ~9.6M+ spans. + // At 1%: 60K × ~1.6 + ~465 distractors → ~96K traces. + expect(result.traces.length).toBeGreaterThan(90_000); + expect(result.traces.length).toBeLessThan(110_000); + }); + + it('produces ~120K templated background logs at 1% (12M at full scale)', () => { + expect(result.logs.length).toBeGreaterThanOrEqual(100_000); + expect(result.logs.length).toBeLessThanOrEqual(140_000); + }); + + it('uses 25 services (high cardinality)', () => { + const services = new Set(result.traces.map(t => t.serviceName)); + expect(services.size).toBeGreaterThanOrEqual(20); + }); + + it('attaches region and tenant.id attributes to background traces', () => { + const sample = result.traces.slice(0, 1000); + const withRegion = sample.filter(t => t.spanAttributes['cloud.region']); + expect(withRegion.length).toBeGreaterThan(500); + const regions = new Set( + sample.map(t => t.spanAttributes['cloud.region']).filter(Boolean), + ); + expect(regions.size).toBeGreaterThanOrEqual(3); + }); + + it('is deterministic for a fixed seed and now', () => { + const b = run(42); + expect(b.traces.length).toBe(result.traces.length); + expect(b.logs.length).toBe(result.logs.length); + // Cheap structural check — sampling spans by index for byte-equality is too slow at 100x volume. + expect(result.traces[0].spanId).toBe(b.traces[0].spanId); + expect(result.traces[1000].spanId).toBe(b.traces[1000].spanId); + expect(result.logs[5000].body).toBe(b.logs[5000].body); + }); + + it('plants exactly 8 payment-service db.payment.connect error spans in the anomaly window (small relative to decoy bursts)', () => { + const anomalyStart = NOW_MS - 5 * 60 * 1000; + const dbErrors = result.traces.filter( + t => + t.serviceName === 'payment-service' && + t.spanName === 'db.payment.connect' && + t.statusCode === 'STATUS_CODE_ERROR' && + t.timestampMs >= anomalyStart, + ); + expect(dbErrors.length).toBe(8); + }); + + it('uses 3 timeout message variants on the planted db.payment.connect spans', () => { + const anomalyStart = NOW_MS - 5 * 60 * 1000; + const dbErrors = result.traces.filter( + t => + t.serviceName === 'payment-service' && + t.spanName === 'db.payment.connect' && + t.statusCode === 'STATUS_CODE_ERROR' && + t.timestampMs >= anomalyStart, + ); + const variants = new Set(dbErrors.map(t => t.statusMessage)); + expect(variants.size).toBe(3); + expect( + [...variants].every(v => v.toLowerCase().includes('db-payment')), + ).toBe(true); + }); + + it('propagates the error up to checkout-api root spans in the anomaly window', () => { + const anomalyStart = NOW_MS - 5 * 60 * 1000; + const checkoutErrors = result.traces.filter( + t => + t.serviceName === 'checkout-api' && + t.spanName === 'POST /api/checkout' && + t.statusCode === 'STATUS_CODE_ERROR' && + t.timestampMs >= anomalyStart, + ); + expect(checkoutErrors.length).toBe(8); + }); + + it('emits correlated payment-service error logs with the same TraceId', () => { + const erroringTraceIds = new Set( + result.traces + .filter( + t => + t.serviceName === 'payment-service' && + t.statusCode === 'STATUS_CODE_ERROR', + ) + .map(t => t.traceId), + ); + const correlated = result.logs.filter( + l => + l.serviceName === 'payment-service' && + l.severityText === 'ERROR' && + l.body.includes('db-payment') && + l.traceId && + erroringTraceIds.has(l.traceId), + ); + expect(correlated.length).toBe(8); + }); + + it('plants the api-gateway TLS distractor at T-25min (not in current incident window)', () => { + const distractorStart = NOW_MS - 25 * 60 * 1000; + const distractorEnd = distractorStart + 5 * 60 * 1000; + const tls = result.traces.filter( + t => + t.serviceName === 'api-gateway' && + t.spanAttributes['error.type'] === 'TLSHandshakeError' && + t.timestampMs >= distractorStart && + t.timestampMs <= distractorEnd, + ); + expect(tls.length).toBeGreaterThanOrEqual(40); + // Should NOT be in the current incident window (last 10 min) + const recent = tls.filter(t => t.timestampMs >= NOW_MS - 10 * 60 * 1000); + expect(recent.length).toBe(0); + }); + + it('plants the auth-service rate-limit distractor at T-40min (resolved before incident)', () => { + const auth = result.traces.filter( + t => + t.serviceName === 'auth-service' && + t.spanAttributes['error.type'] === 'RateLimitExceeded', + ); + expect(auth.length).toBeGreaterThanOrEqual(15); + const recent = auth.filter(t => t.timestampMs >= NOW_MS - 35 * 60 * 1000); + expect(recent.length).toBe(0); + }); + + it('plants slow-but-OK search-service queries throughout the hour (not errors)', () => { + const slowSearch = result.traces.filter( + t => + t.serviceName === 'search-service' && + t.spanName === 'search.query' && + t.statusCode === 'STATUS_CODE_OK', + ); + expect(slowSearch.length).toBeGreaterThanOrEqual(70); + }); + + it('keeps payment-service free of DB-cascade errors outside the anomaly window', () => { + const anomalyStart = NOW_MS - 5 * 60 * 1000; + const preAnomalyCascade = result.traces.filter( + t => + t.serviceName === 'payment-service' && + t.spanName === 'db.payment.connect' && + t.timestampMs < anomalyStart, + ); + expect(preAnomalyCascade.length).toBe(0); + }); + + it('plants 200 background payment.charge declines distributed across the hour (operational, not infra)', () => { + const declines = result.traces.filter( + t => + t.serviceName === 'payment-service' && + t.spanName === 'payment.charge' && + t.spanAttributes['error.type'] === 'PaymentDeclined', + ); + expect(declines.length).toBe(200); + const reasons = new Set(declines.map(t => t.statusMessage)); + expect(reasons.size).toBe(5); + }); + + it('plants 80 concurrent cdn origin-fetch errors in last 5 min, separate from checkout traces (dominates raw 5xx counts)', () => { + const anomalyStart = NOW_MS - 5 * 60 * 1000; + const cdn = result.traces.filter( + t => + t.serviceName === 'cdn' && + t.spanAttributes['error.type'] === 'OriginUnreachableError' && + t.timestampMs >= anomalyStart, + ); + expect(cdn.length).toBe(80); + const checkoutCascadeTraceIds = new Set( + result.traces + .filter( + t => + t.serviceName === 'checkout-api' && + t.statusCode === 'STATUS_CODE_ERROR' && + t.timestampMs >= anomalyStart, + ) + .map(t => t.traceId), + ); + const overlap = cdn.filter(t => checkoutCascadeTraceIds.has(t.traceId)); + expect(overlap.length).toBe(0); + }); + + it('plants 25 concurrent notification-service smtp errors in last 5 min, separate from checkout traces', () => { + const anomalyStart = NOW_MS - 5 * 60 * 1000; + const smtp = result.traces.filter( + t => + t.serviceName === 'notification-service' && + t.spanAttributes['error.type'] === 'SMTPConnectionError' && + t.timestampMs >= anomalyStart, + ); + expect(smtp.length).toBe(25); + // None of these should share traceids with the planted checkout cascade. + const checkoutCascadeTraceIds = new Set( + result.traces + .filter( + t => + t.serviceName === 'checkout-api' && + t.statusCode === 'STATUS_CODE_ERROR' && + t.timestampMs >= anomalyStart, + ) + .map(t => t.traceId), + ); + const overlap = smtp.filter(t => checkoutCascadeTraceIds.has(t.traceId)); + expect(overlap.length).toBe(0); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/grade.test.ts b/packages/hdx-eval/src/__tests__/grade.test.ts new file mode 100644 index 0000000000..762d58c340 --- /dev/null +++ b/packages/hdx-eval/src/__tests__/grade.test.ts @@ -0,0 +1,169 @@ +/** + * Tests for the tool-error penalty applied by gradeBatch. We can't easily + * test the full pipeline (it touches the LLM judge), so instead these tests + * cover the pure computeToolErrorStats helper by re-implementing the call + * pattern: build a synthetic RunRecord, write it to a temp batch dir, run + * gradeBatch with --no-judge, and read back the grade JSON. + */ + +import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'fs'; +import { join } from 'path'; + +import { gradeBatch } from '../grading/grade'; +import type { GradeRecord } from '../grading/types'; +import type { RunRecord } from '../harness/types'; + +function buildRun(args: { + scenario: string; + toolCalls: Array<{ name: string; isError: boolean; output?: string }>; + finalAnswer: string; +}): RunRecord { + return { + schemaVersion: 1, + runId: `${args.scenario}-test-0`, + scenario: args.scenario, + mcp: 'hyperdx', + model: 'claude-sonnet-4-6', + runIndex: 0, + seed: 42, + startedAt: '2026-05-10T00:00:00.000Z', + endedAt: '2026-05-10T00:01:00.000Z', + durationMs: 60_000, + agentPrompt: 'p', + systemPromptAppend: 's', + termination: 'final_answer', + exitCode: 0, + tools: [], + toolCalls: args.toolCalls.map(c => ({ + name: c.name, + input: null, + output: c.output ?? null, + isError: c.isError, + startedAt: '', + endedAt: null, + durationMs: null, + })), + messages: [], + finalAnswer: args.finalAnswer, + tokens: { input: 0, output: 0, cacheCreation: 0, cacheRead: 0 }, + totalCostUsd: 0, + stderr: '', + }; +} + +function writeBatch( + batchDir: string, + scenario: string, + mcp: 'hyperdx' | 'clickhouse', + run: RunRecord, +): void { + const dir = join(batchDir, scenario, mcp); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, '0.json'), JSON.stringify(run, null, 2)); +} + +describe('gradeBatch tool-error penalty', () => { + const tmpRoot = join('/tmp', `hdx-eval-grade-test-${Date.now()}`); + // Answer that hits every positive check in the error-root-cause rubric: + // service, error.type, full db hostname, specific db span, cascade name, + // ruled-out distractor — without triggering any negative blame-pattern. + const ANSWER = + 'Root cause: payment-service ConnectionTimeoutError on db.payment.connect ' + + 'reaching db-payment.internal — DB connection timeout — cascading into ' + + 'checkout-api 5xx errors. Ruled out concurrent SMTP and CDN bursts ' + + '(separate trace trees, no checkout parent).'; + + afterAll(() => { + rmSync(tmpRoot, { recursive: true, force: true }); + }); + + it('records zero penalty when all tool calls succeeded', async () => { + const batchDir = join(tmpRoot, 'ok'); + writeBatch( + batchDir, + 'error-root-cause', + 'hyperdx', + buildRun({ + scenario: 'error-root-cause', + toolCalls: [ + { name: 'mcp__hyperdx__hyperdx_list_sources', isError: false }, + { name: 'mcp__hyperdx__hyperdx_query', isError: false }, + ], + finalAnswer: ANSWER, + }), + ); + const result = await gradeBatch(batchDir, { skipJudge: true }); + expect(result.graded).toHaveLength(1); + const grade = result.graded[0]; + expect(grade.toolErrors.total).toBe(2); + expect(grade.toolErrors.errors).toBe(0); + expect(grade.toolErrors.rate).toBe(0); + expect(grade.toolErrors.penalty).toBe(0); + }); + + it('penalizes runs where tool calls failed', async () => { + const batchDir = join(tmpRoot, 'failed'); + writeBatch( + batchDir, + 'error-root-cause', + 'hyperdx', + buildRun({ + scenario: 'error-root-cause', + toolCalls: [ + { + name: 'mcp__hyperdx__hyperdx_list_sources', + isError: true, + output: 'Maximum call stack size exceeded', + }, + { + name: 'mcp__hyperdx__hyperdx_list_sources', + isError: true, + output: 'Maximum call stack size exceeded', + }, + { + name: 'mcp__hyperdx__hyperdx_query', + isError: true, + output: 'sourceId must be a 24-char hex ObjectId', + }, + { name: 'mcp__hyperdx__hyperdx_query', isError: false }, + ], + finalAnswer: ANSWER, + }), + ); + const result = await gradeBatch(batchDir, { skipJudge: true }); + const grade = result.graded[0]; + expect(grade.toolErrors.total).toBe(4); + expect(grade.toolErrors.errors).toBe(3); + expect(grade.toolErrors.rate).toBeCloseTo(0.75, 5); + // Penalty is capped at MAX_ERROR_PENALTY (0.2) + expect(grade.toolErrors.penalty).toBeCloseTo(0.2, 5); + expect(grade.toolErrors.samples).toHaveLength(3); + expect(grade.toolErrors.samples[0].name).toMatch(/list_sources/); + }); + + it('subtracts the penalty from combinedScore (and clamps to [0,1])', async () => { + const batchDir = join(tmpRoot, 'clamped'); + // Programmatic rubric will hit, but every tool call fails. + writeBatch( + batchDir, + 'error-root-cause', + 'hyperdx', + buildRun({ + scenario: 'error-root-cause', + toolCalls: Array.from({ length: 5 }, () => ({ + name: 'mcp__hyperdx__hyperdx_list_sources', + isError: true, + output: 'fail', + })), + finalAnswer: ANSWER, + }), + ); + const result = await gradeBatch(batchDir, { skipJudge: true }); + const grade = result.graded[0]; + // Programmatic score = 1.0 (answer hits every positive check). Without + // judge, raw combined is programmatic.score. Penalty is 0.2. Final: + // 1.0 - 0.2 = 0.8. + expect(grade.programmatic.score).toBeCloseTo(1, 5); + expect(grade.combinedScore).toBeCloseTo(0.8, 5); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/judgePrompt.test.ts b/packages/hdx-eval/src/__tests__/judgePrompt.test.ts new file mode 100644 index 0000000000..2012176aa1 --- /dev/null +++ b/packages/hdx-eval/src/__tests__/judgePrompt.test.ts @@ -0,0 +1,62 @@ +import { + buildJudgeSystem, + buildJudgeUser, + formatGroundTruthFacts, +} from '../grading/judgePrompt'; +import { loadScenarioRubric } from '../grading/rubric'; +import { getScenario } from '../scenarios'; + +describe('buildJudgeSystem', () => { + it('includes the scenario name and every criterion in the rubric', () => { + const rubric = loadScenarioRubric('error-root-cause'); + const sys = buildJudgeSystem('error-root-cause', rubric); + expect(sys).toContain('SCENARIO: error-root-cause'); + for (const c of rubric.judge.criteria) { + expect(sys).toContain(`"${c.id}"`); + expect(sys).toContain(c.description); + } + }); + + it('instructs strict JSON output', () => { + const rubric = loadScenarioRubric('latency-spike'); + const sys = buildJudgeSystem('latency-spike', rubric); + expect(sys).toMatch(/STRICT JSON/); + expect(sys).toMatch(/"scores"/); + }); + + it('asks the judge to ignore tool choice', () => { + const rubric = loadScenarioRubric('noisy-signals'); + const sys = buildJudgeSystem('noisy-signals', rubric); + expect(sys.toLowerCase()).toContain('tool choice'); + }); +}); + +describe('buildJudgeUser', () => { + it('includes the scenario question, ground truth, and candidate answer', () => { + const user = buildJudgeUser({ + scenarioPrompt: 'Why is checkout failing?', + groundTruthFacts: '- root cause: payment-service', + candidateAnswer: 'It is the payment service.', + }); + expect(user).toContain('Why is checkout failing?'); + expect(user).toContain('root cause: payment-service'); + expect(user).toContain('It is the payment service.'); + }); +}); + +describe('formatGroundTruthFacts', () => { + it('renders nested expected fields as bullets', () => { + const scenario = getScenario('error-root-cause'); + const out = formatGroundTruthFacts(scenario.groundTruth); + expect(out).toContain('rootCauseService: payment-service'); + expect(out).toContain('errorBodyPattern: db-payment'); + }); + + it('handles arrays/objects via JSON for non-scalar values', () => { + const scenario = getScenario('noisy-signals'); + const out = formatGroundTruthFacts(scenario.groundTruth); + // compositeCells is an array — it gets JSON-stringified + expect(out).toContain('compositeCells:'); + expect(out).toContain('inventory'); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/latency-spike.test.ts b/packages/hdx-eval/src/__tests__/latency-spike.test.ts new file mode 100644 index 0000000000..38f147266f --- /dev/null +++ b/packages/hdx-eval/src/__tests__/latency-spike.test.ts @@ -0,0 +1,200 @@ +import { mulberry32 } from '../rng/seeded'; +import { latencySpikeScenario } from '../scenarios/latency-spike/generate'; +import { collectScenario } from '../scenarios/types'; + +const NOW_MS = Date.parse('2026-05-08T12:00:00.000Z'); +// Tests run at 1% volume to keep memory tractable (full v5 ≈ 6M base +// traces / ~12M+ spans). Pattern shares scale linearly so structural +// invariants still hold at 1%. +const TEST_VOLUME_FACTOR = 0.01; + +function run(seed: number) { + return collectScenario( + latencySpikeScenario.generate({ + rng: mulberry32(seed), + nowMs: NOW_MS, + volumeFactor: TEST_VOLUME_FACTOR, + }), + ); +} + +function nsToMs(ns: number): number { + return ns / 1_000_000; +} + +describe('latency-spike scenario', () => { + const result = run(42); + + it('produces high-volume trace data at 1% (~60K planted + ~40K background = ~200K spans)', () => { + // Full v6: 6M planted × ~2 spans + 4M background × 2 spans + extras + // → ~20M+ spans. At 1%: ~120K + ~80K + extras → ~200-215K spans. + expect(result.traces.length).toBeGreaterThan(195_000); + expect(result.traces.length).toBeLessThan(220_000); + }); + + it('uses 12 api-server endpoints and 5 regions', () => { + const sample = result.traces + .filter(t => t.serviceName === 'api-server') + .slice(0, 5000); + const endpoints = new Set(sample.map(t => t.spanName)); + const regions = new Set(sample.map(t => t.spanAttributes['cloud.region'])); + expect(endpoints.size).toBe(12); + expect(regions.size).toBe(5); + const tenantIds = new Set(sample.map(t => t.spanAttributes['tenant.id'])); + expect(tenantIds.size).toBeGreaterThan(2000); + }); + + it('plants production-shaped (ServiceName, SpanName) cardinality', () => { + // Background traffic should produce hundreds of distinct + // (Service, SpanName) combinations so naive + // `GROUP BY ServiceName, SpanName` returns too many rows to scan. + const combos = new Set( + result.traces.map(t => `${t.serviceName}::${t.spanName}`), + ); + expect(combos.size).toBeGreaterThan(800); + // Background should yield at least 80 distinct service names beyond + // the 4-service planted/decoy pool. + const services = new Set(result.traces.map(t => t.serviceName)); + expect(services.size).toBeGreaterThan(80); + }); + + it('is deterministic for a fixed seed and now', () => { + const b = run(42); + expect(b.traces.length).toBe(result.traces.length); + expect(result.traces[0].spanId).toBe(b.traces[0].spanId); + expect(result.traces[10_000].spanId).toBe(b.traces[10_000].spanId); + }); + + it('produces enterprise spans in the anomaly window with ~80% slow / ~20% cache-hit', () => { + const anomalyStart = NOW_MS - 10 * 60 * 1000; + const enterpriseInWindow = result.traces.filter( + t => + t.spanName === 'GET /api/orders/search' && + t.timestampMs >= anomalyStart && + t.spanAttributes['tenant.tier'] === 'enterprise', + ); + expect(enterpriseInWindow.length).toBeGreaterThan(20); + const slow = enterpriseInWindow.filter(t => nsToMs(t.durationNs) >= 1500); + const fast = enterpriseInWindow.filter(t => nsToMs(t.durationNs) < 100); + const slowFraction = slow.length / enterpriseInWindow.length; + expect(slowFraction).toBeGreaterThan(0.65); + expect(slowFraction).toBeLessThan(0.95); + expect(fast.length).toBeGreaterThan(0); // cache-hit subset exists + }); + + it('keeps enterprise traffic on /api/orders/search fast outside the anomaly window', () => { + const anomalyStart = NOW_MS - 10 * 60 * 1000; + const enterprisePreAnomaly = result.traces.filter( + t => + t.spanName === 'GET /api/orders/search' && + t.timestampMs < anomalyStart && + t.spanAttributes['tenant.tier'] === 'enterprise' && + // Exclude distractors that legitimately add latency + !t.spanAttributes['runtime.event'], + ); + if (enterprisePreAnomaly.length > 50) { + const tail = enterprisePreAnomaly + .map(t => nsToMs(t.durationNs)) + .sort((a, b) => b - a)[0]; + expect(tail).toBeLessThan(1500); + } + }); + + it('plants GC-pause distractors uniformly across tiers (not enterprise-concentrated)', () => { + const gcPauses = result.traces.filter( + t => t.spanAttributes['runtime.event'] === 'gc_pause', + ); + expect(gcPauses.length).toBeGreaterThan(50); + const enterprisePauseFraction = + gcPauses.filter(t => t.spanAttributes['tenant.tier'] === 'enterprise') + .length / gcPauses.length; + // Enterprise tier weight is 5%; GC pauses should match that distribution. + expect(enterprisePauseFraction).toBeLessThan(0.15); + }); + + it('plants cold-start distractors only at the start of the hour', () => { + const coldStarts = result.traces.filter( + t => t.spanAttributes['runtime.event'] === 'cold_start', + ); + expect(coldStarts.length).toBeGreaterThan(20); + const inLast10Min = coldStarts.filter( + t => t.timestampMs >= NOW_MS - 10 * 60 * 1000, + ); + expect(inLast10Min.length).toBe(0); + }); + + it('emits cache.lookup distractor spans on a fraction of traces', () => { + const cacheLookups = result.traces.filter( + t => t.spanName === 'cache.lookup', + ); + expect(cacheLookups.length).toBeGreaterThan(500); + }); + + it('includes GET /api/orders/search among the endpoints', () => { + const endpoints = new Set( + result.traces + .filter(t => t.serviceName === 'api-server') + .map(t => t.spanName), + ); + expect(endpoints.has('GET /api/orders/search')).toBe(true); + }); + + it('every parent span has a child query (db or elasticsearch) with the same TraceId', () => { + const parents = result.traces.filter(t => t.serviceName === 'api-server'); + const childTraceIds = new Set( + result.traces + .filter( + t => + t.spanName === 'database.query' || + t.spanName === 'elasticsearch.query', + ) + .map(t => t.traceId), + ); + let missing = 0; + for (let i = 0; i < parents.length; i += Math.ceil(parents.length / 1000)) { + if (!childTraceIds.has(parents[i].traceId)) missing++; + } + expect(missing).toBe(0); + }); + + it('plants concurrent products/search hot-shard slowness in same window, all tiers', () => { + const anomalyStart = NOW_MS - 10 * 60 * 1000; + const productsHotShard = result.traces.filter( + t => + t.spanName === 'elasticsearch.query' && + t.spanAttributes['elasticsearch.shard.hot'] === 'true' && + t.timestampMs >= anomalyStart, + ); + expect(productsHotShard.length).toBeGreaterThan(50); + // Affects all tiers — collect tiers via parent traceId + const productsHotTraceIds = new Set(productsHotShard.map(t => t.traceId)); + const parents = result.traces.filter( + t => + t.spanName === 'GET /api/products/search' && + productsHotTraceIds.has(t.traceId), + ); + const tiers = new Set(parents.map(t => t.spanAttributes['tenant.tier'])); + expect(tiers.size).toBeGreaterThanOrEqual(2); + }); + + it('feature_flag.experiment_a is correlated with enterprise but also enabled on free/pro', () => { + const sample = result.traces + .filter(t => t.serviceName === 'api-server') + .slice(0, 5000); + const enterprise = sample.filter( + t => t.spanAttributes['tenant.tier'] === 'enterprise', + ); + const free = sample.filter(t => t.spanAttributes['tenant.tier'] === 'free'); + const enterpriseOnRate = + enterprise.filter( + t => t.spanAttributes['feature_flag.experiment_a'] === 'enabled', + ).length / Math.max(enterprise.length, 1); + const freeOnRate = + free.filter( + t => t.spanAttributes['feature_flag.experiment_a'] === 'enabled', + ).length / Math.max(free.length, 1); + expect(enterpriseOnRate).toBeGreaterThan(0.6); + expect(freeOnRate).toBeGreaterThan(0.15); + expect(freeOnRate).toBeLessThan(0.5); // confounder, not perfectly correlated + }); +}); diff --git a/packages/hdx-eval/src/__tests__/markdown.test.ts b/packages/hdx-eval/src/__tests__/markdown.test.ts new file mode 100644 index 0000000000..2be7dec2cc --- /dev/null +++ b/packages/hdx-eval/src/__tests__/markdown.test.ts @@ -0,0 +1,113 @@ +import type { GradeRecord } from '../grading/types'; +import type { RunRecord } from '../harness/types'; +import { buildAggregate, type GradedRunPair } from '../reports/aggregate'; +import { renderMarkdownReport } from '../reports/markdown'; + +function pair( + scenario: string, + mcp: 'hyperdx' | 'clickhouse', + i: number, +): GradedRunPair { + const run: RunRecord = { + schemaVersion: 1, + runId: `${scenario}-${mcp}-${i}`, + scenario, + mcp, + model: 'claude-sonnet-4-6', + runIndex: i, + seed: 42, + startedAt: '2026-05-09T07:50:00.000Z', + endedAt: '2026-05-09T07:51:00.000Z', + durationMs: 60_000, + agentPrompt: 'p', + systemPromptAppend: 's', + termination: 'final_answer', + exitCode: 0, + tools: [], + toolCalls: [], + messages: [], + finalAnswer: 'a', + tokens: { input: 50, output: 1000, cacheCreation: 0, cacheRead: 5000 }, + totalCostUsd: 0.01, + stderr: '', + }; + const grade: GradeRecord = { + schemaVersion: 2, + runId: run.runId, + scenario, + mcp, + programmatic: { + hits: [{ id: 'names_root', weight: 1, matched: true, satisfied: true }], + score: 1, + }, + judge: { + model: 'claude-opus-4-7', + scores: { + correctness: { score: 5, rationale: '' }, + completeness: { score: 4, rationale: '' }, + }, + weightedScore: 0.9, + rawResponse: '', + durationMs: 1000, + tokens: { input: 0, output: 0, cacheCreation: 0, cacheRead: 0 }, + }, + toolErrors: { total: 1, errors: 0, rate: 0, penalty: 0, samples: [] }, + combinedScore: 0.94, + gradedAt: '', + judgeModel: 'claude-opus-4-7', + }; + return { run, grade }; +} + +describe('renderMarkdownReport', () => { + const summary = buildAggregate({ + batchDir: '/tmp/2026-05-09T07-50-58-566Z', + pairs: [ + pair('error-root-cause', 'hyperdx', 0), + pair('error-root-cause', 'clickhouse', 0), + pair('latency-spike', 'hyperdx', 0), + pair('latency-spike', 'clickhouse', 0), + ], + }); + const md = renderMarkdownReport(summary); + + it('renders a top-line verdict table', () => { + expect(md).toContain('### Top-line verdict'); + expect(md).toContain('| Scenario | HyperDX | ClickHouse | Δ (combined) |'); + }); + + it('emits a section per scenario', () => { + expect(md).toContain('## error-root-cause'); + expect(md).toContain('## latency-spike'); + }); + + it('includes all summary metrics in each scenario table', () => { + expect(md).toContain('Combined score'); + expect(md).toContain('Programmatic score'); + expect(md).toContain('Judge mean (weighted)'); + expect(md).toContain('Tool calls (mean)'); + expect(md).toContain('Output tokens (mean)'); + expect(md).toContain('Wall clock (s, mean)'); + expect(md).toContain('Termination'); + }); + + it('formats deltas with explicit sign', () => { + // hyperdx and clickhouse have identical fixtures → delta is 0 + expect(md).toMatch(/\+0%|\+0\.0/); + }); + + it('emits judge per-criterion breakdown when judge data is present', () => { + expect(md).toContain('Judge per-criterion (mean 0–5)'); + expect(md).toContain('correctness'); + expect(md).toContain('completeness'); + }); + + it('emits programmatic per-check breakdown', () => { + expect(md).toContain('Programmatic per-check (pass rate)'); + expect(md).toContain('names_root'); + }); + + it('uses the batch basename in the title', () => { + expect(md.split('\n')[0]).toContain('2026-05-09T07-50-58-566Z'); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/mcpConfig.test.ts b/packages/hdx-eval/src/__tests__/mcpConfig.test.ts new file mode 100644 index 0000000000..8e9747c928 --- /dev/null +++ b/packages/hdx-eval/src/__tests__/mcpConfig.test.ts @@ -0,0 +1,99 @@ +import { + allowedToolsPattern, + buildMcpConfig, + MCP_CLICKHOUSE_VERSION, +} from '../harness/mcpConfig'; +import { buildSettings, DENIED_BUILT_IN_TOOLS } from '../harness/settingsFile'; +import type { EvalConfig } from '../hyperdx/config'; + +const cfg: EvalConfig = { + hyperdx: { + apiUrl: 'http://localhost:30196', + mcpUrl: 'http://localhost:30196/mcp', + accessKey: 'test-access-key', + connectionId: 'conn-id-1', + scenarios: { + 'error-root-cause': { tracesSourceId: 't1', logsSourceId: 'l1' }, + 'latency-spike': { tracesSourceId: 't2', logsSourceId: 'l2' }, + 'noisy-signals': { tracesSourceId: 't3', logsSourceId: 'l3' }, + }, + }, + clickhouse: { + host: 'localhost', + port: '30596', + user: 'default', + password: '', + }, +}; + +describe('buildMcpConfig', () => { + it('emits HyperDX HTTP transport with bearer token header', () => { + const json = buildMcpConfig(cfg, 'hyperdx'); + expect(json).toEqual({ + mcpServers: { + hyperdx: { + type: 'http', + url: 'http://localhost:30196/mcp', + headers: { Authorization: 'Bearer test-access-key' }, + }, + }, + }); + }); + + it('emits ClickHouse stdio transport via uv with pinned version + env', () => { + const json = buildMcpConfig(cfg, 'clickhouse') as { + mcpServers: { + clickhouse: { + command: string; + args: string[]; + env: Record; + }; + }; + }; + expect(json.mcpServers.clickhouse.command).toBe('uv'); + expect(json.mcpServers.clickhouse.args).toEqual([ + 'run', + '--with', + `mcp-clickhouse==${MCP_CLICKHOUSE_VERSION}`, + '--python', + '3.10', + 'mcp-clickhouse', + ]); + expect(json.mcpServers.clickhouse.env.CLICKHOUSE_HOST).toBe('localhost'); + expect(json.mcpServers.clickhouse.env.CLICKHOUSE_PORT).toBe('30596'); + expect(json.mcpServers.clickhouse.env.CLICKHOUSE_SECURE).toBe('false'); + }); +}); + +describe('allowedToolsPattern', () => { + it('restricts to the right MCP namespace', () => { + expect(allowedToolsPattern('hyperdx')).toBe('mcp__hyperdx__*'); + expect(allowedToolsPattern('clickhouse')).toBe('mcp__clickhouse__*'); + }); +}); + +describe('buildSettings', () => { + it("allowlists only the run condition's MCP tools", () => { + expect(buildSettings('hyperdx')).toEqual({ + permissions: { + allow: ['mcp__hyperdx__*'], + deny: [...DENIED_BUILT_IN_TOOLS], + }, + }); + expect(buildSettings('clickhouse')).toEqual({ + permissions: { + allow: ['mcp__clickhouse__*'], + deny: [...DENIED_BUILT_IN_TOOLS], + }, + }); + }); + + it('denies the built-in filesystem/shell tools so the agent cannot read prior runs', () => { + const settings = buildSettings('hyperdx') as { + permissions: { deny: string[] }; + }; + expect(settings.permissions.deny).toEqual( + expect.arrayContaining(['Bash', 'Read', 'Glob', 'Grep', 'Write', 'Edit']), + ); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/noisy-signals.test.ts b/packages/hdx-eval/src/__tests__/noisy-signals.test.ts new file mode 100644 index 0000000000..8014e3ce9d --- /dev/null +++ b/packages/hdx-eval/src/__tests__/noisy-signals.test.ts @@ -0,0 +1,273 @@ +import { mulberry32 } from '../rng/seeded'; +import { noisySignalsScenario } from '../scenarios/noisy-signals/generate'; +import { collectScenario } from '../scenarios/types'; + +const NOW_MS = Date.parse('2026-05-08T12:00:00.000Z'); +// Tests run at 1% volume (15M → 150K) to keep memory reasonable. Pattern +// shares are preserved by the volumeFactor scaling. +const TEST_VOLUME_FACTOR = 0.01; +const TARGET_TOTAL = 16_000_000; +const SCALED_TOTAL = TARGET_TOTAL * TEST_VOLUME_FACTOR; // ~160K + +function run(seed: number) { + return collectScenario( + noisySignalsScenario.generate({ + rng: mulberry32(seed), + nowMs: NOW_MS, + volumeFactor: TEST_VOLUME_FACTOR, + }), + ); +} + +describe('noisy-signals scenario v6 (composite cells)', () => { + const result = run(42); + + it(`emits ~${SCALED_TOTAL.toLocaleString()} logs at volumeFactor ${TEST_VOLUME_FACTOR} (target ${TARGET_TOTAL.toLocaleString()} at full scale)`, () => { + // Allow 5% slack since each phase rounds independently. + expect(result.logs.length).toBeGreaterThan(SCALED_TOTAL * 0.95); + expect(result.logs.length).toBeLessThan(SCALED_TOTAL * 1.05); + }); + + it('is deterministic for a fixed seed and now', () => { + const b = run(42); + expect(b.logs.length).toBe(result.logs.length); + for (const i of [0, 1000, result.logs.length - 1]) { + expect(result.logs[i].body).toBe(b.logs[i].body); + expect(result.logs[i].timestampMs).toBe(b.logs[i].timestampMs); + } + }); + + // ─── Per-pattern shares ──────────────────────────────────────────────── + + function shareByEvent(eventName: string): number { + return ( + result.logs.filter(l => l.logAttributes['event.name'] === eventName) + .length / result.logs.length + ); + } + + it('cell 1: notification-service × DEBUG carries BOTH cache.hit (~6.7%) AND notification.delivery (~6.7%)', () => { + expect(shareByEvent('cache.hit')).toBeCloseTo(0.067, 1); + expect(shareByEvent('notification.delivery')).toBeCloseTo(0.067, 1); + }); + + it('cell 2: billing-service × INFO carries BOTH subscription metric-as-log (~6.7%) AND legit financial events (~6.7%)', () => { + expect(shareByEvent('subscription.metric.calculated')).toBeCloseTo( + 0.067, + 1, + ); + const billingEvents = [ + 'invoice.charged', + 'invoice.refunded', + 'subscription.activated', + 'subscription.cancelled', + 'payment.failed', + ]; + const share = + result.logs.filter(l => + billingEvents.includes(l.logAttributes['event.name'] ?? ''), + ).length / result.logs.length; + expect(share).toBeCloseTo(0.067, 1); + }); + + it('cell 3: worker × ERROR carries BOTH NonFatalRetryableError caught (~2.7%) AND job.failed.permanent (~2.7%)', () => { + expect(shareByEvent('job.caught_exception')).toBeCloseTo(0.027, 1); + expect(shareByEvent('job.failed.permanent')).toBeCloseTo(0.027, 1); + }); + + it('cell 4: 4-service × DEBUG carries BOTH health.check (~13.3%) AND legit per-service ops debug (~13.3%)', () => { + expect(shareByEvent('health.check')).toBeCloseTo(0.133, 1); + const opsEvents = [ + 'inventory.stock_lookup', + 'pricing.calculation', + 'shipping.rate.quote', + 'recommendation.score', + ]; + const share = + result.logs.filter(l => + opsEvents.includes(l.logAttributes['event.name'] ?? ''), + ).length / result.logs.length; + expect(share).toBeCloseTo(0.133, 1); + }); + + it('cell 5: frontend-proxy × INFO carries BOTH lb.health.probe (~6.7%) AND proxy.access (~6.7%)', () => { + expect(shareByEvent('lb.health.probe')).toBeCloseTo(0.067, 1); + expect(shareByEvent('proxy.access')).toBeCloseTo(0.067, 1); + }); + + it('cell 6: frontend × INFO carries BOTH console.log_dump (~5.3%) AND page.render (~5.3%)', () => { + expect(shareByEvent('frontend.console.log_dump')).toBeCloseTo(0.053, 1); + expect(shareByEvent('page.render')).toBeCloseTo(0.053, 1); + }); + + it('cell 7: search-service × INFO carries BOTH search.cache.miss (~4%) AND index.shard.flush (~4%)', () => { + expect(shareByEvent('search.cache.miss')).toBeCloseTo(0.04, 1); + expect(shareByEvent('index.shard.flush')).toBeCloseTo(0.04, 1); + }); + + // ─── The whole point: GROUP BY (Service, Severity) does NOT separate + // noise from load-bearing — the cells are composite. ─────────────── + + it('inside notification-service × DEBUG, cache.hit and notification.delivery are within 30% of each other in volume', () => { + const cell = result.logs.filter( + l => + l.serviceName === 'notification-service' && l.severityText === 'DEBUG', + ); + const cacheHits = cell.filter( + l => l.logAttributes['event.name'] === 'cache.hit', + ).length; + const deliveries = cell.filter( + l => l.logAttributes['event.name'] === 'notification.delivery', + ).length; + const ratio = + Math.min(cacheHits, deliveries) / Math.max(cacheHits, deliveries); + expect(ratio).toBeGreaterThan(0.7); + }); + + it('inside billing-service × INFO, subscription metric-as-log and financial events are within 30% of each other', () => { + const cell = result.logs.filter( + l => l.serviceName === 'billing-service' && l.severityText === 'INFO', + ); + const subs = cell.filter( + l => l.logAttributes['event.name'] === 'subscription.metric.calculated', + ).length; + const billingEventsList = [ + 'invoice.charged', + 'invoice.refunded', + 'subscription.activated', + 'subscription.cancelled', + 'payment.failed', + ]; + const fin = cell.filter(l => + billingEventsList.includes(l.logAttributes['event.name'] ?? ''), + ).length; + const ratio = Math.min(subs, fin) / Math.max(subs, fin); + expect(ratio).toBeGreaterThan(0.7); + }); + + it('inside worker × ERROR, caught-exception and permanent-failure are within 30% of each other', () => { + const cell = result.logs.filter( + l => l.serviceName === 'worker' && l.severityText === 'ERROR', + ); + const caught = cell.filter( + l => l.logAttributes['event.name'] === 'job.caught_exception', + ).length; + const perm = cell.filter( + l => l.logAttributes['event.name'] === 'job.failed.permanent', + ).length; + const ratio = Math.min(caught, perm) / Math.max(caught, perm); + expect(ratio).toBeGreaterThan(0.7); + }); + + it('inside frontend-proxy × INFO, lb.health.probe and proxy.access are within 30% of each other', () => { + const cell = result.logs.filter( + l => l.serviceName === 'frontend-proxy' && l.severityText === 'INFO', + ); + const probes = cell.filter( + l => l.logAttributes['event.name'] === 'lb.health.probe', + ).length; + const access = cell.filter( + l => l.logAttributes['event.name'] === 'proxy.access', + ).length; + const ratio = Math.min(probes, access) / Math.max(probes, access); + expect(ratio).toBeGreaterThan(0.7); + }); + + it('inside inventory-service × DEBUG, heartbeats and stock_lookups are within 30% of each other (sample cell)', () => { + const cell = result.logs.filter( + l => l.serviceName === 'inventory-service' && l.severityText === 'DEBUG', + ); + const heart = cell.filter( + l => l.logAttributes['event.name'] === 'health.check', + ).length; + const ops = cell.filter( + l => l.logAttributes['event.name'] === 'inventory.stock_lookup', + ).length; + const ratio = Math.min(heart, ops) / Math.max(heart, ops); + expect(ratio).toBeGreaterThan(0.7); + }); + + // ─── Body cardinality + attribute discoverability ────────────────────── + + it('heartbeat / cache.hit / notification.delivery — each has variable bodies (>50% unique in 500-row sample)', () => { + for (const ev of ['health.check', 'cache.hit', 'notification.delivery']) { + const sample = result.logs + .filter(l => l.logAttributes['event.name'] === ev) + .slice(0, 500); + const unique = new Set(sample.map(l => l.body)); + expect(unique.size / sample.length).toBeGreaterThan(0.5); + } + }); + + it('subscription metric-as-log keeps metric.kind=gauge attribute (the conversion signal)', () => { + const subs = result.logs.filter( + l => l.logAttributes['metric.name'] === 'subscription_mrr', + ); + expect(subs.length).toBeGreaterThan(0); + expect(subs[0].logAttributes['metric.kind']).toBe('gauge'); + }); + + it('NonFatalRetryableError caught-exception has stable error.kind attr; permanent has different error.kind', () => { + const caught = result.logs.filter( + l => + l.logAttributes['event.name'] === 'job.caught_exception' && + l.logAttributes['error.kind'] === 'NonFatalRetryableError', + ); + expect(caught.length).toBeGreaterThan(0); + const perm = result.logs.filter( + l => l.logAttributes['event.name'] === 'job.failed.permanent', + ); + expect(perm.length).toBeGreaterThan(0); + // The two patterns share serviceName + severityText but must be + // distinguishable by event.name (and have different error.kind). + const permKinds = new Set(perm.map(l => l.logAttributes['error.kind'])); + expect(permKinds.has('NonFatalRetryableError')).toBe(false); + }); + + it('lb.health.probe uses kube-probe user-agent (the signal that it is a probe)', () => { + const probes = result.logs.filter( + l => l.logAttributes['event.name'] === 'lb.health.probe', + ); + expect(probes.length).toBeGreaterThan(0); + expect(probes[0].logAttributes['user_agent.original']).toContain( + 'kube-probe', + ); + }); + + it('background variety (~9.3%) emits many distinct body templates across many services', () => { + const bg = result.logs.filter( + l => l.logAttributes['event.name'] === 'app.background', + ); + expect(bg.length / result.logs.length).toBeCloseTo(0.093, 1); + const services = new Set(bg.slice(0, 5000).map(l => l.serviceName)); + expect(services.size).toBeGreaterThan(8); + const bodies = new Set(bg.slice(0, 5000).map(l => l.body)); + expect(bodies.size).toBeGreaterThan(500); + }); + + it('severity is messy (lowercase + alias variants present in _severity_raw attr)', () => { + const rawSevs = new Set( + result.logs + .map(l => l.logAttributes._severity_raw) + .filter(Boolean) as string[], + ); + expect(rawSevs.has('info')).toBe(true); + expect(rawSevs.size).toBeGreaterThanOrEqual(3); + }); + + it('every log has rich resource attributes (k8s + service.* keys)', () => { + const sample = result.logs.slice(0, 500); + for (const l of sample) { + expect(l.resourceAttributes['service.name']).toBeDefined(); + expect(l.resourceAttributes['k8s.pod.name']).toBeDefined(); + expect(l.resourceAttributes['cloud.region']).toBeDefined(); + } + }); + + it('overall body cardinality is high (>50% unique in a 5K sample)', () => { + const sample = result.logs.slice(0, 5000); + const unique = new Set(sample.map(l => l.body)); + const ratio = unique.size / sample.length; + expect(ratio).toBeGreaterThan(0.5); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/programmatic.test.ts b/packages/hdx-eval/src/__tests__/programmatic.test.ts new file mode 100644 index 0000000000..a4c1eecfdb --- /dev/null +++ b/packages/hdx-eval/src/__tests__/programmatic.test.ts @@ -0,0 +1,140 @@ +import { runProgrammaticChecks } from '../grading/programmatic'; +import { loadScenarioRubric } from '../grading/rubric'; + +describe('runProgrammaticChecks', () => { + it('hits all checks when answer mentions every required fact', () => { + const checks = [ + { id: 'a', weight: 1, pattern: 'foo' }, + { id: 'b', weight: 2, pattern: 'bar' }, + ]; + const result = runProgrammaticChecks('foo and bar are present', checks); + expect(result.score).toBeCloseTo(1, 5); + expect(result.hits.every(h => h.matched)).toBe(true); + }); + + it('weights hits proportionally', () => { + const checks = [ + { id: 'a', weight: 1, pattern: 'foo' }, // matches → +1 + { id: 'b', weight: 3, pattern: 'BAR', flags: '' }, // case-sensitive miss + ]; + const result = runProgrammaticChecks('only foo here', checks); + // 1 / (1+3) = 0.25 + expect(result.score).toBeCloseTo(0.25, 5); + }); + + it('defaults flags to case-insensitive', () => { + const checks = [{ id: 'a', weight: 1, pattern: 'PAYMENT-SERVICE' }]; + const result = runProgrammaticChecks( + 'mentions payment-service here', + checks, + ); + expect(result.hits[0].matched).toBe(true); + }); + + it('throws on invalid regex', () => { + expect(() => + runProgrammaticChecks('any', [{ id: 'a', weight: 1, pattern: '(' }]), + ).toThrow(); + }); + + it('returns 0 score for empty checks list', () => { + expect(runProgrammaticChecks('anything', []).score).toBe(0); + }); + + it('error-root-cause rubric scores 100% on a strong answer that hits every multi-criterion check', () => { + const rubric = loadScenarioRubric('error-root-cause'); + const answer = + 'Root cause: payment-service ConnectionTimeoutError on db.payment.connect ' + + 'reaching db-payment.internal — DB connection timeout — cascading into ' + + 'checkout-api 5xx errors. Ruled out concurrent SMTP and CDN origin bursts ' + + '(separate trace trees, no checkout parent) and the historical TLS / ' + + 'rate-limit incidents.'; + const result = runProgrammaticChecks(answer, rubric.programmatic); + expect(result.score).toBeCloseTo(1, 5); + }); + + it('error-root-cause rubric drops below saturation when the answer is generic (no error.type / db host / distractor rule-out)', () => { + const rubric = loadScenarioRubric('error-root-cause'); + const genericAnswer = + 'Root cause is payment-service connection timeout to db-payment, ' + + 'cascading into checkout-api 5xx errors.'; + const result = runProgrammaticChecks(genericAnswer, rubric.programmatic); + // Pre-tightening, this answer scored 1.0. Now it misses + // names_db_host_fqdn (full db-payment.internal), names_specific_db_span + // (db.payment.connect), and ruled_out_a_distractor — should land + // measurably below 1.0. + expect(result.score).toBeLessThan(0.9); + expect(result.score).toBeGreaterThan(0.5); + }); + + it('error-root-cause rubric scores noticeably worse on an irrelevant answer than a correct one', () => { + const rubric = loadScenarioRubric('error-root-cause'); + const irrelevant = runProgrammaticChecks( + 'I think the database needs more memory.', + rubric.programmatic, + ); + const correct = runProgrammaticChecks( + 'Root cause is payment-service connection timeout to db-payment, ' + + 'cascading into checkout-api 5xx errors.', + rubric.programmatic, + ); + // Irrelevant answers vacuously satisfy the negative checks (they don't + // blame anything) but miss every positive — leaves a clear gap below the + // correct answer. + expect(correct.score - irrelevant.score).toBeGreaterThan(0.3); + expect(irrelevant.score).toBeLessThan(correct.score); + }); + + it('negative check is satisfied when the pattern does NOT match', () => { + const checks = [ + { id: 'a', weight: 1, pattern: 'foo' }, // positive, miss + { id: 'b', weight: 1, pattern: 'baz', negative: true }, // negative, miss → satisfied + ]; + const result = runProgrammaticChecks('only bar here', checks); + expect(result.hits[0]).toMatchObject({ matched: false, satisfied: false }); + expect(result.hits[1]).toMatchObject({ + matched: false, + satisfied: true, + negative: true, + }); + // 1 / (1+1) = 0.5 + expect(result.score).toBeCloseTo(0.5, 5); + }); + + it('negative check is unsatisfied when the pattern DOES match', () => { + const checks = [ + { id: 'a', weight: 1, pattern: 'foo' }, // positive, hit + { id: 'b', weight: 2, pattern: 'BAR', negative: true }, // negative, hit → unsatisfied + ]; + const result = runProgrammaticChecks('foo and bar', checks); + expect(result.hits[1]).toMatchObject({ + matched: true, + satisfied: false, + negative: true, + }); + // 1 / (1+2) = 0.333 + expect(result.score).toBeCloseTo(1 / 3, 5); + }); + + it('error-root-cause rubric penalizes blaming a distractor as root cause', () => { + const rubric = loadScenarioRubric('error-root-cause'); + const goodAnswer = + 'Root cause: payment-service ConnectionTimeoutError on db.payment.connect ' + + 'reaching db-payment.internal — DB connection timeout — cascading into ' + + 'checkout-api 5xx. Ruled out concurrent SMTP and CDN bursts ' + + '(separate trace trees) and historical TLS / rate-limit incidents.'; + const goodResult = runProgrammaticChecks(goodAnswer, rubric.programmatic); + + const wrongBlameAnswer = + 'Root cause is the SMTP connection refused error on notification-service. ' + + 'payment-service db-payment connection timeout into checkout-api also seen.'; + const wrongResult = runProgrammaticChecks( + wrongBlameAnswer, + rubric.programmatic, + ); + + expect(goodResult.score).toBeGreaterThan(0.95); + // Wrong blame should drop score below 1 even though all positive checks still hit. + expect(wrongResult.score).toBeLessThan(goodResult.score); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/runRun.test.ts b/packages/hdx-eval/src/__tests__/runRun.test.ts new file mode 100644 index 0000000000..9d0d85621c --- /dev/null +++ b/packages/hdx-eval/src/__tests__/runRun.test.ts @@ -0,0 +1,138 @@ +import { assembleRecord } from '../harness/runRun'; +import type { ParsedEvent } from '../harness/streamParser'; + +function event(raw: object): ParsedEvent | null { + // Round-trip through the JSON parser so we exercise the same code path + // the runtime would. + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { parseStreamLine } = require('../harness/streamParser'); + return parseStreamLine(JSON.stringify(raw)); +} + +function evs(...raws: object[]): ParsedEvent[] { + return raws.map(r => event(r)).filter((e): e is ParsedEvent => e !== null); +} + +const baseInput = { + scenario: 'error-root-cause', + agentPrompt: 'find root cause', + systemPromptAppend: 'sys', + mcp: 'hyperdx' as const, + model: 'claude-sonnet-4-6', + runIndex: 0, + seed: 42, + startedAtIso: '2026-05-09T10:00:00.000Z', + endedAtIso: '2026-05-09T10:01:30.000Z', + durationMs: 90_000, + stderr: '', +}; + +describe('assembleRecord', () => { + it('builds a successful run with one tool call and a final answer', () => { + const events = evs( + { + type: 'system', + subtype: 'init', + session_id: 's1', + tools: [{ name: 'mcp__hyperdx__hyperdx_query' }], + }, + { + type: 'assistant', + message: { + content: [ + { type: 'text', text: 'looking up' }, + { + type: 'tool_use', + id: 'tu_1', + name: 'mcp__hyperdx__hyperdx_query', + input: { sql: 'SELECT 1' }, + }, + ], + usage: { input_tokens: 100, output_tokens: 25 }, + }, + }, + { + type: 'user', + message: { + content: [ + { type: 'tool_result', tool_use_id: 'tu_1', content: 'rows...' }, + ], + }, + }, + { + type: 'assistant', + message: { + content: [{ type: 'text', text: 'Root cause is payment-service.' }], + }, + }, + { + type: 'result', + subtype: 'success', + is_error: false, + duration_ms: 90_000, + total_cost_usd: 0.05, + result: 'Root cause is payment-service.', + usage: { + input_tokens: 5000, + output_tokens: 200, + cache_read_input_tokens: 1500, + }, + }, + ); + + const record = assembleRecord({ + ...baseInput, + events, + exitCode: 0, + timedOut: false, + }); + expect(record.termination).toBe('final_answer'); + expect(record.toolCalls).toHaveLength(1); + expect(record.toolCalls[0].name).toBe('mcp__hyperdx__hyperdx_query'); + expect(record.toolCalls[0].output).toBe('rows...'); + expect(record.tokens.input).toBe(5000); + expect(record.tokens.cacheRead).toBe(1500); + expect(record.totalCostUsd).toBe(0.05); + expect(record.finalAnswer).toBe('Root cause is payment-service.'); + expect(record.tools.map(t => t.name)).toEqual([ + 'mcp__hyperdx__hyperdx_query', + ]); + }); + + it('marks termination as max_turns when result subtype indicates it', () => { + const events = evs({ + type: 'result', + subtype: 'error_max_turns', + is_error: true, + result: '', + usage: { input_tokens: 10, output_tokens: 0 }, + }); + const record = assembleRecord({ + ...baseInput, + events, + exitCode: 0, + timedOut: false, + }); + expect(record.termination).toBe('max_turns'); + }); + + it('marks termination as timeout when killed', () => { + const record = assembleRecord({ + ...baseInput, + events: [], + exitCode: null, + timedOut: true, + }); + expect(record.termination).toBe('timeout'); + }); + + it('marks termination as error on non-zero exit with no answer', () => { + const record = assembleRecord({ + ...baseInput, + events: [], + exitCode: 1, + timedOut: false, + }); + expect(record.termination).toBe('error'); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/seeded.test.ts b/packages/hdx-eval/src/__tests__/seeded.test.ts new file mode 100644 index 0000000000..fcd191c4bb --- /dev/null +++ b/packages/hdx-eval/src/__tests__/seeded.test.ts @@ -0,0 +1,66 @@ +import { mulberry32 } from '../rng/seeded'; + +describe('mulberry32', () => { + it('produces an identical sequence for the same seed', () => { + const a = mulberry32(42); + const b = mulberry32(42); + const seqA = Array.from({ length: 10 }, () => a.next()); + const seqB = Array.from({ length: 10 }, () => b.next()); + expect(seqA).toEqual(seqB); + }); + + it('produces a different sequence for a different seed', () => { + const a = mulberry32(42); + const b = mulberry32(43); + const seqA = Array.from({ length: 10 }, () => a.next()); + const seqB = Array.from({ length: 10 }, () => b.next()); + expect(seqA).not.toEqual(seqB); + }); + + it('range() stays in [lo, hi)', () => { + const rng = mulberry32(1); + for (let i = 0; i < 1000; i++) { + const v = rng.range(5, 10); + expect(v).toBeGreaterThanOrEqual(5); + expect(v).toBeLessThan(10); + } + }); + + it('intRange() stays in [lo, hi) and returns integers', () => { + const rng = mulberry32(2); + for (let i = 0; i < 1000; i++) { + const v = rng.intRange(0, 5); + expect(Number.isInteger(v)).toBe(true); + expect(v).toBeGreaterThanOrEqual(0); + expect(v).toBeLessThan(5); + } + }); + + it('pick() returns a deterministic element for a fixed seed', () => { + const items = ['a', 'b', 'c', 'd'] as const; + const v1 = mulberry32(99).pick(items); + const v2 = mulberry32(99).pick(items); + expect(v1).toBe(v2); + }); + + it('weightedPick() approximates the weight distribution', () => { + const rng = mulberry32(7); + const counts = { a: 0, b: 0 }; + const N = 5000; + for (let i = 0; i < N; i++) { + const v = rng.weightedPick<'a' | 'b'>([ + { value: 'a', weight: 90 }, + { value: 'b', weight: 10 }, + ]); + counts[v]++; + } + const ratioA = counts.a / N; + expect(ratioA).toBeGreaterThan(0.85); + expect(ratioA).toBeLessThan(0.94); + }); + + it('hex() returns the requested byte length as hex', () => { + expect(mulberry32(1).hex(8)).toMatch(/^[0-9a-f]{16}$/); + expect(mulberry32(1).hex(16)).toMatch(/^[0-9a-f]{32}$/); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/segmented-regression.test.ts b/packages/hdx-eval/src/__tests__/segmented-regression.test.ts new file mode 100644 index 0000000000..ee9816a021 --- /dev/null +++ b/packages/hdx-eval/src/__tests__/segmented-regression.test.ts @@ -0,0 +1,157 @@ +import { mulberry32 } from '../rng/seeded'; +import { segmentedRegressionScenario } from '../scenarios/segmented-regression/generate'; +import { collectScenario } from '../scenarios/types'; + +const NOW_MS = Date.parse('2026-05-10T20:00:00.000Z'); +// 1% volume keeps the test cheap (~60K traces) while preserving the +// (tier, cache) cross-tab signal — distractors are still at fixed counts. +const TEST_VOLUME_FACTOR = 0.01; +const ANOMALY_WINDOW_MS = 10 * 60 * 1000; + +function run(seed: number) { + return collectScenario( + segmentedRegressionScenario.generate({ + rng: mulberry32(seed), + nowMs: NOW_MS, + volumeFactor: TEST_VOLUME_FACTOR, + }), + ); +} + +describe('segmented-regression scenario', () => { + const result = run(42); + + it('produces ~60K api-server traces at 1% volume', () => { + const apiTraces = result.traces.filter(t => t.serviceName === 'api-server'); + expect(apiTraces.length).toBeGreaterThan(50_000); + expect(apiTraces.length).toBeLessThan(70_000); + }); + + it('plants exactly 220 recommendation-service distractor errors', () => { + const rec = result.traces.filter( + t => t.serviceName === 'recommendation-service', + ); + expect(rec.length).toBe(220); + for (const r of rec) { + expect(r.statusCode).toBe('STATUS_CODE_ERROR'); + expect(r.statusMessage).toMatch(/upstream cf-feed unreachable/); + } + }); + + it('is deterministic for a fixed seed', () => { + const b = run(42); + expect(b.traces.length).toBe(result.traces.length); + expect(b.logs.length).toBe(result.logs.length); + expect(b.traces[0].spanId).toBe(result.traces[0].spanId); + expect(b.traces[5000].spanId).toBe(result.traces[5000].spanId); + }); + + describe('the planted intersection bug — enterprise × cache.hit=false', () => { + const anomalyStart = NOW_MS - ANOMALY_WINDOW_MS; + const apiAnomaly = result.traces.filter( + t => t.serviceName === 'api-server' && t.timestampMs >= anomalyStart, + ); + const bySegment = ( + tier: string, + hit: 'true' | 'false', + ): { total: number; errors: number } => { + const slice = apiAnomaly.filter( + t => + t.spanAttributes['tenant.tier'] === tier && + t.spanAttributes['cache.hit'] === hit, + ); + const errors = slice.filter( + t => t.statusCode === 'STATUS_CODE_ERROR', + ).length; + return { total: slice.length, errors }; + }; + + it('elevates error rate for enterprise × cache-miss to ~12% (16x baseline)', () => { + const seg = bySegment('enterprise', 'false'); + expect(seg.total).toBeGreaterThan(100); // enough sample at 1% volume + const rate = seg.errors / seg.total; + expect(rate).toBeGreaterThan(0.05); // well above baseline + expect(rate).toBeLessThan(0.2); // not so dominant it's obvious + }); + + it('leaves OTHER segments at the 0.5% baseline error rate', () => { + for (const [tier, hit] of [ + ['enterprise', 'true'], + ['pro', 'false'], + ['pro', 'true'], + ['free', 'false'], + ['free', 'true'], + ] as const) { + const seg = bySegment(tier, hit); + if (seg.total < 30) continue; // skip cells without enough sample + const rate = seg.errors / seg.total; + expect(rate).toBeLessThan(0.04); // close to baseline 0.5% + } + }); + + it('uses the distinctive FallbackHandlerError body for the planted errors', () => { + const fallbackErrors = apiAnomaly.filter( + t => + t.spanAttributes['error.type'] === 'FallbackHandlerError' && + t.statusCode === 'STATUS_CODE_ERROR', + ); + // Every fallback error must be enterprise × miss. + for (const f of fallbackErrors) { + expect(f.spanAttributes['tenant.tier']).toBe('enterprise'); + expect(f.spanAttributes['cache.hit']).toBe('false'); + } + // Body has the schema-mismatch signature. + expect(fallbackErrors[0].statusMessage).toMatch( + /schema mismatch in enterprise_v2_response/, + ); + }); + + it('emits correlated ERROR logs for each fallback error (same traceId)', () => { + const fallbackTraceIds = new Set( + apiAnomaly + .filter( + t => t.spanAttributes['error.type'] === 'FallbackHandlerError', + ) + .map(t => t.traceId), + ); + const fallbackLogs = result.logs.filter( + l => l.traceId && fallbackTraceIds.has(l.traceId), + ); + // Every planted error must have a log row attached. + expect(fallbackLogs.length).toBe(fallbackTraceIds.size); + expect(fallbackLogs[0].body).toMatch(/schema mismatch/); + }); + }); + + it('single-axis aggregates dilute the signal vs the cross-tab', () => { + const anomalyStart = NOW_MS - ANOMALY_WINDOW_MS; + const apiAnomaly = result.traces.filter( + t => t.serviceName === 'api-server' && t.timestampMs >= anomalyStart, + ); + const rateBy = ( + predicate: (t: (typeof apiAnomaly)[number]) => boolean, + ): number => { + const slice = apiAnomaly.filter(predicate); + if (slice.length === 0) return 0; + return ( + slice.filter(t => t.statusCode === 'STATUS_CODE_ERROR').length / + slice.length + ); + }; + const enterpriseRate = rateBy( + t => t.spanAttributes['tenant.tier'] === 'enterprise', + ); + const cacheMissRate = rateBy( + t => t.spanAttributes['cache.hit'] === 'false', + ); + const intersectionRate = rateBy( + t => + t.spanAttributes['tenant.tier'] === 'enterprise' && + t.spanAttributes['cache.hit'] === 'false', + ); + // Each single-axis rate is materially lower than the cross-tab — proves + // the agent has to combine them to find the actual signal. + expect(intersectionRate).toBeGreaterThan(enterpriseRate * 1.5); + expect(intersectionRate).toBeGreaterThan(cacheMissRate * 1.5); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/service-health-check.test.ts b/packages/hdx-eval/src/__tests__/service-health-check.test.ts new file mode 100644 index 0000000000..cd34dd933d --- /dev/null +++ b/packages/hdx-eval/src/__tests__/service-health-check.test.ts @@ -0,0 +1,208 @@ +import { mulberry32 } from '../rng/seeded'; +import { serviceHealthCheckScenario } from '../scenarios/service-health-check/generate'; +import { collectScenario } from '../scenarios/types'; + +const NOW_MS = Date.parse('2026-05-10T20:00:00.000Z'); + +// 1% volume keeps the test cheap (~120K traces, ~240K logs) while preserving +// the planted-signal invariants. The blip count (~250 errors / minute) is +// derived from baseline traffic density × 1% volume, so it stays +// proportional to TEST_VOLUME_FACTOR. +const TEST_VOLUME_FACTOR = 0.01; + +const FOUR_HOURS_MS = 4 * 60 * 60 * 1000; +const ONE_HOUR_MS = 60 * 60 * 1000; +const REPORT_START_MS = NOW_MS - ONE_HOUR_MS; +const BASELINE_START_MS = NOW_MS - FOUR_HOURS_MS; +const BLIP_START_MS = REPORT_START_MS + 3 * 60 * 1000; +const BLIP_END_MS = BLIP_START_MS + 60 * 1000; +const NEW_TEMPLATE_START_MS = NOW_MS - 70 * 60 * 1000; + +function run(seed: number) { + return collectScenario( + serviceHealthCheckScenario.generate({ + rng: mulberry32(seed), + nowMs: NOW_MS, + volumeFactor: TEST_VOLUME_FACTOR, + }), + ); +} + +describe('service-health-check scenario', () => { + const result = run(42); + const apiTraces = result.traces; + const apiLogs = result.logs; + + it('produces traces and logs over the 4-hour window', () => { + // 12M × 0.01 = 120K traces target; 24M × 0.01 = 240K logs. + expect(apiTraces.length).toBeGreaterThan(100_000); + expect(apiTraces.length).toBeLessThan(140_000); + expect(apiLogs.length).toBeGreaterThan(220_000); + expect(apiLogs.length).toBeLessThan(260_000); + // All rows belong to api-server. + expect(apiTraces.every(t => t.serviceName === 'api-server')).toBe(true); + expect(apiLogs.every(l => l.serviceName === 'api-server')).toBe(true); + }); + + it('covers the full 4-hour window with the report window at the end', () => { + let minTs = Infinity; + let maxTs = -Infinity; + for (const t of apiTraces) { + if (t.timestampMs < minTs) minTs = t.timestampMs; + if (t.timestampMs > maxTs) maxTs = t.timestampMs; + } + // First sample within 1 min of baseline start; last within 1 min of now. + expect(minTs - BASELINE_START_MS).toBeLessThan(60_000); + expect(NOW_MS - maxTs).toBeLessThan(60_000); + }); + + it('is deterministic for a fixed seed', () => { + const b = run(42); + expect(b.traces.length).toBe(apiTraces.length); + expect(b.logs.length).toBe(apiLogs.length); + expect(b.traces[100].spanId).toBe(apiTraces[100].spanId); + expect(b.logs[100].body).toBe(apiLogs[100].body); + }); + + describe('baseline characteristics', () => { + it('error rate is ~0.5% outside the blip window', () => { + const nonBlip = apiTraces.filter( + t => !(t.timestampMs >= BLIP_START_MS && t.timestampMs < BLIP_END_MS), + ); + const errors = nonBlip.filter( + t => t.statusCode === 'STATUS_CODE_ERROR', + ).length; + const rate = errors / nonBlip.length; + expect(rate).toBeGreaterThan(0.002); + expect(rate).toBeLessThan(0.008); + }); + + it('log Body cardinality is high — naive GROUP BY Body is useless', () => { + // Sample 5K logs and check unique-body ratio. >= 90% unique means + // GROUP BY Body returns ~1 row per log row, so the agent must use + // hyperdx_log_patterns (Drain) to find templates. + const sample = apiLogs.slice(0, 5000); + const uniqueBodies = new Set(sample.map(l => l.body)); + expect(uniqueBodies.size / sample.length).toBeGreaterThan(0.9); + }); + }); + + describe('planted signal: new log template', () => { + it('feature-flag template appears ONLY in the planted window (~last 70 min)', () => { + const planted = apiLogs.filter(l => + /feature_flag\.new_recommendation_engine/.test(l.body), + ); + expect(planted.length).toBeGreaterThan(0); + // None in the baseline (the first 170 minutes of the dataset). + const baselinePlanted = planted.filter( + l => l.timestampMs < NEW_TEMPLATE_START_MS, + ); + expect(baselinePlanted.length).toBe(0); + // Approximately 0.1% of total log volume. At 1% volume that's ~240 + // logs; relax to a generous band to absorb sampling noise. + expect(planted.length).toBeGreaterThan(50); + expect(planted.length).toBeLessThan(600); + }); + }); + + describe('planted signal: v2 endpoint rollout', () => { + it('GET /api/v2/products appears ONLY in the report window and ramps up', () => { + const v2 = apiTraces.filter(t => t.spanName === 'GET /api/v2/products'); + expect(v2.length).toBeGreaterThan(0); + // None in the baseline. + const baselineV2 = v2.filter(t => t.timestampMs < REPORT_START_MS); + expect(baselineV2.length).toBe(0); + // Ramp check: more v2 in the second half of the report window than + // the first half. (The ramp is linear so 2nd half should be ~3× 1st.) + const halfMs = REPORT_START_MS + ONE_HOUR_MS / 2; + const firstHalf = v2.filter(t => t.timestampMs < halfMs).length; + const secondHalf = v2.filter(t => t.timestampMs >= halfMs).length; + expect(secondHalf).toBeGreaterThan(firstHalf); + }); + }); + + describe('planted signal: latency drift', () => { + it('p99 in the last 10 min of report > p99 in the first 10 min of report', () => { + const firstMinMs = REPORT_START_MS; + const firstMaxMs = REPORT_START_MS + 10 * 60 * 1000; + const lastMinMs = NOW_MS - 10 * 60 * 1000; + const firstSlice = apiTraces + .filter(t => t.timestampMs >= firstMinMs && t.timestampMs < firstMaxMs) + .map(t => t.durationNs / 1e6); + const lastSlice = apiTraces + .filter(t => t.timestampMs >= lastMinMs && t.timestampMs <= NOW_MS) + .map(t => t.durationNs / 1e6); + const p99 = (arr: number[]): number => { + const sorted = [...arr].sort((a, b) => a - b); + return sorted[Math.floor(sorted.length * 0.99)] ?? 0; + }; + const p99First = p99(firstSlice); + const p99Last = p99(lastSlice); + // Drift should be visible: last p99 > first p99. At low volume the + // tail is noisy so we only require a positive direction with a + // moderate floor. + expect(p99Last).toBeGreaterThan(p99First); + expect(p99Last - p99First).toBeGreaterThan(10); + }); + }); + + describe('planted signal: brief error blip', () => { + it('blip minute has ~5% error rate, ~10× baseline', () => { + const blip = apiTraces.filter( + t => t.timestampMs >= BLIP_START_MS && t.timestampMs < BLIP_END_MS, + ); + expect(blip.length).toBeGreaterThan(100); + const blipErrors = blip.filter( + t => t.statusCode === 'STATUS_CODE_ERROR', + ).length; + const blipRate = blipErrors / blip.length; + expect(blipRate).toBeGreaterThan(0.025); + expect(blipRate).toBeLessThan(0.1); + }); + + it('all blip errors use the "upstream connect timeout" message', () => { + const blipErrors = apiTraces.filter( + t => + t.timestampMs >= BLIP_START_MS && + t.timestampMs < BLIP_END_MS && + t.statusCode === 'STATUS_CODE_ERROR', + ); + const timeoutErrors = blipErrors.filter( + t => t.statusMessage === 'upstream connect timeout', + ); + expect(timeoutErrors.length).toBe(blipErrors.length); + }); + }); + + describe('distractor: recurring batch-sync spikes', () => { + it('appears in BOTH the baseline window and the report window', () => { + const batch = apiTraces.filter( + t => t.spanName === 'POST /api/internal/batch-sync', + ); + expect(batch.length).toBeGreaterThan(0); + const inBaseline = batch.filter( + t => t.timestampMs < REPORT_START_MS, + ).length; + const inReport = batch.filter( + t => t.timestampMs >= REPORT_START_MS, + ).length; + // 3 hours of baseline vs 1 hour of report → roughly 3:1 ratio. + expect(inBaseline).toBeGreaterThan(inReport); // present in both + expect(inReport).toBeGreaterThan(0); + }); + + it('repeats on a ~15-minute cadence (16 spike-windows across 4 hours)', () => { + const batch = apiTraces.filter( + t => t.spanName === 'POST /api/internal/batch-sync', + ); + // Bucket batch-sync spans by 15-minute cycle and count occupied buckets. + // We expect 16 buckets to each have at least some spans (one per cycle). + const occupied = new Set(); + for (const t of batch) { + const offset = t.timestampMs - BASELINE_START_MS; + occupied.add(Math.floor(offset / (15 * 60 * 1000))); + } + expect(occupied.size).toBeGreaterThanOrEqual(15); + }); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/streamParser.test.ts b/packages/hdx-eval/src/__tests__/streamParser.test.ts new file mode 100644 index 0000000000..4d20b9587d --- /dev/null +++ b/packages/hdx-eval/src/__tests__/streamParser.test.ts @@ -0,0 +1,127 @@ +import { chunkToLines, parseStreamLine } from '../harness/streamParser'; + +describe('parseStreamLine', () => { + it('returns null for blank/invalid input', () => { + expect(parseStreamLine('')).toBeNull(); + expect(parseStreamLine(' ')).toBeNull(); + expect(parseStreamLine('not json')).toBeNull(); + expect(parseStreamLine('null')).toBeNull(); + }); + + it('parses a system init event', () => { + const line = JSON.stringify({ + type: 'system', + subtype: 'init', + session_id: 'abc-123', + model: 'claude-sonnet-4-6', + tools: [ + { name: 'mcp__hyperdx__hyperdx_query', description: 'Query' }, + 'plain_string_tool', + ], + }); + const ev = parseStreamLine(line); + expect(ev?.kind).toBe('system_init'); + if (ev?.kind !== 'system_init') throw new Error('type narrow'); + expect(ev.sessionId).toBe('abc-123'); + expect(ev.model).toBe('claude-sonnet-4-6'); + expect(ev.tools).toHaveLength(2); + expect(ev.tools?.[0]?.name).toBe('mcp__hyperdx__hyperdx_query'); + expect(ev.tools?.[1]?.name).toBe('plain_string_tool'); + }); + + it('parses an assistant message with text + tool_use blocks', () => { + const line = JSON.stringify({ + type: 'assistant', + message: { + role: 'assistant', + content: [ + { type: 'text', text: 'Looking up sources' }, + { + type: 'tool_use', + id: 'tu_1', + name: 'mcp__hyperdx__hyperdx_list_sources', + input: {}, + }, + ], + usage: { input_tokens: 100, output_tokens: 50 }, + }, + }); + const ev = parseStreamLine(line); + expect(ev?.kind).toBe('assistant_message'); + if (ev?.kind !== 'assistant_message') throw new Error('type narrow'); + expect(ev.content).toHaveLength(2); + expect(ev.usage?.input_tokens).toBe(100); + }); + + it('parses a user (tool_result) event', () => { + const line = JSON.stringify({ + type: 'user', + message: { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'tu_1', + content: 'OK', + }, + ], + }, + }); + const ev = parseStreamLine(line); + expect(ev?.kind).toBe('user_message'); + if (ev?.kind !== 'user_message') throw new Error('type narrow'); + expect(ev.content).toHaveLength(1); + }); + + it('parses a final result event', () => { + const line = JSON.stringify({ + type: 'result', + subtype: 'success', + is_error: false, + duration_ms: 12345, + total_cost_usd: 0.0123, + result: 'Root cause is payment-service.', + usage: { + input_tokens: 5000, + output_tokens: 200, + cache_read_input_tokens: 1000, + }, + }); + const ev = parseStreamLine(line); + expect(ev?.kind).toBe('result'); + if (ev?.kind !== 'result') throw new Error('type narrow'); + expect(ev.subtype).toBe('success'); + expect(ev.isError).toBe(false); + expect(ev.totalCostUsd).toBe(0.0123); + expect(ev.usage?.cache_read_input_tokens).toBe(1000); + }); + + it('classifies an unknown event type without throwing', () => { + const line = JSON.stringify({ type: 'futuristic_event', payload: 42 }); + const ev = parseStreamLine(line); + expect(ev?.kind).toBe('unknown'); + }); +}); + +describe('chunkToLines', () => { + it('splits a complete buffer into lines with empty remainder', () => { + const { events, remainder } = chunkToLines('', 'a\nb\nc\n'); + expect(events).toEqual(['a', 'b', 'c']); + expect(remainder).toBe(''); + }); + + it('preserves the trailing partial line as remainder', () => { + const { events, remainder } = chunkToLines('', 'a\nb\npart'); + expect(events).toEqual(['a', 'b']); + expect(remainder).toBe('part'); + }); + + it('combines a leftover buffer with the new chunk', () => { + const r1 = chunkToLines('', '{"a":'); + expect(r1.events).toHaveLength(0); + expect(r1.remainder).toBe('{"a":'); + const r2 = chunkToLines(r1.remainder, '1}\n{"b":2}\n'); + expect(r2.events).toEqual(['{"a":1}', '{"b":2}']); + expect(r2.remainder).toBe(''); + }); +}); diff --git a/packages/hdx-eval/src/__tests__/systemPrompt.test.ts b/packages/hdx-eval/src/__tests__/systemPrompt.test.ts new file mode 100644 index 0000000000..4ed22ade58 --- /dev/null +++ b/packages/hdx-eval/src/__tests__/systemPrompt.test.ts @@ -0,0 +1,36 @@ +import { buildSystemPrompt } from '../harness/systemPrompt'; + +describe('buildSystemPrompt', () => { + it('includes scenario-specific table names', () => { + const p = buildSystemPrompt('error-root-cause', 'hyperdx'); + expect(p).toContain('default.eval_error_root_cause_otel_traces'); + expect(p).toContain('default.eval_error_root_cause_otel_logs'); + }); + + it('hints HyperDX MCP tools when condition is hyperdx', () => { + const p = buildSystemPrompt('latency-spike', 'hyperdx'); + // The prompt now ships a small tools catalog so the agent doesn't have + // to ToolSearch every tool by name. The catalog covers list_sources, + // describe_source, and the four investigation tools we want the agent + // to know about. + expect(p).toContain('hyperdx_list_sources'); + expect(p).toContain('hyperdx_describe_source'); + expect(p).toContain('hyperdx_event_deltas'); + expect(p).toContain('hyperdx_log_patterns'); + expect(p).not.toContain('run_query'); + }); + + it('hints ClickHouse MCP tools when condition is clickhouse', () => { + const p = buildSystemPrompt('noisy-signals', 'clickhouse'); + expect(p).toContain('run_query'); + expect(p).toContain('list_tables'); + expect(p).not.toContain('hyperdx_query'); + }); + + it('asks for service, operation, and root cause in the final answer', () => { + const p = buildSystemPrompt('error-root-cause', 'clickhouse'); + expect(p.toLowerCase()).toContain('service'); + expect(p.toLowerCase()).toContain('operation'); + expect(p.toLowerCase()).toContain('root cause'); + }); +}); diff --git a/packages/hdx-eval/src/cli.ts b/packages/hdx-eval/src/cli.ts new file mode 100644 index 0000000000..a5ee1ef22c --- /dev/null +++ b/packages/hdx-eval/src/cli.ts @@ -0,0 +1,670 @@ +#!/usr/bin/env node +import { Command } from 'commander'; +import { resolve } from 'path'; + +import { createEvalClient, defaultClickHouseUrl } from './clickhouse/client'; +import { dropScenarioTables, scenarioTables } from './clickhouse/schema'; +import { gradeBatch, resolveBatchDir } from './grading/grade'; +import { runCell } from './harness/runRun'; +import type { McpKind, PromptVariant } from './harness/types'; +import { configExists, configPath, readConfig } from './hyperdx/config'; +import { runCheck, runSetup } from './hyperdx/setup'; +import { writeBatchSummary } from './reports/store'; +import { instrumentBatch, summarizeTimingRecord } from './runs/instrument'; +import { batchDirName } from './runs/path'; +import { writeRun } from './runs/store'; +import { listBatches, listRunsInBatch, readRun } from './runs/store'; +import { getScenario, SCENARIO_NAMES, SCENARIOS } from './scenarios'; +import { seedScenario } from './scenarios/seedScenario'; + +type GlobalOpts = { + chUrl?: string; + chUser?: string; + chPassword?: string; +}; + +function buildClient(opts: GlobalOpts) { + return createEvalClient({ + url: opts.chUrl ?? defaultClickHouseUrl(), + username: opts.chUser, + password: opts.chPassword, + }); +} + +function buildClientFromConfig( + cfg: import('./hyperdx/config').EvalConfig, + globals: GlobalOpts, +) { + return createEvalClient({ + url: + globals.chUrl ?? `http://${cfg.clickhouse.host}:${cfg.clickhouse.port}`, + username: globals.chUser ?? cfg.clickhouse.user, + password: globals.chPassword ?? cfg.clickhouse.password, + }); +} + +function defaultApiUrl(): string { + if (process.env.HDX_EVAL_API_URL) return process.env.HDX_EVAL_API_URL; + if (process.env.HYPERDX_API_PORT) { + return `http://localhost:${process.env.HYPERDX_API_PORT}`; + } + if (process.env.HDX_DEV_API_PORT) { + return `http://localhost:${process.env.HDX_DEV_API_PORT}`; + } + return 'http://localhost:8000'; +} + +const program = new Command(); + +program + .name('hdx-eval') + .description( + 'HyperDX eval framework — synthetic telemetry generators + agent harness', + ) + .option('--ch-url ', 'ClickHouse HTTP URL') + .option('--ch-user ', 'ClickHouse username', 'default') + .option('--ch-password ', 'ClickHouse password', ''); + +program + .command('list') + .description('List available scenarios') + .action(() => { + for (const name of SCENARIO_NAMES) { + const s = SCENARIOS[name]; + const tables = scenarioTables(name); + console.log(`\n${name}`); + console.log(` prompt: ${s.agentPrompt}`); + console.log(` description: ${s.description}`); + console.log(` tables: default.${tables.traces}, default.${tables.logs}`); + } + console.log(); + }); + +program + .command('seed ') + .description('Seed ClickHouse with synthetic data for the given scenario') + .option('--seed ', 'PRNG seed', '42') + .option('--now ', 'Anchor "now" to a fixed ISO timestamp') + .option( + '--volume-factor ', + 'Scale background row counts (e.g. 0.1 for 10% of full volume)', + ) + .action( + async ( + scenarioName: string, + cmdOpts: { seed: string; now?: string; volumeFactor?: string }, + ) => { + const opts = program.opts(); + const seedNum = Number(cmdOpts.seed); + if (!Number.isFinite(seedNum)) { + throw new Error(`--seed must be a number, got: ${cmdOpts.seed}`); + } + const nowMs = cmdOpts.now ? Date.parse(cmdOpts.now) : Date.now(); + if (cmdOpts.now && !Number.isFinite(nowMs)) { + throw new Error( + `--now must be a valid ISO timestamp, got: ${cmdOpts.now}`, + ); + } + let volumeFactor: number | undefined; + if (cmdOpts.volumeFactor !== undefined) { + volumeFactor = Number(cmdOpts.volumeFactor); + if (!Number.isFinite(volumeFactor) || volumeFactor <= 0) { + throw new Error( + `--volume-factor must be a positive number, got: ${cmdOpts.volumeFactor}`, + ); + } + } + + const scenario = getScenario(scenarioName); + console.log(`Seeding scenario "${scenario.name}"`); + console.log(` seed: ${seedNum}`); + console.log(` now: ${new Date(nowMs).toISOString()}`); + if (volumeFactor !== undefined) { + console.log(` volumeFactor: ${volumeFactor}`); + } + + const client = buildClient(opts); + try { + const result = await seedScenario({ + client, + scenarioName: scenario.name, + seed: seedNum, + nowMs, + volumeFactor, + }); + console.log( + `Inserted ${result.tracesInserted} trace rows → default.${result.tables.traces}`, + ); + console.log( + `Inserted ${result.logsInserted} log rows → default.${result.tables.logs}`, + ); + } finally { + await client.close(); + } + }, + ); + +program + .command('drop ') + .description('Drop the eval tables for the given scenario') + .action(async (scenarioName: string) => { + const opts = program.opts(); + getScenario(scenarioName); + const client = buildClient(opts); + try { + await dropScenarioTables(client, scenarioName); + const tables = scenarioTables(scenarioName); + console.log( + `Dropped default.${tables.traces} and default.${tables.logs}`, + ); + } finally { + await client.close(); + } + }); + +program + .command('setup-hyperdx') + .description( + 'One-time setup: register/login + create Connection + per-scenario Sources', + ) + .option('--api-url ', 'HyperDX API URL', defaultApiUrl()) + .option('--email ', 'Account email', 'eval@local.test') + .option('--password ', 'Account password', 'EvalPass123!#') + .option( + '--creds-file ', + 'JSON file with {"email":..., "password":...} (overrides --email/--password)', + ) + .option('--check', 'Validate existing config instead of creating') + .option( + '--reset-sources', + 'Delete and recreate all eval-* Sources (e.g. to apply new querySettings)', + ) + .action( + async (cmdOpts: { + apiUrl: string; + email: string; + password: string; + credsFile?: string; + check?: boolean; + resetSources?: boolean; + }) => { + const opts = program.opts(); + let email = cmdOpts.email; + let password = cmdOpts.password; + if (cmdOpts.credsFile) { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const fs = require('fs') as typeof import('fs'); + const expanded = cmdOpts.credsFile.replace( + /^~(?=$|\/|\\)/, + process.env.HOME ?? '~', + ); + const raw = JSON.parse(fs.readFileSync(expanded, 'utf8')); + if (typeof raw.email !== 'string' || typeof raw.password !== 'string') { + throw new Error( + `--creds-file must be a JSON object with string fields "email" and "password"`, + ); + } + email = raw.email; + password = raw.password; + } + if (cmdOpts.check) { + const result = await runCheck(); + console.log( + 'Config: ', + result.configOk ? 'OK' : 'MISSING/INVALID', + ); + console.log( + 'HyperDX MCP: ', + result.mcpReachable ? 'OK' : 'UNREACHABLE', + ); + console.log( + 'ClickHouse: ', + result.clickhouseReachable ? 'OK' : 'UNREACHABLE', + ); + console.log( + 'uv installed: ', + result.uvAvailable ? 'OK' : 'MISSING', + ); + if (result.errors.length > 0) { + console.log('\nErrors:'); + for (const e of result.errors) console.log(' -', e); + process.exit(1); + } + return; + } + + const chUrl = opts.chUrl ?? defaultClickHouseUrl(); + const url = new URL(chUrl); + console.log(`Setting up HyperDX eval account at ${cmdOpts.apiUrl}`); + const result = await runSetup({ + apiUrl: cmdOpts.apiUrl, + email, + password, + clickhouse: { + host: url.hostname, + port: url.port || '8123', + user: opts.chUser ?? 'default', + password: opts.chPassword ?? '', + }, + resetSources: cmdOpts.resetSources ?? false, + }); + console.log(`Wrote ${result.configPath}`); + console.log( + ` connection: ${result.created.connection ? 'created' : 'already existed'}`, + ); + if (result.created.sources.length === 0) { + console.log(' sources: all already existed'); + } else { + console.log(' sources: created', result.created.sources.join(', ')); + } + }, + ); + +program + .command('run ') + .description( + 'Run Claude Code as the agent against one or both MCPs and capture trajectories', + ) + .option('--mcp ', 'hyperdx | clickhouse | both', 'both') + .option('--runs ', 'Number of runs per (scenario,MCP) cell', '3') + .option( + '--model ', + 'Model ID to pass to Claude Code', + 'claude-sonnet-4-6', + ) + // Lower than the previous 25 — tightens the budget so sloppy agents that + // make 20+ exploratory calls can't paper over correctness with volume. + // Override with --max-turns if a specific scenario needs more. + .option('--max-turns ', 'Max tool-use turns', '15') + .option('--seed ', 'PRNG seed for re-seeding', '42') + .option( + '--timeout ', + 'Per-run wall-clock timeout in milliseconds', + '300000', + ) + .option('--no-reseed', 'Skip the re-seed step (use existing data as-is)') + .option( + '--anchor-time ', + 'Fixed "now" anchor for both seed and agent system prompt. When set, ' + + 'the agent treats this timestamp as the current time for relative ' + + 'window references in the user prompt. Required when running with ' + + '--no-reseed against a shared-anchor batch seed.', + ) + .option( + '--concurrency ', + 'Run up to N cells concurrently within this scenario. Default 1 ' + + '(sequential). Each cell is one (mcp, runIndex) pair.', + '1', + ) + .option( + '--prompt-variant ', + 'Prompt variant: "baseline" (default) or "hypothesis" (forces the ' + + 'agent to enumerate 2-4 hypotheses up front and spawn one Task ' + + 'subagent per hypothesis to investigate in parallel). The hypothesis ' + + 'variant also allows the Task built-in tool; the baseline variant ' + + 'denies it.', + 'baseline', + ) + .action( + async ( + scenarioName: string, + cmdOpts: { + mcp: string; + runs: string; + model: string; + maxTurns: string; + seed: string; + timeout: string; + reseed: boolean; + anchorTime?: string; + concurrency: string; + promptVariant: string; + }, + ) => { + const opts = program.opts(); + const scenario = getScenario(scenarioName); + const apiKeyEnv = process.env.ANTHROPIC_API_KEY; + if (!apiKeyEnv) { + throw new Error( + 'ANTHROPIC_API_KEY is not set. Export it before running `hdx-eval run`.', + ); + } + const apiKey: string = apiKeyEnv; + if (!configExists()) { + throw new Error( + `Eval config missing. Run \`hdx-eval setup-hyperdx\` first. Expected: ${configPath()}`, + ); + } + const config = readConfig(); + + const mcpKinds: McpKind[] = parseMcpFlag(cmdOpts.mcp); + const promptVariant: PromptVariant = parsePromptVariant( + cmdOpts.promptVariant, + ); + const runs = Number(cmdOpts.runs); + const maxTurns = Number(cmdOpts.maxTurns); + const timeoutMs = Number(cmdOpts.timeout); + const seedNum = Number(cmdOpts.seed); + const concurrency = Number(cmdOpts.concurrency); + for (const [k, v] of Object.entries({ + runs, + maxTurns, + timeoutMs, + seedNum, + concurrency, + })) { + if (!Number.isFinite(v) || v <= 0) { + throw new Error( + `--${k.replace(/Num$/, '')} must be a positive number`, + ); + } + } + + let anchorTimeIso: string | undefined; + let anchorMs: number; + if (cmdOpts.anchorTime) { + anchorMs = Date.parse(cmdOpts.anchorTime); + if (!Number.isFinite(anchorMs)) { + throw new Error( + `--anchor-time must be a valid ISO timestamp, got: ${cmdOpts.anchorTime}`, + ); + } + anchorTimeIso = new Date(anchorMs).toISOString(); + } else { + anchorMs = Date.now(); + anchorTimeIso = undefined; // no anchor injection when running live + } + + // Re-seed once before running so timestamps are anchored to the chosen + // "now". Skip if --no-reseed was passed (e.g. shared-anchor batch seed). + if (cmdOpts.reseed !== false) { + const client = buildClientFromConfig(config, opts); + try { + const r = await seedScenario({ + client, + scenarioName: scenario.name, + seed: seedNum, + nowMs: anchorMs, + }); + console.log( + `Re-seeded ${scenario.name}: ${r.tracesInserted} traces, ${r.logsInserted} logs (now=${new Date(anchorMs).toISOString()})`, + ); + } finally { + await client.close(); + } + } + + const batchDir = batchDirName(); + console.log(`\nBatch: runs/${batchDir}`); + console.log(`Scenario: ${scenario.name}`); + console.log(`MCPs: ${mcpKinds.join(', ')}`); + console.log( + `Runs/cell: ${runs}, model: ${cmdOpts.model}, max-turns: ${maxTurns}, concurrency: ${concurrency}, prompt-variant: ${promptVariant}\n`, + ); + + type SummaryRow = { + mcp: McpKind; + i: number; + toolCalls: number; + inputTokens: number; + outputTokens: number; + durationS: number; + termination: string; + path: string; + }; + + // Flatten the (mcp, runIndex) matrix into a single queue and pull from it + // with a worker pool of size `concurrency`. + const cells: Array<{ mcp: McpKind; i: number }> = []; + for (const mcp of mcpKinds) { + for (let i = 0; i < runs; i++) cells.push({ mcp, i }); + } + + const summary: SummaryRow[] = []; + let cursor = 0; + async function worker(workerId: number): Promise { + while (true) { + const idx = cursor++; + if (idx >= cells.length) return; + const { mcp, i } = cells[idx]; + const label = concurrency > 1 ? `[w${workerId}] ` : ' '; + process.stdout.write( + `${label}${mcp} run ${i + 1}/${runs}... starting\n`, + ); + const startedMs = Date.now(); + const record = await runCell({ + config, + scenario: scenario.name, + agentPrompt: scenario.agentPrompt, + mcp, + model: cmdOpts.model, + maxTurns, + timeoutMs, + runIndex: i, + seed: seedNum, + apiKey, + anchorTimeIso, + promptVariant, + }); + const path = writeRun({ record, batchDir }); + const seconds = ((Date.now() - startedMs) / 1000).toFixed(1); + console.log( + `${label}${mcp} run ${i + 1}/${runs}: ${record.termination} · ${record.toolCalls.length} tool calls · ${record.tokens.input}+${record.tokens.output} tok · ${seconds}s`, + ); + summary.push({ + mcp, + i, + toolCalls: record.toolCalls.length, + inputTokens: record.tokens.input, + outputTokens: record.tokens.output, + durationS: Math.round((Date.now() - startedMs) / 100) / 10, + termination: record.termination, + path, + }); + } + } + const workerCount = Math.min(concurrency, cells.length); + await Promise.all( + Array.from({ length: workerCount }, (_, w) => worker(w + 1)), + ); + + console.log('\nResults written under runs/' + batchDir); + // Sort by (mcp, i) for stable summary output even with parallel workers. + summary.sort((a, b) => a.mcp.localeCompare(b.mcp) || a.i - b.i); + for (const row of summary) { + console.log( + ` ${row.mcp.padEnd(10)} #${row.i} ${row.termination.padEnd(13)} tools=${row.toolCalls} tokens=${row.inputTokens}+${row.outputTokens} ${row.durationS}s`, + ); + } + }, + ); + +program + .command('runs-list') + .description('List recorded run batches') + .action(() => { + const batches = listBatches(); + if (batches.length === 0) { + console.log('No runs recorded yet.'); + return; + } + for (const b of batches) { + const files = listRunsInBatch(b); + console.log( + `${b} (${files.length} run${files.length === 1 ? '' : 's'})`, + ); + } + }); + +program + .command('runs-show ') + .description('Pretty-print a run record file') + .option( + '--queries', + 'Print every tool call with its full input (SQL / builder spec) and any matching query_log rows from the timing sidecar', + ) + .option( + '--final-answer', + 'Print the final answer (omitted by default to keep --queries output focused)', + ) + .action( + (path: string, cmdOpts: { queries?: boolean; finalAnswer?: boolean }) => { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const fs = require('fs') as typeof import('fs'); + const r = readRun(path); + console.log(`Run ${r.runId}`); + console.log(` scenario: ${r.scenario}`); + console.log(` mcp: ${r.mcp}`); + console.log(` model: ${r.model}`); + console.log( + ` termination: ${r.termination} (exit ${r.exitCode ?? '?'})`, + ); + console.log(` duration: ${(r.durationMs / 1000).toFixed(1)}s`); + console.log( + ` tokens: in=${r.tokens.input} out=${r.tokens.output} cacheRead=${r.tokens.cacheRead}`, + ); + console.log(` tool calls: ${r.toolCalls.length}`); + + if (cmdOpts.queries) { + const timingPath = path.replace(/\.json$/, '.timing.json'); + const timing = fs.existsSync(timingPath) + ? (JSON.parse(fs.readFileSync(timingPath, 'utf8')) as { + toolCalls: Array<{ + index: number; + serverQueries: Array<{ + queryDurationMs: number; + readRows: number; + queryPreview: string; + }>; + }>; + } | null) + : null; + console.log('\n--- Tool calls ---'); + r.toolCalls.forEach((c, i) => { + const inp = c.input as Record; + let summary: string; + if (c.name === 'mcp__clickhouse__run_query') { + summary = `query=${String(inp?.query ?? '') + .replace(/\s+/g, ' ') + .trim()}`; + } else if (c.name === 'mcp__hyperdx__hyperdx_query') { + if (inp?.displayType === 'sql') { + summary = `[sql] ${String(inp?.sql ?? '') + .replace(/\s+/g, ' ') + .trim()}`; + } else { + summary = `[${inp?.displayType ?? '?'}] where=${inp?.where ?? '—'} groupBy=${inp?.groupBy ?? '—'} select=${JSON.stringify(inp?.select ?? null)}`; + } + } else { + summary = JSON.stringify(inp).slice(0, 200); + } + console.log(`\n#${i + 1} ${c.name}`); + console.log(` ${summary}`); + const t = timing?.toolCalls?.[i]; + if (t && t.serverQueries.length > 0) { + for (const q of t.serverQueries) { + console.log( + ` └─ server: ${q.queryDurationMs}ms, read=${q.readRows} ${q.queryPreview.replace(/\s+/g, ' ').slice(0, 140)}`, + ); + } + } + }); + } + + if (cmdOpts.finalAnswer) { + console.log(`\n--- Final answer ---\n${r.finalAnswer}\n`); + } + }, + ); + +program + .command('grade ') + .description( + 'Grade trajectories: programmatic checks + LLM-as-judge (Opus 4.7 by default)', + ) + .option('--judge-model ', 'Judge model ID', 'claude-opus-4-7') + .option('--rerun-judge', 'Re-call judge even if a grade JSON already exists') + .option('--no-judge', 'Run programmatic checks only (cheap regrade)') + .action( + async ( + batch: string, + cmdOpts: { judgeModel: string; rerunJudge?: boolean; judge: boolean }, + ) => { + const dir = resolveBatchDir(batch); + console.log(`Grading batch at ${dir}`); + const summary = await gradeBatch(dir, { + judgeModel: cmdOpts.judgeModel, + rerunJudge: cmdOpts.rerunJudge ?? false, + skipJudge: cmdOpts.judge === false, + }); + console.log( + `\nGraded ${summary.graded.length} run${summary.graded.length === 1 ? '' : 's'}; ${summary.errors.length} error${summary.errors.length === 1 ? '' : 's'}.`, + ); + if (summary.errors.length > 0) { + for (const e of summary.errors) console.log(' -', e.runPath, e.error); + process.exit(1); + } + }, + ); + +program + .command('runs-instrument ') + .description( + 'Enrich each run with per-tool-call query_log timings from ClickHouse', + ) + .action(async (batch: string) => { + const opts = program.opts(); + const cfgPath = configExists() ? configPath() : null; + let chUrl = opts.chUrl; + let chUser = opts.chUser; + let chPassword = opts.chPassword; + if (!chUrl && cfgPath) { + const cfg = readConfig(); + chUrl = `http://${cfg.clickhouse.host}:${cfg.clickhouse.port}`; + chUser = chUser ?? cfg.clickhouse.user; + chPassword = chPassword ?? cfg.clickhouse.password; + } + chUrl = chUrl ?? defaultClickHouseUrl(); + const records = await instrumentBatch(batch, { + clickhouseUrl: chUrl, + username: chUser, + password: chPassword, + }); + for (const r of records) { + console.log(' ' + summarizeTimingRecord(r)); + } + console.log(`\nWrote ${records.length} *.timing.json sidecars.`); + }); + +program + .command('report ') + .description('Render markdown + JSON summary from grade JSONs in the batch') + .option('--out ', 'Output markdown path (default: /_summary.md)') + .action(async (batch: string, cmdOpts: { out?: string }) => { + const dir = resolveBatchDir(batch); + const out = resolve(cmdOpts.out ?? `${dir}/_summary.md`); + const result = writeBatchSummary(dir, out); + console.log(`Wrote ${result.mdPath}`); + console.log(`Wrote ${result.jsonPath}`); + console.log( + `Aggregated ${result.pairsCount} run${result.pairsCount === 1 ? '' : 's'}.`, + ); + }); + +function parseMcpFlag(v: string): McpKind[] { + if (v === 'both') return ['hyperdx', 'clickhouse']; + if (v === 'hyperdx' || v === 'clickhouse') return [v]; + throw new Error(`--mcp must be hyperdx | clickhouse | both, got: ${v}`); +} + +function parsePromptVariant(v: string): PromptVariant { + if (v === 'baseline' || v === 'hypothesis') return v; + throw new Error( + `--prompt-variant must be "baseline" or "hypothesis", got: ${v}`, + ); +} + +program.parseAsync(process.argv).catch(err => { + console.error(err instanceof Error ? err.message : String(err)); + process.exit(1); +}); diff --git a/packages/hdx-eval/src/clickhouse/client.ts b/packages/hdx-eval/src/clickhouse/client.ts new file mode 100644 index 0000000000..d699dbb181 --- /dev/null +++ b/packages/hdx-eval/src/clickhouse/client.ts @@ -0,0 +1,35 @@ +import { type ClickHouseClient, createClient } from '@clickhouse/client'; +import ms from 'ms'; + +export type EvalClickHouseConfig = { + url: string; + username?: string; + password?: string; +}; + +export function createEvalClient( + cfg: EvalClickHouseConfig, + application = 'hdx-eval', +): ClickHouseClient { + return createClient({ + url: cfg.url, + username: cfg.username ?? 'default', + password: cfg.password ?? '', + application, + request_timeout: ms('2m'), + compression: { request: false, response: false }, + clickhouse_settings: { + date_time_input_format: 'best_effort', + date_time_output_format: 'iso', + max_execution_time: ms('2m') / 1000, + }, + }); +} + +export function defaultClickHouseUrl(): string { + if (process.env.HDX_EVAL_CH_URL) return process.env.HDX_EVAL_CH_URL; + if (process.env.HDX_DEV_CH_HTTP_PORT) { + return `http://localhost:${process.env.HDX_DEV_CH_HTTP_PORT}`; + } + return 'http://localhost:8123'; +} diff --git a/packages/hdx-eval/src/clickhouse/insert.ts b/packages/hdx-eval/src/clickhouse/insert.ts new file mode 100644 index 0000000000..fd855d22ba --- /dev/null +++ b/packages/hdx-eval/src/clickhouse/insert.ts @@ -0,0 +1,106 @@ +import type { ClickHouseClient } from '@clickhouse/client'; + +import type { LogRow, TraceRow } from '../generators/types'; +import { EVAL_DATABASE } from './schema'; + +const BATCH_SIZE = 5_000; + +function msToDateTime64(ms: number): string { + const date = new Date(ms); + const iso = date.toISOString(); + const base = iso.slice(0, 19).replace('T', ' '); + const msPart = iso.slice(20, 23); + return `${base}.${msPart}000000`; +} + +function chunk(rows: T[], size: number): T[][] { + const out: T[][] = []; + for (let i = 0; i < rows.length; i += size) { + out.push(rows.slice(i, i + size)); + } + return out; +} + +function traceRowToCHObject(r: TraceRow): Record { + return { + Timestamp: msToDateTime64(r.timestampMs), + TraceId: r.traceId, + SpanId: r.spanId, + ParentSpanId: r.parentSpanId, + TraceState: '', + SpanName: r.spanName, + SpanKind: r.spanKind, + ServiceName: r.serviceName, + ResourceAttributes: r.resourceAttributes, + ScopeName: '', + ScopeVersion: '', + SpanAttributes: r.spanAttributes, + Duration: String(r.durationNs), + StatusCode: r.statusCode, + StatusMessage: r.statusMessage, + 'Events.Timestamp': [], + 'Events.Name': [], + 'Events.Attributes': [], + 'Links.TraceId': [], + 'Links.SpanId': [], + 'Links.TraceState': [], + 'Links.Attributes': [], + }; +} + +function logRowToCHObject(r: LogRow): Record { + return { + Timestamp: msToDateTime64(r.timestampMs), + TraceId: r.traceId ?? '', + SpanId: r.spanId ?? '', + TraceFlags: 0, + SeverityText: r.severityText, + SeverityNumber: r.severityNumber, + ServiceName: r.serviceName, + Body: r.body, + ResourceSchemaUrl: '', + ResourceAttributes: r.resourceAttributes, + ScopeSchemaUrl: '', + ScopeName: '', + ScopeVersion: '', + ScopeAttributes: {}, + LogAttributes: r.logAttributes, + EventName: '', + }; +} + +export async function insertTraceRows( + client: ClickHouseClient, + table: string, + rows: TraceRow[], +): Promise { + if (rows.length === 0) return 0; + let inserted = 0; + for (const batch of chunk(rows, BATCH_SIZE)) { + await client.insert({ + table: `${EVAL_DATABASE}.${table}`, + values: batch.map(traceRowToCHObject), + format: 'JSONEachRow', + }); + inserted += batch.length; + } + return inserted; +} + +export async function insertLogRows( + client: ClickHouseClient, + table: string, + rows: LogRow[], +): Promise { + if (rows.length === 0) return 0; + let inserted = 0; + for (const batch of chunk(rows, BATCH_SIZE)) { + await client.insert({ + table: `${EVAL_DATABASE}.${table}`, + values: batch.map(logRowToCHObject), + format: 'JSONEachRow', + }); + inserted += batch.length; + } + return inserted; +} diff --git a/packages/hdx-eval/src/clickhouse/schema.ts b/packages/hdx-eval/src/clickhouse/schema.ts new file mode 100644 index 0000000000..e2807a663e --- /dev/null +++ b/packages/hdx-eval/src/clickhouse/schema.ts @@ -0,0 +1,116 @@ +import type { ClickHouseClient } from '@clickhouse/client'; + +export const EVAL_DATABASE = 'default'; +export const SOURCE_TRACES_TABLE = 'otel_traces'; +export const SOURCE_LOGS_TABLE = 'otel_logs'; + +export type ScenarioTables = { + traces: string; + logs: string; +}; + +function scenarioSlug(scenario: string): string { + return scenario.replace(/-/g, '_'); +} + +export function scenarioTables(scenario: string): ScenarioTables { + const slug = scenarioSlug(scenario); + return { + traces: `eval_${slug}_otel_traces`, + logs: `eval_${slug}_otel_logs`, + }; +} + +async function tableExists( + client: ClickHouseClient, + database: string, + table: string, +): Promise { + const rs = await client.query({ + query: `SELECT 1 FROM system.tables WHERE database = {db:String} AND name = {tbl:String}`, + query_params: { db: database, tbl: table }, + format: 'JSONEachRow', + }); + const rows = await rs.json(); + return rows.length > 0; +} + +async function assertSourceTablesExist( + client: ClickHouseClient, +): Promise { + for (const t of [SOURCE_TRACES_TABLE, SOURCE_LOGS_TABLE]) { + if (!(await tableExists(client, EVAL_DATABASE, t))) { + throw new Error( + `Required source table ${EVAL_DATABASE}.${t} does not exist. ` + + `Start the dev stack (\`yarn dev\`) so the OTel collector can ` + + `create the standard otel_* tables before seeding eval data.`, + ); + } + } +} + +export async function ensureScenarioTables( + client: ClickHouseClient, + scenario: string, +): Promise { + await assertSourceTablesExist(client); + const tables = scenarioTables(scenario); + + await client.command({ + query: `CREATE TABLE IF NOT EXISTS ${EVAL_DATABASE}.${tables.traces} AS ${EVAL_DATABASE}.${SOURCE_TRACES_TABLE}`, + }); + await client.command({ + query: `CREATE TABLE IF NOT EXISTS ${EVAL_DATABASE}.${tables.logs} AS ${EVAL_DATABASE}.${SOURCE_LOGS_TABLE}`, + }); + + // The source tables inherit a 1-day TTL from HyperDX migrations. Eval data + // is anchored to a chosen `now` (often historical) and we control its + // lifecycle via the seed/drop CLI, so strip TTL on the eval copies. Without + // this, seeding with `--now` >1 day in the past silently drops the part. + // Idempotent: subsequent runs error with "no TTL to remove" — ignore. + await removeTtlIfPresent(client, tables.traces); + await removeTtlIfPresent(client, tables.logs); + + return tables; +} + +async function removeTtlIfPresent( + client: ClickHouseClient, + table: string, +): Promise { + try { + await client.command({ + query: `ALTER TABLE ${EVAL_DATABASE}.${table} REMOVE TTL`, + }); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + if (/cannot remove|doesn't have any table TTL/i.test(msg)) return; + throw err; + } +} + +export async function truncateScenarioTables( + client: ClickHouseClient, + scenario: string, +): Promise { + const tables = scenarioTables(scenario); + await client.command({ + query: `TRUNCATE TABLE IF EXISTS ${EVAL_DATABASE}.${tables.traces}`, + }); + await client.command({ + query: `TRUNCATE TABLE IF EXISTS ${EVAL_DATABASE}.${tables.logs}`, + }); +} + +export async function dropScenarioTables( + client: ClickHouseClient, + scenario: string, +): Promise { + const tables = scenarioTables(scenario); + await client.command({ + query: `DROP TABLE IF EXISTS ${EVAL_DATABASE}.${tables.traces}`, + }); + await client.command({ + query: `DROP TABLE IF EXISTS ${EVAL_DATABASE}.${tables.logs}`, + }); +} diff --git a/packages/hdx-eval/src/generators/logs.ts b/packages/hdx-eval/src/generators/logs.ts new file mode 100644 index 0000000000..1e5b6b25da --- /dev/null +++ b/packages/hdx-eval/src/generators/logs.ts @@ -0,0 +1,30 @@ +import type { LogRow } from './types'; +import { SEVERITY_NUMBER } from './types'; + +export type LogInput = { + timestampMs: number; + serviceName: string; + severityText: LogRow['severityText']; + body: string; + traceId?: string; + spanId?: string; + resourceAttributes?: Record; + logAttributes?: Record; +}; + +export function makeLog(input: LogInput): LogRow { + return { + timestampMs: input.timestampMs, + traceId: input.traceId, + spanId: input.spanId, + serviceName: input.serviceName, + severityText: input.severityText, + severityNumber: SEVERITY_NUMBER[input.severityText], + body: input.body, + resourceAttributes: { + 'service.name': input.serviceName, + ...(input.resourceAttributes ?? {}), + }, + logAttributes: input.logAttributes ?? {}, + }; +} diff --git a/packages/hdx-eval/src/generators/templates.ts b/packages/hdx-eval/src/generators/templates.ts new file mode 100644 index 0000000000..8157658936 --- /dev/null +++ b/packages/hdx-eval/src/generators/templates.ts @@ -0,0 +1,1191 @@ +/** + * Realistic body/attribute generators inspired by the public ClickHouse + * otel_v2 demo dataset (~68M log rows, ~55% unique bodies). Each helper + * produces a body string with embedded dynamic values plus an attribute + * map — the goal is high body cardinality with stable identifiable + * templates. + */ +import type { SeededRng } from '../rng/seeded'; + +// ─── Severity ──────────────────────────────────────────────────────────── +// +// Real-world OTel collectors aggregate logs from many runtimes and emit +// inconsistent severity strings. Mirror that mess so agents have to handle +// case folding and aliasing. + +const SEVERITY_DISTRIBUTION: { value: string; weight: number }[] = [ + { value: 'info', weight: 60 }, + { value: 'INFO', weight: 8 }, + { value: 'information', weight: 8 }, + { value: 'debug', weight: 5 }, + { value: 'DEBUG', weight: 2 }, + { value: 'warn', weight: 8 }, + { value: 'WARN', weight: 1.5 }, + { value: 'warning', weight: 1 }, + { value: 'error', weight: 3 }, + { value: 'ERROR', weight: 1.5 }, + { value: 'fatal', weight: 0.4 }, + { value: 'trace', weight: 1.6 }, +]; + +const SEVERITY_NUMBER_BY_TEXT: Record = { + trace: 1, + TRACE: 1, + debug: 5, + DEBUG: 5, + info: 9, + INFO: 9, + information: 9, + warn: 13, + WARN: 13, + warning: 13, + error: 17, + ERROR: 17, + fatal: 21, + FATAL: 21, +}; + +export function pickSeverity(rng: SeededRng): { + text: string; + number: number; +} { + const text = rng.weightedPick(SEVERITY_DISTRIBUTION); + return { text, number: SEVERITY_NUMBER_BY_TEXT[text] ?? 9 }; +} + +/** + * Pick a severity but bias toward a target band ('info'|'warn'|'error'). + * Useful when emitting service-level chatter where the level is determined + * by the event itself but should still vary case/alias. + */ +export function pickSeverityIn( + rng: SeededRng, + band: 'info' | 'debug' | 'warn' | 'error' | 'trace', +): { text: string; number: number } { + const families: Record = { + info: [ + { value: 'info', weight: 70 }, + { value: 'INFO', weight: 15 }, + { value: 'information', weight: 15 }, + ], + debug: [ + { value: 'debug', weight: 75 }, + { value: 'DEBUG', weight: 25 }, + ], + warn: [ + { value: 'warn', weight: 70 }, + { value: 'WARN', weight: 15 }, + { value: 'warning', weight: 15 }, + ], + error: [ + { value: 'error', weight: 70 }, + { value: 'ERROR', weight: 25 }, + { value: 'fatal', weight: 5 }, + ], + trace: [{ value: 'trace', weight: 100 }], + }; + const text = rng.weightedPick(families[band]); + return { text, number: SEVERITY_NUMBER_BY_TEXT[text] ?? 9 }; +} + +// ─── Resource attributes (k8s/OTel-style) ──────────────────────────────── + +const NAMESPACES = [ + 'production', + 'staging', + 'platform', + 'observability', + 'edge', + 'data-plane', +]; +const NODES = [ + 'gke-prod-pool-0a1b', + 'gke-prod-pool-2c3d', + 'gke-prod-pool-4e5f', + 'gke-prod-pool-6g7h', +]; +const SERVICE_VERSIONS = ['1.42.3', '1.42.4', '1.43.0', '1.43.1', '2.0.0-rc1']; + +export function buildResourceAttrs(args: { + rng: SeededRng; + serviceName: string; + region?: string; +}): Record { + const { rng, serviceName, region } = args; + const podShortId = rng.hex(5); + const podHash = rng.hex(8); + const ns = rng.pick(NAMESPACES); + const node = rng.pick(NODES); + return { + 'service.name': serviceName, + 'service.namespace': ns, + 'service.version': rng.pick(SERVICE_VERSIONS), + 'service.instance.id': `${serviceName}-${podHash}-${podShortId}`, + 'k8s.namespace.name': ns, + 'k8s.pod.name': `${serviceName}-${podHash}-${podShortId}`, + 'k8s.pod.uid': uuidv4(rng), + 'k8s.node.name': node, + 'k8s.deployment.name': serviceName, + 'k8s.container.name': serviceName, + 'host.name': node, + 'cloud.region': + region ?? rng.pick(['us-east-1', 'us-west-2', 'eu-west-1', 'ap-south-1']), + 'telemetry.sdk.name': 'opentelemetry', + 'telemetry.sdk.language': rng.pick([ + 'go', + 'java', + 'nodejs', + 'python', + 'rust', + ]), + 'telemetry.sdk.version': rng.pick(['1.27.0', '1.28.0', '1.29.0']), + 'deployment.environment.name': ns, + }; +} + +/** + * Pre-build a small pool of resource-attribute objects per service so we can + * sample at row-emit time without paying ~30 attribute-construction lookups + * per span. Mirrors reality (small fixed set of pods/replicas per service). + */ +export function buildResourcePool(args: { + rng: SeededRng; + services: readonly string[]; + instancesPerService?: number; + region?: string; +}): Record[]> { + const { rng, services, region } = args; + const n = args.instancesPerService ?? 24; + const out: Record[]> = {}; + for (const svc of services) { + out[svc] = []; + for (let i = 0; i < n; i++) { + out[svc].push(buildResourceAttrs({ rng, serviceName: svc, region })); + } + } + return out; +} + +export function pickResource( + rng: SeededRng, + pool: Record[]>, + serviceName: string, +): Record { + const list = pool[serviceName]; + if (!list || list.length === 0) { + return buildResourceAttrs({ rng, serviceName }); + } + return rng.pick(list); +} + +// ─── Body templates ────────────────────────────────────────────────────── + +export function uuidv4(rng: SeededRng): string { + // Deterministic v4-shaped UUID (no real randomness — we use seeded RNG). + const h = rng.hex(16); + return `${h.slice(0, 8)}-${h.slice(8, 12)}-4${h.slice(13, 16)}-${ + '89ab'[rng.intRange(0, 4)] + }${h.slice(17, 20)}-${h.slice(20, 32)}`; +} + +const HTTP_METHODS = ['GET', 'POST', 'PUT', 'DELETE', 'PATCH']; +const HTTP_PATHS = [ + '/api/cart', + '/api/checkout', + '/api/orders', + '/api/orders/{id}', + '/api/products', + '/api/products/{id}', + '/api/search', + '/api/recommendations', + '/api/users/{id}', + '/api/inventory/{sku}', + '/api/health', + '/api/auth/refresh', +]; +const USER_AGENTS = [ + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36', + 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Mobile/15E148 Safari/604.1', + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36', + 'python-requests/2.32.3', + 'curl/8.6.0', + 'Mozilla/5.0 (compatible; CheckpointBot/1.0)', +]; + +/** Envoy-style access log body — high cardinality (timestamps, IDs, sizes). */ +export function envoyAccessLog(args: { rng: SeededRng; nowMs: number }): { + body: string; + attrs: Record; +} { + const { rng, nowMs } = args; + const ts = new Date(nowMs).toISOString(); + const method = rng.pick(HTTP_METHODS); + const path = rng.pick(HTTP_PATHS); + const status = rng.weightedPick([ + { value: 200, weight: 90 }, + { value: 304, weight: 3 }, + { value: 404, weight: 3 }, + { value: 401, weight: 1 }, + { value: 500, weight: 2 }, + { value: 503, weight: 1 }, + ]); + const reqBytes = rng.intRange(0, 4096); + const respBytes = rng.intRange(0, 32768); + const totalMs = rng.intRange(2, 800); + const upstreamMs = rng.intRange(1, totalMs); + const reqId = uuidv4(rng); + const ua = rng.pick(USER_AGENTS); + const upstreamIp = `10.${rng.intRange(0, 256)}.${rng.intRange(0, 256)}.${rng.intRange(1, 254)}`; + const sourceIp = `34.${rng.intRange(100, 256)}.${rng.intRange(0, 256)}.${rng.intRange(1, 254)}`; + const body = + `[${ts}] "${method} ${path} HTTP/1.1" ${status} - via_upstream - "-" ` + + `${reqBytes} ${respBytes} ${totalMs} ${upstreamMs} "-" "${ua}" ` + + `"${reqId}" "frontend-proxy:8080" "${upstreamIp}:80" frontend-proxy ` + + `${sourceIp}:0 ${upstreamIp}:80 - - default`; + return { + body, + attrs: { + 'event.name': 'proxy.access', + 'http.method': method, + 'url.template': path, + 'url.path': path, + 'url.full': `https://example.com${path}`, + 'http.status_code': String(status), + 'user_agent.original': ua, + 'net.peer_address': upstreamIp, + 'source.address': sourceIp, + 'server.address': 'frontend-proxy', + 'upstream.host': `${upstreamIp}:80`, + 'upstream.cluster': 'default', + 'destination.address': `${upstreamIp}:80`, + 'http.request.body.size': String(reqBytes), + 'http.response.body.size': String(respBytes), + 'http.duration_ms': String(totalMs), + request_id: reqId, + 'log.iostream': 'stdout', + 'log.file.path': '/var/log/envoy/access.log', + logtag: 'F', + }, + }; +} + +/** logfmt body (popular Go/Python convention). */ +export function logfmtBody(args: { + rng: SeededRng; + nowMs: number; + level: string; + msg: string; + fields: Record; +}): string { + const ts = new Date(args.nowMs).toISOString(); + const parts = [`time="${ts}"`, `level=${args.level}`, `msg="${args.msg}"`]; + for (const [k, v] of Object.entries(args.fields)) { + if (typeof v === 'string' && /[\s"=]/.test(v)) { + parts.push(`${k}="${v.replace(/"/g, '\\"')}"`); + } else { + parts.push(`${k}=${v}`); + } + } + return parts.join(' '); +} + +const BRANDS = [ + 'Nikon', + 'Canon', + 'Sony', + 'Vixen', + 'Bushnell', + 'Orion', + 'Sky-Watcher', + 'Bresser', + 'Leica', + 'Celestron', + 'Meade', +]; +const CATEGORIES = [ + 'binoculars', + 'telescopes', + 'eyepieces', + 'mounts', + 'tripods', + 'filters', + 'cameras', + 'accessories', + 'books', + 'travel', + 'assembly', +]; + +/** Product catalog "Product found" log — logfmt with rich attrs. */ +export function catalogLog(args: { + rng: SeededRng; + nowMs: number; + level?: string; +}): { body: string; attrs: Record } { + const { rng, nowMs } = args; + const level = args.level ?? 'info'; + const productId = rng.hex(5).toUpperCase().slice(0, 10); + const brand = rng.pick(BRANDS); + const category = rng.pick(CATEGORIES); + const cacheHit = rng.next() < 0.78; + const inStock = rng.next() < 0.82; + const featured = rng.next() < 0.12; + const onSale = rng.next() < 0.18; + const price = rng.intRange(2999, 89999); + const fields = { + 'catalog.product_id': productId, + 'catalog.brand': brand, + 'catalog.cache_hit': cacheHit, + 'catalog.category': category, + 'catalog.in_stock': inStock, + 'catalog.is_featured': featured, + 'catalog.is_on_sale': onSale, + 'catalog.price_cents': price, + 'catalog.lookup_ms': rng.intRange(1, 80), + }; + return { + body: logfmtBody({ + rng, + nowMs, + level, + msg: 'Product found', + fields, + }), + attrs: { + 'event.name': 'catalog.lookup', + 'code.function.name': 'GetProduct', + 'code.line.number': String(rng.intRange(40, 320)), + 'code.file.path': '/app/internal/catalog/server.go', + 'product.id': productId, + 'product.brand': brand, + 'product.category': category, + 'log.iostream': 'stdout', + logtag: 'F', + }, + }; +} + +/** JSON-event body, common for app-tier services. */ +export function jsonEventBody(args: { + rng: SeededRng; + fields: Record; +}): string { + return JSON.stringify(args.fields); +} + +const EMAIL_PROVIDERS = ['ses', 'sendgrid', 'mailgun', 'postmark']; +const EMAIL_TEMPLATES = [ + 'order_confirmation', + 'shipping_update', + 'receipt', + 'password_reset', + 'welcome', +]; + +/** checkout/order JSON event with many app.* attrs. */ +export function checkoutEventLog(args: { rng: SeededRng; nowMs: number }): { + body: string; + attrs: Record; + level: string; +} { + const { rng, nowMs } = args; + const checkoutId = uuidv4(rng); + const orderId = uuidv4(rng); + const userId = uuidv4(rng); + const amountNanos = rng.intRange(1_000_000_000, 999_999_999_999); + const amountUsd = (amountNanos / 1e9).toFixed(2); + const itemCount = rng.intRange(1, 12); + const fields = { + 'app.checkout.id': checkoutId, + 'app.order.id': orderId, + 'app.order.user_id': userId, + 'app.payment.amount_nanos': amountNanos, + 'app.payment.amount_usd': amountUsd, + 'app.payment.currency': 'USD', + 'app.payment.method': rng.pick(['card', 'wallet', 'ach']), + 'app.payment.3ds_used': rng.next() < 0.4, + 'app.payment.processor': rng.pick(['stripe', 'braintree', 'adyen']), + 'app.email.provider': rng.pick(EMAIL_PROVIDERS), + 'app.email.template': rng.pick(EMAIL_TEMPLATES), + 'app.email.delivery_ms': rng.intRange(120, 4500), + 'app.email.recipient': `user${rng.intRange(1, 999_999)}@example.com`, + 'app.email.status': rng.weightedPick([ + { value: 'sent', weight: 92 }, + { value: 'queued', weight: 5 }, + { value: 'failed', weight: 3 }, + ]), + 'app.checkout.items_count': itemCount, + timestamp: new Date(nowMs).toISOString(), + }; + const level = fields['app.email.status'] === 'failed' ? 'warn' : 'info'; + return { + body: jsonEventBody({ rng, fields }), + attrs: { + 'event.name': 'checkout.complete', + 'app.checkout.id': checkoutId, + 'app.order.id': orderId, + 'code.function.name': 'placeOrder', + 'code.line.number': String(rng.intRange(40, 220)), + 'code.file.path': '/app/internal/checkout/handler.go', + 'log.iostream': 'stdout', + logtag: 'F', + }, + level, + }; +} + +const PAGES = [ + '/home', + '/product', + '/cart', + '/checkout', + '/profile', + '/orders', + '/recommendations', + '/search', +]; + +/** Frontend page render timing. Keep — it's RUM data. */ +export function pageRenderLog(args: { rng: SeededRng; nowMs: number }): { + body: string; + attrs: Record; +} { + const { rng, nowMs } = args; + const path = rng.pick(PAGES); + const renderMs = rng.intRange(40, 1200); + const ttfb = rng.intRange(20, renderMs / 2); + const fcp = rng.intRange(ttfb, renderMs); + const lcp = rng.intRange(fcp, renderMs); + const userId = `u${rng.intRange(1, 1_000_000)}`; + const sessionId = uuidv4(rng); + return { + body: logfmtBody({ + rng, + nowMs, + level: 'info', + msg: 'page.render', + fields: { + path, + render_ms: renderMs, + ttfb_ms: ttfb, + fcp_ms: fcp, + lcp_ms: lcp, + user_id: userId, + session_id: sessionId, + viewport: rng.pick(['1920x1080', '1366x768', '375x812', '1280x800']), + }, + }), + attrs: { + 'event.name': 'page.render', + 'http.route': path, + 'render.ms': String(renderMs), + 'rum.session_id': sessionId, + 'user.id': userId, + 'log.iostream': 'stdout', + logtag: 'F', + }, + }; +} + +/** search-service indexing progress. KEEP — operational signal. */ +export function indexingLog(args: { rng: SeededRng; nowMs: number }): { + body: string; + attrs: Record; +} { + const { rng, nowMs } = args; + const docs = rng.intRange(1, 5000); + const shard = rng.intRange(0, 32); + const lagMs = rng.intRange(50, 1500); + const seq = rng.intRange(1_000_000, 9_999_999); + return { + body: logfmtBody({ + rng, + nowMs, + level: 'info', + msg: 'index.shard.flush', + fields: { + 'search.docs_indexed': docs, + 'search.shard': shard, + 'search.lag_ms': lagMs, + 'search.seq': seq, + 'search.bytes_written': rng.intRange(2048, 524288), + }, + }), + attrs: { + 'event.name': 'index.shard.flush', + 'search.shard': String(shard), + 'log.iostream': 'stdout', + logtag: 'F', + }, + }; +} + +const ANALYTICS_EVENTS = [ + 'product.view', + 'cart.add', + 'cart.remove', + 'checkout.start', + 'checkout.complete', + 'search.perform', + 'search.click', + 'recommendation.click', + 'page.scroll', + 'video.play', + 'wishlist.add', +]; + +/** Analytics event log — JSON, KEEP (product signal). */ +export function analyticsEventLog(args: { rng: SeededRng; nowMs: number }): { + body: string; + attrs: Record; +} { + const { rng, nowMs } = args; + const event = rng.pick(ANALYTICS_EVENTS); + const userId = `u${rng.intRange(1, 5_000_000)}`; + const sessionId = uuidv4(rng); + const productId = rng.hex(5).toUpperCase().slice(0, 10); + const fields: Record = { + 'event.name': event, + user_id: userId, + session_id: sessionId, + timestamp: new Date(nowMs).toISOString(), + referrer: rng.weightedPick([ + { value: 'organic', weight: 50 }, + { value: 'paid', weight: 20 }, + { value: 'direct', weight: 25 }, + { value: 'email', weight: 5 }, + ]), + device: rng.pick(['desktop', 'mobile', 'tablet']), + }; + if ( + event.startsWith('product.') || + event.startsWith('cart.') || + event === 'checkout.start' + ) { + fields.product_id = productId; + fields.price_cents = rng.intRange(999, 99999); + } + if (event === 'search.perform') { + fields.query = rng.pick([ + 'nikon binoculars', + 'celestron telescope', + 'tripod', + 'eyepiece 1.25', + 'star chart', + ]); + fields.results_count = rng.intRange(0, 250); + } + return { + body: jsonEventBody({ rng, fields }), + attrs: { + 'event.name': event, + 'user.id': userId, + 'rum.session_id': sessionId, + 'log.iostream': 'stdout', + logtag: 'F', + }, + }; +} + +/** Health check / heartbeat — DROP. High volume but body has dynamic mem/cpu/uptime. */ +export function heartbeatLog(args: { + rng: SeededRng; + nowMs: number; + serviceName: string; +}): { body: string; attrs: Record; level: string } { + const { rng, nowMs } = args; + const uptimeS = rng.intRange(60, 8 * 24 * 3600); + const cpu = rng.range(0.5, 78).toFixed(2); + const mem = rng.intRange(128, 4096); + const queue = rng.intRange(0, 50); + const inflight = rng.intRange(0, 200); + return { + body: logfmtBody({ + rng, + nowMs, + level: 'debug', + msg: 'health.check', + fields: { + component: 'heartbeat', + status: 'ok', + 'infra.uptime_sec': uptimeS, + 'infra.cpu_pct': cpu, + 'infra.memory_usage_mb': mem, + 'infra.queue_depth': queue, + 'infra.inflight_requests': inflight, + }, + }), + attrs: { + 'event.name': 'health.check', + 'infra.cpu_pct': cpu, + 'infra.memory_usage_mb': String(mem), + 'infra.queue_depth': String(queue), + 'log.iostream': 'stdout', + logtag: 'F', + component: 'heartbeat', + }, + level: 'debug', + }; +} + +/** Cache hit chatter — DROP. JSON with high-cardinality keys. */ +export function cacheHitLog(args: { rng: SeededRng; nowMs: number }): { + body: string; + attrs: Record; + level: string; +} { + const { rng, nowMs } = args; + const keyKind = rng.pick([ + 'user_profile', + 'session', + 'product', + 'campaign', + 'feature_flag', + 'rate_limit', + 'auth_token', + ]); + const keyId = rng.intRange(1, 10_000_000); + const key = `${keyKind}:${keyId}`; + const layer = rng.pick(['memcached', 'redis', 'in-memory']); + const node = rng.pick(['cache-0', 'cache-1', 'cache-2', 'cache-3']); + const fields = { + event: 'cache.hit', + key, + layer, + node, + 'cache.ttl_remaining_sec': rng.intRange(1, 86400), + 'cache.value_size_bytes': rng.intRange(8, 65536), + timestamp: new Date(nowMs).toISOString(), + }; + return { + body: jsonEventBody({ rng, fields }), + attrs: { + 'event.name': 'cache.hit', + 'cache.layer': layer, + 'cache.key': key, + 'log.iostream': 'stdout', + logtag: 'F', + }, + level: 'debug', + }; +} + +const STACK_FILES = [ + '/app/worker.js', + '/app/runner.js', + '/app/lib/retry.js', + '/app/lib/queue.js', + '/app/handlers/job.js', +]; + +/** Recurring caught exception — DROP. Same exception class, varying frames. */ +export function caughtExceptionLog(args: { rng: SeededRng; nowMs: number }): { + body: string; + attrs: Record; + level: string; +} { + const { rng, nowMs } = args; + const cause = rng.pick([ + 'transient blip', + 'connection reset', + 'upstream backoff', + 'queue full', + 'rate limited downstream', + 'response timeout', + ]); + const frames = Array.from({ length: rng.intRange(3, 6) }, () => { + const file = rng.pick(STACK_FILES); + const line = rng.intRange(20, 480); + const fn = rng.pick([ + 'handleJob', + 'run', + 'process', + 'enqueue', + 'flush', + 'tick', + 'consume', + ]); + return ` at Worker.${fn} (${file}:${line}:${rng.intRange(2, 80)})`; + }); + const body = [ + `NonFatalRetryableError: ${cause} (caught)`, + ...frames, + ` at processTicksAndRejections (node:internal/process/task_queues:96:5)`, + ].join('\n'); + return { + body, + attrs: { + 'event.name': 'job.caught_exception', + 'error.kind': 'NonFatalRetryableError', + 'error.cause': cause, + handled: 'true', + 'code.function.name': 'handleJob', + 'log.iostream': 'stderr', + logtag: 'F', + // Make this look enough like a real error to fool naive severity-based filters + // but the cardinality is constrained — agent should recognize the pattern. + _ts: new Date(nowMs).toISOString(), + }, + level: 'error', + }; +} + +/** Subscription metric-as-log — DROP, convert to gauge metric. */ +export function subscriptionMetricLog(args: { + rng: SeededRng; + nowMs: number; +}): { body: string; attrs: Record; level: string } { + const { rng, nowMs } = args; + const tenant = rng.intRange(1, 50_000); + const mrrCents = rng.intRange(0, 1_000_000); + const seats = rng.intRange(1, 5000); + const trialDaysLeft = rng.intRange(0, 30); + return { + body: logfmtBody({ + rng, + nowMs, + level: 'info', + msg: 'subscription.metric.calculated', + fields: { + 'tenant.id': tenant, + 'metric.name': 'subscription_mrr', + 'metric.kind': 'gauge', + mrr_cents: mrrCents, + seats, + trial_days_left: trialDaysLeft, + plan: rng.pick(['starter', 'team', 'business', 'enterprise']), + currency: 'USD', + }, + }), + attrs: { + 'event.name': 'subscription.metric.calculated', + 'metric.name': 'subscription_mrr', + 'metric.kind': 'gauge', + 'tenant.id': String(tenant), + 'log.iostream': 'stdout', + logtag: 'F', + }, + level: 'info', + }; +} + +const BACKGROUND_BODIES_TEMPLATES = [ + (rng: SeededRng) => + `Connection pool stats: idle=${rng.intRange(0, 10)} active=${rng.intRange(0, 30)} waiting=${rng.intRange(0, 5)}`, + (rng: SeededRng) => + `Refunded order ${uuidv4(rng)} amount=${(rng.intRange(100, 99999) / 100).toFixed(2)} reason="${rng.pick(['customer request', 'duplicate charge', 'fraud review', 'pricing error'])}"`, + (rng: SeededRng) => + `Webhook delivered to https://hooks.example.com/${rng.hex(8)} status=${rng.pick([200, 201, 204, 502, 503])} retry=${rng.intRange(0, 3)}`, + (rng: SeededRng) => + `OAuth token refreshed for client_id=${rng.hex(12)} grant_type=refresh_token expires_in=${rng.intRange(300, 7200)}`, + (rng: SeededRng) => + `Slow query detected duration_ms=${rng.intRange(500, 4500)} table=${rng.pick(['orders', 'users', 'invoices', 'sessions'])} rows_scanned=${rng.intRange(10_000, 5_000_000)}`, + (rng: SeededRng) => + `Kafka producer flushed batch_size=${rng.intRange(50, 500)} bytes=${rng.intRange(1024, 1048576)} topic=${rng.pick(['orders', 'audit', 'analytics', 'notifications'])} partition=${rng.intRange(0, 32)}`, + (rng: SeededRng) => + `Migration step completed name="${rng.pick(['add_index_orders_tenant', 'backfill_users_email_lower', 'drop_legacy_sessions'])}" rows_affected=${rng.intRange(0, 10_000_000)} duration_s=${rng.intRange(1, 300)}`, + (rng: SeededRng) => + `CSRF token mismatch on POST /checkout origin=${rng.pick(['https://example.com', 'https://api.example.com', 'null'])} ip=${rng.intRange(1, 254)}.${rng.intRange(0, 256)}.${rng.intRange(0, 256)}.${rng.intRange(1, 254)}`, + (rng: SeededRng) => + `JWT verification failed reason="${rng.pick(['expired', 'invalid signature', 'unknown kid', 'malformed'])}" sub=user-${rng.intRange(1, 999999)}`, + (rng: SeededRng) => + `GDPR deletion request processed user_id=u${rng.intRange(1, 9_999_999)} records_purged=${rng.intRange(0, 50_000)} duration_ms=${rng.intRange(50, 8000)}`, + (rng: SeededRng) => + `Circuit breaker tripped service="${rng.pick(['payment-gateway', 'fraud-detection', 'shipping-api', 'tax-service'])}" failures_in_window=${rng.intRange(5, 200)} cooldown_s=${rng.intRange(15, 300)}`, + (rng: SeededRng) => + `Feature flag evaluated flag="${rng.pick(['new_checkout_flow', 'fast_search', 'price_experiment_b', 'reco_v2'])}" tenant=${rng.intRange(1, 50000)} value=${rng.next() < 0.5 ? 'true' : 'false'}`, + (rng: SeededRng) => + `Health probe response upstream="${rng.pick(['payment-service', 'inventory-service', 'cart-service'])}" code=${rng.pick([200, 200, 200, 503, 504])} latency_ms=${rng.intRange(2, 500)}`, +]; + +/** Random varied background log. */ +export function backgroundLog(args: { rng: SeededRng; nowMs: number }): string { + const tmpl = args.rng.pick(BACKGROUND_BODIES_TEMPLATES); + return tmpl(args.rng); +} + +// ─── Composite-cell helpers ───────────────────────────────────────────── +// +// These helpers exist so that each (ServiceName, SeverityText) cell can +// contain BOTH a noisy pattern and a legitimate load-bearing pattern at +// comparable volumes. That prevents a `GROUP BY ServiceName, SeverityText` +// agent from cheaply identifying noise without looking at event.name / +// body templates. + +/** + * Push-notification delivery record — KEEP. + * Lives in `notification-service × DEBUG` next to the cache.hit chatter. + * Used by support / compliance to confirm whether a user received a push. + */ +export function notificationDeliveryLog(args: { + rng: SeededRng; + nowMs: number; +}): { body: string; attrs: Record; level: string } { + const { rng, nowMs } = args; + const channel = rng.pick(['apns', 'fcm', 'webpush', 'sms']); + const template = rng.pick([ + 'order_shipped', + 'order_delivered', + 'cart_abandoned', + 'price_drop', + 'back_in_stock', + 'security_alert', + 'subscription_expiring', + ]); + const userId = `u${rng.intRange(1, 9_000_000)}`; + const deviceId = uuidv4(rng); + const messageId = uuidv4(rng); + const status = rng.weightedPick([ + { value: 'delivered', weight: 88 }, + { value: 'queued', weight: 5 }, + { value: 'throttled', weight: 4 }, + { value: 'unregistered', weight: 3 }, + ]); + return { + body: logfmtBody({ + rng, + nowMs, + level: 'debug', + msg: 'notification.delivery', + fields: { + 'notification.channel': channel, + 'notification.template': template, + 'notification.message_id': messageId, + 'notification.status': status, + 'user.id': userId, + 'device.id': deviceId, + 'notification.delivery_ms': rng.intRange(40, 2200), + 'notification.retry_count': rng.intRange(0, 3), + }, + }), + attrs: { + 'event.name': 'notification.delivery', + 'notification.channel': channel, + 'notification.status': status, + 'notification.template': template, + 'user.id': userId, + 'message.id': messageId, + 'log.iostream': 'stdout', + logtag: 'F', + }, + level: 'debug', + }; +} + +/** + * Billing financial events — KEEP. + * Lives in `billing-service × INFO` next to the subscription_mrr metric-as-log. + * These are audit/finance-critical records. + */ +export function billingEventLog(args: { rng: SeededRng; nowMs: number }): { + body: string; + attrs: Record; + level: string; +} { + const { rng, nowMs } = args; + const event = rng.weightedPick< + | 'invoice.charged' + | 'invoice.refunded' + | 'subscription.activated' + | 'subscription.cancelled' + | 'payment.failed' + >([ + { value: 'invoice.charged', weight: 55 }, + { value: 'invoice.refunded', weight: 8 }, + { value: 'subscription.activated', weight: 18 }, + { value: 'subscription.cancelled', weight: 9 }, + { value: 'payment.failed', weight: 10 }, + ]); + const tenant = rng.intRange(1, 50_000); + const invoiceId = `inv_${rng.hex(12)}`; + const customerId = `cus_${rng.hex(10)}`; + const amountCents = rng.intRange(500, 999_999); + return { + body: logfmtBody({ + rng, + nowMs, + level: 'info', + msg: event, + fields: { + 'tenant.id': tenant, + 'billing.invoice_id': invoiceId, + 'billing.customer_id': customerId, + 'billing.amount_cents': amountCents, + 'billing.currency': 'USD', + 'billing.processor': rng.pick(['stripe', 'braintree', 'adyen']), + 'billing.payment_method': rng.pick([ + 'card', + 'wallet', + 'ach', + 'invoice', + ]), + }, + }), + attrs: { + 'event.name': event, + 'tenant.id': String(tenant), + 'billing.invoice_id': invoiceId, + 'billing.customer_id': customerId, + 'log.iostream': 'stdout', + logtag: 'F', + }, + level: 'info', + }; +} + +/** + * Real, unhandled worker failure — KEEP. + * Lives in `worker × ERROR` next to the NonFatalRetryableError chatter. + * These are the errors that actually represent jobs that gave up. + */ +export function jobFailedPermanentLog(args: { + rng: SeededRng; + nowMs: number; +}): { body: string; attrs: Record; level: string } { + const { rng } = args; + const jobType = rng.pick([ + 'image_resize', + 'email_send', + 'webhook_dispatch', + 'report_export', + 'invoice_generate', + 'data_purge', + ]); + const jobId = uuidv4(rng); + const cause = rng.pick([ + 'JobPoisonPillError: input checksum mismatch', + 'JobTimeoutError: exceeded 300s deadline', + 'JobDataLossError: source row deleted mid-flight', + 'JobIntegrityError: foreign key constraint failed after retries', + ]); + const attempts = rng.intRange(5, 12); + const body = [ + `job.failed.permanent type=${jobType} id=${jobId} attempts=${attempts}`, + `${cause}`, + ` at Worker.handleJob (/app/worker.js:${rng.intRange(20, 480)}:${rng.intRange(2, 80)})`, + ` at Worker.process (/app/runner.js:${rng.intRange(20, 480)}:${rng.intRange(2, 80)})`, + ].join('\n'); + return { + body, + attrs: { + 'event.name': 'job.failed.permanent', + 'job.type': jobType, + 'job.id': jobId, + 'job.attempts': String(attempts), + 'error.kind': cause.split(':')[0], + handled: 'false', + 'log.iostream': 'stderr', + logtag: 'F', + }, + level: 'error', + }; +} + +/** + * Per-service operational debug — KEEP. + * Lives in inventory/pricing/shipping/recommendation × DEBUG next to the + * health.check heartbeats. Each service emits a distinct event.name. + */ +export function serviceOpsDebugLog(args: { + rng: SeededRng; + nowMs: number; + serviceName: + | 'inventory-service' + | 'pricing-service' + | 'shipping-service' + | 'recommendation-service'; +}): { body: string; attrs: Record; level: string } { + const { rng, nowMs, serviceName } = args; + let msg: string; + let extra: Record; + let eventName: string; + switch (serviceName) { + case 'inventory-service': { + eventName = 'inventory.stock_lookup'; + msg = 'inventory.stock_lookup'; + const sku = `sku-${rng.hex(8)}`; + extra = { + 'inventory.sku': sku, + 'inventory.warehouse': rng.pick(['us-east', 'us-west', 'eu', 'apac']), + 'inventory.on_hand': rng.intRange(0, 1200), + 'inventory.reserved': rng.intRange(0, 300), + 'inventory.lookup_ms': rng.intRange(2, 90), + }; + break; + } + case 'pricing-service': { + eventName = 'pricing.calculation'; + msg = 'pricing.calculation'; + extra = { + 'pricing.product_id': `prd-${rng.hex(8)}`, + 'pricing.tier': rng.pick(['retail', 'wholesale', 'partner']), + 'pricing.list_cents': rng.intRange(199, 199999), + 'pricing.discount_pct': rng.intRange(0, 60), + 'pricing.tax_jurisdiction': rng.pick(['US-CA', 'US-NY', 'EU-DE', 'JP']), + 'pricing.calc_ms': rng.intRange(1, 35), + }; + break; + } + case 'shipping-service': { + eventName = 'shipping.rate.quote'; + msg = 'shipping.rate.quote'; + extra = { + 'shipping.origin': rng.pick(['LAX', 'JFK', 'DFW', 'SEA', 'ATL']), + 'shipping.dest_zip': String(rng.intRange(10000, 99999)), + 'shipping.carrier': rng.pick(['ups', 'fedex', 'usps', 'dhl']), + 'shipping.weight_g': rng.intRange(50, 18000), + 'shipping.rate_cents': rng.intRange(299, 4999), + 'shipping.eta_days': rng.intRange(1, 9), + }; + break; + } + case 'recommendation-service': { + eventName = 'recommendation.score'; + msg = 'recommendation.score'; + extra = { + 'reco.model': rng.pick(['als_v3', 'content_v2', 'hybrid_v1']), + 'reco.user_id': `u${rng.intRange(1, 8_000_000)}`, + 'reco.candidates': rng.intRange(10, 500), + 'reco.returned': rng.intRange(5, 25), + 'reco.score_ms': rng.intRange(8, 220), + }; + break; + } + } + return { + body: logfmtBody({ rng, nowMs, level: 'debug', msg, fields: extra }), + attrs: { + 'event.name': eventName, + 'log.iostream': 'stdout', + logtag: 'F', + // Surface the most identifying field as an attribute too so + // attribute-grouping agents can find this pattern. + ...Object.fromEntries( + Object.entries(extra) + .filter(([, v]) => typeof v !== 'number') + .map(([k, v]) => [k, String(v)]), + ), + }, + level: 'debug', + }; +} + +/** + * Load-balancer health-probe spam — DROP (or filter at the LB layer). + * Lives in `frontend-proxy × INFO` next to the real Envoy access logs. + * Looks like an access log to a naive grouping but represents internal + * `/healthz` traffic that adds noise without diagnostic value. + */ +export function upstreamHealthProbeLog(args: { + rng: SeededRng; + nowMs: number; +}): { body: string; attrs: Record } { + const { rng, nowMs } = args; + const ts = new Date(nowMs).toISOString(); + const probe = rng.pick(['/healthz', '/-/ready', '/healthcheck', '/lb/probe']); + const upstreamMs = rng.intRange(0, 8); + const sourceIp = `10.${rng.intRange(0, 256)}.${rng.intRange(0, 256)}.${rng.intRange(1, 254)}`; + return { + body: + `[${ts}] "GET ${probe} HTTP/1.1" 200 - via_upstream - "-" ` + + `0 4 ${upstreamMs} ${upstreamMs} "-" "kube-probe/1.30" ` + + `"-" "frontend-proxy:8080" "127.0.0.1:80" frontend-proxy ` + + `${sourceIp}:0 127.0.0.1:80 - - default`, + attrs: { + 'event.name': 'lb.health.probe', + 'http.method': 'GET', + 'url.path': probe, + 'http.status_code': '200', + 'user_agent.original': 'kube-probe/1.30', + 'http.duration_ms': String(upstreamMs), + 'source.address': sourceIp, + 'server.address': 'frontend-proxy', + 'lb.probe_kind': probe.replace(/^\//, ''), + 'log.iostream': 'stdout', + 'log.file.path': '/var/log/envoy/access.log', + logtag: 'F', + }, + }; +} + +/** + * Client-side console.log dumps flushed to the backend — DROP. + * Lives in `frontend × INFO` next to the page.render RUM logs. Looks like + * a frontend log but is just chatty debug-style console output never used + * for anything. + */ +export function consoleLogDumpLog(args: { rng: SeededRng; nowMs: number }): { + body: string; + attrs: Record; +} { + const { rng, nowMs } = args; + const lines = rng.intRange(2, 6); + const components = ['Cart', 'Header', 'ProductCard', 'Footer', 'Search']; + const messages = [ + 'render', + 'state hydrated', + 'effect ran', + 'memoize hit', + 'prop drilled', + 'click handler attached', + 'noop', + ]; + const dump: string[] = []; + for (let i = 0; i < lines; i++) { + const comp = rng.pick(components); + const msg = rng.pick(messages); + dump.push( + `[console.log] ${comp}.${msg} ts=${nowMs - rng.intRange(0, 300)}`, + ); + } + const sessionId = uuidv4(rng); + return { + body: `console.log_dump session=${sessionId}\n` + dump.join('\n'), + attrs: { + 'event.name': 'frontend.console.log_dump', + 'rum.session_id': sessionId, + 'frontend.line_count': String(lines), + 'log.iostream': 'stdout', + logtag: 'F', + }, + }; +} + +/** + * Verbose cache-miss event from search-service — DROP. + * Lives in `search-service × INFO` next to the legit index.shard.flush + * KEEP logs. Looks like a normal INFO log but is chatty per-query. + */ +export function searchCacheMissVerboseLog(args: { + rng: SeededRng; + nowMs: number; +}): { body: string; attrs: Record } { + const { rng, nowMs } = args; + const query = `q-${rng.hex(10)}`; + return { + body: logfmtBody({ + rng, + nowMs, + level: 'info', + msg: 'search.cache.miss', + fields: { + 'search.query_hash': query, + 'search.cache_layer': rng.pick(['l1', 'l2', 'edge']), + 'search.shard': rng.intRange(0, 32), + 'search.miss_reason': rng.pick([ + 'ttl_expired', + 'evicted', + 'not_present', + 'invalidated', + ]), + 'search.fallback_ms': rng.intRange(5, 90), + }, + }), + attrs: { + 'event.name': 'search.cache.miss', + 'search.query_hash': query, + 'log.iostream': 'stdout', + logtag: 'F', + }, + }; +} diff --git a/packages/hdx-eval/src/generators/traces.ts b/packages/hdx-eval/src/generators/traces.ts new file mode 100644 index 0000000000..f433f888f2 --- /dev/null +++ b/packages/hdx-eval/src/generators/traces.ts @@ -0,0 +1,50 @@ +import type { SeededRng } from '../rng/seeded'; +import type { TraceRow } from './types'; + +export function newTraceId(rng: SeededRng): string { + return rng.hex(16); +} + +export function newSpanId(rng: SeededRng): string { + return rng.hex(8); +} + +export type SpanInput = { + rng: SeededRng; + timestampMs: number; + traceId: string; + spanId: string; + parentSpanId?: string; + spanName: string; + spanKind?: TraceRow['spanKind']; + serviceName: string; + durationNs: number; + statusCode?: TraceRow['statusCode']; + statusMessage?: string; + resourceAttributes?: Record; + spanAttributes?: Record; +}; + +export function makeSpan(input: SpanInput): TraceRow { + return { + timestampMs: input.timestampMs, + traceId: input.traceId, + spanId: input.spanId, + parentSpanId: input.parentSpanId ?? '', + spanName: input.spanName, + spanKind: input.spanKind ?? 'SPAN_KIND_INTERNAL', + serviceName: input.serviceName, + durationNs: input.durationNs, + statusCode: input.statusCode ?? 'STATUS_CODE_OK', + statusMessage: input.statusMessage ?? '', + resourceAttributes: { + 'service.name': input.serviceName, + ...(input.resourceAttributes ?? {}), + }, + spanAttributes: input.spanAttributes ?? {}, + }; +} + +export function msToNs(ms: number): number { + return Math.floor(ms * 1_000_000); +} diff --git a/packages/hdx-eval/src/generators/types.ts b/packages/hdx-eval/src/generators/types.ts new file mode 100644 index 0000000000..d0d70ee780 --- /dev/null +++ b/packages/hdx-eval/src/generators/types.ts @@ -0,0 +1,39 @@ +export type TraceRow = { + timestampMs: number; + traceId: string; + spanId: string; + parentSpanId: string; + spanName: string; + spanKind: + | 'SPAN_KIND_SERVER' + | 'SPAN_KIND_CLIENT' + | 'SPAN_KIND_INTERNAL' + | 'SPAN_KIND_PRODUCER' + | 'SPAN_KIND_CONSUMER'; + serviceName: string; + durationNs: number; + statusCode: 'STATUS_CODE_OK' | 'STATUS_CODE_ERROR' | 'STATUS_CODE_UNSET'; + statusMessage: string; + resourceAttributes: Record; + spanAttributes: Record; +}; + +export type LogRow = { + timestampMs: number; + traceId?: string; + spanId?: string; + serviceName: string; + severityText: 'TRACE' | 'DEBUG' | 'INFO' | 'WARN' | 'ERROR'; + severityNumber: number; + body: string; + resourceAttributes: Record; + logAttributes: Record; +}; + +export const SEVERITY_NUMBER: Record = { + TRACE: 1, + DEBUG: 5, + INFO: 9, + WARN: 13, + ERROR: 17, +}; diff --git a/packages/hdx-eval/src/grading/blind.ts b/packages/hdx-eval/src/grading/blind.ts new file mode 100644 index 0000000000..6a4d8da490 --- /dev/null +++ b/packages/hdx-eval/src/grading/blind.ts @@ -0,0 +1,24 @@ +/** + * Blind candidate answers before showing them to the judge so the judge + * can't tell which MCP the agent used. We replace MCP-identifying tool + * names and brand mentions with neutral placeholders. We do NOT touch + * data references like `payment-service` or `database.query` — those + * are part of the answer being graded. + */ + +const TOOL_PREFIX = /\bmcp__(hyperdx|clickhouse)__/gi; +const HYPERDX_TOOL_NAMES = + /\bhyperdx_(query|list_sources|get_dashboard|save_dashboard|query_tile|delete_dashboard)\b/gi; +const CLICKHOUSE_TOOL_NAMES = + /\b(run_query|run_select_query|list_tables|list_databases|describe_table|run_chdb_select_query)\b/gi; +const HYPERDX_BRAND = /\bHyperDX(\s+MCP)?\b/g; +const CLICKHOUSE_BRAND_MCP = /\bClickHouse\s+MCP\b/g; + +export function blindAnswer(text: string): string { + return text + .replace(TOOL_PREFIX, 'mcp__redacted__') + .replace(HYPERDX_TOOL_NAMES, 'mcp_query') + .replace(CLICKHOUSE_TOOL_NAMES, 'mcp_query') + .replace(HYPERDX_BRAND, 'MCP A') + .replace(CLICKHOUSE_BRAND_MCP, 'MCP B'); +} diff --git a/packages/hdx-eval/src/grading/grade.ts b/packages/hdx-eval/src/grading/grade.ts new file mode 100644 index 0000000000..24da24882d --- /dev/null +++ b/packages/hdx-eval/src/grading/grade.ts @@ -0,0 +1,261 @@ +import Anthropic from '@anthropic-ai/sdk'; +import { existsSync, readdirSync, readFileSync, writeFileSync } from 'fs'; +import { dirname, join } from 'path'; + +import type { RunRecord } from '../harness/types'; +import { runsRoot } from '../runs/path'; +import { readRun } from '../runs/store'; +import { getScenario, SCENARIO_NAMES } from '../scenarios'; +import { judgeTrajectory } from './judge'; +import { runProgrammaticChecks } from './programmatic'; +import { loadScenarioRubric } from './rubric'; +import { + COMBINED_SCORE_JUDGE_WEIGHT, + COMBINED_SCORE_PROGRAMMATIC_WEIGHT, + type GradeRecord, + type JudgeResult, + MAX_ERROR_PENALTY, + type ToolErrorStats, +} from './types'; + +// Infrastructure-error patterns that come from the MCP/CH server itself +// rather than from agent mistakes. We exclude these from the tool-error +// penalty so a server hiccup doesn't get charged against the agent. +const INFRA_ERROR_PATTERNS: RegExp[] = [ + /too many simultaneous queries/i, + /stream has been already consumed/i, + /rate.?limit/i, + /429/, + /503\b/, + /504\b/, + /econn(refused|reset|aborted)/i, + /socket hang up/i, + /timeout.{0,20}(connecting|while waiting)/i, +]; + +function isInfraError(text: string): boolean { + return INFRA_ERROR_PATTERNS.some(rx => rx.test(text)); +} + +function computeToolErrorStats(record: RunRecord): ToolErrorStats { + const total = record.toolCalls.length; + const errored = record.toolCalls.filter(c => c.isError); + // Separate agent-attributable errors from infrastructure failures (rate + // limits, server-side bugs, transient 5xx). Only agent errors feed the + // penalty — the agent has no control over server hiccups. + const agentErrored = errored.filter(c => { + const text = + typeof c.output === 'string' ? c.output : JSON.stringify(c.output ?? ''); + return !isInfraError(text); + }); + const errors = agentErrored.length; + const rate = total > 0 ? errors / total : 0; + // Linear penalty up to MAX_ERROR_PENALTY when ALL tool calls fail. + // Mathematically: penalty = min(rate, MAX_ERROR_PENALTY). + // We intentionally use rate (not errors/maxTurns) so a run that calls + // 1 tool which fails (rate=1.0) is penalized as harshly as a run that + // fails 20 of 20 tool calls — both are equally broken sessions. + const penalty = Math.min(rate, MAX_ERROR_PENALTY); + const samples = agentErrored.slice(0, 3).map(c => { + const text = + typeof c.output === 'string' ? c.output : JSON.stringify(c.output ?? ''); + return { + name: c.name, + preview: text.length > 160 ? text.slice(0, 160) + '…' : text, + }; + }); + return { total, errors, rate, penalty, samples }; +} + +export type GradeBatchOptions = { + judgeModel?: string; + rerunJudge?: boolean; + skipJudge?: boolean; + apiKey?: string; +}; + +export type GradeBatchSummary = { + batchDir: string; + graded: GradeRecord[]; + errors: { runPath: string; error: string }[]; +}; + +export async function gradeBatch( + batchDir: string, + opts: GradeBatchOptions = {}, +): Promise { + const resolved = resolveBatchDir(batchDir); + if (!existsSync(resolved)) { + throw new Error(`Batch directory does not exist: ${resolved}`); + } + const runFiles = listRunFiles(resolved); + + // Decide whether we'll actually need the judge: skipJudge wins, otherwise + // we need it if any run is missing a cached judge OR rerun was requested. + const needsJudge = + !opts.skipJudge && + runFiles.some(p => { + if (opts.rerunJudge) return true; + const existing = readExistingGrade(gradeFilePath(p)); + return !existing?.judge; + }); + + if (needsJudge && !opts.apiKey && !process.env.ANTHROPIC_API_KEY) { + throw new Error( + 'ANTHROPIC_API_KEY is not set; pass --no-judge to skip the LLM judge ' + + 'or supply a key via env.', + ); + } + + const client = needsJudge + ? new Anthropic({ + apiKey: opts.apiKey ?? process.env.ANTHROPIC_API_KEY, + }) + : undefined; + const graded: GradeRecord[] = []; + const errors: { runPath: string; error: string }[] = []; + + for (const runPath of runFiles) { + try { + const record = readRun(runPath); + const gradePath = gradeFilePath(runPath); + const existing = readExistingGrade(gradePath); + + const grade = await gradeOne({ + record, + client, + existing, + opts, + }); + writeGradeFile(gradePath, grade); + graded.push(grade); + const judgeBit = grade.judge + ? `judge=${(grade.judge.weightedScore * 5).toFixed(1)}/5` + : 'judge=skipped'; + const errBit = grade.toolErrors.errors + ? ` errs=${grade.toolErrors.errors}/${grade.toolErrors.total}` + + (grade.toolErrors.penalty > 0 + ? ` (-${(grade.toolErrors.penalty * 100).toFixed(0)}pp)` + : '') + : ''; + console.log( + ` ${grade.scenario}/${grade.mcp}/${grade.runId.split('-').slice(-1)[0]} prog=${( + grade.programmatic.score * 100 + ).toFixed( + 0, + )}% ${judgeBit} combined=${(grade.combinedScore * 100).toFixed(0)}%${errBit}`, + ); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + errors.push({ runPath, error: msg }); + console.error(` ${runPath}: ${msg}`); + } + } + + return { batchDir: resolved, graded, errors }; +} + +async function gradeOne(args: { + record: RunRecord; + client?: Anthropic; + existing: GradeRecord | null; + opts: GradeBatchOptions; +}): Promise { + const { record, client, existing, opts } = args; + const scenario = getScenario(record.scenario); + const rubric = loadScenarioRubric(record.scenario); + + const programmatic = runProgrammaticChecks( + record.finalAnswer, + rubric.programmatic, + ); + + let judge: JudgeResult | null = existing?.judge ?? null; + if (!opts.skipJudge && client && (!judge || opts.rerunJudge)) { + judge = await judgeTrajectory({ + scenarioName: scenario.name, + scenarioPrompt: scenario.agentPrompt, + groundTruth: scenario.groundTruth, + rubric, + finalAnswer: record.finalAnswer, + judgeModel: opts.judgeModel, + client, + }); + } + + const judgeScore = judge?.weightedScore ?? 0; + const rawCombined = judge + ? COMBINED_SCORE_PROGRAMMATIC_WEIGHT * programmatic.score + + COMBINED_SCORE_JUDGE_WEIGHT * judgeScore + : programmatic.score; + const toolErrors = computeToolErrorStats(record); + // Apply the tool-error penalty AFTER scoring the answer. Clamp to [0,1]. + const combinedScore = Math.max( + 0, + Math.min(1, rawCombined - toolErrors.penalty), + ); + + return { + schemaVersion: 2, + runId: record.runId, + scenario: record.scenario, + mcp: record.mcp, + programmatic, + judge, + toolErrors, + combinedScore, + gradedAt: new Date().toISOString(), + judgeModel: judge?.model ?? opts.judgeModel ?? 'skipped', + }; +} + +function listRunFiles(batchDir: string): string[] { + const out: string[] = []; + for (const scenario of safeReaddir(batchDir)) { + if (!SCENARIO_NAMES.includes(scenario)) continue; + const sceneDir = join(batchDir, scenario); + for (const mcp of safeReaddir(sceneDir)) { + const mcpDir = join(sceneDir, mcp); + for (const file of safeReaddir(mcpDir)) { + if (file.endsWith('.grade.json')) continue; + if (file.endsWith('.json')) out.push(join(mcpDir, file)); + } + } + } + return out.sort(); +} + +function safeReaddir(path: string): string[] { + try { + return readdirSync(path); + } catch { + return []; + } +} + +function readExistingGrade(path: string): GradeRecord | null { + if (!existsSync(path)) return null; + try { + return JSON.parse(readFileSync(path, 'utf8')) as GradeRecord; + } catch { + return null; + } +} + +function writeGradeFile(path: string, grade: GradeRecord): void { + writeFileSync(path, JSON.stringify(grade, null, 2) + '\n', 'utf8'); +} + +function gradeFilePath(runPath: string): string { + return runPath.replace(/\.json$/, '.grade.json'); +} + +export function resolveBatchDir(input: string): string { + // Accept absolute path, relative path, or just a batch basename. + if (existsSync(input)) return input; + const root = runsRoot(); + const candidate = join(root, input); + if (existsSync(candidate)) return candidate; + // Allow user to drop the bare basename even if not yet present. + return dirname(input) === '.' ? candidate : input; +} diff --git a/packages/hdx-eval/src/grading/judge.ts b/packages/hdx-eval/src/grading/judge.ts new file mode 100644 index 0000000000..6cb1b52a82 --- /dev/null +++ b/packages/hdx-eval/src/grading/judge.ts @@ -0,0 +1,259 @@ +import Anthropic from '@anthropic-ai/sdk'; + +import { blindAnswer } from './blind'; +import { + buildJudgeSystem, + buildJudgeUser, + formatGroundTruthFacts, +} from './judgePrompt'; +import type { JudgeCriterionScore, JudgeResult, Rubric } from './types'; + +const DEFAULT_JUDGE_MODEL = 'claude-opus-4-7'; +const MAX_TOKENS = 1500; + +export type JudgeOptions = { + scenarioName: string; + scenarioPrompt: string; + groundTruth: unknown; + rubric: Rubric; + finalAnswer: string; + judgeModel?: string; + client?: Anthropic; +}; + +export async function judgeTrajectory( + opts: JudgeOptions, +): Promise { + const model = opts.judgeModel ?? DEFAULT_JUDGE_MODEL; + const client = + opts.client ?? new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY }); + + const systemPrompt = buildJudgeSystem(opts.scenarioName, opts.rubric); + const userPrompt = buildJudgeUser({ + scenarioPrompt: opts.scenarioPrompt, + groundTruthFacts: formatGroundTruthFacts(opts.groundTruth), + candidateAnswer: blindAnswer(opts.finalAnswer), + }); + + const startedMs = Date.now(); + const result = await callJudgeWithRetry({ + client, + model, + systemPrompt, + userPrompt, + }); + const durationMs = Date.now() - startedMs; + + if (result.error) { + return { + model, + scores: {}, + weightedScore: 0, + rawResponse: result.rawResponse, + durationMs, + tokens: result.tokens, + error: result.error, + }; + } + + const validated = validateAndScore(result.parsed, opts.rubric); + if (validated.error) { + return { + model, + scores: validated.scores, + weightedScore: 0, + rawResponse: result.rawResponse, + durationMs, + tokens: result.tokens, + error: validated.error, + }; + } + + return { + model, + scores: validated.scores, + weightedScore: validated.weightedScore, + rawResponse: result.rawResponse, + durationMs, + tokens: result.tokens, + }; +} + +type JudgeApiOutcome = { + parsed: unknown; + rawResponse: string; + tokens: JudgeResult['tokens']; + error?: string; +}; + +async function callJudgeWithRetry(args: { + client: Anthropic; + model: string; + systemPrompt: string; + userPrompt: string; +}): Promise { + const first = await callJudgeOnce(args); + if (!first.error) return first; + + // Retry once with a stricter reminder appended. + const retry = await callJudgeOnce({ + ...args, + userPrompt: + args.userPrompt + + '\n\nReturn STRICT JSON only — no Markdown fences, no prose.', + }); + if (retry.error) { + // Surface the second error but accumulate tokens from both attempts. + retry.tokens = sumTokens(first.tokens, retry.tokens); + } + return retry; +} + +async function callJudgeOnce(args: { + client: Anthropic; + model: string; + systemPrompt: string; + userPrompt: string; +}): Promise { + let response: Anthropic.Message; + try { + response = await args.client.messages.create({ + model: args.model, + max_tokens: MAX_TOKENS, + system: [ + { + type: 'text', + text: args.systemPrompt, + cache_control: { type: 'ephemeral' }, + }, + ], + messages: [{ role: 'user', content: args.userPrompt }], + }); + } catch (err) { + return { + parsed: null, + rawResponse: '', + tokens: { input: 0, output: 0, cacheCreation: 0, cacheRead: 0 }, + error: + err instanceof Anthropic.APIError + ? `${err.constructor.name} (${err.status}): ${err.message}` + : err instanceof Error + ? err.message + : String(err), + }; + } + + const text = response.content + .filter((b): b is Anthropic.TextBlock => b.type === 'text') + .map(b => b.text) + .join('\n'); + + const tokens = { + input: response.usage.input_tokens, + output: response.usage.output_tokens, + cacheCreation: response.usage.cache_creation_input_tokens ?? 0, + cacheRead: response.usage.cache_read_input_tokens ?? 0, + }; + + const parsed = tryParseJson(text); + if (parsed === null) { + return { + parsed: null, + rawResponse: text, + tokens, + error: 'judge response was not valid JSON', + }; + } + return { parsed, rawResponse: text, tokens }; +} + +function tryParseJson(text: string): unknown { + const trimmed = text.trim(); + // Direct parse first. + try { + return JSON.parse(trimmed); + } catch { + // fall through + } + // Extract the first {...} block in case the model wrapped JSON in prose. + const start = trimmed.indexOf('{'); + const end = trimmed.lastIndexOf('}'); + if (start === -1 || end <= start) return null; + try { + return JSON.parse(trimmed.slice(start, end + 1)); + } catch { + return null; + } +} + +function validateAndScore( + parsed: unknown, + rubric: Rubric, +): { + scores: Record; + weightedScore: number; + error?: string; +} { + if (!parsed || typeof parsed !== 'object') { + return { scores: {}, weightedScore: 0, error: 'judge JSON not an object' }; + } + const obj = parsed as Record; + const rawScores = obj.scores; + if (!rawScores || typeof rawScores !== 'object') { + return { + scores: {}, + weightedScore: 0, + error: 'judge response missing `scores` object', + }; + } + const scoresMap = rawScores as Record; + + const scores: Record = {}; + let weightedSum = 0; + let totalWeight = 0; + const missing: string[] = []; + + for (const criterion of rubric.judge.criteria) { + const entry = scoresMap[criterion.id]; + if (!entry || typeof entry !== 'object') { + missing.push(criterion.id); + continue; + } + const e = entry as Record; + const rawScore = typeof e.score === 'number' ? e.score : Number(e.score); + if (!Number.isFinite(rawScore)) { + missing.push(criterion.id); + continue; + } + const clamped = Math.max(0, Math.min(5, Math.round(rawScore))); + scores[criterion.id] = { + score: clamped, + rationale: typeof e.rationale === 'string' ? e.rationale : '', + }; + weightedSum += clamped * criterion.weight; + totalWeight += criterion.weight; + } + + if (missing.length > 0) { + return { + scores, + weightedScore: 0, + error: `judge omitted criteria: ${missing.join(', ')}`, + }; + } + + const weightedScore = totalWeight === 0 ? 0 : weightedSum / (5 * totalWeight); + return { scores, weightedScore }; +} + +function sumTokens( + a: JudgeResult['tokens'], + b: JudgeResult['tokens'], +): JudgeResult['tokens'] { + return { + input: a.input + b.input, + output: a.output + b.output, + cacheCreation: a.cacheCreation + b.cacheCreation, + cacheRead: a.cacheRead + b.cacheRead, + }; +} diff --git a/packages/hdx-eval/src/grading/judgePrompt.ts b/packages/hdx-eval/src/grading/judgePrompt.ts new file mode 100644 index 0000000000..082ca8639c --- /dev/null +++ b/packages/hdx-eval/src/grading/judgePrompt.ts @@ -0,0 +1,77 @@ +import type { Rubric } from './types'; + +const SYSTEM_PREAMBLE = `You are evaluating an SRE investigation. You will receive: +- the scenario question (what the candidate was asked) +- the ground-truth facts (the planted answer the candidate did NOT see) +- a rubric with weighted criteria +- the candidate's final answer (anonymized — tool names and product brands have been redacted so you cannot tell which tool the candidate used) + +For each rubric criterion, output an integer score from 0 to 5 plus a one-sentence rationale. Do not consider tool choice, query syntax, or implementation details — score only the quality of the candidate's final answer relative to the ground truth. + +Return STRICT JSON of shape: +{ "scores": { "": { "score": N, "rationale": "..." } } } +No prose outside the JSON. Include every criterion id from the rubric.`; + +export function buildJudgeSystem(scenarioName: string, rubric: Rubric): string { + const criteriaSection = rubric.judge.criteria + .map(c => `- "${c.id}" (weight ${c.weight}): ${c.description}`) + .join('\n'); + return [ + SYSTEM_PREAMBLE, + '', + `SCENARIO: ${scenarioName}`, + '', + 'RUBRIC CRITERIA:', + criteriaSection, + ].join('\n'); +} + +export function buildJudgeUser(args: { + scenarioPrompt: string; + groundTruthFacts: string; + candidateAnswer: string; +}): string { + return [ + 'SCENARIO QUESTION:', + args.scenarioPrompt, + '', + 'GROUND-TRUTH FACTS (internal only — the candidate did not see these):', + args.groundTruthFacts, + '', + 'CANDIDATE FINAL ANSWER (anonymized):', + args.candidateAnswer, + ].join('\n'); +} + +/** + * Renders the `expected` and `anomalyAttributes` portions of a scenario + * ground-truth.json as a human-readable bulleted summary for the judge. + * Strips internal-only fields (rubric, agentPrompt, scenario name). + */ +export function formatGroundTruthFacts(groundTruth: unknown): string { + if (!groundTruth || typeof groundTruth !== 'object') return '(none)'; + const gt = groundTruth as Record; + const lines: string[] = []; + const expected = gt.expected; + if (expected && typeof expected === 'object') { + lines.push('Expected:'); + for (const [k, v] of Object.entries(expected)) { + lines.push(` - ${k}: ${formatValue(v)}`); + } + } + const anomaly = gt.anomalyAttributes; + if (anomaly && typeof anomaly === 'object') { + lines.push('Anomaly attributes:'); + for (const [k, v] of Object.entries(anomaly)) { + lines.push(` - ${k}: ${formatValue(v)}`); + } + } + return lines.join('\n'); +} + +function formatValue(v: unknown): string { + if (v === null || v === undefined) return 'null'; + if (typeof v === 'string') return v; + if (typeof v === 'number' || typeof v === 'boolean') return String(v); + return JSON.stringify(v); +} diff --git a/packages/hdx-eval/src/grading/programmatic.ts b/packages/hdx-eval/src/grading/programmatic.ts new file mode 100644 index 0000000000..769081d1ae --- /dev/null +++ b/packages/hdx-eval/src/grading/programmatic.ts @@ -0,0 +1,42 @@ +import type { + ProgrammaticCheck, + ProgrammaticHit, + ProgrammaticResult, +} from './types'; + +export function runProgrammaticChecks( + answer: string, + checks: ProgrammaticCheck[], +): ProgrammaticResult { + const hits: ProgrammaticHit[] = []; + let totalWeight = 0; + let hitWeight = 0; + + for (const check of checks) { + totalWeight += check.weight; + const flags = check.flags ?? 'i'; + let regex: RegExp; + try { + regex = new RegExp(check.pattern, flags); + } catch (err) { + throw new Error( + `Invalid regex in programmatic check '${check.id}': ${ + err instanceof Error ? err.message : String(err) + }`, + ); + } + const matched = regex.test(answer); + const satisfied = check.negative ? !matched : matched; + if (satisfied) hitWeight += check.weight; + hits.push({ + id: check.id, + weight: check.weight, + matched, + satisfied, + negative: check.negative, + }); + } + + const score = totalWeight === 0 ? 0 : hitWeight / totalWeight; + return { hits, score }; +} diff --git a/packages/hdx-eval/src/grading/rubric.ts b/packages/hdx-eval/src/grading/rubric.ts new file mode 100644 index 0000000000..07f3f44c6e --- /dev/null +++ b/packages/hdx-eval/src/grading/rubric.ts @@ -0,0 +1,77 @@ +import { getScenario } from '../scenarios'; +import type { Rubric } from './types'; + +export function loadScenarioRubric(scenarioName: string): Rubric { + const scenario = getScenario(scenarioName); + const gt = scenario.groundTruth as { rubric?: unknown }; + if (!gt || typeof gt !== 'object' || !gt.rubric) { + throw new Error( + `Scenario '${scenarioName}' has no \`rubric\` block in ground-truth.json`, + ); + } + return validateRubric(gt.rubric, scenarioName); +} + +export function getScenarioGroundTruth(scenarioName: string): unknown { + return getScenario(scenarioName).groundTruth; +} + +function validateRubric(raw: unknown, scenarioName: string): Rubric { + if (!raw || typeof raw !== 'object') { + throw new Error(`rubric for '${scenarioName}' must be an object`); + } + const obj = raw as Record; + if (!Array.isArray(obj.programmatic)) { + throw new Error( + `rubric.programmatic for '${scenarioName}' must be an array`, + ); + } + for (const c of obj.programmatic as unknown[]) { + if (!c || typeof c !== 'object') { + throw new Error( + `rubric.programmatic for '${scenarioName}' has a non-object entry`, + ); + } + const check = c as Record; + if (typeof check.id !== 'string' || typeof check.pattern !== 'string') { + throw new Error( + `rubric.programmatic for '${scenarioName}': each check needs string 'id' and 'pattern'`, + ); + } + if (typeof check.weight !== 'number' || check.weight <= 0) { + throw new Error( + `rubric.programmatic for '${scenarioName}': check '${check.id}' weight must be a positive number`, + ); + } + if (check.negative !== undefined && typeof check.negative !== 'boolean') { + throw new Error( + `rubric.programmatic for '${scenarioName}': check '${check.id}' 'negative' must be a boolean`, + ); + } + } + const judge = obj.judge as { criteria?: unknown } | undefined; + if (!judge || !Array.isArray(judge.criteria) || judge.criteria.length === 0) { + throw new Error( + `rubric.judge.criteria for '${scenarioName}' must be a non-empty array`, + ); + } + for (const c of judge.criteria as unknown[]) { + if (!c || typeof c !== 'object') { + throw new Error( + `rubric.judge.criteria for '${scenarioName}' has a non-object entry`, + ); + } + const crit = c as Record; + if (typeof crit.id !== 'string' || typeof crit.description !== 'string') { + throw new Error( + `rubric.judge.criteria for '${scenarioName}': each criterion needs string 'id' and 'description'`, + ); + } + if (typeof crit.weight !== 'number' || crit.weight <= 0) { + throw new Error( + `rubric.judge.criteria for '${scenarioName}': criterion '${crit.id}' weight must be a positive number`, + ); + } + } + return raw as Rubric; +} diff --git a/packages/hdx-eval/src/grading/types.ts b/packages/hdx-eval/src/grading/types.ts new file mode 100644 index 0000000000..80e15cec58 --- /dev/null +++ b/packages/hdx-eval/src/grading/types.ts @@ -0,0 +1,100 @@ +import type { McpKind } from '../harness/types'; + +export type ProgrammaticCheck = { + id: string; + weight: number; + pattern: string; + flags?: string; + // When true, the check is satisfied when the pattern does NOT match the + // answer. Used to penalize wrong attributions / blamed distractors. + negative?: boolean; +}; + +export type JudgeCriterion = { + id: string; + weight: number; + description: string; +}; + +export type Rubric = { + programmatic: ProgrammaticCheck[]; + judge: { criteria: JudgeCriterion[] }; +}; + +export type ProgrammaticHit = { + id: string; + weight: number; + matched: boolean; + satisfied: boolean; + negative?: boolean; +}; + +export type ProgrammaticResult = { + hits: ProgrammaticHit[]; + score: number; // 0..1 +}; + +export type JudgeCriterionScore = { + score: number; // 0..5 + rationale: string; +}; + +export type JudgeResult = { + model: string; + scores: Record; + weightedScore: number; // 0..1, sum(score*weight) / (5*sum(weight)) + rawResponse: string; + durationMs: number; + tokens: { + input: number; + output: number; + cacheCreation: number; + cacheRead: number; + }; + error?: string; +}; + +/** + * Per-run tool-error stats — computed from the saved RunRecord, not the + * agent's answer. Bad runs (lots of failed tool calls) get a combined-score + * penalty even if the final answer happens to be correct. + */ +export type ToolErrorStats = { + /** Total tool calls observed. */ + total: number; + /** Tool calls flagged isError by the harness/MCP. */ + errors: number; + /** errors / total (0 when total = 0). */ + rate: number; + /** Penalty applied to combinedScore: clamp(rate, 0, MAX_ERROR_PENALTY). */ + penalty: number; + /** First few error tool names + a short snippet of the error text. */ + samples: Array<{ name: string; preview: string }>; +}; + +export type GradeRecord = { + schemaVersion: 2; + runId: string; + scenario: string; + mcp: McpKind; + programmatic: ProgrammaticResult; + judge: JudgeResult | null; + toolErrors: ToolErrorStats; + /** + * combinedScore = clamp01( + * PROGRAMMATIC_WEIGHT * programmatic + JUDGE_WEIGHT * judge + * - toolErrors.penalty + * ) + * The penalty is at most MAX_ERROR_PENALTY (0.2) — a run can't be reduced + * to zero from a high error rate alone, but spamming bad tool calls makes + * an otherwise-good answer measurably worse. + */ + combinedScore: number; + gradedAt: string; + judgeModel: string; +}; + +export const COMBINED_SCORE_PROGRAMMATIC_WEIGHT = 0.4; +export const COMBINED_SCORE_JUDGE_WEIGHT = 0.6; +/** Maximum penalty applied for tool errors (subtracted from combined score). */ +export const MAX_ERROR_PENALTY = 0.2; diff --git a/packages/hdx-eval/src/harness/claudeSpawn.ts b/packages/hdx-eval/src/harness/claudeSpawn.ts new file mode 100644 index 0000000000..68d5f3d405 --- /dev/null +++ b/packages/hdx-eval/src/harness/claudeSpawn.ts @@ -0,0 +1,146 @@ +import { spawn } from 'child_process'; +import { mkdtempSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; + +import type { EvalConfig } from '../hyperdx/config'; +import { allowedToolsPattern, buildMcpConfig } from './mcpConfig'; +import { buildSettings, deniedToolsFor } from './settingsFile'; +import { + chunkToLines, + type ParsedEvent, + parseStreamLine, +} from './streamParser'; +import type { McpKind, PromptVariant } from './types'; + +export type SpawnOptions = { + config: EvalConfig; + scenario: string; + mcp: McpKind; + model: string; + maxTurns: number; + timeoutMs: number; + agentPrompt: string; + systemPromptAppend: string; + apiKey: string; + /** Prompt variant — affects which built-in tools are denied. The + * `hypothesis` variant allows the Task tool so the agent can spawn + * subagents. Default 'baseline'. */ + promptVariant?: PromptVariant; +}; + +export type SpawnResult = { + events: ParsedEvent[]; + stderr: string; + exitCode: number | null; + signal: NodeJS.Signals | null; + timedOut: boolean; + argv: string[]; + tempdir: string; +}; + +export async function runClaude(opts: SpawnOptions): Promise { + const tempdir = mkdtempSync(join(tmpdir(), 'hdx-eval-')); + const mcpConfigPath = join(tempdir, 'mcp-config.json'); + const settingsPath = join(tempdir, 'settings.json'); + + writeFileSync( + mcpConfigPath, + JSON.stringify(buildMcpConfig(opts.config, opts.mcp), null, 2), + ); + const promptVariant: PromptVariant = opts.promptVariant ?? 'baseline'; + writeFileSync( + settingsPath, + JSON.stringify(buildSettings(opts.mcp, promptVariant), null, 2), + ); + + const argv = [ + '-p', + '--mcp-config', + mcpConfigPath, + '--allowedTools', + allowedToolsPattern(opts.mcp), + '--disallowedTools', + deniedToolsFor(promptVariant).join(','), + '--dangerously-skip-permissions', + '--setting-sources', + 'local', + '--settings', + settingsPath, + '--append-system-prompt', + opts.systemPromptAppend, + '--output-format', + 'stream-json', + '--verbose', + '--model', + opts.model, + '--max-turns', + String(opts.maxTurns), + opts.agentPrompt, + ]; + + // Run claude with cwd=tempdir (a fresh isolated dir). The agent can't + // Bash/Read/Glob into the repo to peek at ground-truth, prior eval runs, + // or the eval config — the only state visible is the mcp-config + settings + // files we just wrote here. + const proc = spawn('claude', argv, { + env: { ...process.env, ANTHROPIC_API_KEY: opts.apiKey }, + cwd: tempdir, + stdio: ['ignore', 'pipe', 'pipe'], + }); + + const events: ParsedEvent[] = []; + let stdoutBuffer = ''; + let stderr = ''; + let timedOut = false; + + const stdoutDone = new Promise(resolve => { + proc.stdout.setEncoding('utf8'); + proc.stdout.on('data', (chunk: string) => { + const { events: lines, remainder } = chunkToLines(stdoutBuffer, chunk); + stdoutBuffer = remainder; + for (const line of lines) { + const parsed = parseStreamLine(line); + if (parsed) events.push(parsed); + } + }); + proc.stdout.on('end', () => { + if (stdoutBuffer.trim()) { + const parsed = parseStreamLine(stdoutBuffer); + if (parsed) events.push(parsed); + } + resolve(); + }); + }); + + proc.stderr.setEncoding('utf8'); + proc.stderr.on('data', (chunk: string) => { + stderr += chunk; + }); + + const timeout = setTimeout(() => { + timedOut = true; + proc.kill('SIGTERM'); + setTimeout(() => proc.kill('SIGKILL'), 5_000).unref(); + }, opts.timeoutMs); + + const exitInfo = await new Promise<{ + code: number | null; + signal: NodeJS.Signals | null; + }>((resolve, reject) => { + proc.on('error', reject); + proc.on('exit', (code, signal) => resolve({ code, signal })); + }); + clearTimeout(timeout); + await stdoutDone; + + return { + events, + stderr, + exitCode: exitInfo.code, + signal: exitInfo.signal, + timedOut, + argv, + tempdir, + }; +} diff --git a/packages/hdx-eval/src/harness/mcpConfig.ts b/packages/hdx-eval/src/harness/mcpConfig.ts new file mode 100644 index 0000000000..49769d87d0 --- /dev/null +++ b/packages/hdx-eval/src/harness/mcpConfig.ts @@ -0,0 +1,59 @@ +import type { EvalConfig } from '../hyperdx/config'; +import type { McpKind } from './types'; + +export const MCP_CLICKHOUSE_VERSION = '0.3.0'; + +// Simulate a production observability backend: queries that scan the whole +// table or don't use filters/indexes time out. Forces agents to write +// efficient queries instead of `SELECT * FROM huge_table`. +export const QUERY_TIMEOUT_SECONDS = 10; + +export function buildMcpConfig( + cfg: EvalConfig, + kind: McpKind, +): Record { + if (kind === 'hyperdx') { + return { + mcpServers: { + hyperdx: { + type: 'http', + url: cfg.hyperdx.mcpUrl, + headers: { + Authorization: `Bearer ${cfg.hyperdx.accessKey}`, + }, + }, + }, + }; + } + return { + mcpServers: { + clickhouse: { + command: 'uv', + args: [ + 'run', + '--with', + `mcp-clickhouse==${MCP_CLICKHOUSE_VERSION}`, + '--python', + '3.10', + 'mcp-clickhouse', + ], + env: { + CLICKHOUSE_HOST: cfg.clickhouse.host, + CLICKHOUSE_PORT: cfg.clickhouse.port, + CLICKHOUSE_USER: cfg.clickhouse.user, + CLICKHOUSE_PASSWORD: cfg.clickhouse.password, + CLICKHOUSE_SECURE: 'false', + CLICKHOUSE_VERIFY: 'false', + // Tighten the client-side timeout to match the per-query + // max_execution_time set on the HyperDX side, so both MCPs face + // the same constraint. + CLICKHOUSE_SEND_RECEIVE_TIMEOUT: String(QUERY_TIMEOUT_SECONDS), + }, + }, + }, + }; +} + +export function allowedToolsPattern(kind: McpKind): string { + return kind === 'hyperdx' ? 'mcp__hyperdx__*' : 'mcp__clickhouse__*'; +} diff --git a/packages/hdx-eval/src/harness/runRun.ts b/packages/hdx-eval/src/harness/runRun.ts new file mode 100644 index 0000000000..d54ba4f9c1 --- /dev/null +++ b/packages/hdx-eval/src/harness/runRun.ts @@ -0,0 +1,233 @@ +import type { EvalConfig } from '../hyperdx/config'; +import { runClaude } from './claudeSpawn'; +import type { ParsedEvent } from './streamParser'; +import { buildSystemPrompt } from './systemPrompt'; +import type { + McpKind, + PromptVariant, + RunRecord, + Termination, + ToolCallRecord, +} from './types'; + +export type RunCellOptions = { + config: EvalConfig; + scenario: string; + agentPrompt: string; + mcp: McpKind; + model: string; + maxTurns: number; + timeoutMs: number; + runIndex: number; + seed: number; + apiKey: string; + /** When set, the agent's system prompt declares this ISO timestamp as the + * fixed "now" for any relative-time reference in the user prompt. */ + anchorTimeIso?: string; + /** Prompt variant — 'baseline' (default) or 'hypothesis' (forces + * hypothesis-enumeration + parallel Task subagents). */ + promptVariant?: PromptVariant; +}; + +export async function runCell(opts: RunCellOptions): Promise { + const startedAtIso = new Date().toISOString(); + const startedAtMs = Date.now(); + const promptVariant: PromptVariant = opts.promptVariant ?? 'baseline'; + const systemPromptAppend = buildSystemPrompt( + opts.scenario, + opts.mcp, + opts.anchorTimeIso, + promptVariant, + ); + + const result = await runClaude({ + config: opts.config, + scenario: opts.scenario, + mcp: opts.mcp, + model: opts.model, + maxTurns: opts.maxTurns, + timeoutMs: opts.timeoutMs, + agentPrompt: opts.agentPrompt, + systemPromptAppend, + apiKey: opts.apiKey, + promptVariant, + }); + + const endedAtIso = new Date().toISOString(); + const endedAtMs = Date.now(); + + return assembleRecord({ + events: result.events, + stderr: result.stderr, + exitCode: result.exitCode, + timedOut: result.timedOut, + scenario: opts.scenario, + agentPrompt: opts.agentPrompt, + systemPromptAppend, + mcp: opts.mcp, + model: opts.model, + runIndex: opts.runIndex, + seed: opts.seed, + startedAtIso, + endedAtIso, + durationMs: endedAtMs - startedAtMs, + }); +} + +type AssembleInput = { + events: ParsedEvent[]; + stderr: string; + exitCode: number | null; + timedOut: boolean; + scenario: string; + agentPrompt: string; + systemPromptAppend: string; + mcp: McpKind; + model: string; + runIndex: number; + seed: number; + startedAtIso: string; + endedAtIso: string; + durationMs: number; +}; + +export function assembleRecord(input: AssembleInput): RunRecord { + const toolCalls: ToolCallRecord[] = []; + const callsById = new Map(); + const tools: { name: string; description?: string }[] = []; + let lastAssistantText = ''; + let resultUsage: + | { + input_tokens?: number; + output_tokens?: number; + cache_creation_input_tokens?: number; + cache_read_input_tokens?: number; + } + | undefined; + let resultText: string | undefined; + let resultSubtype: string | undefined; + let resultIsError = false; + let totalCostUsd: number | null = null; + + for (const ev of input.events) { + if (ev.kind === 'system_init') { + if (ev.tools) tools.push(...ev.tools); + } else if (ev.kind === 'assistant_message') { + const ts = new Date().toISOString(); + for (const block of ev.content) { + if (!block || typeof block !== 'object') continue; + const b = block as Record; + if (b.type === 'text' && typeof b.text === 'string') { + lastAssistantText = b.text; + } else if (b.type === 'tool_use') { + const id = typeof b.id === 'string' ? b.id : null; + if (!id) continue; + const call: ToolCallRecord = { + name: typeof b.name === 'string' ? b.name : 'unknown', + input: b.input ?? null, + output: null, + isError: false, + startedAt: ts, + endedAt: null, + durationMs: null, + }; + callsById.set(id, call); + toolCalls.push(call); + } + } + } else if (ev.kind === 'user_message') { + const ts = new Date().toISOString(); + for (const block of ev.content) { + if (!block || typeof block !== 'object') continue; + const b = block as Record; + if (b.type !== 'tool_result') continue; + const id = typeof b.tool_use_id === 'string' ? b.tool_use_id : null; + if (!id) continue; + const call = callsById.get(id); + if (!call) continue; + call.output = stringifyToolResult(b.content); + call.isError = b.is_error === true; + call.endedAt = ts; + const startMs = Date.parse(call.startedAt); + const endMs = Date.parse(ts); + if (Number.isFinite(startMs) && Number.isFinite(endMs)) { + call.durationMs = Math.max(0, endMs - startMs); + } + } + } else if (ev.kind === 'result') { + resultUsage = ev.usage; + resultText = ev.resultText; + resultSubtype = ev.subtype; + resultIsError = ev.isError; + if (typeof ev.totalCostUsd === 'number') totalCostUsd = ev.totalCostUsd; + } + } + + const finalAnswer = resultText ?? lastAssistantText; + const termination = determineTermination({ + timedOut: input.timedOut, + exitCode: input.exitCode, + resultSubtype, + resultIsError, + hasFinalAnswer: !!finalAnswer, + }); + + return { + schemaVersion: 1, + runId: `${input.startedAtIso}-${input.scenario}-${input.mcp}-${input.runIndex}`, + scenario: input.scenario, + mcp: input.mcp, + model: input.model, + runIndex: input.runIndex, + seed: input.seed, + startedAt: input.startedAtIso, + endedAt: input.endedAtIso, + durationMs: input.durationMs, + agentPrompt: input.agentPrompt, + systemPromptAppend: input.systemPromptAppend, + termination, + exitCode: input.exitCode, + tools, + toolCalls, + messages: input.events.map(e => e.raw), + finalAnswer, + tokens: { + input: resultUsage?.input_tokens ?? 0, + output: resultUsage?.output_tokens ?? 0, + cacheCreation: resultUsage?.cache_creation_input_tokens ?? 0, + cacheRead: resultUsage?.cache_read_input_tokens ?? 0, + }, + totalCostUsd, + stderr: input.stderr, + }; +} + +function stringifyToolResult(content: unknown): string { + if (typeof content === 'string') return content; + if (Array.isArray(content)) { + return content + .map(c => { + if (!c || typeof c !== 'object') return JSON.stringify(c); + const o = c as Record; + if (typeof o.text === 'string') return o.text; + return JSON.stringify(o); + }) + .join('\n'); + } + return JSON.stringify(content); +} + +function determineTermination(args: { + timedOut: boolean; + exitCode: number | null; + resultSubtype?: string; + resultIsError: boolean; + hasFinalAnswer: boolean; +}): Termination { + if (args.timedOut) return 'timeout'; + if (args.resultSubtype === 'error_max_turns') return 'max_turns'; + if (args.resultIsError) return 'error'; + if (args.exitCode !== null && args.exitCode !== 0) return 'error'; + if (args.hasFinalAnswer) return 'final_answer'; + return 'error'; +} diff --git a/packages/hdx-eval/src/harness/settingsFile.ts b/packages/hdx-eval/src/harness/settingsFile.ts new file mode 100644 index 0000000000..2ea1dd0e6e --- /dev/null +++ b/packages/hdx-eval/src/harness/settingsFile.ts @@ -0,0 +1,51 @@ +import { allowedToolsPattern } from './mcpConfig'; +import type { McpKind, PromptVariant } from './types'; + +/** + * Filesystem / shell / built-in tools we never want the eval agent to use. + * The agent's working directory is a fresh tempdir (no answer key) — but + * Bash/Read/Glob/Grep would still let it shell out to /home/ and read prior + * runs or the source tree. Lock it to the MCP tools only. + * + * `Task` (Claude Code's subagent-spawning tool) is denied by default but + * ALLOWED in the `hypothesis` prompt variant — that variant explicitly + * asks the agent to spawn parallel subagents to investigate multiple + * hypotheses concurrently. + */ +export const DENIED_BUILT_IN_TOOLS_BASE = [ + 'Bash', + 'Read', + 'Write', + 'Edit', + 'MultiEdit', + 'NotebookEdit', + 'Glob', + 'Grep', + 'WebFetch', + 'WebSearch', + 'TodoWrite', +] as const; + +/** Retained for backward-compatibility with the existing claudeSpawn import. */ +export const DENIED_BUILT_IN_TOOLS = [ + ...DENIED_BUILT_IN_TOOLS_BASE, + 'Task', +] as const; + +export function deniedToolsFor(variant: PromptVariant): readonly string[] { + return variant === 'hypothesis' + ? DENIED_BUILT_IN_TOOLS_BASE + : DENIED_BUILT_IN_TOOLS; +} + +export function buildSettings( + kind: McpKind, + variant: PromptVariant = 'baseline', +): Record { + return { + permissions: { + allow: [allowedToolsPattern(kind)], + deny: [...deniedToolsFor(variant)], + }, + }; +} diff --git a/packages/hdx-eval/src/harness/streamParser.ts b/packages/hdx-eval/src/harness/streamParser.ts new file mode 100644 index 0000000000..bed4669d9c --- /dev/null +++ b/packages/hdx-eval/src/harness/streamParser.ts @@ -0,0 +1,156 @@ +/** + * Parser for `claude -p --output-format stream-json` event lines. + * Tolerant of unknown event shapes — passes through as Unknown. + */ + +export type Usage = { + input_tokens?: number; + output_tokens?: number; + cache_creation_input_tokens?: number; + cache_read_input_tokens?: number; +}; + +export type SystemInitEvent = { + kind: 'system_init'; + sessionId?: string; + model?: string; + tools?: { name: string; description?: string }[]; + raw: unknown; +}; + +export type AssistantMessageEvent = { + kind: 'assistant_message'; + content: unknown[]; + usage?: Usage; + raw: unknown; +}; + +export type UserMessageEvent = { + kind: 'user_message'; + content: unknown[]; + raw: unknown; +}; + +export type ResultEvent = { + kind: 'result'; + subtype?: string; + isError: boolean; + durationMs?: number; + totalCostUsd?: number; + usage?: Usage; + resultText?: string; + raw: unknown; +}; + +export type UnknownEvent = { + kind: 'unknown'; + type?: string; + raw: unknown; +}; + +export type ParsedEvent = + | SystemInitEvent + | AssistantMessageEvent + | UserMessageEvent + | ResultEvent + | UnknownEvent; + +export function parseStreamLine(line: string): ParsedEvent | null { + const trimmed = line.trim(); + if (!trimmed) return null; + let raw: unknown; + try { + raw = JSON.parse(trimmed); + } catch { + return null; + } + if (!raw || typeof raw !== 'object') return null; + const obj = raw as Record; + const type = typeof obj.type === 'string' ? obj.type : undefined; + + if (type === 'system' && obj.subtype === 'init') { + return { + kind: 'system_init', + sessionId: + typeof obj.session_id === 'string' ? obj.session_id : undefined, + model: typeof obj.model === 'string' ? obj.model : undefined, + tools: Array.isArray(obj.tools) + ? (obj.tools as unknown[]) + .map(toToolDescriptor) + .filter((t): t is { name: string } => t !== null) + : undefined, + raw, + }; + } + + if (type === 'assistant') { + const message = (obj.message ?? {}) as Record; + return { + kind: 'assistant_message', + content: Array.isArray(message.content) + ? (message.content as unknown[]) + : [], + usage: message.usage as Usage | undefined, + raw, + }; + } + + if (type === 'user') { + const message = (obj.message ?? {}) as Record; + return { + kind: 'user_message', + content: Array.isArray(message.content) + ? (message.content as unknown[]) + : [], + raw, + }; + } + + if (type === 'result') { + return { + kind: 'result', + subtype: typeof obj.subtype === 'string' ? obj.subtype : undefined, + isError: obj.is_error === true, + durationMs: + typeof obj.duration_ms === 'number' ? obj.duration_ms : undefined, + totalCostUsd: + typeof obj.total_cost_usd === 'number' ? obj.total_cost_usd : undefined, + usage: obj.usage as Usage | undefined, + resultText: typeof obj.result === 'string' ? obj.result : undefined, + raw, + }; + } + + return { kind: 'unknown', type, raw }; +} + +function toToolDescriptor( + v: unknown, +): { name: string; description?: string } | null { + if (typeof v === 'string') return { name: v }; + if (v && typeof v === 'object') { + const obj = v as Record; + if (typeof obj.name === 'string') { + return { + name: obj.name, + description: + typeof obj.description === 'string' ? obj.description : undefined, + }; + } + } + return null; +} + +/** + * Splits a chunk of streamed bytes into complete JSON-line events plus a + * remainder. Preserves the trailing partial line for the next chunk. + */ +export function chunkToLines( + buffer: string, + chunk: string, +): { events: string[]; remainder: string } { + const combined = buffer + chunk; + const parts = combined.split('\n'); + const remainder = parts.pop() ?? ''; + return { events: parts.filter(p => p.length > 0), remainder }; +} diff --git a/packages/hdx-eval/src/harness/systemPrompt.ts b/packages/hdx-eval/src/harness/systemPrompt.ts new file mode 100644 index 0000000000..9ed3776586 --- /dev/null +++ b/packages/hdx-eval/src/harness/systemPrompt.ts @@ -0,0 +1,139 @@ +import { scenarioTables } from '../clickhouse/schema'; +import type { McpKind, PromptVariant } from './types'; + +export function buildSystemPrompt( + scenario: string, + mcp: McpKind, + anchorTimeIso?: string, + variant: PromptVariant = 'baseline', +): string { + const { traces, logs } = scenarioTables(scenario); + const sharedSchema = ''; + // `These follow the standard OpenTelemetry ClickHouse schema: + // - traces have Timestamp DateTime64(9), TraceId, SpanId, ParentSpanId, + // ServiceName, SpanName, SpanKind, Duration (nanoseconds), StatusCode, + // StatusMessage, plus Map(String,String) columns ResourceAttributes and + // SpanAttributes. + // - logs have Timestamp DateTime64(9), TraceId, SpanId, ServiceName, + // SeverityText, SeverityNumber, Body, plus Map(String,String) columns + // ResourceAttributes and LogAttributes.`; + + const mcpSpecific = + mcp === 'hyperdx' + ? `HyperDX Sources are pre-configured for these tables (named +\`eval-${scenario}-traces\` and \`eval-${scenario}-logs\`). Call +\`hyperdx_list_sources\` first to discover source IDs. + +Available HyperDX investigation tools (call directly — names + purposes +below so you don't need to ToolSearch every tool): +- hyperdx_list_sources Catalog: sources + connections (their IDs, kinds, key columns) +- hyperdx_describe_source Full column schema + map attribute keys + low-cardinality values for one source +- hyperdx_timeseries Plot a metric over time; supports groupBy + granularity +- hyperdx_table Group-by aggregations. Can take MULTIPLE select items in one call (count + p50 + p99 in one shot). Map attributes work in both groupBy and valueExpression — \`SpanAttributes['']\` is a valid groupBy expression. Default shape="table" works for everything; only use number for a single scalar. +- hyperdx_search Browse individual rows ordered by timestamp +- hyperdx_sql Raw ClickHouse SQL. Only reach for this when the builder tools genuinely cannot express the query — multi-aggregate and attribute-map access are both supported by hyperdx_table. +- hyperdx_log_patterns Cluster log bodies into Drain templates, ranked by frequency, with per-pattern trend buckets +- hyperdx_event_deltas *** PREFER THIS over manual GROUP BYs when the question is "which attribute separates these two row groups". Examples: errors vs successes, slow vs fast spans, after-deploy vs before-deploy, one cohort vs the rest. One call ranks ALL attributes by signal strength. Whenever you find yourself running a sequence of GROUP BYs on different columns trying to find a discriminator, that's the cue to use event_deltas instead. +- hyperdx_trace_waterfall Full span tree for ONE trace (by TraceId or auto-picked via pickFilter + pickBy) +- hyperdx_trace_top_time_consuming_operations + Aggregate child-span breakdown across MANY traces matching a parent filter. Ranks operations by total time spent — answers "where is the time going inside these slow requests?" + +ToolSearch is still needed to load the full input schema for tools you've +never used in this session, but you don't need to ToolSearch for the +list itself — the catalog above is complete.` + : `Use \`run_query\` to issue SQL against the tables above. Schema +introspection is available via \`list_databases\` and \`list_tables\`. + +Note: ClickHouse will NOT implicitly cast a bare ISO-8601 string to +\`DateTime64\`. \`WHERE Timestamp >= ''\` fails with "Cannot convert +string ... to type DateTime64(9)". Wrap with +\`parseDateTime64BestEffortOrNull('')\` or use \`toDateTime64('', 9)\`.`; + + // When the harness anchors a run to a fixed past time, the agent must use + // that anchor as "now" for any relative window in the user's prompt. Without + // this, the model uses today's date and queries an empty future window. + const anchorBlock = anchorTimeIso + ? `\nFIXED CURRENT TIME: ${anchorTimeIso} +All "now", "recently", "in the last N minutes/hours" references in the user's +prompt are anchored to this timestamp. When you query, use absolute ISO +timestamps relative to this anchor — do NOT use today's wall-clock date.\n` + : ''; + + // Hypothesis-variant playbook — encourages hypothesis enumeration and + // (optionally) parallel subagent investigation. Goal: break the + // "commit to the first thing you see" anchoring failure we observe on + // incident scenarios, without paying the subagent overhead when a + // straightforward investigation will do. + const playbookBlock = + variant === 'hypothesis' + ? ` + +INVESTIGATION GUIDANCE: + +1. Before investigating, briefly list the candidate hypotheses about what + could be causing what the user is asking about. List as few or as many + as the problem warrants — one if it's obvious, several if it's + ambiguous. Each hypothesis should be a specific testable claim + ("X is caused by Y in component Z"). Consider whether what looks like + one problem might actually be two or more independent issues happening + at once — if that's plausible, include "multiple independent causes" + as one of the hypotheses worth investigating. + +2. Decide whether to investigate sequentially yourself or to fan out via + parallel Task subagents. The built-in Task tool is available — use it + when you have several plausible hypotheses that you'd otherwise have + to investigate one at a time, or when the question is ambiguous enough + that you'd benefit from having multiple framings explored + independently. Skip it when the answer is likely a single short query + chain. Subagents inherit the same MCP tools you have. If you spawn + subagents, send them in the same response so they run concurrently and + give each a focused prompt that says what would confirm or refute the + hypothesis and what unexpected findings to surface. + +3. Synthesize what you (and your subagents) found. Watch specifically + for: (a) two independent things going wrong at once that look like + one — the loudest signal can mask a quieter co-occurring issue, + (b) subagent unanimous agreement that's actually all four latching + onto the same loud signal rather than independent confirmation, + (c) evidence that refutes the framing entirely and points somewhere + you didn't list. + +4. Then write the final answer. + +The goal is to avoid both common failure modes: committing to the first +plausible story (anchoring) AND ignoring the obvious in favor of busywork +(over-investigation). Spend the budget where it actually pays. +` + : ''; + + return `You are an SRE answering an operational question using +observability data. The OpenTelemetry data lives in ClickHouse: + +- Traces: default.${traces} +- Logs: default.${logs} +${anchorBlock} +${sharedSchema} + +${mcpSpecific} +${playbookBlock} +Investigate the user's question. In your final answer, identify: +- the specific service(s) and operation(s) involved +- the root cause (or, for cleanup-style questions, the top concrete actions) +- supporting evidence from the data (counts, time ranges, sample messages) + +Stay open to the possibility that what looks like one issue is actually +multiple independent issues — when two components look similarly degraded, +confirm they share the same underlying cause before declaring one. + +Before finalizing a root cause, briefly check whether your explanation +fully accounts for the data you've seen. Note any datapoints that don't +fit, and decide whether the gap is benign noise or evidence that the +real story is more complex. + +When the user's question is about understanding why something is slow +or what's gone wrong, drill into the cause rather than stopping at the +symptom. Use tools that aggregate the underlying contributors rather +than reading individual events one by one. + +Be concise. No filler — the reader is technical.`; +} diff --git a/packages/hdx-eval/src/harness/types.ts b/packages/hdx-eval/src/harness/types.ts new file mode 100644 index 0000000000..d6a85bd94c --- /dev/null +++ b/packages/hdx-eval/src/harness/types.ts @@ -0,0 +1,51 @@ +export type McpKind = 'hyperdx' | 'clickhouse'; + +/** + * Prompt variants for the system-prompt A/B. + * - `baseline`: the default investigative prompt, no subagent affordance. + * - `hypothesis`: hard playbook that asks the agent to enumerate 2–4 + * hypotheses up front, spawn a Task subagent per hypothesis to + * investigate in parallel, then synthesize. + */ +export type PromptVariant = 'baseline' | 'hypothesis'; + +export type Termination = 'final_answer' | 'max_turns' | 'timeout' | 'error'; + +export type ToolCallRecord = { + name: string; + input: unknown; + output: string | null; + isError: boolean; + startedAt: string; + endedAt: string | null; + durationMs: number | null; +}; + +export type RunRecord = { + schemaVersion: 1; + runId: string; + scenario: string; + mcp: McpKind; + model: string; + runIndex: number; + seed: number; + startedAt: string; + endedAt: string; + durationMs: number; + agentPrompt: string; + systemPromptAppend: string; + termination: Termination; + exitCode: number | null; + tools: { name: string; description?: string }[]; + toolCalls: ToolCallRecord[]; + messages: unknown[]; + finalAnswer: string; + tokens: { + input: number; + output: number; + cacheCreation: number; + cacheRead: number; + }; + totalCostUsd: number | null; + stderr: string; +}; diff --git a/packages/hdx-eval/src/hyperdx/api.ts b/packages/hdx-eval/src/hyperdx/api.ts new file mode 100644 index 0000000000..2243c8a58f --- /dev/null +++ b/packages/hdx-eval/src/hyperdx/api.ts @@ -0,0 +1,197 @@ +type CookieJar = Map; + +export type HyperdxConnection = { + _id: string; + name: string; + host: string; + username: string; + password?: string; +}; + +export type HyperdxSource = { + _id: string; + id: string; + name: string; + kind: 'log' | 'trace' | 'session' | 'metric'; + connection: string; + from: { databaseName: string; tableName: string }; + timestampValueExpression: string; + [key: string]: unknown; +}; + +export type MeResponse = { + id: string; + email: string; + accessKey: string; + team: { _id: string; name: string }; +}; + +export class HyperdxApiClient { + private cookies: CookieJar = new Map(); + + constructor(private readonly apiUrl: string) {} + + private url(path: string): string { + return `${this.apiUrl.replace(/\/$/, '')}${path}`; + } + + private cookieHeader(): string { + return Array.from(this.cookies.entries()) + .map(([k, v]) => `${k}=${v}`) + .join('; '); + } + + private absorbSetCookie(res: Response): void { + // Node's fetch exposes Set-Cookie via getSetCookie() (Node 22+). + const headers = res.headers as Headers & { + getSetCookie?: () => string[]; + }; + const list = headers.getSetCookie?.() ?? []; + for (const raw of list) { + const [pair] = raw.split(';'); + const eq = pair.indexOf('='); + if (eq <= 0) continue; + const name = pair.slice(0, eq).trim(); + const value = pair.slice(eq + 1).trim(); + this.cookies.set(name, value); + } + } + + private async request( + method: string, + path: string, + body?: unknown, + ): Promise<{ status: number; body: T }> { + const headers: Record = { + Accept: 'application/json', + }; + if (body !== undefined) headers['Content-Type'] = 'application/json'; + const cookie = this.cookieHeader(); + if (cookie) headers['Cookie'] = cookie; + + const res = await fetch(this.url(path), { + method, + headers, + body: body !== undefined ? JSON.stringify(body) : undefined, + redirect: 'manual', + }); + this.absorbSetCookie(res); + + const text = await res.text(); + let parsed: unknown = text; + if (text && (res.headers.get('content-type') ?? '').includes('json')) { + try { + parsed = JSON.parse(text); + } catch { + // keep as text + } + } + + if (res.status >= 400) { + const snippet = + typeof parsed === 'string' + ? parsed.slice(0, 500) + : JSON.stringify(parsed).slice(0, 500); + throw new Error( + `HyperDX API ${method} ${path} → ${res.status}: ${snippet}`, + ); + } + return { status: res.status, body: parsed as T }; + } + + async ping(): Promise { + try { + const res = await fetch(this.url('/'), { method: 'GET' }); + return res.status < 500; + } catch { + return false; + } + } + + async register( + email: string, + password: string, + ): Promise<'created' | 'exists'> { + const res = await fetch(this.url('/register/password'), { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ email, password, confirmPassword: password }), + redirect: 'manual', + }); + this.absorbSetCookie(res); + if (res.status === 200) return 'created'; + // Existing accounts return 400 with { error: 'invalid' }; treat as 'exists' + // and let login retry verify the credentials match. + return 'exists'; + } + + async login(email: string, password: string): Promise { + const res = await fetch(this.url('/login/password'), { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ email, password }), + redirect: 'manual', + }); + this.absorbSetCookie(res); + // Passport login redirects to /login?err=authFail on bad credentials. + // The redirect itself is 3xx (not 4xx), so we have to inspect the + // Location header to detect failure. + const location = res.headers.get('location') ?? ''; + if (/err=/.test(location)) { + throw new Error( + `HyperDX login failed: passport returned auth-failure redirect to ${location}`, + ); + } + if (res.status >= 400) { + const text = await res.text(); + throw new Error( + `HyperDX login failed (${res.status}): ${text.slice(0, 300)}`, + ); + } + } + + async me(): Promise { + const { body } = await this.request('GET', '/me'); + return body; + } + + async listConnections(): Promise { + const { body } = await this.request( + 'GET', + '/connections', + ); + return body; + } + + async createConnection(input: { + name: string; + host: string; + username: string; + password?: string; + }): Promise<{ id: string }> { + const { body } = await this.request<{ id: string }>( + 'POST', + '/connections', + input, + ); + return body; + } + + async listSources(): Promise { + const { body } = await this.request('GET', '/sources'); + return body; + } + + async createSource(input: Record): Promise { + const { body } = await this.request( + 'POST', + '/sources', + input, + ); + return body; + } + + async deleteSource(id: string): Promise { + await this.request('DELETE', `/sources/${id}`); + } +} diff --git a/packages/hdx-eval/src/hyperdx/config.ts b/packages/hdx-eval/src/hyperdx/config.ts new file mode 100644 index 0000000000..770988a53b --- /dev/null +++ b/packages/hdx-eval/src/hyperdx/config.ts @@ -0,0 +1,103 @@ +import { existsSync, readFileSync, writeFileSync } from 'fs'; +import { dirname, resolve } from 'path'; + +import { SCENARIO_NAMES } from '../scenarios'; + +export type ScenarioSourceIds = { + tracesSourceId: string; + logsSourceId: string; +}; + +export type EvalConfig = { + hyperdx: { + apiUrl: string; + mcpUrl: string; + accessKey: string; + connectionId: string; + scenarios: Record; + }; + clickhouse: { + host: string; + port: string; + user: string; + password: string; + }; +}; + +export const CONFIG_FILENAME = 'eval.config.json'; + +export function configPath(): string { + // Resolve relative to the package root regardless of cwd. + // dist/cli.js → dist/ → packageRoot. src/cli.ts via tsx → src/ → packageRoot. + return resolve(__dirname, '..', '..', CONFIG_FILENAME); +} + +export function configExists(path: string = configPath()): boolean { + return existsSync(path); +} + +export function readConfig(path: string = configPath()): EvalConfig { + if (!existsSync(path)) { + throw new Error( + `Eval config not found at ${path}. Run \`hdx-eval setup-hyperdx\` first.`, + ); + } + const raw = JSON.parse(readFileSync(path, 'utf8')); + return validateConfig(raw, path); +} + +export function writeConfig( + config: EvalConfig, + path: string = configPath(), +): void { + validateConfig(config, path); + writeFileSync(path, JSON.stringify(config, null, 2) + '\n', 'utf8'); +} + +function validateConfig(raw: unknown, path: string): EvalConfig { + if (!raw || typeof raw !== 'object') { + throw new Error(`Eval config at ${path} is not an object.`); + } + const obj = raw as Record; + const hdx = obj.hyperdx as Record | undefined; + const ch = obj.clickhouse as Record | undefined; + if (!hdx) throw new Error(`Eval config missing 'hyperdx' section`); + if (!ch) throw new Error(`Eval config missing 'clickhouse' section`); + + for (const f of ['apiUrl', 'mcpUrl', 'accessKey', 'connectionId'] as const) { + if (typeof hdx[f] !== 'string' || !hdx[f]) { + throw new Error(`Eval config 'hyperdx.${f}' must be a non-empty string`); + } + } + const scenarios = hdx.scenarios as Record; + if (!scenarios || typeof scenarios !== 'object') { + throw new Error(`Eval config 'hyperdx.scenarios' must be an object`); + } + for (const name of SCENARIO_NAMES) { + const s = scenarios[name]; + if (!s || !s.tracesSourceId || !s.logsSourceId) { + throw new Error( + `Eval config missing source IDs for scenario '${name}'. ` + + `Re-run \`hdx-eval setup-hyperdx\`.`, + ); + } + } + + for (const f of ['host', 'port', 'user'] as const) { + if (typeof ch[f] !== 'string') { + throw new Error(`Eval config 'clickhouse.${f}' must be a string`); + } + } + if (typeof ch.password !== 'string') { + throw new Error(`Eval config 'clickhouse.password' must be a string`); + } + + return raw as EvalConfig; +} + +export function ensureConfigDir(path: string = configPath()): void { + const dir = dirname(path); + if (!existsSync(dir)) { + throw new Error(`Config directory does not exist: ${dir}`); + } +} diff --git a/packages/hdx-eval/src/hyperdx/setup.ts b/packages/hdx-eval/src/hyperdx/setup.ts new file mode 100644 index 0000000000..38f008d0a6 --- /dev/null +++ b/packages/hdx-eval/src/hyperdx/setup.ts @@ -0,0 +1,297 @@ +import { scenarioTables } from '../clickhouse/schema'; +import { QUERY_TIMEOUT_SECONDS } from '../harness/mcpConfig'; +import { SCENARIO_NAMES } from '../scenarios'; +import { HyperdxApiClient, type HyperdxSource } from './api'; +import { + configPath, + type EvalConfig, + readConfig as readEvalConfig, + writeConfig, +} from './config'; + +const EVAL_CONNECTION_NAME = 'hdx-eval-clickhouse'; +const TRACE_DEFAULT_SELECT = + 'Timestamp, ServiceName, SpanName, Duration, StatusCode'; +const LOG_DEFAULT_SELECT = 'Timestamp, ServiceName, SeverityText, Body'; + +export type SetupOptions = { + apiUrl: string; + email: string; + password: string; + clickhouse: { + host: string; + port: string; + user: string; + password: string; + }; + resetSources?: boolean; +}; + +export type SetupResult = { + configPath: string; + config: EvalConfig; + created: { connection: boolean; sources: string[] }; +}; + +export async function runSetup(opts: SetupOptions): Promise { + const api = new HyperdxApiClient(opts.apiUrl); + + // 1. Register (idempotent — accepts "already exists") then login. + await api.register(opts.email, opts.password); + await api.login(opts.email, opts.password); + + // 2. Fetch user accessKey for MCP Bearer auth. + const me = await api.me(); + + // 3. Ensure eval Connection exists pointing at the local ClickHouse. + const connections = await api.listConnections(); + const chHost = `http://${opts.clickhouse.host}:${opts.clickhouse.port}`; + let connection = connections.find(c => c.name === EVAL_CONNECTION_NAME); + let createdConnection = false; + if (!connection) { + const { id } = await api.createConnection({ + name: EVAL_CONNECTION_NAME, + host: chHost, + username: opts.clickhouse.user, + password: opts.clickhouse.password, + }); + connection = { + _id: id, + name: EVAL_CONNECTION_NAME, + host: chHost, + username: opts.clickhouse.user, + }; + createdConnection = true; + } + + // 4. Ensure one Trace Source + one Log Source per scenario. + let sources = await api.listSources(); + if (opts.resetSources) { + const evalSources = sources.filter(s => s.name.startsWith('eval-')); + for (const s of evalSources) { + await api.deleteSource(sourceId(s)); + } + sources = await api.listSources(); + } + const sourcesByName = new Map(sources.map(s => [s.name, s])); + const created: string[] = []; + const scenarioIds: Record< + string, + { tracesSourceId: string; logsSourceId: string } + > = {}; + + for (const name of SCENARIO_NAMES) { + const tables = scenarioTables(name); + const traceName = `eval-${name}-traces`; + const logName = `eval-${name}-logs`; + + let traceSource = sourcesByName.get(traceName); + if (!traceSource) { + traceSource = await api.createSource( + buildTraceSourceBody(traceName, connection._id, tables.traces), + ); + created.push(traceName); + } + + let logSource = sourcesByName.get(logName); + if (!logSource) { + logSource = await api.createSource( + buildLogSourceBody(logName, connection._id, tables.logs), + ); + created.push(logName); + } + + scenarioIds[name] = { + tracesSourceId: sourceId(traceSource), + logsSourceId: sourceId(logSource), + }; + } + + const config: EvalConfig = { + hyperdx: { + apiUrl: opts.apiUrl, + mcpUrl: `${opts.apiUrl.replace(/\/$/, '')}/mcp`, + accessKey: me.accessKey, + connectionId: connection._id, + scenarios: scenarioIds, + }, + clickhouse: opts.clickhouse, + }; + writeConfig(config); + + return { + configPath: configPath(), + config, + created: { connection: createdConnection, sources: created }, + }; +} + +function sourceId(source: HyperdxSource): string { + return source.id ?? source._id; +} + +// Apply a per-query max_execution_time to every Source so the HyperDX MCP +// faces the same query timeout as the ClickHouse MCP. Simulates a busy +// production cluster where queries that scan everything time out. +const EVAL_QUERY_SETTINGS = [ + { setting: 'max_execution_time', value: String(QUERY_TIMEOUT_SECONDS) }, +]; + +function buildTraceSourceBody( + name: string, + connectionId: string, + tableName: string, +): Record { + return { + name, + kind: 'trace', + connection: connectionId, + from: { databaseName: 'default', tableName }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: TRACE_DEFAULT_SELECT, + durationExpression: 'Duration', + durationPrecision: 9, + traceIdExpression: 'TraceId', + spanIdExpression: 'SpanId', + parentSpanIdExpression: 'ParentSpanId', + spanNameExpression: 'SpanName', + spanKindExpression: 'SpanKind', + statusCodeExpression: 'StatusCode', + statusMessageExpression: 'StatusMessage', + serviceNameExpression: 'ServiceName', + resourceAttributesExpression: 'ResourceAttributes', + eventAttributesExpression: 'SpanAttributes', + implicitColumnExpression: 'SpanName', + querySettings: EVAL_QUERY_SETTINGS, + }; +} + +function buildLogSourceBody( + name: string, + connectionId: string, + tableName: string, +): Record { + return { + name, + kind: 'log', + connection: connectionId, + from: { databaseName: 'default', tableName }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: LOG_DEFAULT_SELECT, + serviceNameExpression: 'ServiceName', + severityTextExpression: 'SeverityText', + bodyExpression: 'Body', + traceIdExpression: 'TraceId', + spanIdExpression: 'SpanId', + resourceAttributesExpression: 'ResourceAttributes', + eventAttributesExpression: 'LogAttributes', + implicitColumnExpression: 'Body', + querySettings: EVAL_QUERY_SETTINGS, + }; +} + +export type CheckResult = { + configOk: boolean; + mcpReachable: boolean; + clickhouseReachable: boolean; + uvAvailable: boolean; + errors: string[]; +}; + +export async function runCheck( + apiBearerCheck: ( + mcpUrl: string, + accessKey: string, + ) => Promise = defaultMcpCheck, + uvCheck: () => Promise = defaultUvCheck, +): Promise { + const errors: string[] = []; + let configOk = false; + let cfg: EvalConfig | null = null; + try { + cfg = readEvalConfig(); + configOk = true; + } catch (e) { + errors.push(e instanceof Error ? e.message : String(e)); + } + + let mcpReachable = false; + let chReachable = false; + if (cfg) { + mcpReachable = await apiBearerCheck( + cfg.hyperdx.mcpUrl, + cfg.hyperdx.accessKey, + ); + if (!mcpReachable) { + errors.push( + `HyperDX MCP at ${cfg.hyperdx.mcpUrl} did not respond as expected`, + ); + } + try { + const chUrl = `http://${cfg.clickhouse.host}:${cfg.clickhouse.port}/ping`; + const res = await fetch(chUrl); + chReachable = res.ok; + if (!chReachable) errors.push(`ClickHouse ping → ${res.status}`); + } catch (e) { + errors.push( + `ClickHouse not reachable: ${e instanceof Error ? e.message : e}`, + ); + } + } + + const uvAvailable = await uvCheck(); + if (!uvAvailable) { + errors.push( + `\`uv\` not found on PATH. Install via \`brew install uv\` or follow https://docs.astral.sh/uv/`, + ); + } + + return { + configOk, + mcpReachable, + clickhouseReachable: chReachable, + uvAvailable, + errors, + }; +} + +async function defaultMcpCheck( + mcpUrl: string, + accessKey: string, +): Promise { + try { + // The MCP HTTP transport responds to a GET with 405 (method not allowed) + // when not given a JSON-RPC body, but it must accept the bearer token. + // POST a minimal initialize message instead. + const res = await fetch(mcpUrl, { + method: 'POST', + headers: { + Authorization: `Bearer ${accessKey}`, + 'Content-Type': 'application/json', + Accept: 'application/json, text/event-stream', + }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: '2024-11-05', + capabilities: {}, + clientInfo: { name: 'hdx-eval-check', version: '0.1.0' }, + }, + }), + }); + return res.ok || res.status === 200 || res.status === 202; + } catch { + return false; + } +} + +async function defaultUvCheck(): Promise { + const { spawn } = await import('child_process'); + return new Promise(resolve => { + const proc = spawn('uv', ['--version'], { stdio: 'ignore' }); + proc.on('error', () => resolve(false)); + proc.on('exit', code => resolve(code === 0)); + }); +} diff --git a/packages/hdx-eval/src/index.ts b/packages/hdx-eval/src/index.ts new file mode 100644 index 0000000000..1307108093 --- /dev/null +++ b/packages/hdx-eval/src/index.ts @@ -0,0 +1,10 @@ +export { scenarioTables } from './clickhouse/schema'; +export type { LogRow, TraceRow } from './generators/types'; +export type { SeededRng } from './rng/seeded'; +export { mulberry32 } from './rng/seeded'; +export { getScenario, SCENARIO_NAMES, SCENARIOS } from './scenarios'; +export type { + GenerateContext, + Scenario, + ScenarioOutput, +} from './scenarios/types'; diff --git a/packages/hdx-eval/src/reports/aggregate.ts b/packages/hdx-eval/src/reports/aggregate.ts new file mode 100644 index 0000000000..378e1590d5 --- /dev/null +++ b/packages/hdx-eval/src/reports/aggregate.ts @@ -0,0 +1,201 @@ +import type { GradeRecord } from '../grading/types'; +import type { McpKind, RunRecord } from '../harness/types'; + +export type CellSummary = { + scenario: string; + mcp: McpKind; + n: number; + programmatic: { mean: number; perCheck: Record }; + judge: { + weightedMean: number; + perCriterion: Record; + judged: number; + }; + combinedScore: { mean: number }; + toolCalls: { mean: number }; + toolErrors: { mean: number; rateMean: number; penaltyMean: number }; + tokens: { + input: number; + output: number; + cacheRead: number; + cacheCreation: number; + }; + durationMs: { mean: number }; + termination: Record; +}; + +export type ScenarioSummary = { + scenario: string; + cells: Partial>; + delta: { + combinedScore: number | null; + programmaticScore: number | null; + judgeWeightedMean: number | null; + toolCalls: number | null; + outputTokens: number | null; + durationMs: number | null; + }; +}; + +export type BatchSummary = { + batchDir: string; + generatedAt: string; + scenarios: ScenarioSummary[]; +}; + +export type GradedRunPair = { + run: RunRecord; + grade: GradeRecord; +}; + +export function buildAggregate(args: { + batchDir: string; + pairs: GradedRunPair[]; +}): BatchSummary { + const byScenario = new Map(); + for (const p of args.pairs) { + if (!byScenario.has(p.run.scenario)) byScenario.set(p.run.scenario, []); + byScenario.get(p.run.scenario)!.push(p); + } + + const scenarios: ScenarioSummary[] = []; + for (const [name, ps] of byScenario.entries()) { + const byMcp = new Map(); + for (const p of ps) { + if (!byMcp.has(p.run.mcp)) byMcp.set(p.run.mcp, []); + byMcp.get(p.run.mcp)!.push(p); + } + const cells: Partial> = {}; + for (const [mcp, list] of byMcp.entries()) { + cells[mcp] = buildCellSummary(name, mcp, list); + } + scenarios.push({ + scenario: name, + cells, + delta: computeDelta(cells.hyperdx, cells.clickhouse), + }); + } + + scenarios.sort((a, b) => a.scenario.localeCompare(b.scenario)); + return { + batchDir: args.batchDir, + generatedAt: new Date().toISOString(), + scenarios, + }; +} + +function buildCellSummary( + scenario: string, + mcp: McpKind, + pairs: GradedRunPair[], +): CellSummary { + const n = pairs.length; + const programmaticScore = mean(pairs.map(p => p.grade.programmatic.score)); + + const judgeable = pairs.filter(p => p.grade.judge); + const judgeMean = + judgeable.length === 0 + ? 0 + : mean(judgeable.map(p => p.grade.judge!.weightedScore)); + + const combinedMean = mean(pairs.map(p => p.grade.combinedScore)); + const toolCallsMean = mean(pairs.map(p => p.run.toolCalls.length)); + const toolErrorsMean = mean(pairs.map(p => p.grade.toolErrors?.errors ?? 0)); + const toolErrorRateMean = mean(pairs.map(p => p.grade.toolErrors?.rate ?? 0)); + const toolErrorPenaltyMean = mean( + pairs.map(p => p.grade.toolErrors?.penalty ?? 0), + ); + const durationMean = mean(pairs.map(p => p.run.durationMs)); + + const tokens = { + input: mean(pairs.map(p => p.run.tokens.input)), + output: mean(pairs.map(p => p.run.tokens.output)), + cacheRead: mean(pairs.map(p => p.run.tokens.cacheRead)), + cacheCreation: mean(pairs.map(p => p.run.tokens.cacheCreation)), + }; + + const perCheck: Record = {}; + if (pairs.length > 0) { + const checkIds = new Set(); + for (const p of pairs) + for (const h of p.grade.programmatic.hits) checkIds.add(h.id); + for (const id of checkIds) { + const satisfiedHits = pairs.map( + p => + p.grade.programmatic.hits.find(h => h.id === id)?.satisfied ?? false, + ); + perCheck[id] = + satisfiedHits.filter(Boolean).length / satisfiedHits.length; + } + } + + const perCriterion: Record = {}; + if (judgeable.length > 0) { + const critIds = new Set(); + for (const p of judgeable) + for (const k of Object.keys(p.grade.judge!.scores)) critIds.add(k); + for (const id of critIds) { + const scores = judgeable + .map(p => p.grade.judge!.scores[id]?.score) + .filter((s): s is number => typeof s === 'number'); + if (scores.length > 0) perCriterion[id] = mean(scores); + } + } + + const termination: Record = {}; + for (const p of pairs) { + termination[p.run.termination] = (termination[p.run.termination] ?? 0) + 1; + } + + return { + scenario, + mcp, + n, + programmatic: { mean: programmaticScore, perCheck }, + judge: { + weightedMean: judgeMean, + perCriterion, + judged: judgeable.length, + }, + combinedScore: { mean: combinedMean }, + toolCalls: { mean: toolCallsMean }, + toolErrors: { + mean: toolErrorsMean, + rateMean: toolErrorRateMean, + penaltyMean: toolErrorPenaltyMean, + }, + tokens, + durationMs: { mean: durationMean }, + termination, + }; +} + +function computeDelta( + hyperdx: CellSummary | undefined, + clickhouse: CellSummary | undefined, +): ScenarioSummary['delta'] { + if (!hyperdx || !clickhouse) { + return { + combinedScore: null, + programmaticScore: null, + judgeWeightedMean: null, + toolCalls: null, + outputTokens: null, + durationMs: null, + }; + } + return { + combinedScore: hyperdx.combinedScore.mean - clickhouse.combinedScore.mean, + programmaticScore: hyperdx.programmatic.mean - clickhouse.programmatic.mean, + judgeWeightedMean: + hyperdx.judge.weightedMean - clickhouse.judge.weightedMean, + toolCalls: hyperdx.toolCalls.mean - clickhouse.toolCalls.mean, + outputTokens: hyperdx.tokens.output - clickhouse.tokens.output, + durationMs: hyperdx.durationMs.mean - clickhouse.durationMs.mean, + }; +} + +function mean(xs: number[]): number { + if (xs.length === 0) return 0; + return xs.reduce((s, x) => s + x, 0) / xs.length; +} diff --git a/packages/hdx-eval/src/reports/markdown.ts b/packages/hdx-eval/src/reports/markdown.ts new file mode 100644 index 0000000000..a10608ba2f --- /dev/null +++ b/packages/hdx-eval/src/reports/markdown.ts @@ -0,0 +1,244 @@ +import type { BatchSummary, ScenarioSummary } from './aggregate'; + +export function renderMarkdownReport(summary: BatchSummary): string { + const lines: string[] = []; + lines.push(`# Eval Batch — ${basename(summary.batchDir)}`); + lines.push(''); + lines.push(`Generated: ${summary.generatedAt}`); + lines.push(''); + lines.push(renderTopVerdict(summary)); + lines.push(''); + + for (const scenario of summary.scenarios) { + lines.push(`## ${scenario.scenario}`); + lines.push(''); + lines.push(renderScenarioTable(scenario)); + lines.push(''); + const judgeBreakdown = renderJudgeBreakdown(scenario); + if (judgeBreakdown) { + lines.push(judgeBreakdown); + lines.push(''); + } + const programmaticBreakdown = renderProgrammaticBreakdown(scenario); + if (programmaticBreakdown) { + lines.push(programmaticBreakdown); + lines.push(''); + } + } + return lines.join('\n'); +} + +function renderTopVerdict(summary: BatchSummary): string { + const rows = [ + '| Scenario | HyperDX | ClickHouse | Δ (combined) |', + '|---|---|---|---|', + ]; + for (const s of summary.scenarios) { + const h = s.cells.hyperdx; + const c = s.cells.clickhouse; + const hCell = h ? pct(h.combinedScore.mean) : '—'; + const cCell = c ? pct(c.combinedScore.mean) : '—'; + const delta = + s.delta.combinedScore === null ? '—' : signedPct(s.delta.combinedScore); + rows.push(`| ${s.scenario} | ${hCell} | ${cCell} | ${delta} |`); + } + return ['### Top-line verdict', '', ...rows].join('\n'); +} + +function renderScenarioTable(scenario: ScenarioSummary): string { + const h = scenario.cells.hyperdx; + const c = scenario.cells.clickhouse; + const rows = [ + '| Metric | HyperDX | ClickHouse | Δ (HDX − CH) |', + '|---|---|---|---|', + ]; + rows.push( + row( + 'Combined score', + val(h?.combinedScore.mean, pct), + val(c?.combinedScore.mean, pct), + signedPct(scenario.delta.combinedScore), + ), + ); + rows.push( + row( + 'Programmatic score', + val(h?.programmatic.mean, pct), + val(c?.programmatic.mean, pct), + signedPct(scenario.delta.programmaticScore), + ), + ); + rows.push( + row( + 'Judge mean (weighted)', + val(h?.judge.weightedMean, pct), + val(c?.judge.weightedMean, pct), + signedPct(scenario.delta.judgeWeightedMean), + ), + ); + rows.push( + row( + 'Tool calls (mean)', + val(h?.toolCalls.mean, oneDecimal), + val(c?.toolCalls.mean, oneDecimal), + signedNumber(scenario.delta.toolCalls, 1), + ), + ); + rows.push( + row( + 'Tool errors (mean)', + val(h?.toolErrors?.mean, oneDecimal), + val(c?.toolErrors?.mean, oneDecimal), + h && c + ? signedNumber((h.toolErrors?.mean ?? 0) - (c.toolErrors?.mean ?? 0), 1) + : '—', + ), + ); + rows.push( + row( + 'Tool-error penalty', + val(h?.toolErrors?.penaltyMean, x => `${(x * 100).toFixed(0)}pp`), + val(c?.toolErrors?.penaltyMean, x => `${(x * 100).toFixed(0)}pp`), + '—', + ), + ); + rows.push( + row( + 'Output tokens (mean)', + val(h?.tokens.output, intStr), + val(c?.tokens.output, intStr), + signedNumber(scenario.delta.outputTokens, 0), + ), + ); + rows.push( + row( + 'Cache reads (mean)', + val(h?.tokens.cacheRead, intStr), + val(c?.tokens.cacheRead, intStr), + h && c ? signedNumber(h.tokens.cacheRead - c.tokens.cacheRead, 0) : '—', + ), + ); + rows.push( + row( + 'Wall clock (s, mean)', + val(h?.durationMs.mean, secondsFromMs), + val(c?.durationMs.mean, secondsFromMs), + scenario.delta.durationMs === null + ? '—' + : signedNumber(scenario.delta.durationMs / 1000, 1), + ), + ); + rows.push( + row( + 'Termination', + h ? terminationBreakdown(h.termination) : '—', + c ? terminationBreakdown(c.termination) : '—', + '—', + ), + ); + rows.push(row('N', h ? String(h.n) : '—', c ? String(c.n) : '—', '—')); + return rows.join('\n'); +} + +function renderJudgeBreakdown(scenario: ScenarioSummary): string | null { + const allCriteria = new Set(); + for (const cell of [scenario.cells.hyperdx, scenario.cells.clickhouse]) { + if (!cell) continue; + for (const id of Object.keys(cell.judge.perCriterion)) allCriteria.add(id); + } + if (allCriteria.size === 0) return null; + const rows = [ + '#### Judge per-criterion (mean 0–5)', + '', + '| Criterion | HyperDX | ClickHouse |', + '|---|---|---|', + ]; + for (const id of [...allCriteria].sort()) { + const h = scenario.cells.hyperdx?.judge.perCriterion[id]; + const c = scenario.cells.clickhouse?.judge.perCriterion[id]; + rows.push(`| ${id} | ${fmtScore(h)} | ${fmtScore(c)} |`); + } + return rows.join('\n'); +} + +function renderProgrammaticBreakdown(scenario: ScenarioSummary): string | null { + const allChecks = new Set(); + for (const cell of [scenario.cells.hyperdx, scenario.cells.clickhouse]) { + if (!cell) continue; + for (const id of Object.keys(cell.programmatic.perCheck)) allChecks.add(id); + } + if (allChecks.size === 0) return null; + const rows = [ + '#### Programmatic per-check (pass rate)', + '', + 'Pass rate = positive checks matched + negative checks not matched.', + '', + '| Check | HyperDX | ClickHouse |', + '|---|---|---|', + ]; + for (const id of [...allChecks].sort()) { + const h = scenario.cells.hyperdx?.programmatic.perCheck[id]; + const c = scenario.cells.clickhouse?.programmatic.perCheck[id]; + const isNegative = id.startsWith('false_'); + const label = isNegative ? `${id} (neg)` : id; + rows.push(`| ${label} | ${fmtRate(h)} | ${fmtRate(c)} |`); + } + return rows.join('\n'); +} + +function row(label: string, h: string, c: string, delta: string): string { + return `| ${label} | ${h} | ${c} | ${delta} |`; +} + +function val(v: T | undefined, fmt: (x: T) => string): string { + return v === undefined ? '—' : fmt(v); +} + +function pct(x: number | undefined): string { + if (x === undefined) return '—'; + return `${(x * 100).toFixed(0)}%`; +} + +function signedPct(x: number | null | undefined): string { + if (x === null || x === undefined) return '—'; + const sign = x >= 0 ? '+' : ''; + return `${sign}${(x * 100).toFixed(0)}%`; +} + +function oneDecimal(x: number): string { + return x.toFixed(1); +} + +function intStr(x: number): string { + return Math.round(x).toString(); +} + +function secondsFromMs(x: number): string { + return (x / 1000).toFixed(1); +} + +function signedNumber(x: number | null | undefined, decimals: number): string { + if (x === null || x === undefined) return '—'; + const sign = x >= 0 ? '+' : ''; + return `${sign}${x.toFixed(decimals)}`; +} + +function fmtScore(x: number | undefined): string { + if (x === undefined) return '—'; + return `${x.toFixed(2)}/5`; +} + +function fmtRate(x: number | undefined): string { + if (x === undefined) return '—'; + return `${(x * 100).toFixed(0)}%`; +} + +function terminationBreakdown(rec: Record): string { + const entries = Object.entries(rec).sort((a, b) => b[1] - a[1]); + return entries.map(([k, v]) => `${k}: ${v}`).join(', '); +} + +function basename(path: string): string { + const parts = path.split(/[\\/]/); + return parts[parts.length - 1] || path; +} diff --git a/packages/hdx-eval/src/reports/store.ts b/packages/hdx-eval/src/reports/store.ts new file mode 100644 index 0000000000..049361db0b --- /dev/null +++ b/packages/hdx-eval/src/reports/store.ts @@ -0,0 +1,59 @@ +import { existsSync, readdirSync, readFileSync, writeFileSync } from 'fs'; +import { join } from 'path'; + +import type { GradeRecord } from '../grading/types'; +import type { RunRecord } from '../harness/types'; +import { SCENARIO_NAMES } from '../scenarios'; +import { buildAggregate, type GradedRunPair } from './aggregate'; +import { renderMarkdownReport } from './markdown'; + +export function loadGradedPairs(batchDir: string): GradedRunPair[] { + const pairs: GradedRunPair[] = []; + for (const scenario of safeReaddir(batchDir)) { + if (!SCENARIO_NAMES.includes(scenario)) continue; + const sceneDir = join(batchDir, scenario); + for (const mcp of safeReaddir(sceneDir)) { + const mcpDir = join(sceneDir, mcp); + for (const file of safeReaddir(mcpDir)) { + if (!file.endsWith('.json')) continue; + if (file.endsWith('.grade.json')) continue; + const runPath = join(mcpDir, file); + const gradePath = runPath.replace(/\.json$/, '.grade.json'); + if (!existsSync(gradePath)) continue; + try { + const run = JSON.parse(readFileSync(runPath, 'utf8')) as RunRecord; + const grade = JSON.parse( + readFileSync(gradePath, 'utf8'), + ) as GradeRecord; + pairs.push({ run, grade }); + } catch { + // skip malformed + } + } + } + } + return pairs; +} + +export function writeBatchSummary( + batchDir: string, + outPath: string, +): { jsonPath: string; mdPath: string; pairsCount: number } { + const pairs = loadGradedPairs(batchDir); + const summary = buildAggregate({ batchDir, pairs }); + + const jsonPath = outPath.replace(/\.md$/, '.json'); + writeFileSync(jsonPath, JSON.stringify(summary, null, 2) + '\n', 'utf8'); + const md = renderMarkdownReport(summary); + writeFileSync(outPath, md, 'utf8'); + + return { jsonPath, mdPath: outPath, pairsCount: pairs.length }; +} + +function safeReaddir(path: string): string[] { + try { + return readdirSync(path); + } catch { + return []; + } +} diff --git a/packages/hdx-eval/src/rng/seeded.ts b/packages/hdx-eval/src/rng/seeded.ts new file mode 100644 index 0000000000..9dac8fa9a4 --- /dev/null +++ b/packages/hdx-eval/src/rng/seeded.ts @@ -0,0 +1,57 @@ +export type SeededRng = { + next(): number; + range(minInclusive: number, maxExclusive: number): number; + intRange(minInclusive: number, maxExclusive: number): number; + pick(items: readonly T[]): T; + weightedPick(items: readonly { value: T; weight: number }[]): T; + hex(bytes: number): string; +}; + +export function mulberry32(seed: number): SeededRng { + let state = seed >>> 0; + + const next = (): number => { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; + + const range = (lo: number, hi: number): number => lo + next() * (hi - lo); + + const intRange = (lo: number, hi: number): number => + Math.floor(range(lo, hi)); + + const pick = (items: readonly T[]): T => { + if (items.length === 0) { + throw new Error('pick() called with empty array'); + } + return items[intRange(0, items.length)]; + }; + + const weightedPick = ( + items: readonly { value: T; weight: number }[], + ): T => { + const total = items.reduce((s, i) => s + i.weight, 0); + if (total <= 0) { + throw new Error('weightedPick() requires positive total weight'); + } + let roll = next() * total; + for (const item of items) { + roll -= item.weight; + if (roll < 0) return item.value; + } + return items[items.length - 1].value; + }; + + const hex = (bytes: number): string => { + let out = ''; + for (let i = 0; i < bytes; i++) { + out += intRange(0, 256).toString(16).padStart(2, '0'); + } + return out; + }; + + return { next, range, intRange, pick, weightedPick, hex }; +} diff --git a/packages/hdx-eval/src/scenarios/error-root-cause/generate.ts b/packages/hdx-eval/src/scenarios/error-root-cause/generate.ts new file mode 100644 index 0000000000..e692fb642e --- /dev/null +++ b/packages/hdx-eval/src/scenarios/error-root-cause/generate.ts @@ -0,0 +1,725 @@ +import { makeLog } from '../../generators/logs'; +import { + analyticsEventLog, + backgroundLog, + buildResourcePool, + catalogLog, + envoyAccessLog, + heartbeatLog, + pageRenderLog, + pickResource, + pickSeverityIn, + uuidv4, +} from '../../generators/templates'; +import { + makeSpan, + msToNs, + newSpanId, + newTraceId, +} from '../../generators/traces'; +import type { LogRow, TraceRow } from '../../generators/types'; +import type { GenerateContext, Scenario, ScenarioBatch } from '../types'; +import groundTruth from './ground-truth.json'; + +const BACKGROUND_SERVICES = [ + 'checkout-api', + 'payment-service', + 'cart-service', + 'inventory-service', + 'notification-service', + 'search-service', + 'recommendation-service', + 'frontend', + 'api-gateway', + 'database', + 'auth-service', + 'billing-service', + 'analytics-service', + 'image-service', + 'cdn', + 'session-service', + 'pricing-service', + 'review-service', + 'shipping-service', + 'tax-service', + 'email-service', + 'feature-flags', + 'rum-collector', + 'webhook-relay', + 'metrics-pipeline', +] as const; + +const REGIONS = ['us-east-1', 'us-west-2', 'eu-west-1', 'ap-south-1'] as const; + +const BACKGROUND_OPS: Record = { + 'checkout-api': ['POST /api/checkout', 'GET /api/cart', 'GET /api/order'], + 'payment-service': ['payment.charge', 'payment.refund', 'payment.lookup'], + 'cart-service': ['cart.add', 'cart.list', 'cart.remove'], + 'inventory-service': ['inventory.check', 'inventory.reserve'], + 'notification-service': ['notify.email', 'notify.sms', 'notify.push'], + 'search-service': ['search.autocomplete', 'search.click'], + 'recommendation-service': ['rec.compute', 'rec.fetch'], + frontend: ['GET /home', 'GET /product', 'GET /checkout', 'GET /account'], + 'api-gateway': ['route.dispatch', 'route.auth'], + database: ['database.query', 'database.tx'], + 'auth-service': ['auth.login', 'auth.refresh', 'auth.verify'], + 'billing-service': ['billing.invoice', 'billing.subscription'], + 'analytics-service': ['analytics.event', 'analytics.batch'], + 'image-service': ['image.resize', 'image.fetch'], + cdn: ['cdn.purge', 'cdn.fetch'], + 'session-service': ['session.create', 'session.validate', 'session.expire'], + 'pricing-service': ['pricing.compute', 'pricing.lookup'], + 'review-service': ['review.list', 'review.create'], + 'shipping-service': ['shipping.quote', 'shipping.track'], + 'tax-service': ['tax.compute'], + 'email-service': ['email.send', 'email.queue'], + 'feature-flags': ['flag.evaluate', 'flag.list'], + 'rum-collector': ['rum.ingest'], + 'webhook-relay': ['webhook.dispatch', 'webhook.retry'], + 'metrics-pipeline': ['metrics.ingest', 'metrics.flush'], +}; + +// v6 — 6M base background traces (~12M+ spans w/ child) + 12M logs. +// FAILING_TRACE_COUNT is intentionally low so a naive +// `WHERE StatusCode='ERROR'` query returns the planted failures as a small +// minority of the total recent error count (most of which is decoy traffic). +const BACKGROUND_TRACE_COUNT = 6_000_000; +const BACKGROUND_LOG_COUNT = 12_000_000; +const FAILING_TRACE_COUNT = 8; + +const ANOMALY_WINDOW_MS = 5 * 60 * 1000; // last 5 min +const HISTORY_WINDOW_MS = 60 * 60 * 1000; + +const TLS_SPIKE_OFFSET_FROM_NOW_MS = 25 * 60 * 1000; // T-25min +const TLS_SPIKE_DURATION_MS = 5 * 60 * 1000; +const TLS_SPIKE_COUNT = 50; + +const AUTH_RATE_LIMIT_OFFSET_FROM_NOW_MS = 40 * 60 * 1000; // T-40min +const AUTH_RATE_LIMIT_DURATION_MS = 2 * 60 * 1000; +const AUTH_RATE_LIMIT_COUNT = 20; + +const SEARCH_SLOW_COUNT = 80; + +// Background "operational" payment.charge errors throughout the hour — these +// are NOT the planted infra incident. The decline reasons rotate so an agent +// blanket-grepping payment-service errors will get a noisy result that mixes +// legitimate user-level failures with the actual root cause. +const BACKGROUND_PAYMENT_ERROR_COUNT = 200; +const PAYMENT_DECLINE_REASONS = [ + 'card declined', + 'insufficient funds', + 'CVV mismatch', + 'expired card', + 'payment method not supported', +] as const; + +// Concurrent decoy: notification-service smtp errors in the SAME 5-min window +// as the planted cascade. Looks like a similar service-can't-reach-dependency +// pattern, but isn't part of any checkout-api trace — different traceids. +const SMTP_DECOY_COUNT = 25; + +// Concurrent decoy: cdn origin-fetch failures in the SAME 5-min window. Same +// shape as the smtp decoy (separate trace tree, no checkout-api parent) but +// at higher count, so a status-code-only query in the recent window returns +// far more cdn rows than planted-cascade rows. Forces the agent to do +// causal/cascade analysis instead of leaning on raw error counts. +const CDN_ORIGIN_DECOY_COUNT = 80; + +// Three variants of the planted timeout message — same root cause, resists +// naive exact-string matching. Counts sum to FAILING_TRACE_COUNT. +const TIMEOUT_VARIANTS = [ + { count: 3, body: 'connection timeout to db-payment' }, + { count: 3, body: 'timeout connecting to db-payment.internal' }, + { count: 2, body: 'db-payment unreachable after 5 retries' }, +] as const; + +type Rng = GenerateContext['rng']; + +function generateTlsSpike(rng: Rng, nowMs: number): TraceRow[] { + const traces: TraceRow[] = []; + const startMs = nowMs - TLS_SPIKE_OFFSET_FROM_NOW_MS; + for (let i = 0; i < TLS_SPIKE_COUNT; i++) { + const t = + startMs + + (i / Math.max(TLS_SPIKE_COUNT - 1, 1)) * (TLS_SPIKE_DURATION_MS - 1000); + traces.push( + makeSpan({ + rng, + timestampMs: t, + traceId: newTraceId(rng), + spanId: newSpanId(rng), + spanName: 'route.dispatch', + spanKind: 'SPAN_KIND_SERVER', + serviceName: 'api-gateway', + durationNs: msToNs(rng.range(50, 300)), + statusCode: 'STATUS_CODE_ERROR', + statusMessage: 'tls handshake timeout: peer reset', + spanAttributes: { + 'http.method': 'GET', + 'http.status_code': '525', + 'error.type': 'TLSHandshakeError', + 'tls.version': '1.3', + }, + }), + ); + } + return traces; +} + +function generateAuthRateLimitBlip(rng: Rng, nowMs: number): TraceRow[] { + const traces: TraceRow[] = []; + const startMs = nowMs - AUTH_RATE_LIMIT_OFFSET_FROM_NOW_MS; + for (let i = 0; i < AUTH_RATE_LIMIT_COUNT; i++) { + const t = + startMs + + (i / Math.max(AUTH_RATE_LIMIT_COUNT - 1, 1)) * + (AUTH_RATE_LIMIT_DURATION_MS - 1000); + traces.push( + makeSpan({ + rng, + timestampMs: t, + traceId: newTraceId(rng), + spanId: newSpanId(rng), + spanName: 'auth.refresh', + spanKind: 'SPAN_KIND_SERVER', + serviceName: 'auth-service', + durationNs: msToNs(rng.range(8, 25)), + statusCode: 'STATUS_CODE_ERROR', + statusMessage: 'rate limit exceeded: 1000 req/min on /v2/token', + spanAttributes: { + 'http.method': 'POST', + 'http.status_code': '429', + 'error.type': 'RateLimitExceeded', + }, + }), + ); + } + return traces; +} + +function generateSearchSlowQueries(rng: Rng, nowMs: number): TraceRow[] { + const traces: TraceRow[] = []; + const startMs = nowMs - HISTORY_WINDOW_MS; + // Slow but successful — distractor for latency-style queries. + for (let i = 0; i < SEARCH_SLOW_COUNT; i++) { + const t = + startMs + + (i / Math.max(SEARCH_SLOW_COUNT - 1, 1)) * (HISTORY_WINDOW_MS - 30_000); + traces.push( + makeSpan({ + rng, + timestampMs: t, + traceId: newTraceId(rng), + spanId: newSpanId(rng), + spanName: 'search.query', + spanKind: 'SPAN_KIND_SERVER', + serviceName: 'search-service', + durationNs: msToNs(rng.range(1500, 3000)), + statusCode: 'STATUS_CODE_OK', + spanAttributes: { + 'http.method': 'GET', + 'db.system': 'elasticsearch', + 'db.statement': + "SELECT * FROM products WHERE description LIKE '%query%'", + 'search.results_count': String(rng.intRange(0, 1000)), + }, + }), + ); + } + return traces; +} + +function generateBackgroundPaymentErrors(rng: Rng, nowMs: number): TraceRow[] { + const traces: TraceRow[] = []; + const startMs = nowMs - HISTORY_WINDOW_MS; + for (let i = 0; i < BACKGROUND_PAYMENT_ERROR_COUNT; i++) { + const t = + startMs + + (i / Math.max(BACKGROUND_PAYMENT_ERROR_COUNT - 1, 1)) * + (HISTORY_WINDOW_MS - 30_000); + const reason = rng.pick(PAYMENT_DECLINE_REASONS); + traces.push( + makeSpan({ + rng, + timestampMs: t, + traceId: newTraceId(rng), + spanId: newSpanId(rng), + spanName: 'payment.charge', + spanKind: 'SPAN_KIND_SERVER', + serviceName: 'payment-service', + durationNs: msToNs(rng.range(50, 250)), + statusCode: 'STATUS_CODE_ERROR', + statusMessage: reason, + spanAttributes: { + 'rpc.system': 'grpc', + 'payment.amount_cents': String(rng.intRange(100, 50000)), + 'payment.method': rng.pick(['card', 'ach', 'wallet']), + 'error.type': 'PaymentDeclined', + 'decline.reason': reason, + }, + }), + ); + } + return traces; +} + +function generateSmtpDecoy( + rng: Rng, + nowMs: number, +): { traces: TraceRow[]; logs: LogRow[] } { + const traces: TraceRow[] = []; + const logs: LogRow[] = []; + const onsetMs = nowMs - ANOMALY_WINDOW_MS; + for (let i = 0; i < SMTP_DECOY_COUNT; i++) { + const t = + onsetMs + + (i / Math.max(SMTP_DECOY_COUNT - 1, 1)) * (ANOMALY_WINDOW_MS - 5_000); + const traceId = newTraceId(rng); + const spanId = newSpanId(rng); + traces.push( + makeSpan({ + rng, + timestampMs: t, + traceId, + spanId, + spanName: 'notify.email', + spanKind: 'SPAN_KIND_CLIENT', + serviceName: 'notification-service', + durationNs: msToNs(rng.range(800, 2000)), + statusCode: 'STATUS_CODE_ERROR', + statusMessage: 'smtp connection refused: smtp.internal:587', + spanAttributes: { + 'net.peer.name': 'smtp.internal', + 'net.peer.port': '587', + 'error.type': 'SMTPConnectionError', + 'email.recipient_count': '1', + }, + }), + ); + logs.push( + makeLog({ + timestampMs: t + rng.intRange(5, 30), + serviceName: 'notification-service', + severityText: 'ERROR', + body: 'smtp connection refused: smtp.internal:587', + traceId, + spanId, + logAttributes: { + 'error.kind': 'SMTPConnectionError', + component: 'notification-service', + }, + }), + ); + } + return { traces, logs }; +} + +function generateCdnOriginDecoy( + rng: Rng, + nowMs: number, +): { traces: TraceRow[]; logs: LogRow[] } { + const traces: TraceRow[] = []; + const logs: LogRow[] = []; + const onsetMs = nowMs - ANOMALY_WINDOW_MS; + for (let i = 0; i < CDN_ORIGIN_DECOY_COUNT; i++) { + const t = + onsetMs + + (i / Math.max(CDN_ORIGIN_DECOY_COUNT - 1, 1)) * + (ANOMALY_WINDOW_MS - 5_000); + const traceId = newTraceId(rng); + const spanId = newSpanId(rng); + traces.push( + makeSpan({ + rng, + timestampMs: t, + traceId, + spanId, + spanName: 'cdn.cache.fetch', + spanKind: 'SPAN_KIND_CLIENT', + serviceName: 'cdn', + durationNs: msToNs(rng.range(150, 600)), + statusCode: 'STATUS_CODE_ERROR', + statusMessage: 'origin connection refused: origin-static.internal:443', + spanAttributes: { + 'http.method': 'GET', + 'http.status_code': '502', + 'net.peer.name': 'origin-static.internal', + 'net.peer.port': '443', + 'error.type': 'OriginUnreachableError', + 'cdn.edge_pop': rng.pick([ + 'edge-iad', + 'edge-sfo', + 'edge-fra', + 'edge-syd', + ]), + }, + }), + ); + logs.push( + makeLog({ + timestampMs: t + rng.intRange(5, 30), + serviceName: 'cdn', + severityText: 'ERROR', + body: 'origin connection refused: origin-static.internal:443', + traceId, + spanId, + logAttributes: { + 'error.kind': 'OriginUnreachableError', + component: 'cdn', + 'origin.host': 'origin-static.internal', + }, + }), + ); + } + return { traces, logs }; +} + +function generateAnomaly( + rng: Rng, + nowMs: number, +): { traces: TraceRow[]; logs: LogRow[] } { + const traces: TraceRow[] = []; + const logs: LogRow[] = []; + const onsetMs = nowMs - ANOMALY_WINDOW_MS; + const slots: { body: string }[] = []; + for (const v of TIMEOUT_VARIANTS) { + for (let k = 0; k < v.count; k++) slots.push({ body: v.body }); + } + // Shuffle slots deterministically using the seeded RNG. + for (let i = slots.length - 1; i > 0; i--) { + const j = rng.intRange(0, i + 1); + [slots[i], slots[j]] = [slots[j], slots[i]]; + } + + for (let i = 0; i < FAILING_TRACE_COUNT; i++) { + const t = + onsetMs + + (i / Math.max(FAILING_TRACE_COUNT - 1, 1)) * (ANOMALY_WINDOW_MS - 5_000); + const traceId = newTraceId(rng); + const variantBody = slots[i].body; + + const rootSpanId = newSpanId(rng); + traces.push( + makeSpan({ + rng, + timestampMs: t, + traceId, + spanId: rootSpanId, + spanName: 'POST /api/checkout', + spanKind: 'SPAN_KIND_SERVER', + serviceName: 'checkout-api', + durationNs: msToNs(rng.range(2200, 3200)), + statusCode: 'STATUS_CODE_ERROR', + statusMessage: 'payment_service_unavailable', + spanAttributes: { + 'http.method': 'POST', + 'http.route': '/api/checkout', + 'http.status_code': '500', + 'user.id': `user-${rng.intRange(1000, 9999)}`, + }, + }), + ); + + const midSpanId = newSpanId(rng); + traces.push( + makeSpan({ + rng, + timestampMs: t + rng.intRange(1, 5), + traceId, + spanId: midSpanId, + parentSpanId: rootSpanId, + spanName: 'payment.charge', + spanKind: 'SPAN_KIND_CLIENT', + serviceName: 'payment-service', + durationNs: msToNs(rng.range(2000, 2900)), + statusCode: 'STATUS_CODE_ERROR', + statusMessage: 'db connection timeout', + spanAttributes: { + 'rpc.system': 'grpc', + 'payment.amount_cents': String(rng.intRange(500, 50000)), + 'error.type': 'ConnectionTimeoutError', + }, + }), + ); + + traces.push( + makeSpan({ + rng, + timestampMs: t + rng.intRange(5, 15), + traceId, + spanId: newSpanId(rng), + parentSpanId: midSpanId, + spanName: 'db.payment.connect', + spanKind: 'SPAN_KIND_CLIENT', + serviceName: 'payment-service', + durationNs: msToNs(rng.range(1900, 2500)), + statusCode: 'STATUS_CODE_ERROR', + statusMessage: variantBody, + spanAttributes: { + 'db.system': 'postgres', + 'db.name': 'payments', + 'db.statement': 'SELECT 1', + 'net.peer.name': 'db-payment.internal', + 'error.type': 'ConnectionTimeoutError', + }, + }), + ); + + logs.push( + makeLog({ + timestampMs: t + rng.intRange(10, 50), + serviceName: 'payment-service', + severityText: 'ERROR', + body: `${variantBody} (trace_id=${traceId})`, + traceId, + spanId: midSpanId, + logAttributes: { + 'error.kind': 'DBTimeout', + component: 'payment-service', + 'db.host': 'db-payment.internal', + }, + }), + ); + } + return { traces, logs }; +} + +export const errorRootCauseScenario: Scenario = { + name: 'error-root-cause', + agentPrompt: groundTruth.agentPrompt, + description: + 'payment-service DB connection timeout cascading into checkout-api 5xx, buried in 12M+ background spans + 12M templated log bodies + 5 distractor anomalies', + *generate(ctx): Iterable { + const factor = ctx.volumeFactor ?? 1; + const batchSize = ctx.batchSize ?? 10_000; + const bgTraceCount = Math.max( + 50, + Math.round(BACKGROUND_TRACE_COUNT * factor), + ); + const bgLogCount = Math.max(50, Math.round(BACKGROUND_LOG_COUNT * factor)); + + const resourcePool = buildResourcePool({ + rng: ctx.rng, + services: BACKGROUND_SERVICES, + instancesPerService: 16, + }); + + yield* streamBackground( + ctx.rng, + ctx.nowMs, + bgTraceCount, + bgLogCount, + batchSize, + resourcePool, + ); + + // Distractors and the planted anomaly stay at fixed counts so the + // structural invariants remain identical regardless of volumeFactor. + const tls = generateTlsSpike(ctx.rng, ctx.nowMs); + const auth = generateAuthRateLimitBlip(ctx.rng, ctx.nowMs); + const slow = generateSearchSlowQueries(ctx.rng, ctx.nowMs); + const bgPayment = generateBackgroundPaymentErrors(ctx.rng, ctx.nowMs); + const smtp = generateSmtpDecoy(ctx.rng, ctx.nowMs); + const cdn = generateCdnOriginDecoy(ctx.rng, ctx.nowMs); + const anomaly = generateAnomaly(ctx.rng, ctx.nowMs); + yield { + traces: [ + ...tls, + ...auth, + ...slow, + ...bgPayment, + ...smtp.traces, + ...cdn.traces, + ...anomaly.traces, + ], + logs: [...smtp.logs, ...cdn.logs, ...anomaly.logs], + }; + }, + groundTruth, +}; + +type LogTemplateKind = + | 'page_render' + | 'envoy_access' + | 'analytics' + | 'catalog' + | 'heartbeat' + | 'background'; + +const LOG_TEMPLATE_MIX: { value: LogTemplateKind; weight: number }[] = [ + { value: 'page_render', weight: 25 }, + { value: 'envoy_access', weight: 15 }, + { value: 'analytics', weight: 15 }, + { value: 'catalog', weight: 15 }, + { value: 'heartbeat', weight: 10 }, + { value: 'background', weight: 20 }, +]; + +const LOG_TEMPLATE_SERVICE: Record = { + page_render: 'frontend', + envoy_access: 'api-gateway', + analytics: 'analytics-service', + catalog: 'image-service', + heartbeat: 'inventory-service', + background: 'session-service', +}; + +function* streamBackground( + rng: Rng, + nowMs: number, + traceCount: number, + logCount: number, + batchSize: number, + resourcePool: Record[]>, +) { + const bufTraces: TraceRow[] = []; + const bufLogs: LogRow[] = []; + const startMs = nowMs - HISTORY_WINDOW_MS; + + for (let i = 0; i < traceCount; i++) { + const t = + traceCount > 1 + ? startMs + (i / (traceCount - 1)) * (HISTORY_WINDOW_MS - 60_000) + : startMs; + const traceId = newTraceId(rng); + const parentService = rng.pick(BACKGROUND_SERVICES); + const parentOp = rng.pick(BACKGROUND_OPS[parentService]); + const parentSpanId = newSpanId(rng); + const isError = parentService !== 'payment-service' && rng.next() < 0.005; + const region = rng.pick(REGIONS); + const tenantId = `tenant-${rng.intRange(1, 5000)}`; + const requestId = uuidv4(rng); + const userId = `u${rng.intRange(1, 5_000_000)}`; + const sessionId = uuidv4(rng); + bufTraces.push( + makeSpan({ + rng, + timestampMs: t, + traceId, + spanId: parentSpanId, + spanName: parentOp, + spanKind: 'SPAN_KIND_SERVER', + serviceName: parentService, + durationNs: msToNs(rng.range(15, 200)), + statusCode: isError ? 'STATUS_CODE_ERROR' : 'STATUS_CODE_OK', + resourceAttributes: pickResource(rng, resourcePool, parentService), + spanAttributes: { + 'http.method': 'GET', + 'http.status_code': isError ? '500' : '200', + 'http.route': `/internal/${parentOp.replace(/[^a-z0-9]+/gi, '/')}`, + 'cloud.region': region, + 'tenant.id': tenantId, + 'user.id': userId, + 'session.id': sessionId, + 'request.id': requestId, + 'rpc.system': 'http', + }, + }), + ); + if (rng.next() < 0.6) { + const candidates = BACKGROUND_SERVICES.filter(s => s !== parentService); + const childService = rng.pick(candidates); + const childOp = rng.pick(BACKGROUND_OPS[childService]); + bufTraces.push( + makeSpan({ + rng, + timestampMs: t + rng.intRange(1, 10), + traceId, + spanId: newSpanId(rng), + parentSpanId, + spanName: childOp, + spanKind: 'SPAN_KIND_CLIENT', + serviceName: childService, + durationNs: msToNs(rng.range(5, 80)), + statusCode: 'STATUS_CODE_OK', + resourceAttributes: pickResource(rng, resourcePool, childService), + spanAttributes: { + 'rpc.system': 'http', + 'cloud.region': region, + 'request.id': requestId, + 'user.id': userId, + 'net.peer.name': `${childService}.svc.cluster.local`, + }, + }), + ); + } + if (bufTraces.length >= batchSize) { + yield { traces: bufTraces.splice(0, bufTraces.length), logs: [] }; + } + } + if (bufTraces.length) { + yield { traces: bufTraces.splice(0, bufTraces.length), logs: [] }; + } + + for (let i = 0; i < logCount; i++) { + const t = + logCount > 1 + ? startMs + (i / (logCount - 1)) * (HISTORY_WINDOW_MS - 60_000) + : startMs; + const kind = rng.weightedPick(LOG_TEMPLATE_MIX); + const service = LOG_TEMPLATE_SERVICE[kind]; + const sev = pickSeverityIn( + rng, + kind === 'heartbeat' + ? 'debug' + : rng.weightedPick<'info' | 'warn' | 'error'>([ + { value: 'info', weight: 88 }, + { value: 'warn', weight: 9 }, + { value: 'error', weight: 3 }, + ]), + ); + let body: string; + let attrs: Record; + if (kind === 'page_render') { + const tmpl = pageRenderLog({ rng, nowMs: t }); + body = tmpl.body; + attrs = tmpl.attrs; + } else if (kind === 'envoy_access') { + const tmpl = envoyAccessLog({ rng, nowMs: t }); + body = tmpl.body; + attrs = tmpl.attrs; + } else if (kind === 'analytics') { + const tmpl = analyticsEventLog({ rng, nowMs: t }); + body = tmpl.body; + attrs = tmpl.attrs; + } else if (kind === 'catalog') { + const tmpl = catalogLog({ rng, nowMs: t, level: sev.text.toLowerCase() }); + body = tmpl.body; + attrs = tmpl.attrs; + } else if (kind === 'heartbeat') { + const tmpl = heartbeatLog({ rng, nowMs: t, serviceName: service }); + body = tmpl.body; + attrs = tmpl.attrs; + } else { + body = backgroundLog({ rng, nowMs: t }); + attrs = { + 'event.name': 'app.background', + 'log.iostream': 'stdout', + logtag: 'F', + }; + } + const sevText = sev.text.toUpperCase().startsWith('WARN') + ? 'WARN' + : sev.text.toUpperCase().startsWith('ERR') || sev.text === 'fatal' + ? 'ERROR' + : sev.text.toUpperCase().startsWith('DEB') + ? 'DEBUG' + : sev.text.toUpperCase() === 'TRACE' + ? 'TRACE' + : 'INFO'; + bufLogs.push( + makeLog({ + timestampMs: t, + serviceName: service, + severityText: sevText, + body, + resourceAttributes: pickResource(rng, resourcePool, service), + logAttributes: { ...attrs, _severity_raw: sev.text }, + }), + ); + if (bufLogs.length >= batchSize) { + yield { traces: [], logs: bufLogs.splice(0, bufLogs.length) }; + } + } + if (bufLogs.length) yield { traces: [], logs: bufLogs }; +} diff --git a/packages/hdx-eval/src/scenarios/error-root-cause/ground-truth.json b/packages/hdx-eval/src/scenarios/error-root-cause/ground-truth.json new file mode 100644 index 0000000000..2d0930df1c --- /dev/null +++ b/packages/hdx-eval/src/scenarios/error-root-cause/ground-truth.json @@ -0,0 +1,202 @@ +{ + "scenario": "error-root-cause", + "agentPrompt": "Checkout requests are failing for some users in the last 10 minutes. Find the root cause.", + "expected": { + "rootCauseService": "payment-service", + "rootCauseOperation": "payment.charge", + "downstreamFailure": "db.payment.connect", + "errorBodyPattern": "db-payment", + "errorMessageVariants": [ + "connection timeout to db-payment", + "timeout connecting to db-payment.internal", + "db-payment unreachable after 5 retries" + ], + "userFacingService": "checkout-api", + "userFacingOperation": "POST /api/checkout", + "onsetMinutesAgo": 5, + "approxFailingTraces": 8 + }, + "distractors": [ + { + "kind": "api-gateway TLS handshake spike", + "service": "api-gateway", + "windowMinutesAgo": [25, 20], + "approxCount": 50, + "errorMessage": "tls handshake timeout: peer reset", + "whyNotRootCause": "Resolved 20 minutes ago; unrelated to current checkout failures." + }, + { + "kind": "auth-service rate-limit blip", + "service": "auth-service", + "windowMinutesAgo": [40, 38], + "approxCount": 20, + "errorMessage": "rate limit exceeded: 1000 req/min on /v2/token", + "whyNotRootCause": "Resolved 38 minutes ago; not in the current incident window." + }, + { + "kind": "search-service slow queries", + "service": "search-service", + "windowMinutesAgo": "throughout-hour", + "approxCount": 80, + "whyNotRootCause": "Status OK, not errors. Slow but successful. Unrelated to checkout cascade." + }, + { + "kind": "background payment.charge declines", + "service": "payment-service", + "spanName": "payment.charge", + "windowMinutesAgo": "throughout-hour", + "approxCount": 200, + "errorTypes": [ + "card declined", + "insufficient funds", + "CVV mismatch", + "expired card", + "payment method not supported" + ], + "whyNotRootCause": "Operational user-level declines, not infrastructure failures. error.type='PaymentDeclined' (not ConnectionTimeoutError). Distributed throughout the hour, not concentrated in the recent window." + }, + { + "kind": "notification-service smtp failure (concurrent)", + "service": "notification-service", + "spanName": "notify.email", + "windowMinutesAgo": [5, 0], + "approxCount": 25, + "errorMessage": "smtp connection refused: smtp.internal:587", + "errorType": "SMTPConnectionError", + "whyNotRootCause": "Concurrent but separate incident — these traces have no checkout-api parent and don't appear in any checkout trace tree. Different dependency (smtp.internal vs db-payment.internal)." + }, + { + "kind": "cdn origin-fetch burst (concurrent)", + "service": "cdn", + "spanName": "cdn.cache.fetch", + "windowMinutesAgo": [5, 0], + "approxCount": 80, + "errorMessage": "origin connection refused: origin-static.internal:443", + "errorType": "OriginUnreachableError", + "whyNotRootCause": "Concurrent recent 5xx spike, but on cdn traces with no checkout-api parent. Looks superficially similar to the planted cascade (service-can't-reach-dependency) but lives in entirely separate trace trees and has no link to payment-service or checkout-api. Status-code-only queries will return these alongside the planted failures." + } + ], + "anomalyAttributes": { + "spanAttributes": { + "db.system": "postgres", + "db.name": "payments", + "error.type": "ConnectionTimeoutError", + "net.peer.name": "db-payment.internal" + }, + "logAttributes": { + "error.kind": "DBTimeout", + "component": "payment-service" + } + }, + "rubric": { + "programmatic": [ + { + "id": "names_root_cause_service", + "weight": 2, + "pattern": "payment-service", + "flags": "i" + }, + { + "id": "names_underlying_failure", + "weight": 2, + "pattern": "(connection timeout|db connection|connect.*timeout|timeout.*connect|unreachable)", + "flags": "i" + }, + { + "id": "names_user_facing_service", + "weight": 1, + "pattern": "checkout(-api)?", + "flags": "i" + }, + { + "id": "names_db_target", + "weight": 1, + "pattern": "db-payment", + "flags": "i" + }, + { + "id": "names_error_type", + "weight": 2, + "pattern": "(ConnectionTimeoutError|connection.?timeout|DBTimeout|error\\.type[^a-z]+(connection|db))", + "flags": "i" + }, + { + "id": "names_db_host_fqdn", + "weight": 1, + "pattern": "db-payment\\.internal", + "flags": "i" + }, + { + "id": "names_specific_db_span", + "weight": 1, + "pattern": "(db\\.payment\\.connect|payment\\.charge)", + "flags": "i" + }, + { + "id": "ruled_out_a_distractor", + "weight": 2, + "pattern": "(rul(ed|ing) out|not (the )?(root )?cause|concurrent (but )?(separate|unrelated)|distractor|separate (incident|trace tree|burst))[\\s\\S]{0,250}?\\b(tls|api-gateway|auth-service|rate.?limit|smtp|notification-service|cdn|origin|card declin|insufficient funds|search-service)", + "flags": "i" + }, + { + "id": "false_blame_tls_handshake", + "weight": 2, + "negative": true, + "pattern": "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\b(tls handshake|api-gateway[^.\\n]{0,40}?(tls|handshake|peer reset))", + "flags": "i" + }, + { + "id": "false_blame_auth_rate_limit", + "weight": 2, + "negative": true, + "pattern": "(root cause|caused by|primary issue|the issue is|because of|the cause is)[^.\\n]{0,80}?\\b(rate.?limit|auth-service[^.\\n]{0,40}?(rate.?limit|429|throttl))", + "flags": "i" + }, + { + "id": "false_blame_smtp", + "weight": 2, + "negative": true, + "pattern": "(root cause|caused by|primary issue|the issue is|because of|the cause is)[^.\\n]{0,80}?\\b(smtp|notification-service[^.\\n]{0,40}?(smtp|email))", + "flags": "i" + }, + { + "id": "false_blame_payment_decline", + "weight": 2, + "negative": true, + "pattern": "(root cause|caused by|primary issue|the issue is|because of|the cause is)[^.\\n]{0,80}?\\b(card declined|insufficient funds|cvv mismatch|expired card|payment.?declined)", + "flags": "i" + }, + { + "id": "false_blame_cdn_origin", + "weight": 2, + "negative": true, + "pattern": "(root cause|caused by|primary issue|the issue is|because of|the cause is)[^.\\n]{0,80}?\\b(cdn|origin[^.\\n]{0,40}?(refused|unreachable|timeout)|cache.?miss[^.\\n]{0,40}?upstream)", + "flags": "i" + } + ], + "judge": { + "criteria": [ + { + "id": "correctness", + "weight": 3, + "description": "Did the answer correctly identify payment-service as the root-cause service AND the DB connection timeout to db-payment as the underlying failure? The error message has 3 variants — recognizing them as one cause counts." + }, + { + "id": "completeness", + "weight": 2, + "description": "Did it explain the full failure chain — payment-service db.payment.connect timeout → payment.charge fails → checkout-api 5xx — rather than just naming one layer?" + }, + { + "id": "calibration", + "weight": 2, + "description": "Did the answer correctly distinguish the planted timeout cascade from the distractors? Specifically: (a) Resolved historical incidents — api-gateway TLS spike at T-25min, auth-service rate-limit blip at T-40min — are NOT the current cause. (b) search-service slow queries are OK status, not errors. (c) payment-service payment.charge declines (card declined/insufficient funds) are operational user-level failures, NOT the infrastructure incident. (d) notification-service smtp errors and cdn origin-fetch errors are CONCURRENT bursts in the same window but live on separate traces with no checkout-api parent — not the cause of checkout failures. Status-code-only queries will surface all of these together; the agent must trace the causal chain rather than rely on raw error counts. Mistaking any of these for the root cause is a calibration failure." + }, + { + "id": "conciseness", + "weight": 1, + "description": "Was the answer concise, well-structured, and free of filler or unnecessary speculation?" + } + ] + } + } +} diff --git a/packages/hdx-eval/src/scenarios/index.ts b/packages/hdx-eval/src/scenarios/index.ts new file mode 100644 index 0000000000..6387da339e --- /dev/null +++ b/packages/hdx-eval/src/scenarios/index.ts @@ -0,0 +1,26 @@ +import { errorRootCauseScenario } from './error-root-cause/generate'; +import { latencySpikeScenario } from './latency-spike/generate'; +import { noisySignalsScenario } from './noisy-signals/generate'; +import { segmentedRegressionScenario } from './segmented-regression/generate'; +import { serviceHealthCheckScenario } from './service-health-check/generate'; +import type { Scenario } from './types'; + +export const SCENARIOS: Record = { + [errorRootCauseScenario.name]: errorRootCauseScenario, + [latencySpikeScenario.name]: latencySpikeScenario, + [noisySignalsScenario.name]: noisySignalsScenario, + [segmentedRegressionScenario.name]: segmentedRegressionScenario, + [serviceHealthCheckScenario.name]: serviceHealthCheckScenario, +}; + +export const SCENARIO_NAMES = Object.keys(SCENARIOS); + +export function getScenario(name: string): Scenario { + const s = SCENARIOS[name]; + if (!s) { + throw new Error( + `Unknown scenario: ${name}. Known: ${SCENARIO_NAMES.join(', ')}`, + ); + } + return s; +} diff --git a/packages/hdx-eval/src/scenarios/latency-spike/generate.ts b/packages/hdx-eval/src/scenarios/latency-spike/generate.ts new file mode 100644 index 0000000000..e6e3161eb5 --- /dev/null +++ b/packages/hdx-eval/src/scenarios/latency-spike/generate.ts @@ -0,0 +1,559 @@ +import { + buildResourcePool, + pickResource, + uuidv4, +} from '../../generators/templates'; +import { + makeSpan, + msToNs, + newSpanId, + newTraceId, +} from '../../generators/traces'; +import type { LogRow, TraceRow } from '../../generators/types'; +import type { GenerateContext, Scenario, ScenarioBatch } from '../types'; +import groundTruth from './ground-truth.json'; + +// v6 — 6M base traces × ~2 child spans + 4M background traces × 2 spans +// + extras → ~20M+ total spans. Background traffic plants ~100 services +// × ~15 operations each so the (ServiceName, SpanName) cardinality is +// production-shaped (~1500 distinct combos) and naive +// `GROUP BY ServiceName, SpanName` returns too many rows to scan. +const TOTAL_TRACES = 6_000_000; +const BACKGROUND_TRACES = 4_000_000; +const HISTORY_WINDOW_MS = 60 * 60 * 1000; +const ANOMALY_WINDOW_MS = 10 * 60 * 1000; + +const ENDPOINTS = [ + { name: 'GET /api/orders/search', weight: 12 }, + { name: 'GET /api/orders/{id}', weight: 18 }, + { name: 'POST /api/orders', weight: 6 }, + { name: 'GET /api/products/search', weight: 10 }, + { name: 'GET /api/products/{id}', weight: 12 }, + { name: 'GET /api/inventory/{sku}', weight: 8 }, + { name: 'POST /api/cart/items', weight: 8 }, + { name: 'GET /api/cart/{id}', weight: 8 }, + { name: 'GET /api/users/{id}', weight: 6 }, + { name: 'POST /api/checkout', weight: 4 }, + { name: 'GET /api/recommendations', weight: 4 }, + { name: 'GET /api/health', weight: 4 }, +]; + +const REGIONS = [ + 'us-east-1', + 'us-west-2', + 'eu-west-1', + 'ap-south-1', + 'sa-east-1', +] as const; +const TENANT_RANGE = 5_000; // was 200 + +const TENANT_TIERS = [ + { value: 'free', weight: 60 }, + { value: 'pro', weight: 35 }, + { value: 'enterprise', weight: 5 }, +] as const; + +// Normal-traffic latency distribution. Each tuple is (lo_ms, hi_ms, weight). +const NORMAL_LATENCY_BUCKETS = [ + { lo: 25, hi: 80, weight: 65 }, + { lo: 80, hi: 180, weight: 25 }, + { lo: 180, hi: 350, weight: 8 }, + { lo: 350, hi: 600, weight: 2 }, +]; + +// Distractors +const GC_PAUSE_RATE = 0.003; // 0.3% of all traces — uniform across tiers/endpoints +const CACHE_LOOKUP_RATE = 0.03; // 3% of traces add an extra cache.lookup child span (200-400ms) + +const COLD_START_WINDOW_END_MS = 55 * 60 * 1000; // T-55min +const COLD_START_WINDOW_START_MS = 60 * 60 * 1000; // T-60min +const COLD_START_RATE_IN_WINDOW = 0.4; // 40% of traces in the early window are cold-start + +// Anomaly +const ENTERPRISE_CACHE_HIT_RATE = 0.2; // 20% of enterprise window requests still fast (cache hit) + +// Concurrent decoy: GET /api/products/search has its own slowness in the +// same window, all tiers, caused by an elasticsearch hot shard. Real, but +// unrelated to the orders/search × enterprise × missing-index issue. +const PRODUCTS_SEARCH_HOT_SHARD_RATE = 0.5; // 50% of products/search calls in window are slow + +// Confounder: feature_flag.experiment_a is enabled at very different rates +// across tiers — naive correlation analysis would suggest the flag causes +// the slowness, but the actual cause is the missing index on tenant_id. +const FEATURE_FLAG_RATE_BY_TIER: Record = { + enterprise: 0.8, + pro: 0.3, + free: 0.3, +}; + +type Rng = GenerateContext['rng']; + +function pickNormalLatencyMs(rng: Rng): number { + const bucket = rng.weightedPick( + NORMAL_LATENCY_BUCKETS.map(b => ({ value: b, weight: b.weight })), + ); + return rng.range(bucket.lo, bucket.hi); +} + +const SERVICE_POOL_LIST = [ + 'api-server', + 'database', + 'search-index', + 'cache', +] as const; + +// ─── Background traffic — production-shaped cardinality ───────────────────── +// +// In production environments a single application emits traces from dozens of +// services with thousands of distinct (ServiceName, SpanName) combinations. +// To prevent agents from trivially solving the eval via +// `GROUP BY ServiceName, SpanName ORDER BY p99 DESC LIMIT 20`, we plant a +// pool of ~100 background services × ~15 operations each into the trace +// stream alongside the focal incident pattern. Background traces have +// normal-distribution latency (no spikes) and zero overlap with the +// planted/decoy span names, so they don't confound the rubric — they just +// raise the noise floor of group-by enumerations. + +const BG_SERVICE_PREFIXES = [ + 'auth', + 'billing', + 'payments', + 'notifications', + 'emails', + 'sms', + 'user-profile', + 'user-prefs', + 'user-events', + 'analytics', + 'metrics', + 'tracing', + 'logging', + 'audit', + 'compliance', + 'gdpr', + 'pii-scan', + 'feature-flag', + 'config', + 'secrets', + 'kms', + 'queue', + 'worker', + 'cron', + 'scheduler', + 'retry', + 'ml-inference', + 'recommendation', + 'personalization', + 'fraud', + 'risk', + 'rate-limiter', + 'throttle', + 'cdn', + 'image-resizer', + 'thumbnailer', + 'media', + 'webhook', + 'integration', + 'partner-api', + 'graph', + 'recommender', + 'feed', + 'timeline', + 'shipping', + 'tax', + 'currency', + 'i18n', + 'inventory-sync', + 'price-sync', + 'catalog-sync', +] as const; +const BG_SERVICE_SUFFIXES = ['v1', 'v2', 'worker', 'api', 'svc'] as const; +const BG_OPERATIONS = [ + 'GET /health', + 'GET /ready', + 'GET /metrics', + 'GET /v1/get', + 'POST /v1/create', + 'PUT /v1/update', + 'DELETE /v1/delete', + 'GET /v1/list', + 'GET /v1/search', + 'POST /v1/batch', + 'http.send', + 'http.receive', + 'grpc.call', + 'rpc.invoke', + 'db.query', + 'db.write', + 'db.connect', + 'cache.get', + 'cache.set', + 'cache.evict', + 'queue.publish', + 'queue.consume', + 'queue.ack', + 'compute', + 'transform', + 'serialize', + 'validate', + 'render', + 'enrich', + 'normalize', + 'aggregate', +] as const; +const BG_CHILD_OPERATIONS = [ + 'db.select', + 'db.insert', + 'db.update', + 'cache.lookup', + 'cache.store', + 'http.dependency', + 'grpc.dependency', + 'queue.fetch', + 'queue.send', + 'compute.task', +] as const; +const BG_SERVICE_COUNT = 100; +const BG_OPS_PER_SERVICE = 15; + +type BgService = { + name: string; + operations: readonly string[]; + childOps: readonly string[]; +}; + +function buildBackgroundServices(rng: Rng): BgService[] { + const services: BgService[] = []; + const used = new Set(); + // Cap at BG_SERVICE_COUNT distinct names. + let attempts = 0; + while (services.length < BG_SERVICE_COUNT && attempts < 10_000) { + attempts++; + const prefix = rng.pick(BG_SERVICE_PREFIXES); + const suffix = rng.pick(BG_SERVICE_SUFFIXES); + const name = `${prefix}-${suffix}`; + if (used.has(name)) continue; + used.add(name); + // Each service emits ~BG_OPS_PER_SERVICE distinct operations. + const ops: string[] = []; + const opsUsed = new Set(); + for (let i = 0; i < BG_OPS_PER_SERVICE; i++) { + const o = rng.pick(BG_OPERATIONS); + if (!opsUsed.has(o)) { + opsUsed.add(o); + ops.push(o); + } + } + services.push({ + name, + operations: ops, + childOps: rng.pick([ + BG_CHILD_OPERATIONS.slice(0, 3), + BG_CHILD_OPERATIONS.slice(2, 6), + BG_CHILD_OPERATIONS.slice(5, 10), + ]), + }); + } + return services; +} + +export const latencySpikeScenario: Scenario = { + name: 'latency-spike', + agentPrompt: groundTruth.agentPrompt, + description: + 'api-server p99 spike on /api/orders/search for enterprise tenants (missing-index DB query). 12M+ spans across 12 endpoints, 5 regions, 5K tenants, with GC-pause / cold-start / cache-lookup / hot-shard distractors.', + *generate(ctx): Iterable { + const { rng, nowMs } = ctx; + const factor = ctx.volumeFactor ?? 1; + const batchSize = ctx.batchSize ?? 10_000; + const totalTraces = Math.max(50, Math.round(TOTAL_TRACES * factor)); + const resourcePool = buildResourcePool({ + rng, + services: SERVICE_POOL_LIST, + instancesPerService: 24, + }); + const traces: TraceRow[] = []; + const logs: LogRow[] = []; + const startMs = nowMs - HISTORY_WINDOW_MS; + const anomalyStartMs = nowMs - ANOMALY_WINDOW_MS; + + for (let i = 0; i < totalTraces; i++) { + const t = + totalTraces > 1 + ? startMs + (i / (totalTraces - 1)) * (HISTORY_WINDOW_MS - 30_000) + : startMs; + const traceId = newTraceId(rng); + const endpoint = rng.weightedPick( + ENDPOINTS.map(e => ({ value: e.name, weight: e.weight })), + ); + const tenantTier = rng.weightedPick( + TENANT_TIERS.map(t => ({ value: t.value, weight: t.weight })), + ); + const tenantId = `${tenantTier}-${rng.intRange(1, TENANT_RANGE)}`; + const region = rng.pick(REGIONS); + + const inAnomalyWindow = t >= anomalyStartMs; + const isAffectedSegment = + inAnomalyWindow && + endpoint === 'GET /api/orders/search' && + tenantTier === 'enterprise'; + const isCacheHit = + isAffectedSegment && rng.next() < ENTERPRISE_CACHE_HIT_RATE; + const isSlowAnomaly = isAffectedSegment && !isCacheHit; + + // Concurrent decoy: products/search hot-shard slowness in the SAME + // window, all tiers. A real incident on a different endpoint — + // calibration test for not conflating with the orders/search anomaly. + const isProductsSearchHotShard = + inAnomalyWindow && + endpoint === 'GET /api/products/search' && + rng.next() < PRODUCTS_SEARCH_HOT_SHARD_RATE; + + // Distractors only fire when not the planted anomaly nor the + // products/search decoy (so duration profiles don't overlap). + const inColdStartWindow = + t >= nowMs - COLD_START_WINDOW_START_MS && + t <= nowMs - COLD_START_WINDOW_END_MS; + const isColdStart = + !isAffectedSegment && + !isProductsSearchHotShard && + inColdStartWindow && + rng.next() < COLD_START_RATE_IN_WINDOW; + const isGcPause = + !isAffectedSegment && + !isProductsSearchHotShard && + !isColdStart && + rng.next() < GC_PAUSE_RATE; + + // Feature-flag confounder — correlated with tier but not the cause. + const flagRate = FEATURE_FLAG_RATE_BY_TIER[tenantTier] ?? 0.3; + const flagOn = rng.next() < flagRate; + + let dbDurationMs: number; + let parentDurationMs: number; + const requestId = uuidv4(rng); + const userId = `u${rng.intRange(1, 5_000_000)}`; + const sessionId = uuidv4(rng); + const route = endpoint.replace(/^[A-Z]+ /, ''); + const queryParam = + endpoint.includes('search') || endpoint.includes('?') + ? `?q=${rng.hex(4)}&page=${rng.intRange(1, 50)}&limit=${rng.intRange(10, 100)}` + : ''; + const parentAttrs: Record = { + 'http.method': endpoint.startsWith('POST') ? 'POST' : 'GET', + 'http.route': route, + 'http.target': `${route}${queryParam}`, + 'url.path': route, + 'url.full': `https://api.example.com${route}${queryParam}`, + 'url.scheme': 'https', + 'http.status_code': '200', + 'http.request.body.size': String(rng.intRange(0, 4096)), + 'http.response.body.size': String(rng.intRange(0, 32768)), + 'tenant.id': tenantId, + 'tenant.tier': tenantTier, + 'user.id': userId, + 'session.id': sessionId, + 'cloud.region': region, + 'feature_flag.experiment_a': flagOn ? 'enabled' : 'disabled', + 'request.id': requestId, + 'net.peer.name': 'api-server.svc.cluster.local', + 'rpc.system': 'http', + }; + const childAttrs: Record = { + 'db.system': 'postgres', + 'db.name': 'orders', + 'tenant.id': tenantId, + 'tenant.tier': tenantTier, + 'user.id': userId, + 'request.id': requestId, + 'net.peer.name': 'db-orders-primary.internal', + 'net.peer.port': '5432', + }; + let childSpanName = 'database.query'; + let childServiceName = 'database'; + + if (isSlowAnomaly) { + dbDurationMs = rng.range(1500, 2500); + parentDurationMs = dbDurationMs + rng.range(15, 35); + childAttrs['db.statement'] = 'SELECT * FROM orders WHERE tenant_id = ?'; + } else if (isProductsSearchHotShard) { + dbDurationMs = rng.range(1500, 2500); + parentDurationMs = dbDurationMs + rng.range(15, 35); + childServiceName = 'search-index'; + childSpanName = 'elasticsearch.query'; + childAttrs['db.system'] = 'elasticsearch'; + childAttrs['db.name'] = 'products'; + childAttrs['db.statement'] = 'POST /products/_search'; + childAttrs['elasticsearch.shard.hot'] = 'true'; + childAttrs['elasticsearch.shard.id'] = '17'; + } else if (isCacheHit) { + // Enterprise cache-hit subset — still fast despite being in window. + dbDurationMs = rng.range(2, 8); + parentDurationMs = rng.range(8, 30); + childAttrs['cache.hit'] = 'true'; + childAttrs['db.statement'] = + 'SELECT * FROM orders WHERE tenant_id = ? -- cached'; + } else if (isGcPause) { + dbDurationMs = rng.range(20, 60); + parentDurationMs = rng.range(1500, 2200); + parentAttrs['runtime.event'] = 'gc_pause'; + parentAttrs['runtime.gc.duration_ms'] = String( + Math.round(parentDurationMs - dbDurationMs), + ); + } else if (isColdStart) { + dbDurationMs = rng.range(20, 60); + parentDurationMs = rng.range(2000, 3000); + parentAttrs['runtime.event'] = 'cold_start'; + parentAttrs['runtime.cold_start_ms'] = String( + Math.round(parentDurationMs - dbDurationMs), + ); + } else { + dbDurationMs = rng.range(5, 50); + parentDurationMs = pickNormalLatencyMs(rng); + // Endpoint-appropriate db statement + if (endpoint.startsWith('GET /api/products')) { + childAttrs['db.name'] = 'products'; + childAttrs['db.statement'] = 'SELECT * FROM products WHERE id = ?'; + } else if (endpoint.startsWith('GET /api/inventory')) { + childAttrs['db.name'] = 'inventory'; + childAttrs['db.statement'] = 'SELECT * FROM inventory WHERE sku = ?'; + } else { + childAttrs['db.statement'] = 'SELECT * FROM orders WHERE id = ?'; + } + } + + const parentSpanId = newSpanId(rng); + traces.push( + makeSpan({ + rng, + timestampMs: t, + traceId, + spanId: parentSpanId, + spanName: endpoint, + spanKind: 'SPAN_KIND_SERVER', + serviceName: 'api-server', + durationNs: msToNs(parentDurationMs), + resourceAttributes: pickResource(rng, resourcePool, 'api-server'), + spanAttributes: parentAttrs, + }), + ); + + traces.push( + makeSpan({ + rng, + timestampMs: t + rng.intRange(1, 5), + traceId, + spanId: newSpanId(rng), + parentSpanId, + spanName: childSpanName, + spanKind: 'SPAN_KIND_CLIENT', + serviceName: childServiceName, + durationNs: msToNs(dbDurationMs), + resourceAttributes: pickResource(rng, resourcePool, childServiceName), + spanAttributes: childAttrs, + }), + ); + + // 3% of traces also get a cache.lookup child (200-400ms) — adds tail + // latency that an agent might mistake for the spike. + if (!isAffectedSegment && rng.next() < CACHE_LOOKUP_RATE) { + traces.push( + makeSpan({ + rng, + timestampMs: t + rng.intRange(2, 8), + traceId, + spanId: newSpanId(rng), + parentSpanId, + spanName: 'cache.lookup', + spanKind: 'SPAN_KIND_CLIENT', + serviceName: 'cache', + durationNs: msToNs(rng.range(200, 400)), + resourceAttributes: pickResource(rng, resourcePool, 'cache'), + spanAttributes: { + 'cache.system': 'redis', + 'cache.key': `cache:${rng.hex(8)}`, + 'cache.hit': rng.next() < 0.7 ? 'true' : 'false', + 'request.id': requestId, + 'net.peer.name': 'redis-cache.internal', + 'net.peer.port': '6379', + }, + }), + ); + } + // Stream traces in batches to avoid OOM at high volume. + if (traces.length >= batchSize) { + yield { traces: traces.splice(0, traces.length), logs: [] }; + } + } + + // ── Background traffic ──────────────────────────────────────────────── + // Production-shaped cardinality: ~100 services × ~15 operations each. + // Normal latency, no slow tails, no overlap with planted span names. + // Builds a pool from the same rng so the result is deterministic for a + // given seed. + const bgServices = buildBackgroundServices(rng); + const bgResourcePool = buildResourcePool({ + rng, + services: bgServices.map(s => s.name), + instancesPerService: 4, + }); + const bgTotal = Math.max(0, Math.round(BACKGROUND_TRACES * factor)); + for (let i = 0; i < bgTotal; i++) { + const t = + bgTotal > 1 + ? startMs + (i / (bgTotal - 1)) * (HISTORY_WINDOW_MS - 30_000) + : startMs; + const svc = bgServices[i % bgServices.length]; + const op = svc.operations[rng.intRange(0, svc.operations.length - 1)]; + const childOp = svc.childOps[rng.intRange(0, svc.childOps.length - 1)]; + const traceId = newTraceId(rng); + const parentSpanId = newSpanId(rng); + const parentDurationMs = pickNormalLatencyMs(rng); + const childDurationMs = Math.round( + parentDurationMs * rng.range(0.1, 0.7), + ); + traces.push( + makeSpan({ + rng, + timestampMs: t, + traceId, + spanId: parentSpanId, + spanName: op, + spanKind: 'SPAN_KIND_SERVER', + serviceName: svc.name, + durationNs: msToNs(parentDurationMs), + resourceAttributes: pickResource(rng, bgResourcePool, svc.name), + spanAttributes: { + 'http.status_code': '200', + 'rpc.system': op.includes('grpc') ? 'grpc' : 'http', + }, + }), + ); + traces.push( + makeSpan({ + rng, + timestampMs: t + rng.intRange(1, 5), + traceId, + spanId: newSpanId(rng), + parentSpanId, + spanName: childOp, + spanKind: 'SPAN_KIND_CLIENT', + serviceName: svc.name, + durationNs: msToNs(childDurationMs), + resourceAttributes: pickResource(rng, bgResourcePool, svc.name), + spanAttributes: { + 'net.peer.name': `${svc.name}.internal`, + }, + }), + ); + if (traces.length >= batchSize) { + yield { traces: traces.splice(0, traces.length), logs: [] }; + } + } + + if (traces.length || logs.length) { + yield { traces, logs }; + } + }, + groundTruth, +}; diff --git a/packages/hdx-eval/src/scenarios/latency-spike/ground-truth.json b/packages/hdx-eval/src/scenarios/latency-spike/ground-truth.json new file mode 100644 index 0000000000..44f9a914af --- /dev/null +++ b/packages/hdx-eval/src/scenarios/latency-spike/ground-truth.json @@ -0,0 +1,162 @@ +{ + "scenario": "latency-spike", + "agentPrompt": "p99 latency on api-server has jumped in the last 15 minutes. What's slow and why?", + "expected": { + "affectedService": "api-server", + "affectedOperation": "GET /api/orders/search", + "slowChildService": "database", + "slowChildOperation": "database.query", + "segmentAttribute": "tenant.tier", + "segmentValue": "enterprise", + "onsetMinutesAgo": 10, + "baselineP99Ms": 600, + "anomalyP99Ms": 2200, + "cacheHitRateInWindow": 0.2, + "endpointsInWorld": [ + "GET /api/orders/search", + "GET /api/orders/{id}", + "POST /api/orders", + "GET /api/products/search", + "GET /api/inventory/{sku}" + ] + }, + "distractors": [ + { + "kind": "GC pause distractor", + "approxRate": 0.003, + "marker": "runtime.event=gc_pause", + "durationMs": [1500, 2200], + "whyNotRootCause": "Uniform across tenants and endpoints — not concentrated on enterprise GET /api/orders/search." + }, + { + "kind": "Cold-start spike", + "windowMinutesAgo": [60, 55], + "marker": "runtime.event=cold_start", + "durationMs": [2000, 3000], + "whyNotRootCause": "Bounded to the first 5 minutes of the hour; not present in the recent window." + }, + { + "kind": "cache.lookup latency tail", + "approxRate": 0.03, + "spanName": "cache.lookup", + "durationMs": [200, 400], + "whyNotRootCause": "Adds tail latency but cap is well below the 1500ms+ spike." + }, + { + "kind": "products/search elasticsearch hot shard (concurrent)", + "endpoint": "GET /api/products/search", + "approxRate": 0.5, + "marker": "elasticsearch.shard.hot=true", + "windowMinutesAgo": [10, 0], + "durationMs": [1500, 2500], + "whyNotRootCause": "Real but unrelated incident on a different endpoint with a different root cause (hot shard, not missing index). Affects all tiers, not enterprise-specific." + }, + { + "kind": "feature_flag.experiment_a confounder", + "marker": "feature_flag.experiment_a=enabled", + "tierCorrelation": { "enterprise": 0.8, "pro": 0.3, "free": 0.3 }, + "whyNotRootCause": "Highly correlated with enterprise tier (because experiment was rolled out there first), so naive correlation analysis would suggest the flag causes the slowness. The flag is also enabled on many pro/free traces that are NOT slow — so it's a correlation, not a cause. The actual cause is the missing index on tenant_id." + } + ], + "anomalyAttributes": { + "spanAttributes": { + "db.system": "postgres", + "db.name": "orders", + "db.statement": "SELECT * FROM orders WHERE tenant_id = ?" + }, + "rootCauseHint": "missing index on orders.tenant_id" + }, + "rubric": { + "programmatic": [ + { + "id": "names_affected_service", + "weight": 1, + "pattern": "api-server", + "flags": "i" + }, + { + "id": "names_affected_endpoint", + "weight": 1, + "pattern": "(/api/orders/search|orders/search)", + "flags": "i" + }, + { + "id": "names_slow_child", + "weight": 2, + "pattern": "(database\\.query|SELECT.*FROM\\s+orders)", + "flags": "i" + }, + { + "id": "names_segment", + "weight": 2, + "pattern": "(enterprise|tenant\\.tier)", + "flags": "i" + }, + { + "id": "names_index_or_query_root_cause", + "weight": 1, + "pattern": "(index|tenant_id|missing.*index)", + "flags": "i" + }, + { + "id": "false_blame_hot_shard", + "weight": 2, + "negative": true, + "pattern": "(root cause|caused by|primary issue|the issue is|the cause is|main issue)[^.\\n]{0,80}?\\b(hot.shard|elasticsearch.shard.hot|shard.{0,30}?hot)", + "flags": "i" + }, + { + "id": "false_blame_products_search", + "weight": 2, + "negative": true, + "pattern": "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\bproducts.search", + "flags": "i" + }, + { + "id": "false_blame_feature_flag", + "weight": 2, + "negative": true, + "pattern": "(root cause|caused by|primary issue|the issue is|because of|the cause is)[^.\\n]{0,80}?\\b(feature.flag|experiment_a)", + "flags": "i" + }, + { + "id": "false_blame_gc_pause", + "weight": 1, + "negative": true, + "pattern": "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\bgc.pause", + "flags": "i" + }, + { + "id": "false_blame_cold_start", + "weight": 1, + "negative": true, + "pattern": "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\bcold.start", + "flags": "i" + } + ], + "judge": { + "criteria": [ + { + "id": "correctness", + "weight": 3, + "description": "Did the answer correctly identify GET /api/orders/search on api-server as affected AND localize the slow path to the database.query child for enterprise tenants? With 5 endpoints in play, naming the wrong endpoint is a correctness failure." + }, + { + "id": "completeness", + "weight": 2, + "description": "Did it surface the segment (tenant.tier=enterprise), describe how p99 changes (only for enterprise on /api/orders/search, other tiers and endpoints unchanged), and propose a plausible root cause (missing index on tenant_id or equivalent)?" + }, + { + "id": "calibration", + "weight": 2, + "description": "Did the answer avoid mistaking distractors for the cause? Specifically: (a) GC pauses are uniform across tiers and would not concentrate on enterprise. (b) cold starts only appear in the first 5 minutes of the window. (c) cache.lookup latency caps at 400ms, well below the 1500ms+ spike. (d) GET /api/products/search has its own concurrent slowness (hot shard, all tiers) — that is a SEPARATE incident from the enterprise orders/search issue, not the same cause. (e) feature_flag.experiment_a is correlated with enterprise tier (~80%) but is also enabled on many free/pro traces that are NOT slow — it is a confounder, not a cause. Blaming the flag, conflating products/search with orders/search, or claiming a single root cause for both endpoint spikes is a calibration failure." + }, + { + "id": "conciseness", + "weight": 1, + "description": "Was the answer concise, well-structured, and free of filler?" + } + ] + } + } +} diff --git a/packages/hdx-eval/src/scenarios/noisy-signals/generate.ts b/packages/hdx-eval/src/scenarios/noisy-signals/generate.ts new file mode 100644 index 0000000000..bc63663d1d --- /dev/null +++ b/packages/hdx-eval/src/scenarios/noisy-signals/generate.ts @@ -0,0 +1,719 @@ +import { makeLog } from '../../generators/logs'; +import { + analyticsEventLog, + backgroundLog, + billingEventLog, + buildResourceAttrs, + cacheHitLog, + catalogLog, + caughtExceptionLog, + consoleLogDumpLog, + envoyAccessLog, + heartbeatLog, + indexingLog, + jobFailedPermanentLog, + notificationDeliveryLog, + pageRenderLog, + pickSeverity, + pickSeverityIn, + searchCacheMissVerboseLog, + serviceOpsDebugLog, + subscriptionMetricLog, + upstreamHealthProbeLog, +} from '../../generators/templates'; +import type { LogRow } from '../../generators/types'; +import type { GenerateContext, Scenario, ScenarioBatch } from '../types'; +import groundTruth from './ground-truth.json'; + +const HISTORY_WINDOW_MS = 60 * 60 * 1000; +const DEFAULT_BATCH_SIZE = 10_000; + +// Composite-cell design: every (ServiceName, SeverityText) cell that holds a +// noisy pattern also holds a high-volume LEGITIMATE pattern of comparable +// size. `GROUP BY ServiceName, SeverityText` no longer identifies noise — +// the agent has to look at event.name / body templates. +// +// Total ≈ 16M logs. +// +// cell NOISE LOAD-BEARING +// notification-service × DEBUG cache.hit notification.delivery +// billing-service × INFO subscription_mrr-log invoice / refund / sub events +// worker × ERROR NonFatalRetryableError job.failed.permanent +// {inv,pri,ship,rec}-svc × DEBUG health.check per-svc ops debug +// frontend-proxy × INFO lb.health.probe envoy access +// frontend × INFO console.log_dump page.render +// search-service × INFO search.cache.miss index.shard.flush +const DEFAULTS = { + // Composite cell 1 (notification-service × DEBUG) + CACHE_HIT: 1_000_000, // ~6.7% NOISE + NOTIFICATION_DELIVERY: 1_000_000, // ~6.7% LOAD-BEARING + + // Composite cell 2 (billing-service × INFO) + SUBSCRIPTION: 1_000_000, // ~6.7% NOISE (metric-as-log) + BILLING_EVENTS: 1_000_000, // ~6.7% LOAD-BEARING (invoice/refund/sub) + + // Composite cell 3 (worker × ERROR) + STACKTRACE: 400_000, // ~2.7% NOISE (caught) + JOB_FAILED: 400_000, // ~2.7% LOAD-BEARING (permanent) + + // Composite cell 4 ({inv,pri,ship,rec}-service × DEBUG) + HEARTBEAT: 2_000_000, // ~13.3% NOISE across 4 services + SERVICE_OPS: 2_000_000, // ~13.3% LOAD-BEARING across same 4 services + + // Composite cell 5 (frontend-proxy × INFO) + LB_PROBE: 1_000_000, // ~6.7% NOISE + ACCESS_LOG: 1_000_000, // ~6.7% LOAD-BEARING + + // Composite cell 6 (frontend × INFO) + CONSOLE_DUMP: 800_000, // ~5.3% NOISE + PAGE_RENDER: 800_000, // ~5.3% LOAD-BEARING + + // Composite cell 7 (search-service × INFO) + SEARCH_MISS: 600_000, // ~4% NOISE + INDEXING: 600_000, // ~4% LOAD-BEARING + + // Single-pattern cells (analytics is load-bearing-only; catalog + bg + // contribute variety so the cells above aren't the only thing in the + // table). + ANALYTICS: 600_000, // ~4% LOAD-BEARING (no noise twin) + CATALOG_LOOKUP: 400_000, // ~2.7% mixed-info background + BACKGROUND_VARIED: 1_400_000, // ~9.3% genuine variety +}; + +type Rng = GenerateContext['rng']; + +function* emitBatched( + rows: LogRow[], + batchSize: number, +): Iterable { + for (let i = 0; i < rows.length; i += batchSize) { + yield { traces: [], logs: rows.slice(i, i + batchSize) }; + } +} + +function spread(i: number, total: number, startMs: number): number { + return total > 1 + ? startMs + (i / (total - 1)) * (HISTORY_WINDOW_MS - 30_000) + : startMs; +} + +export const noisySignalsScenario: Scenario = { + name: 'noisy-signals', + agentPrompt: groundTruth.agentPrompt, + description: + '~16M logs across composite (service, severity) cells. Each cell that contains a noise pattern (drop/throttle candidate) also contains a high-volume LOAD-BEARING pattern of comparable size. Grouping by service+severity alone cannot identify noise — the agent must inspect event.name / body templates to disambiguate keep-vs-drop within each cell.', + *generate(ctx): Iterable { + const { rng, nowMs } = ctx; + const startMs = nowMs - HISTORY_WINDOW_MS; + const batchSize = ctx.batchSize ?? DEFAULT_BATCH_SIZE; + const factor = ctx.volumeFactor ?? 1; + const counts = scaleCounts(DEFAULTS, factor); + + // Composite cell 1: notification-service × DEBUG + yield* phaseCacheHit(rng, startMs, counts.CACHE_HIT, batchSize); + yield* phaseNotificationDelivery( + rng, + startMs, + counts.NOTIFICATION_DELIVERY, + batchSize, + ); + // Composite cell 2: billing-service × INFO + yield* phaseSubscription(rng, startMs, counts.SUBSCRIPTION, batchSize); + yield* phaseBillingEvents(rng, startMs, counts.BILLING_EVENTS, batchSize); + // Composite cell 3: worker × ERROR + yield* phaseStacktrace(rng, startMs, counts.STACKTRACE, batchSize); + yield* phaseJobFailed(rng, startMs, counts.JOB_FAILED, batchSize); + // Composite cell 4: 4-service × DEBUG + yield* phaseHeartbeat(rng, startMs, counts.HEARTBEAT, batchSize); + yield* phaseServiceOps(rng, startMs, counts.SERVICE_OPS, batchSize); + // Composite cell 5: frontend-proxy × INFO + yield* phaseLbProbe(rng, startMs, counts.LB_PROBE, batchSize); + yield* phaseAccessLog(rng, startMs, counts.ACCESS_LOG, batchSize); + // Composite cell 6: frontend × INFO + yield* phaseConsoleDump(rng, startMs, counts.CONSOLE_DUMP, batchSize); + yield* phasePageRender(rng, startMs, counts.PAGE_RENDER, batchSize); + // Composite cell 7: search-service × INFO + yield* phaseSearchMiss(rng, startMs, counts.SEARCH_MISS, batchSize); + yield* phaseIndexing(rng, startMs, counts.INDEXING, batchSize); + // Single-pattern cells + yield* phaseAnalytics(rng, startMs, counts.ANALYTICS, batchSize); + yield* phaseCatalog(rng, startMs, counts.CATALOG_LOOKUP, batchSize); + yield* phaseBackgroundVaried( + rng, + startMs, + counts.BACKGROUND_VARIED, + batchSize, + ); + }, + groundTruth, +}; + +function scaleCounts>( + defaults: T, + factor: number, +): T { + const out = {} as T; + for (const k of Object.keys(defaults) as (keyof T)[]) { + (out[k] as number) = Math.max(1, Math.round(defaults[k] * factor)); + } + return out; +} + +function* phaseHeartbeat( + rng: Rng, + startMs: number, + count: number, + batchSize: number, +): Iterable { + const buf: LogRow[] = []; + const services = [ + 'inventory-service', + 'pricing-service', + 'shipping-service', + 'recommendation-service', + ]; + for (let i = 0; i < count; i++) { + const t = spread(i, count, startMs); + const svc = services[i % services.length]; + const tmpl = heartbeatLog({ rng, nowMs: t, serviceName: svc }); + buf.push( + makeLog({ + timestampMs: t, + serviceName: svc, + severityText: 'DEBUG', + body: tmpl.body, + resourceAttributes: buildResourceAttrs({ rng, serviceName: svc }), + logAttributes: tmpl.attrs, + }), + ); + if (buf.length >= batchSize) { + yield { traces: [], logs: buf.splice(0, buf.length) }; + } + } + if (buf.length) yield { traces: [], logs: buf }; +} + +function* phaseServiceOps( + rng: Rng, + startMs: number, + count: number, + batchSize: number, +): Iterable { + const buf: LogRow[] = []; + const services = [ + 'inventory-service', + 'pricing-service', + 'shipping-service', + 'recommendation-service', + ] as const; + for (let i = 0; i < count; i++) { + const t = spread(i, count, startMs); + const svc = services[i % services.length]; + const tmpl = serviceOpsDebugLog({ rng, nowMs: t, serviceName: svc }); + buf.push( + makeLog({ + timestampMs: t, + serviceName: svc, + severityText: 'DEBUG', + body: tmpl.body, + resourceAttributes: buildResourceAttrs({ rng, serviceName: svc }), + logAttributes: tmpl.attrs, + }), + ); + if (buf.length >= batchSize) { + yield { traces: [], logs: buf.splice(0, buf.length) }; + } + } + if (buf.length) yield { traces: [], logs: buf }; +} + +function* phaseCacheHit( + rng: Rng, + startMs: number, + count: number, + batchSize: number, +): Iterable { + const buf: LogRow[] = []; + for (let i = 0; i < count; i++) { + const t = spread(i, count, startMs); + const tmpl = cacheHitLog({ rng, nowMs: t }); + buf.push( + makeLog({ + timestampMs: t, + serviceName: 'notification-service', + severityText: 'DEBUG', + body: tmpl.body, + resourceAttributes: buildResourceAttrs({ + rng, + serviceName: 'notification-service', + }), + logAttributes: tmpl.attrs, + }), + ); + if (buf.length >= batchSize) { + yield { traces: [], logs: buf.splice(0, buf.length) }; + } + } + if (buf.length) yield { traces: [], logs: buf }; +} + +function* phaseNotificationDelivery( + rng: Rng, + startMs: number, + count: number, + batchSize: number, +): Iterable { + const buf: LogRow[] = []; + for (let i = 0; i < count; i++) { + const t = spread(i, count, startMs); + const tmpl = notificationDeliveryLog({ rng, nowMs: t }); + buf.push( + makeLog({ + timestampMs: t, + serviceName: 'notification-service', + severityText: 'DEBUG', + body: tmpl.body, + resourceAttributes: buildResourceAttrs({ + rng, + serviceName: 'notification-service', + }), + logAttributes: tmpl.attrs, + }), + ); + if (buf.length >= batchSize) { + yield { traces: [], logs: buf.splice(0, buf.length) }; + } + } + if (buf.length) yield { traces: [], logs: buf }; +} + +function* phaseStacktrace( + rng: Rng, + startMs: number, + count: number, + batchSize: number, +): Iterable { + const buf: LogRow[] = []; + for (let i = 0; i < count; i++) { + const t = spread(i, count, startMs); + const tmpl = caughtExceptionLog({ rng, nowMs: t }); + buf.push( + makeLog({ + timestampMs: t, + serviceName: 'worker', + severityText: 'ERROR', + body: tmpl.body, + resourceAttributes: buildResourceAttrs({ rng, serviceName: 'worker' }), + logAttributes: tmpl.attrs, + }), + ); + if (buf.length >= batchSize) { + yield { traces: [], logs: buf.splice(0, buf.length) }; + } + } + if (buf.length) yield { traces: [], logs: buf }; +} + +function* phaseJobFailed( + rng: Rng, + startMs: number, + count: number, + batchSize: number, +): Iterable { + const buf: LogRow[] = []; + for (let i = 0; i < count; i++) { + const t = spread(i, count, startMs); + const tmpl = jobFailedPermanentLog({ rng, nowMs: t }); + buf.push( + makeLog({ + timestampMs: t, + serviceName: 'worker', + severityText: 'ERROR', + body: tmpl.body, + resourceAttributes: buildResourceAttrs({ rng, serviceName: 'worker' }), + logAttributes: tmpl.attrs, + }), + ); + if (buf.length >= batchSize) { + yield { traces: [], logs: buf.splice(0, buf.length) }; + } + } + if (buf.length) yield { traces: [], logs: buf }; +} + +function* phaseSubscription( + rng: Rng, + startMs: number, + count: number, + batchSize: number, +): Iterable { + const buf: LogRow[] = []; + for (let i = 0; i < count; i++) { + const t = spread(i, count, startMs); + const tmpl = subscriptionMetricLog({ rng, nowMs: t }); + buf.push( + makeLog({ + timestampMs: t, + serviceName: 'billing-service', + severityText: 'INFO', + body: tmpl.body, + resourceAttributes: buildResourceAttrs({ + rng, + serviceName: 'billing-service', + }), + logAttributes: tmpl.attrs, + }), + ); + if (buf.length >= batchSize) { + yield { traces: [], logs: buf.splice(0, buf.length) }; + } + } + if (buf.length) yield { traces: [], logs: buf }; +} + +function* phaseBillingEvents( + rng: Rng, + startMs: number, + count: number, + batchSize: number, +): Iterable { + const buf: LogRow[] = []; + for (let i = 0; i < count; i++) { + const t = spread(i, count, startMs); + const tmpl = billingEventLog({ rng, nowMs: t }); + buf.push( + makeLog({ + timestampMs: t, + serviceName: 'billing-service', + severityText: 'INFO', + body: tmpl.body, + resourceAttributes: buildResourceAttrs({ + rng, + serviceName: 'billing-service', + }), + logAttributes: tmpl.attrs, + }), + ); + if (buf.length >= batchSize) { + yield { traces: [], logs: buf.splice(0, buf.length) }; + } + } + if (buf.length) yield { traces: [], logs: buf }; +} + +function* phaseAccessLog( + rng: Rng, + startMs: number, + count: number, + batchSize: number, +): Iterable { + const buf: LogRow[] = []; + for (let i = 0; i < count; i++) { + const t = spread(i, count, startMs); + const tmpl = envoyAccessLog({ rng, nowMs: t }); + const sev = pickSeverityIn(rng, 'info'); + buf.push( + makeLog({ + timestampMs: t, + serviceName: 'frontend-proxy', + severityText: 'INFO', + body: tmpl.body, + resourceAttributes: buildResourceAttrs({ + rng, + serviceName: 'frontend-proxy', + }), + logAttributes: { ...tmpl.attrs, _severity_raw: sev.text }, + }), + ); + if (buf.length >= batchSize) { + yield { traces: [], logs: buf.splice(0, buf.length) }; + } + } + if (buf.length) yield { traces: [], logs: buf }; +} + +function* phaseLbProbe( + rng: Rng, + startMs: number, + count: number, + batchSize: number, +): Iterable { + const buf: LogRow[] = []; + for (let i = 0; i < count; i++) { + const t = spread(i, count, startMs); + const tmpl = upstreamHealthProbeLog({ rng, nowMs: t }); + buf.push( + makeLog({ + timestampMs: t, + serviceName: 'frontend-proxy', + severityText: 'INFO', + body: tmpl.body, + resourceAttributes: buildResourceAttrs({ + rng, + serviceName: 'frontend-proxy', + }), + logAttributes: tmpl.attrs, + }), + ); + if (buf.length >= batchSize) { + yield { traces: [], logs: buf.splice(0, buf.length) }; + } + } + if (buf.length) yield { traces: [], logs: buf }; +} + +function* phasePageRender( + rng: Rng, + startMs: number, + count: number, + batchSize: number, +): Iterable { + const buf: LogRow[] = []; + for (let i = 0; i < count; i++) { + const t = spread(i, count, startMs); + const tmpl = pageRenderLog({ rng, nowMs: t }); + buf.push( + makeLog({ + timestampMs: t, + serviceName: 'frontend', + severityText: 'INFO', + body: tmpl.body, + resourceAttributes: buildResourceAttrs({ + rng, + serviceName: 'frontend', + }), + logAttributes: tmpl.attrs, + }), + ); + if (buf.length >= batchSize) { + yield { traces: [], logs: buf.splice(0, buf.length) }; + } + } + if (buf.length) yield { traces: [], logs: buf }; +} + +function* phaseConsoleDump( + rng: Rng, + startMs: number, + count: number, + batchSize: number, +): Iterable { + const buf: LogRow[] = []; + for (let i = 0; i < count; i++) { + const t = spread(i, count, startMs); + const tmpl = consoleLogDumpLog({ rng, nowMs: t }); + buf.push( + makeLog({ + timestampMs: t, + serviceName: 'frontend', + severityText: 'INFO', + body: tmpl.body, + resourceAttributes: buildResourceAttrs({ + rng, + serviceName: 'frontend', + }), + logAttributes: tmpl.attrs, + }), + ); + if (buf.length >= batchSize) { + yield { traces: [], logs: buf.splice(0, buf.length) }; + } + } + if (buf.length) yield { traces: [], logs: buf }; +} + +function* phaseIndexing( + rng: Rng, + startMs: number, + count: number, + batchSize: number, +): Iterable { + const buf: LogRow[] = []; + for (let i = 0; i < count; i++) { + const t = spread(i, count, startMs); + const tmpl = indexingLog({ rng, nowMs: t }); + buf.push( + makeLog({ + timestampMs: t, + serviceName: 'search-service', + severityText: 'INFO', + body: tmpl.body, + resourceAttributes: buildResourceAttrs({ + rng, + serviceName: 'search-service', + }), + logAttributes: tmpl.attrs, + }), + ); + if (buf.length >= batchSize) { + yield { traces: [], logs: buf.splice(0, buf.length) }; + } + } + if (buf.length) yield { traces: [], logs: buf }; +} + +function* phaseSearchMiss( + rng: Rng, + startMs: number, + count: number, + batchSize: number, +): Iterable { + const buf: LogRow[] = []; + for (let i = 0; i < count; i++) { + const t = spread(i, count, startMs); + const tmpl = searchCacheMissVerboseLog({ rng, nowMs: t }); + buf.push( + makeLog({ + timestampMs: t, + serviceName: 'search-service', + severityText: 'INFO', + body: tmpl.body, + resourceAttributes: buildResourceAttrs({ + rng, + serviceName: 'search-service', + }), + logAttributes: tmpl.attrs, + }), + ); + if (buf.length >= batchSize) { + yield { traces: [], logs: buf.splice(0, buf.length) }; + } + } + if (buf.length) yield { traces: [], logs: buf }; +} + +function* phaseAnalytics( + rng: Rng, + startMs: number, + count: number, + batchSize: number, +): Iterable { + const buf: LogRow[] = []; + for (let i = 0; i < count; i++) { + const t = spread(i, count, startMs); + const tmpl = analyticsEventLog({ rng, nowMs: t }); + buf.push( + makeLog({ + timestampMs: t, + serviceName: 'analytics-service', + severityText: 'INFO', + body: tmpl.body, + resourceAttributes: buildResourceAttrs({ + rng, + serviceName: 'analytics-service', + }), + logAttributes: tmpl.attrs, + }), + ); + if (buf.length >= batchSize) { + yield { traces: [], logs: buf.splice(0, buf.length) }; + } + } + if (buf.length) yield { traces: [], logs: buf }; +} + +function* phaseCatalog( + rng: Rng, + startMs: number, + count: number, + batchSize: number, +): Iterable { + const buf: LogRow[] = []; + for (let i = 0; i < count; i++) { + const t = spread(i, count, startMs); + const sev = pickSeverityIn(rng, 'info'); + const tmpl = catalogLog({ rng, nowMs: t, level: sev.text.toLowerCase() }); + buf.push( + makeLog({ + timestampMs: t, + serviceName: 'product-catalog', + severityText: 'INFO', + body: tmpl.body, + resourceAttributes: buildResourceAttrs({ + rng, + serviceName: 'product-catalog', + }), + logAttributes: { ...tmpl.attrs, _severity_raw: sev.text }, + }), + ); + if (buf.length >= batchSize) { + yield { traces: [], logs: buf.splice(0, buf.length) }; + } + } + if (buf.length) yield { traces: [], logs: buf }; +} + +function* phaseBackgroundVaried( + rng: Rng, + startMs: number, + count: number, + batchSize: number, +): Iterable { + const buf: LogRow[] = []; + const services = [ + 'auth-service', + 'shipping-service', + 'tax-service', + 'fraud-detection', + 'recommendation-service', + 'kafka', + 'payment-service', + 'cart-service', + 'session-service', + 'webhook-relay', + 'feature-flags', + 'image-service', + ]; + for (let i = 0; i < count; i++) { + const t = spread(i, count, startMs); + const svc = services[i % services.length]; + // Mostly info, with realistic warn/error sprinkle + const sev = rng.weightedPick<'info' | 'warn' | 'error' | 'debug' | 'trace'>( + [ + { value: 'info', weight: 78 }, + { value: 'warn', weight: 14 }, + { value: 'error', weight: 5 }, + { value: 'debug', weight: 2 }, + { value: 'trace', weight: 1 }, + ], + ); + const sevPick = pickSeverityIn(rng, sev); + const body = backgroundLog({ rng, nowMs: t }); + buf.push( + makeLog({ + timestampMs: t, + serviceName: svc, + // The schema uses uppercase severityText; we encode the messy + // raw severity in LogAttributes._severity_raw so agents can see it + // (and have to handle case-folding to count by severity). + severityText: sevPick.text.toUpperCase().startsWith('WARN') + ? 'WARN' + : sevPick.text.toUpperCase().startsWith('ERR') || + sevPick.text === 'fatal' + ? 'ERROR' + : sevPick.text.toUpperCase().startsWith('DEB') + ? 'DEBUG' + : sevPick.text.toUpperCase() === 'TRACE' + ? 'TRACE' + : 'INFO', + body, + resourceAttributes: buildResourceAttrs({ rng, serviceName: svc }), + logAttributes: { + 'event.name': 'app.background', + 'log.iostream': 'stdout', + logtag: 'F', + _severity_raw: sevPick.text, + }, + }), + ); + if (buf.length >= batchSize) { + yield { traces: [], logs: buf.splice(0, buf.length) }; + } + } + if (buf.length) yield { traces: [], logs: buf }; +} + +// Keep these unused imports tidy for the linter (they're conditionally +// referenced via dynamic phase functions above). +void emitBatched; +void pickSeverity; diff --git a/packages/hdx-eval/src/scenarios/noisy-signals/ground-truth.json b/packages/hdx-eval/src/scenarios/noisy-signals/ground-truth.json new file mode 100644 index 0000000000..b64afbe1a1 --- /dev/null +++ b/packages/hdx-eval/src/scenarios/noisy-signals/ground-truth.json @@ -0,0 +1,276 @@ +{ + "scenario": "noisy-signals", + "agentPrompt": "We want to cut log ingest cost. What are the top noisy signals we should drop or throttle?", + "expected": { + "totalLogVolume": 16000000, + "designNote": "Every (ServiceName, SeverityText) cell that contains a noise pattern also contains a high-volume LOAD-BEARING pattern of comparable size. The agent has to inspect event.name / body templates to disambiguate noise from legitimate logs WITHIN each cell — grouping by service+severity alone is insufficient. A blanket drop on any of these cells would destroy load-bearing data.", + "compositeCells": [ + { + "cell": "notification-service × DEBUG", + "noise": { + "eventName": "cache.hit", + "approxCount": 1000000, + "remediation": "drop debug-level cache hit logs" + }, + "loadBearing": { + "eventName": "notification.delivery", + "approxCount": 1000000, + "whyKeep": "Push delivery audit trail — required for support and compliance." + } + }, + { + "cell": "billing-service × INFO", + "noise": { + "eventName": "subscription.metric.calculated", + "metricName": "subscription_mrr", + "approxCount": 1000000, + "remediation": "convert to a gauge metric (subscription_mrr); stop emitting as a log" + }, + "loadBearing": { + "eventName": "invoice.charged / invoice.refunded / subscription.activated / subscription.cancelled / payment.failed", + "approxCount": 1000000, + "whyKeep": "Financial / audit events — required for compliance and revenue recognition." + } + }, + { + "cell": "worker × ERROR", + "noise": { + "eventName": "job.caught_exception", + "errorKind": "NonFatalRetryableError", + "approxCount": 400000, + "remediation": "downgrade severity or throttle — caught/handled error" + }, + "loadBearing": { + "eventName": "job.failed.permanent", + "approxCount": 400000, + "whyKeep": "Real, unhandled failures — must be kept for incident response." + } + }, + { + "cell": "{inventory,pricing,shipping,recommendation}-service × DEBUG", + "noise": { + "eventName": "health.check", + "approxCount": 2000000, + "remediation": "drop entirely; replace with a metric" + }, + "loadBearing": { + "eventName": "inventory.stock_lookup / pricing.calculation / shipping.rate.quote / recommendation.score", + "approxCount": 2000000, + "whyKeep": "Operational debug used by SREs to diagnose hot-path slowness in each service." + } + }, + { + "cell": "frontend-proxy × INFO", + "noise": { + "eventName": "lb.health.probe", + "approxCount": 1000000, + "remediation": "filter out kube-probe / LB internal health probes at the proxy" + }, + "loadBearing": { + "eventName": "proxy.access", + "approxCount": 1000000, + "whyKeep": "Envoy access logs — request-level auditing and debugging." + } + }, + { + "cell": "frontend × INFO", + "noise": { + "eventName": "frontend.console.log_dump", + "approxCount": 800000, + "remediation": "stop flushing client-side console.log dumps to backend logs" + }, + "loadBearing": { + "eventName": "page.render", + "approxCount": 800000, + "whyKeep": "RUM data — frontend performance signal." + } + }, + { + "cell": "search-service × INFO", + "noise": { + "eventName": "search.cache.miss", + "approxCount": 600000, + "remediation": "drop verbose per-query cache-miss logs; replace with a counter" + }, + "loadBearing": { + "eventName": "index.shard.flush", + "approxCount": 600000, + "whyKeep": "Indexer health signal — needed to track pipeline lag." + } + } + ], + "loadBearingOnlyCells": [ + { + "cell": "analytics-service × INFO", + "eventName": "product.view / cart.add / checkout.* / search.* / etc.", + "approxCount": 600000, + "whyKeep": "Product analytics feeding business dashboards." + } + ] + }, + "rubric": { + "programmatic": [ + { + "id": "names_heartbeat_pattern", + "weight": 2, + "pattern": "(health\\.check|heartbeat)", + "flags": "i" + }, + { + "id": "names_cache_hit_pattern", + "weight": 2, + "pattern": "cache\\.?hit", + "flags": "i" + }, + { + "id": "names_subscription_metric_pattern", + "weight": 2, + "pattern": "(subscription_mrr|metric\\.name=subscription|metric.{0,15}calculated|mrr)", + "flags": "i" + }, + { + "id": "names_stacktrace_pattern", + "weight": 2, + "pattern": "(NonFatalRetryableError|job\\.caught_exception|caught.{0,8}exception)", + "flags": "i" + }, + { + "id": "names_lb_probe_pattern", + "weight": 2, + "pattern": "(lb\\.health\\.probe|kube[- ]?probe|/healthz|health.?probe|LB.{0,20}probe|internal.{0,10}health.?check)", + "flags": "i" + }, + { + "id": "names_console_dump_pattern", + "weight": 2, + "pattern": "(console\\.log_dump|console\\.log dump|console.{0,10}dump|client.?side.{0,15}console|frontend.{0,10}console)", + "flags": "i" + }, + { + "id": "names_search_miss_pattern", + "weight": 1, + "pattern": "(search\\.cache\\.miss|cache.?miss.{0,30}(search|verbose|per.?query)|verbose.{0,20}cache.?miss)", + "flags": "i" + }, + { + "id": "names_keep_notification_delivery", + "weight": 1, + "pattern": "(notification\\.delivery|push.{0,10}delivery|delivery.{0,15}(audit|record|trail))", + "flags": "i" + }, + { + "id": "names_keep_billing_events", + "weight": 1, + "pattern": "(invoice\\.(charged|refunded)|subscription\\.(activated|cancelled)|payment\\.failed|billing.{0,15}(events?|financial|audit))", + "flags": "i" + }, + { + "id": "names_keep_job_failed_permanent", + "weight": 1, + "pattern": "(job\\.failed\\.permanent|permanent.{0,15}(failure|fail)|unhandled.{0,20}(failure|error|job))", + "flags": "i" + }, + { + "id": "false_blanket_drop_notification_debug", + "weight": 3, + "negative": true, + "pattern": "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\bnotification-service[^.\\n]{0,40}?\\bdebug\\b(?![^.\\n]*?(cache\\.?hit|cache.?lookup))", + "flags": "i" + }, + { + "id": "false_blanket_drop_billing_info", + "weight": 3, + "negative": true, + "pattern": "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\bbilling-service[^.\\n]{0,40}?\\binfo\\b(?![^.\\n]*?(subscription|metric|mrr))", + "flags": "i" + }, + { + "id": "false_blanket_drop_worker_errors", + "weight": 3, + "negative": true, + "pattern": "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\bworker[^.\\n]{0,40}?\\berror\\b(?![^.\\n]*?(NonFatalRetryable|caught|retryable|handled))", + "flags": "i" + }, + { + "id": "false_drop_envoy_access", + "weight": 2, + "negative": true, + "pattern": "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(envoy.{0,20}?access|proxy\\.access|access.?log)\\b", + "flags": "i" + }, + { + "id": "false_drop_page_render", + "weight": 2, + "negative": true, + "pattern": "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(page\\.render|rum|frontend.{0,15}render)", + "flags": "i" + }, + { + "id": "false_drop_indexing", + "weight": 1, + "negative": true, + "pattern": "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(index\\.shard\\.flush|indexing.?progress|indexer.?health)", + "flags": "i" + }, + { + "id": "false_drop_analytics_events", + "weight": 2, + "negative": true, + "pattern": "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(analytics-service|product\\.view|cart\\.add|checkout\\.[a-z]+|event tracking|product analytics)", + "flags": "i" + }, + { + "id": "false_drop_notification_delivery", + "weight": 2, + "negative": true, + "pattern": "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(notification\\.delivery|push.{0,10}delivery|delivery.?(audit|record|trail))", + "flags": "i" + }, + { + "id": "false_drop_billing_events", + "weight": 2, + "negative": true, + "pattern": "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(invoice\\.(charged|refunded)|subscription\\.(activated|cancelled)|payment\\.failed|financial.?event)", + "flags": "i" + }, + { + "id": "false_drop_job_failed_permanent", + "weight": 2, + "negative": true, + "pattern": "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(job\\.failed\\.permanent|permanent.{0,15}(failure|fail))", + "flags": "i" + }, + { + "id": "false_drop_service_ops_debug", + "weight": 2, + "negative": true, + "pattern": "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(inventory\\.stock_lookup|pricing\\.calculation|shipping\\.rate\\.quote|recommendation\\.score)", + "flags": "i" + } + ], + "judge": { + "criteria": [ + { + "id": "correctness", + "weight": 3, + "description": "Did the answer correctly identify the SEVEN noise patterns and name them by event.name or body template? (1) health.check heartbeats across inventory/pricing/shipping/recommendation services × DEBUG, (2) notification-service cache.hit chatter, (3) billing-service subscription_mrr metric-as-log, (4) worker NonFatalRetryableError caught-exception, (5) frontend-proxy lb.health.probe internal probes, (6) frontend console.log_dump client-side dumps, (7) search-service search.cache.miss verbose. Missing the billing metric-as-log (most subtle — it's INFO not DEBUG) is a correctness failure. Missing pattern 5/6/7 (the new mixed-cell noises) is also a correctness failure." + }, + { + "id": "completeness", + "weight": 2, + "description": "Did it propose a concrete remediation for each noise pattern (drop, throttle, sample, downgrade severity, or convert to metric)? The billing-service one specifically should be flagged as 'convert to a gauge metric'." + }, + { + "id": "calibration", + "weight": 3, + "description": "CRITICAL — every (ServiceName, SeverityText) cell that contains a noise pattern ALSO contains a high-volume LOAD-BEARING pattern of comparable size. The agent must DISTINGUISH noise from legitimate within each cell. Specifically: notification-service DEBUG has cache.hit (drop) AND notification.delivery (KEEP, push delivery audit); billing-service INFO has subscription_mrr-as-log (drop/convert) AND invoice/refund/subscription financial events (KEEP); worker ERROR has NonFatalRetryableError (drop/downgrade) AND job.failed.permanent (KEEP, real failures); inventory/pricing/shipping/recommendation × DEBUG each have heartbeats (drop) AND legit per-service ops debug (KEEP); frontend-proxy INFO has lb.health.probe (drop/filter) AND envoy access (KEEP); frontend INFO has console.log_dump (drop) AND page.render (KEEP, RUM); search-service INFO has search.cache.miss (drop) AND index.shard.flush (KEEP). A blanket 'drop notification-service DEBUG' or 'drop worker ERROR' recommendation that does not name the specific event.name to drop is a calibration failure — it would destroy load-bearing data. The recommendation must be event.name-specific, not service-and-severity-specific." + }, + { + "id": "conciseness", + "weight": 1, + "description": "Was the answer concise, well-structured, and ranked clearly?" + } + ] + } + } +} diff --git a/packages/hdx-eval/src/scenarios/seedScenario.ts b/packages/hdx-eval/src/scenarios/seedScenario.ts new file mode 100644 index 0000000000..758dc3d551 --- /dev/null +++ b/packages/hdx-eval/src/scenarios/seedScenario.ts @@ -0,0 +1,48 @@ +import type { ClickHouseClient } from '@clickhouse/client'; + +import { insertLogRows, insertTraceRows } from '../clickhouse/insert'; +import { + ensureScenarioTables, + truncateScenarioTables, +} from '../clickhouse/schema'; +import { mulberry32 } from '../rng/seeded'; +import { getScenario } from './index'; + +export type SeedResult = { + tracesInserted: number; + logsInserted: number; + tables: { traces: string; logs: string }; +}; + +export async function seedScenario(args: { + client: ClickHouseClient; + scenarioName: string; + seed: number; + nowMs: number; + volumeFactor?: number; +}): Promise { + const scenario = getScenario(args.scenarioName); + const tables = await ensureScenarioTables(args.client, scenario.name); + await truncateScenarioTables(args.client, scenario.name); + const rng = mulberry32(args.seed); + + let tracesInserted = 0; + let logsInserted = 0; + for (const batch of scenario.generate({ + rng, + nowMs: args.nowMs, + volumeFactor: args.volumeFactor, + })) { + if (batch.traces.length > 0) { + tracesInserted += await insertTraceRows( + args.client, + tables.traces, + batch.traces, + ); + } + if (batch.logs.length > 0) { + logsInserted += await insertLogRows(args.client, tables.logs, batch.logs); + } + } + return { tracesInserted, logsInserted, tables }; +} diff --git a/packages/hdx-eval/src/scenarios/segmented-regression/generate.ts b/packages/hdx-eval/src/scenarios/segmented-regression/generate.ts new file mode 100644 index 0000000000..7e6adf86cf --- /dev/null +++ b/packages/hdx-eval/src/scenarios/segmented-regression/generate.ts @@ -0,0 +1,297 @@ +/** + * segmented-regression scenario. + * + * Story: api-server error rate has gone up in the last 10 minutes, but only + * for the intersection of `tenant.tier=enterprise` AND `cache.hit=false`. + * A newly deployed enterprise-only fallback handler has a schema-mismatch + * bug; it only fires when both conditions hold. Single-axis aggregates + * (group-by tier only, or group-by cache state only) dilute the signal + * because most enterprise traffic is cache-hit (normal) and most cache-miss + * traffic is non-enterprise (normal). + * + * Distractors: + * - Background 0.5% error rate everywhere (uniform noise). + * - Concurrent recommendation-service 502 burst (different service — + * real but unrelated incident, tempting because of the timing overlap). + * + * What a successful agent does: + * 1. Spots the elevated error rate on api-server in the last 10 min. + * 2. Notices that aggregating by tenant.tier alone OR by cache.hit alone + * shows only a modest bump. + * 3. Cross-tabs on (tenant.tier, cache.hit), sees enterprise × miss at + * ~12% vs ~0.5% baseline elsewhere. + * 4. Identifies the schema-mismatch / FallbackHandlerError body pattern. + * 5. Calls out that recommendation-service is on different traces and + * not the cause. + */ +import { makeLog } from '../../generators/logs'; +import { + buildResourcePool, + pickResource, + uuidv4, +} from '../../generators/templates'; +import { + makeSpan, + msToNs, + newSpanId, + newTraceId, +} from '../../generators/traces'; +import type { LogRow, TraceRow } from '../../generators/types'; +import type { GenerateContext, Scenario, ScenarioBatch } from '../types'; +import groundTruth from './ground-truth.json'; + +// ─── Volumes ─────────────────────────────────────────────────────────────── + +const TOTAL_API_SERVER_TRACES = 6_000_000; +const HISTORY_WINDOW_MS = 60 * 60 * 1000; +const ANOMALY_WINDOW_MS = 10 * 60 * 1000; + +// Concurrent distractor burst on a different service. +const REC_DECOY_COUNT = 220; +const REC_DECOY_WINDOW_MS = 12 * 60 * 1000; + +// ─── Distribution knobs ─────────────────────────────────────────────────── + +const TENANT_TIERS = [ + { value: 'free', weight: 60 }, + { value: 'pro', weight: 30 }, + { value: 'enterprise', weight: 10 }, +] as const; + +const CACHE_HIT_RATE = 0.7; // 70% cache hits, 30% misses + +const REGIONS = ['us-east-1', 'us-west-2', 'eu-west-1', 'ap-south-1'] as const; + +const ENDPOINTS = [ + 'GET /api/products/{id}', + 'GET /api/products/search', + 'POST /api/cart/items', + 'GET /api/cart/{id}', + 'GET /api/users/{id}', + 'GET /api/orders/{id}', + 'POST /api/orders', + 'POST /api/checkout', + 'GET /api/recommendations', + 'GET /api/health', +] as const; + +// Background error rate everywhere (0.5%) — pure noise, no signal. +const BASELINE_ERROR_RATE = 0.005; + +// Anomaly: enterprise × cache.hit=false in the LAST 10 min errors at 12%. +// All other (tier, hit) combinations stay at BASELINE_ERROR_RATE. +const ANOMALY_ERROR_RATE = 0.12; + +// Generic background error messages — heterogeneous so simple body grouping +// doesn't immediately reveal the anomaly pattern. +const BACKGROUND_ERROR_BODIES = [ + 'upstream connect timeout', + 'context deadline exceeded', + 'temporary unavailable: retry', + 'rate limited by upstream', + 'failed to read response body', + 'invalid request: missing header', +] as const; + +// ─── Generator ──────────────────────────────────────────────────────────── + +type Rng = GenerateContext['rng']; + +// Streaming generator: yield batches of `batchSize` rows as we go instead of +// buffering all 6M traces in memory. Without this, full-volume seed runs OOM +// the Node heap even with --max-old-space-size=8192. +function* streamBackgroundRows( + rng: Rng, + nowMs: number, + totalTraces: number, + batchSize: number, +): Generator { + const startMs = nowMs - HISTORY_WINDOW_MS; + const anomalyStartMs = nowMs - ANOMALY_WINDOW_MS; + let tracesBuf: TraceRow[] = []; + let logsBuf: LogRow[] = []; + + const resourcePool = buildResourcePool({ + rng, + services: ['api-server'], + instancesPerService: 24, + }); + + for (let i = 0; i < totalTraces; i++) { + const t = + totalTraces > 1 + ? startMs + (i / (totalTraces - 1)) * (HISTORY_WINDOW_MS - 30_000) + : startMs; + + const tenantTier = rng.weightedPick( + TENANT_TIERS.map(tt => ({ value: tt.value, weight: tt.weight })), + ); + const tenantId = `${tenantTier}-${rng.intRange(1, 5_000)}`; + const cacheHit = rng.next() < CACHE_HIT_RATE; + const region = rng.pick(REGIONS); + const endpoint = rng.pick(ENDPOINTS); + const inAnomalyWindow = t >= anomalyStartMs; + const isAffectedSegment = + inAnomalyWindow && tenantTier === 'enterprise' && !cacheHit; + const errorRate = isAffectedSegment + ? ANOMALY_ERROR_RATE + : BASELINE_ERROR_RATE; + const isError = rng.next() < errorRate; + + const traceId = newTraceId(rng); + const spanId = newSpanId(rng); + const resource = pickResource(rng, resourcePool, 'api-server'); + const requestId = uuidv4(rng); + const userId = `u${rng.intRange(1, 5_000_000)}`; + const route = endpoint.replace(/^[A-Z]+ /, ''); + const httpMethod = endpoint.startsWith('POST') ? 'POST' : 'GET'; + + const spanAttributes: Record = { + 'http.method': httpMethod, + 'http.route': route, + 'http.target': route, + 'http.status_code': isError ? '500' : '200', + 'tenant.id': tenantId, + 'tenant.tier': tenantTier, + 'user.id': userId, + 'cloud.region': region, + 'cache.hit': cacheHit ? 'true' : 'false', + 'request.id': requestId, + }; + + let statusCode: TraceRow['statusCode'] = 'STATUS_CODE_OK'; + let statusMessage = ''; + if (isError) { + statusCode = 'STATUS_CODE_ERROR'; + if (isAffectedSegment) { + statusMessage = + 'fallback handler returned 500: schema mismatch in enterprise_v2_response'; + spanAttributes['error.type'] = 'FallbackHandlerError'; + spanAttributes['fallback.path'] = 'enterprise_v2'; + } else { + statusMessage = rng.pick(BACKGROUND_ERROR_BODIES); + spanAttributes['error.type'] = 'BackgroundError'; + } + } + + tracesBuf.push( + makeSpan({ + rng, + timestampMs: t, + traceId, + spanId, + spanName: endpoint, + spanKind: 'SPAN_KIND_SERVER', + serviceName: 'api-server', + durationNs: msToNs(rng.range(20, 250)), + statusCode, + statusMessage, + resourceAttributes: resource, + spanAttributes, + }), + ); + + // Emit a correlated ERROR log for affected-segment failures so an agent + // searching logs can find the distinctive body too. + if (isError && isAffectedSegment) { + logsBuf.push( + makeLog({ + timestampMs: t + rng.intRange(5, 30), + serviceName: 'api-server', + severityText: 'ERROR', + body: `fallback handler returned 500: schema mismatch in enterprise_v2_response tenant=${tenantId} request_id=${requestId}`, + traceId, + spanId, + logAttributes: { + 'error.kind': 'FallbackHandlerError', + 'tenant.tier': 'enterprise', + 'cache.hit': 'false', + }, + }), + ); + } + + if (tracesBuf.length >= batchSize) { + yield { traces: tracesBuf, logs: logsBuf }; + tracesBuf = []; + logsBuf = []; + } + } + if (tracesBuf.length > 0 || logsBuf.length > 0) { + yield { traces: tracesBuf, logs: logsBuf }; + } +} + +function buildRecommendationDecoy( + rng: Rng, + nowMs: number, +): { traces: TraceRow[]; logs: LogRow[] } { + const startMs = nowMs - REC_DECOY_WINDOW_MS; + const traces: TraceRow[] = []; + const logs: LogRow[] = []; + for (let i = 0; i < REC_DECOY_COUNT; i++) { + const t = + startMs + + (i / Math.max(REC_DECOY_COUNT - 1, 1)) * (REC_DECOY_WINDOW_MS - 5_000); + const traceId = newTraceId(rng); + const spanId = newSpanId(rng); + traces.push( + makeSpan({ + rng, + timestampMs: t, + traceId, + spanId, + spanName: 'rec.compute', + spanKind: 'SPAN_KIND_SERVER', + serviceName: 'recommendation-service', + durationNs: msToNs(rng.range(400, 1200)), + statusCode: 'STATUS_CODE_ERROR', + statusMessage: 'rec.compute timed out: upstream cf-feed unreachable', + spanAttributes: { + 'http.status_code': '502', + 'error.type': 'UpstreamTimeout', + 'upstream.host': 'cf-feed.internal', + }, + }), + ); + logs.push( + makeLog({ + timestampMs: t + rng.intRange(5, 30), + serviceName: 'recommendation-service', + severityText: 'ERROR', + body: 'rec.compute timed out: upstream cf-feed.internal unreachable after 1500ms', + traceId, + spanId, + logAttributes: { + 'error.kind': 'UpstreamTimeout', + component: 'recommendation-service', + }, + }), + ); + } + return { traces, logs }; +} + +export const segmentedRegressionScenario: Scenario = { + name: 'segmented-regression', + agentPrompt: groundTruth.agentPrompt, + description: + '6M api-server traces over an hour. Last 10 min: enterprise × cache-miss errors at ~12% (schema mismatch in a newly deployed fallback). All other (tier, cache) combinations stay at ~0.5% baseline. Single-axis aggregates dilute the signal — the agent must cross-tab. Concurrent recommendation-service 502 burst as distractor.', + *generate(ctx): Iterable { + const factor = ctx.volumeFactor ?? 1; + const batchSize = ctx.batchSize ?? 10_000; + const totalTraces = Math.max( + 50, + Math.round(TOTAL_API_SERVER_TRACES * factor), + ); + + // Stream the background api-server traces in batches so we don't buffer + // 6M rows in memory. + yield* streamBackgroundRows(ctx.rng, ctx.nowMs, totalTraces, batchSize); + // Decoy is small (220 rows) — fine to build in memory. + const decoy = buildRecommendationDecoy(ctx.rng, ctx.nowMs); + if (decoy.traces.length > 0) yield { traces: decoy.traces, logs: [] }; + if (decoy.logs.length > 0) yield { traces: [], logs: decoy.logs }; + }, + groundTruth, +}; diff --git a/packages/hdx-eval/src/scenarios/segmented-regression/ground-truth.json b/packages/hdx-eval/src/scenarios/segmented-regression/ground-truth.json new file mode 100644 index 0000000000..d04873a857 --- /dev/null +++ b/packages/hdx-eval/src/scenarios/segmented-regression/ground-truth.json @@ -0,0 +1,138 @@ +{ + "scenario": "segmented-regression", + "agentPrompt": "API error rate has gone up in the last 10 minutes for some users. Find the segment(s) where the regression is concentrated and what's going wrong.", + "expected": { + "affectedService": "api-server", + "segmentAttributeA": "tenant.tier", + "segmentValueA": "enterprise", + "segmentAttributeB": "cache.hit", + "segmentValueB": "false", + "anomalyWindowMinutesAgo": 10, + "errorRateInSegment": 0.12, + "errorRateBaseline": 0.005, + "errorBodyPattern": "schema mismatch in enterprise_v2_response", + "errorType": "FallbackHandlerError", + "rootCauseHint": "newly deployed enterprise-only cache-miss fallback path has a schema mismatch — only fires when tier=enterprise AND cache.hit=false" + }, + "distractors": [ + { + "kind": "background uniform 5xx", + "approxRate": 0.005, + "whyNotRootCause": "Uniform 0.5% error rate across all segments and time — normal background noise, NOT the regression." + }, + { + "kind": "concurrent recommendation-service incident", + "service": "recommendation-service", + "windowMinutesAgo": [12, 0], + "approxCount": 220, + "errorMessage": "rec.compute timed out: upstream cf-feed unreachable", + "whyNotRootCause": "Concurrent unrelated 502 burst on a different service. Tempting because it overlaps in time, but the prompt is about api-server (the customer-facing tier) and the affected traces have no recommendation-service span." + }, + { + "kind": "non-anomaly enterprise errors", + "approxRate": 0.005, + "tierFilter": "enterprise", + "whyNotRootCause": "Enterprise tier has the same 0.5% baseline error rate as other tiers OUTSIDE the cache-miss path. Blaming enterprise on its own (without the cache axis) is incomplete — enterprise + cache.hit=true errors at baseline." + }, + { + "kind": "non-anomaly cache-miss errors", + "approxRate": 0.005, + "cacheFilter": "false", + "whyNotRootCause": "Cache misses on non-enterprise tiers error at the same 0.5% baseline. Blaming cache misses on their own (without the tier axis) is incomplete — pro/free + cache.hit=false errors at baseline." + } + ], + "anomalyAttributes": { + "spanAttributes": { + "tenant.tier": "enterprise", + "cache.hit": "false", + "http.status_code": "500", + "error.type": "FallbackHandlerError" + } + }, + "rubric": { + "programmatic": [ + { + "id": "names_affected_service", + "weight": 1, + "pattern": "api-server", + "flags": "i" + }, + { + "id": "names_tenant_segment", + "weight": 2, + "pattern": "(tenant\\.?tier|enterprise)", + "flags": "i" + }, + { + "id": "names_cache_segment", + "weight": 2, + "pattern": "(cache\\.?(hit|miss)|cache hit:?\\s*false|cache miss|miss(es)?)", + "flags": "i" + }, + { + "id": "names_both_axes_combined", + "weight": 3, + "pattern": "(enterprise[^.\\n]{0,80}?(cache|miss)|cache[^.\\n]{0,40}?miss[^.\\n]{0,80}?enterprise|tier\\s*=\\s*['\"]?enterprise['\"]?[^.\\n]{0,80}?cache|enterprise[^.\\n]{0,40}?AND[^.\\n]{0,40}?(cache|hit\\s*=\\s*false|miss))", + "flags": "i" + }, + { + "id": "names_error_pattern", + "weight": 1, + "pattern": "(schema mismatch|enterprise_v2|FallbackHandlerError|fallback (handler|path))", + "flags": "i" + }, + { + "id": "false_blame_global", + "weight": 2, + "negative": true, + "pattern": "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\b(global|api-server (alone|generally)|all (tiers|users|traffic)|service[- ]wide)", + "flags": "i" + }, + { + "id": "false_blame_tier_only", + "weight": 2, + "negative": true, + "pattern": "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\benterprise[^.\\n]{0,40}?(tier|customers|tenants?)(?![^.\\n]{0,200}?(cache|miss|hit\\s*=\\s*false))", + "flags": "i" + }, + { + "id": "false_blame_cache_only", + "weight": 2, + "negative": true, + "pattern": "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\bcache[ \\-]?(miss(es)?|lookup|hit\\s*=\\s*false)(?![^.\\n]{0,200}?(enterprise|tier))", + "flags": "i" + }, + { + "id": "false_blame_recommendation", + "weight": 2, + "negative": true, + "pattern": "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\brecommendation-service", + "flags": "i" + } + ], + "judge": { + "criteria": [ + { + "id": "correctness", + "weight": 3, + "description": "Did the answer correctly identify the affected segment as the INTERSECTION of `tenant.tier=enterprise` AND `cache.hit=false` on `api-server`? Identifying only ONE axis (just the tier OR just the cache state) is a correctness failure — the bug only fires on the intersection, and quoting an aggregate over a single axis (which dilutes the signal) misses the actual problem." + }, + { + "id": "completeness", + "weight": 2, + "description": "Did the answer (a) note the rough magnitude (e.g. ~12% error rate in the affected segment vs ~0.5% baseline), (b) reference the specific error signature (schema mismatch / enterprise_v2 / FallbackHandlerError or the fallback handler path), and (c) explain why neither single-axis aggregation reveals the issue?" + }, + { + "id": "calibration", + "weight": 2, + "description": "Did the answer rule out the distractors? Specifically: (a) the concurrent recommendation-service incident is on a different service and unrelated. (b) Enterprise-tier alone is NOT the cause — enterprise + cache.hit=true errors at the baseline rate. (c) Cache misses alone are NOT the cause — pro/free + cache.hit=false error at baseline. (d) The global api-server error rate is dominated by baseline noise; the regression is invisible without segmenting. Mistaking any of these for the root cause is a calibration failure." + }, + { + "id": "conciseness", + "weight": 1, + "description": "Was the answer concise, well-structured, and free of filler or unnecessary speculation?" + } + ] + } + } +} diff --git a/packages/hdx-eval/src/scenarios/service-health-check/generate.ts b/packages/hdx-eval/src/scenarios/service-health-check/generate.ts new file mode 100644 index 0000000000..f925ec37bc --- /dev/null +++ b/packages/hdx-eval/src/scenarios/service-health-check/generate.ts @@ -0,0 +1,428 @@ +/** + * service-health-check scenario. + * + * Peace-time SLI report on `api-server`. The dataset is healthy — no + * incident — but the report window contains four genuinely-novel-but- + * non-critical signals the agent should call out, plus a couple of + * recurring/baseline patterns it should NOT escalate. + * + * Dataset spans 4 hours ending at `nowMs`: + * - Baseline window: nowMs-240min .. nowMs-60min (steady, no planted signals) + * - Report window: nowMs-60min .. nowMs (what the prompt covers) + * + * Planted novel signals (in the report window): + * 1. New log template "feature_flag.new_recommendation_engine: shadow eval + * succeeded …" at ~0.1% of total log volume, starting 70 min before + * nowMs. Absent from the baseline. + * 2. `GET /api/v2/products` endpoint ramps from 0% → ~4% of products + * traffic across the report window. Absent from the baseline. + * 3. p99 latency drifts ~340ms → ~380ms over the report window. Still + * well under the stated 500ms SLO. Steady ~340ms in the baseline. + * 4. 60-second error blip at reportStart+3min: ~5% error rate (~250 + * errors), all "upstream connect timeout". Outside that minute the + * error rate is the baseline 0.5%. + * + * Distractors (in BOTH baseline and report window — should NOT be flagged): + * 5. POST /api/internal/batch-sync 30-second burst every 15 minutes. + * Recurring; an agent comparing to baseline can see it's routine. + * 6. Constant 0.5% background error rate uniformly across endpoints + * and time. NOT a regression. + * + * Hardness: + * - Subtle signals (0.1% / 30ms / 4%) all below typical chart thresholds. + * - Log Body has per-row IDs/UUIDs/timestamps so `GROUP BY Body` returns + * millions of distinct rows — only hyperdx_log_patterns (Drain) gives + * a coherent view. + * - Strong calibration penalty: false-alarming on the baseline noise or + * escalating the brief blip costs as much as missing a real finding. + * - Baseline comparison required: distractor #5 is visible in the report + * window only as a "spike"; an agent that queries only the report + * window may flag it as novel. + */ +import { makeLog } from '../../generators/logs'; +import { + buildResourcePool, + pickResource, + uuidv4, +} from '../../generators/templates'; +import { + makeSpan, + msToNs, + newSpanId, + newTraceId, +} from '../../generators/traces'; +import type { LogRow, TraceRow } from '../../generators/types'; +import type { GenerateContext, Scenario, ScenarioBatch } from '../types'; +import groundTruth from './ground-truth.json'; + +// ─── Volumes ───────────────────────────────────────────────────────────────── + +const TOTAL_TRACES = 12_000_000; // 50K/min × 240 min +const TOTAL_LOGS = 24_000_000; // 100K/min × 240 min + +const BASELINE_WINDOW_MS = 4 * 60 * 60 * 1000; // 4 hours total +const REPORT_WINDOW_MS = 60 * 60 * 1000; // last hour + +// ─── Distractor: recurring batch-sync spike ───────────────────────────────── +const BATCH_SPIKE_PERIOD_MS = 15 * 60 * 1000; // every 15 min +const BATCH_SPIKE_DURATION_MS = 30 * 1000; // 30s burst +// During a spike window, this fraction of traffic is batch-sync. Tuned so +// the spike adds ~5K extra batch-sync calls per cycle on top of baseline. +const BATCH_SPIKE_SHARE = 0.4; + +// ─── Planted #1: new log template ────────────────────────────────────────── +// Appears starting 70 min before nowMs (10 min before the report window). +const NEW_TEMPLATE_LEAD_MS = 70 * 60 * 1000; +const NEW_TEMPLATE_VOLUME_SHARE = 0.001; // 0.1% of TOTAL log volume + +// ─── Planted #2: v2 endpoint ramp ───────────────────────────────────────── +// Linear ramp from 0% at reportStart to 4% of products traffic by nowMs. +const V2_PRODUCTS_MAX_SHARE = 0.04; + +// ─── Planted #3: latency drift in the report window ─────────────────────── +// Scale baseline latency by 1.0 at reportStart, 1.12 at nowMs. +// Baseline p99 ~340ms → end-of-window p99 ~380ms. +const LATENCY_DRIFT_MAX = 0.12; + +// ─── Planted #4: brief 60-second error blip ─────────────────────────────── +const BLIP_OFFSET_FROM_REPORT_START_MS = 3 * 60 * 1000; // T+3min +const BLIP_DURATION_MS = 60 * 1000; // 60s +const BLIP_ERROR_RATE = 0.05; // 5% (vs 0.5% baseline) + +// ─── Baseline distribution ────────────────────────────────────────────────── + +const BASELINE_ERROR_RATE = 0.005; // 0.5% pure background noise + +// Endpoint mix (products is intentionally large so v2 has room to ramp +// visibly within "4% of products traffic"). +const ENDPOINTS = [ + { spanName: 'GET /api/v1/products', method: 'GET', weight: 35 }, + { spanName: 'GET /api/cart/{id}', method: 'GET', weight: 25 }, + { spanName: 'POST /api/checkout', method: 'POST', weight: 20 }, + { spanName: 'GET /api/users/{id}', method: 'GET', weight: 20 }, +] as const; + +const REGIONS = ['us-east-1', 'us-west-2', 'eu-west-1', 'ap-south-1'] as const; +const TABLES = [ + 'orders', + 'users', + 'products', + 'sessions', + 'inventory', +] as const; +const UPSTREAMS = [ + 'cf-feed.internal', + 'recsys.internal', + 'auth.internal', + 'inventory.internal', +] as const; + +// ─── Latency ──────────────────────────────────────────────────────────────── + +type Rng = GenerateContext['rng']; + +/** + * Piecewise distribution chosen so baseline (scale=1) has p50 ~80ms and + * p99 ~340ms. `scale` multiplies everything — used to drift latency + * during the report window. + */ +function sampleLatencyMs(rng: Rng, scale: number): number { + const r = rng.next(); + let base: number; + if (r < 0.5) + base = 30 + rng.next() * 70; // 30..100 (p50 ~65) + else if (r < 0.9) + base = 60 + rng.next() * 140; // 60..200 (p90 ~190) + else if (r < 0.99) + base = 180 + rng.next() * 120; // 180..300 (p99 entry) + else base = 280 + rng.next() * 80; // 280..360 (p99 tail) + return base * scale; +} + +function latencyScaleAt( + t: number, + reportStartMs: number, + nowMs: number, +): number { + if (t < reportStartMs) return 1; + const frac = Math.min(1, (t - reportStartMs) / (nowMs - reportStartMs)); + return 1 + frac * LATENCY_DRIFT_MAX; +} + +// ─── Log body templates ───────────────────────────────────────────────────── +// Every body embeds per-row dynamic values so naive `GROUP BY Body` +// returns ~1 row per log entry. Drain clusters them into the templates +// below. + +function buildAccessBody(rng: Rng): string { + const reqId = uuidv4(rng); + const method = rng.pick(['GET', 'GET', 'GET', 'POST'] as const); + const path = `/api/v1/${rng.pick(['products', 'cart', 'users', 'checkout'] as const)}/${rng.intRange(1, 100_000)}`; + const status = rng.weightedPick([ + { value: 200, weight: 90 }, + { value: 404, weight: 5 }, + { value: 500, weight: 3 }, + { value: 401, weight: 2 }, + ]); + const latency = rng.intRange(20, 400); + return `req_id=${reqId} ${method} ${path} -> ${status} in ${latency}ms`; +} + +function buildCacheBody(rng: Rng): string { + const key = `${rng.pick(['user', 'product', 'session'] as const)}:${rng.intRange(1, 1_000_000)}`; + const hit = rng.next() < 0.85 ? 'true' : 'false'; + const region = rng.pick(REGIONS); + return `cache.lookup key=${key} hit=${hit} region=${region}`; +} + +function buildAuthBody(rng: Rng): string { + const userId = `u${rng.intRange(1, 5_000_000)}`; + const sessionId = uuidv4(rng); + const ttl = rng.intRange(60, 3600); + return `auth.verified user=${userId} session=${sessionId} ttl=${ttl}s`; +} + +function buildRetryBody(rng: Rng): string { + const upstream = rng.pick(UPSTREAMS); + const retry = rng.intRange(1, 4); + const latency = rng.intRange(100, 1500); + const tenant = `t-${rng.intRange(1, 50_000)}`; + return `upstream=${upstream} retry=${retry} latency_ms=${latency} tenant=${tenant}`; +} + +function buildSlowQueryBody(rng: Rng): string { + const duration = rng.intRange(200, 1500); + const table = rng.pick(TABLES); + const rows = rng.intRange(1000, 1_000_000); + return `slow.query duration_ms=${duration} table=${table} rows_scanned=${rows}`; +} + +function buildNewTemplateBody(rng: Rng): string { + const corrId = uuidv4(rng); + const latency = rng.intRange(15, 90); + return `feature_flag.new_recommendation_engine: shadow eval succeeded correlation_id=${corrId} latency_ms=${latency}`; +} + +// Weighted pick for the baseline log template. Add up to 100. +type LogTemplateKind = 'access' | 'cache' | 'auth' | 'retry' | 'slow'; +const LOG_TEMPLATES: Array<{ + value: { kind: LogTemplateKind; severity: 'INFO' | 'WARN' }; + weight: number; +}> = [ + { value: { kind: 'access', severity: 'INFO' }, weight: 60 }, + { value: { kind: 'cache', severity: 'INFO' }, weight: 20 }, + { value: { kind: 'auth', severity: 'INFO' }, weight: 10 }, + { value: { kind: 'retry', severity: 'INFO' }, weight: 5 }, + { value: { kind: 'slow', severity: 'WARN' }, weight: 5 }, +]; + +function buildBaselineLogBody(rng: Rng): { + body: string; + severityText: 'INFO' | 'WARN'; +} { + const pick = rng.weightedPick(LOG_TEMPLATES); + switch (pick.kind) { + case 'access': + return { body: buildAccessBody(rng), severityText: pick.severity }; + case 'cache': + return { body: buildCacheBody(rng), severityText: pick.severity }; + case 'auth': + return { body: buildAuthBody(rng), severityText: pick.severity }; + case 'retry': + return { body: buildRetryBody(rng), severityText: pick.severity }; + case 'slow': + return { body: buildSlowQueryBody(rng), severityText: pick.severity }; + } +} + +// ─── Streaming generators ────────────────────────────────────────────────── + +function* streamTraces( + rng: Rng, + nowMs: number, + totalTraces: number, + batchSize: number, +): Generator { + const baselineStartMs = nowMs - BASELINE_WINDOW_MS; + const reportStartMs = nowMs - REPORT_WINDOW_MS; + const blipStartMs = reportStartMs + BLIP_OFFSET_FROM_REPORT_START_MS; + const blipEndMs = blipStartMs + BLIP_DURATION_MS; + + const resourcePool = buildResourcePool({ + rng, + services: ['api-server'], + instancesPerService: 24, + }); + + let buf: TraceRow[] = []; + for (let i = 0; i < totalTraces; i++) { + const t = + totalTraces > 1 + ? baselineStartMs + + (i / (totalTraces - 1)) * (BASELINE_WINDOW_MS - 30_000) + : baselineStartMs; + const inReportWindow = t >= reportStartMs; + const cycleOffset = (t - baselineStartMs) % BATCH_SPIKE_PERIOD_MS; + const inBatchSpike = cycleOffset < BATCH_SPIKE_DURATION_MS; + const inBlip = t >= blipStartMs && t < blipEndMs; + + // Endpoint selection + let spanName: string; + let httpMethod: string; + if (inBatchSpike && rng.next() < BATCH_SPIKE_SHARE) { + spanName = 'POST /api/internal/batch-sync'; + httpMethod = 'POST'; + } else { + const endpoint = rng.weightedPick( + ENDPOINTS.map(e => ({ value: e, weight: e.weight })), + ); + spanName = endpoint.spanName; + httpMethod = endpoint.method; + // Planted #2: v2 endpoint ramp. Only inside the report window, and + // only for the products endpoint. + if ( + inReportWindow && + spanName === 'GET /api/v1/products' && + rng.next() < + ((t - reportStartMs) / (nowMs - reportStartMs)) * + V2_PRODUCTS_MAX_SHARE + ) { + spanName = 'GET /api/v2/products'; + } + } + + const errorRate = inBlip ? BLIP_ERROR_RATE : BASELINE_ERROR_RATE; + const isError = rng.next() < errorRate; + + const traceId = newTraceId(rng); + const spanId = newSpanId(rng); + const resource = pickResource(rng, resourcePool, 'api-server'); + const requestId = uuidv4(rng); + const userId = `u${rng.intRange(1, 5_000_000)}`; + const region = rng.pick(REGIONS); + + let statusCode: TraceRow['statusCode'] = 'STATUS_CODE_OK'; + let statusMessage = ''; + if (isError) { + statusCode = 'STATUS_CODE_ERROR'; + statusMessage = inBlip + ? 'upstream connect timeout' + : rng.pick([ + 'connection reset by peer', + 'context deadline exceeded', + 'temporary unavailable: retry', + 'rate limited by upstream', + ] as const); + } + + const durationMs = sampleLatencyMs( + rng, + latencyScaleAt(t, reportStartMs, nowMs), + ); + + const spanAttributes: Record = { + 'http.method': httpMethod, + 'http.route': spanName.replace(/^[A-Z]+ /, ''), + 'http.status_code': isError ? '500' : '200', + 'user.id': userId, + 'cloud.region': region, + 'request.id': requestId, + }; + if (inBlip) spanAttributes['error.type'] = 'UpstreamTimeout'; + + buf.push( + makeSpan({ + rng, + timestampMs: t, + traceId, + spanId, + spanName, + spanKind: 'SPAN_KIND_SERVER', + serviceName: 'api-server', + durationNs: msToNs(durationMs), + statusCode, + statusMessage, + resourceAttributes: resource, + spanAttributes, + }), + ); + + if (buf.length >= batchSize) { + yield { traces: buf, logs: [] }; + buf = []; + } + } + if (buf.length > 0) yield { traces: buf, logs: [] }; +} + +function* streamLogs( + rng: Rng, + nowMs: number, + totalLogs: number, + batchSize: number, +): Generator { + const baselineStartMs = nowMs - BASELINE_WINDOW_MS; + const newTemplateStartMs = nowMs - NEW_TEMPLATE_LEAD_MS; + + // Within the planted window, what fraction of rows are the planted + // template? Calibrated so the planted rows are 0.1% of TOTAL log volume. + const plantedWindowMs = NEW_TEMPLATE_LEAD_MS; + const plantedWindowShareOfTotal = plantedWindowMs / BASELINE_WINDOW_MS; + const plantedShareWithinWindow = + plantedWindowShareOfTotal > 0 + ? NEW_TEMPLATE_VOLUME_SHARE / plantedWindowShareOfTotal + : 0; + + let buf: LogRow[] = []; + for (let i = 0; i < totalLogs; i++) { + const t = + totalLogs > 1 + ? baselineStartMs + + (i / (totalLogs - 1)) * (BASELINE_WINDOW_MS - 30_000) + : baselineStartMs; + + let body: string; + let severityText: 'INFO' | 'WARN'; + if (t >= newTemplateStartMs && rng.next() < plantedShareWithinWindow) { + body = buildNewTemplateBody(rng); + severityText = 'INFO'; + } else { + const baseline = buildBaselineLogBody(rng); + body = baseline.body; + severityText = baseline.severityText; + } + + buf.push( + makeLog({ + timestampMs: t, + serviceName: 'api-server', + severityText, + body, + logAttributes: {}, + }), + ); + + if (buf.length >= batchSize) { + yield { traces: [], logs: buf }; + buf = []; + } + } + if (buf.length > 0) yield { traces: [], logs: buf }; +} + +export const serviceHealthCheckScenario: Scenario = { + name: 'service-health-check', + agentPrompt: groundTruth.agentPrompt, + description: + '4-hour dataset on api-server (12M traces + 24M logs). 3-hour steady baseline + 1-hour report window. Healthy service. Four subtle novel signals (new log template, v2 endpoint rollout, p99 latency drift within SLO, brief 60s error blip) plus two distractors (recurring batch-sync spikes, baseline 0.5% error noise). Tests calibrated SLI reporting without false alarms.', + *generate(ctx): Iterable { + const factor = ctx.volumeFactor ?? 1; + const batchSize = ctx.batchSize ?? 10_000; + const totalTraces = Math.max(200, Math.round(TOTAL_TRACES * factor)); + const totalLogs = Math.max(400, Math.round(TOTAL_LOGS * factor)); + + yield* streamTraces(ctx.rng, ctx.nowMs, totalTraces, batchSize); + yield* streamLogs(ctx.rng, ctx.nowMs, totalLogs, batchSize); + }, + groundTruth, +}; diff --git a/packages/hdx-eval/src/scenarios/service-health-check/ground-truth.json b/packages/hdx-eval/src/scenarios/service-health-check/ground-truth.json new file mode 100644 index 0000000000..ad7b25fd4d --- /dev/null +++ b/packages/hdx-eval/src/scenarios/service-health-check/ground-truth.json @@ -0,0 +1,183 @@ +{ + "scenario": "service-health-check", + "agentPrompt": "Routine status check on `api-server` over the last hour. Summarize key SLIs (traffic, error rate, latency) and call out anything novel or worth a closer look. Don't escalate routine variance. Be concise.", + "expected": { + "service": "api-server", + "reportWindowMinutes": 60, + "baselineWindowMinutes": 180, + "sliBaselines": { + "trafficPerMinute": 50000, + "errorRate": 0.005, + "latencyP50Ms": 80, + "latencyP99StartMs": 340, + "latencyP99EndMs": 380, + "latencySloMs": 500 + }, + "noveltiesToReport": [ + { + "kind": "new log template", + "approxVolumeShare": 0.001, + "bodyPattern": "feature_flag.new_recommendation_engine: shadow eval succeeded …", + "framing": "new pattern observed, looks like a feature-flag shadow-eval rollout — informational" + }, + { + "kind": "traffic shift", + "endpoint": "GET /api/v2/products", + "approxEndOfWindowShare": 0.04, + "framing": "v2 rollout in progress, ramps to ~4% of products traffic" + }, + { + "kind": "latency drift within SLO", + "fromP99Ms": 340, + "toP99Ms": 380, + "sloMs": 500, + "framing": "p99 trending up modestly across the hour, still well under SLO" + }, + { + "kind": "brief error blip", + "offsetIntoReportWindowMinutes": 3, + "durationSeconds": 60, + "approxErrorCount": 250, + "errorMessage": "upstream connect timeout", + "framing": "brief 1-minute spike of upstream timeouts, recovered immediately — transient, not a pattern" + } + ], + "distractorsToNotEscalate": [ + { + "kind": "recurring batch-sync spike", + "endpoint": "POST /api/internal/batch-sync", + "periodMinutes": 15, + "durationSeconds": 30, + "whyNotEscalate": "Visible throughout the 3-hour baseline; recurring batch job, not novel." + }, + { + "kind": "baseline 0.5% error rate", + "whyNotEscalate": "Constant across the full 4-hour dataset, uniformly distributed. Background noise, not a regression." + } + ] + }, + "rubric": { + "programmatic": [ + { + "id": "names_service", + "weight": 1, + "pattern": "api-server", + "flags": "i" + }, + { + "id": "reports_traffic_rate", + "weight": 1, + "pattern": "(requests?/min|requests/s|req/min|traffic|throughput|call volume|qps|rps)", + "flags": "i" + }, + { + "id": "reports_error_rate", + "weight": 1, + "pattern": "(error rate|errors?/min|0\\.5\\s?%?|~?0\\.5|baseline.{0,15}error)", + "flags": "i" + }, + { + "id": "reports_latency_metrics", + "weight": 1, + "pattern": "(p50|p95|p99|latency)[\\s\\S]{0,200}?\\d+(\\.\\d+)?\\s?(ms|millisecond)", + "flags": "i" + }, + { + "id": "notes_new_log_pattern", + "weight": 2, + "pattern": "(shadow.?eval|new[_\\s]+recommendation|recommendation.engine|feature.?flag)", + "flags": "i" + }, + { + "id": "notes_v2_endpoint", + "weight": 2, + "pattern": "(/api/v2/products|/v2/|v2\\s+(endpoint|rollout|rolling))", + "flags": "i" + }, + { + "id": "notes_latency_drift", + "weight": 1, + "pattern": "(p50|p95|p99|latency)[\\s\\S]{0,60}?\\b(trend|drift|trending|creep|creeping|rising|rising|climb|increasing|monotonic)", + "flags": "i" + }, + { + "id": "notes_brief_blip", + "weight": 1, + "pattern": "(brief|transient|1.?(min(ute)?|second)|60.?second|short.lived).{0,80}(spike|blip|errors?|timeouts?)", + "flags": "i" + }, + { + "id": "frames_as_healthy", + "weight": 1, + "pattern": "(healthy|stable|within SLO|no concerns|operating normally|nominal|steady)", + "flags": "i" + }, + { + "id": "false_alarm_incident", + "weight": 2, + "negative": true, + "pattern": "\\b(incident|outage|service.degradation|degraded|critical|severe|emergency)\\b", + "flags": "i" + }, + { + "id": "false_alarm_paging", + "weight": 2, + "negative": true, + "pattern": "\\b(page (the|on-call)|on-call|escalat(e|ion)|wake.{0,5}up|urgent(ly)?)\\b", + "flags": "i" + }, + { + "id": "false_alarm_blip_as_pattern", + "weight": 2, + "negative": true, + "pattern": "\\b(sustained|ongoing|persistent|recurring|continuing)\\b[^.\\n]{0,60}(error|timeout)", + "flags": "i" + }, + { + "id": "false_alarm_baseline_errors", + "weight": 2, + "negative": true, + "pattern": "error rate[^.\\n]{0,60}\\b(elevated|high|concerning|problem(atic)?|investigate)\\b", + "flags": "i" + }, + { + "id": "false_alarm_slo_breach", + "weight": 2, + "negative": true, + "pattern": "(SLO[^.\\n]{0,20}(breach|violation|exceed)|latency[^.\\n]{0,30}(critical|breach|violation))", + "flags": "i" + }, + { + "id": "false_flag_batch_job", + "weight": 1, + "negative": true, + "pattern": "batch.?sync[^.\\n]{0,80}\\b(novel|unusual|new|anomal(y|ous)|concern|investigate)\\b", + "flags": "i" + } + ], + "judge": { + "criteria": [ + { + "id": "correctness", + "weight": 3, + "description": "Did the answer identify ALL FOUR planted signals? (a) new log template (feature-flag shadow eval rollout), (b) GET /api/v2/products traffic rollout, (c) p99 latency drift within SLO (~340 → ~380ms), (d) brief 60-second upstream-timeout error blip. Each missing item is a correctness hit. False positives (claiming a signal that isn't there — e.g. saying the batch-sync recurrence is new, or that the 0.5% baseline error rate is elevated) are also correctness hits." + }, + { + "id": "completeness", + "weight": 2, + "description": "Does the answer cover all four SLI dimensions of a routine status check: traffic rate, error rate, latency (p50/p99), and a section on novel observations? Missing a dimension is a completeness hit." + }, + { + "id": "calibration", + "weight": 3, + "description": "Are magnitudes and framing appropriate for a healthy service? Specifically: the p99 latency drift must be described as a small trend within SLO (NOT as a breach, problem, or risk); the brief 60-second blip must be described as transient (NOT as a pattern or ongoing); the 0.5% baseline error rate must be described as baseline / background / nominal (NOT as elevated or concerning); the recurring batch-sync spikes must be acknowledged as recurring/expected (NOT as novel). BONUS for explicitly comparing the report window against the 3-hour baseline to verify novelty vs recurrence." + }, + { + "id": "conciseness", + "weight": 1, + "description": "Is the answer a structured status report rather than an exhaustive dump? It should fit on a screen, lead with the headline (service is healthy), then SLI numbers, then 'noted observations' for the four novelties — without long stack-by-stack drill-downs." + } + ] + } + } +} diff --git a/packages/hdx-eval/src/scenarios/types.ts b/packages/hdx-eval/src/scenarios/types.ts new file mode 100644 index 0000000000..1cdd635017 --- /dev/null +++ b/packages/hdx-eval/src/scenarios/types.ts @@ -0,0 +1,49 @@ +import type { LogRow, TraceRow } from '../generators/types'; +import type { SeededRng } from '../rng/seeded'; + +export type GenerateContext = { + rng: SeededRng; + nowMs: number; + /** + * Test-only override. Scenarios should multiply background counts by this + * factor. Planted anomaly counts (the diagnostic signal) stay fixed so + * structural invariants still hold at low volume. Default 1. + */ + volumeFactor?: number; + /** + * Rows per batch for streaming generation. Default 10_000. Tests can pass + * a larger value to reduce overhead when collecting. + */ + batchSize?: number; +}; + +export type ScenarioBatch = { + traces: TraceRow[]; + logs: LogRow[]; +}; + +export type ScenarioOutput = ScenarioBatch; + +export type Scenario = { + name: string; + agentPrompt: string; + description: string; + /** + * Yields batches of rows for insertion. Implementations should respect + * `ctx.batchSize` (cap per-yield row count) and `ctx.volumeFactor` + * (scale background volumes — planted anomalies stay constant). + */ + generate(ctx: GenerateContext): Iterable; + groundTruth: unknown; +}; + +/** Helper for tests: drain an iterable into one combined batch. */ +export function collectScenario(iter: Iterable): ScenarioOutput { + const traces: TraceRow[] = []; + const logs: LogRow[] = []; + for (const b of iter) { + if (b.traces.length) traces.push(...b.traces); + if (b.logs.length) logs.push(...b.logs); + } + return { traces, logs }; +} diff --git a/packages/hdx-eval/tsconfig.build.json b/packages/hdx-eval/tsconfig.build.json new file mode 100644 index 0000000000..a9862d7ad9 --- /dev/null +++ b/packages/hdx-eval/tsconfig.build.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.json", + "exclude": ["node_modules", "dist", "**/__tests__/**"] +} diff --git a/packages/hdx-eval/tsconfig.json b/packages/hdx-eval/tsconfig.json new file mode 100644 index 0000000000..48370bd1d7 --- /dev/null +++ b/packages/hdx-eval/tsconfig.json @@ -0,0 +1,23 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "module": "CommonJS", + "moduleResolution": "Node", + "baseUrl": "./src", + "paths": { + "@/*": ["./*"] + }, + "types": ["jest", "node"], + "outDir": "dist", + "rootDir": "src", + "declaration": true, + "isolatedModules": true, + "esModuleInterop": true, + "resolveJsonModule": true, + "strict": true, + "skipLibCheck": true, + "sourceMap": true + }, + "include": ["src"], + "exclude": ["node_modules", "dist"] +} diff --git a/yarn.lock b/yarn.lock index 7b481164c2..1d62b98f0d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -111,6 +111,22 @@ __metadata: languageName: node linkType: hard +"@anthropic-ai/sdk@npm:^0.92.0": + version: 0.92.0 + resolution: "@anthropic-ai/sdk@npm:0.92.0" + dependencies: + json-schema-to-ts: "npm:^3.1.1" + peerDependencies: + zod: ^3.25.0 || ^4.0.0 + peerDependenciesMeta: + zod: + optional: true + bin: + anthropic-ai-sdk: bin/cli + checksum: 10c0/a06f2ebc4dce6e3ee36e3cc938b9fb952a6eebaa3b4478351770d0bf264aa3cb141a5a6854cdc93d1693218415d626338f66d0c446e1d1fa2402a02dcd890610 + languageName: node + linkType: hard + "@apidevtools/json-schema-ref-parser@npm:^9.0.6": version: 9.1.2 resolution: "@apidevtools/json-schema-ref-parser@npm:9.1.2" @@ -2609,6 +2625,13 @@ __metadata: languageName: node linkType: hard +"@babel/runtime@npm:^7.18.3": + version: 7.29.2 + resolution: "@babel/runtime@npm:7.29.2" + checksum: 10c0/30b80a0140d16467792e1bbeb06f655b0dab70407da38dfac7fedae9c859f9ae9d846ef14ad77bd3814c064295fe9b1bc551f1541ea14646ae9f22b71a8bc17a + languageName: node + linkType: hard + "@babel/runtime@npm:^7.18.6": version: 7.24.0 resolution: "@babel/runtime@npm:7.24.0" @@ -4611,6 +4634,28 @@ __metadata: languageName: unknown linkType: soft +"@hyperdx/hdx-eval@workspace:packages/hdx-eval": + version: 0.0.0-use.local + resolution: "@hyperdx/hdx-eval@workspace:packages/hdx-eval" + dependencies: + "@anthropic-ai/sdk": "npm:^0.92.0" + "@clickhouse/client": "npm:^1.12.1" + "@hyperdx/common-utils": "npm:^0.19.0" + "@types/jest": "npm:^29.5.14" + "@types/ms": "npm:^0.7.31" + "@types/node": "npm:^22.15.18" + commander: "npm:^12.1.0" + jest: "npm:^30.2.0" + ms: "npm:^2.1.3" + rimraf: "npm:^4.4.1" + ts-jest: "npm:^29.4.5" + tsx: "npm:^4.19.0" + typescript: "npm:^5.9.3" + bin: + hdx-eval: ./dist/cli.js + languageName: unknown + linkType: soft + "@hyperdx/instrumentation-exception@npm:^0.1.0": version: 0.1.0 resolution: "@hyperdx/instrumentation-exception@npm:0.1.0" @@ -19704,6 +19749,16 @@ __metadata: languageName: node linkType: hard +"json-schema-to-ts@npm:^3.1.1": + version: 3.1.1 + resolution: "json-schema-to-ts@npm:3.1.1" + dependencies: + "@babel/runtime": "npm:^7.18.3" + ts-algebra: "npm:^2.0.0" + checksum: 10c0/609bae04aa5e860a11b6d30ccf41445fae1c7f66fb600c1d170257cf33aa468aa9d03aa046428c3688aff0ff450c2b0c76584b66fa4a5d0da8e33799e4c439a6 + languageName: node + linkType: hard + "json-schema-traverse@npm:^0.4.1": version: 0.4.1 resolution: "json-schema-traverse@npm:0.4.1" @@ -27249,6 +27304,13 @@ __metadata: languageName: node linkType: hard +"ts-algebra@npm:^2.0.0": + version: 2.0.0 + resolution: "ts-algebra@npm:2.0.0" + checksum: 10c0/4ae93bec1bada635bba425854eec323dad50b6ffe86bc04ad2d7f9ce3fb129d673dcf483e19a6e70d07a3a9083e6a0a7f4e004bb8d2164cddc60cc9540ba187f + languageName: node + linkType: hard + "ts-api-utils@npm:^2.1.0": version: 2.1.0 resolution: "ts-api-utils@npm:2.1.0" From d93aa98a08f7b260e7a5e7a25af9ef8883c5bd73 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Wed, 20 May 2026 15:57:24 -0600 Subject: [PATCH 02/22] stream seeding results --- packages/hdx-eval/src/cli.ts | 36 +++++++++++++++++-- .../hdx-eval/src/scenarios/seedScenario.ts | 10 ++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/packages/hdx-eval/src/cli.ts b/packages/hdx-eval/src/cli.ts index a5ee1ef22c..0cd713226f 100644 --- a/packages/hdx-eval/src/cli.ts +++ b/packages/hdx-eval/src/cli.ts @@ -15,7 +15,27 @@ import { batchDirName } from './runs/path'; import { writeRun } from './runs/store'; import { listBatches, listRunsInBatch, readRun } from './runs/store'; import { getScenario, SCENARIO_NAMES, SCENARIOS } from './scenarios'; -import { seedScenario } from './scenarios/seedScenario'; +import { type SeedProgress, seedScenario } from './scenarios/seedScenario'; + +if (!process.env.ANTHROPIC_API_KEY && process.env.AI_API_KEY) { + process.env.ANTHROPIC_API_KEY = process.env.AI_API_KEY; +} + +function formatCount(n: number): string { + if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`; + if (n >= 1_000) return `${(n / 1_000).toFixed(1)}K`; + return String(n); +} + +function logSeedProgress(prefix: string, startMs: number) { + return (p: SeedProgress) => { + const elapsed = ((Date.now() - startMs) / 1000).toFixed(0); + const total = p.tracesInserted + p.logsInserted; + process.stdout.write( + `\r${prefix}${formatCount(total)} rows (${formatCount(p.tracesInserted)} traces, ${formatCount(p.logsInserted)} logs) · ${elapsed}s`, + ); + }; +} type GlobalOpts = { chUrl?: string; @@ -125,19 +145,24 @@ program const client = buildClient(opts); try { + const seedStart = Date.now(); const result = await seedScenario({ client, scenarioName: scenario.name, seed: seedNum, nowMs, volumeFactor, + onProgress: logSeedProgress(' seeding: ', seedStart), }); + const seedSecs = ((Date.now() - seedStart) / 1000).toFixed(1); + process.stdout.write('\n'); console.log( `Inserted ${result.tracesInserted} trace rows → default.${result.tables.traces}`, ); console.log( `Inserted ${result.logsInserted} log rows → default.${result.tables.logs}`, ); + console.log(`Done in ${seedSecs}s`); } finally { await client.close(); } @@ -378,6 +403,10 @@ program // Re-seed once before running so timestamps are anchored to the chosen // "now". Skip if --no-reseed was passed (e.g. shared-anchor batch seed). if (cmdOpts.reseed !== false) { + console.log( + `Seeding ${scenario.name} (seed=${seedNum}, now=${new Date(anchorMs).toISOString()})...`, + ); + const seedStart = Date.now(); const client = buildClientFromConfig(config, opts); try { const r = await seedScenario({ @@ -385,9 +414,12 @@ program scenarioName: scenario.name, seed: seedNum, nowMs: anchorMs, + onProgress: logSeedProgress(' ', seedStart), }); + const seedSecs = ((Date.now() - seedStart) / 1000).toFixed(1); + process.stdout.write('\n'); console.log( - `Re-seeded ${scenario.name}: ${r.tracesInserted} traces, ${r.logsInserted} logs (now=${new Date(anchorMs).toISOString()})`, + `Seeded ${scenario.name}: ${formatCount(r.tracesInserted)} traces, ${formatCount(r.logsInserted)} logs in ${seedSecs}s`, ); } finally { await client.close(); diff --git a/packages/hdx-eval/src/scenarios/seedScenario.ts b/packages/hdx-eval/src/scenarios/seedScenario.ts index 758dc3d551..3ba13c913f 100644 --- a/packages/hdx-eval/src/scenarios/seedScenario.ts +++ b/packages/hdx-eval/src/scenarios/seedScenario.ts @@ -8,6 +8,12 @@ import { import { mulberry32 } from '../rng/seeded'; import { getScenario } from './index'; +export type SeedProgress = { + tracesInserted: number; + logsInserted: number; + batchesCompleted: number; +}; + export type SeedResult = { tracesInserted: number; logsInserted: number; @@ -20,6 +26,7 @@ export async function seedScenario(args: { seed: number; nowMs: number; volumeFactor?: number; + onProgress?: (progress: SeedProgress) => void; }): Promise { const scenario = getScenario(args.scenarioName); const tables = await ensureScenarioTables(args.client, scenario.name); @@ -28,6 +35,7 @@ export async function seedScenario(args: { let tracesInserted = 0; let logsInserted = 0; + let batchesCompleted = 0; for (const batch of scenario.generate({ rng, nowMs: args.nowMs, @@ -43,6 +51,8 @@ export async function seedScenario(args: { if (batch.logs.length > 0) { logsInserted += await insertLogRows(args.client, tables.logs, batch.logs); } + batchesCompleted++; + args.onProgress?.({ tracesInserted, logsInserted, batchesCompleted }); } return { tracesInserted, logsInserted, tables }; } From 37f9abee34c6ce75ba841e3458253e7742bdee1f Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Wed, 20 May 2026 16:34:08 -0600 Subject: [PATCH 03/22] general setup fixes --- packages/hdx-eval/README.md | 182 +++++++++++++++++++++++++------ packages/hdx-eval/package.json | 3 +- packages/hdx-eval/scripts/env.sh | 20 ++++ yarn.lock | 1 + 4 files changed, 171 insertions(+), 35 deletions(-) create mode 100755 packages/hdx-eval/scripts/env.sh diff --git a/packages/hdx-eval/README.md b/packages/hdx-eval/README.md index 57d9b801aa..96dc150740 100644 --- a/packages/hdx-eval/README.md +++ b/packages/hdx-eval/README.md @@ -1,60 +1,174 @@ # @hyperdx/hdx-eval -Synthetic telemetry data generators for the HyperDX observability eval -framework. Phase 1 deliverable: deterministic, planted-anomaly traces and -logs across three investigation scenarios. +Eval framework for benchmarking AI agents against HyperDX's observability +platform. Generates deterministic synthetic telemetry (traces + logs) with +planted anomalies, spawns Claude Code as an SRE agent with access to either +the HyperDX MCP server or a raw ClickHouse MCP, records full trajectories, +and grades answers using programmatic checks + an LLM-as-judge. + +## Prerequisites + +- **`yarn dev` running** so ClickHouse, MongoDB, and the HyperDX API are up +- **`claude` CLI installed** (the harness spawns it in streaming JSON mode) +- **`ANTHROPIC_API_KEY`** exported (for the `run` and `grade` commands) +- **[`uv`](https://docs.astral.sh/uv/)** installed (the ClickHouse MCP server + is launched via `uv run --with mcp-clickhouse`; without `uv` on `PATH` the + MCP process silently fails to start and the agent gets no tools) + +Port detection is automatic — the CLI computes the same worktree-based slot +as `scripts/dev-env.sh`, so no manual `source` step is needed. + +## Quick Start + +```bash +# 1. One-time setup: register eval account + create Sources +yarn workspace @hyperdx/hdx-eval dev setup-hyperdx + +# 2. Seed a scenario (use --volume-factor 0.1 for faster iteration) +yarn workspace @hyperdx/hdx-eval dev seed error-root-cause --volume-factor 0.1 + +# 3. Run the agent eval (both MCPs, 3 runs each) +ANTHROPIC_API_KEY=sk-... yarn workspace @hyperdx/hdx-eval dev run error-root-cause + +# 4. Grade the batch +yarn workspace @hyperdx/hdx-eval dev grade + +# 5. Generate a markdown report +yarn workspace @hyperdx/hdx-eval dev report +``` ## Scenarios -- `error-root-cause` — checkout 5xx spike caused by a `payment-service` → - DB connection timeout, propagated up through `checkout-api`. -- `latency-spike` — `api-server` p99 jump on `GET /api/orders/search` for - enterprise tenants (slow `database.query` child span). -- `noisy-signals` — log volume dominated by 3 patterns: `inventory-service` - heartbeats, `notification-service` cache-hit chatter, `worker` - caught-exception stack traces. +Each scenario writes into its own ClickHouse tables +(`eval__otel_traces` / `eval__otel_logs`) whose schema +mirrors HyperDX's `otel_traces` / `otel_logs` exactly. -Each scenario writes into its own ClickHouse tables in the `default` -database: `eval__otel_traces` and `eval__otel_logs`. -Schema mirrors HyperDX's `otel_traces` / `otel_logs` exactly via -`CREATE TABLE ... AS otel_traces`. +| Scenario | Agent Prompt | What's Planted | +|---|---|---| +| `error-root-cause` | "Checkout requests are failing for some users in the last 10 minutes. Find the root cause." | `payment-service` DB connection timeout cascading into `checkout-api` 5xx. 6M+ background spans, 12M logs, 5 distractor anomalies. | +| `latency-spike` | "p99 latency on api-server has jumped in the last 15 minutes. What's slow and why?" | `/api/orders/search` p99 spike for enterprise tenants (missing-index DB query). 12M+ spans, 5 regions, 5K tenants, GC/cold-start/cache distractors. | +| `noisy-signals` | "We want to cut log ingest cost. What are the top noisy signals we should drop or throttle?" | ~16M logs across composite (service, severity) cells. Each noisy cell also has a high-volume load-bearing pattern — grouping by service+severity alone won't work. | +| `segmented-regression` | "API error rate has gone up in the last 10 minutes for some users. Find the segment(s)." | Enterprise x cache-miss errors at ~12% from a schema-mismatch bug. Single-axis aggregates dilute the signal. Concurrent recommendation-service 502 burst as distractor. | +| `service-health-check` | "Generate a service health report for api-server for the last hour." | Peace-time report — no incident, but 4 novel-but-non-critical signals to call out, plus recurring baseline patterns the agent should NOT escalate. | -## Usage +Use `--volume-factor` to scale background row counts for faster iteration. +Planted anomaly counts stay fixed regardless of the factor. -Pre-req: `yarn dev` running so `default.otel_traces` and -`default.otel_logs` exist (created by the OTel collector migrations). +```bash +# Full volume (default) — realistic needle-in-haystack difficulty +yarn workspace @hyperdx/hdx-eval dev seed error-root-cause + +# 10% volume — much faster, still structurally valid +yarn workspace @hyperdx/hdx-eval dev seed error-root-cause --volume-factor 0.1 +``` + +## CLI Reference + +All commands are run via `yarn workspace @hyperdx/hdx-eval dev `. + +### Setup ```bash -# List scenarios + their tables -yarn workspace @hyperdx/hdx-eval dev list +# One-time: register account, create ClickHouse connection, create Sources +dev setup-hyperdx +dev setup-hyperdx --api-url http://localhost:30176 # explicit port +dev setup-hyperdx --check # validate existing config +dev setup-hyperdx --reset-sources # recreate all Sources +``` -# Seed a scenario (default seed=42, now=Date.now()) -yarn workspace @hyperdx/hdx-eval dev seed error-root-cause -yarn workspace @hyperdx/hdx-eval dev seed latency-spike --seed 7 -yarn workspace @hyperdx/hdx-eval dev seed noisy-signals \ - --now 2026-05-08T15:00:00Z +### Data Management -# Drop a scenario's tables -yarn workspace @hyperdx/hdx-eval dev drop error-root-cause +```bash +dev list # list scenarios + tables +dev seed # seed with defaults (seed=42, now=Date.now()) +dev seed --seed 7 # custom PRNG seed +dev seed --now 2026-05-08T15:00:00Z # fixed anchor time +dev seed --volume-factor 0.1 # 10% background volume +dev drop # drop scenario tables ``` -Connection defaults to `http://localhost:${HDX_DEV_CH_HTTP_PORT}` (set by -`scripts/dev-env.sh`); override with `--ch-url`. +### Running Evals + +```bash +dev run # both MCPs, 3 runs each +dev run --mcp hyperdx # HyperDX MCP only +dev run --mcp clickhouse # raw ClickHouse MCP only +dev run --runs 5 # 5 runs per cell +dev run --model claude-sonnet-4-6 +dev run --max-turns 20 # default: 15 +dev run --concurrency 4 # parallel cells +dev run --prompt-variant hypothesis # hypothesis-first variant +dev run --no-reseed --anchor-time 2026-05-08T15:00:00Z +``` + +### Inspecting Results + +```bash +dev runs-list # list recorded batches +dev runs-show # summary of a run +dev runs-show --queries # every tool call + SQL +dev runs-show --final-answer # print the agent's answer +dev runs-instrument # enrich with ClickHouse query_log timings +``` + +### Grading & Reporting + +```bash +dev grade # programmatic + LLM-as-judge (Opus 4.7) +dev grade --judge-model claude-sonnet-4-6 +dev grade --no-judge # programmatic checks only (no LLM cost) +dev grade --rerun-judge # re-run judge even if grades exist +dev report # render _summary.md + _summary.json +``` + +### Viewer + +```bash +yarn workspace @hyperdx/hdx-eval viewer # web UI at http://localhost:3333 +``` + +## Scoring + +Combined score = `0.4 * programmatic + 0.6 * judge - toolErrorPenalty` + +- **Programmatic (40%)** — regex-based keyword checks against the final + answer, defined per-scenario in `ground-truth.json`. +- **LLM-as-judge (60%)** — multi-criteria rubric scored 0-5 by Claude Opus 4.7 + (configurable). Criteria are scenario-specific. +- **Tool error penalty** — up to 20% deducted for high tool-call error rates. + Prevents agents from papering over correctness with volume. + +## Bulk Eval + +`scripts/fast-eval.sh` seeds all scenarios at a shared anchor time in +parallel, then runs them concurrently. Reads the API key from +`~/.config/hdx-eval-anthropic-key`. + +```bash +# Defaults: 4 scenarios, 2 runs/cell, concurrency 2 +./packages/hdx-eval/scripts/fast-eval.sh + +# Override +SCENARIOS="error-root-cause latency-spike" RUNS=3 \ + ./packages/hdx-eval/scripts/fast-eval.sh +``` -After seeding, manually create a HyperDX Source pointing at the eval -tables (one-time per scenario) so the HyperDX MCP server can query them. +There are also scripts for ablation studies (`scripts/ablation.sh`) and +prompt variant comparison (`scripts/compare-prompt-variants.sh`). ## Determinism All randomness flows through a `mulberry32(seed)` PRNG. Same `(seed, now)` produces byte-identical row content. This is what makes side-by-side -comparison of MCP A vs MCP B meaningful. +comparison of HyperDX MCP vs ClickHouse MCP meaningful. ## Tests ```bash -yarn workspace @hyperdx/hdx-eval ci:unit +yarn workspace @hyperdx/hdx-eval ci:unit # all unit tests +yarn workspace @hyperdx/hdx-eval dev:unit # watch mode ``` -Unit tests verify (a) PRNG determinism and (b) each scenario's planted -anomaly invariants hold in the generated rows (no ClickHouse needed). +Unit tests verify PRNG determinism, planted anomaly invariants, grading +pipeline, stream parsing, prompt construction, and report aggregation. +No ClickHouse needed. diff --git a/packages/hdx-eval/package.json b/packages/hdx-eval/package.json index 9dcd1c9327..55774c8cf3 100644 --- a/packages/hdx-eval/package.json +++ b/packages/hdx-eval/package.json @@ -20,6 +20,7 @@ "ms": "^2.1.3" }, "devDependencies": { + "@dotenvx/dotenvx": "^1.51.1", "@types/jest": "^29.5.14", "@types/ms": "^0.7.31", "@types/node": "^22.15.18", @@ -31,7 +32,7 @@ }, "scripts": { "build": "rimraf ./dist && tsc -p tsconfig.build.json", - "dev": "NODE_OPTIONS=--max-old-space-size=8192 tsx src/cli.ts", + "dev": "NODE_OPTIONS=--max-old-space-size=8192 ./scripts/env.sh tsx src/cli.ts", "lint": "npx eslint --quiet . --ext .ts", "lint:fix": "npx eslint . --ext .ts --fix", "ci:lint": "yarn lint && yarn tsc --noEmit", diff --git a/packages/hdx-eval/scripts/env.sh b/packages/hdx-eval/scripts/env.sh new file mode 100755 index 0000000000..a30d58c915 --- /dev/null +++ b/packages/hdx-eval/scripts/env.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Set up the full environment for hdx-eval, then exec the remaining args. +# +# 1. Source dev-env.sh from the monorepo root for slot-based port vars +# 2. Wrap the command with dotenvx to load .env / .env.local +# +# Usage: +# scripts/env.sh tsx src/cli.ts seed error-root-cause +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" + +# Slot-based port env vars (HYPERDX_API_PORT, HDX_DEV_CH_HTTP_PORT, etc.) +_HDX_EVAL_CWD="$PWD" +cd "$REPO_ROOT" && source ./scripts/dev-env.sh > /dev/null 2>&1 +cd "$_HDX_EVAL_CWD" +unset _HDX_EVAL_CWD + +# Load .env.local from the monorepo root (existing vars take precedence) +exec "$REPO_ROOT/node_modules/.bin/dotenvx" run \ + -f "$REPO_ROOT/.env.local" \ + --ignore=MISSING_ENV_FILE --quiet -- "$@" diff --git a/yarn.lock b/yarn.lock index 1d62b98f0d..8d4f69f9d9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4640,6 +4640,7 @@ __metadata: dependencies: "@anthropic-ai/sdk": "npm:^0.92.0" "@clickhouse/client": "npm:^1.12.1" + "@dotenvx/dotenvx": "npm:^1.51.1" "@hyperdx/common-utils": "npm:^0.19.0" "@types/jest": "npm:^29.5.14" "@types/ms": "npm:^0.7.31" From e07e4c7c3e48d094a740439c7d7276aae2e00001 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Fri, 22 May 2026 10:25:55 -0600 Subject: [PATCH 04/22] general eval improvements --- .../src/__tests__/systemPrompt.test.ts | 27 ++++------ packages/hdx-eval/src/cli.ts | 51 ++++++++++++++++++- packages/hdx-eval/src/harness/claudeSpawn.ts | 2 +- packages/hdx-eval/src/harness/runRun.ts | 1 + packages/hdx-eval/src/harness/settingsFile.ts | 37 ++++++++++++-- packages/hdx-eval/src/harness/systemPrompt.ts | 32 ------------ 6 files changed, 93 insertions(+), 57 deletions(-) diff --git a/packages/hdx-eval/src/__tests__/systemPrompt.test.ts b/packages/hdx-eval/src/__tests__/systemPrompt.test.ts index 4ed22ade58..0eb9610d31 100644 --- a/packages/hdx-eval/src/__tests__/systemPrompt.test.ts +++ b/packages/hdx-eval/src/__tests__/systemPrompt.test.ts @@ -7,24 +7,15 @@ describe('buildSystemPrompt', () => { expect(p).toContain('default.eval_error_root_cause_otel_logs'); }); - it('hints HyperDX MCP tools when condition is hyperdx', () => { - const p = buildSystemPrompt('latency-spike', 'hyperdx'); - // The prompt now ships a small tools catalog so the agent doesn't have - // to ToolSearch every tool by name. The catalog covers list_sources, - // describe_source, and the four investigation tools we want the agent - // to know about. - expect(p).toContain('hyperdx_list_sources'); - expect(p).toContain('hyperdx_describe_source'); - expect(p).toContain('hyperdx_event_deltas'); - expect(p).toContain('hyperdx_log_patterns'); - expect(p).not.toContain('run_query'); - }); - - it('hints ClickHouse MCP tools when condition is clickhouse', () => { - const p = buildSystemPrompt('noisy-signals', 'clickhouse'); - expect(p).toContain('run_query'); - expect(p).toContain('list_tables'); - expect(p).not.toContain('hyperdx_query'); + it('includes generic tool-environment guidance for both MCPs', () => { + const hdx = buildSystemPrompt('latency-spike', 'hyperdx'); + const ch = buildSystemPrompt('noisy-signals', 'clickhouse'); + // Both prompts should warn that Read/Bash/Grep are unavailable + for (const p of [hdx, ch]) { + expect(p).toContain('TOOL ENVIRONMENT'); + expect(p).toContain('NO'); + expect(p).toContain('file-reading tools'); + } }); it('asks for service, operation, and root cause in the final answer', () => { diff --git a/packages/hdx-eval/src/cli.ts b/packages/hdx-eval/src/cli.ts index 0cd713226f..b9588d7ec7 100644 --- a/packages/hdx-eval/src/cli.ts +++ b/packages/hdx-eval/src/cli.ts @@ -4,7 +4,11 @@ import { resolve } from 'path'; import { createEvalClient, defaultClickHouseUrl } from './clickhouse/client'; import { dropScenarioTables, scenarioTables } from './clickhouse/schema'; -import { gradeBatch, resolveBatchDir } from './grading/grade'; +import { + gradeBatch, + type GradeBatchOptions, + resolveBatchDir, +} from './grading/grade'; import { runCell } from './harness/runRun'; import type { McpKind, PromptVariant } from './harness/types'; import { configExists, configPath, readConfig } from './hyperdx/config'; @@ -330,6 +334,17 @@ program 'denies it.', 'baseline', ) + .option('--no-grade', 'Skip automatic grading after runs complete') + .option('--no-report', 'Skip automatic report generation after grading') + .option( + '--judge-model ', + 'Judge model ID (used when auto-grading)', + 'claude-opus-4-7', + ) + .option( + '--no-judge', + 'Run programmatic checks only during auto-grading (skip LLM judge)', + ) .action( async ( scenarioName: string, @@ -344,6 +359,10 @@ program anchorTime?: string; concurrency: string; promptVariant: string; + grade: boolean; + report: boolean; + judgeModel: string; + judge: boolean; }, ) => { const opts = program.opts(); @@ -508,6 +527,36 @@ program ` ${row.mcp.padEnd(10)} #${row.i} ${row.termination.padEnd(13)} tools=${row.toolCalls} tokens=${row.inputTokens}+${row.outputTokens} ${row.durationS}s`, ); } + + // ─── Auto-grade ────────────────────────────────────────────── + if (cmdOpts.grade !== false) { + console.log('\n--- Auto-grading ---'); + const gradeOpts: GradeBatchOptions = { + judgeModel: cmdOpts.judgeModel, + skipJudge: cmdOpts.judge === false, + }; + const gradeResult = await gradeBatch(batchDir, gradeOpts); + console.log( + `\nGraded ${gradeResult.graded.length} run${gradeResult.graded.length === 1 ? '' : 's'}; ${gradeResult.errors.length} error${gradeResult.errors.length === 1 ? '' : 's'}.`, + ); + if (gradeResult.errors.length > 0) { + for (const e of gradeResult.errors) + console.log(' -', e.runPath, e.error); + } + + // ─── Auto-report ──────────────────────────────────────────── + if (cmdOpts.report !== false && gradeResult.graded.length > 0) { + console.log('\n--- Auto-report ---'); + const resolvedDir = resolveBatchDir(batchDir); + const outPath = `${resolvedDir}/_summary.md`; + const reportResult = writeBatchSummary(resolvedDir, outPath); + console.log(`Wrote ${reportResult.mdPath}`); + console.log(`Wrote ${reportResult.jsonPath}`); + console.log( + `Aggregated ${reportResult.pairsCount} run${reportResult.pairsCount === 1 ? '' : 's'}.`, + ); + } + } }, ); diff --git a/packages/hdx-eval/src/harness/claudeSpawn.ts b/packages/hdx-eval/src/harness/claudeSpawn.ts index 68d5f3d405..ec886ea17f 100644 --- a/packages/hdx-eval/src/harness/claudeSpawn.ts +++ b/packages/hdx-eval/src/harness/claudeSpawn.ts @@ -61,7 +61,7 @@ export async function runClaude(opts: SpawnOptions): Promise { '--allowedTools', allowedToolsPattern(opts.mcp), '--disallowedTools', - deniedToolsFor(promptVariant).join(','), + deniedToolsFor(promptVariant, opts.mcp).join(','), '--dangerously-skip-permissions', '--setting-sources', 'local', diff --git a/packages/hdx-eval/src/harness/runRun.ts b/packages/hdx-eval/src/harness/runRun.ts index d54ba4f9c1..e5602ea344 100644 --- a/packages/hdx-eval/src/harness/runRun.ts +++ b/packages/hdx-eval/src/harness/runRun.ts @@ -38,6 +38,7 @@ export async function runCell(opts: RunCellOptions): Promise { opts.mcp, opts.anchorTimeIso, promptVariant, + opts.maxTurns, ); const result = await runClaude({ diff --git a/packages/hdx-eval/src/harness/settingsFile.ts b/packages/hdx-eval/src/harness/settingsFile.ts index 2ea1dd0e6e..944395bb9a 100644 --- a/packages/hdx-eval/src/harness/settingsFile.ts +++ b/packages/hdx-eval/src/harness/settingsFile.ts @@ -32,10 +32,37 @@ export const DENIED_BUILT_IN_TOOLS = [ 'Task', ] as const; -export function deniedToolsFor(variant: PromptVariant): readonly string[] { - return variant === 'hypothesis' - ? DENIED_BUILT_IN_TOOLS_BASE - : DENIED_BUILT_IN_TOOLS; +/** + * HyperDX MCP tools that are irrelevant to eval investigation scenarios. + * Denying these reduces the total visible tool count so Claude Code is more + * likely to load schemas eagerly (instead of deferring them behind + * ToolSearch), saving 1-2 turns per run. + */ +const DENIED_HYPERDX_NON_INVESTIGATION_TOOLS = [ + 'mcp__hyperdx__hyperdx_delete_dashboard', + 'mcp__hyperdx__hyperdx_get_dashboard', + 'mcp__hyperdx__hyperdx_save_dashboard', + 'mcp__hyperdx__hyperdx_query_tile', + 'mcp__hyperdx__hyperdx_get_saved_search', + 'mcp__hyperdx__hyperdx_save_saved_search', + 'mcp__hyperdx__hyperdx_get_alert', + 'mcp__hyperdx__hyperdx_get_webhook', + 'mcp__hyperdx__hyperdx_save_alert', +] as const; + +export function deniedToolsFor( + variant: PromptVariant, + kind?: McpKind, +): readonly string[] { + const builtIn = + variant === 'hypothesis' + ? DENIED_BUILT_IN_TOOLS_BASE + : DENIED_BUILT_IN_TOOLS; + + if (kind === 'hyperdx') { + return [...builtIn, ...DENIED_HYPERDX_NON_INVESTIGATION_TOOLS]; + } + return builtIn; } export function buildSettings( @@ -45,7 +72,7 @@ export function buildSettings( return { permissions: { allow: [allowedToolsPattern(kind)], - deny: [...deniedToolsFor(variant)], + deny: [...deniedToolsFor(variant, kind)], }, }; } diff --git a/packages/hdx-eval/src/harness/systemPrompt.ts b/packages/hdx-eval/src/harness/systemPrompt.ts index 9ed3776586..a2fcacb3f0 100644 --- a/packages/hdx-eval/src/harness/systemPrompt.ts +++ b/packages/hdx-eval/src/harness/systemPrompt.ts @@ -18,37 +18,6 @@ export function buildSystemPrompt( // SeverityText, SeverityNumber, Body, plus Map(String,String) columns // ResourceAttributes and LogAttributes.`; - const mcpSpecific = - mcp === 'hyperdx' - ? `HyperDX Sources are pre-configured for these tables (named -\`eval-${scenario}-traces\` and \`eval-${scenario}-logs\`). Call -\`hyperdx_list_sources\` first to discover source IDs. - -Available HyperDX investigation tools (call directly — names + purposes -below so you don't need to ToolSearch every tool): -- hyperdx_list_sources Catalog: sources + connections (their IDs, kinds, key columns) -- hyperdx_describe_source Full column schema + map attribute keys + low-cardinality values for one source -- hyperdx_timeseries Plot a metric over time; supports groupBy + granularity -- hyperdx_table Group-by aggregations. Can take MULTIPLE select items in one call (count + p50 + p99 in one shot). Map attributes work in both groupBy and valueExpression — \`SpanAttributes['']\` is a valid groupBy expression. Default shape="table" works for everything; only use number for a single scalar. -- hyperdx_search Browse individual rows ordered by timestamp -- hyperdx_sql Raw ClickHouse SQL. Only reach for this when the builder tools genuinely cannot express the query — multi-aggregate and attribute-map access are both supported by hyperdx_table. -- hyperdx_log_patterns Cluster log bodies into Drain templates, ranked by frequency, with per-pattern trend buckets -- hyperdx_event_deltas *** PREFER THIS over manual GROUP BYs when the question is "which attribute separates these two row groups". Examples: errors vs successes, slow vs fast spans, after-deploy vs before-deploy, one cohort vs the rest. One call ranks ALL attributes by signal strength. Whenever you find yourself running a sequence of GROUP BYs on different columns trying to find a discriminator, that's the cue to use event_deltas instead. -- hyperdx_trace_waterfall Full span tree for ONE trace (by TraceId or auto-picked via pickFilter + pickBy) -- hyperdx_trace_top_time_consuming_operations - Aggregate child-span breakdown across MANY traces matching a parent filter. Ranks operations by total time spent — answers "where is the time going inside these slow requests?" - -ToolSearch is still needed to load the full input schema for tools you've -never used in this session, but you don't need to ToolSearch for the -list itself — the catalog above is complete.` - : `Use \`run_query\` to issue SQL against the tables above. Schema -introspection is available via \`list_databases\` and \`list_tables\`. - -Note: ClickHouse will NOT implicitly cast a bare ISO-8601 string to -\`DateTime64\`. \`WHERE Timestamp >= ''\` fails with "Cannot convert -string ... to type DateTime64(9)". Wrap with -\`parseDateTime64BestEffortOrNull('')\` or use \`toDateTime64('', 9)\`.`; - // When the harness anchors a run to a fixed past time, the agent must use // that anchor as "now" for any relative window in the user's prompt. Without // this, the model uses today's date and queries an empty future window. @@ -114,7 +83,6 @@ observability data. The OpenTelemetry data lives in ClickHouse: ${anchorBlock} ${sharedSchema} -${mcpSpecific} ${playbookBlock} Investigate the user's question. In your final answer, identify: - the specific service(s) and operation(s) involved From 2fdc6d73e585a24c8cfbcad5915a2304b56310e8 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Fri, 22 May 2026 17:42:59 -0600 Subject: [PATCH 05/22] improvements to framework - leverage kv rollup mvs - allow claude access to read only in temp dir - tweaks to sysprompt --- .../hdx-eval/src/__tests__/mcpConfig.test.ts | 30 ++- .../src/__tests__/systemPrompt.test.ts | 9 +- packages/hdx-eval/src/clickhouse/schema.ts | 226 ++++++++++++++++++ packages/hdx-eval/src/harness/claudeSpawn.ts | 4 +- packages/hdx-eval/src/harness/settingsFile.ts | 20 +- packages/hdx-eval/src/harness/systemPrompt.ts | 23 ++ packages/hdx-eval/src/hyperdx/setup.ts | 22 +- 7 files changed, 318 insertions(+), 16 deletions(-) diff --git a/packages/hdx-eval/src/__tests__/mcpConfig.test.ts b/packages/hdx-eval/src/__tests__/mcpConfig.test.ts index 8e9747c928..4251a1337d 100644 --- a/packages/hdx-eval/src/__tests__/mcpConfig.test.ts +++ b/packages/hdx-eval/src/__tests__/mcpConfig.test.ts @@ -3,7 +3,11 @@ import { buildMcpConfig, MCP_CLICKHOUSE_VERSION, } from '../harness/mcpConfig'; -import { buildSettings, DENIED_BUILT_IN_TOOLS } from '../harness/settingsFile'; +import { + buildSettings, + DENIED_BUILT_IN_TOOLS, + deniedToolsFor, +} from '../harness/settingsFile'; import type { EvalConfig } from '../hyperdx/config'; const cfg: EvalConfig = { @@ -73,11 +77,11 @@ describe('allowedToolsPattern', () => { }); describe('buildSettings', () => { - it("allowlists only the run condition's MCP tools", () => { + it("allowlists only the run condition's MCP tools when no tempdir given", () => { expect(buildSettings('hyperdx')).toEqual({ permissions: { allow: ['mcp__hyperdx__*'], - deny: [...DENIED_BUILT_IN_TOOLS], + deny: [...deniedToolsFor('baseline', 'hyperdx')], }, }); expect(buildSettings('clickhouse')).toEqual({ @@ -88,12 +92,28 @@ describe('buildSettings', () => { }); }); - it('denies the built-in filesystem/shell tools so the agent cannot read prior runs', () => { + it('adds a scoped Read allow rule when tempdir is provided', () => { + const settings = buildSettings( + 'hyperdx', + 'baseline', + '/tmp/hdx-eval-abc', + ) as { + permissions: { allow: string[] }; + }; + expect(settings.permissions.allow).toEqual([ + 'mcp__hyperdx__*', + 'Read(/tmp/hdx-eval-abc/*)', + ]); + }); + + it('denies the built-in filesystem/shell tools so the agent cannot access the source tree', () => { const settings = buildSettings('hyperdx') as { permissions: { deny: string[] }; }; expect(settings.permissions.deny).toEqual( - expect.arrayContaining(['Bash', 'Read', 'Glob', 'Grep', 'Write', 'Edit']), + expect.arrayContaining(['Bash', 'Glob', 'Grep', 'Write', 'Edit']), ); + // Read is NOT denied — it is allowed with a path scope via the allow list + expect(settings.permissions.deny).not.toContain('Read'); }); }); diff --git a/packages/hdx-eval/src/__tests__/systemPrompt.test.ts b/packages/hdx-eval/src/__tests__/systemPrompt.test.ts index 0eb9610d31..7e9981f85d 100644 --- a/packages/hdx-eval/src/__tests__/systemPrompt.test.ts +++ b/packages/hdx-eval/src/__tests__/systemPrompt.test.ts @@ -10,18 +10,19 @@ describe('buildSystemPrompt', () => { it('includes generic tool-environment guidance for both MCPs', () => { const hdx = buildSystemPrompt('latency-spike', 'hyperdx'); const ch = buildSystemPrompt('noisy-signals', 'clickhouse'); - // Both prompts should warn that Read/Bash/Grep are unavailable + // Both prompts should explain tool environment and Read availability for (const p of [hdx, ch]) { expect(p).toContain('TOOL ENVIRONMENT'); - expect(p).toContain('NO'); - expect(p).toContain('file-reading tools'); + expect(p).toContain('Read tool'); + expect(p).toContain('oversized tool responses'); } }); - it('asks for service, operation, and root cause in the final answer', () => { + it('asks for service, operation, root cause, and ruled-out section', () => { const p = buildSystemPrompt('error-root-cause', 'clickhouse'); expect(p.toLowerCase()).toContain('service'); expect(p.toLowerCase()).toContain('operation'); expect(p.toLowerCase()).toContain('root cause'); + expect(p.toLowerCase()).toContain("what's not the cause"); }); }); diff --git a/packages/hdx-eval/src/clickhouse/schema.ts b/packages/hdx-eval/src/clickhouse/schema.ts index e2807a663e..4e789721ee 100644 --- a/packages/hdx-eval/src/clickhouse/schema.ts +++ b/packages/hdx-eval/src/clickhouse/schema.ts @@ -7,6 +7,10 @@ export const SOURCE_LOGS_TABLE = 'otel_logs'; export type ScenarioTables = { traces: string; logs: string; + tracesKvRollup: string; + tracesKeyRollup: string; + logsKvRollup: string; + logsKeyRollup: string; }; function scenarioSlug(scenario: string): string { @@ -18,6 +22,10 @@ export function scenarioTables(scenario: string): ScenarioTables { return { traces: `eval_${slug}_otel_traces`, logs: `eval_${slug}_otel_logs`, + tracesKvRollup: `eval_${slug}_otel_traces_kv_rollup_15m`, + tracesKeyRollup: `eval_${slug}_otel_traces_key_rollup_15m`, + logsKvRollup: `eval_${slug}_otel_logs_kv_rollup_15m`, + logsKeyRollup: `eval_${slug}_otel_logs_key_rollup_15m`, }; } @@ -71,6 +79,11 @@ export async function ensureScenarioTables( await removeTtlIfPresent(client, tables.traces); await removeTtlIfPresent(client, tables.logs); + // Create KV/Key rollup tables and materialized views so the HyperDX MCP + // can use fast metadata discovery instead of scanning the full raw tables. + // MVs are created *before* seeding so they capture inserts as they happen. + await ensureRollupTables(client, tables); + return tables; } @@ -94,6 +107,19 @@ export async function truncateScenarioTables( scenario: string, ): Promise { const tables = scenarioTables(scenario); + // Truncate rollup tables alongside raw tables. + await client.command({ + query: `TRUNCATE TABLE IF EXISTS ${EVAL_DATABASE}.${tables.tracesKvRollup}`, + }); + await client.command({ + query: `TRUNCATE TABLE IF EXISTS ${EVAL_DATABASE}.${tables.tracesKeyRollup}`, + }); + await client.command({ + query: `TRUNCATE TABLE IF EXISTS ${EVAL_DATABASE}.${tables.logsKvRollup}`, + }); + await client.command({ + query: `TRUNCATE TABLE IF EXISTS ${EVAL_DATABASE}.${tables.logsKeyRollup}`, + }); await client.command({ query: `TRUNCATE TABLE IF EXISTS ${EVAL_DATABASE}.${tables.traces}`, }); @@ -107,6 +133,8 @@ export async function dropScenarioTables( scenario: string, ): Promise { const tables = scenarioTables(scenario); + // Drop MVs first (they depend on the tables), then rollup tables, then raw. + await dropRollupTables(client, tables); await client.command({ query: `DROP TABLE IF EXISTS ${EVAL_DATABASE}.${tables.traces}`, }); @@ -114,3 +142,201 @@ export async function dropScenarioTables( query: `DROP TABLE IF EXISTS ${EVAL_DATABASE}.${tables.logs}`, }); } + +// --------------------------------------------------------------------------- +// Rollup tables & materialized views +// --------------------------------------------------------------------------- + +const ROLLUP_TABLE_DDL = `( + \`Timestamp\` DateTime, + \`ColumnIdentifier\` LowCardinality(String), + \`Key\` LowCardinality(String), + \`Value\` String, + \`count\` UInt64, + INDEX idx_count_minmax count TYPE minmax GRANULARITY 1, + INDEX idx_timestamp_minmax Timestamp TYPE minmax GRANULARITY 1 +) +ENGINE = SummingMergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (ColumnIdentifier, Key, Timestamp, Value) +SETTINGS index_granularity = 8192`; + +const KEY_ROLLUP_TABLE_DDL = `( + \`Timestamp\` DateTime, + \`ColumnIdentifier\` LowCardinality(String), + \`Key\` LowCardinality(String), + \`count\` UInt64, + INDEX idx_count_minmax count TYPE minmax GRANULARITY 1, + INDEX idx_timestamp_minmax Timestamp TYPE minmax GRANULARITY 1 +) +ENGINE = SummingMergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (ColumnIdentifier, Key, Timestamp) +SETTINGS index_granularity = 8192`; + +function mvName(rollupTable: string): string { + return `${rollupTable}_mv`; +} + +function keyMvName(keyRollupTable: string): string { + return `${keyRollupTable}_mv`; +} + +/** Build the KV rollup MV SELECT for a traces table. */ +function tracesKvMvSelect(db: string, rawTable: string): string { + return ` +WITH elements AS ( + SELECT + 'ResourceAttributes' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + replaceRegexpAll(entry.1, '\\\\[\\\\d+\\\\]', '[*]') AS Key, + CAST(entry.2 AS String) AS Value + FROM ${db}.${rawTable} + ARRAY JOIN ResourceAttributes AS entry + UNION ALL + SELECT + 'SpanAttributes' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + replaceRegexpAll(entry.1, '\\\\[\\\\d+\\\\]', '[*]') AS Key, + CAST(entry.2 AS String) AS Value + FROM ${db}.${rawTable} + ARRAY JOIN SpanAttributes AS entry + UNION ALL + SELECT 'NativeColumn', toStartOfFifteenMinutes(Timestamp), 'ServiceName', CAST(ServiceName AS String) FROM ${db}.${rawTable} + UNION ALL + SELECT 'NativeColumn', toStartOfFifteenMinutes(Timestamp), 'SpanName', CAST(SpanName AS String) FROM ${db}.${rawTable} + UNION ALL + SELECT 'NativeColumn', toStartOfFifteenMinutes(Timestamp), 'SpanKind', CAST(SpanKind AS String) FROM ${db}.${rawTable} + UNION ALL + SELECT 'NativeColumn', toStartOfFifteenMinutes(Timestamp), 'StatusCode', CAST(StatusCode AS String) FROM ${db}.${rawTable} + UNION ALL + SELECT 'NativeColumn', toStartOfFifteenMinutes(Timestamp), 'ScopeName', CAST(ScopeName AS String) FROM ${db}.${rawTable} + UNION ALL + SELECT 'NativeColumn', toStartOfFifteenMinutes(Timestamp), 'ScopeVersion', CAST(ScopeVersion AS String) FROM ${db}.${rawTable} +) +SELECT Timestamp, ColumnIdentifier, Key, Value, count() AS count FROM elements +GROUP BY Timestamp, ColumnIdentifier, Key, Value`; +} + +/** Build the KV rollup MV SELECT for a logs table. */ +function logsKvMvSelect(db: string, rawTable: string): string { + return ` +WITH elements AS ( + SELECT + 'ResourceAttributes' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + replaceRegexpAll(entry.1, '\\\\[\\\\d+\\\\]', '[*]') AS Key, + CAST(entry.2 AS String) AS Value + FROM ${db}.${rawTable} + ARRAY JOIN ResourceAttributes AS entry + UNION ALL + SELECT + 'LogAttributes' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + replaceRegexpAll(entry.1, '\\\\[\\\\d+\\\\]', '[*]') AS Key, + CAST(entry.2 AS String) AS Value + FROM ${db}.${rawTable} + ARRAY JOIN LogAttributes AS entry + UNION ALL + SELECT + 'ScopeAttributes' AS ColumnIdentifier, + toStartOfFifteenMinutes(Timestamp) AS Timestamp, + replaceRegexpAll(entry.1, '\\\\[\\\\d+\\\\]', '[*]') AS Key, + CAST(entry.2 AS String) AS Value + FROM ${db}.${rawTable} + ARRAY JOIN ScopeAttributes AS entry + UNION ALL + SELECT 'NativeColumn', toStartOfFifteenMinutes(Timestamp), 'SeverityText', CAST(SeverityText AS String) FROM ${db}.${rawTable} + UNION ALL + SELECT 'NativeColumn', toStartOfFifteenMinutes(Timestamp), 'ServiceName', CAST(ServiceName AS String) FROM ${db}.${rawTable} + UNION ALL + SELECT 'NativeColumn', toStartOfFifteenMinutes(Timestamp), 'ScopeName', CAST(ScopeName AS String) FROM ${db}.${rawTable} + UNION ALL + SELECT 'NativeColumn', toStartOfFifteenMinutes(Timestamp), 'ScopeVersion', CAST(ScopeVersion AS String) FROM ${db}.${rawTable} + UNION ALL + SELECT 'NativeColumn', toStartOfFifteenMinutes(Timestamp), 'ResourceSchemaUrl', CAST(ResourceSchemaUrl AS String) FROM ${db}.${rawTable} + UNION ALL + SELECT 'NativeColumn', toStartOfFifteenMinutes(Timestamp), 'ScopeSchemaUrl', CAST(ScopeSchemaUrl AS String) FROM ${db}.${rawTable} +) +SELECT Timestamp, ColumnIdentifier, Key, Value, count() AS count FROM elements +GROUP BY Timestamp, ColumnIdentifier, Key, Value`; +} + +/** Key rollup MV SELECT (shared for both traces and logs). */ +function keyRollupMvSelect(db: string, kvRollupTable: string): string { + return ` +SELECT + Timestamp, + ColumnIdentifier, + Key, + sum(count) as count +FROM ${db}.${kvRollupTable} +GROUP BY ColumnIdentifier, Key, Timestamp`; +} + +async function ensureRollupTables( + client: ClickHouseClient, + tables: ScenarioTables, +): Promise { + const db = EVAL_DATABASE; + + // -- Traces rollups -- + await client.command({ + query: `CREATE TABLE IF NOT EXISTS ${db}.${tables.tracesKvRollup} ${ROLLUP_TABLE_DDL}`, + }); + await client.command({ + query: `CREATE TABLE IF NOT EXISTS ${db}.${tables.tracesKeyRollup} ${KEY_ROLLUP_TABLE_DDL}`, + }); + // KV rollup MV: raw traces → kv rollup + await client.command({ + query: `CREATE MATERIALIZED VIEW IF NOT EXISTS ${db}.${mvName(tables.tracesKvRollup)} TO ${db}.${tables.tracesKvRollup} AS ${tracesKvMvSelect(db, tables.traces)}`, + }); + // Key rollup MV: kv rollup → key rollup + await client.command({ + query: `CREATE MATERIALIZED VIEW IF NOT EXISTS ${db}.${keyMvName(tables.tracesKeyRollup)} TO ${db}.${tables.tracesKeyRollup} AS ${keyRollupMvSelect(db, tables.tracesKvRollup)}`, + }); + + // -- Logs rollups -- + await client.command({ + query: `CREATE TABLE IF NOT EXISTS ${db}.${tables.logsKvRollup} ${ROLLUP_TABLE_DDL}`, + }); + await client.command({ + query: `CREATE TABLE IF NOT EXISTS ${db}.${tables.logsKeyRollup} ${KEY_ROLLUP_TABLE_DDL}`, + }); + // KV rollup MV: raw logs → kv rollup + await client.command({ + query: `CREATE MATERIALIZED VIEW IF NOT EXISTS ${db}.${mvName(tables.logsKvRollup)} TO ${db}.${tables.logsKvRollup} AS ${logsKvMvSelect(db, tables.logs)}`, + }); + // Key rollup MV: kv rollup → key rollup + await client.command({ + query: `CREATE MATERIALIZED VIEW IF NOT EXISTS ${db}.${keyMvName(tables.logsKeyRollup)} TO ${db}.${tables.logsKeyRollup} AS ${keyRollupMvSelect(db, tables.logsKvRollup)}`, + }); +} + +async function dropRollupTables( + client: ClickHouseClient, + tables: ScenarioTables, +): Promise { + const db = EVAL_DATABASE; + // Drop MVs first (they reference the tables). + for (const mv of [ + mvName(tables.tracesKvRollup), + keyMvName(tables.tracesKeyRollup), + mvName(tables.logsKvRollup), + keyMvName(tables.logsKeyRollup), + ]) { + await client.command({ + query: `DROP VIEW IF EXISTS ${db}.${mv}`, + }); + } + for (const tbl of [ + tables.tracesKvRollup, + tables.tracesKeyRollup, + tables.logsKvRollup, + tables.logsKeyRollup, + ]) { + await client.command({ + query: `DROP TABLE IF EXISTS ${db}.${tbl}`, + }); + } +} diff --git a/packages/hdx-eval/src/harness/claudeSpawn.ts b/packages/hdx-eval/src/harness/claudeSpawn.ts index ec886ea17f..9e8d995700 100644 --- a/packages/hdx-eval/src/harness/claudeSpawn.ts +++ b/packages/hdx-eval/src/harness/claudeSpawn.ts @@ -51,7 +51,7 @@ export async function runClaude(opts: SpawnOptions): Promise { const promptVariant: PromptVariant = opts.promptVariant ?? 'baseline'; writeFileSync( settingsPath, - JSON.stringify(buildSettings(opts.mcp, promptVariant), null, 2), + JSON.stringify(buildSettings(opts.mcp, promptVariant, tempdir), null, 2), ); const argv = [ @@ -59,7 +59,7 @@ export async function runClaude(opts: SpawnOptions): Promise { '--mcp-config', mcpConfigPath, '--allowedTools', - allowedToolsPattern(opts.mcp), + `${allowedToolsPattern(opts.mcp)},Read(${tempdir}/*)`, '--disallowedTools', deniedToolsFor(promptVariant, opts.mcp).join(','), '--dangerously-skip-permissions', diff --git a/packages/hdx-eval/src/harness/settingsFile.ts b/packages/hdx-eval/src/harness/settingsFile.ts index 944395bb9a..be0932046f 100644 --- a/packages/hdx-eval/src/harness/settingsFile.ts +++ b/packages/hdx-eval/src/harness/settingsFile.ts @@ -4,9 +4,14 @@ import type { McpKind, PromptVariant } from './types'; /** * Filesystem / shell / built-in tools we never want the eval agent to use. * The agent's working directory is a fresh tempdir (no answer key) — but - * Bash/Read/Glob/Grep would still let it shell out to /home/ and read prior + * Bash/Glob/Grep would still let it shell out to /home/ and read prior * runs or the source tree. Lock it to the MCP tools only. * + * Read is NOT denied here — it is allowed with a path scope restricted to + * the agent's tempdir so the agent can recover oversized tool responses + * that Claude Code saves to disk. The scoped allow rule is added in + * buildSettings() using the concrete tempdir path. + * * `Task` (Claude Code's subagent-spawning tool) is denied by default but * ALLOWED in the `hypothesis` prompt variant — that variant explicitly * asks the agent to spawn parallel subagents to investigate multiple @@ -14,7 +19,6 @@ import type { McpKind, PromptVariant } from './types'; */ export const DENIED_BUILT_IN_TOOLS_BASE = [ 'Bash', - 'Read', 'Write', 'Edit', 'MultiEdit', @@ -68,10 +72,20 @@ export function deniedToolsFor( export function buildSettings( kind: McpKind, variant: PromptVariant = 'baseline', + tempdir?: string, ): Record { + const allow: string[] = [allowedToolsPattern(kind)]; + + // Allow Read scoped to the agent's tempdir so it can recover oversized + // tool responses that Claude Code saves to disk. The tempdir is an + // empty mkdtemp directory — no ground truth, source, or prior runs. + if (tempdir) { + allow.push(`Read(${tempdir}/*)`); + } + return { permissions: { - allow: [allowedToolsPattern(kind)], + allow, deny: [...deniedToolsFor(variant, kind)], }, }; diff --git a/packages/hdx-eval/src/harness/systemPrompt.ts b/packages/hdx-eval/src/harness/systemPrompt.ts index a2fcacb3f0..f5506235dd 100644 --- a/packages/hdx-eval/src/harness/systemPrompt.ts +++ b/packages/hdx-eval/src/harness/systemPrompt.ts @@ -6,6 +6,7 @@ export function buildSystemPrompt( mcp: McpKind, anchorTimeIso?: string, variant: PromptVariant = 'baseline', + maxTurns?: number, ): string { const { traces, logs } = scenarioTables(scenario); const sharedSchema = ''; @@ -84,10 +85,32 @@ ${anchorBlock} ${sharedSchema} ${playbookBlock} +TOOL ENVIRONMENT: Only MCP query tools and the Read tool are available. +There are NO shell, write, or search tools (Bash, Write, Edit, Glob, Grep, +etc.). The Read tool is restricted to your working directory — you can only +use it to read files that were saved there by oversized tool responses. +If a tool response is too large and gets saved to a file, use Read to +retrieve its contents. Do NOT ToolSearch for Bash or other file tools. +If the saved file is still too large for Read, re-run the query with +narrower filters or smaller limits. + +TURN BUDGET: You have a limited number of tool calls. Manage them wisely: +- After ~${maxTurns ?? 15} tool calls, if you have identified a primary root cause with + supporting evidence, write your final answer. You can note additional + areas worth investigating without exhaustively querying them. +- Do not keep exploring tangential signals once you have a strong causal + chain for the user's reported symptom. +- High log/error volume alone does not indicate relevance. Before + investigating a service further, verify it is in the causal chain of + the user's reported symptom — not just noisy in the same time window. + Investigate the user's question. In your final answer, identify: - the specific service(s) and operation(s) involved - the root cause (or, for cleanup-style questions, the top concrete actions) - supporting evidence from the data (counts, time ranges, sample messages) +- a brief "What's not the cause" section that explicitly names any + investigated signals you ruled out and why — this helps the reader + trust your conclusion Stay open to the possibility that what looks like one issue is actually multiple independent issues — when two components look similarly degraded, diff --git a/packages/hdx-eval/src/hyperdx/setup.ts b/packages/hdx-eval/src/hyperdx/setup.ts index 38f008d0a6..007b8c75e7 100644 --- a/packages/hdx-eval/src/hyperdx/setup.ts +++ b/packages/hdx-eval/src/hyperdx/setup.ts @@ -88,7 +88,10 @@ export async function runSetup(opts: SetupOptions): Promise { let traceSource = sourcesByName.get(traceName); if (!traceSource) { traceSource = await api.createSource( - buildTraceSourceBody(traceName, connection._id, tables.traces), + buildTraceSourceBody(traceName, connection._id, tables.traces, { + kvRollupTable: tables.tracesKvRollup, + keyRollupTable: tables.tracesKeyRollup, + }), ); created.push(traceName); } @@ -96,7 +99,10 @@ export async function runSetup(opts: SetupOptions): Promise { let logSource = sourcesByName.get(logName); if (!logSource) { logSource = await api.createSource( - buildLogSourceBody(logName, connection._id, tables.logs), + buildLogSourceBody(logName, connection._id, tables.logs, { + kvRollupTable: tables.logsKvRollup, + keyRollupTable: tables.logsKeyRollup, + }), ); created.push(logName); } @@ -141,6 +147,7 @@ function buildTraceSourceBody( name: string, connectionId: string, tableName: string, + rollup: { kvRollupTable: string; keyRollupTable: string }, ): Record { return { name, @@ -163,6 +170,11 @@ function buildTraceSourceBody( eventAttributesExpression: 'SpanAttributes', implicitColumnExpression: 'SpanName', querySettings: EVAL_QUERY_SETTINGS, + metadataMaterializedViews: { + keyRollupTable: rollup.keyRollupTable, + kvRollupTable: rollup.kvRollupTable, + granularity: '15 minute', + }, }; } @@ -170,6 +182,7 @@ function buildLogSourceBody( name: string, connectionId: string, tableName: string, + rollup: { kvRollupTable: string; keyRollupTable: string }, ): Record { return { name, @@ -187,6 +200,11 @@ function buildLogSourceBody( eventAttributesExpression: 'LogAttributes', implicitColumnExpression: 'Body', querySettings: EVAL_QUERY_SETTINGS, + metadataMaterializedViews: { + keyRollupTable: rollup.keyRollupTable, + kvRollupTable: rollup.kvRollupTable, + granularity: '15 minute', + }, }; } From 019db69fe894fa17183c650a6d278031c55d6cb1 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Fri, 29 May 2026 12:43:30 -0700 Subject: [PATCH 06/22] add eval skill --- .opencode/commands/eval-summary.md | 157 +++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 .opencode/commands/eval-summary.md diff --git a/.opencode/commands/eval-summary.md b/.opencode/commands/eval-summary.md new file mode 100644 index 0000000000..5f2dfede07 --- /dev/null +++ b/.opencode/commands/eval-summary.md @@ -0,0 +1,157 @@ +--- +description: Analyze HyperDX eval benchmark results and produce a summary report +--- + +You are an eval benchmark analyst for HyperDX. Your job is to review eval run +results and produce a structured summary comparing HyperDX MCP performance +against the ClickHouse baseline MCP. + +## Input + +The user will provide either: + +- Raw eval output (pasted terminal logs from running the eval harness) +- A batch directory path under `packages/hdx-eval/runs/` +- A `$ARGUMENTS` referencing a specific batch timestamp (e.g. + `2026-05-21T16-34-35-417Z`) + +If the user provides a batch timestamp or directory path, locate the run files +under `packages/hdx-eval/runs//`. If they paste terminal output, +extract the batch path from the "Results written under" line. + +If no batch is specified, list `packages/hdx-eval/runs/` and use the **most +recent** batch directory (sorted by timestamp). + +## Analysis Steps + +Perform these steps in order. Use the Task tool with parallel subagents where +possible to speed up analysis. + +### 1. Read Summary Files + +Read `_summary.md` and `_summary.json` from the batch directory. These contain +aggregated metrics across all runs. + +### 2. Analyze Run Transcripts + +Use **parallel Task subagents** — one per MCP (hyperdx, clickhouse). Each +subagent reads all run JSON files for its MCP +(`//0.json`, `1.json`, `2.json`, etc.) and extracts: + +- **Tool call sequence** — ordered list of tool names called with index numbers +- **Tool errors** — any failed tool calls, including error messages and index +- **Backtracking patterns** — sequences where the model retried, searched for + unavailable tools, or re-ran queries with different parameters +- **Final answer** — whether the run produced one (final_answer vs max_turns) +- **Wasted tool calls** — calls that didn't contribute to the final answer (e.g. + ToolSearch for tools that don't exist in the sandbox, queries on empty tables, + dead-end hypothesis investigations) +- **Turn budget usage** — how many calls were used vs budget, and whether the + model showed awareness of remaining turns +- **Interesting observations** — anything unusual, surprising, or noteworthy + about the run (clever investigation strategies, unexpected tool behavior, + novel failure modes, etc.) + +### 3. Read Grade Files and Ground Truth + +Use a **single Task subagent** to read all `*.grade.json` files, the scenario's +`ground-truth.json` from +`packages/hdx-eval/src/scenarios//ground-truth.json`, and the system +prompt at `packages/hdx-eval/src/harness/systemPrompt.ts`. Extract: + +- Per-run combined, programmatic, and judge scores +- Per-check pass/fail with weights +- Judge reasoning/rationale per criterion +- Ground truth expected answers, checks, and distractors +- System prompt text and any variant-specific guidance + +## Default Output + +By default, produce the **Quick Stats** output below. Then prompt the user +asking if they want to see more. + +### Quick Stats + +Four bullets covering: + +- **Accuracy** — combined score comparison with relative percentage +- **Consistency** — completion rate (final_answer vs max_turns) and score + variance across runs +- **Speed** — wall clock, tool calls, and token usage with relative percentages +- **Overall** — one-line headline with the key deltas + +Then a **Notable Findings** section: 3-5 short bullets highlighting the most +interesting or surprising things you found in the transcripts. These should be +specific, concrete observations — not generic patterns. Examples: + +- "HyperDX Run 1 scored 79.8% despite 2 SQL errors — it recovered in 2 calls" +- "ClickHouse Run 2 was the single best run (89.8%) but the other 2 CH runs + both hit max_turns" +- "The feature_flag distractor tripped HyperDX Run 1 because event_deltas + surfaced it as a top-changing attribute" + +Finally, end with: + +> Want to dig deeper? I can show: +> - **Detailed stats** — full per-run breakdown, per-check wins, and failure +> analysis +> - **Recommendations** — specific tool/prompt changes to improve HyperDX scores +> - **Both** + +### Detailed Stats (on request) + +When the user asks for detailed stats, produce: + +#### Section 1: Top-Line Stats + +A table of metrics where HyperDX beat ClickHouse. For each, compute the +percentage improvement (relative, not absolute). Focus on: + +- Combined score delta +- Completion rate (final_answer vs max_turns) +- Programmatic accuracy delta +- Judge criteria where HyperDX scored notably higher +- Tool call efficiency (fewer calls = better) + +Format as a markdown table with columns: Metric, HyperDX, ClickHouse, Delta, +Relative. + +#### Section 2: Per-Check Wins + +A table of the programmatic checks where HyperDX outperformed ClickHouse by the +largest margin. Include both positive checks (identified the right pattern) and +negative checks (avoided false positives). Only include checks with a gap of +20pp or more. + +For each check with a large gap, add a brief explanation of **why** the gap +exists. + +#### Section 3: Failure Analysis + +For any run (either MCP) that scored below 50% combined or hit max_turns, +explain what went wrong. Reference specific tool call sequences and errors. +Contrast with how the other MCP handled the same challenge. + +### Recommendations (on request) + +When the user asks for recommendations, analyze the run transcripts and produce +concrete improvement opportunities for the HyperDX MCP tools or system prompt. +For each opportunity: + +1. **Problem** — What happened in the runs (cite specific tool call sequences) +2. **Impact** — How many tool calls / seconds / score points this costs +3. **Fix** — A specific, actionable change to the tool, system prompt, or eval + harness + +Do NOT use a pre-built checklist of patterns to look for. Derive all findings +directly from the transcript data for this specific batch. Each recommendation +should be grounded in something that actually happened in the runs. + +## Guidelines + +- Be precise with numbers. Always compute relative deltas (not just absolute). +- Cite specific run IDs and tool call numbers when describing patterns. +- Focus on actionable insights, not just observations. +- If the eval only has one scenario, that's fine — analyze it thoroughly. If + there are multiple scenarios, analyze each separately and then synthesize. +- Keep the tone technical and direct. No filler. From 3256e47dc323fce8fdf752c7f766a9212a33b2db Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Mon, 1 Jun 2026 10:49:45 -0600 Subject: [PATCH 07/22] fix: track src/runs/ files and anchor gitignore to root runs/ only The runs/ gitignore pattern was matching src/runs/ (source code) in addition to the intended /runs/ (data directory). Anchor the pattern so only the top-level runs/ directory is ignored, and commit the three missing source files: instrument.ts, path.ts, store.ts. --- packages/hdx-eval/.gitignore | 2 +- packages/hdx-eval/src/runs/instrument.ts | 324 +++++++++++++++++++++++ packages/hdx-eval/src/runs/path.ts | 26 ++ packages/hdx-eval/src/runs/store.ts | 59 +++++ 4 files changed, 410 insertions(+), 1 deletion(-) create mode 100644 packages/hdx-eval/src/runs/instrument.ts create mode 100644 packages/hdx-eval/src/runs/path.ts create mode 100644 packages/hdx-eval/src/runs/store.ts diff --git a/packages/hdx-eval/.gitignore b/packages/hdx-eval/.gitignore index 9a31e16621..8452b93777 100644 --- a/packages/hdx-eval/.gitignore +++ b/packages/hdx-eval/.gitignore @@ -1,4 +1,4 @@ -runs/ +/runs/ eval.config.json # Ablation scratch / run artifacts diff --git a/packages/hdx-eval/src/runs/instrument.ts b/packages/hdx-eval/src/runs/instrument.ts new file mode 100644 index 0000000000..f061471d39 --- /dev/null +++ b/packages/hdx-eval/src/runs/instrument.ts @@ -0,0 +1,324 @@ +import type { ClickHouseClient } from '@clickhouse/client'; +import { existsSync, readdirSync, readFileSync, writeFileSync } from 'fs'; +import { join } from 'path'; + +import { createEvalClient, defaultClickHouseUrl } from '../clickhouse/client'; +import { scenarioTables } from '../clickhouse/schema'; +import type { RunRecord } from '../harness/types'; +import { SCENARIO_NAMES } from '../scenarios'; +import { runsRoot } from './path'; + +const QUERY_LOG_BUFFER_MS = 5_000; + +export type ServerQuery = { + eventTime: string; + queryDurationMs: number; + readRows: number; + resultRows: number; + memoryUsageBytes: number; + queryPreview: string; +}; + +export type ToolCallTiming = { + index: number; + name: string; + wallStartTs: string; + wallEndTs: string; + wallMs: number; + serverQueries: ServerQuery[]; + serverMs: number; + inferenceMs: number; +}; + +export type TimingRecord = { + schemaVersion: 1; + runId: string; + scenario: string; + mcp: 'hyperdx' | 'clickhouse'; + toolCalls: ToolCallTiming[]; + totalWallMs: number; + totalServerMs: number; + totalInferenceMs: number; + unmatchedQueriesCount: number; +}; + +type QueryLogRow = { + event_time_microseconds: string; + query_duration_ms: number | string; + read_rows: number | string; + result_rows: number | string; + memory_usage: number | string; + query: string; +}; + +export type InstrumentOptions = { + clickhouseUrl?: string; + username?: string; + password?: string; +}; + +/** + * Walk each run's `messages` array to derive per-tool-call wall-clock + * windows from user-message timestamps (which are present) plus + * tool_use ids in the preceding assistant message. + */ +function extractToolCallWindows(record: RunRecord): Array<{ + index: number; + name: string; + startTs: string; + endTs: string; +}> { + const out: Array<{ + index: number; + name: string; + startTs: string; + endTs: string; + }> = []; + let prevTs: string | null = record.startedAt; + let lastToolUseName: string | null = null; + let idx = 0; + for (const ev of record.messages as Array>) { + const t = ev?.type; + if (t === 'system' && ev.subtype === 'init') { + const stamp = ev.timestamp; + if (typeof stamp === 'string') prevTs = stamp; + } else if (t === 'assistant') { + const msg = (ev.message as Record) ?? {}; + const content = Array.isArray(msg.content) + ? (msg.content as unknown[]) + : []; + for (const block of content) { + if ( + block && + typeof block === 'object' && + (block as Record).type === 'tool_use' + ) { + const name = (block as Record).name; + if (typeof name === 'string') lastToolUseName = name; + } + } + } else if (t === 'user') { + const ts = ev.timestamp; + if (typeof ts !== 'string') continue; + const msg = (ev.message as Record) ?? {}; + const content = Array.isArray(msg.content) + ? (msg.content as unknown[]) + : []; + const hasToolResult = content.some( + b => + b && + typeof b === 'object' && + (b as Record).type === 'tool_result', + ); + if (!hasToolResult) continue; + out.push({ + index: idx++, + name: lastToolUseName ?? 'unknown', + startTs: prevTs ?? record.startedAt, + endTs: ts, + }); + prevTs = ts; + } + } + return out; +} + +async function fetchQueryLog( + client: ClickHouseClient, + startTs: string, + endTs: string, + tablePatterns: string[], +): Promise { + const startMs = Date.parse(startTs) - QUERY_LOG_BUFFER_MS; + const endMs = Date.parse(endTs) + QUERY_LOG_BUFFER_MS; + const startStr = new Date(startMs) + .toISOString() + .slice(0, 19) + .replace('T', ' '); + const endStr = new Date(endMs).toISOString().slice(0, 19).replace('T', ' '); + // Match queries that touch any of the given table patterns. + const orClauses = tablePatterns + .map(p => `query LIKE '%${p.replace(/'/g, "''")}%'`) + .join(' OR '); + // Restrict to SELECT-shaped queries — agents don't issue INSERTs, but the + // re-seed step right before the run does, and we don't want those in the + // per-call attribution. + const sql = ` + SELECT + toString(event_time_microseconds) AS event_time_microseconds, + query_duration_ms, + read_rows, + result_rows, + memory_usage, + substring(query, 1, 240) AS query + FROM system.query_log + WHERE type = 'QueryFinish' + AND event_time >= toDateTime('${startStr}') + AND event_time <= toDateTime('${endStr}') + AND query_kind = 'Select' + AND (${orClauses}) + ORDER BY event_time_microseconds + SETTINGS max_execution_time = 30 + `; + const rs = await client.query({ query: sql, format: 'JSONEachRow' }); + return (await rs.json()) as QueryLogRow[]; +} + +function matchQueriesToCalls( + windows: ReturnType, + queries: QueryLogRow[], +): { calls: ToolCallTiming[]; unmatched: number } { + const callTimings: ToolCallTiming[] = windows.map(w => ({ + index: w.index, + name: w.name, + wallStartTs: w.startTs, + wallEndTs: w.endTs, + wallMs: Math.max(0, Date.parse(w.endTs) - Date.parse(w.startTs)), + serverQueries: [], + serverMs: 0, + inferenceMs: 0, + })); + + let unmatched = 0; + for (const q of queries) { + // ClickHouse returns toString(DateTime64(6)) as 'YYYY-MM-DD HH:MM:SS.uuuuuu' + // (UTC, space-separated). Convert to ISO so Date.parse() accepts it. + const isoForm = q.event_time_microseconds.replace(' ', 'T') + 'Z'; + const ts = Date.parse(isoForm); + if (!Number.isFinite(ts)) { + unmatched++; + continue; + } + // Walk calls in reverse: prefer the LATEST call whose start <= ts. This + // assigns queries to the call that issued them even when there's a small + // network/processing lag between query event_time and the user-message + // timestamp that closes the window. + let owner: ToolCallTiming | undefined; + for (let i = callTimings.length - 1; i >= 0; i--) { + const c = callTimings[i]; + const start = Date.parse(c.wallStartTs); + const end = Date.parse(c.wallEndTs); + if (ts >= start && ts <= end + QUERY_LOG_BUFFER_MS) { + owner = c; + break; + } + } + if (!owner) { + unmatched++; + continue; + } + const durMs = Number(q.query_duration_ms); + owner.serverQueries.push({ + eventTime: q.event_time_microseconds, + queryDurationMs: Number.isFinite(durMs) ? durMs : 0, + readRows: Number(q.read_rows), + resultRows: Number(q.result_rows), + memoryUsageBytes: Number(q.memory_usage), + queryPreview: q.query, + }); + owner.serverMs += Number.isFinite(durMs) ? durMs : 0; + } + + for (const c of callTimings) { + c.inferenceMs = Math.max(0, c.wallMs - c.serverMs); + } + return { calls: callTimings, unmatched }; +} + +export async function instrumentRun(args: { + runPath: string; + client: ClickHouseClient; +}): Promise { + const record = JSON.parse(readFileSync(args.runPath, 'utf8')) as RunRecord; + const windows = extractToolCallWindows(record); + + // Match query_log on either the eval-specific tables OR system.tables + // calls (mcp-clickhouse does list_tables internally). + const tables = scenarioTables(record.scenario); + const tablePatterns = [tables.traces, tables.logs, 'system.tables']; + + const queries = await fetchQueryLog( + args.client, + record.startedAt, + record.endedAt, + tablePatterns, + ); + const matched = matchQueriesToCalls(windows, queries); + + const totalWall = matched.calls.reduce((s, c) => s + c.wallMs, 0); + const totalServer = matched.calls.reduce((s, c) => s + c.serverMs, 0); + + const timing: TimingRecord = { + schemaVersion: 1, + runId: record.runId, + scenario: record.scenario, + mcp: record.mcp, + toolCalls: matched.calls, + totalWallMs: totalWall, + totalServerMs: totalServer, + totalInferenceMs: Math.max(0, totalWall - totalServer), + unmatchedQueriesCount: matched.unmatched, + }; + const sidecarPath = args.runPath.replace(/\.json$/, '.timing.json'); + writeFileSync(sidecarPath, JSON.stringify(timing, null, 2) + '\n', 'utf8'); + return timing; +} + +export async function instrumentBatch( + batchDir: string, + opts: InstrumentOptions = {}, +): Promise { + const resolved = existsSync(batchDir) ? batchDir : join(runsRoot(), batchDir); + if (!existsSync(resolved)) { + throw new Error(`Batch not found: ${resolved}`); + } + const client = createEvalClient({ + url: opts.clickhouseUrl ?? defaultClickHouseUrl(), + username: opts.username, + password: opts.password, + }); + try { + const out: TimingRecord[] = []; + for (const scenario of safeReaddir(resolved)) { + if (!SCENARIO_NAMES.includes(scenario)) continue; + const sceneDir = join(resolved, scenario); + for (const mcp of safeReaddir(sceneDir)) { + const mcpDir = join(sceneDir, mcp); + for (const file of safeReaddir(mcpDir)) { + if (!file.endsWith('.json')) continue; + if (file.endsWith('.grade.json') || file.endsWith('.timing.json')) { + continue; + } + const timing = await instrumentRun({ + runPath: join(mcpDir, file), + client, + }); + out.push(timing); + } + } + } + return out; + } finally { + await client.close(); + } +} + +function safeReaddir(path: string): string[] { + try { + return readdirSync(path); + } catch { + return []; + } +} + +export function summarizeTimingRecord(t: TimingRecord): string { + const wall = (t.totalWallMs / 1000).toFixed(1); + const server = (t.totalServerMs / 1000).toFixed(2); + const inference = (t.totalInferenceMs / 1000).toFixed(1); + const serverPct = + t.totalWallMs > 0 ? Math.round((t.totalServerMs / t.totalWallMs) * 100) : 0; + return ( + `${t.scenario}/${t.mcp}: wall=${wall}s server=${server}s (${serverPct}%) ` + + `inference=${inference}s calls=${t.toolCalls.length} unmatched=${t.unmatchedQueriesCount}` + ); +} diff --git a/packages/hdx-eval/src/runs/path.ts b/packages/hdx-eval/src/runs/path.ts new file mode 100644 index 0000000000..b69c48a8f6 --- /dev/null +++ b/packages/hdx-eval/src/runs/path.ts @@ -0,0 +1,26 @@ +import { resolve } from 'path'; + +import type { McpKind } from '../harness/types'; + +export function runsRoot(): string { + return resolve(__dirname, '..', '..', 'runs'); +} + +export function batchDirName(date: Date = new Date()): string { + return date.toISOString().replace(/[:.]/g, '-'); +} + +export function runFilePath(args: { + batchDir: string; + scenario: string; + mcp: McpKind; + runIndex: number; +}): string { + return resolve( + runsRoot(), + args.batchDir, + args.scenario, + args.mcp, + `${args.runIndex}.json`, + ); +} diff --git a/packages/hdx-eval/src/runs/store.ts b/packages/hdx-eval/src/runs/store.ts new file mode 100644 index 0000000000..15c2499c1e --- /dev/null +++ b/packages/hdx-eval/src/runs/store.ts @@ -0,0 +1,59 @@ +import { mkdirSync, readdirSync, readFileSync, writeFileSync } from 'fs'; +import { dirname, join } from 'path'; + +import type { RunRecord } from '../harness/types'; +import { runFilePath, runsRoot } from './path'; + +export function writeRun(args: { + record: RunRecord; + batchDir: string; +}): string { + const path = runFilePath({ + batchDir: args.batchDir, + scenario: args.record.scenario, + mcp: args.record.mcp, + runIndex: args.record.runIndex, + }); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, JSON.stringify(args.record, null, 2) + '\n', 'utf8'); + return path; +} + +export function readRun(path: string): RunRecord { + return JSON.parse(readFileSync(path, 'utf8')) as RunRecord; +} + +export function listBatches(): string[] { + try { + return readdirSync(runsRoot()).sort(); + } catch { + return []; + } +} + +export function listRunsInBatch(batchDir: string): string[] { + const root = join(runsRoot(), batchDir); + const out: string[] = []; + try { + for (const scenario of readdirSync(root)) { + const sceneDir = join(root, scenario); + try { + for (const mcp of readdirSync(sceneDir)) { + const mcpDir = join(sceneDir, mcp); + try { + for (const file of readdirSync(mcpDir)) { + if (file.endsWith('.json')) out.push(join(mcpDir, file)); + } + } catch { + // skip + } + } + } catch { + // skip + } + } + } catch { + // none + } + return out.sort(); +} From a4df2b3c6d5c5bff00125fab9c8bb3d5792ebe85 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Mon, 1 Jun 2026 13:52:51 -0600 Subject: [PATCH 08/22] chore: remove ablation scripts, experiment artifacts, and notes Remove files that are dev-specific or experimental scratch work: - ablation/ directory (REPORT.md, manifest.tsv) - scripts/ablation.sh, ablation-report.ts - scripts/compare-prompt-variants.sh, fast-eval.sh - MCP_IMPROVEMENTS.md Also clean up README references and ablation .gitignore patterns. --- packages/hdx-eval/.gitignore | 9 - packages/hdx-eval/MCP_IMPROVEMENTS.md | 138 --------- packages/hdx-eval/README.md | 18 -- packages/hdx-eval/ablation/REPORT.md | 59 ---- packages/hdx-eval/ablation/manifest.tsv | 2 - packages/hdx-eval/scripts/ablation-report.ts | 266 ------------------ packages/hdx-eval/scripts/ablation.sh | 179 ------------ .../scripts/compare-prompt-variants.sh | 125 -------- packages/hdx-eval/scripts/fast-eval.sh | 125 -------- 9 files changed, 921 deletions(-) delete mode 100644 packages/hdx-eval/MCP_IMPROVEMENTS.md delete mode 100644 packages/hdx-eval/ablation/REPORT.md delete mode 100644 packages/hdx-eval/ablation/manifest.tsv delete mode 100644 packages/hdx-eval/scripts/ablation-report.ts delete mode 100755 packages/hdx-eval/scripts/ablation.sh delete mode 100755 packages/hdx-eval/scripts/compare-prompt-variants.sh delete mode 100755 packages/hdx-eval/scripts/fast-eval.sh diff --git a/packages/hdx-eval/.gitignore b/packages/hdx-eval/.gitignore index 8452b93777..423f3d8fde 100644 --- a/packages/hdx-eval/.gitignore +++ b/packages/hdx-eval/.gitignore @@ -1,11 +1,2 @@ /runs/ eval.config.json - -# Ablation scratch / run artifacts -ablation/_*.pgid -ablation/_*.log -ablation/autonomous-*.md -ablation/autonomous-loop*-*.md -ablation/MAIN_VS_*.md -ablation/PROMPT_VARIANT*.md -ablation/prompt-variant-*/ diff --git a/packages/hdx-eval/MCP_IMPROVEMENTS.md b/packages/hdx-eval/MCP_IMPROVEMENTS.md deleted file mode 100644 index 527a606153..0000000000 --- a/packages/hdx-eval/MCP_IMPROVEMENTS.md +++ /dev/null @@ -1,138 +0,0 @@ -# HyperDX MCP Server — Next Improvements - -Based on the n=1 rerun trajectories (`runs/2026-05-10T04-*`) after adding -`hyperdx_trace` and `hyperdx_log_patterns`, plus the schema audit (per-select -conditional `where`, derived-expression `groupBy`). - -## High leverage - -### 1. `hyperdx_segment_compare` — the missing primitive - -**Where it shows up:** the 25-call ClickHouse `max_turns` failure on -latency-spike. About 10 of those 25 calls were variations of "is the slowness -concentrated on `tenant.tier`? on `feature_flag`? on `http.route`? on -`backend`? on `service.version`?" — manual iteration through candidate segment -columns. HDX wins this scenario partly because it finds the segment by -accident, partly because it bails to `hyperdx_trace`. Neither MCP makes the -actual question first-class. - -**Shape:** - -```ts -hyperdx_segment_compare({ - source, metric, spanFilter, timeWindow, - segmentColumns?: string[] // omit → use mapAttributeLowCardinalityValues -}) -``` - -Returns each candidate segment ranked by metric spread (max/min ratio across -values) with a top-2 callout. `hyperdx_list_sources` already exposes -`mapAttributeLowCardinalityValues` — this tool just leverages that to -pre-compute the obvious comparisons. - -### 2. `hyperdx_anomaly_window` — when did the change happen - -ClickHouse burned 4 consecutive calls (~#7–#10 of the latency-spike run) doing -minute-bucketed p99 queries to find the inflection. HDX did the same in -calls #5 + #7. Both arms reach for this and both grind through it. - -**Shape:** - -```ts -hyperdx_anomaly_window({ source, metric, spanFilter, timeWindow, granularity?: 'auto' }) -``` - -Returns the bucketed timeseries plus a detected onset timestamp and magnitude. -Replaces the "GROUP BY toStartOfMinute, eyeball the result" pattern. - -### 3. `hyperdx_log_patterns` should support `compareWindow` - -**Where it shows up:** noisy-signals is solved with one call now, but for -incident response the natural follow-up is "which patterns *spiked*?" The -agent currently has to call `hyperdx_log_patterns` twice and diff manually. - -**Shape addition:** - -```ts -hyperdx_log_patterns({ - ..., - compareWindow?: { startTime, endTime } -}) -``` - -Returns each cluster with `count_now`, `count_baseline`, `delta_pct`. Cheap to -implement (run the sample twice). - -## Medium leverage - -### 4. `hyperdx_trace` should auto-include correlated logs - -The HDX agent's call #11 walked a trace and got 16 spans. To answer "what error -message" it would normally need a second log query joining on `TraceId`. Trace -sources have a configured `logSourceId` already (`source.logSourceId` in the -schema). The trace tool can pull correlated log lines and inline them at the -right span depth. - -### 5. Single combined "first-look" tool - -```ts -hyperdx_overview({ - source, focus: 'service' | 'span' | 'pattern', value, timeWindow -}) -``` - -Returns: - -- Timeseries of the focus's primary metric -- Top 5 error templates -- Top 3 differentiating segments -- Top 5 slowest `TraceId`s - -**One call** that replaces the agent's ~5-call exploration phase. This is the -biggest tool-call savings — both agents spend 4–6 calls warming up. - -### 6. Default a `limit` on builder `table` queries - -Looking at HDX's `groupBy: "SpanAttributes['error.type'], -SpanAttributes['decline.reason'], ServiceName"` — that returns every -combination, which can be hundreds of rows. None of the runs hit this in a bad -way, but it's a footgun. Default to `limit: 50` with `orderBy` first column -desc unless explicitly set, and surface "result truncated" in the response. - -## Description / discoverability - -### 7. ToolSearch doesn't pre-load `hyperdx_trace` / `hyperdx_log_patterns` - -In the error-root-cause run, the HDX agent did -`ToolSearch select:list_sources,query` first (call #1), then **a second** -`ToolSearch select:hyperdx_trace` later (call #10). That's a wasted turn. -**The fix isn't on the MCP server** — it's on the harness side or in the -system prompt. But: bundling the description so an agent loading -`hyperdx_query` *also* sees a pointer to `hyperdx_trace` / `hyperdx_log_patterns` -mentioned in the *first* response would help. Right now the cross-references -are in the description only. - -### 8. `pickBy` description for `hyperdx_trace` slightly off - -Doc says "earliest trace whose root span has `STATUS_CODE_ERROR`" but the -implementation actually filters by *any* span with that status (since the -picker groups by `TraceId`). Either tighten the implementation or fix the doc. - -## What to skip - -- **Don't add a `hyperdx_explain` ("what changed at time T") tool** — too - generic, hard to make accurate, and `segment_compare` + `anomaly_window` - together cover most of the value. -- **Don't split `hyperdx_query` into per-displayType tools** — same reasoning - as before; one entry point is good. -- **Don't add an `attribute_keys` tool** — `list_sources` already returns - `mapAttributeKeys` and `mapAttributeLowCardinalityValues`, which is exactly - the right shape. - -## Honest caveat on the n=1 result - -The latency-spike flip (HDX 43% → 82%, CH 60% → 26%) looks dramatic but n=1 -is high variance and CH hitting `max_turns` is partly bad luck. The real test -is whether at n≥3 the HDX advantage holds *without* CH happening to time out. -Worth running n=3 on latency-spike specifically to confirm before claiming the -new tools deliver that magnitude of improvement. diff --git a/packages/hdx-eval/README.md b/packages/hdx-eval/README.md index 96dc150740..a99535e7b2 100644 --- a/packages/hdx-eval/README.md +++ b/packages/hdx-eval/README.md @@ -138,24 +138,6 @@ Combined score = `0.4 * programmatic + 0.6 * judge - toolErrorPenalty` - **Tool error penalty** — up to 20% deducted for high tool-call error rates. Prevents agents from papering over correctness with volume. -## Bulk Eval - -`scripts/fast-eval.sh` seeds all scenarios at a shared anchor time in -parallel, then runs them concurrently. Reads the API key from -`~/.config/hdx-eval-anthropic-key`. - -```bash -# Defaults: 4 scenarios, 2 runs/cell, concurrency 2 -./packages/hdx-eval/scripts/fast-eval.sh - -# Override -SCENARIOS="error-root-cause latency-spike" RUNS=3 \ - ./packages/hdx-eval/scripts/fast-eval.sh -``` - -There are also scripts for ablation studies (`scripts/ablation.sh`) and -prompt variant comparison (`scripts/compare-prompt-variants.sh`). - ## Determinism All randomness flows through a `mulberry32(seed)` PRNG. Same `(seed, now)` diff --git a/packages/hdx-eval/ablation/REPORT.md b/packages/hdx-eval/ablation/REPORT.md deleted file mode 100644 index 1b51689b8e..0000000000 --- a/packages/hdx-eval/ablation/REPORT.md +++ /dev/null @@ -1,59 +0,0 @@ -# Ablation Report — Split Tools / TOON Output - -Generated: 2026-05-10T11:32:58.357Z - -Each cell: HDX-only, n=see below; metric is the HyperDX MCP arm. - -## Top-line — Combined Score - -| Scenario | Baseline | Split | TOON | Both | Δ split | Δ toon | Δ both | -|---|---:|---:|---:|---:|---:|---:|---:| -| error-root-cause | 97% (n=2) | 98% (n=2) | 100% (n=2) | 97% (n=2) | +1pp | +3pp | +0pp | -| latency-spike | 38% (n=2) | 58% (n=2) | 33% (n=2) | 62% (n=2) | +21pp | -5pp | +24pp | -| noisy-signals | 82% (n=2) | 90% (n=1) | 82% (n=2) | 78% (n=2) | +7pp | -1pp | -5pp | - -## error-root-cause - -| Metric | Baseline | Split | TOON | Both | -|---|---:|---:|---:|---:| -| Combined score | 97% | 98% | 100% | 97% | -| Programmatic score | 100% | 100% | 100% | 100% | -| Judge weighted | 95% | 96% | 100% | 95% | -| Tool calls (mean) | 18.0 | 18.5 | 26.0 | 25.0 | -| Output tokens (mean) | 8943 | 7748 | 6879 | 9177 | -| Wall clock s (mean) | 158.8 | 141.6 | 152.2 | 209.3 | -| N runs | 2 | 2 | 2 | 2 | - -## latency-spike - -| Metric | Baseline | Split | TOON | Both | -|---|---:|---:|---:|---:| -| Combined score | 38% | 58% | 33% | 62% | -| Programmatic score | 70% | 80% | 63% | 77% | -| Judge weighted | 16% | 44% | 13% | 53% | -| Tool calls (mean) | 14.0 | 17.0 | 16.0 | 14.5 | -| Output tokens (mean) | 8985 | 8704 | 7617 | 4861 | -| Wall clock s (mean) | 165.2 | 163.0 | 152.3 | 125.7 | -| N runs | 2 | 2 | 2 | 2 | - -## noisy-signals - -| Metric | Baseline | Split | TOON | Both | -|---|---:|---:|---:|---:| -| Combined score | 82% | 90% | 82% | 78% | -| Programmatic score | 97% | 100% | 92% | 89% | -| Judge weighted | 73% | 83% | 75% | 70% | -| Tool calls (mean) | 8.5 | 9.0 | 14.5 | 16.0 | -| Output tokens (mean) | 5338 | 4671 | 6177 | 3621 | -| Wall clock s (mean) | 105.6 | 89.6 | 147.6 | 149.0 | -| N runs | 2 | 1 | 2 | 2 | - -## Attribution - -Per-scenario combined-score deltas vs baseline (positive = better): - -- **error-root-cause**: split=1pp, toon=3pp, both=0pp -- **latency-spike**: split=21pp, toon=-5pp, both=24pp -- **noisy-signals**: split=7pp, toon=-1pp, both=-5pp - -*N≤2 per cell — single-digit deltas are not significant. Treat any signal of magnitude < 10pp as noise.* diff --git a/packages/hdx-eval/ablation/manifest.tsv b/packages/hdx-eval/ablation/manifest.tsv deleted file mode 100644 index 11073b0f3d..0000000000 --- a/packages/hdx-eval/ablation/manifest.tsv +++ /dev/null @@ -1,2 +0,0 @@ -variant scenario batchDir startedAt -split-fix error-root-cause runs/2026-05-11T04-23-56-949Z 2026-05-11T04:10:34Z diff --git a/packages/hdx-eval/scripts/ablation-report.ts b/packages/hdx-eval/scripts/ablation-report.ts deleted file mode 100644 index c74ca6055b..0000000000 --- a/packages/hdx-eval/scripts/ablation-report.ts +++ /dev/null @@ -1,266 +0,0 @@ -/** - * Read packages/hdx-eval/ablation/manifest.tsv (written by ablation.sh), - * grade every batch listed there, and emit a comparison markdown that - * attributes performance changes to each variant (split / toon / both) - * relative to the baseline. - * - * Usage: - * yarn workspace @hyperdx/hdx-eval dev scripts/ablation-report.ts - */ - -import { existsSync, readFileSync, writeFileSync } from 'fs'; -import { resolve } from 'path'; - -import { gradeBatch, resolveBatchDir } from '../src/grading/grade'; -import type { BatchSummary } from '../src/reports/aggregate'; -import { writeBatchSummary } from '../src/reports/store'; - -type ManifestRow = { - variant: string; - scenario: string; - batchDir: string; - startedAt: string; -}; - -const PKG_ROOT = resolve(__dirname, '..'); -const MANIFEST_PATH = resolve(PKG_ROOT, 'ablation', 'manifest.tsv'); -const REPORT_PATH = resolve(PKG_ROOT, 'ablation', 'REPORT.md'); - -const VARIANT_ORDER = ['baseline', 'split', 'toon', 'both']; -const VARIANT_LABEL: Record = { - baseline: 'Baseline', - split: 'Split', - toon: 'TOON', - both: 'Both', -}; - -function readManifest(): ManifestRow[] { - const raw = readFileSync(MANIFEST_PATH, 'utf8'); - const lines = raw.trim().split('\n'); - const out: ManifestRow[] = []; - for (const line of lines.slice(1)) { - const parts = line.split('\t'); - if (parts.length < 4) continue; - if (!parts[2] || parts[2] === '') continue; // missing batch dir - out.push({ - variant: parts[0], - scenario: parts[1], - batchDir: parts[2], - startedAt: parts[3], - }); - } - return out; -} - -async function main() { - if (!existsSync(MANIFEST_PATH)) { - console.error(`Manifest not found: ${MANIFEST_PATH}`); - process.exit(1); - } - - const rows = readManifest(); - console.log(`Manifest has ${rows.length} batch entries.`); - - // ── Grade every batch (idempotent — skips runs that already have grade.json) ── - for (const row of rows) { - const dir = resolveBatchDir(row.batchDir); - if (!existsSync(dir)) { - console.warn( - ` skip ${row.variant}/${row.scenario}: batch dir missing (${dir})`, - ); - continue; - } - console.log( - ` grading ${row.variant}/${row.scenario} (${row.batchDir})...`, - ); - try { - await gradeBatch(dir, { - judgeModel: 'claude-opus-4-7', - rerunJudge: false, - skipJudge: false, - }); - } catch (e) { - console.error( - ` grade failed: ${e instanceof Error ? e.message : String(e)}`, - ); - } - } - - // ── Render each batch's _summary.{md,json} ── - for (const row of rows) { - const dir = resolveBatchDir(row.batchDir); - if (!existsSync(dir)) continue; - try { - writeBatchSummary(dir, resolve(dir, '_summary.md')); - } catch (e) { - console.error(` summary failed for ${row.batchDir}: ${e}`); - } - } - - // ── Load per-cell summaries (HDX side only — split/toon/both don't change CH) ── - type Cell = { - variant: string; - scenario: string; - n: number; - combined: number; - programmatic: number; - judge: number; - toolCalls: number; - outputTokens: number; - wallClockS: number; - perCheck: Record; - }; - const cells: Cell[] = []; - for (const row of rows) { - const dir = resolveBatchDir(row.batchDir); - const summaryPath = resolve(dir, '_summary.json'); - if (!existsSync(summaryPath)) { - console.warn(` no _summary.json in ${row.batchDir}`); - continue; - } - const summary = JSON.parse( - readFileSync(summaryPath, 'utf8'), - ) as BatchSummary; - for (const sc of summary.scenarios) { - const hdx = sc.cells.hyperdx; - if (!hdx) continue; - cells.push({ - variant: row.variant, - scenario: sc.scenario, - n: hdx.n, - combined: hdx.combinedScore.mean, - programmatic: hdx.programmatic.mean, - judge: hdx.judge.weightedMean, - toolCalls: hdx.toolCalls.mean, - outputTokens: hdx.tokens.output, - wallClockS: hdx.durationMs.mean / 1000, - perCheck: hdx.programmatic.perCheck, - }); - } - } - - // ── Build report ── - const scenarios = Array.from(new Set(cells.map(c => c.scenario))).sort(); - const lines: string[] = []; - lines.push('# Ablation Report — Split Tools / TOON Output'); - lines.push(''); - lines.push(`Generated: ${new Date().toISOString()}`); - lines.push(''); - lines.push( - `Each cell: HDX-only, n=${rows[0]?.batchDir ? 'see below' : '?'}; metric is the HyperDX MCP arm.`, - ); - lines.push(''); - - // Top-line: combined score per (variant, scenario) - lines.push('## Top-line — Combined Score'); - lines.push(''); - lines.push( - '| Scenario | ' + - VARIANT_ORDER.map(v => VARIANT_LABEL[v]).join(' | ') + - ' | Δ split | Δ toon | Δ both |', - ); - lines.push( - '|---|' + VARIANT_ORDER.map(() => '---:').join('|') + '|---:|---:|---:|', - ); - for (const sc of scenarios) { - const byVariant: Record = {}; - for (const c of cells) { - if (c.scenario === sc) byVariant[c.variant] = c; - } - const baseline = byVariant.baseline; - const cellStr = (c?: Cell) => - c ? `${(c.combined * 100).toFixed(0)}% (n=${c.n})` : '—'; - const deltaStr = (c?: Cell) => { - if (!c || !baseline) return '—'; - const d = (c.combined - baseline.combined) * 100; - const sign = d >= 0 ? '+' : ''; - return `${sign}${d.toFixed(0)}pp`; - }; - lines.push( - '| ' + - sc + - ' | ' + - VARIANT_ORDER.map(v => cellStr(byVariant[v])).join(' | ') + - ' | ' + - deltaStr(byVariant.split) + - ' | ' + - deltaStr(byVariant.toon) + - ' | ' + - deltaStr(byVariant.both) + - ' |', - ); - } - lines.push(''); - - // Per-scenario detail tables - for (const sc of scenarios) { - lines.push(`## ${sc}`); - lines.push(''); - lines.push('| Metric | Baseline | Split | TOON | Both |'); - lines.push('|---|---:|---:|---:|---:|'); - const byVariant: Record = {}; - for (const c of cells) { - if (c.scenario === sc) byVariant[c.variant] = c; - } - const row = (label: string, fn: (c: Cell) => string): string => - '| ' + - label + - ' | ' + - VARIANT_ORDER.map(v => { - const c = byVariant[v]; - return c ? fn(c) : '—'; - }).join(' | ') + - ' |'; - lines.push(row('Combined score', c => `${(c.combined * 100).toFixed(0)}%`)); - lines.push( - row('Programmatic score', c => `${(c.programmatic * 100).toFixed(0)}%`), - ); - lines.push(row('Judge weighted', c => `${(c.judge * 100).toFixed(0)}%`)); - lines.push(row('Tool calls (mean)', c => c.toolCalls.toFixed(1))); - lines.push( - row('Output tokens (mean)', c => Math.round(c.outputTokens).toString()), - ); - lines.push(row('Wall clock s (mean)', c => c.wallClockS.toFixed(1))); - lines.push(row('N runs', c => String(c.n))); - lines.push(''); - } - - // Summary commentary - lines.push('## Attribution'); - lines.push(''); - lines.push( - 'Per-scenario combined-score deltas vs baseline (positive = better):', - ); - lines.push(''); - for (const sc of scenarios) { - const byVariant: Record = {}; - for (const c of cells) { - if (c.scenario === sc) byVariant[c.variant] = c; - } - const b = byVariant.baseline; - const s = byVariant.split; - const t = byVariant.toon; - const both = byVariant.both; - if (!b) continue; - const delta = (c?: Cell) => - c ? `${((c.combined - b.combined) * 100).toFixed(0)}pp` : '—'; - lines.push( - `- **${sc}**: split=${delta(s)}, toon=${delta(t)}, both=${delta(both)}`, - ); - } - lines.push(''); - lines.push( - '*N≤2 per cell — single-digit deltas are not significant. Treat any signal of magnitude < 10pp as noise.*', - ); - - const out = lines.join('\n') + '\n'; - writeFileSync(REPORT_PATH, out); - console.log(`\nWrote ${REPORT_PATH}`); - console.log('\n--- preview ---\n'); - console.log(out.slice(0, 4000)); -} - -main().catch(err => { - console.error(err); - process.exit(1); -}); diff --git a/packages/hdx-eval/scripts/ablation.sh b/packages/hdx-eval/scripts/ablation.sh deleted file mode 100755 index 4c7d155a5a..0000000000 --- a/packages/hdx-eval/scripts/ablation.sh +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/env bash -# Ablation runner for the MCP query-tool experiments. -# -# Variants: -# baseline — current MCP (hyperdx_query, JSON) -# split — HDX_MCP_SPLIT_QUERY=1 (hyperdx_timeseries / table / search / sql) -# toon — HDX_MCP_OUTPUT=toon (TOON output, ClickHouse metadata stripped) -# both — split + toon -# -# Per variant: kill the running API process, restart yarn app:dev with the -# variant env vars, wait for /health, run hdx-eval (HDX MCP only) on every -# scenario at n=$RUNS, and stash the resulting batch dirs under -# ablation//. -# -# Reads ANTHROPIC_API_KEY from ~/.config/hdx-eval-anthropic-key. - -set -uo pipefail - -REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" -cd "$REPO_ROOT" - -VARIANTS="${VARIANTS:-baseline split toon both}" -SCENARIOS="${SCENARIOS:-error-root-cause latency-spike noisy-signals}" -RUNS="${RUNS:-2}" - -OUT_DIR="$REPO_ROOT/packages/hdx-eval/ablation" -mkdir -p "$OUT_DIR" -MANIFEST="$OUT_DIR/manifest.tsv" -echo -e "variant\tscenario\tbatchDir\tstartedAt" > "$MANIFEST" - -source ~/.nvm/nvm.sh > /dev/null 2>&1 || true -nvm use 22.21.1 > /dev/null 2>&1 || true -export PATH="/usr/local/bin:$HOME/.local/bin:$PATH" - -if [[ ! -f "$HOME/.config/hdx-eval-anthropic-key" ]]; then - echo "Missing ~/.config/hdx-eval-anthropic-key — aborting" >&2 - exit 1 -fi -export ANTHROPIC_API_KEY=$(cat "$HOME/.config/hdx-eval-anthropic-key") - -# ─── dev-env (port slot, etc) ────────────────────────────────────────────── -source ./scripts/dev-env.sh > /dev/null - -# ─── ensure docker stack is up (idempotent) ──────────────────────────────── -# We bypass `yarn dev` because that script runs `docker compose down` when -# yarn app:dev exits, which would tear down ClickHouse between variants. -# Bring docker up once here; leave it running across variants. -ensure_docker() { - echo "[$(date +%H:%M:%S)] Ensuring docker stack is up (project=$HDX_DEV_PROJECT)..." - sg docker -c "docker compose -p '$HDX_DEV_PROJECT' -f docker-compose.dev.yml up -d" \ - > "$OUT_DIR/_docker.log" 2>&1 -} -ensure_docker - -# ─── env-vars per variant ────────────────────────────────────────────────── -variant_env() { - case "$1" in - baseline) echo "" ;; - split) echo "HDX_MCP_SPLIT_QUERY=1" ;; - toon) echo "HDX_MCP_OUTPUT=toon" ;; - both) echo "HDX_MCP_SPLIT_QUERY=1 HDX_MCP_OUTPUT=toon" ;; - *) echo "unknown variant: $1" >&2; exit 1 ;; - esac -} - -# ─── kill the API process group ──────────────────────────────────────────── -# We start the API in its own process group via setsid so we can kill the -# whole tree (yarn → nodemon → ts-node) with `kill -- -`. Falling back -# to pattern-pkill in case the pgid isn't tracked (first invocation). -API_PGID_FILE="$OUT_DIR/_api.pgid" - -kill_api() { - echo "[$(date +%H:%M:%S)] Killing existing API processes..." - if [[ -f "$API_PGID_FILE" ]]; then - local pgid - pgid="$(cat "$API_PGID_FILE")" - if [[ -n "$pgid" ]] && kill -0 "-$pgid" 2>/dev/null; then - kill -TERM -- "-$pgid" 2>/dev/null || true - sleep 3 - kill -KILL -- "-$pgid" 2>/dev/null || true - fi - rm -f "$API_PGID_FILE" - fi - # Best-effort cleanup of anything that survived. - pkill -9 -f 'nodemon.*src/index' >/dev/null 2>&1 || true - pkill -9 -f 'ts-node.*src/index' >/dev/null 2>&1 || true - # Wait until the API port is actually free before returning. - for _ in {1..15}; do - if ! ss -tln 2>/dev/null | grep -q ":${HYPERDX_API_PORT}\b"; then - break - fi - sleep 1 - done - ensure_docker -} - -# ─── start ONLY the @hyperdx/api workspace dev process ───────────────────── -# We bypass yarn app:dev (which also brings up Next.js, the alerts-task, and -# a common-utils nodemon — none of which the eval needs). Just -# `yarn workspace @hyperdx/api dev` is enough to host /mcp. -# -# Started via `setsid` so the process gets its own group; pgid is recorded so -# kill_api can take down the whole tree at once. -start_api() { - local env_extra="$1" - local label="$2" - local logfile="$OUT_DIR/$label.api.log" - echo "[$(date +%H:%M:%S)] Starting API with: $env_extra" - setsid bash -c "source ~/.nvm/nvm.sh && nvm use 22.21.1 > /dev/null && source ./scripts/dev-env.sh && export $env_extra && yarn workspace @hyperdx/api dev" \ - > "$logfile" 2>&1 & - local pid=$! - echo "$pid" > "$API_PGID_FILE" # setsid makes pid == pgid - echo " pgid=$pid log=$logfile" -} - -wait_for_api() { - echo -n " Waiting for API" - local deadline=$(( $(date +%s) + 240 )) - while (( $(date +%s) < deadline )); do - if curl -fsS "http://localhost:${HYPERDX_API_PORT}/health" > /dev/null 2>&1; then - echo " — ready" - return 0 - fi - echo -n "." - sleep 3 - done - echo " — TIMEOUT" - return 1 -} - -# ─── per-scenario eval invocation ────────────────────────────────────────── -run_variant() { - local variant="$1" - local env_extra - env_extra="$(variant_env "$variant")" - local started - started="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - echo - echo "==========================================================" - echo " VARIANT: $variant ($started)" - echo "==========================================================" - - kill_api - start_api "$env_extra" "$variant" - if ! wait_for_api; then - echo "API failed to start for variant=$variant; skipping" >&2 - return 1 - fi - - for scenario in $SCENARIOS; do - echo "[$(date +%H:%M:%S)] $variant / $scenario — running n=$RUNS..." - local logfile="$OUT_DIR/$variant.$scenario.run.log" - yarn workspace @hyperdx/hdx-eval dev run "$scenario" \ - --runs "$RUNS" \ - --mcp hyperdx \ - > "$logfile" 2>&1 - local rc=$? - local batch_dir - batch_dir="$(grep -oE 'runs/[0-9TZ:.-]+' "$logfile" | head -1)" - echo -e "${variant}\t${scenario}\t${batch_dir}\t${started}" >> "$MANIFEST" - echo " -> rc=$rc batch=$batch_dir" - done -} - -# ─── main ────────────────────────────────────────────────────────────────── -echo "Repo: $REPO_ROOT" -echo "Variants: $VARIANTS" -echo "Scenarios: $SCENARIOS" -echo "Runs/cell: $RUNS" -echo "Manifest: $MANIFEST" - -for variant in $VARIANTS; do - run_variant "$variant" -done - -echo -echo "All variants done at $(date +%H:%M:%S). Manifest:" -cat "$MANIFEST" diff --git a/packages/hdx-eval/scripts/compare-prompt-variants.sh b/packages/hdx-eval/scripts/compare-prompt-variants.sh deleted file mode 100755 index e4b916aedd..0000000000 --- a/packages/hdx-eval/scripts/compare-prompt-variants.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/env bash -# compare-prompt-variants.sh — A/B the baseline vs hypothesis system prompt -# on a fixed set of scenarios at a shared anchor. -# -# Runs N×scenarios×{baseline,hypothesis} agents against the already-seeded -# tables (no reseed), then grades the resulting batches. -# -# Usage: -# ./packages/hdx-eval/scripts/compare-prompt-variants.sh -# [SCENARIOS="latency-spike segmented-regression error-root-cause"] -# [RUNS=3] [MCP=both] [PER_SCENARIO_CONCURRENCY=10] -# [ANCHOR_TIME=2026-05-11T04:49:00Z] -# [VARIANTS="baseline hypothesis"] -# -# Defaults tuned for fast iteration: n=3 per cell, 10 agents in parallel per -# (scenario, variant), grading parallelized across all batches. Override any -# of those env vars to revert to a slower / more thorough sweep. - -set -uo pipefail - -REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" -cd "$REPO_ROOT" - -source ~/.nvm/nvm.sh > /dev/null 2>&1 || true -nvm use 22.21.1 > /dev/null 2>&1 || true -export PATH="/usr/local/bin:$HOME/.local/bin:$PATH" - -if [[ ! -f "$HOME/.config/hdx-eval-anthropic-key" ]]; then - echo "Missing ~/.config/hdx-eval-anthropic-key — aborting" >&2 - exit 1 -fi -export ANTHROPIC_API_KEY=$(cat "$HOME/.config/hdx-eval-anthropic-key") - -source ./scripts/dev-env.sh > /dev/null - -SCENARIOS="${SCENARIOS:-latency-spike segmented-regression error-root-cause}" -RUNS="${RUNS:-3}" -MCP="${MCP:-both}" -MODEL="${MODEL:-claude-sonnet-4-6}" -PER_SCENARIO_CONCURRENCY="${PER_SCENARIO_CONCURRENCY:-5}" -ANCHOR_TIME="${ANCHOR_TIME:-2026-05-11T04:49:00Z}" -VARIANTS="${VARIANTS:-baseline hypothesis}" - -OUT_DIR="$REPO_ROOT/packages/hdx-eval/ablation" -mkdir -p "$OUT_DIR" -LOGDIR="$OUT_DIR/prompt-variant-$(date -u +%Y%m%dT%H%M%SZ)" -mkdir -p "$LOGDIR" - -echo "==========================================================" -echo " compare-prompt-variants" -echo " scenarios : $SCENARIOS" -echo " variants : $VARIANTS" -echo " runs per cell : $RUNS" -echo " mcp : $MCP" -echo " per-scenario conc : $PER_SCENARIO_CONCURRENCY" -echo " anchor time : $ANCHOR_TIME" -echo " log dir : $LOGDIR" -echo "==========================================================" - -run_one_variant() { - local variant="$1" - echo - echo "[$(date +%H:%M:%S)] === variant=$variant ===" - pids=() - for scenario in $SCENARIOS; do - ( - log="$LOGDIR/run.$variant.$scenario.log" - echo " [$(date +%H:%M:%S)] start $variant/$scenario → $log" - yarn workspace @hyperdx/hdx-eval dev run "$scenario" \ - --runs "$RUNS" --mcp "$MCP" --no-reseed \ - --anchor-time "$ANCHOR_TIME" \ - --concurrency "$PER_SCENARIO_CONCURRENCY" \ - --prompt-variant "$variant" \ - --model "$MODEL" \ - --ch-url "http://localhost:${HDX_DEV_CH_HTTP_PORT}" \ - > "$log" 2>&1 - rc=$? - batch=$(grep -oE 'runs/[^ ]+' "$log" | head -1) - echo " [$(date +%H:%M:%S)] done $variant/$scenario exit=$rc batch=$batch" - if [[ -n "$batch" ]]; then - echo "$variant $scenario $batch" >> "$LOGDIR/batches.tsv" - fi - ) & - pids+=($!) - done - for pid in "${pids[@]}"; do - wait "$pid" || true - done -} - -for v in $VARIANTS; do - run_one_variant "$v" -done - -echo -echo "[$(date +%H:%M:%S)] All runs done. Batches:" -cat "$LOGDIR/batches.tsv" 2>/dev/null || echo " (no batches recorded)" - -# ─── Grade each batch (in parallel) ──────────────────────────────────────── -# Judge calls are independent per (run, scenario, mcp); fan them out instead -# of grinding through batches serially. With one batch per (variant, scenario) -# this is up to 6 concurrent grade invocations on the default 3-scenario -# × 2-variant sweep. -echo -echo "[$(date +%H:%M:%S)] Grading all batches (parallel)..." -grade_pids=() -while read -r variant scenario batch; do - [[ -z "$batch" ]] && continue - ( - log="$LOGDIR/grade.$variant.$scenario.log" - echo " [$(date +%H:%M:%S)] start grade $variant/$scenario → $log" - yarn workspace @hyperdx/hdx-eval dev grade "$batch" > "$log" 2>&1 - rc=$? - echo " [$(date +%H:%M:%S)] done grade $variant/$scenario exit=$rc" - ) & - grade_pids+=($!) -done < "$LOGDIR/batches.tsv" -for pid in "${grade_pids[@]}"; do - wait "$pid" || true -done - -echo -echo "[$(date +%H:%M:%S)] ALL DONE" -echo "Logs: $LOGDIR" -echo "Batches: $LOGDIR/batches.tsv" diff --git a/packages/hdx-eval/scripts/fast-eval.sh b/packages/hdx-eval/scripts/fast-eval.sh deleted file mode 100755 index 9ca5d0592c..0000000000 --- a/packages/hdx-eval/scripts/fast-eval.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/env bash -# fast-eval.sh — bulk seed all scenarios at a shared anchor, then run the -# eval in parallel across scenarios. -# -# Speedup vs the legacy sequential per-scenario re-seed: -# - seed once, share across all (mcp, run) cells (-50 min) -# - run scenarios concurrently, agents in parallel (variable) -# -# Usage: -# ./packages/hdx-eval/scripts/fast-eval.sh -# [SCENARIOS="error-root-cause latency-spike ..."] -# [RUNS=2] [PER_SCENARIO_CONCURRENCY=2] [SCENARIO_CONCURRENCY=2] -# -# Reads ANTHROPIC_API_KEY from ~/.config/hdx-eval-anthropic-key. -# Assumes the dev API + ClickHouse are already up at the slot's ports. - -set -uo pipefail - -REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" -cd "$REPO_ROOT" - -source ~/.nvm/nvm.sh > /dev/null 2>&1 || true -nvm use 22.21.1 > /dev/null 2>&1 || true -export PATH="/usr/local/bin:$HOME/.local/bin:$PATH" - -if [[ ! -f "$HOME/.config/hdx-eval-anthropic-key" ]]; then - echo "Missing ~/.config/hdx-eval-anthropic-key — aborting" >&2 - exit 1 -fi -export ANTHROPIC_API_KEY=$(cat "$HOME/.config/hdx-eval-anthropic-key") - -source ./scripts/dev-env.sh > /dev/null - -SCENARIOS="${SCENARIOS:-error-root-cause latency-spike noisy-signals segmented-regression}" -RUNS="${RUNS:-2}" -PER_SCENARIO_CONCURRENCY="${PER_SCENARIO_CONCURRENCY:-2}" -SCENARIO_CONCURRENCY="${SCENARIO_CONCURRENCY:-2}" -# Anchor at "now" rounded to the nearest minute. All scenarios will seed at -# this same timestamp and the agent's system prompt will use it as "now". -ANCHOR_TIME="${ANCHOR_TIME:-$(date -u -d "$(date -u +%Y-%m-%dT%H:%M:00)" +%Y-%m-%dT%H:%M:%SZ)}" - -OUT_DIR="$REPO_ROOT/packages/hdx-eval/ablation" -mkdir -p "$OUT_DIR" -LOGDIR="$OUT_DIR/fast-eval-$(date -u +%Y%m%dT%H%M%SZ)" -mkdir -p "$LOGDIR" - -echo "==========================================================" -echo " fast-eval" -echo " scenarios : $SCENARIOS" -echo " runs per cell : $RUNS" -echo " per-scenario conc : $PER_SCENARIO_CONCURRENCY" -echo " scenario conc : $SCENARIO_CONCURRENCY" -echo " anchor time : $ANCHOR_TIME" -echo " log dir : $LOGDIR" -echo "==========================================================" - -# ─── Phase 1: seed all scenarios at the shared anchor ───────────────────── -echo -echo "[$(date +%H:%M:%S)] Phase 1: seeding all scenarios at $ANCHOR_TIME (parallel)" -seed_pids=() -for scenario in $SCENARIOS; do - ( - log="$LOGDIR/seed.$scenario.log" - echo " [$(date +%H:%M:%S)] start seed $scenario → $log" - yarn workspace @hyperdx/hdx-eval dev seed "$scenario" \ - --seed 42 --now "$ANCHOR_TIME" \ - --ch-url "http://localhost:${HDX_DEV_CH_HTTP_PORT}" \ - > "$log" 2>&1 - rc=$? - echo " [$(date +%H:%M:%S)] done seed $scenario exit=$rc" - ) & - seed_pids+=($!) -done -seed_failed=0 -for pid in "${seed_pids[@]}"; do - wait "$pid" || seed_failed=1 -done -if (( seed_failed )); then - echo "One or more seeds failed — see $LOGDIR/seed.*.log" >&2 - exit 1 -fi - -# ─── Phase 2: run scenarios in parallel, each cell pool also parallel ───── -echo -echo "[$(date +%H:%M:%S)] Phase 2: running scenarios (concurrency=$SCENARIO_CONCURRENCY × $PER_SCENARIO_CONCURRENCY agents/scenario)" - -# Pool of scenario-runners. Limit to SCENARIO_CONCURRENCY in flight. -running=0 -run_pids=() -run_scens=() -for scenario in $SCENARIOS; do - while (( running >= SCENARIO_CONCURRENCY )); do - # Wait for any one to finish. - for idx in "${!run_pids[@]}"; do - pid="${run_pids[$idx]}" - if ! kill -0 "$pid" 2>/dev/null; then - wait "$pid" || true - unset 'run_pids[idx]' - unset 'run_scens[idx]' - running=$((running - 1)) - fi - done - sleep 1 - done - ( - log="$LOGDIR/run.$scenario.log" - echo " [$(date +%H:%M:%S)] start run $scenario → $log" - yarn workspace @hyperdx/hdx-eval dev run "$scenario" \ - --runs "$RUNS" --mcp both --no-reseed \ - --anchor-time "$ANCHOR_TIME" \ - --concurrency "$PER_SCENARIO_CONCURRENCY" \ - --ch-url "http://localhost:${HDX_DEV_CH_HTTP_PORT}" \ - > "$log" 2>&1 - rc=$? - batch=$(grep -oE 'runs/[^ ]+' "$log" | head -1) - echo " [$(date +%H:%M:%S)] done run $scenario exit=$rc batch=$batch" - ) & - run_pids+=($!) - run_scens+=("$scenario") - running=$((running + 1)) -done -wait -echo -echo "[$(date +%H:%M:%S)] ALL DONE" -echo "Logs: $LOGDIR" From b74bf0761fada911d9f7a63270d8e72a60eb980e Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Mon, 1 Jun 2026 15:37:28 -0600 Subject: [PATCH 09/22] refactor: DRY scenario generators and compact rubric format (-772 lines) - Extract spreadTimestamp() and normalizeSeverityText() shared helpers - Collapse 14 identical phase functions into streamLogPhase() generic - Compact ground-truth programmatic checks to tuple format [id, weight, pattern, neg?] - Delete dead checkoutEventLog export, unexport internal logfmtBody/jsonEventBody --- packages/hdx-eval/src/generators/templates.ts | 97 +-- packages/hdx-eval/src/grading/rubric.ts | 73 +- .../scenarios/error-root-cause/generate.ts | 95 ++- .../error-root-cause/ground-truth.json | 137 ++-- .../src/scenarios/latency-spike/generate.ts | 11 +- .../scenarios/latency-spike/ground-truth.json | 104 +-- .../src/scenarios/noisy-signals/generate.ts | 721 +++++------------- .../scenarios/noisy-signals/ground-truth.json | 245 +++--- .../segmented-regression/generate.ts | 16 +- .../segmented-regression/ground-truth.json | 99 +-- .../service-health-check/generate.ts | 23 +- .../service-health-check/ground-truth.json | 173 ++--- 12 files changed, 669 insertions(+), 1125 deletions(-) diff --git a/packages/hdx-eval/src/generators/templates.ts b/packages/hdx-eval/src/generators/templates.ts index 8157658936..d12c2ff6e1 100644 --- a/packages/hdx-eval/src/generators/templates.ts +++ b/packages/hdx-eval/src/generators/templates.ts @@ -88,6 +88,38 @@ export function pickSeverityIn( return { text, number: SEVERITY_NUMBER_BY_TEXT[text] ?? 9 }; } +/** + * Normalize messy OTel severity text (case variants, aliases) into the + * canonical uppercase form used by ClickHouse schema columns. + */ +export function normalizeSeverityText( + raw: string, +): 'TRACE' | 'DEBUG' | 'INFO' | 'WARN' | 'ERROR' { + const u = raw.toUpperCase(); + if (u.startsWith('WARN')) return 'WARN'; + if (u.startsWith('ERR') || raw === 'fatal') return 'ERROR'; + if (u.startsWith('DEB')) return 'DEBUG'; + if (u === 'TRACE') return 'TRACE'; + return 'INFO'; +} + +/** + * Spread `count` timestamps linearly across a time window. + * `marginMs` is subtracted from the window end to avoid landing exactly + * on the boundary. + */ +export function spreadTimestamp( + i: number, + count: number, + startMs: number, + windowMs: number, + marginMs = 30_000, +): number { + return count > 1 + ? startMs + (i / (count - 1)) * (windowMs - marginMs) + : startMs; +} + // ─── Resource attributes (k8s/OTel-style) ──────────────────────────────── const NAMESPACES = [ @@ -270,7 +302,7 @@ export function envoyAccessLog(args: { rng: SeededRng; nowMs: number }): { } /** logfmt body (popular Go/Python convention). */ -export function logfmtBody(args: { +function logfmtBody(args: { rng: SeededRng; nowMs: number; level: string; @@ -366,74 +398,13 @@ export function catalogLog(args: { } /** JSON-event body, common for app-tier services. */ -export function jsonEventBody(args: { +function jsonEventBody(args: { rng: SeededRng; fields: Record; }): string { return JSON.stringify(args.fields); } -const EMAIL_PROVIDERS = ['ses', 'sendgrid', 'mailgun', 'postmark']; -const EMAIL_TEMPLATES = [ - 'order_confirmation', - 'shipping_update', - 'receipt', - 'password_reset', - 'welcome', -]; - -/** checkout/order JSON event with many app.* attrs. */ -export function checkoutEventLog(args: { rng: SeededRng; nowMs: number }): { - body: string; - attrs: Record; - level: string; -} { - const { rng, nowMs } = args; - const checkoutId = uuidv4(rng); - const orderId = uuidv4(rng); - const userId = uuidv4(rng); - const amountNanos = rng.intRange(1_000_000_000, 999_999_999_999); - const amountUsd = (amountNanos / 1e9).toFixed(2); - const itemCount = rng.intRange(1, 12); - const fields = { - 'app.checkout.id': checkoutId, - 'app.order.id': orderId, - 'app.order.user_id': userId, - 'app.payment.amount_nanos': amountNanos, - 'app.payment.amount_usd': amountUsd, - 'app.payment.currency': 'USD', - 'app.payment.method': rng.pick(['card', 'wallet', 'ach']), - 'app.payment.3ds_used': rng.next() < 0.4, - 'app.payment.processor': rng.pick(['stripe', 'braintree', 'adyen']), - 'app.email.provider': rng.pick(EMAIL_PROVIDERS), - 'app.email.template': rng.pick(EMAIL_TEMPLATES), - 'app.email.delivery_ms': rng.intRange(120, 4500), - 'app.email.recipient': `user${rng.intRange(1, 999_999)}@example.com`, - 'app.email.status': rng.weightedPick([ - { value: 'sent', weight: 92 }, - { value: 'queued', weight: 5 }, - { value: 'failed', weight: 3 }, - ]), - 'app.checkout.items_count': itemCount, - timestamp: new Date(nowMs).toISOString(), - }; - const level = fields['app.email.status'] === 'failed' ? 'warn' : 'info'; - return { - body: jsonEventBody({ rng, fields }), - attrs: { - 'event.name': 'checkout.complete', - 'app.checkout.id': checkoutId, - 'app.order.id': orderId, - 'code.function.name': 'placeOrder', - 'code.line.number': String(rng.intRange(40, 220)), - 'code.file.path': '/app/internal/checkout/handler.go', - 'log.iostream': 'stdout', - logtag: 'F', - }, - level, - }; -} - const PAGES = [ '/home', '/product', diff --git a/packages/hdx-eval/src/grading/rubric.ts b/packages/hdx-eval/src/grading/rubric.ts index 07f3f44c6e..33040e9f3b 100644 --- a/packages/hdx-eval/src/grading/rubric.ts +++ b/packages/hdx-eval/src/grading/rubric.ts @@ -1,5 +1,5 @@ import { getScenario } from '../scenarios'; -import type { Rubric } from './types'; +import type { ProgrammaticCheck, Rubric } from './types'; export function loadScenarioRubric(scenarioName: string): Rubric { const scenario = getScenario(scenarioName); @@ -16,6 +16,49 @@ export function getScenarioGroundTruth(scenarioName: string): unknown { return getScenario(scenarioName).groundTruth; } +/** + * Hydrate a compact tuple `[id, weight, pattern, negative?]` into a + * ProgrammaticCheck object. The `flags` field always defaults to `"i"`. + */ +function hydrateCheck(entry: unknown, scenarioName: string): ProgrammaticCheck { + if (Array.isArray(entry)) { + const [id, weight, pattern, negative] = entry; + if ( + typeof id !== 'string' || + typeof weight !== 'number' || + typeof pattern !== 'string' + ) { + throw new Error( + `rubric.programmatic for '${scenarioName}': tuple must be [string, number, string, boolean?]`, + ); + } + return { + id, + weight, + pattern, + flags: 'i', + ...(negative === true ? { negative: true } : {}), + }; + } + if (!entry || typeof entry !== 'object') { + throw new Error( + `rubric.programmatic for '${scenarioName}' has a non-object/non-array entry`, + ); + } + const check = entry as Record; + if (typeof check.id !== 'string' || typeof check.pattern !== 'string') { + throw new Error( + `rubric.programmatic for '${scenarioName}': each check needs string 'id' and 'pattern'`, + ); + } + if (typeof check.weight !== 'number' || check.weight <= 0) { + throw new Error( + `rubric.programmatic for '${scenarioName}': check '${check.id}' weight must be a positive number`, + ); + } + return entry as ProgrammaticCheck; +} + function validateRubric(raw: unknown, scenarioName: string): Rubric { if (!raw || typeof raw !== 'object') { throw new Error(`rubric for '${scenarioName}' must be an object`); @@ -26,29 +69,9 @@ function validateRubric(raw: unknown, scenarioName: string): Rubric { `rubric.programmatic for '${scenarioName}' must be an array`, ); } - for (const c of obj.programmatic as unknown[]) { - if (!c || typeof c !== 'object') { - throw new Error( - `rubric.programmatic for '${scenarioName}' has a non-object entry`, - ); - } - const check = c as Record; - if (typeof check.id !== 'string' || typeof check.pattern !== 'string') { - throw new Error( - `rubric.programmatic for '${scenarioName}': each check needs string 'id' and 'pattern'`, - ); - } - if (typeof check.weight !== 'number' || check.weight <= 0) { - throw new Error( - `rubric.programmatic for '${scenarioName}': check '${check.id}' weight must be a positive number`, - ); - } - if (check.negative !== undefined && typeof check.negative !== 'boolean') { - throw new Error( - `rubric.programmatic for '${scenarioName}': check '${check.id}' 'negative' must be a boolean`, - ); - } - } + const programmatic = (obj.programmatic as unknown[]).map(c => + hydrateCheck(c, scenarioName), + ); const judge = obj.judge as { criteria?: unknown } | undefined; if (!judge || !Array.isArray(judge.criteria) || judge.criteria.length === 0) { throw new Error( @@ -73,5 +96,5 @@ function validateRubric(raw: unknown, scenarioName: string): Rubric { ); } } - return raw as Rubric; + return { programmatic, judge: judge as Rubric['judge'] }; } diff --git a/packages/hdx-eval/src/scenarios/error-root-cause/generate.ts b/packages/hdx-eval/src/scenarios/error-root-cause/generate.ts index e692fb642e..dcc1a60b49 100644 --- a/packages/hdx-eval/src/scenarios/error-root-cause/generate.ts +++ b/packages/hdx-eval/src/scenarios/error-root-cause/generate.ts @@ -6,9 +6,11 @@ import { catalogLog, envoyAccessLog, heartbeatLog, + normalizeSeverityText, pageRenderLog, pickResource, pickSeverityIn, + spreadTimestamp, uuidv4, } from '../../generators/templates'; import { @@ -139,9 +141,13 @@ function generateTlsSpike(rng: Rng, nowMs: number): TraceRow[] { const traces: TraceRow[] = []; const startMs = nowMs - TLS_SPIKE_OFFSET_FROM_NOW_MS; for (let i = 0; i < TLS_SPIKE_COUNT; i++) { - const t = - startMs + - (i / Math.max(TLS_SPIKE_COUNT - 1, 1)) * (TLS_SPIKE_DURATION_MS - 1000); + const t = spreadTimestamp( + i, + TLS_SPIKE_COUNT, + startMs, + TLS_SPIKE_DURATION_MS, + 1000, + ); traces.push( makeSpan({ rng, @@ -170,10 +176,13 @@ function generateAuthRateLimitBlip(rng: Rng, nowMs: number): TraceRow[] { const traces: TraceRow[] = []; const startMs = nowMs - AUTH_RATE_LIMIT_OFFSET_FROM_NOW_MS; for (let i = 0; i < AUTH_RATE_LIMIT_COUNT; i++) { - const t = - startMs + - (i / Math.max(AUTH_RATE_LIMIT_COUNT - 1, 1)) * - (AUTH_RATE_LIMIT_DURATION_MS - 1000); + const t = spreadTimestamp( + i, + AUTH_RATE_LIMIT_COUNT, + startMs, + AUTH_RATE_LIMIT_DURATION_MS, + 1000, + ); traces.push( makeSpan({ rng, @@ -202,9 +211,7 @@ function generateSearchSlowQueries(rng: Rng, nowMs: number): TraceRow[] { const startMs = nowMs - HISTORY_WINDOW_MS; // Slow but successful — distractor for latency-style queries. for (let i = 0; i < SEARCH_SLOW_COUNT; i++) { - const t = - startMs + - (i / Math.max(SEARCH_SLOW_COUNT - 1, 1)) * (HISTORY_WINDOW_MS - 30_000); + const t = spreadTimestamp(i, SEARCH_SLOW_COUNT, startMs, HISTORY_WINDOW_MS); traces.push( makeSpan({ rng, @@ -233,10 +240,12 @@ function generateBackgroundPaymentErrors(rng: Rng, nowMs: number): TraceRow[] { const traces: TraceRow[] = []; const startMs = nowMs - HISTORY_WINDOW_MS; for (let i = 0; i < BACKGROUND_PAYMENT_ERROR_COUNT; i++) { - const t = - startMs + - (i / Math.max(BACKGROUND_PAYMENT_ERROR_COUNT - 1, 1)) * - (HISTORY_WINDOW_MS - 30_000); + const t = spreadTimestamp( + i, + BACKGROUND_PAYMENT_ERROR_COUNT, + startMs, + HISTORY_WINDOW_MS, + ); const reason = rng.pick(PAYMENT_DECLINE_REASONS); traces.push( makeSpan({ @@ -271,9 +280,13 @@ function generateSmtpDecoy( const logs: LogRow[] = []; const onsetMs = nowMs - ANOMALY_WINDOW_MS; for (let i = 0; i < SMTP_DECOY_COUNT; i++) { - const t = - onsetMs + - (i / Math.max(SMTP_DECOY_COUNT - 1, 1)) * (ANOMALY_WINDOW_MS - 5_000); + const t = spreadTimestamp( + i, + SMTP_DECOY_COUNT, + onsetMs, + ANOMALY_WINDOW_MS, + 5_000, + ); const traceId = newTraceId(rng); const spanId = newSpanId(rng); traces.push( @@ -322,10 +335,13 @@ function generateCdnOriginDecoy( const logs: LogRow[] = []; const onsetMs = nowMs - ANOMALY_WINDOW_MS; for (let i = 0; i < CDN_ORIGIN_DECOY_COUNT; i++) { - const t = - onsetMs + - (i / Math.max(CDN_ORIGIN_DECOY_COUNT - 1, 1)) * - (ANOMALY_WINDOW_MS - 5_000); + const t = spreadTimestamp( + i, + CDN_ORIGIN_DECOY_COUNT, + onsetMs, + ANOMALY_WINDOW_MS, + 5_000, + ); const traceId = newTraceId(rng); const spanId = newSpanId(rng); traces.push( @@ -392,9 +408,13 @@ function generateAnomaly( } for (let i = 0; i < FAILING_TRACE_COUNT; i++) { - const t = - onsetMs + - (i / Math.max(FAILING_TRACE_COUNT - 1, 1)) * (ANOMALY_WINDOW_MS - 5_000); + const t = spreadTimestamp( + i, + FAILING_TRACE_COUNT, + onsetMs, + ANOMALY_WINDOW_MS, + 5_000, + ); const traceId = newTraceId(rng); const variantBody = slots[i].body; @@ -577,10 +597,13 @@ function* streamBackground( const startMs = nowMs - HISTORY_WINDOW_MS; for (let i = 0; i < traceCount; i++) { - const t = - traceCount > 1 - ? startMs + (i / (traceCount - 1)) * (HISTORY_WINDOW_MS - 60_000) - : startMs; + const t = spreadTimestamp( + i, + traceCount, + startMs, + HISTORY_WINDOW_MS, + 60_000, + ); const traceId = newTraceId(rng); const parentService = rng.pick(BACKGROUND_SERVICES); const parentOp = rng.pick(BACKGROUND_OPS[parentService]); @@ -652,10 +675,7 @@ function* streamBackground( } for (let i = 0; i < logCount; i++) { - const t = - logCount > 1 - ? startMs + (i / (logCount - 1)) * (HISTORY_WINDOW_MS - 60_000) - : startMs; + const t = spreadTimestamp(i, logCount, startMs, HISTORY_WINDOW_MS, 60_000); const kind = rng.weightedPick(LOG_TEMPLATE_MIX); const service = LOG_TEMPLATE_SERVICE[kind]; const sev = pickSeverityIn( @@ -698,20 +718,11 @@ function* streamBackground( logtag: 'F', }; } - const sevText = sev.text.toUpperCase().startsWith('WARN') - ? 'WARN' - : sev.text.toUpperCase().startsWith('ERR') || sev.text === 'fatal' - ? 'ERROR' - : sev.text.toUpperCase().startsWith('DEB') - ? 'DEBUG' - : sev.text.toUpperCase() === 'TRACE' - ? 'TRACE' - : 'INFO'; bufLogs.push( makeLog({ timestampMs: t, serviceName: service, - severityText: sevText, + severityText: normalizeSeverityText(sev.text), body, resourceAttributes: pickResource(rng, resourcePool, service), logAttributes: { ...attrs, _severity_raw: sev.text }, diff --git a/packages/hdx-eval/src/scenarios/error-root-cause/ground-truth.json b/packages/hdx-eval/src/scenarios/error-root-cause/ground-truth.json index 2d0930df1c..8b9fa25a9b 100644 --- a/packages/hdx-eval/src/scenarios/error-root-cause/ground-truth.json +++ b/packages/hdx-eval/src/scenarios/error-root-cause/ground-truth.json @@ -90,89 +90,60 @@ }, "rubric": { "programmatic": [ - { - "id": "names_root_cause_service", - "weight": 2, - "pattern": "payment-service", - "flags": "i" - }, - { - "id": "names_underlying_failure", - "weight": 2, - "pattern": "(connection timeout|db connection|connect.*timeout|timeout.*connect|unreachable)", - "flags": "i" - }, - { - "id": "names_user_facing_service", - "weight": 1, - "pattern": "checkout(-api)?", - "flags": "i" - }, - { - "id": "names_db_target", - "weight": 1, - "pattern": "db-payment", - "flags": "i" - }, - { - "id": "names_error_type", - "weight": 2, - "pattern": "(ConnectionTimeoutError|connection.?timeout|DBTimeout|error\\.type[^a-z]+(connection|db))", - "flags": "i" - }, - { - "id": "names_db_host_fqdn", - "weight": 1, - "pattern": "db-payment\\.internal", - "flags": "i" - }, - { - "id": "names_specific_db_span", - "weight": 1, - "pattern": "(db\\.payment\\.connect|payment\\.charge)", - "flags": "i" - }, - { - "id": "ruled_out_a_distractor", - "weight": 2, - "pattern": "(rul(ed|ing) out|not (the )?(root )?cause|concurrent (but )?(separate|unrelated)|distractor|separate (incident|trace tree|burst))[\\s\\S]{0,250}?\\b(tls|api-gateway|auth-service|rate.?limit|smtp|notification-service|cdn|origin|card declin|insufficient funds|search-service)", - "flags": "i" - }, - { - "id": "false_blame_tls_handshake", - "weight": 2, - "negative": true, - "pattern": "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\b(tls handshake|api-gateway[^.\\n]{0,40}?(tls|handshake|peer reset))", - "flags": "i" - }, - { - "id": "false_blame_auth_rate_limit", - "weight": 2, - "negative": true, - "pattern": "(root cause|caused by|primary issue|the issue is|because of|the cause is)[^.\\n]{0,80}?\\b(rate.?limit|auth-service[^.\\n]{0,40}?(rate.?limit|429|throttl))", - "flags": "i" - }, - { - "id": "false_blame_smtp", - "weight": 2, - "negative": true, - "pattern": "(root cause|caused by|primary issue|the issue is|because of|the cause is)[^.\\n]{0,80}?\\b(smtp|notification-service[^.\\n]{0,40}?(smtp|email))", - "flags": "i" - }, - { - "id": "false_blame_payment_decline", - "weight": 2, - "negative": true, - "pattern": "(root cause|caused by|primary issue|the issue is|because of|the cause is)[^.\\n]{0,80}?\\b(card declined|insufficient funds|cvv mismatch|expired card|payment.?declined)", - "flags": "i" - }, - { - "id": "false_blame_cdn_origin", - "weight": 2, - "negative": true, - "pattern": "(root cause|caused by|primary issue|the issue is|because of|the cause is)[^.\\n]{0,80}?\\b(cdn|origin[^.\\n]{0,40}?(refused|unreachable|timeout)|cache.?miss[^.\\n]{0,40}?upstream)", - "flags": "i" - } + ["names_root_cause_service", 2, "payment-service"], + [ + "names_underlying_failure", + 2, + "(connection timeout|db connection|connect.*timeout|timeout.*connect|unreachable)" + ], + ["names_user_facing_service", 1, "checkout(-api)?"], + ["names_db_target", 1, "db-payment"], + [ + "names_error_type", + 2, + "(ConnectionTimeoutError|connection.?timeout|DBTimeout|error\\.type[^a-z]+(connection|db))" + ], + ["names_db_host_fqdn", 1, "db-payment\\.internal"], + [ + "names_specific_db_span", + 1, + "(db\\.payment\\.connect|payment\\.charge)" + ], + [ + "ruled_out_a_distractor", + 2, + "(rul(ed|ing) out|not (the )?(root )?cause|concurrent (but )?(separate|unrelated)|distractor|separate (incident|trace tree|burst))[\\s\\S]{0,250}?\\b(tls|api-gateway|auth-service|rate.?limit|smtp|notification-service|cdn|origin|card declin|insufficient funds|search-service)" + ], + [ + "false_blame_tls_handshake", + 2, + "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\b(tls handshake|api-gateway[^.\\n]{0,40}?(tls|handshake|peer reset))", + true + ], + [ + "false_blame_auth_rate_limit", + 2, + "(root cause|caused by|primary issue|the issue is|because of|the cause is)[^.\\n]{0,80}?\\b(rate.?limit|auth-service[^.\\n]{0,40}?(rate.?limit|429|throttl))", + true + ], + [ + "false_blame_smtp", + 2, + "(root cause|caused by|primary issue|the issue is|because of|the cause is)[^.\\n]{0,80}?\\b(smtp|notification-service[^.\\n]{0,40}?(smtp|email))", + true + ], + [ + "false_blame_payment_decline", + 2, + "(root cause|caused by|primary issue|the issue is|because of|the cause is)[^.\\n]{0,80}?\\b(card declined|insufficient funds|cvv mismatch|expired card|payment.?declined)", + true + ], + [ + "false_blame_cdn_origin", + 2, + "(root cause|caused by|primary issue|the issue is|because of|the cause is)[^.\\n]{0,80}?\\b(cdn|origin[^.\\n]{0,40}?(refused|unreachable|timeout)|cache.?miss[^.\\n]{0,40}?upstream)", + true + ] ], "judge": { "criteria": [ diff --git a/packages/hdx-eval/src/scenarios/latency-spike/generate.ts b/packages/hdx-eval/src/scenarios/latency-spike/generate.ts index e6e3161eb5..4822524ef7 100644 --- a/packages/hdx-eval/src/scenarios/latency-spike/generate.ts +++ b/packages/hdx-eval/src/scenarios/latency-spike/generate.ts @@ -1,6 +1,7 @@ import { buildResourcePool, pickResource, + spreadTimestamp, uuidv4, } from '../../generators/templates'; import { @@ -278,10 +279,7 @@ export const latencySpikeScenario: Scenario = { const anomalyStartMs = nowMs - ANOMALY_WINDOW_MS; for (let i = 0; i < totalTraces; i++) { - const t = - totalTraces > 1 - ? startMs + (i / (totalTraces - 1)) * (HISTORY_WINDOW_MS - 30_000) - : startMs; + const t = spreadTimestamp(i, totalTraces, startMs, HISTORY_WINDOW_MS); const traceId = newTraceId(rng); const endpoint = rng.weightedPick( ENDPOINTS.map(e => ({ value: e.name, weight: e.weight })), @@ -499,10 +497,7 @@ export const latencySpikeScenario: Scenario = { }); const bgTotal = Math.max(0, Math.round(BACKGROUND_TRACES * factor)); for (let i = 0; i < bgTotal; i++) { - const t = - bgTotal > 1 - ? startMs + (i / (bgTotal - 1)) * (HISTORY_WINDOW_MS - 30_000) - : startMs; + const t = spreadTimestamp(i, bgTotal, startMs, HISTORY_WINDOW_MS); const svc = bgServices[i % bgServices.length]; const op = svc.operations[rng.intRange(0, svc.operations.length - 1)]; const childOp = svc.childOps[rng.intRange(0, svc.childOps.length - 1)]; diff --git a/packages/hdx-eval/src/scenarios/latency-spike/ground-truth.json b/packages/hdx-eval/src/scenarios/latency-spike/ground-truth.json index 44f9a914af..bd2b93bced 100644 --- a/packages/hdx-eval/src/scenarios/latency-spike/ground-truth.json +++ b/packages/hdx-eval/src/scenarios/latency-spike/ground-truth.json @@ -68,71 +68,45 @@ }, "rubric": { "programmatic": [ - { - "id": "names_affected_service", - "weight": 1, - "pattern": "api-server", - "flags": "i" - }, - { - "id": "names_affected_endpoint", - "weight": 1, - "pattern": "(/api/orders/search|orders/search)", - "flags": "i" - }, - { - "id": "names_slow_child", - "weight": 2, - "pattern": "(database\\.query|SELECT.*FROM\\s+orders)", - "flags": "i" - }, - { - "id": "names_segment", - "weight": 2, - "pattern": "(enterprise|tenant\\.tier)", - "flags": "i" - }, - { - "id": "names_index_or_query_root_cause", - "weight": 1, - "pattern": "(index|tenant_id|missing.*index)", - "flags": "i" - }, - { - "id": "false_blame_hot_shard", - "weight": 2, - "negative": true, - "pattern": "(root cause|caused by|primary issue|the issue is|the cause is|main issue)[^.\\n]{0,80}?\\b(hot.shard|elasticsearch.shard.hot|shard.{0,30}?hot)", - "flags": "i" - }, - { - "id": "false_blame_products_search", - "weight": 2, - "negative": true, - "pattern": "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\bproducts.search", - "flags": "i" - }, - { - "id": "false_blame_feature_flag", - "weight": 2, - "negative": true, - "pattern": "(root cause|caused by|primary issue|the issue is|because of|the cause is)[^.\\n]{0,80}?\\b(feature.flag|experiment_a)", - "flags": "i" - }, - { - "id": "false_blame_gc_pause", - "weight": 1, - "negative": true, - "pattern": "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\bgc.pause", - "flags": "i" - }, - { - "id": "false_blame_cold_start", - "weight": 1, - "negative": true, - "pattern": "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\bcold.start", - "flags": "i" - } + ["names_affected_service", 1, "api-server"], + ["names_affected_endpoint", 1, "(/api/orders/search|orders/search)"], + ["names_slow_child", 2, "(database\\.query|SELECT.*FROM\\s+orders)"], + ["names_segment", 2, "(enterprise|tenant\\.tier)"], + [ + "names_index_or_query_root_cause", + 1, + "(index|tenant_id|missing.*index)" + ], + [ + "false_blame_hot_shard", + 2, + "(root cause|caused by|primary issue|the issue is|the cause is|main issue)[^.\\n]{0,80}?\\b(hot.shard|elasticsearch.shard.hot|shard.{0,30}?hot)", + true + ], + [ + "false_blame_products_search", + 2, + "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\bproducts.search", + true + ], + [ + "false_blame_feature_flag", + 2, + "(root cause|caused by|primary issue|the issue is|because of|the cause is)[^.\\n]{0,80}?\\b(feature.flag|experiment_a)", + true + ], + [ + "false_blame_gc_pause", + 1, + "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\bgc.pause", + true + ], + [ + "false_blame_cold_start", + 1, + "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\bcold.start", + true + ] ], "judge": { "criteria": [ diff --git a/packages/hdx-eval/src/scenarios/noisy-signals/generate.ts b/packages/hdx-eval/src/scenarios/noisy-signals/generate.ts index bc63663d1d..c04bdc22c3 100644 --- a/packages/hdx-eval/src/scenarios/noisy-signals/generate.ts +++ b/packages/hdx-eval/src/scenarios/noisy-signals/generate.ts @@ -12,12 +12,14 @@ import { heartbeatLog, indexingLog, jobFailedPermanentLog, + normalizeSeverityText, notificationDeliveryLog, pageRenderLog, pickSeverity, pickSeverityIn, searchCacheMissVerboseLog, serviceOpsDebugLog, + spreadTimestamp, subscriptionMetricLog, upstreamHealthProbeLog, } from '../../generators/templates'; @@ -82,19 +84,55 @@ const DEFAULTS = { type Rng = GenerateContext['rng']; -function* emitBatched( - rows: LogRow[], - batchSize: number, -): Iterable { - for (let i = 0; i < rows.length; i += batchSize) { - yield { traces: [], logs: rows.slice(i, i + batchSize) }; - } +function spread(i: number, total: number, startMs: number): number { + return spreadTimestamp(i, total, startMs, HISTORY_WINDOW_MS); } -function spread(i: number, total: number, startMs: number): number { - return total > 1 - ? startMs + (i / (total - 1)) * (HISTORY_WINDOW_MS - 30_000) - : startMs; +// ─── Generic log-phase generator ────────────────────────────────────────── +// Most phases follow the same skeleton: loop N times, spread timestamps, +// call a template function, push a log row, yield in batches. This generic +// eliminates the per-phase boilerplate. + +type TemplateResult = { body: string; attrs: Record }; + +function* streamLogPhase(opts: { + rng: Rng; + startMs: number; + count: number; + batchSize: number; + serviceName: string | ((i: number) => string); + severityText: LogRow['severityText']; + template: (rng: Rng, t: number, svc: string) => TemplateResult; + extraAttrs?: (rng: Rng, t: number) => Record; +}): Iterable { + const { rng, startMs, count, batchSize, severityText, template, extraAttrs } = + opts; + const resolveSvc = + typeof opts.serviceName === 'function' + ? opts.serviceName + : () => opts.serviceName as string; + const buf: LogRow[] = []; + for (let i = 0; i < count; i++) { + const t = spread(i, count, startMs); + const svc = resolveSvc(i); + const tmpl = template(rng, t, svc); + buf.push( + makeLog({ + timestampMs: t, + serviceName: svc, + severityText, + body: tmpl.body, + resourceAttributes: buildResourceAttrs({ rng, serviceName: svc }), + logAttributes: extraAttrs + ? { ...tmpl.attrs, ...extraAttrs(rng, t) } + : tmpl.attrs, + }), + ); + if (buf.length >= batchSize) { + yield { traces: [], logs: buf.splice(0, buf.length) }; + } + } + if (buf.length) yield { traces: [], logs: buf }; } export const noisySignalsScenario: Scenario = { @@ -109,35 +147,154 @@ export const noisySignalsScenario: Scenario = { const factor = ctx.volumeFactor ?? 1; const counts = scaleCounts(DEFAULTS, factor); + const FOUR_SERVICES = [ + 'inventory-service', + 'pricing-service', + 'shipping-service', + 'recommendation-service', + ] as const; + + const base = { rng, startMs, batchSize }; + // Composite cell 1: notification-service × DEBUG - yield* phaseCacheHit(rng, startMs, counts.CACHE_HIT, batchSize); - yield* phaseNotificationDelivery( - rng, - startMs, - counts.NOTIFICATION_DELIVERY, - batchSize, - ); + yield* streamLogPhase({ + ...base, + count: counts.CACHE_HIT, + serviceName: 'notification-service', + severityText: 'DEBUG', + template: (r, t) => cacheHitLog({ rng: r, nowMs: t }), + }); + yield* streamLogPhase({ + ...base, + count: counts.NOTIFICATION_DELIVERY, + serviceName: 'notification-service', + severityText: 'DEBUG', + template: (r, t) => notificationDeliveryLog({ rng: r, nowMs: t }), + }); // Composite cell 2: billing-service × INFO - yield* phaseSubscription(rng, startMs, counts.SUBSCRIPTION, batchSize); - yield* phaseBillingEvents(rng, startMs, counts.BILLING_EVENTS, batchSize); + yield* streamLogPhase({ + ...base, + count: counts.SUBSCRIPTION, + serviceName: 'billing-service', + severityText: 'INFO', + template: (r, t) => subscriptionMetricLog({ rng: r, nowMs: t }), + }); + yield* streamLogPhase({ + ...base, + count: counts.BILLING_EVENTS, + serviceName: 'billing-service', + severityText: 'INFO', + template: (r, t) => billingEventLog({ rng: r, nowMs: t }), + }); // Composite cell 3: worker × ERROR - yield* phaseStacktrace(rng, startMs, counts.STACKTRACE, batchSize); - yield* phaseJobFailed(rng, startMs, counts.JOB_FAILED, batchSize); + yield* streamLogPhase({ + ...base, + count: counts.STACKTRACE, + serviceName: 'worker', + severityText: 'ERROR', + template: (r, t) => caughtExceptionLog({ rng: r, nowMs: t }), + }); + yield* streamLogPhase({ + ...base, + count: counts.JOB_FAILED, + serviceName: 'worker', + severityText: 'ERROR', + template: (r, t) => jobFailedPermanentLog({ rng: r, nowMs: t }), + }); // Composite cell 4: 4-service × DEBUG - yield* phaseHeartbeat(rng, startMs, counts.HEARTBEAT, batchSize); - yield* phaseServiceOps(rng, startMs, counts.SERVICE_OPS, batchSize); + yield* streamLogPhase({ + ...base, + count: counts.HEARTBEAT, + serviceName: (i: number) => FOUR_SERVICES[i % FOUR_SERVICES.length], + severityText: 'DEBUG', + template: (r, t, svc) => + heartbeatLog({ rng: r, nowMs: t, serviceName: svc }), + }); + yield* streamLogPhase({ + ...base, + count: counts.SERVICE_OPS, + serviceName: (i: number) => FOUR_SERVICES[i % FOUR_SERVICES.length], + severityText: 'DEBUG', + template: (r, t, svc) => + serviceOpsDebugLog({ + rng: r, + nowMs: t, + serviceName: svc as (typeof FOUR_SERVICES)[number], + }), + }); // Composite cell 5: frontend-proxy × INFO - yield* phaseLbProbe(rng, startMs, counts.LB_PROBE, batchSize); - yield* phaseAccessLog(rng, startMs, counts.ACCESS_LOG, batchSize); + yield* streamLogPhase({ + ...base, + count: counts.LB_PROBE, + serviceName: 'frontend-proxy', + severityText: 'INFO', + template: (r, t) => upstreamHealthProbeLog({ rng: r, nowMs: t }), + }); + yield* streamLogPhase({ + ...base, + count: counts.ACCESS_LOG, + serviceName: 'frontend-proxy', + severityText: 'INFO', + template: (r, t) => envoyAccessLog({ rng: r, nowMs: t }), + extraAttrs: r => ({ _severity_raw: pickSeverityIn(r, 'info').text }), + }); // Composite cell 6: frontend × INFO - yield* phaseConsoleDump(rng, startMs, counts.CONSOLE_DUMP, batchSize); - yield* phasePageRender(rng, startMs, counts.PAGE_RENDER, batchSize); + yield* streamLogPhase({ + ...base, + count: counts.CONSOLE_DUMP, + serviceName: 'frontend', + severityText: 'INFO', + template: (r, t) => consoleLogDumpLog({ rng: r, nowMs: t }), + }); + yield* streamLogPhase({ + ...base, + count: counts.PAGE_RENDER, + serviceName: 'frontend', + severityText: 'INFO', + template: (r, t) => pageRenderLog({ rng: r, nowMs: t }), + }); // Composite cell 7: search-service × INFO - yield* phaseSearchMiss(rng, startMs, counts.SEARCH_MISS, batchSize); - yield* phaseIndexing(rng, startMs, counts.INDEXING, batchSize); + yield* streamLogPhase({ + ...base, + count: counts.SEARCH_MISS, + serviceName: 'search-service', + severityText: 'INFO', + template: (r, t) => searchCacheMissVerboseLog({ rng: r, nowMs: t }), + }); + yield* streamLogPhase({ + ...base, + count: counts.INDEXING, + serviceName: 'search-service', + severityText: 'INFO', + template: (r, t) => indexingLog({ rng: r, nowMs: t }), + }); // Single-pattern cells - yield* phaseAnalytics(rng, startMs, counts.ANALYTICS, batchSize); - yield* phaseCatalog(rng, startMs, counts.CATALOG_LOOKUP, batchSize); + yield* streamLogPhase({ + ...base, + count: counts.ANALYTICS, + serviceName: 'analytics-service', + severityText: 'INFO', + template: (r, t) => analyticsEventLog({ rng: r, nowMs: t }), + }); + yield* streamLogPhase({ + ...base, + count: counts.CATALOG_LOOKUP, + serviceName: 'product-catalog', + severityText: 'INFO', + template: (r, t) => { + const sev = pickSeverityIn(r, 'info'); + const tmpl = catalogLog({ + rng: r, + nowMs: t, + level: sev.text.toLowerCase(), + }); + return { + body: tmpl.body, + attrs: { ...tmpl.attrs, _severity_raw: sev.text }, + }; + }, + }); + // Background varied — custom logic, uses phaseBackgroundVaried yield* phaseBackgroundVaried( rng, startMs, @@ -159,490 +316,6 @@ function scaleCounts>( return out; } -function* phaseHeartbeat( - rng: Rng, - startMs: number, - count: number, - batchSize: number, -): Iterable { - const buf: LogRow[] = []; - const services = [ - 'inventory-service', - 'pricing-service', - 'shipping-service', - 'recommendation-service', - ]; - for (let i = 0; i < count; i++) { - const t = spread(i, count, startMs); - const svc = services[i % services.length]; - const tmpl = heartbeatLog({ rng, nowMs: t, serviceName: svc }); - buf.push( - makeLog({ - timestampMs: t, - serviceName: svc, - severityText: 'DEBUG', - body: tmpl.body, - resourceAttributes: buildResourceAttrs({ rng, serviceName: svc }), - logAttributes: tmpl.attrs, - }), - ); - if (buf.length >= batchSize) { - yield { traces: [], logs: buf.splice(0, buf.length) }; - } - } - if (buf.length) yield { traces: [], logs: buf }; -} - -function* phaseServiceOps( - rng: Rng, - startMs: number, - count: number, - batchSize: number, -): Iterable { - const buf: LogRow[] = []; - const services = [ - 'inventory-service', - 'pricing-service', - 'shipping-service', - 'recommendation-service', - ] as const; - for (let i = 0; i < count; i++) { - const t = spread(i, count, startMs); - const svc = services[i % services.length]; - const tmpl = serviceOpsDebugLog({ rng, nowMs: t, serviceName: svc }); - buf.push( - makeLog({ - timestampMs: t, - serviceName: svc, - severityText: 'DEBUG', - body: tmpl.body, - resourceAttributes: buildResourceAttrs({ rng, serviceName: svc }), - logAttributes: tmpl.attrs, - }), - ); - if (buf.length >= batchSize) { - yield { traces: [], logs: buf.splice(0, buf.length) }; - } - } - if (buf.length) yield { traces: [], logs: buf }; -} - -function* phaseCacheHit( - rng: Rng, - startMs: number, - count: number, - batchSize: number, -): Iterable { - const buf: LogRow[] = []; - for (let i = 0; i < count; i++) { - const t = spread(i, count, startMs); - const tmpl = cacheHitLog({ rng, nowMs: t }); - buf.push( - makeLog({ - timestampMs: t, - serviceName: 'notification-service', - severityText: 'DEBUG', - body: tmpl.body, - resourceAttributes: buildResourceAttrs({ - rng, - serviceName: 'notification-service', - }), - logAttributes: tmpl.attrs, - }), - ); - if (buf.length >= batchSize) { - yield { traces: [], logs: buf.splice(0, buf.length) }; - } - } - if (buf.length) yield { traces: [], logs: buf }; -} - -function* phaseNotificationDelivery( - rng: Rng, - startMs: number, - count: number, - batchSize: number, -): Iterable { - const buf: LogRow[] = []; - for (let i = 0; i < count; i++) { - const t = spread(i, count, startMs); - const tmpl = notificationDeliveryLog({ rng, nowMs: t }); - buf.push( - makeLog({ - timestampMs: t, - serviceName: 'notification-service', - severityText: 'DEBUG', - body: tmpl.body, - resourceAttributes: buildResourceAttrs({ - rng, - serviceName: 'notification-service', - }), - logAttributes: tmpl.attrs, - }), - ); - if (buf.length >= batchSize) { - yield { traces: [], logs: buf.splice(0, buf.length) }; - } - } - if (buf.length) yield { traces: [], logs: buf }; -} - -function* phaseStacktrace( - rng: Rng, - startMs: number, - count: number, - batchSize: number, -): Iterable { - const buf: LogRow[] = []; - for (let i = 0; i < count; i++) { - const t = spread(i, count, startMs); - const tmpl = caughtExceptionLog({ rng, nowMs: t }); - buf.push( - makeLog({ - timestampMs: t, - serviceName: 'worker', - severityText: 'ERROR', - body: tmpl.body, - resourceAttributes: buildResourceAttrs({ rng, serviceName: 'worker' }), - logAttributes: tmpl.attrs, - }), - ); - if (buf.length >= batchSize) { - yield { traces: [], logs: buf.splice(0, buf.length) }; - } - } - if (buf.length) yield { traces: [], logs: buf }; -} - -function* phaseJobFailed( - rng: Rng, - startMs: number, - count: number, - batchSize: number, -): Iterable { - const buf: LogRow[] = []; - for (let i = 0; i < count; i++) { - const t = spread(i, count, startMs); - const tmpl = jobFailedPermanentLog({ rng, nowMs: t }); - buf.push( - makeLog({ - timestampMs: t, - serviceName: 'worker', - severityText: 'ERROR', - body: tmpl.body, - resourceAttributes: buildResourceAttrs({ rng, serviceName: 'worker' }), - logAttributes: tmpl.attrs, - }), - ); - if (buf.length >= batchSize) { - yield { traces: [], logs: buf.splice(0, buf.length) }; - } - } - if (buf.length) yield { traces: [], logs: buf }; -} - -function* phaseSubscription( - rng: Rng, - startMs: number, - count: number, - batchSize: number, -): Iterable { - const buf: LogRow[] = []; - for (let i = 0; i < count; i++) { - const t = spread(i, count, startMs); - const tmpl = subscriptionMetricLog({ rng, nowMs: t }); - buf.push( - makeLog({ - timestampMs: t, - serviceName: 'billing-service', - severityText: 'INFO', - body: tmpl.body, - resourceAttributes: buildResourceAttrs({ - rng, - serviceName: 'billing-service', - }), - logAttributes: tmpl.attrs, - }), - ); - if (buf.length >= batchSize) { - yield { traces: [], logs: buf.splice(0, buf.length) }; - } - } - if (buf.length) yield { traces: [], logs: buf }; -} - -function* phaseBillingEvents( - rng: Rng, - startMs: number, - count: number, - batchSize: number, -): Iterable { - const buf: LogRow[] = []; - for (let i = 0; i < count; i++) { - const t = spread(i, count, startMs); - const tmpl = billingEventLog({ rng, nowMs: t }); - buf.push( - makeLog({ - timestampMs: t, - serviceName: 'billing-service', - severityText: 'INFO', - body: tmpl.body, - resourceAttributes: buildResourceAttrs({ - rng, - serviceName: 'billing-service', - }), - logAttributes: tmpl.attrs, - }), - ); - if (buf.length >= batchSize) { - yield { traces: [], logs: buf.splice(0, buf.length) }; - } - } - if (buf.length) yield { traces: [], logs: buf }; -} - -function* phaseAccessLog( - rng: Rng, - startMs: number, - count: number, - batchSize: number, -): Iterable { - const buf: LogRow[] = []; - for (let i = 0; i < count; i++) { - const t = spread(i, count, startMs); - const tmpl = envoyAccessLog({ rng, nowMs: t }); - const sev = pickSeverityIn(rng, 'info'); - buf.push( - makeLog({ - timestampMs: t, - serviceName: 'frontend-proxy', - severityText: 'INFO', - body: tmpl.body, - resourceAttributes: buildResourceAttrs({ - rng, - serviceName: 'frontend-proxy', - }), - logAttributes: { ...tmpl.attrs, _severity_raw: sev.text }, - }), - ); - if (buf.length >= batchSize) { - yield { traces: [], logs: buf.splice(0, buf.length) }; - } - } - if (buf.length) yield { traces: [], logs: buf }; -} - -function* phaseLbProbe( - rng: Rng, - startMs: number, - count: number, - batchSize: number, -): Iterable { - const buf: LogRow[] = []; - for (let i = 0; i < count; i++) { - const t = spread(i, count, startMs); - const tmpl = upstreamHealthProbeLog({ rng, nowMs: t }); - buf.push( - makeLog({ - timestampMs: t, - serviceName: 'frontend-proxy', - severityText: 'INFO', - body: tmpl.body, - resourceAttributes: buildResourceAttrs({ - rng, - serviceName: 'frontend-proxy', - }), - logAttributes: tmpl.attrs, - }), - ); - if (buf.length >= batchSize) { - yield { traces: [], logs: buf.splice(0, buf.length) }; - } - } - if (buf.length) yield { traces: [], logs: buf }; -} - -function* phasePageRender( - rng: Rng, - startMs: number, - count: number, - batchSize: number, -): Iterable { - const buf: LogRow[] = []; - for (let i = 0; i < count; i++) { - const t = spread(i, count, startMs); - const tmpl = pageRenderLog({ rng, nowMs: t }); - buf.push( - makeLog({ - timestampMs: t, - serviceName: 'frontend', - severityText: 'INFO', - body: tmpl.body, - resourceAttributes: buildResourceAttrs({ - rng, - serviceName: 'frontend', - }), - logAttributes: tmpl.attrs, - }), - ); - if (buf.length >= batchSize) { - yield { traces: [], logs: buf.splice(0, buf.length) }; - } - } - if (buf.length) yield { traces: [], logs: buf }; -} - -function* phaseConsoleDump( - rng: Rng, - startMs: number, - count: number, - batchSize: number, -): Iterable { - const buf: LogRow[] = []; - for (let i = 0; i < count; i++) { - const t = spread(i, count, startMs); - const tmpl = consoleLogDumpLog({ rng, nowMs: t }); - buf.push( - makeLog({ - timestampMs: t, - serviceName: 'frontend', - severityText: 'INFO', - body: tmpl.body, - resourceAttributes: buildResourceAttrs({ - rng, - serviceName: 'frontend', - }), - logAttributes: tmpl.attrs, - }), - ); - if (buf.length >= batchSize) { - yield { traces: [], logs: buf.splice(0, buf.length) }; - } - } - if (buf.length) yield { traces: [], logs: buf }; -} - -function* phaseIndexing( - rng: Rng, - startMs: number, - count: number, - batchSize: number, -): Iterable { - const buf: LogRow[] = []; - for (let i = 0; i < count; i++) { - const t = spread(i, count, startMs); - const tmpl = indexingLog({ rng, nowMs: t }); - buf.push( - makeLog({ - timestampMs: t, - serviceName: 'search-service', - severityText: 'INFO', - body: tmpl.body, - resourceAttributes: buildResourceAttrs({ - rng, - serviceName: 'search-service', - }), - logAttributes: tmpl.attrs, - }), - ); - if (buf.length >= batchSize) { - yield { traces: [], logs: buf.splice(0, buf.length) }; - } - } - if (buf.length) yield { traces: [], logs: buf }; -} - -function* phaseSearchMiss( - rng: Rng, - startMs: number, - count: number, - batchSize: number, -): Iterable { - const buf: LogRow[] = []; - for (let i = 0; i < count; i++) { - const t = spread(i, count, startMs); - const tmpl = searchCacheMissVerboseLog({ rng, nowMs: t }); - buf.push( - makeLog({ - timestampMs: t, - serviceName: 'search-service', - severityText: 'INFO', - body: tmpl.body, - resourceAttributes: buildResourceAttrs({ - rng, - serviceName: 'search-service', - }), - logAttributes: tmpl.attrs, - }), - ); - if (buf.length >= batchSize) { - yield { traces: [], logs: buf.splice(0, buf.length) }; - } - } - if (buf.length) yield { traces: [], logs: buf }; -} - -function* phaseAnalytics( - rng: Rng, - startMs: number, - count: number, - batchSize: number, -): Iterable { - const buf: LogRow[] = []; - for (let i = 0; i < count; i++) { - const t = spread(i, count, startMs); - const tmpl = analyticsEventLog({ rng, nowMs: t }); - buf.push( - makeLog({ - timestampMs: t, - serviceName: 'analytics-service', - severityText: 'INFO', - body: tmpl.body, - resourceAttributes: buildResourceAttrs({ - rng, - serviceName: 'analytics-service', - }), - logAttributes: tmpl.attrs, - }), - ); - if (buf.length >= batchSize) { - yield { traces: [], logs: buf.splice(0, buf.length) }; - } - } - if (buf.length) yield { traces: [], logs: buf }; -} - -function* phaseCatalog( - rng: Rng, - startMs: number, - count: number, - batchSize: number, -): Iterable { - const buf: LogRow[] = []; - for (let i = 0; i < count; i++) { - const t = spread(i, count, startMs); - const sev = pickSeverityIn(rng, 'info'); - const tmpl = catalogLog({ rng, nowMs: t, level: sev.text.toLowerCase() }); - buf.push( - makeLog({ - timestampMs: t, - serviceName: 'product-catalog', - severityText: 'INFO', - body: tmpl.body, - resourceAttributes: buildResourceAttrs({ - rng, - serviceName: 'product-catalog', - }), - logAttributes: { ...tmpl.attrs, _severity_raw: sev.text }, - }), - ); - if (buf.length >= batchSize) { - yield { traces: [], logs: buf.splice(0, buf.length) }; - } - } - if (buf.length) yield { traces: [], logs: buf }; -} - function* phaseBackgroundVaried( rng: Rng, startMs: number, @@ -667,7 +340,6 @@ function* phaseBackgroundVaried( for (let i = 0; i < count; i++) { const t = spread(i, count, startMs); const svc = services[i % services.length]; - // Mostly info, with realistic warn/error sprinkle const sev = rng.weightedPick<'info' | 'warn' | 'error' | 'debug' | 'trace'>( [ { value: 'info', weight: 78 }, @@ -683,19 +355,7 @@ function* phaseBackgroundVaried( makeLog({ timestampMs: t, serviceName: svc, - // The schema uses uppercase severityText; we encode the messy - // raw severity in LogAttributes._severity_raw so agents can see it - // (and have to handle case-folding to count by severity). - severityText: sevPick.text.toUpperCase().startsWith('WARN') - ? 'WARN' - : sevPick.text.toUpperCase().startsWith('ERR') || - sevPick.text === 'fatal' - ? 'ERROR' - : sevPick.text.toUpperCase().startsWith('DEB') - ? 'DEBUG' - : sevPick.text.toUpperCase() === 'TRACE' - ? 'TRACE' - : 'INFO', + severityText: normalizeSeverityText(sevPick.text), body, resourceAttributes: buildResourceAttrs({ rng, serviceName: svc }), logAttributes: { @@ -715,5 +375,4 @@ function* phaseBackgroundVaried( // Keep these unused imports tidy for the linter (they're conditionally // referenced via dynamic phase functions above). -void emitBatched; void pickSeverity; diff --git a/packages/hdx-eval/src/scenarios/noisy-signals/ground-truth.json b/packages/hdx-eval/src/scenarios/noisy-signals/ground-truth.json index b64afbe1a1..0046e311d2 100644 --- a/packages/hdx-eval/src/scenarios/noisy-signals/ground-truth.json +++ b/packages/hdx-eval/src/scenarios/noisy-signals/ground-truth.json @@ -110,143 +110,114 @@ }, "rubric": { "programmatic": [ - { - "id": "names_heartbeat_pattern", - "weight": 2, - "pattern": "(health\\.check|heartbeat)", - "flags": "i" - }, - { - "id": "names_cache_hit_pattern", - "weight": 2, - "pattern": "cache\\.?hit", - "flags": "i" - }, - { - "id": "names_subscription_metric_pattern", - "weight": 2, - "pattern": "(subscription_mrr|metric\\.name=subscription|metric.{0,15}calculated|mrr)", - "flags": "i" - }, - { - "id": "names_stacktrace_pattern", - "weight": 2, - "pattern": "(NonFatalRetryableError|job\\.caught_exception|caught.{0,8}exception)", - "flags": "i" - }, - { - "id": "names_lb_probe_pattern", - "weight": 2, - "pattern": "(lb\\.health\\.probe|kube[- ]?probe|/healthz|health.?probe|LB.{0,20}probe|internal.{0,10}health.?check)", - "flags": "i" - }, - { - "id": "names_console_dump_pattern", - "weight": 2, - "pattern": "(console\\.log_dump|console\\.log dump|console.{0,10}dump|client.?side.{0,15}console|frontend.{0,10}console)", - "flags": "i" - }, - { - "id": "names_search_miss_pattern", - "weight": 1, - "pattern": "(search\\.cache\\.miss|cache.?miss.{0,30}(search|verbose|per.?query)|verbose.{0,20}cache.?miss)", - "flags": "i" - }, - { - "id": "names_keep_notification_delivery", - "weight": 1, - "pattern": "(notification\\.delivery|push.{0,10}delivery|delivery.{0,15}(audit|record|trail))", - "flags": "i" - }, - { - "id": "names_keep_billing_events", - "weight": 1, - "pattern": "(invoice\\.(charged|refunded)|subscription\\.(activated|cancelled)|payment\\.failed|billing.{0,15}(events?|financial|audit))", - "flags": "i" - }, - { - "id": "names_keep_job_failed_permanent", - "weight": 1, - "pattern": "(job\\.failed\\.permanent|permanent.{0,15}(failure|fail)|unhandled.{0,20}(failure|error|job))", - "flags": "i" - }, - { - "id": "false_blanket_drop_notification_debug", - "weight": 3, - "negative": true, - "pattern": "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\bnotification-service[^.\\n]{0,40}?\\bdebug\\b(?![^.\\n]*?(cache\\.?hit|cache.?lookup))", - "flags": "i" - }, - { - "id": "false_blanket_drop_billing_info", - "weight": 3, - "negative": true, - "pattern": "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\bbilling-service[^.\\n]{0,40}?\\binfo\\b(?![^.\\n]*?(subscription|metric|mrr))", - "flags": "i" - }, - { - "id": "false_blanket_drop_worker_errors", - "weight": 3, - "negative": true, - "pattern": "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\bworker[^.\\n]{0,40}?\\berror\\b(?![^.\\n]*?(NonFatalRetryable|caught|retryable|handled))", - "flags": "i" - }, - { - "id": "false_drop_envoy_access", - "weight": 2, - "negative": true, - "pattern": "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(envoy.{0,20}?access|proxy\\.access|access.?log)\\b", - "flags": "i" - }, - { - "id": "false_drop_page_render", - "weight": 2, - "negative": true, - "pattern": "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(page\\.render|rum|frontend.{0,15}render)", - "flags": "i" - }, - { - "id": "false_drop_indexing", - "weight": 1, - "negative": true, - "pattern": "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(index\\.shard\\.flush|indexing.?progress|indexer.?health)", - "flags": "i" - }, - { - "id": "false_drop_analytics_events", - "weight": 2, - "negative": true, - "pattern": "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(analytics-service|product\\.view|cart\\.add|checkout\\.[a-z]+|event tracking|product analytics)", - "flags": "i" - }, - { - "id": "false_drop_notification_delivery", - "weight": 2, - "negative": true, - "pattern": "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(notification\\.delivery|push.{0,10}delivery|delivery.?(audit|record|trail))", - "flags": "i" - }, - { - "id": "false_drop_billing_events", - "weight": 2, - "negative": true, - "pattern": "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(invoice\\.(charged|refunded)|subscription\\.(activated|cancelled)|payment\\.failed|financial.?event)", - "flags": "i" - }, - { - "id": "false_drop_job_failed_permanent", - "weight": 2, - "negative": true, - "pattern": "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(job\\.failed\\.permanent|permanent.{0,15}(failure|fail))", - "flags": "i" - }, - { - "id": "false_drop_service_ops_debug", - "weight": 2, - "negative": true, - "pattern": "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(inventory\\.stock_lookup|pricing\\.calculation|shipping\\.rate\\.quote|recommendation\\.score)", - "flags": "i" - } + ["names_heartbeat_pattern", 2, "(health\\.check|heartbeat)"], + ["names_cache_hit_pattern", 2, "cache\\.?hit"], + [ + "names_subscription_metric_pattern", + 2, + "(subscription_mrr|metric\\.name=subscription|metric.{0,15}calculated|mrr)" + ], + [ + "names_stacktrace_pattern", + 2, + "(NonFatalRetryableError|job\\.caught_exception|caught.{0,8}exception)" + ], + [ + "names_lb_probe_pattern", + 2, + "(lb\\.health\\.probe|kube[- ]?probe|/healthz|health.?probe|LB.{0,20}probe|internal.{0,10}health.?check)" + ], + [ + "names_console_dump_pattern", + 2, + "(console\\.log_dump|console\\.log dump|console.{0,10}dump|client.?side.{0,15}console|frontend.{0,10}console)" + ], + [ + "names_search_miss_pattern", + 1, + "(search\\.cache\\.miss|cache.?miss.{0,30}(search|verbose|per.?query)|verbose.{0,20}cache.?miss)" + ], + [ + "names_keep_notification_delivery", + 1, + "(notification\\.delivery|push.{0,10}delivery|delivery.{0,15}(audit|record|trail))" + ], + [ + "names_keep_billing_events", + 1, + "(invoice\\.(charged|refunded)|subscription\\.(activated|cancelled)|payment\\.failed|billing.{0,15}(events?|financial|audit))" + ], + [ + "names_keep_job_failed_permanent", + 1, + "(job\\.failed\\.permanent|permanent.{0,15}(failure|fail)|unhandled.{0,20}(failure|error|job))" + ], + [ + "false_blanket_drop_notification_debug", + 3, + "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\bnotification-service[^.\\n]{0,40}?\\bdebug\\b(?![^.\\n]*?(cache\\.?hit|cache.?lookup))", + true + ], + [ + "false_blanket_drop_billing_info", + 3, + "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\bbilling-service[^.\\n]{0,40}?\\binfo\\b(?![^.\\n]*?(subscription|metric|mrr))", + true + ], + [ + "false_blanket_drop_worker_errors", + 3, + "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\bworker[^.\\n]{0,40}?\\berror\\b(?![^.\\n]*?(NonFatalRetryable|caught|retryable|handled))", + true + ], + [ + "false_drop_envoy_access", + 2, + "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(envoy.{0,20}?access|proxy\\.access|access.?log)\\b", + true + ], + [ + "false_drop_page_render", + 2, + "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(page\\.render|rum|frontend.{0,15}render)", + true + ], + [ + "false_drop_indexing", + 1, + "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(index\\.shard\\.flush|indexing.?progress|indexer.?health)", + true + ], + [ + "false_drop_analytics_events", + 2, + "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(analytics-service|product\\.view|cart\\.add|checkout\\.[a-z]+|event tracking|product analytics)", + true + ], + [ + "false_drop_notification_delivery", + 2, + "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(notification\\.delivery|push.{0,10}delivery|delivery.?(audit|record|trail))", + true + ], + [ + "false_drop_billing_events", + 2, + "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(invoice\\.(charged|refunded)|subscription\\.(activated|cancelled)|payment\\.failed|financial.?event)", + true + ], + [ + "false_drop_job_failed_permanent", + 2, + "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(job\\.failed\\.permanent|permanent.{0,15}(failure|fail))", + true + ], + [ + "false_drop_service_ops_debug", + 2, + "(drop|throttle|sample.?out|stop emitting|remove)[^.\\n]{0,80}?\\b(inventory\\.stock_lookup|pricing\\.calculation|shipping\\.rate\\.quote|recommendation\\.score)", + true + ] ], "judge": { "criteria": [ diff --git a/packages/hdx-eval/src/scenarios/segmented-regression/generate.ts b/packages/hdx-eval/src/scenarios/segmented-regression/generate.ts index 7e6adf86cf..084a205018 100644 --- a/packages/hdx-eval/src/scenarios/segmented-regression/generate.ts +++ b/packages/hdx-eval/src/scenarios/segmented-regression/generate.ts @@ -28,6 +28,7 @@ import { makeLog } from '../../generators/logs'; import { buildResourcePool, pickResource, + spreadTimestamp, uuidv4, } from '../../generators/templates'; import { @@ -118,10 +119,7 @@ function* streamBackgroundRows( }); for (let i = 0; i < totalTraces; i++) { - const t = - totalTraces > 1 - ? startMs + (i / (totalTraces - 1)) * (HISTORY_WINDOW_MS - 30_000) - : startMs; + const t = spreadTimestamp(i, totalTraces, startMs, HISTORY_WINDOW_MS); const tenantTier = rng.weightedPick( TENANT_TIERS.map(tt => ({ value: tt.value, weight: tt.weight })), @@ -230,9 +228,13 @@ function buildRecommendationDecoy( const traces: TraceRow[] = []; const logs: LogRow[] = []; for (let i = 0; i < REC_DECOY_COUNT; i++) { - const t = - startMs + - (i / Math.max(REC_DECOY_COUNT - 1, 1)) * (REC_DECOY_WINDOW_MS - 5_000); + const t = spreadTimestamp( + i, + REC_DECOY_COUNT, + startMs, + REC_DECOY_WINDOW_MS, + 5_000, + ); const traceId = newTraceId(rng); const spanId = newSpanId(rng); traces.push( diff --git a/packages/hdx-eval/src/scenarios/segmented-regression/ground-truth.json b/packages/hdx-eval/src/scenarios/segmented-regression/ground-truth.json index d04873a857..59169bee16 100644 --- a/packages/hdx-eval/src/scenarios/segmented-regression/ground-truth.json +++ b/packages/hdx-eval/src/scenarios/segmented-regression/ground-truth.json @@ -51,64 +51,47 @@ }, "rubric": { "programmatic": [ - { - "id": "names_affected_service", - "weight": 1, - "pattern": "api-server", - "flags": "i" - }, - { - "id": "names_tenant_segment", - "weight": 2, - "pattern": "(tenant\\.?tier|enterprise)", - "flags": "i" - }, - { - "id": "names_cache_segment", - "weight": 2, - "pattern": "(cache\\.?(hit|miss)|cache hit:?\\s*false|cache miss|miss(es)?)", - "flags": "i" - }, - { - "id": "names_both_axes_combined", - "weight": 3, - "pattern": "(enterprise[^.\\n]{0,80}?(cache|miss)|cache[^.\\n]{0,40}?miss[^.\\n]{0,80}?enterprise|tier\\s*=\\s*['\"]?enterprise['\"]?[^.\\n]{0,80}?cache|enterprise[^.\\n]{0,40}?AND[^.\\n]{0,40}?(cache|hit\\s*=\\s*false|miss))", - "flags": "i" - }, - { - "id": "names_error_pattern", - "weight": 1, - "pattern": "(schema mismatch|enterprise_v2|FallbackHandlerError|fallback (handler|path))", - "flags": "i" - }, - { - "id": "false_blame_global", - "weight": 2, - "negative": true, - "pattern": "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\b(global|api-server (alone|generally)|all (tiers|users|traffic)|service[- ]wide)", - "flags": "i" - }, - { - "id": "false_blame_tier_only", - "weight": 2, - "negative": true, - "pattern": "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\benterprise[^.\\n]{0,40}?(tier|customers|tenants?)(?![^.\\n]{0,200}?(cache|miss|hit\\s*=\\s*false))", - "flags": "i" - }, - { - "id": "false_blame_cache_only", - "weight": 2, - "negative": true, - "pattern": "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\bcache[ \\-]?(miss(es)?|lookup|hit\\s*=\\s*false)(?![^.\\n]{0,200}?(enterprise|tier))", - "flags": "i" - }, - { - "id": "false_blame_recommendation", - "weight": 2, - "negative": true, - "pattern": "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\brecommendation-service", - "flags": "i" - } + ["names_affected_service", 1, "api-server"], + ["names_tenant_segment", 2, "(tenant\\.?tier|enterprise)"], + [ + "names_cache_segment", + 2, + "(cache\\.?(hit|miss)|cache hit:?\\s*false|cache miss|miss(es)?)" + ], + [ + "names_both_axes_combined", + 3, + "(enterprise[^.\\n]{0,80}?(cache|miss)|cache[^.\\n]{0,40}?miss[^.\\n]{0,80}?enterprise|tier\\s*=\\s*['\"]?enterprise['\"]?[^.\\n]{0,80}?cache|enterprise[^.\\n]{0,40}?AND[^.\\n]{0,40}?(cache|hit\\s*=\\s*false|miss))" + ], + [ + "names_error_pattern", + 1, + "(schema mismatch|enterprise_v2|FallbackHandlerError|fallback (handler|path))" + ], + [ + "false_blame_global", + 2, + "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\b(global|api-server (alone|generally)|all (tiers|users|traffic)|service[- ]wide)", + true + ], + [ + "false_blame_tier_only", + 2, + "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\benterprise[^.\\n]{0,40}?(tier|customers|tenants?)(?![^.\\n]{0,200}?(cache|miss|hit\\s*=\\s*false))", + true + ], + [ + "false_blame_cache_only", + 2, + "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\bcache[ \\-]?(miss(es)?|lookup|hit\\s*=\\s*false)(?![^.\\n]{0,200}?(enterprise|tier))", + true + ], + [ + "false_blame_recommendation", + 2, + "(root cause|caused by|primary issue|the issue is|the cause is)[^.\\n]{0,80}?\\brecommendation-service", + true + ] ], "judge": { "criteria": [ diff --git a/packages/hdx-eval/src/scenarios/service-health-check/generate.ts b/packages/hdx-eval/src/scenarios/service-health-check/generate.ts index f925ec37bc..c6b30cc7fb 100644 --- a/packages/hdx-eval/src/scenarios/service-health-check/generate.ts +++ b/packages/hdx-eval/src/scenarios/service-health-check/generate.ts @@ -43,6 +43,7 @@ import { makeLog } from '../../generators/logs'; import { buildResourcePool, pickResource, + spreadTimestamp, uuidv4, } from '../../generators/templates'; import { @@ -256,11 +257,12 @@ function* streamTraces( let buf: TraceRow[] = []; for (let i = 0; i < totalTraces; i++) { - const t = - totalTraces > 1 - ? baselineStartMs + - (i / (totalTraces - 1)) * (BASELINE_WINDOW_MS - 30_000) - : baselineStartMs; + const t = spreadTimestamp( + i, + totalTraces, + baselineStartMs, + BASELINE_WINDOW_MS, + ); const inReportWindow = t >= reportStartMs; const cycleOffset = (t - baselineStartMs) % BATCH_SPIKE_PERIOD_MS; const inBatchSpike = cycleOffset < BATCH_SPIKE_DURATION_MS; @@ -375,11 +377,12 @@ function* streamLogs( let buf: LogRow[] = []; for (let i = 0; i < totalLogs; i++) { - const t = - totalLogs > 1 - ? baselineStartMs + - (i / (totalLogs - 1)) * (BASELINE_WINDOW_MS - 30_000) - : baselineStartMs; + const t = spreadTimestamp( + i, + totalLogs, + baselineStartMs, + BASELINE_WINDOW_MS, + ); let body: string; let severityText: 'INFO' | 'WARN'; diff --git a/packages/hdx-eval/src/scenarios/service-health-check/ground-truth.json b/packages/hdx-eval/src/scenarios/service-health-check/ground-truth.json index ad7b25fd4d..1024569f27 100644 --- a/packages/hdx-eval/src/scenarios/service-health-check/ground-truth.json +++ b/packages/hdx-eval/src/scenarios/service-health-check/ground-truth.json @@ -58,102 +58,83 @@ }, "rubric": { "programmatic": [ - { - "id": "names_service", - "weight": 1, - "pattern": "api-server", - "flags": "i" - }, - { - "id": "reports_traffic_rate", - "weight": 1, - "pattern": "(requests?/min|requests/s|req/min|traffic|throughput|call volume|qps|rps)", - "flags": "i" - }, - { - "id": "reports_error_rate", - "weight": 1, - "pattern": "(error rate|errors?/min|0\\.5\\s?%?|~?0\\.5|baseline.{0,15}error)", - "flags": "i" - }, - { - "id": "reports_latency_metrics", - "weight": 1, - "pattern": "(p50|p95|p99|latency)[\\s\\S]{0,200}?\\d+(\\.\\d+)?\\s?(ms|millisecond)", - "flags": "i" - }, - { - "id": "notes_new_log_pattern", - "weight": 2, - "pattern": "(shadow.?eval|new[_\\s]+recommendation|recommendation.engine|feature.?flag)", - "flags": "i" - }, - { - "id": "notes_v2_endpoint", - "weight": 2, - "pattern": "(/api/v2/products|/v2/|v2\\s+(endpoint|rollout|rolling))", - "flags": "i" - }, - { - "id": "notes_latency_drift", - "weight": 1, - "pattern": "(p50|p95|p99|latency)[\\s\\S]{0,60}?\\b(trend|drift|trending|creep|creeping|rising|rising|climb|increasing|monotonic)", - "flags": "i" - }, - { - "id": "notes_brief_blip", - "weight": 1, - "pattern": "(brief|transient|1.?(min(ute)?|second)|60.?second|short.lived).{0,80}(spike|blip|errors?|timeouts?)", - "flags": "i" - }, - { - "id": "frames_as_healthy", - "weight": 1, - "pattern": "(healthy|stable|within SLO|no concerns|operating normally|nominal|steady)", - "flags": "i" - }, - { - "id": "false_alarm_incident", - "weight": 2, - "negative": true, - "pattern": "\\b(incident|outage|service.degradation|degraded|critical|severe|emergency)\\b", - "flags": "i" - }, - { - "id": "false_alarm_paging", - "weight": 2, - "negative": true, - "pattern": "\\b(page (the|on-call)|on-call|escalat(e|ion)|wake.{0,5}up|urgent(ly)?)\\b", - "flags": "i" - }, - { - "id": "false_alarm_blip_as_pattern", - "weight": 2, - "negative": true, - "pattern": "\\b(sustained|ongoing|persistent|recurring|continuing)\\b[^.\\n]{0,60}(error|timeout)", - "flags": "i" - }, - { - "id": "false_alarm_baseline_errors", - "weight": 2, - "negative": true, - "pattern": "error rate[^.\\n]{0,60}\\b(elevated|high|concerning|problem(atic)?|investigate)\\b", - "flags": "i" - }, - { - "id": "false_alarm_slo_breach", - "weight": 2, - "negative": true, - "pattern": "(SLO[^.\\n]{0,20}(breach|violation|exceed)|latency[^.\\n]{0,30}(critical|breach|violation))", - "flags": "i" - }, - { - "id": "false_flag_batch_job", - "weight": 1, - "negative": true, - "pattern": "batch.?sync[^.\\n]{0,80}\\b(novel|unusual|new|anomal(y|ous)|concern|investigate)\\b", - "flags": "i" - } + ["names_service", 1, "api-server"], + [ + "reports_traffic_rate", + 1, + "(requests?/min|requests/s|req/min|traffic|throughput|call volume|qps|rps)" + ], + [ + "reports_error_rate", + 1, + "(error rate|errors?/min|0\\.5\\s?%?|~?0\\.5|baseline.{0,15}error)" + ], + [ + "reports_latency_metrics", + 1, + "(p50|p95|p99|latency)[\\s\\S]{0,200}?\\d+(\\.\\d+)?\\s?(ms|millisecond)" + ], + [ + "notes_new_log_pattern", + 2, + "(shadow.?eval|new[_\\s]+recommendation|recommendation.engine|feature.?flag)" + ], + [ + "notes_v2_endpoint", + 2, + "(/api/v2/products|/v2/|v2\\s+(endpoint|rollout|rolling))" + ], + [ + "notes_latency_drift", + 1, + "(p50|p95|p99|latency)[\\s\\S]{0,60}?\\b(trend|drift|trending|creep|creeping|rising|rising|climb|increasing|monotonic)" + ], + [ + "notes_brief_blip", + 1, + "(brief|transient|1.?(min(ute)?|second)|60.?second|short.lived).{0,80}(spike|blip|errors?|timeouts?)" + ], + [ + "frames_as_healthy", + 1, + "(healthy|stable|within SLO|no concerns|operating normally|nominal|steady)" + ], + [ + "false_alarm_incident", + 2, + "\\b(incident|outage|service.degradation|degraded|critical|severe|emergency)\\b", + true + ], + [ + "false_alarm_paging", + 2, + "\\b(page (the|on-call)|on-call|escalat(e|ion)|wake.{0,5}up|urgent(ly)?)\\b", + true + ], + [ + "false_alarm_blip_as_pattern", + 2, + "\\b(sustained|ongoing|persistent|recurring|continuing)\\b[^.\\n]{0,60}(error|timeout)", + true + ], + [ + "false_alarm_baseline_errors", + 2, + "error rate[^.\\n]{0,60}\\b(elevated|high|concerning|problem(atic)?|investigate)\\b", + true + ], + [ + "false_alarm_slo_breach", + 2, + "(SLO[^.\\n]{0,20}(breach|violation|exceed)|latency[^.\\n]{0,30}(critical|breach|violation))", + true + ], + [ + "false_flag_batch_job", + 1, + "batch.?sync[^.\\n]{0,80}\\b(novel|unusual|new|anomal(y|ous)|concern|investigate)\\b", + true + ] ], "judge": { "criteria": [ From ac37fe1a6d629596fb9e1e938b1aa889ef64b1a0 Mon Sep 17 00:00:00 2001 From: Brandon Pereira Date: Wed, 3 Jun 2026 16:00:11 -0600 Subject: [PATCH 10/22] feat: MCP-agnostic eval framework with N-way comparison support - Generalize from fixed HyperDX+ClickHouse pair to config-driven MCP registry - Add dual-slot eval setup docs for A/B branch comparison - Add baseline+challengers reporting model with delta columns - Expand README with MCP config reference, field tables, and examples - Improve viewer with comparison dashboard and drill-down - Update blinding to handle arbitrary brand terms per MCP - Add --baseline, --ch-url, --no-grade, --no-judge CLI flags --- packages/hdx-eval/README.md | 273 +++++++++-- packages/hdx-eval/eval.config.example.json | 80 +++- packages/hdx-eval/package.json | 2 +- .../hdx-eval/scripts/viewer/public/app.js | 424 +++++++++++++++++- .../hdx-eval/scripts/viewer/public/index.html | 7 +- .../hdx-eval/scripts/viewer/public/style.css | 218 +++++++++ .../hdx-eval/src/__tests__/aggregate.test.ts | 121 ++++- packages/hdx-eval/src/__tests__/blind.test.ts | 102 +++-- packages/hdx-eval/src/__tests__/grade.test.ts | 2 +- .../hdx-eval/src/__tests__/markdown.test.ts | 15 +- .../hdx-eval/src/__tests__/mcpConfig.test.ts | 138 ++++-- .../src/__tests__/systemPrompt.test.ts | 24 +- packages/hdx-eval/src/cli.ts | 166 +++++-- packages/hdx-eval/src/grading/blind.ts | 67 ++- packages/hdx-eval/src/grading/grade.ts | 4 + packages/hdx-eval/src/grading/judge.ts | 6 +- packages/hdx-eval/src/harness/claudeSpawn.ts | 13 +- packages/hdx-eval/src/harness/mcpConfig.ts | 56 +-- packages/hdx-eval/src/harness/runRun.ts | 5 +- packages/hdx-eval/src/harness/settingsFile.ts | 33 +- packages/hdx-eval/src/harness/systemPrompt.ts | 3 +- packages/hdx-eval/src/harness/types.ts | 45 +- packages/hdx-eval/src/hyperdx/config.ts | 122 +++-- packages/hdx-eval/src/hyperdx/setup.ts | 96 +++- packages/hdx-eval/src/reports/aggregate.ts | 76 +++- packages/hdx-eval/src/reports/markdown.ts | 256 ++++++----- packages/hdx-eval/src/reports/store.ts | 5 +- packages/hdx-eval/src/runs/instrument.ts | 2 +- yarn.lock | 2 +- 29 files changed, 1888 insertions(+), 475 deletions(-) diff --git a/packages/hdx-eval/README.md b/packages/hdx-eval/README.md index a99535e7b2..673a3c9f5e 100644 --- a/packages/hdx-eval/README.md +++ b/packages/hdx-eval/README.md @@ -1,40 +1,192 @@ # @hyperdx/hdx-eval -Eval framework for benchmarking AI agents against HyperDX's observability -platform. Generates deterministic synthetic telemetry (traces + logs) with -planted anomalies, spawns Claude Code as an SRE agent with access to either -the HyperDX MCP server or a raw ClickHouse MCP, records full trajectories, -and grades answers using programmatic checks + an LLM-as-judge. +Eval framework for benchmarking AI agents against observability MCP servers. +Generates deterministic synthetic telemetry (traces + logs) with planted +anomalies, spawns Claude Code as an SRE agent with access to any configured +MCP, records full trajectories, and grades answers using programmatic checks and +an LLM-as-judge. The framework is **MCP-agnostic** — compare any combination of +MCPs: HyperDX vs ClickHouse, feature-branch vs main, two different HyperDX +instances, or any N-way comparison. MCPs are defined in a config registry and +selected at run time. ## Prerequisites -- **`yarn dev` running** so ClickHouse, MongoDB, and the HyperDX API are up +- **`yarn dev` running** on at least one worktree (ClickHouse, MongoDB, API) - **`claude` CLI installed** (the harness spawns it in streaming JSON mode) -- **`ANTHROPIC_API_KEY`** exported (for the `run` and `grade` commands) -- **[`uv`](https://docs.astral.sh/uv/)** installed (the ClickHouse MCP server - is launched via `uv run --with mcp-clickhouse`; without `uv` on `PATH` the - MCP process silently fails to start and the agent gets no tools) - -Port detection is automatic — the CLI computes the same worktree-based slot -as `scripts/dev-env.sh`, so no manual `source` step is needed. +- **`ANTHROPIC_API_KEY`** or **`AI_API_KEY`** in `.env.local` (for `run` and + `grade` commands) +- **[`uv`](https://docs.astral.sh/uv/)** installed (only needed if you + configure a `stdio`-type ClickHouse MCP — it launches via + `uv run --with mcp-clickhouse`) ## Quick Start ```bash -# 1. One-time setup: register eval account + create Sources +# 1. One-time setup: register eval account + create Sources + write config yarn workspace @hyperdx/hdx-eval dev setup-hyperdx # 2. Seed a scenario (use --volume-factor 0.1 for faster iteration) yarn workspace @hyperdx/hdx-eval dev seed error-root-cause --volume-factor 0.1 -# 3. Run the agent eval (both MCPs, 3 runs each) -ANTHROPIC_API_KEY=sk-... yarn workspace @hyperdx/hdx-eval dev run error-root-cause +# 3. Run the eval (all enabled MCPs, 3 runs each) +yarn workspace @hyperdx/hdx-eval dev run error-root-cause -# 4. Grade the batch +# 4. Grade + report (auto-runs after step 3 by default) yarn workspace @hyperdx/hdx-eval dev grade - -# 5. Generate a markdown report yarn workspace @hyperdx/hdx-eval dev report + +# 5. Browse results in the web viewer +yarn workspace @hyperdx/hdx-eval viewer +``` + +## Recommended: Dual-Slot Eval Setup + +For reliable A/B comparison of two HyperDX branches (e.g. feature vs main), +we recommend reserving two fixed dev slots. This gives you two fully isolated +HyperDX stacks (separate ClickHouse, MongoDB, API) that can run +simultaneously. + +### 1. Start two dev stacks with fixed slots + +```bash +# Slot 98: "baseline" — run from your main branch checkout +HDX_DEV_SLOT=98 yarn dev + +# Slot 99: "candidate" — run from your feature branch checkout +HDX_DEV_SLOT=99 yarn dev +``` + +This gives you: + +| | Slot 98 (baseline) | Slot 99 (candidate) | +|---|---|---| +| API | `http://localhost:30198` | `http://localhost:30199` | +| App | `http://localhost:30298` | `http://localhost:30299` | +| ClickHouse | `http://localhost:30598` | `http://localhost:30599` | +| MCP | `http://localhost:30198/mcp` | `http://localhost:30199/mcp` | + +### 2. Run `setup-hyperdx` against each instance + +```bash +# Setup baseline (slot 98) +yarn workspace @hyperdx/hdx-eval dev setup-hyperdx --api-url http://localhost:30198 + +# Setup candidate (slot 99) +yarn workspace @hyperdx/hdx-eval dev setup-hyperdx --api-url http://localhost:30199 +``` + +After setup, `eval.config.json` will have the `hyperdx` MCP pointing at the +slot it was set up against. You can then manually add the second MCP entry +(see [MCP Configuration](#mcp-configuration) below). + +### 3. Seed both ClickHouse instances + +Both instances need the eval data: + +```bash +yarn workspace @hyperdx/hdx-eval dev seed error-root-cause --ch-url http://localhost:30598 +yarn workspace @hyperdx/hdx-eval dev seed error-root-cause --ch-url http://localhost:30599 +``` + +### 4. Run the comparison + +```bash +yarn workspace @hyperdx/hdx-eval dev run error-root-cause \ + --mcp hyperdx-main,hyperdx-feature \ + --baseline hyperdx-main \ + --runs 5 +``` + +## MCP Configuration + +MCPs are defined in `eval.config.json` under the `mcps` key. Each entry +fully specifies how to connect, which tools to allow/deny, and how to +anonymize the MCP identity for fair LLM judging. + +### Config structure + +```json +{ + "mcps": { + "": { + "type": "http" | "stdio", + "url": "...", // http only + "headers": { ... }, // http only + "command": "...", // stdio only + "args": ["..."], // stdio only + "env": { ... }, // stdio only + "toolPattern": "mcp____*", + "label": "Display Name", + "brandTerms": ["Brand", "brand"], + "deniedTools": ["mcp____tool_to_hide"], + "enabled": true + } + }, + "scenarios": { ... }, + "hyperdxApi": { ... }, + "clickhouse": { ... } +} +``` + +### Field reference + +| Field | Required | Description | +|---|---|---| +| `type` | yes | `"http"` for HTTP/SSE transport, `"stdio"` for subprocess | +| `url` | http | MCP server URL | +| `headers` | no | HTTP headers (e.g. `Authorization: Bearer ...`) | +| `command` | stdio | Executable to spawn | +| `args` | no | Command arguments | +| `env` | no | Environment variables for the subprocess | +| `toolPattern` | yes | Glob for allowed tools (e.g. `mcp__myname__*`) | +| `label` | yes | Human-readable name for reports | +| `brandTerms` | no | Terms to redact when blinding answers for the LLM judge | +| `deniedTools` | no | Tool names to deny (reduces tool count for faster schema loading) | +| `enabled` | no | `false` to exclude from `--mcp all` (default: `true`). Can still be explicitly named with `--mcp name`. | + +### Top-level config sections + +| Section | Required | Purpose | +|---|---|---| +| `mcps` | yes | MCP registry (see above) | +| `scenarios` | no | Per-scenario HyperDX Source IDs (populated by `setup-hyperdx`) | +| `hyperdxApi` | no | HyperDX API URL + access key (used only by `setup-hyperdx`) | +| `clickhouse` | no | ClickHouse connection for `seed`, `drop`, `runs-instrument` commands | + +### Example: HTTP MCP (HyperDX) + +```json +{ + "type": "http", + "url": "http://localhost:30198/mcp", + "headers": { "Authorization": "Bearer " }, + "toolPattern": "mcp__hyperdx__*", + "label": "HyperDX", + "brandTerms": ["HyperDX", "hyperdx"], + "deniedTools": [ + "mcp__hyperdx__hyperdx_delete_dashboard", + "mcp__hyperdx__hyperdx_get_dashboard" + ] +} +``` + +### Example: stdio MCP (raw ClickHouse) + +```json +{ + "type": "stdio", + "command": "uv", + "args": ["run", "--with", "mcp-clickhouse==0.3.0", "--python", "3.10", "mcp-clickhouse"], + "env": { + "CLICKHOUSE_HOST": "localhost", + "CLICKHOUSE_PORT": "30598", + "CLICKHOUSE_USER": "default", + "CLICKHOUSE_PASSWORD": "" + }, + "toolPattern": "mcp__clickhouse__*", + "label": "ClickHouse MCP", + "brandTerms": ["ClickHouse MCP", "clickhouse"] +} ``` ## Scenarios @@ -45,14 +197,14 @@ mirrors HyperDX's `otel_traces` / `otel_logs` exactly. | Scenario | Agent Prompt | What's Planted | |---|---|---| -| `error-root-cause` | "Checkout requests are failing for some users in the last 10 minutes. Find the root cause." | `payment-service` DB connection timeout cascading into `checkout-api` 5xx. 6M+ background spans, 12M logs, 5 distractor anomalies. | -| `latency-spike` | "p99 latency on api-server has jumped in the last 15 minutes. What's slow and why?" | `/api/orders/search` p99 spike for enterprise tenants (missing-index DB query). 12M+ spans, 5 regions, 5K tenants, GC/cold-start/cache distractors. | -| `noisy-signals` | "We want to cut log ingest cost. What are the top noisy signals we should drop or throttle?" | ~16M logs across composite (service, severity) cells. Each noisy cell also has a high-volume load-bearing pattern — grouping by service+severity alone won't work. | -| `segmented-regression` | "API error rate has gone up in the last 10 minutes for some users. Find the segment(s)." | Enterprise x cache-miss errors at ~12% from a schema-mismatch bug. Single-axis aggregates dilute the signal. Concurrent recommendation-service 502 burst as distractor. | -| `service-health-check` | "Generate a service health report for api-server for the last hour." | Peace-time report — no incident, but 4 novel-but-non-critical signals to call out, plus recurring baseline patterns the agent should NOT escalate. | +| `error-root-cause` | "Checkout requests are failing..." | `payment-service` DB timeout cascading into `checkout-api` 5xx. 6M+ spans, 12M logs, 5 distractors. | +| `latency-spike` | "p99 latency on api-server has jumped..." | `/api/orders/search` p99 spike for enterprise tenants. 12M+ spans, 5 regions, 5 distractors. | +| `noisy-signals` | "We want to cut log ingest cost..." | ~16M logs across composite cells. Each noisy cell has a load-bearing pattern mixed in. | +| `segmented-regression` | "API error rate has gone up..." | Enterprise x cache-miss errors at ~12%. Single-axis aggregates dilute the signal. | +| `service-health-check` | "Generate a service health report..." | Peace-time report — no incident, but 4 novel signals to call out. | Use `--volume-factor` to scale background row counts for faster iteration. -Planted anomaly counts stay fixed regardless of the factor. +Planted anomaly counts stay fixed. ```bash # Full volume (default) — realistic needle-in-haystack difficulty @@ -69,11 +221,10 @@ All commands are run via `yarn workspace @hyperdx/hdx-eval dev `. ### Setup ```bash -# One-time: register account, create ClickHouse connection, create Sources -dev setup-hyperdx -dev setup-hyperdx --api-url http://localhost:30176 # explicit port -dev setup-hyperdx --check # validate existing config -dev setup-hyperdx --reset-sources # recreate all Sources +dev setup-hyperdx # one-time: register + create Sources +dev setup-hyperdx --api-url http://... # explicit API URL +dev setup-hyperdx --check # validate existing config +dev setup-hyperdx --reset-sources # recreate all Sources ``` ### Data Management @@ -84,21 +235,26 @@ dev seed # seed with defaults (seed=42, now=D dev seed --seed 7 # custom PRNG seed dev seed --now 2026-05-08T15:00:00Z # fixed anchor time dev seed --volume-factor 0.1 # 10% background volume +dev seed --ch-url http://localhost:30599 # target specific CH dev drop # drop scenario tables ``` ### Running Evals ```bash -dev run # both MCPs, 3 runs each -dev run --mcp hyperdx # HyperDX MCP only -dev run --mcp clickhouse # raw ClickHouse MCP only +dev run # all enabled MCPs, 3 runs each +dev run --mcp hyperdx # single MCP +dev run --mcp hyperdx-main,hyperdx-feature # compare two +dev run --mcp all # all enabled MCPs (default) +dev run --baseline hyperdx-main # set baseline for delta reports dev run --runs 5 # 5 runs per cell -dev run --model claude-sonnet-4-6 +dev run --model claude-opus-4-6 dev run --max-turns 20 # default: 15 dev run --concurrency 4 # parallel cells dev run --prompt-variant hypothesis # hypothesis-first variant dev run --no-reseed --anchor-time 2026-05-08T15:00:00Z +dev run --no-grade # skip auto-grading +dev run --no-judge # programmatic checks only (faster) ``` ### Inspecting Results @@ -114,35 +270,62 @@ dev runs-instrument # enrich with ClickHouse query_log t ### Grading & Reporting ```bash -dev grade # programmatic + LLM-as-judge (Opus 4.7) +dev grade # programmatic + LLM-as-judge dev grade --judge-model claude-sonnet-4-6 -dev grade --no-judge # programmatic checks only (no LLM cost) +dev grade --no-judge # programmatic checks only dev grade --rerun-judge # re-run judge even if grades exist dev report # render _summary.md + _summary.json +dev report --baseline hyperdx-main # override baseline for deltas ``` ### Viewer ```bash -yarn workspace @hyperdx/hdx-eval viewer # web UI at http://localhost:3333 +yarn workspace @hyperdx/hdx-eval viewer # web UI at http://localhost:5176 ``` +The viewer shows a **comparison dashboard** when you select a batch: + +- **Top-line verdict table** — combined scores per MCP with delta columns +- **Per-scenario breakdown** — metrics, judge criterion heatmap, programmatic + check pass rates +- **Side-by-side runs** — click any run to drill into the full trajectory + ## Scoring Combined score = `0.4 * programmatic + 0.6 * judge - toolErrorPenalty` - **Programmatic (40%)** — regex-based keyword checks against the final - answer, defined per-scenario in `ground-truth.json`. -- **LLM-as-judge (60%)** — multi-criteria rubric scored 0-5 by Claude Opus 4.7 - (configurable). Criteria are scenario-specific. + answer, defined per-scenario in `ground-truth.json`. Includes both positive + checks (must find the right thing) and negative checks (must not blame a + distractor). +- **LLM-as-judge (60%)** — multi-criteria rubric scored 0-5 by Claude Opus + (configurable). Criteria are scenario-specific. Answers are **blinded** — + MCP tool names and brand terms are redacted so the judge can't tell which + MCP produced the answer. - **Tool error penalty** — up to 20% deducted for high tool-call error rates. - Prevents agents from papering over correctness with volume. + +### Baseline + Challengers + +Reports use a **baseline + challengers** model. One MCP is designated as the +baseline (default: first MCP alphabetically, or set explicitly with +`--baseline`). All other MCPs show deltas relative to it: + +``` +| Metric | hyperdx-main | hyperdx-feature | Δ (hyperdx-feature) | +|----------------|-------------|-----------------|---------------------| +| Combined score | 85% | 91% | +6% | +``` ## Determinism All randomness flows through a `mulberry32(seed)` PRNG. Same `(seed, now)` -produces byte-identical row content. This is what makes side-by-side -comparison of HyperDX MCP vs ClickHouse MCP meaningful. +produces byte-identical row content. This makes side-by-side comparison +meaningful — both MCPs query the exact same data. + +When using `--anchor-time` with `--no-reseed`, the agent's system prompt +treats the anchor as "now" for relative time references. This is required +when running against pre-seeded data from an earlier session. ## Tests @@ -152,5 +335,5 @@ yarn workspace @hyperdx/hdx-eval dev:unit # watch mode ``` Unit tests verify PRNG determinism, planted anomaly invariants, grading -pipeline, stream parsing, prompt construction, and report aggregation. -No ClickHouse needed. +pipeline, stream parsing, prompt construction, blinding, and report +aggregation. No ClickHouse needed. diff --git a/packages/hdx-eval/eval.config.example.json b/packages/hdx-eval/eval.config.example.json index cf5949288a..ab1b8af4c8 100644 --- a/packages/hdx-eval/eval.config.example.json +++ b/packages/hdx-eval/eval.config.example.json @@ -1,25 +1,71 @@ { - "_comment": "This file is generated by `hdx-eval setup-hyperdx`. Don't edit by hand.", - "hyperdx": { - "apiUrl": "http://localhost:30196", - "mcpUrl": "http://localhost:30196/mcp", - "accessKey": "uuid-from-/me-endpoint", - "connectionId": "mongo-objectid", - "scenarios": { - "error-root-cause": { - "tracesSourceId": "mongo-objectid", - "logsSourceId": "mongo-objectid" + "_comment": "This file is generated by `hdx-eval setup-hyperdx`. You can also edit it by hand to add/modify MCPs.", + "mcps": { + "hyperdx": { + "type": "http", + "url": "http://localhost:30196/mcp", + "headers": { + "Authorization": "Bearer uuid-from-/me-endpoint" }, - "latency-spike": { - "tracesSourceId": "mongo-objectid", - "logsSourceId": "mongo-objectid" + "toolPattern": "mcp__hyperdx__*", + "label": "HyperDX", + "brandTerms": ["HyperDX", "hyperdx"], + "deniedTools": [ + "mcp__hyperdx__hyperdx_delete_dashboard", + "mcp__hyperdx__hyperdx_get_dashboard", + "mcp__hyperdx__hyperdx_save_dashboard", + "mcp__hyperdx__hyperdx_query_tile", + "mcp__hyperdx__hyperdx_get_saved_search", + "mcp__hyperdx__hyperdx_save_saved_search", + "mcp__hyperdx__hyperdx_get_alert", + "mcp__hyperdx__hyperdx_get_webhook", + "mcp__hyperdx__hyperdx_save_alert" + ] + }, + "clickhouse": { + "type": "stdio", + "command": "uv", + "args": [ + "run", + "--with", + "mcp-clickhouse==0.3.0", + "--python", + "3.10", + "mcp-clickhouse" + ], + "env": { + "CLICKHOUSE_HOST": "localhost", + "CLICKHOUSE_PORT": "30596", + "CLICKHOUSE_USER": "default", + "CLICKHOUSE_PASSWORD": "", + "CLICKHOUSE_SECURE": "false", + "CLICKHOUSE_VERIFY": "false", + "CLICKHOUSE_SEND_RECEIVE_TIMEOUT": "10" }, - "noisy-signals": { - "tracesSourceId": "mongo-objectid", - "logsSourceId": "mongo-objectid" - } + "toolPattern": "mcp__clickhouse__*", + "label": "ClickHouse MCP", + "brandTerms": ["ClickHouse MCP", "clickhouse"] + } + }, + "scenarios": { + "error-root-cause": { + "tracesSourceId": "mongo-objectid", + "logsSourceId": "mongo-objectid" + }, + "latency-spike": { + "tracesSourceId": "mongo-objectid", + "logsSourceId": "mongo-objectid" + }, + "noisy-signals": { + "tracesSourceId": "mongo-objectid", + "logsSourceId": "mongo-objectid" } }, + "hyperdxApi": { + "apiUrl": "http://localhost:30196", + "accessKey": "uuid-from-/me-endpoint", + "connectionId": "mongo-objectid" + }, "clickhouse": { "host": "localhost", "port": "30596", diff --git a/packages/hdx-eval/package.json b/packages/hdx-eval/package.json index 55774c8cf3..67b51ae79d 100644 --- a/packages/hdx-eval/package.json +++ b/packages/hdx-eval/package.json @@ -15,7 +15,7 @@ "dependencies": { "@anthropic-ai/sdk": "^0.92.0", "@clickhouse/client": "^1.12.1", - "@hyperdx/common-utils": "^0.19.0", + "@hyperdx/common-utils": "^0.20.0", "commander": "^12.1.0", "ms": "^2.1.3" }, diff --git a/packages/hdx-eval/scripts/viewer/public/app.js b/packages/hdx-eval/scripts/viewer/public/app.js index a1dce8c5bf..c88fb29936 100644 --- a/packages/hdx-eval/scripts/viewer/public/app.js +++ b/packages/hdx-eval/scripts/viewer/public/app.js @@ -20,9 +20,11 @@ batches: [], batch: null, cells: [], + summary: null, selectedRun: null, // { scenario, mcp, idx } runData: null, // { trajectory, grade } activeTab: 'trajectory', + view: 'summary', // 'summary' | 'run' }; // ----- formatting helpers ----- @@ -169,8 +171,11 @@ state.batch = data.batch; state.cells = data.cells || []; state.summary = data.summary; + state.selectedRun = null; + state.runData = null; renderBatchMeta(); renderNav(); + showView('summary'); } async function loadRun(scenario, mcp, idx) { @@ -178,6 +183,7 @@ const u = `/api/batches/${encodeURIComponent(state.batch)}/runs/${encodeURIComponent(scenario)}/${encodeURIComponent(mcp)}/${idx}`; state.runData = await fetchJson(u); renderNav(); + showView('run'); renderRun(); } @@ -195,7 +201,20 @@ function renderBatchMeta() { const sc = state.summary?.scenarios?.length ?? state.cells.length; const cells = state.cells.length; - $('#batch-meta').textContent = `${sc} scenario(s) · ${cells} cell(s)`; + const meta = $('#batch-meta'); + meta.innerHTML = ''; + meta.appendChild(document.createTextNode(`${sc} scenario(s) \u00B7 ${cells} cell(s) `)); + const link = el('a', { + href: '#', + class: 'back-link', + onclick: (e) => { + e.preventDefault(); + state.selectedRun = null; + renderNav(); + showView('summary'); + }, + }, '\u2190 Summary'); + meta.appendChild(link); } // ----- rendering: sidebar nav ----- @@ -267,6 +286,7 @@ // ----- rendering: run ----- function renderRun() { + $('#summary').hidden = true; $('#empty').hidden = true; $('#run').hidden = false; @@ -665,6 +685,406 @@ ); } + // ----- view switching ----- + + function showView(view) { + state.view = view; + const summaryEl = $('#summary'); + const runEl = $('#run'); + const emptyEl = $('#empty'); + if (view === 'summary') { + summaryEl.hidden = false; + runEl.hidden = true; + emptyEl.hidden = true; + renderSummary(); + } else { + summaryEl.hidden = true; + runEl.hidden = false; + emptyEl.hidden = true; + } + } + + // ----- rendering: summary comparison dashboard ----- + + const pct = (x) => (x == null ? '—' : `${(x * 100).toFixed(0)}%`); + const signedPct = (x) => { + if (x == null) return '—'; + const sign = x >= 0 ? '+' : ''; + return `${sign}${(x * 100).toFixed(0)}%`; + }; + const signedNum = (x, d = 1) => { + if (x == null) return '—'; + const sign = x >= 0 ? '+' : ''; + return `${sign}${x.toFixed(d)}`; + }; + const deltaClass = (x, invert = false) => { + if (x == null) return ''; + const v = invert ? -x : x; + if (Math.abs(v) < 0.005) return 'delta-neutral'; + return v > 0 ? 'delta-pos' : 'delta-neg'; + }; + + function renderSummary() { + const root = $('#summary-content'); + root.innerHTML = ''; + + const s = state.summary; + if (!s || !s.scenarios || s.scenarios.length === 0) { + root.className = 'muted center'; + root.textContent = 'No summary data. Run grading and report first.'; + return; + } + root.className = 'summary-dashboard'; + + const mcpOrder = s.mcpOrder || Object.keys(s.scenarios[0]?.cells || {}).sort(); + const baseline = s.baseline || mcpOrder[0]; + const challengers = mcpOrder.filter((m) => m !== baseline); + + // Header info + root.appendChild( + el( + 'div', + { class: 'summary-header' }, + el('span', { class: 'label' }, 'Baseline:'), + el('span', { class: 'value accent' }, baseline), + el('span', { class: 'label' }, 'MCPs:'), + el('span', { class: 'value' }, mcpOrder.join(', ')), + s.generatedAt + ? el( + 'span', + { class: 'muted small' }, + new Date(s.generatedAt).toLocaleString(), + ) + : null, + ), + ); + + // Top-line verdict table + root.appendChild(renderVerdictTable(s, mcpOrder, baseline, challengers)); + + // Per-scenario breakdowns + for (const scenario of s.scenarios) { + root.appendChild( + renderScenarioBreakdown(scenario, mcpOrder, baseline, challengers), + ); + } + } + + function renderVerdictTable(s, mcpOrder, baseline, challengers) { + const headerCells = [el('th', {}, 'Scenario')]; + for (const m of mcpOrder) { + headerCells.push( + el('th', { class: m === baseline ? 'baseline-col' : '' }, m), + ); + } + for (const m of challengers) { + headerCells.push(el('th', { class: 'delta-col' }, `\u0394 ${m}`)); + } + + const rows = s.scenarios.map((sc) => { + const cells = [el('td', { class: 'scenario-name' }, sc.scenario)]; + for (const m of mcpOrder) { + const c = sc.cells?.[m]; + const score = c?.combinedScore?.mean; + cells.push( + el( + 'td', + { class: `score-cell ${scoreClass(score)}` }, + pct(score), + ), + ); + } + for (const m of challengers) { + const d = sc.deltas?.[m]?.combinedScore; + cells.push( + el('td', { class: `delta-cell ${deltaClass(d)}` }, signedPct(d)), + ); + } + return el('tr', {}, ...cells); + }); + + return el( + 'div', + { class: 'summary-section' }, + el('h2', {}, 'Top-line Verdict'), + el( + 'table', + { class: 'summary-table' }, + el('thead', {}, el('tr', {}, ...headerCells)), + el('tbody', {}, ...rows), + ), + ); + } + + function renderScenarioBreakdown(scenario, mcpOrder, baseline, challengers) { + const section = el('div', { class: 'summary-section scenario-breakdown' }); + section.appendChild(el('h2', {}, scenario.scenario)); + + // Metrics table + const metrics = [ + { + label: 'Combined score', + get: (c) => c?.combinedScore?.mean, + fmt: pct, + dKey: 'combinedScore', + dFmt: signedPct, + }, + { + label: 'Programmatic', + get: (c) => c?.programmatic?.mean, + fmt: pct, + dKey: 'programmaticScore', + dFmt: signedPct, + }, + { + label: 'Judge (weighted)', + get: (c) => c?.judge?.weightedMean, + fmt: pct, + dKey: 'judgeWeightedMean', + dFmt: signedPct, + }, + { + label: 'Tool calls', + get: (c) => c?.toolCalls?.mean, + fmt: (x) => (x == null ? '—' : x.toFixed(1)), + dKey: 'toolCalls', + dFmt: (x) => signedNum(x, 1), + invert: true, + }, + { + label: 'Tool errors', + get: (c) => c?.toolErrors?.mean, + fmt: (x) => (x == null ? '—' : x.toFixed(1)), + }, + { + label: 'Error penalty', + get: (c) => c?.toolErrors?.penaltyMean, + fmt: (x) => (x == null ? '—' : `${(x * 100).toFixed(0)}pp`), + }, + { + label: 'Output tokens', + get: (c) => c?.tokens?.output, + fmt: (x) => (x == null ? '—' : Math.round(x).toLocaleString()), + dKey: 'outputTokens', + dFmt: (x) => signedNum(x, 0), + invert: true, + }, + { + label: 'Duration (s)', + get: (c) => c?.durationMs?.mean, + fmt: (x) => (x == null ? '—' : (x / 1000).toFixed(1)), + dKey: 'durationMs', + dFmt: (x) => (x == null ? '—' : signedNum(x / 1000, 1)), + invert: true, + }, + { + label: 'Termination', + get: (c) => c?.termination, + fmt: (x) => { + if (!x) return '—'; + return Object.entries(x) + .sort((a, b) => b[1] - a[1]) + .map(([k, v]) => `${k}: ${v}`) + .join(', '); + }, + }, + { + label: 'N', + get: (c) => c?.n, + fmt: (x) => (x == null ? '—' : String(x)), + }, + ]; + + const headerCells = [el('th', {}, 'Metric')]; + for (const m of mcpOrder) { + headerCells.push( + el('th', { class: m === baseline ? 'baseline-col' : '' }, m), + ); + } + for (const m of challengers) { + headerCells.push(el('th', { class: 'delta-col' }, `\u0394 ${m}`)); + } + + const metricRows = metrics.map((metric) => { + const cells = [el('td', { class: 'metric-label' }, metric.label)]; + for (const m of mcpOrder) { + const c = scenario.cells?.[m]; + const v = metric.get(c); + cells.push(el('td', { class: 'metric-value' }, metric.fmt(v))); + } + for (const m of challengers) { + if (metric.dKey) { + const d = scenario.deltas?.[m]?.[metric.dKey]; + cells.push( + el( + 'td', + { class: `delta-cell ${deltaClass(d, metric.invert)}` }, + metric.dFmt(d), + ), + ); + } else { + cells.push(el('td', { class: 'delta-cell' }, '—')); + } + } + return el('tr', {}, ...cells); + }); + + section.appendChild( + el( + 'table', + { class: 'summary-table metrics-table' }, + el('thead', {}, el('tr', {}, ...headerCells)), + el('tbody', {}, ...metricRows), + ), + ); + + // Judge per-criterion heatmap + const allCriteria = new Set(); + for (const cell of Object.values(scenario.cells || {})) { + if (cell?.judge?.perCriterion) { + for (const id of Object.keys(cell.judge.perCriterion)) allCriteria.add(id); + } + } + if (allCriteria.size > 0) { + const jHeader = [el('th', {}, 'Criterion')]; + for (const m of mcpOrder) jHeader.push(el('th', {}, m)); + const jRows = [...allCriteria].sort().map((id) => { + const cells = [el('td', { class: 'metric-label' }, id)]; + for (const m of mcpOrder) { + const v = scenario.cells?.[m]?.judge?.perCriterion?.[id]; + const cls = criterionHeatClass(v); + cells.push( + el('td', { class: `heatmap-cell ${cls}` }, v != null ? `${v.toFixed(1)}/5` : '—'), + ); + } + return el('tr', {}, ...cells); + }); + section.appendChild( + el( + 'div', + { class: 'sub-table' }, + el('h3', {}, 'Judge per-criterion (0\u20135)'), + el( + 'table', + { class: 'summary-table heatmap' }, + el('thead', {}, el('tr', {}, ...jHeader)), + el('tbody', {}, ...jRows), + ), + ), + ); + } + + // Programmatic per-check + const allChecks = new Set(); + for (const cell of Object.values(scenario.cells || {})) { + if (cell?.programmatic?.perCheck) { + for (const id of Object.keys(cell.programmatic.perCheck)) allChecks.add(id); + } + } + if (allChecks.size > 0) { + const pHeader = [el('th', {}, 'Check')]; + for (const m of mcpOrder) pHeader.push(el('th', {}, m)); + const pRows = [...allChecks].sort().map((id) => { + const isNeg = id.startsWith('false_'); + const label = isNeg ? `${id} (neg)` : id; + const cells = [el('td', { class: 'metric-label' }, label)]; + for (const m of mcpOrder) { + const v = scenario.cells?.[m]?.programmatic?.perCheck?.[id]; + const cls = v != null ? (v >= 0.99 ? 'check-pass' : v >= 0.5 ? 'check-partial' : 'check-fail') : ''; + cells.push( + el('td', { class: `heatmap-cell ${cls}` }, v != null ? pct(v) : '—'), + ); + } + return el('tr', {}, ...cells); + }); + section.appendChild( + el( + 'div', + { class: 'sub-table' }, + el('h3', {}, 'Programmatic per-check (pass rate)'), + el( + 'table', + { class: 'summary-table heatmap' }, + el('thead', {}, el('tr', {}, ...pHeader)), + el('tbody', {}, ...pRows), + ), + ), + ); + } + + // Side-by-side run comparison + const scenarioCells = state.cells.filter((c) => c.scenario === scenario.scenario); + if (scenarioCells.length > 0) { + const maxRuns = Math.max(...scenarioCells.map((c) => c.runs.length)); + const runHeader = [el('th', {}, 'Run')]; + for (const c of scenarioCells) { + runHeader.push(el('th', {}, c.mcp)); + } + const runRows = []; + for (let i = 0; i < maxRuns; i++) { + const cells = [el('td', { class: 'metric-label' }, `#${i}`)]; + for (const c of scenarioCells) { + const r = c.runs[i]; + if (!r) { + cells.push(el('td', {}, '—')); + continue; + } + const chip = el( + 'span', + { class: `score-chip ${scoreClass(r.combinedScore)}` }, + fmtScore(r.combinedScore), + ); + const errBit = + r.toolErrors > 0 + ? el('span', { class: 'tag error small' }, `${r.toolErrors}err`) + : null; + const termBit = + r.termination !== 'final_answer' + ? el('span', { class: 'tag small' }, r.termination) + : null; + cells.push( + el( + 'td', + { + class: 'run-cell clickable', + onclick: () => loadRun(scenario.scenario, c.mcp, r.idx), + }, + chip, + ' ', + el('span', { class: 'muted small' }, `${r.toolCalls ?? '?'}c ${fmtMs(r.durationMs)}`), + termBit ? el('span', {}, ' ', termBit) : null, + errBit ? el('span', {}, ' ', errBit) : null, + ), + ); + } + runRows.push(el('tr', {}, ...cells)); + } + section.appendChild( + el( + 'div', + { class: 'sub-table' }, + el('h3', {}, 'Runs (click to inspect)'), + el( + 'table', + { class: 'summary-table runs-table' }, + el('thead', {}, el('tr', {}, ...runHeader)), + el('tbody', {}, ...runRows), + ), + ), + ); + } + + return section; + } + + function criterionHeatClass(v) { + if (v == null) return ''; + if (v >= 4) return 'heat-high'; + if (v >= 3) return 'heat-mid'; + if (v >= 2) return 'heat-low'; + return 'heat-bad'; + } + // ----- wiring ----- for (const btn of document.querySelectorAll('.tab')) { @@ -680,6 +1100,6 @@ }); loadBatches().catch((e) => { - $('#empty').textContent = `Error loading batches: ${e.message}`; + $('#summary-content').textContent = `Error loading batches: ${e.message}`; }); })(); diff --git a/packages/hdx-eval/scripts/viewer/public/index.html b/packages/hdx-eval/scripts/viewer/public/index.html index 53f583e6e2..ea1b626c65 100644 --- a/packages/hdx-eval/scripts/viewer/public/index.html +++ b/packages/hdx-eval/scripts/viewer/public/index.html @@ -23,7 +23,12 @@
-
+
+
+ Loading... +
+
+