From b423ce3c3432a1867eb28985313d56726f90525f Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Wed, 23 Sep 2026 04:44:35 +0000 Subject: [PATCH 1/3] fix(bench): stop upstream calls after qualification timeout --- bench/src/scorer-eval.test.ts | 48 +++++++++++++++++++++++++++++++++++ bench/src/scorer-eval.ts | 7 +++++ 2 files changed, 55 insertions(+) diff --git a/bench/src/scorer-eval.test.ts b/bench/src/scorer-eval.test.ts index 701f87e..5e664f5 100644 --- a/bench/src/scorer-eval.test.ts +++ b/bench/src/scorer-eval.test.ts @@ -1117,6 +1117,54 @@ describe("scorer proxy and isolated runtime", () => { } }); + for (const phase of ["scorer", "adjudication"] as const) { + test(`latches ${phase} admission timeout before a retry can dispatch upstream`, async () => { + let dispatches = 0; + const upstream = createServer(async (req: IncomingMessage, res: ServerResponse) => { + await requestBody(req); + dispatches++; + if (dispatches === 1) return; + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ usage: { prompt_tokens: 3, completion_tokens: 2, cost: 0.001 } })); + }); + const upstreamBase = await listen(upstream); + const proxy = await startScorerProxy( + fixture("clean-docs-only"), "falseFinding", upstreamBase, crypto.randomUUID(), 100, + ); + const request = (body: object) => fetch(`${proxy.baseUrl}/chat/completions`, { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); + try { + const first = await request(phase === "scorer" ? scorerRequest() : adjudicationRequest()); + expect(first.status).toBe(504); + await first.text(); + for (const body of [scorerRequest(), adjudicationRequest()]) { + const retry = await request(body); + expect(retry.status).toBe(400); + expect(await retry.json()).toEqual({ error: "qualification admission already failed after an upstream timeout" }); + } + expect(dispatches).toBe(1); + expect(proxy.attempts).toHaveLength(1); + expect(proxy.attempts[0]).toMatchObject({ + phase, outcome: "timedOut", costUsd: null, costProviderDecimal: null, + usageValid: false, usagePresent: false, httpStatus: null, + }); + const diagnostics = scorerCaseDiagnostics({ + child: { exitCode: 1, stderr: "", timedOut: false }, attempts: proxy.attempts, + }); + expect(diagnostics.failureSignals).toContain("upstreamTimeout"); + expect(diagnostics.responses[0]).toMatchObject({ + exactCost: "unavailable", accountingIssues: ["responseUnavailable"], + }); + } finally { + await proxy.close(); + upstream.closeAllConnections(); + if (upstream.listening) await close(upstream); + } + }); + } + test("aborts an in-flight upstream request before proxy teardown waits", async () => { let markUpstreamStarted: (() => void) | undefined; const upstreamStarted = new Promise((resolve) => { diff --git a/bench/src/scorer-eval.ts b/bench/src/scorer-eval.ts index 1add2f5..8061d44 100644 --- a/bench/src/scorer-eval.ts +++ b/bench/src/scorer-eval.ts @@ -1846,6 +1846,7 @@ export async function startScorerProxy( }> = []; const upstreamControllers = new Set(); let upstreamOrdinal = 0; + let admissionTimedOut = false; let closing = false; const server = createServer(async (req: IncomingMessage, res: ServerResponse) => { if (req.method !== "POST" || req.url !== "/chat/completions") { @@ -2014,12 +2015,18 @@ export async function startScorerProxy( return; } + if (admissionTimedOut) { + res.writeHead(400, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: "qualification admission already failed after an upstream timeout" })); + return; + } const ordinal = ++upstreamOrdinal; const controller = new AbortController(); upstreamControllers.add(controller); let deadlineExceeded = false; const timeout = setTimeout(() => { deadlineExceeded = true; + admissionTimedOut = true; controller.abort(); }, upstreamTimeoutMs); const startedAt = performance.now(); From ba55395a7104e09c979c0bd22a4170de1fbc8810 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Wed, 23 Sep 2026 04:52:01 +0000 Subject: [PATCH 2/3] fix(bench): require complete accounting for aggregate cost --- bench/src/scorer-eval.test.ts | 35 +++++++++++++++++++++++++++++++++++ bench/src/scorer-eval.ts | 5 ++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/bench/src/scorer-eval.test.ts b/bench/src/scorer-eval.test.ts index 5e664f5..26cef34 100644 --- a/bench/src/scorer-eval.test.ts +++ b/bench/src/scorer-eval.test.ts @@ -2130,6 +2130,41 @@ describe("qualification utilities", () => { }); describe("formatReport", () => { + test("reports estimated timeout cost as unknown until exact accounting is complete", () => { + const cases = qualificationCases(1); + const reportFor = (entries: ScorerEvalCase[]): ScorerEvalReport => ({ + generatedAt: "2026-07-11T00:00:00.000Z", + qualificationSourceSha: "a".repeat(40), cliBinarySha256: "b".repeat(64), + apiBase: "https://example.test/v1", upstreamProvider: "test-provider", + upstreamProviderRoute: "test-provider/route", ...scorerReportContract(), + repeats: 1, completedCases: entries.length, totalCases: entries.length, + matrixComplete: true, passed: false, + models: [aggregate("scorer/model", entries, 1)], cases: entries, + }); + const complete = reportFor(cases); + expect(complete.models[0]!.pricingKnown).toBe(true); + expect(complete.models[0]!.passed).toBe(true); + expect(formatReport(complete)).toContain("$0.000100"); + expect(formatReport(complete)).toContain("Observed provider cost: $0.0012 (complete accounting)"); + for (const accounting of [ + { costProviderDecimal: null, usageAccountingComplete: false }, + { costProviderDecimal: null, usageAccountingComplete: true }, + { costProviderDecimal: "0.0001", usageAccountingComplete: false }, + ]) { + const entries = [...cases]; + entries[0] = { ...entries[0]!, ...accounting, timedOut: true, passed: false, costUsd: 0.001 }; + const report = reportFor(entries); + expect(report.models[0]!.pricingKnown).toBe(false); + expect(report.models[0]!.admissionFailures).toContain("pricing missing for one or more cases"); + expect(report.models[0]!.passed).toBe(false); + const text = formatReport(report); + expect(text.split("\n").find((line) => line.startsWith("scorer/model"))).toContain("unknown"); + expect(text).toContain("Observed provider cost: incomplete accounting"); + expect(text).not.toContain("(complete accounting)"); + } + }); + + test("prints comparable scorer metrics", () => { const report: ScorerEvalReport = { generatedAt: "2026-07-11T00:00:00.000Z", diff --git a/bench/src/scorer-eval.ts b/bench/src/scorer-eval.ts index 8061d44..a3b6b05 100644 --- a/bench/src/scorer-eval.ts +++ b/bench/src/scorer-eval.ts @@ -2578,7 +2578,9 @@ export function aggregate( } } } - const pricingKnown = costs.length === cases.length && cases.length > 0; + const pricingKnown = costs.length === cases.length && cases.length > 0 && + cases.every((item) => item.usageAccountingComplete === true && + typeof item.costProviderDecimal === "string"); if (!pricingKnown) admissionFailures.push("pricing missing for one or more cases"); if (structuralPass && p50DurationMs > SCORER_MAX_P50_MS) { admissionFailures.push(`p50 latency ${p50DurationMs.toFixed(0)}ms exceeds ${SCORER_MAX_P50_MS}ms`); @@ -2646,6 +2648,7 @@ export function formatReport(report: ScorerEvalReport): string { const exactCosts = report.cases.map((item) => item.costProviderDecimal); if ( exactCosts.length > 0 && + report.cases.every((item) => item.usageAccountingComplete === true) && exactCosts.every((cost): cost is string => typeof cost === "string") ) { lines.push( From 4c5d22d952a2468a73d9a8cf107d6b017485b638 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Wed, 23 Sep 2026 09:07:27 +0000 Subject: [PATCH 3/3] fix(bench): bound scorer qualification at thirty seconds --- Cargo.lock | 2 +- Cargo.toml | 2 +- bench/src/scorer-eval.test.ts | 21 ++++++++++++++++++++- bench/src/scorer-eval.ts | 2 +- 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 84a86c3..70aa69f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1358,7 +1358,7 @@ checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "postil-cli" -version = "0.9.15" +version = "0.9.16" dependencies = [ "aho-corasick", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 14c36a2..a046969 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "postil-cli" -version = "0.9.15" +version = "0.9.16" edition = "2024" description = "Postil: a low-noise AI review gate. Silent on clean PRs, hard gate on real risk." license = "Apache-2.0" diff --git a/bench/src/scorer-eval.test.ts b/bench/src/scorer-eval.test.ts index 26cef34..cd9c4fa 100644 --- a/bench/src/scorer-eval.test.ts +++ b/bench/src/scorer-eval.test.ts @@ -18,6 +18,7 @@ import { SCORER_CASE_EXEC_TIMEOUT_MS, SCORER_CASE_HARNESS_ALLOWANCE_MS, SCORER_MAX_CASE_MS, + SCORER_PROXY_UPSTREAM_TIMEOUT_MS, SCORER_REASON_SCHEMA_PATTERN, SCORER_DIAGNOSTIC_RESPONSE_LIMIT, TRUE_FINDING_CASES, @@ -1956,6 +1957,24 @@ describe("aggregate", () => { }); }); + test("admits a 30-second maximum without relaxing percentile limits", () => { + expect(SCORER_MAX_CASE_MS).toBe(30_000); + expect(SCORER_PROXY_UPSTREAM_TIMEOUT_MS).toBe(30_000); + expect(SCORER_CASE_EXEC_TIMEOUT_MS).toBe(65_000); + const cases = qualificationCases(3); + cases[0]!.durationMs = 30_000; + expect(aggregate("scorer/model", cases, 3)).toMatchObject({ + passed: true, maxDurationMs: 30_000, p50DurationMs: 1000, p95DurationMs: 1000, + }); + cases[0]!.durationMs = 30_001; + expect(aggregate("scorer/model", cases, 3).admissionFailures) + .toContain("max latency 30001ms exceeds 30000ms"); + for (const entry of cases) entry.durationMs = 10_001; + const failures = aggregate("scorer/model", cases, 3).admissionFailures; + expect(failures).toContain("p50 latency 10001ms exceeds 5000ms"); + expect(failures).toContain("p95 latency 10001ms exceeds 10000ms"); + }); + test("reports a timed-out scorer case without double-counting a structured failure", () => { const cases = qualificationCases(1); cases[0] = result({ @@ -2005,7 +2024,7 @@ describe("aggregate", () => { c.findingPublished = true; c.passed = false; } - cases[0]!.durationMs = 20_001; + cases[0]!.durationMs = SCORER_MAX_CASE_MS + 1; cases[1]!.costUsd = null; const aggregateResult = aggregate("scorer/model", cases, 5); expect(aggregateResult.passed).toBe(false); diff --git a/bench/src/scorer-eval.ts b/bench/src/scorer-eval.ts index a3b6b05..c10114f 100644 --- a/bench/src/scorer-eval.ts +++ b/bench/src/scorer-eval.ts @@ -55,7 +55,7 @@ export const DEFAULT_QUALIFICATION_REPEATS = 5; export const SCORER_REASON_MAX_BYTES = 240; export const SCORER_MAX_P50_MS = 5_000; export const SCORER_MAX_P95_MS = 10_000; -export const SCORER_MAX_CASE_MS = 20_000; +export const SCORER_MAX_CASE_MS = 30_000; export const SCORER_CASE_HARNESS_ALLOWANCE_MS = 5_000; export const SCORER_CASE_EXEC_TIMEOUT_MS = 2 * SCORER_MAX_CASE_MS + SCORER_CASE_HARNESS_ALLOWANCE_MS;