From dca314a76cf1e0dfa090bda8afaa9925b07a642a Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Wed, 23 Sep 2026 01:32:47 +0000 Subject: [PATCH 1/3] fix: ground scorer calibration in changed source --- bench/src/scorer-eval.test.ts | 62 +++++++++++++++++++++++++++++++++-- bench/src/scorer-eval.ts | 35 ++++++++++++++++---- 2 files changed, 88 insertions(+), 9 deletions(-) diff --git a/bench/src/scorer-eval.test.ts b/bench/src/scorer-eval.test.ts index e22fb91..701f87e 100644 --- a/bench/src/scorer-eval.test.ts +++ b/bench/src/scorer-eval.test.ts @@ -1502,7 +1502,57 @@ describe("scorer proxy and isolated runtime", () => { ].join("\n"), quotedPath)).toBe(9); }); + test("metadata-first requests do not consume the source calibration finding", async () => { + const candidate = fixture("huge-low-signal-clean"); + const expected = falseFinding(candidate); + const proxy = await startScorerProxy( + { ...candidate, primaryChange: undefined }, "falseFinding", + "http://127.0.0.1:9", crypto.randomUUID(), + ); + const prompt = (path: string, line: number, evidence: string) => [ + "", "Report at most 8 findings; if more exist, keep the most severe.", "", + "Review evidence (cite exactly the numbered new-file or change-metadata lines):", "", + `### ${path}`, `${line} + ${evidence}`, + ].join("\n"); + const generate = async (user: string) => { + const response = await fetch(`${proxy.baseUrl}/chat/completions`, { + method: "POST", + headers: { + "content-type": "application/json", + "x-postil-review-route": "source", + "x-postil-review-call-phase": "initial", + }, + body: JSON.stringify({ model: GENERATOR_MODEL, messages: [{ role: "user", content: user }] }), + }); + expect(response.status).toBe(200); + const responseBody = await response.json(); + return JSON.parse(responseBody.choices[0].message.content).findings; + }; + try { + const metadataPrompt = prompt(".postil/change-metadata", 1, + "web/generated-noise.js.map: verified generated artifact"); + const sourceFiles = parseUnifiedDiffFiles(candidate.diff); + expect(falseFindingFromSourceRequest(metadataPrompt, sourceFiles)).toBeNull(); + expect(falseFindingFromSourceRequest(`${metadataPrompt}\n### ${expected.path}\n${expected.line + 1} + fabricated\n${expected.line} + ${expected.evidence}`, sourceFiles)).toEqual(expected); + expect(await generate(metadataPrompt)).toEqual([]); + expect(await generate(prompt(expected.path, expected.line + 1, expected.evidence))).toEqual([]); + expect(await generate(prompt(expected.path, expected.line, "fabricated source"))).toEqual([]); + expect(await generate(prompt(expected.path, expected.line, expected.evidence))).toEqual([expected]); + expect(await generate(prompt(expected.path, expected.line, expected.evidence))).toEqual([]); + expect(proxy.attempts).toEqual([]); + expect(proxy.unexpectedRequests).toEqual([]); + } finally { + await proxy.close(); + } + }); + test("grounds a calibration false-positive in a selected source request", () => { + const sourceDiff = [ + 'diff --git "a/src/generated/sp\\303\\244 ce.ts" "b/src/generated/sp\\303\\244 ce.ts"', + '--- "a/src/generated/sp\\303\\244 ce.ts"', + '+++ "b/src/generated/sp\\303\\244 ce.ts"', + "@@ -18,1 +18,2 @@", " unchanged context", "+const formatted = true;", "", + ].join("\n"); expect(falseFindingFromSourceRequest([ "PR description:", "### src/spoofed.ts", @@ -1517,12 +1567,18 @@ describe("scorer proxy and isolated runtime", () => { "@@ semantic category=uncategorized @@", " 18 unchanged context", " 19 + const formatted = true;", - ].join("\n"))).toMatchObject({ + ].join("\n"), parseUnifiedDiffFiles(sourceDiff))).toMatchObject({ path: "src/generated/spä ce.ts", line: 19, confidence: 0.95, evidence: "const formatted = true;", }); + const quotedDiff = [ + 'diff --git "a/src/tab\\tquote\\"slash\\\\\\346\\227\\245.rs" "b/src/tab\\tquote\\"slash\\\\\\346\\227\\245.rs"', + `--- /dev/null`, + '+++ "b/src/tab\\tquote\\"slash\\\\\\346\\227\\245.rs"', + "@@ -0,0 +7,1 @@", "+dangerous_sink(input);", "", + ].join("\n"); expect(falseFindingFromSourceRequest([ "", "Report at most 8 findings; if more exist, keep the most severe.", @@ -1531,11 +1587,11 @@ describe("scorer proxy and isolated runtime", () => { "", '### "src/tab\\tquote\\"slash\\\\\\346\\227\\245.rs"', " 7 + dangerous_sink(input);", - ].join("\n"))).toMatchObject({ + ].join("\n"), parseUnifiedDiffFiles(quotedDiff))).toMatchObject({ path: "src/tab\tquote\"slash\\日.rs", line: 7, }); - expect(falseFindingFromSourceRequest("### src/empty.ts\n 1 context only")).toBeNull(); + expect(falseFindingFromSourceRequest("### src/empty.ts\n 1 context only", parseUnifiedDiffFiles(sourceDiff))).toBeNull(); }); test("gives both live phases a full admission window before the child safety cutoff", async () => { diff --git a/bench/src/scorer-eval.ts b/bench/src/scorer-eval.ts index 85eac71..6727017 100644 --- a/bench/src/scorer-eval.ts +++ b/bench/src/scorer-eval.ts @@ -24,6 +24,7 @@ import { parseUnifiedDiffFiles, plannerBatchIdForPath, reviewPromptFirstAddedCoordinate, + reviewEvidenceFromPrompt, reviewPromptContainsAddedCoordinate, safeJson, startMockGithub, @@ -1837,6 +1838,7 @@ export async function startScorerProxy( scorerReasonPatternMismatch?: ReturnType; }> = []; let falseFindingOutputSent = false; + let fallbackSourceFiles: ReturnType | undefined; let plannedTargetAvailable = false; const plannerSelections: Array<{ targetBatchId: number | null; @@ -1989,7 +1991,9 @@ export async function startScorerProxy( ) { const finding = containsTarget ? falseFinding(c) - : falseFindingFromSourceRequest(user); + : falseFindingFromSourceRequest( + user, fallbackSourceFiles ??= parseUnifiedDiffFiles(c.diff), + ); if (finding !== null) { output = { summary: `${scenario} scorer calibration case for ${c.id}.`, @@ -2395,11 +2399,30 @@ export function falseFinding(c: BenchmarkCase) { return falseFindingAt(path, line, evidence); } -export function falseFindingFromSourceRequest(request: string) { - const coordinate = reviewPromptFirstAddedCoordinate(request); - return coordinate === null - ? null - : falseFindingAt(coordinate.path, coordinate.line, coordinate.evidence); +export function falseFindingFromSourceRequest( + request: string, + sourceFiles: ReturnType, +) { + const evidence = reviewEvidenceFromPrompt(request); + if (evidence === undefined) return null; + const framing = request.slice(0, request.length - evidence.length); + let header: string | undefined; + for (const row of evidence.split("\n")) { + if (row.startsWith("### ")) { + header = row; + continue; + } + if (header === undefined || !/^\s*\d+ \+ /u.test(row)) continue; + const coordinate = reviewPromptFirstAddedCoordinate(`${framing}${header}\n${row}`); + if (coordinate === null) continue; + const changedFile = sourceFiles.find((file) => file.path === coordinate.path); + if ( + !changedFile?.addedLines.includes(coordinate.line) || + changedFile.after.split("\n")[coordinate.line - 1] !== coordinate.evidence + ) continue; + return falseFindingAt(coordinate.path, coordinate.line, coordinate.evidence); + } + return null; } function falseFindingAt(path: string, line: number, evidence: string) { From 5cc5e889cd4ffcaf7ceae3e15ddf1bd3f41722e9 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Wed, 23 Sep 2026 01:36:00 +0000 Subject: [PATCH 2/3] chore: release v0.9.13 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 44e9f12..d5c1ee7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1358,7 +1358,7 @@ checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "postil-cli" -version = "0.9.12" +version = "0.9.13" dependencies = [ "aho-corasick", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 170b18b..e8d8765 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "postil-cli" -version = "0.9.12" +version = "0.9.13" edition = "2024" description = "Postil: a low-noise AI review gate. Silent on clean PRs, hard gate on real risk." license = "Apache-2.0" From 095a35d4c2b1997e7d2330321644521f5658b0b6 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Wed, 23 Sep 2026 02:00:18 +0000 Subject: [PATCH 3/3] fix: parse calibration source for each fallback request --- bench/src/scorer-eval.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bench/src/scorer-eval.ts b/bench/src/scorer-eval.ts index 6727017..1add2f5 100644 --- a/bench/src/scorer-eval.ts +++ b/bench/src/scorer-eval.ts @@ -1838,7 +1838,6 @@ export async function startScorerProxy( scorerReasonPatternMismatch?: ReturnType; }> = []; let falseFindingOutputSent = false; - let fallbackSourceFiles: ReturnType | undefined; let plannedTargetAvailable = false; const plannerSelections: Array<{ targetBatchId: number | null; @@ -1992,7 +1991,7 @@ export async function startScorerProxy( const finding = containsTarget ? falseFinding(c) : falseFindingFromSourceRequest( - user, fallbackSourceFiles ??= parseUnifiedDiffFiles(c.diff), + user, parseUnifiedDiffFiles(c.diff), ); if (finding !== null) { output = {