Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "postil-cli"
version = "0.9.11"
version = "0.9.12"
edition = "2024"
description = "Postil: a low-noise AI review gate. Silent on clean PRs, hard gate on real risk."
license = "Apache-2.0"
Expand Down
76 changes: 75 additions & 1 deletion bench/src/scorer-eval.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -448,6 +448,75 @@ describe("scorer case diagnostics", () => {
attempts: [accountedAttempt], envelope: { findings: [] }, passed: false,
};

const validationPrefix = "postil: finding adjudication validation failed; preserving all generated findings: ";
const validationCases = [
["causal change is not exact changed-source evidence", "causalChangeNotExact"],
["causal change source evidence is ambiguous without an exact position", "causalPositionAmbiguous"],
["refuted adjudication must cite candidate-specific contradictory evidence", "refutationUnsupported"],
["publication anchor has no verified source role", "anchorSourceRoleMissing"],
["added anchor evidence is not an exact source slice", "addedAnchorEvidenceNotExact"],
["scope evidence has an incomplete diff hunk", "incompleteScopeHunk"],
["scope assessment requires a bounded reason", "scopeReasonInvalid"],
["context-anchored confirmation requires a causal change reference", "causalChangeMissing"],
["adjudication returned an unknown candidate identity", "unknownCandidateIdentity"],
["adjudication returned a duplicate candidate identity", "duplicateCandidateIdentity"],
] as const;

test("maps only complete static validation lines and preserves existing failure signals", () => {
for (const [message, code] of validationCases) {
for (const ending of ["\n", "\r\n"]) {
const diagnostics = scorerCaseDiagnostics({ ...diagnosticInput,
child: { ...diagnosticInput.child, stderr: `${validationPrefix}${message}${ending}` },
});
expect(diagnostics.adjudicationValidationCodes).toEqual([code]);
expect(diagnostics.adjudicationValidationCodesOmitted).toBe(0);
expect(diagnostics.failureSignals).toEqual([
code === "refutationUnsupported" ? "unsupportedRefutation" : "adjudicationValidation",
]);
}
}
});

test("redacts unknown, suffixed, embedded and control-modified validation text", () => {
const marker = crypto.randomUUID();
const known = validationCases[0][0];
for (const message of ["", marker, `${known} ${marker}`, `${known}: ${marker}`, ` ${known}`, `${known}\r`, `\u001b[31m${known}`, `${known}\0${marker}`]) {
const diagnostics = scorerCaseDiagnostics({ ...diagnosticInput,
child: { ...diagnosticInput.child, stderr: `${validationPrefix}${message}` },
});
expect(diagnostics.adjudicationValidationCodes).toEqual(["unknownValidationReason"]);
expect(JSON.stringify(diagnostics)).not.toContain(marker);
expect(JSON.stringify(diagnostics)).not.toContain(known);
}
for (const line of [marker, `prefix ${validationPrefix}${known}`, `\u001b[31m${validationPrefix}${known}`, known, ""]) {
const diagnostics = scorerCaseDiagnostics({ ...diagnosticInput,
child: { ...diagnosticInput.child, stderr: line },
});
expect(diagnostics.adjudicationValidationCodes).toEqual([]);
expect(diagnostics.adjudicationValidationCodesOmitted).toBe(0);
expect(JSON.stringify(diagnostics)).not.toContain(marker);
}
});

test("deduplicates and bounds validation codes independently of stderr order", () => {
const lines = validationCases.map(([message]) => `${validationPrefix}${message}`);
lines.push(`${validationPrefix}${crypto.randomUUID()}`);
const expected = [...validationCases.map(([, code]) => code), "unknownValidationReason"].sort();
const build = (stderr: string) => scorerCaseDiagnostics({ ...diagnosticInput,
child: { ...diagnosticInput.child, stderr },
});
const first = build(lines.join("\n"));
const reversed = build([...lines, ...lines].reverse().join("\r\n"));
expect(first.adjudicationValidationCodes).toEqual(expected.slice(0, 8));
expect(first.adjudicationValidationCodesOmitted).toBe(expected.length - 8);
expect(reversed.adjudicationValidationCodes).toEqual(first.adjudicationValidationCodes);
expect(reversed.adjudicationValidationCodesOmitted).toBe(first.adjudicationValidationCodesOmitted);
const original = result({ usageAccountingComplete: true, usageValid: false, upstreamRequests: 1, passed: false });
const observed = { ...original, diagnostics: first };
expect(aggregate(original.model, [observed], 1)).toEqual(aggregate(original.model, [original], 1));
expect(isAdmissionFatalStructuralResult(observed, original.model)).toBe(isAdmissionFatalStructuralResult(original, original.model));
});

test("retains error then recovery in dispatch order without changing qualification", () => {
const failed = {
...accountedAttempt, ordinal: 1, usagePresent: false, usageValid: false,
Expand Down Expand Up @@ -823,9 +892,14 @@ describe("scorer proxy and isolated runtime", () => {
}
expect(requestIndex).toBe(3);
const diagnostics = scorerCaseDiagnostics({
child: { exitCode: 0, timedOut: false, stderr: "" }, attempts: proxy.attempts,
child: { exitCode: 0, timedOut: false, stderr: [
"postil: finding adjudication validation failed; preserving all generated findings: causal change is not exact changed-source evidence",
`postil: finding adjudication validation failed; preserving all generated findings: ${marker}`,
].join("\r\n") }, attempts: proxy.attempts,
envelope: { findings: [] }, passed: false,
});
expect(diagnostics.adjudicationValidationCodes).toEqual(["causalChangeNotExact", "unknownValidationReason"]);
expect(diagnostics.adjudicationValidationCodesOmitted).toBe(0);
expect(diagnostics.responses.map(({ ordinal, httpStatus, numericErrorCodes, exactCost, accountingIssues }) =>
({ ordinal, httpStatus, numericErrorCodes, exactCost, accountingIssues }))).toEqual([
{ ordinal: 1, httpStatus: 429, numericErrorCodes: [429], exactCost: "unavailable", accountingIssues: ["usageMissing", "costUnavailable"] },
Expand Down
59 changes: 59 additions & 0 deletions bench/src/scorer-eval.ts
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,57 @@ type ScorerFailureSignal = "childTimeout" | "invalidEnvelope" |
"adjudicationProvider" | "adjudicationUnavailable" | "upstreamTimeout" |
"upstreamFailure" | "upstreamHttp" | "unknown";

const ADJUDICATION_VALIDATION_CODES = new Map<string, string>([
["adjudication candidate count exceeds its hard bound", "candidateCountExceeded"],
["adjudication must return exactly one result per candidate", "resultCountMismatch"],
["adjudication candidate identity count mismatch", "candidateIdentityCountMismatch"],
["adjudication candidate identities are not unique", "candidateIdentitiesNotUnique"],
["adjudication returned an unknown candidate identity", "unknownCandidateIdentity"],
["adjudication returned a duplicate candidate identity", "duplicateCandidateIdentity"],
["factual refutation cannot carry a scope assessment", "refutationHasScope"],
["pre-existing scope cannot declare a duplicate", "preExistingDuplicate"],
["pre-existing scope requires an explicit factual assessment", "preExistingAssessmentMissing"],
["adjudication candidate cannot duplicate itself", "selfDuplicate"],
["adjudication duplicate references an unknown candidate identity", "unknownDuplicateIdentity"],
["only a confirmed candidate can be collapsed as a duplicate", "unconfirmedDuplicate"],
["adjudication omitted a candidate identity", "candidateIdentityMissing"],
["duplicate primary disappeared", "duplicatePrimaryMissing"],
["duplicate primary must be a retained confirmed candidate", "duplicatePrimaryInvalid"],
["semantic duplicates must establish one identical canonical defect", "duplicateCanonicalMismatch"],
["semantic duplicate must retain the more concrete primary kind", "duplicatePrimaryKindInvalid"],
["adjudication direct-source receipt snapshot mismatch", "snapshotMismatch"],
["adjudication candidate origin receipt mismatch", "candidateOriginMismatch"],
["confirmed adjudication must include revised publication text and evidence", "confirmationTextMissing"],
["confirmed adjudication evidence is not in a supplied evidence window or structured receipt", "confirmationEvidenceMissing"],
["repository-dependent finding is not supported by an exact complete receipt", "repositoryClaimUnsupported"],
["confirmed adjudication describes evidence boundaries", "publicationEvidenceBoundary"],
["confirmed adjudication makes an undeclared repository-wide claim", "undeclaredRepositoryClaim"],
["refuted adjudication cannot publish revised finding text", "refutationHasPublicationText"],
["refuted adjudication must cite candidate-specific contradictory evidence", "refutationUnsupported"],
["unresolved adjudication cannot publish text, evidence, or duplicate identity", "unresolvedPublication"],
["scope coordinate overflow", "scopeCoordinateOverflow"],
["scope evidence has an incomplete diff hunk", "incompleteScopeHunk"],
["scope source coordinate is ambiguous", "ambiguousScopeCoordinate"],
["scope evidence has an invalid diff hunk", "invalidScopeHunk"],
["scope evidence corpus identity mismatch", "scopeCorpusMismatch"],
["scope assessment requires a bounded reason", "scopeReasonInvalid"],
["causal change reference exceeds its bounds", "causalReferenceBounds"],
["causal change must reference an addition or deletion", "causalRoleInvalid"],
["publication anchor has conflicting source roles", "anchorSourceRoleConflict"],
["publication anchor has no verified source role", "anchorSourceRoleMissing"],
["pre-existing scope requires an unchanged anchor and no causal change", "preExistingAnchorInvalid"],
["pre-existing scope requires exact unchanged-anchor evidence", "preExistingEvidenceMissing"],
["pre-existing scope requires complete candidate evidence", "preExistingEvidenceIncomplete"],
["causal change is not exact changed-source evidence", "causalChangeNotExact"],
["causal change source evidence is ambiguous without an exact position", "causalPositionAmbiguous"],
["long added anchor requires a bounded exact causal source slice", "causalSliceMissing"],
["added anchor evidence is not an exact source slice", "addedAnchorEvidenceNotExact"],
["metadata cause exceeds its evidence bound", "metadataCauseBounds"],
["context-anchored confirmation requires a causal change reference", "causalChangeMissing"],
["validated adjudication result disappeared", "validatedResultMissing"],
]);
const ADJUDICATION_VALIDATION_CODE_LIMIT = 8;

interface ScorerPhaseDiagnostics {
attempts: number;
collectedResponses: number;
Expand All @@ -143,6 +194,8 @@ export interface ScorerCaseDiagnostics {
adjudication: ScorerPhaseDiagnostics;
scorer: ScorerPhaseDiagnostics;
failureSignals: ScorerFailureSignal[];
adjudicationValidationCodes: string[];
adjudicationValidationCodesOmitted: number;
publicationFailureCodes: string[];
responses: ScorerResponseDiagnostics[];
responsesOmitted: number;
Expand Down Expand Up @@ -251,6 +304,10 @@ export function scorerCaseDiagnostics(input: {
const signals = new Set<ScorerFailureSignal>();
const lines = new Set(input.child.stderr.split(/\r?\n/u));
const validationPrefix = "postil: finding adjudication validation failed; preserving all generated findings: ";
const validationCodes = [...new Set([...lines]
.filter((line) => line.startsWith(validationPrefix))
.map((line) => ADJUDICATION_VALIDATION_CODES.get(line.slice(validationPrefix.length))
?? "unknownValidationReason"))].sort();
if (lines.has(`${validationPrefix}refuted adjudication must cite candidate-specific contradictory evidence`)) {
signals.add("unsupportedRefutation");
} else if ([...lines].some((line) => line.startsWith(validationPrefix))) {
Expand Down Expand Up @@ -295,6 +352,8 @@ export function scorerCaseDiagnostics(input: {
adjudication: phaseCounts("adjudication"),
scorer: phaseCounts("scorer"),
failureSignals: [...signals],
adjudicationValidationCodes: validationCodes.slice(0, ADJUDICATION_VALIDATION_CODE_LIMIT),
adjudicationValidationCodesOmitted: Math.max(0, validationCodes.length - ADJUDICATION_VALIDATION_CODE_LIMIT),
publicationFailureCodes: ["check-run-state", "review-count", "comment-count", "missing-anchor"]
.filter((code) => input.publicationFailureCodes?.includes(code)),
responses: input.attempts
Expand Down
Loading
Loading