From 1d44eff30860d88fdbbe7cb4258c106d999146ac Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Wed, 23 Sep 2026 05:38:12 +0000 Subject: [PATCH 1/5] feat(bench): bind calibration and release to Azure US --- .github/workflows/benchmark-calibration.yml | 13 +-- .github/workflows/release.yml | 11 +-- Cargo.lock | 2 +- Cargo.toml | 2 +- bench/README.md | 2 + bench/baseline-us.json | 13 +++ bench/src/cohort.test.ts | 20 +++++ bench/src/compare-baseline.test.ts | 90 ++++++++++++++++++++- bench/src/compare-baseline.ts | 12 +-- bench/src/livemodels.test.ts | 16 +++- bench/src/scorer-eval.test.ts | 83 +++++++++++++++++++ bench/src/scorer-eval.ts | 12 ++- provisional-models-eu.json | 20 +++++ provisional-models.json | 2 +- src/config.rs | 2 +- src/llm.rs | 2 +- 16 files changed, 274 insertions(+), 28 deletions(-) create mode 100644 bench/baseline-us.json create mode 100644 provisional-models-eu.json diff --git a/.github/workflows/benchmark-calibration.yml b/.github/workflows/benchmark-calibration.yml index 451533c..3bf3064 100644 --- a/.github/workflows/benchmark-calibration.yml +++ b/.github/workflows/benchmark-calibration.yml @@ -45,10 +45,10 @@ jobs: - name: Verify embedded model admission working-directory: bench run: bun run verify-admission --allow-provisional - - name: Require an unpopulated Luna baseline + - name: Require an unpopulated Azure/US Luna baseline run: >- jq -e '.profiles["openai/gpt-5.6-luna"].populated == false' - bench/baseline.json >/dev/null + bench/baseline-us.json >/dev/null prepare: needs: validate @@ -373,6 +373,7 @@ jobs: run: | set -euo pipefail bun run bench:compare -- \ + --baseline baseline-us.json \ --binary "${RUNNER_TEMP}/prepared/postil" \ --screen-profile ../provisional-models.json \ --cohort-manifest "${RUNNER_TEMP}/prepared/cohort.json" \ @@ -411,17 +412,17 @@ jobs: id: attest-baseline uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4 with: - subject-path: ${{ github.workspace }}/bench/baseline.json + subject-path: ${{ github.workspace }}/bench/baseline-us.json - name: Stage populated baseline evidence run: | mkdir -p "${RUNNER_TEMP}/populated-baseline" - cp bench/baseline.json "${RUNNER_TEMP}/populated-baseline/baseline.json" + cp bench/baseline-us.json "${RUNNER_TEMP}/populated-baseline/baseline-us.json" cp "${{ steps.attest-baseline.outputs.bundle-path }}" \ - "${RUNNER_TEMP}/populated-baseline/baseline.attestation.json" + "${RUNNER_TEMP}/populated-baseline/baseline-us.attestation.json" - name: Upload populated baseline uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: benchmark-calibration-baseline-${{ github.run_id }} + name: benchmark-calibration-us-baseline-${{ github.run_id }} path: ${{ runner.temp }}/populated-baseline if-no-files-found: error retention-days: 90 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7a788a0..a06ab09 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -71,13 +71,13 @@ jobs: run: | set -euo pipefail profile="openai/gpt-5.6-luna" - test -s bench/baseline.attestation.json || { + test -s bench/baseline-us.attestation.json || { echo "::error::The populated Luna baseline has no committed calibration attestation." exit 1 } source_sha="$(jq -er --arg profile "$profile" \ '.profiles[$profile] | select(.populated == true) | .calibration.sourceSha' \ - bench/baseline.json)" + bench/baseline-us.json)" [[ "$source_sha" =~ ^[0-9a-f]{40,64}$ ]] || { echo "::error::The Luna baseline has an invalid calibration source digest." exit 1 @@ -89,8 +89,8 @@ jobs: echo "::error::The Luna baseline source is not protected by the calibration registry." exit 1 } - gh attestation verify bench/baseline.json \ - --bundle bench/baseline.attestation.json \ + gh attestation verify bench/baseline-us.json \ + --bundle bench/baseline-us.attestation.json \ --repo postil-dev/postil-cli \ --signer-workflow postil-dev/postil-cli/.github/workflows/benchmark-calibration.yml \ --signer-digest "$source_sha" \ @@ -213,7 +213,7 @@ jobs: POSTIL_BIN: ${{ github.workspace }}/target/release/postil POSTIL_SCORER_EVAL_REPEATS: "3" POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER: Azure - POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER_ROUTE: azure/eu + POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER_ROUTE: azure/us POSTIL_QUALIFICATION_SOURCE_SHA: ${{ github.sha }} run: >- bun run scorer-eval --json-out @@ -530,6 +530,7 @@ jobs: set -euo pipefail comparison=0 bun run bench:compare -- \ + --baseline baseline-us.json \ --binary "${{ github.workspace }}/target/release/postil" \ --screen-profile ../provisional-models.json \ --cohort-manifest "${{ runner.temp }}/bench-live-cohort.json" \ diff --git a/Cargo.lock b/Cargo.lock index 84a86c3..70aa69f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1358,7 +1358,7 @@ checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "postil-cli" -version = "0.9.15" +version = "0.9.16" dependencies = [ "aho-corasick", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 14c36a2..a046969 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "postil-cli" -version = "0.9.15" +version = "0.9.16" edition = "2024" description = "Postil: a low-noise AI review gate. Silent on clean PRs, hard gate on real risk." license = "Apache-2.0" diff --git a/bench/README.md b/bench/README.md index 4e6b3d4..07a57f6 100644 --- a/bench/README.md +++ b/bench/README.md @@ -102,6 +102,8 @@ bun run bench:cohort-run -- --mode execute --manifest --slot < Use [the release workflow](../.github/workflows/release.yml) for the five-sample comparison and [the calibration workflow](../.github/workflows/benchmark-calibration.yml) for the ten-sample recorded baseline. Both verify attestations, receipts, and provider generation evidence before comparison or recording. +Calibration and release use the Azure/US route in `provisional-models.json` and the separate `baseline-us.json` dataset. Release requires a populated US baseline with its calibration attestation. The preserved Azure/EU baseline has a different evaluator digest and cannot qualify the US evaluator. Calibration requires ten complete attested samples; its credential balance check is not a total spending cap. + ## Boundaries and deeper reference diff --git a/bench/baseline-us.json b/bench/baseline-us.json new file mode 100644 index 0000000..8a76733 --- /dev/null +++ b/bench/baseline-us.json @@ -0,0 +1,13 @@ +{ + "schemaVersion": 2, + "corpus": { + "fixtureCorpusSha256": "8e4c2cb9ad5a7efdfe6a875566d20133e905155b6f693a873595adf6c069e065", + "evaluatorSha256": "bc108dcce8628c7a15d08d2ba0024a08dff577b504284f3833158e3b63814ed6" + }, + "profiles": { + "openai/gpt-5.6-luna": { + "populated": false, + "instructions": "Record a predeclared ten-slot calibration cohort with provisional-models.json and exact Azure/US provider evidence." + } + } +} diff --git a/bench/src/cohort.test.ts b/bench/src/cohort.test.ts index 2e56bff..deafdf0 100644 --- a/bench/src/cohort.test.ts +++ b/bench/src/cohort.test.ts @@ -95,6 +95,26 @@ describe("cohort manifests", () => { ).rejects.toThrow("evaluatorSha256 is not bound"); }); + test("US calibration receipts cannot be used with the preserved EU profile", async () => { + const execution = calibrationExecution(); + const usProfile = screeningProfilePath; + const euProfile = resolve(import.meta.dir, "..", "..", "provisional-models-eu.json"); + const manifest = await createCohortManifest({ + purpose: "calibration", + binaryPath: process.execPath, + screeningProfilePath: usProfile, + runPrefix: "calibration-us", + execution, + }); + expect(manifest.slots).toHaveLength(10); + await expect(assertManifestBoundToInputs( + manifest, process.execPath, usProfile, githubEnvironment(execution), + )).resolves.toBeUndefined(); + await expect(assertManifestBoundToInputs( + manifest, process.execPath, euProfile, githubEnvironment(execution), + )).rejects.toThrow("screeningProfileSha256 is not bound"); + }); + test("rejects wrong counts, unordered slots, and unbound release execution", async () => { let sequence = 100; const execution = calibrationExecution(); diff --git a/bench/src/compare-baseline.test.ts b/bench/src/compare-baseline.test.ts index ac1d052..0bb02ae 100644 --- a/bench/src/compare-baseline.test.ts +++ b/bench/src/compare-baseline.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from "bun:test"; import { createHash } from "node:crypto"; -import { readFile } from "node:fs/promises"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; import { resolve } from "node:path"; import { cases } from "../fixtures/cases"; import { @@ -47,7 +48,7 @@ import { type ProviderContractEvidence, } from "./livemodels-score"; -test("committed baseline authority matches the current benchmark sources", async () => { +test("EU baseline preserves its fixture identity but is stale for the calibration evaluator", async () => { const baseline = parseBaselineFile(JSON.parse( await readFile(resolve(import.meta.dir, "..", "baseline.json"), "utf8"), )); @@ -56,7 +57,7 @@ test("committed baseline authority matches the current benchmark sources", async .digest("hex"); expect(baseline.corpus.fixtureCorpusSha256).toBe(fixtureCorpusSha256); - expect(baseline.corpus.evaluatorSha256).toBe(await evaluatorSourceSha256()); + expect(baseline.corpus.evaluatorSha256).not.toBe(await evaluatorSourceSha256()); }); test("committed Luna baseline is either fail-closed or has a valid ten-report calibration", async () => { @@ -79,6 +80,41 @@ test("committed Luna baseline is either fail-closed or has a valid ten-report ca expect(() => assertBaselineCalibrationIntegrity(profile)).not.toThrow(); }); +test("US calibration target binds the current evaluator and active provider", async () => { + const root = resolve(import.meta.dir, "..", ".."); + const [euBytes, usBytes, eu, us] = await Promise.all([ + readFile(resolve(root, "bench/baseline.json"), "utf8"), + readFile(resolve(root, "bench/baseline-us.json"), "utf8"), + screeningProfileMetadata(resolve(root, "provisional-models-eu.json")), + screeningProfileMetadata(resolve(root, "provisional-models.json")), + ]); + const euBaseline = parseBaselineFile(JSON.parse(euBytes)); + const usBaseline = parseBaselineFile(JSON.parse(usBytes)); + expect(euBaseline.profiles["openai/gpt-5.6-luna"]?.populated).toBe(true); + const usBaselineProfile = usBaseline.profiles["openai/gpt-5.6-luna"]; + expect(usBaselineProfile).toBeDefined(); + if (usBaselineProfile?.populated) { + expect(isCalibratedBaselineProfile(usBaselineProfile)).toBe(true); + if (!isCalibratedBaselineProfile(usBaselineProfile)) throw new Error("US calibration evidence is missing"); + expect(() => assertBaselineCalibrationIntegrity(usBaselineProfile)).not.toThrow(); + expect(usBaselineProfile.screeningProfileSha256).toBe(us.sha256); + expect(usBaselineProfile.calibration.providerContractSha256).toBe(us.providerContractSha256); + } else { + expect(usBaselineProfile?.instructions).toContain("predeclared ten-slot calibration cohort"); + } + expect(usBaseline.corpus.fixtureCorpusSha256).toBe(euBaseline.corpus.fixtureCorpusSha256); + expect(usBaseline.corpus.evaluatorSha256).not.toBe(euBaseline.corpus.evaluatorSha256); + expect(usBaseline.corpus.evaluatorSha256).toBe(await evaluatorSourceSha256()); + expect(eu.upstreamProviderRoute).toBe("azure/eu"); + expect(us.upstreamProviderRoute).toBe("azure/us"); + expect(us.sha256).not.toBe(eu.sha256); + expect(us.providerContractSha256).not.toBe(eu.providerContractSha256); + expect(us.providerContract).toEqual({ + ...eu.providerContract, + upstreamProviderRoute: "azure/us", + }); +}); + const PROVIDER_CONTRACT: ProviderContractEvidence = { version: 1, benchmarkProviderIdentity: "openrouter:managed-routing", @@ -191,6 +227,33 @@ function fakeReport(options: FakeReportOptions = {}): LiveReportForComparison { }; } +test("comparison CLI rejects preserved EU evidence for the current evaluator", async () => { + const directory = await mkdtemp(resolve(tmpdir(), "postil-stale-baseline-")); + try { + const paths = await Promise.all(Array.from({ length: 1 }, async (_, index) => { + const report = await inputBoundReport(); + const path = resolve(directory, `report-${index}.json`); + await writeFile(path, JSON.stringify(report)); + return { path, runId: report.summary.runId }; + })); + const child = Bun.spawn([ + process.execPath, "--no-env-file", resolve(import.meta.dir, "compare-baseline.ts"), + "--binary", process.execPath, + "--baseline", resolve(import.meta.dir, "..", "baseline.json"), + "--screen-profile", resolve(import.meta.dir, "..", "..", "provisional-models.json"), + ...paths.flatMap(({ path, runId }) => ["--expected-run-id", runId, "--result", path]), + ], { stdout: "pipe", stderr: "pipe", timeout: 30_000 }); + const [exitCode, stderr] = await Promise.all([ + child.exited, new Response(child.stderr).text(), new Response(child.stdout).text(), + ]); + expect(exitCode).toBe(1); + expect(stderr).toContain("FIXTURE CORPUS MISMATCH"); + expect(stderr).toContain("evaluator source than the recorded baseline"); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + function cloneReport(report: LiveReportForComparison): LiveReportForComparison { return structuredClone(report); } @@ -913,6 +976,27 @@ describe("three-report aggregation compatibility", () => { ); }); + test("valid US reports cannot qualify against an EU calibration", async () => { + const us = await screeningProfileMetadata( + resolve(import.meta.dir, "..", "..", "provisional-models.json"), + ); + const reports = Array.from({ length: 5 }, () => { + const report = fakeReport(); + Object.assign(report.summary, { + screeningProfileSha256: us.sha256, + upstreamProviderRoute: us.upstreamProviderRoute, + providerContract: us.providerContract, + providerContractSha256: us.providerContractSha256, + }); + expect(() => assertValidReleaseReport(report)).not.toThrow(); + return report; + }); + const observed = aggregateObservedMetrics(reports); + expect(() => compareMetrics(populatedBaseline, observed)).toThrow( + "baseline calibration execution identity does not match the candidate cohort", + ); + }); + test("cost remains blocking when the provider profile differs", () => { const observed = aggregateObservedMetrics([ fakeReport({ costPerCase: "1" }), diff --git a/bench/src/compare-baseline.ts b/bench/src/compare-baseline.ts index 8d7506c..bdef18a 100644 --- a/bench/src/compare-baseline.ts +++ b/bench/src/compare-baseline.ts @@ -3,7 +3,7 @@ // // Compare mode consumes one, three, or five LiveReport JSON artifacts written by // `bun run bench:live --json-out ` and compares their metrics against the -// committed `bench/baseline.json`. Record mode requires a predeclared ten-report +// committed `bench/baseline-us.json`. Record mode requires a predeclared ten-report // calibration cohort. Every report must be complete full-corpus evidence. A // multi-report operation additionally requires one identical benchmark cohort // and distinct raw artifacts. Exits non-zero on invalid evidence or a material @@ -18,9 +18,9 @@ // --expected-run-id ... --expected-run-id // --result ... --result // -// Record mode writes the ten-sample calibration cohort into baseline.json as the new +// Record mode writes the ten-sample calibration cohort into baseline-us.json as the new // baseline for the reports' model. This is the deliberate re-baseline path: -// nothing updates baseline.json except an explicit --record invocation. +// nothing updates baseline-us.json except an explicit --record invocation. // // bun run bench:compare -- --binary --screen-profile // --expected-run-id ... --expected-run-id @@ -1112,7 +1112,7 @@ function requiredFlagValue(args: readonly string[], index: number, flag: string) } function defaultBaselinePath(): string { - return resolve(import.meta.dir, "..", "baseline.json"); + return resolve(import.meta.dir, "..", "baseline-us.json"); } export function assertDistinctResultPaths(paths: readonly string[]): void { @@ -1604,7 +1604,7 @@ async function main() { baselineFile.corpus.evaluatorSha256 !== observed.evaluatorSha256) { console.error( "FIXTURE CORPUS MISMATCH: this report was scored against a different fixture set or\n" + - "evaluator source than baseline.json was recorded against, so its metrics are not\n" + + "evaluator source than the recorded baseline, so its metrics are not\n" + "comparable.\n" + ` baseline fixtureCorpusSha256 ${baselineFile.corpus.fixtureCorpusSha256}\n` + ` observed fixtureCorpusSha256 ${observed.fixtureCorpusSha256}\n` + @@ -1708,7 +1708,7 @@ async function main() { console.log(formatComparisonTable(comparison.rows)); if (!comparison.ok) { console.error( - "\nRELEASE BLOCKED: the live benchmark regressed past tolerance against bench/baseline.json.\n" + + "\nRELEASE BLOCKED: the live benchmark regressed past tolerance against the recorded baseline.\n" + "Fix the regression, or if the new numbers are an accepted tradeoff, re-baseline\n" + "deliberately with:\n" + rebaselineGuidance, diff --git a/bench/src/livemodels.test.ts b/bench/src/livemodels.test.ts index 06b446d..34297b8 100644 --- a/bench/src/livemodels.test.ts +++ b/bench/src/livemodels.test.ts @@ -1393,6 +1393,18 @@ describe("managed admission workflow", () => { ); expect([...calibrationGenerationVerification.matchAll(/--screen-profile \.\.\/provisional-models\.json/gu)]).toHaveLength(1); expect(calibration).toContain("--record"); + expect(calibration).not.toContain("bench/baseline.json"); + expect(calibration.match(/--screen-profile \.\.\/provisional-models\.json/gu)).toHaveLength(5); + expect(calibration).toContain("--baseline baseline-us.json"); + expect(calibration).toContain("subject-path: ${{ github.workspace }}/bench/baseline-us.json"); + expect(calibration).toContain('cp bench/baseline-us.json "${RUNNER_TEMP}/populated-baseline/baseline-us.json"'); + expect(calibration).toContain("/populated-baseline/baseline-us.attestation.json"); + expect(calibration).toContain("name: benchmark-calibration-us-baseline-${{ github.run_id }}"); + expect(release).not.toContain("bench/baseline.json"); + expect(release).toContain("bench/baseline-us.json"); + expect(release).toContain("--baseline baseline-us.json"); + expect(release).toContain("--bundle bench/baseline-us.attestation.json"); + expect(release).toContain("--signer-workflow postil-dev/postil-cli/.github/workflows/benchmark-calibration.yml"); expect(release).not.toContain("workflow_dispatch"); expect(release).toContain("name: Require the unique first release run for this tag"); expect(release).toContain('if [[ "${GITHUB_RUN_ATTEMPT}" != "1" ]]'); @@ -1401,7 +1413,7 @@ describe("managed admission workflow", () => { expect(release).toContain("group: release-${{ github.ref_name }}"); expect(release).toContain('gh release view "${GITHUB_REF_NAME}"'); expect(release).toContain("name: Verify the attested Luna calibration baseline"); - expect(release).toContain("bench/baseline.attestation.json"); + expect(release).toContain("bench/baseline-us.attestation.json"); expect(release).toContain('git/ref/tags/postil-calibration-${source_sha}'); expect(release).toContain( "--signer-workflow postil-dev/postil-cli/.github/workflows/benchmark-calibration.yml", @@ -1418,7 +1430,7 @@ describe("managed admission workflow", () => { expect(release).not.toContain("POSTIL_SCORER_EVAL_MODELS:"); expect(release).toContain('POSTIL_SCORER_EVAL_REPEATS: "3"'); expect(release).toContain("POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER: Azure"); - expect(release).toContain("POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER_ROUTE: azure/eu"); + expect(release).toContain("POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER_ROUTE: azure/us"); expect(release).toContain("POSTIL_BIN: ${{ github.workspace }}/target/release/postil"); expect(release).not.toContain("Build scorer qualification binary"); expect(release).toContain("bun run scorer-eval --json-out"); diff --git a/bench/src/scorer-eval.test.ts b/bench/src/scorer-eval.test.ts index 701f87e..26cef34 100644 --- a/bench/src/scorer-eval.test.ts +++ b/bench/src/scorer-eval.test.ts @@ -1117,6 +1117,54 @@ describe("scorer proxy and isolated runtime", () => { } }); + for (const phase of ["scorer", "adjudication"] as const) { + test(`latches ${phase} admission timeout before a retry can dispatch upstream`, async () => { + let dispatches = 0; + const upstream = createServer(async (req: IncomingMessage, res: ServerResponse) => { + await requestBody(req); + dispatches++; + if (dispatches === 1) return; + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ usage: { prompt_tokens: 3, completion_tokens: 2, cost: 0.001 } })); + }); + const upstreamBase = await listen(upstream); + const proxy = await startScorerProxy( + fixture("clean-docs-only"), "falseFinding", upstreamBase, crypto.randomUUID(), 100, + ); + const request = (body: object) => fetch(`${proxy.baseUrl}/chat/completions`, { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); + try { + const first = await request(phase === "scorer" ? scorerRequest() : adjudicationRequest()); + expect(first.status).toBe(504); + await first.text(); + for (const body of [scorerRequest(), adjudicationRequest()]) { + const retry = await request(body); + expect(retry.status).toBe(400); + expect(await retry.json()).toEqual({ error: "qualification admission already failed after an upstream timeout" }); + } + expect(dispatches).toBe(1); + expect(proxy.attempts).toHaveLength(1); + expect(proxy.attempts[0]).toMatchObject({ + phase, outcome: "timedOut", costUsd: null, costProviderDecimal: null, + usageValid: false, usagePresent: false, httpStatus: null, + }); + const diagnostics = scorerCaseDiagnostics({ + child: { exitCode: 1, stderr: "", timedOut: false }, attempts: proxy.attempts, + }); + expect(diagnostics.failureSignals).toContain("upstreamTimeout"); + expect(diagnostics.responses[0]).toMatchObject({ + exactCost: "unavailable", accountingIssues: ["responseUnavailable"], + }); + } finally { + await proxy.close(); + upstream.closeAllConnections(); + if (upstream.listening) await close(upstream); + } + }); + } + test("aborts an in-flight upstream request before proxy teardown waits", async () => { let markUpstreamStarted: (() => void) | undefined; const upstreamStarted = new Promise((resolve) => { @@ -2082,6 +2130,41 @@ describe("qualification utilities", () => { }); describe("formatReport", () => { + test("reports estimated timeout cost as unknown until exact accounting is complete", () => { + const cases = qualificationCases(1); + const reportFor = (entries: ScorerEvalCase[]): ScorerEvalReport => ({ + generatedAt: "2026-07-11T00:00:00.000Z", + qualificationSourceSha: "a".repeat(40), cliBinarySha256: "b".repeat(64), + apiBase: "https://example.test/v1", upstreamProvider: "test-provider", + upstreamProviderRoute: "test-provider/route", ...scorerReportContract(), + repeats: 1, completedCases: entries.length, totalCases: entries.length, + matrixComplete: true, passed: false, + models: [aggregate("scorer/model", entries, 1)], cases: entries, + }); + const complete = reportFor(cases); + expect(complete.models[0]!.pricingKnown).toBe(true); + expect(complete.models[0]!.passed).toBe(true); + expect(formatReport(complete)).toContain("$0.000100"); + expect(formatReport(complete)).toContain("Observed provider cost: $0.0012 (complete accounting)"); + for (const accounting of [ + { costProviderDecimal: null, usageAccountingComplete: false }, + { costProviderDecimal: null, usageAccountingComplete: true }, + { costProviderDecimal: "0.0001", usageAccountingComplete: false }, + ]) { + const entries = [...cases]; + entries[0] = { ...entries[0]!, ...accounting, timedOut: true, passed: false, costUsd: 0.001 }; + const report = reportFor(entries); + expect(report.models[0]!.pricingKnown).toBe(false); + expect(report.models[0]!.admissionFailures).toContain("pricing missing for one or more cases"); + expect(report.models[0]!.passed).toBe(false); + const text = formatReport(report); + expect(text.split("\n").find((line) => line.startsWith("scorer/model"))).toContain("unknown"); + expect(text).toContain("Observed provider cost: incomplete accounting"); + expect(text).not.toContain("(complete accounting)"); + } + }); + + test("prints comparable scorer metrics", () => { const report: ScorerEvalReport = { generatedAt: "2026-07-11T00:00:00.000Z", diff --git a/bench/src/scorer-eval.ts b/bench/src/scorer-eval.ts index 1add2f5..a3b6b05 100644 --- a/bench/src/scorer-eval.ts +++ b/bench/src/scorer-eval.ts @@ -1846,6 +1846,7 @@ export async function startScorerProxy( }> = []; const upstreamControllers = new Set(); let upstreamOrdinal = 0; + let admissionTimedOut = false; let closing = false; const server = createServer(async (req: IncomingMessage, res: ServerResponse) => { if (req.method !== "POST" || req.url !== "/chat/completions") { @@ -2014,12 +2015,18 @@ export async function startScorerProxy( return; } + if (admissionTimedOut) { + res.writeHead(400, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: "qualification admission already failed after an upstream timeout" })); + return; + } const ordinal = ++upstreamOrdinal; const controller = new AbortController(); upstreamControllers.add(controller); let deadlineExceeded = false; const timeout = setTimeout(() => { deadlineExceeded = true; + admissionTimedOut = true; controller.abort(); }, upstreamTimeoutMs); const startedAt = performance.now(); @@ -2571,7 +2578,9 @@ export function aggregate( } } } - const pricingKnown = costs.length === cases.length && cases.length > 0; + const pricingKnown = costs.length === cases.length && cases.length > 0 && + cases.every((item) => item.usageAccountingComplete === true && + typeof item.costProviderDecimal === "string"); if (!pricingKnown) admissionFailures.push("pricing missing for one or more cases"); if (structuralPass && p50DurationMs > SCORER_MAX_P50_MS) { admissionFailures.push(`p50 latency ${p50DurationMs.toFixed(0)}ms exceeds ${SCORER_MAX_P50_MS}ms`); @@ -2639,6 +2648,7 @@ export function formatReport(report: ScorerEvalReport): string { const exactCosts = report.cases.map((item) => item.costProviderDecimal); if ( exactCosts.length > 0 && + report.cases.every((item) => item.usageAccountingComplete === true) && exactCosts.every((cost): cost is string => typeof cost === "string") ) { lines.push( diff --git a/provisional-models-eu.json b/provisional-models-eu.json new file mode 100644 index 0000000..f1fa9fc --- /dev/null +++ b/provisional-models-eu.json @@ -0,0 +1,20 @@ +{ + "benchmarkProviderIdentity": "openrouter:managed-routing", + "upstreamProviderIdentity": "Azure", + "upstreamProviderRoute": "azure/eu", + "apiBase": "https://openrouter.ai:443/api/v1", + "apiFormat": "openai-compatible", + "generatorChain": ["openai/gpt-5.6-luna"], + "consensus": 1, + "scorerChain": ["openai/gpt-5.6-luna"], + "providerGenerationModels": { + "openai/gpt-5.6-luna": "openai/gpt-5.6-luna-20260709" + }, + "modelPriceBounds": [ + { + "model": "openai/gpt-5.6-luna", + "inputMicrosPerMillionTokens": 220000, + "outputMicrosPerMillionTokens": 1320000 + } + ] +} diff --git a/provisional-models.json b/provisional-models.json index f1fa9fc..ae5ed97 100644 --- a/provisional-models.json +++ b/provisional-models.json @@ -1,7 +1,7 @@ { "benchmarkProviderIdentity": "openrouter:managed-routing", "upstreamProviderIdentity": "Azure", - "upstreamProviderRoute": "azure/eu", + "upstreamProviderRoute": "azure/us", "apiBase": "https://openrouter.ai:443/api/v1", "apiFormat": "openai-compatible", "generatorChain": ["openai/gpt-5.6-luna"], diff --git a/src/config.rs b/src/config.rs index c7e2d39..d618711 100644 --- a/src/config.rs +++ b/src/config.rs @@ -3954,7 +3954,7 @@ scorer = { enabled = true, default_model = "provider/scorer", reasoning_effort = Some(profile.clone()) ); assert_eq!(profile.upstream_provider_identity, "Azure"); - assert_eq!(profile.upstream_provider_route, "azure/eu"); + assert_eq!(profile.upstream_provider_route, "azure/us"); assert_eq!(profile.generator_chain, vec!["openai/gpt-5.6-luna"]); assert_eq!(profile.scorer_chain, vec!["openai/gpt-5.6-luna"]); assert_eq!( diff --git a/src/llm.rs b/src/llm.rs index d1ee967..08d7014 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -10960,7 +10960,7 @@ mod tests { 0.0, LlmPhase::Review, ); - assert_eq!(body["provider"]["order"], json!(["azure/eu"])); + assert_eq!(body["provider"]["order"], json!(["azure/us"])); assert_eq!(body["provider"]["allow_fallbacks"], false); assert_eq!(body["provider"]["data_collection"], "deny"); assert_eq!(body["provider"]["zdr"], true); From 1a5fe3f7357bb3750ca779d96a109539429eb38a Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Wed, 23 Sep 2026 05:45:46 +0000 Subject: [PATCH 2/5] test(bench): make US calibration evidence primary --- bench/src/compare-baseline.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bench/src/compare-baseline.test.ts b/bench/src/compare-baseline.test.ts index 0bb02ae..1036d3c 100644 --- a/bench/src/compare-baseline.test.ts +++ b/bench/src/compare-baseline.test.ts @@ -82,11 +82,11 @@ test("committed Luna baseline is either fail-closed or has a valid ten-report ca test("US calibration target binds the current evaluator and active provider", async () => { const root = resolve(import.meta.dir, "..", ".."); - const [euBytes, usBytes, eu, us] = await Promise.all([ - readFile(resolve(root, "bench/baseline.json"), "utf8"), + const [usBytes, euBytes, us, eu] = await Promise.all([ readFile(resolve(root, "bench/baseline-us.json"), "utf8"), - screeningProfileMetadata(resolve(root, "provisional-models-eu.json")), + readFile(resolve(root, "bench/baseline.json"), "utf8"), screeningProfileMetadata(resolve(root, "provisional-models.json")), + screeningProfileMetadata(resolve(root, "provisional-models-eu.json")), ]); const euBaseline = parseBaselineFile(JSON.parse(euBytes)); const usBaseline = parseBaselineFile(JSON.parse(usBytes)); From 068f29cd1f6b15eab46fb41cb8109ff1d98640e3 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Wed, 23 Sep 2026 13:11:18 +0000 Subject: [PATCH 3/5] Separate US calibration from the embedded EU release route --- .github/workflows/benchmark-calibration.yml | 10 +++--- .github/workflows/release.yml | 12 +++---- bench/README.md | 10 +++--- bench/baseline-us.json | 4 +-- bench/src/cohort.test.ts | 4 +-- bench/src/compare-baseline.test.ts | 11 +++++-- bench/src/livemodels.test.ts | 19 +++++------ ...dels-eu.json => provisional-models-us.json | 2 +- provisional-models.json | 2 +- src/config.rs | 2 +- src/llm.rs | 32 ++++++++++++------- 11 files changed, 61 insertions(+), 47 deletions(-) rename provisional-models-eu.json => provisional-models-us.json (93%) diff --git a/.github/workflows/benchmark-calibration.yml b/.github/workflows/benchmark-calibration.yml index 3bf3064..3dbc661 100644 --- a/.github/workflows/benchmark-calibration.yml +++ b/.github/workflows/benchmark-calibration.yml @@ -98,7 +98,7 @@ jobs: bun run bench:cohort-create -- --purpose calibration --binary "${{ github.workspace }}/target/release/postil" - --screen-profile ../provisional-models.json + --screen-profile ../provisional-models-us.json --run-prefix "calibration-${{ github.run_id }}" --out "${{ runner.temp }}/benchmark-calibration-cohort.json" - name: Attest the calibration binary and cohort @@ -180,7 +180,7 @@ jobs: --manifest "${{ runner.temp }}/prepared/cohort.json" --slot "${{ matrix.sample }}" --binary "${{ runner.temp }}/prepared/postil" - --screen-profile ../provisional-models.json + --screen-profile ../provisional-models-us.json - name: Attest benchmark sample reservation id: attest-reservation uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4 @@ -203,7 +203,7 @@ jobs: --manifest "${{ runner.temp }}/prepared/cohort.json" --slot "${{ matrix.sample }}" --binary "${{ runner.temp }}/prepared/postil" - --screen-profile ../provisional-models.json + --screen-profile ../provisional-models-us.json - name: Attest benchmark sample result id: attest-result if: success() @@ -347,7 +347,7 @@ jobs: OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} run: | bun run bench:verify-generations -- \ - --screen-profile ../provisional-models.json \ + --screen-profile ../provisional-models-us.json \ --result "${RUNNER_TEMP}/verified-slots/01/report.json" \ --receipt "${RUNNER_TEMP}/verified-slots/01/receipt.json" \ --result "${RUNNER_TEMP}/verified-slots/02/report.json" \ @@ -375,7 +375,7 @@ jobs: bun run bench:compare -- \ --baseline baseline-us.json \ --binary "${RUNNER_TEMP}/prepared/postil" \ - --screen-profile ../provisional-models.json \ + --screen-profile ../provisional-models-us.json \ --cohort-manifest "${RUNNER_TEMP}/prepared/cohort.json" \ --expected-run-id "calibration-${GITHUB_RUN_ID}-01" \ --expected-run-id "calibration-${GITHUB_RUN_ID}-02" \ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a06ab09..33ce794 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -71,13 +71,13 @@ jobs: run: | set -euo pipefail profile="openai/gpt-5.6-luna" - test -s bench/baseline-us.attestation.json || { + test -s bench/baseline.attestation.json || { echo "::error::The populated Luna baseline has no committed calibration attestation." exit 1 } source_sha="$(jq -er --arg profile "$profile" \ '.profiles[$profile] | select(.populated == true) | .calibration.sourceSha' \ - bench/baseline-us.json)" + bench/baseline.json)" [[ "$source_sha" =~ ^[0-9a-f]{40,64}$ ]] || { echo "::error::The Luna baseline has an invalid calibration source digest." exit 1 @@ -89,8 +89,8 @@ jobs: echo "::error::The Luna baseline source is not protected by the calibration registry." exit 1 } - gh attestation verify bench/baseline-us.json \ - --bundle bench/baseline-us.attestation.json \ + gh attestation verify bench/baseline.json \ + --bundle bench/baseline.attestation.json \ --repo postil-dev/postil-cli \ --signer-workflow postil-dev/postil-cli/.github/workflows/benchmark-calibration.yml \ --signer-digest "$source_sha" \ @@ -213,7 +213,7 @@ jobs: POSTIL_BIN: ${{ github.workspace }}/target/release/postil POSTIL_SCORER_EVAL_REPEATS: "3" POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER: Azure - POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER_ROUTE: azure/us + POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER_ROUTE: azure/eu POSTIL_QUALIFICATION_SOURCE_SHA: ${{ github.sha }} run: >- bun run scorer-eval --json-out @@ -530,7 +530,7 @@ jobs: set -euo pipefail comparison=0 bun run bench:compare -- \ - --baseline baseline-us.json \ + --baseline baseline.json \ --binary "${{ github.workspace }}/target/release/postil" \ --screen-profile ../provisional-models.json \ --cohort-manifest "${{ runner.temp }}/bench-live-cohort.json" \ diff --git a/bench/README.md b/bench/README.md index 07a57f6..b75942e 100644 --- a/bench/README.md +++ b/bench/README.md @@ -87,22 +87,22 @@ bun run scorer-eval --json-out It receives the scorer models, repeat count, provider identity, route, credential, and release binary from the [release workflow](../.github/workflows/release.yml). The scorer screen can reject a scorer but cannot admit a hosted profile. -Release and calibration use an immutable cohort before model calls: +Release and calibration use an immutable cohort before model calls. Use `../provisional-models.json` for release and `../provisional-models-us.json` for calibration: ```sh bun run bench:cohort-create -- \ --purpose \ --binary \ - --screen-profile ../provisional-models.json \ + --screen-profile \ --run-prefix \ --out -bun run bench:cohort-run -- --mode reserve --manifest --slot --binary --screen-profile ../provisional-models.json -bun run bench:cohort-run -- --mode execute --manifest --slot --binary --screen-profile ../provisional-models.json +bun run bench:cohort-run -- --mode reserve --manifest --slot --binary --screen-profile +bun run bench:cohort-run -- --mode execute --manifest --slot --binary --screen-profile ``` Use [the release workflow](../.github/workflows/release.yml) for the five-sample comparison and [the calibration workflow](../.github/workflows/benchmark-calibration.yml) for the ten-sample recorded baseline. Both verify attestations, receipts, and provider generation evidence before comparison or recording. -Calibration and release use the Azure/US route in `provisional-models.json` and the separate `baseline-us.json` dataset. Release requires a populated US baseline with its calibration attestation. The preserved Azure/EU baseline has a different evaluator digest and cannot qualify the US evaluator. Calibration requires ten complete attested samples; its credential balance check is not a total spending cap. +Release and the embedded hosted profile use Azure/EU in `provisional-models.json` with `baseline.json` and its attestation. US calibration uses `provisional-models-us.json` and writes the separate `baseline-us.json` dataset. Calibration requires ten complete attested samples; its credential balance check is not a total spending cap. Changes to the evaluator invalidate comparison against an older baseline, including the preserved EU evidence. diff --git a/bench/baseline-us.json b/bench/baseline-us.json index 8a76733..b76e011 100644 --- a/bench/baseline-us.json +++ b/bench/baseline-us.json @@ -2,12 +2,12 @@ "schemaVersion": 2, "corpus": { "fixtureCorpusSha256": "8e4c2cb9ad5a7efdfe6a875566d20133e905155b6f693a873595adf6c069e065", - "evaluatorSha256": "bc108dcce8628c7a15d08d2ba0024a08dff577b504284f3833158e3b63814ed6" + "evaluatorSha256": "2fcc625aa1ff4a32cc520152b222d617875b496bf0dc72ab7800997345e211c4" }, "profiles": { "openai/gpt-5.6-luna": { "populated": false, - "instructions": "Record a predeclared ten-slot calibration cohort with provisional-models.json and exact Azure/US provider evidence." + "instructions": "Record a predeclared ten-slot calibration cohort with provisional-models-us.json and exact Azure/US provider evidence." } } } diff --git a/bench/src/cohort.test.ts b/bench/src/cohort.test.ts index deafdf0..65a268f 100644 --- a/bench/src/cohort.test.ts +++ b/bench/src/cohort.test.ts @@ -97,8 +97,8 @@ describe("cohort manifests", () => { test("US calibration receipts cannot be used with the preserved EU profile", async () => { const execution = calibrationExecution(); - const usProfile = screeningProfilePath; - const euProfile = resolve(import.meta.dir, "..", "..", "provisional-models-eu.json"); + const usProfile = resolve(import.meta.dir, "..", "..", "provisional-models-us.json"); + const euProfile = resolve(import.meta.dir, "..", "..", "provisional-models.json"); const manifest = await createCohortManifest({ purpose: "calibration", binaryPath: process.execPath, diff --git a/bench/src/compare-baseline.test.ts b/bench/src/compare-baseline.test.ts index 1036d3c..d6539e1 100644 --- a/bench/src/compare-baseline.test.ts +++ b/bench/src/compare-baseline.test.ts @@ -80,13 +80,13 @@ test("committed Luna baseline is either fail-closed or has a valid ten-report ca expect(() => assertBaselineCalibrationIntegrity(profile)).not.toThrow(); }); -test("US calibration target binds the current evaluator and active provider", async () => { +test("US calibration target binds the current evaluator separately from the embedded EU provider", async () => { const root = resolve(import.meta.dir, "..", ".."); const [usBytes, euBytes, us, eu] = await Promise.all([ readFile(resolve(root, "bench/baseline-us.json"), "utf8"), readFile(resolve(root, "bench/baseline.json"), "utf8"), + screeningProfileMetadata(resolve(root, "provisional-models-us.json")), screeningProfileMetadata(resolve(root, "provisional-models.json")), - screeningProfileMetadata(resolve(root, "provisional-models-eu.json")), ]); const euBaseline = parseBaselineFile(JSON.parse(euBytes)); const usBaseline = parseBaselineFile(JSON.parse(usBytes)); @@ -107,6 +107,11 @@ test("US calibration target binds the current evaluator and active provider", as expect(usBaseline.corpus.evaluatorSha256).toBe(await evaluatorSourceSha256()); expect(eu.upstreamProviderRoute).toBe("azure/eu"); expect(us.upstreamProviderRoute).toBe("azure/us"); + for (const profile of [us, eu]) { + expect(profile.providerContract.dataCollection).toBe("deny"); + expect(profile.providerContract.zeroDataRetention).toBe(true); + expect(profile.providerContract.allowFallbacks).toBe(false); + } expect(us.sha256).not.toBe(eu.sha256); expect(us.providerContractSha256).not.toBe(eu.providerContractSha256); expect(us.providerContract).toEqual({ @@ -978,7 +983,7 @@ describe("three-report aggregation compatibility", () => { test("valid US reports cannot qualify against an EU calibration", async () => { const us = await screeningProfileMetadata( - resolve(import.meta.dir, "..", "..", "provisional-models.json"), + resolve(import.meta.dir, "..", "..", "provisional-models-us.json"), ); const reports = Array.from({ length: 5 }, () => { const report = fakeReport(); diff --git a/bench/src/livemodels.test.ts b/bench/src/livemodels.test.ts index 34297b8..76c2719 100644 --- a/bench/src/livemodels.test.ts +++ b/bench/src/livemodels.test.ts @@ -1389,21 +1389,22 @@ describe("managed admission workflow", () => { }; const calibrationGenerationVerification = generationVerificationCommand(calibration); expect(calibrationGenerationVerification).toMatch( - /bun run bench:verify-generations -- \\\n\s+--screen-profile \.\.\/provisional-models\.json \\\n\s+--result /u, + /bun run bench:verify-generations -- \\\n\s+--screen-profile \.\.\/provisional-models-us\.json \\\n\s+--result /u, ); - expect([...calibrationGenerationVerification.matchAll(/--screen-profile \.\.\/provisional-models\.json/gu)]).toHaveLength(1); + expect([...calibrationGenerationVerification.matchAll(/--screen-profile \.\.\/provisional-models-us\.json/gu)]).toHaveLength(1); expect(calibration).toContain("--record"); expect(calibration).not.toContain("bench/baseline.json"); - expect(calibration.match(/--screen-profile \.\.\/provisional-models\.json/gu)).toHaveLength(5); + expect(calibration.match(/--screen-profile \.\.\/provisional-models-us\.json/gu)).toHaveLength(5); expect(calibration).toContain("--baseline baseline-us.json"); expect(calibration).toContain("subject-path: ${{ github.workspace }}/bench/baseline-us.json"); expect(calibration).toContain('cp bench/baseline-us.json "${RUNNER_TEMP}/populated-baseline/baseline-us.json"'); expect(calibration).toContain("/populated-baseline/baseline-us.attestation.json"); expect(calibration).toContain("name: benchmark-calibration-us-baseline-${{ github.run_id }}"); - expect(release).not.toContain("bench/baseline.json"); - expect(release).toContain("bench/baseline-us.json"); - expect(release).toContain("--baseline baseline-us.json"); - expect(release).toContain("--bundle bench/baseline-us.attestation.json"); + expect(release).not.toContain("baseline-us"); + expect(release).not.toContain("provisional-models-us.json"); + expect(release).toContain("bench/baseline.json"); + expect(release).toContain("--baseline baseline.json"); + expect(release).toContain("--bundle bench/baseline.attestation.json"); expect(release).toContain("--signer-workflow postil-dev/postil-cli/.github/workflows/benchmark-calibration.yml"); expect(release).not.toContain("workflow_dispatch"); expect(release).toContain("name: Require the unique first release run for this tag"); @@ -1413,7 +1414,7 @@ describe("managed admission workflow", () => { expect(release).toContain("group: release-${{ github.ref_name }}"); expect(release).toContain('gh release view "${GITHUB_REF_NAME}"'); expect(release).toContain("name: Verify the attested Luna calibration baseline"); - expect(release).toContain("bench/baseline-us.attestation.json"); + expect(release).toContain("bench/baseline.attestation.json"); expect(release).toContain('git/ref/tags/postil-calibration-${source_sha}'); expect(release).toContain( "--signer-workflow postil-dev/postil-cli/.github/workflows/benchmark-calibration.yml", @@ -1430,7 +1431,7 @@ describe("managed admission workflow", () => { expect(release).not.toContain("POSTIL_SCORER_EVAL_MODELS:"); expect(release).toContain('POSTIL_SCORER_EVAL_REPEATS: "3"'); expect(release).toContain("POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER: Azure"); - expect(release).toContain("POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER_ROUTE: azure/us"); + expect(release).toContain("POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER_ROUTE: azure/eu"); expect(release).toContain("POSTIL_BIN: ${{ github.workspace }}/target/release/postil"); expect(release).not.toContain("Build scorer qualification binary"); expect(release).toContain("bun run scorer-eval --json-out"); diff --git a/provisional-models-eu.json b/provisional-models-us.json similarity index 93% rename from provisional-models-eu.json rename to provisional-models-us.json index f1fa9fc..ae5ed97 100644 --- a/provisional-models-eu.json +++ b/provisional-models-us.json @@ -1,7 +1,7 @@ { "benchmarkProviderIdentity": "openrouter:managed-routing", "upstreamProviderIdentity": "Azure", - "upstreamProviderRoute": "azure/eu", + "upstreamProviderRoute": "azure/us", "apiBase": "https://openrouter.ai:443/api/v1", "apiFormat": "openai-compatible", "generatorChain": ["openai/gpt-5.6-luna"], diff --git a/provisional-models.json b/provisional-models.json index ae5ed97..f1fa9fc 100644 --- a/provisional-models.json +++ b/provisional-models.json @@ -1,7 +1,7 @@ { "benchmarkProviderIdentity": "openrouter:managed-routing", "upstreamProviderIdentity": "Azure", - "upstreamProviderRoute": "azure/us", + "upstreamProviderRoute": "azure/eu", "apiBase": "https://openrouter.ai:443/api/v1", "apiFormat": "openai-compatible", "generatorChain": ["openai/gpt-5.6-luna"], diff --git a/src/config.rs b/src/config.rs index d618711..c7e2d39 100644 --- a/src/config.rs +++ b/src/config.rs @@ -3954,7 +3954,7 @@ scorer = { enabled = true, default_model = "provider/scorer", reasoning_effort = Some(profile.clone()) ); assert_eq!(profile.upstream_provider_identity, "Azure"); - assert_eq!(profile.upstream_provider_route, "azure/us"); + assert_eq!(profile.upstream_provider_route, "azure/eu"); assert_eq!(profile.generator_chain, vec!["openai/gpt-5.6-luna"]); assert_eq!(profile.scorer_chain, vec!["openai/gpt-5.6-luna"]); assert_eq!( diff --git a/src/llm.rs b/src/llm.rs index 08d7014..3cc5e41 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -10907,28 +10907,33 @@ mod tests { fn benchmark_screening_enforces_the_exact_provisional_provider_contract() { const CHILD_ENV: &str = "POSTIL_TEST_BENCHMARK_SCREENING_CHILD"; if std::env::var_os(CHILD_ENV).is_none() { - let directory = tempfile::tempdir().unwrap(); - let profile_path = directory.path().join("screen-profile.json"); - std::fs::write(&profile_path, include_str!("../provisional-models.json")).unwrap(); - let output = std::process::Command::new(std::env::current_exe().unwrap()) + for (route, profile) in [ + ("azure/eu", include_str!("../provisional-models.json")), + ("azure/us", include_str!("../provisional-models-us.json")), + ] { + let directory = tempfile::tempdir().unwrap(); + let profile_path = directory.path().join("screen-profile.json"); + std::fs::write(&profile_path, profile).unwrap(); + let output = std::process::Command::new(std::env::current_exe().unwrap()) .args([ "--exact", "llm::tests::benchmark_screening_enforces_the_exact_provisional_provider_contract", "--nocapture", ]) - .env(CHILD_ENV, "1") + .env(CHILD_ENV, route) .env("POSTIL_BENCH_SCREEN_PROFILE", &profile_path) .env("POSTIL_BENCH_REQUIRE_HOSTED_PROVIDER_PRIVACY", "1") .env_remove("POSTIL_HOSTED_MODE") .env_remove("POSTIL_QUALIFICATION_CANDIDATE_PROFILE") .output() .unwrap(); - assert!( - output.status.success(), - "isolated screening test failed\nstdout:\n{}\nstderr:\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr), - ); + assert!( + output.status.success(), + "isolated screening test failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + } return; } @@ -10960,7 +10965,10 @@ mod tests { 0.0, LlmPhase::Review, ); - assert_eq!(body["provider"]["order"], json!(["azure/us"])); + assert_eq!( + body["provider"]["order"], + json!([std::env::var(CHILD_ENV).unwrap()]) + ); assert_eq!(body["provider"]["allow_fallbacks"], false); assert_eq!(body["provider"]["data_collection"], "deny"); assert_eq!(body["provider"]["zdr"], true); From f0099c76c938723ce62c252f0cbe347a5842cd24 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Wed, 23 Sep 2026 13:31:26 +0000 Subject: [PATCH 4/5] Freeze US qualification contract behind attested calibration --- .github/workflows/release.yml | 27 +++++++++--------- bench/README.md | 4 +-- bench/baseline-us.json | 2 +- bench/src/cohort.test.ts | 2 +- bench/src/compare-baseline.test.ts | 45 ++++++++++++++++++++++++++++-- bench/src/livemodels.test.ts | 27 +++++++++--------- provisional-models-eu.json | 20 +++++++++++++ src/config.rs | 10 ++++++- src/llm.rs | 2 +- 9 files changed, 104 insertions(+), 35 deletions(-) create mode 100644 provisional-models-eu.json diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 33ce794..df78f4b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -65,19 +65,20 @@ jobs: - name: Verify embedded model admission working-directory: bench run: bun run verify-admission --allow-provisional - - name: Verify the attested Luna calibration baseline + - name: Verify the attested Azure/US Luna calibration baseline env: GH_TOKEN: ${{ github.token }} run: | set -euo pipefail profile="openai/gpt-5.6-luna" - test -s bench/baseline.attestation.json || { - echo "::error::The populated Luna baseline has no committed calibration attestation." + jq -e --arg profile "$profile" '.profiles[$profile].populated == true' \ + bench/baseline-us.json >/dev/null && test -s bench/baseline-us.attestation.json || { + echo "::error::New releases require the populated Azure/US baseline and its main-branch first-attempt calibration attestation." exit 1 } source_sha="$(jq -er --arg profile "$profile" \ '.profiles[$profile] | select(.populated == true) | .calibration.sourceSha' \ - bench/baseline.json)" + bench/baseline-us.json)" [[ "$source_sha" =~ ^[0-9a-f]{40,64}$ ]] || { echo "::error::The Luna baseline has an invalid calibration source digest." exit 1 @@ -89,8 +90,8 @@ jobs: echo "::error::The Luna baseline source is not protected by the calibration registry." exit 1 } - gh attestation verify bench/baseline.json \ - --bundle bench/baseline.attestation.json \ + gh attestation verify bench/baseline-us.json \ + --bundle bench/baseline-us.attestation.json \ --repo postil-dev/postil-cli \ --signer-workflow postil-dev/postil-cli/.github/workflows/benchmark-calibration.yml \ --signer-digest "$source_sha" \ @@ -180,7 +181,7 @@ jobs: bun run bench:cohort-create -- --purpose release --binary "${{ github.workspace }}/target/release/postil" - --screen-profile ../provisional-models.json + --screen-profile ../provisional-models-us.json --run-prefix "release-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}" --out "${{ runner.temp }}/bench-live-cohort.json" @@ -213,7 +214,7 @@ jobs: POSTIL_BIN: ${{ github.workspace }}/target/release/postil POSTIL_SCORER_EVAL_REPEATS: "3" POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER: Azure - POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER_ROUTE: azure/eu + POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER_ROUTE: azure/us POSTIL_QUALIFICATION_SOURCE_SHA: ${{ github.sha }} run: >- bun run scorer-eval --json-out @@ -304,7 +305,7 @@ jobs: --manifest "${{ runner.temp }}/bench-live-cohort.json" --slot "${{ matrix.sample }}" --binary "${{ github.workspace }}/target/release/postil" - --screen-profile ../provisional-models.json + --screen-profile ../provisional-models-us.json - name: Attest benchmark sample reservation id: attest-reservation uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4 @@ -327,7 +328,7 @@ jobs: --manifest "${{ runner.temp }}/bench-live-cohort.json" --slot "${{ matrix.sample }}" --binary "${{ github.workspace }}/target/release/postil" - --screen-profile ../provisional-models.json + --screen-profile ../provisional-models-us.json - name: Verify successful benchmark sample evidence if: success() run: | @@ -511,7 +512,7 @@ jobs: OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} run: | bun run bench:verify-generations -- \ - --screen-profile ../provisional-models.json \ + --screen-profile ../provisional-models-us.json \ --result "${RUNNER_TEMP}/bench-live-reports/slots/01/report.json" \ --receipt "${RUNNER_TEMP}/bench-live-reports/slots/01/receipt.json" \ --result "${RUNNER_TEMP}/bench-live-reports/slots/02/report.json" \ @@ -530,9 +531,9 @@ jobs: set -euo pipefail comparison=0 bun run bench:compare -- \ - --baseline baseline.json \ + --baseline baseline-us.json \ --binary "${{ github.workspace }}/target/release/postil" \ - --screen-profile ../provisional-models.json \ + --screen-profile ../provisional-models-us.json \ --cohort-manifest "${{ runner.temp }}/bench-live-cohort.json" \ --expected-run-id "release-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}-01" \ --expected-run-id "release-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}-02" \ diff --git a/bench/README.md b/bench/README.md index b75942e..e58dcfd 100644 --- a/bench/README.md +++ b/bench/README.md @@ -87,7 +87,7 @@ bun run scorer-eval --json-out It receives the scorer models, repeat count, provider identity, route, credential, and release binary from the [release workflow](../.github/workflows/release.yml). The scorer screen can reject a scorer but cannot admit a hosted profile. -Release and calibration use an immutable cohort before model calls. Use `../provisional-models.json` for release and `../provisional-models-us.json` for calibration: +Release and calibration use an immutable cohort before model calls. Both use `../provisional-models-us.json`: ```sh bun run bench:cohort-create -- \ @@ -102,7 +102,7 @@ bun run bench:cohort-run -- --mode execute --manifest --slot < Use [the release workflow](../.github/workflows/release.yml) for the five-sample comparison and [the calibration workflow](../.github/workflows/benchmark-calibration.yml) for the ten-sample recorded baseline. Both verify attestations, receipts, and provider generation evidence before comparison or recording. -Release and the embedded hosted profile use Azure/EU in `provisional-models.json` with `baseline.json` and its attestation. US calibration uses `provisional-models-us.json` and writes the separate `baseline-us.json` dataset. Calibration requires ten complete attested samples; its credential balance check is not a total spending cap. Changes to the evaluator invalidate comparison against an older baseline, including the preserved EU evidence. +Release and calibration use Azure/US in `provisional-models-us.json`. The embedded hosted profile remains Azure/EU in `provisional-models.json` until authenticated US calibration evidence is imported. Calibration records `baseline-us.json`. New releases fail closed until the populated baseline and its attestation are committed. Calibration requires ten complete attested samples from the first attempt on main; its credential balance check is not a total spending cap. Baseline imports preserve the artifact bytes and every evaluator contract source, including both workflows. Any change to those sources requires calibration of the changed contract. The EU profile, baseline, and attestation remain separate evidence and cannot qualify the US route. diff --git a/bench/baseline-us.json b/bench/baseline-us.json index b76e011..cff0982 100644 --- a/bench/baseline-us.json +++ b/bench/baseline-us.json @@ -2,7 +2,7 @@ "schemaVersion": 2, "corpus": { "fixtureCorpusSha256": "8e4c2cb9ad5a7efdfe6a875566d20133e905155b6f693a873595adf6c069e065", - "evaluatorSha256": "2fcc625aa1ff4a32cc520152b222d617875b496bf0dc72ab7800997345e211c4" + "evaluatorSha256": "962efe39771505e5d45428837e59a38dc6577a1821cba2c4a24358bdd581c71c" }, "profiles": { "openai/gpt-5.6-luna": { diff --git a/bench/src/cohort.test.ts b/bench/src/cohort.test.ts index 65a268f..fa6b913 100644 --- a/bench/src/cohort.test.ts +++ b/bench/src/cohort.test.ts @@ -98,7 +98,7 @@ describe("cohort manifests", () => { test("US calibration receipts cannot be used with the preserved EU profile", async () => { const execution = calibrationExecution(); const usProfile = resolve(import.meta.dir, "..", "..", "provisional-models-us.json"); - const euProfile = resolve(import.meta.dir, "..", "..", "provisional-models.json"); + const euProfile = resolve(import.meta.dir, "..", "..", "provisional-models-eu.json"); const manifest = await createCohortManifest({ purpose: "calibration", binaryPath: process.execPath, diff --git a/bench/src/compare-baseline.test.ts b/bench/src/compare-baseline.test.ts index d6539e1..63cd9d2 100644 --- a/bench/src/compare-baseline.test.ts +++ b/bench/src/compare-baseline.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from "bun:test"; import { createHash } from "node:crypto"; -import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { resolve } from "node:path"; import { cases } from "../fixtures/cases"; @@ -80,20 +80,59 @@ test("committed Luna baseline is either fail-closed or has a valid ten-report ca expect(() => assertBaselineCalibrationIntegrity(profile)).not.toThrow(); }); -test("US calibration target binds the current evaluator separately from the embedded EU provider", async () => { +test("release prerequisite rejects an unpopulated US baseline before attestation or model calls", async () => { + const workflow = await readFile(resolve(import.meta.dir, "..", "..", ".github/workflows/release.yml"), "utf8"); + const start = workflow.indexOf(' jq -e --arg profile "$profile"'); + const end = workflow.indexOf(' source_sha=', start); + expect(start).toBeGreaterThan(0); + expect(end).toBeGreaterThan(start); + const prerequisite = workflow.slice(start, end); + expect(prerequisite).toContain("main-branch first-attempt calibration attestation"); + const directory = await mkdtemp(resolve(tmpdir(), "postil-calibration-prerequisite-")); + try { + await mkdir(resolve(directory, "bench")); + const check = async (): Promise<{ code: number; output: string }> => { + const child = Bun.spawn(["bash", "-c", 'set -euo pipefail\nprofile="openai/gpt-5.6-luna"\n' + prerequisite], { + cwd: directory, stdout: "pipe", stderr: "pipe", timeout: 10_000, + }); + const [code, stdout, stderr] = await Promise.all([child.exited, new Response(child.stdout).text(), new Response(child.stderr).text()]); + return { code, output: stdout + stderr }; + }; + const baseline = (populated: boolean) => JSON.stringify({ profiles: { "openai/gpt-5.6-luna": { populated } } }); + await writeFile(resolve(directory, "bench/baseline-us.json"), baseline(false)); + const missing = await check(); + expect(missing.code).toBe(1); + expect(missing.output).toContain("New releases require the populated Azure/US baseline"); + await writeFile(resolve(directory, "bench/baseline-us.attestation.json"), JSON.stringify({ fixture: true })); + expect((await check()).code).toBe(1); + await writeFile(resolve(directory, "bench/baseline-us.json"), baseline(true)); + expect((await check()).code).toBe(0); + await writeFile(resolve(directory, "bench/baseline-us.attestation.json"), ""); + expect((await check()).code).toBe(1); + expect(workflow.indexOf("gh attestation verify", end)).toBeGreaterThan(end); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test("US calibration target binds the final evaluator separately from the embedded EU provider", async () => { const root = resolve(import.meta.dir, "..", ".."); const [usBytes, euBytes, us, eu] = await Promise.all([ readFile(resolve(root, "bench/baseline-us.json"), "utf8"), readFile(resolve(root, "bench/baseline.json"), "utf8"), screeningProfileMetadata(resolve(root, "provisional-models-us.json")), - screeningProfileMetadata(resolve(root, "provisional-models.json")), + screeningProfileMetadata(resolve(root, "provisional-models-eu.json")), ]); + const euBaseline = parseBaselineFile(JSON.parse(euBytes)); const usBaseline = parseBaselineFile(JSON.parse(usBytes)); expect(euBaseline.profiles["openai/gpt-5.6-luna"]?.populated).toBe(true); const usBaselineProfile = usBaseline.profiles["openai/gpt-5.6-luna"]; expect(usBaselineProfile).toBeDefined(); + const expectedEmbeddedProfile = usBaselineProfile?.populated ? "provisional-models-us.json" : "provisional-models-eu.json"; + expect(await readFile(resolve(root, "provisional-models.json"), "utf8")).toBe(await readFile(resolve(root, expectedEmbeddedProfile), "utf8")); if (usBaselineProfile?.populated) { + expect((await readFile(resolve(root, "bench/baseline-us.attestation.json"))).length).toBeGreaterThan(0); expect(isCalibratedBaselineProfile(usBaselineProfile)).toBe(true); if (!isCalibratedBaselineProfile(usBaselineProfile)) throw new Error("US calibration evidence is missing"); expect(() => assertBaselineCalibrationIntegrity(usBaselineProfile)).not.toThrow(); diff --git a/bench/src/livemodels.test.ts b/bench/src/livemodels.test.ts index 76c2719..1b80999 100644 --- a/bench/src/livemodels.test.ts +++ b/bench/src/livemodels.test.ts @@ -1400,11 +1400,12 @@ describe("managed admission workflow", () => { expect(calibration).toContain('cp bench/baseline-us.json "${RUNNER_TEMP}/populated-baseline/baseline-us.json"'); expect(calibration).toContain("/populated-baseline/baseline-us.attestation.json"); expect(calibration).toContain("name: benchmark-calibration-us-baseline-${{ github.run_id }}"); - expect(release).not.toContain("baseline-us"); - expect(release).not.toContain("provisional-models-us.json"); - expect(release).toContain("bench/baseline.json"); - expect(release).toContain("--baseline baseline.json"); - expect(release).toContain("--bundle bench/baseline.attestation.json"); + expect(release).toContain("New releases require the populated Azure/US baseline and its main-branch first-attempt calibration attestation."); + expect(release).not.toContain("provisional-models.json"); + expect(release).toContain("--screen-profile ../provisional-models-us.json"); + expect(release).not.toContain("bench/baseline.json"); + expect(release).toContain("--baseline baseline-us.json"); + expect(release).toContain("--bundle bench/baseline-us.attestation.json"); expect(release).toContain("--signer-workflow postil-dev/postil-cli/.github/workflows/benchmark-calibration.yml"); expect(release).not.toContain("workflow_dispatch"); expect(release).toContain("name: Require the unique first release run for this tag"); @@ -1413,8 +1414,8 @@ describe("managed admission workflow", () => { expect(release).toContain("This version tag already has another release run."); expect(release).toContain("group: release-${{ github.ref_name }}"); expect(release).toContain('gh release view "${GITHUB_REF_NAME}"'); - expect(release).toContain("name: Verify the attested Luna calibration baseline"); - expect(release).toContain("bench/baseline.attestation.json"); + expect(release).toContain("name: Verify the attested Azure/US Luna calibration baseline"); + expect(release).toContain("bench/baseline-us.attestation.json"); expect(release).toContain('git/ref/tags/postil-calibration-${source_sha}'); expect(release).toContain( "--signer-workflow postil-dev/postil-cli/.github/workflows/benchmark-calibration.yml", @@ -1423,7 +1424,7 @@ describe("managed admission workflow", () => { expect(release).toContain('--source-digest "$source_sha"'); expect(release).toContain("--source-ref refs/heads/main"); expect(release).toMatch(/validate-tag:\n[\s\S]*?permissions:\n\s+contents: read\n\s+actions: read/u); - expect(release).toMatch(/validate-tag:\n[\s\S]*?fetch-depth: 0[\s\S]*?bun-version: 1\.3\.14[\s\S]*?bun install --frozen-lockfile[\s\S]*?bun run verify-admission[\s\S]*?name: Verify the attested Luna calibration baseline[\s\S]*?\n bench-live-prepare:\n/u); + expect(release).toMatch(/validate-tag:\n[\s\S]*?fetch-depth: 0[\s\S]*?bun-version: 1\.3\.14[\s\S]*?bun install --frozen-lockfile[\s\S]*?bun run verify-admission[\s\S]*?name: Verify the attested Azure\/US Luna calibration baseline[\s\S]*?\n bench-live-prepare:\n/u); // The gate derives its model from the binary's embedded configuration so // caller drift cannot benchmark a model absent from the release. expect(release).not.toContain("REVIEW_MODEL:"); @@ -1431,7 +1432,7 @@ describe("managed admission workflow", () => { expect(release).not.toContain("POSTIL_SCORER_EVAL_MODELS:"); expect(release).toContain('POSTIL_SCORER_EVAL_REPEATS: "3"'); expect(release).toContain("POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER: Azure"); - expect(release).toContain("POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER_ROUTE: azure/eu"); + expect(release).toContain("POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER_ROUTE: azure/us"); expect(release).toContain("POSTIL_BIN: ${{ github.workspace }}/target/release/postil"); expect(release).not.toContain("Build scorer qualification binary"); expect(release).toContain("bun run scorer-eval --json-out"); @@ -1486,7 +1487,7 @@ describe("managed admission workflow", () => { expect(sample).toContain( '--manifest "${{ runner.temp }}/bench-live-cohort.json"', ); - expect(sample).toContain("--screen-profile ../provisional-models.json"); + expect(sample).toContain("--screen-profile ../provisional-models-us.json"); expect(sample).not.toContain("--report-out"); expect(sample).not.toContain("--receipt-out"); expect(sample).toContain("uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4"); @@ -1513,9 +1514,9 @@ describe("managed admission workflow", () => { expect(final).toContain("name: Verify independent release generations"); const releaseGenerationVerification = generationVerificationCommand(final); expect(releaseGenerationVerification).toMatch( - /bun run bench:verify-generations -- \\\n\s+--screen-profile \.\.\/provisional-models\.json \\\n\s+--result /u, + /bun run bench:verify-generations -- \\\n\s+--screen-profile \.\.\/provisional-models-us\.json \\\n\s+--result /u, ); - expect([...releaseGenerationVerification.matchAll(/--screen-profile \.\.\/provisional-models\.json/gu)]).toHaveLength(1); + expect([...releaseGenerationVerification.matchAll(/--screen-profile \.\.\/provisional-models-us\.json/gu)]).toHaveLength(1); expect(final).toContain("gh attestation verify"); expect(final).toContain("--deny-self-hosted-runners"); expect(final).toContain("name: bench-live-cohort-${{ github.run_attempt }}"); @@ -1535,7 +1536,7 @@ describe("managed admission workflow", () => { expect([...final.matchAll(/--result /gu)]).toHaveLength(10); expect([...final.matchAll(/--receipt /gu)]).toHaveLength(10); expect(final).toMatch( - /bun run bench:compare --[\s\S]*--binary "\$\{\{ github\.workspace \}\}\/target\/release\/postil"[\s\S]*--screen-profile \.\.\/provisional-models\.json/u, + /bun run bench:compare --[\s\S]*--binary "\$\{\{ github\.workspace \}\}\/target\/release\/postil"[\s\S]*--screen-profile \.\.\/provisional-models-us\.json/u, ); expect(final).toContain("SAMPLE_JOB_RESULT: ${{ needs.bench-live-sample.result }}"); expect(final).toContain('if [[ "${SAMPLE_JOB_RESULT}" != "success" ]]'); diff --git a/provisional-models-eu.json b/provisional-models-eu.json new file mode 100644 index 0000000..f1fa9fc --- /dev/null +++ b/provisional-models-eu.json @@ -0,0 +1,20 @@ +{ + "benchmarkProviderIdentity": "openrouter:managed-routing", + "upstreamProviderIdentity": "Azure", + "upstreamProviderRoute": "azure/eu", + "apiBase": "https://openrouter.ai:443/api/v1", + "apiFormat": "openai-compatible", + "generatorChain": ["openai/gpt-5.6-luna"], + "consensus": 1, + "scorerChain": ["openai/gpt-5.6-luna"], + "providerGenerationModels": { + "openai/gpt-5.6-luna": "openai/gpt-5.6-luna-20260709" + }, + "modelPriceBounds": [ + { + "model": "openai/gpt-5.6-luna", + "inputMicrosPerMillionTokens": 220000, + "outputMicrosPerMillionTokens": 1320000 + } + ] +} diff --git a/src/config.rs b/src/config.rs index c7e2d39..a3d0d5c 100644 --- a/src/config.rs +++ b/src/config.rs @@ -3954,7 +3954,15 @@ scorer = { enabled = true, default_model = "provider/scorer", reasoning_effort = Some(profile.clone()) ); assert_eq!(profile.upstream_provider_identity, "Azure"); - assert_eq!(profile.upstream_provider_route, "azure/eu"); + let calibration: serde_json::Value = + serde_json::from_str(include_str!("../bench/baseline-us.json")).unwrap(); + let calibrated = calibration["profiles"]["openai/gpt-5.6-luna"]["populated"] + .as_bool() + .unwrap(); + assert_eq!( + profile.upstream_provider_route, + if calibrated { "azure/us" } else { "azure/eu" } + ); assert_eq!(profile.generator_chain, vec!["openai/gpt-5.6-luna"]); assert_eq!(profile.scorer_chain, vec!["openai/gpt-5.6-luna"]); assert_eq!( diff --git a/src/llm.rs b/src/llm.rs index 3cc5e41..680b58e 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -10908,7 +10908,7 @@ mod tests { const CHILD_ENV: &str = "POSTIL_TEST_BENCHMARK_SCREENING_CHILD"; if std::env::var_os(CHILD_ENV).is_none() { for (route, profile) in [ - ("azure/eu", include_str!("../provisional-models.json")), + ("azure/eu", include_str!("../provisional-models-eu.json")), ("azure/us", include_str!("../provisional-models-us.json")), ] { let directory = tempfile::tempdir().unwrap(); From 1ad6aa4bef63a263102e7ad36ebc532cf86cd6b4 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Wed, 23 Sep 2026 18:45:18 +0000 Subject: [PATCH 5/5] test(config): pin preparation provider routes independently --- bench/src/compare-baseline.test.ts | 17 ++++------------- src/config.rs | 20 +++++++++++++++----- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/bench/src/compare-baseline.test.ts b/bench/src/compare-baseline.test.ts index 63cd9d2..30234d9 100644 --- a/bench/src/compare-baseline.test.ts +++ b/bench/src/compare-baseline.test.ts @@ -115,7 +115,7 @@ test("release prerequisite rejects an unpopulated US baseline before attestation } }); -test("US calibration target binds the final evaluator separately from the embedded EU provider", async () => { +test("unpopulated US calibration target keeps the embedded provider on Azure/EU", async () => { const root = resolve(import.meta.dir, "..", ".."); const [usBytes, euBytes, us, eu] = await Promise.all([ readFile(resolve(root, "bench/baseline-us.json"), "utf8"), @@ -129,18 +129,9 @@ test("US calibration target binds the final evaluator separately from the embedd expect(euBaseline.profiles["openai/gpt-5.6-luna"]?.populated).toBe(true); const usBaselineProfile = usBaseline.profiles["openai/gpt-5.6-luna"]; expect(usBaselineProfile).toBeDefined(); - const expectedEmbeddedProfile = usBaselineProfile?.populated ? "provisional-models-us.json" : "provisional-models-eu.json"; - expect(await readFile(resolve(root, "provisional-models.json"), "utf8")).toBe(await readFile(resolve(root, expectedEmbeddedProfile), "utf8")); - if (usBaselineProfile?.populated) { - expect((await readFile(resolve(root, "bench/baseline-us.attestation.json"))).length).toBeGreaterThan(0); - expect(isCalibratedBaselineProfile(usBaselineProfile)).toBe(true); - if (!isCalibratedBaselineProfile(usBaselineProfile)) throw new Error("US calibration evidence is missing"); - expect(() => assertBaselineCalibrationIntegrity(usBaselineProfile)).not.toThrow(); - expect(usBaselineProfile.screeningProfileSha256).toBe(us.sha256); - expect(usBaselineProfile.calibration.providerContractSha256).toBe(us.providerContractSha256); - } else { - expect(usBaselineProfile?.instructions).toContain("predeclared ten-slot calibration cohort"); - } + expect(usBaselineProfile?.populated).toBe(false); + expect(usBaselineProfile?.instructions).toContain("predeclared ten-slot calibration cohort"); + expect(await readFile(resolve(root, "provisional-models.json"), "utf8")).toBe(await readFile(resolve(root, "provisional-models-eu.json"), "utf8")); expect(usBaseline.corpus.fixtureCorpusSha256).toBe(euBaseline.corpus.fixtureCorpusSha256); expect(usBaseline.corpus.evaluatorSha256).not.toBe(euBaseline.corpus.evaluatorSha256); expect(usBaseline.corpus.evaluatorSha256).toBe(await evaluatorSourceSha256()); diff --git a/src/config.rs b/src/config.rs index a3d0d5c..1498b91 100644 --- a/src/config.rs +++ b/src/config.rs @@ -3956,13 +3956,11 @@ scorer = { enabled = true, default_model = "provider/scorer", reasoning_effort = assert_eq!(profile.upstream_provider_identity, "Azure"); let calibration: serde_json::Value = serde_json::from_str(include_str!("../bench/baseline-us.json")).unwrap(); - let calibrated = calibration["profiles"]["openai/gpt-5.6-luna"]["populated"] - .as_bool() - .unwrap(); assert_eq!( - profile.upstream_provider_route, - if calibrated { "azure/us" } else { "azure/eu" } + calibration["profiles"]["openai/gpt-5.6-luna"]["populated"], + false ); + assert_eq!(profile.upstream_provider_route, "azure/eu"); assert_eq!(profile.generator_chain, vec!["openai/gpt-5.6-luna"]); assert_eq!(profile.scorer_chain, vec!["openai/gpt-5.6-luna"]); assert_eq!( @@ -4010,6 +4008,18 @@ scorer = { enabled = true, default_model = "provider/scorer", reasoning_effort = } } + #[test] + fn us_calibration_profile_has_its_own_provider_route() { + let calibration_profile = + parse_provisional_hosted_profile(include_str!("../provisional-models-us.json")) + .unwrap(); + let embedded_profile = provisional_hosted_profile(); + + assert_eq!(calibration_profile.upstream_provider_identity, "Azure"); + assert_eq!(calibration_profile.upstream_provider_route, "azure/us"); + assert_eq!(embedded_profile.upstream_provider_route, "azure/eu"); + } + #[test] fn benchmark_screening_projects_an_exact_profile_when_scoring_is_disabled() { let profile = provisional_hosted_profile();