diff --git a/.github/workflows/benchmark-calibration.yml b/.github/workflows/benchmark-calibration.yml index 451533c..3dbc661 100644 --- a/.github/workflows/benchmark-calibration.yml +++ b/.github/workflows/benchmark-calibration.yml @@ -45,10 +45,10 @@ jobs: - name: Verify embedded model admission working-directory: bench run: bun run verify-admission --allow-provisional - - name: Require an unpopulated Luna baseline + - name: Require an unpopulated Azure/US Luna baseline run: >- jq -e '.profiles["openai/gpt-5.6-luna"].populated == false' - bench/baseline.json >/dev/null + bench/baseline-us.json >/dev/null prepare: needs: validate @@ -98,7 +98,7 @@ jobs: bun run bench:cohort-create -- --purpose calibration --binary "${{ github.workspace }}/target/release/postil" - --screen-profile ../provisional-models.json + --screen-profile ../provisional-models-us.json --run-prefix "calibration-${{ github.run_id }}" --out "${{ runner.temp }}/benchmark-calibration-cohort.json" - name: Attest the calibration binary and cohort @@ -180,7 +180,7 @@ jobs: --manifest "${{ runner.temp }}/prepared/cohort.json" --slot "${{ matrix.sample }}" --binary "${{ runner.temp }}/prepared/postil" - --screen-profile ../provisional-models.json + --screen-profile ../provisional-models-us.json - name: Attest benchmark sample reservation id: attest-reservation uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4 @@ -203,7 +203,7 @@ jobs: --manifest "${{ runner.temp }}/prepared/cohort.json" --slot "${{ matrix.sample }}" --binary "${{ runner.temp }}/prepared/postil" - --screen-profile ../provisional-models.json + --screen-profile ../provisional-models-us.json - name: Attest benchmark sample result id: attest-result if: success() @@ -347,7 +347,7 @@ jobs: OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} run: | bun run bench:verify-generations -- \ - --screen-profile ../provisional-models.json \ + --screen-profile ../provisional-models-us.json \ --result "${RUNNER_TEMP}/verified-slots/01/report.json" \ --receipt "${RUNNER_TEMP}/verified-slots/01/receipt.json" \ --result "${RUNNER_TEMP}/verified-slots/02/report.json" \ @@ -373,8 +373,9 @@ jobs: run: | set -euo pipefail bun run bench:compare -- \ + --baseline baseline-us.json \ --binary "${RUNNER_TEMP}/prepared/postil" \ - --screen-profile ../provisional-models.json \ + --screen-profile ../provisional-models-us.json \ --cohort-manifest "${RUNNER_TEMP}/prepared/cohort.json" \ --expected-run-id "calibration-${GITHUB_RUN_ID}-01" \ --expected-run-id "calibration-${GITHUB_RUN_ID}-02" \ @@ -411,17 +412,17 @@ jobs: id: attest-baseline uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4 with: - subject-path: ${{ github.workspace }}/bench/baseline.json + subject-path: ${{ github.workspace }}/bench/baseline-us.json - name: Stage populated baseline evidence run: | mkdir -p "${RUNNER_TEMP}/populated-baseline" - cp bench/baseline.json "${RUNNER_TEMP}/populated-baseline/baseline.json" + cp bench/baseline-us.json "${RUNNER_TEMP}/populated-baseline/baseline-us.json" cp "${{ steps.attest-baseline.outputs.bundle-path }}" \ - "${RUNNER_TEMP}/populated-baseline/baseline.attestation.json" + "${RUNNER_TEMP}/populated-baseline/baseline-us.attestation.json" - name: Upload populated baseline uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: benchmark-calibration-baseline-${{ github.run_id }} + name: benchmark-calibration-us-baseline-${{ github.run_id }} path: ${{ runner.temp }}/populated-baseline if-no-files-found: error retention-days: 90 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7a788a0..df78f4b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -65,19 +65,20 @@ jobs: - name: Verify embedded model admission working-directory: bench run: bun run verify-admission --allow-provisional - - name: Verify the attested Luna calibration baseline + - name: Verify the attested Azure/US Luna calibration baseline env: GH_TOKEN: ${{ github.token }} run: | set -euo pipefail profile="openai/gpt-5.6-luna" - test -s bench/baseline.attestation.json || { - echo "::error::The populated Luna baseline has no committed calibration attestation." + jq -e --arg profile "$profile" '.profiles[$profile].populated == true' \ + bench/baseline-us.json >/dev/null && test -s bench/baseline-us.attestation.json || { + echo "::error::New releases require the populated Azure/US baseline and its main-branch first-attempt calibration attestation." exit 1 } source_sha="$(jq -er --arg profile "$profile" \ '.profiles[$profile] | select(.populated == true) | .calibration.sourceSha' \ - bench/baseline.json)" + bench/baseline-us.json)" [[ "$source_sha" =~ ^[0-9a-f]{40,64}$ ]] || { echo "::error::The Luna baseline has an invalid calibration source digest." exit 1 @@ -89,8 +90,8 @@ jobs: echo "::error::The Luna baseline source is not protected by the calibration registry." exit 1 } - gh attestation verify bench/baseline.json \ - --bundle bench/baseline.attestation.json \ + gh attestation verify bench/baseline-us.json \ + --bundle bench/baseline-us.attestation.json \ --repo postil-dev/postil-cli \ --signer-workflow postil-dev/postil-cli/.github/workflows/benchmark-calibration.yml \ --signer-digest "$source_sha" \ @@ -180,7 +181,7 @@ jobs: bun run bench:cohort-create -- --purpose release --binary "${{ github.workspace }}/target/release/postil" - --screen-profile ../provisional-models.json + --screen-profile ../provisional-models-us.json --run-prefix "release-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}" --out "${{ runner.temp }}/bench-live-cohort.json" @@ -213,7 +214,7 @@ jobs: POSTIL_BIN: ${{ github.workspace }}/target/release/postil POSTIL_SCORER_EVAL_REPEATS: "3" POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER: Azure - POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER_ROUTE: azure/eu + POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER_ROUTE: azure/us POSTIL_QUALIFICATION_SOURCE_SHA: ${{ github.sha }} run: >- bun run scorer-eval --json-out @@ -304,7 +305,7 @@ jobs: --manifest "${{ runner.temp }}/bench-live-cohort.json" --slot "${{ matrix.sample }}" --binary "${{ github.workspace }}/target/release/postil" - --screen-profile ../provisional-models.json + --screen-profile ../provisional-models-us.json - name: Attest benchmark sample reservation id: attest-reservation uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4 @@ -327,7 +328,7 @@ jobs: --manifest "${{ runner.temp }}/bench-live-cohort.json" --slot "${{ matrix.sample }}" --binary "${{ github.workspace }}/target/release/postil" - --screen-profile ../provisional-models.json + --screen-profile ../provisional-models-us.json - name: Verify successful benchmark sample evidence if: success() run: | @@ -511,7 +512,7 @@ jobs: OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} run: | bun run bench:verify-generations -- \ - --screen-profile ../provisional-models.json \ + --screen-profile ../provisional-models-us.json \ --result "${RUNNER_TEMP}/bench-live-reports/slots/01/report.json" \ --receipt "${RUNNER_TEMP}/bench-live-reports/slots/01/receipt.json" \ --result "${RUNNER_TEMP}/bench-live-reports/slots/02/report.json" \ @@ -530,8 +531,9 @@ jobs: set -euo pipefail comparison=0 bun run bench:compare -- \ + --baseline baseline-us.json \ --binary "${{ github.workspace }}/target/release/postil" \ - --screen-profile ../provisional-models.json \ + --screen-profile ../provisional-models-us.json \ --cohort-manifest "${{ runner.temp }}/bench-live-cohort.json" \ --expected-run-id "release-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}-01" \ --expected-run-id "release-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}-02" \ diff --git a/bench/README.md b/bench/README.md index 4e6b3d4..e58dcfd 100644 --- a/bench/README.md +++ b/bench/README.md @@ -87,21 +87,23 @@ bun run scorer-eval --json-out It receives the scorer models, repeat count, provider identity, route, credential, and release binary from the [release workflow](../.github/workflows/release.yml). The scorer screen can reject a scorer but cannot admit a hosted profile. -Release and calibration use an immutable cohort before model calls: +Release and calibration use an immutable cohort before model calls. Both use `../provisional-models-us.json`: ```sh bun run bench:cohort-create -- \ --purpose \ --binary \ - --screen-profile ../provisional-models.json \ + --screen-profile \ --run-prefix \ --out -bun run bench:cohort-run -- --mode reserve --manifest --slot --binary --screen-profile ../provisional-models.json -bun run bench:cohort-run -- --mode execute --manifest --slot --binary --screen-profile ../provisional-models.json +bun run bench:cohort-run -- --mode reserve --manifest --slot --binary --screen-profile +bun run bench:cohort-run -- --mode execute --manifest --slot --binary --screen-profile ``` Use [the release workflow](../.github/workflows/release.yml) for the five-sample comparison and [the calibration workflow](../.github/workflows/benchmark-calibration.yml) for the ten-sample recorded baseline. Both verify attestations, receipts, and provider generation evidence before comparison or recording. +Release and calibration use Azure/US in `provisional-models-us.json`. The embedded hosted profile remains Azure/EU in `provisional-models.json` until authenticated US calibration evidence is imported. Calibration records `baseline-us.json`. New releases fail closed until the populated baseline and its attestation are committed. Calibration requires ten complete attested samples from the first attempt on main; its credential balance check is not a total spending cap. Baseline imports preserve the artifact bytes and every evaluator contract source, including both workflows. Any change to those sources requires calibration of the changed contract. The EU profile, baseline, and attestation remain separate evidence and cannot qualify the US route. + ## Boundaries and deeper reference diff --git a/bench/baseline-us.json b/bench/baseline-us.json new file mode 100644 index 0000000..cff0982 --- /dev/null +++ b/bench/baseline-us.json @@ -0,0 +1,13 @@ +{ + "schemaVersion": 2, + "corpus": { + "fixtureCorpusSha256": "8e4c2cb9ad5a7efdfe6a875566d20133e905155b6f693a873595adf6c069e065", + "evaluatorSha256": "962efe39771505e5d45428837e59a38dc6577a1821cba2c4a24358bdd581c71c" + }, + "profiles": { + "openai/gpt-5.6-luna": { + "populated": false, + "instructions": "Record a predeclared ten-slot calibration cohort with provisional-models-us.json and exact Azure/US provider evidence." + } + } +} diff --git a/bench/src/cohort.test.ts b/bench/src/cohort.test.ts index 2e56bff..fa6b913 100644 --- a/bench/src/cohort.test.ts +++ b/bench/src/cohort.test.ts @@ -95,6 +95,26 @@ describe("cohort manifests", () => { ).rejects.toThrow("evaluatorSha256 is not bound"); }); + test("US calibration receipts cannot be used with the preserved EU profile", async () => { + const execution = calibrationExecution(); + const usProfile = resolve(import.meta.dir, "..", "..", "provisional-models-us.json"); + const euProfile = resolve(import.meta.dir, "..", "..", "provisional-models-eu.json"); + const manifest = await createCohortManifest({ + purpose: "calibration", + binaryPath: process.execPath, + screeningProfilePath: usProfile, + runPrefix: "calibration-us", + execution, + }); + expect(manifest.slots).toHaveLength(10); + await expect(assertManifestBoundToInputs( + manifest, process.execPath, usProfile, githubEnvironment(execution), + )).resolves.toBeUndefined(); + await expect(assertManifestBoundToInputs( + manifest, process.execPath, euProfile, githubEnvironment(execution), + )).rejects.toThrow("screeningProfileSha256 is not bound"); + }); + test("rejects wrong counts, unordered slots, and unbound release execution", async () => { let sequence = 100; const execution = calibrationExecution(); diff --git a/bench/src/compare-baseline.test.ts b/bench/src/compare-baseline.test.ts index ac1d052..30234d9 100644 --- a/bench/src/compare-baseline.test.ts +++ b/bench/src/compare-baseline.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from "bun:test"; import { createHash } from "node:crypto"; -import { readFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; import { resolve } from "node:path"; import { cases } from "../fixtures/cases"; import { @@ -47,7 +48,7 @@ import { type ProviderContractEvidence, } from "./livemodels-score"; -test("committed baseline authority matches the current benchmark sources", async () => { +test("EU baseline preserves its fixture identity but is stale for the calibration evaluator", async () => { const baseline = parseBaselineFile(JSON.parse( await readFile(resolve(import.meta.dir, "..", "baseline.json"), "utf8"), )); @@ -56,7 +57,7 @@ test("committed baseline authority matches the current benchmark sources", async .digest("hex"); expect(baseline.corpus.fixtureCorpusSha256).toBe(fixtureCorpusSha256); - expect(baseline.corpus.evaluatorSha256).toBe(await evaluatorSourceSha256()); + expect(baseline.corpus.evaluatorSha256).not.toBe(await evaluatorSourceSha256()); }); test("committed Luna baseline is either fail-closed or has a valid ten-report calibration", async () => { @@ -79,6 +80,76 @@ test("committed Luna baseline is either fail-closed or has a valid ten-report ca expect(() => assertBaselineCalibrationIntegrity(profile)).not.toThrow(); }); +test("release prerequisite rejects an unpopulated US baseline before attestation or model calls", async () => { + const workflow = await readFile(resolve(import.meta.dir, "..", "..", ".github/workflows/release.yml"), "utf8"); + const start = workflow.indexOf(' jq -e --arg profile "$profile"'); + const end = workflow.indexOf(' source_sha=', start); + expect(start).toBeGreaterThan(0); + expect(end).toBeGreaterThan(start); + const prerequisite = workflow.slice(start, end); + expect(prerequisite).toContain("main-branch first-attempt calibration attestation"); + const directory = await mkdtemp(resolve(tmpdir(), "postil-calibration-prerequisite-")); + try { + await mkdir(resolve(directory, "bench")); + const check = async (): Promise<{ code: number; output: string }> => { + const child = Bun.spawn(["bash", "-c", 'set -euo pipefail\nprofile="openai/gpt-5.6-luna"\n' + prerequisite], { + cwd: directory, stdout: "pipe", stderr: "pipe", timeout: 10_000, + }); + const [code, stdout, stderr] = await Promise.all([child.exited, new Response(child.stdout).text(), new Response(child.stderr).text()]); + return { code, output: stdout + stderr }; + }; + const baseline = (populated: boolean) => JSON.stringify({ profiles: { "openai/gpt-5.6-luna": { populated } } }); + await writeFile(resolve(directory, "bench/baseline-us.json"), baseline(false)); + const missing = await check(); + expect(missing.code).toBe(1); + expect(missing.output).toContain("New releases require the populated Azure/US baseline"); + await writeFile(resolve(directory, "bench/baseline-us.attestation.json"), JSON.stringify({ fixture: true })); + expect((await check()).code).toBe(1); + await writeFile(resolve(directory, "bench/baseline-us.json"), baseline(true)); + expect((await check()).code).toBe(0); + await writeFile(resolve(directory, "bench/baseline-us.attestation.json"), ""); + expect((await check()).code).toBe(1); + expect(workflow.indexOf("gh attestation verify", end)).toBeGreaterThan(end); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test("unpopulated US calibration target keeps the embedded provider on Azure/EU", async () => { + const root = resolve(import.meta.dir, "..", ".."); + const [usBytes, euBytes, us, eu] = await Promise.all([ + readFile(resolve(root, "bench/baseline-us.json"), "utf8"), + readFile(resolve(root, "bench/baseline.json"), "utf8"), + screeningProfileMetadata(resolve(root, "provisional-models-us.json")), + screeningProfileMetadata(resolve(root, "provisional-models-eu.json")), + ]); + + const euBaseline = parseBaselineFile(JSON.parse(euBytes)); + const usBaseline = parseBaselineFile(JSON.parse(usBytes)); + expect(euBaseline.profiles["openai/gpt-5.6-luna"]?.populated).toBe(true); + const usBaselineProfile = usBaseline.profiles["openai/gpt-5.6-luna"]; + expect(usBaselineProfile).toBeDefined(); + expect(usBaselineProfile?.populated).toBe(false); + expect(usBaselineProfile?.instructions).toContain("predeclared ten-slot calibration cohort"); + expect(await readFile(resolve(root, "provisional-models.json"), "utf8")).toBe(await readFile(resolve(root, "provisional-models-eu.json"), "utf8")); + expect(usBaseline.corpus.fixtureCorpusSha256).toBe(euBaseline.corpus.fixtureCorpusSha256); + expect(usBaseline.corpus.evaluatorSha256).not.toBe(euBaseline.corpus.evaluatorSha256); + expect(usBaseline.corpus.evaluatorSha256).toBe(await evaluatorSourceSha256()); + expect(eu.upstreamProviderRoute).toBe("azure/eu"); + expect(us.upstreamProviderRoute).toBe("azure/us"); + for (const profile of [us, eu]) { + expect(profile.providerContract.dataCollection).toBe("deny"); + expect(profile.providerContract.zeroDataRetention).toBe(true); + expect(profile.providerContract.allowFallbacks).toBe(false); + } + expect(us.sha256).not.toBe(eu.sha256); + expect(us.providerContractSha256).not.toBe(eu.providerContractSha256); + expect(us.providerContract).toEqual({ + ...eu.providerContract, + upstreamProviderRoute: "azure/us", + }); +}); + const PROVIDER_CONTRACT: ProviderContractEvidence = { version: 1, benchmarkProviderIdentity: "openrouter:managed-routing", @@ -191,6 +262,33 @@ function fakeReport(options: FakeReportOptions = {}): LiveReportForComparison { }; } +test("comparison CLI rejects preserved EU evidence for the current evaluator", async () => { + const directory = await mkdtemp(resolve(tmpdir(), "postil-stale-baseline-")); + try { + const paths = await Promise.all(Array.from({ length: 1 }, async (_, index) => { + const report = await inputBoundReport(); + const path = resolve(directory, `report-${index}.json`); + await writeFile(path, JSON.stringify(report)); + return { path, runId: report.summary.runId }; + })); + const child = Bun.spawn([ + process.execPath, "--no-env-file", resolve(import.meta.dir, "compare-baseline.ts"), + "--binary", process.execPath, + "--baseline", resolve(import.meta.dir, "..", "baseline.json"), + "--screen-profile", resolve(import.meta.dir, "..", "..", "provisional-models.json"), + ...paths.flatMap(({ path, runId }) => ["--expected-run-id", runId, "--result", path]), + ], { stdout: "pipe", stderr: "pipe", timeout: 30_000 }); + const [exitCode, stderr] = await Promise.all([ + child.exited, new Response(child.stderr).text(), new Response(child.stdout).text(), + ]); + expect(exitCode).toBe(1); + expect(stderr).toContain("FIXTURE CORPUS MISMATCH"); + expect(stderr).toContain("evaluator source than the recorded baseline"); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + function cloneReport(report: LiveReportForComparison): LiveReportForComparison { return structuredClone(report); } @@ -913,6 +1011,27 @@ describe("three-report aggregation compatibility", () => { ); }); + test("valid US reports cannot qualify against an EU calibration", async () => { + const us = await screeningProfileMetadata( + resolve(import.meta.dir, "..", "..", "provisional-models-us.json"), + ); + const reports = Array.from({ length: 5 }, () => { + const report = fakeReport(); + Object.assign(report.summary, { + screeningProfileSha256: us.sha256, + upstreamProviderRoute: us.upstreamProviderRoute, + providerContract: us.providerContract, + providerContractSha256: us.providerContractSha256, + }); + expect(() => assertValidReleaseReport(report)).not.toThrow(); + return report; + }); + const observed = aggregateObservedMetrics(reports); + expect(() => compareMetrics(populatedBaseline, observed)).toThrow( + "baseline calibration execution identity does not match the candidate cohort", + ); + }); + test("cost remains blocking when the provider profile differs", () => { const observed = aggregateObservedMetrics([ fakeReport({ costPerCase: "1" }), diff --git a/bench/src/compare-baseline.ts b/bench/src/compare-baseline.ts index 8d7506c..bdef18a 100644 --- a/bench/src/compare-baseline.ts +++ b/bench/src/compare-baseline.ts @@ -3,7 +3,7 @@ // // Compare mode consumes one, three, or five LiveReport JSON artifacts written by // `bun run bench:live --json-out ` and compares their metrics against the -// committed `bench/baseline.json`. Record mode requires a predeclared ten-report +// committed `bench/baseline-us.json`. Record mode requires a predeclared ten-report // calibration cohort. Every report must be complete full-corpus evidence. A // multi-report operation additionally requires one identical benchmark cohort // and distinct raw artifacts. Exits non-zero on invalid evidence or a material @@ -18,9 +18,9 @@ // --expected-run-id ... --expected-run-id // --result ... --result // -// Record mode writes the ten-sample calibration cohort into baseline.json as the new +// Record mode writes the ten-sample calibration cohort into baseline-us.json as the new // baseline for the reports' model. This is the deliberate re-baseline path: -// nothing updates baseline.json except an explicit --record invocation. +// nothing updates baseline-us.json except an explicit --record invocation. // // bun run bench:compare -- --binary --screen-profile // --expected-run-id ... --expected-run-id @@ -1112,7 +1112,7 @@ function requiredFlagValue(args: readonly string[], index: number, flag: string) } function defaultBaselinePath(): string { - return resolve(import.meta.dir, "..", "baseline.json"); + return resolve(import.meta.dir, "..", "baseline-us.json"); } export function assertDistinctResultPaths(paths: readonly string[]): void { @@ -1604,7 +1604,7 @@ async function main() { baselineFile.corpus.evaluatorSha256 !== observed.evaluatorSha256) { console.error( "FIXTURE CORPUS MISMATCH: this report was scored against a different fixture set or\n" + - "evaluator source than baseline.json was recorded against, so its metrics are not\n" + + "evaluator source than the recorded baseline, so its metrics are not\n" + "comparable.\n" + ` baseline fixtureCorpusSha256 ${baselineFile.corpus.fixtureCorpusSha256}\n` + ` observed fixtureCorpusSha256 ${observed.fixtureCorpusSha256}\n` + @@ -1708,7 +1708,7 @@ async function main() { console.log(formatComparisonTable(comparison.rows)); if (!comparison.ok) { console.error( - "\nRELEASE BLOCKED: the live benchmark regressed past tolerance against bench/baseline.json.\n" + + "\nRELEASE BLOCKED: the live benchmark regressed past tolerance against the recorded baseline.\n" + "Fix the regression, or if the new numbers are an accepted tradeoff, re-baseline\n" + "deliberately with:\n" + rebaselineGuidance, diff --git a/bench/src/livemodels.test.ts b/bench/src/livemodels.test.ts index 06b446d..1b80999 100644 --- a/bench/src/livemodels.test.ts +++ b/bench/src/livemodels.test.ts @@ -1389,10 +1389,24 @@ describe("managed admission workflow", () => { }; const calibrationGenerationVerification = generationVerificationCommand(calibration); expect(calibrationGenerationVerification).toMatch( - /bun run bench:verify-generations -- \\\n\s+--screen-profile \.\.\/provisional-models\.json \\\n\s+--result /u, + /bun run bench:verify-generations -- \\\n\s+--screen-profile \.\.\/provisional-models-us\.json \\\n\s+--result /u, ); - expect([...calibrationGenerationVerification.matchAll(/--screen-profile \.\.\/provisional-models\.json/gu)]).toHaveLength(1); + expect([...calibrationGenerationVerification.matchAll(/--screen-profile \.\.\/provisional-models-us\.json/gu)]).toHaveLength(1); expect(calibration).toContain("--record"); + expect(calibration).not.toContain("bench/baseline.json"); + expect(calibration.match(/--screen-profile \.\.\/provisional-models-us\.json/gu)).toHaveLength(5); + expect(calibration).toContain("--baseline baseline-us.json"); + expect(calibration).toContain("subject-path: ${{ github.workspace }}/bench/baseline-us.json"); + expect(calibration).toContain('cp bench/baseline-us.json "${RUNNER_TEMP}/populated-baseline/baseline-us.json"'); + expect(calibration).toContain("/populated-baseline/baseline-us.attestation.json"); + expect(calibration).toContain("name: benchmark-calibration-us-baseline-${{ github.run_id }}"); + expect(release).toContain("New releases require the populated Azure/US baseline and its main-branch first-attempt calibration attestation."); + expect(release).not.toContain("provisional-models.json"); + expect(release).toContain("--screen-profile ../provisional-models-us.json"); + expect(release).not.toContain("bench/baseline.json"); + expect(release).toContain("--baseline baseline-us.json"); + expect(release).toContain("--bundle bench/baseline-us.attestation.json"); + expect(release).toContain("--signer-workflow postil-dev/postil-cli/.github/workflows/benchmark-calibration.yml"); expect(release).not.toContain("workflow_dispatch"); expect(release).toContain("name: Require the unique first release run for this tag"); expect(release).toContain('if [[ "${GITHUB_RUN_ATTEMPT}" != "1" ]]'); @@ -1400,8 +1414,8 @@ describe("managed admission workflow", () => { expect(release).toContain("This version tag already has another release run."); expect(release).toContain("group: release-${{ github.ref_name }}"); expect(release).toContain('gh release view "${GITHUB_REF_NAME}"'); - expect(release).toContain("name: Verify the attested Luna calibration baseline"); - expect(release).toContain("bench/baseline.attestation.json"); + expect(release).toContain("name: Verify the attested Azure/US Luna calibration baseline"); + expect(release).toContain("bench/baseline-us.attestation.json"); expect(release).toContain('git/ref/tags/postil-calibration-${source_sha}'); expect(release).toContain( "--signer-workflow postil-dev/postil-cli/.github/workflows/benchmark-calibration.yml", @@ -1410,7 +1424,7 @@ describe("managed admission workflow", () => { expect(release).toContain('--source-digest "$source_sha"'); expect(release).toContain("--source-ref refs/heads/main"); expect(release).toMatch(/validate-tag:\n[\s\S]*?permissions:\n\s+contents: read\n\s+actions: read/u); - expect(release).toMatch(/validate-tag:\n[\s\S]*?fetch-depth: 0[\s\S]*?bun-version: 1\.3\.14[\s\S]*?bun install --frozen-lockfile[\s\S]*?bun run verify-admission[\s\S]*?name: Verify the attested Luna calibration baseline[\s\S]*?\n bench-live-prepare:\n/u); + expect(release).toMatch(/validate-tag:\n[\s\S]*?fetch-depth: 0[\s\S]*?bun-version: 1\.3\.14[\s\S]*?bun install --frozen-lockfile[\s\S]*?bun run verify-admission[\s\S]*?name: Verify the attested Azure\/US Luna calibration baseline[\s\S]*?\n bench-live-prepare:\n/u); // The gate derives its model from the binary's embedded configuration so // caller drift cannot benchmark a model absent from the release. expect(release).not.toContain("REVIEW_MODEL:"); @@ -1418,7 +1432,7 @@ describe("managed admission workflow", () => { expect(release).not.toContain("POSTIL_SCORER_EVAL_MODELS:"); expect(release).toContain('POSTIL_SCORER_EVAL_REPEATS: "3"'); expect(release).toContain("POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER: Azure"); - expect(release).toContain("POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER_ROUTE: azure/eu"); + expect(release).toContain("POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER_ROUTE: azure/us"); expect(release).toContain("POSTIL_BIN: ${{ github.workspace }}/target/release/postil"); expect(release).not.toContain("Build scorer qualification binary"); expect(release).toContain("bun run scorer-eval --json-out"); @@ -1473,7 +1487,7 @@ describe("managed admission workflow", () => { expect(sample).toContain( '--manifest "${{ runner.temp }}/bench-live-cohort.json"', ); - expect(sample).toContain("--screen-profile ../provisional-models.json"); + expect(sample).toContain("--screen-profile ../provisional-models-us.json"); expect(sample).not.toContain("--report-out"); expect(sample).not.toContain("--receipt-out"); expect(sample).toContain("uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4"); @@ -1500,9 +1514,9 @@ describe("managed admission workflow", () => { expect(final).toContain("name: Verify independent release generations"); const releaseGenerationVerification = generationVerificationCommand(final); expect(releaseGenerationVerification).toMatch( - /bun run bench:verify-generations -- \\\n\s+--screen-profile \.\.\/provisional-models\.json \\\n\s+--result /u, + /bun run bench:verify-generations -- \\\n\s+--screen-profile \.\.\/provisional-models-us\.json \\\n\s+--result /u, ); - expect([...releaseGenerationVerification.matchAll(/--screen-profile \.\.\/provisional-models\.json/gu)]).toHaveLength(1); + expect([...releaseGenerationVerification.matchAll(/--screen-profile \.\.\/provisional-models-us\.json/gu)]).toHaveLength(1); expect(final).toContain("gh attestation verify"); expect(final).toContain("--deny-self-hosted-runners"); expect(final).toContain("name: bench-live-cohort-${{ github.run_attempt }}"); @@ -1522,7 +1536,7 @@ describe("managed admission workflow", () => { expect([...final.matchAll(/--result /gu)]).toHaveLength(10); expect([...final.matchAll(/--receipt /gu)]).toHaveLength(10); expect(final).toMatch( - /bun run bench:compare --[\s\S]*--binary "\$\{\{ github\.workspace \}\}\/target\/release\/postil"[\s\S]*--screen-profile \.\.\/provisional-models\.json/u, + /bun run bench:compare --[\s\S]*--binary "\$\{\{ github\.workspace \}\}\/target\/release\/postil"[\s\S]*--screen-profile \.\.\/provisional-models-us\.json/u, ); expect(final).toContain("SAMPLE_JOB_RESULT: ${{ needs.bench-live-sample.result }}"); expect(final).toContain('if [[ "${SAMPLE_JOB_RESULT}" != "success" ]]'); diff --git a/provisional-models-eu.json b/provisional-models-eu.json new file mode 100644 index 0000000..f1fa9fc --- /dev/null +++ b/provisional-models-eu.json @@ -0,0 +1,20 @@ +{ + "benchmarkProviderIdentity": "openrouter:managed-routing", + "upstreamProviderIdentity": "Azure", + "upstreamProviderRoute": "azure/eu", + "apiBase": "https://openrouter.ai:443/api/v1", + "apiFormat": "openai-compatible", + "generatorChain": ["openai/gpt-5.6-luna"], + "consensus": 1, + "scorerChain": ["openai/gpt-5.6-luna"], + "providerGenerationModels": { + "openai/gpt-5.6-luna": "openai/gpt-5.6-luna-20260709" + }, + "modelPriceBounds": [ + { + "model": "openai/gpt-5.6-luna", + "inputMicrosPerMillionTokens": 220000, + "outputMicrosPerMillionTokens": 1320000 + } + ] +} diff --git a/provisional-models-us.json b/provisional-models-us.json new file mode 100644 index 0000000..ae5ed97 --- /dev/null +++ b/provisional-models-us.json @@ -0,0 +1,20 @@ +{ + "benchmarkProviderIdentity": "openrouter:managed-routing", + "upstreamProviderIdentity": "Azure", + "upstreamProviderRoute": "azure/us", + "apiBase": "https://openrouter.ai:443/api/v1", + "apiFormat": "openai-compatible", + "generatorChain": ["openai/gpt-5.6-luna"], + "consensus": 1, + "scorerChain": ["openai/gpt-5.6-luna"], + "providerGenerationModels": { + "openai/gpt-5.6-luna": "openai/gpt-5.6-luna-20260709" + }, + "modelPriceBounds": [ + { + "model": "openai/gpt-5.6-luna", + "inputMicrosPerMillionTokens": 220000, + "outputMicrosPerMillionTokens": 1320000 + } + ] +} diff --git a/src/config.rs b/src/config.rs index c7e2d39..1498b91 100644 --- a/src/config.rs +++ b/src/config.rs @@ -3954,6 +3954,12 @@ scorer = { enabled = true, default_model = "provider/scorer", reasoning_effort = Some(profile.clone()) ); assert_eq!(profile.upstream_provider_identity, "Azure"); + let calibration: serde_json::Value = + serde_json::from_str(include_str!("../bench/baseline-us.json")).unwrap(); + assert_eq!( + calibration["profiles"]["openai/gpt-5.6-luna"]["populated"], + false + ); assert_eq!(profile.upstream_provider_route, "azure/eu"); assert_eq!(profile.generator_chain, vec!["openai/gpt-5.6-luna"]); assert_eq!(profile.scorer_chain, vec!["openai/gpt-5.6-luna"]); @@ -4002,6 +4008,18 @@ scorer = { enabled = true, default_model = "provider/scorer", reasoning_effort = } } + #[test] + fn us_calibration_profile_has_its_own_provider_route() { + let calibration_profile = + parse_provisional_hosted_profile(include_str!("../provisional-models-us.json")) + .unwrap(); + let embedded_profile = provisional_hosted_profile(); + + assert_eq!(calibration_profile.upstream_provider_identity, "Azure"); + assert_eq!(calibration_profile.upstream_provider_route, "azure/us"); + assert_eq!(embedded_profile.upstream_provider_route, "azure/eu"); + } + #[test] fn benchmark_screening_projects_an_exact_profile_when_scoring_is_disabled() { let profile = provisional_hosted_profile(); diff --git a/src/llm.rs b/src/llm.rs index d1ee967..680b58e 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -10907,28 +10907,33 @@ mod tests { fn benchmark_screening_enforces_the_exact_provisional_provider_contract() { const CHILD_ENV: &str = "POSTIL_TEST_BENCHMARK_SCREENING_CHILD"; if std::env::var_os(CHILD_ENV).is_none() { - let directory = tempfile::tempdir().unwrap(); - let profile_path = directory.path().join("screen-profile.json"); - std::fs::write(&profile_path, include_str!("../provisional-models.json")).unwrap(); - let output = std::process::Command::new(std::env::current_exe().unwrap()) + for (route, profile) in [ + ("azure/eu", include_str!("../provisional-models-eu.json")), + ("azure/us", include_str!("../provisional-models-us.json")), + ] { + let directory = tempfile::tempdir().unwrap(); + let profile_path = directory.path().join("screen-profile.json"); + std::fs::write(&profile_path, profile).unwrap(); + let output = std::process::Command::new(std::env::current_exe().unwrap()) .args([ "--exact", "llm::tests::benchmark_screening_enforces_the_exact_provisional_provider_contract", "--nocapture", ]) - .env(CHILD_ENV, "1") + .env(CHILD_ENV, route) .env("POSTIL_BENCH_SCREEN_PROFILE", &profile_path) .env("POSTIL_BENCH_REQUIRE_HOSTED_PROVIDER_PRIVACY", "1") .env_remove("POSTIL_HOSTED_MODE") .env_remove("POSTIL_QUALIFICATION_CANDIDATE_PROFILE") .output() .unwrap(); - assert!( - output.status.success(), - "isolated screening test failed\nstdout:\n{}\nstderr:\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr), - ); + assert!( + output.status.success(), + "isolated screening test failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + } return; } @@ -10960,7 +10965,10 @@ mod tests { 0.0, LlmPhase::Review, ); - assert_eq!(body["provider"]["order"], json!(["azure/eu"])); + assert_eq!( + body["provider"]["order"], + json!([std::env::var(CHILD_ENV).unwrap()]) + ); assert_eq!(body["provider"]["allow_fallbacks"], false); assert_eq!(body["provider"]["data_collection"], "deny"); assert_eq!(body["provider"]["zdr"], true);