Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 12 additions & 11 deletions .github/workflows/benchmark-calibration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,10 @@ jobs:
- name: Verify embedded model admission
working-directory: bench
run: bun run verify-admission --allow-provisional
- name: Require an unpopulated Luna baseline
- name: Require an unpopulated Azure/US Luna baseline
run: >-
jq -e '.profiles["openai/gpt-5.6-luna"].populated == false'
bench/baseline.json >/dev/null
bench/baseline-us.json >/dev/null

prepare:
needs: validate
Expand Down Expand Up @@ -98,7 +98,7 @@ jobs:
bun run bench:cohort-create --
--purpose calibration
--binary "${{ github.workspace }}/target/release/postil"
--screen-profile ../provisional-models.json
--screen-profile ../provisional-models-us.json
--run-prefix "calibration-${{ github.run_id }}"
--out "${{ runner.temp }}/benchmark-calibration-cohort.json"
- name: Attest the calibration binary and cohort
Expand Down Expand Up @@ -180,7 +180,7 @@ jobs:
--manifest "${{ runner.temp }}/prepared/cohort.json"
--slot "${{ matrix.sample }}"
--binary "${{ runner.temp }}/prepared/postil"
--screen-profile ../provisional-models.json
--screen-profile ../provisional-models-us.json
- name: Attest benchmark sample reservation
id: attest-reservation
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4
Expand All @@ -203,7 +203,7 @@ jobs:
--manifest "${{ runner.temp }}/prepared/cohort.json"
--slot "${{ matrix.sample }}"
--binary "${{ runner.temp }}/prepared/postil"
--screen-profile ../provisional-models.json
--screen-profile ../provisional-models-us.json
- name: Attest benchmark sample result
id: attest-result
if: success()
Expand Down Expand Up @@ -347,7 +347,7 @@ jobs:
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
run: |
bun run bench:verify-generations -- \
--screen-profile ../provisional-models.json \
--screen-profile ../provisional-models-us.json \
--result "${RUNNER_TEMP}/verified-slots/01/report.json" \
--receipt "${RUNNER_TEMP}/verified-slots/01/receipt.json" \
--result "${RUNNER_TEMP}/verified-slots/02/report.json" \
Expand All @@ -373,8 +373,9 @@ jobs:
run: |
set -euo pipefail
bun run bench:compare -- \
--baseline baseline-us.json \
--binary "${RUNNER_TEMP}/prepared/postil" \
--screen-profile ../provisional-models.json \
--screen-profile ../provisional-models-us.json \
--cohort-manifest "${RUNNER_TEMP}/prepared/cohort.json" \
--expected-run-id "calibration-${GITHUB_RUN_ID}-01" \
--expected-run-id "calibration-${GITHUB_RUN_ID}-02" \
Expand Down Expand Up @@ -411,17 +412,17 @@ jobs:
id: attest-baseline
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4
with:
subject-path: ${{ github.workspace }}/bench/baseline.json
subject-path: ${{ github.workspace }}/bench/baseline-us.json
- name: Stage populated baseline evidence
run: |
mkdir -p "${RUNNER_TEMP}/populated-baseline"
cp bench/baseline.json "${RUNNER_TEMP}/populated-baseline/baseline.json"
cp bench/baseline-us.json "${RUNNER_TEMP}/populated-baseline/baseline-us.json"
cp "${{ steps.attest-baseline.outputs.bundle-path }}" \
"${RUNNER_TEMP}/populated-baseline/baseline.attestation.json"
"${RUNNER_TEMP}/populated-baseline/baseline-us.attestation.json"
- name: Upload populated baseline
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: benchmark-calibration-baseline-${{ github.run_id }}
name: benchmark-calibration-us-baseline-${{ github.run_id }}
path: ${{ runner.temp }}/populated-baseline
if-no-files-found: error
retention-days: 90
26 changes: 14 additions & 12 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -65,19 +65,20 @@ jobs:
- name: Verify embedded model admission
working-directory: bench
run: bun run verify-admission --allow-provisional
- name: Verify the attested Luna calibration baseline
- name: Verify the attested Azure/US Luna calibration baseline
env:
GH_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
profile="openai/gpt-5.6-luna"
test -s bench/baseline.attestation.json || {
echo "::error::The populated Luna baseline has no committed calibration attestation."
jq -e --arg profile "$profile" '.profiles[$profile].populated == true' \
Comment thread
morgaesis marked this conversation as resolved.
bench/baseline-us.json >/dev/null && test -s bench/baseline-us.attestation.json || {
echo "::error::New releases require the populated Azure/US baseline and its main-branch first-attempt calibration attestation."
exit 1
}
source_sha="$(jq -er --arg profile "$profile" \
'.profiles[$profile] | select(.populated == true) | .calibration.sourceSha' \
bench/baseline.json)"
bench/baseline-us.json)"
[[ "$source_sha" =~ ^[0-9a-f]{40,64}$ ]] || {
echo "::error::The Luna baseline has an invalid calibration source digest."
exit 1
Expand All @@ -89,8 +90,8 @@ jobs:
echo "::error::The Luna baseline source is not protected by the calibration registry."
exit 1
}
gh attestation verify bench/baseline.json \
--bundle bench/baseline.attestation.json \
gh attestation verify bench/baseline-us.json \
--bundle bench/baseline-us.attestation.json \
--repo postil-dev/postil-cli \
--signer-workflow postil-dev/postil-cli/.github/workflows/benchmark-calibration.yml \
--signer-digest "$source_sha" \
Expand Down Expand Up @@ -180,7 +181,7 @@ jobs:
bun run bench:cohort-create --
--purpose release
--binary "${{ github.workspace }}/target/release/postil"
--screen-profile ../provisional-models.json
--screen-profile ../provisional-models-us.json
--run-prefix "release-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}"
--out "${{ runner.temp }}/bench-live-cohort.json"

Expand Down Expand Up @@ -213,7 +214,7 @@ jobs:
POSTIL_BIN: ${{ github.workspace }}/target/release/postil
POSTIL_SCORER_EVAL_REPEATS: "3"
POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER: Azure
POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER_ROUTE: azure/eu
POSTIL_SCORER_EVAL_UPSTREAM_PROVIDER_ROUTE: azure/us
POSTIL_QUALIFICATION_SOURCE_SHA: ${{ github.sha }}
run: >-
bun run scorer-eval --json-out
Expand Down Expand Up @@ -304,7 +305,7 @@ jobs:
--manifest "${{ runner.temp }}/bench-live-cohort.json"
--slot "${{ matrix.sample }}"
--binary "${{ github.workspace }}/target/release/postil"
--screen-profile ../provisional-models.json
--screen-profile ../provisional-models-us.json
- name: Attest benchmark sample reservation
id: attest-reservation
uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4
Expand All @@ -327,7 +328,7 @@ jobs:
--manifest "${{ runner.temp }}/bench-live-cohort.json"
--slot "${{ matrix.sample }}"
--binary "${{ github.workspace }}/target/release/postil"
--screen-profile ../provisional-models.json
--screen-profile ../provisional-models-us.json
- name: Verify successful benchmark sample evidence
if: success()
run: |
Expand Down Expand Up @@ -511,7 +512,7 @@ jobs:
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
run: |
bun run bench:verify-generations -- \
--screen-profile ../provisional-models.json \
--screen-profile ../provisional-models-us.json \
--result "${RUNNER_TEMP}/bench-live-reports/slots/01/report.json" \
--receipt "${RUNNER_TEMP}/bench-live-reports/slots/01/receipt.json" \
--result "${RUNNER_TEMP}/bench-live-reports/slots/02/report.json" \
Expand All @@ -530,8 +531,9 @@ jobs:
set -euo pipefail
comparison=0
bun run bench:compare -- \
--baseline baseline-us.json \
--binary "${{ github.workspace }}/target/release/postil" \
--screen-profile ../provisional-models.json \
--screen-profile ../provisional-models-us.json \
--cohort-manifest "${{ runner.temp }}/bench-live-cohort.json" \
--expected-run-id "release-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}-01" \
--expected-run-id "release-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}-02" \
Expand Down
10 changes: 6 additions & 4 deletions bench/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -87,21 +87,23 @@ bun run scorer-eval --json-out <report-path>

It receives the scorer models, repeat count, provider identity, route, credential, and release binary from the [release workflow](../.github/workflows/release.yml). The scorer screen can reject a scorer but cannot admit a hosted profile.

Release and calibration use an immutable cohort before model calls:
Release and calibration use an immutable cohort before model calls. Both use `../provisional-models-us.json`:

```sh
bun run bench:cohort-create -- \
--purpose <release-or-calibration> \
--binary <release-binary> \
--screen-profile ../provisional-models.json \
--screen-profile <screen-profile> \
--run-prefix <workflow-bound-prefix> \
--out <cohort-manifest>
bun run bench:cohort-run -- --mode reserve --manifest <cohort-manifest> --slot <slot> --binary <release-binary> --screen-profile ../provisional-models.json
bun run bench:cohort-run -- --mode execute --manifest <cohort-manifest> --slot <slot> --binary <release-binary> --screen-profile ../provisional-models.json
bun run bench:cohort-run -- --mode reserve --manifest <cohort-manifest> --slot <slot> --binary <release-binary> --screen-profile <screen-profile>
bun run bench:cohort-run -- --mode execute --manifest <cohort-manifest> --slot <slot> --binary <release-binary> --screen-profile <screen-profile>
```

Use [the release workflow](../.github/workflows/release.yml) for the five-sample comparison and [the calibration workflow](../.github/workflows/benchmark-calibration.yml) for the ten-sample recorded baseline. Both verify attestations, receipts, and provider generation evidence before comparison or recording.

Release and calibration use Azure/US in `provisional-models-us.json`. The embedded hosted profile remains Azure/EU in `provisional-models.json` until authenticated US calibration evidence is imported. Calibration records `baseline-us.json`. New releases fail closed until the populated baseline and its attestation are committed. Calibration requires ten complete attested samples from the first attempt on main; its credential balance check is not a total spending cap. Baseline imports preserve the artifact bytes and every evaluator contract source, including both workflows. Any change to those sources requires calibration of the changed contract. The EU profile, baseline, and attestation remain separate evidence and cannot qualify the US route.

</details>

## Boundaries and deeper reference
Expand Down
13 changes: 13 additions & 0 deletions bench/baseline-us.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{
"schemaVersion": 2,
"corpus": {
"fixtureCorpusSha256": "8e4c2cb9ad5a7efdfe6a875566d20133e905155b6f693a873595adf6c069e065",
"evaluatorSha256": "962efe39771505e5d45428837e59a38dc6577a1821cba2c4a24358bdd581c71c"
},
"profiles": {
"openai/gpt-5.6-luna": {
"populated": false,
"instructions": "Record a predeclared ten-slot calibration cohort with provisional-models-us.json and exact Azure/US provider evidence."
}
}
}
20 changes: 20 additions & 0 deletions bench/src/cohort.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,26 @@ describe("cohort manifests", () => {
).rejects.toThrow("evaluatorSha256 is not bound");
});

test("US calibration receipts cannot be used with the preserved EU profile", async () => {
const execution = calibrationExecution();
const usProfile = resolve(import.meta.dir, "..", "..", "provisional-models-us.json");
const euProfile = resolve(import.meta.dir, "..", "..", "provisional-models-eu.json");
const manifest = await createCohortManifest({
purpose: "calibration",
binaryPath: process.execPath,
screeningProfilePath: usProfile,
runPrefix: "calibration-us",
execution,
});
expect(manifest.slots).toHaveLength(10);
await expect(assertManifestBoundToInputs(
manifest, process.execPath, usProfile, githubEnvironment(execution),
)).resolves.toBeUndefined();
await expect(assertManifestBoundToInputs(
manifest, process.execPath, euProfile, githubEnvironment(execution),
)).rejects.toThrow("screeningProfileSha256 is not bound");
});

test("rejects wrong counts, unordered slots, and unbound release execution", async () => {
let sequence = 100;
const execution = calibrationExecution();
Expand Down
Loading
Loading