diff --git a/Cargo.lock b/Cargo.lock index a8c7939..8691848 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1358,7 +1358,7 @@ checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "postil-cli" -version = "0.9.17" +version = "0.9.18" dependencies = [ "aho-corasick", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 255e342..4251efa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "postil-cli" -version = "0.9.17" +version = "0.9.18" edition = "2024" description = "Postil: a low-noise AI review gate. Silent on clean PRs, hard gate on real risk." license = "Apache-2.0" diff --git a/bench/README.md b/bench/README.md index e58dcfd..fe90965 100644 --- a/bench/README.md +++ b/bench/README.md @@ -102,7 +102,7 @@ bun run bench:cohort-run -- --mode execute --manifest --slot < Use [the release workflow](../.github/workflows/release.yml) for the five-sample comparison and [the calibration workflow](../.github/workflows/benchmark-calibration.yml) for the ten-sample recorded baseline. Both verify attestations, receipts, and provider generation evidence before comparison or recording. -Release and calibration use Azure/US in `provisional-models-us.json`. The embedded hosted profile remains Azure/EU in `provisional-models.json` until authenticated US calibration evidence is imported. Calibration records `baseline-us.json`. New releases fail closed until the populated baseline and its attestation are committed. Calibration requires ten complete attested samples from the first attempt on main; its credential balance check is not a total spending cap. Baseline imports preserve the artifact bytes and every evaluator contract source, including both workflows. Any change to those sources requires calibration of the changed contract. The EU profile, baseline, and attestation remain separate evidence and cannot qualify the US route. +The embedded hosted profile uses Azure/EU in `provisional-models.json`. Release and calibration use Azure/US in `provisional-models-us.json`, with separate baseline evidence in `baseline-us.json` and its attestation. New releases require that populated baseline and its attestation. Calibration requires ten complete attested samples from the first attempt on main; its credential balance check is not a total spending cap. Baseline imports preserve the artifact bytes and every evaluator contract source, including both workflows. Any change to those sources requires calibration of the changed contract. The EU profile, baseline, and attestation remain separate evidence and cannot qualify the US route. diff --git a/bench/baseline-us.attestation.json b/bench/baseline-us.attestation.json new file mode 100644 index 0000000..804b34b --- /dev/null +++ b/bench/baseline-us.attestation.json @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"certificate":{"rawBytes":"MIIHUDCCBtegAwIBAgIUFh9+ZN5DmT7t4LjhIMBY2rZpExkwCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYwOTI0MDM1NDM1WhcNMjYwOTI0MDQwNDM1WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE0zBB9yyAiVIph3R0+tAPrOsm5QtoL7nmdgfgtVz9n8wSVjDvZiI9Prfy257IiDf2ZYR3JtxCo7vIxFSpDxLb5KOCBfYwggXyMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAdBgNVHQ4EFgQURhbRHWuWsW64XGdI6dOcuJ0C8VkwHwYDVR0jBBgwFoAU39Ppz1YkEZb5qNjpKFWixi4YZD8wcgYDVR0RAQH/BGgwZoZkaHR0cHM6Ly9naXRodWIuY29tL3Bvc3RpbC1kZXYvcG9zdGlsLWNsaS8uZ2l0aHViL3dvcmtmbG93cy9iZW5jaG1hcmstY2FsaWJyYXRpb24ueW1sQHJlZnMvaGVhZHMvbWFpbjA5BgorBgEEAYO/MAEBBCtodHRwczovL3Rva2VuLmFjdGlvbnMuZ2l0aHVidXNlcmNvbnRlbnQuY29tMB8GCisGAQQBg78wAQIEEXdvcmtmbG93X2Rpc3BhdGNoMDYGCisGAQQBg78wAQMEKDM1NmJhNjkxOTk5YzczNzU1MTUwNjAzOWQyYWQ0YmMyNmM2YzEzNTQwIwYKKwYBBAGDvzABBAQVQmVuY2htYXJrIGNhbGlicmF0aW9uMCMGCisGAQQBg78wAQUEFXBvc3RpbC1kZXYvcG9zdGlsLWNsaTAdBgorBgEEAYO/MAEGBA9yZWZzL2hlYWRzL21haW4wOwYKKwYBBAGDvzABCAQtDCtodHRwczovL3Rva2VuLmFjdGlvbnMuZ2l0aHVidXNlcmNvbnRlbnQuY29tMHQGCisGAQQBg78wAQkEZgxkaHR0cHM6Ly9naXRodWIuY29tL3Bvc3RpbC1kZXYvcG9zdGlsLWNsaS8uZ2l0aHViL3dvcmtmbG93cy9iZW5jaG1hcmstY2FsaWJyYXRpb24ueW1sQHJlZnMvaGVhZHMvbWFpbjA4BgorBgEEAYO/MAEKBCoMKDM1NmJhNjkxOTk5YzczNzU1MTUwNjAzOWQyYWQ0YmMyNmM2YzEzNTQwHQYKKwYBBAGDvzABCwQPDA1naXRodWItaG9zdGVkMDgGCisGAQQBg78wAQwEKgwoaHR0cHM6Ly9naXRodWIuY29tL3Bvc3RpbC1kZXYvcG9zdGlsLWNsaTA4BgorBgEEAYO/MAENBCoMKDM1NmJhNjkxOTk5YzczNzU1MTUwNjAzOWQyYWQ0YmMyNmM2YzEzNTQwHwYKKwYBBAGDvzABDgQRDA9yZWZzL2hlYWRzL21haW4wGgYKKwYBBAGDvzABDwQMDAoxMjQ2NjkyMjA4MC0GCisGAQQBg78wARAEHwwdaHR0cHM6Ly9naXRodWIuY29tL3Bvc3RpbC1kZXYwGQYKKwYBBAGDvzABEQQLDAkyNzgwNzUwOTEwdAYKKwYBBAGDvzABEgRmDGRodHRwczovL2dpdGh1Yi5jb20vcG9zdGlsLWRldi9wb3N0aWwtY2xpLy5naXRodWIvd29ya2Zsb3dzL2JlbmNobWFyay1jYWxpYnJhdGlvbi55bWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wARMEKgwoMzU2YmE2OTE5OTljNzM3NTUxNTA2MDM5ZDJhZDRiYzI2YzZjMTM1NDAhBgorBgEEAYO/MAEUBBMMEXdvcmtmbG93X2Rpc3BhdGNoMFwGCisGAQQBg78wARUETgxMaHR0cHM6Ly9naXRodWIuY29tL3Bvc3RpbC1kZXYvcG9zdGlsLWNsaS9hY3Rpb25zL3J1bnMvMzU5NTEyMjc4MTgvYXR0ZW1wdHMvMTAWBgorBgEEAYO/MAEWBAgMBnB1YmxpYzA+BgorBgEEAYO/MAEYBDAMLnJlcG86cG9zdGlsLWRldi9wb3N0aWwtY2xpOnJlZjpyZWZzL2hlYWRzL21haW4wgYkGCisGAQQB1nkCBAIEewR5AHcAdQDdPTBqxscRMmMZHhyZZzcCokpeuN48rf+HinKALynujgAAAaDRjQg8AAAEAwBGMEQCIA25leOyKmRw17F05hlr5J6P3ixCzN5StQKY9VYlEbOBAiAxnOGPyzGL0UZXGz1Xsuf1F37DlEX8gWa6ecFqu9DO+jAKBggqhkjOPQQDAwNnADBkAjBfwUKXDInAL+Zw92YC7dtA9QsFKocFN64xLmqmgt/XFikF2bqvnjxXwKHjNLN097kCMFw7dEeB5Mow57buQNz5w7aNVLiyQUj4yMkZHYMG1jeyLNyX6TH3puVkUkCyZ451Rg=="},"tlogEntries":[{"logIndex":"2931301373","logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="},"kindVersion":{"kind":"dsse","version":"0.0.1"},"integratedTime":"1790222076","inclusionPromise":{"signedEntryTimestamp":"MEUCIQDCgfzKH8/bVMuCjJMQOJv3i0MQQr07MNHb1Uwf9xp3pgIgBo1ezPSUeVGkGtzmJk3kyMBs09PSvZbzz0Dx2k0Dz8M="},"inclusionProof":{"logIndex":"2809397111","rootHash":"Vn15TkT5PZEm3yaYi6nQb9fUHmJvOF0hbxG8sKTrrsk=","treeSize":"2809397114","hashes":["yPZuojy48N6T5MCTrHRUV8FmxIK2IV/eLV9oQW+q9xw=","T7x6x9536whoyj2NOINS3o62W6xkpe9a62PBEk2H5PM=","30Zde/VlanNqIAJ4xVEmsRyOXNJfbxivKdbERb1jwCY=","jikTFf4uJ/5mPDwHZu6zi1lvb98dwCFUMrUl1m5H1zQ=","d3kUmYWFW6h8odnI8y2vD0HrgJ9y3CT1X9AzXFONAXU=","tiFqHZ8lUTDprmxsXljr+amQdu4zggjDW7vkX06cCeQ=","M/75hVppVMxSO4Rm+Jm5tQM4beBI0Y4H+gD88oC5SWE=","YynjcohjWYCaMsBUNecGGt7CcGOPUp4ptp+qIwqWtYU=","AjecGVzWCGvv1ReQ6bfKJRYHPYj8qkcAq/FKIL7QlCE=","fvkKNvqMS9VjKt2w1xPPoWL3rNKRiCMoL+9HaAxta6s=","2AcaJeW6o2XkehfHemfX92mnBLOgOfOFZ3akWx3AafU=","YNUMG9prkoawfQBmaFY7qsDP19cMqN0IZCWa+e4iCMo=","zWw6+c7S7fX78W88J88XZesFr7oyySw0jy6z+AeJQLY=","rB/kBDdOKYlOkeIX+pOncBiSlfvjbBDkvzrUbbRWAPI=","w/W66dfaBH625xILHkiQlUbOJ5QelZSp+C6GqYnB6MI=","MvQ+iLog0baE7oTRB31sH0woXyIDAaRd+xen1XrPpYg=","ebG2CsH5m5FuJQ0PR5vTPsSKI4I0VzKQZab+C6nZY+0=","eHW2pk/Q5c7ktN0pxWuY595q3Xtfd15g1SW9qFcyFx4=","JRIcZD2G8h7Iq0hZuQz5n2EZlMoRl+CidVyXLz5qg6Q=","Q3KSjpgGMyRso12cTtkfLjhxmsEOe02yzxJhFEq9+FE=","z5EmMdp/Wd0IVl0nUx8KTZCspZnHuBgCWIdw5mKBkek=","jJJsjr/pA/yBIChBkuFvUPlTHgDHtR/HJRcV60HXdRc=","mfWjSdNaEfNXxDww6AAOw/NoK9e6z1yKaZm5xvmMabs=","qxzHanAzz57SDdmJe0B7bJK72NTIBbwEMGBBKvDOROw=","xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="],"checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n2809397114\nVn15TkT5PZEm3yaYi6nQb9fUHmJvOF0hbxG8sKTrrsk=\n\n— rekor.sigstore.dev wNI9ajBFAiBnhAU0eVH5gmsULQxBawvEzuZqGfaVSawOOkL+uuxk7QIhALMuEouKEr+AO9eLSGXBV6K3mJ2io/G15j24Oulo6nW7\n"}},"canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiZHNzZSIsInNwZWMiOnsiZW52ZWxvcGVIYXNoIjp7ImFsZ29yaXRobSI6InNoYTI1NiIsInZhbHVlIjoiYzA1OTA3ZGY3NDUzNmNiN2NmMjFhMDZmZjRlMzJiOGRkM2NlNmIwYjA0NGU0M2RhNDE3NTkxNTk0YWY2YzJkOCJ9LCJwYXlsb2FkSGFzaCI6eyJhbGdvcml0aG0iOiJzaGEyNTYiLCJ2YWx1ZSI6IjY2YjUzNTU0MzJmNzUzYzQ4NDBjOWJlNTNkMmJlODdjZmY5MmM1MjAyNWFhNTEzZTEzODc4YTI3OWRhMTIxZDkifSwic2lnbmF0dXJlcyI6W3sic2lnbmF0dXJlIjoiTUVVQ0lETW9lTDVtQUoyazZSY2p4dENabE9mTHI0RVl1QVhLM0diNG5iclE4cVE2QWlFQXpjSWh4d0FESytBOHJDaGlGaUQvUjE2VXNrZXdZbzVqZS92ZjNDR1J2TWc9IiwidmVyaWZpZXIiOiJMUzB0TFMxQ1JVZEpUaUJEUlZKVVNVWkpRMEZVUlMwdExTMHRDazFKU1VoVlJFTkRRblJsWjBGM1NVSkJaMGxWUm1nNUsxcE9OVVJ0VkRkME5FeHFhRWxOUWxreWNscHdSWGhyZDBObldVbExiMXBKZW1vd1JVRjNUWGNLVG5wRlZrMUNUVWRCTVZWRlEyaE5UV015Ykc1ak0xSjJZMjFWZFZwSFZqSk5ValIzU0VGWlJGWlJVVVJGZUZaNllWZGtlbVJIT1hsYVV6RndZbTVTYkFwamJURnNXa2RzYUdSSFZYZElhR05PVFdwWmQwOVVTVEJOUkUweFRrUk5NVmRvWTA1TmFsbDNUMVJKTUUxRVVYZE9SRTB4VjJwQlFVMUdhM2RGZDFsSUNrdHZXa2w2YWpCRFFWRlpTVXR2V2tsNmFqQkVRVkZqUkZGblFVVXdla0pDT1hsNVFXbFdTWEJvTTFJd0szUkJVSEpQYzIwMVVYUnZURGR1YldSblptY0tkRlo2T1c0NGQxTldha1IyV21sSk9WQnlabmt5TlRkSmFVUm1NbHBaVWpOS2RIaERiemQyU1hoR1UzQkVlRXhpTlV0UFEwSm1XWGRuWjFoNVRVRTBSd3BCTVZWa1JIZEZRaTkzVVVWQmQwbElaMFJCVkVKblRsWklVMVZGUkVSQlMwSm5aM0pDWjBWR1FsRmpSRUY2UVdSQ1owNVdTRkUwUlVablVWVlNhR0pTQ2toWGRWZHpWelkwV0Vka1NUWmtUMk4xU2pCRE9GWnJkMGgzV1VSV1VqQnFRa0puZDBadlFWVXpPVkJ3ZWpGWmEwVmFZalZ4VG1wd1MwWlhhWGhwTkZrS1drUTRkMk5uV1VSV1VqQlNRVkZJTDBKSFozZGFiMXByWVVoU01HTklUVFpNZVRsdVlWaFNiMlJYU1hWWk1qbDBURE5DZG1NelVuQmlRekZyV2xoWmRncGpSemw2WkVkc2MweFhUbk5oVXpoMVdqSnNNR0ZJVm1sTU0yUjJZMjEwYldKSE9UTmplVGxwV2xjMWFtRkhNV2hqYlhOMFdUSkdjMkZYU25sWldGSndDbUl5TkhWbFZ6RnpVVWhLYkZwdVRYWmhSMVpvV2toTmRtSlhSbkJpYWtFMVFtZHZja0puUlVWQldVOHZUVUZGUWtKRGRHOWtTRkozWTNwdmRrd3pVbllLWVRKV2RVeHRSbXBrUjJ4MlltNU5kVm95YkRCaFNGWnBaRmhPYkdOdFRuWmlibEpzWW01UmRWa3lPWFJOUWpoSFEybHpSMEZSVVVKbk56aDNRVkZKUlFwRldHUjJZMjEwYldKSE9UTllNbEp3WXpOQ2FHUkhUbTlOUkZsSFEybHpSMEZSVVVKbk56aDNRVkZOUlV0RVRURk9iVXBvVG1wcmVFOVVhelZaZW1ONkNrNTZWVEZOVkZWM1RtcEJlazlYVVhsWlYxRXdXVzFOZVU1dFRUSlpla1Y2VGxSUmQwbDNXVXRMZDFsQ1FrRkhSSFo2UVVKQ1FWRldVVzFXZFZreWFIUUtXVmhLY2tsSFRtaGlSMnhwWTIxR01HRlhPWFZOUTAxSFEybHpSMEZSVVVKbk56aDNRVkZWUlVaWVFuWmpNMUp3WWtNeGExcFlXWFpqUnpsNlpFZHNjd3BNVjA1ellWUkJaRUpuYjNKQ1owVkZRVmxQTDAxQlJVZENRVGw1V2xkYWVrd3lhR3haVjFKNlRESXhhR0ZYTkhkUGQxbExTM2RaUWtKQlIwUjJla0ZDQ2tOQlVYUkVRM1J2WkVoU2QyTjZiM1pNTTFKMllUSldkVXh0Um1wa1IyeDJZbTVOZFZveWJEQmhTRlpwWkZoT2JHTnRUblppYmxKc1ltNVJkVmt5T1hRS1RVaFJSME5wYzBkQlVWRkNaemM0ZDBGUmEwVmFaM2hyWVVoU01HTklUVFpNZVRsdVlWaFNiMlJYU1hWWk1qbDBURE5DZG1NelVuQmlRekZyV2xoWmRncGpSemw2WkVkc2MweFhUbk5oVXpoMVdqSnNNR0ZJVm1sTU0yUjJZMjEwYldKSE9UTmplVGxwV2xjMWFtRkhNV2hqYlhOMFdUSkdjMkZYU25sWldGSndDbUl5TkhWbFZ6RnpVVWhLYkZwdVRYWmhSMVpvV2toTmRtSlhSbkJpYWtFMFFtZHZja0puUlVWQldVOHZUVUZGUzBKRGIwMUxSRTB4VG0xS2FFNXFhM2dLVDFSck5WbDZZM3BPZWxVeFRWUlZkMDVxUVhwUFYxRjVXVmRSTUZsdFRYbE9iVTB5V1hwRmVrNVVVWGRJVVZsTFMzZFpRa0pCUjBSMmVrRkNRM2RSVUFwRVFURnVZVmhTYjJSWFNYUmhSemw2WkVkV2EwMUVaMGREYVhOSFFWRlJRbWMzT0hkQlVYZEZTMmQzYjJGSVVqQmpTRTAyVEhrNWJtRllVbTlrVjBsMUNsa3lPWFJNTTBKMll6TlNjR0pETVd0YVdGbDJZMGM1ZW1SSGJITk1WMDV6WVZSQk5FSm5iM0pDWjBWRlFWbFBMMDFCUlU1Q1EyOU5TMFJOTVU1dFNtZ0tUbXByZUU5VWF6VlplbU42VG5wVk1VMVVWWGRPYWtGNlQxZFJlVmxYVVRCWmJVMTVUbTFOTWxsNlJYcE9WRkYzU0hkWlMwdDNXVUpDUVVkRWRucEJRZ3BFWjFGU1JFRTVlVnBYV25wTU1taHNXVmRTZWt3eU1XaGhWelIzUjJkWlMwdDNXVUpDUVVkRWRucEJRa1IzVVUxRVFXOTRUV3BSTWs1cWEzbE5ha0UwQ2sxRE1FZERhWE5IUVZGUlFtYzNPSGRCVWtGRlNIZDNaR0ZJVWpCalNFMDJUSGs1Ym1GWVVtOWtWMGwxV1RJNWRFd3pRblpqTTFKd1lrTXhhMXBZV1hjS1IxRlpTMHQzV1VKQ1FVZEVkbnBCUWtWUlVVeEVRV3Q1VG5wbmQwNTZWWGRQVkVWM1pFRlpTMHQzV1VKQ1FVZEVkbnBCUWtWblVtMUVSMUp2WkVoU2R3cGplbTkyVERKa2NHUkhhREZaYVRWcVlqSXdkbU5IT1hwa1IyeHpURmRTYkdScE9YZGlNMDR3WVZkM2RGa3llSEJNZVRWdVlWaFNiMlJYU1haa01qbDVDbUV5V25OaU0yUjZUREpLYkdKdFRtOWlWMFo1WVhreGFsbFhlSEJaYmtwb1pFZHNkbUpwTlRWaVYzaEJZMjFXYldONU9XOWFWMFpyWTNrNWRGbFhiSFVLVFVSblIwTnBjMGRCVVZGQ1p6YzRkMEZTVFVWTFozZHZUWHBWTWxsdFJUSlBWRVUxVDFSc2FrNTZUVE5PVkZWNFRsUkJNazFFVFRWYVJFcG9Xa1JTYVFwWmVra3lXWHBhYWsxVVRURk9SRUZvUW1kdmNrSm5SVVZCV1U4dlRVRkZWVUpDVFUxRldHUjJZMjEwYldKSE9UTllNbEp3WXpOQ2FHUkhUbTlOUm5kSENrTnBjMGRCVVZGQ1p6YzRkMEZTVlVWVVozaE5ZVWhTTUdOSVRUWk1lVGx1WVZoU2IyUlhTWFZaTWpsMFRETkNkbU16VW5CaVF6RnJXbGhaZG1OSE9Yb0taRWRzYzB4WFRuTmhVemxvV1ROU2NHSXlOWHBNTTBveFltNU5kazE2VlRWT1ZFVjVUV3BqTkUxVVozWlpXRkl3V2xjeGQyUklUWFpOVkVGWFFtZHZjZ3BDWjBWRlFWbFBMMDFCUlZkQ1FXZE5RbTVDTVZsdGVIQlpla0VyUW1kdmNrSm5SVVZCV1U4dlRVRkZXVUpFUVUxTWJrcHNZMGM0Tm1OSE9YcGtSMnh6Q2t4WFVteGthVGwzWWpOT01HRlhkM1JaTW5od1QyNUtiRnBxY0hsYVYxcDZUREpvYkZsWFVucE1NakZvWVZjMGQyZFphMGREYVhOSFFWRlJRakZ1YTBNS1FrRkpSV1YzVWpWQlNHTkJaRkZFWkZCVVFuRjRjMk5TVFcxTldraG9lVnBhZW1ORGIydHdaWFZPTkRoeVppdElhVzVMUVV4NWJuVnFaMEZCUVdGRVVncHFVV2M0UVVGQlJVRjNRa2ROUlZGRFNVRXlOV3hsVDNsTGJWSjNNVGRHTURWb2JISTFTalpRTTJsNFEzcE9OVk4wVVV0Wk9WWlpiRVZpVDBKQmFVRjRDbTVQUjFCNWVrZE1NRlZhV0VkNk1WaHpkV1l4UmpNM1JHeEZXRGhuVjJFMlpXTkdjWFU1UkU4cmFrRkxRbWRuY1docmFrOVFVVkZFUVhkT2JrRkVRbXNLUVdwQ1puZFZTMWhFU1c1QlRDdGFkemt5V1VNM1pIUkJPVkZ6Umt0dlkwWk9OalI0VEcxeGJXZDBMMWhHYVd0R01tSnhkbTVxZUZoM1MwaHFUa3hPTUFvNU4ydERUVVozTjJSRlpVSTFUVzkzTlRkaWRWRk9lalYzTjJGT1ZreHBlVkZWYWpSNVRXdGFTRmxOUnpGcVpYbE1UbmxZTmxSSU0zQjFWbXRWYTBONUNsbzBOVEZTWnowOUNpMHRMUzB0UlU1RUlFTkZVbFJKUmtsRFFWUkZMUzB0TFMwSyJ9XX19"}],"timestampVerificationData":{}},"dsseEnvelope":{"payload":"eyJfdHlwZSI6Imh0dHBzOi8vaW4tdG90by5pby9TdGF0ZW1lbnQvdjEiLCJzdWJqZWN0IjpbeyJuYW1lIjoiYmFzZWxpbmUtdXMuanNvbiIsImRpZ2VzdCI6eyJzaGEyNTYiOiJhMjg3ZmFkZmQ4NGJhMDcyNTA3YzUwMDQyNDQyZmM4MmM5N2QzN2Q5MmU5Mjg3ZjdjM2ZjMjVmMGQ3MTM2NWE1In19XSwicHJlZGljYXRlVHlwZSI6Imh0dHBzOi8vc2xzYS5kZXYvcHJvdmVuYW5jZS92MSIsInByZWRpY2F0ZSI6eyJidWlsZERlZmluaXRpb24iOnsiYnVpbGRUeXBlIjoiaHR0cHM6Ly9hY3Rpb25zLmdpdGh1Yi5pby9idWlsZHR5cGVzL3dvcmtmbG93L3YxIiwiZXh0ZXJuYWxQYXJhbWV0ZXJzIjp7IndvcmtmbG93Ijp7InJlZiI6InJlZnMvaGVhZHMvbWFpbiIsInJlcG9zaXRvcnkiOiJodHRwczovL2dpdGh1Yi5jb20vcG9zdGlsLWRldi9wb3N0aWwtY2xpIiwicGF0aCI6Ii5naXRodWIvd29ya2Zsb3dzL2JlbmNobWFyay1jYWxpYnJhdGlvbi55bWwifX0sImludGVybmFsUGFyYW1ldGVycyI6eyJnaXRodWIiOnsiZXZlbnRfbmFtZSI6IndvcmtmbG93X2Rpc3BhdGNoIiwicmVwb3NpdG9yeV9pZCI6IjEyNDY2OTIyMDgiLCJyZXBvc2l0b3J5X293bmVyX2lkIjoiMjc4MDc1MDkxIiwicnVubmVyX2Vudmlyb25tZW50IjoiZ2l0aHViLWhvc3RlZCJ9fSwicmVzb2x2ZWREZXBlbmRlbmNpZXMiOlt7InVyaSI6ImdpdCtodHRwczovL2dpdGh1Yi5jb20vcG9zdGlsLWRldi9wb3N0aWwtY2xpQHJlZnMvaGVhZHMvbWFpbiIsImRpZ2VzdCI6eyJnaXRDb21taXQiOiIzNTZiYTY5MTk5OWM3Mzc1NTE1MDYwMzlkMmFkNGJjMjZjNmMxMzU0In19XX0sInJ1bkRldGFpbHMiOnsiYnVpbGRlciI6eyJpZCI6Imh0dHBzOi8vZ2l0aHViLmNvbS9wb3N0aWwtZGV2L3Bvc3RpbC1jbGkvLmdpdGh1Yi93b3JrZmxvd3MvYmVuY2htYXJrLWNhbGlicmF0aW9uLnltbEByZWZzL2hlYWRzL21haW4ifSwibWV0YWRhdGEiOnsiaW52b2NhdGlvbklkIjoiaHR0cHM6Ly9naXRodWIuY29tL3Bvc3RpbC1kZXYvcG9zdGlsLWNsaS9hY3Rpb25zL3J1bnMvMzU5NTEyMjc4MTgvYXR0ZW1wdHMvMSJ9fX19","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MEUCIDMoeL5mAJ2k6RcjxtCZlOfLr4EYuAXK3Gb4nbrQ8qQ6AiEAzcIhxwADK+A8rChiFiD/R16UskewYo5je/vf3CGRvMg="}]}} diff --git a/bench/baseline-us.json b/bench/baseline-us.json index cff0982..878fbc4 100644 --- a/bench/baseline-us.json +++ b/bench/baseline-us.json @@ -6,8 +6,198 @@ }, "profiles": { "openai/gpt-5.6-luna": { - "populated": false, - "instructions": "Record a predeclared ten-slot calibration cohort with provisional-models-us.json and exact Azure/US provider evidence." + "populated": true, + "generatedAt": "2026-09-24T03:54:35.279Z", + "reviewMode": "exhaustive", + "sourceRunAt": "2026-09-24T03:42:49.468Z", + "providerContractEnforced": true, + "screeningProfileSha256": "486f261a29c6f6ece01cafac681c082f94b5f0e3e7eb8ca432a5898ef0a0efe2", + "upstreamProviderIdentity": "Azure", + "calibration": { + "reportCount": 10, + "cohortId": "6cd32b7f-7c92-472e-81ac-a86e661d2834", + "manifestSha256": "0b295c46a6ca0bf5ee655d1b40d20a74507c9827e338c6cb1688be2fac8227c4", + "sourceSha": "356ba691999c737551506039d2ad4bc26c6c1354", + "workflowRunId": "35951227818", + "binarySha256": "91044194b101d3fa66f94a95635c459ce209104b33b638a891c1e66408e48e18", + "providerContractSha256": "d48fd24c0d2f762046ca9162acaa076dad4dcb8f2155ba0b40b4198aee3b27d1", + "comparisonCohortSha256": "d7a85f880d6a7c1c27510db39580bc189366add1c489b0a5598ac9141a0617bc", + "reports": [ + { + "slot": 1, + "nonce": "aae97988-912c-421e-b132-53a91c000515", + "runId": "calibration-35951227818-01", + "ranAt": "2026-09-24T03:29:22.237Z", + "rawSha256": "47d5deb43ccb7e7fa5bb80de1cc881237ce2df5fb253eb92fe75b2c01ee058d8", + "semanticSha256": "aa11da9bb7636883668572dd4c6a07d0dd645b18f3033c4c4cf899f82e867fa3", + "receiptRawSha256": "a5a4bcfc45d4fca986226759aa41f21a8bdb599718e35ab0671a956e4d550b70", + "binarySha256": "91044194b101d3fa66f94a95635c459ce209104b33b638a891c1e66408e48e18", + "detected": 52, + "falsePositives": 0, + "gateVerdictCorrect": 61, + "totalCostUsdDecimal": "0.074824013", + "p50LatencyMs": 8673, + "p95LatencyMs": 21854 + }, + { + "slot": 2, + "nonce": "dc52f143-78af-4baf-9583-b84d4e08159b", + "runId": "calibration-35951227818-02", + "ranAt": "2026-09-24T03:32:01.131Z", + "rawSha256": "01c4cf74adc8bc7ed85845ddd167ac6aa015ff76d0027763dd8f8d8355bd2747", + "semanticSha256": "ee67790a52a45785a654a77ee3399dc8dfbbd8e75bcf5486da131343678ce0a2", + "receiptRawSha256": "2481af51649ec419f621c28fc825f3726c34e9204ef6d57ed0e069ec5b47e2c9", + "binarySha256": "91044194b101d3fa66f94a95635c459ce209104b33b638a891c1e66408e48e18", + "detected": 50, + "falsePositives": 0, + "gateVerdictCorrect": 60, + "totalCostUsdDecimal": "0.074190094", + "p50LatencyMs": 9028, + "p95LatencyMs": 21123 + }, + { + "slot": 3, + "nonce": "ec0db226-8c8f-4e7c-b9e7-6931f2d4c5a5", + "runId": "calibration-35951227818-03", + "ranAt": "2026-09-24T03:34:17.191Z", + "rawSha256": "43fc6a4c5d73a27ae047850025e5c6cd2e07b3e373df77998e9b1e884e8da3e9", + "semanticSha256": "f0431fadfb2912ded203a52ae101e2da5ecbc4eef2ef8886f960482204770976", + "receiptRawSha256": "eccc872e07260bbc888ac80c3697d705b49cee94ebf6d06766080a144a422267", + "binarySha256": "91044194b101d3fa66f94a95635c459ce209104b33b638a891c1e66408e48e18", + "detected": 53, + "falsePositives": 1, + "gateVerdictCorrect": 59, + "totalCostUsdDecimal": "0.069530736", + "p50LatencyMs": 7707, + "p95LatencyMs": 17639 + }, + { + "slot": 4, + "nonce": "6716647b-d1f5-4f15-9fd6-6ff193c2d3d5", + "runId": "calibration-35951227818-04", + "ranAt": "2026-09-24T03:37:45.174Z", + "rawSha256": "333aa5bd6be63b11f8433e9497c001fd282ff992bbfceb3f5548c8deae753b6c", + "semanticSha256": "39a9d168e1f319658b984ce27e556bc76e2c7eb3107e2190702ac8b7ebe03be3", + "receiptRawSha256": "85a3ad2ccfd1e186e9eef2667e848167b35e26884eadfb08b5f7c089f459b218", + "binarySha256": "91044194b101d3fa66f94a95635c459ce209104b33b638a891c1e66408e48e18", + "detected": 50, + "falsePositives": 0, + "gateVerdictCorrect": 58, + "totalCostUsdDecimal": "0.071011028", + "p50LatencyMs": 8555, + "p95LatencyMs": 18714 + }, + { + "slot": 5, + "nonce": "f286fd8a-d9e7-45dc-86e7-0ddb825fb313", + "runId": "calibration-35951227818-05", + "ranAt": "2026-09-24T03:40:10.195Z", + "rawSha256": "9a27d429535b774e61b7d0ad9b2a541c3dcc0fc54ad24dc66c47f4f662a259fd", + "semanticSha256": "8cbf046c2d0fc279b1d4558b5d73364063d99d3f9d2c6cf9eb2f35677b45521c", + "receiptRawSha256": "8cd6ecae4597ff7d2b3a75f179bd090b24c154ef94a18d4f22182a7f7dda702e", + "binarySha256": "91044194b101d3fa66f94a95635c459ce209104b33b638a891c1e66408e48e18", + "detected": 50, + "falsePositives": 0, + "gateVerdictCorrect": 61, + "totalCostUsdDecimal": "0.068661021", + "p50LatencyMs": 8268, + "p95LatencyMs": 19374 + }, + { + "slot": 6, + "nonce": "7a28a03a-fdad-4c36-8970-855366ce4374", + "runId": "calibration-35951227818-06", + "ranAt": "2026-09-24T03:42:49.468Z", + "rawSha256": "69b53fd99fa907a5bb33843518b5486cf5153a946b78d0808ed6f33379e2d1f5", + "semanticSha256": "fbf2d79ab6b8ac5e1e2e3e8d13482ae708546bb12bc5ee398eecdeb2b03ef8f8", + "receiptRawSha256": "435e3e9e22efce56bb60474e750422cefdc1f28eea58ae7d87651f62d36727b2", + "binarySha256": "91044194b101d3fa66f94a95635c459ce209104b33b638a891c1e66408e48e18", + "detected": 51, + "falsePositives": 0, + "gateVerdictCorrect": 63, + "totalCostUsdDecimal": "0.066395373", + "p50LatencyMs": 8887, + "p95LatencyMs": 19671 + }, + { + "slot": 7, + "nonce": "1a53c9bc-14ea-49b3-b790-97e797ee7aef", + "runId": "calibration-35951227818-07", + "ranAt": "2026-09-24T03:45:23.989Z", + "rawSha256": "f12b2e905ba9a6f6923c1f80f38854aa05846d1ed7665821465f3111df3cae8f", + "semanticSha256": "28282ea58917e8afbbfb26a99222adb3f74ca42fbedaab92f32c9eb5af88370d", + "receiptRawSha256": "10edf63a709c64af94b37c021fa17ddb453aa849cf9869fc75b00bf3b9ad06c2", + "binarySha256": "91044194b101d3fa66f94a95635c459ce209104b33b638a891c1e66408e48e18", + "detected": 54, + "falsePositives": 0, + "gateVerdictCorrect": 60, + "totalCostUsdDecimal": "0.071498603", + "p50LatencyMs": 9092, + "p95LatencyMs": 19537 + }, + { + "slot": 8, + "nonce": "a0a8e9fe-eea7-476d-b07f-1d999235786e", + "runId": "calibration-35951227818-08", + "ranAt": "2026-09-24T03:47:33.865Z", + "rawSha256": "559830057dd3679865017a74c92c730b299c0536774be10a0029c339ca7ecd8c", + "semanticSha256": "6a98a0c1c3551048a4e38acadea7076f487506956b99e1c05a71454ffe807e71", + "receiptRawSha256": "73324cb2d82312ff9a688ad8d56a84b15d18156f8cbc2bac8925cc2d2a230da7", + "binarySha256": "91044194b101d3fa66f94a95635c459ce209104b33b638a891c1e66408e48e18", + "detected": 51, + "falsePositives": 0, + "gateVerdictCorrect": 59, + "totalCostUsdDecimal": "0.068616196", + "p50LatencyMs": 8189, + "p95LatencyMs": 13723 + }, + { + "slot": 9, + "nonce": "8994cc67-a2fd-4f97-989c-2191be47544c", + "runId": "calibration-35951227818-09", + "ranAt": "2026-09-24T03:49:54.898Z", + "rawSha256": "fbf0b432bd97ea63424df1b1286b0e80a1994eec21c437ea20ba764a840d9d19", + "semanticSha256": "a6083ebd4eacd4b6cefdfc563fccb17c42a90e94382c5a29eaf96662976c714d", + "receiptRawSha256": "e2512cc28147922b89289d9911262e206462392854e8dc95beea28dfb8984672", + "binarySha256": "91044194b101d3fa66f94a95635c459ce209104b33b638a891c1e66408e48e18", + "detected": 51, + "falsePositives": 0, + "gateVerdictCorrect": 63, + "totalCostUsdDecimal": "0.072017022", + "p50LatencyMs": 7967, + "p95LatencyMs": 19305 + }, + { + "slot": 10, + "nonce": "6561aebb-3965-41f5-9587-c5834d820a17", + "runId": "calibration-35951227818-10", + "ranAt": "2026-09-24T03:52:18.608Z", + "rawSha256": "fe7c67a4d7153afd96b3501954c3d2bffce860023b751d3a9662d04cd8b018d0", + "semanticSha256": "9f3a46b48f71e6e773c4d47dc20a033ebeb0d6426acbdd2a458e22c6c29b6ada", + "receiptRawSha256": "33c21bf7d1c7c98554852f677a148b1b8b943841eb0425d6efb7d97b07d45d74", + "binarySha256": "91044194b101d3fa66f94a95635c459ce209104b33b638a891c1e66408e48e18", + "detected": 51, + "falsePositives": 0, + "gateVerdictCorrect": 62, + "totalCostUsdDecimal": "0.069771053", + "p50LatencyMs": 8680, + "p95LatencyMs": 21209 + } + ] + }, + "totalCases": 70, + "scoredCases": 70, + "defectCases": 57, + "detectionRate": 0.9, + "falsePositives": 0, + "gateVerdictCorrectness": 0.8642857142857143, + "meanCostUsdPerCase": 0.0010689144714285713, + "maximumRunCostUsdDecimal": "0.074824013", + "costCaseCount": 70, + "latencyMs": { + "p50": 8614, + "p95": 19455.5 + } } } } diff --git a/bench/src/compare-baseline.test.ts b/bench/src/compare-baseline.test.ts index 30234d9..38764d4 100644 --- a/bench/src/compare-baseline.test.ts +++ b/bench/src/compare-baseline.test.ts @@ -115,11 +115,12 @@ test("release prerequisite rejects an unpopulated US baseline before attestation } }); -test("unpopulated US calibration target keeps the embedded provider on Azure/EU", async () => { +test("attested US calibration binds its source and route while the embedded provider stays on Azure/EU", async () => { const root = resolve(import.meta.dir, "..", ".."); - const [usBytes, euBytes, us, eu] = await Promise.all([ + const [usBytes, euBytes, attestationBytes, us, eu] = await Promise.all([ readFile(resolve(root, "bench/baseline-us.json"), "utf8"), readFile(resolve(root, "bench/baseline.json"), "utf8"), + readFile(resolve(root, "bench/baseline-us.attestation.json"), "utf8"), screeningProfileMetadata(resolve(root, "provisional-models-us.json")), screeningProfileMetadata(resolve(root, "provisional-models-eu.json")), ]); @@ -129,8 +130,36 @@ test("unpopulated US calibration target keeps the embedded provider on Azure/EU" expect(euBaseline.profiles["openai/gpt-5.6-luna"]?.populated).toBe(true); const usBaselineProfile = usBaseline.profiles["openai/gpt-5.6-luna"]; expect(usBaselineProfile).toBeDefined(); - expect(usBaselineProfile?.populated).toBe(false); - expect(usBaselineProfile?.instructions).toContain("predeclared ten-slot calibration cohort"); + expect(usBaselineProfile?.populated).toBe(true); + if (!usBaselineProfile?.populated || !isCalibratedBaselineProfile(usBaselineProfile)) { + throw new Error("US baseline must contain calibration evidence"); + } + expect(usBaselineProfile.calibration.reportCount).toBe(10); + expect(() => assertBaselineCalibrationIntegrity(usBaselineProfile)).not.toThrow(); + expect(usBaselineProfile.calibration.sourceSha).toMatch(/^[0-9a-f]{40}(?:[0-9a-f]{24})?$/u); + expect(usBaselineProfile.screeningProfileSha256).toBe(us.sha256); + expect(usBaselineProfile.calibration.providerContractSha256).toBe(us.providerContractSha256); + expect(usBaselineProfile.upstreamProviderIdentity).toBe("Azure"); + const attestation = JSON.parse(attestationBytes); + expect(attestation.mediaType).toBe("application/vnd.dev.sigstore.bundle.v0.3+json"); + expect(attestation.dsseEnvelope).toBeDefined(); + expect(attestation.verificationMaterial).toBeDefined(); + expect(attestation.dsseEnvelope.payloadType).toBe("application/vnd.in-toto+json"); + const statement = JSON.parse(Buffer.from(attestation.dsseEnvelope.payload, "base64").toString("utf8")); + expect(statement.predicateType).toBe("https://slsa.dev/provenance/v1"); + expect(statement.subject).toContainEqual({ + name: "baseline-us.json", + digest: { sha256: createHash("sha256").update(usBytes).digest("hex") }, + }); + expect(statement.predicate.buildDefinition.externalParameters.workflow).toEqual({ + ref: "refs/heads/main", + repository: "https://github.com/postil-dev/postil-cli", + path: ".github/workflows/benchmark-calibration.yml", + }); + expect(statement.predicate.buildDefinition.resolvedDependencies).toContainEqual({ + uri: "git+https://github.com/postil-dev/postil-cli@refs/heads/main", + digest: { gitCommit: usBaselineProfile.calibration.sourceSha }, + }); expect(await readFile(resolve(root, "provisional-models.json"), "utf8")).toBe(await readFile(resolve(root, "provisional-models-eu.json"), "utf8")); expect(usBaseline.corpus.fixtureCorpusSha256).toBe(euBaseline.corpus.fixtureCorpusSha256); expect(usBaseline.corpus.evaluatorSha256).not.toBe(euBaseline.corpus.evaluatorSha256); diff --git a/src/config.rs b/src/config.rs index 1498b91..09aa5fa 100644 --- a/src/config.rs +++ b/src/config.rs @@ -3958,7 +3958,25 @@ scorer = { enabled = true, default_model = "provider/scorer", reasoning_effort = serde_json::from_str(include_str!("../bench/baseline-us.json")).unwrap(); assert_eq!( calibration["profiles"]["openai/gpt-5.6-luna"]["populated"], - false + true + ); + assert_eq!( + calibration["profiles"]["openai/gpt-5.6-luna"]["upstreamProviderIdentity"], + "Azure" + ); + assert_eq!( + calibration["profiles"]["openai/gpt-5.6-luna"]["calibration"]["reportCount"], + 10 + ); + let calibration_source = + calibration["profiles"]["openai/gpt-5.6-luna"]["calibration"]["sourceSha"] + .as_str() + .unwrap(); + assert!(matches!(calibration_source.len(), 40 | 64)); + assert!( + calibration_source + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) ); assert_eq!(profile.upstream_provider_route, "azure/eu"); assert_eq!(profile.generator_chain, vec!["openai/gpt-5.6-luna"]);