From ea45566fc377709b3ba2b6e7bd0d4b65f911c063 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Fri, 13 Nov 2026 18:43:40 -0700 Subject: [PATCH] feat(network): optional PUMAS_GITHUB_TOKEN for GitHub API release requests Anonymous GitHub API requests from shared CI runners hit the 60/hour limit while listing ONNX Runtime releases, failing the packaged-ONNX verifiers. Pumas now reads an optional PUMAS_GITHUB_TOKEN and sends it as a bearer token only to the real GitHub API origin; loopback fixtures and other hosts never receive it, Debug output redacts it, and the child-process allowlist keeps it out of runtimes. The ONNX verifiers pass it through their sanitized environment, and CI supplies the job's short-lived github.token, so no credential lives in the repo. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build.yml | 3 + CHANGELOG.md | 9 ++ rust/crates/pumas-core/src/network/github.rs | 121 +++++++++++++++--- .../src/platform/child_environment.rs | 1 + scripts/release/packaged_onnx_support.py | 6 + scripts/release/test_inference_release.py | 10 ++ 6 files changed, 130 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0f38a455..e61a220b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -291,6 +291,8 @@ jobs: run: python scripts/release/synthetic_onnx_fixture.py --output-dir "${{ runner.temp }}/native-synthetic-fixture" - name: Verify untrained model lifecycle on ONNX Runtime installed by the packaged Pumas timeout-minutes: 20 + env: + PUMAS_GITHUB_TOKEN: ${{ github.token }} run: python scripts/release/verify-packaged-onnx.py "${{ runner.temp }}/inference-candidate/extracted/${{ matrix.binary }}" "${{ runner.temp }}/native-synthetic-fixture" > "${{ runner.temp }}/inference-candidate/native-synthetic-lifecycle.log" - name: Verify required inference archive run: node scripts/release/check-artifacts.mjs "${{ runner.temp }}/inference-candidate" headless-inference-${{ matrix.check }} @@ -359,6 +361,7 @@ jobs: timeout-minutes: 20 env: EXPECTED_SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + PUMAS_GITHUB_TOKEN: ${{ github.token }} shell: bash run: | set -euo pipefail diff --git a/CHANGELOG.md b/CHANGELOG.md index 757e0b88..b02fe296 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Added + +- Pumas reads an optional `PUMAS_GITHUB_TOKEN` and sends it only to the + GitHub API (`api.github.com`) when listing or resolving runtime releases, + which lifts the anonymous rate limit. Pumas never stores the token, never + logs it, and never passes it to runtimes or other child processes. Without + it, requests stay anonymous as before. CI passes the job's short-lived + `github.token`, so no credential is committed. + ### Fixed - A copied import no longer reaches the file watcher as an external change, diff --git a/rust/crates/pumas-core/src/network/github.rs b/rust/crates/pumas-core/src/network/github.rs index 428a0b38..1c8e928e 100644 --- a/rust/crates/pumas-core/src/network/github.rs +++ b/rust/crates/pumas-core/src/network/github.rs @@ -36,6 +36,44 @@ use url::Url; // Re-export for convenience pub use crate::models::{GitHubAsset, GitHubRelease}; +/// Optional token for GitHub API requests. It raises the anonymous rate limit +/// and is never stored by Pumas; an unset or empty variable means anonymous. +pub const GITHUB_API_TOKEN_ENV: &str = "PUMAS_GITHUB_TOKEN"; + +/// A GitHub API token. It is sent only to the GitHub API origin and never +/// appears in `Debug` output, logs or errors. +#[derive(Clone)] +struct GitHubApiToken(String); + +impl std::fmt::Debug for GitHubApiToken { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("GitHubApiToken()") + } +} + +impl GitHubApiToken { + /// Accept a token only for the real GitHub API origin, and only when it + /// is a single header-safe word. Anything else stays anonymous. + fn for_api_base(api_base: &str, raw: Option) -> Option { + if api_base.trim_end_matches('/') != NetworkConfig::GITHUB_API_BASE { + return None; + } + let raw = raw?; + let Some(value) = raw.to_str().map(str::trim) else { + warn!("{GITHUB_API_TOKEN_ENV} is not valid UTF-8; using anonymous GitHub API requests"); + return None; + }; + if value.is_empty() { + return None; + } + if !value.bytes().all(|byte| byte.is_ascii_graphic()) { + warn!("{GITHUB_API_TOKEN_ENV} contains whitespace or control characters; using anonymous GitHub API requests"); + return None; + } + Some(Self(value.to_string())) + } +} + #[derive(serde::Serialize, serde::Deserialize)] struct StoredReleasesCache { #[serde(flatten)] @@ -507,6 +545,7 @@ pub struct GitHubClient { loopback_fixture: bool, http: Arc, api_base: String, + api_token: Option, cache: ReleasesCache, /// Whether we're currently fetching releases. is_fetching: AtomicBool, @@ -608,12 +647,23 @@ impl GitHubClient { } fn with_config(cache_dir: PathBuf, ttl: Duration, api_base: String) -> Result { - Ok(Self::with_http( - HttpClient::new()?, - cache_dir, - ttl, - api_base, - )) + let api_token = + GitHubApiToken::for_api_base(&api_base, std::env::var_os(GITHUB_API_TOKEN_ENV)); + let mut client = Self::with_http(HttpClient::new()?, cache_dir, ttl, api_base); + client.api_token = api_token; + Ok(client) + } + + /// Headers for a request to the GitHub API origin. + fn api_headers(&self) -> Vec<(String, String)> { + let mut headers = vec![( + "Accept".to_string(), + "application/vnd.github.v3+json".to_string(), + )]; + if let Some(GitHubApiToken(token)) = &self.api_token { + headers.push(("Authorization".to_string(), format!("Bearer {token}"))); + } + headers } fn with_http(http: HttpClient, cache_dir: PathBuf, ttl: Duration, api_base: String) -> Self { @@ -622,6 +672,7 @@ impl GitHubClient { loopback_fixture: false, http: Arc::new(http), api_base: api_base.trim_end_matches('/').to_string(), + api_token: None, cache: ReleasesCache::new(cache_dir, ttl), is_fetching: AtomicBool::new(false), fetch_lock: RwLock::new(()), @@ -1005,19 +1056,15 @@ impl GitHubClient { .with_base_delay(Duration::from_secs(2)); let http = self.http.clone(); let url_string = url.to_string(); + let headers = self.api_headers(); let (result, stats) = retry_async( &retry_config, || { let http = http.clone(); let url = url_string.clone(); - async move { - let headers = vec![( - "Accept".to_string(), - "application/vnd.github.v3+json".to_string(), - )]; - http.get_with_headers(&url, &headers).await - } + let headers = headers.clone(); + async move { http.get_with_headers(&url, &headers).await } }, |error| error.is_retryable(), ) @@ -1207,19 +1254,15 @@ impl GitHubClient { let http = self.http.clone(); let url_clone = url.clone(); + let headers = self.api_headers(); let (result, stats) = retry_async( &retry_config, || { let http = http.clone(); let url = url_clone.clone(); - async move { - let headers = vec![( - "Accept".to_string(), - "application/vnd.github.v3+json".to_string(), - )]; - http.get_with_headers(&url, &headers).await - } + let headers = headers.clone(); + async move { http.get_with_headers(&url, &headers).await } }, |e| e.is_retryable(), ) @@ -1304,6 +1347,44 @@ impl WebSource for GitHubClient { #[cfg(test)] mod tests { + #[test] + fn api_token_applies_only_to_the_github_api_origin() { + use super::GitHubApiToken; + let token = Some(std::ffi::OsString::from(" ghs_example123 ")); + let accepted = GitHubApiToken::for_api_base("https://api.github.com/", token.clone()) + .expect("token for the GitHub API"); + assert_eq!(accepted.0, "ghs_example123"); + assert_eq!(format!("{accepted:?}"), "GitHubApiToken()"); + assert!(GitHubApiToken::for_api_base("http://127.0.0.1:9/api", token).is_none()); + for rejected in ["", " ", "two words", "line\nbreak"] { + assert!( + GitHubApiToken::for_api_base("https://api.github.com", Some(rejected.into())) + .is_none(), + "{rejected:?}" + ); + } + assert!(GitHubApiToken::for_api_base("https://api.github.com", None).is_none()); + } + + #[test] + fn api_headers_carry_the_token_only_when_present() { + let root = TempDir::new().unwrap(); + let mut client = GitHubClient::with_http( + HttpClient::new().unwrap(), + root.path().to_path_buf(), + Duration::from_secs(60), + "https://api.github.com".into(), + ); + assert!(client + .api_headers() + .iter() + .all(|(name, _)| name != "Authorization")); + client.api_token = Some(super::GitHubApiToken("ghs_example".into())); + assert!(client + .api_headers() + .contains(&("Authorization".into(), "Bearer ghs_example".into()))); + } + use super::*; use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering}; use tempfile::TempDir; diff --git a/rust/crates/pumas-core/src/platform/child_environment.rs b/rust/crates/pumas-core/src/platform/child_environment.rs index 6a48c73c..92042981 100644 --- a/rust/crates/pumas-core/src/platform/child_environment.rs +++ b/rust/crates/pumas-core/src/platform/child_environment.rs @@ -222,6 +222,7 @@ mod tests { "PYTHONSTARTUP", "HF_TOKEN", "PUMAS_RELEASE_API_TOKEN", + "PUMAS_GITHUB_TOKEN", "AWS_SECRET_ACCESS_KEY", ] .map(|name| (OsString::from(name), OsString::from("value"))); diff --git a/scripts/release/packaged_onnx_support.py b/scripts/release/packaged_onnx_support.py index 8f0b0619..f30bcb2a 100644 --- a/scripts/release/packaged_onnx_support.py +++ b/scripts/release/packaged_onnx_support.py @@ -115,6 +115,12 @@ def child_environment(root): WINDIR=buffer.value, PATH=str(Path(buffer.value) / "System32"), ) + # The GitHub API token is the one deliberate pass-through: Pumas sends it + # only to the GitHub API, and it lifts the shared runner's anonymous + # rate limit on the ONNX Runtime release listing. + token = os.environ.get("PUMAS_GITHUB_TOKEN") + if token: + environment["PUMAS_GITHUB_TOKEN"] = token return environment diff --git a/scripts/release/test_inference_release.py b/scripts/release/test_inference_release.py index 5b0a7c4b..83056daa 100644 --- a/scripts/release/test_inference_release.py +++ b/scripts/release/test_inference_release.py @@ -26,6 +26,16 @@ class ReleaseTests(unittest.TestCase): + def test_child_environment_passes_only_the_github_api_token(self): + ambient = {"PUMAS_GITHUB_TOKEN": "ghs_fixture", "GITHUB_TOKEN": "x", "HF_TOKEN": "y"} + with patch.dict(os.environ, ambient): + environment = support.child_environment(tempfile.gettempdir()) + self.assertEqual(environment["PUMAS_GITHUB_TOKEN"], "ghs_fixture") + self.assertNotIn("GITHUB_TOKEN", environment) + self.assertNotIn("HF_TOKEN", environment) + with patch.dict(os.environ, {"PUMAS_GITHUB_TOKEN": ""}): + self.assertNotIn("PUMAS_GITHUB_TOKEN", support.child_environment(tempfile.gettempdir())) + def test_only_shared_libraries_count_as_a_mapped_onnx_runtime(self): # ONNX Runtime 1.31 maps its own cache database; it is not a library. for name, expected in [