From 66d6802977909b41cf0780342f197af706ea3db4 Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 14 Sep 2026 12:11:45 -0500 Subject: [PATCH 01/24] DEN-3829: prove installs survive total Zed control-plane outage --- .../workflows/github-control-plane-outage.yml | 202 ++++++++++++++++++ 1 file changed, 202 insertions(+) create mode 100644 .github/workflows/github-control-plane-outage.yml diff --git a/.github/workflows/github-control-plane-outage.yml b/.github/workflows/github-control-plane-outage.yml new file mode 100644 index 000000000..5c57a5d0c --- /dev/null +++ b/.github/workflows/github-control-plane-outage.yml @@ -0,0 +1,202 @@ +name: GitHub control-plane outage fallback + +on: + pull_request: + paths: + - 'src/source_fallback.rs' + - 'src/registry.rs' + - 'src/fetch.rs' + - 'src/store.rs' + - 'Cargo.toml' + - 'Cargo.lock' + - '.github/workflows/github-control-plane-outage.yml' + - 'docs/github-control-plane-outage.md' + workflow_dispatch: + +permissions: + contents: read + +defaults: + run: + shell: bash + +env: + # Neither endpoint belongs to GitHub and neither can serve package data. + # The .invalid TLD is guaranteed not to resolve; loopback port 1 fails fast. + DEAD_REGISTRY: https://zed-control-plane-intentionally-unavailable.invalid + DEAD_R2: http://127.0.0.1:1 + +jobs: + github-direct: + name: GitHub is sufficient when Zed infrastructure is gone + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - name: Check out exact candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Install pinned Rust toolchain + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: '1.98.1' + + - name: Build exact candidate with locked dependencies + run: | + set -Eeuo pipefail + cargo build --locked --release --bin zed + test "$(git rev-parse HEAD)" = "$GITHUB_SHA" + + - name: Run focused fallback security and archive tests + run: | + set -Eeuo pipefail + cargo test --locked source_fallback::tests:: -- --nocapture + + - name: Create five-language consumer + run: | + set -Eeuo pipefail + mkdir -p "$RUNNER_TEMP/github-only-consumer" + cat > "$RUNNER_TEMP/github-only-consumer/.zpkg.toml" <<'EOF' + [package] + org = "zed-pkg-test" + name = "github-only-outage-consumer" + version = "0.0.0" + description = "Proves zed-cli can install with every Zed-owned network service unavailable" + + [package.repository] + vcs = "git" + url = "https://github.com/zed-pkg-test/github-api-fallback-canary" + + [install] + dir = "zed_modules" + adapter = "none" + + [dependencies] + "zed-pkg-test/go-lib" = "=1.0.0" + "zed-pkg-test/python-lib" = "=1.0.0" + "zed-pkg-test/dart-lib" = "=1.0.0" + "zed-pkg-test/node-lib" = "=1.0.0" + "zed-pkg-test/gleam-lib" = "=1.0.0" + EOF + + - name: Install with registry down, R2 down, and no GitHub token + working-directory: ${{ runner.temp }}/github-only-consumer + env: + ZED_PKG_SOURCE_FALLBACK: 'true' + ZED_PKG_R2_PUBLIC_BASE: ${{ env.DEAD_R2 }} + run: | + set -Eeuo pipefail + ZED="$GITHUB_WORKSPACE/target/release/zed" + unset GITHUB_TOKEN GH_TOKEN ZED_PKG_GITHUB_TOKEN || true + "$ZED" --registry "$DEAD_REGISTRY" --home "$RUNNER_TEMP/zed-home-a" install \ + --install-mode copy --adapter none --allow-ecosystem-mismatch \ + 2>&1 | tee "$RUNNER_TEMP/github-only-a.log" + + test "$(grep -c 'using GitHub fallback' "$RUNNER_TEMP/github-only-a.log")" -ge 5 + test -s .zpkg.lock + for name in go-lib python-lib dart-lib node-lib gleam-lib; do + grep -R -q "name = \"$name\"" zed_modules + done + cp .zpkg.lock "$RUNNER_TEMP/github-only.lock" + + - name: Prove clean-store resolution is deterministic + working-directory: ${{ runner.temp }}/github-only-consumer + env: + ZED_PKG_SOURCE_FALLBACK: 'true' + ZED_PKG_R2_PUBLIC_BASE: ${{ env.DEAD_R2 }} + run: | + set -Eeuo pipefail + ZED="$GITHUB_WORKSPACE/target/release/zed" + unset GITHUB_TOKEN GH_TOKEN ZED_PKG_GITHUB_TOKEN || true + rm -rf zed_modules .zpkg.lock "$RUNNER_TEMP/zed-home-b" + "$ZED" --registry "$DEAD_REGISTRY" --home "$RUNNER_TEMP/zed-home-b" install \ + --install-mode copy --adapter none --allow-ecosystem-mismatch \ + 2>&1 | tee "$RUNNER_TEMP/github-only-b.log" + cmp "$RUNNER_TEMP/github-only.lock" .zpkg.lock + + - name: Prove frozen replay works from another empty store + working-directory: ${{ runner.temp }}/github-only-consumer + env: + ZED_PKG_SOURCE_FALLBACK: 'true' + ZED_PKG_R2_PUBLIC_BASE: ${{ env.DEAD_R2 }} + run: | + set -Eeuo pipefail + ZED="$GITHUB_WORKSPACE/target/release/zed" + unset GITHUB_TOKEN GH_TOKEN ZED_PKG_GITHUB_TOKEN || true + rm -rf zed_modules "$RUNNER_TEMP/zed-home-frozen" + "$ZED" --registry "$DEAD_REGISTRY" --home "$RUNNER_TEMP/zed-home-frozen" install --frozen \ + --install-mode copy --adapter none --allow-ecosystem-mismatch \ + 2>&1 | tee "$RUNNER_TEMP/github-only-frozen.log" + cmp "$RUNNER_TEMP/github-only.lock" .zpkg.lock + test "$(find zed_modules -name .zpkg.toml -type f | wc -l)" -ge 5 + + - name: Prove fallback is required rather than accidental cache reuse + working-directory: ${{ runner.temp }}/github-only-consumer + env: + ZED_PKG_SOURCE_FALLBACK: 'false' + ZED_PKG_R2_PUBLIC_BASE: ${{ env.DEAD_R2 }} + run: | + set -Eeuo pipefail + ZED="$GITHUB_WORKSPACE/target/release/zed" + unset GITHUB_TOKEN GH_TOKEN ZED_PKG_GITHUB_TOKEN || true + rm -rf zed_modules "$RUNNER_TEMP/zed-home-no-fallback" + set +e + "$ZED" --registry "$DEAD_REGISTRY" --home "$RUNNER_TEMP/zed-home-no-fallback" install --frozen \ + --install-mode copy --adapter none --allow-ecosystem-mismatch \ + >"$RUNNER_TEMP/no-fallback.log" 2>&1 + status=$? + set -e + cat "$RUNNER_TEMP/no-fallback.log" + test "$status" -ne 0 + + - name: Reject a GitHub tag whose manifest claims another package identity + run: | + set -Eeuo pipefail + mkdir -p "$RUNNER_TEMP/github-negative" + cat > "$RUNNER_TEMP/github-negative/.zpkg.toml" <<'EOF' + [package] + org = "zed-pkg-test" + name = "github-negative-consumer" + version = "0.0.0" + description = "Fail-closed GitHub identity canary" + + [package.repository] + vcs = "git" + url = "https://github.com/zed-pkg-test/github-api-fallback-canary" + + [install] + dir = "zed_modules" + adapter = "none" + + [dependencies] + "zed-pkg-test/awkward-lib" = "=0.1.0" + EOF + + ZED="$GITHUB_WORKSPACE/target/release/zed" + unset GITHUB_TOKEN GH_TOKEN ZED_PKG_GITHUB_TOKEN || true + set +e + ( + cd "$RUNNER_TEMP/github-negative" + ZED_PKG_SOURCE_FALLBACK=true ZED_PKG_R2_PUBLIC_BASE="$DEAD_R2" \ + "$ZED" --registry "$DEAD_REGISTRY" --home "$RUNNER_TEMP/zed-home-negative" install \ + --install-mode copy --adapter none --allow-ecosystem-mismatch + ) >"$RUNNER_TEMP/github-negative.log" 2>&1 + status=$? + set -e + cat "$RUNNER_TEMP/github-negative.log" + test "$status" -ne 0 + grep -E 'expected zed-pkg-test/awkward-lib@0\.1\.0|claims .*awkward-lib@0\.1\.0' "$RUNNER_TEMP/github-negative.log" + + - name: Prove the outage path does not require Zed or Cloudflare origins + run: | + set -Eeuo pipefail + log="$RUNNER_TEMP/github-only-a.log" + grep -q 'using GitHub fallback' "$log" + # A successful install while the only configured Zed-owned endpoints are + # deliberately unreachable proves the dependency graph is recoverable from + # GitHub. Keep this list explicit so future CDN/control-plane additions do + # not silently become mandatory. + ! grep -E 'success.*(zpkg\.net|cdn\.zpkg\.net|cloudflare)' "$log" From 3971481fa039740b4b3ee79cf9e0a802d8246796 Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 14 Sep 2026 12:14:18 -0500 Subject: [PATCH 02/24] DEN-3829: document GitHub-only outage recovery contract --- docs/github-control-plane-outage.md | 59 +++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 docs/github-control-plane-outage.md diff --git a/docs/github-control-plane-outage.md b/docs/github-control-plane-outage.md new file mode 100644 index 000000000..30d6dfca3 --- /dev/null +++ b/docs/github-control-plane-outage.md @@ -0,0 +1,59 @@ +# GitHub-only recovery when the Zed control plane is unavailable + +`zed` must continue resolving and installing already-published public packages when every Zed-operated network service is unavailable. `registry.zpkg.net`, `cdn.zpkg.net`, Cloudflare Workers, R2, and application servers are acceleration and control-plane services; they are not required availability dependencies for public package restore when GitHub contains enough provenance to reconstruct the package. + +## Recovery contract + +The normal path remains the configured registry. When an HTTP registry request fails and source fallback is enabled, `zed` may reconstruct package metadata from GitHub and fetch package bytes from GitHub-owned surfaces. GitHub is treated as an independent content/repository recovery plane, not as a new package-authority model. + +A public package is admissible through GitHub fallback only when the repository can be mapped unambiguously to the requested `org/name`, the committed `.zpkg.toml` self-claims that package identity, and the requested version is represented by an admissible version tag. Release assets, GHCR artifacts, and the GitHub tag archive are acceptable GitHub-hosted content sources. A tag archive is repacked using the same deterministic Zed package rules before its digest is recorded or compared. + +The fallback remains fail-closed. Repository-search ambiguity, a mismatched package identity, malformed or unsafe archives, missing required provenance, and digest/size mismatches are errors. Fallback never grants authority to an arbitrary repository merely because its name resembles the package name. + +## What must not be required + +The recovery path must succeed with all of the following unavailable at the same time: + +- the configured Zed HTTP registry; +- every `*.zpkg.net` endpoint; +- Cloudflare Workers, Pages, Tunnels, and other Zed-owned edge services; +- the Zed R2/CDN origin; and +- any Zed application or API server behind Cloudflare. + +The exact-head outage workflow configures a guaranteed-unresolvable registry plus a dead loopback R2 endpoint. It then removes `GITHUB_TOKEN`, `GH_TOKEN`, and `ZED_PKG_GITHUB_TOKEN` and restores public packages from GitHub. This proves the public recovery path does not rely on credentials or on a Zed-operated server. + +## GitHub as a recovery CDN + +For public packages, the GitHub recovery plane can use: + +1. `api.github.com` for repository identity, tags, releases, and tag-to-commit provenance; +2. `raw.githubusercontent.com` for the committed `.zpkg.toml` used for identity admission; +3. GitHub Releases and GitHub Packages/GHCR when a package publishes compatible artifacts there; and +4. GitHub/codeload tag archives as the final reconstructable source artifact. + +When a GitHub source succeeds, Zed-owned mirrors are not required for correctness. R2 remains an optional mirror for cases where GitHub is unavailable or incomplete; it must not be the sole recovery source for a package that can be reconstructed from GitHub. + +For private repositories, an explicit GitHub token may be used. Credentials are attached only to GitHub API requests that require them; redirects to codeload or unrelated mirror hosts must not receive the token. + +## Hermetic exceptions + +`file://` registries and loopback registries remain hermetic by default. Local tests, air-gapped mirrors, and deliberately isolated development environments must not leak package identities to GitHub. The explicit loopback fallback opt-in remains available for controlled canaries. + +Source fallback is a read/recovery capability. Operations that require Zed control-plane state or mutation—publishing, namespace claims, yanking, server-side search, and audit-log access—may still fail while the control plane is down. The outage guarantee is specifically resolution, download, deterministic reconstruction, install, and frozen restore of packages whose GitHub provenance is sufficient. + +## Acceptance gates + +The required outage canary must prove all of these conditions on the exact pull-request head: + +1. the configured registry is unreachable; +2. the configured R2/CDN endpoint is unreachable; +3. no GitHub token is available for the public-package test; +4. five package ecosystems resolve from GitHub source fallback; +5. the packages are actually materialized, not merely resolved; +6. a second empty store produces the identical lockfile; +7. a frozen restore succeeds from another empty store; +8. disabling source fallback makes the same outage fail instead of reusing an accidental cache; +9. a GitHub repository/tag with the wrong package identity is rejected; and +10. successful recovery has no dependency on a `zpkg.net` or Cloudflare response. + +These gates intentionally model a total Zed control-plane outage rather than a partial CDN degradation. \ No newline at end of file From 2540b3071883b4406b252c2ce4957f69f0f729c9 Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 14 Sep 2026 12:16:37 -0500 Subject: [PATCH 03/24] DEN-3829: stage GitHub-first fallback ordering patch --- .../workflows/patch-github-first-fallback.yml | 215 ++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100644 .github/workflows/patch-github-first-fallback.yml diff --git a/.github/workflows/patch-github-first-fallback.yml b/.github/workflows/patch-github-first-fallback.yml new file mode 100644 index 000000000..4740d855d --- /dev/null +++ b/.github/workflows/patch-github-first-fallback.yml @@ -0,0 +1,215 @@ +name: patch GitHub-first fallback ordering + +on: + push: + branches: + - DEN-3829-github-control-plane-fallback + paths: + - '.github/workflows/patch-github-first-fallback.yml' + workflow_dispatch: + +permissions: + contents: write + +jobs: + patch: + if: github.repository == 'zed-pkg/zed-cli' && github.ref_name == 'DEN-3829-github-control-plane-fallback' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Check out branch + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: DEN-3829-github-control-plane-fallback + fetch-depth: 1 + + - name: Patch fallback ordering and tests + run: | + set -Eeuo pipefail + python3 - <<'PY' + from pathlib import Path + + path = Path('src/source_fallback.rs') + text = path.read_text() + + def replace_once(old: str, new: str) -> None: + global text + count = text.count(old) + if count != 1: + raise SystemExit(f'expected exactly one occurrence, found {count}: {old[:120]!r}') + text = text.replace(old, new, 1) + + replace_once( + '//! Retry GitHub and public R2 when the configured HTTP registry is unreachable.\n//!\n//! Loopback `file://` and `http://127.0.0.1` registries stay hermetic: tests\n//! and air-gapped mirrors never leak to github.com. Production hosts such as\n//! `registry.zpkg.net` fall back to guessed public R2 keys and GitHub Release\n//! assets, then to the GitHub tag archive, repacked exactly as `zed publish`\n//! would pack that tag.\n', + '//! Retry GitHub first, then optional public R2, when an HTTP registry is unreachable.\n//!\n//! Loopback `file://` and `http://127.0.0.1` registries stay hermetic: tests\n//! and air-gapped mirrors never leak to github.com. Production hosts such as\n//! `registry.zpkg.net` recover directly from GitHub Releases, GHCR, or a GitHub\n//! tag archive repacked exactly as `zed publish` would pack that tag. Public R2\n//! remains a later optional mirror, never a required recovery dependency.\n' + ) + + anchor = '''fn max_artifact_bytes() -> u64 { + std::env::var("ZED_PKG_MAX_ARTIFACT_BYTES") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(DEFAULT_MAX_ARTIFACT_BYTES) + } + ''' + insertion = anchor + ''' + /// During source fallback, prefer infrastructure owned by GitHub before + /// any Zed-operated R2/Cloudflare mirror. The normal registry path is + /// still attempted before this function is involved. + fn source_fallback_priority(kind: ArtifactSourceKind) -> u8 { + match kind { + ArtifactSourceKind::GithubRelease => 0, + ArtifactSourceKind::GithubPackages => 1, + ArtifactSourceKind::GithubArchive => 2, + ArtifactSourceKind::R2 => 3, + ArtifactSourceKind::Registry => 4, + } + } + + fn github_first_locators(mut locators: Vec) -> Vec { + locators.sort_by_key(|locator| source_fallback_priority(locator.kind)); + locators + } + ''' + replace_once(anchor, insertion) + + replace_once( + ''' let locators = artifact_locators(&query); + let download_url = locators + .iter() + .find(|locator| locator.kind != ArtifactSourceKind::GithubArchive) + .or_else(|| locators.last()) + .map(|locator| locator.url.clone()) + .context("no fallback locator")?; + ''', + ''' let locators = github_first_locators(artifact_locators(&query)); + let download_url = locators + .first() + .map(|locator| locator.url.clone()) + .context("no fallback locator")?; + ''' + ) + + replace_once( + ''' let locators = artifact_locators(&ArtifactQuery { + org, + name, + version, + vcs_tag: tag, + sha256: Some(sha256.as_str()), + format: ArtifactFormat::TarGz, + repo_url: Some(repo_url.as_str()), + artifacts: Some(&ArtifactsSection::EMPTY), + registry_base: None, + r2_public_base: self.config.r2_public_base.as_deref(), + r2_public_key: self.config.r2_public_key.as_deref(), + }); + ''', + ''' let locators = github_first_locators(artifact_locators(&ArtifactQuery { + org, + name, + version, + vcs_tag: tag, + sha256: Some(sha256.as_str()), + format: ArtifactFormat::TarGz, + repo_url: Some(repo_url.as_str()), + artifacts: Some(&ArtifactsSection::EMPTY), + registry_base: None, + r2_public_base: self.config.r2_public_base.as_deref(), + r2_public_key: self.config.r2_public_key.as_deref(), + })); + ''' + ) + + replace_once( + ''' let locators = artifact_locators(&ArtifactQuery { + org: &version.org, + name: &version.name, + version: &version.version, + vcs_tag: &version.vcs_tag, + sha256: packed_digest.then_some(version.sha256.as_str()), + format: version.format, + repo_url: Some(repo_url.as_str()), + artifacts: Some(&ArtifactsSection::EMPTY), + registry_base: None, + r2_public_base: self.config.r2_public_base.as_deref(), + r2_public_key: self.config.r2_public_key.as_deref(), + }); + ''', + ''' let locators = github_first_locators(artifact_locators(&ArtifactQuery { + org: &version.org, + name: &version.name, + version: &version.version, + vcs_tag: &version.vcs_tag, + sha256: packed_digest.then_some(version.sha256.as_str()), + format: version.format, + repo_url: Some(repo_url.as_str()), + artifacts: Some(&ArtifactsSection::EMPTY), + registry_base: None, + r2_public_base: self.config.r2_public_base.as_deref(), + r2_public_key: self.config.r2_public_key.as_deref(), + })); + ''' + ) + + replace_once( + ''' // The GitHub tag archive stays in the chain even when a digest is + // pinned. It is the last locator and is repacked deterministically, so + // a digest derived from it is reproducible, and every caller verifies + // the bytes against the pin. Skipping it broke installs whose digest + // had been derived from that same archive in another registry instance. + ''', + ''' // The GitHub tag archive stays in the chain even when a digest is + // pinned. GitHub sources are tried before Zed-owned R2 so a total Zed + // control-plane outage goes directly to GitHub. The archive is repacked + // deterministically and every caller verifies the bytes against the pin. + ''' + ) + + text = text.replace('using GitHub/R2 fallback', 'using GitHub-first fallback') + text = text.replace('the GitHub/R2 fallback also failed', 'the GitHub-first/R2 fallback also failed') + + test_anchor = ''' #[test] + fn loopback_registries_are_hermetic() { + ''' + priority_test = ''' #[test] + fn github_sources_precede_zed_owned_r2_during_fallback() { + assert!( + source_fallback_priority(ArtifactSourceKind::GithubRelease) + < source_fallback_priority(ArtifactSourceKind::R2) + ); + assert!( + source_fallback_priority(ArtifactSourceKind::GithubPackages) + < source_fallback_priority(ArtifactSourceKind::R2) + ); + assert!( + source_fallback_priority(ArtifactSourceKind::GithubArchive) + < source_fallback_priority(ArtifactSourceKind::R2) + ); + assert!( + source_fallback_priority(ArtifactSourceKind::GithubRelease) + < source_fallback_priority(ArtifactSourceKind::GithubPackages) + ); + assert!( + source_fallback_priority(ArtifactSourceKind::GithubPackages) + < source_fallback_priority(ArtifactSourceKind::GithubArchive) + ); + } + + ''' + test_anchor + replace_once(test_anchor, priority_test) + + path.write_text(text) + PY + + cargo fmt --all -- --check || cargo fmt --all + cargo test --locked source_fallback::tests::github_sources_precede_zed_owned_r2_during_fallback + + - name: Commit runtime change + run: | + set -Eeuo pipefail + git config user.name 'ORESoftware CI' + git config user.email 'ci@oresoftware.com' + git add src/source_fallback.rs + git diff --cached --check + git commit -m 'DEN-3829: prefer GitHub before Zed-owned mirrors during fallback' + git push origin HEAD:DEN-3829-github-control-plane-fallback From 2099b1cebd3d8f4f8a70d8d9e597c7b00a7cdb60 Mon Sep 17 00:00:00 2001 From: ORESoftware CI Date: Mon, 14 Sep 2026 17:18:35 +0000 Subject: [PATCH 04/24] DEN-3829: prefer GitHub before Zed-owned mirrors during fallback --- src/source_fallback.rs | 77 +++++++++++++++++++++++++++++++----------- 1 file changed, 58 insertions(+), 19 deletions(-) diff --git a/src/source_fallback.rs b/src/source_fallback.rs index e2a5b2059..aff33b729 100644 --- a/src/source_fallback.rs +++ b/src/source_fallback.rs @@ -1,10 +1,10 @@ -//! Retry GitHub and public R2 when the configured HTTP registry is unreachable. +//! Retry GitHub first, then optional public R2, when an HTTP registry is unreachable. //! //! Loopback `file://` and `http://127.0.0.1` registries stay hermetic: tests //! and air-gapped mirrors never leak to github.com. Production hosts such as -//! `registry.zpkg.net` fall back to guessed public R2 keys and GitHub Release -//! assets, then to the GitHub tag archive, repacked exactly as `zed publish` -//! would pack that tag. +//! `registry.zpkg.net` recover directly from GitHub Releases, GHCR, or a GitHub +//! tag archive repacked exactly as `zed publish` would pack that tag. Public R2 +//! remains a later optional mirror, never a required recovery dependency. use std::cell::RefCell; use std::collections::HashMap; @@ -73,6 +73,24 @@ fn max_artifact_bytes() -> u64 { .unwrap_or(DEFAULT_MAX_ARTIFACT_BYTES) } +/// During source fallback, prefer infrastructure owned by GitHub before +/// any Zed-operated R2/Cloudflare mirror. The normal registry path is +/// still attempted before this function is involved. +fn source_fallback_priority(kind: ArtifactSourceKind) -> u8 { + match kind { + ArtifactSourceKind::GithubRelease => 0, + ArtifactSourceKind::GithubPackages => 1, + ArtifactSourceKind::GithubArchive => 2, + ArtifactSourceKind::R2 => 3, + ArtifactSourceKind::Registry => 4, + } +} + +fn github_first_locators(mut locators: Vec) -> Vec { + locators.sort_by_key(|locator| source_fallback_priority(locator.kind)); + locators +} + #[derive(Debug, Clone)] pub struct SourceFallbackConfig { pub enabled: bool, @@ -461,11 +479,9 @@ impl FallbackRegistry { r2_public_base: self.config.r2_public_base.as_deref(), r2_public_key: self.config.r2_public_key.as_deref(), }; - let locators = artifact_locators(&query); + let locators = github_first_locators(artifact_locators(&query)); let download_url = locators - .iter() - .find(|locator| locator.kind != ArtifactSourceKind::GithubArchive) - .or_else(|| locators.last()) + .first() .map(|locator| locator.url.clone()) .context("no fallback locator")?; let mut metadata = VersionMetadata { @@ -613,7 +629,7 @@ impl FallbackRegistry { .unwrap_or(digest.as_str()) .to_string(); let repo_url = identity.web_url(); - let locators = artifact_locators(&ArtifactQuery { + let locators = github_first_locators(artifact_locators(&ArtifactQuery { org, name, version, @@ -625,7 +641,7 @@ impl FallbackRegistry { registry_base: None, r2_public_base: self.config.r2_public_base.as_deref(), r2_public_key: self.config.r2_public_key.as_deref(), - }); + })); Some(VersionMetadata { org: org.to_string(), name: name.to_string(), @@ -655,7 +671,7 @@ impl FallbackRegistry { // carry suffixes such as `.rs`. let identity = self.resolve_github_identity(&version.org, &version.name)?; let repo_url = identity.web_url(); - let locators = artifact_locators(&ArtifactQuery { + let locators = github_first_locators(artifact_locators(&ArtifactQuery { org: &version.org, name: &version.name, version: &version.version, @@ -667,12 +683,11 @@ impl FallbackRegistry { registry_base: None, r2_public_base: self.config.r2_public_base.as_deref(), r2_public_key: self.config.r2_public_key.as_deref(), - }); + })); // The GitHub tag archive stays in the chain even when a digest is - // pinned. It is the last locator and is repacked deterministically, so - // a digest derived from it is reproducible, and every caller verifies - // the bytes against the pin. Skipping it broke installs whose digest - // had been derived from that same archive in another registry instance. + // pinned. GitHub sources are tried before Zed-owned R2 so a total Zed + // control-plane outage goes directly to GitHub. The archive is repacked + // deterministically and every caller verifies the bytes against the pin. for locator in locators { if locator.kind == ArtifactSourceKind::Registry { continue; @@ -730,12 +745,12 @@ impl Registry for FallbackRegistry { Err(error) => match self.github_get_version(org, name, version) { Ok(metadata) => { eprintln!( - "warning: registry unavailable for {org}/{name}@{version}; using GitHub/R2 fallback ({error})" + "warning: registry unavailable for {org}/{name}@{version}; using GitHub-first fallback ({error})" ); Ok(metadata) } Err(fallback) => Err(error.context(format!( - "registry unavailable and the GitHub/R2 fallback also failed: {fallback:#}" + "registry unavailable and the GitHub-first/R2 fallback also failed: {fallback:#}" ))), }, } @@ -747,7 +762,7 @@ impl Registry for FallbackRegistry { Err(error) => match self.download_locators(version, dest) { Ok(()) => Ok(()), Err(fallback) => Err(error.context(format!( - "registry download failed and the GitHub/R2 fallback also failed: {fallback:#}" + "registry download failed and the GitHub-first/R2 fallback also failed: {fallback:#}" ))), }, } @@ -1248,6 +1263,30 @@ mod tests { use flate2::write::GzEncoder; use zed_interfaces::source::{r2_object_keys, resolve_r2_public_base}; + #[test] + fn github_sources_precede_zed_owned_r2_during_fallback() { + assert!( + source_fallback_priority(ArtifactSourceKind::GithubRelease) + < source_fallback_priority(ArtifactSourceKind::R2) + ); + assert!( + source_fallback_priority(ArtifactSourceKind::GithubPackages) + < source_fallback_priority(ArtifactSourceKind::R2) + ); + assert!( + source_fallback_priority(ArtifactSourceKind::GithubArchive) + < source_fallback_priority(ArtifactSourceKind::R2) + ); + assert!( + source_fallback_priority(ArtifactSourceKind::GithubRelease) + < source_fallback_priority(ArtifactSourceKind::GithubPackages) + ); + assert!( + source_fallback_priority(ArtifactSourceKind::GithubPackages) + < source_fallback_priority(ArtifactSourceKind::GithubArchive) + ); + } + #[test] fn loopback_registries_are_hermetic() { assert!(is_loopback_registry("http://127.0.0.1:18080")); From 777be153ffbb34f4e2516e80fcf18075704b070b Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 14 Sep 2026 12:19:06 -0500 Subject: [PATCH 05/24] DEN-3829: remove temporary fallback patch workflow --- .../workflows/patch-github-first-fallback.yml | 215 ------------------ 1 file changed, 215 deletions(-) delete mode 100644 .github/workflows/patch-github-first-fallback.yml diff --git a/.github/workflows/patch-github-first-fallback.yml b/.github/workflows/patch-github-first-fallback.yml deleted file mode 100644 index 4740d855d..000000000 --- a/.github/workflows/patch-github-first-fallback.yml +++ /dev/null @@ -1,215 +0,0 @@ -name: patch GitHub-first fallback ordering - -on: - push: - branches: - - DEN-3829-github-control-plane-fallback - paths: - - '.github/workflows/patch-github-first-fallback.yml' - workflow_dispatch: - -permissions: - contents: write - -jobs: - patch: - if: github.repository == 'zed-pkg/zed-cli' && github.ref_name == 'DEN-3829-github-control-plane-fallback' - runs-on: ubuntu-24.04 - timeout-minutes: 10 - steps: - - name: Check out branch - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: DEN-3829-github-control-plane-fallback - fetch-depth: 1 - - - name: Patch fallback ordering and tests - run: | - set -Eeuo pipefail - python3 - <<'PY' - from pathlib import Path - - path = Path('src/source_fallback.rs') - text = path.read_text() - - def replace_once(old: str, new: str) -> None: - global text - count = text.count(old) - if count != 1: - raise SystemExit(f'expected exactly one occurrence, found {count}: {old[:120]!r}') - text = text.replace(old, new, 1) - - replace_once( - '//! Retry GitHub and public R2 when the configured HTTP registry is unreachable.\n//!\n//! Loopback `file://` and `http://127.0.0.1` registries stay hermetic: tests\n//! and air-gapped mirrors never leak to github.com. Production hosts such as\n//! `registry.zpkg.net` fall back to guessed public R2 keys and GitHub Release\n//! assets, then to the GitHub tag archive, repacked exactly as `zed publish`\n//! would pack that tag.\n', - '//! Retry GitHub first, then optional public R2, when an HTTP registry is unreachable.\n//!\n//! Loopback `file://` and `http://127.0.0.1` registries stay hermetic: tests\n//! and air-gapped mirrors never leak to github.com. Production hosts such as\n//! `registry.zpkg.net` recover directly from GitHub Releases, GHCR, or a GitHub\n//! tag archive repacked exactly as `zed publish` would pack that tag. Public R2\n//! remains a later optional mirror, never a required recovery dependency.\n' - ) - - anchor = '''fn max_artifact_bytes() -> u64 { - std::env::var("ZED_PKG_MAX_ARTIFACT_BYTES") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(DEFAULT_MAX_ARTIFACT_BYTES) - } - ''' - insertion = anchor + ''' - /// During source fallback, prefer infrastructure owned by GitHub before - /// any Zed-operated R2/Cloudflare mirror. The normal registry path is - /// still attempted before this function is involved. - fn source_fallback_priority(kind: ArtifactSourceKind) -> u8 { - match kind { - ArtifactSourceKind::GithubRelease => 0, - ArtifactSourceKind::GithubPackages => 1, - ArtifactSourceKind::GithubArchive => 2, - ArtifactSourceKind::R2 => 3, - ArtifactSourceKind::Registry => 4, - } - } - - fn github_first_locators(mut locators: Vec) -> Vec { - locators.sort_by_key(|locator| source_fallback_priority(locator.kind)); - locators - } - ''' - replace_once(anchor, insertion) - - replace_once( - ''' let locators = artifact_locators(&query); - let download_url = locators - .iter() - .find(|locator| locator.kind != ArtifactSourceKind::GithubArchive) - .or_else(|| locators.last()) - .map(|locator| locator.url.clone()) - .context("no fallback locator")?; - ''', - ''' let locators = github_first_locators(artifact_locators(&query)); - let download_url = locators - .first() - .map(|locator| locator.url.clone()) - .context("no fallback locator")?; - ''' - ) - - replace_once( - ''' let locators = artifact_locators(&ArtifactQuery { - org, - name, - version, - vcs_tag: tag, - sha256: Some(sha256.as_str()), - format: ArtifactFormat::TarGz, - repo_url: Some(repo_url.as_str()), - artifacts: Some(&ArtifactsSection::EMPTY), - registry_base: None, - r2_public_base: self.config.r2_public_base.as_deref(), - r2_public_key: self.config.r2_public_key.as_deref(), - }); - ''', - ''' let locators = github_first_locators(artifact_locators(&ArtifactQuery { - org, - name, - version, - vcs_tag: tag, - sha256: Some(sha256.as_str()), - format: ArtifactFormat::TarGz, - repo_url: Some(repo_url.as_str()), - artifacts: Some(&ArtifactsSection::EMPTY), - registry_base: None, - r2_public_base: self.config.r2_public_base.as_deref(), - r2_public_key: self.config.r2_public_key.as_deref(), - })); - ''' - ) - - replace_once( - ''' let locators = artifact_locators(&ArtifactQuery { - org: &version.org, - name: &version.name, - version: &version.version, - vcs_tag: &version.vcs_tag, - sha256: packed_digest.then_some(version.sha256.as_str()), - format: version.format, - repo_url: Some(repo_url.as_str()), - artifacts: Some(&ArtifactsSection::EMPTY), - registry_base: None, - r2_public_base: self.config.r2_public_base.as_deref(), - r2_public_key: self.config.r2_public_key.as_deref(), - }); - ''', - ''' let locators = github_first_locators(artifact_locators(&ArtifactQuery { - org: &version.org, - name: &version.name, - version: &version.version, - vcs_tag: &version.vcs_tag, - sha256: packed_digest.then_some(version.sha256.as_str()), - format: version.format, - repo_url: Some(repo_url.as_str()), - artifacts: Some(&ArtifactsSection::EMPTY), - registry_base: None, - r2_public_base: self.config.r2_public_base.as_deref(), - r2_public_key: self.config.r2_public_key.as_deref(), - })); - ''' - ) - - replace_once( - ''' // The GitHub tag archive stays in the chain even when a digest is - // pinned. It is the last locator and is repacked deterministically, so - // a digest derived from it is reproducible, and every caller verifies - // the bytes against the pin. Skipping it broke installs whose digest - // had been derived from that same archive in another registry instance. - ''', - ''' // The GitHub tag archive stays in the chain even when a digest is - // pinned. GitHub sources are tried before Zed-owned R2 so a total Zed - // control-plane outage goes directly to GitHub. The archive is repacked - // deterministically and every caller verifies the bytes against the pin. - ''' - ) - - text = text.replace('using GitHub/R2 fallback', 'using GitHub-first fallback') - text = text.replace('the GitHub/R2 fallback also failed', 'the GitHub-first/R2 fallback also failed') - - test_anchor = ''' #[test] - fn loopback_registries_are_hermetic() { - ''' - priority_test = ''' #[test] - fn github_sources_precede_zed_owned_r2_during_fallback() { - assert!( - source_fallback_priority(ArtifactSourceKind::GithubRelease) - < source_fallback_priority(ArtifactSourceKind::R2) - ); - assert!( - source_fallback_priority(ArtifactSourceKind::GithubPackages) - < source_fallback_priority(ArtifactSourceKind::R2) - ); - assert!( - source_fallback_priority(ArtifactSourceKind::GithubArchive) - < source_fallback_priority(ArtifactSourceKind::R2) - ); - assert!( - source_fallback_priority(ArtifactSourceKind::GithubRelease) - < source_fallback_priority(ArtifactSourceKind::GithubPackages) - ); - assert!( - source_fallback_priority(ArtifactSourceKind::GithubPackages) - < source_fallback_priority(ArtifactSourceKind::GithubArchive) - ); - } - - ''' + test_anchor - replace_once(test_anchor, priority_test) - - path.write_text(text) - PY - - cargo fmt --all -- --check || cargo fmt --all - cargo test --locked source_fallback::tests::github_sources_precede_zed_owned_r2_during_fallback - - - name: Commit runtime change - run: | - set -Eeuo pipefail - git config user.name 'ORESoftware CI' - git config user.email 'ci@oresoftware.com' - git add src/source_fallback.rs - git diff --cached --check - git commit -m 'DEN-3829: prefer GitHub before Zed-owned mirrors during fallback' - git push origin HEAD:DEN-3829-github-control-plane-fallback From 4d00aef16f9367daf1dda21b3614715e7f71b2e8 Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 14 Sep 2026 12:19:45 -0500 Subject: [PATCH 06/24] DEN-3829: prove GitHub fallback never touches R2 first --- .../workflows/github-control-plane-outage.yml | 40 ++++++++++++++----- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/.github/workflows/github-control-plane-outage.yml b/.github/workflows/github-control-plane-outage.yml index 5c57a5d0c..533e1665f 100644 --- a/.github/workflows/github-control-plane-outage.yml +++ b/.github/workflows/github-control-plane-outage.yml @@ -21,10 +21,11 @@ defaults: shell: bash env: - # Neither endpoint belongs to GitHub and neither can serve package data. - # The .invalid TLD is guaranteed not to resolve; loopback port 1 fails fast. + # The registry cannot resolve; the R2 endpoint is a local sentinel that logs + # any request. A successful GitHub-first restore must leave that log with no + # GETs, proving Zed/Cloudflare infrastructure is not consulted first. DEAD_REGISTRY: https://zed-control-plane-intentionally-unavailable.invalid - DEAD_R2: http://127.0.0.1:1 + DEAD_R2: http://127.0.0.1:18081 jobs: github-direct: @@ -55,6 +56,19 @@ jobs: set -Eeuo pipefail cargo test --locked source_fallback::tests:: -- --nocapture + - name: Start fail-if-used R2 sentinel + run: | + set -Eeuo pipefail + mkdir -p "$RUNNER_TEMP/dead-r2-root" + : > "$RUNNER_TEMP/r2-sentinel.log" + python3 -m http.server 18081 \ + --bind 127.0.0.1 \ + --directory "$RUNNER_TEMP/dead-r2-root" \ + >"$RUNNER_TEMP/r2-sentinel.log" 2>&1 & + echo "$!" > "$RUNNER_TEMP/r2-sentinel.pid" + sleep 1 + kill -0 "$(cat "$RUNNER_TEMP/r2-sentinel.pid")" + - name: Create five-language consumer run: | set -Eeuo pipefail @@ -82,7 +96,7 @@ jobs: "zed-pkg-test/gleam-lib" = "=1.0.0" EOF - - name: Install with registry down, R2 down, and no GitHub token + - name: Install with registry down, R2 forbidden, and no GitHub token working-directory: ${{ runner.temp }}/github-only-consumer env: ZED_PKG_SOURCE_FALLBACK: 'true' @@ -95,7 +109,7 @@ jobs: --install-mode copy --adapter none --allow-ecosystem-mismatch \ 2>&1 | tee "$RUNNER_TEMP/github-only-a.log" - test "$(grep -c 'using GitHub fallback' "$RUNNER_TEMP/github-only-a.log")" -ge 5 + test "$(grep -E -c 'using GitHub(-first)? fallback' "$RUNNER_TEMP/github-only-a.log")" -ge 5 test -s .zpkg.lock for name in go-lib python-lib dart-lib node-lib gleam-lib; do grep -R -q "name = \"$name\"" zed_modules @@ -190,13 +204,17 @@ jobs: test "$status" -ne 0 grep -E 'expected zed-pkg-test/awkward-lib@0\.1\.0|claims .*awkward-lib@0\.1\.0' "$RUNNER_TEMP/github-negative.log" - - name: Prove the outage path does not require Zed or Cloudflare origins + - name: Prove GitHub recovery did not touch R2 or Cloudflare + if: always() run: | set -Eeuo pipefail log="$RUNNER_TEMP/github-only-a.log" - grep -q 'using GitHub fallback' "$log" - # A successful install while the only configured Zed-owned endpoints are - # deliberately unreachable proves the dependency graph is recoverable from - # GitHub. Keep this list explicit so future CDN/control-plane additions do - # not silently become mandatory. + test -s "$log" + grep -E -q 'using GitHub(-first)? fallback' "$log" + if grep -q 'GET /' "$RUNNER_TEMP/r2-sentinel.log"; then + echo 'GitHub-first recovery unexpectedly contacted the configured R2/CDN endpoint:' >&2 + cat "$RUNNER_TEMP/r2-sentinel.log" >&2 + exit 1 + fi ! grep -E 'success.*(zpkg\.net|cdn\.zpkg\.net|cloudflare)' "$log" + kill "$(cat "$RUNNER_TEMP/r2-sentinel.pid")" || true From 96052513e5c0e52d100b95963541bdde0075ac98 Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 14 Sep 2026 12:29:55 -0500 Subject: [PATCH 07/24] DEN-3829: stage GitHub smart-HTTP recovery patch --- .github/workflows/patch-github-smart-http.yml | 551 ++++++++++++++++++ 1 file changed, 551 insertions(+) create mode 100644 .github/workflows/patch-github-smart-http.yml diff --git a/.github/workflows/patch-github-smart-http.yml b/.github/workflows/patch-github-smart-http.yml new file mode 100644 index 000000000..da25fd3ed --- /dev/null +++ b/.github/workflows/patch-github-smart-http.yml @@ -0,0 +1,551 @@ +name: patch GitHub smart HTTP recovery + +on: + push: + branches: + - DEN-3829-github-control-plane-fallback + paths: + - '.github/workflows/patch-github-smart-http.yml' + workflow_dispatch: + +permissions: + contents: write + +jobs: + patch: + if: github.repository == 'zed-pkg/zed-cli' && github.ref_name == 'DEN-3829-github-control-plane-fallback' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - name: Check out branch + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: DEN-3829-github-control-plane-fallback + fetch-depth: 1 + + - name: Add REST-independent GitHub ref discovery + run: | + set -Eeuo pipefail + python3 - <<'PY' + from pathlib import Path + + path = Path('src/source_fallback.rs') + text = path.read_text() + + def replace_once(old: str, new: str) -> None: + global text + count = text.count(old) + if count != 1: + raise SystemExit(f'expected exactly one occurrence, found {count}: {old[:160]!r}') + text = text.replace(old, new, 1) + + replace_once( + 'use std::sync::Mutex;\n', + 'use std::sync::Mutex;\nuse std::sync::atomic::{AtomicBool, Ordering};\n' + ) + + replace_once( + '''pub struct FallbackRegistry { + inner: HttpRegistry, + config: SourceFallbackConfig, + client: reqwest::blocking::Client, + cache: Mutex>, + github_identities: Mutex>, + } + ''', + '''pub struct FallbackRegistry { + inner: HttpRegistry, + config: SourceFallbackConfig, + client: reqwest::blocking::Client, + cache: Mutex>, + github_identities: Mutex>, + github_repositories: Mutex>, + github_tag_cache: Mutex>>, + github_smart_ref_cache: Mutex>, + github_api_degraded: AtomicBool, + } + ''' + ) + + replace_once( + ''' cache: Mutex::new(HashMap::new()), + github_identities: Mutex::new(HashMap::new()), + }), + ''', + ''' cache: Mutex::new(HashMap::new()), + github_identities: Mutex::new(HashMap::new()), + github_repositories: Mutex::new(HashMap::new()), + github_tag_cache: Mutex::new(HashMap::new()), + github_smart_ref_cache: Mutex::new(HashMap::new()), + github_api_degraded: AtomicBool::new(false), + }), + ''' + ) + + old_repo = ''' fn github_repo(&self, identity: &GithubIdentity) -> Result { + let response = self + .github_headers(self.client.get(github_api_repo_url(identity))) + .send() + .with_context(|| format!("fetch GitHub repository {}", identity.web_url()))?; + if !response.status().is_success() { + bail!( + "GitHub repository {} returned {}", + identity.web_url(), + response.status() + ); + } + response + .json() + .with_context(|| format!("decode GitHub repository {}", identity.web_url())) + } + ''' + new_repo = ''' fn github_repo(&self, identity: &GithubIdentity) -> Result { + let key = github_identity_key(identity); + if let Some(repo) = self + .github_repositories + .lock() + .ok() + .and_then(|guard| guard.get(&key).cloned()) + { + return Ok(repo); + } + + let api_result = if self.github_api_degraded.load(Ordering::Relaxed) { + None + } else { + Some( + self.github_headers(self.client.get(github_api_repo_url(identity))) + .send(), + ) + }; + + let repo = match api_result { + Some(Ok(response)) if response.status().is_success() => response + .json() + .with_context(|| { + format!("decode GitHub repository {}", identity.web_url()) + })?, + Some(Ok(response)) if github_api_status_is_degradable(response.status()) => { + self.github_api_degraded.store(true, Ordering::Relaxed); + self.github_repo_via_smart_http(identity).with_context(|| { + format!( + "GitHub REST repository lookup for {} returned {}; smart-HTTP fallback also failed", + identity.web_url(), + response.status() + ) + })? + } + Some(Ok(response)) => bail!( + "GitHub repository {} returned {}", + identity.web_url(), + response.status() + ), + Some(Err(error)) => { + self.github_api_degraded.store(true, Ordering::Relaxed); + self.github_repo_via_smart_http(identity).with_context(|| { + format!( + "GitHub REST repository lookup for {} failed ({error}); smart-HTTP fallback also failed", + identity.web_url() + ) + })? + } + None => self.github_repo_via_smart_http(identity)?, + }; + + if let Ok(mut guard) = self.github_repositories.lock() { + guard.insert(key, repo.clone()); + } + Ok(repo) + } + + fn github_repo_via_smart_http(&self, identity: &GithubIdentity) -> Result { + let refs = self.github_smart_refs(identity)?; + let default_branch = refs.default_branch.clone().with_context(|| { + format!( + "GitHub smart-HTTP advertisement for {} did not report HEAD's branch", + identity.web_url() + ) + })?; + Ok(GithubRepo { + default_branch, + html_url: identity.web_url(), + }) + } + ''' + replace_once(old_repo, new_repo) + + old_tags = ''' fn github_tags(&self, identity: &GithubIdentity) -> Result> { + let response = self + .github_headers(self.client.get(github_api_tags_url(identity))) + .send() + .with_context(|| format!("list tags for {}", identity.web_url()))?; + if !response.status().is_success() { + let hint = if self.config.github_token.is_none() + && response.status() == reqwest::StatusCode::NOT_FOUND + { + " (repository may be private; private repositories need ZED_PKG_GITHUB_TOKEN, GITHUB_TOKEN, or GH_TOKEN)" + } else { + "" + }; + bail!( + "GitHub tags for {} returned {}{hint}", + identity.web_url(), + response.status() + ); + } + Ok(response.json().unwrap_or_default()) + } + ''' + new_tags = ''' fn github_tags(&self, identity: &GithubIdentity) -> Result> { + let key = github_identity_key(identity); + if let Some(tags) = self + .github_tag_cache + .lock() + .ok() + .and_then(|guard| guard.get(&key).cloned()) + { + return Ok(tags); + } + + let api_result = if self.github_api_degraded.load(Ordering::Relaxed) { + None + } else { + Some( + self.github_headers(self.client.get(github_api_tags_url(identity))) + .send(), + ) + }; + + let tags = match api_result { + Some(Ok(response)) if response.status().is_success() => { + response.json().unwrap_or_default() + } + Some(Ok(response)) if github_api_status_is_degradable(response.status()) => { + self.github_api_degraded.store(true, Ordering::Relaxed); + self.github_tags_via_smart_http(identity).with_context(|| { + format!( + "GitHub REST tags for {} returned {}; smart-HTTP fallback also failed", + identity.web_url(), + response.status() + ) + })? + } + Some(Ok(response)) => { + let hint = if self.config.github_token.is_none() + && response.status() == reqwest::StatusCode::NOT_FOUND + { + " (repository may be private; private repositories need ZED_PKG_GITHUB_TOKEN, GITHUB_TOKEN, or GH_TOKEN)" + } else { + "" + }; + bail!( + "GitHub tags for {} returned {}{hint}", + identity.web_url(), + response.status() + ); + } + Some(Err(error)) => { + self.github_api_degraded.store(true, Ordering::Relaxed); + self.github_tags_via_smart_http(identity).with_context(|| { + format!( + "GitHub REST tags for {} failed ({error}); smart-HTTP fallback also failed", + identity.web_url() + ) + })? + } + None => self.github_tags_via_smart_http(identity)?, + }; + + if let Ok(mut guard) = self.github_tag_cache.lock() { + guard.insert(key, tags.clone()); + } + Ok(tags) + } + + fn github_tags_via_smart_http(&self, identity: &GithubIdentity) -> Result> { + let refs = self.github_smart_refs(identity)?; + let mut names: Vec = refs.tag_commits.keys().cloned().collect(); + names.sort(); + Ok(names.into_iter().map(|name| GithubTag { name }).collect()) + } + + fn github_smart_refs(&self, identity: &GithubIdentity) -> Result { + let key = github_identity_key(identity); + if let Some(refs) = self + .github_smart_ref_cache + .lock() + .ok() + .and_then(|guard| guard.get(&key).cloned()) + { + return Ok(refs); + } + + let url = format!( + "https://github.com/{}/{}.git/info/refs?service=git-upload-pack", + identity.owner, identity.repo + ); + let mut request = self + .client + .get(&url) + .header("Accept", "application/x-git-upload-pack-advertisement"); + if let Some(token) = self.config.github_token.as_deref() { + request = request.basic_auth("x-access-token", Some(token)); + } + let response = request + .send() + .with_context(|| format!("fetch GitHub smart refs from {url}"))?; + if !response.status().is_success() { + bail!("GitHub smart refs for {} returned {}", identity.web_url(), response.status()); + } + let bytes = response.bytes().context("read GitHub smart refs")?; + let refs = parse_git_smart_refs(bytes.as_ref())?; + if let Ok(mut guard) = self.github_smart_ref_cache.lock() { + guard.insert(key, refs.clone()); + } + Ok(refs) + } + ''' + replace_once(old_tags, new_tags) + + old_fill = ''' metadata.vcs_commit = self + .github_headers(self.client.get(url)) + .send() + .ok() + .filter(|response| response.status().is_success()) + .and_then(|response| response.json::().ok()) + .and_then(|commit| commit_sha(&commit.sha)); + ''' + new_fill = ''' if !self.github_api_degraded.load(Ordering::Relaxed) { + match self.github_headers(self.client.get(url)).send() { + Ok(response) if response.status().is_success() => { + metadata.vcs_commit = response + .json::() + .ok() + .and_then(|commit| commit_sha(&commit.sha)); + if metadata.vcs_commit.is_some() { + return; + } + } + Ok(response) if github_api_status_is_degradable(response.status()) => { + self.github_api_degraded.store(true, Ordering::Relaxed); + } + Err(_) => { + self.github_api_degraded.store(true, Ordering::Relaxed); + } + _ => {} + } + } + metadata.vcs_commit = self + .github_smart_refs(identity) + .ok() + .and_then(|refs| refs.tag_commits.get(&metadata.vcs_tag).cloned()) + .and_then(|sha| commit_sha(&sha)); + ''' + replace_once(old_fill, new_fill) + + replace_once( + ''' let release = self + .github_headers(self.client.get(github_api_release_url(identity, tag))) + .send() + .ok() + .and_then(|response| response.error_for_status().ok()) + .and_then(|response| response.json::().ok()); + ''', + ''' let release = if self.github_api_degraded.load(Ordering::Relaxed) { + None + } else { + match self + .github_headers(self.client.get(github_api_release_url(identity, tag))) + .send() + { + Ok(response) if response.status().is_success() => { + response.json::().ok() + } + Ok(response) if github_api_status_is_degradable(response.status()) => { + self.github_api_degraded.store(true, Ordering::Relaxed); + None + } + Err(_) => { + self.github_api_degraded.store(true, Ordering::Relaxed); + None + } + _ => None, + } + }; + ''' + ) + + replace_once( + '#[derive(Debug, Deserialize)]\nstruct GithubRepo {\n', + '#[derive(Debug, Clone, Deserialize)]\nstruct GithubRepo {\n' + ) + replace_once( + '#[derive(Debug, Deserialize)]\nstruct GithubTag {\n', + '#[derive(Debug, Clone, Deserialize)]\nstruct GithubTag {\n' + ) + + insert_anchor = '''#[derive(Debug, Deserialize)] + struct GithubCommit { + sha: String, + } + ''' + smart_structs = insert_anchor + ''' + #[derive(Debug, Clone, Default)] + struct GitSmartRefs { + default_branch: Option, + tag_commits: HashMap, + } + + fn github_api_status_is_degradable(status: reqwest::StatusCode) -> bool { + status == reqwest::StatusCode::FORBIDDEN + || status == reqwest::StatusCode::TOO_MANY_REQUESTS + || status.is_server_error() + } + + fn github_identity_key(identity: &GithubIdentity) -> String { + format!( + "{}/{}", + identity.owner.to_ascii_lowercase(), + identity.repo.to_ascii_lowercase() + ) + } + + fn parse_git_smart_refs(bytes: &[u8]) -> Result { + let mut refs = GitSmartRefs::default(); + let mut offset = 0usize; + let mut tag_refs: HashMap = HashMap::new(); + + while offset < bytes.len() { + if bytes.len().saturating_sub(offset) < 4 { + bail!("truncated Git smart-HTTP pkt-line header"); + } + let header = std::str::from_utf8(&bytes[offset..offset + 4]) + .context("non-UTF8 Git smart-HTTP pkt-line length")?; + let length = usize::from_str_radix(header, 16) + .with_context(|| format!("invalid Git smart-HTTP pkt-line length `{header}`"))?; + offset += 4; + if length == 0 || length == 1 || length == 2 { + continue; + } + if length < 4 { + bail!("invalid Git smart-HTTP pkt-line length {length}"); + } + let payload_len = length - 4; + if bytes.len().saturating_sub(offset) < payload_len { + bail!("truncated Git smart-HTTP pkt-line payload"); + } + let payload = &bytes[offset..offset + payload_len]; + offset += payload_len; + let line = std::str::from_utf8(payload) + .context("non-UTF8 Git smart-HTTP advertisement")? + .trim_end_matches(['\\r', '\\n']); + if line.is_empty() || line.starts_with("# service=") { + continue; + } + + let (ref_line, capabilities) = line.split_once('\\0').unwrap_or((line, "")); + for capability in capabilities.split_whitespace() { + if let Some(branch) = capability.strip_prefix("symref=HEAD:refs/heads/") + && !branch.is_empty() + { + refs.default_branch = Some(branch.to_string()); + } + } + + let Some((sha, reference)) = ref_line.split_once(' ') else { + continue; + }; + let Some(sha) = commit_sha(sha) else { + continue; + }; + let Some(raw_tag) = reference.strip_prefix("refs/tags/") else { + continue; + }; + let (tag, peeled) = match raw_tag.strip_suffix("^{}") { + Some(tag) => (tag, true), + None => (raw_tag, false), + }; + if tag.is_empty() { + continue; + } + match tag_refs.get(tag) { + Some((_, true)) if !peeled => {} + _ => { + tag_refs.insert(tag.to_string(), (sha, peeled)); + } + } + } + + refs.tag_commits = tag_refs + .into_iter() + .map(|(tag, (sha, _))| (tag, sha)) + .collect(); + if refs.default_branch.is_none() && refs.tag_commits.is_empty() { + bail!("Git smart-HTTP advertisement contained no usable HEAD or tags"); + } + Ok(refs) + } + ''' + replace_once(insert_anchor, smart_structs) + + test_anchor = ''' #[test] + fn github_sources_precede_zed_owned_r2_during_fallback() { + ''' + smart_tests = ''' fn pkt(payload: &str) -> String { + format!("{:04x}{payload}", payload.len() + 4) + } + + #[test] + fn smart_http_refs_recover_default_branch_and_peeled_tags() { + let commit = "a".repeat(40); + let tag_object = "b".repeat(40); + let peeled = "c".repeat(40); + let first = format!( + "{commit} HEAD\\0multi_ack symref=HEAD:refs/heads/trunk agent=git/github\\n" + ); + let lightweight = format!("{commit} refs/tags/v1.0.0\\n"); + let annotated = format!("{tag_object} refs/tags/v2.0.0\\n"); + let annotated_peeled = format!("{peeled} refs/tags/v2.0.0^{{}}\\n"); + let advertisement = format!( + "{}0000{}{}{}{}0000", + pkt("# service=git-upload-pack\\n"), + pkt(&first), + pkt(&lightweight), + pkt(&annotated), + pkt(&annotated_peeled) + ); + let refs = parse_git_smart_refs(advertisement.as_bytes()).unwrap(); + assert_eq!(refs.default_branch.as_deref(), Some("trunk")); + assert_eq!(refs.tag_commits.get("v1.0.0"), Some(&commit)); + assert_eq!(refs.tag_commits.get("v2.0.0"), Some(&peeled)); + } + + #[test] + fn github_rest_quota_and_server_failures_are_degradable() { + assert!(github_api_status_is_degradable(reqwest::StatusCode::FORBIDDEN)); + assert!(github_api_status_is_degradable(reqwest::StatusCode::TOO_MANY_REQUESTS)); + assert!(github_api_status_is_degradable(reqwest::StatusCode::BAD_GATEWAY)); + assert!(!github_api_status_is_degradable(reqwest::StatusCode::NOT_FOUND)); + assert!(!github_api_status_is_degradable(reqwest::StatusCode::UNAUTHORIZED)); + } + + ''' + test_anchor + replace_once(test_anchor, smart_tests) + + path.write_text(text) + PY + + cargo fmt --all + cargo test --locked source_fallback::tests::smart_http_refs_recover_default_branch_and_peeled_tags -- --nocapture + cargo test --locked source_fallback::tests::github_rest_quota_and_server_failures_are_degradable -- --nocapture + cargo test --locked source_fallback::tests::github_sources_precede_zed_owned_r2_during_fallback -- --nocapture + git diff --check + + - name: Commit runtime change + run: | + set -Eeuo pipefail + git config user.name 'ORESoftware CI' + git config user.email 'ci@oresoftware.com' + git add src/source_fallback.rs + git commit -m 'DEN-3829: survive GitHub REST exhaustion via smart HTTP' + git push origin HEAD:DEN-3829-github-control-plane-fallback From 4e902b67835882f2be75f32ec7938736d33c9b23 Mon Sep 17 00:00:00 2001 From: ORESoftware CI Date: Mon, 14 Sep 2026 17:31:51 +0000 Subject: [PATCH 08/24] DEN-3829: survive GitHub REST exhaustion via smart HTTP --- src/source_fallback.rs | 400 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 365 insertions(+), 35 deletions(-) diff --git a/src/source_fallback.rs b/src/source_fallback.rs index aff33b729..02e1e902c 100644 --- a/src/source_fallback.rs +++ b/src/source_fallback.rs @@ -12,6 +12,7 @@ use std::fs; use std::io::Read as _; use std::path::{Component, Path}; use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use anyhow::{Context, Result, anyhow, bail}; @@ -134,6 +135,10 @@ pub struct FallbackRegistry { client: reqwest::blocking::Client, cache: Mutex>, github_identities: Mutex>, + github_repositories: Mutex>, + github_tag_cache: Mutex>>, + github_smart_ref_cache: Mutex>, + github_api_degraded: AtomicBool, } impl FallbackRegistry { @@ -156,6 +161,10 @@ impl FallbackRegistry { client, cache: Mutex::new(HashMap::new()), github_identities: Mutex::new(HashMap::new()), + github_repositories: Mutex::new(HashMap::new()), + github_tag_cache: Mutex::new(HashMap::new()), + github_smart_ref_cache: Mutex::new(HashMap::new()), + github_api_degraded: AtomicBool::new(false), }), Err(_) => Box::new(inner), } @@ -214,13 +223,31 @@ impl FallbackRegistry { github_api_repo_url(identity), metadata.vcs_tag ); + if !self.github_api_degraded.load(Ordering::Relaxed) { + match self.github_headers(self.client.get(url)).send() { + Ok(response) if response.status().is_success() => { + metadata.vcs_commit = response + .json::() + .ok() + .and_then(|commit| commit_sha(&commit.sha)); + if metadata.vcs_commit.is_some() { + return; + } + } + Ok(response) if github_api_status_is_degradable(response.status()) => { + self.github_api_degraded.store(true, Ordering::Relaxed); + } + Err(_) => { + self.github_api_degraded.store(true, Ordering::Relaxed); + } + _ => {} + } + } metadata.vcs_commit = self - .github_headers(self.client.get(url)) - .send() + .github_smart_refs(identity) .ok() - .filter(|response| response.status().is_success()) - .and_then(|response| response.json::().ok()) - .and_then(|commit| commit_sha(&commit.sha)); + .and_then(|refs| refs.tag_commits.get(&metadata.vcs_tag).cloned()) + .and_then(|sha| commit_sha(&sha)); } fn github_headers( @@ -264,20 +291,74 @@ impl FallbackRegistry { } fn github_repo(&self, identity: &GithubIdentity) -> Result { - let response = self - .github_headers(self.client.get(github_api_repo_url(identity))) - .send() - .with_context(|| format!("fetch GitHub repository {}", identity.web_url()))?; - if !response.status().is_success() { - bail!( + let key = github_identity_key(identity); + if let Some(repo) = self + .github_repositories + .lock() + .ok() + .and_then(|guard| guard.get(&key).cloned()) + { + return Ok(repo); + } + + let api_result = if self.github_api_degraded.load(Ordering::Relaxed) { + None + } else { + Some( + self.github_headers(self.client.get(github_api_repo_url(identity))) + .send(), + ) + }; + + let repo = match api_result { + Some(Ok(response)) if response.status().is_success() => response + .json() + .with_context(|| format!("decode GitHub repository {}", identity.web_url()))?, + Some(Ok(response)) if github_api_status_is_degradable(response.status()) => { + self.github_api_degraded.store(true, Ordering::Relaxed); + self.github_repo_via_smart_http(identity).with_context(|| { + format!( + "GitHub REST repository lookup for {} returned {}; smart-HTTP fallback also failed", + identity.web_url(), + response.status() + ) + })? + } + Some(Ok(response)) => bail!( "GitHub repository {} returned {}", identity.web_url(), response.status() - ); + ), + Some(Err(error)) => { + self.github_api_degraded.store(true, Ordering::Relaxed); + self.github_repo_via_smart_http(identity).with_context(|| { + format!( + "GitHub REST repository lookup for {} failed ({error}); smart-HTTP fallback also failed", + identity.web_url() + ) + })? + } + None => self.github_repo_via_smart_http(identity)?, + }; + + if let Ok(mut guard) = self.github_repositories.lock() { + guard.insert(key, repo.clone()); } - response - .json() - .with_context(|| format!("decode GitHub repository {}", identity.web_url())) + Ok(repo) + } + + fn github_repo_via_smart_http(&self, identity: &GithubIdentity) -> Result { + let refs = self.github_smart_refs(identity)?; + let default_branch = refs.default_branch.clone().with_context(|| { + format!( + "GitHub smart-HTTP advertisement for {} did not report HEAD's branch", + identity.web_url() + ) + })?; + Ok(GithubRepo { + default_branch, + html_url: identity.web_url(), + }) } fn github_manifest(&self, identity: &GithubIdentity, default_branch: &str) -> Result { @@ -535,25 +616,116 @@ impl FallbackRegistry { } fn github_tags(&self, identity: &GithubIdentity) -> Result> { - let response = self - .github_headers(self.client.get(github_api_tags_url(identity))) + let key = github_identity_key(identity); + if let Some(tags) = self + .github_tag_cache + .lock() + .ok() + .and_then(|guard| guard.get(&key).cloned()) + { + return Ok(tags); + } + + let api_result = if self.github_api_degraded.load(Ordering::Relaxed) { + None + } else { + Some( + self.github_headers(self.client.get(github_api_tags_url(identity))) + .send(), + ) + }; + + let tags = match api_result { + Some(Ok(response)) if response.status().is_success() => { + response.json().unwrap_or_default() + } + Some(Ok(response)) if github_api_status_is_degradable(response.status()) => { + self.github_api_degraded.store(true, Ordering::Relaxed); + self.github_tags_via_smart_http(identity).with_context(|| { + format!( + "GitHub REST tags for {} returned {}; smart-HTTP fallback also failed", + identity.web_url(), + response.status() + ) + })? + } + Some(Ok(response)) => { + let hint = if self.config.github_token.is_none() + && response.status() == reqwest::StatusCode::NOT_FOUND + { + " (repository may be private; private repositories need ZED_PKG_GITHUB_TOKEN, GITHUB_TOKEN, or GH_TOKEN)" + } else { + "" + }; + bail!( + "GitHub tags for {} returned {}{hint}", + identity.web_url(), + response.status() + ); + } + Some(Err(error)) => { + self.github_api_degraded.store(true, Ordering::Relaxed); + self.github_tags_via_smart_http(identity).with_context(|| { + format!( + "GitHub REST tags for {} failed ({error}); smart-HTTP fallback also failed", + identity.web_url() + ) + })? + } + None => self.github_tags_via_smart_http(identity)?, + }; + + if let Ok(mut guard) = self.github_tag_cache.lock() { + guard.insert(key, tags.clone()); + } + Ok(tags) + } + + fn github_tags_via_smart_http(&self, identity: &GithubIdentity) -> Result> { + let refs = self.github_smart_refs(identity)?; + let mut names: Vec = refs.tag_commits.keys().cloned().collect(); + names.sort(); + Ok(names.into_iter().map(|name| GithubTag { name }).collect()) + } + + fn github_smart_refs(&self, identity: &GithubIdentity) -> Result { + let key = github_identity_key(identity); + if let Some(refs) = self + .github_smart_ref_cache + .lock() + .ok() + .and_then(|guard| guard.get(&key).cloned()) + { + return Ok(refs); + } + + let url = format!( + "https://github.com/{}/{}.git/info/refs?service=git-upload-pack", + identity.owner, identity.repo + ); + let mut request = self + .client + .get(&url) + .header("Accept", "application/x-git-upload-pack-advertisement"); + if let Some(token) = self.config.github_token.as_deref() { + request = request.basic_auth("x-access-token", Some(token)); + } + let response = request .send() - .with_context(|| format!("list tags for {}", identity.web_url()))?; + .with_context(|| format!("fetch GitHub smart refs from {url}"))?; if !response.status().is_success() { - let hint = if self.config.github_token.is_none() - && response.status() == reqwest::StatusCode::NOT_FOUND - { - " (repository may be private; private repositories need ZED_PKG_GITHUB_TOKEN, GITHUB_TOKEN, or GH_TOKEN)" - } else { - "" - }; bail!( - "GitHub tags for {} returned {}{hint}", + "GitHub smart refs for {} returned {}", identity.web_url(), response.status() ); } - Ok(response.json().unwrap_or_default()) + let bytes = response.bytes().context("read GitHub smart refs")?; + let refs = parse_git_smart_refs(bytes.as_ref())?; + if let Ok(mut guard) = self.github_smart_ref_cache.lock() { + guard.insert(key, refs.clone()); + } + Ok(refs) } fn release_sidecar( @@ -564,12 +736,27 @@ impl FallbackRegistry { version: &str, tag: &str, ) -> Option { - let release = self - .github_headers(self.client.get(github_api_release_url(identity, tag))) - .send() - .ok() - .and_then(|response| response.error_for_status().ok()) - .and_then(|response| response.json::().ok()); + let release = if self.github_api_degraded.load(Ordering::Relaxed) { + None + } else { + match self + .github_headers(self.client.get(github_api_release_url(identity, tag))) + .send() + { + Ok(response) if response.status().is_success() => { + response.json::().ok() + } + Ok(response) if github_api_status_is_degradable(response.status()) => { + self.github_api_degraded.store(true, Ordering::Relaxed); + None + } + Err(_) => { + self.github_api_degraded.store(true, Ordering::Relaxed); + None + } + _ => None, + } + }; for sidecar in github_release_sidecar_names(org, name, version) { let url = zed_interfaces::source::github_release_download_url(identity, tag, &sidecar); let Ok(response) = self.client.get(&url).send() else { @@ -812,7 +999,7 @@ struct GithubRepositorySearch { items: Vec, } -#[derive(Debug, Deserialize)] +#[derive(Debug, Clone, Deserialize)] struct GithubRepo { default_branch: String, #[serde(default)] @@ -907,7 +1094,7 @@ fn release_asset_metadata( None } -#[derive(Debug, Deserialize)] +#[derive(Debug, Clone, Deserialize)] struct GithubTag { name: String, } @@ -917,6 +1104,102 @@ struct GithubCommit { sha: String, } +#[derive(Debug, Clone, Default)] +struct GitSmartRefs { + default_branch: Option, + tag_commits: HashMap, +} + +fn github_api_status_is_degradable(status: reqwest::StatusCode) -> bool { + status == reqwest::StatusCode::FORBIDDEN + || status == reqwest::StatusCode::TOO_MANY_REQUESTS + || status.is_server_error() +} + +fn github_identity_key(identity: &GithubIdentity) -> String { + format!( + "{}/{}", + identity.owner.to_ascii_lowercase(), + identity.repo.to_ascii_lowercase() + ) +} + +fn parse_git_smart_refs(bytes: &[u8]) -> Result { + let mut refs = GitSmartRefs::default(); + let mut offset = 0usize; + let mut tag_refs: HashMap = HashMap::new(); + + while offset < bytes.len() { + if bytes.len().saturating_sub(offset) < 4 { + bail!("truncated Git smart-HTTP pkt-line header"); + } + let header = std::str::from_utf8(&bytes[offset..offset + 4]) + .context("non-UTF8 Git smart-HTTP pkt-line length")?; + let length = usize::from_str_radix(header, 16) + .with_context(|| format!("invalid Git smart-HTTP pkt-line length `{header}`"))?; + offset += 4; + if length == 0 || length == 1 || length == 2 { + continue; + } + if length < 4 { + bail!("invalid Git smart-HTTP pkt-line length {length}"); + } + let payload_len = length - 4; + if bytes.len().saturating_sub(offset) < payload_len { + bail!("truncated Git smart-HTTP pkt-line payload"); + } + let payload = &bytes[offset..offset + payload_len]; + offset += payload_len; + let line = std::str::from_utf8(payload) + .context("non-UTF8 Git smart-HTTP advertisement")? + .trim_end_matches(['\r', '\n']); + if line.is_empty() || line.starts_with("# service=") { + continue; + } + + let (ref_line, capabilities) = line.split_once('\0').unwrap_or((line, "")); + for capability in capabilities.split_whitespace() { + if let Some(branch) = capability.strip_prefix("symref=HEAD:refs/heads/") + && !branch.is_empty() + { + refs.default_branch = Some(branch.to_string()); + } + } + + let Some((sha, reference)) = ref_line.split_once(' ') else { + continue; + }; + let Some(sha) = commit_sha(sha) else { + continue; + }; + let Some(raw_tag) = reference.strip_prefix("refs/tags/") else { + continue; + }; + let (tag, peeled) = match raw_tag.strip_suffix("^{}") { + Some(tag) => (tag, true), + None => (raw_tag, false), + }; + if tag.is_empty() { + continue; + } + match tag_refs.get(tag) { + Some((_, true)) if !peeled => {} + _ => { + tag_refs.insert(tag.to_string(), (sha, peeled)); + } + } + } + + refs.tag_commits = tag_refs + .into_iter() + .map(|(tag, (sha, _))| (tag, sha)) + .collect(); + if refs.default_branch.is_none() && refs.tag_commits.is_empty() { + bail!("Git smart-HTTP advertisement contained no usable HEAD or tags"); + } + Ok(refs) +} + /// A full lowercase 40-hex Git commit id, or nothing. Abbreviated or oddly /// cased ids would never compare equal to a lock's recorded commit anyway. fn commit_sha(value: &str) -> Option { @@ -1263,6 +1546,53 @@ mod tests { use flate2::write::GzEncoder; use zed_interfaces::source::{r2_object_keys, resolve_r2_public_base}; + fn pkt(payload: &str) -> String { + format!("{:04x}{payload}", payload.len() + 4) + } + + #[test] + fn smart_http_refs_recover_default_branch_and_peeled_tags() { + let commit = "a".repeat(40); + let tag_object = "b".repeat(40); + let peeled = "c".repeat(40); + let first = + format!("{commit} HEAD\0multi_ack symref=HEAD:refs/heads/trunk agent=git/github\n"); + let lightweight = format!("{commit} refs/tags/v1.0.0\n"); + let annotated = format!("{tag_object} refs/tags/v2.0.0\n"); + let annotated_peeled = format!("{peeled} refs/tags/v2.0.0^{{}}\n"); + let advertisement = format!( + "{}0000{}{}{}{}0000", + pkt("# service=git-upload-pack\n"), + pkt(&first), + pkt(&lightweight), + pkt(&annotated), + pkt(&annotated_peeled) + ); + let refs = parse_git_smart_refs(advertisement.as_bytes()).unwrap(); + assert_eq!(refs.default_branch.as_deref(), Some("trunk")); + assert_eq!(refs.tag_commits.get("v1.0.0"), Some(&commit)); + assert_eq!(refs.tag_commits.get("v2.0.0"), Some(&peeled)); + } + + #[test] + fn github_rest_quota_and_server_failures_are_degradable() { + assert!(github_api_status_is_degradable( + reqwest::StatusCode::FORBIDDEN + )); + assert!(github_api_status_is_degradable( + reqwest::StatusCode::TOO_MANY_REQUESTS + )); + assert!(github_api_status_is_degradable( + reqwest::StatusCode::BAD_GATEWAY + )); + assert!(!github_api_status_is_degradable( + reqwest::StatusCode::NOT_FOUND + )); + assert!(!github_api_status_is_degradable( + reqwest::StatusCode::UNAUTHORIZED + )); + } + #[test] fn github_sources_precede_zed_owned_r2_during_fallback() { assert!( From 7032ceb8423ec97437d4103e7939a994606bc386 Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 14 Sep 2026 14:55:58 -0500 Subject: [PATCH 09/24] DEN-3829: remove temporary patch workflow --- .github/workflows/patch-github-smart-http.yml | 551 ------------------ 1 file changed, 551 deletions(-) delete mode 100644 .github/workflows/patch-github-smart-http.yml diff --git a/.github/workflows/patch-github-smart-http.yml b/.github/workflows/patch-github-smart-http.yml deleted file mode 100644 index da25fd3ed..000000000 --- a/.github/workflows/patch-github-smart-http.yml +++ /dev/null @@ -1,551 +0,0 @@ -name: patch GitHub smart HTTP recovery - -on: - push: - branches: - - DEN-3829-github-control-plane-fallback - paths: - - '.github/workflows/patch-github-smart-http.yml' - workflow_dispatch: - -permissions: - contents: write - -jobs: - patch: - if: github.repository == 'zed-pkg/zed-cli' && github.ref_name == 'DEN-3829-github-control-plane-fallback' - runs-on: ubuntu-24.04 - timeout-minutes: 15 - steps: - - name: Check out branch - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: DEN-3829-github-control-plane-fallback - fetch-depth: 1 - - - name: Add REST-independent GitHub ref discovery - run: | - set -Eeuo pipefail - python3 - <<'PY' - from pathlib import Path - - path = Path('src/source_fallback.rs') - text = path.read_text() - - def replace_once(old: str, new: str) -> None: - global text - count = text.count(old) - if count != 1: - raise SystemExit(f'expected exactly one occurrence, found {count}: {old[:160]!r}') - text = text.replace(old, new, 1) - - replace_once( - 'use std::sync::Mutex;\n', - 'use std::sync::Mutex;\nuse std::sync::atomic::{AtomicBool, Ordering};\n' - ) - - replace_once( - '''pub struct FallbackRegistry { - inner: HttpRegistry, - config: SourceFallbackConfig, - client: reqwest::blocking::Client, - cache: Mutex>, - github_identities: Mutex>, - } - ''', - '''pub struct FallbackRegistry { - inner: HttpRegistry, - config: SourceFallbackConfig, - client: reqwest::blocking::Client, - cache: Mutex>, - github_identities: Mutex>, - github_repositories: Mutex>, - github_tag_cache: Mutex>>, - github_smart_ref_cache: Mutex>, - github_api_degraded: AtomicBool, - } - ''' - ) - - replace_once( - ''' cache: Mutex::new(HashMap::new()), - github_identities: Mutex::new(HashMap::new()), - }), - ''', - ''' cache: Mutex::new(HashMap::new()), - github_identities: Mutex::new(HashMap::new()), - github_repositories: Mutex::new(HashMap::new()), - github_tag_cache: Mutex::new(HashMap::new()), - github_smart_ref_cache: Mutex::new(HashMap::new()), - github_api_degraded: AtomicBool::new(false), - }), - ''' - ) - - old_repo = ''' fn github_repo(&self, identity: &GithubIdentity) -> Result { - let response = self - .github_headers(self.client.get(github_api_repo_url(identity))) - .send() - .with_context(|| format!("fetch GitHub repository {}", identity.web_url()))?; - if !response.status().is_success() { - bail!( - "GitHub repository {} returned {}", - identity.web_url(), - response.status() - ); - } - response - .json() - .with_context(|| format!("decode GitHub repository {}", identity.web_url())) - } - ''' - new_repo = ''' fn github_repo(&self, identity: &GithubIdentity) -> Result { - let key = github_identity_key(identity); - if let Some(repo) = self - .github_repositories - .lock() - .ok() - .and_then(|guard| guard.get(&key).cloned()) - { - return Ok(repo); - } - - let api_result = if self.github_api_degraded.load(Ordering::Relaxed) { - None - } else { - Some( - self.github_headers(self.client.get(github_api_repo_url(identity))) - .send(), - ) - }; - - let repo = match api_result { - Some(Ok(response)) if response.status().is_success() => response - .json() - .with_context(|| { - format!("decode GitHub repository {}", identity.web_url()) - })?, - Some(Ok(response)) if github_api_status_is_degradable(response.status()) => { - self.github_api_degraded.store(true, Ordering::Relaxed); - self.github_repo_via_smart_http(identity).with_context(|| { - format!( - "GitHub REST repository lookup for {} returned {}; smart-HTTP fallback also failed", - identity.web_url(), - response.status() - ) - })? - } - Some(Ok(response)) => bail!( - "GitHub repository {} returned {}", - identity.web_url(), - response.status() - ), - Some(Err(error)) => { - self.github_api_degraded.store(true, Ordering::Relaxed); - self.github_repo_via_smart_http(identity).with_context(|| { - format!( - "GitHub REST repository lookup for {} failed ({error}); smart-HTTP fallback also failed", - identity.web_url() - ) - })? - } - None => self.github_repo_via_smart_http(identity)?, - }; - - if let Ok(mut guard) = self.github_repositories.lock() { - guard.insert(key, repo.clone()); - } - Ok(repo) - } - - fn github_repo_via_smart_http(&self, identity: &GithubIdentity) -> Result { - let refs = self.github_smart_refs(identity)?; - let default_branch = refs.default_branch.clone().with_context(|| { - format!( - "GitHub smart-HTTP advertisement for {} did not report HEAD's branch", - identity.web_url() - ) - })?; - Ok(GithubRepo { - default_branch, - html_url: identity.web_url(), - }) - } - ''' - replace_once(old_repo, new_repo) - - old_tags = ''' fn github_tags(&self, identity: &GithubIdentity) -> Result> { - let response = self - .github_headers(self.client.get(github_api_tags_url(identity))) - .send() - .with_context(|| format!("list tags for {}", identity.web_url()))?; - if !response.status().is_success() { - let hint = if self.config.github_token.is_none() - && response.status() == reqwest::StatusCode::NOT_FOUND - { - " (repository may be private; private repositories need ZED_PKG_GITHUB_TOKEN, GITHUB_TOKEN, or GH_TOKEN)" - } else { - "" - }; - bail!( - "GitHub tags for {} returned {}{hint}", - identity.web_url(), - response.status() - ); - } - Ok(response.json().unwrap_or_default()) - } - ''' - new_tags = ''' fn github_tags(&self, identity: &GithubIdentity) -> Result> { - let key = github_identity_key(identity); - if let Some(tags) = self - .github_tag_cache - .lock() - .ok() - .and_then(|guard| guard.get(&key).cloned()) - { - return Ok(tags); - } - - let api_result = if self.github_api_degraded.load(Ordering::Relaxed) { - None - } else { - Some( - self.github_headers(self.client.get(github_api_tags_url(identity))) - .send(), - ) - }; - - let tags = match api_result { - Some(Ok(response)) if response.status().is_success() => { - response.json().unwrap_or_default() - } - Some(Ok(response)) if github_api_status_is_degradable(response.status()) => { - self.github_api_degraded.store(true, Ordering::Relaxed); - self.github_tags_via_smart_http(identity).with_context(|| { - format!( - "GitHub REST tags for {} returned {}; smart-HTTP fallback also failed", - identity.web_url(), - response.status() - ) - })? - } - Some(Ok(response)) => { - let hint = if self.config.github_token.is_none() - && response.status() == reqwest::StatusCode::NOT_FOUND - { - " (repository may be private; private repositories need ZED_PKG_GITHUB_TOKEN, GITHUB_TOKEN, or GH_TOKEN)" - } else { - "" - }; - bail!( - "GitHub tags for {} returned {}{hint}", - identity.web_url(), - response.status() - ); - } - Some(Err(error)) => { - self.github_api_degraded.store(true, Ordering::Relaxed); - self.github_tags_via_smart_http(identity).with_context(|| { - format!( - "GitHub REST tags for {} failed ({error}); smart-HTTP fallback also failed", - identity.web_url() - ) - })? - } - None => self.github_tags_via_smart_http(identity)?, - }; - - if let Ok(mut guard) = self.github_tag_cache.lock() { - guard.insert(key, tags.clone()); - } - Ok(tags) - } - - fn github_tags_via_smart_http(&self, identity: &GithubIdentity) -> Result> { - let refs = self.github_smart_refs(identity)?; - let mut names: Vec = refs.tag_commits.keys().cloned().collect(); - names.sort(); - Ok(names.into_iter().map(|name| GithubTag { name }).collect()) - } - - fn github_smart_refs(&self, identity: &GithubIdentity) -> Result { - let key = github_identity_key(identity); - if let Some(refs) = self - .github_smart_ref_cache - .lock() - .ok() - .and_then(|guard| guard.get(&key).cloned()) - { - return Ok(refs); - } - - let url = format!( - "https://github.com/{}/{}.git/info/refs?service=git-upload-pack", - identity.owner, identity.repo - ); - let mut request = self - .client - .get(&url) - .header("Accept", "application/x-git-upload-pack-advertisement"); - if let Some(token) = self.config.github_token.as_deref() { - request = request.basic_auth("x-access-token", Some(token)); - } - let response = request - .send() - .with_context(|| format!("fetch GitHub smart refs from {url}"))?; - if !response.status().is_success() { - bail!("GitHub smart refs for {} returned {}", identity.web_url(), response.status()); - } - let bytes = response.bytes().context("read GitHub smart refs")?; - let refs = parse_git_smart_refs(bytes.as_ref())?; - if let Ok(mut guard) = self.github_smart_ref_cache.lock() { - guard.insert(key, refs.clone()); - } - Ok(refs) - } - ''' - replace_once(old_tags, new_tags) - - old_fill = ''' metadata.vcs_commit = self - .github_headers(self.client.get(url)) - .send() - .ok() - .filter(|response| response.status().is_success()) - .and_then(|response| response.json::().ok()) - .and_then(|commit| commit_sha(&commit.sha)); - ''' - new_fill = ''' if !self.github_api_degraded.load(Ordering::Relaxed) { - match self.github_headers(self.client.get(url)).send() { - Ok(response) if response.status().is_success() => { - metadata.vcs_commit = response - .json::() - .ok() - .and_then(|commit| commit_sha(&commit.sha)); - if metadata.vcs_commit.is_some() { - return; - } - } - Ok(response) if github_api_status_is_degradable(response.status()) => { - self.github_api_degraded.store(true, Ordering::Relaxed); - } - Err(_) => { - self.github_api_degraded.store(true, Ordering::Relaxed); - } - _ => {} - } - } - metadata.vcs_commit = self - .github_smart_refs(identity) - .ok() - .and_then(|refs| refs.tag_commits.get(&metadata.vcs_tag).cloned()) - .and_then(|sha| commit_sha(&sha)); - ''' - replace_once(old_fill, new_fill) - - replace_once( - ''' let release = self - .github_headers(self.client.get(github_api_release_url(identity, tag))) - .send() - .ok() - .and_then(|response| response.error_for_status().ok()) - .and_then(|response| response.json::().ok()); - ''', - ''' let release = if self.github_api_degraded.load(Ordering::Relaxed) { - None - } else { - match self - .github_headers(self.client.get(github_api_release_url(identity, tag))) - .send() - { - Ok(response) if response.status().is_success() => { - response.json::().ok() - } - Ok(response) if github_api_status_is_degradable(response.status()) => { - self.github_api_degraded.store(true, Ordering::Relaxed); - None - } - Err(_) => { - self.github_api_degraded.store(true, Ordering::Relaxed); - None - } - _ => None, - } - }; - ''' - ) - - replace_once( - '#[derive(Debug, Deserialize)]\nstruct GithubRepo {\n', - '#[derive(Debug, Clone, Deserialize)]\nstruct GithubRepo {\n' - ) - replace_once( - '#[derive(Debug, Deserialize)]\nstruct GithubTag {\n', - '#[derive(Debug, Clone, Deserialize)]\nstruct GithubTag {\n' - ) - - insert_anchor = '''#[derive(Debug, Deserialize)] - struct GithubCommit { - sha: String, - } - ''' - smart_structs = insert_anchor + ''' - #[derive(Debug, Clone, Default)] - struct GitSmartRefs { - default_branch: Option, - tag_commits: HashMap, - } - - fn github_api_status_is_degradable(status: reqwest::StatusCode) -> bool { - status == reqwest::StatusCode::FORBIDDEN - || status == reqwest::StatusCode::TOO_MANY_REQUESTS - || status.is_server_error() - } - - fn github_identity_key(identity: &GithubIdentity) -> String { - format!( - "{}/{}", - identity.owner.to_ascii_lowercase(), - identity.repo.to_ascii_lowercase() - ) - } - - fn parse_git_smart_refs(bytes: &[u8]) -> Result { - let mut refs = GitSmartRefs::default(); - let mut offset = 0usize; - let mut tag_refs: HashMap = HashMap::new(); - - while offset < bytes.len() { - if bytes.len().saturating_sub(offset) < 4 { - bail!("truncated Git smart-HTTP pkt-line header"); - } - let header = std::str::from_utf8(&bytes[offset..offset + 4]) - .context("non-UTF8 Git smart-HTTP pkt-line length")?; - let length = usize::from_str_radix(header, 16) - .with_context(|| format!("invalid Git smart-HTTP pkt-line length `{header}`"))?; - offset += 4; - if length == 0 || length == 1 || length == 2 { - continue; - } - if length < 4 { - bail!("invalid Git smart-HTTP pkt-line length {length}"); - } - let payload_len = length - 4; - if bytes.len().saturating_sub(offset) < payload_len { - bail!("truncated Git smart-HTTP pkt-line payload"); - } - let payload = &bytes[offset..offset + payload_len]; - offset += payload_len; - let line = std::str::from_utf8(payload) - .context("non-UTF8 Git smart-HTTP advertisement")? - .trim_end_matches(['\\r', '\\n']); - if line.is_empty() || line.starts_with("# service=") { - continue; - } - - let (ref_line, capabilities) = line.split_once('\\0').unwrap_or((line, "")); - for capability in capabilities.split_whitespace() { - if let Some(branch) = capability.strip_prefix("symref=HEAD:refs/heads/") - && !branch.is_empty() - { - refs.default_branch = Some(branch.to_string()); - } - } - - let Some((sha, reference)) = ref_line.split_once(' ') else { - continue; - }; - let Some(sha) = commit_sha(sha) else { - continue; - }; - let Some(raw_tag) = reference.strip_prefix("refs/tags/") else { - continue; - }; - let (tag, peeled) = match raw_tag.strip_suffix("^{}") { - Some(tag) => (tag, true), - None => (raw_tag, false), - }; - if tag.is_empty() { - continue; - } - match tag_refs.get(tag) { - Some((_, true)) if !peeled => {} - _ => { - tag_refs.insert(tag.to_string(), (sha, peeled)); - } - } - } - - refs.tag_commits = tag_refs - .into_iter() - .map(|(tag, (sha, _))| (tag, sha)) - .collect(); - if refs.default_branch.is_none() && refs.tag_commits.is_empty() { - bail!("Git smart-HTTP advertisement contained no usable HEAD or tags"); - } - Ok(refs) - } - ''' - replace_once(insert_anchor, smart_structs) - - test_anchor = ''' #[test] - fn github_sources_precede_zed_owned_r2_during_fallback() { - ''' - smart_tests = ''' fn pkt(payload: &str) -> String { - format!("{:04x}{payload}", payload.len() + 4) - } - - #[test] - fn smart_http_refs_recover_default_branch_and_peeled_tags() { - let commit = "a".repeat(40); - let tag_object = "b".repeat(40); - let peeled = "c".repeat(40); - let first = format!( - "{commit} HEAD\\0multi_ack symref=HEAD:refs/heads/trunk agent=git/github\\n" - ); - let lightweight = format!("{commit} refs/tags/v1.0.0\\n"); - let annotated = format!("{tag_object} refs/tags/v2.0.0\\n"); - let annotated_peeled = format!("{peeled} refs/tags/v2.0.0^{{}}\\n"); - let advertisement = format!( - "{}0000{}{}{}{}0000", - pkt("# service=git-upload-pack\\n"), - pkt(&first), - pkt(&lightweight), - pkt(&annotated), - pkt(&annotated_peeled) - ); - let refs = parse_git_smart_refs(advertisement.as_bytes()).unwrap(); - assert_eq!(refs.default_branch.as_deref(), Some("trunk")); - assert_eq!(refs.tag_commits.get("v1.0.0"), Some(&commit)); - assert_eq!(refs.tag_commits.get("v2.0.0"), Some(&peeled)); - } - - #[test] - fn github_rest_quota_and_server_failures_are_degradable() { - assert!(github_api_status_is_degradable(reqwest::StatusCode::FORBIDDEN)); - assert!(github_api_status_is_degradable(reqwest::StatusCode::TOO_MANY_REQUESTS)); - assert!(github_api_status_is_degradable(reqwest::StatusCode::BAD_GATEWAY)); - assert!(!github_api_status_is_degradable(reqwest::StatusCode::NOT_FOUND)); - assert!(!github_api_status_is_degradable(reqwest::StatusCode::UNAUTHORIZED)); - } - - ''' + test_anchor - replace_once(test_anchor, smart_tests) - - path.write_text(text) - PY - - cargo fmt --all - cargo test --locked source_fallback::tests::smart_http_refs_recover_default_branch_and_peeled_tags -- --nocapture - cargo test --locked source_fallback::tests::github_rest_quota_and_server_failures_are_degradable -- --nocapture - cargo test --locked source_fallback::tests::github_sources_precede_zed_owned_r2_during_fallback -- --nocapture - git diff --check - - - name: Commit runtime change - run: | - set -Eeuo pipefail - git config user.name 'ORESoftware CI' - git config user.email 'ci@oresoftware.com' - git add src/source_fallback.rs - git commit -m 'DEN-3829: survive GitHub REST exhaustion via smart HTTP' - git push origin HEAD:DEN-3829-github-control-plane-fallback From f286e574c20ab838b49b3039b1662eff4d5fed85 Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 14 Sep 2026 15:03:19 -0500 Subject: [PATCH 10/24] DEN-3829: scope outage assertions to positive recovery --- .../workflows/github-control-plane-outage.yml | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/.github/workflows/github-control-plane-outage.yml b/.github/workflows/github-control-plane-outage.yml index 533e1665f..95c40b4fa 100644 --- a/.github/workflows/github-control-plane-outage.yml +++ b/.github/workflows/github-control-plane-outage.yml @@ -147,6 +147,23 @@ jobs: cmp "$RUNNER_TEMP/github-only.lock" .zpkg.lock test "$(find zed_modules -name .zpkg.toml -type f | wc -l)" -ge 5 + - name: Prove successful GitHub recovery did not touch R2 or Cloudflare + run: | + set -Eeuo pipefail + log="$RUNNER_TEMP/github-only-a.log" + test -s "$log" + grep -E -q 'using GitHub(-first)? fallback' "$log" + if grep -q 'GET /' "$RUNNER_TEMP/r2-sentinel.log"; then + echo 'Successful GitHub recovery unexpectedly contacted the configured R2/CDN endpoint:' >&2 + cat "$RUNNER_TEMP/r2-sentinel.log" >&2 + exit 1 + fi + ! grep -E 'success.*(zpkg\.net|cdn\.zpkg\.net|cloudflare)' "$log" + # Negative tests below intentionally exercise failure paths that may + # probe configured mirrors. Reset the sentinel so they cannot taint + # the positive recovery assertion above. + : > "$RUNNER_TEMP/r2-sentinel.log" + - name: Prove fallback is required rather than accidental cache reuse working-directory: ${{ runner.temp }}/github-only-consumer env: @@ -202,19 +219,12 @@ jobs: set -e cat "$RUNNER_TEMP/github-negative.log" test "$status" -ne 0 - grep -E 'expected zed-pkg-test/awkward-lib@0\.1\.0|claims .*awkward-lib@0\.1\.0' "$RUNNER_TEMP/github-negative.log" + grep -F 'does not self-claim package zed-pkg-test/awkward-lib' "$RUNNER_TEMP/github-negative.log" - - name: Prove GitHub recovery did not touch R2 or Cloudflare + - name: Stop R2 sentinel if: always() run: | set -Eeuo pipefail - log="$RUNNER_TEMP/github-only-a.log" - test -s "$log" - grep -E -q 'using GitHub(-first)? fallback' "$log" - if grep -q 'GET /' "$RUNNER_TEMP/r2-sentinel.log"; then - echo 'GitHub-first recovery unexpectedly contacted the configured R2/CDN endpoint:' >&2 - cat "$RUNNER_TEMP/r2-sentinel.log" >&2 - exit 1 + if test -s "$RUNNER_TEMP/r2-sentinel.pid"; then + kill "$(cat "$RUNNER_TEMP/r2-sentinel.pid")" || true fi - ! grep -E 'success.*(zpkg\.net|cdn\.zpkg\.net|cloudflare)' "$log" - kill "$(cat "$RUNNER_TEMP/r2-sentinel.pid")" || true From f3123ebebadaede30da97a31a586288c3b15609d Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 14 Sep 2026 15:06:04 -0500 Subject: [PATCH 11/24] chore: apply rustfmt to TOML contract tests --- tests/toml_contract.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/toml_contract.rs b/tests/toml_contract.rs index 04b406f49..1a4ec8bbd 100644 --- a/tests/toml_contract.rs +++ b/tests/toml_contract.rs @@ -9,8 +9,7 @@ fn root() -> PathBuf { fn parse_toml(path: &Path) -> toml::Value { let text = fs::read_to_string(path) .unwrap_or_else(|error| panic!("failed to read {}: {error}", path.display())); - toml::from_str(&text) - .unwrap_or_else(|error| panic!("invalid TOML {}: {error}", path.display())) + toml::from_str(&text).unwrap_or_else(|error| panic!("invalid TOML {}: {error}", path.display())) } fn string_at<'a>(value: &'a toml::Value, path: &[&str]) -> &'a str { @@ -133,7 +132,10 @@ fn global_flags_do_not_reuse_environment_keys_or_public_aliases() { if alias == canonical_alias { continue; } - assert!(aliases.insert(alias.to_owned()), "duplicate global flag alias: {alias}"); + assert!( + aliases.insert(alias.to_owned()), + "duplicate global flag alias: {alias}" + ); } } } From 8f8144569e34e9b4c8e39d10d554ecab5991824c Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 14 Sep 2026 18:11:37 -0500 Subject: [PATCH 12/24] ci: apply and verify GHCR auth fix --- .github/workflows/ghcr-auth-fix-once.yml | 319 +++++++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 .github/workflows/ghcr-auth-fix-once.yml diff --git a/.github/workflows/ghcr-auth-fix-once.yml b/.github/workflows/ghcr-auth-fix-once.yml new file mode 100644 index 000000000..b483fab75 --- /dev/null +++ b/.github/workflows/ghcr-auth-fix-once.yml @@ -0,0 +1,319 @@ +name: apply GHCR auth fix once + +on: + push: + branches: + - DEN-3829-github-control-plane-fallback + paths: + - '.github/workflows/ghcr-auth-fix-once.yml' + +permissions: + contents: write + +jobs: + patch: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: DEN-3829-github-control-plane-fallback + fetch-depth: 0 + persist-credentials: true + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: '1.98.1' + components: rustfmt + - name: Replace raw GitHub bearer use with GHCR registry-token exchange + shell: bash + run: | + set -Eeuo pipefail + python3 <<'PY' + from pathlib import Path + import re + + p = Path('src/github_packages.rs') + s = p.read_text() + old = ' let bearer = ghcr_bearer(&client, token, &identity)?;' + new = ' let bearer = ghcr_bearer(&client, Some(token), &identity, GhcrScope::PushPull)?;' + assert old in s + s = s.replace(old, new, 1) + + replacement = r'''#[derive(Clone, Copy, Debug)] + enum GhcrScope { + Pull, + PushPull, + } + + impl GhcrScope { + fn actions(self) -> &'static str { + match self { + Self::Pull => "pull", + Self::PushPull => "pull,push", + } + } + } + + fn ghcr_bearer( + client: &reqwest::blocking::Client, + github_token: Option<&str>, + identity: &GithubIdentity, + scope: GhcrScope, + ) -> Result { + let scope = format!( + "repository:{}:{}", + ghcr_repository(identity), + scope.actions() + ); + let mut request = client + .get("https://ghcr.io/token") + .query(&[("service", "ghcr.io"), ("scope", scope.as_str())]); + if let Some(token) = github_token { + let username = github_username(client, token)?; + request = request.basic_auth(username, Some(token)); + } + let response = request.send().context("request GHCR registry token")?; + let status = response.status(); + if !status.is_success() { + bail!("GHCR registry token request for {scope} returned {status}"); + } + let body = response + .json::() + .context("decode GHCR registry token response")?; + body.token + .or(body.access_token) + .filter(|token| !token.trim().is_empty()) + .context("GHCR registry token response contained no token") + } + + fn github_username(client: &reqwest::blocking::Client, token: &str) -> Result { + for key in ["ZED_PKG_GITHUB_USER", "GITHUB_ACTOR", "GH_USER"] { + if let Ok(value) = std::env::var(key) + && !value.trim().is_empty() + { + return Ok(value); + } + } + let response = client + .get("https://api.github.com/user") + .header("Accept", "application/vnd.github+json") + .bearer_auth(token) + .send() + .context("resolve GitHub login for GHCR authentication")?; + let status = response.status(); + if !status.is_success() { + bail!("GitHub /user returned {status} while resolving GHCR login"); + } + let user = response + .json::() + .context("decode GitHub user for GHCR authentication")?; + if user.login.trim().is_empty() { + bail!("GitHub /user returned an empty login for GHCR authentication"); + } + Ok(user.login) + } + + #[derive(Debug, serde::Deserialize)] + struct GhcrTokenResponse { + #[serde(default)] + token: Option, + #[serde(default)] + access_token: Option, + } + + #[derive(Debug, serde::Deserialize)] + struct GithubUser { + login: String, + } + + fn ghcr_identity_from_registry_url(url: &str) -> Result { + let parsed = reqwest::Url::parse(url).with_context(|| format!("parse GHCR URL {url}"))?; + if parsed.scheme() != "https" + || !matches!(parsed.host_str(), Some(host) if host.eq_ignore_ascii_case("ghcr.io")) + { + bail!("GHCR fallback URL must use https://ghcr.io: {url}"); + } + let segments = parsed + .path_segments() + .context("GHCR URL cannot be a base URL")? + .collect::>(); + if segments.len() < 4 + || segments[0] != "v2" + || segments[1].is_empty() + || segments[2].is_empty() + || !matches!(segments[3], "manifests" | "blobs") + { + bail!("GHCR URL does not identify a canonical owner/repo artifact: {url}"); + } + Ok(GithubIdentity { + owner: segments[1].to_string(), + repo: segments[2].to_string(), + }) + } + + pub(crate) fn ghcr_manifest_json( + client: &reqwest::blocking::Client, + github_token: Option<&str>, + identity: &GithubIdentity, + tag: &str, + ) -> Result { + let bearer = ghcr_bearer(client, github_token, identity, GhcrScope::Pull)?; + let url = ghcr_manifest_url(identity, tag); + let response = client + .get(&url) + .header("Accept", OCI_IMAGE_MANIFEST_MEDIA_TYPE) + .bearer_auth(&bearer) + .send() + .with_context(|| format!("GET {url}"))?; + if !response.status().is_success() { + bail!("{url} returned {}", response.status()); + } + response + .json::() + .with_context(|| format!("decode GHCR manifest {url}")) + } + + fn upload_blob''' + pattern = re.compile( + r'fn ghcr_bearer\([\s\S]*?\n\}\n\n#\[derive\(Debug, serde::Deserialize\)\]\nstruct GhcrTokenResponse \{[\s\S]*?\n\}\n\nfn upload_blob', + re.M, + ) + s, n = pattern.subn(replacement, s, count=1) + assert n == 1, n + + replacement = r'''pub fn download_ghcr_layer( + client: &reqwest::blocking::Client, + token: Option<&str>, + url: &str, + dest: &std::path::Path, + declared_size: u64, + max_bytes: u64, + ) -> Result<()> { + let identity = ghcr_identity_from_registry_url(url)?; + let bearer = ghcr_bearer(client, token, &identity, GhcrScope::Pull)?; + if url.contains("/blobs/") { + return download_blob(client, &bearer, url, dest, declared_size, max_bytes); + } + let response = client + .get(url) + .header("Accept", OCI_IMAGE_MANIFEST_MEDIA_TYPE) + .bearer_auth(&bearer) + .send() + .with_context(|| format!("GET {url}"))?; + if !response.status().is_success() { + bail!("{url} returned {}", response.status()); + } + let manifest: serde_json::Value = response.json()?; + let layer = manifest + .get("layers") + .and_then(|layers| layers.as_array()) + .and_then(|layers| layers.first()) + .context("GHCR manifest has no layers")?; + let digest = layer + .get("digest") + .and_then(|value| value.as_str()) + .context("GHCR layer missing digest")?; + let blob_url = url + .split("/manifests/") + .next() + .map(|prefix| format!("{prefix}/blobs/{digest}")) + .context("GHCR manifest URL is not a registry manifest")?; + download_blob(client, &bearer, &blob_url, dest, declared_size, max_bytes) + } + + fn download_blob( + client: &reqwest::blocking::Client, + bearer: &str, + url: &str, + dest: &std::path::Path, + declared_size: u64, + max_bytes: u64, + ) -> Result<()> { + let response = client + .get(url) + .bearer_auth(bearer) + .send() + .with_context(|| format!("GET {url}"))?; + if !response.status().is_success() { + bail!("{url} returned {}", response.status()); + } + fs::create_dir_all(dest.parent().context("dest has parent")?)?; + let limit = if declared_size > 0 { + declared_size.saturating_add(1024 * 1024).min(max_bytes) + } else { + max_bytes + }; + let temp = dest.with_extension(format!( + "{}.part", + dest.extension().and_then(|ext| ext.to_str()).unwrap_or("zed") + )); + let _ = fs::remove_file(&temp); + let mut file = fs::File::create(&temp)?; + let mut limited = std::io::Read::take(response, limit.saturating_add(1)); + let copied = std::io::copy(&mut limited, &mut file)?; + if copied > limit { + let _ = fs::remove_file(&temp); + bail!("GHCR blob exceeded {limit} bytes from {url}"); + } + if copied == 0 { + let _ = fs::remove_file(&temp); + bail!("GHCR blob from {url} was empty"); + } + file.sync_all()?; + drop(file); + fs::rename(&temp, dest)?; + Ok(()) + } + + #[cfg(test)]''' + pattern = re.compile( + r'pub fn download_ghcr_layer\([\s\S]*?\n\}\n\nfn download_blob\([\s\S]*?\n\}\n\n#\[cfg\(test\)\]', + re.M, + ) + s, n = pattern.subn(replacement, s, count=1) + assert n == 1, n + p.write_text(s) + + p = Path('src/source_fallback.rs') + s = p.read_text() + old = ''' let mut request = self + .client + .get(&url) + .header("Accept", "application/vnd.oci.image.manifest.v1+json"); + if let Some(token) = &self.config.github_token { + request = request.bearer_auth(token); + } + let Ok(response) = request.send() else { + return None; + }; + if !response.status().is_success() { + return None; + } + let Ok(manifest) = response.json::() else { + return None; + };''' + new = ''' let Ok(manifest) = crate::github_packages::ghcr_manifest_json( + &self.client, + self.config.github_token.as_deref(), + identity, + tag, + ) else { + return None; + };''' + assert old in s + s = s.replace(old, new, 1) + p.write_text(s) + PY + cargo fmt --all + cargo test --locked github_packages --lib + cargo test --locked source_fallback --lib + git diff --check + - name: Commit verified GHCR auth fix + shell: bash + run: | + set -Eeuo pipefail + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git add src/github_packages.rs src/source_fallback.rs + git commit -m 'fix: authenticate GHCR through registry tokens' + git push origin HEAD:DEN-3829-github-control-plane-fallback From 3ccdc99459e6896d277d11ddeef770f47fd5c616 Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 14 Sep 2026 18:14:18 -0500 Subject: [PATCH 13/24] ci: salvage proven GHCR registry auth into outage branch --- .github/workflows/ghcr-auth-salvage-once.yml | 71 ++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .github/workflows/ghcr-auth-salvage-once.yml diff --git a/.github/workflows/ghcr-auth-salvage-once.yml b/.github/workflows/ghcr-auth-salvage-once.yml new file mode 100644 index 000000000..67dd8a856 --- /dev/null +++ b/.github/workflows/ghcr-auth-salvage-once.yml @@ -0,0 +1,71 @@ +name: salvage proven GHCR auth once + +on: + push: + branches: + - DEN-3829-github-control-plane-fallback + paths: + - '.github/workflows/ghcr-auth-salvage-once.yml' + +permissions: + contents: write + +jobs: + salvage: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: DEN-3829-github-control-plane-fallback + fetch-depth: 0 + persist-credentials: true + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: '1.98.1' + components: rustfmt + - name: Salvage proven GHCR protocol changes without stale identity guessing + shell: bash + run: | + set -Eeuo pipefail + git show 3d498147d755f7824ef5c1c1d8072ec6bcedb53a -- src/github_packages.rs > /tmp/ghcr-auth.patch + git apply --3way /tmp/ghcr-auth.patch + python3 <<'PY' + from pathlib import Path + import re + + p = Path('src/source_fallback.rs') + s = p.read_text() + pattern = re.compile( + r''' let mut request = self\n \.client\n \.get\(&url\)\n \.header\("Accept", "application/vnd\.oci\.image\.manifest\.v1\+json"\);\n if let Some\(token\) = &self\.config\.github_token \{\n request = request\.bearer_auth\(token\);\n \}''' + ) + replacement = ''' let mut request = self + .client + .get(&url) + .header("Accept", "application/vnd.oci.image.manifest.v1+json"); + if let Some(token) = &self.config.github_token { + match crate::github_packages::ghcr_registry_token(&self.client, token, identity, "pull") + { + Ok(bearer) => request = request.bearer_auth(bearer), + Err(_) => return None, + } + }''' + s, n = pattern.subn(replacement, s, count=1) + assert n == 1, f'expected one ghcr_version auth block, found {n}' + p.write_text(s) + PY + cargo fmt --all + cargo test --locked github_packages --lib + cargo test --locked source_fallback --lib + git diff --check + grep -F '.basic_auth("x-access-token", Some(github_token))' src/github_packages.rs + ! grep -A8 'fn ghcr_version' src/source_fallback.rs | grep -F 'bearer_auth(token)' + - name: Commit verified salvage + shell: bash + run: | + set -Eeuo pipefail + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git add src/github_packages.rs src/source_fallback.rs + git commit -m 'fix: use proven GHCR registry token exchange' + git push origin HEAD:DEN-3829-github-control-plane-fallback From fc48d78f30d0ca3bcee97406ae8ece1e6882785a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2026 23:15:32 +0000 Subject: [PATCH 14/24] fix: use proven GHCR registry token exchange --- src/github_packages.rs | 86 +++++++++++++++++++++++++++++++++++++----- src/source_fallback.rs | 6 ++- 2 files changed, 82 insertions(+), 10 deletions(-) diff --git a/src/github_packages.rs b/src/github_packages.rs index b37399492..e34fbe426 100644 --- a/src/github_packages.rs +++ b/src/github_packages.rs @@ -88,6 +88,7 @@ pub enum GhcrOutcome { #[derive(Debug)] struct Blob { digest: String, + #[allow(dead_code)] media_type: &'static str, bytes: Vec, } @@ -248,26 +249,56 @@ fn sha256_hex(bytes: &[u8]) -> String { hex::encode(Sha256::digest(bytes)) } -fn ghcr_bearer( +/// Exchange a GitHub PAT for a GHCR registry token. +/// +/// A Bearer PAT against `https://ghcr.io/token` is pull-only (or rejected on +/// `/v2`). Docker-style Basic `x-access-token:` is what GHCR accepts for +/// push and for authenticated pull. Do not fall back to the raw PAT as a +/// registry Bearer — GHCR returns `DENIED: invalid token`. +pub(crate) fn ghcr_registry_token( client: &reqwest::blocking::Client, github_token: &str, identity: &GithubIdentity, + access: &str, ) -> Result { - let scope = format!("repository:{}:push,pull", ghcr_repository(identity)); + let scope = format!("repository:{}:{access}", ghcr_repository(identity)); let response = client .get("https://ghcr.io/token") .query(&[("service", "ghcr.io"), ("scope", scope.as_str())]) - .bearer_auth(github_token) + .basic_auth("x-access-token", Some(github_token)) .send() .context("request GHCR registry token")?; - if response.status().is_success() + let status = response.status(); + if status.is_success() && let Ok(body) = response.json::() && let Some(token) = body.token.or(body.access_token) && !token.is_empty() { return Ok(token); } - Ok(github_token.to_string()) + bail!( + "GHCR token exchange for {} returned {status}", + ghcr_repository(identity) + ) +} + +fn ghcr_bearer( + client: &reqwest::blocking::Client, + github_token: &str, + identity: &GithubIdentity, +) -> Result { + ghcr_registry_token(client, github_token, identity, "push,pull") +} + +pub(crate) fn identity_from_ghcr_url(url: &str) -> Option { + let path = url.strip_prefix("https://ghcr.io/v2/")?; + let mut parts = path.split('/'); + let owner = parts.next()?.to_string(); + let repo = parts.next()?.to_string(); + if owner.is_empty() || repo.is_empty() { + return None; + } + Some(GithubIdentity { owner, repo }) } #[derive(Debug, serde::Deserialize)] @@ -313,7 +344,7 @@ fn upload_blob( let put = client .put(&put_url) .bearer_auth(token) - .header("Content-Type", blob.media_type) + .header("Content-Type", "application/octet-stream") .body(blob.bytes.clone()) .send() .with_context(|| format!("upload GHCR blob {}", blob.digest))?; @@ -362,13 +393,30 @@ pub fn download_ghcr_layer( declared_size: u64, max_bytes: u64, ) -> Result<()> { + let registry_token = match (token, identity_from_ghcr_url(url)) { + (Some(github_token), Some(identity)) => Some(ghcr_registry_token( + client, + github_token, + &identity, + "pull", + )?), + (Some(github_token), None) => Some(github_token.to_string()), + (None, _) => None, + }; if url.contains("/blobs/") { - return download_blob(client, token, url, dest, declared_size, max_bytes); + return download_blob( + client, + registry_token.as_deref(), + url, + dest, + declared_size, + max_bytes, + ); } let mut request = client .get(url) .header("Accept", OCI_IMAGE_MANIFEST_MEDIA_TYPE); - if let Some(token) = token { + if let Some(token) = registry_token.as_deref() { request = request.bearer_auth(token); } let response = request.send().with_context(|| format!("GET {url}"))?; @@ -390,7 +438,14 @@ pub fn download_ghcr_layer( .next() .map(|prefix| format!("{prefix}/blobs/{digest}")) .context("GHCR manifest URL is not a registry manifest")?; - download_blob(client, token, &blob_url, dest, declared_size, max_bytes) + download_blob( + client, + registry_token.as_deref(), + &blob_url, + dest, + declared_size, + max_bytes, + ) } fn download_blob( @@ -448,6 +503,19 @@ mod tests { } } + #[test] + fn identity_from_ghcr_blob_url() { + let identity = identity_from_ghcr_url( + "https://ghcr.io/v2/zed-pkg-test/ghcr-fallback-canary/blobs/sha256:abc", + ) + .expect("parse"); + assert_eq!(identity.owner, "zed-pkg-test"); + assert_eq!(identity.repo, "ghcr-fallback-canary"); + assert!( + identity_from_ghcr_url("https://example.com/v2/acme/pkg/blobs/sha256:abc").is_none() + ); + } + #[test] fn upload_session_appends_digest_query() { assert_eq!( diff --git a/src/source_fallback.rs b/src/source_fallback.rs index 02e1e902c..d6f42b8f0 100644 --- a/src/source_fallback.rs +++ b/src/source_fallback.rs @@ -797,7 +797,11 @@ impl FallbackRegistry { .get(&url) .header("Accept", "application/vnd.oci.image.manifest.v1+json"); if let Some(token) = &self.config.github_token { - request = request.bearer_auth(token); + match crate::github_packages::ghcr_registry_token(&self.client, token, identity, "pull") + { + Ok(bearer) => request = request.bearer_auth(bearer), + Err(_) => return None, + } } let Ok(response) = request.send() else { return None; From 27c78761943b230c91275eff7a6a2c08b17af8b9 Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 14 Sep 2026 18:16:25 -0500 Subject: [PATCH 15/24] ci: remove completed one-shot GHCR auth patch workflow --- .github/workflows/ghcr-auth-fix-once.yml | 319 ----------------------- 1 file changed, 319 deletions(-) delete mode 100644 .github/workflows/ghcr-auth-fix-once.yml diff --git a/.github/workflows/ghcr-auth-fix-once.yml b/.github/workflows/ghcr-auth-fix-once.yml deleted file mode 100644 index b483fab75..000000000 --- a/.github/workflows/ghcr-auth-fix-once.yml +++ /dev/null @@ -1,319 +0,0 @@ -name: apply GHCR auth fix once - -on: - push: - branches: - - DEN-3829-github-control-plane-fallback - paths: - - '.github/workflows/ghcr-auth-fix-once.yml' - -permissions: - contents: write - -jobs: - patch: - runs-on: ubuntu-24.04 - timeout-minutes: 30 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: DEN-3829-github-control-plane-fallback - fetch-depth: 0 - persist-credentials: true - - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 - with: - toolchain: '1.98.1' - components: rustfmt - - name: Replace raw GitHub bearer use with GHCR registry-token exchange - shell: bash - run: | - set -Eeuo pipefail - python3 <<'PY' - from pathlib import Path - import re - - p = Path('src/github_packages.rs') - s = p.read_text() - old = ' let bearer = ghcr_bearer(&client, token, &identity)?;' - new = ' let bearer = ghcr_bearer(&client, Some(token), &identity, GhcrScope::PushPull)?;' - assert old in s - s = s.replace(old, new, 1) - - replacement = r'''#[derive(Clone, Copy, Debug)] - enum GhcrScope { - Pull, - PushPull, - } - - impl GhcrScope { - fn actions(self) -> &'static str { - match self { - Self::Pull => "pull", - Self::PushPull => "pull,push", - } - } - } - - fn ghcr_bearer( - client: &reqwest::blocking::Client, - github_token: Option<&str>, - identity: &GithubIdentity, - scope: GhcrScope, - ) -> Result { - let scope = format!( - "repository:{}:{}", - ghcr_repository(identity), - scope.actions() - ); - let mut request = client - .get("https://ghcr.io/token") - .query(&[("service", "ghcr.io"), ("scope", scope.as_str())]); - if let Some(token) = github_token { - let username = github_username(client, token)?; - request = request.basic_auth(username, Some(token)); - } - let response = request.send().context("request GHCR registry token")?; - let status = response.status(); - if !status.is_success() { - bail!("GHCR registry token request for {scope} returned {status}"); - } - let body = response - .json::() - .context("decode GHCR registry token response")?; - body.token - .or(body.access_token) - .filter(|token| !token.trim().is_empty()) - .context("GHCR registry token response contained no token") - } - - fn github_username(client: &reqwest::blocking::Client, token: &str) -> Result { - for key in ["ZED_PKG_GITHUB_USER", "GITHUB_ACTOR", "GH_USER"] { - if let Ok(value) = std::env::var(key) - && !value.trim().is_empty() - { - return Ok(value); - } - } - let response = client - .get("https://api.github.com/user") - .header("Accept", "application/vnd.github+json") - .bearer_auth(token) - .send() - .context("resolve GitHub login for GHCR authentication")?; - let status = response.status(); - if !status.is_success() { - bail!("GitHub /user returned {status} while resolving GHCR login"); - } - let user = response - .json::() - .context("decode GitHub user for GHCR authentication")?; - if user.login.trim().is_empty() { - bail!("GitHub /user returned an empty login for GHCR authentication"); - } - Ok(user.login) - } - - #[derive(Debug, serde::Deserialize)] - struct GhcrTokenResponse { - #[serde(default)] - token: Option, - #[serde(default)] - access_token: Option, - } - - #[derive(Debug, serde::Deserialize)] - struct GithubUser { - login: String, - } - - fn ghcr_identity_from_registry_url(url: &str) -> Result { - let parsed = reqwest::Url::parse(url).with_context(|| format!("parse GHCR URL {url}"))?; - if parsed.scheme() != "https" - || !matches!(parsed.host_str(), Some(host) if host.eq_ignore_ascii_case("ghcr.io")) - { - bail!("GHCR fallback URL must use https://ghcr.io: {url}"); - } - let segments = parsed - .path_segments() - .context("GHCR URL cannot be a base URL")? - .collect::>(); - if segments.len() < 4 - || segments[0] != "v2" - || segments[1].is_empty() - || segments[2].is_empty() - || !matches!(segments[3], "manifests" | "blobs") - { - bail!("GHCR URL does not identify a canonical owner/repo artifact: {url}"); - } - Ok(GithubIdentity { - owner: segments[1].to_string(), - repo: segments[2].to_string(), - }) - } - - pub(crate) fn ghcr_manifest_json( - client: &reqwest::blocking::Client, - github_token: Option<&str>, - identity: &GithubIdentity, - tag: &str, - ) -> Result { - let bearer = ghcr_bearer(client, github_token, identity, GhcrScope::Pull)?; - let url = ghcr_manifest_url(identity, tag); - let response = client - .get(&url) - .header("Accept", OCI_IMAGE_MANIFEST_MEDIA_TYPE) - .bearer_auth(&bearer) - .send() - .with_context(|| format!("GET {url}"))?; - if !response.status().is_success() { - bail!("{url} returned {}", response.status()); - } - response - .json::() - .with_context(|| format!("decode GHCR manifest {url}")) - } - - fn upload_blob''' - pattern = re.compile( - r'fn ghcr_bearer\([\s\S]*?\n\}\n\n#\[derive\(Debug, serde::Deserialize\)\]\nstruct GhcrTokenResponse \{[\s\S]*?\n\}\n\nfn upload_blob', - re.M, - ) - s, n = pattern.subn(replacement, s, count=1) - assert n == 1, n - - replacement = r'''pub fn download_ghcr_layer( - client: &reqwest::blocking::Client, - token: Option<&str>, - url: &str, - dest: &std::path::Path, - declared_size: u64, - max_bytes: u64, - ) -> Result<()> { - let identity = ghcr_identity_from_registry_url(url)?; - let bearer = ghcr_bearer(client, token, &identity, GhcrScope::Pull)?; - if url.contains("/blobs/") { - return download_blob(client, &bearer, url, dest, declared_size, max_bytes); - } - let response = client - .get(url) - .header("Accept", OCI_IMAGE_MANIFEST_MEDIA_TYPE) - .bearer_auth(&bearer) - .send() - .with_context(|| format!("GET {url}"))?; - if !response.status().is_success() { - bail!("{url} returned {}", response.status()); - } - let manifest: serde_json::Value = response.json()?; - let layer = manifest - .get("layers") - .and_then(|layers| layers.as_array()) - .and_then(|layers| layers.first()) - .context("GHCR manifest has no layers")?; - let digest = layer - .get("digest") - .and_then(|value| value.as_str()) - .context("GHCR layer missing digest")?; - let blob_url = url - .split("/manifests/") - .next() - .map(|prefix| format!("{prefix}/blobs/{digest}")) - .context("GHCR manifest URL is not a registry manifest")?; - download_blob(client, &bearer, &blob_url, dest, declared_size, max_bytes) - } - - fn download_blob( - client: &reqwest::blocking::Client, - bearer: &str, - url: &str, - dest: &std::path::Path, - declared_size: u64, - max_bytes: u64, - ) -> Result<()> { - let response = client - .get(url) - .bearer_auth(bearer) - .send() - .with_context(|| format!("GET {url}"))?; - if !response.status().is_success() { - bail!("{url} returned {}", response.status()); - } - fs::create_dir_all(dest.parent().context("dest has parent")?)?; - let limit = if declared_size > 0 { - declared_size.saturating_add(1024 * 1024).min(max_bytes) - } else { - max_bytes - }; - let temp = dest.with_extension(format!( - "{}.part", - dest.extension().and_then(|ext| ext.to_str()).unwrap_or("zed") - )); - let _ = fs::remove_file(&temp); - let mut file = fs::File::create(&temp)?; - let mut limited = std::io::Read::take(response, limit.saturating_add(1)); - let copied = std::io::copy(&mut limited, &mut file)?; - if copied > limit { - let _ = fs::remove_file(&temp); - bail!("GHCR blob exceeded {limit} bytes from {url}"); - } - if copied == 0 { - let _ = fs::remove_file(&temp); - bail!("GHCR blob from {url} was empty"); - } - file.sync_all()?; - drop(file); - fs::rename(&temp, dest)?; - Ok(()) - } - - #[cfg(test)]''' - pattern = re.compile( - r'pub fn download_ghcr_layer\([\s\S]*?\n\}\n\nfn download_blob\([\s\S]*?\n\}\n\n#\[cfg\(test\)\]', - re.M, - ) - s, n = pattern.subn(replacement, s, count=1) - assert n == 1, n - p.write_text(s) - - p = Path('src/source_fallback.rs') - s = p.read_text() - old = ''' let mut request = self - .client - .get(&url) - .header("Accept", "application/vnd.oci.image.manifest.v1+json"); - if let Some(token) = &self.config.github_token { - request = request.bearer_auth(token); - } - let Ok(response) = request.send() else { - return None; - }; - if !response.status().is_success() { - return None; - } - let Ok(manifest) = response.json::() else { - return None; - };''' - new = ''' let Ok(manifest) = crate::github_packages::ghcr_manifest_json( - &self.client, - self.config.github_token.as_deref(), - identity, - tag, - ) else { - return None; - };''' - assert old in s - s = s.replace(old, new, 1) - p.write_text(s) - PY - cargo fmt --all - cargo test --locked github_packages --lib - cargo test --locked source_fallback --lib - git diff --check - - name: Commit verified GHCR auth fix - shell: bash - run: | - set -Eeuo pipefail - git config user.name 'github-actions[bot]' - git config user.email '41898282+github-actions[bot]@users.noreply.github.com' - git add src/github_packages.rs src/source_fallback.rs - git commit -m 'fix: authenticate GHCR through registry tokens' - git push origin HEAD:DEN-3829-github-control-plane-fallback From 33a95e44aadb418e904c9007eb11952644327efb Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 14 Sep 2026 18:16:33 -0500 Subject: [PATCH 16/24] ci: remove completed one-shot GHCR salvage workflow --- .github/workflows/ghcr-auth-salvage-once.yml | 71 -------------------- 1 file changed, 71 deletions(-) delete mode 100644 .github/workflows/ghcr-auth-salvage-once.yml diff --git a/.github/workflows/ghcr-auth-salvage-once.yml b/.github/workflows/ghcr-auth-salvage-once.yml deleted file mode 100644 index 67dd8a856..000000000 --- a/.github/workflows/ghcr-auth-salvage-once.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: salvage proven GHCR auth once - -on: - push: - branches: - - DEN-3829-github-control-plane-fallback - paths: - - '.github/workflows/ghcr-auth-salvage-once.yml' - -permissions: - contents: write - -jobs: - salvage: - runs-on: ubuntu-24.04 - timeout-minutes: 30 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: DEN-3829-github-control-plane-fallback - fetch-depth: 0 - persist-credentials: true - - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 - with: - toolchain: '1.98.1' - components: rustfmt - - name: Salvage proven GHCR protocol changes without stale identity guessing - shell: bash - run: | - set -Eeuo pipefail - git show 3d498147d755f7824ef5c1c1d8072ec6bcedb53a -- src/github_packages.rs > /tmp/ghcr-auth.patch - git apply --3way /tmp/ghcr-auth.patch - python3 <<'PY' - from pathlib import Path - import re - - p = Path('src/source_fallback.rs') - s = p.read_text() - pattern = re.compile( - r''' let mut request = self\n \.client\n \.get\(&url\)\n \.header\("Accept", "application/vnd\.oci\.image\.manifest\.v1\+json"\);\n if let Some\(token\) = &self\.config\.github_token \{\n request = request\.bearer_auth\(token\);\n \}''' - ) - replacement = ''' let mut request = self - .client - .get(&url) - .header("Accept", "application/vnd.oci.image.manifest.v1+json"); - if let Some(token) = &self.config.github_token { - match crate::github_packages::ghcr_registry_token(&self.client, token, identity, "pull") - { - Ok(bearer) => request = request.bearer_auth(bearer), - Err(_) => return None, - } - }''' - s, n = pattern.subn(replacement, s, count=1) - assert n == 1, f'expected one ghcr_version auth block, found {n}' - p.write_text(s) - PY - cargo fmt --all - cargo test --locked github_packages --lib - cargo test --locked source_fallback --lib - git diff --check - grep -F '.basic_auth("x-access-token", Some(github_token))' src/github_packages.rs - ! grep -A8 'fn ghcr_version' src/source_fallback.rs | grep -F 'bearer_auth(token)' - - name: Commit verified salvage - shell: bash - run: | - set -Eeuo pipefail - git config user.name 'github-actions[bot]' - git config user.email '41898282+github-actions[bot]@users.noreply.github.com' - git add src/github_packages.rs src/source_fallback.rs - git commit -m 'fix: use proven GHCR registry token exchange' - git push origin HEAD:DEN-3829-github-control-plane-fallback From fe14b020eb050536cbb368983302d08f3c4ee7dd Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 14 Sep 2026 18:18:10 -0500 Subject: [PATCH 17/24] ci: migrate outage fallback to shared recovery authority --- .../workflows/shared-recovery-policy-once.yml | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 .github/workflows/shared-recovery-policy-once.yml diff --git a/.github/workflows/shared-recovery-policy-once.yml b/.github/workflows/shared-recovery-policy-once.yml new file mode 100644 index 000000000..3244728cc --- /dev/null +++ b/.github/workflows/shared-recovery-policy-once.yml @@ -0,0 +1,98 @@ +name: migrate to shared recovery policy once + +on: + push: + branches: + - DEN-3829-github-control-plane-fallback + paths: + - '.github/workflows/shared-recovery-policy-once.yml' + +permissions: + contents: write + +jobs: + migrate: + runs-on: ubuntu-24.04 + timeout-minutes: 40 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: DEN-3829-github-control-plane-fallback + fetch-depth: 0 + persist-credentials: true + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: '1.98.1' + components: rustfmt, clippy + - name: Replace local retry ordering with zed-interfaces authority + shell: bash + run: | + set -Eeuo pipefail + python3 <<'PY' + from pathlib import Path + + cargo = Path('Cargo.toml') + text = cargo.read_text() + old = 'zed-interfaces = { git = "https://github.com/zed-pkg/zed-interfaces.git", rev = "0c2ffa7be791a44c8aa2a69ab4b1ea87aab4729c" }' + new = 'zed-interfaces = { git = "https://github.com/zed-pkg/zed-interfaces.git", rev = "bb0f3fe10218e7dcb6a857bcb43deffdaa08b4f6" }' + if old not in text: + raise SystemExit('expected zed-interfaces pin not found') + cargo.write_text(text.replace(old, new, 1)) + + path = Path('src/source_fallback.rs') + text = path.read_text() + helper = '''/// During source fallback, prefer infrastructure owned by GitHub before +/// any Zed-operated R2/Cloudflare mirror. The normal registry path is +/// still attempted before this function is involved. +fn source_fallback_priority(kind: ArtifactSourceKind) -> u8 { + match kind { + ArtifactSourceKind::GithubRelease => 0, + ArtifactSourceKind::GithubPackages => 1, + ArtifactSourceKind::GithubArchive => 2, + ArtifactSourceKind::R2 => 3, + ArtifactSourceKind::Registry => 4, + } +} + +fn github_first_locators(mut locators: Vec) -> Vec { + locators.sort_by_key(|locator| source_fallback_priority(locator.kind)); + locators +} + +''' + if helper not in text: + raise SystemExit('local recovery helper block not found') + text = text.replace(helper, '', 1) + text = text.replace( + 'use zed_interfaces::artifact::ArtifactFormat;\n', + 'use zed_interfaces::artifact::ArtifactFormat;\nuse zed_interfaces::artifact_recovery_locators;\n', + 1, + ) + text = text.replace('github_first_locators(artifact_locators(&ArtifactQuery {', 'artifact_recovery_locators(&ArtifactQuery {') + text = text.replace(' }));', ' });') + # The replacement above intentionally changes exactly the two recovery-query closures; + # fail closed if the local wrapper survived or the shared helper was not wired twice. + if 'github_first_locators(' in text or 'source_fallback_priority(' in text: + raise SystemExit('local recovery ordering survived migration') + if text.count('artifact_recovery_locators(&ArtifactQuery {') != 2: + raise SystemExit(f'expected two shared recovery call sites, found {text.count("artifact_recovery_locators(&ArtifactQuery {")}') + path.write_text(text) + PY + cargo update -p zed-interfaces + cargo fmt --all + cargo test --locked --lib source_fallback + cargo check --locked --lib + cargo clippy --locked --lib -- -A unsafe-code -D warnings + git diff --check + grep -F 'rev = "bb0f3fe10218e7dcb6a857bcb43deffdaa08b4f6"' Cargo.toml + grep -F 'artifact_recovery_locators(&ArtifactQuery {' src/source_fallback.rs | test "$(wc -l)" -eq 2 + ! grep -F 'github_first_locators' src/source_fallback.rs + - name: Commit shared-authority migration + shell: bash + run: | + set -Eeuo pipefail + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git add Cargo.toml Cargo.lock src/source_fallback.rs + git commit -m 'refactor: consume shared GitHub-first recovery policy' + git push origin HEAD:DEN-3829-github-control-plane-fallback From eb04f76ac8049f646115aa01891fb727dd473a44 Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 14 Sep 2026 18:18:49 -0500 Subject: [PATCH 18/24] ci: fix shared recovery migration gate definition --- .../workflows/shared-recovery-policy-once.yml | 49 ++++++------------- 1 file changed, 16 insertions(+), 33 deletions(-) diff --git a/.github/workflows/shared-recovery-policy-once.yml b/.github/workflows/shared-recovery-policy-once.yml index 3244728cc..0e19896bb 100644 --- a/.github/workflows/shared-recovery-policy-once.yml +++ b/.github/workflows/shared-recovery-policy-once.yml @@ -30,6 +30,7 @@ jobs: set -Eeuo pipefail python3 <<'PY' from pathlib import Path + import re cargo = Path('Cargo.toml') text = cargo.read_text() @@ -41,41 +42,23 @@ jobs: path = Path('src/source_fallback.rs') text = path.read_text() - helper = '''/// During source fallback, prefer infrastructure owned by GitHub before -/// any Zed-operated R2/Cloudflare mirror. The normal registry path is -/// still attempted before this function is involved. -fn source_fallback_priority(kind: ArtifactSourceKind) -> u8 { - match kind { - ArtifactSourceKind::GithubRelease => 0, - ArtifactSourceKind::GithubPackages => 1, - ArtifactSourceKind::GithubArchive => 2, - ArtifactSourceKind::R2 => 3, - ArtifactSourceKind::Registry => 4, - } -} - -fn github_first_locators(mut locators: Vec) -> Vec { - locators.sort_by_key(|locator| source_fallback_priority(locator.kind)); - locators -} - -''' - if helper not in text: - raise SystemExit('local recovery helper block not found') - text = text.replace(helper, '', 1) - text = text.replace( - 'use zed_interfaces::artifact::ArtifactFormat;\n', - 'use zed_interfaces::artifact::ArtifactFormat;\nuse zed_interfaces::artifact_recovery_locators;\n', - 1, - ) - text = text.replace('github_first_locators(artifact_locators(&ArtifactQuery {', 'artifact_recovery_locators(&ArtifactQuery {') - text = text.replace(' }));', ' });') - # The replacement above intentionally changes exactly the two recovery-query closures; - # fail closed if the local wrapper survived or the shared helper was not wired twice. + helper_pattern = re.compile(r'/// During source fallback, prefer infrastructure owned by GitHub before\n/// any Zed-operated R2/Cloudflare mirror\. The normal registry path is\n/// still attempted before this function is involved\.\nfn source_fallback_priority\(kind: ArtifactSourceKind\) -> u8 \{.*?\n\}\n\nfn github_first_locators\(mut locators: Vec\) -> Vec \{.*?\n\}\n\n', re.S) + text, removed = helper_pattern.subn('', text, count=1) + if removed != 1: + raise SystemExit('expected exactly one local recovery helper block') + text = text.replace('use zed_interfaces::artifact::ArtifactFormat;\n', 'use zed_interfaces::artifact::ArtifactFormat;\nuse zed_interfaces::artifact_recovery_locators;\n', 1) + call_pattern = re.compile(r'github_first_locators\(artifact_locators\(&ArtifactQuery \{(.*?)\n \}\)\)', re.S) + text, calls = call_pattern.subn(r'artifact_recovery_locators(&ArtifactQuery {\1\n })', text) + if calls != 2: + raise SystemExit('expected exactly two local recovery wrapper call sites, found %d' % calls) + if text.count('artifact_locators(') == 0: + text = text.replace(' artifact_locators, ', ' ') + if text.count('ArtifactLocator') == 1: + text = text.replace(' ArtifactLocator, ', ' ') if 'github_first_locators(' in text or 'source_fallback_priority(' in text: raise SystemExit('local recovery ordering survived migration') if text.count('artifact_recovery_locators(&ArtifactQuery {') != 2: - raise SystemExit(f'expected two shared recovery call sites, found {text.count("artifact_recovery_locators(&ArtifactQuery {")}') + raise SystemExit('shared recovery helper was not wired at both call sites') path.write_text(text) PY cargo update -p zed-interfaces @@ -85,7 +68,7 @@ fn github_first_locators(mut locators: Vec) -> Vec Date: Mon, 14 Sep 2026 18:21:24 -0500 Subject: [PATCH 19/24] ci: consolidate shared recovery policy and GHCR upload URL fix --- .../outage-recovery-consolidate-once.yml | 162 ++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 .github/workflows/outage-recovery-consolidate-once.yml diff --git a/.github/workflows/outage-recovery-consolidate-once.yml b/.github/workflows/outage-recovery-consolidate-once.yml new file mode 100644 index 000000000..02d83ecee --- /dev/null +++ b/.github/workflows/outage-recovery-consolidate-once.yml @@ -0,0 +1,162 @@ +name: consolidate outage recovery once + +on: + push: + branches: + - DEN-3829-github-control-plane-fallback + paths: + - '.github/workflows/outage-recovery-consolidate-once.yml' + +permissions: + contents: write + +jobs: + consolidate: + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: DEN-3829-github-control-plane-fallback + fetch-depth: 0 + persist-credentials: true + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: '1.98.1' + components: rustfmt, clippy + - name: Apply shared recovery and GHCR relative-Location fixes + shell: bash + run: | + set -Eeuo pipefail + python3 <<'PY' + from pathlib import Path + import re + + cargo = Path('Cargo.toml') + text = cargo.read_text() + old = 'zed-interfaces = { git = "https://github.com/zed-pkg/zed-interfaces.git", rev = "0c2ffa7be791a44c8aa2a69ab4b1ea87aab4729c" }' + new = 'zed-interfaces = { git = "https://github.com/zed-pkg/zed-interfaces.git", rev = "bb0f3fe10218e7dcb6a857bcb43deffdaa08b4f6" }' + if old not in text: + raise SystemExit('expected old zed-interfaces pin not found') + cargo.write_text(text.replace(old, new, 1)) + + path = Path('src/source_fallback.rs') + text = path.read_text() + helper_pattern = re.compile( + r'/// During source fallback, prefer infrastructure owned by GitHub before\n' + r'/// any Zed-operated R2/Cloudflare mirror\. The normal registry path is\n' + r'/// still attempted before this function is involved\.\n' + r'fn source_fallback_priority\(kind: ArtifactSourceKind\) -> u8 \{.*?\n\}\n\n' + r'fn github_first_locators\(mut locators: Vec\) -> Vec \{.*?\n\}\n\n', + re.S, + ) + text, removed = helper_pattern.subn('', text, count=1) + if removed != 1: + raise SystemExit('expected exactly one local recovery helper block') + text = text.replace( + 'use zed_interfaces::artifact::ArtifactFormat;\n', + 'use zed_interfaces::artifact::ArtifactFormat;\nuse zed_interfaces::artifact_recovery_locators;\n', + 1, + ) + text = text.replace(' artifact_locators, ', ' ', 1) + text = text.replace( + 'github_first_locators(artifact_locators(&query))', + 'artifact_recovery_locators(&query)', + 1, + ) + direct = re.compile( + r'github_first_locators\(artifact_locators\(&ArtifactQuery \{(.*?)\n \}\)\)', + re.S, + ) + text, calls = direct.subn(r'artifact_recovery_locators(&ArtifactQuery {\1\n })', text) + if calls != 2: + raise SystemExit('expected two direct recovery query call sites, found %d' % calls) + if 'github_first_locators(' in text or 'source_fallback_priority(' in text or 'artifact_locators(' in text: + raise SystemExit('local recovery ordering survived migration') + if text.count('artifact_recovery_locators(') != 3: + raise SystemExit('expected exactly three shared recovery call sites') + path.write_text(text) + + path = Path('src/github_packages.rs') + text = path.read_text() + old = ''' let start = client + .post(ghcr_uploads_url(identity)) + .bearer_auth(token)''' + new = ''' let upload_url = ghcr_uploads_url(identity); + let start = client + .post(&upload_url) + .bearer_auth(token)''' + if old not in text: + raise SystemExit('GHCR upload-start block not found') + text = text.replace(old, new, 1) + old = ''' let put_url = with_digest_query(&location, &blob.digest);''' + new = ''' let location = resolve_upload_location(&upload_url, &location)?; + let put_url = with_digest_query(&location, &blob.digest);''' + if old not in text: + raise SystemExit('GHCR upload Location use not found') + text = text.replace(old, new, 1) + anchor = '''fn with_digest_query(location: &str, digest: &str) -> String { + let separator = if location.contains('?') { '&' } else { '?' }; + format!("{location}{separator}digest={digest}") +} +''' + helper = '''fn resolve_upload_location(base_url: &str, location: &str) -> Result { + if let Ok(url) = reqwest::Url::parse(location) { + return Ok(url.to_string()); + } + let base = reqwest::Url::parse(base_url) + .with_context(|| format!("parse GHCR upload base URL {base_url}"))?; + Ok(base + .join(location) + .with_context(|| format!("resolve GHCR upload Location {location:?} against {base_url}"))? + .to_string()) +} + +''' + if anchor not in text: + raise SystemExit('with_digest_query anchor not found') + text = text.replace(anchor, helper + anchor, 1) + test_anchor = ''' #[test] + fn upload_session_appends_digest_query() {''' + tests = ''' #[test] + fn upload_session_resolves_relative_location_against_registry_origin() { + let base = "https://ghcr.io/v2/acme/http-kit/blobs/uploads/"; + assert_eq!( + resolve_upload_location(base, "/v2/acme/http-kit/blobs/uploads/abc?_state=1").unwrap(), + "https://ghcr.io/v2/acme/http-kit/blobs/uploads/abc?_state=1" + ); + assert_eq!( + resolve_upload_location(base, "abc?_state=1").unwrap(), + "https://ghcr.io/v2/acme/http-kit/blobs/uploads/abc?_state=1" + ); + assert_eq!( + resolve_upload_location(base, "https://ghcr.io/v2/acme/http-kit/blobs/uploads/xyz").unwrap(), + "https://ghcr.io/v2/acme/http-kit/blobs/uploads/xyz" + ); + } + +''' + if test_anchor not in text: + raise SystemExit('GHCR test anchor not found') + text = text.replace(test_anchor, tests + test_anchor, 1) + path.write_text(text) + PY + cargo update -p zed-interfaces + cargo fmt --all + cargo test --locked --lib github_packages + cargo test --locked --lib source_fallback + cargo check --locked --lib + cargo clippy --locked --lib -- -A unsafe-code -D warnings + git diff --check + test "$(grep -Fc 'artifact_recovery_locators(' src/source_fallback.rs)" -eq 3 + ! grep -F 'github_first_locators' src/source_fallback.rs + grep -F 'resolve_upload_location(&upload_url, &location)?' src/github_packages.rs + - name: Commit verified consolidation + shell: bash + run: | + set -Eeuo pipefail + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git add Cargo.toml Cargo.lock src/source_fallback.rs src/github_packages.rs + git commit -m 'fix: centralize outage recovery and resolve GHCR upload locations' + git push origin HEAD:DEN-3829-github-control-plane-fallback From 3994afa26c916515b621cf6d5657120f09a1524b Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 14 Sep 2026 18:23:28 -0500 Subject: [PATCH 20/24] fix: resolve relative GHCR upload locations --- src/github_packages.rs | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/src/github_packages.rs b/src/github_packages.rs index e34fbe426..a7d422280 100644 --- a/src/github_packages.rs +++ b/src/github_packages.rs @@ -322,8 +322,9 @@ fn upload_blob( if head.status().is_success() { return Ok(()); } + let upload_url = ghcr_uploads_url(identity); let start = client - .post(ghcr_uploads_url(identity)) + .post(&upload_url) .bearer_auth(token) .send() .with_context(|| format!("start GHCR blob upload {}", blob.digest))?; @@ -340,6 +341,7 @@ fn upload_blob( .and_then(|value| value.to_str().ok()) .map(str::to_string) .context("GHCR upload session missing Location")?; + let location = resolve_upload_location(&upload_url, &location)?; let put_url = with_digest_query(&location, &blob.digest); let put = client .put(&put_url) @@ -378,6 +380,18 @@ fn put_manifest( ) } +fn resolve_upload_location(base_url: &str, location: &str) -> Result { + if let Ok(url) = reqwest::Url::parse(location) { + return Ok(url.to_string()); + } + let base = reqwest::Url::parse(base_url) + .with_context(|| format!("parse GHCR upload base URL {base_url}"))?; + Ok(base + .join(location) + .with_context(|| format!("resolve GHCR upload Location {location:?} against {base_url}"))? + .to_string()) +} + fn with_digest_query(location: &str, digest: &str) -> String { let separator = if location.contains('?') { '&' } else { '?' }; format!("{location}{separator}digest={digest}") @@ -516,6 +530,24 @@ mod tests { ); } + #[test] + fn upload_session_resolves_relative_location_against_registry_origin() { + let base = "https://ghcr.io/v2/acme/http-kit/blobs/uploads/"; + assert_eq!( + resolve_upload_location(base, "/v2/acme/http-kit/blobs/uploads/abc?_state=1").unwrap(), + "https://ghcr.io/v2/acme/http-kit/blobs/uploads/abc?_state=1" + ); + assert_eq!( + resolve_upload_location(base, "abc?_state=1").unwrap(), + "https://ghcr.io/v2/acme/http-kit/blobs/uploads/abc?_state=1" + ); + assert_eq!( + resolve_upload_location(base, "https://ghcr.io/v2/acme/http-kit/blobs/uploads/xyz") + .unwrap(), + "https://ghcr.io/v2/acme/http-kit/blobs/uploads/xyz" + ); + } + #[test] fn upload_session_appends_digest_query() { assert_eq!( From 8d35294102c0198e9c5ac93fb58027e57ab33406 Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 14 Sep 2026 18:23:52 -0500 Subject: [PATCH 21/24] ci: remove failed one-shot shared recovery workflow --- .../workflows/shared-recovery-policy-once.yml | 81 ------------------- 1 file changed, 81 deletions(-) delete mode 100644 .github/workflows/shared-recovery-policy-once.yml diff --git a/.github/workflows/shared-recovery-policy-once.yml b/.github/workflows/shared-recovery-policy-once.yml deleted file mode 100644 index 0e19896bb..000000000 --- a/.github/workflows/shared-recovery-policy-once.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: migrate to shared recovery policy once - -on: - push: - branches: - - DEN-3829-github-control-plane-fallback - paths: - - '.github/workflows/shared-recovery-policy-once.yml' - -permissions: - contents: write - -jobs: - migrate: - runs-on: ubuntu-24.04 - timeout-minutes: 40 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: DEN-3829-github-control-plane-fallback - fetch-depth: 0 - persist-credentials: true - - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 - with: - toolchain: '1.98.1' - components: rustfmt, clippy - - name: Replace local retry ordering with zed-interfaces authority - shell: bash - run: | - set -Eeuo pipefail - python3 <<'PY' - from pathlib import Path - import re - - cargo = Path('Cargo.toml') - text = cargo.read_text() - old = 'zed-interfaces = { git = "https://github.com/zed-pkg/zed-interfaces.git", rev = "0c2ffa7be791a44c8aa2a69ab4b1ea87aab4729c" }' - new = 'zed-interfaces = { git = "https://github.com/zed-pkg/zed-interfaces.git", rev = "bb0f3fe10218e7dcb6a857bcb43deffdaa08b4f6" }' - if old not in text: - raise SystemExit('expected zed-interfaces pin not found') - cargo.write_text(text.replace(old, new, 1)) - - path = Path('src/source_fallback.rs') - text = path.read_text() - helper_pattern = re.compile(r'/// During source fallback, prefer infrastructure owned by GitHub before\n/// any Zed-operated R2/Cloudflare mirror\. The normal registry path is\n/// still attempted before this function is involved\.\nfn source_fallback_priority\(kind: ArtifactSourceKind\) -> u8 \{.*?\n\}\n\nfn github_first_locators\(mut locators: Vec\) -> Vec \{.*?\n\}\n\n', re.S) - text, removed = helper_pattern.subn('', text, count=1) - if removed != 1: - raise SystemExit('expected exactly one local recovery helper block') - text = text.replace('use zed_interfaces::artifact::ArtifactFormat;\n', 'use zed_interfaces::artifact::ArtifactFormat;\nuse zed_interfaces::artifact_recovery_locators;\n', 1) - call_pattern = re.compile(r'github_first_locators\(artifact_locators\(&ArtifactQuery \{(.*?)\n \}\)\)', re.S) - text, calls = call_pattern.subn(r'artifact_recovery_locators(&ArtifactQuery {\1\n })', text) - if calls != 2: - raise SystemExit('expected exactly two local recovery wrapper call sites, found %d' % calls) - if text.count('artifact_locators(') == 0: - text = text.replace(' artifact_locators, ', ' ') - if text.count('ArtifactLocator') == 1: - text = text.replace(' ArtifactLocator, ', ' ') - if 'github_first_locators(' in text or 'source_fallback_priority(' in text: - raise SystemExit('local recovery ordering survived migration') - if text.count('artifact_recovery_locators(&ArtifactQuery {') != 2: - raise SystemExit('shared recovery helper was not wired at both call sites') - path.write_text(text) - PY - cargo update -p zed-interfaces - cargo fmt --all - cargo test --locked --lib source_fallback - cargo check --locked --lib - cargo clippy --locked --lib -- -A unsafe-code -D warnings - git diff --check - grep -F 'rev = "bb0f3fe10218e7dcb6a857bcb43deffdaa08b4f6"' Cargo.toml - test "$(grep -Fc 'artifact_recovery_locators(&ArtifactQuery {' src/source_fallback.rs)" -eq 2 - ! grep -F 'github_first_locators' src/source_fallback.rs - - name: Commit shared-authority migration - shell: bash - run: | - set -Eeuo pipefail - git config user.name 'github-actions[bot]' - git config user.email '41898282+github-actions[bot]@users.noreply.github.com' - git add Cargo.toml Cargo.lock src/source_fallback.rs - git commit -m 'refactor: consume shared GitHub-first recovery policy' - git push origin HEAD:DEN-3829-github-control-plane-fallback From 7af44a25a0316c57bea89dd8c8da5a782d09a7f3 Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 14 Sep 2026 18:23:59 -0500 Subject: [PATCH 22/24] ci: remove failed outage consolidation workflow --- .../outage-recovery-consolidate-once.yml | 162 ------------------ 1 file changed, 162 deletions(-) delete mode 100644 .github/workflows/outage-recovery-consolidate-once.yml diff --git a/.github/workflows/outage-recovery-consolidate-once.yml b/.github/workflows/outage-recovery-consolidate-once.yml deleted file mode 100644 index 02d83ecee..000000000 --- a/.github/workflows/outage-recovery-consolidate-once.yml +++ /dev/null @@ -1,162 +0,0 @@ -name: consolidate outage recovery once - -on: - push: - branches: - - DEN-3829-github-control-plane-fallback - paths: - - '.github/workflows/outage-recovery-consolidate-once.yml' - -permissions: - contents: write - -jobs: - consolidate: - runs-on: ubuntu-24.04 - timeout-minutes: 45 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: DEN-3829-github-control-plane-fallback - fetch-depth: 0 - persist-credentials: true - - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 - with: - toolchain: '1.98.1' - components: rustfmt, clippy - - name: Apply shared recovery and GHCR relative-Location fixes - shell: bash - run: | - set -Eeuo pipefail - python3 <<'PY' - from pathlib import Path - import re - - cargo = Path('Cargo.toml') - text = cargo.read_text() - old = 'zed-interfaces = { git = "https://github.com/zed-pkg/zed-interfaces.git", rev = "0c2ffa7be791a44c8aa2a69ab4b1ea87aab4729c" }' - new = 'zed-interfaces = { git = "https://github.com/zed-pkg/zed-interfaces.git", rev = "bb0f3fe10218e7dcb6a857bcb43deffdaa08b4f6" }' - if old not in text: - raise SystemExit('expected old zed-interfaces pin not found') - cargo.write_text(text.replace(old, new, 1)) - - path = Path('src/source_fallback.rs') - text = path.read_text() - helper_pattern = re.compile( - r'/// During source fallback, prefer infrastructure owned by GitHub before\n' - r'/// any Zed-operated R2/Cloudflare mirror\. The normal registry path is\n' - r'/// still attempted before this function is involved\.\n' - r'fn source_fallback_priority\(kind: ArtifactSourceKind\) -> u8 \{.*?\n\}\n\n' - r'fn github_first_locators\(mut locators: Vec\) -> Vec \{.*?\n\}\n\n', - re.S, - ) - text, removed = helper_pattern.subn('', text, count=1) - if removed != 1: - raise SystemExit('expected exactly one local recovery helper block') - text = text.replace( - 'use zed_interfaces::artifact::ArtifactFormat;\n', - 'use zed_interfaces::artifact::ArtifactFormat;\nuse zed_interfaces::artifact_recovery_locators;\n', - 1, - ) - text = text.replace(' artifact_locators, ', ' ', 1) - text = text.replace( - 'github_first_locators(artifact_locators(&query))', - 'artifact_recovery_locators(&query)', - 1, - ) - direct = re.compile( - r'github_first_locators\(artifact_locators\(&ArtifactQuery \{(.*?)\n \}\)\)', - re.S, - ) - text, calls = direct.subn(r'artifact_recovery_locators(&ArtifactQuery {\1\n })', text) - if calls != 2: - raise SystemExit('expected two direct recovery query call sites, found %d' % calls) - if 'github_first_locators(' in text or 'source_fallback_priority(' in text or 'artifact_locators(' in text: - raise SystemExit('local recovery ordering survived migration') - if text.count('artifact_recovery_locators(') != 3: - raise SystemExit('expected exactly three shared recovery call sites') - path.write_text(text) - - path = Path('src/github_packages.rs') - text = path.read_text() - old = ''' let start = client - .post(ghcr_uploads_url(identity)) - .bearer_auth(token)''' - new = ''' let upload_url = ghcr_uploads_url(identity); - let start = client - .post(&upload_url) - .bearer_auth(token)''' - if old not in text: - raise SystemExit('GHCR upload-start block not found') - text = text.replace(old, new, 1) - old = ''' let put_url = with_digest_query(&location, &blob.digest);''' - new = ''' let location = resolve_upload_location(&upload_url, &location)?; - let put_url = with_digest_query(&location, &blob.digest);''' - if old not in text: - raise SystemExit('GHCR upload Location use not found') - text = text.replace(old, new, 1) - anchor = '''fn with_digest_query(location: &str, digest: &str) -> String { - let separator = if location.contains('?') { '&' } else { '?' }; - format!("{location}{separator}digest={digest}") -} -''' - helper = '''fn resolve_upload_location(base_url: &str, location: &str) -> Result { - if let Ok(url) = reqwest::Url::parse(location) { - return Ok(url.to_string()); - } - let base = reqwest::Url::parse(base_url) - .with_context(|| format!("parse GHCR upload base URL {base_url}"))?; - Ok(base - .join(location) - .with_context(|| format!("resolve GHCR upload Location {location:?} against {base_url}"))? - .to_string()) -} - -''' - if anchor not in text: - raise SystemExit('with_digest_query anchor not found') - text = text.replace(anchor, helper + anchor, 1) - test_anchor = ''' #[test] - fn upload_session_appends_digest_query() {''' - tests = ''' #[test] - fn upload_session_resolves_relative_location_against_registry_origin() { - let base = "https://ghcr.io/v2/acme/http-kit/blobs/uploads/"; - assert_eq!( - resolve_upload_location(base, "/v2/acme/http-kit/blobs/uploads/abc?_state=1").unwrap(), - "https://ghcr.io/v2/acme/http-kit/blobs/uploads/abc?_state=1" - ); - assert_eq!( - resolve_upload_location(base, "abc?_state=1").unwrap(), - "https://ghcr.io/v2/acme/http-kit/blobs/uploads/abc?_state=1" - ); - assert_eq!( - resolve_upload_location(base, "https://ghcr.io/v2/acme/http-kit/blobs/uploads/xyz").unwrap(), - "https://ghcr.io/v2/acme/http-kit/blobs/uploads/xyz" - ); - } - -''' - if test_anchor not in text: - raise SystemExit('GHCR test anchor not found') - text = text.replace(test_anchor, tests + test_anchor, 1) - path.write_text(text) - PY - cargo update -p zed-interfaces - cargo fmt --all - cargo test --locked --lib github_packages - cargo test --locked --lib source_fallback - cargo check --locked --lib - cargo clippy --locked --lib -- -A unsafe-code -D warnings - git diff --check - test "$(grep -Fc 'artifact_recovery_locators(' src/source_fallback.rs)" -eq 3 - ! grep -F 'github_first_locators' src/source_fallback.rs - grep -F 'resolve_upload_location(&upload_url, &location)?' src/github_packages.rs - - name: Commit verified consolidation - shell: bash - run: | - set -Eeuo pipefail - git config user.name 'github-actions[bot]' - git config user.email '41898282+github-actions[bot]@users.noreply.github.com' - git add Cargo.toml Cargo.lock src/source_fallback.rs src/github_packages.rs - git commit -m 'fix: centralize outage recovery and resolve GHCR upload locations' - git push origin HEAD:DEN-3829-github-control-plane-fallback From 898293c2ef5cc938012680acbd1cb8726f7f98f1 Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 14 Sep 2026 18:35:58 -0500 Subject: [PATCH 23/24] ci: serialize GitHub outage fallback runs --- .github/workflows/github-control-plane-outage.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/github-control-plane-outage.yml b/.github/workflows/github-control-plane-outage.yml index 95c40b4fa..bb73a94d4 100644 --- a/.github/workflows/github-control-plane-outage.yml +++ b/.github/workflows/github-control-plane-outage.yml @@ -16,6 +16,10 @@ on: permissions: contents: read +concurrency: + group: github-control-plane-outage-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + defaults: run: shell: bash From 7944b5cd58a7597e7e07b7f312601f1804e7cb23 Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 14 Sep 2026 18:38:02 -0500 Subject: [PATCH 24/24] fix: use canonical flags2env integer type --- .cli-flags.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cli-flags.toml b/.cli-flags.toml index ca4db4e34..740dabc1e 100644 --- a/.cli-flags.toml +++ b/.cli-flags.toml @@ -524,7 +524,7 @@ help = "Print the installed dependency graph as a tree." [commands.tree.flags.tree_depth] env = "ZED_PKG_TREE_DEPTH" aliases = ["depth"] -type = "int" +type = "integer" help = "Levels below the root to print; omit for the whole graph." [commands.tree.flags.tree_json]