From a9838439aa391fe009bd2a979ddf8a1eaedd9fe0 Mon Sep 17 00:00:00 2001 From: AlexReal Date: Sat, 12 Sep 2026 17:13:49 -0500 Subject: [PATCH 1/3] fix(fallback): install from GitHub tag archives when zpkg.net is down The registry-down fallback could not install a package that has a git tag but no published zpkg Release asset (zed-pkg/zed-cli#378): - download_locators skipped the GitHub tag archive whenever a digest was known, including the digest the fallback had just derived from that archive, so every install failed after metadata resolved. - A raw `git archive` tarball has a `-/` root and a pax_global_header entry, which the store extractor rejects. It is now repacked into the deterministic `pkg/` artifact layout (sorted entries, zeroed headers), so the derived digest names the tree, not GitHub's compressor. Links and specials still fail closed. - Fallback metadata turned full R2 object URLs into object-store mirror bases, producing doubled URLs such as `.tar.gz/artifacts/`. R2 locators now collapse to the public base and registry locators are dropped. Also drops the duplicate `registry_for` import that stopped the lib test target from compiling on main. Verified live against the 2026-09-12 registry.zpkg.net 503 outage: ores-wasm-loaders/owls-interfaces@0.1.1 now downloads and unpacks from GitHub. cargo test --lib: 604 passed. Co-Authored-By: Claude Opus 5 (1M context) --- src/install_graph/hardening_tests.rs | 1 - src/source_fallback.rs | 357 +++++++++++++++++++++++++-- 2 files changed, 334 insertions(+), 24 deletions(-) diff --git a/src/install_graph/hardening_tests.rs b/src/install_graph/hardening_tests.rs index 1cce2fc22..ccfe6d52f 100644 --- a/src/install_graph/hardening_tests.rs +++ b/src/install_graph/hardening_tests.rs @@ -156,4 +156,3 @@ fn sequenced_receive_buffers_later_failures_until_earlier_results_arrive() { .to_string(); assert!(error.contains("later lockfile entry failed"), "{error}"); } -use crate::registry::registry_for; diff --git a/src/source_fallback.rs b/src/source_fallback.rs index b845f1f43..5b3327459 100644 --- a/src/source_fallback.rs +++ b/src/source_fallback.rs @@ -8,25 +8,29 @@ use std::cell::RefCell; use std::collections::HashMap; use std::fs; -use std::io::Read as _; -use std::path::Path; +use std::io::{Read as _, Write as _}; +use std::path::{Component, Path}; use std::sync::Mutex; use std::time::Duration; use anyhow::{Context, Result, anyhow, bail}; +use flate2::Compression; +use flate2::read::GzDecoder; +use flate2::write::GzEncoder; use serde::Deserialize; use zed_interfaces::artifact::ArtifactFormat; use zed_interfaces::manifest::{Manifest, is_slug}; -use zed_interfaces::paths::MANIFEST_FILE; +use zed_interfaces::mirror::MirrorDescriptorV1; +use zed_interfaces::paths::{ARCHIVE_ROOT, MANIFEST_FILE}; use zed_interfaces::registry::{ AuditLogResponse, ClaimOrgResponse, PackageMetadata, PublishMeta, PublishResponse, SearchResponse, VersionMetadata, YankResponse, }; use zed_interfaces::source::{ - ArtifactQuery, ArtifactSourceKind, ArtifactsSection, GithubIdentity, artifact_locators, - github_api_release_url, github_api_repo_url, github_api_tags_url, github_raw_manifest_url, - github_release_asset_names, github_release_sidecar_names, parse_github_identity, - version_from_git_tag, + ArtifactLocator, ArtifactQuery, ArtifactSourceKind, ArtifactsSection, GithubIdentity, + artifact_locators, github_api_release_url, github_api_repo_url, github_api_tags_url, + github_raw_manifest_url, github_release_asset_names, github_release_sidecar_names, + parse_github_identity, resolve_r2_public_base, version_from_git_tag, }; use zed_interfaces::vcs::Vcs; @@ -140,6 +144,39 @@ impl FallbackRegistry { } } + fn r2_base(&self) -> String { + resolve_r2_public_base( + None, + self.config.r2_public_base.as_deref(), + self.config.r2_public_key.as_deref(), + ) + } + + /// Fetch a GitHub tag archive and repack it as a Zed artifact. The raw + /// archive's size is unrelated to the packed artifact's, so only the + /// global cap bounds the download; the caller verifies the packed digest. + fn download_tag_archive( + &self, + identity: &GithubIdentity, + url: &str, + version: &VersionMetadata, + dest: &Path, + ) -> Result<()> { + let raw = tempfile::NamedTempFile::new().context("tag archive tempfile")?; + match self.config.github_token.as_deref() { + // github.com///archive/... is anonymous-only, so a + // private repository 404s there even with a token. The REST + // tarball endpoint honours the token and redirects to a signed + // codeload URL for the same tag. + Some(token) => { + let api_url = github_api_tarball_url(identity, &version.vcs_tag); + download_url(&self.client, &api_url, raw.path(), 0, Some(token))?; + } + None => download_url(&self.client, url, raw.path(), 0, None)?, + } + normalize_tag_archive(raw.path(), dest) + } + fn github_headers( &self, request: reqwest::blocking::RequestBuilder, @@ -413,10 +450,7 @@ impl FallbackRegistry { download_url, published_at: "1970-01-01T00:00:00Z".to_string(), yanked: false, - mirrors: locators - .iter() - .map(zed_interfaces::mirror::MirrorDescriptorV1::from_locator) - .collect(), + mirrors: fallback_mirrors(&locators, &self.r2_base()), signatures: Vec::new(), }; self.fill_digest(&mut metadata)?; @@ -603,10 +637,12 @@ impl FallbackRegistry { r2_public_base: self.config.r2_public_base.as_deref(), r2_public_key: self.config.r2_public_key.as_deref(), }); + // The GitHub tag archive stays in the chain even when a digest is + // pinned. It is the last locator and is repacked deterministically, so + // a digest derived from it is reproducible, and every caller verifies + // the bytes against the pin. Skipping it broke installs whose digest + // had been derived from that same archive in another registry instance. for locator in locators { - if locator.kind == ArtifactSourceKind::GithubArchive && packed_digest { - continue; - } if locator.kind == ArtifactSourceKind::Registry { continue; } @@ -619,15 +655,8 @@ impl FallbackRegistry { version.size, max_artifact_bytes(), ) - } else if locator.kind == ArtifactSourceKind::GithubArchive - && let Some(token) = self.config.github_token.as_deref() - { - // github.com///archive/... is anonymous-only, so a - // private repository 404s there even with a token. The REST - // tarball endpoint honours the token and redirects to a signed - // codeload URL for the same tag. - let api_url = github_api_tarball_url(&identity, &version.vcs_tag); - download_url(&self.client, &api_url, dest, version.size, Some(token)) + } else if locator.kind == ArtifactSourceKind::GithubArchive { + self.download_tag_archive(&identity, &locator.url, version, dest) } else { download_url(&self.client, &locator.url, dest, version.size, None) }; @@ -837,6 +866,130 @@ struct GithubTag { name: String, } +/// Ceiling on files taken from one GitHub tag archive (inode-exhaustion guard, +/// matching the store's extraction bound). +const MAX_TAG_ARCHIVE_FILES: usize = 200_000; + +/// Repack a GitHub tag archive into the deterministic Zed artifact layout. +/// +/// `git archive` output has a `-/` root, a `pax_global_header` +/// entry, and gzip bytes GitHub does not promise to keep stable; the store +/// accepts none of that. The top-level directory is replaced by `pkg/`, +/// metadata-only entries are dropped, and files are written sorted with the +/// zeroed headers `pack` uses — so the digest names the tree, not GitHub's +/// compressor, and every machine derives the same one. Links and specials +/// fail closed, exactly as the store's extractor would. +fn normalize_tag_archive(raw: &Path, dest: &Path) -> Result<()> { + let budget = max_artifact_bytes(); + let mut total: u64 = 0; + let mut files: Vec<(String, u32, Vec)> = Vec::new(); + let file = fs::File::open(raw).context("open GitHub tag archive")?; + let mut archive = tar::Archive::new(GzDecoder::new(file)); + for entry in archive.entries().context("read GitHub tag archive")? { + let mut entry = entry?; + let path = entry.path()?.into_owned(); + match entry.header().entry_type() { + tar::EntryType::Regular => {} + tar::EntryType::Directory | tar::EntryType::XGlobalHeader | tar::EntryType::XHeader => { + continue; + } + other => bail!( + "GitHub tag archive entry `{}` has unsupported type {other:?} \ + (only files and directories are allowed)", + path.display() + ), + } + let mut components = path.components(); + if !matches!(components.next(), Some(Component::Normal(_))) { + bail!( + "GitHub tag archive entry `{}` has no root directory", + path.display() + ); + } + let rel = components.as_path(); + if rel.as_os_str().is_empty() + || !rel.components().all(|c| matches!(c, Component::Normal(_))) + { + bail!( + "GitHub tag archive entry `{}` escapes the archive root", + path.display() + ); + } + if files.len() >= MAX_TAG_ARCHIVE_FILES { + bail!("GitHub tag archive has more than {MAX_TAG_ARCHIVE_FILES} files; refusing"); + } + let size = entry.header().size()?; + total = total.saturating_add(size); + if total > budget { + bail!("GitHub tag archive expands past the {budget}-byte cap; refusing"); + } + let mode = if entry.header().mode()? & 0o111 != 0 { + 0o755 + } else { + 0o644 + }; + let mut data = Vec::new(); + (&mut entry).take(size).read_to_end(&mut data)?; + let rel = rel + .components() + .map(|c| c.as_os_str().to_string_lossy()) + .collect::>() + .join("/"); + files.push((rel, mode, data)); + } + if files.is_empty() { + bail!("GitHub tag archive contains no files"); + } + files.sort_by(|left, right| left.0.cmp(&right.0)); + if let Some(pair) = files.windows(2).find(|pair| pair[0].0 == pair[1].0) { + bail!("GitHub tag archive repeats entry `{}`", pair[0].0); + } + + let out = fs::File::create(dest)?; + let mut builder = tar::Builder::new(GzEncoder::new(out, Compression::default())); + for (rel, mode, data) in &files { + let mut header = tar::Header::new_gnu(); + header.set_size(data.len() as u64); + header.set_mtime(0); + header.set_uid(0); + header.set_gid(0); + header.set_mode(*mode); + builder.append_data( + &mut header, + format!("{ARCHIVE_ROOT}/{rel}"), + data.as_slice(), + )?; + } + let mut out = builder.into_inner()?.finish()?; + out.flush()?; + Ok(()) +} + +/// Mirrors recorded on fallback-derived version metadata. An R2 locator is a +/// complete object URL, but an object-store mirror is a *base* that +/// `artifact_urls` appends keys to, so every R2 locator collapses to the one +/// public base. Registry locators are dropped: the registry already heads the +/// mirror chain. +fn fallback_mirrors(locators: &[ArtifactLocator], r2_base: &str) -> Vec { + let mut mirrors: Vec = Vec::new(); + for locator in locators { + let mirror = match locator.kind { + ArtifactSourceKind::Registry => continue, + ArtifactSourceKind::R2 => { + MirrorDescriptorV1::object_store(r2_base.trim_end_matches('/')) + } + _ => MirrorDescriptorV1::from_locator(locator), + }; + if !mirrors + .iter() + .any(|seen| seen.kind == mirror.kind && seen.url == mirror.url) + { + mirrors.push(mirror); + } + } + mirrors +} + fn versions_from_tags(tags: &[GithubTag]) -> Vec { let mut versions: Vec = tags .iter() @@ -1199,4 +1352,162 @@ url = "https://github.com/ores-otel/ores-otel-sidecar.rs" .any(|key| key == "github/zed-pkg/zed-cli/v0.1.0/zed-cli-0.1.0.tar.gz") ); } + + #[test] + fn pinned_downloads_try_the_tag_archive_last() { + let sha256 = "ab".repeat(32); + let query = ArtifactQuery { + org: "ores-wasm-loaders", + name: "owls-interfaces", + version: "0.1.1", + vcs_tag: "v0.1.1", + sha256: Some(&sha256), + format: ArtifactFormat::TarGz, + repo_url: Some("https://github.com/ores-wasm-loaders/owls-interfaces"), + artifacts: Some(&ArtifactsSection::EMPTY), + registry_base: None, + r2_public_base: Some("https://cdn.zpkg.net"), + r2_public_key: None, + }; + let locators = artifact_locators(&query); + let archive = locators + .iter() + .position(|locator| locator.kind == ArtifactSourceKind::GithubArchive) + .expect("a pinned query still yields the GitHub tag archive"); + assert_eq!(archive, locators.len() - 1); + } + + /// A `git archive`-shaped tarball: pax global header, `/` directory, + /// a nested file, and an executable, all with non-zero mtimes. + fn git_archive_fixture(root: &str, mtime: u64, level: u32) -> tempfile::NamedTempFile { + let fixture = tempfile::NamedTempFile::new().unwrap(); + let encoder = GzEncoder::new( + fs::File::create(fixture.path()).unwrap(), + Compression::new(level), + ); + let mut builder = tar::Builder::new(encoder); + let comment = b"52 comment=291406191b55606e34b7980b42112e9c77ab690b\n"; + let mut pax = tar::Header::new_ustar(); + pax.set_entry_type(tar::EntryType::XGlobalHeader); + pax.set_size(comment.len() as u64); + builder + .append_data(&mut pax, "pax_global_header", comment.as_slice()) + .unwrap(); + let mut dir = tar::Header::new_ustar(); + dir.set_entry_type(tar::EntryType::Directory); + dir.set_size(0); + dir.set_mode(0o775); + dir.set_mtime(mtime); + builder + .append_data(&mut dir, format!("{root}/"), std::io::empty()) + .unwrap(); + for (path, mode, body) in [ + ("rust/src/lib.rs", 0o664, b"pub fn owls() {}\n".as_slice()), + (".zpkg.toml", 0o664, b"[package]\n".as_slice()), + ("scripts/run.sh", 0o775, b"#!/bin/sh\n".as_slice()), + ] { + let mut header = tar::Header::new_ustar(); + header.set_size(body.len() as u64); + header.set_mode(mode); + header.set_mtime(mtime); + builder + .append_data(&mut header, format!("{root}/{path}"), body) + .unwrap(); + } + builder.into_inner().unwrap().finish().unwrap(); + fixture + } + + #[test] + fn tag_archives_repack_into_a_reproducible_pkg_artifact() { + let first = git_archive_fixture("owls-interfaces-0.1.1", 1_757_080_440, 9); + let second = git_archive_fixture("owls-interfaces-291406191b55", 42, 1); + let out = tempfile::tempdir().unwrap(); + let (a, b) = (out.path().join("a.tar.gz"), out.path().join("b.tar.gz")); + normalize_tag_archive(first.path(), &a).unwrap(); + normalize_tag_archive(second.path(), &b).unwrap(); + assert_eq!(sha256_and_size(&a).unwrap(), sha256_and_size(&b).unwrap()); + + let mut archive = tar::Archive::new(GzDecoder::new(fs::File::open(&a).unwrap())); + let entries: Vec<(String, u32, u64)> = archive + .entries() + .unwrap() + .map(|entry| { + let entry = entry.unwrap(); + let header = entry.header(); + ( + entry.path().unwrap().to_string_lossy().into_owned(), + header.mode().unwrap(), + header.mtime().unwrap(), + ) + }) + .collect(); + assert_eq!( + entries, + vec![ + ("pkg/.zpkg.toml".to_string(), 0o644, 0), + ("pkg/rust/src/lib.rs".to_string(), 0o644, 0), + ("pkg/scripts/run.sh".to_string(), 0o755, 0), + ] + ); + } + + #[test] + fn tag_archive_links_fail_closed() { + let fixture = tempfile::NamedTempFile::new().unwrap(); + let mut builder = tar::Builder::new(GzEncoder::new( + fs::File::create(fixture.path()).unwrap(), + Compression::default(), + )); + let mut link = tar::Header::new_ustar(); + link.set_entry_type(tar::EntryType::Symlink); + link.set_size(0); + builder + .append_link(&mut link, "repo-1.0.0/escape", "/etc/passwd") + .unwrap(); + builder.into_inner().unwrap().finish().unwrap(); + let out = tempfile::tempdir().unwrap(); + let error = normalize_tag_archive(fixture.path(), &out.path().join("a.tar.gz")) + .unwrap_err() + .to_string(); + assert!(error.contains("unsupported type"), "{error}"); + } + + #[test] + fn fallback_mirrors_use_object_store_bases_not_object_urls() { + use zed_interfaces::mirror::MirrorKindV1; + let query = ArtifactQuery { + org: "ores-wasm-loaders", + name: "owls-interfaces", + version: "0.1.1", + vcs_tag: "v0.1.1", + sha256: None, + format: ArtifactFormat::TarGz, + repo_url: Some("https://github.com/ores-wasm-loaders/owls-interfaces"), + artifacts: None, + registry_base: None, + r2_public_base: Some("https://cdn.zpkg.net"), + r2_public_key: None, + }; + let locators = artifact_locators(&query); + assert!( + locators + .iter() + .filter(|locator| locator.kind == ArtifactSourceKind::R2) + .count() + > 1 + ); + let mirrors = fallback_mirrors(&locators, "https://cdn.zpkg.net/"); + let stores: Vec<_> = mirrors + .iter() + .filter(|mirror| mirror.kind == MirrorKindV1::ObjectStore) + .collect(); + assert_eq!(stores.len(), 1); + assert_eq!(stores[0].url.as_deref(), Some("https://cdn.zpkg.net")); + assert!( + mirrors + .iter() + .all(|mirror| mirror.kind != MirrorKindV1::ZedRegistry) + ); + } } From 4649b3d45319bef2f2263aec20bcc9c53414dd08 Mon Sep 17 00:00:00 2001 From: AlexReal Date: Sat, 12 Sep 2026 21:23:45 -0500 Subject: [PATCH 2/3] fix(fallback): repack GitHub tag archives exactly as zed publish packs them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to a983843 (zed-pkg/zed-cli#378). A raw tag archive still could not install: its polyglot manifest keeps `[targets]`, so a consumer that infers `rust` was refused, and its digest could never match a lockfile. The fallback now extracts the tag tree (links, specials, escaping paths and oversize input fail closed), checks the declared org/name/version, applies the same manifest hardening `zed pack` uses, and packs it with `pack::pack_all`, keeping the root package. Packing is deterministic, so the fallback reproduces the published artifact byte for byte: - owls-interfaces v0.1.0 tag -> 3ed5a04a…, the published Release asset - owls-interfaces v0.1.1 tag -> fea1f81d…, the digest pinned in ores-wasm-loaders/owls-runtime.rs/.zpkg.lock Verified live while registry.zpkg.net returns 503 and cdn.zpkg.net 404: `zed install --frozen --adapter none --install-mode copy` in a fresh owls-runtime.rs clone installs from github.com, and `cargo check --locked` passes. cargo test --lib: 605 passed. Co-Authored-By: Claude Opus 5 --- src/source_fallback.rs | 261 +++++++++++++++++++++++++++-------------- 1 file changed, 174 insertions(+), 87 deletions(-) diff --git a/src/source_fallback.rs b/src/source_fallback.rs index 5b3327459..0db34af90 100644 --- a/src/source_fallback.rs +++ b/src/source_fallback.rs @@ -3,25 +3,24 @@ //! Loopback `file://` and `http://127.0.0.1` registries stay hermetic: tests //! and air-gapped mirrors never leak to github.com. Production hosts such as //! `registry.zpkg.net` fall back to guessed public R2 keys and GitHub Release -//! assets, then to a tagged source archive only when no packed digest is known. +//! assets, then to the GitHub tag archive, repacked exactly as `zed publish` +//! would pack that tag. use std::cell::RefCell; use std::collections::HashMap; use std::fs; -use std::io::{Read as _, Write as _}; +use std::io::Read as _; use std::path::{Component, Path}; use std::sync::Mutex; use std::time::Duration; use anyhow::{Context, Result, anyhow, bail}; -use flate2::Compression; use flate2::read::GzDecoder; -use flate2::write::GzEncoder; use serde::Deserialize; use zed_interfaces::artifact::ArtifactFormat; use zed_interfaces::manifest::{Manifest, is_slug}; use zed_interfaces::mirror::MirrorDescriptorV1; -use zed_interfaces::paths::{ARCHIVE_ROOT, MANIFEST_FILE}; +use zed_interfaces::paths::MANIFEST_FILE; use zed_interfaces::registry::{ AuditLogResponse, ClaimOrgResponse, PackageMetadata, PublishMeta, PublishResponse, SearchResponse, VersionMetadata, YankResponse, @@ -174,7 +173,13 @@ impl FallbackRegistry { } None => download_url(&self.client, url, raw.path(), 0, None)?, } - normalize_tag_archive(raw.path(), dest) + repack_tag_archive( + raw.path(), + &version.org, + &version.name, + &version.version, + dest, + ) } fn github_headers( @@ -870,34 +875,75 @@ struct GithubTag { /// matching the store's extraction bound). const MAX_TAG_ARCHIVE_FILES: usize = 200_000; -/// Repack a GitHub tag archive into the deterministic Zed artifact layout. +/// Rebuild, from a GitHub tag archive, the artifact `zed publish` would have +/// produced for that tag. /// /// `git archive` output has a `-/` root, a `pax_global_header` /// entry, and gzip bytes GitHub does not promise to keep stable; the store -/// accepts none of that. The top-level directory is replaced by `pkg/`, -/// metadata-only entries are dropped, and files are written sorted with the -/// zeroed headers `pack` uses — so the digest names the tree, not GitHub's -/// compressor, and every machine derives the same one. Links and specials -/// fail closed, exactly as the store's extractor would. -fn normalize_tag_archive(raw: &Path, dest: &Path) -> Result<()> { +/// accepts none of that, and a raw polyglot manifest still declares targets +/// the published artifact does not. The tree is extracted and packed with the +/// same [`crate::pack::pack_all`] publish uses, keeping the root package. +/// Packing is deterministic, so the digest names the tree rather than +/// GitHub's compressor, and it equals the published digest for a tag packed +/// by this packer — a pinned lockfile can be satisfied from github.com alone. +fn repack_tag_archive(raw: &Path, org: &str, name: &str, version: &str, dest: &Path) -> Result<()> { + let staging = tempfile::tempdir().context("tag archive staging directory")?; + let tree = staging.path().join("tree"); + extract_tag_archive(raw, &tree)?; + let manifest = crate::config::read_manifest(&tree) + .context("GitHub tag archive has no valid package manifest")?; + let declared = &manifest.package; + if declared.org != org || declared.name != name || declared.version != version { + bail!( + "GitHub tag archive declares {}/{}@{}, expected {org}/{name}@{version}", + declared.org, + declared.name, + declared.version + ); + } + // The same manifest hardening `zed pack` / `zed publish` apply before + // packing; without it the derived `.zpkg.toml` — and so the digest — + // differs from the published artifact. Their preflights guard a local + // checkout's untracked inputs, which a tag archive cannot contain. + let manifest = + crate::pack_inputs::harden_manifest(crate::pack_guard::harden_manifest(manifest)); + let packages = crate::pack::pack_all(&tree, &manifest, Some(&staging.path().join("out"))) + .context("repack GitHub tag archive")?; + let root = packages + .into_iter() + .find(|package| package.manifest.package.name == name) + .with_context(|| { + format!("GitHub tag archive for {org}/{name}@{version} publishes no root package") + })?; + fs::copy(&root.packed.path, dest).context("stage repacked GitHub tag archive")?; + Ok(()) +} + +/// Extract a GitHub tag archive's tree, without its `-/` root, into +/// `tree`. Links, specials, and escaping paths fail closed and sizes are +/// capped, exactly as in the store's extractor: the bytes come from the network. +fn extract_tag_archive(raw: &Path, tree: &Path) -> Result<()> { let budget = max_artifact_bytes(); let mut total: u64 = 0; - let mut files: Vec<(String, u32, Vec)> = Vec::new(); + let mut files = 0usize; let file = fs::File::open(raw).context("open GitHub tag archive")?; let mut archive = tar::Archive::new(GzDecoder::new(file)); for entry in archive.entries().context("read GitHub tag archive")? { let mut entry = entry?; let path = entry.path()?.into_owned(); - match entry.header().entry_type() { - tar::EntryType::Regular => {} - tar::EntryType::Directory | tar::EntryType::XGlobalHeader | tar::EntryType::XHeader => { - continue; - } - other => bail!( - "GitHub tag archive entry `{}` has unsupported type {other:?} \ + let kind = entry.header().entry_type(); + if matches!( + kind, + tar::EntryType::Directory | tar::EntryType::XGlobalHeader | tar::EntryType::XHeader + ) { + continue; + } + if kind != tar::EntryType::Regular { + bail!( + "GitHub tag archive entry `{}` has unsupported type {kind:?} \ (only files and directories are allowed)", path.display() - ), + ); } let mut components = path.components(); if !matches!(components.next(), Some(Component::Normal(_))) { @@ -915,7 +961,8 @@ fn normalize_tag_archive(raw: &Path, dest: &Path) -> Result<()> { path.display() ); } - if files.len() >= MAX_TAG_ARCHIVE_FILES { + files += 1; + if files > MAX_TAG_ARCHIVE_FILES { bail!("GitHub tag archive has more than {MAX_TAG_ARCHIVE_FILES} files; refusing"); } let size = entry.header().size()?; @@ -923,45 +970,32 @@ fn normalize_tag_archive(raw: &Path, dest: &Path) -> Result<()> { if total > budget { bail!("GitHub tag archive expands past the {budget}-byte cap; refusing"); } - let mode = if entry.header().mode()? & 0o111 != 0 { - 0o755 - } else { - 0o644 - }; - let mut data = Vec::new(); - (&mut entry).take(size).read_to_end(&mut data)?; - let rel = rel - .components() - .map(|c| c.as_os_str().to_string_lossy()) - .collect::>() - .join("/"); - files.push((rel, mode, data)); + let target = tree.join(rel); + if target.exists() { + bail!("GitHub tag archive repeats entry `{}`", path.display()); + } + if let Some(parent) = target.parent() { + fs::create_dir_all(parent)?; + } + let mut out = fs::File::create(&target)?; + let copied = std::io::copy(&mut (&mut entry).take(size), &mut out)?; + if copied != size { + bail!("GitHub tag archive entry `{}` is truncated", path.display()); + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = if entry.header().mode()? & 0o111 != 0 { + 0o755 + } else { + 0o644 + }; + fs::set_permissions(&target, fs::Permissions::from_mode(mode))?; + } } - if files.is_empty() { + if files == 0 { bail!("GitHub tag archive contains no files"); } - files.sort_by(|left, right| left.0.cmp(&right.0)); - if let Some(pair) = files.windows(2).find(|pair| pair[0].0 == pair[1].0) { - bail!("GitHub tag archive repeats entry `{}`", pair[0].0); - } - - let out = fs::File::create(dest)?; - let mut builder = tar::Builder::new(GzEncoder::new(out, Compression::default())); - for (rel, mode, data) in &files { - let mut header = tar::Header::new_gnu(); - header.set_size(data.len() as u64); - header.set_mtime(0); - header.set_uid(0); - header.set_gid(0); - header.set_mode(*mode); - builder.append_data( - &mut header, - format!("{ARCHIVE_ROOT}/{rel}"), - data.as_slice(), - )?; - } - let mut out = builder.into_inner()?.finish()?; - out.flush()?; Ok(()) } @@ -978,7 +1012,9 @@ fn fallback_mirrors(locators: &[ArtifactLocator], r2_base: &str) -> Vec { MirrorDescriptorV1::object_store(r2_base.trim_end_matches('/')) } - _ => MirrorDescriptorV1::from_locator(locator), + ArtifactSourceKind::GithubRelease + | ArtifactSourceKind::GithubPackages + | ArtifactSourceKind::GithubArchive => MirrorDescriptorV1::from_locator(locator), }; if !mirrors .iter() @@ -1161,6 +1197,8 @@ pub fn github_guess_is_safe(org: &str, name: &str) -> bool { #[cfg(test)] mod tests { use super::*; + use flate2::Compression; + use flate2::write::GzEncoder; use zed_interfaces::source::{r2_object_keys, resolve_r2_public_base}; #[test] @@ -1377,8 +1415,24 @@ url = "https://github.com/ores-otel/ores-otel-sidecar.rs" assert_eq!(archive, locators.len() - 1); } - /// A `git archive`-shaped tarball: pax global header, `/` directory, - /// a nested file, and an executable, all with non-zero mtimes. + const FIXTURE_MANIFEST: &[u8] = br#"[package] +org = "ores-wasm-loaders" +name = "owls-interfaces" +version = "0.1.1" +description = "Release manifests and host contracts for shared WASM loaders" +license = "MIT" + +[package.repository] +vcs = "git" +url = "https://github.com/ores-wasm-loaders/owls-interfaces" + +[targets.repository] +dir = "." +"#; + + /// A `git archive`-shaped tarball of a polyglot repository: pax global + /// header, `/` directory, a nested file, and an executable, all + /// with non-zero mtimes. fn git_archive_fixture(root: &str, mtime: u64, level: u32) -> tempfile::NamedTempFile { let fixture = tempfile::NamedTempFile::new().unwrap(); let encoder = GzEncoder::new( @@ -1403,7 +1457,7 @@ url = "https://github.com/ores-otel/ores-otel-sidecar.rs" .unwrap(); for (path, mode, body) in [ ("rust/src/lib.rs", 0o664, b"pub fn owls() {}\n".as_slice()), - (".zpkg.toml", 0o664, b"[package]\n".as_slice()), + (".zpkg.toml", 0o664, FIXTURE_MANIFEST), ("scripts/run.sh", 0o775, b"#!/bin/sh\n".as_slice()), ] { let mut header = tar::Header::new_ustar(); @@ -1418,37 +1472,70 @@ url = "https://github.com/ores-otel/ores-otel-sidecar.rs" fixture } + fn repack(fixture: &Path, name: &str, dest: &Path) -> Result<()> { + repack_tag_archive(fixture, "ores-wasm-loaders", name, "0.1.1", dest) + } + #[test] - fn tag_archives_repack_into_a_reproducible_pkg_artifact() { + fn tag_archives_repack_into_the_published_root_artifact() { let first = git_archive_fixture("owls-interfaces-0.1.1", 1_757_080_440, 9); let second = git_archive_fixture("owls-interfaces-291406191b55", 42, 1); let out = tempfile::tempdir().unwrap(); let (a, b) = (out.path().join("a.tar.gz"), out.path().join("b.tar.gz")); - normalize_tag_archive(first.path(), &a).unwrap(); - normalize_tag_archive(second.path(), &b).unwrap(); + repack(first.path(), "owls-interfaces", &a).unwrap(); + repack(second.path(), "owls-interfaces", &b).unwrap(); assert_eq!(sha256_and_size(&a).unwrap(), sha256_and_size(&b).unwrap()); let mut archive = tar::Archive::new(GzDecoder::new(fs::File::open(&a).unwrap())); - let entries: Vec<(String, u32, u64)> = archive - .entries() - .unwrap() - .map(|entry| { - let entry = entry.unwrap(); - let header = entry.header(); - ( - entry.path().unwrap().to_string_lossy().into_owned(), - header.mode().unwrap(), - header.mtime().unwrap(), - ) - }) - .collect(); - assert_eq!( - entries, - vec![ - ("pkg/.zpkg.toml".to_string(), 0o644, 0), - ("pkg/rust/src/lib.rs".to_string(), 0o644, 0), - ("pkg/scripts/run.sh".to_string(), 0o755, 0), - ] + let mut manifest = None; + let mut entries = Vec::new(); + for entry in archive.entries().unwrap() { + let mut entry = entry.unwrap(); + let path = entry.path().unwrap().to_string_lossy().into_owned(); + assert_eq!(entry.header().mtime().unwrap(), 0, "{path}"); + if path == "pkg/scripts/run.sh" { + assert_eq!(entry.header().mode().unwrap(), 0o755); + } + if path == "pkg/.zpkg.toml" { + let mut text = String::new(); + entry.read_to_string(&mut text).unwrap(); + manifest = Some(Manifest::parse(&text).unwrap()); + } + entries.push(path); + } + for expected in [ + "pkg/.zpkg.toml", + "pkg/rust/src/lib.rs", + "pkg/scripts/run.sh", + ] { + assert!(entries.iter().any(|path| path == expected), "{entries:?}"); + } + // The published root artifact carries a single-target manifest, so a + // consumer that requests `rust` is not refused over `[targets]`. + let manifest = manifest.expect("packed manifest"); + assert_eq!(manifest.package.name, "owls-interfaces"); + assert!(manifest.targets.is_empty()); + // `zed pack` hardens the manifest before packing; skipping that changes + // the derived manifest bytes and so the digest a lockfile pins. + for hardened in [".zedinclude", "**/.git/**"] { + assert!( + manifest.publish.exclude.iter().any(|rule| rule == hardened), + "{:?}", + manifest.publish.exclude + ); + } + } + + #[test] + fn tag_archive_identity_must_match_the_request() { + let fixture = git_archive_fixture("other-0.1.1", 1, 6); + let out = tempfile::tempdir().unwrap(); + let error = repack(fixture.path(), "other", &out.path().join("a.tar.gz")) + .unwrap_err() + .to_string(); + assert!( + error.contains("expected ores-wasm-loaders/other@0.1.1"), + "{error}" ); } @@ -1467,7 +1554,7 @@ url = "https://github.com/ores-otel/ores-otel-sidecar.rs" .unwrap(); builder.into_inner().unwrap().finish().unwrap(); let out = tempfile::tempdir().unwrap(); - let error = normalize_tag_archive(fixture.path(), &out.path().join("a.tar.gz")) + let error = repack(fixture.path(), "repo", &out.path().join("a.tar.gz")) .unwrap_err() .to_string(); assert!(error.contains("unsupported type"), "{error}"); From 2f6075da1001368f8a2bd2910f9d247ecb828f86 Mon Sep 17 00:00:00 2001 From: AlexReal Date: Sat, 12 Sep 2026 21:39:30 -0500 Subject: [PATCH 3/3] fix(fetch): restore frozen fetches from github.com when zpkg.net is down MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `zed fetch --frozen` (the Nix/offline bundle path) still could not restore a lock during the registry outage (zed-pkg/zed-cli#378): - A committed lock can name a machine-local `file://` registry that does not exist here (owls-runtime.rs pins `file:///tmp/ores-wasm-registry`). When that directory is absent, fetch now reads the pinned version from the configured registry, and through its fallback from GitHub. It records a degraded note that does not echo the path. `immutable-nix-store-input` sources never fall back. Documented in docs/frozen-fetch.md. - Fallback metadata carried no VCS commit, so the exact provenance check refused a digest-identical artifact. The fallback now records the commit the tag points at, from the GitHub commits API; a full 40-hex id only, otherwise the field stays empty and the check fails closed. - The repacked tag archive was copied into a store cache directory that a fresh isolated fetch store has not created yet. Create it first, and keep full error chains in the fallback attempt list. - Cargo.lock: take the zed-lock rev Cargo.toml already pins (1db0da0), as 17 open branches do. main's stale lock fails every `--locked` job. Verified live (registry.zpkg.net 503, cdn.zpkg.net 404) in a fresh owls-runtime.rs clone: - `zed fetch --frozen` publishes a bundle with sha256 fea1f81d…, commit 291406191b… - `zed install --frozen --adapter none --install-mode copy` succeeds. cargo test --locked --lib: 607 passed; --all-targets builds under --locked. Co-Authored-By: Claude Opus 5 --- Cargo.lock | 3 +- docs/frozen-fetch.md | 8 ++++ src/fetch.rs | 84 +++++++++++++++++++++++++++++++++++++++++- src/source_fallback.rs | 73 ++++++++++++++++++++++++++++++++++-- 4 files changed, 161 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 34d224b12..9d03f946d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2229,11 +2229,10 @@ dependencies = [ [[package]] name = "zed-lock" version = "0.1.1" -source = "git+https://github.com/zed-pkg/zed-lock.git?rev=1a95bb87272ac7b08895cea1f61743cf40321213#1a95bb87272ac7b08895cea1f61743cf40321213" +source = "git+https://github.com/zed-pkg/zed-lock.git?rev=1db0da00d30fcf2e0762f50eedb1f88458020b52#1db0da00d30fcf2e0762f50eedb1f88458020b52" dependencies = [ "anyhow", "fs2", - "libc", ] [[package]] diff --git a/docs/frozen-fetch.md b/docs/frozen-fetch.md index 1342fd1d8..e6605e775 100644 --- a/docs/frozen-fetch.md +++ b/docs/frozen-fetch.md @@ -84,6 +84,14 @@ authority is the URL-parser domain `localhost` (or no host at all); loopback IP literals are rejected before `Url::to_file_path()` so Windows cannot reinterpret them as UNC. +A well-formed `file://` lock source whose registry directory does not exist on +this machine (for example a throwaway publish directory recorded on another +machine) is not a trust input: the pinned digest is. Fetch then reads the pinned +version from the configured registry, and through that registry's GitHub +fallback when zpkg.net is unavailable. It verifies every field above exactly as +for the locked source and reports the substitution as a degraded note that does +not echo the source path. `immutable-nix-store-input` sources never fall back. + ## Nix use A Nix fixed-output derivation should ask `zed fetch` to publish into an existing diff --git a/src/fetch.rs b/src/fetch.rs index 52a53eaef..ea054aa57 100644 --- a/src/fetch.rs +++ b/src/fetch.rs @@ -252,7 +252,24 @@ pub fn run(requested_root: &Path, cfg: &Config, options: FetchArgs) -> Result = Vec::new(); for locked in &packages { - let source = effective_source(locked, &cfg.registry); + let locked_source = effective_source(locked, &cfg.registry); + let configured = cfg.registry.trim_end_matches('/'); + // A committed lock can name a machine-local `file://` registry that + // does not exist here (a throwaway publish directory, another + // developer's path). The pin, not that path, is the authority, so read + // the pinned version from the configured registry — and through its + // fallback, GitHub — and verify it exactly as before. + let source = if locked_source != configured && missing_local_source(locked_source) { + degraded.push(format!( + "{}@{}: the locked local registry is not present on this machine; \ + reading the pinned version from the configured registry", + locked.full_name(), + locked.version + )); + configured + } else { + locked_source + }; if !registries.contains_key(source) { registries.insert(source.to_string(), context.open(source)?); } @@ -396,6 +413,17 @@ fn validate_locked_packages( Ok(packages) } +/// A `file://` lock source whose registry directory is absent on this machine. +/// Immutable Nix store inputs never qualify: a missing store path is a broken +/// derivation, not an outage to route around, and must stay hermetic. +fn missing_local_source(source: &str) -> bool { + source_kind(source) == "file" + && reqwest::Url::parse(source) + .ok() + .and_then(|url| url.to_file_path().ok()) + .is_some_and(|path| !path.is_dir()) +} + fn effective_source<'a>(package: &'a LockedPackage, fallback_registry: &'a str) -> &'a str { let source = package.source.trim(); if source.is_empty() { @@ -1164,6 +1192,60 @@ mod tests { assert!(no_fetch_staging(outputs.path())); } + #[test] + fn missing_local_lock_source_reads_the_pin_from_the_configured_registry() { + let project = tempfile::tempdir().unwrap(); + let registry = tempfile::tempdir().unwrap(); + let outputs = tempfile::tempdir().unwrap(); + let mut locked = seed_package( + registry.path(), + "ores-wasm-loaders", + "owls-interfaces", + "0.1.1", + &[("rust/src/lib.rs", b"pub fn owls() {}\n")], + ); + // The lock was written on a machine whose publish directory is gone. + let vanished = outputs.path().join("vanished-publish-registry"); + locked.source = format!("file://{}", vanished.display()); + write_lock(project.path(), vec![locked.clone()]); + let mut cfg = config(outputs.path().join("home")); + cfg.registry = format!("file://{}", registry.path().display()); + + let bundle = outputs.path().join("bundle"); + let report = run( + project.path(), + &cfg, + FetchArgs { + frozen: true, + output: bundle.clone(), + }, + ) + .unwrap(); + + assert_eq!(report.packages, 1); + assert_eq!(report.degraded.len(), 1, "{:?}", report.degraded); + let vanished_text = vanished.to_string_lossy().to_string(); + assert!( + !report.degraded[0].contains(&vanished_text), + "diagnostics must not echo the lock source: {:?}", + report.degraded + ); + assert!( + bundle + .join("packages") + .join(&locked.sha256) + .join("pkg/rust/src/lib.rs") + .is_file() + ); + + assert!(missing_local_source(&locked.source)); + assert!(!missing_local_source(&cfg.registry)); + assert!(!missing_local_source( + "file:///nix/store/00000000000000000000000000000000-missing-zed-registry" + )); + assert!(!missing_local_source("https://registry.zpkg.net")); + } + #[test] fn tampered_lock_digest_fails_before_atomic_publish() { let project = tempfile::tempdir().unwrap(); diff --git a/src/source_fallback.rs b/src/source_fallback.rs index 0db34af90..e2a5b2059 100644 --- a/src/source_fallback.rs +++ b/src/source_fallback.rs @@ -182,6 +182,29 @@ impl FallbackRegistry { ) } + /// Record the commit a version's tag points at. Fallback metadata otherwise + /// carries no commit, and a frozen restore compares VCS provenance against + /// the lock exactly, so it would refuse a digest-identical artifact. The + /// commit comes from GitHub, never from the archive; if GitHub cannot say, + /// the field stays empty and that comparison fails closed as before. + fn fill_tag_commit(&self, identity: &GithubIdentity, metadata: &mut VersionMetadata) { + if metadata.vcs_commit.is_some() { + return; + } + let url = format!( + "{}/commits/{}", + github_api_repo_url(identity), + metadata.vcs_tag + ); + metadata.vcs_commit = self + .github_headers(self.client.get(url)) + .send() + .ok() + .filter(|response| response.status().is_success()) + .and_then(|response| response.json::().ok()) + .and_then(|commit| commit_sha(&commit.sha)); + } + fn github_headers( &self, request: reqwest::blocking::RequestBuilder, @@ -401,11 +424,13 @@ impl FallbackRegistry { } let identity = self.resolve_github_identity(org, name)?; let tag = format!("v{version}"); - if let Some(metadata) = self.release_sidecar(&identity, org, name, version, &tag) { + if let Some(mut metadata) = self.release_sidecar(&identity, org, name, version, &tag) { + self.fill_tag_commit(&identity, &mut metadata); self.remember(metadata.clone()); return Ok(metadata); } - if let Some(metadata) = self.ghcr_version(&identity, org, name, version, &tag) { + if let Some(mut metadata) = self.ghcr_version(&identity, org, name, version, &tag) { + self.fill_tag_commit(&identity, &mut metadata); self.remember(metadata.clone()); return Ok(metadata); } @@ -458,6 +483,7 @@ impl FallbackRegistry { mirrors: fallback_mirrors(&locators, &self.r2_base()), signatures: Vec::new(), }; + self.fill_tag_commit(&identity, &mut metadata); self.fill_digest(&mut metadata)?; self.remember(metadata.clone()); Ok(metadata) @@ -667,7 +693,7 @@ impl FallbackRegistry { }; match result { Ok(()) => return Ok(()), - Err(error) => errors.push(format!("{}: {error}", locator.url)), + Err(error) => errors.push(format!("{}: {error:#}", locator.url)), } } bail!( @@ -871,6 +897,21 @@ struct GithubTag { name: String, } +#[derive(Debug, Deserialize)] +struct GithubCommit { + sha: String, +} + +/// A full lowercase 40-hex Git commit id, or nothing. Abbreviated or oddly +/// cased ids would never compare equal to a lock's recorded commit anyway. +fn commit_sha(value: &str) -> Option { + (value.len() == 40 + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))) + .then(|| value.to_owned()) +} + /// Ceiling on files taken from one GitHub tag archive (inode-exhaustion guard, /// matching the store's extraction bound). const MAX_TAG_ARCHIVE_FILES: usize = 200_000; @@ -915,6 +956,12 @@ fn repack_tag_archive(raw: &Path, org: &str, name: &str, version: &str, dest: &P .with_context(|| { format!("GitHub tag archive for {org}/{name}@{version} publishes no root package") })?; + // Callers hand over a store cache path whose directory may not exist yet + // (a frozen fetch uses a fresh isolated store); `download_url` creates it + // for the other locators, so this one must too. + if let Some(parent) = dest.parent() { + fs::create_dir_all(parent).context("create repacked artifact directory")?; + } fs::copy(&root.packed.path, dest).context("stage repacked GitHub tag archive")?; Ok(()) } @@ -1481,7 +1528,12 @@ dir = "." let first = git_archive_fixture("owls-interfaces-0.1.1", 1_757_080_440, 9); let second = git_archive_fixture("owls-interfaces-291406191b55", 42, 1); let out = tempfile::tempdir().unwrap(); - let (a, b) = (out.path().join("a.tar.gz"), out.path().join("b.tar.gz")); + // `a` lands in a cache directory that does not exist yet, as in a + // frozen fetch's fresh isolated store. + let (a, b) = ( + out.path().join("cache/artifacts/a.tar.gz"), + out.path().join("b.tar.gz"), + ); repack(first.path(), "owls-interfaces", &a).unwrap(); repack(second.path(), "owls-interfaces", &b).unwrap(); assert_eq!(sha256_and_size(&a).unwrap(), sha256_and_size(&b).unwrap()); @@ -1526,6 +1578,19 @@ dir = "." } } + #[test] + fn tag_commits_must_be_full_lowercase_git_ids() { + let sha = "291406191b55606e34b7980b42112e9c77ab690b"; + assert_eq!(commit_sha(sha).as_deref(), Some(sha)); + assert_eq!(commit_sha("2914061"), None); + assert_eq!(commit_sha(&sha.to_ascii_uppercase()), None); + assert_eq!(commit_sha(&format!("{sha}0")), None); + assert_eq!( + commit_sha("../../../etc/passwd-padding-to-forty-chars"), + None + ); + } + #[test] fn tag_archive_identity_must_match_the_request() { let fixture = git_archive_fixture("other-0.1.1", 1, 6);