chore(DEN-1505/DEN-1585): integrate recursive installer stack with current Nix main

.github/workflows/blocking-store-process-locks.yml

@@ -0,0 +1,70 @@
+name: blocking store process locks
+
+on:
+  pull_request:
+    paths:
+      - src/store.rs
+      - src/install_graph.rs
+      - src/install_graph/**
+      - Cargo.toml
+      - Cargo.lock
+      - .github/workflows/blocking-store-process-locks.yml
+  push:
+    branches: [main]
+    paths:
+      - src/store.rs
+      - src/install_graph.rs
+      - src/install_graph/**
+      - Cargo.toml
+      - Cargo.lock
+      - .github/workflows/blocking-store-process-locks.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: blocking-store-process-locks-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: "1"
+
+jobs:
+  linux-kernel-lock:
+    name: Linux process-lock and recursive-cache contracts
+    runs-on: ubuntu-24.04
+    timeout-minutes: 35
+    steps:
+      - name: Check out zed-cli
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 1
+          persist-credentials: false
+          show-progress: false
+
+      - name: Install stable Rust with formatting and lint support
+        run: |
+          rustup toolchain install stable --profile minimal --component rustfmt,clippy
+          rustup default stable
+          rustc --version --verbose
+          cargo --version --verbose
+
+      - name: Check formatting
+        run: cargo fmt --all --check
+
+      - name: Prove a separate process blocks until orderly release
+        run: cargo test --locked --lib store::tests::install_process_lock_blocks_without_polling_until_owner_release -- --exact --nocapture
+
+      - name: Prove owner termination releases the descriptor lock
+        run: cargo test --locked --lib store::tests::install_process_lock_is_released_when_owner_process_is_terminated -- --exact --nocapture
+
+      - name: Run all store process-lock tests
+        run: cargo test --locked --lib store::tests
+
+      - name: Run recursive acquisition and cache-race tests
+        run: cargo test --locked --lib install_graph::tests
+
+      - name: Deny Rust warnings
+        run: cargo clippy --locked --all-targets -- -D warnings

.github/workflows/recursive-install-windows.yml

@@ -0,0 +1,87 @@
+name: recursive install windows locking
+
+on:
+  pull_request:
+    paths:
+      - src/install_graph.rs
+      - src/install_graph/**
+      - src/store.rs
+      - Cargo.toml
+      - Cargo.lock
+      - .github/workflows/recursive-install-windows.yml
+  push:
+    branches: [main]
+    paths:
+      - src/install_graph.rs
+      - src/install_graph/**
+      - src/store.rs
+      - Cargo.toml
+      - Cargo.lock
+      - .github/workflows/recursive-install-windows.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: recursive-install-windows-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: "1"
+
+jobs:
+  windows-kernel-lock:
+    name: Windows process lock and shared-home races
+    runs-on: windows-latest
+    timeout-minutes: 35
+    defaults:
+      run:
+        shell: pwsh
+    steps:
+      - name: Check out zed-cli
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          path: zed-cli
+          fetch-depth: 1
+          persist-credentials: false
+          show-progress: false
+
+      - name: Check out pinned zed-interfaces sibling
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          repository: zed-pkg/zed-interfaces
+          ref: dc0e0a0620b9462817950b552d3d334a184b1cb1
+          path: zed-interfaces
+          fetch-depth: 1
+          persist-credentials: false
+          show-progress: false
+
+      - name: Install stable Rust with formatting support
+        run: |
+          rustup toolchain install stable --profile minimal --component rustfmt
+          rustup default stable
+          rustc --version --verbose
+          cargo --version --verbose
+
+      - name: Check formatting
+        run: cargo fmt --all --check --manifest-path zed-cli/Cargo.toml
+
+      - name: Prove a separate process blocks on the same artifact lock
+        run: cargo test --locked --manifest-path zed-cli/Cargo.toml --lib install_graph::tests::artifact_waiters_block_until_the_owner_releases_the_kernel_lock -- --exact --nocapture
+
+      - name: Prove concurrent prefetches deduplicate a shared artifact
+        run: cargo test --locked --manifest-path zed-cli/Cargo.toml --lib install_graph::tests::concurrent_prefetches_share_one_artifact_download -- --exact --nocapture
+
+      - name: Prove recursive and transactional paths share one lock
+        run: cargo test --locked --manifest-path zed-cli/Cargo.toml --lib install_graph::tests::legacy_and_recursive_acquisition_paths_share_one_download_lock -- --exact --nocapture
+
+      - name: Prove failed downloads leave no cache publication
+        run: cargo test --locked --manifest-path zed-cli/Cargo.toml --lib install_graph::tests::failed_downloads_remove_staging_files_and_never_publish_cache_entries -- --exact --nocapture
+
+      - name: Prove the lower-level store lock blocks across processes
+        run: cargo test --locked --manifest-path zed-cli/Cargo.toml --lib store::tests::install_process_lock_blocks_without_polling_until_owner_release -- --exact --nocapture
+
+      - name: Prove owner termination releases the lower-level store lock
+        run: cargo test --locked --manifest-path zed-cli/Cargo.toml --lib store::tests::install_process_lock_is_released_when_owner_process_is_terminated -- --exact --nocapture

docs/recursive-installs.md

@@ -0,0 +1,89 @@
+# Recursive and concurrent installs
+
+`zed install` resolves dependencies from every installed package manifest, not
+only the consumer's direct dependency table. The resolver keeps one selected
+version per `org/name`, detects incompatible requirements, terminates cycles,
+and deduplicates diamond-shaped graphs before the project tree is changed.
+
+## Concurrency
+
+Artifact download, integrity verification, and extraction run through a bounded
+worker queue. The default is five concurrent artifact installs:
+
+```text
+ZED_PKG_INSTALL_CONCURRENCY=5 zed install
+```
+
+`ZED_PKG_INSTALL_CONCURRENCY` may be set from 1 through 32. Invalid values and
+zero use the default. Registry metadata resolution stays deterministic on the
+coordinator thread; only independent artifact work is concurrent.
+
+## Cross-process locking
+
+Each content hash owns a lock file under:
+
+```text
+$ZED_PKG_HOME/locks/artifact-<sha256>.lock
+```
+
+The lock uses the operating system's blocking file-lock primitive. A waiter
+sleeps in the kernel until the owner exits or releases the descriptor; there is
+no retry timer or filesystem polling loop. The lock covers download, SHA-256
+verification, atomic publication into the download cache, and extraction into
+the content-addressed store. Two terminals or CI jobs sharing a Zed home can
+therefore install unrelated artifacts concurrently while downloading any one
+artifact at most once.
+
+Recursive prefetch and the transactional installer use the same artifact
+acquisition function. Frozen installs, build preparation, `zed add`, and
+`zed remove` therefore cannot bypass the per-hash lock through an older direct
+cache-download path.
+
+Downloads are written to a temporary file in the cache directory, verified,
+and atomically renamed. A corrupt cache file left by an interrupted older
+client is removed and fetched again while the artifact lock is held.
+
+## Materialization
+
+The concurrent phase only warms the global store. The existing project
+transaction remains responsible for lockfile writes, adapter wiring, rollback,
+and dependency materialization. Unix installations therefore continue to use
+store-backed symlinks by default; explicit copy mode and the non-Unix fallback
+remain self-contained copies.
+
+## Frozen and workspace compatibility
+
+Recursive acquisition is composed with the strict frozen-lock boundary. A
+frozen install may verify artifacts, repair deleted links, reconstruct workspace
+projections, and refresh generated adapter wiring, but it preserves the caller's
+committed lockfile bytes exactly. It does not rewrite comments, provenance, or
+fields introduced by a newer CLI.
+
+Workspace packages still enter the ordinary adapter and transaction lifecycle
+through the `ops_entry` dispatcher. This keeps direct and transitive workspace
+links, copy mode, uninstall cleanup, and post-uninstall frozen restoration on
+the same fail-closed path as registry packages.
+
+The recursive install modules coexist with the resolver-only frozen fetch path,
+Nix fixed-output bundle generation, canonical export planning, and standalone
+pure flake rendering. Those export surfaces consume the same validated lock and
+content identities; they do not bypass recursive integrity checks or per-hash
+locking.
+
+## Validation
+
+The unit suite races the recursive and transactional acquisition paths against
+the same absent content hash. The first download is held open while the second
+path attempts acquisition; the test proves that the second path never enters
+its registry download and that the aggregate download count remains one.
+
+A separate parent/child-process regression holds an artifact lock in the parent,
+proves that the child remains blocked after reaching acquisition, releases the
+owner descriptor, and then requires the child to wake and acquire successfully.
+The same contract runs on Windows together with the shared-home deduplication,
+mixed recursive/transactional acquisition, and failed-download cleanup tests.
+
+The companion end-to-end suite also runs multiple independent CLI processes
+against one shared Zed home, verifies the complete transitive lock graph, and
+checks that project packages are store-backed symlinks rather than copied
+package trees.

src/config.rs

@@ -156,6 +156,7 @@ struct ManifestOverride {
 
 thread_local! {
     static MANIFEST_OVERRIDE: RefCell<Option<ManifestOverride>> = const { RefCell::new(None) };
+    static INSTALL_PREFETCH_CONFIG: RefCell<Option<Config>> = const { RefCell::new(None) };
 }
 
 struct ManifestOverrideGuard;
@@ -168,10 +169,44 @@ impl Drop for ManifestOverrideGuard {
     }
 }
 
+struct InstallPrefetchGuard;
+
+impl Drop for InstallPrefetchGuard {
+    fn drop(&mut self) {
+        INSTALL_PREFETCH_CONFIG.with(|slot| {
+            slot.borrow_mut().take();
+        });
+    }
+}
+
 fn normalized_project(project: &Path) -> PathBuf {
     fs::canonicalize(project).unwrap_or_else(|_| project.to_path_buf())
 }
 
+/// Mark one facade operation so an in-memory manifest proposed by `zed add`
+/// or `zed remove` is recursively prefetched before the implementation-local
+/// reinstall begins. The context is thread-local and panic-safe; ordinary
+/// manifest overrides remain side-effect free.
+pub(crate) fn with_install_prefetch<T>(
+    cfg: &Config,
+    operation: impl FnOnce() -> Result<T>,
+) -> Result<T> {
+    INSTALL_PREFETCH_CONFIG.with(|slot| {
+        let mut slot = slot.borrow_mut();
+        if slot.is_some() {
+            return Err(anyhow!(
+                "an install prefetch context is already active on this thread"
+            ));
+        }
+        *slot = Some(cfg.clone());
+        Ok(())
+    })?;
+    let guard = InstallPrefetchGuard;
+    let result = operation();
+    drop(guard);
+    result
+}
+
 /// Run one installer operation with an in-memory root manifest. Package
 /// manifests read from the store still come from disk; only the exact consumer
 /// directory is overridden. The guard is thread-local and panic-safe, so a
@@ -189,10 +224,17 @@ pub(crate) fn with_manifest_override<T>(
                 "a manifest override is already active on this thread"
             ));
         }
-        *slot = Some(ManifestOverride { project, text });
+        *slot = Some(ManifestOverride {
+            project: project.clone(),
+            text,
+        });
         Ok(())
     })?;
     let guard = ManifestOverrideGuard;
+    let prefetch_cfg = INSTALL_PREFETCH_CONFIG.with(|slot| slot.borrow().clone());
+    if let Some(cfg) = prefetch_cfg {
+        crate::install_graph::prefetch(&project, &cfg, false)?;
+    }
     let result = operation();
     drop(guard);
     result
@@ -282,6 +324,32 @@ url = "https://localhost/manifestless/consumer"
         assert!(error.to_string().contains("already active"));
     }
 
+    #[test]
+    fn install_prefetch_context_is_scoped_and_nested_contexts_fail_closed() {
+        let home = tempfile::tempdir().unwrap();
+        let cfg = Config {
+            registry: "file:///unused".to_string(),
+            home: home.path().to_path_buf(),
+            token: None,
+            auth_url: "https://localhost/unused".to_string(),
+            supabase_url: None,
+            supabase_key: None,
+            interactive: false,
+        };
+
+        with_install_prefetch(&cfg, || {
+            let active = INSTALL_PREFETCH_CONFIG.with(|slot| slot.borrow().is_some());
+            assert!(active);
+            let nested = with_install_prefetch(&cfg, || Ok(())).unwrap_err();
+            assert!(nested.to_string().contains("already active"));
+            Ok(())
+        })
+        .unwrap();
+
+        let active = INSTALL_PREFETCH_CONFIG.with(|slot| slot.borrow().is_some());
+        assert!(!active);
+    }
+
     #[test]
     fn relative_home_is_resolved_from_the_invocation_directory() {
         let globals = Globals {