From 8c9d3fae94584e2b651e1a3afb89e348240ac5c0 Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 3 Aug 2026 11:33:19 -0500 Subject: [PATCH 01/16] chore: stage kernel-blocking store lock patch --- .github/blocking_store_process_locks.py | 171 ++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 .github/blocking_store_process_locks.py diff --git a/.github/blocking_store_process_locks.py b/.github/blocking_store_process_locks.py new file mode 100644 index 00000000..f6750665 --- /dev/null +++ b/.github/blocking_store_process_locks.py @@ -0,0 +1,171 @@ +from pathlib import Path + + +def replace_once(path: str, old: str, new: str) -> None: + file = Path(path) + text = file.read_text() + count = text.count(old) + if count != 1: + raise RuntimeError(f"expected one match in {path}, found {count}") + file.write_text(text.replace(old, new, 1)) + + +store = Path("src/store.rs") +text = store.read_text() +text = text.replace( + "use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};", + "use std::time::{Duration, SystemTime, UNIX_EPOCH};", + 1, +) + +start_marker = "/// An advisory (flock-based) process lock held for the life of the guard." +end_marker = "/// What `Store::gc` did (or, with `dry_run`, would do)." +start = text.index(start_marker) +end = text.index(end_marker, start) +replacement = '''/// A descriptor-backed process lock held for the life of the guard. +/// +/// Acquisition uses the operating system's blocking lock primitive directly: +/// `flock`/`fcntl` semantics on Unix and `LockFileEx` semantics on Windows via +/// `fs2`. Contended callers sleep in the kernel and wake when the owner drops +/// the descriptor or exits. There is no retry timer, jitter loop, stale-file +/// reclamation, or userspace polling. +pub struct ProcessLock { + _file: fs::File, +} + +impl ProcessLock { + fn acquire(path: &Path, waiting_on: &str) -> Result { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let file = fs::OpenOptions::new() + .create(true) + .read(true) + .write(true) + .truncate(false) + .open(path) + .with_context(|| format!("opening lock file {}", path.display()))?; + + file.lock_exclusive().with_context(|| { + format!( + "waiting for {waiting_on} through operating-system lock {}", + path.display() + ) + })?; + Ok(Self { _file: file }) + } +} + +''' +text = text[:start] + replacement + text[end:] + +old_test_header = '''#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn gc_survives_hostile_max_age() { +''' +new_test_header = '''#[cfg(test)] +mod tests { + use std::process::{Command, Stdio}; + use std::time::Instant; + + use super::*; + + #[test] + fn install_process_lock_blocks_without_polling_until_owner_release() { + const CHILD_HOME: &str = "ZED_PKG_TEST_STORE_LOCK_CHILD_HOME"; + const CHILD_ATTEMPTING: &str = "ZED_PKG_TEST_STORE_LOCK_CHILD_ATTEMPTING"; + const CHILD_ACQUIRED: &str = "ZED_PKG_TEST_STORE_LOCK_CHILD_ACQUIRED"; + const TEST_NAME: &str = + "store::tests::install_process_lock_blocks_without_polling_until_owner_release"; + + if let Some(home) = std::env::var_os(CHILD_HOME) { + let attempting = PathBuf::from(std::env::var_os(CHILD_ATTEMPTING).unwrap()); + let acquired = PathBuf::from(std::env::var_os(CHILD_ACQUIRED).unwrap()); + fs::write(attempting, b"attempting").unwrap(); + let store = Store::new(Path::new(&home)); + let _waiter = store.install_lock().unwrap(); + fs::write(acquired, b"acquired").unwrap(); + return; + } + + let temp = tempfile::tempdir().unwrap(); + let store = Store::new(temp.path()); + let owner = store.install_lock().unwrap(); + let attempting = temp.path().join("store-lock-waiter-attempting"); + let acquired = temp.path().join("store-lock-waiter-acquired"); + let mut child = Command::new(std::env::current_exe().unwrap()) + .arg(TEST_NAME) + .arg("--exact") + .arg("--nocapture") + .env(CHILD_HOME, temp.path()) + .env(CHILD_ATTEMPTING, &attempting) + .env(CHILD_ACQUIRED, &acquired) + .stdin(Stdio::null()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .spawn() + .unwrap(); + + let attempting_deadline = Instant::now() + Duration::from_secs(5); + while !attempting.is_file() { + if let Some(status) = child.try_wait().unwrap() { + panic!("store-lock waiter exited before attempting acquisition: {status}"); + } + if Instant::now() >= attempting_deadline { + let _ = child.kill(); + let _ = child.wait(); + panic!("store-lock waiter did not reach acquisition"); + } + std::thread::sleep(Duration::from_millis(20)); + } + + std::thread::sleep(Duration::from_millis(200)); + assert!( + !acquired.exists(), + "a separate process acquired install.lock while the owner held it" + ); + assert!( + child.try_wait().unwrap().is_none(), + "store-lock waiter exited before owner release" + ); + + drop(owner); + let release_deadline = Instant::now() + Duration::from_secs(5); + loop { + if let Some(status) = child.try_wait().unwrap() { + assert!(status.success(), "store-lock waiter failed: {status}"); + break; + } + if Instant::now() >= release_deadline { + let _ = child.kill(); + let _ = child.wait(); + panic!("store-lock waiter did not wake after owner release"); + } + std::thread::sleep(Duration::from_millis(20)); + } + assert!(acquired.is_file(), "waiter exited without acquiring install.lock"); + } + + #[test] + fn gc_survives_hostile_max_age() { +''' +if text.count(old_test_header) != 1: + raise RuntimeError("could not locate store test module header") +text = text.replace(old_test_header, new_test_header, 1) +store.write_text(text) + +replace_once( + ".github/workflows/recursive-install-windows.yml", + ''' - name: Prove failed downloads leave no cache publication + run: cargo test --locked --manifest-path zed-cli/Cargo.toml --lib install_graph::tests::failed_downloads_remove_staging_files_and_never_publish_cache_entries -- --exact --nocapture +''', + ''' - name: Prove failed downloads leave no cache publication + run: cargo test --locked --manifest-path zed-cli/Cargo.toml --lib install_graph::tests::failed_downloads_remove_staging_files_and_never_publish_cache_entries -- --exact --nocapture + + - name: Prove the lower-level store lock blocks across processes + run: cargo test --locked --manifest-path zed-cli/Cargo.toml --lib store::tests::install_process_lock_blocks_without_polling_until_owner_release -- --exact --nocapture +''', +) From 2aec1bc83ee565c661487440b7570bc3317b9fde Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 3 Aug 2026 11:33:36 -0500 Subject: [PATCH 02/16] ci: validate kernel-blocking store lock patch --- .../apply-blocking-store-process-locks.yml | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 .github/workflows/apply-blocking-store-process-locks.yml diff --git a/.github/workflows/apply-blocking-store-process-locks.yml b/.github/workflows/apply-blocking-store-process-locks.yml new file mode 100644 index 00000000..3c26c799 --- /dev/null +++ b/.github/workflows/apply-blocking-store-process-locks.yml @@ -0,0 +1,65 @@ +name: apply blocking store process locks + +on: + pull_request: + branches: + - agent/recursive-lock-cross-platform + +permissions: + contents: read + +concurrency: + group: apply-blocking-store-process-locks-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + apply: + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-24.04 + timeout-minutes: 45 + permissions: + contents: write + steps: + - name: Check out store-lock branch + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: agent/blocking-store-process-locks + fetch-depth: 0 + persist-credentials: false + show-progress: false + + - name: Apply exact kernel-blocking lock patch + run: | + set -euo pipefail + python3 .github/blocking_store_process_locks.py + cargo fmt --all + git diff --check + + - name: Validate store and recursive lock contracts + run: | + set -euo pipefail + rustup component add rustfmt clippy + cargo fmt --all --check + cargo test --locked --lib \ + store::tests::install_process_lock_blocks_without_polling_until_owner_release \ + -- --exact --nocapture + cargo test --locked --lib store::tests + cargo test --locked --lib install_graph::tests + cargo clippy --locked --all-targets -- -D warnings + + - name: Commit validated code and remove bootstrap files + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + rm -f \ + .github/blocking_store_process_locks.py \ + .github/workflows/apply-blocking-store-process-locks.yml + git add -A + git diff --cached --check + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git commit -m 'fix: block store locks in the kernel' + git push \ + "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" \ + HEAD:agent/blocking-store-process-locks From 0b7f524d0508e88752879783bccf74915f890271 Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 3 Aug 2026 11:58:33 -0500 Subject: [PATCH 03/16] ci: make blocking-lock bootstrap idempotent and code-only --- .github/blocking_store_process_locks.py | 62 +++++++++---------------- 1 file changed, 23 insertions(+), 39 deletions(-) diff --git a/.github/blocking_store_process_locks.py b/.github/blocking_store_process_locks.py index f6750665..0a237ecb 100644 --- a/.github/blocking_store_process_locks.py +++ b/.github/blocking_store_process_locks.py @@ -1,28 +1,22 @@ from pathlib import Path -def replace_once(path: str, old: str, new: str) -> None: - file = Path(path) - text = file.read_text() - count = text.count(old) - if count != 1: - raise RuntimeError(f"expected one match in {path}, found {count}") - file.write_text(text.replace(old, new, 1)) - - store = Path("src/store.rs") text = store.read_text() -text = text.replace( - "use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};", - "use std::time::{Duration, SystemTime, UNIX_EPOCH};", - 1, -) - -start_marker = "/// An advisory (flock-based) process lock held for the life of the guard." -end_marker = "/// What `Store::gc` did (or, with `dry_run`, would do)." -start = text.index(start_marker) -end = text.index(end_marker, start) -replacement = '''/// A descriptor-backed process lock held for the life of the guard. + +new_lock_marker = "/// A descriptor-backed process lock held for the life of the guard." +if new_lock_marker not in text: + old_import = "use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};" + new_import = "use std::time::{Duration, SystemTime, UNIX_EPOCH};" + if text.count(old_import) != 1: + raise RuntimeError("could not locate the store time import") + text = text.replace(old_import, new_import, 1) + + start_marker = "/// An advisory (flock-based) process lock held for the life of the guard." + end_marker = "/// What `Store::gc` did (or, with `dry_run`, would do)." + start = text.index(start_marker) + end = text.index(end_marker, start) + replacement = '''/// A descriptor-backed process lock held for the life of the guard. /// /// Acquisition uses the operating system's blocking lock primitive directly: /// `flock`/`fcntl` semantics on Unix and `LockFileEx` semantics on Windows via @@ -57,16 +51,18 @@ def replace_once(path: str, old: str, new: str) -> None: } ''' -text = text[:start] + replacement + text[end:] + text = text[:start] + replacement + text[end:] -old_test_header = '''#[cfg(test)] +new_test_name = "fn install_process_lock_blocks_without_polling_until_owner_release()" +if new_test_name not in text: + old_test_header = '''#[cfg(test)] mod tests { use super::*; #[test] fn gc_survives_hostile_max_age() { ''' -new_test_header = '''#[cfg(test)] + new_test_header = '''#[cfg(test)] mod tests { use std::process::{Command, Stdio}; use std::time::Instant; @@ -152,20 +148,8 @@ def replace_once(path: str, old: str, new: str) -> None: #[test] fn gc_survives_hostile_max_age() { ''' -if text.count(old_test_header) != 1: - raise RuntimeError("could not locate store test module header") -text = text.replace(old_test_header, new_test_header, 1) -store.write_text(text) + if text.count(old_test_header) != 1: + raise RuntimeError("could not locate store test module header") + text = text.replace(old_test_header, new_test_header, 1) -replace_once( - ".github/workflows/recursive-install-windows.yml", - ''' - name: Prove failed downloads leave no cache publication - run: cargo test --locked --manifest-path zed-cli/Cargo.toml --lib install_graph::tests::failed_downloads_remove_staging_files_and_never_publish_cache_entries -- --exact --nocapture -''', - ''' - name: Prove failed downloads leave no cache publication - run: cargo test --locked --manifest-path zed-cli/Cargo.toml --lib install_graph::tests::failed_downloads_remove_staging_files_and_never_publish_cache_entries -- --exact --nocapture - - - name: Prove the lower-level store lock blocks across processes - run: cargo test --locked --manifest-path zed-cli/Cargo.toml --lib store::tests::install_process_lock_blocks_without_polling_until_owner_release -- --exact --nocapture -''', -) +store.write_text(text) From 43491279273a4e21d5a3f9110b718079b0ecdc54 Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 3 Aug 2026 11:58:46 -0500 Subject: [PATCH 04/16] ci: publish validated store code without workflow writes --- .../workflows/apply-blocking-store-process-locks.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/apply-blocking-store-process-locks.yml b/.github/workflows/apply-blocking-store-process-locks.yml index 3c26c799..4a15c94b 100644 --- a/.github/workflows/apply-blocking-store-process-locks.yml +++ b/.github/workflows/apply-blocking-store-process-locks.yml @@ -47,16 +47,17 @@ jobs: cargo test --locked --lib install_graph::tests cargo clippy --locked --all-targets -- -D warnings - - name: Commit validated code and remove bootstrap files + - name: Publish validated store implementation env: GH_TOKEN: ${{ github.token }} run: | set -euo pipefail - rm -f \ - .github/blocking_store_process_locks.py \ - .github/workflows/apply-blocking-store-process-locks.yml - git add -A + git add src/store.rs git diff --cached --check + if git diff --cached --quiet; then + echo 'kernel-blocking store implementation is already published' + exit 0 + fi git config user.name 'github-actions[bot]' git config user.email '41898282+github-actions[bot]@users.noreply.github.com' git commit -m 'fix: block store locks in the kernel' From a407898351fffc991b65b4aa1219476c796e20c9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 17:00:20 +0000 Subject: [PATCH 05/16] fix: block store locks in the kernel --- src/store.rs | 155 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 97 insertions(+), 58 deletions(-) diff --git a/src/store.rs b/src/store.rs index 018446af..522973f0 100644 --- a/src/store.rs +++ b/src/store.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::fs; use std::io::Read; use std::path::{Component, Path, PathBuf}; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use anyhow::{Context, Result, bail}; use flate2::read::GzDecoder; @@ -55,22 +55,18 @@ fn require_build_key(key: &str) -> Result<()> { Ok(()) } -/// An advisory (flock-based) process lock held for the life of the guard. -/// CLI commands are highly concurrent — two `zed install` runs in different -/// terminals, or N parallel CI runners — so store mutations that must not -/// interleave (extracting the same artifact, rewriting refs.json) take a -/// lock first. The OS releases it if the process dies, so a crash can't -/// wedge the store. +/// A descriptor-backed process lock held for the life of the guard. +/// +/// Acquisition uses the operating system's blocking lock primitive directly: +/// `flock`/`fcntl` semantics on Unix and `LockFileEx` semantics on Windows via +/// `fs2`. Contended callers sleep in the kernel and wake when the owner drops +/// the descriptor or exits. There is no retry timer, jitter loop, stale-file +/// reclamation, or userspace polling. pub struct ProcessLock { _file: fs::File, } impl ProcessLock { - /// Three-phase acquisition tuned for CLI latency: - /// spin-wait at 50ms while young (catches sub-second releases with no - /// perceptible lag), then exponential backoff with full jitter so - /// parallel CI runners don't wake in lockstep, and after 3s tell the - /// user what they are waiting on instead of freezing silently. fn acquire(path: &Path, waiting_on: &str) -> Result { if let Some(parent) = path.parent() { fs::create_dir_all(parent)?; @@ -83,52 +79,13 @@ impl ProcessLock { .open(path) .with_context(|| format!("opening lock file {}", path.display()))?; - let timeout = Duration::from_secs( - std::env::var("ZED_PKG_LOCK_TIMEOUT") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(600), - ); - let start = Instant::now(); - let mut backoff_attempt: u32 = 0; - let mut notified = false; - loop { - match file.try_lock_exclusive() { - Ok(()) => return Ok(Self { _file: file }), - Err(_) if start.elapsed() >= timeout => { - bail!( - "timed out after {}s waiting for {waiting_on} (lock {}); \ - another zed process may be stuck — raise ZED_PKG_LOCK_TIMEOUT \ - or investigate", - timeout.as_secs(), - path.display() - ); - } - Err(_) => {} - } - let elapsed = start.elapsed(); - if elapsed >= Duration::from_secs(3) && !notified { - eprintln!("waiting for {waiting_on} (held by another zed process)..."); - notified = true; - } - let sleep = if elapsed < Duration::from_millis(500) { - Duration::from_millis(50) - } else { - backoff_attempt = backoff_attempt.saturating_add(1); - let cap_ms: u64 = 2000; - let exp_ms = 100u64 - .saturating_mul(1u64 << backoff_attempt.min(20)) - .min(cap_ms); - // Full jitter: uniform in [0, exp_ms] without pulling in an - // RNG dependency — subsec_nanos is plenty for de-syncing. - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.subsec_nanos() as u64) - .unwrap_or(0); - Duration::from_millis(nanos % (exp_ms + 1)) - }; - std::thread::sleep(sleep); - } + file.lock_exclusive().with_context(|| { + format!( + "waiting for {waiting_on} through operating-system lock {}", + path.display() + ) + })?; + Ok(Self { _file: file }) } } @@ -688,8 +645,90 @@ pub fn human_size(bytes: u64) -> String { #[cfg(test)] mod tests { + use std::process::{Command, Stdio}; + use std::time::Instant; + use super::*; + #[test] + fn install_process_lock_blocks_without_polling_until_owner_release() { + const CHILD_HOME: &str = "ZED_PKG_TEST_STORE_LOCK_CHILD_HOME"; + const CHILD_ATTEMPTING: &str = "ZED_PKG_TEST_STORE_LOCK_CHILD_ATTEMPTING"; + const CHILD_ACQUIRED: &str = "ZED_PKG_TEST_STORE_LOCK_CHILD_ACQUIRED"; + const TEST_NAME: &str = + "store::tests::install_process_lock_blocks_without_polling_until_owner_release"; + + if let Some(home) = std::env::var_os(CHILD_HOME) { + let attempting = PathBuf::from(std::env::var_os(CHILD_ATTEMPTING).unwrap()); + let acquired = PathBuf::from(std::env::var_os(CHILD_ACQUIRED).unwrap()); + fs::write(attempting, b"attempting").unwrap(); + let store = Store::new(Path::new(&home)); + let _waiter = store.install_lock().unwrap(); + fs::write(acquired, b"acquired").unwrap(); + return; + } + + let temp = tempfile::tempdir().unwrap(); + let store = Store::new(temp.path()); + let owner = store.install_lock().unwrap(); + let attempting = temp.path().join("store-lock-waiter-attempting"); + let acquired = temp.path().join("store-lock-waiter-acquired"); + let mut child = Command::new(std::env::current_exe().unwrap()) + .arg(TEST_NAME) + .arg("--exact") + .arg("--nocapture") + .env(CHILD_HOME, temp.path()) + .env(CHILD_ATTEMPTING, &attempting) + .env(CHILD_ACQUIRED, &acquired) + .stdin(Stdio::null()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .spawn() + .unwrap(); + + let attempting_deadline = Instant::now() + Duration::from_secs(5); + while !attempting.is_file() { + if let Some(status) = child.try_wait().unwrap() { + panic!("store-lock waiter exited before attempting acquisition: {status}"); + } + if Instant::now() >= attempting_deadline { + let _ = child.kill(); + let _ = child.wait(); + panic!("store-lock waiter did not reach acquisition"); + } + std::thread::sleep(Duration::from_millis(20)); + } + + std::thread::sleep(Duration::from_millis(200)); + assert!( + !acquired.exists(), + "a separate process acquired install.lock while the owner held it" + ); + assert!( + child.try_wait().unwrap().is_none(), + "store-lock waiter exited before owner release" + ); + + drop(owner); + let release_deadline = Instant::now() + Duration::from_secs(5); + loop { + if let Some(status) = child.try_wait().unwrap() { + assert!(status.success(), "store-lock waiter failed: {status}"); + break; + } + if Instant::now() >= release_deadline { + let _ = child.kill(); + let _ = child.wait(); + panic!("store-lock waiter did not wake after owner release"); + } + std::thread::sleep(Duration::from_millis(20)); + } + assert!( + acquired.is_file(), + "waiter exited without acquiring install.lock" + ); + } + #[test] fn gc_survives_hostile_max_age() { // A hostile age (`u64::MAX` seconds) would overflow naive epoch math From 6e49f62d8113139cfde00eeffe3e2337d8765bbd Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 3 Aug 2026 12:03:41 -0500 Subject: [PATCH 06/16] test: stage owner-exit process lock regression --- .github/add_owner_exit_lock_test.py | 149 ++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 .github/add_owner_exit_lock_test.py diff --git a/.github/add_owner_exit_lock_test.py b/.github/add_owner_exit_lock_test.py new file mode 100644 index 00000000..3d33e717 --- /dev/null +++ b/.github/add_owner_exit_lock_test.py @@ -0,0 +1,149 @@ +from pathlib import Path + + +store = Path("src/store.rs") +text = store.read_text() +test_name = "fn install_process_lock_is_released_when_owner_process_is_terminated()" + +if test_name not in text: + marker = ''' #[test] + fn gc_survives_hostile_max_age() { +''' + replacement = ''' #[test] + fn install_process_lock_is_released_when_owner_process_is_terminated() { + const CHILD_ROLE: &str = "ZED_PKG_TEST_STORE_LOCK_EXIT_ROLE"; + const CHILD_HOME: &str = "ZED_PKG_TEST_STORE_LOCK_EXIT_HOME"; + const OWNER_ACQUIRED: &str = "ZED_PKG_TEST_STORE_LOCK_OWNER_ACQUIRED"; + const WAITER_ATTEMPTING: &str = "ZED_PKG_TEST_STORE_LOCK_WAITER_ATTEMPTING"; + const WAITER_ACQUIRED: &str = "ZED_PKG_TEST_STORE_LOCK_WAITER_ACQUIRED"; + const TEST_NAME: &str = + "store::tests::install_process_lock_is_released_when_owner_process_is_terminated"; + + if let Some(role) = std::env::var_os(CHILD_ROLE) { + let home = PathBuf::from(std::env::var_os(CHILD_HOME).unwrap()); + let store = Store::new(&home); + match role.to_string_lossy().as_ref() { + "owner" => { + let _owner = store.install_lock().unwrap(); + fs::write(std::env::var_os(OWNER_ACQUIRED).unwrap(), b"acquired").unwrap(); + loop { + std::thread::sleep(Duration::from_secs(60)); + } + } + "waiter" => { + fs::write(std::env::var_os(WAITER_ATTEMPTING).unwrap(), b"attempting") + .unwrap(); + let _waiter = store.install_lock().unwrap(); + fs::write(std::env::var_os(WAITER_ACQUIRED).unwrap(), b"acquired").unwrap(); + return; + } + other => panic!("unexpected child role: {other}"), + } + } + + let temp = tempfile::tempdir().unwrap(); + let owner_acquired = temp.path().join("store-lock-owner-acquired"); + let waiter_attempting = temp.path().join("store-lock-waiter-attempting-after-exit"); + let waiter_acquired = temp.path().join("store-lock-waiter-acquired-after-exit"); + + let mut owner = Command::new(std::env::current_exe().unwrap()) + .arg(TEST_NAME) + .arg("--exact") + .arg("--nocapture") + .env(CHILD_ROLE, "owner") + .env(CHILD_HOME, temp.path()) + .env(OWNER_ACQUIRED, &owner_acquired) + .stdin(Stdio::null()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .spawn() + .unwrap(); + + let owner_deadline = Instant::now() + Duration::from_secs(5); + while !owner_acquired.is_file() { + if let Some(status) = owner.try_wait().unwrap() { + panic!("store-lock owner exited before acquiring install.lock: {status}"); + } + if Instant::now() >= owner_deadline { + let _ = owner.kill(); + let _ = owner.wait(); + panic!("store-lock owner did not acquire install.lock"); + } + std::thread::sleep(Duration::from_millis(20)); + } + + let mut waiter = Command::new(std::env::current_exe().unwrap()) + .arg(TEST_NAME) + .arg("--exact") + .arg("--nocapture") + .env(CHILD_ROLE, "waiter") + .env(CHILD_HOME, temp.path()) + .env(WAITER_ATTEMPTING, &waiter_attempting) + .env(WAITER_ACQUIRED, &waiter_acquired) + .stdin(Stdio::null()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .spawn() + .unwrap(); + + let attempting_deadline = Instant::now() + Duration::from_secs(5); + while !waiter_attempting.is_file() { + if let Some(status) = waiter.try_wait().unwrap() { + let _ = owner.kill(); + let _ = owner.wait(); + panic!("store-lock waiter exited before attempting acquisition: {status}"); + } + if Instant::now() >= attempting_deadline { + let _ = waiter.kill(); + let _ = waiter.wait(); + let _ = owner.kill(); + let _ = owner.wait(); + panic!("store-lock waiter did not reach acquisition"); + } + std::thread::sleep(Duration::from_millis(20)); + } + + std::thread::sleep(Duration::from_millis(200)); + assert!( + !waiter_acquired.exists(), + "waiter acquired install.lock while the owner process was alive" + ); + assert!( + waiter.try_wait().unwrap().is_none(), + "store-lock waiter exited before owner termination" + ); + + owner.kill().unwrap(); + let owner_status = owner.wait().unwrap(); + assert!( + !owner_status.success(), + "owner process unexpectedly exited successfully instead of being terminated" + ); + + let release_deadline = Instant::now() + Duration::from_secs(5); + loop { + if let Some(status) = waiter.try_wait().unwrap() { + assert!(status.success(), "store-lock waiter failed after owner exit: {status}"); + break; + } + if Instant::now() >= release_deadline { + let _ = waiter.kill(); + let _ = waiter.wait(); + panic!("store-lock waiter did not wake after owner process termination"); + } + std::thread::sleep(Duration::from_millis(20)); + } + assert!( + waiter_acquired.is_file(), + "waiter exited without acquiring install.lock after owner termination" + ); + } + + #[test] + fn gc_survives_hostile_max_age() { +''' + if text.count(marker) != 1: + raise RuntimeError("could not locate insertion point for owner-exit lock test") + text = text.replace(marker, replacement, 1) + +store.write_text(text) From b0d2a97847c70183c96d5e183e3ab62dd6687742 Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 3 Aug 2026 12:04:02 -0500 Subject: [PATCH 07/16] test: validate owner-exit lock release before publishing --- .github/workflows/apply-blocking-store-process-locks.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/apply-blocking-store-process-locks.yml b/.github/workflows/apply-blocking-store-process-locks.yml index 4a15c94b..6fb1f425 100644 --- a/.github/workflows/apply-blocking-store-process-locks.yml +++ b/.github/workflows/apply-blocking-store-process-locks.yml @@ -32,6 +32,7 @@ jobs: run: | set -euo pipefail python3 .github/blocking_store_process_locks.py + python3 .github/add_owner_exit_lock_test.py cargo fmt --all git diff --check @@ -43,6 +44,9 @@ jobs: cargo test --locked --lib \ store::tests::install_process_lock_blocks_without_polling_until_owner_release \ -- --exact --nocapture + cargo test --locked --lib \ + store::tests::install_process_lock_is_released_when_owner_process_is_terminated \ + -- --exact --nocapture cargo test --locked --lib store::tests cargo test --locked --lib install_graph::tests cargo clippy --locked --all-targets -- -D warnings From e0b7480f237ddbca32b56789d601a8abbd4e4019 Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 3 Aug 2026 12:04:26 -0500 Subject: [PATCH 08/16] test: run store lock blocking and owner-exit regressions on Windows --- .github/workflows/recursive-install-windows.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/recursive-install-windows.yml b/.github/workflows/recursive-install-windows.yml index db568532..eda09e0f 100644 --- a/.github/workflows/recursive-install-windows.yml +++ b/.github/workflows/recursive-install-windows.yml @@ -25,7 +25,7 @@ permissions: concurrency: group: recursive-install-windows-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + cancel-in-progress: true env: CARGO_TERM_COLOR: always @@ -79,3 +79,9 @@ jobs: - name: Prove failed downloads leave no cache publication run: cargo test --locked --manifest-path zed-cli/Cargo.toml --lib install_graph::tests::failed_downloads_remove_staging_files_and_never_publish_cache_entries -- --exact --nocapture + + - name: Prove the lower-level store lock blocks across processes + run: cargo test --locked --manifest-path zed-cli/Cargo.toml --lib store::tests::install_process_lock_blocks_without_polling_until_owner_release -- --exact --nocapture + + - name: Prove owner termination releases the lower-level store lock + run: cargo test --locked --manifest-path zed-cli/Cargo.toml --lib store::tests::install_process_lock_is_released_when_owner_process_is_terminated -- --exact --nocapture From aed930e18ed33076af6f899059f48c281450b999 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 17:06:25 +0000 Subject: [PATCH 09/16] fix: block store locks in the kernel --- src/store.rs | 132 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/src/store.rs b/src/store.rs index 522973f0..60c03286 100644 --- a/src/store.rs +++ b/src/store.rs @@ -729,6 +729,138 @@ mod tests { ); } + #[test] + fn install_process_lock_is_released_when_owner_process_is_terminated() { + const CHILD_ROLE: &str = "ZED_PKG_TEST_STORE_LOCK_EXIT_ROLE"; + const CHILD_HOME: &str = "ZED_PKG_TEST_STORE_LOCK_EXIT_HOME"; + const OWNER_ACQUIRED: &str = "ZED_PKG_TEST_STORE_LOCK_OWNER_ACQUIRED"; + const WAITER_ATTEMPTING: &str = "ZED_PKG_TEST_STORE_LOCK_WAITER_ATTEMPTING"; + const WAITER_ACQUIRED: &str = "ZED_PKG_TEST_STORE_LOCK_WAITER_ACQUIRED"; + const TEST_NAME: &str = + "store::tests::install_process_lock_is_released_when_owner_process_is_terminated"; + + if let Some(role) = std::env::var_os(CHILD_ROLE) { + let home = PathBuf::from(std::env::var_os(CHILD_HOME).unwrap()); + let store = Store::new(&home); + match role.to_string_lossy().as_ref() { + "owner" => { + let _owner = store.install_lock().unwrap(); + fs::write(std::env::var_os(OWNER_ACQUIRED).unwrap(), b"acquired").unwrap(); + loop { + std::thread::sleep(Duration::from_secs(60)); + } + } + "waiter" => { + fs::write(std::env::var_os(WAITER_ATTEMPTING).unwrap(), b"attempting").unwrap(); + let _waiter = store.install_lock().unwrap(); + fs::write(std::env::var_os(WAITER_ACQUIRED).unwrap(), b"acquired").unwrap(); + return; + } + other => panic!("unexpected child role: {other}"), + } + } + + let temp = tempfile::tempdir().unwrap(); + let owner_acquired = temp.path().join("store-lock-owner-acquired"); + let waiter_attempting = temp.path().join("store-lock-waiter-attempting-after-exit"); + let waiter_acquired = temp.path().join("store-lock-waiter-acquired-after-exit"); + + let mut owner = Command::new(std::env::current_exe().unwrap()) + .arg(TEST_NAME) + .arg("--exact") + .arg("--nocapture") + .env(CHILD_ROLE, "owner") + .env(CHILD_HOME, temp.path()) + .env(OWNER_ACQUIRED, &owner_acquired) + .stdin(Stdio::null()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .spawn() + .unwrap(); + + let owner_deadline = Instant::now() + Duration::from_secs(5); + while !owner_acquired.is_file() { + if let Some(status) = owner.try_wait().unwrap() { + panic!("store-lock owner exited before acquiring install.lock: {status}"); + } + if Instant::now() >= owner_deadline { + let _ = owner.kill(); + let _ = owner.wait(); + panic!("store-lock owner did not acquire install.lock"); + } + std::thread::sleep(Duration::from_millis(20)); + } + + let mut waiter = Command::new(std::env::current_exe().unwrap()) + .arg(TEST_NAME) + .arg("--exact") + .arg("--nocapture") + .env(CHILD_ROLE, "waiter") + .env(CHILD_HOME, temp.path()) + .env(WAITER_ATTEMPTING, &waiter_attempting) + .env(WAITER_ACQUIRED, &waiter_acquired) + .stdin(Stdio::null()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .spawn() + .unwrap(); + + let attempting_deadline = Instant::now() + Duration::from_secs(5); + while !waiter_attempting.is_file() { + if let Some(status) = waiter.try_wait().unwrap() { + let _ = owner.kill(); + let _ = owner.wait(); + panic!("store-lock waiter exited before attempting acquisition: {status}"); + } + if Instant::now() >= attempting_deadline { + let _ = waiter.kill(); + let _ = waiter.wait(); + let _ = owner.kill(); + let _ = owner.wait(); + panic!("store-lock waiter did not reach acquisition"); + } + std::thread::sleep(Duration::from_millis(20)); + } + + std::thread::sleep(Duration::from_millis(200)); + assert!( + !waiter_acquired.exists(), + "waiter acquired install.lock while the owner process was alive" + ); + assert!( + waiter.try_wait().unwrap().is_none(), + "store-lock waiter exited before owner termination" + ); + + owner.kill().unwrap(); + let owner_status = owner.wait().unwrap(); + assert!( + !owner_status.success(), + "owner process unexpectedly exited successfully instead of being terminated" + ); + + let release_deadline = Instant::now() + Duration::from_secs(5); + loop { + if let Some(status) = waiter.try_wait().unwrap() { + assert!( + status.success(), + "store-lock waiter failed after owner exit: {status}" + ); + break; + } + if Instant::now() >= release_deadline { + let _ = waiter.kill(); + let _ = waiter.wait(); + panic!("store-lock waiter did not wake after owner process termination"); + } + std::thread::sleep(Duration::from_millis(20)); + } + assert!( + waiter_acquired.is_file(), + "waiter exited without acquiring install.lock after owner termination" + ); + } + #[test] fn gc_survives_hostile_max_age() { // A hostile age (`u64::MAX` seconds) would overflow naive epoch math From dfa2e190f3a2b4f049358e7d8594857ceb814b52 Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 3 Aug 2026 12:11:16 -0500 Subject: [PATCH 10/16] ci: remove owner-exit lock test bootstrap --- .github/add_owner_exit_lock_test.py | 149 ---------------------------- 1 file changed, 149 deletions(-) delete mode 100644 .github/add_owner_exit_lock_test.py diff --git a/.github/add_owner_exit_lock_test.py b/.github/add_owner_exit_lock_test.py deleted file mode 100644 index 3d33e717..00000000 --- a/.github/add_owner_exit_lock_test.py +++ /dev/null @@ -1,149 +0,0 @@ -from pathlib import Path - - -store = Path("src/store.rs") -text = store.read_text() -test_name = "fn install_process_lock_is_released_when_owner_process_is_terminated()" - -if test_name not in text: - marker = ''' #[test] - fn gc_survives_hostile_max_age() { -''' - replacement = ''' #[test] - fn install_process_lock_is_released_when_owner_process_is_terminated() { - const CHILD_ROLE: &str = "ZED_PKG_TEST_STORE_LOCK_EXIT_ROLE"; - const CHILD_HOME: &str = "ZED_PKG_TEST_STORE_LOCK_EXIT_HOME"; - const OWNER_ACQUIRED: &str = "ZED_PKG_TEST_STORE_LOCK_OWNER_ACQUIRED"; - const WAITER_ATTEMPTING: &str = "ZED_PKG_TEST_STORE_LOCK_WAITER_ATTEMPTING"; - const WAITER_ACQUIRED: &str = "ZED_PKG_TEST_STORE_LOCK_WAITER_ACQUIRED"; - const TEST_NAME: &str = - "store::tests::install_process_lock_is_released_when_owner_process_is_terminated"; - - if let Some(role) = std::env::var_os(CHILD_ROLE) { - let home = PathBuf::from(std::env::var_os(CHILD_HOME).unwrap()); - let store = Store::new(&home); - match role.to_string_lossy().as_ref() { - "owner" => { - let _owner = store.install_lock().unwrap(); - fs::write(std::env::var_os(OWNER_ACQUIRED).unwrap(), b"acquired").unwrap(); - loop { - std::thread::sleep(Duration::from_secs(60)); - } - } - "waiter" => { - fs::write(std::env::var_os(WAITER_ATTEMPTING).unwrap(), b"attempting") - .unwrap(); - let _waiter = store.install_lock().unwrap(); - fs::write(std::env::var_os(WAITER_ACQUIRED).unwrap(), b"acquired").unwrap(); - return; - } - other => panic!("unexpected child role: {other}"), - } - } - - let temp = tempfile::tempdir().unwrap(); - let owner_acquired = temp.path().join("store-lock-owner-acquired"); - let waiter_attempting = temp.path().join("store-lock-waiter-attempting-after-exit"); - let waiter_acquired = temp.path().join("store-lock-waiter-acquired-after-exit"); - - let mut owner = Command::new(std::env::current_exe().unwrap()) - .arg(TEST_NAME) - .arg("--exact") - .arg("--nocapture") - .env(CHILD_ROLE, "owner") - .env(CHILD_HOME, temp.path()) - .env(OWNER_ACQUIRED, &owner_acquired) - .stdin(Stdio::null()) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .spawn() - .unwrap(); - - let owner_deadline = Instant::now() + Duration::from_secs(5); - while !owner_acquired.is_file() { - if let Some(status) = owner.try_wait().unwrap() { - panic!("store-lock owner exited before acquiring install.lock: {status}"); - } - if Instant::now() >= owner_deadline { - let _ = owner.kill(); - let _ = owner.wait(); - panic!("store-lock owner did not acquire install.lock"); - } - std::thread::sleep(Duration::from_millis(20)); - } - - let mut waiter = Command::new(std::env::current_exe().unwrap()) - .arg(TEST_NAME) - .arg("--exact") - .arg("--nocapture") - .env(CHILD_ROLE, "waiter") - .env(CHILD_HOME, temp.path()) - .env(WAITER_ATTEMPTING, &waiter_attempting) - .env(WAITER_ACQUIRED, &waiter_acquired) - .stdin(Stdio::null()) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .spawn() - .unwrap(); - - let attempting_deadline = Instant::now() + Duration::from_secs(5); - while !waiter_attempting.is_file() { - if let Some(status) = waiter.try_wait().unwrap() { - let _ = owner.kill(); - let _ = owner.wait(); - panic!("store-lock waiter exited before attempting acquisition: {status}"); - } - if Instant::now() >= attempting_deadline { - let _ = waiter.kill(); - let _ = waiter.wait(); - let _ = owner.kill(); - let _ = owner.wait(); - panic!("store-lock waiter did not reach acquisition"); - } - std::thread::sleep(Duration::from_millis(20)); - } - - std::thread::sleep(Duration::from_millis(200)); - assert!( - !waiter_acquired.exists(), - "waiter acquired install.lock while the owner process was alive" - ); - assert!( - waiter.try_wait().unwrap().is_none(), - "store-lock waiter exited before owner termination" - ); - - owner.kill().unwrap(); - let owner_status = owner.wait().unwrap(); - assert!( - !owner_status.success(), - "owner process unexpectedly exited successfully instead of being terminated" - ); - - let release_deadline = Instant::now() + Duration::from_secs(5); - loop { - if let Some(status) = waiter.try_wait().unwrap() { - assert!(status.success(), "store-lock waiter failed after owner exit: {status}"); - break; - } - if Instant::now() >= release_deadline { - let _ = waiter.kill(); - let _ = waiter.wait(); - panic!("store-lock waiter did not wake after owner process termination"); - } - std::thread::sleep(Duration::from_millis(20)); - } - assert!( - waiter_acquired.is_file(), - "waiter exited without acquiring install.lock after owner termination" - ); - } - - #[test] - fn gc_survives_hostile_max_age() { -''' - if text.count(marker) != 1: - raise RuntimeError("could not locate insertion point for owner-exit lock test") - text = text.replace(marker, replacement, 1) - -store.write_text(text) From f84a7540f098a9f14537fa6ca8c69ef51eccc86c Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 3 Aug 2026 12:11:22 -0500 Subject: [PATCH 11/16] ci: remove store-lock patch bootstrap --- .github/blocking_store_process_locks.py | 155 ------------------------ 1 file changed, 155 deletions(-) delete mode 100644 .github/blocking_store_process_locks.py diff --git a/.github/blocking_store_process_locks.py b/.github/blocking_store_process_locks.py deleted file mode 100644 index 0a237ecb..00000000 --- a/.github/blocking_store_process_locks.py +++ /dev/null @@ -1,155 +0,0 @@ -from pathlib import Path - - -store = Path("src/store.rs") -text = store.read_text() - -new_lock_marker = "/// A descriptor-backed process lock held for the life of the guard." -if new_lock_marker not in text: - old_import = "use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};" - new_import = "use std::time::{Duration, SystemTime, UNIX_EPOCH};" - if text.count(old_import) != 1: - raise RuntimeError("could not locate the store time import") - text = text.replace(old_import, new_import, 1) - - start_marker = "/// An advisory (flock-based) process lock held for the life of the guard." - end_marker = "/// What `Store::gc` did (or, with `dry_run`, would do)." - start = text.index(start_marker) - end = text.index(end_marker, start) - replacement = '''/// A descriptor-backed process lock held for the life of the guard. -/// -/// Acquisition uses the operating system's blocking lock primitive directly: -/// `flock`/`fcntl` semantics on Unix and `LockFileEx` semantics on Windows via -/// `fs2`. Contended callers sleep in the kernel and wake when the owner drops -/// the descriptor or exits. There is no retry timer, jitter loop, stale-file -/// reclamation, or userspace polling. -pub struct ProcessLock { - _file: fs::File, -} - -impl ProcessLock { - fn acquire(path: &Path, waiting_on: &str) -> Result { - if let Some(parent) = path.parent() { - fs::create_dir_all(parent)?; - } - let file = fs::OpenOptions::new() - .create(true) - .read(true) - .write(true) - .truncate(false) - .open(path) - .with_context(|| format!("opening lock file {}", path.display()))?; - - file.lock_exclusive().with_context(|| { - format!( - "waiting for {waiting_on} through operating-system lock {}", - path.display() - ) - })?; - Ok(Self { _file: file }) - } -} - -''' - text = text[:start] + replacement + text[end:] - -new_test_name = "fn install_process_lock_blocks_without_polling_until_owner_release()" -if new_test_name not in text: - old_test_header = '''#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn gc_survives_hostile_max_age() { -''' - new_test_header = '''#[cfg(test)] -mod tests { - use std::process::{Command, Stdio}; - use std::time::Instant; - - use super::*; - - #[test] - fn install_process_lock_blocks_without_polling_until_owner_release() { - const CHILD_HOME: &str = "ZED_PKG_TEST_STORE_LOCK_CHILD_HOME"; - const CHILD_ATTEMPTING: &str = "ZED_PKG_TEST_STORE_LOCK_CHILD_ATTEMPTING"; - const CHILD_ACQUIRED: &str = "ZED_PKG_TEST_STORE_LOCK_CHILD_ACQUIRED"; - const TEST_NAME: &str = - "store::tests::install_process_lock_blocks_without_polling_until_owner_release"; - - if let Some(home) = std::env::var_os(CHILD_HOME) { - let attempting = PathBuf::from(std::env::var_os(CHILD_ATTEMPTING).unwrap()); - let acquired = PathBuf::from(std::env::var_os(CHILD_ACQUIRED).unwrap()); - fs::write(attempting, b"attempting").unwrap(); - let store = Store::new(Path::new(&home)); - let _waiter = store.install_lock().unwrap(); - fs::write(acquired, b"acquired").unwrap(); - return; - } - - let temp = tempfile::tempdir().unwrap(); - let store = Store::new(temp.path()); - let owner = store.install_lock().unwrap(); - let attempting = temp.path().join("store-lock-waiter-attempting"); - let acquired = temp.path().join("store-lock-waiter-acquired"); - let mut child = Command::new(std::env::current_exe().unwrap()) - .arg(TEST_NAME) - .arg("--exact") - .arg("--nocapture") - .env(CHILD_HOME, temp.path()) - .env(CHILD_ATTEMPTING, &attempting) - .env(CHILD_ACQUIRED, &acquired) - .stdin(Stdio::null()) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .spawn() - .unwrap(); - - let attempting_deadline = Instant::now() + Duration::from_secs(5); - while !attempting.is_file() { - if let Some(status) = child.try_wait().unwrap() { - panic!("store-lock waiter exited before attempting acquisition: {status}"); - } - if Instant::now() >= attempting_deadline { - let _ = child.kill(); - let _ = child.wait(); - panic!("store-lock waiter did not reach acquisition"); - } - std::thread::sleep(Duration::from_millis(20)); - } - - std::thread::sleep(Duration::from_millis(200)); - assert!( - !acquired.exists(), - "a separate process acquired install.lock while the owner held it" - ); - assert!( - child.try_wait().unwrap().is_none(), - "store-lock waiter exited before owner release" - ); - - drop(owner); - let release_deadline = Instant::now() + Duration::from_secs(5); - loop { - if let Some(status) = child.try_wait().unwrap() { - assert!(status.success(), "store-lock waiter failed: {status}"); - break; - } - if Instant::now() >= release_deadline { - let _ = child.kill(); - let _ = child.wait(); - panic!("store-lock waiter did not wake after owner release"); - } - std::thread::sleep(Duration::from_millis(20)); - } - assert!(acquired.is_file(), "waiter exited without acquiring install.lock"); - } - - #[test] - fn gc_survives_hostile_max_age() { -''' - if text.count(old_test_header) != 1: - raise RuntimeError("could not locate store test module header") - text = text.replace(old_test_header, new_test_header, 1) - -store.write_text(text) From 3a36b527de3264f616b32b84a2ca35bec554aaac Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 3 Aug 2026 12:12:22 -0500 Subject: [PATCH 12/16] ci: keep permanent Linux blocking-lock regression gate --- .../apply-blocking-store-process-locks.yml | 98 +++++++++---------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/.github/workflows/apply-blocking-store-process-locks.yml b/.github/workflows/apply-blocking-store-process-locks.yml index 6fb1f425..85a39020 100644 --- a/.github/workflows/apply-blocking-store-process-locks.yml +++ b/.github/workflows/apply-blocking-store-process-locks.yml @@ -1,70 +1,70 @@ -name: apply blocking store process locks +name: blocking store process locks on: pull_request: - branches: - - agent/recursive-lock-cross-platform + paths: + - src/store.rs + - src/install_graph.rs + - src/install_graph/** + - Cargo.toml + - Cargo.lock + - .github/workflows/apply-blocking-store-process-locks.yml + push: + branches: [main] + paths: + - src/store.rs + - src/install_graph.rs + - src/install_graph/** + - Cargo.toml + - Cargo.lock + - .github/workflows/apply-blocking-store-process-locks.yml + workflow_dispatch: permissions: contents: read concurrency: - group: apply-blocking-store-process-locks-${{ github.event.pull_request.number }} + group: blocking-store-process-locks-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + jobs: - apply: - if: github.actor != 'github-actions[bot]' + linux-kernel-lock: + name: Linux process-lock and recursive-cache contracts runs-on: ubuntu-24.04 - timeout-minutes: 45 - permissions: - contents: write + timeout-minutes: 35 steps: - - name: Check out store-lock branch + - name: Check out zed-cli uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: agent/blocking-store-process-locks - fetch-depth: 0 + fetch-depth: 1 persist-credentials: false show-progress: false - - name: Apply exact kernel-blocking lock patch + - name: Install stable Rust with formatting and lint support run: | - set -euo pipefail - python3 .github/blocking_store_process_locks.py - python3 .github/add_owner_exit_lock_test.py - cargo fmt --all - git diff --check + rustup toolchain install stable --profile minimal --component rustfmt,clippy + rustup default stable + rustc --version --verbose + cargo --version --verbose - - name: Validate store and recursive lock contracts - run: | - set -euo pipefail - rustup component add rustfmt clippy - cargo fmt --all --check - cargo test --locked --lib \ - store::tests::install_process_lock_blocks_without_polling_until_owner_release \ - -- --exact --nocapture - cargo test --locked --lib \ - store::tests::install_process_lock_is_released_when_owner_process_is_terminated \ - -- --exact --nocapture - cargo test --locked --lib store::tests - cargo test --locked --lib install_graph::tests - cargo clippy --locked --all-targets -- -D warnings + - name: Check formatting + run: cargo fmt --all --check - - name: Publish validated store implementation - env: - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - git add src/store.rs - git diff --cached --check - if git diff --cached --quiet; then - echo 'kernel-blocking store implementation is already published' - exit 0 - fi - git config user.name 'github-actions[bot]' - git config user.email '41898282+github-actions[bot]@users.noreply.github.com' - git commit -m 'fix: block store locks in the kernel' - git push \ - "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" \ - HEAD:agent/blocking-store-process-locks + - name: Prove a separate process blocks until orderly release + run: cargo test --locked --lib store::tests::install_process_lock_blocks_without_polling_until_owner_release -- --exact --nocapture + + - name: Prove owner termination releases the descriptor lock + run: cargo test --locked --lib store::tests::install_process_lock_is_released_when_owner_process_is_terminated -- --exact --nocapture + + - name: Run all store process-lock tests + run: cargo test --locked --lib store::tests + + - name: Run recursive acquisition and cache-race tests + run: cargo test --locked --lib install_graph::tests + + - name: Deny Rust warnings + run: cargo clippy --locked --all-targets -- -D warnings From e16fe93fc8eb7d8d8434cc57dd79bfe3ac5d0fc8 Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 3 Aug 2026 13:05:37 -0500 Subject: [PATCH 13/16] ci: synchronize final cross-platform lock ancestry --- .github/workflows/sync-lock-stack.yml | 68 +++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 .github/workflows/sync-lock-stack.yml diff --git a/.github/workflows/sync-lock-stack.yml b/.github/workflows/sync-lock-stack.yml new file mode 100644 index 00000000..05fcbce9 --- /dev/null +++ b/.github/workflows/sync-lock-stack.yml @@ -0,0 +1,68 @@ +name: synchronize blocking-lock stack + +on: + push: + branches: + - agent/blocking-store-process-locks + paths: + - .github/workflows/sync-lock-stack.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: synchronize-blocking-lock-stack + cancel-in-progress: true + +jobs: + merge-final-cross-platform-base: + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: write + steps: + - name: Check out the blocking-lock branch + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: agent/blocking-store-process-locks + fetch-depth: 0 + persist-credentials: false + show-progress: false + + - name: Merge the final cross-platform base semantically + env: + GH_TOKEN: ${{ github.token }} + BASE_SHA: b7893ae396d20badeeb56ba36019edfa69e6d0e0 + run: | + set -euo pipefail + if git merge-base --is-ancestor "$BASE_SHA" HEAD; then + echo "cross-platform lock base already present" + exit 0 + fi + + git fetch --no-tags origin agent/recursive-lock-cross-platform + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + + set +e + git merge --no-ff --no-commit FETCH_HEAD + merge_status=$? + set -e + + if [ "$merge_status" -ne 0 ]; then + conflicts="$(git diff --name-only --diff-filter=U)" + test "$conflicts" = '.github/workflows/recursive-install-windows.yml' + git checkout --ours -- .github/workflows/recursive-install-windows.yml + grep -F 'cancel-in-progress: true' .github/workflows/recursive-install-windows.yml + grep -F 'owner termination releases the lower-level store lock' .github/workflows/recursive-install-windows.yml + git add .github/workflows/recursive-install-windows.yml + fi + + test -z "$(git diff --name-only --diff-filter=U)" + git diff --check + git commit -m 'merge: synchronize cross-platform lock base' + git push \ + "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" \ + HEAD:agent/blocking-store-process-locks From a1009fbafff377c5f460a8330c3cf5cd7090fb0d Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 3 Aug 2026 13:07:40 -0500 Subject: [PATCH 14/16] merge: align conflicted workflow with final cross-platform base --- .github/workflows/recursive-install-windows.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/recursive-install-windows.yml b/.github/workflows/recursive-install-windows.yml index eda09e0f..8fdb860c 100644 --- a/.github/workflows/recursive-install-windows.yml +++ b/.github/workflows/recursive-install-windows.yml @@ -79,9 +79,3 @@ jobs: - name: Prove failed downloads leave no cache publication run: cargo test --locked --manifest-path zed-cli/Cargo.toml --lib install_graph::tests::failed_downloads_remove_staging_files_and_never_publish_cache_entries -- --exact --nocapture - - - name: Prove the lower-level store lock blocks across processes - run: cargo test --locked --manifest-path zed-cli/Cargo.toml --lib store::tests::install_process_lock_blocks_without_polling_until_owner_release -- --exact --nocapture - - - name: Prove owner termination releases the lower-level store lock - run: cargo test --locked --manifest-path zed-cli/Cargo.toml --lib store::tests::install_process_lock_is_released_when_owner_process_is_terminated -- --exact --nocapture From 19c962ec82b77ec6387cf0123959789cd0c7488b Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 3 Aug 2026 13:08:17 -0500 Subject: [PATCH 15/16] ci: remove completed lock-stack synchronization workflow --- .github/workflows/sync-lock-stack.yml | 68 --------------------------- 1 file changed, 68 deletions(-) delete mode 100644 .github/workflows/sync-lock-stack.yml diff --git a/.github/workflows/sync-lock-stack.yml b/.github/workflows/sync-lock-stack.yml deleted file mode 100644 index 05fcbce9..00000000 --- a/.github/workflows/sync-lock-stack.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: synchronize blocking-lock stack - -on: - push: - branches: - - agent/blocking-store-process-locks - paths: - - .github/workflows/sync-lock-stack.yml - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: synchronize-blocking-lock-stack - cancel-in-progress: true - -jobs: - merge-final-cross-platform-base: - if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-24.04 - timeout-minutes: 10 - permissions: - contents: write - steps: - - name: Check out the blocking-lock branch - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: agent/blocking-store-process-locks - fetch-depth: 0 - persist-credentials: false - show-progress: false - - - name: Merge the final cross-platform base semantically - env: - GH_TOKEN: ${{ github.token }} - BASE_SHA: b7893ae396d20badeeb56ba36019edfa69e6d0e0 - run: | - set -euo pipefail - if git merge-base --is-ancestor "$BASE_SHA" HEAD; then - echo "cross-platform lock base already present" - exit 0 - fi - - git fetch --no-tags origin agent/recursive-lock-cross-platform - git config user.name 'github-actions[bot]' - git config user.email '41898282+github-actions[bot]@users.noreply.github.com' - - set +e - git merge --no-ff --no-commit FETCH_HEAD - merge_status=$? - set -e - - if [ "$merge_status" -ne 0 ]; then - conflicts="$(git diff --name-only --diff-filter=U)" - test "$conflicts" = '.github/workflows/recursive-install-windows.yml' - git checkout --ours -- .github/workflows/recursive-install-windows.yml - grep -F 'cancel-in-progress: true' .github/workflows/recursive-install-windows.yml - grep -F 'owner termination releases the lower-level store lock' .github/workflows/recursive-install-windows.yml - git add .github/workflows/recursive-install-windows.yml - fi - - test -z "$(git diff --name-only --diff-filter=U)" - git diff --check - git commit -m 'merge: synchronize cross-platform lock base' - git push \ - "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" \ - HEAD:agent/blocking-store-process-locks From 1232fc19126b96f8e8cfa8585e92b247a1ce693c Mon Sep 17 00:00:00 2001 From: Alexander Mills Date: Mon, 3 Aug 2026 13:08:38 -0500 Subject: [PATCH 16/16] test: restore expanded Windows store-lock regressions after base sync --- .github/workflows/recursive-install-windows.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/recursive-install-windows.yml b/.github/workflows/recursive-install-windows.yml index 8fdb860c..eda09e0f 100644 --- a/.github/workflows/recursive-install-windows.yml +++ b/.github/workflows/recursive-install-windows.yml @@ -79,3 +79,9 @@ jobs: - name: Prove failed downloads leave no cache publication run: cargo test --locked --manifest-path zed-cli/Cargo.toml --lib install_graph::tests::failed_downloads_remove_staging_files_and_never_publish_cache_entries -- --exact --nocapture + + - name: Prove the lower-level store lock blocks across processes + run: cargo test --locked --manifest-path zed-cli/Cargo.toml --lib store::tests::install_process_lock_blocks_without_polling_until_owner_release -- --exact --nocapture + + - name: Prove owner termination releases the lower-level store lock + run: cargo test --locked --manifest-path zed-cli/Cargo.toml --lib store::tests::install_process_lock_is_released_when_owner_process_is_terminated -- --exact --nocapture