From 2d24e0897a299ebcc5d4353535959ee14a4676b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:36:27 +0000 Subject: [PATCH 1/3] Checkpoint task recovery and bound worker lifetimes (#333) --- oxa/docs/work-state.md | 56 +++ oxa/docs/work.md | 77 ++-- oxa/oxa-cli/src/work_checks.rs | 39 ++ oxa/oxa-cli/src/work_cmd.rs | 116 ++++-- oxa/oxa-cli/tests/work_cli.rs | 15 +- oxa/oxa-cli/tests/work_continue.rs | 5 +- oxa/oxa-cli/tests/work_state.rs | 555 ++++++++++++++++++++++++++ oxa/oxa-work/src/checkpoint.rs | 231 +++++++++++ oxa/oxa-work/src/execute.rs | 254 ++++++++++++ oxa/oxa-work/src/lib.rs | 359 ++++++++++------- oxa/oxa-work/src/state.rs | 603 +++++++++++++++++++++++++++++ oxa/oxa-work/src/status.rs | 178 +++++++++ oxa/oxa-work/src/tree.rs | 13 +- oxa/oxa-work/src/worker.rs | 89 ++++- oxa/oxa-work/src/worker_wait.rs | 261 +++++++++++++ 15 files changed, 2632 insertions(+), 219 deletions(-) create mode 100644 oxa/docs/work-state.md create mode 100644 oxa/oxa-cli/tests/work_state.rs create mode 100644 oxa/oxa-work/src/checkpoint.rs create mode 100644 oxa/oxa-work/src/execute.rs create mode 100644 oxa/oxa-work/src/state.rs create mode 100644 oxa/oxa-work/src/status.rs create mode 100644 oxa/oxa-work/src/worker_wait.rs diff --git a/oxa/docs/work-state.md b/oxa/docs/work-state.md new file mode 100644 index 000000000..462a72bbe --- /dev/null +++ b/oxa/docs/work-state.md @@ -0,0 +1,56 @@ +# Runner checkpoint integration + +The runner is the sole writer of task state. `run_task_with_observer` accepts a +`CheckpointObserver: Sync` with `checkpoint(&Record) -> Result<()>`. Observers +consume public typed records; they never read or rewrite private `State` files. +The CLI currently supplies a no-op while the separately owned facts store is +integrated. The hook is not itself an event journal or situation implementation. + +Checkpoints live outside writable Git binds in the protected per-repository +runner root. `.json` stores a versioned private state with its task spec, +frozen execution inputs, offline policy, deadline and public record. `.lock` +is a stable inherited kernel lock. Status is read-only and does not acquire it. +Facts storage receives a separate `facts/` child of this protected root, avoiding +collisions with task files. Standing data also remains outside Git. + +`Record.checkpoint_revision` is repository-wide and strictly increases. The event +producer domain is the fixed string `runner`, not one source per task. Event +sequence numbers belong to the facts store and are distinct from checkpoint +revisions. Reservations that did not reach the task rename can leave revision +gaps; they must not create event-sequence gaps. + +A short repository kernel lock serializes revision allocation, task-file +replacement, observation and acknowledgement. `.checkpoint-counter` contains +the last reserved revision and at most one pending task ID. A new writer must +recover that pending durable checkpoint before reserving another revision: + +1. Reserve the revision and pending identity atomically, then sync the directory. +2. Write and sync the complete task checkpoint, replace its canonical name + atomically, and sync the directory. The first task publication is create-only. +3. Call the observer with the durable public record. +4. Atomically acknowledge that observation in the repository counter. + +A failure stops the batch and poisons that writer. Recovery replays the same +durable revision, including a lost acknowledgement after an append. The observer +must deduplicate that identity and reject different contents under it. Older +acknowledged task records are not replayed: the source table needs one bounded +runner high-water mark, without completed-task tombstones. A missing/corrupt +initialized counter fails closed rather than resetting revisions. +`recover_checkpoints(root, observer)` exposes that pending-observation recovery +before a lead reads its facts view; it never acquires a task lock, changes a task +file or starts a worker. The plain `status` command remains entirely read-only. + +`checkpoint_kind` distinguishes `task_transition` from `worker_finished`. The +worker result is saved at acceptance/recheck with the latter kind; a separate +revision enters that phase before running checks. A crash between the two does +not restart the worker. A preparing transition can establish a `TaskStarted` +location; subsequent transitions update the closed step and optional session. +Worker completion, local acceptance, publication, external acceptance and merge +remain separate observations. Only a merge is landed work. + +Records expose fixed outcomes and typed deadline interruption. Their `detail` +is bounded diagnostic context for the owner, not a facts payload: event adapters +select typed fields and fixed reason codes and never emit prompts or raw logs. +Saved head/tree IDs describe the actual saved commit. They do not prove that an +ignored helper or build artifact was absent while local checks ran; publication +must verify commit-bound acceptance separately. diff --git a/oxa/docs/work.md b/oxa/docs/work.md index 73e973578..12fc7d393 100644 --- a/oxa/docs/work.md +++ b/oxa/docs/work.md @@ -7,7 +7,14 @@ Run it inside the project repository. The JSON file is an array: [{"id":"docs-fix","model":"your-model","branch":"agent/docs-fix","base":"main","prompt":"Fix documentation links."}] ``` -`base` defaults to `main`. IDs contain letters, digits, hyphens or underscores. +`base` defaults to `main`. IDs contain 1–128 letters, digits, hyphens or underscores. +Optional `max_time` is a positive integer number of seconds, defaulting to 2700 +(45 minutes). The task's saved deadline is shared by its worker, repair and crash +recovery; a repair does not get a fresh budget. Each child receives its remaining +allowance as `--max-time` for one graceful wrap-up turn. If it remains alive 60 +seconds after that allowance, the runner sends SIGTERM, then SIGKILL after another +10 seconds. The result is interrupted and its worktree is retained. `kill` must be +available as a trusted executable outside the project; status reads do not need it. IDs and branches must be unique within one file. Existing task branches can be continued. For a GitHub issue use `oxa work --issue 123 --model your-model`. This explicitly invokes your installed, @@ -36,7 +43,7 @@ creates detached Git metadata with callbacks suppressed on the host, then create the task branch and populates the fresh worktree inside the normal sandbox. Reference hooks, checkout hooks and smudge filters stay inside it; a reference hook that rejects the task branch still blocks the task. -missing bwrap blocks before checkout or a worker starts. Configured hook directories +Missing bwrap blocks before checkout or a worker starts. Configured hook directories keep their normal read-only protection: a checkout that needs to populate a tracked protected hook directory blocks with the worktree retained. Acceptance commands are frozen from the captured base commit before checkout @@ -53,37 +60,52 @@ error blocks the task rather than producing a clean verdict. Agent-internal done checks may also run when the project is trusted. The runner reads the child's exit code, terminal `oxa_result` and session header, -without retaining its event stream. A blocked, failed or interrupted worker remains -blocked and does not consume a repair attempt. Failed acceptance after a successful +without retaining its event stream. It checkpoints the session ID as soon as the +header arrives. A blocked or failed worker remains blocked; an interrupted worker +remains interrupted. Neither consumes a repair attempt. Failed acceptance after a successful worker continues that same saved session once, then acceptance runs once more. -JSON lines include the worker result, session ID and captured `base_sha` and report `done`, `nothing_to_do` (no -commits or file changes), `red` (checks still fail), or `blocked` (setup or agent -execution failed). Exit codes are 0 for success, 1 if any task is red or blocked, -and 2 for invalid input or output errors. A task setup failure produces a blocked +JSON lines include the worker result, session ID, captured `base_sha`, saved +`head_sha`/`tree_sha`, step, checkpoint revision and time. Outcomes are `done`, +`nothing_to_do` (no worker commits or file changes), `red` (checks still fail), +`blocked` (setup or execution failed), or `interrupted`. Deadline interruptions +carry a typed `interruption: "deadline"`. Exit codes are 0 for success, 1 if any +task is red, blocked or interrupted, and 2 for invalid input, output or checkpoint +errors. A checkpoint/event-append error stops the entire batch before another +action. An ordinary task setup failure produces a blocked JSON line and does not skip later tasks. If a task cannot be locked or has an incompatible receipt, its existing state is preserved; a pre-worktree failure has -an empty `worktree` in the result. Locks use kernel ownership, so an abandoned lock file -after process death does not permanently block a task. +an empty `worktree` in the result. Locks use kernel ownership and are inherited +by the worker and check/save supervisors. Killing the runner does not allow a +second runner to start while its child still holds that lock. -Worktrees and uncommitted changes remain in `.oxa-worktrees//` +Worktrees remain in `.oxa-worktrees//` beside the repository (or the separate git directory). Private result receipts and locks live under `~/.oxa/agent/work//` (or the configured agent directory), outside the worker's writable Git bind. New receipts are published atomically without overwriting an existing receipt; a validated matching receipt is replaced atomically under the task lock when continuing. Worktrees stay outside that git -directory so the normal sandbox can bind git objects and refs for commits. No automatic -cleanup, commit, push or PR creation is implemented yet. Inspect -retained work before removing it with `git worktree remove`. Review task prompts -and bases before running them. +directory so the normal sandbox can bind git objects and refs for commits. +Before finishing, the runner saves staged, unstaged and untracked work with an +ordinary sandboxed commit, including red, blocked and interrupted work. Normal +hooks remain active. Unresolved merges are kept in progress; a rejecting hook or +remaining dirty files block completion. No automatic cleanup, push or PR creation +is implemented. Keep retained work until its saved commit has been published and +inspected. Review task prompts and bases before running them. -`done` here describes accepted local work, which may be uncommitted. This slice -does not publish it. The saving/publication slices must bind acceptance to a clean -committed tree and verify its recorded commit before pushing. +`done` here describes locally checked and saved work. It does not mean published, +externally accepted or merged. Publication must verify a clean committed export, +bind checks to that commit and verify base-object integrity before pushing; +ignored files in a retained working directory are not commit-bound evidence. -To continue a task, run the same task file again. A matching retained receipt and -linked worktree are reused. The source checkout is never used as the task worktree. -An existing branch checked out elsewhere is blocked. Dirty retained work is kept -and blocked until committed. A clean retained branch gets an ordinary sandboxed +To recover a task, run the same task file again. An unfinished checkpoint resumes +its saved step with the original base, commands, session and offline policy. +Finished successful work is returned without another worker or check run; add +`--rerun` to the task-file or issue command to start another attempt. A task spec +change requires a new ID. A receipt from before durable checkpoints needs an +explicit `--rerun` before adoption. A matching linked worktree is reused; the +source checkout is never used as the task worktree. An existing branch checked +out elsewhere is blocked. Dirty retained work is preserved and saved before +retrying. A clean retained branch gets an ordinary sandboxed merge of the captured base; no reset, stash, force checkout or automatic fetch runs. An unfinished merge is passed to the worker with its old pending commit, the @@ -91,3 +113,14 @@ captured current base and at most twenty conflicted paths. The worker must finis that merge and merge the captured base too. Aborted or unresolved merges and branch switches remain blocked even if acceptance passes. Worker, merge and acceptance failures are retained together. Receipt replacement keeps private modes. + +`oxa work status --json` (also `--status`) reads checkpoints without acquiring a +task lock, changing state, constructing a model or calling GitHub. Its separate +`liveness` is `running`, `finished`, `interrupted`, or `unknown` when the kernel +cannot establish ownership. A missing original parent is not proof of a stopped +child: status also checks the inherited stdin lock. Without `--json` it prints +one short line per task. + +The [checkpoint integration contract](work-state.md) defines the runner's public +event hook. Event/situation/standing stores and their CLI adapters are a separate +integration; this slice does not infer facts from worker prose or raw logs. diff --git a/oxa/oxa-cli/src/work_checks.rs b/oxa/oxa-cli/src/work_checks.rs index e2a3a3ffa..8b265c526 100644 --- a/oxa/oxa-cli/src/work_checks.rs +++ b/oxa/oxa-cli/src/work_checks.rs @@ -78,6 +78,45 @@ pub(crate) fn run( } return Ok(i32::from(result.exit != Some(0))); } + SupervisorOperation::Save { task_id } => { + let sandbox = crate::netguard::with_net(Arc::new(b), &settings); + let message = format!("oxa work: save task {task_id}").replace('\'', "'\\''"); + let spec = ExecSpec { + command: format!( + r#"git --no-replace-objects diff --cached --quiet --no-ext-diff --no-textconv --diff-filter=U +work_index_status=$? +if test "$work_index_status" -ne 0; then exit 2; fi +git --no-replace-objects add --all || exit 2 +git --no-replace-objects rev-parse --verify --quiet MERGE_HEAD >/dev/null +work_merge_status=$? +if test "$work_merge_status" -ne 1; then exit 2; fi +git --no-replace-objects diff --cached --quiet --no-ext-diff --no-textconv +work_changes=$? +if test "$work_changes" -eq 0; then exit 0; fi +if test "$work_changes" -ne 1; then exit 2; fi +git --no-replace-objects commit -m '{message}' +"# + ), + cwd: root, + writable: true, + net: sandbox.network(), + timeout_s: Some(600), + env: git_environment(), + }; + let mut tail = Vec::new(); + let result = sandbox.run_streaming(&spec, &CancelToken::default(), &mut |bytes| { + keep_tail(&mut tail, bytes, 8192); + }); + if result.exit != Some(0) || result.timed_out { + return Err(format!( + "save exit {:?}, timeout {}: {}", + result.exit, + result.timed_out, + String::from_utf8_lossy(&tail) + )); + } + return Ok(0); + } SupervisorOperation::Checks { commands } => commands, }; let shell = oxa_tools::ShellConfig { diff --git a/oxa/oxa-cli/src/work_cmd.rs b/oxa/oxa-cli/src/work_cmd.rs index c73f7aeda..28409514e 100644 --- a/oxa/oxa-cli/src/work_cmd.rs +++ b/oxa/oxa-cli/src/work_cmd.rs @@ -68,33 +68,6 @@ pub(crate) fn run( if root != project { return Err("Git root differs from the source project boundary".into()); } - let is_trusted = trusted(&root, env, dirs)?; - let tasks = match args { - [file] if !file.starts_with('-') => oxa_work::read_tasks(&cwd.join(file))?, - [flag, number, model_flag, model] if flag == "--issue" && model_flag == "--model" => { - if oxa_proto::offline() { - return Err("GitHub issue lookup is unavailable in offline mode".into()); - } - if !is_trusted { - return Err("source project is not trusted; approve it before issue lookup".into()); - } - let gh = crate::netguard::trusted_executable_for( - "gh", - Some(host_path.as_os_str()), - Some(&root), - Some(&dirs.home), - ) - .ok_or("no trusted gh executable outside the source project")?; - vec![oxa_work::issue_task( - &root, - &gh, - &host_path, - number.parse().map_err(|_| "invalid issue number")?, - model.clone(), - )?] - } - _ => return Err("usage: oxa work | --issue N --model MODEL".into()), - }; let common = oxa_work::metadata_git( &git_binary, &host_path, @@ -113,12 +86,79 @@ pub(crate) fn run( .iter() .map(|b| format!("{b:02x}")) .collect::(); + let terminate_binary = crate::netguard::trusted_executable_for( + "kill", + Some(host_path.as_os_str()), + Some(&root), + Some(&dirs.home), + ); let policy = oxa_work::RunPolicy { git_binary, host_path, + terminate_binary, offline: oxa_proto::offline(), storage: dirs.agent_dir.join("work").join(key), }; + if matches!(args.first().map(String::as_str), Some("status" | "--status")) { + let json = match args { + [_] => false, + [_, flag] if flag == "--json" => true, + _ => return Err("usage: oxa work status [--json]".into()), + }; + let rows = oxa_work::status(&policy.storage, &binary)?; + if json { + serde_json::to_writer(&mut *out, &rows).map_err(|e| e.to_string())?; + writeln!(out).map_err(|e| e.to_string())?; + } else { + for row in rows { + let outcome = serde_json::to_value(row.record.outcome).map_err(|e| e.to_string())?; + let step = serde_json::to_value(row.record.step).map_err(|e| e.to_string())?; + writeln!( + out, + "{} {} {} {} {:?}", + row.record.id, + outcome.as_str().unwrap_or("unknown"), + row.record.checkpoint_at_ms, + step.as_str().unwrap_or("unknown"), + row.liveness + ) + .map_err(|e| e.to_string())?; + } + } + return Ok(0); + } + let rerun = args.last().is_some_and(|arg| arg == "--rerun"); + let task_args = if rerun { &args[..args.len() - 1] } else { args }; + let is_trusted = trusted(&root, env, dirs)?; + let tasks = match task_args { + [file] if !file.starts_with('-') => oxa_work::read_tasks(&cwd.join(file))?, + [flag, number, model_flag, model] if flag == "--issue" && model_flag == "--model" => { + if oxa_proto::offline() { + return Err("GitHub issue lookup is unavailable in offline mode".into()); + } + if !is_trusted { + return Err("source project is not trusted; approve it before issue lookup".into()); + } + let gh = crate::netguard::trusted_executable_for( + "gh", + Some(policy.host_path.as_os_str()), + Some(&root), + Some(&dirs.home), + ) + .ok_or("no trusted gh executable outside the source project")?; + vec![oxa_work::issue_task( + &root, + &gh, + &policy.host_path, + number.parse().map_err(|_| "invalid issue number")?, + model.clone(), + )?] + } + _ => return Err( + "usage: oxa work [--rerun] | --issue N --model MODEL [--rerun] | status [--json]" + .into(), + ), + }; // Freeze every task before checkout hooks or a first worker can change // shared refs. A bad plan remains a per-task error, not a batch abort. let planned: Vec<_> = tasks @@ -139,16 +179,28 @@ pub(crate) fn run( for task in planned { let record = match task { Ok(plan) => { - let mut record = oxa_work::run_task_with_policy(&root, &binary, &plan, &policy) - .unwrap_or_else(|detail| oxa_work::blocked_record(plan.task(), detail)); - record.base_sha = Some(plan.base_sha().to_owned()); - record + match oxa_work::run_task_with_observer( + &root, + &binary, + &plan, + &policy, + rerun, + &|_: &oxa_work::Record| Ok(()), + ) { + Ok(record) => record, + Err(oxa_work::RunError::Blocked(detail)) => { + let mut record = oxa_work::blocked_record(plan.task(), detail); + record.base_sha = Some(plan.base_sha().to_owned()); + record + } + Err(oxa_work::RunError::Checkpoint(error)) => return Err(error), + } } Err(record) => *record, }; if matches!( record.outcome, - oxa_work::Outcome::Red | oxa_work::Outcome::Blocked + oxa_work::Outcome::Red | oxa_work::Outcome::Blocked | oxa_work::Outcome::Interrupted ) { code = 1; } diff --git a/oxa/oxa-cli/tests/work_cli.rs b/oxa/oxa-cli/tests/work_cli.rs index 0f222ac4c..960687cb2 100644 --- a/oxa/oxa-cli/tests/work_cli.rs +++ b/oxa/oxa-cli/tests/work_cli.rs @@ -33,6 +33,8 @@ fn setup(env: &TestEnv, done: &str) { git(env, &env.work(), &["init", "--bare", "remote.git"]); git(env, &env.work(), &["init", "-b", "main", "project"]); let project = env.work().join("project"); + git(env, &project, &["config", "user.name", "Test"]); + git(env, &project, &["config", "user.email", "test@example.invalid"]); env.write( "work/project/.oxa/settings.json", &format!(r#"{{"done":{done}}}"#), @@ -1011,7 +1013,7 @@ fn worktree_reference_hooks_run_only_inside_the_sandbox() { setup(&env, "[]"); let hook = env.write( "work/project/.git/hooks/reference-transaction", - "#!/bin/sh\nprintf escaped > \"$HOME/worktree-reference-escape\" 2>/dev/null || true\nprintf ran >> reference-hook-ran\ncat >/dev/null\n", + "#!/bin/sh\nprintf escaped > \"$HOME/worktree-reference-escape\" 2>/dev/null || true\nprintf ran > reference-hook-ran\ncat >/dev/null\n", ); std::fs::set_permissions(hook, std::fs::Permissions::from_mode(0o755)).unwrap(); let record = run(&env, vec![ScriptTurn::text("finished")]); @@ -1340,8 +1342,11 @@ fn security_failed_receipt_write_never_publishes_a_torn_record() { ); let out = child.wait_with_output().unwrap(); assert!(!out.status.success()); - assert!( - !storage(&env).join("test.json").exists(), - "interrupted receipt write left a canonical partial record" - ); + let retained: serde_json::Value = + serde_json::from_slice(&std::fs::read(storage(&env).join("test.json")).unwrap()) + .expect("the previously durable checkpoint remains complete JSON"); + assert_eq!(retained["version"], 1); + assert_eq!(retained["record"]["step"], "worker", "{retained}"); + assert_eq!(retained["record"]["outcome"], "running", "{retained}"); + assert!(retained["record"]["checkpoint_revision"].as_u64().unwrap() > 0); } diff --git a/oxa/oxa-cli/tests/work_continue.rs b/oxa/oxa-cli/tests/work_continue.rs index a0d7de429..dc7492baf 100644 --- a/oxa/oxa-cli/tests/work_continue.rs +++ b/oxa/oxa-cli/tests/work_continue.rs @@ -160,12 +160,13 @@ fn dirty_retained_worktree_is_blocked_without_dropping_edits() { ], ); assert_eq!(first["outcome"], "done", "{first}"); - let (record, requests) = run_observed(&env, vec![], &["work", "../tasks.json"], None); + std::fs::write(tree(&first).join("notes.txt"), "new retained edits").unwrap(); + let (record, requests) = run_observed(&env, vec![], &["work", "../tasks.json", "--rerun"], None); assert_eq!(record["outcome"], "blocked", "{record}"); assert!(requests.is_empty()); assert_eq!( std::fs::read_to_string(tree(&record).join("notes.txt")).unwrap(), - "keep me" + "new retained edits" ); } #[test] diff --git a/oxa/oxa-cli/tests/work_state.rs b/oxa/oxa-cli/tests/work_state.rs new file mode 100644 index 000000000..ce4bff1ad --- /dev/null +++ b/oxa/oxa-cli/tests/work_state.rs @@ -0,0 +1,555 @@ +//! Durable runner recovery through the real CLI, sandbox, local Git and fake engine. +#[path = "common/work.rs"] +mod work; +use oxa_fake::{FakeOpts, FakeServer, ScriptTurn, TestEnv}; +use sha2::{Digest, Sha256}; +use std::{ + fs, + path::{Path, PathBuf}, + process::{Command, Stdio}, + time::{Duration, Instant}, +}; +use work::{git, run, run_args, run_observed, setup}; + +fn project(env: &TestEnv) -> PathBuf { + env.work().join("project") +} +fn storage(env: &TestEnv) -> PathBuf { + let common = project(env).join(".git").canonicalize().unwrap(); + let key: String = Sha256::digest(common.as_os_str().as_encoded_bytes()) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + env.agent_dir().join("work").join(key) +} +fn path(env: &TestEnv) -> PathBuf { + storage(env).join("test.json") +} +fn load(env: &TestEnv) -> serde_json::Value { + serde_json::from_slice(&fs::read(path(env)).unwrap()).unwrap() +} +fn replace(env: &TestEnv, state: &serde_json::Value) { + fs::write(path(env), serde_json::to_vec(state).unwrap()).unwrap(); +} +fn tree(record: &serde_json::Value) -> &Path { + Path::new(record["worktree"].as_str().unwrap()) +} +fn write_turn() -> ScriptTurn { + ScriptTurn::tool("write", "write", r#"{"path":"answer.txt","content":"retained"}"#) +} +fn until(mut ready: impl FnMut() -> bool) { + let end = Instant::now() + Duration::from_secs(15); + while !ready() { + assert!(Instant::now() < end, "fixture did not reach its barrier"); + std::thread::sleep(Duration::from_millis(10)); + } +} +fn status(env: &TestEnv) -> serde_json::Value { + let output = Command::new(env!("CARGO_BIN_EXE_oxa")) + .args(["work", "status", "--json"]) + .current_dir(project(env)) + .env_clear() + .envs(env.envs("http://127.0.0.1:1")) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + serde_json::from_slice(&output.stdout).unwrap() +} + +#[test] +fn dirty_success_is_saved_and_records_the_actual_head_and_tree() { + use std::os::unix::fs::PermissionsExt; + let env = TestEnv::new(); + setup(&env, "[]"); + let record = run(&env, vec![write_turn(), ScriptTurn::text("done")]); + assert_eq!(record["outcome"], "done", "{record}"); + assert_eq!(record["step"], "finished"); + assert_eq!(git(&env, tree(&record), &["show", "HEAD:answer.txt"]), "retained"); + assert!(git(&env, tree(&record), &["status", "--porcelain"]).is_empty()); + assert_eq!( + record["head_sha"], + git(&env, tree(&record), &["rev-parse", "HEAD"]) + ); + assert_eq!( + record["tree_sha"], + git(&env, tree(&record), &["rev-parse", "HEAD^{tree}"]) + ); + assert!(record["checkpoint_revision"].as_u64().unwrap() > 0); + assert_eq!( + fs::metadata(path(&env)).unwrap().permissions().mode() & 0o777, + 0o600 + ); +} + +#[test] +fn red_work_is_saved_without_reporting_local_acceptance() { + let env = TestEnv::new(); + setup(&env, r#"["printf failed > failure.txt; exit 1"]"#); + let record = run( + &env, + vec![ScriptTurn::text("done"), ScriptTurn::text("cannot repair")], + ); + assert_eq!(record["outcome"], "red", "{record}"); + assert_eq!(git(&env, tree(&record), &["show", "HEAD:failure.txt"]), "failed"); + assert!(git(&env, tree(&record), &["status", "--porcelain"]).is_empty()); +} + +#[test] +fn finished_success_is_cached_and_explicit_rerun_keeps_revisions_monotonic() { + let env = TestEnv::new(); + setup(&env, "[]"); + let first = run(&env, vec![ScriptTurn::text("done")]); + let (again, requests) = run_observed(&env, vec![], &["work", "../tasks.json"], None); + assert_eq!(first, again); + assert!(requests.is_empty()); + let repeated = run_args( + &env, + vec![ScriptTurn::text("repeated")], + &["work", "../tasks.json", "--rerun"], + None, + ); + assert_eq!(repeated["outcome"], "nothing_to_do", "{repeated}"); + assert!( + repeated["checkpoint_revision"].as_u64().unwrap() > first["checkpoint_revision"].as_u64().unwrap() + ); +} + +#[test] +fn restart_at_saving_preserves_the_verdict_and_does_not_replay_the_worker() { + let env = TestEnv::new(); + setup(&env, "[]"); + let first = run(&env, vec![ScriptTurn::text("done")]); + fs::write(tree(&first).join("recovered.txt"), "recovered").unwrap(); + let mut state = load(&env); + state["record"]["step"] = "saving".into(); + state["record"]["outcome"] = "red".into(); + state["record"]["detail"] = "previous acceptance failed".into(); + replace(&env, &state); + let (record, requests) = run_observed(&env, vec![], &["work", "../tasks.json"], None); + assert_eq!(record["outcome"], "red", "{record}"); + assert!(requests.is_empty()); + assert_eq!( + git(&env, tree(&record), &["show", "HEAD:recovered.txt"]), + "recovered" + ); +} + +#[test] +fn restart_at_acceptance_uses_the_durable_base_and_commands() { + let env = TestEnv::new(); + setup(&env, r#"["printf ran >> attempts"]"#); + let first = run(&env, vec![ScriptTurn::text("done")]); + let old_base = git(&env, &project(&env), &["rev-parse", "main"]); + env.write("work/project/.oxa/settings.json", r#"{"done":["exit 7"]}"#); + git(&env, &project(&env), &["add", "."]); + git(&env, &project(&env), &["commit", "-m", "advance base checks"]); + let new_base = git(&env, &project(&env), &["rev-parse", "main"]); + assert_ne!(old_base, new_base); + // Source resources remain exactly what the owner approved. The mutable base + // now differs, so planning sees its new commands but recovery must ignore them. + git(&env, &project(&env), &["checkout", "--detach", &old_base]); + let mut state = load(&env); + state["record"]["step"] = "acceptance".into(); + state["record"]["outcome"] = "running".into(); + replace(&env, &state); + let (record, requests) = run_observed(&env, vec![], &["work", "../tasks.json"], None); + assert_eq!(record["outcome"], "done", "{record}"); + assert_eq!(record["base_sha"], old_base); + assert!(requests.is_empty()); + assert_eq!( + fs::read_to_string(tree(&first).join("attempts")).unwrap(), + "ranran" + ); +} + +#[test] +fn failed_commit_hook_preserves_files_and_blocks() { + use std::os::unix::fs::PermissionsExt; + let env = TestEnv::new(); + setup(&env, "[]"); + let hook = env.write("work/project/.git/hooks/pre-commit", "#!/bin/sh\nexit 1\n"); + fs::set_permissions(hook, fs::Permissions::from_mode(0o755)).unwrap(); + let record = run(&env, vec![write_turn(), ScriptTurn::text("done")]); + assert_eq!(record["outcome"], "blocked", "{record}"); + assert_eq!( + fs::read_to_string(tree(&record).join("answer.txt")).unwrap(), + "retained" + ); + assert!(!git(&env, tree(&record), &["status", "--porcelain"]).is_empty()); +} + +#[test] +fn saving_hooks_run_inside_the_real_sandbox() { + use std::os::unix::fs::PermissionsExt; + let env = TestEnv::new(); + setup(&env, "[]"); + let hook = env.write( + "work/project/.git/hooks/pre-commit", + r#"#!/bin/sh +printf escaped > "$HOME/save-hook-escape" 2>/dev/null || true +printf ran > "$(git rev-parse --git-path save-hook-ran)" +"#, + ); + fs::set_permissions(hook, fs::Permissions::from_mode(0o755)).unwrap(); + let record = run(&env, vec![write_turn(), ScriptTurn::text("done")]); + assert_eq!(record["outcome"], "done", "{record}"); + assert!(!env.home().join("save-hook-escape").exists()); + let marker = PathBuf::from(git( + &env, + tree(&record), + &[ + "rev-parse", + "--path-format=absolute", + "--git-path", + "save-hook-ran", + ], + )); + assert_eq!(fs::read_to_string(marker).unwrap(), "ran"); +} + +#[test] +fn a_parent_crash_retains_session_and_child_lock_then_resumes_the_same_session() { + let env = TestEnv::new(); + setup(&env, "[]"); + let server = FakeServer::start( + vec![ScriptTurn::text("first")], + FakeOpts { + hold_headers_ms: 1500, + ..FakeOpts::default() + }, + ) + .unwrap(); + let mut child = Command::new(env!("CARGO_BIN_EXE_oxa")) + .args(["work", "../tasks.json"]) + .current_dir(project(&env)) + .env_clear() + .envs(env.envs(&server.url())) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + until(|| !server.requests().is_empty() && load(&env)["record"]["session_id"].is_string()); + let before = fs::read(path(&env)).unwrap(); + let id = load(&env)["record"]["session_id"].as_str().unwrap().to_owned(); + child.kill().unwrap(); + child.wait().unwrap(); + let blocked = run(&env, vec![]); + assert_eq!(blocked["outcome"], "blocked", "{blocked}"); + assert!(blocked["detail"].as_str().unwrap().contains("already running")); + assert_eq!(fs::read(path(&env)).unwrap(), before); + assert_eq!(status(&env)[0]["liveness"], "running"); + let lock_path = storage(&env).join("test.lock"); + until(|| { + fs::OpenOptions::new() + .read(true) + .write(true) + .open(&lock_path) + .unwrap() + .try_lock() + .is_ok() + }); + let observed = status(&env); + assert!( + matches!(observed[0]["liveness"].as_str(), Some("interrupted" | "unknown")), + "{observed}" + ); + assert_eq!(fs::read(path(&env)).unwrap(), before); + let resumed = run(&env, vec![ScriptTurn::text("resumed")]); + assert_eq!(resumed["outcome"], "nothing_to_do", "{resumed}"); + assert_eq!(resumed["session_id"], id); + assert_eq!(resumed["step"], "finished"); +} + +#[test] +fn persistence_failure_aborts_the_batch_before_checks_or_saving() { + use std::os::unix::fs::PermissionsExt; + struct Restore(PathBuf); + impl Drop for Restore { + fn drop(&mut self) { + let _ = fs::set_permissions(&self.0, fs::Permissions::from_mode(0o700)); + } + } + let env = TestEnv::new(); + setup(&env, r#"["printf forbidden > acceptance-ran"]"#); + env.write("work/tasks.json", r#"[{"id":"test","branch":"agent/test","model":"fake","prompt":"Finish task"},{"id":"later","branch":"agent/later","model":"fake","prompt":"Later task"}]"#); + let server = FakeServer::start( + vec![ScriptTurn::text("first")], + FakeOpts { + hold_headers_ms: 700, + ..FakeOpts::default() + }, + ) + .unwrap(); + let child = Command::new(env!("CARGO_BIN_EXE_oxa")) + .args(["work", "../tasks.json"]) + .current_dir(project(&env)) + .env_clear() + .envs(env.envs(&server.url())) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + until(|| !server.requests().is_empty()); + let restore = Restore(storage(&env)); + fs::set_permissions(&restore.0, fs::Permissions::from_mode(0o500)).unwrap(); + let output = child.wait_with_output().unwrap(); + drop(restore); + assert_eq!( + output.status.code(), + Some(2), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!( + output.stdout.is_empty(), + "fatal persistence is not a normal per-task block" + ); + assert!(!env + .work() + .join(".oxa-worktrees/project/test/acceptance-ran") + .exists()); + assert!(!env.work().join(".oxa-worktrees/project/later").exists()); + assert_eq!(load(&env)["record"]["step"], "worker"); +} + +#[test] +fn offline_policy_survives_a_later_online_invocation() { + let env = TestEnv::new(); + setup(&env, "[]"); + env.write( + "work/tasks.json", + r#"[{"id":"test","branch":"agent/test","model":"cloud-model","prompt":"Finish task"}]"#, + ); + let server = FakeServer::start( + vec![ScriptTurn::text("never")], + FakeOpts { + model: "cloud-model".into(), + remote: Some("openrouter"), + models: vec![oxa_fake::FakeModel { + alias: "cloud-model".into(), + cap: 64000, + max_output: Some(4096), + caps: None, + }], + ..FakeOpts::default() + }, + ) + .unwrap(); + for args in [ + vec!["--offline", "work", "../tasks.json"], + vec!["work", "../tasks.json"], + ] { + let output = Command::new(env!("CARGO_BIN_EXE_oxa")) + .args(args) + .current_dir(project(&env)) + .env_clear() + .envs(env.envs(&server.url())) + .output() + .unwrap(); + let record: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(record["outcome"], "blocked", "{record}"); + assert!( + record["detail"].as_str().unwrap().contains("offline mode is on"), + "{record}" + ); + assert!(load(&env)["offline"].as_bool().unwrap()); + } + assert!(server.requests().is_empty()); +} + +#[test] +fn corrupt_checkpoints_are_preserved_and_status_reports_an_error() { + let env = TestEnv::new(); + setup(&env, "[]"); + run(&env, vec![ScriptTurn::text("done")]); + let mut state = load(&env); + state["version"] = 99.into(); + replace(&env, &state); + let before = fs::read(path(&env)).unwrap(); + let (record, requests) = run_observed(&env, vec![], &["work", "../tasks.json"], None); + assert_eq!(record["outcome"], "blocked", "{record}"); + assert!(requests.is_empty()); + assert_eq!(fs::read(path(&env)).unwrap(), before); + let output = Command::new(env!("CARGO_BIN_EXE_oxa")) + .args(["work", "status", "--json"]) + .current_dir(project(&env)) + .env_clear() + .envs(env.envs("http://127.0.0.1:1")) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(2)); + assert!(output.stdout.is_empty()); +} + +#[test] +#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] +fn status_works_when_the_kernel_refuses_every_flock_syscall() { + let env = TestEnv::new(); + setup(&env, "[]"); + run(&env, vec![ScriptTurn::text("done")]); + let mut state = load(&env); + state["record"]["step"] = "worker".into(); + state["record"]["outcome"] = "running".into(); + replace(&env, &state); + let before = fs::read(path(&env)).unwrap(); + let syscall = if cfg!(target_arch = "aarch64") { + 32_u32 + } else { + 73_u32 + }; + let mut program = Vec::new(); + for (code, yes, no, value) in [ + (0x20_u16, 0_u8, 0_u8, 0_u32), + (0x15, 0, 1, syscall), + (0x06, 0, 0, 0x80000000), + (0x06, 0, 0, 0x7fff0000), + ] { + program.extend_from_slice(&code.to_ne_bytes()); + program.push(yes); + program.push(no); + program.extend_from_slice(&value.to_ne_bytes()); + } + let filter = env.tmp().join("deny-flock.bpf"); + fs::write(&filter, program).unwrap(); + let output = Command::new("bwrap") + .args([ + "--bind", + "/", + "/", + "--tmpfs", + "/run", + "--proc", + "/proc", + "--dev-bind", + "/dev", + "/dev", + "--seccomp", + "0", + "--", + env!("CARGO_BIN_EXE_oxa"), + "work", + "status", + "--json", + ]) + .stdin(fs::File::open(filter).unwrap()) + .current_dir(project(&env)) + .env_clear() + .envs(env.envs("http://127.0.0.1:1")) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let rows: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert!( + matches!(rows[0]["liveness"].as_str(), Some("interrupted" | "unknown")), + "{rows}" + ); + assert_eq!(fs::read(path(&env)).unwrap(), before); +} + +#[test] +fn the_owner_time_limit_reaches_the_real_child_and_keeps_its_wrap_up_work() { + let env = TestEnv::new(); + setup(&env, "[]"); + env.write( + "work/tasks.json", + r#"[{"id":"test","branch":"agent/test","model":"fake","prompt":"Finish task","max_time":1}]"#, + ); + let (record, requests) = run_observed( + &env, + vec![ + ScriptTurn::tool( + "edit", + "bash", + r#"{"command":"printf draft > change.txt; sleep 1.2; printf finished > tool-finished"}"#, + ), + ScriptTurn::tool( + "save", + "bash", + r#"{"command":"git add change.txt tool-finished && git commit -m 'Save interrupted work'"}"#, + ), + ], + &["work", "../tasks.json"], + None, + ); + assert_eq!(record["outcome"], "interrupted", "{record}"); + assert_eq!(record["worker"]["outcome"], "interrupted", "{record}"); + assert_eq!(requests.len(), 2, "one graceful wrap-up turn"); + assert_eq!( + fs::read_to_string(tree(&record).join("tool-finished")).unwrap(), + "finished" + ); + assert_eq!(git(&env, tree(&record), &["show", "HEAD:change.txt"]), "draft"); + assert_eq!(load(&env)["task"]["max_time"], 1); + assert_eq!(record["step"], "finished"); + assert_eq!(record["interruption"], "deadline"); + assert_eq!(record["interruption"], "deadline"); +} + +#[test] +fn a_crash_between_worker_finished_and_acceptance_entry_does_not_restart_the_worker() { + let env = TestEnv::new(); + setup(&env, r#"["printf ran >> attempts"]"#); + run(&env, vec![ScriptTurn::text("done")]); + let mut state = load(&env); + state["record"]["step"] = "acceptance".into(); + state["record"]["outcome"] = "running".into(); + state["record"]["checkpoint_kind"] = "worker_finished".into(); + replace(&env, &state); + let (record, requests) = run_observed(&env, vec![], &["work", "../tasks.json"], None); + assert!(requests.is_empty()); + assert_eq!(record["outcome"], "done", "{record}"); + assert_eq!(record["checkpoint_kind"], "task_transition"); + assert_eq!( + fs::read_to_string(tree(&record).join("attempts")).unwrap(), + "ranran" + ); +} + +#[test] +fn acceptance_time_cannot_give_the_repair_worker_a_fresh_time_budget() { + let env = TestEnv::new(); + setup(&env, r#"["sleep 2.2; printf failed > failure.txt; exit 1"]"#); + env.write( + "work/tasks.json", + r#"[{"id":"test","branch":"agent/test","model":"fake","prompt":"Finish task","max_time":2}]"#, + ); + let (record, requests) = run_observed( + &env, + vec![ScriptTurn::text("first"), ScriptTurn::text("must not run")], + &["work", "../tasks.json"], + None, + ); + assert_eq!(requests.len(), 1, "expired task budget refuses repair admission"); + assert_eq!(record["outcome"], "interrupted", "{record}"); + assert_eq!(record["interruption"], "deadline"); + assert_eq!( + record["worker"]["outcome"], "done", + "worker success remains distinct from task interruption" + ); + assert_eq!(git(&env, tree(&record), &["show", "HEAD:failure.txt"]), "failed"); +} + +#[test] +fn a_restart_keeps_the_original_deadline_instead_of_starting_another_budget() { + let env = TestEnv::new(); + setup(&env, "[]"); + run(&env, vec![ScriptTurn::text("done")]); + let mut state = load(&env); + state["record"]["step"] = "worker".into(); + state["record"]["outcome"] = "running".into(); + state["deadline_at_ms"] = 1.into(); + replace(&env, &state); + let (record, requests) = run_observed(&env, vec![], &["work", "../tasks.json"], None); + assert!(requests.is_empty()); + assert_eq!(record["outcome"], "interrupted", "{record}"); + assert_eq!(record["interruption"], "deadline"); + assert_eq!(load(&env)["deadline_at_ms"], 1); +} diff --git a/oxa/oxa-work/src/checkpoint.rs b/oxa/oxa-work/src/checkpoint.rs new file mode 100644 index 000000000..8a3435e60 --- /dev/null +++ b/oxa/oxa-work/src/checkpoint.rs @@ -0,0 +1,231 @@ +//! One repository-wide revision source and a bounded pending checkpoint. +//! A later writer must recover the prior durable event before advancing the source. +use crate::{state, CheckpointObserver, Record, Result}; +use serde::{Deserialize, Serialize}; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +use std::os::unix::fs::OpenOptionsExt; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +const COUNTER: &str = ".checkpoint-counter"; +const MARKER: &str = ".checkpoint-initialized"; + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct Cursor { + version: u64, + revision: u64, + pending: Option, +} + +pub(crate) struct Repository { + root: PathBuf, + _lock: File, + cursor: Cursor, +} +impl Repository { + pub(crate) fn open(root: &Path, observer: &dyn CheckpointObserver) -> Result { + let lock = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .mode(0o600) + .open(root.join(".checkpoint-lock")) + .map_err(|e| e.to_string())?; + lock.lock().map_err(|e| e.to_string())?; + let mut bytes = Vec::new(); + let cursor = match File::open(root.join(COUNTER)) { + Ok(file) => { + file.take(4097) + .read_to_end(&mut bytes) + .map_err(|e| e.to_string())?; + if bytes.len() > 4096 { + return Err("checkpoint counter exceeds capacity".into()); + } + let value: serde_json::Value = + serde_json::from_slice(&bytes).map_err(|e| format!("checkpoint counter: {e}"))?; + let object = value.as_object().ok_or("invalid checkpoint counter")?; + if object.len() != 3 + || ["version", "revision", "pending"] + .iter() + .any(|key| !object.contains_key(*key)) + { + return Err("incomplete or unknown checkpoint counter fields".into()); + } + let cursor: Cursor = + serde_json::from_value(value).map_err(|e| format!("checkpoint counter: {e}"))?; + if cursor.version != 1 + || (cursor.pending.is_some() && cursor.revision == 0) + || cursor + .pending + .as_deref() + .is_some_and(|id| !crate::valid_task_id(id)) + { + return Err("invalid checkpoint counter".into()); + } + cursor + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + if root.join(MARKER).exists() || has_durable_state(root)? { + return Err("checkpoint counter is missing; repository recovery is required".into()); + } + let cursor = Cursor { + version: 1, + revision: 0, + pending: None, + }; + write_atomic( + &root.join(COUNTER), + &serde_json::to_vec(&cursor).map_err(|e| e.to_string())?, + false, + )?; + cursor + } + Err(error) => return Err(error.to_string()), + }; + if !root.join(MARKER).exists() { + write_atomic(&root.join(MARKER), b"1\n", false)?; + } + let mut marker = Vec::new(); + File::open(root.join(MARKER)) + .map_err(|e| e.to_string())? + .take(3) + .read_to_end(&mut marker) + .map_err(|e| e.to_string())?; + if marker != b"1\n" { + return Err("invalid checkpoint initialization marker".into()); + } + let mut repository = Self { + root: root.to_path_buf(), + _lock: lock, + cursor, + }; + repository.recover(observer)?; + Ok(repository) + } + + fn store(&self) -> Result<()> { + write_atomic( + &self.root.join(COUNTER), + &serde_json::to_vec(&self.cursor).map_err(|e| e.to_string())?, + true, + ) + } + + fn recover(&mut self, observer: &dyn CheckpointObserver) -> Result<()> { + let Some(id) = &self.cursor.pending else { + return Ok(()); + }; + let path = self.root.join(format!("{id}.json")); + let record = match fs::symlink_metadata(&path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(error.to_string()), + Ok(metadata) if !metadata.is_file() => { + return Err("pending checkpoint is not a regular file".into()) + } + Ok(_) => match state::load(&path)? { + state::Stored::Current(state) => Some(state.record), + state::Stored::Legacy(_) => None, + }, + }; + if let Some(record) = record { + if record.id != *id || record.checkpoint_revision > self.cursor.revision { + return Err("pending checkpoint identity disagrees with the counter".into()); + } + if record.checkpoint_revision == self.cursor.revision { + // Complete any failed directory sync before emitting the event. + File::open(&path) + .and_then(|file| file.sync_all()) + .map_err(|e| e.to_string())?; + File::open(&self.root) + .and_then(|directory| directory.sync_all()) + .map_err(|e| e.to_string())?; + observer + .checkpoint(&record) + .map_err(|e| format!("checkpoint event replay: {e}"))?; + } + } + self.acknowledge() + } + + pub(crate) fn reserve(&mut self, record: &Record) -> Result { + if self.cursor.pending.is_some() || !crate::valid_task_id(&record.id) { + return Err("checkpoint source is not ready".into()); + } + let revision = self + .cursor + .revision + .checked_add(1) + .ok_or("task checkpoint revision exhausted")?; + self.cursor.revision = revision; + self.cursor.pending = Some(record.id.clone()); + self.store()?; + Ok(revision) + } + + pub(crate) fn verify(&self, record: &Record) -> Result<()> { + if record.checkpoint_revision == 0 || record.checkpoint_revision > self.cursor.revision { + return Err("task checkpoint is ahead of its repository counter".into()); + } + Ok(()) + } + + pub(crate) fn acknowledge(&mut self) -> Result<()> { + self.cursor.pending = None; + self.store() + } +} + +fn has_durable_state(root: &Path) -> Result { + for entry in fs::read_dir(root).map_err(|e| e.to_string())? { + let path = entry.map_err(|e| e.to_string())?.path(); + if path.extension().is_none_or(|suffix| suffix != "json") { + continue; + } + let mut bytes = Vec::new(); + File::open(path) + .map_err(|e| e.to_string())? + .take(4 * 1024 * 1024 + 1) + .read_to_end(&mut bytes) + .map_err(|e| e.to_string())?; + if serde_json::from_slice::(&bytes) + .is_ok_and(|value| value.get("version").is_some()) + { + return Ok(true); + } + } + Ok(false) +} + +pub(crate) fn write_atomic(path: &Path, bytes: &[u8], replacing: bool) -> Result<()> { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + let temporary = path.with_extension(format!("tmp-{}-{nonce}", std::process::id())); + let result = (|| -> Result<()> { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&temporary) + .map_err(|e| e.to_string())?; + file.write_all(bytes).map_err(|e| e.to_string())?; + file.sync_all().map_err(|e| e.to_string())?; + if replacing { + fs::rename(&temporary, path).map_err(|e| e.to_string())?; + } else { + fs::hard_link(&temporary, path).map_err(|e| e.to_string())?; + fs::remove_file(&temporary).map_err(|e| e.to_string())?; + } + File::open(path.parent().ok_or("checkpoint has no parent")?) + .and_then(|directory| directory.sync_all()) + .map_err(|e| e.to_string()) + })(); + if result.is_err() { + let _ = fs::remove_file(temporary); + } + result +} diff --git a/oxa/oxa-work/src/execute.rs b/oxa/oxa-work/src/execute.rs new file mode 100644 index 000000000..f0efb221e --- /dev/null +++ b/oxa/oxa-work/src/execute.rs @@ -0,0 +1,254 @@ +//! Deterministic step recovery; every next action follows a durable checkpoint. +use crate::{acceptance, checked, dirty_worktree, git, output, state, supervise, tree, worker}; +use crate::{Outcome, Record, Result, RunError, RunPolicy, SupervisorOperation, SupervisorRequest}; +use state::{CheckpointKind, CheckpointObserver, State, Step, Writer}; +use std::fs::File; +use std::path::Path; + +pub(crate) struct Runner<'a> { + pub repo: &'a Path, + pub binary: &'a Path, + pub policy: &'a RunPolicy, + pub lock: &'a File, + pub writer: &'a mut Writer, + pub observer: &'a dyn CheckpointObserver, +} +impl Runner<'_> { + fn advance(&mut self, state: &mut State, step: Step) -> std::result::Result<(), RunError> { + self.writer + .save(state, step, self.observer) + .map_err(RunError::Checkpoint) + } + + pub(crate) fn drive(&mut self, state: &mut State) -> std::result::Result<(), RunError> { + let tree = state.record.worktree.clone(); + loop { + match state.record.step { + Step::Preparing => { + let base = point(state)?.base.clone(); + let prepared = tree::prepare( + self.repo, + &tree, + &state.task, + &base, + self.binary, + self.policy, + self.lock, + )?; + let point = state + .execution + .as_mut() + .ok_or("missing task execution checkpoint")?; + point.initial = Some(prepared.initial); + point.prompt = prepared.prompt; + self.advance(state, Step::Worker)?; + } + Step::Worker | Step::Repair => { + tree::validate(self.policy, self.repo, &tree, &state.task)?; + let step = state.record.step; + let remaining = std::time::Duration::from_millis( + state.deadline_at_ms.saturating_sub(state::now_ms()), + ); + if remaining.is_zero() { + state.record.outcome = Outcome::Interrupted; + state.record.interruption = Some(crate::WorkerInterruption::Deadline); + return Err(RunError::Blocked( + "task deadline exhausted before another worker; worktree retained".into(), + )); + } + let point = point(state)?; + let prompt = if step == Step::Repair { + format!( + "External acceptance checks failed. Repair once:\n{}", + point.failures + ) + } else if state.record.session_id.is_some() { + "Continue the interrupted task from this session; preserve the retained work.".into() + } else { + point.prompt.clone() + }; + let session = state.record.session_id.clone(); + let task = state.task.clone(); + let result = worker::run( + self.binary, + &tree, + &task, + &prompt, + self.policy, + self.lock, + session.as_deref(), + remaining, + &mut |id| { + state.record.session_id = Some(id.to_owned()); + self.writer.save(state, step, self.observer) + }, + ); + // Session persistence fails while the child is active. Its reader + // requests immediate termination; no merge/check/save follows it. + if self.writer.failed() { + return Err(RunError::Checkpoint( + result + .err() + .unwrap_or_else(|| "worker session checkpoint failed".into()), + )); + } + let error = match &result { + Ok(worker) => { + if matches!(worker.result.outcome, crate::WorkerOutcome::Interrupted) + && state::now_ms() >= state.deadline_at_ms + { + state.record.interruption = Some(crate::WorkerInterruption::Deadline); + } else { + state.record.interruption = worker.result.interruption; + } + state.record.worker = Some(worker.result.clone()); + state.record.session_id = worker.session.clone(); + worker.failure() + } + Err(error) => { + state.record.worker = None; + Some(format!("worker: {error}")) + } + }; + let point = state + .execution + .as_mut() + .ok_or("missing task execution checkpoint")?; + point.worker_error = error; + point.merge_error = + tree::ensure_merged(self.policy, &tree, &point.base, &task.branch).err(); + self.writer + .save_kind( + state, + if step == Step::Worker { + Step::Acceptance + } else { + Step::Recheck + }, + CheckpointKind::WorkerFinished, + self.observer, + ) + .map_err(RunError::Checkpoint)?; + } + Step::Acceptance | Step::Recheck => { + if state.record.checkpoint_kind == CheckpointKind::WorkerFinished { + self.advance(state, state.record.step)?; + } + tree::validate(self.policy, self.repo, &tree, &state.task)?; + let point = point(state)?; + let merge = + tree::ensure_merged(self.policy, &tree, &point.base, &state.task.branch).err(); + let checks = acceptance( + self.binary, + &tree, + self.repo, + &point.commands, + self.policy, + self.lock, + ); + let mut errors: Vec<_> = point.worker_error.iter().cloned().collect(); + errors.extend( + point + .merge_error + .iter() + .chain(merge.as_ref()) + .map(|error| format!("merge verification: {error}")), + ); + errors.dedup(); + let failures = match checks { + Ok(failures) => failures, + Err(error) => { + errors.push(error); + String::new() + } + }; + if !errors.is_empty() { + if !failures.is_empty() { + errors.push(format!("Acceptance checks:\n{failures}")); + } + return Err(RunError::Blocked(errors.join("\n"))); + } + if failures.is_empty() { + let initial = point.initial.as_ref().ok_or("missing initial task commit")?; + let head = checked(git(self.policy, &tree, &["rev-parse", "HEAD"]))?; + let dirty = dirty_worktree(self.binary, &tree, self.repo, self.policy, self.lock)?; + state.record.outcome = if head == *initial && !dirty { + Outcome::NothingToDo + } else { + Outcome::Done + }; + self.advance(state, Step::Saving)?; + } else if state.record.step == Step::Acceptance { + state + .execution + .as_mut() + .ok_or("missing task execution checkpoint")? + .failures = failures; + self.advance(state, Step::Repair)?; + } else { + state.record.outcome = Outcome::Red; + state.record.detail = failures; + self.advance(state, Step::Saving)?; + } + } + Step::Saving | Step::Finished => return Ok(()), + } + } + } +} + +fn point(state: &State) -> Result<&state::Execution> { + state + .execution + .as_ref() + .ok_or_else(|| "missing task execution checkpoint".into()) +} + +pub(crate) fn save_work( + repo: &Path, + binary: &Path, + task: &crate::Task, + policy: &RunPolicy, + lock: &File, + record: &mut Record, +) -> Result<()> { + let tree = &record.worktree; + if !tree.exists() { + return Ok(()); + } + tree::validate(policy, repo, tree, task)?; + let merging = output(git( + policy, + tree, + &["rev-parse", "--verify", "--quiet", "MERGE_HEAD"], + ))?; + match merging.status.code() { + Some(0) => return Err("unresolved merge retained in progress; files were not discarded".into()), + Some(1) => {} + _ => return Err("cannot inspect pending task merge".into()), + } + let saved = supervise( + binary, + tree, + SupervisorRequest { + source: repo.to_path_buf(), + operation: SupervisorOperation::Save { + task_id: task.id.clone(), + }, + }, + policy, + lock, + )?; + if !saved.status.success() { + return Err(format!( + "sandboxed save failed: {}", + String::from_utf8_lossy(&saved.stderr) + )); + } + if dirty_worktree(binary, tree, repo, policy, lock)? { + return Err("uncommitted changes remain after saving; retained for inspection".into()); + } + record.head_sha = Some(checked(git(policy, tree, &["rev-parse", "HEAD"]))?); + record.tree_sha = Some(checked(git(policy, tree, &["rev-parse", "HEAD^{tree}"]))?); + Ok(()) +} diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index 76e638467..2265e717a 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -1,18 +1,24 @@ //! Opt-in, sequential unattended task execution. Worktrees are retained on every outcome. use serde::{Deserialize, Serialize}; use std::fs; -use std::io::Write; use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt}; use std::path::{Path, PathBuf}; use std::process::{Command, Output, Stdio}; +mod checkpoint; +mod execute; +mod state; +mod status; +pub use status::{status, Liveness, Status}; mod tree; +pub use state::{CheckpointKind, CheckpointObserver, Step}; mod worker; -pub use worker::{WorkerOutcome, WorkerResult}; +mod worker_wait; +pub use worker::{WorkerInterruption, WorkerOutcome, WorkerResult}; pub type Result = std::result::Result; -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct Task { pub id: String, @@ -21,6 +27,11 @@ pub struct Task { #[serde(default = "default_base")] pub base: String, pub prompt: String, + #[serde(default = "default_max_time")] + pub max_time: u64, +} +fn default_max_time() -> u64 { + 45 * 60 } fn default_base() -> String { "main".into() @@ -29,13 +40,15 @@ fn default_base() -> String { #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum Outcome { + Running, + Interrupted, Done, NothingToDo, Red, Blocked, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct Record { pub id: String, pub branch: String, @@ -45,6 +58,20 @@ pub struct Record { pub worker: Option, pub session_id: Option, pub base_sha: Option, + #[serde(default)] + pub checkpoint_revision: u64, + #[serde(default)] + pub checkpoint_kind: CheckpointKind, + #[serde(default)] + pub checkpoint_at_ms: u64, + #[serde(default)] + pub step: Step, + #[serde(default)] + pub head_sha: Option, + #[serde(default)] + pub tree_sha: Option, + #[serde(default)] + pub interruption: Option, } /// An immutable task/base/check snapshot, built before any task in its batch runs. @@ -142,14 +169,32 @@ pub fn read_tasks(path: &Path) -> Result> { } Ok(tasks) } -fn validate(task: &Task) -> Result<()> { - if task.id.is_empty() - || !task - .id +fn valid_task_id(id: &str) -> bool { + !id.is_empty() + && id.len() <= 128 + && id .bytes() - .all(|b| b.is_ascii_alphanumeric() || b"-_".contains(&b)) + .all(|byte| byte.is_ascii_alphanumeric() || b"-_".contains(&byte)) +} +fn validate(task: &Task) -> Result<()> { + if !valid_task_id(&task.id) { + return Err("task id must contain 1..128 letters, digits, - or _".into()); + } + let allowance = + std::time::Duration::from_secs(task.max_time).checked_add(std::time::Duration::from_secs(70)); + if task.max_time == 0 + || allowance.is_none_or(|duration| std::time::Instant::now().checked_add(duration).is_none()) { - return Err("task id must contain only letters, digits, - or _".into()); + return Err("max_time must be a positive number of seconds that fits the runtime clock".into()); + } + if task + .max_time + .checked_mul(1000) + .and_then(|ms| ms.checked_add(70_000)) + .and_then(|ms| ms.checked_add(state::now_ms())) + .is_none() + { + return Err("max_time is too large for a durable deadline".into()); } if task.model.trim().is_empty() || task.prompt.trim().is_empty() { return Err("model and prompt are required".into()); @@ -192,6 +237,7 @@ pub fn issue_task( branch: format!("agent/{number}-work"), base: default_base(), prompt: format!("Implement issue #{number}: {title}\n\n{body}"), + max_time: default_max_time(), }) } @@ -209,11 +255,13 @@ fn done_commands(policy: &RunPolicy, root: &Path, base: &str) -> Result, pub offline: bool, pub storage: PathBuf, } @@ -230,6 +278,7 @@ pub struct SupervisorRequest { pub enum SupervisorOperation { Checkout { branch: String, create: bool }, Merge { base: String }, + Save { task_id: String }, Checks { commands: Vec }, Inspect, } @@ -318,33 +367,28 @@ fn dirty_worktree( } } -fn worker_and_checks( - worker: &Result, - merge: Result<()>, - checks: Result, -) -> Result { - let mut errors = Vec::new(); - match worker { - Ok(worker) => errors.extend(worker.failure()), - Err(error) => errors.push(format!("worker: {error}")), - } - if let Err(error) = merge { - errors.push(format!("merge verification: {error}")); - } - let failures = match checks { - Ok(failures) => failures, - Err(error) => { - errors.push(error); - String::new() +#[derive(Debug)] +pub enum RunError { + Blocked(String), + Checkpoint(String), +} +impl std::fmt::Display for RunError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Blocked(error) | Self::Checkpoint(error) => f.write_str(error), } - }; - if errors.is_empty() { - return Ok(failures); } - if !failures.is_empty() { - errors.push(format!("Acceptance checks:\n{failures}")); +} +impl std::error::Error for RunError {} +impl From for RunError { + fn from(value: String) -> Self { + Self::Blocked(value) + } +} +impl From<&str> for RunError { + fn from(value: &str) -> Self { + Self::Blocked(value.into()) } - Err(errors.join("\n")) } pub fn blocked_record(task: &Task, detail: String) -> Record { @@ -357,7 +401,33 @@ pub fn blocked_record(task: &Task, detail: String) -> Record { worker: None, session_id: None, base_sha: None, + checkpoint_revision: 0, + checkpoint_kind: CheckpointKind::TaskTransition, + checkpoint_at_ms: 0, + step: Step::Finished, + head_sha: None, + tree_sha: None, + interruption: None, + } +} + +/// Recover the bounded pending observation before opening a lead's facts view. +/// This never acquires a task lock, changes a task file or starts a worker. +pub fn recover_checkpoints( + storage: &Path, + observer: &dyn CheckpointObserver, +) -> std::result::Result<(), RunError> { + match fs::metadata(storage) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(RunError::Checkpoint(error.to_string())), + Ok(metadata) if !metadata.is_dir() => { + return Err(RunError::Checkpoint("checkpoint root is not a directory".into())) + } + Ok(_) => {} } + checkpoint::Repository::open(storage, observer) + .map(|_| ()) + .map_err(RunError::Checkpoint) } pub fn run_task_with_policy( repo: &Path, @@ -365,23 +435,34 @@ pub fn run_task_with_policy( planned: &PlannedTask, policy: &RunPolicy, ) -> Result { + run_task_with_observer(repo, binary, planned, policy, false, &|_: &Record| Ok(())) + .map_err(|e| e.to_string()) +} + +/// Checkpoint/observer failures are fatal to the batch; ordinary task failures +/// remain per-task blocked records. No observer can read or write private State. +pub fn run_task_with_observer( + repo: &Path, + binary: &Path, + planned: &PlannedTask, + policy: &RunPolicy, + rerun: bool, + observer: &dyn CheckpointObserver, +) -> std::result::Result { let task = &planned.task; validate(task)?; - let root = checked(git(policy, repo, &["rev-parse", "--show-toplevel"]))?; - let root = PathBuf::from(root); - let common = checked(git( + let root = PathBuf::from(checked(git(policy, repo, &["rev-parse", "--show-toplevel"]))?); + let common = PathBuf::from(checked(git( policy, &root, &["rev-parse", "--path-format=absolute", "--git-common-dir"], - ))?; - let common = PathBuf::from(common); - let storage = &policy.storage; + ))?); fs::DirBuilder::new() .recursive(true) .mode(0o700) - .create(storage) + .create(&policy.storage) .map_err(|e| e.to_string())?; - let lock_path = storage.join(format!("{}.lock", task.id)); + let lock_path = policy.storage.join(format!("{}.lock", task.id)); let lock_file = fs::OpenOptions::new() .read(true) .write(true) @@ -393,12 +474,9 @@ pub fn run_task_with_policy( lock_file .try_lock() .map_err(|e| format!("task {} is already running: {e}", task.id))?; - // Keep the inode so a crashed runner leaves no stale ownership and concurrent - // runners cannot acquire locks on two different files with the same name. - let receipt = storage.join(format!("{}.json", task.id)); + // The inherited lock retains its inode; status never tries to acquire it. + let receipt = policy.storage.join(format!("{}.json", task.id)); let replacing = receipt.exists(); - // The common git directory must not be an ancestor of the child cwd: the sandbox - // deliberately refuses writable git binds with that layout. let repo_dir = if common.file_name().is_some_and(|name| name == ".git") { common.parent().ok_or("git directory has no parent")? } else { @@ -409,104 +487,107 @@ pub fn run_task_with_policy( .ok_or("repository has no parent for worktrees")?; let name = repo_dir.file_name().ok_or("repository has no directory name")?; let trees = parent.join(".oxa-worktrees").join(name); - fs::create_dir_all(&trees).map_err(|e| e.to_string())?; let tree = trees.join(&task.id); - tree::check_receipt(&receipt, &tree, task)?; - let mut record = Record { - id: task.id.clone(), - branch: task.branch.clone(), - worktree: tree.clone(), - outcome: Outcome::Blocked, - detail: String::new(), - worker: None, - session_id: None, - base_sha: Some(planned.base_sha.clone()), + let fresh = |revision| { + let mut record = blocked_record(task, String::new()); + record.worktree = tree.clone(); + record.base_sha = Some(planned.base_sha.clone()); + record.checkpoint_revision = revision; + let mut state = state::State::new(task.clone(), record, policy.offline); + state.execution = Some(state::Execution { + initial: None, + base: planned.base_sha.clone(), + prompt: task.prompt.clone(), + commands: planned.commands.clone(), + worker_error: None, + merge_error: None, + failures: String::new(), + }); + state }; - let result = (|| -> Result { - let base = &planned.base_sha; - let commands = &planned.commands; - let prepared = tree::prepare(&root, &tree, task, base, binary, policy, &lock_file)?; - let tree = tree.canonicalize().map_err(|e| e.to_string())?; - let first = worker::run(binary, &tree, task, &prepared.prompt, policy, &lock_file, None); - if let Ok(worker) = &first { - record.worker = Some(worker.result.clone()); - record.session_id = worker.session.clone(); - } - let failures = worker_and_checks( - &first, - tree::ensure_merged(policy, &tree, base, &task.branch), - acceptance(binary, &tree, &root, commands, policy, &lock_file), - )?; - let first = first?; - if !failures.is_empty() { - let repair = worker::run( - binary, - &tree, - task, - &format!("External acceptance checks failed. Repair once:\n{failures}"), - policy, - &lock_file, - first.session.as_deref(), - ); - if let Ok(worker) = &repair { - record.worker = Some(worker.result.clone()); - record.session_id = worker.session.clone(); + let mut writer = state::Writer::new(receipt.clone(), replacing); + let mut state = if replacing { + match state::load(&receipt)? { + state::Stored::Current(old) => { + if old.task != *task || old.record.worktree != tree { + return Err("task specification changed; use a new task id".into()); + } + writer.replay(&old, observer).map_err(RunError::Checkpoint)?; + if old.record.step == Step::Finished { + if !rerun && matches!(old.record.outcome, Outcome::Done | Outcome::NothingToDo) { + return Ok(old.record); + } + let mut new = fresh(old.record.checkpoint_revision); + new.offline |= old.offline; + new + } else { + *old + } } - let failures = worker_and_checks( - &repair, - tree::ensure_merged(policy, &tree, base, &task.branch), - acceptance(binary, &tree, &root, commands, policy, &lock_file), - )?; - if !failures.is_empty() { - record.detail = failures; - return Ok(Outcome::Red); + state::Stored::Legacy(old) => { + if old.id != task.id || old.branch != task.branch || old.worktree != tree { + return Err("retained receipt belongs to a different task branch or worktree".into()); + } + if !rerun { + return Err("receipt predates durable checkpoints; use --rerun to continue it".into()); + } + fresh(0) } } - let head = checked(git(policy, &tree, &["rev-parse", "HEAD"]))?; - let dirty = dirty_worktree(binary, &tree, &root, policy, &lock_file)?; - Ok(if head == prepared.initial && !dirty { - Outcome::NothingToDo - } else { - Outcome::Done - }) - })(); - match result { - Ok(outcome) => record.outcome = outcome, - Err(e) => record.detail = e, + } else { + fresh(0) + }; + state.offline |= policy.offline; + let policy = RunPolicy { + offline: state.offline, + ..policy.clone() + }; + // Replaying the prior checkpoint precedes this first new checkpoint or action. + let step = state.record.step; + writer + .save(&mut state, step, observer) + .map_err(RunError::Checkpoint)?; + match (execute::Runner { + repo: &root, + binary, + policy: &policy, + lock: &lock_file, + writer: &mut writer, + observer, + }) + .drive(&mut state) + { + Ok(()) => {} + Err(RunError::Checkpoint(error)) => return Err(RunError::Checkpoint(error)), + Err(RunError::Blocked(error)) => { + state.record.outcome = if state.record.outcome == Outcome::Interrupted + || state + .record + .worker + .as_ref() + .is_some_and(|worker| matches!(worker.outcome, WorkerOutcome::Interrupted)) + { + Outcome::Interrupted + } else { + Outcome::Blocked + }; + state.record.detail = error; + } } - save_receipt(&receipt, &record, replacing)?; - Ok(record) -} - -fn save_receipt(path: &Path, record: &Record, replacing: bool) -> Result<()> { - let nonce = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_nanos(); - let temporary = path.with_extension(format!("tmp-{}-{nonce}", std::process::id())); - let result = (|| -> Result<()> { - let mut file = fs::OpenOptions::new() - .write(true) - .create_new(true) - .mode(0o600) - .open(&temporary) - .map_err(|e| e.to_string())?; - file.write_all(&serde_json::to_vec_pretty(record).map_err(|e| e.to_string())?) - .map_err(|e| e.to_string())?; - file.sync_all().map_err(|e| e.to_string())?; - if replacing { - // Only the task's already validated receipt is replaced under its lock. - fs::rename(&temporary, path).map_err(|e| e.to_string())?; - } else { - fs::hard_link(&temporary, path).map_err(|e| e.to_string())?; - fs::remove_file(&temporary).map_err(|e| e.to_string())?; + writer + .save(&mut state, Step::Saving, observer) + .map_err(RunError::Checkpoint)?; + if let Err(error) = execute::save_work(&root, binary, task, &policy, &lock_file, &mut state.record) { + if state.record.outcome != Outcome::Interrupted { + state.record.outcome = Outcome::Blocked; + } + if !state.record.detail.is_empty() { + state.record.detail.push('\n'); } - fs::File::open(path.parent().ok_or("receipt has no parent")?) - .and_then(|parent| parent.sync_all()) - .map_err(|e| e.to_string()) - })(); - if result.is_err() { - let _ = fs::remove_file(temporary); + state.record.detail.push_str(&format!("save: {error}")); } - result + writer + .save(&mut state, Step::Finished, observer) + .map_err(RunError::Checkpoint)?; + Ok(state.record) } diff --git a/oxa/oxa-work/src/state.rs b/oxa/oxa-work/src/state.rs new file mode 100644 index 000000000..14ef50a88 --- /dev/null +++ b/oxa/oxa-work/src/state.rs @@ -0,0 +1,603 @@ +//! Private durable task checkpoints. Observers never receive or mutate State. +use crate::{checkpoint, Outcome, Record, Result, Task}; +use serde::{Deserialize, Serialize}; +use std::fs::File; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +const VERSION: u64 = 1; +const MAX_STATE_BYTES: u64 = 4 * 1024 * 1024; + +#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum Step { + Preparing, + Worker, + Acceptance, + Repair, + Recheck, + Saving, + #[default] + Finished, +} + +#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum CheckpointKind { + #[default] + TaskTransition, + WorkerFinished, +} + +/// Called after the checkpoint and directory are synced, and again on recovery. +/// The consumer deduplicates by repository and checkpoint_revision (source runner). +/// An error must stop the runner before another action or checkpoint. +pub trait CheckpointObserver: Sync { + fn checkpoint(&self, record: &Record) -> Result<()>; +} +impl Result<()> + Sync> CheckpointObserver for F { + fn checkpoint(&self, record: &Record) -> Result<()> { + self(record) + } +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct Execution { + pub initial: Option, + pub base: String, + pub prompt: String, + pub commands: Vec, + pub worker_error: Option, + pub merge_error: Option, + pub failures: String, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct State { + pub version: u64, + pub record: Record, + pub task: Task, + pub execution: Option, + pub offline: bool, + pub deadline_at_ms: u64, +} +impl State { + pub(crate) fn new(task: Task, mut record: Record, offline: bool) -> Self { + record.outcome = Outcome::Running; + record.step = Step::Preparing; + let deadline_at_ms = now_ms().saturating_add(task.max_time.saturating_mul(1000)); + Self { + version: VERSION, + record, + task, + execution: None, + offline, + deadline_at_ms, + } + } +} + +pub(crate) fn now_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + .try_into() + .unwrap_or(u64::MAX) +} + +pub(crate) enum Stored { + Current(Box), + Legacy(Box), +} +#[cfg(test)] +pub(crate) fn read(path: &Path) -> Result { + match load(path)? { + Stored::Current(state) => Ok(*state), + Stored::Legacy(_) => Err("receipt predates durable checkpoints".into()), + } +} + +pub(crate) fn load(path: &Path) -> Result { + let file = File::open(path).map_err(|e| format!("task checkpoint: {e}"))?; + let mut bytes = Vec::new(); + file.take(MAX_STATE_BYTES + 1) + .read_to_end(&mut bytes) + .map_err(|e| e.to_string())?; + if bytes.len() as u64 > MAX_STATE_BYTES { + return Err("task checkpoint exceeds capacity".into()); + } + let value: serde_json::Value = + serde_json::from_slice(&bytes).map_err(|e| format!("task checkpoint: {e}"))?; + if value.get("version").is_none() { + let record: Record = serde_json::from_value(value).map_err(|e| format!("retained receipt: {e}"))?; + if record.checkpoint_revision != 0 { + return Err("unknown task checkpoint format".into()); + } + return Ok(Stored::Legacy(Box::new(record))); + } + let fields = value.as_object().ok_or("invalid task checkpoint")?; + if fields.len() != 6 + || [ + "version", + "record", + "task", + "execution", + "offline", + "deadline_at_ms", + ] + .iter() + .any(|key| !fields.contains_key(*key)) + { + return Err("incomplete or unknown task checkpoint fields".into()); + } + // Public Record accepts older receipt defaults; a durable checkpoint must + // carry every field explicitly and cannot silently become Finished. + let keys = [ + "id", + "branch", + "worktree", + "outcome", + "detail", + "worker", + "session_id", + "base_sha", + "checkpoint_revision", + "checkpoint_kind", + "checkpoint_at_ms", + "step", + "head_sha", + "tree_sha", + "interruption", + ]; + let record = value + .get("record") + .and_then(serde_json::Value::as_object) + .ok_or("task checkpoint record is missing")?; + if record.len() != keys.len() || keys.iter().any(|key| !record.contains_key(*key)) { + return Err("incomplete or unknown task checkpoint record fields".into()); + } + let state: State = serde_json::from_value(value).map_err(|e| format!("task checkpoint: {e}"))?; + if state.version != VERSION || state.record.checkpoint_revision == 0 || state.deadline_at_ms == 0 { + return Err("unsupported or invalid task checkpoint".into()); + } + if state.record.id != state.task.id || state.record.branch != state.task.branch { + return Err("task checkpoint identity disagrees with its task".into()); + } + Ok(Stored::Current(Box::new(state))) +} + +/// A writer is constructed only after validating an existing file under its task lock. +pub(crate) struct Writer { + path: PathBuf, + present: bool, + failed: bool, +} +impl Writer { + pub(crate) fn new(path: PathBuf, present: bool) -> Self { + Self { + path, + present, + failed: false, + } + } + + pub(crate) fn failed(&self) -> bool { + self.failed + } + + pub(crate) fn replay(&mut self, state: &State, observer: &dyn CheckpointObserver) -> Result<()> { + if self.failed { + return Err("checkpoint persistence is blocked; restart to replay".into()); + } + let result = + checkpoint::Repository::open(self.path.parent().ok_or("checkpoint has no parent")?, observer) + .and_then(|repository| repository.verify(&state.record)); + self.failed = result.is_err(); + result + } + + pub(crate) fn save( + &mut self, + state: &mut State, + step: Step, + observer: &dyn CheckpointObserver, + ) -> Result<()> { + self.save_kind(state, step, CheckpointKind::TaskTransition, observer) + } + + pub(crate) fn save_kind( + &mut self, + state: &mut State, + step: Step, + kind: CheckpointKind, + observer: &dyn CheckpointObserver, + ) -> Result<()> { + if self.failed { + return Err("checkpoint persistence is blocked; restart to replay".into()); + } + let result = self.persist(state, step, kind, observer); + self.failed = result.is_err(); + result + } + + fn persist( + &mut self, + state: &mut State, + step: Step, + kind: CheckpointKind, + observer: &dyn CheckpointObserver, + ) -> Result<()> { + if self + .path + .file_name() + .is_none_or(|name| name != std::ffi::OsStr::new(&format!("{}.json", state.record.id))) + { + return Err("task checkpoint path disagrees with its identity".into()); + } + let mut repository = + checkpoint::Repository::open(self.path.parent().ok_or("checkpoint has no parent")?, observer)?; + let revision = repository.reserve(&state.record)?; + state.record.checkpoint_revision = revision; + state.record.checkpoint_at_ms = now_ms(); + state.record.step = step; + state.record.checkpoint_kind = kind; + let bytes = serde_json::to_vec(state).map_err(|e| e.to_string())?; + if bytes.len() as u64 > MAX_STATE_BYTES { + return Err("task checkpoint exceeds capacity".into()); + } + checkpoint::write_atomic(&self.path, &bytes, self.present)?; + self.present = true; + observer + .checkpoint(&state.record) + .map_err(|e| format!("checkpoint event append: {e}"))?; + repository.acknowledge() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs::{self, OpenOptions}; + use std::os::unix::fs::{DirBuilderExt, PermissionsExt}; + use std::sync::{ + atomic::{AtomicU64, Ordering}, + Mutex, + }; + + struct Fixture(PathBuf); + impl Fixture { + fn new() -> Self { + let path = std::env::temp_dir().join(format!( + "oxa-checkpoint-{}-{}", + std::process::id(), + SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos() + )); + fs::DirBuilder::new().mode(0o700).create(&path).unwrap(); + Self(path) + } + fn path(&self) -> PathBuf { + self.0.join("test.json") + } + } + impl Drop for Fixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + fn state() -> State { + let task = Task { + id: "test".into(), + branch: "agent/test".into(), + base: "main".into(), + model: "fake".into(), + prompt: "Synthetic task".into(), + max_time: crate::default_max_time(), + }; + let record = crate::blocked_record(&task, String::new()); + State::new(task, record, false) + } + + #[test] + fn observer_sees_durable_private_checkpoint_and_monotonic_revisions() { + let fixture = Fixture::new(); + let revisions = Mutex::new(Vec::new()); + let observer = |record: &Record| -> Result<()> { + let stored = read(&fixture.path())?; + assert_eq!(stored.record.checkpoint_revision, record.checkpoint_revision); + assert_eq!(stored.record.step, record.step); + assert_eq!( + fs::metadata(fixture.path()).unwrap().permissions().mode() & 0o777, + 0o600 + ); + revisions.lock().unwrap().push(record.checkpoint_revision); + Ok(()) + }; + let mut state = state(); + let mut writer = Writer::new(fixture.path(), false); + writer.save(&mut state, Step::Preparing, &observer).unwrap(); + writer.save(&mut state, Step::Worker, &observer).unwrap(); + let mut recovered = read(&fixture.path()).unwrap(); + let mut writer = Writer::new(fixture.path(), true); + writer.replay(&recovered, &observer).unwrap(); + writer.save(&mut recovered, Step::Acceptance, &observer).unwrap(); + assert_eq!(*revisions.lock().unwrap(), [1, 2, 3]); + } + + #[test] + fn append_failure_leaves_latest_checkpoint_replayable_without_new_revision() { + let fixture = Fixture::new(); + let mut writer = Writer::new(fixture.path(), false); + let mut state = state(); + let unavailable = |_: &Record| Err("synthetic append failure".into()); + assert!(writer.save(&mut state, Step::Worker, &unavailable).is_err()); + let recovered = read(&fixture.path()).unwrap(); + assert_eq!(recovered.record.step, Step::Worker); + assert_eq!(recovered.record.checkpoint_revision, 1); + let replayed = AtomicU64::new(0); + let mut writer = Writer::new(fixture.path(), true); + writer + .replay(&recovered, &|record: &Record| { + replayed.store(record.checkpoint_revision, Ordering::Relaxed); + Ok(()) + }) + .unwrap(); + assert_eq!(replayed.load(Ordering::Relaxed), 1); + assert_eq!(read(&fixture.path()).unwrap().record.checkpoint_revision, 1); + } + + #[test] + fn failed_append_refuses_another_checkpoint_until_restart_and_replay() { + let fixture = Fixture::new(); + let mut writer = Writer::new(fixture.path(), false); + let mut state = state(); + assert!(writer + .save(&mut state, Step::Worker, &|_: &Record| { + Err("synthetic append failure".into()) + }) + .is_err()); + let durable = fs::read(fixture.path()).unwrap(); + let called = AtomicU64::new(0); + assert!(writer + .save(&mut state, Step::Saving, &|_: &Record| { + called.fetch_add(1, Ordering::Relaxed); + Ok(()) + }) + .is_err()); + assert_eq!(fs::read(fixture.path()).unwrap(), durable); + assert_eq!(called.load(Ordering::Relaxed), 0); + } + + #[test] + fn a_new_checkpoint_cannot_replace_an_unexpected_existing_file() { + let fixture = Fixture::new(); + fs::write(fixture.path(), b"existing").unwrap(); + let mut writer = Writer::new(fixture.path(), false); + let called = AtomicU64::new(0); + assert!(writer + .save(&mut state(), Step::Preparing, &|_: &Record| { + called.fetch_add(1, Ordering::Relaxed); + Ok(()) + }) + .is_err()); + assert_eq!(fs::read(fixture.path()).unwrap(), b"existing"); + assert_eq!(called.load(Ordering::Relaxed), 0); + } + + #[test] + fn revision_exhaustion_preserves_the_previous_file_and_never_calls_observer() { + let fixture = Fixture::new(); + let mut state = state(); + state.record.checkpoint_revision = u64::MAX; + let bytes = serde_json::to_vec(&state).unwrap(); + fs::write(fixture.path(), &bytes).unwrap(); + fs::write( + fixture.0.join(".checkpoint-counter"), + serde_json::to_vec(&serde_json::json!({"version":1,"revision":u64::MAX,"pending":null})).unwrap(), + ) + .unwrap(); + let mut writer = Writer::new(fixture.path(), true); + assert!(writer + .save(&mut state, Step::Worker, &|_: &Record| -> Result<()> { + panic!("exhausted revision was emitted") + }) + .unwrap_err() + .contains("revision exhausted")); + assert_eq!(fs::read(fixture.path()).unwrap(), bytes); + } + + #[test] + fn missing_corrupt_oversized_and_future_checkpoints_are_errors() { + let fixture = Fixture::new(); + assert!(read(&fixture.path()).is_err()); + fs::write(fixture.path(), b"{").unwrap(); + assert!(read(&fixture.path()).is_err()); + let mut state = state(); + state.version = VERSION + 1; + state.record.checkpoint_revision = 1; + fs::write(fixture.path(), serde_json::to_vec(&state).unwrap()).unwrap(); + assert!(read(&fixture.path()).is_err()); + OpenOptions::new() + .write(true) + .open(fixture.path()) + .unwrap() + .set_len(MAX_STATE_BYTES + 1) + .unwrap(); + assert!(read(&fixture.path()).unwrap_err().contains("capacity")); + } + + #[test] + fn durable_records_reject_missing_step_revision_and_base_even_with_legacy_defaults() { + let fixture = Fixture::new(); + let mut state = state(); + let mut writer = Writer::new(fixture.path(), false); + writer + .save(&mut state, Step::Preparing, &|_: &Record| Ok(())) + .unwrap(); + let complete: serde_json::Value = serde_json::from_slice(&fs::read(fixture.path()).unwrap()).unwrap(); + for field in ["step", "checkpoint_revision", "base_sha"] { + let mut damaged = complete.clone(); + damaged["record"].as_object_mut().unwrap().remove(field); + fs::write(fixture.path(), serde_json::to_vec(&damaged).unwrap()).unwrap(); + assert!(read(&fixture.path()).is_err(), "missing {field} was accepted"); + } + } + + #[test] + fn repository_recovery_emits_pending_other_task_before_allocating_again() { + let fixture = Fixture::new(); + let mut first = state(); + let mut writer = Writer::new(fixture.path(), false); + assert!(writer + .save(&mut first, Step::Worker, &|_: &Record| Err("unavailable".into())) + .is_err()); + let mut second = state(); + second.task.id = "second".into(); + second.record.id = "second".into(); + second.task.branch = "agent/second".into(); + second.record.branch = "agent/second".into(); + let second_path = fixture.0.join("second.json"); + let mut still_blocked = Writer::new(second_path.clone(), false); + assert!(still_blocked + .save(&mut second, Step::Preparing, &|_: &Record| Err( + "unavailable".into() + )) + .is_err()); + assert!(!second_path.exists()); + let events = Mutex::new(Vec::new()); + let observe = |record: &Record| { + events + .lock() + .unwrap() + .push((record.id.clone(), record.checkpoint_revision)); + Ok(()) + }; + let mut restarted = Writer::new(second_path, false); + restarted.save(&mut second, Step::Preparing, &observe).unwrap(); + assert_eq!( + *events.lock().unwrap(), + [("test".into(), 1), ("second".into(), 2)] + ); + // An acknowledged older task has no replay: the one emitter high-water mark suffices. + Writer::new(fixture.path(), true) + .replay(&first, &observe) + .unwrap(); + assert_eq!(events.lock().unwrap().len(), 2); + } + + #[test] + fn a_reservation_without_a_task_rename_creates_only_a_revision_gap() { + let fixture = Fixture::new(); + let state = state(); + { + let mut repository = checkpoint::Repository::open(&fixture.0, &|_: &Record| Ok(())).unwrap(); + assert_eq!(repository.reserve(&state.record).unwrap(), 1); + } + assert!(!fixture.path().exists()); + let revisions = Mutex::new(Vec::new()); + let mut writer = Writer::new(fixture.path(), false); + let mut state = state; + writer + .save(&mut state, Step::Preparing, &|record: &Record| { + revisions.lock().unwrap().push(record.checkpoint_revision); + Ok(()) + }) + .unwrap(); + assert_eq!(*revisions.lock().unwrap(), [2]); + } + + #[test] + fn missing_or_corrupt_repository_counter_cannot_reset_revisions() { + let fixture = Fixture::new(); + let mut state = state(); + Writer::new(fixture.path(), false) + .save(&mut state, Step::Preparing, &|_: &Record| Ok(())) + .unwrap(); + let durable = fs::read(fixture.path()).unwrap(); + fs::remove_file(fixture.0.join(".checkpoint-counter")).unwrap(); + let mut writer = Writer::new(fixture.path(), true); + assert!(writer + .save(&mut state, Step::Worker, &|_: &Record| -> Result<()> { + panic!("missing counter must not emit") + }) + .is_err()); + assert_eq!(fs::read(fixture.path()).unwrap(), durable); + fs::write(fixture.0.join(".checkpoint-counter"), b"{").unwrap(); + let mut writer = Writer::new(fixture.path(), true); + assert!(writer + .save(&mut state, Step::Worker, &|_: &Record| -> Result<()> { + panic!("corrupt counter must not emit") + }) + .is_err()); + assert_eq!(fs::read(fixture.path()).unwrap(), durable); + fs::write( + fixture.0.join(".checkpoint-counter"), + br#"{"version":1,"revision":1}"#, + ) + .unwrap(); + let mut writer = Writer::new(fixture.path(), true); + assert!( + writer + .save(&mut state, Step::Worker, &|_: &Record| Ok(())) + .is_err(), + "a missing acknowledgement field must not become an empty pending slot" + ); + assert_eq!(fs::read(fixture.path()).unwrap(), durable); + } + + #[test] + fn completed_tasks_do_not_grow_the_repository_replay_cursor() { + let fixture = Fixture::new(); + let mut last = 0; + for number in 0..140 { + let id = format!("task-{number}"); + let mut state = state(); + state.task.id = id.clone(); + state.record.id = id.clone(); + state.task.branch = format!("agent/{id}"); + state.record.branch = state.task.branch.clone(); + Writer::new(fixture.0.join(format!("{id}.json")), false) + .save(&mut state, Step::Preparing, &|_: &Record| Ok(())) + .unwrap(); + assert!(state.record.checkpoint_revision > last); + last = state.record.checkpoint_revision; + assert!(fs::metadata(fixture.0.join(".checkpoint-counter")).unwrap().len() < 256); + } + assert_eq!(last, 140); + } + + #[test] + fn worker_result_and_acceptance_entry_are_distinct_durable_observations() { + let fixture = Fixture::new(); + let mut state = state(); + let mut writer = Writer::new(fixture.path(), false); + let seen = Mutex::new(Vec::new()); + let observe = |record: &Record| { + let durable = read(&fixture.path()).unwrap(); + assert_eq!(durable.record.checkpoint_kind, record.checkpoint_kind); + seen.lock() + .unwrap() + .push((record.checkpoint_revision, record.checkpoint_kind, record.step)); + Ok(()) + }; + writer + .save_kind( + &mut state, + Step::Acceptance, + CheckpointKind::WorkerFinished, + &observe, + ) + .unwrap(); + writer.save(&mut state, Step::Acceptance, &observe).unwrap(); + assert_eq!( + *seen.lock().unwrap(), + [ + (1, CheckpointKind::WorkerFinished, Step::Acceptance), + (2, CheckpointKind::TaskTransition, Step::Acceptance) + ] + ); + } +} diff --git a/oxa/oxa-work/src/status.rs b/oxa/oxa-work/src/status.rs new file mode 100644 index 000000000..52345e656 --- /dev/null +++ b/oxa/oxa-work/src/status.rs @@ -0,0 +1,178 @@ +//! Read-only checkpoint observations; never acquire, rewrite or delete a task lock. +use crate::{state, Outcome, Record, Result, Step}; +use serde::Serialize; +use std::fs; +use std::os::unix::fs::MetadataExt; +use std::path::Path; + +#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum Liveness { + Finished, + Running, + Interrupted, + Unknown, +} + +#[derive(Debug, Serialize)] +pub struct Status { + #[serde(flatten)] + pub record: Record, + pub liveness: Liveness, +} + +pub fn status(storage: &Path, binary: &Path) -> Result> { + let directory = match fs::read_dir(storage) { + Ok(directory) => directory, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(error.to_string()), + }; + let locks = fs::read_to_string("/proc/locks"); + let mut records = Vec::new(); + for entry in directory { + let path = entry.map_err(|e| e.to_string())?.path(); + if path.extension().is_none_or(|suffix| suffix != "json") { + continue; + } + let mut record = match state::load(&path)? { + state::Stored::Current(state) => state.record, + state::Stored::Legacy(record) => *record, + }; + let liveness = if record.step == Step::Finished { + Liveness::Finished + } else { + match locked(&path.with_extension("lock"), locks.as_deref().ok(), binary) { + Some(true) => { + record.outcome = Outcome::Running; + Liveness::Running + } + Some(false) => { + record.outcome = Outcome::Blocked; + record.detail = "task was interrupted; rerun its task file to resume".into(); + Liveness::Interrupted + } + None => Liveness::Unknown, + } + }; + records.push(Status { record, liveness }); + } + records.sort_by(|a, b| a.record.id.cmp(&b.record.id)); + Ok(records) +} + +fn locked(path: &Path, locks: Option<&str>, binary: &Path) -> Option { + let metadata = match fs::metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Some(false), + Err(_) => return None, + }; + let device = metadata.dev(); + let major = ((device >> 8) & 0xfff) | ((device >> 32) & 0xfffff000); + let minor = (device & 0xff) | ((device >> 12) & 0xffffff00); + let matches_lock = |text: &str| { + text.lines().any(|line| { + let fields: Vec<_> = line.split_ascii_whitespace().collect(); + let Some(at) = fields.iter().position(|field| *field == "FLOCK") else { + return false; + }; + if fields.get(at + 2).copied() != Some("WRITE") { + return false; + } + let Some(identity) = fields.get(at + 4) else { + return false; + }; + let mut parts = identity.split(':'); + let parsed = (|| { + Some(( + u64::from_str_radix(parts.next()?, 16).ok()?, + u64::from_str_radix(parts.next()?, 16).ok()?, + parts.next()?.parse::().ok()?, + )) + })(); + parsed == Some((major, minor, metadata.ino())) + }) + }; + if locks.is_some_and(matches_lock) { + return Some(true); + } + // /proc/locks omits an inherited flock when its original owner has died. + // Every worker/supervisor inherits this same description as stdin; fdinfo + // still exposes the held lock. Inspect only descriptors of the file owner. + let mut uncertain = locks.is_none(); + let executable = binary.file_name()?.as_encoded_bytes(); + let comm = &executable[..executable.len().min(15)]; + for process in fs::read_dir("/proc").ok()? { + let process = process.ok()?; + if !process + .file_name() + .as_encoded_bytes() + .iter() + .all(u8::is_ascii_digit) + { + continue; + } + let Ok(owner) = process.metadata() else { + continue; + }; + if owner.uid() != metadata.uid() { + continue; + } + // Relevant keepers are the same oxa worker/supervisor executable. + // Ignore unrelated same-user sandboxes whose descriptors are private. + let name = match fs::read(process.path().join("comm")) { + Ok(name) => name, + Err(_) => { + uncertain = true; + continue; + } + }; + if name.strip_suffix(b"\n").unwrap_or(&name) != comm { + continue; + } + // Zombies have closed every descriptor, but Linux denies fd lookup + // for them. Their inaccessible stdin is not an unknown live keeper. + match fs::read_to_string(process.path().join("stat")) { + Ok(stat) + if stat + .rsplit_once(')') + .and_then(|(_, tail)| tail.split_ascii_whitespace().next()) + .is_some_and(|state| matches!(state, "Z" | "X")) => + { + continue + } + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(_) => { + uncertain = true; + continue; + } + } + let fd = process.path().join("fd/0"); + let inherited = match fs::metadata(fd) { + Ok(inherited) => inherited, + Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied => { + uncertain = true; + continue; + } + Err(_) => continue, + }; + if (inherited.dev(), inherited.ino()) != (device, metadata.ino()) { + continue; + } + let info = match fs::read_to_string(process.path().join("fdinfo/0")) { + Ok(info) => info, + Err(_) => { + uncertain = true; + continue; + } + }; + if matches_lock(&info) { + return Some(true); + } + } + if uncertain { + None + } else { + Some(false) + } +} diff --git a/oxa/oxa-work/src/tree.rs b/oxa/oxa-work/src/tree.rs index faa47df7d..fb135b4ae 100644 --- a/oxa/oxa-work/src/tree.rs +++ b/oxa/oxa-work/src/tree.rs @@ -148,7 +148,7 @@ pub(crate) fn prepare( }); } if retained && dirty_worktree(binary, tree, repo, policy, lock)? { - return Err("retained worktree has uncommitted changes; commit them before continuing".into()); + return Err("retained worktree has uncommitted changes; worker was not restarted".into()); } let merged = supervise( binary, @@ -217,14 +217,3 @@ pub(crate) fn ensure_merged(policy: &RunPolicy, tree: &Path, base: &str, branch: )), } } - -pub(crate) fn check_receipt(receipt: &Path, tree: &Path, task: &Task) -> Result<()> { - if receipt.exists() { - let record: super::Record = serde_json::from_slice(&fs::read(receipt).map_err(|e| e.to_string())?) - .map_err(|e| format!("retained receipt: {e}"))?; - if record.id != task.id || record.branch != task.branch || record.worktree != tree { - return Err("retained receipt belongs to a different task branch or worktree".into()); - } - } - Ok(()) -} diff --git a/oxa/oxa-work/src/worker.rs b/oxa/oxa-work/src/worker.rs index cb802ac31..f079095b0 100644 --- a/oxa/oxa-work/src/worker.rs +++ b/oxa/oxa-work/src/worker.rs @@ -3,8 +3,10 @@ use crate::{Result, RunPolicy, Task}; use serde::{Deserialize, Serialize}; use std::fs::File; use std::io::{BufRead, BufReader, Read}; +use std::os::unix::process::ExitStatusExt; use std::path::Path; use std::process::{Command, Stdio}; +use std::time::Duration; const LINE_CAP: usize = 64 * 1024; const DIAGNOSTIC_CAP: usize = 8192; @@ -19,11 +21,19 @@ pub enum WorkerOutcome { Interrupted, } +#[derive(Clone, Copy, Debug, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum WorkerInterruption { + Deadline, +} + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct WorkerResult { pub outcome: WorkerOutcome, pub exit_code: i32, pub error_class: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub interruption: Option, } impl WorkerResult { @@ -63,14 +73,27 @@ pub(crate) fn run( policy: &RunPolicy, lock: &File, session: Option<&str>, + remaining: Duration, + on_session: &mut (dyn FnMut(&str) -> Result<()> + Send), ) -> Result { + let terminate = policy + .terminate_binary + .as_deref() + .ok_or("no trusted kill executable outside the source project")?; let mut command = Command::new(binary); crate::isolate_git_environment(&mut command); command .env("PATH", &policy.host_path) .env_remove("OXA_TRUST_PROJECT") .current_dir(root) - .args(["-na", "--mode=json", "--model", &task.model]); + .args([ + "-na", + "--mode=json", + "--model", + &task.model, + "--max-time", + &remaining.as_secs_f64().to_string(), + ]); if policy.offline { command.arg("--offline"); } @@ -87,11 +110,61 @@ pub(crate) fn run( let stdout = child.stdout.take().ok_or("worker stdout missing")?; let stderr = child.stderr.take().ok_or("worker stderr missing")?; std::thread::scope(|scope| { - let header = scope.spawn(|| read_session(stdout)); - let terminal = scope.spawn(|| read_result(stderr)); - let status = child.wait().map_err(|e| format!("wait for worker: {e}"))?; - let session = header.join().map_err(|_| "worker header reader failed")??; - let (result, diagnostic) = terminal.join().map_err(|_| "worker result reader failed")??; + let (notice_tx, notice_rx) = std::sync::mpsc::channel(); + let header_tx = notice_tx.clone(); + let header = scope.spawn(move || { + read_session(stdout, &mut |id| { + let result = on_session(id); + if let Err(error) = &result { + let _ = header_tx.send(Some(error.clone())); + } + result + }) + }); + let terminal = scope.spawn(move || { + let result = read_result(stderr); + let _ = notice_tx.send(None); + result + }); + let waited = match crate::worker_wait::wait( + &mut child, + ¬ice_rx, + remaining, + Duration::from_secs(60), + Duration::from_secs(10), + terminate, + ) { + Ok(waited) => waited, + Err(error) => { + let _ = child.kill(); + let _ = child.wait(); + let _ = header.join(); + let _ = terminal.join(); + return Err(error); + } + }; + let session = header.join().map_err(|_| "worker header reader failed")?; + let terminal = terminal.join().map_err(|_| "worker result reader failed")?; + if let Some(error) = waited.failure { + return Err(error); + } + let status = waited.status; + if waited.deadline { + return Ok(Worker { + result: WorkerResult { + outcome: WorkerOutcome::Interrupted, + exit_code: status + .code() + .unwrap_or_else(|| 128 + status.signal().unwrap_or(9)), + error_class: Some("worker_deadline".into()), + interruption: Some(WorkerInterruption::Deadline), + }, + session: session.ok().flatten(), + diagnostic: "worker exceeded max_time and its wrap-up grace; worktree retained".into(), + }); + } + let session = session?; + let (result, diagnostic) = terminal?; if status.code() != Some(result.exit_code) { return Err("worker exit code disagrees with its oxa_result".into()); } @@ -106,7 +179,7 @@ pub(crate) fn run( }) } -fn read_session(pipe: impl Read) -> Result> { +fn read_session(pipe: impl Read, on_session: &mut dyn FnMut(&str) -> Result<()>) -> Result> { let mut reader = BufReader::new(pipe); let session = line(&mut reader)?.and_then(|line| { let value: serde_json::Value = serde_json::from_slice(&line).ok()?; @@ -121,7 +194,9 @@ fn read_session(pipe: impl Read) -> Result> { .all(|b| b.is_ascii_alphanumeric() || b"-_".contains(&b)) }) }); + let recorded = session.as_deref().map(on_session).transpose(); std::io::copy(&mut reader, &mut std::io::sink()).map_err(|e| e.to_string())?; + recorded?; Ok(session) } diff --git a/oxa/oxa-work/src/worker_wait.rs b/oxa/oxa-work/src/worker_wait.rs new file mode 100644 index 000000000..b0c4bff3f --- /dev/null +++ b/oxa/oxa-work/src/worker_wait.rs @@ -0,0 +1,261 @@ +//! Event-driven child waiting with one wall deadline and two escalation stages. +use crate::Result; +use std::path::Path; +use std::process::{Child, Command, ExitStatus, Stdio}; +use std::sync::mpsc::{Receiver, RecvTimeoutError}; +use std::time::{Duration, Instant}; + +pub(crate) struct Waited { + pub status: ExitStatus, + pub deadline: bool, + pub failure: Option, +} + +struct Sender(Option); +impl Drop for Sender { + fn drop(&mut self) { + if let Some(child) = &mut self.0 { + if child.try_wait().ok().flatten().is_none() { + let _ = child.kill(); + let _ = child.wait(); + } + } + } +} + +pub(crate) fn wait( + child: &mut Child, + notice: &Receiver>, + max_time: Duration, + grace: Duration, + kill_grace: Duration, + terminate: &Path, +) -> Result { + let term_at = Instant::now() + .checked_add(max_time) + .and_then(|at| at.checked_add(grace)) + .ok_or("worker deadline is too large")?; + let kill_at = term_at + .checked_add(kill_grace) + .ok_or("worker deadline is too large")?; + let mut deadline = false; + let mut streams_ended = false; + let mut sender = Sender(None); + loop { + if let Some(status) = child.try_wait().map_err(|e| e.to_string())? { + return Ok(Waited { + status, + deadline, + failure: None, + }); + } + let now = Instant::now(); + if !deadline && now >= term_at { + // The child remains unreaped: its positive PID cannot be reused. + // This absolute executable was validated before worker admission. + sender.0 = Command::new(terminate) + .args(["-TERM", "--", &child.id().to_string()]) + .env_clear() + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .ok(); + deadline = true; + } + if now >= kill_at { + let _ = child.kill(); + let status = child.wait().map_err(|e| e.to_string())?; + return Ok(Waited { + status, + deadline: true, + failure: None, + }); + } + let pause = (if deadline { kill_at } else { term_at }).saturating_duration_since(Instant::now()); + let pause = if streams_ended { + pause.min(Duration::from_millis(20)) + } else { + pause + }; + match notice.recv_timeout(pause) { + Ok(Some(error)) => { + let _ = child.kill(); + let status = child.wait().map_err(|e| e.to_string())?; + return Ok(Waited { + status, + deadline, + failure: Some(error), + }); + } + Ok(None) => streams_ended = true, + Err(RecvTimeoutError::Timeout) => {} + Err(RecvTimeoutError::Disconnected) => { + streams_ended = true; + std::thread::sleep(pause.min(Duration::from_millis(20))); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + use std::os::unix::{ + fs::{DirBuilderExt, PermissionsExt}, + process::ExitStatusExt, + }; + use std::path::PathBuf; + use std::sync::mpsc; + use std::time::{SystemTime, UNIX_EPOCH}; + + struct Fixture(PathBuf); + impl Fixture { + fn new() -> Self { + let root = std::env::temp_dir().join(format!( + "oxa-worker-deadline-{}-{}", + std::process::id(), + SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos() + )); + fs::DirBuilder::new().mode(0o700).create(&root).unwrap(); + Self(root) + } + fn child(&self, script: &str) -> Owned { + let child = Command::new("/bin/sh") + .args(["-c", script, "deadline-fixture"]) + .arg(self.0.join("ready")) + .arg(self.0.join("term")) + .env_clear() + .env("HOME", &self.0) + .env("PATH", "/usr/bin:/bin") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let child = Owned(child); + let end = Instant::now() + Duration::from_secs(5); + while !self.0.join("ready").exists() { + assert!(Instant::now() < end, "child installed its signal handler"); + std::thread::sleep(Duration::from_millis(5)); + } + child + } + } + impl Drop for Fixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + struct Owned(Child); + impl Drop for Owned { + fn drop(&mut self) { + if self.0.try_wait().ok().flatten().is_none() { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } + } + + fn notices(child: &mut Owned) -> (mpsc::Sender>, Receiver>) { + let (sender, receiver) = mpsc::channel(); + let done = sender.clone(); + let mut pipe = child.0.stderr.take().unwrap(); + std::thread::spawn(move || { + let _ = std::io::copy(&mut pipe, &mut std::io::sink()); + let _ = done.send(None); + }); + (sender, receiver) + } + + #[test] + fn wall_deadline_sends_sigterm_after_the_wrap_up_grace() { + let fixture = Fixture::new(); + let mut child = fixture.child("trap 'exit 42' TERM\nprintf ready > \"$1\"\nwhile :; do :; done\n"); + let (_sender, receiver) = notices(&mut child); + let started = Instant::now(); + let result = wait( + &mut child.0, + &receiver, + Duration::from_millis(30), + Duration::from_millis(30), + Duration::from_secs(2), + Path::new("/usr/bin/kill"), + ) + .unwrap(); + assert!(result.deadline); + assert_eq!(result.status.code(), Some(42)); + assert!(result.failure.is_none()); + assert!(started.elapsed() >= Duration::from_millis(60)); + } + + #[test] + fn a_child_that_keeps_running_after_sigterm_is_killed_even_after_stream_eof() { + let fixture = Fixture::new(); + let mut child = + fixture.child("exec 2>/dev/null\ntrap 'printf term > \"$2\"' TERM\nprintf ready > \"$1\"\nwhile :; do :; done\n"); + let (_sender, receiver) = notices(&mut child); + let started = Instant::now(); + let result = wait( + &mut child.0, + &receiver, + Duration::from_millis(30), + Duration::from_millis(30), + Duration::from_millis(200), + Path::new("/usr/bin/kill"), + ) + .unwrap(); + assert!(result.deadline); + assert_eq!(result.status.signal(), Some(9)); + assert_eq!(fs::read_to_string(fixture.0.join("term")).unwrap(), "term"); + assert!(started.elapsed() >= Duration::from_millis(260)); + } + + #[test] + fn a_checkpoint_failure_terminates_the_current_child_without_waiting_for_the_deadline() { + let fixture = Fixture::new(); + let mut child = fixture.child("printf ready > \"$1\"\nwhile :; do :; done\n"); + let (sender, receiver) = notices(&mut child); + sender.send(Some("synthetic checkpoint failure".into())).unwrap(); + let started = Instant::now(); + let result = wait( + &mut child.0, + &receiver, + Duration::from_secs(30), + Duration::from_secs(60), + Duration::from_secs(10), + Path::new("/usr/bin/kill"), + ) + .unwrap(); + assert!(!result.deadline); + assert_eq!(result.status.signal(), Some(9)); + assert_eq!(result.failure.as_deref(), Some("synthetic checkpoint failure")); + assert!(started.elapsed() < Duration::from_secs(5)); + } + + #[test] + fn a_stuck_term_sender_cannot_delay_the_final_kill() { + let fixture = Fixture::new(); + let sender = fixture.0.join("slow-terminate"); + fs::write(&sender, "#!/bin/sh\n/usr/bin/sleep 2\n").unwrap(); + fs::set_permissions(&sender, fs::Permissions::from_mode(0o700)).unwrap(); + let mut child = fixture.child("printf ready > \"$1\"\nwhile :; do :; done\n"); + let (_sender, receiver) = notices(&mut child); + let started = Instant::now(); + let result = wait( + &mut child.0, + &receiver, + Duration::from_millis(20), + Duration::from_millis(20), + Duration::from_millis(100), + &sender, + ) + .unwrap(); + assert_eq!(result.status.signal(), Some(9)); + assert!( + started.elapsed() < Duration::from_secs(1), + "TERM sender blocked SIGKILL" + ); + } +} From 4a12234b16ec333ad9a6f6ee8e04b40f6a134cea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:53:05 +0000 Subject: [PATCH 2/3] Require current checkpoint format and retain saving-lock coverage --- oxa/docs/work.md | 4 +-- oxa/oxa-cli/tests/work_state.rs | 55 ++++++++++++++++++++++++++++++++- oxa/oxa-work/src/checkpoint.rs | 5 +-- oxa/oxa-work/src/lib.rs | 45 ++++++++------------------- oxa/oxa-work/src/state.rs | 26 ++++------------ oxa/oxa-work/src/status.rs | 5 +-- 6 files changed, 77 insertions(+), 63 deletions(-) diff --git a/oxa/docs/work.md b/oxa/docs/work.md index 12fc7d393..f7ccddf70 100644 --- a/oxa/docs/work.md +++ b/oxa/docs/work.md @@ -101,8 +101,8 @@ To recover a task, run the same task file again. An unfinished checkpoint resume its saved step with the original base, commands, session and offline policy. Finished successful work is returned without another worker or check run; add `--rerun` to the task-file or issue command to start another attempt. A task spec -change requires a new ID. A receipt from before durable checkpoints needs an -explicit `--rerun` before adoption. A matching linked worktree is reused; the +change requires a new ID. Unsupported or corrupt checkpoints are preserved and +refused. A matching linked worktree is reused; the source checkout is never used as the task worktree. An existing branch checked out elsewhere is blocked. Dirty retained work is preserved and saved before retrying. A clean retained branch gets an ordinary sandboxed diff --git a/oxa/oxa-cli/tests/work_state.rs b/oxa/oxa-cli/tests/work_state.rs index ce4bff1ad..282a15068 100644 --- a/oxa/oxa-cli/tests/work_state.rs +++ b/oxa/oxa-cli/tests/work_state.rs @@ -211,6 +211,60 @@ printf ran > "$(git rev-parse --git-path save-hook-ran)" assert_eq!(fs::read_to_string(marker).unwrap(), "ran"); } +#[test] +fn a_parent_crash_during_saving_keeps_the_task_locked_until_the_commit_finishes() { + use std::os::unix::fs::PermissionsExt; + let env = TestEnv::new(); + setup(&env, "[]"); + env.write( + "work/project/.git/info/exclude", + "/.save-hook-started\n/.release-save-hook\n", + ); + let hook = env.write( + "work/project/.git/hooks/pre-commit", + "#!/bin/sh\nprintf ready > .save-hook-started\nwhile ! test -e .release-save-hook; do sleep 0.05; done\n", + ); + fs::set_permissions(hook, fs::Permissions::from_mode(0o755)).unwrap(); + let server = + FakeServer::start(vec![write_turn(), ScriptTurn::text("done")], FakeOpts::default()).unwrap(); + let mut child = Command::new(env!("CARGO_BIN_EXE_oxa")) + .args(["work", "../tasks.json"]) + .current_dir(project(&env)) + .env_clear() + .envs(env.envs(&server.url())) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let retained = env.work().join(".oxa-worktrees/project/test"); + until(|| retained.join(".save-hook-started").exists()); + let before = fs::read(path(&env)).unwrap(); + assert_eq!(load(&env)["record"]["step"], "saving"); + child.kill().unwrap(); + child.wait().unwrap(); + let blocked = run(&env, vec![]); + // Always release the orphaned save before checking the competing result. + fs::write(retained.join(".release-save-hook"), "release").unwrap(); + assert_eq!(blocked["outcome"], "blocked", "{blocked}"); + assert!(blocked["detail"].as_str().unwrap().contains("already running")); + assert_eq!(fs::read(path(&env)).unwrap(), before); + let lock_path = storage(&env).join("test.lock"); + until(|| { + fs::OpenOptions::new() + .read(true) + .write(true) + .open(&lock_path) + .unwrap() + .try_lock() + .is_ok() + }); + assert_eq!(git(&env, &retained, &["show", "HEAD:answer.txt"]), "retained"); + let (recovered, requests) = run_observed(&env, vec![], &["work", "../tasks.json"], None); + assert!(requests.is_empty()); + assert_eq!(recovered["outcome"], "done", "{recovered}"); + assert!(git(&env, &retained, &["status", "--porcelain"]).is_empty()); +} + #[test] fn a_parent_crash_retains_session_and_child_lock_then_resumes_the_same_session() { let env = TestEnv::new(); @@ -490,7 +544,6 @@ fn the_owner_time_limit_reaches_the_real_child_and_keeps_its_wrap_up_work() { assert_eq!(load(&env)["task"]["max_time"], 1); assert_eq!(record["step"], "finished"); assert_eq!(record["interruption"], "deadline"); - assert_eq!(record["interruption"], "deadline"); } #[test] diff --git a/oxa/oxa-work/src/checkpoint.rs b/oxa/oxa-work/src/checkpoint.rs index 8a3435e60..f2b080b68 100644 --- a/oxa/oxa-work/src/checkpoint.rs +++ b/oxa/oxa-work/src/checkpoint.rs @@ -125,10 +125,7 @@ impl Repository { Ok(metadata) if !metadata.is_file() => { return Err("pending checkpoint is not a regular file".into()) } - Ok(_) => match state::load(&path)? { - state::Stored::Current(state) => Some(state.record), - state::Stored::Legacy(_) => None, - }, + Ok(_) => Some(state::load(&path)?.record), }; if let Some(record) = record { if record.id != *id || record.checkpoint_revision > self.cursor.revision { diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index 2265e717a..05803289e 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -58,19 +58,12 @@ pub struct Record { pub worker: Option, pub session_id: Option, pub base_sha: Option, - #[serde(default)] pub checkpoint_revision: u64, - #[serde(default)] pub checkpoint_kind: CheckpointKind, - #[serde(default)] pub checkpoint_at_ms: u64, - #[serde(default)] pub step: Step, - #[serde(default)] pub head_sha: Option, - #[serde(default)] pub tree_sha: Option, - #[serde(default)] pub interruption: Option, } @@ -507,32 +500,20 @@ pub fn run_task_with_observer( }; let mut writer = state::Writer::new(receipt.clone(), replacing); let mut state = if replacing { - match state::load(&receipt)? { - state::Stored::Current(old) => { - if old.task != *task || old.record.worktree != tree { - return Err("task specification changed; use a new task id".into()); - } - writer.replay(&old, observer).map_err(RunError::Checkpoint)?; - if old.record.step == Step::Finished { - if !rerun && matches!(old.record.outcome, Outcome::Done | Outcome::NothingToDo) { - return Ok(old.record); - } - let mut new = fresh(old.record.checkpoint_revision); - new.offline |= old.offline; - new - } else { - *old - } - } - state::Stored::Legacy(old) => { - if old.id != task.id || old.branch != task.branch || old.worktree != tree { - return Err("retained receipt belongs to a different task branch or worktree".into()); - } - if !rerun { - return Err("receipt predates durable checkpoints; use --rerun to continue it".into()); - } - fresh(0) + let old = state::load(&receipt)?; + if old.task != *task || old.record.worktree != tree { + return Err("task specification changed; use a new task id".into()); + } + writer.replay(&old, observer).map_err(RunError::Checkpoint)?; + if old.record.step == Step::Finished { + if !rerun && matches!(old.record.outcome, Outcome::Done | Outcome::NothingToDo) { + return Ok(old.record); } + let mut new = fresh(old.record.checkpoint_revision); + new.offline |= old.offline; + new + } else { + old } } else { fresh(0) diff --git a/oxa/oxa-work/src/state.rs b/oxa/oxa-work/src/state.rs index 14ef50a88..7822578ab 100644 --- a/oxa/oxa-work/src/state.rs +++ b/oxa/oxa-work/src/state.rs @@ -89,19 +89,12 @@ pub(crate) fn now_ms() -> u64 { .unwrap_or(u64::MAX) } -pub(crate) enum Stored { - Current(Box), - Legacy(Box), -} #[cfg(test)] pub(crate) fn read(path: &Path) -> Result { - match load(path)? { - Stored::Current(state) => Ok(*state), - Stored::Legacy(_) => Err("receipt predates durable checkpoints".into()), - } + load(path) } -pub(crate) fn load(path: &Path) -> Result { +pub(crate) fn load(path: &Path) -> Result { let file = File::open(path).map_err(|e| format!("task checkpoint: {e}"))?; let mut bytes = Vec::new(); file.take(MAX_STATE_BYTES + 1) @@ -112,13 +105,6 @@ pub(crate) fn load(path: &Path) -> Result { } let value: serde_json::Value = serde_json::from_slice(&bytes).map_err(|e| format!("task checkpoint: {e}"))?; - if value.get("version").is_none() { - let record: Record = serde_json::from_value(value).map_err(|e| format!("retained receipt: {e}"))?; - if record.checkpoint_revision != 0 { - return Err("unknown task checkpoint format".into()); - } - return Ok(Stored::Legacy(Box::new(record))); - } let fields = value.as_object().ok_or("invalid task checkpoint")?; if fields.len() != 6 || [ @@ -134,8 +120,8 @@ pub(crate) fn load(path: &Path) -> Result { { return Err("incomplete or unknown task checkpoint fields".into()); } - // Public Record accepts older receipt defaults; a durable checkpoint must - // carry every field explicitly and cannot silently become Finished. + // Every field is explicit in the current format. Missing metadata cannot + // silently turn an unfinished task into Finished. let keys = [ "id", "branch", @@ -167,7 +153,7 @@ pub(crate) fn load(path: &Path) -> Result { if state.record.id != state.task.id || state.record.branch != state.task.branch { return Err("task checkpoint identity disagrees with its task".into()); } - Ok(Stored::Current(Box::new(state))) + Ok(state) } /// A writer is constructed only after validating an existing file under its task lock. @@ -430,7 +416,7 @@ mod tests { } #[test] - fn durable_records_reject_missing_step_revision_and_base_even_with_legacy_defaults() { + fn durable_records_require_explicit_metadata_fields() { let fixture = Fixture::new(); let mut state = state(); let mut writer = Writer::new(fixture.path(), false); diff --git a/oxa/oxa-work/src/status.rs b/oxa/oxa-work/src/status.rs index 52345e656..cf5522843 100644 --- a/oxa/oxa-work/src/status.rs +++ b/oxa/oxa-work/src/status.rs @@ -34,10 +34,7 @@ pub fn status(storage: &Path, binary: &Path) -> Result> { if path.extension().is_none_or(|suffix| suffix != "json") { continue; } - let mut record = match state::load(&path)? { - state::Stored::Current(state) => state.record, - state::Stored::Legacy(record) => *record, - }; + let mut record = state::load(&path)?.record; let liveness = if record.step == Step::Finished { Liveness::Finished } else { From 11dc511079791d986a4b179115e9fc9ccbc51fd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:08:15 +0000 Subject: [PATCH 3/3] Recover saved task plans before resolving mutable base inputs --- oxa/docs/work.md | 2 ++ oxa/oxa-cli/src/work_cmd.rs | 2 +- oxa/oxa-cli/tests/work_state.rs | 37 +++++++++++++++++++++++- oxa/oxa-work/src/lib.rs | 45 +++++++++++++++++++++++++++++ oxa/oxa-work/src/state.rs | 50 +++++++++++++++++++++++++++++++++ 5 files changed, 134 insertions(+), 2 deletions(-) diff --git a/oxa/docs/work.md b/oxa/docs/work.md index f7ccddf70..ac5c39b40 100644 --- a/oxa/docs/work.md +++ b/oxa/docs/work.md @@ -99,6 +99,8 @@ ignored files in a retained working directory are not commit-bound evidence. To recover a task, run the same task file again. An unfinished checkpoint resumes its saved step with the original base, commands, session and offline policy. +Recovery loads those inputs before resolving the current base branch or settings, +so a deleted base ref or invalid new checks cannot strand an existing attempt. Finished successful work is returned without another worker or check run; add `--rerun` to the task-file or issue command to start another attempt. A task spec change requires a new ID. Unsupported or corrupt checkpoints are preserved and diff --git a/oxa/oxa-cli/src/work_cmd.rs b/oxa/oxa-cli/src/work_cmd.rs index 28409514e..3f48661ec 100644 --- a/oxa/oxa-cli/src/work_cmd.rs +++ b/oxa/oxa-cli/src/work_cmd.rs @@ -165,7 +165,7 @@ pub(crate) fn run( .into_iter() .map(|task| { if is_trusted { - oxa_work::plan_task(&root, task.clone(), &policy) + oxa_work::plan_task_for_run(&root, task.clone(), &policy, rerun) .map_err(|detail| Box::new(oxa_work::blocked_record(&task, detail))) } else { Err(Box::new(oxa_work::blocked_record( diff --git a/oxa/oxa-cli/tests/work_state.rs b/oxa/oxa-cli/tests/work_state.rs index 282a15068..5b26ea8d9 100644 --- a/oxa/oxa-cli/tests/work_state.rs +++ b/oxa/oxa-cli/tests/work_state.rs @@ -144,7 +144,7 @@ fn restart_at_acceptance_uses_the_durable_base_and_commands() { setup(&env, r#"["printf ran >> attempts"]"#); let first = run(&env, vec![ScriptTurn::text("done")]); let old_base = git(&env, &project(&env), &["rev-parse", "main"]); - env.write("work/project/.oxa/settings.json", r#"{"done":["exit 7"]}"#); + env.write("work/project/.oxa/settings.json", r#"{"done":false}"#); git(&env, &project(&env), &["add", "."]); git(&env, &project(&env), &["commit", "-m", "advance base checks"]); let new_base = git(&env, &project(&env), &["rev-parse", "main"]); @@ -166,6 +166,41 @@ fn restart_at_acceptance_uses_the_durable_base_and_commands() { ); } +#[test] +fn a_deleted_base_cannot_strand_saving_or_a_cached_success() { + let env = TestEnv::new(); + setup(&env, "[]"); + let first = run(&env, vec![ScriptTurn::text("done")]); + git(&env, &project(&env), &["checkout", "--detach"]); + git(&env, &project(&env), &["branch", "-D", "main"]); + let (cached, requests) = run_observed(&env, vec![], &["work", "../tasks.json"], None); + assert_eq!(cached, first); + assert!(requests.is_empty()); + + fs::write(tree(&first).join("recovered.txt"), "kept").unwrap(); + let mut state = load(&env); + state["record"]["step"] = "saving".into(); + state["record"]["outcome"] = "red".into(); + state["record"]["detail"] = "saved acceptance failed".into(); + replace(&env, &state); + let (recovered, requests) = run_observed(&env, vec![], &["work", "../tasks.json"], None); + assert_eq!(recovered["outcome"], "red", "{recovered}"); + assert!(requests.is_empty()); + assert_eq!( + git(&env, tree(&recovered), &["show", "HEAD:recovered.txt"]), + "kept" + ); + + let before = fs::read(path(&env)).unwrap(); + let (fresh, requests) = run_observed(&env, vec![], &["work", "../tasks.json", "--rerun"], None); + assert_eq!( + fresh["outcome"], "blocked", + "new attempts still need a valid current base" + ); + assert!(requests.is_empty()); + assert_eq!(fs::read(path(&env)).unwrap(), before); +} + #[test] fn failed_commit_hook_preserves_files_and_blocks() { use std::os::unix::fs::PermissionsExt; diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index 05803289e..5fd59e43f 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -72,6 +72,7 @@ pub struct PlannedTask { task: Task, base_sha: String, commands: Vec, + checkpoint_revision: Option, } impl PlannedTask { pub fn task(&self) -> &Task { @@ -98,9 +99,44 @@ pub fn plan_task(repo: &Path, task: Task, policy: &RunPolicy) -> Result Result { + validate(&task)?; + let path = policy.storage.join(format!("{}.json", task.id)); + match fs::symlink_metadata(&path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return plan_task(repo, task, policy); + } + Err(error) => return Err(format!("task checkpoint: {error}")), + Ok(metadata) if !metadata.is_file() => return Err("unsafe task checkpoint".into()), + Ok(_) => {} + } + let saved = state::load(&path)?; + if saved.task != task { + return Err("task specification changed; use a new task id".into()); + } + if saved.record.step != Step::Finished + || (!rerun && matches!(saved.record.outcome, Outcome::Done | Outcome::NothingToDo)) + { + let execution = saved.execution.ok_or("missing task execution checkpoint")?; + if saved.record.base_sha.as_deref() != Some(&execution.base) { + return Err("saved task base disagrees with execution checkpoint".into()); + } + return Ok(PlannedTask { + task, + base_sha: execution.base, + commands: execution.commands, + checkpoint_revision: Some(saved.record.checkpoint_revision), + }); + } + plan_task(repo, task, policy) +} + fn output(mut command: Command) -> Result { command.stdin(Stdio::null()).output().map_err(|e| e.to_string()) } @@ -470,6 +506,9 @@ pub fn run_task_with_observer( // The inherited lock retains its inode; status never tries to acquire it. let receipt = policy.storage.join(format!("{}.json", task.id)); let replacing = receipt.exists(); + if planned.checkpoint_revision.is_some() && !replacing { + return Err("task checkpoint disappeared after planning; retry the task file".into()); + } let repo_dir = if common.file_name().is_some_and(|name| name == ".git") { common.parent().ok_or("git directory has no parent")? } else { @@ -501,6 +540,12 @@ pub fn run_task_with_observer( let mut writer = state::Writer::new(receipt.clone(), replacing); let mut state = if replacing { let old = state::load(&receipt)?; + if planned + .checkpoint_revision + .is_some_and(|revision| revision != old.record.checkpoint_revision) + { + return Err("task checkpoint changed after planning; retry the task file".into()); + } if old.task != *task || old.record.worktree != tree { return Err("task specification changed; use a new task id".into()); } diff --git a/oxa/oxa-work/src/state.rs b/oxa/oxa-work/src/state.rs index 7822578ab..73f80ee3a 100644 --- a/oxa/oxa-work/src/state.rs +++ b/oxa/oxa-work/src/state.rs @@ -287,6 +287,56 @@ mod tests { State::new(task, record, false) } + #[test] + fn recovery_planning_does_not_need_current_refs_or_settings() { + for step in [Step::Worker, Step::Saving, Step::Finished] { + let fixture = Fixture::new(); + let mut state = state(); + let base = "1".repeat(40); + state.record.base_sha = Some(base.clone()); + state.record.outcome = if step == Step::Finished { + Outcome::Done + } else { + Outcome::Running + }; + state.execution = Some(Execution { + initial: Some(base.clone()), + base: base.clone(), + prompt: "saved prompt".into(), + commands: vec!["saved command".into()], + worker_error: None, + merge_error: None, + failures: String::new(), + }); + Writer::new(fixture.path(), false) + .save(&mut state, step, &|_: &Record| Ok(())) + .unwrap(); + let before = fs::read(fixture.path()).unwrap(); + let policy = crate::RunPolicy { + git_binary: fixture.0.join("unavailable-git"), + host_path: "/usr/bin:/bin".into(), + terminate_binary: None, + offline: true, + storage: fixture.0.clone(), + }; + for rerun in [false, true] { + let planned = crate::plan_task_for_run(&fixture.0, state.task.clone(), &policy, rerun); + if step == Step::Finished && rerun { + assert!(planned.is_err(), "explicit new attempts require a fresh plan"); + } else { + let planned = planned.unwrap(); + assert_eq!(planned.base_sha, base); + assert_eq!(planned.commands, ["saved command"]); + assert_eq!( + planned.checkpoint_revision, + Some(state.record.checkpoint_revision) + ); + } + } + assert_eq!(fs::read(fixture.path()).unwrap(), before); + } + } + #[test] fn observer_sees_durable_private_checkpoint_and_monotonic_revisions() { let fixture = Fixture::new();