From 3936b70cee6f783bec8f0670c6e8a982e299ad38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:18:50 +0000 Subject: [PATCH 01/52] Register oxa-work crate for issue #333 --- oxa/Cargo.lock | 9 +++++++++ oxa/Cargo.toml | 3 ++- oxa/oxa-cli/Cargo.toml | 1 + oxa/oxa-work/Cargo.toml | 14 ++++++++++++++ oxa/oxa-work/src/lib.rs | 1 + 5 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 oxa/oxa-work/Cargo.toml create mode 100644 oxa/oxa-work/src/lib.rs diff --git a/oxa/Cargo.lock b/oxa/Cargo.lock index 485000b98..c5f1ba2d8 100644 --- a/oxa/Cargo.lock +++ b/oxa/Cargo.lock @@ -431,6 +431,7 @@ dependencies = [ "oxa-tools", "oxa-tui", "oxa-web", + "oxa-work", "regex", "serde_json", "sha2", @@ -694,6 +695,14 @@ dependencies = [ "ureq", ] +[[package]] +name = "oxa-work" +version = "0.2.0" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "parking_lot" version = "0.12.5" diff --git a/oxa/Cargo.toml b/oxa/Cargo.toml index b4a683d90..27d254cff 100644 --- a/oxa/Cargo.toml +++ b/oxa/Cargo.toml @@ -12,7 +12,7 @@ members = [ "oxa-core", "oxa-tui", "oxa-interactive", "oxa-rpc", "oxa-export", - "oxa-cli", + "oxa-cli", "oxa-work", "oxa-fake", "oxa-parity", ] @@ -45,6 +45,7 @@ oxa-interactive = { path = "oxa-interactive" } oxa-rpc = { path = "oxa-rpc" } oxa-export = { path = "oxa-export" } oxa-cli = { path = "oxa-cli" } +oxa-work = { path = "oxa-work" } oxa-fake = { path = "oxa-fake" } # external (plan sec. 4); no async runtime anywhere serde = { version = "1", features = ["derive"] } diff --git a/oxa/oxa-cli/Cargo.toml b/oxa/oxa-cli/Cargo.toml index f962b13ad..a6085a4db 100644 --- a/oxa/oxa-cli/Cargo.toml +++ b/oxa/oxa-cli/Cargo.toml @@ -17,6 +17,7 @@ name = "oxa" path = "src/main.rs" [dependencies] +oxa-work.workspace = true oxa-proto.workspace = true oxa-engine.workspace = true oxa-session.workspace = true diff --git a/oxa/oxa-work/Cargo.toml b/oxa/oxa-work/Cargo.toml new file mode 100644 index 000000000..6097461ce --- /dev/null +++ b/oxa/oxa-work/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "oxa-work" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +serde.workspace = true +serde_json.workspace = true + +[lints] +workspace = true diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs new file mode 100644 index 000000000..90f7f11b9 --- /dev/null +++ b/oxa/oxa-work/src/lib.rs @@ -0,0 +1 @@ +//! Unattended task runner. From fb25b362d9be2fe7c59d4ce3f12e9ce17e50124c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:23:27 +0000 Subject: [PATCH 02/52] Add opt-in work task runner with external acceptance checks (#333) --- oxa/docs/work.md | 33 +++++ oxa/oxa-cli/src/args.rs | 7 + oxa/oxa-cli/src/dispatch.rs | 1 + oxa/oxa-cli/src/lib.rs | 1 + oxa/oxa-cli/src/work_cmd.rs | 39 ++++++ oxa/oxa-cli/tests/work_cli.rs | 190 ++++++++++++++++++++++++++ oxa/oxa-work/src/lib.rs | 250 +++++++++++++++++++++++++++++++++- 7 files changed, 520 insertions(+), 1 deletion(-) create mode 100644 oxa/docs/work.md create mode 100644 oxa/oxa-cli/src/work_cmd.rs create mode 100644 oxa/oxa-cli/tests/work_cli.rs diff --git a/oxa/docs/work.md b/oxa/docs/work.md new file mode 100644 index 000000000..4bee57969 --- /dev/null +++ b/oxa/docs/work.md @@ -0,0 +1,33 @@ +# Unattended work + +`oxa work tasks.json` opts into sequential tasks in separate git worktrees. +Run it inside the project repository. The JSON file is an array: + +```json +[{"id":"docs-fix","model":"your-model","branch":"agent/docs-fix","base":"main","prompt":"Fix documentation links."}] +``` + +`base` defaults to `main`. IDs contain letters, digits, hyphens or underscores. +IDs and branches must be unique; branches must be new. For a GitHub issue use +`oxa work --issue 123 --model your-model`. This explicitly invokes your installed, +authenticated `gh` CLI to fetch the issue title and body. + +Each task runs a child `oxa -p` with its model. Normal agent permission, trust and +sandbox rules apply. Configure project trust before unattended use. The runner +executes the base version of `.oxa/settings.json`'s `done` commands outside the +agent after it exits, even without edits. These are trusted project shell commands: +invoking `work` authorizes their execution as your user. `in subdirectory: command` +runs in that worktree subdirectory; escaping it with the directory is rejected. +Agent-internal done checks may also run when the project is trusted. + +Failed acceptance output goes to one additional child run with the original prompt, +then acceptance runs once more. JSON lines report `done`, `nothing_to_do` (no +commits or file changes), `red` (checks still fail), or `blocked` (setup or agent +execution failed). Exit codes are 0 for success, 1 if any task is red or blocked, +and 2 for invalid input or runner errors. + +Worktrees and uncommitted changes remain under the common git directory's +`oxa-work/`. Result receipts are beside them at `.json`. No automatic +cleanup, commit, push, PR creation or continuation is implemented yet. Inspect +retained work before removing it with `git worktree remove`. Review task prompts +and bases before running them. diff --git a/oxa/oxa-cli/src/args.rs b/oxa/oxa-cli/src/args.rs index d62d19272..ecf283560 100644 --- a/oxa/oxa-cli/src/args.rs +++ b/oxa/oxa-cli/src/args.rs @@ -14,6 +14,8 @@ pub enum ModeArg { /// A subcommand (first positional word). #[derive(Debug, Clone, PartialEq, Eq)] pub enum Subcommand { + /// `oxa work | --issue N --model MODEL`. + Work(Vec), /// `oxa mcp add|remove|list|login|logout ..`: the words after `mcp`. Mcp(Vec), /// `oxa install|remove|update|list ..`: the verb and the words after it. @@ -174,6 +176,10 @@ const COMMANDS: &[(&str, &str)] = &[ "oxa report-issue ", "Draft a scrubbed ox-llm issue; submit posts it only after you type yes", ), + ( + "oxa work | --issue N --model MODEL", + "Run tasks in isolated worktrees with external acceptance checks", + ), ( "oxa wipe --session | --all [--yes] [--engine]", "Delete one session (or every session) and its spill files; --engine also drops the engine's state", @@ -340,6 +346,7 @@ fn subcommand(word: &str, rest: &[String]) -> Result, OxaErro "identity" => Subcommand::Identity(rest.to_vec()), "report-issue" => Subcommand::ReportIssue(rest.to_vec()), "wipe" => Subcommand::Wipe(rest.to_vec()), + "work" => Subcommand::Work(rest.to_vec()), "auth" => return Err(OxaError::unsupported("the auth command")), // pi's provider sign-in (`pi login [--api-key] [--no-browser]`, `pi logout`). "login" | "logout" => { diff --git a/oxa/oxa-cli/src/dispatch.rs b/oxa/oxa-cli/src/dispatch.rs index 101f58fe4..4bdc4b7b0 100644 --- a/oxa/oxa-cli/src/dispatch.rs +++ b/oxa/oxa-cli/src/dispatch.rs @@ -125,6 +125,7 @@ pub fn run(argv: &[String], env: &Env, io: Stdio) -> i32 { Err(e) => return report(&mut *stderr, &e), }; let result = match sub { + Subcommand::Work(words) => return crate::work_cmd::run(&words, &cwd, &mut *stdout, &mut *stderr), Subcommand::Mcp(words) => run_mcp(&words, &dirs, &cwd, &mut *stdout, &mut *stderr), Subcommand::Pkg(verb, words) => run_pkg(&verb, &words, &dirs, &cwd, &mut *stdout, &mut *stderr), Subcommand::Config(words) => run_config(&words, &dirs, &cwd, &mut *stdout), diff --git a/oxa/oxa-cli/src/lib.rs b/oxa/oxa-cli/src/lib.rs index b7096b398..f5cc30f16 100644 --- a/oxa/oxa-cli/src/lib.rs +++ b/oxa/oxa-cli/src/lib.rs @@ -31,6 +31,7 @@ mod report_cmd; mod sandbox_layout; mod themes; mod wipe_cmd; +mod work_cmd; pub use args::{parse_args, usage_text, ApiKey, Args, ModeArg, Subcommand}; pub use config_cmd::run_config; diff --git a/oxa/oxa-cli/src/work_cmd.rs b/oxa/oxa-cli/src/work_cmd.rs new file mode 100644 index 000000000..3d8769c5e --- /dev/null +++ b/oxa/oxa-cli/src/work_cmd.rs @@ -0,0 +1,39 @@ +use std::io::Write; +use std::path::{Path, PathBuf}; + +pub(crate) fn run(args: &[String], cwd: &Path, out: &mut dyn Write, err: &mut dyn Write) -> i32 { + let result = (|| -> Result { + let tasks = match args { + [file] if !file.starts_with('-') => oxa_work::read_tasks(&cwd.join(file))?, + [flag, number, model_flag, model] if flag == "--issue" && model_flag == "--model" => { + vec![oxa_work::issue_task( + cwd, + number.parse().map_err(|_| "invalid issue number")?, + model.clone(), + )?] + } + _ => return Err("usage: oxa work | --issue N --model MODEL".into()), + }; + let binary: PathBuf = std::env::current_exe().map_err(|e| e.to_string())?; + let mut code = 0; + for task in tasks { + let record = oxa_work::run_task(cwd, &binary, &task)?; + if matches!( + record.outcome, + oxa_work::Outcome::Red | oxa_work::Outcome::Blocked + ) { + code = 1; + } + serde_json::to_writer(&mut *out, &record).map_err(|e| e.to_string())?; + writeln!(out).map_err(|e| e.to_string())?; + } + Ok(code) + })(); + match result { + Ok(code) => code, + Err(e) => { + let _ = writeln!(err, "oxa work: {e}"); + 2 + } + } +} diff --git a/oxa/oxa-cli/tests/work_cli.rs b/oxa/oxa-cli/tests/work_cli.rs new file mode 100644 index 000000000..fa28e7b54 --- /dev/null +++ b/oxa/oxa-cli/tests/work_cli.rs @@ -0,0 +1,190 @@ +//! Real child oxa with fake engine and local bare git remote; no network. +use oxa_fake::{FakeOpts, FakeServer, ScriptTurn, TestEnv}; +use std::{path::Path, process::Command}; +fn git(env: &TestEnv, dir: &Path, args: &[&str]) { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .env_clear() + .envs(env.envs("http://127.0.0.1:1")) + .env("GIT_AUTHOR_NAME", "Test") + .env("GIT_AUTHOR_EMAIL", "test@example.invalid") + .env("GIT_COMMITTER_NAME", "Test") + .env("GIT_COMMITTER_EMAIL", "test@example.invalid") + .output() + .unwrap(); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); +} +fn setup(env: &TestEnv, done: &str) { + git(env, &env.work(), &["init", "--bare", "remote.git"]); + git(env, &env.work(), &["init", "-b", "main", "project"]); + let project = env.work().join("project"); + env.write( + "work/project/.oxa/settings.json", + &format!(r#"{{"done":{done}}}"#), + ); + git(env, &project, &["add", "."]); + git(env, &project, &["commit", "-m", "fixture"]); + git(env, &project, &["remote", "add", "origin", "../remote.git"]); + git(env, &project, &["push", "origin", "main"]); + env.write( + "work/tasks.json", + r#"[{"id":"test","branch":"agent/test","model":"fake","prompt":"Finish task"}]"#, + ); +} +fn run(env: &TestEnv, turns: Vec) -> serde_json::Value { + run_args(env, turns, &["work", "../tasks.json"], None) +} +fn run_args(env: &TestEnv, turns: Vec, args: &[&str], path: Option) -> serde_json::Value { + let server = FakeServer::start(turns, FakeOpts::default()).unwrap(); + let mut command = Command::new(env!("CARGO_BIN_EXE_oxa")); + command + .args(args) + .current_dir(env.work().join("project")) + .env_clear() + .envs(env.envs(&server.url())); + if let Some(path) = path { + command.env("PATH", path); + } + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + tx.send(command.output().unwrap()).unwrap(); + }); + let out = rx + .recv_timeout(std::time::Duration::from_secs(90)) + .expect("runner finished"); + serde_json::from_slice(&out.stdout).unwrap_or_else(|e| { + panic!( + "{e}: {} / {}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ) + }) +} +#[test] +fn external_checks_run_without_edits_and_retain_worktree() { + let env = TestEnv::new(); + setup(&env, r#"["printf checked > receipt"]"#); + let record = run(&env, vec![ScriptTurn::text("finished")]); + assert_eq!(record["outcome"], "done", "{record}"); + let tree = Path::new(record["worktree"].as_str().unwrap()); + assert_eq!(std::fs::read_to_string(tree.join("receipt")).unwrap(), "checked"); + assert!(env.work().join("project/.git/oxa-work/test.json").exists()); +} +#[test] +fn clean_task_is_nothing_to_do() { + let env = TestEnv::new(); + setup(&env, "[]"); + assert_eq!( + run(&env, vec![ScriptTurn::text("nothing")])["outcome"], + "nothing_to_do" + ); +} +#[test] +fn failed_checks_get_exactly_one_retry() { + let env = TestEnv::new(); + setup( + &env, + r#"["printf attempt >> attempts; echo external-check; exit 1"]"#, + ); + let record = run( + &env, + vec![ScriptTurn::text("finished"), ScriptTurn::text("cannot repair")], + ); + assert_eq!(record["outcome"], "red", "{record}"); + assert!(record["detail"].as_str().unwrap().contains("external-check")); + let tree = Path::new(record["worktree"].as_str().unwrap()); + assert_eq!( + std::fs::read_to_string(tree.join("attempts")).unwrap(), + "attemptattempt" + ); +} +#[test] +fn retry_can_make_acceptance_green() { + let env = TestEnv::new(); + setup( + &env, + r#"["if test -f once; then true; else touch once; exit 1; fi"]"#, + ); + let record = run( + &env, + vec![ScriptTurn::text("finished"), ScriptTurn::text("retry")], + ); + assert_eq!(record["outcome"], "done", "{record}"); +} + +#[test] +fn agent_edits_stay_in_worktree() { + let env = TestEnv::new(); + setup(&env, "[]"); + env.write( + "home/.oxa/agent/settings.json", + r#"{"defaultProjectTrust":"always"}"#, + ); + let record = run( + &env, + vec![ + ScriptTurn::tool("edit", "write", r#"{"path":"answer.txt","content":"finished"}"#), + ScriptTurn::text("done"), + ], + ); + assert_eq!(record["outcome"], "done", "{record}"); + let tree = Path::new(record["worktree"].as_str().unwrap()); + assert_eq!( + std::fs::read_to_string(tree.join("answer.txt")).unwrap(), + "finished" + ); + assert!(!env.work().join("project/answer.txt").exists()); +} +#[test] +fn invalid_task_file_is_rejected_before_creating_worktree() { + let env = TestEnv::new(); + setup(&env, "[]"); + env.write( + "work/tasks.json", + r#"[{"id":"../escape","branch":"agent/test","model":"fake","prompt":"go"}]"#, + ); + let out = Command::new(env!("CARGO_BIN_EXE_oxa")) + .args(["work", "../tasks.json"]) + .current_dir(env.work().join("project")) + .env_clear() + .envs(env.envs("http://127.0.0.1:1")) + .output() + .unwrap(); + assert_eq!(out.status.code(), Some(2)); + assert!(!env.work().join("project/.git/oxa-work").exists()); +} + +#[test] +fn github_issue_input_uses_mock_gh() { + use std::os::unix::fs::PermissionsExt; + let env = TestEnv::new(); + setup(&env, "[]"); + let gh = env.write( + "tmp/gh", + r##"#!/bin/sh +[ "$*" = 'issue view 42 --json title,body' ] || exit 9 +printf '%s' '{"title":"Synthetic task","body":"Fix the test fixture"}' +"##, + ); + std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap(); + let path = format!("{}:{}", env.tmp().display(), std::env::var("PATH").unwrap()); + let record = run_args( + &env, + vec![ScriptTurn::text("done")], + &["work", "--issue", "42", "--model", "fake"], + Some(path), + ); + assert_eq!(record["id"], "issue-42"); + assert_eq!(record["branch"], "agent/42-work"); + assert_eq!(record["outcome"], "nothing_to_do", "{record}"); +} +#[test] +fn setup_failure_records_blocked_and_keeps_existing_branch() { + let env = TestEnv::new(); + setup(&env, "[]"); + git(&env, &env.work().join("project"), &["branch", "agent/test"]); + let record = run(&env, vec![]); + assert_eq!(record["outcome"], "blocked", "{record}"); + assert!(record["detail"].as_str().unwrap().contains("already exists")); +} diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index 90f7f11b9..5f1dbd487 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -1 +1,249 @@ -//! Unattended task runner. +//! Opt-in, sequential unattended task execution. Worktrees are retained on every outcome. +use serde::{Deserialize, Serialize}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output, Stdio}; + +pub type Result = std::result::Result; + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Task { + pub id: String, + pub model: String, + pub branch: String, + #[serde(default = "default_base")] + pub base: String, + pub prompt: String, +} +fn default_base() -> String { + "main".into() +} + +#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum Outcome { + Done, + NothingToDo, + Red, + Blocked, +} + +#[derive(Debug, Serialize)] +pub struct Record { + pub id: String, + pub branch: String, + pub worktree: PathBuf, + pub outcome: Outcome, + pub detail: String, +} + +fn output(mut command: Command) -> Result { + command.stdin(Stdio::null()).output().map_err(|e| e.to_string()) +} +fn checked(command: Command) -> Result { + let out = output(command)?; + if !out.status.success() { + return Err(String::from_utf8_lossy(&out.stderr).into_owned()); + } + Ok(String::from_utf8_lossy(&out.stdout).trim().into()) +} +fn git(repo: &Path, args: &[&str]) -> Command { + let mut c = Command::new("git"); + c.current_dir(repo).args(args); + c +} + +pub fn read_tasks(path: &Path) -> Result> { + let tasks: Vec = serde_json::from_slice(&fs::read(path).map_err(|e| e.to_string())?) + .map_err(|e| format!("task file: {e}"))?; + if tasks.is_empty() { + return Err("task file is empty".into()); + } + let mut ids = std::collections::HashSet::new(); + let mut branches = std::collections::HashSet::new(); + for task in &tasks { + validate(task)?; + if !ids.insert(&task.id) || !branches.insert(&task.branch) { + return Err("duplicate task id or branch".into()); + } + } + Ok(tasks) +} +fn validate(task: &Task) -> Result<()> { + if task.id.is_empty() + || !task + .id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"-_".contains(&b)) + { + return Err("task id must contain only letters, digits, - or _".into()); + } + if task.model.trim().is_empty() || task.prompt.trim().is_empty() { + return Err("model and prompt are required".into()); + } + for reference in [&task.branch, &task.base] { + if reference.is_empty() || reference.starts_with('-') { + return Err("invalid branch or base".into()); + } + } + Ok(()) +} + +/// Issue text is data, passed as one prompt argument; gh never invokes a shell. +pub fn issue_task(repo: &Path, number: u64, model: String) -> Result { + if number == 0 { + return Err("issue number must be positive".into()); + } + let mut c = Command::new("gh"); + c.current_dir(repo) + .args(["issue", "view", &number.to_string(), "--json", "title,body"]); + let text = checked(c)?; + let issue: serde_json::Value = serde_json::from_str(&text).map_err(|e| e.to_string())?; + let title = issue["title"].as_str().ok_or("issue title missing")?; + let body = issue["body"].as_str().ok_or("issue body missing")?; + Ok(Task { + id: format!("issue-{number}"), + model, + branch: format!("agent/{number}-work"), + base: default_base(), + prompt: format!("Implement issue #{number}: {title}\n\n{body}"), + }) +} + +/// Acceptance commands are frozen from the base before the agent can edit settings. +fn done_commands(root: &Path) -> Result> { + let path = root.join(".oxa/settings.json"); + let bytes = match fs::read(path) { + Ok(bytes) => bytes, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(vec![]), + Err(e) => return Err(e.to_string()), + }; + let value: serde_json::Value = serde_json::from_slice(&bytes).map_err(|e| e.to_string())?; + match value.get("done") { + None => Ok(vec![]), + Some(value) => serde_json::from_value(value.clone()).map_err(|e| format!("done commands: {e}")), + } +} +fn acceptance(root: &Path, commands: &[String]) -> Result { + let mut failures = String::new(); + for entry in commands { + let (dir, shell) = match entry.strip_prefix("in ").and_then(|s| s.split_once(':')) { + Some((dir, cmd)) => (root.join(dir.trim()), cmd.trim()), + None => (root.to_path_buf(), entry.as_str()), + }; + let dir = dir.canonicalize().map_err(|e| e.to_string())?; + if !dir.starts_with(root) { + return Err("done directory escapes worktree".into()); + } + let mut c = Command::new("sh"); + c.current_dir(dir).args(["-c", shell]); + let out = output(c)?; + if !out.status.success() { + failures.push_str(&format!( + "{entry}: {}\n{}\n{}\n", + out.status, + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + )); + } + } + Ok(failures) +} +fn agent(binary: &Path, root: &Path, task: &Task, prompt: &str) -> Result<()> { + let mut c = Command::new(binary); + c.current_dir(root) + .args(["-p", "--model", &task.model, "--no-session", "--", prompt]); + checked(c).map(|_| ()) +} + +struct TaskLock(PathBuf); +impl Drop for TaskLock { + fn drop(&mut self) { + let _ = fs::remove_file(&self.0); + } +} + +pub fn run_task(repo: &Path, binary: &Path, task: &Task) -> Result { + validate(task)?; + let root = checked(git(repo, &["rev-parse", "--show-toplevel"]))?; + let root = PathBuf::from(root); + checked(git(&root, &["check-ref-format", "--branch", &task.branch]))?; + let common = checked(git( + &root, + &["rev-parse", "--path-format=absolute", "--git-common-dir"], + ))?; + let storage = PathBuf::from(common).join("oxa-work"); + fs::create_dir_all(&storage).map_err(|e| e.to_string())?; + let lock_path = storage.join(format!("{}.lock", task.id)); + let _lock_file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&lock_path) + .map_err(|e| format!("cannot lock task {}: {e}", task.id))?; + let _lock = TaskLock(lock_path); + let receipt = storage.join(format!("{}.json", task.id)); + if receipt.exists() { + return Err("task already has a receipt; retained for inspection".into()); + } + let tree = storage.join(&task.id); + let mut record = Record { + id: task.id.clone(), + branch: task.branch.clone(), + worktree: tree.clone(), + outcome: Outcome::Blocked, + detail: String::new(), + }; + let result = (|| -> Result { + if tree.exists() { + return Err("task worktree already exists; retained for inspection".into()); + } + let tree_arg = tree.to_str().ok_or("non-UTF8 worktree path")?; + checked(git( + &root, + &["worktree", "add", "-b", &task.branch, tree_arg, &task.base], + ))?; + let tree = tree.canonicalize().map_err(|e| e.to_string())?; + let initial = checked(git(&tree, &["rev-parse", "HEAD"]))?; + let commands = done_commands(&tree)?; + let first_agent = agent(binary, &tree, task, &task.prompt); + let failures = acceptance(&tree, &commands)?; + if !failures.is_empty() { + let repair_agent = agent( + binary, + &tree, + task, + &format!( + "{}\n\nExternal acceptance checks failed. Repair once:\n{failures}", + task.prompt + ), + ); + let failures = acceptance(&tree, &commands)?; + if !failures.is_empty() { + record.detail = failures; + return Ok(Outcome::Red); + } + repair_agent?; + } else { + first_agent?; + } + let head = checked(git(&tree, &["rev-parse", "HEAD"]))?; + let dirty = checked(git(&tree, &["status", "--porcelain"]))?; + Ok(if head == initial && dirty.is_empty() { + Outcome::NothingToDo + } else { + Outcome::Done + }) + })(); + match result { + Ok(outcome) => record.outcome = outcome, + Err(e) => record.detail = e, + } + // Separate receipts avoid overwriting another task's result. Never remove its worktree. + fs::write( + receipt, + serde_json::to_vec_pretty(&record).map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + Ok(record) +} From cdfab279d60b4ae005dcc413507d9bac49a32948 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:48:01 +0000 Subject: [PATCH 03/52] Fix sandbox commits and offline issue lookup in work runner --- oxa/docs/work.md | 9 ++-- oxa/oxa-cli/src/work_cmd.rs | 3 ++ oxa/oxa-cli/tests/work_cli.rs | 80 +++++++++++++++++++++++++++++++++++ oxa/oxa-work/src/lib.rs | 18 +++++++- 4 files changed, 105 insertions(+), 5 deletions(-) diff --git a/oxa/docs/work.md b/oxa/docs/work.md index 4bee57969..ed74c8ab8 100644 --- a/oxa/docs/work.md +++ b/oxa/docs/work.md @@ -10,7 +10,8 @@ Run it inside the project repository. The JSON file is an array: `base` defaults to `main`. IDs contain letters, digits, hyphens or underscores. IDs and branches must be unique; branches must be new. For a GitHub issue use `oxa work --issue 123 --model your-model`. This explicitly invokes your installed, -authenticated `gh` CLI to fetch the issue title and body. +authenticated `gh` CLI to fetch the issue title and body. Issue lookup is refused +when the environment or user settings enable offline mode. Each task runs a child `oxa -p` with its model. Normal agent permission, trust and sandbox rules apply. Configure project trust before unattended use. The runner @@ -26,8 +27,10 @@ commits or file changes), `red` (checks still fail), or `blocked` (setup or agen execution failed). Exit codes are 0 for success, 1 if any task is red or blocked, and 2 for invalid input or runner errors. -Worktrees and uncommitted changes remain under the common git directory's -`oxa-work/`. Result receipts are beside them at `.json`. No automatic +Worktrees and uncommitted changes remain in `.oxa-worktrees//` +beside the repository (or the separate git directory). Result receipts live inside +the common git directory at `oxa-work/.json`. Worktrees stay outside that git +directory so the normal sandbox can bind git objects and refs for commits. No automatic cleanup, commit, push, PR creation or continuation is implemented yet. Inspect retained work before removing it with `git worktree remove`. Review task prompts and bases before running them. diff --git a/oxa/oxa-cli/src/work_cmd.rs b/oxa/oxa-cli/src/work_cmd.rs index 3d8769c5e..af2d67257 100644 --- a/oxa/oxa-cli/src/work_cmd.rs +++ b/oxa/oxa-cli/src/work_cmd.rs @@ -6,6 +6,9 @@ pub(crate) fn run(args: &[String], cwd: &Path, out: &mut dyn Write, err: &mut dy let tasks = match args { [file] if !file.starts_with('-') => oxa_work::read_tasks(&cwd.join(file))?, [flag, number, model_flag, model] if flag == "--issue" && model_flag == "--model" => { + if oxa_proto::offline() { + return Err("GitHub issue lookup is unavailable in offline mode".into()); + } vec![oxa_work::issue_task( cwd, number.parse().map_err(|_| "invalid issue number")?, diff --git a/oxa/oxa-cli/tests/work_cli.rs b/oxa/oxa-cli/tests/work_cli.rs index fa28e7b54..ec6279eef 100644 --- a/oxa/oxa-cli/tests/work_cli.rs +++ b/oxa/oxa-cli/tests/work_cli.rs @@ -188,3 +188,83 @@ fn setup_failure_records_blocked_and_keeps_existing_branch() { assert_eq!(record["outcome"], "blocked", "{record}"); assert!(record["detail"].as_str().unwrap().contains("already exists")); } + +#[test] +fn child_can_create_a_commit_with_the_real_sandbox() { + let env = TestEnv::new(); + setup(&env, "[]"); + env.write( + "home/.oxa/agent/settings.json", + r#"{"defaultProjectTrust":"always"}"#, + ); + let record = run( + &env, + vec![ + ScriptTurn::tool("write", "write", r#"{"path":"answer.txt","content":"committed"}"#), + ScriptTurn::tool( + "commit", + "bash", + r#"{"command":"git add answer.txt && git -c user.name=Test -c user.email=test@example.invalid commit -m task"}"#, + ), + ScriptTurn::text("done"), + ], + ); + assert_eq!(record["outcome"], "done", "{record}"); + let tree = Path::new(record["worktree"].as_str().unwrap()); + let status = Command::new("git") + .args(["status", "--porcelain"]) + .current_dir(tree) + .env_clear() + .envs(env.envs("http://127.0.0.1:1")) + .output() + .unwrap(); + assert!(status.status.success()); + assert!( + status.stdout.is_empty(), + "commit must persist: {}", + String::from_utf8_lossy(&status.stdout) + ); + let subject = Command::new("git") + .args(["log", "-1", "--format=%s"]) + .current_dir(tree) + .env_clear() + .envs(env.envs("http://127.0.0.1:1")) + .output() + .unwrap(); + assert_eq!(String::from_utf8_lossy(&subject.stdout).trim(), "task"); +} +#[test] +fn offline_issue_input_never_spawns_gh() { + use std::os::unix::fs::PermissionsExt; + for setting in [false, true] { + let env = TestEnv::new(); + setup(&env, "[]"); + let gh = env.write( + "tmp/gh", + "#!/bin/sh\nprintf invoked > \"$HOME/gh-invoked\"\nexit 7\n", + ); + std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap(); + if setting { + env.write("home/.oxa/agent/settings.json", r#"{"oxa":{"offline":true}}"#); + } + let mut cmd = Command::new(env!("CARGO_BIN_EXE_oxa")); + cmd.args(["work", "--issue", "42", "--model", "fake"]) + .current_dir(env.work().join("project")) + .env_clear() + .envs(env.envs("http://127.0.0.1:1")) + .env( + "PATH", + format!("{}:{}", env.tmp().display(), std::env::var("PATH").unwrap()), + ); + if !setting { + cmd.env("OXA_OFFLINE", "1"); + } + let out = cmd.output().unwrap(); + assert_eq!(out.status.code(), Some(2)); + assert!( + !env.home().join("gh-invoked").exists(), + "offline lookup ran gh, setting={setting}" + ); + assert!(String::from_utf8_lossy(&out.stderr).contains("offline")); + } +} diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index 5f1dbd487..4ea8a1551 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -173,7 +173,8 @@ pub fn run_task(repo: &Path, binary: &Path, task: &Task) -> Result { &root, &["rev-parse", "--path-format=absolute", "--git-common-dir"], ))?; - let storage = PathBuf::from(common).join("oxa-work"); + let common = PathBuf::from(common); + let storage = common.join("oxa-work"); fs::create_dir_all(&storage).map_err(|e| e.to_string())?; let lock_path = storage.join(format!("{}.lock", task.id)); let _lock_file = fs::OpenOptions::new() @@ -186,7 +187,20 @@ pub fn run_task(repo: &Path, binary: &Path, task: &Task) -> Result { if receipt.exists() { return Err("task already has a receipt; retained for inspection".into()); } - let tree = storage.join(&task.id); + // The common git directory must not be an ancestor of the child cwd: the sandbox + // deliberately refuses writable git binds with that layout. + let repo_dir = if common.file_name().is_some_and(|name| name == ".git") { + common.parent().ok_or("git directory has no parent")? + } else { + &common + }; + let parent = repo_dir + .parent() + .ok_or("repository has no parent for worktrees")?; + let name = repo_dir.file_name().ok_or("repository has no directory name")?; + let trees = parent.join(".oxa-worktrees").join(name); + fs::create_dir_all(&trees).map_err(|e| e.to_string())?; + let tree = trees.join(&task.id); let mut record = Record { id: task.id.clone(), branch: task.branch.clone(), From c86fa8b9bf3df4ae49612e9458f2fa51aad354bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:50:25 +0000 Subject: [PATCH 04/52] Continue work branches with ordinary base merges (#333) --- oxa/docs/work.md | 16 ++- oxa/oxa-cli/tests/common/work.rs | 79 +++++++++++ oxa/oxa-cli/tests/work_cli.rs | 75 ++-------- oxa/oxa-cli/tests/work_continue.rs | 215 +++++++++++++++++++++++++++++ oxa/oxa-work/src/lib.rs | 43 ++---- oxa/oxa-work/src/tree.rs | 147 ++++++++++++++++++++ 6 files changed, 477 insertions(+), 98 deletions(-) create mode 100644 oxa/oxa-cli/tests/common/work.rs create mode 100644 oxa/oxa-cli/tests/work_continue.rs create mode 100644 oxa/oxa-work/src/tree.rs diff --git a/oxa/docs/work.md b/oxa/docs/work.md index ed74c8ab8..0772b2b76 100644 --- a/oxa/docs/work.md +++ b/oxa/docs/work.md @@ -8,7 +8,9 @@ Run it inside the project repository. The JSON file is an array: ``` `base` defaults to `main`. IDs contain letters, digits, hyphens or underscores. -IDs and branches must be unique; branches must be new. For a GitHub issue use +IDs and branches must be unique within one file. A new branch starts from `base`; +an existing branch is continued with a normal merge of that base before the agent +starts. No remote fetch happens automatically. For a GitHub issue use `oxa work --issue 123 --model your-model`. This explicitly invokes your installed, authenticated `gh` CLI to fetch the issue title and body. Issue lookup is refused when the environment or user settings enable offline mode. @@ -20,6 +22,8 @@ agent after it exits, even without edits. These are trusted project shell comman invoking `work` authorizes their execution as your user. `in subdirectory: command` runs in that worktree subdirectory; escaping it with the directory is rejected. Agent-internal done checks may also run when the project is trusted. +On continued branches, acceptance commands are read from the captured base commit, +even when the task branch has changed its own settings. Failed acceptance output goes to one additional child run with the original prompt, then acceptance runs once more. JSON lines report `done`, `nothing_to_do` (no @@ -31,6 +35,14 @@ Worktrees and uncommitted changes remain in `.oxa-worktrees//` beside the repository (or the separate git directory). Result receipts live inside the common git directory at `oxa-work/.json`. Worktrees stay outside that git directory so the normal sandbox can bind git objects and refs for commits. No automatic -cleanup, commit, push, PR creation or continuation is implemented yet. Inspect +cleanup, commit, push or PR creation is implemented yet. + +To continue a task, run the same file again. The retained receipt must match its +ID, branch and worktree. A clean worktree is reused; a branch checked out elsewhere +is blocked. Uncommitted changes are retained and blocked until you commit them. +Merge conflicts are passed to the agent with the merge still in progress, the +conflicted file list and merge output. An unfinished merge can be continued on a +later run. The agent must finish the merge commit and keep the task branch selected; +unresolved or aborted base merges remain blocked even when checks would pass. Inspect retained work before removing it with `git worktree remove`. Review task prompts and bases before running them. diff --git a/oxa/oxa-cli/tests/common/work.rs b/oxa/oxa-cli/tests/common/work.rs new file mode 100644 index 000000000..44062a99f --- /dev/null +++ b/oxa/oxa-cli/tests/common/work.rs @@ -0,0 +1,79 @@ +use oxa_fake::{FakeOpts, FakeServer, ScriptTurn, TestEnv}; +use std::{path::Path, process::Command}; +pub fn git(env: &TestEnv, dir: &Path, args: &[&str]) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .env_clear() + .envs(env.envs("http://127.0.0.1:1")) + .env("GIT_AUTHOR_NAME", "Test") + .env("GIT_AUTHOR_EMAIL", "test@example.invalid") + .env("GIT_COMMITTER_NAME", "Test") + .env("GIT_COMMITTER_EMAIL", "test@example.invalid") + .output() + .unwrap(); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + String::from_utf8_lossy(&out.stdout).trim().into() +} +pub fn setup(env: &TestEnv, done: &str) { + git(env, &env.work(), &["init", "--bare", "remote.git"]); + git(env, &env.work(), &["init", "-b", "main", "project"]); + let project = env.work().join("project"); + git(env, &project, &["config", "user.name", "Test"]); + git(env, &project, &["config", "user.email", "test@example.invalid"]); + env.write( + "work/project/.oxa/settings.json", + &format!(r#"{{"done":{done}}}"#), + ); + git(env, &project, &["add", "."]); + git(env, &project, &["commit", "-m", "fixture"]); + git(env, &project, &["remote", "add", "origin", "../remote.git"]); + git(env, &project, &["push", "origin", "main"]); + env.write( + "work/tasks.json", + r#"[{"id":"test","branch":"agent/test","model":"fake","prompt":"Finish task"}]"#, + ); +} +pub fn run(env: &TestEnv, turns: Vec) -> serde_json::Value { + run_args(env, turns, &["work", "../tasks.json"], None) +} +pub fn run_args( + env: &TestEnv, + turns: Vec, + args: &[&str], + path: Option, +) -> serde_json::Value { + run_observed(env, turns, args, path).0 +} +pub fn run_observed( + env: &TestEnv, + turns: Vec, + args: &[&str], + path: Option, +) -> (serde_json::Value, Vec) { + let server = FakeServer::start(turns, FakeOpts::default()).unwrap(); + let mut command = Command::new(env!("CARGO_BIN_EXE_oxa")); + command + .args(args) + .current_dir(env.work().join("project")) + .env_clear() + .envs(env.envs(&server.url())); + if let Some(path) = path { + command.env("PATH", path); + } + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + tx.send(command.output().unwrap()).unwrap(); + }); + let out = rx + .recv_timeout(std::time::Duration::from_secs(90)) + .expect("runner finished"); + let record = serde_json::from_slice(&out.stdout).unwrap_or_else(|e| { + panic!( + "{e}: {} / {}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ) + }); + (record, server.requests()) +} diff --git a/oxa/oxa-cli/tests/work_cli.rs b/oxa/oxa-cli/tests/work_cli.rs index ec6279eef..52dc5f2ee 100644 --- a/oxa/oxa-cli/tests/work_cli.rs +++ b/oxa/oxa-cli/tests/work_cli.rs @@ -1,66 +1,9 @@ //! Real child oxa with fake engine and local bare git remote; no network. -use oxa_fake::{FakeOpts, FakeServer, ScriptTurn, TestEnv}; +#[path = "common/work.rs"] +mod work; +use oxa_fake::{ScriptTurn, TestEnv}; use std::{path::Path, process::Command}; -fn git(env: &TestEnv, dir: &Path, args: &[&str]) { - let out = Command::new("git") - .args(args) - .current_dir(dir) - .env_clear() - .envs(env.envs("http://127.0.0.1:1")) - .env("GIT_AUTHOR_NAME", "Test") - .env("GIT_AUTHOR_EMAIL", "test@example.invalid") - .env("GIT_COMMITTER_NAME", "Test") - .env("GIT_COMMITTER_EMAIL", "test@example.invalid") - .output() - .unwrap(); - assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); -} -fn setup(env: &TestEnv, done: &str) { - git(env, &env.work(), &["init", "--bare", "remote.git"]); - git(env, &env.work(), &["init", "-b", "main", "project"]); - let project = env.work().join("project"); - env.write( - "work/project/.oxa/settings.json", - &format!(r#"{{"done":{done}}}"#), - ); - git(env, &project, &["add", "."]); - git(env, &project, &["commit", "-m", "fixture"]); - git(env, &project, &["remote", "add", "origin", "../remote.git"]); - git(env, &project, &["push", "origin", "main"]); - env.write( - "work/tasks.json", - r#"[{"id":"test","branch":"agent/test","model":"fake","prompt":"Finish task"}]"#, - ); -} -fn run(env: &TestEnv, turns: Vec) -> serde_json::Value { - run_args(env, turns, &["work", "../tasks.json"], None) -} -fn run_args(env: &TestEnv, turns: Vec, args: &[&str], path: Option) -> serde_json::Value { - let server = FakeServer::start(turns, FakeOpts::default()).unwrap(); - let mut command = Command::new(env!("CARGO_BIN_EXE_oxa")); - command - .args(args) - .current_dir(env.work().join("project")) - .env_clear() - .envs(env.envs(&server.url())); - if let Some(path) = path { - command.env("PATH", path); - } - let (tx, rx) = std::sync::mpsc::channel(); - std::thread::spawn(move || { - tx.send(command.output().unwrap()).unwrap(); - }); - let out = rx - .recv_timeout(std::time::Duration::from_secs(90)) - .expect("runner finished"); - serde_json::from_slice(&out.stdout).unwrap_or_else(|e| { - panic!( - "{e}: {} / {}", - String::from_utf8_lossy(&out.stdout), - String::from_utf8_lossy(&out.stderr) - ) - }) -} +use work::{git, run, run_args, setup}; #[test] fn external_checks_run_without_edits_and_retain_worktree() { let env = TestEnv::new(); @@ -180,13 +123,17 @@ printf '%s' '{"title":"Synthetic task","body":"Fix the test fixture"}' assert_eq!(record["outcome"], "nothing_to_do", "{record}"); } #[test] -fn setup_failure_records_blocked_and_keeps_existing_branch() { +fn branch_checked_out_elsewhere_is_blocked() { let env = TestEnv::new(); setup(&env, "[]"); - git(&env, &env.work().join("project"), &["branch", "agent/test"]); + git( + &env, + &env.work().join("project"), + &["worktree", "add", "-b", "agent/test", "../other"], + ); let record = run(&env, vec![]); assert_eq!(record["outcome"], "blocked", "{record}"); - assert!(record["detail"].as_str().unwrap().contains("already exists")); + assert!(record["detail"].as_str().unwrap().contains("other")); } #[test] diff --git a/oxa/oxa-cli/tests/work_continue.rs b/oxa/oxa-cli/tests/work_continue.rs new file mode 100644 index 000000000..5c0c244a2 --- /dev/null +++ b/oxa/oxa-cli/tests/work_continue.rs @@ -0,0 +1,215 @@ +//! Continued branches merge the base before the real child agent starts. +#[path = "common/work.rs"] +mod work; +use oxa_fake::{ScriptTurn, TestEnv}; +use std::path::{Path, PathBuf}; +use work::{git, run, run_observed, setup}; + +fn project(env: &TestEnv) -> PathBuf { + env.work().join("project") +} +fn commit_file(env: &TestEnv, file: &str, content: &str) { + env.write(&format!("work/project/{file}"), content); + git(env, &project(env), &["add", file]); + git(env, &project(env), &["commit", "-m", "fixture update"]); +} +fn diverge(env: &TestEnv, conflict: bool) { + setup(env, "[]"); + commit_file(env, "shared.txt", "original\n"); + git(env, &project(env), &["checkout", "-b", "agent/test"]); + commit_file(env, "shared.txt", "task version\n"); + git(env, &project(env), &["checkout", "main"]); + commit_file( + env, + if conflict { "shared.txt" } else { "base.txt" }, + "base version\n", + ); +} +fn tree(record: &serde_json::Value) -> &Path { + Path::new(record["worktree"].as_str().unwrap()) +} +fn merge_head(env: &TestEnv, root: &Path) -> PathBuf { + PathBuf::from(git( + env, + root, + &["rev-parse", "--path-format=absolute", "--git-path", "MERGE_HEAD"], + )) +} +fn resolve_turns() -> Vec { + vec![ + ScriptTurn::tool( + "resolve", + "write", + r#"{"path":"shared.txt","content":"resolved\n"}"#, + ), + ScriptTurn::tool( + "commit", + "bash", + r#"{"command":"git add shared.txt && git commit --no-edit"}"#, + ), + ScriptTurn::text("merge finished"), + ] +} +fn trust(env: &TestEnv) { + env.write( + "home/.oxa/agent/settings.json", + r#"{"defaultProjectTrust":"always"}"#, + ); +} +#[test] +fn existing_branch_fast_forwards_to_base_before_agent() { + let env = TestEnv::new(); + setup(&env, "[]"); + git(&env, &project(&env), &["branch", "agent/test"]); + commit_file(&env, "base.txt", "from main\n"); + let record = run(&env, vec![ScriptTurn::text("nothing")]); + assert_eq!(record["outcome"], "nothing_to_do", "{record}"); + assert_eq!( + std::fs::read_to_string(tree(&record).join("base.txt")).unwrap(), + "from main\n" + ); + git( + &env, + &project(&env), + &["merge-base", "--is-ancestor", "main", "agent/test"], + ); +} +#[test] +fn divergent_branch_gets_normal_merge_without_losing_task_commit() { + let env = TestEnv::new(); + diverge(&env, false); + let record = run(&env, vec![ScriptTurn::text("nothing")]); + assert_eq!(record["outcome"], "nothing_to_do", "{record}"); + assert_eq!( + git(&env, tree(&record), &["rev-list", "--parents", "-n", "1", "HEAD"]) + .split_whitespace() + .count(), + 3 + ); + assert_eq!( + std::fs::read_to_string(tree(&record).join("shared.txt")).unwrap(), + "task version\n" + ); + assert!(tree(&record).join("base.txt").exists()); +} +#[test] +fn conflicts_reach_agent_with_merge_in_progress_and_stay_blocked_if_unresolved() { + let env = TestEnv::new(); + diverge(&env, true); + let (record, requests) = run_observed( + &env, + vec![ScriptTurn::text("cannot resolve")], + &["work", "../tasks.json"], + None, + ); + assert_eq!(record["outcome"], "blocked", "{record}"); + assert!(record["detail"].as_str().unwrap().contains("merge")); + assert!(merge_head(&env, tree(&record)).exists()); + assert!(std::fs::read_to_string(tree(&record).join("shared.txt")) + .unwrap() + .contains("<<<<<<<")); + let user = requests + .iter() + .flat_map(|r| &r.messages) + .filter_map(|m| m.content.as_deref()) + .collect::>() + .join("\n"); + assert!( + user.contains("Finish task") && user.contains("merge in progress") && user.contains("shared.txt"), + "{user}" + ); +} +#[test] +fn agent_can_resolve_and_commit_merge_inside_real_sandbox() { + let env = TestEnv::new(); + diverge(&env, true); + trust(&env); + let record = run(&env, resolve_turns()); + assert_eq!(record["outcome"], "done", "{record}"); + assert!(!merge_head(&env, tree(&record)).exists()); + assert_eq!( + std::fs::read_to_string(tree(&record).join("shared.txt")).unwrap(), + "resolved\n" + ); + assert_eq!( + std::fs::read_to_string(project(&env).join("shared.txt")).unwrap(), + "base version\n" + ); +} +#[test] +fn unresolved_merge_can_be_continued_without_resetting_it() { + let env = TestEnv::new(); + diverge(&env, true); + trust(&env); + let first = run(&env, vec![ScriptTurn::text("blocked")]); + assert_eq!(first["outcome"], "blocked", "{first}"); + let second = run(&env, resolve_turns()); + assert_eq!(second["outcome"], "done", "{second}"); + assert!(!merge_head(&env, tree(&second)).exists()); +} +#[test] +fn dirty_retained_worktree_is_blocked_without_dropping_edits() { + let env = TestEnv::new(); + setup(&env, "[]"); + trust(&env); + let first = run( + &env, + vec![ + ScriptTurn::tool("write", "write", r#"{"path":"notes.txt","content":"keep me"}"#), + ScriptTurn::text("done"), + ], + ); + assert_eq!(first["outcome"], "done", "{first}"); + let (record, requests) = run_observed(&env, vec![], &["work", "../tasks.json"], None); + assert_eq!(record["outcome"], "blocked", "{record}"); + assert!(requests.is_empty()); + assert_eq!( + std::fs::read_to_string(tree(&record).join("notes.txt")).unwrap(), + "keep me" + ); +} +#[test] +fn continued_branch_cannot_replace_base_acceptance_commands() { + let env = TestEnv::new(); + setup(&env, r#"["echo base-acceptance; exit 1"]"#); + git(&env, &project(&env), &["checkout", "-b", "agent/test"]); + commit_file(&env, ".oxa/settings.json", r#"{"done":[]}"#); + git(&env, &project(&env), &["checkout", "main"]); + let record = run( + &env, + vec![ScriptTurn::text("done"), ScriptTurn::text("still done")], + ); + assert_eq!(record["outcome"], "red", "{record}"); + assert!(record["detail"].as_str().unwrap().contains("base-acceptance")); +} + +#[test] +fn aborting_merge_cannot_report_green() { + let env = TestEnv::new(); + diverge(&env, true); + trust(&env); + let record = run( + &env, + vec![ + ScriptTurn::tool("abort", "bash", r#"{"command":"git merge --abort"}"#), + ScriptTurn::text("done"), + ], + ); + assert_eq!(record["outcome"], "blocked", "{record}"); + assert!(record["detail"].as_str().unwrap().contains("base is not merged")); +} +#[test] +fn changing_branch_cannot_report_green_for_task_branch() { + let env = TestEnv::new(); + setup(&env, "[]"); + trust(&env); + let record = run( + &env, + vec![ + ScriptTurn::tool("switch", "bash", r#"{"command":"git checkout -b alternate"}"#), + ScriptTurn::text("done"), + ], + ); + assert_eq!(record["outcome"], "blocked", "{record}"); + assert!(record["detail"].as_str().unwrap().contains("different branch")); +} diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index 4ea8a1551..02e9d52e9 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -1,4 +1,5 @@ //! Opt-in, sequential unattended task execution. Worktrees are retained on every outcome. +mod tree; use serde::{Deserialize, Serialize}; use std::fs; use std::path::{Path, PathBuf}; @@ -20,7 +21,7 @@ fn default_base() -> String { "main".into() } -#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum Outcome { Done, @@ -29,7 +30,7 @@ pub enum Outcome { Blocked, } -#[derive(Debug, Serialize)] +#[derive(Debug, Deserialize, Serialize)] pub struct Record { pub id: String, pub branch: String, @@ -111,20 +112,6 @@ pub fn issue_task(repo: &Path, number: u64, model: String) -> Result { }) } -/// Acceptance commands are frozen from the base before the agent can edit settings. -fn done_commands(root: &Path) -> Result> { - let path = root.join(".oxa/settings.json"); - let bytes = match fs::read(path) { - Ok(bytes) => bytes, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(vec![]), - Err(e) => return Err(e.to_string()), - }; - let value: serde_json::Value = serde_json::from_slice(&bytes).map_err(|e| e.to_string())?; - match value.get("done") { - None => Ok(vec![]), - Some(value) => serde_json::from_value(value.clone()).map_err(|e| format!("done commands: {e}")), - } -} fn acceptance(root: &Path, commands: &[String]) -> Result { let mut failures = String::new(); for entry in commands { @@ -184,9 +171,6 @@ pub fn run_task(repo: &Path, binary: &Path, task: &Task) -> Result { .map_err(|e| format!("cannot lock task {}: {e}", task.id))?; let _lock = TaskLock(lock_path); let receipt = storage.join(format!("{}.json", task.id)); - if receipt.exists() { - return Err("task already has a receipt; retained for inspection".into()); - } // The common git directory must not be an ancestor of the child cwd: the sandbox // deliberately refuses writable git binds with that layout. let repo_dir = if common.file_name().is_some_and(|name| name == ".git") { @@ -201,6 +185,7 @@ pub fn run_task(repo: &Path, binary: &Path, task: &Task) -> Result { let trees = parent.join(".oxa-worktrees").join(name); fs::create_dir_all(&trees).map_err(|e| e.to_string())?; let tree = trees.join(&task.id); + tree::check_receipt(&receipt, &tree, task)?; let mut record = Record { id: task.id.clone(), branch: task.branch.clone(), @@ -209,18 +194,11 @@ pub fn run_task(repo: &Path, binary: &Path, task: &Task) -> Result { detail: String::new(), }; let result = (|| -> Result { - if tree.exists() { - return Err("task worktree already exists; retained for inspection".into()); - } - let tree_arg = tree.to_str().ok_or("non-UTF8 worktree path")?; - checked(git( - &root, - &["worktree", "add", "-b", &task.branch, tree_arg, &task.base], - ))?; + let prepared = tree::prepare(&root, &tree, task)?; let tree = tree.canonicalize().map_err(|e| e.to_string())?; - let initial = checked(git(&tree, &["rev-parse", "HEAD"]))?; - let commands = done_commands(&tree)?; - let first_agent = agent(binary, &tree, task, &task.prompt); + let commands = tree::done_from_base(&tree, &prepared.base)?; + let first_agent = agent(binary, &tree, task, &prepared.prompt); + tree::ensure_merged(&tree, &prepared.base, &task.branch)?; let failures = acceptance(&tree, &commands)?; if !failures.is_empty() { let repair_agent = agent( @@ -229,9 +207,10 @@ pub fn run_task(repo: &Path, binary: &Path, task: &Task) -> Result { task, &format!( "{}\n\nExternal acceptance checks failed. Repair once:\n{failures}", - task.prompt + prepared.prompt ), ); + tree::ensure_merged(&tree, &prepared.base, &task.branch)?; let failures = acceptance(&tree, &commands)?; if !failures.is_empty() { record.detail = failures; @@ -243,7 +222,7 @@ pub fn run_task(repo: &Path, binary: &Path, task: &Task) -> Result { } let head = checked(git(&tree, &["rev-parse", "HEAD"]))?; let dirty = checked(git(&tree, &["status", "--porcelain"]))?; - Ok(if head == initial && dirty.is_empty() { + Ok(if head == prepared.initial && dirty.is_empty() { Outcome::NothingToDo } else { Outcome::Done diff --git a/oxa/oxa-work/src/tree.rs b/oxa/oxa-work/src/tree.rs new file mode 100644 index 000000000..48c8e5561 --- /dev/null +++ b/oxa/oxa-work/src/tree.rs @@ -0,0 +1,147 @@ +//! Worktree continuation uses ordinary merges; no reset, stash or force checkout. +use super::{checked, git, output, Result, Task}; +use std::fs; +use std::path::{Path, PathBuf}; + +pub(crate) struct Prepared { + pub initial: String, + pub base: String, + pub prompt: String, +} +fn git_path(tree: &Path, name: &str) -> Result { + checked(git( + tree, + &["rev-parse", "--path-format=absolute", "--git-path", name], + )) + .map(PathBuf::from) +} +fn merging(tree: &Path) -> Result { + Ok(git_path(tree, "MERGE_HEAD")?.exists()) +} +fn merge_prompt(tree: &Path, task: &Task, output: &str) -> Result { + let paths = checked(git(tree, &["diff", "--name-only", "--diff-filter=U"]))?; + Ok(format!("{}\n\nThere is a merge in progress from base {}. Resolve the conflicts, stage the resolutions, and finish the merge commit before completing the task. Preserve the task's existing work.\nConflicted paths:\n{paths}\nMerge output:\n{output}", task.prompt, task.base)) +} +pub(crate) fn prepare(repo: &Path, tree: &Path, task: &Task) -> Result { + let base = checked(git( + repo, + &["rev-parse", "--verify", &format!("{}^{{commit}}", task.base)], + ))?; + if !tree.exists() { + let branch = format!("refs/heads/{}", task.branch); + let exists = output(git(repo, &["show-ref", "--verify", "--quiet", &branch]))?; + let tree_arg = tree.to_str().ok_or("non-UTF8 worktree path")?; + match exists.status.code() { + Some(0) => { + checked(git(repo, &["worktree", "add", tree_arg, &task.branch]))?; + } + Some(1) => { + checked(git( + repo, + &["worktree", "add", "-b", &task.branch, tree_arg, &base], + ))?; + } + _ => return Err(String::from_utf8_lossy(&exists.stderr).into()), + } + } + let canonical = tree.canonicalize().map_err(|e| e.to_string())?; + let actual = PathBuf::from(checked(git(tree, &["rev-parse", "--show-toplevel"]))?); + let common = |p: &Path| -> Result { + PathBuf::from(checked(git( + p, + &["rev-parse", "--path-format=absolute", "--git-common-dir"], + ))?) + .canonicalize() + .map_err(|e| e.to_string()) + }; + if actual.canonicalize().map_err(|e| e.to_string())? != canonical || common(tree)? != common(repo)? { + return Err("retained directory is not this repository's task worktree".into()); + } + let branch = checked(git(tree, &["symbolic-ref", "--quiet", "HEAD"]))?; + if branch != format!("refs/heads/{}", task.branch) { + return Err("retained worktree is on a different branch".into()); + } + for operation in ["rebase-merge", "rebase-apply", "CHERRY_PICK_HEAD", "REVERT_HEAD"] { + if git_path(tree, operation)?.exists() { + return Err(format!("retained worktree has {operation} in progress")); + } + } + let initial = checked(git(tree, &["rev-parse", "HEAD"]))?; + if merging(tree)? { + return Ok(Prepared { + initial, + base, + prompt: merge_prompt(tree, task, "Resuming the retained merge.")?, + }); + } + if !checked(git(tree, &["status", "--porcelain"]))?.is_empty() { + return Err("retained worktree has uncommitted changes; commit them before continuing".into()); + } + let merged = output(git(tree, &["merge", "--no-edit", &base]))?; + if !merged.status.success() { + if !merging(tree)? { + return Err(format!( + "base merge failed: {}\n{}", + String::from_utf8_lossy(&merged.stdout), + String::from_utf8_lossy(&merged.stderr) + )); + } + return Ok(Prepared { + initial, + base, + prompt: merge_prompt( + tree, + task, + &format!( + "{}\n{}", + String::from_utf8_lossy(&merged.stdout), + String::from_utf8_lossy(&merged.stderr) + ), + )?, + }); + } + // A clean base-only merge is preparation, not new task work. + Ok(Prepared { + initial: checked(git(tree, &["rev-parse", "HEAD"]))?, + base, + prompt: task.prompt.clone(), + }) +} +pub(crate) fn ensure_merged(tree: &Path, base: &str, branch: &str) -> Result<()> { + if checked(git(tree, &["symbolic-ref", "--quiet", "HEAD"]))? != format!("refs/heads/{branch}") { + return Err("agent left the worktree on a different branch".into()); + } + if merging(tree)? || !checked(git(tree, &["diff", "--name-only", "--diff-filter=U"]))?.is_empty() { + return Err("merge remains in progress; retained for resolution".into()); + } + if !output(git(tree, &["merge-base", "--is-ancestor", base, "HEAD"]))? + .status + .success() + { + return Err("base is not merged into the task branch; retained for continuation".into()); + } + Ok(()) +} +/// Read acceptance settings from the captured base commit, even on an older task branch. +pub(crate) fn done_from_base(repo: &Path, base: &str) -> Result> { + let path = ".oxa/settings.json"; + if checked(git(repo, &["ls-tree", "--name-only", base, "--", path]))?.is_empty() { + return Ok(vec![]); + } + let text = checked(git(repo, &["show", &format!("{base}:{path}")]))?; + let value: serde_json::Value = serde_json::from_str(&text).map_err(|e| e.to_string())?; + match value.get("done") { + None => Ok(vec![]), + Some(value) => serde_json::from_value(value.clone()).map_err(|e| format!("done commands: {e}")), + } +} +pub(crate) fn check_receipt(receipt: &Path, tree: &Path, task: &Task) -> Result<()> { + if receipt.exists() { + let record: super::Record = serde_json::from_slice(&fs::read(receipt).map_err(|e| e.to_string())?) + .map_err(|e| format!("retained receipt: {e}"))?; + if record.id != task.id || record.branch != task.branch || record.worktree != tree { + return Err("retained receipt belongs to a different task branch or worktree".into()); + } + } + Ok(()) +} From d05391082f1edb8a76d76664f90912743853b2e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:56:43 +0000 Subject: [PATCH 05/52] Include the current base when resuming an older merge --- oxa/oxa-cli/tests/work_continue.rs | 39 ++++++++++++++++++++++++++++++ oxa/oxa-work/src/tree.rs | 12 +++++---- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/oxa/oxa-cli/tests/work_continue.rs b/oxa/oxa-cli/tests/work_continue.rs index 5c0c244a2..bc0c252d4 100644 --- a/oxa/oxa-cli/tests/work_continue.rs +++ b/oxa/oxa-cli/tests/work_continue.rs @@ -213,3 +213,42 @@ fn changing_branch_cannot_report_green_for_task_branch() { assert_eq!(record["outcome"], "blocked", "{record}"); assert!(record["detail"].as_str().unwrap().contains("different branch")); } + +#[test] +fn resumed_old_merge_is_told_to_integrate_the_newly_captured_base() { + let env = TestEnv::new(); + diverge(&env, true); + trust(&env); + let first = run(&env, vec![ScriptTurn::text("blocked")]); + assert_eq!(first["outcome"], "blocked", "{first}"); + let old_base = git(&env, &project(&env), &["rev-parse", "main"]); + commit_file(&env, "new-base.txt", "new base commit\n"); + let new_base = git(&env, &project(&env), &["rev-parse", "main"]); + let mut turns = resolve_turns(); + turns.insert( + 2, + ScriptTurn::tool( + "latest", + "bash", + &serde_json::json!({"command": format!("git merge --no-edit {new_base}")}).to_string(), + ), + ); + let (record, requests) = run_observed(&env, turns, &["work", "../tasks.json"], None); + assert_eq!(record["outcome"], "done", "{record}"); + let messages = requests + .iter() + .flat_map(|r| &r.messages) + .filter_map(|m| m.content.as_deref()) + .collect::>() + .join("\n"); + assert!( + messages.contains(&old_base) && messages.contains(&format!("git merge --no-edit {new_base}")), + "{messages}" + ); + assert!(tree(&record).join("new-base.txt").exists()); + git( + &env, + &project(&env), + &["merge-base", "--is-ancestor", &new_base, "agent/test"], + ); +} diff --git a/oxa/oxa-work/src/tree.rs b/oxa/oxa-work/src/tree.rs index 48c8e5561..122fe61a7 100644 --- a/oxa/oxa-work/src/tree.rs +++ b/oxa/oxa-work/src/tree.rs @@ -18,9 +18,10 @@ fn git_path(tree: &Path, name: &str) -> Result { fn merging(tree: &Path) -> Result { Ok(git_path(tree, "MERGE_HEAD")?.exists()) } -fn merge_prompt(tree: &Path, task: &Task, output: &str) -> Result { +fn merge_prompt(tree: &Path, task: &Task, base: &str, output: &str) -> Result { let paths = checked(git(tree, &["diff", "--name-only", "--diff-filter=U"]))?; - Ok(format!("{}\n\nThere is a merge in progress from base {}. Resolve the conflicts, stage the resolutions, and finish the merge commit before completing the task. Preserve the task's existing work.\nConflicted paths:\n{paths}\nMerge output:\n{output}", task.prompt, task.base)) + let pending = checked(git(tree, &["rev-parse", "MERGE_HEAD"]))?; + Ok(format!("{}\n\nThere is a merge in progress from base {} at commit {pending}. The current base for this run is {base}. Resolve the conflicts, stage the resolutions, and finish the merge commit before completing the task. Then run `git merge --no-edit {base}` to integrate the captured current base as well, resolving and committing any further conflicts. Preserve the task's existing work.\nConflicted paths:\n{paths}\nMerge output:\n{output}", task.prompt, task.base)) } pub(crate) fn prepare(repo: &Path, tree: &Path, task: &Task) -> Result { let base = checked(git( @@ -70,8 +71,8 @@ pub(crate) fn prepare(repo: &Path, tree: &Path, task: &Task) -> Result if merging(tree)? { return Ok(Prepared { initial, - base, - prompt: merge_prompt(tree, task, "Resuming the retained merge.")?, + base: base.clone(), + prompt: merge_prompt(tree, task, &base, "Resuming the retained merge.")?, }); } if !checked(git(tree, &["status", "--porcelain"]))?.is_empty() { @@ -88,10 +89,11 @@ pub(crate) fn prepare(repo: &Path, tree: &Path, task: &Task) -> Result } return Ok(Prepared { initial, - base, + base: base.clone(), prompt: merge_prompt( tree, task, + &base, &format!( "{}\n{}", String::from_utf8_lossy(&merged.stdout), From 1e3917272e6b67660b34ae7f9795a55cad9b3c11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:39:53 +0000 Subject: [PATCH 06/52] Preserve offline policy and continue work batches after task failures --- oxa/docs/work.md | 9 +++- oxa/oxa-cli/src/args.rs | 21 +++++---- oxa/oxa-cli/src/work_cmd.rs | 3 +- oxa/oxa-cli/tests/work_cli.rs | 82 +++++++++++++++++++++++++++++++++++ oxa/oxa-work/src/lib.rs | 37 +++++++++++----- 5 files changed, 130 insertions(+), 22 deletions(-) diff --git a/oxa/docs/work.md b/oxa/docs/work.md index ed74c8ab8..a04b88a26 100644 --- a/oxa/docs/work.md +++ b/oxa/docs/work.md @@ -13,7 +13,8 @@ IDs and branches must be unique; branches must be new. For a GitHub issue use authenticated `gh` CLI to fetch the issue title and body. Issue lookup is refused when the environment or user settings enable offline mode. -Each task runs a child `oxa -p` with its model. Normal agent permission, trust and +Each task runs a child `oxa -p` with its model. The installed offline policy, +including a parent `--offline` flag, is passed to every child. Normal agent permission, trust and sandbox rules apply. Configure project trust before unattended use. The runner executes the base version of `.oxa/settings.json`'s `done` commands outside the agent after it exits, even without edits. These are trusted project shell commands: @@ -25,7 +26,11 @@ Failed acceptance output goes to one additional child run with the original prom then acceptance runs once more. JSON lines report `done`, `nothing_to_do` (no commits or file changes), `red` (checks still fail), or `blocked` (setup or agent execution failed). Exit codes are 0 for success, 1 if any task is red or blocked, -and 2 for invalid input or runner errors. +and 2 for invalid input or output errors. A task setup failure produces a blocked +JSON line and does not skip later tasks. If a task cannot be locked or already has +a receipt, its existing state is preserved; a pre-worktree failure has an empty +`worktree` in the result. Locks use kernel ownership, so an abandoned lock file +after process death does not permanently block a task. Worktrees and uncommitted changes remain in `.oxa-worktrees//` beside the repository (or the separate git directory). Result receipts live inside diff --git a/oxa/oxa-cli/src/args.rs b/oxa/oxa-cli/src/args.rs index ecf283560..b4a6942ed 100644 --- a/oxa/oxa-cli/src/args.rs +++ b/oxa/oxa-cli/src/args.rs @@ -458,19 +458,24 @@ fn positional(a: &mut Args, word: &str) { /// `unsupported_in_oxa`. pub fn parse_args(argv: &[String], ext_flags: &[FlagSpec]) -> Result { let mut a = Args::default(); - if let Some(first) = argv.first() { - if !first.starts_with('-') && !first.starts_with('@') { - if let Some(sub) = subcommand(first, &argv[1..])? { - a.subcommand = Some(sub); - return Ok(a); - } - } - } let mut i = 0; let mut stop = false; while i < argv.len() { let arg = argv[i].as_str(); i += 1; + if !stop + && !a.print + && a.mode.is_none() + && !arg.starts_with('-') + && !arg.starts_with('@') + && a.messages.is_empty() + && a.files.is_empty() + { + if let Some(sub) = subcommand(arg, &argv[i..])? { + a.subcommand = Some(sub); + break; + } + } if stop || !arg.starts_with('-') { positional(&mut a, arg); continue; diff --git a/oxa/oxa-cli/src/work_cmd.rs b/oxa/oxa-cli/src/work_cmd.rs index af2d67257..4afd4b9d1 100644 --- a/oxa/oxa-cli/src/work_cmd.rs +++ b/oxa/oxa-cli/src/work_cmd.rs @@ -20,7 +20,8 @@ pub(crate) fn run(args: &[String], cwd: &Path, out: &mut dyn Write, err: &mut dy let binary: PathBuf = std::env::current_exe().map_err(|e| e.to_string())?; let mut code = 0; for task in tasks { - let record = oxa_work::run_task(cwd, &binary, &task)?; + let record = oxa_work::run_task_with_policy(cwd, &binary, &task, oxa_proto::offline()) + .unwrap_or_else(|detail| oxa_work::blocked_record(&task, detail)); if matches!( record.outcome, oxa_work::Outcome::Red | oxa_work::Outcome::Blocked diff --git a/oxa/oxa-cli/tests/work_cli.rs b/oxa/oxa-cli/tests/work_cli.rs index ec6279eef..3cc488928 100644 --- a/oxa/oxa-cli/tests/work_cli.rs +++ b/oxa/oxa-cli/tests/work_cli.rs @@ -268,3 +268,85 @@ fn offline_issue_input_never_spawns_gh() { assert!(String::from_utf8_lossy(&out.stderr).contains("offline")); } } + +#[test] +fn command_line_offline_policy_reaches_child_agent() { + let env = TestEnv::new(); + setup(&env, "[]"); + env.write( + "work/tasks.json", + r#"[{"id":"test","branch":"agent/test","model":"cloud-model","prompt":"go"}]"#, + ); + let server = FakeServer::start( + vec![ScriptTurn::text("done")], + FakeOpts { + model: "cloud-model".into(), + remote: Some("openrouter"), + models: vec![oxa_fake::FakeModel { + alias: "cloud-model".into(), + cap: 64000, + max_output: Some(4096), + caps: None, + }], + ..FakeOpts::default() + }, + ) + .unwrap(); + let out = Command::new(env!("CARGO_BIN_EXE_oxa")) + .args(["--offline", "work", "../tasks.json"]) + .current_dir(env.work().join("project")) + .env_clear() + .envs(env.envs(&server.url())) + .output() + .unwrap(); + let record: serde_json::Value = serde_json::from_slice(&out.stdout).unwrap(); + assert_eq!(record["outcome"], "blocked", "{record}"); + assert!( + record["detail"].as_str().unwrap().contains("offline mode is on"), + "{record}" + ); + assert!( + server.requests().is_empty(), + "no chat reached even the mock remote" + ); +} +#[test] +fn batch_reports_one_task_error_and_runs_later_tasks() { + let env = TestEnv::new(); + setup(&env, "[]"); + env.write("work/project/.git/oxa-work/test.json", "{}"); + env.write("work/tasks.json", r#"[{"id":"test","branch":"agent/test","model":"fake","prompt":"go"},{"id":"next","branch":"agent/next","model":"fake","prompt":"go"}]"#); + let server = FakeServer::start(vec![ScriptTurn::text("done")], FakeOpts::default()).unwrap(); + let out = Command::new(env!("CARGO_BIN_EXE_oxa")) + .args(["work", "../tasks.json"]) + .current_dir(env.work().join("project")) + .env_clear() + .envs(env.envs(&server.url())) + .output() + .unwrap(); + assert_eq!( + out.status.code(), + Some(1), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + let rows: Vec = String::from_utf8_lossy(&out.stdout) + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + assert_eq!(rows.len(), 2); + assert_eq!(rows[0]["outcome"], "blocked"); + assert_eq!(rows[1]["outcome"], "nothing_to_do"); + assert_eq!( + std::fs::read_to_string(env.work().join("project/.git/oxa-work/test.json")).unwrap(), + "{}" + ); +} +#[test] +fn abandoned_lock_file_does_not_block_a_task() { + let env = TestEnv::new(); + setup(&env, "[]"); + env.write("work/project/.git/oxa-work/test.lock", ""); + let record = run(&env, vec![ScriptTurn::text("done")]); + assert_eq!(record["outcome"], "nothing_to_do", "{record}"); +} diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index 4ea8a1551..2a862c1e6 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -150,21 +150,30 @@ fn acceptance(root: &Path, commands: &[String]) -> Result { } Ok(failures) } -fn agent(binary: &Path, root: &Path, task: &Task, prompt: &str) -> Result<()> { +fn agent(binary: &Path, root: &Path, task: &Task, prompt: &str, offline: bool) -> Result<()> { let mut c = Command::new(binary); c.current_dir(root) - .args(["-p", "--model", &task.model, "--no-session", "--", prompt]); + .args(["-p", "--model", &task.model, "--no-session"]); + if offline { + c.arg("--offline"); + } + c.args(["--", prompt]); checked(c).map(|_| ()) } -struct TaskLock(PathBuf); -impl Drop for TaskLock { - fn drop(&mut self) { - let _ = fs::remove_file(&self.0); +pub fn blocked_record(task: &Task, detail: String) -> Record { + Record { + id: task.id.clone(), + branch: task.branch.clone(), + worktree: PathBuf::new(), + outcome: Outcome::Blocked, + detail, } } - pub fn run_task(repo: &Path, binary: &Path, task: &Task) -> Result { + run_task_with_policy(repo, binary, task, false) +} +pub fn run_task_with_policy(repo: &Path, binary: &Path, task: &Task, offline: bool) -> Result { validate(task)?; let root = checked(git(repo, &["rev-parse", "--show-toplevel"]))?; let root = PathBuf::from(root); @@ -177,12 +186,17 @@ pub fn run_task(repo: &Path, binary: &Path, task: &Task) -> Result { let storage = common.join("oxa-work"); fs::create_dir_all(&storage).map_err(|e| e.to_string())?; let lock_path = storage.join(format!("{}.lock", task.id)); - let _lock_file = fs::OpenOptions::new() + let lock_file = fs::OpenOptions::new() .write(true) - .create_new(true) + .create(true) + .truncate(false) .open(&lock_path) .map_err(|e| format!("cannot lock task {}: {e}", task.id))?; - let _lock = TaskLock(lock_path); + lock_file + .try_lock() + .map_err(|e| format!("task {} is already running: {e}", task.id))?; + // Keep the inode so a crashed runner leaves no stale ownership and concurrent + // runners cannot acquire locks on two different files with the same name. let receipt = storage.join(format!("{}.json", task.id)); if receipt.exists() { return Err("task already has a receipt; retained for inspection".into()); @@ -220,7 +234,7 @@ pub fn run_task(repo: &Path, binary: &Path, task: &Task) -> Result { let tree = tree.canonicalize().map_err(|e| e.to_string())?; let initial = checked(git(&tree, &["rev-parse", "HEAD"]))?; let commands = done_commands(&tree)?; - let first_agent = agent(binary, &tree, task, &task.prompt); + let first_agent = agent(binary, &tree, task, &task.prompt, offline); let failures = acceptance(&tree, &commands)?; if !failures.is_empty() { let repair_agent = agent( @@ -231,6 +245,7 @@ pub fn run_task(repo: &Path, binary: &Path, task: &Task) -> Result { "{}\n\nExternal acceptance checks failed. Repair once:\n{failures}", task.prompt ), + offline, ); let failures = acceptance(&tree, &commands)?; if !failures.is_empty() { From 77100aa4ab4b65d096885590ec99e5250cb819a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:47:42 +0000 Subject: [PATCH 07/52] Checkpoint work tasks and save unfinished work as commits (#333) --- oxa/docs/work.md | 38 ++++-- oxa/oxa-cli/src/args.rs | 2 +- oxa/oxa-cli/src/work_cmd.rs | 27 +++- oxa/oxa-cli/tests/work_continue.rs | 5 +- oxa/oxa-cli/tests/work_state.rs | 210 +++++++++++++++++++++++++++++ oxa/oxa-work/src/execute.rs | 124 +++++++++++++++++ oxa/oxa-work/src/lib.rs | 186 +++++++++++++++---------- oxa/oxa-work/src/state.rs | 94 +++++++++++++ oxa/oxa-work/src/tree.rs | 51 +++---- 9 files changed, 622 insertions(+), 115 deletions(-) create mode 100644 oxa/oxa-cli/tests/work_state.rs create mode 100644 oxa/oxa-work/src/execute.rs create mode 100644 oxa/oxa-work/src/state.rs diff --git a/oxa/docs/work.md b/oxa/docs/work.md index 7983dd97d..896fc3792 100644 --- a/oxa/docs/work.md +++ b/oxa/docs/work.md @@ -1,6 +1,7 @@ # Unattended work `oxa work tasks.json` opts into sequential tasks in separate git worktrees. +Completed tasks are skipped on restart; add `--rerun` to deliberately run them again. Run it inside the project repository. The JSON file is an array: ```json @@ -16,8 +17,8 @@ authenticated `gh` CLI to fetch the issue title and body. Issue lookup is refuse when the environment or user settings enable offline mode. Each task runs a child `oxa -p` with its model. The installed offline policy, -including a parent `--offline` flag, is passed to every child. Normal agent permission, trust and -sandbox rules apply. Configure project trust before unattended use. The runner +including a parent `--offline` flag, is passed to every child. Normal agent +permission, trust and sandbox rules apply. Configure project trust before unattended use. The runner executes the base version of `.oxa/settings.json`'s `done` commands outside the agent after it exits, even without edits. These are trusted project shell commands: invoking `work` authorizes their execution as your user. `in subdirectory: command` @@ -34,20 +35,41 @@ and 2 for invalid input or output errors. A task setup failure produces a blocke JSON line and does not skip later tasks. If a task cannot be locked or already has a receipt, its existing state is preserved; a pre-worktree failure has an empty `worktree` in the result. Locks use kernel ownership, so an abandoned lock file -after process death does not permanently block a task. +after process death does not permanently block a task. A surviving child keeps the +lock until it exits, preventing overlap with a restarted runner. Worktrees and uncommitted changes remain in `.oxa-worktrees//` beside the repository (or the separate git directory). Result receipts live inside the common git directory at `oxa-work/.json`. Worktrees stay outside that git -directory so the normal sandbox can bind git objects and refs for commits. No automatic -cleanup, commit, push or PR creation is implemented yet. +directory so the normal sandbox can bind git objects and refs for commits. +Uncommitted task work, including red work, is saved as a commit with the repository's +configured identity. Commit hooks run normally. A missing identity or rejected commit +blocks the task and retains every file. Unresolved conflicts remain in progress for +the agent; they are never staged as a fabricated resolution. There is no automatic +cleanup, push or PR creation yet. -To continue a task, run the same file again. The retained receipt must match its -ID, branch and worktree. A clean worktree is reused; a branch checked out elsewhere -is blocked. Uncommitted changes are retained and blocked until you commit them. +To continue an interrupted, red or blocked task, run the same file again. Use +`--rerun` to continue a completed task after its base changes. The retained receipt +must match its ID, branch and worktree. A clean worktree is reused; a branch checked out elsewhere +is blocked. Existing dirty work is saved when possible before a later retry. Merge conflicts are passed to the agent with the merge still in progress, the conflicted file list and merge output. An unfinished merge can be continued on a later run. The agent must finish the merge commit and keep the task branch selected; unresolved or aborted base merges remain blocked even when checks would pass. Inspect retained work before removing it with `git worktree remove`. Review task prompts and bases before running them. + +`oxa work --status` prints one line per recorded task, sorted by ID: + +```text +docs-fix done 1800000000 finished +``` + +Columns are task ID, state, last step-progress time (Unix seconds), and current step. +Running tasks report steps such as `agent`, `acceptance`, `repair` or `saving`. +The state file is replaced atomically with owner-only permissions at each transition. +It records the task specification, captured base/checks, repair output and current +step. Reusing an ID with a different specification is refused; use a new ID. +Restart continues the saved step. An agent or check interrupted within its step +may run again; tasks and checks should tolerate that. Checkpoints after an agent +step avoid replaying it when only acceptance or saving remains. diff --git a/oxa/oxa-cli/src/args.rs b/oxa/oxa-cli/src/args.rs index b4a6942ed..ec2fe79aa 100644 --- a/oxa/oxa-cli/src/args.rs +++ b/oxa/oxa-cli/src/args.rs @@ -177,7 +177,7 @@ const COMMANDS: &[(&str, &str)] = &[ "Draft a scrubbed ox-llm issue; submit posts it only after you type yes", ), ( - "oxa work | --issue N --model MODEL", + "oxa work [--rerun] | --issue N --model MODEL | --status", "Run tasks in isolated worktrees with external acceptance checks", ), ( diff --git a/oxa/oxa-cli/src/work_cmd.rs b/oxa/oxa-cli/src/work_cmd.rs index 4afd4b9d1..6978aa9f6 100644 --- a/oxa/oxa-cli/src/work_cmd.rs +++ b/oxa/oxa-cli/src/work_cmd.rs @@ -3,6 +3,24 @@ use std::path::{Path, PathBuf}; pub(crate) fn run(args: &[String], cwd: &Path, out: &mut dyn Write, err: &mut dyn Write) -> i32 { let result = (|| -> Result { + if args == ["--status"] { + for record in oxa_work::status(cwd)? { + let state = serde_json::to_value(record.outcome).map_err(|e| e.to_string())?; + let step = serde_json::to_value(record.step).map_err(|e| e.to_string())?; + writeln!( + out, + "{} {} {} {}", + record.id, + state.as_str().unwrap_or("unknown"), + record.progress_at, + step.as_str().unwrap_or("unknown") + ) + .map_err(|e| e.to_string())?; + } + return Ok(0); + } + let rerun = args.last().is_some_and(|s| s == "--rerun"); + let args = if rerun { &args[..args.len() - 1] } else { args }; let tasks = match args { [file] if !file.starts_with('-') => oxa_work::read_tasks(&cwd.join(file))?, [flag, number, model_flag, model] if flag == "--issue" && model_flag == "--model" => { @@ -15,12 +33,17 @@ pub(crate) fn run(args: &[String], cwd: &Path, out: &mut dyn Write, err: &mut dy model.clone(), )?] } - _ => return Err("usage: oxa work | --issue N --model MODEL".into()), + _ => { + return Err( + "usage: oxa work [--rerun] | --issue N --model MODEL [--rerun] | --status" + .into(), + ) + } }; let binary: PathBuf = std::env::current_exe().map_err(|e| e.to_string())?; let mut code = 0; for task in tasks { - let record = oxa_work::run_task_with_policy(cwd, &binary, &task, oxa_proto::offline()) + let record = oxa_work::run_task_with_options(cwd, &binary, &task, rerun, oxa_proto::offline()) .unwrap_or_else(|detail| oxa_work::blocked_record(&task, detail)); if matches!( record.outcome, diff --git a/oxa/oxa-cli/tests/work_continue.rs b/oxa/oxa-cli/tests/work_continue.rs index bc0c252d4..6e437bdf5 100644 --- a/oxa/oxa-cli/tests/work_continue.rs +++ b/oxa/oxa-cli/tests/work_continue.rs @@ -148,7 +148,7 @@ fn unresolved_merge_can_be_continued_without_resetting_it() { assert!(!merge_head(&env, tree(&second)).exists()); } #[test] -fn dirty_retained_worktree_is_blocked_without_dropping_edits() { +fn completed_work_is_saved_and_not_replayed() { let env = TestEnv::new(); setup(&env, "[]"); trust(&env); @@ -161,8 +161,9 @@ fn dirty_retained_worktree_is_blocked_without_dropping_edits() { ); assert_eq!(first["outcome"], "done", "{first}"); let (record, requests) = run_observed(&env, vec![], &["work", "../tasks.json"], None); - assert_eq!(record["outcome"], "blocked", "{record}"); + assert_eq!(record["outcome"], "done", "{record}"); assert!(requests.is_empty()); + assert!(git(&env, tree(&record), &["status", "--porcelain"]).is_empty()); assert_eq!( std::fs::read_to_string(tree(&record).join("notes.txt")).unwrap(), "keep me" diff --git a/oxa/oxa-cli/tests/work_state.rs b/oxa/oxa-cli/tests/work_state.rs new file mode 100644 index 000000000..c5464e2fd --- /dev/null +++ b/oxa/oxa-cli/tests/work_state.rs @@ -0,0 +1,210 @@ +//! Durable outcomes through the real runner, fake engine and local bare remote. +#[path = "common/work.rs"] +mod work; +use oxa_fake::{FakeOpts, FakeServer, ScriptTurn, TestEnv}; +use std::{ + fs, + path::{Path, PathBuf}, + process::{Command, Stdio}, + time::{Duration, Instant}, +}; +use work::{git, run, run_args, run_observed, setup}; +fn project(env: &TestEnv) -> PathBuf { + env.work().join("project") +} +fn receipt(env: &TestEnv) -> PathBuf { + project(env).join(".git/oxa-work/test.json") +} +fn tree(record: &serde_json::Value) -> &Path { + Path::new(record["worktree"].as_str().unwrap()) +} +fn trust(env: &TestEnv) { + env.write( + "home/.oxa/agent/settings.json", + r#"{"defaultProjectTrust":"always"}"#, + ); +} +fn write_turn() -> ScriptTurn { + ScriptTurn::tool("write", "write", r#"{"path":"answer.txt","content":"retained"}"#) +} +fn load(env: &TestEnv) -> serde_json::Value { + serde_json::from_slice(&fs::read(receipt(env)).unwrap()).unwrap() +} +fn replace_state(env: &TestEnv, value: &serde_json::Value) { + fs::write(receipt(env), serde_json::to_vec(value).unwrap()).unwrap(); +} +#[test] +fn uncommitted_success_is_saved_as_a_commit() { + let env = TestEnv::new(); + setup(&env, "[]"); + trust(&env); + let record = run(&env, vec![write_turn(), ScriptTurn::text("done")]); + assert_eq!(record["outcome"], "done", "{record}"); + assert_eq!(git(&env, tree(&record), &["show", "HEAD:answer.txt"]), "retained"); + assert!(git(&env, tree(&record), &["status", "--porcelain"]).is_empty()); + use std::os::unix::fs::PermissionsExt; + assert_eq!( + fs::metadata(receipt(&env)).unwrap().permissions().mode() & 0o077, + 0 + ); + assert_eq!(record["step"], "finished"); + assert!(record["progress_at"].as_u64().unwrap() > 0); +} +#[test] +fn red_work_is_saved_too() { + let env = TestEnv::new(); + setup(&env, r#"["printf failed > failure.txt; exit 1"]"#); + let record = run( + &env, + vec![ScriptTurn::text("done"), ScriptTurn::text("cannot repair")], + ); + assert_eq!(record["outcome"], "red", "{record}"); + assert_eq!(git(&env, tree(&record), &["show", "HEAD:failure.txt"]), "failed"); + assert!(git(&env, tree(&record), &["status", "--porcelain"]).is_empty()); +} +#[test] +fn completed_task_is_skipped_unless_rerun_is_explicit() { + let env = TestEnv::new(); + setup(&env, "[]"); + let first = run(&env, vec![ScriptTurn::text("done")]); + let (again, requests) = run_observed(&env, vec![], &["work", "../tasks.json"], None); + assert_eq!(first, again); + assert!(requests.is_empty()); + let repeated = run_args( + &env, + vec![ScriptTurn::text("repeated")], + &["work", "../tasks.json", "--rerun"], + None, + ); + assert_eq!(repeated["outcome"], "nothing_to_do", "{repeated}"); +} +#[test] +fn restart_at_saving_preserves_outcome_without_replaying_agent() { + let env = TestEnv::new(); + setup(&env, "[]"); + let first = run(&env, vec![ScriptTurn::text("done")]); + fs::write(tree(&first).join("recovered.txt"), "recovered").unwrap(); + let mut state = load(&env); + state["step"] = "saving".into(); + state["outcome"] = "red".into(); + state["detail"] = "previous acceptance failed".into(); + replace_state(&env, &state); + let (record, requests) = run_observed(&env, vec![], &["work", "../tasks.json"], None); + assert_eq!(record["outcome"], "red"); + assert!(requests.is_empty()); + assert_eq!( + git(&env, tree(&record), &["show", "HEAD:recovered.txt"]), + "recovered" + ); +} +#[test] +fn restart_at_acceptance_keeps_captured_checks_and_base() { + let env = TestEnv::new(); + setup(&env, r#"["printf ran >> attempts"]"#); + let first = run(&env, vec![ScriptTurn::text("done")]); + env.write("work/project/.oxa/settings.json", r#"{"done":["exit 7"]}"#); + git(&env, &project(&env), &["add", ".oxa/settings.json"]); + git(&env, &project(&env), &["commit", "-m", "advance base checks"]); + let mut state = load(&env); + state["step"] = "acceptance".into(); + state["outcome"] = "running".into(); + replace_state(&env, &state); + let (record, requests) = run_observed(&env, vec![], &["work", "../tasks.json"], None); + assert_eq!(record["outcome"], "done", "{record}"); + assert!(requests.is_empty()); + assert_eq!( + fs::read_to_string(tree(&first).join("attempts")).unwrap(), + "ranran" + ); +} +#[test] +fn status_reports_state_progress_time_and_current_step() { + let env = TestEnv::new(); + setup(&env, "[]"); + let record = run(&env, vec![ScriptTurn::text("done")]); + let out = Command::new(env!("CARGO_BIN_EXE_oxa")) + .args(["work", "--status"]) + .current_dir(project(&env)) + .env_clear() + .envs(env.envs("http://127.0.0.1:1")) + .output() + .unwrap(); + assert!(out.status.success()); + assert_eq!( + String::from_utf8_lossy(&out.stdout).trim(), + format!("test nothing_to_do {} finished", record["progress_at"]) + ); +} +#[test] +fn failed_auto_commit_retains_every_file_and_blocks() { + let env = TestEnv::new(); + setup(&env, "[]"); + trust(&env); + // A user hook refuses the automatic commit. The runner must not bypass it. + use std::os::unix::fs::PermissionsExt; + let hook = env.write("work/project/.git/hooks/pre-commit", "#!/bin/sh\nexit 1\n"); + fs::set_permissions(hook, fs::Permissions::from_mode(0o755)).unwrap(); + let record = run(&env, vec![write_turn(), ScriptTurn::text("done")]); + assert_eq!(record["outcome"], "blocked", "{record}"); + assert_eq!( + fs::read_to_string(tree(&record).join("answer.txt")).unwrap(), + "retained" + ); + assert!(!git(&env, tree(&record), &["status", "--porcelain"]).is_empty()); +} +fn until(mut predicate: impl FnMut() -> bool) { + let deadline = Instant::now() + Duration::from_secs(15); + while !predicate() { + assert!(Instant::now() < deadline, "timed out"); + std::thread::sleep(Duration::from_millis(10)); + } +} +#[test] +fn child_keeps_lock_after_runner_is_killed_then_task_restarts() { + let env = TestEnv::new(); + setup(&env, "[]"); + let server = FakeServer::start( + vec![ScriptTurn::text("first")], + FakeOpts { + hold_headers_ms: 1500, + ..FakeOpts::default() + }, + ) + .unwrap(); + let mut child = Command::new(env!("CARGO_BIN_EXE_oxa")) + .args(["work", "../tasks.json"]) + .current_dir(project(&env)) + .env_clear() + .envs(env.envs(&server.url())) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + until(|| !server.requests().is_empty()); + assert_eq!(load(&env)["step"], "agent"); + child.kill().unwrap(); + child.wait().unwrap(); + let blocked = run(&env, vec![]); + assert_eq!(blocked["outcome"], "blocked", "{blocked}"); + assert!( + blocked["detail"].as_str().unwrap().contains("already running"), + "{blocked}" + ); + assert_eq!( + load(&env)["step"], + "agent", + "busy retry must not clobber the checkpoint" + ); + let lock_path = project(&env).join(".git/oxa-work/test.lock"); + until(|| { + let file = fs::OpenOptions::new() + .read(true) + .write(true) + .open(&lock_path) + .unwrap(); + file.try_lock().is_ok() + }); + let restarted = run(&env, vec![ScriptTurn::text("resumed")]); + assert_eq!(restarted["outcome"], "nothing_to_do", "{restarted}"); + assert_eq!(restarted["step"], "finished"); +} diff --git a/oxa/oxa-work/src/execute.rs b/oxa/oxa-work/src/execute.rs new file mode 100644 index 000000000..115ec7317 --- /dev/null +++ b/oxa/oxa-work/src/execute.rs @@ -0,0 +1,124 @@ +use super::{acceptance, agent, checked, git, state, tree, Outcome, Result, Task}; +use state::{Checkpoint, State, Step}; +use std::fs::File; +use std::path::Path; + +fn complete(tree: &Path, state: &mut State) -> Result<()> { + let point = state.checkpoint.as_ref().ok_or("missing task checkpoint")?; + if let Some(error) = &point.agent_error { + return Err(error.clone()); + } + let head = checked(git(tree, &["rev-parse", "HEAD"]))?; + let dirty = checked(git(tree, &["status", "--porcelain"]))?; + state.record.outcome = if head == point.initial && dirty.is_empty() { + Outcome::NothingToDo + } else { + Outcome::Done + }; + Ok(()) +} +pub(crate) fn execute( + repo: &Path, + binary: &Path, + task: &Task, + path: &Path, + state: &mut State, + lock: &File, +) -> Result<()> { + let tree = state.record.worktree.clone(); + loop { + match state.record.step { + Step::Preparing => { + let prepared = tree::prepare(repo, &tree, task)?; + let commands = tree::done_from_base(&tree, &prepared.base)?; + state.checkpoint = Some(Checkpoint { + initial: prepared.initial, + base: prepared.base, + prompt: prepared.prompt, + commands, + agent_error: None, + failures: String::new(), + }); + state::save(path, state, Step::Agent)?; + } + Step::Agent | Step::Repair => { + tree::validate(repo, &tree, task)?; + let step = state.record.step; + let point = state.checkpoint.as_ref().ok_or("missing task checkpoint")?; + let prompt = if step == Step::Repair { + format!( + "{}\n\nExternal acceptance checks failed. Repair once:\n{}", + point.prompt, point.failures + ) + } else { + point.prompt.clone() + }; + let result = agent(binary, &tree, task, &prompt, lock, state.offline); + tree::ensure_merged(&tree, &point.base, &task.branch)?; + state + .checkpoint + .as_mut() + .ok_or("missing task checkpoint")? + .agent_error = result.err(); + state::save( + path, + state, + if step == Step::Agent { + Step::Acceptance + } else { + Step::Recheck + }, + )?; + } + Step::Acceptance | Step::Recheck => { + tree::validate(repo, &tree, task)?; + let point = state.checkpoint.as_ref().ok_or("missing task checkpoint")?; + tree::ensure_merged(&tree, &point.base, &task.branch)?; + let failures = acceptance( + &tree.canonicalize().map_err(|e| e.to_string())?, + &point.commands, + lock, + state.offline, + )?; + if failures.is_empty() { + complete(&tree, state)?; + state::save(path, state, Step::Saving)?; + } else if state.record.step == Step::Acceptance { + state + .checkpoint + .as_mut() + .ok_or("missing task checkpoint")? + .failures = failures; + state::save(path, state, Step::Repair)?; + } else { + state.record.outcome = Outcome::Red; + state.record.detail = failures; + state::save(path, state, Step::Saving)?; + } + } + Step::Saving | Step::Finished => return Ok(()), + } + } +} +/// Save all staged, unstaged and untracked files. Conflicts are kept in progress for the agent. +pub(crate) fn save_work(repo: &Path, tree: &Path, task: &Task) -> Result<()> { + if !tree.exists() { + return Ok(()); + } + tree::validate(repo, tree, task)?; + if tree::merging(tree)? || !checked(git(tree, &["diff", "--name-only", "--diff-filter=U"]))?.is_empty() { + return Err("unresolved merge retained in progress; files were not discarded".into()); + } + if checked(git(tree, &["status", "--porcelain"]))?.is_empty() { + return Ok(()); + } + checked(git(tree, &["add", "--all"]))?; + checked(git( + tree, + &["commit", "-m", &format!("oxa work: save task {}", task.id)], + ))?; + if !checked(git(tree, &["status", "--porcelain"]))?.is_empty() { + return Err("uncommitted changes remain after saving; retained for inspection".into()); + } + Ok(()) +} diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index abe10485b..0ddd10e97 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -1,13 +1,16 @@ //! Opt-in, sequential unattended task execution. Worktrees are retained on every outcome. +mod execute; +mod state; mod tree; use serde::{Deserialize, Serialize}; +pub use state::Step; use std::fs; use std::path::{Path, PathBuf}; use std::process::{Command, Output, Stdio}; pub type Result = std::result::Result; -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct Task { pub id: String, @@ -24,6 +27,7 @@ fn default_base() -> String { #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum Outcome { + Running, Done, NothingToDo, Red, @@ -37,13 +41,17 @@ pub struct Record { pub worktree: PathBuf, pub outcome: Outcome, pub detail: String, + pub step: Step, + pub progress_at: u64, } fn output(mut command: Command) -> Result { command.stdin(Stdio::null()).output().map_err(|e| e.to_string()) } fn checked(command: Command) -> Result { - let out = output(command)?; + checked_output(output(command)?) +} +fn checked_output(out: Output) -> Result { if !out.status.success() { return Err(String::from_utf8_lossy(&out.stderr).into_owned()); } @@ -112,7 +120,7 @@ pub fn issue_task(repo: &Path, number: u64, model: String) -> Result { }) } -fn acceptance(root: &Path, commands: &[String]) -> Result { +fn acceptance(root: &Path, commands: &[String], lock: &fs::File, offline: bool) -> Result { let mut failures = String::new(); for entry in commands { let (dir, shell) = match entry.strip_prefix("in ").and_then(|s| s.split_once(':')) { @@ -124,8 +132,13 @@ fn acceptance(root: &Path, commands: &[String]) -> Result { return Err("done directory escapes worktree".into()); } let mut c = Command::new("sh"); - c.current_dir(dir).args(["-c", shell]); - let out = output(c)?; + c.current_dir(dir) + .args(["-c", shell]) + .stdin(lock.try_clone().map_err(|e| e.to_string())?); + if offline { + c.env("OXA_OFFLINE", "1"); + } + let out = c.output().map_err(|e| e.to_string())?; if !out.status.success() { failures.push_str(&format!( "{entry}: {}\n{}\n{}\n", @@ -137,15 +150,25 @@ fn acceptance(root: &Path, commands: &[String]) -> Result { } Ok(failures) } -fn agent(binary: &Path, root: &Path, task: &Task, prompt: &str, offline: bool) -> Result<()> { +fn agent( + binary: &Path, + root: &Path, + task: &Task, + prompt: &str, + lock: &fs::File, + offline: bool, +) -> Result<()> { let mut c = Command::new(binary); c.current_dir(root) .args(["-p", "--model", &task.model, "--no-session"]); if offline { c.arg("--offline"); } - c.args(["--", prompt]); - checked(c).map(|_| ()) + // Stdin was null for unattended runs. An empty locked file still supplies EOF, + // and fd 0 keeps the shared kernel lock alive if the runner dies before its child. + c.args(["--", prompt]) + .stdin(lock.try_clone().map_err(|e| e.to_string())?); + checked_output(c.output().map_err(|e| e.to_string())?).map(|_| ()) } pub fn blocked_record(task: &Task, detail: String) -> Record { @@ -155,12 +178,23 @@ pub fn blocked_record(task: &Task, detail: String) -> Record { worktree: PathBuf::new(), outcome: Outcome::Blocked, detail, + step: Step::Finished, + progress_at: state::now(), } } +pub fn run_task_with_policy(repo: &Path, binary: &Path, task: &Task, offline: bool) -> Result { + run_task_with_options(repo, binary, task, false, offline) +} pub fn run_task(repo: &Path, binary: &Path, task: &Task) -> Result { - run_task_with_policy(repo, binary, task, false) + run_task_with_options(repo, binary, task, false, false) } -pub fn run_task_with_policy(repo: &Path, binary: &Path, task: &Task, offline: bool) -> Result { +pub fn run_task_with_options( + repo: &Path, + binary: &Path, + task: &Task, + rerun: bool, + offline: bool, +) -> Result { validate(task)?; let root = checked(git(repo, &["rev-parse", "--show-toplevel"]))?; let root = PathBuf::from(root); @@ -173,17 +207,7 @@ pub fn run_task_with_policy(repo: &Path, binary: &Path, task: &Task, offline: bo let storage = common.join("oxa-work"); fs::create_dir_all(&storage).map_err(|e| e.to_string())?; let lock_path = storage.join(format!("{}.lock", task.id)); - let lock_file = fs::OpenOptions::new() - .write(true) - .create(true) - .truncate(false) - .open(&lock_path) - .map_err(|e| format!("cannot lock task {}: {e}", task.id))?; - lock_file - .try_lock() - .map_err(|e| format!("task {} is already running: {e}", task.id))?; - // Keep the inode so a crashed runner leaves no stale ownership and concurrent - // runners cannot acquire locks on two different files with the same name. + let lock = state::lock(&lock_path)?; let receipt = storage.join(format!("{}.json", task.id)); // The common git directory must not be an ancestor of the child cwd: the sandbox // deliberately refuses writable git binds with that layout. @@ -199,59 +223,75 @@ pub fn run_task_with_policy(repo: &Path, binary: &Path, task: &Task, offline: bo let trees = parent.join(".oxa-worktrees").join(name); fs::create_dir_all(&trees).map_err(|e| e.to_string())?; let tree = trees.join(&task.id); - tree::check_receipt(&receipt, &tree, task)?; - let mut record = Record { - id: task.id.clone(), - branch: task.branch.clone(), - worktree: tree.clone(), - outcome: Outcome::Blocked, - detail: String::new(), - }; - let result = (|| -> Result { - let prepared = tree::prepare(&root, &tree, task)?; - let tree = tree.canonicalize().map_err(|e| e.to_string())?; - let commands = tree::done_from_base(&tree, &prepared.base)?; - let first_agent = agent(binary, &tree, task, &prepared.prompt, offline); - tree::ensure_merged(&tree, &prepared.base, &task.branch)?; - let failures = acceptance(&tree, &commands)?; - if !failures.is_empty() { - let repair_agent = agent( - binary, - &tree, - task, - &format!( - "{}\n\nExternal acceptance checks failed. Repair once:\n{failures}", - prepared.prompt - ), - offline, - ); - tree::ensure_merged(&tree, &prepared.base, &task.branch)?; - let failures = acceptance(&tree, &commands)?; - if !failures.is_empty() { - record.detail = failures; - return Ok(Outcome::Red); - } - repair_agent?; - } else { - first_agent?; + let mut state = if receipt.exists() { + let old = state::read(&receipt)?; + if old.task != *task + || old.record.worktree != tree + || old.record.id != task.id + || old.record.branch != task.branch + { + return Err("task specification changed; use a new task id".into()); } - let head = checked(git(&tree, &["rev-parse", "HEAD"]))?; - let dirty = checked(git(&tree, &["status", "--porcelain"]))?; - Ok(if head == prepared.initial && dirty.is_empty() { - Outcome::NothingToDo + if old.record.step != Step::Finished { + old + } else if !rerun && matches!(old.record.outcome, Outcome::Done | Outcome::NothingToDo) { + return Ok(old.record); } else { - Outcome::Done - }) - })(); - match result { - Ok(outcome) => record.outcome = outcome, - Err(e) => record.detail = e, + new_state(task, tree, offline || old.offline) + } + } else { + new_state(task, tree, offline) + }; + state.offline |= offline; + let step = state.record.step; + state::save(&receipt, &mut state, step)?; + if let Err(error) = execute::execute(&root, binary, task, &receipt, &mut state, &lock) { + state.record.outcome = Outcome::Blocked; + state.record.detail = error; + } + state::save(&receipt, &mut state, Step::Saving)?; + if let Err(error) = execute::save_work(&root, &state.record.worktree, task) { + state.record.outcome = Outcome::Blocked; + if !state.record.detail.is_empty() { + state.record.detail.push('\n'); + } + state.record.detail.push_str(&format!("save: {error}")); + } + state::save(&receipt, &mut state, Step::Finished)?; + Ok(state.record) +} +fn new_state(task: &Task, worktree: PathBuf, offline: bool) -> state::State { + state::State { + record: Record { + id: task.id.clone(), + branch: task.branch.clone(), + worktree, + outcome: Outcome::Running, + detail: String::new(), + step: Step::Preparing, + progress_at: state::now(), + }, + task: task.clone(), + checkpoint: None, + offline, + } +} +pub fn status(repo: &Path) -> Result> { + let common = checked(git( + repo, + &["rev-parse", "--path-format=absolute", "--git-common-dir"], + ))?; + let directory = PathBuf::from(common).join("oxa-work"); + if !directory.exists() { + return Ok(vec![]); + } + let mut records = vec![]; + for entry in fs::read_dir(directory).map_err(|e| e.to_string())? { + let path = entry.map_err(|e| e.to_string())?.path(); + if path.extension().is_some_and(|s| s == "json") { + records.push(state::read(&path)?.record); + } } - // Separate receipts avoid overwriting another task's result. Never remove its worktree. - fs::write( - receipt, - serde_json::to_vec_pretty(&record).map_err(|e| e.to_string())?, - ) - .map_err(|e| e.to_string())?; - Ok(record) + records.sort_by(|a, b| a.id.cmp(&b.id)); + Ok(records) } diff --git a/oxa/oxa-work/src/state.rs b/oxa/oxa-work/src/state.rs new file mode 100644 index 000000000..ff0b433e9 --- /dev/null +++ b/oxa/oxa-work/src/state.rs @@ -0,0 +1,94 @@ +//! Durable step checkpoints and a kernel lock that recovers after process death. +use super::{Record, Result, Task}; +use serde::{Deserialize, Serialize}; +use std::fs::{self, File, OpenOptions}; +use std::io::Write; +use std::os::unix::fs::OpenOptionsExt; +use std::path::Path; +use std::time::{SystemTime, UNIX_EPOCH}; + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum Step { + Preparing, + Agent, + Acceptance, + Repair, + Recheck, + Saving, + Finished, +} + +#[derive(Debug, Deserialize, Serialize)] +pub(crate) struct Checkpoint { + pub initial: String, + pub base: String, + pub prompt: String, + pub commands: Vec, + pub agent_error: Option, + pub failures: String, +} +#[derive(Debug, Deserialize, Serialize)] +pub(crate) struct State { + #[serde(flatten)] + pub record: Record, + pub task: Task, + pub checkpoint: Option, + pub offline: bool, +} +pub(crate) fn now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} +pub(crate) fn lock(path: &Path) -> Result { + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .mode(0o600) + .open(path) + .map_err(|e| e.to_string())?; + file.try_lock() + .map_err(|e| format!("task is already running: {e}"))?; + file.set_len(0).map_err(|e| e.to_string())?; + // Keep the inode: removing a lock file permits two processes to lock different inodes. + Ok(file) +} +pub(crate) fn read(path: &Path) -> Result { + serde_json::from_slice(&fs::read(path).map_err(|e| e.to_string())?) + .map_err(|e| format!("task state: {e}")) +} +pub(crate) fn save(path: &Path, state: &mut State, step: Step) -> Result<()> { + state.record.step = step; + state.record.progress_at = now(); + let temporary = path.with_extension(format!( + "tmp-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + )); + let result = (|| -> Result<()> { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&temporary) + .map_err(|e| e.to_string())?; + file.write_all(&serde_json::to_vec_pretty(state).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + file.sync_all().map_err(|e| e.to_string())?; + fs::rename(&temporary, path).map_err(|e| e.to_string())?; + File::open(path.parent().ok_or("state has no parent")?) + .and_then(|p| p.sync_all()) + .map_err(|e| e.to_string()) + })(); + if result.is_err() { + let _ = fs::remove_file(temporary); + } + result +} diff --git a/oxa/oxa-work/src/tree.rs b/oxa/oxa-work/src/tree.rs index 122fe61a7..341282286 100644 --- a/oxa/oxa-work/src/tree.rs +++ b/oxa/oxa-work/src/tree.rs @@ -1,6 +1,5 @@ //! Worktree continuation uses ordinary merges; no reset, stash or force checkout. use super::{checked, git, output, Result, Task}; -use std::fs; use std::path::{Path, PathBuf}; pub(crate) struct Prepared { @@ -15,7 +14,7 @@ fn git_path(tree: &Path, name: &str) -> Result { )) .map(PathBuf::from) } -fn merging(tree: &Path) -> Result { +pub(crate) fn merging(tree: &Path) -> Result { Ok(git_path(tree, "MERGE_HEAD")?.exists()) } fn merge_prompt(tree: &Path, task: &Task, base: &str, output: &str) -> Result { @@ -45,23 +44,7 @@ pub(crate) fn prepare(repo: &Path, tree: &Path, task: &Task) -> Result _ => return Err(String::from_utf8_lossy(&exists.stderr).into()), } } - let canonical = tree.canonicalize().map_err(|e| e.to_string())?; - let actual = PathBuf::from(checked(git(tree, &["rev-parse", "--show-toplevel"]))?); - let common = |p: &Path| -> Result { - PathBuf::from(checked(git( - p, - &["rev-parse", "--path-format=absolute", "--git-common-dir"], - ))?) - .canonicalize() - .map_err(|e| e.to_string()) - }; - if actual.canonicalize().map_err(|e| e.to_string())? != canonical || common(tree)? != common(repo)? { - return Err("retained directory is not this repository's task worktree".into()); - } - let branch = checked(git(tree, &["symbolic-ref", "--quiet", "HEAD"]))?; - if branch != format!("refs/heads/{}", task.branch) { - return Err("retained worktree is on a different branch".into()); - } + validate(repo, tree, task)?; for operation in ["rebase-merge", "rebase-apply", "CHERRY_PICK_HEAD", "REVERT_HEAD"] { if git_path(tree, operation)?.exists() { return Err(format!("retained worktree has {operation} in progress")); @@ -109,6 +92,26 @@ pub(crate) fn prepare(repo: &Path, tree: &Path, task: &Task) -> Result prompt: task.prompt.clone(), }) } +pub(crate) fn validate(repo: &Path, tree: &Path, task: &Task) -> Result<()> { + let canonical = tree.canonicalize().map_err(|e| e.to_string())?; + let actual = PathBuf::from(checked(git(tree, &["rev-parse", "--show-toplevel"]))?); + let common = |p: &Path| -> Result { + PathBuf::from(checked(git( + p, + &["rev-parse", "--path-format=absolute", "--git-common-dir"], + ))?) + .canonicalize() + .map_err(|e| e.to_string()) + }; + if actual.canonicalize().map_err(|e| e.to_string())? != canonical || common(tree)? != common(repo)? { + return Err("retained directory is not this repository's task worktree".into()); + } + let branch = checked(git(tree, &["symbolic-ref", "--quiet", "HEAD"]))?; + if branch != format!("refs/heads/{}", task.branch) { + return Err("retained worktree is on a different branch".into()); + } + Ok(()) +} pub(crate) fn ensure_merged(tree: &Path, base: &str, branch: &str) -> Result<()> { if checked(git(tree, &["symbolic-ref", "--quiet", "HEAD"]))? != format!("refs/heads/{branch}") { return Err("agent left the worktree on a different branch".into()); @@ -137,13 +140,3 @@ pub(crate) fn done_from_base(repo: &Path, base: &str) -> Result> { Some(value) => serde_json::from_value(value.clone()).map_err(|e| format!("done commands: {e}")), } } -pub(crate) fn check_receipt(receipt: &Path, tree: &Path, task: &Task) -> Result<()> { - if receipt.exists() { - let record: super::Record = serde_json::from_slice(&fs::read(receipt).map_err(|e| e.to_string())?) - .map_err(|e| format!("retained receipt: {e}"))?; - if record.id != task.id || record.branch != task.branch || record.worktree != tree { - return Err("retained receipt belongs to a different task branch or worktree".into()); - } - } - Ok(()) -} From f6321b0c5ee42508cc0795930060dc07b72c0fc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:03:57 +0000 Subject: [PATCH 08/52] Keep task ownership throughout saving and detect interrupted status --- oxa/docs/work.md | 3 ++ oxa/oxa-cli/tests/work_state.rs | 80 +++++++++++++++++++++++++++++++++ oxa/oxa-work/src/execute.rs | 20 +++++---- oxa/oxa-work/src/lib.rs | 29 +++++++++++- 4 files changed, 121 insertions(+), 11 deletions(-) diff --git a/oxa/docs/work.md b/oxa/docs/work.md index 896fc3792..d6089db74 100644 --- a/oxa/docs/work.md +++ b/oxa/docs/work.md @@ -73,3 +73,6 @@ step. Reusing an ID with a different specification is refused; use a new ID. Restart continues the saved step. An agent or check interrupted within its step may run again; tasks and checks should tolerate that. Checkpoints after an agent step avoid replaying it when only acceptance or saving remains. + +Status probes the task lock. An interrupted phase with no surviving worker is shown +as blocked; the checkpoint stays intact for the next task-file invocation. diff --git a/oxa/oxa-cli/tests/work_state.rs b/oxa/oxa-cli/tests/work_state.rs index c5464e2fd..8101df4f5 100644 --- a/oxa/oxa-cli/tests/work_state.rs +++ b/oxa/oxa-cli/tests/work_state.rs @@ -208,3 +208,83 @@ fn child_keeps_lock_after_runner_is_killed_then_task_restarts() { assert_eq!(restarted["outcome"], "nothing_to_do", "{restarted}"); assert_eq!(restarted["step"], "finished"); } + +#[test] +fn saving_git_keeps_lock_after_runner_dies_during_commit_hook() { + use std::os::unix::fs::PermissionsExt; + struct Release(PathBuf); + impl Drop for Release { + fn drop(&mut self) { + let _ = fs::write(&self.0, "release"); + } + } + let env = TestEnv::new(); + setup(&env, "[]"); + trust(&env); + let release = Release(env.home().join("release-hook")); + let hook = env.write( + "work/project/.git/hooks/pre-commit", + r#"#!/bin/sh +touch "$HOME/hook-started" +attempt=0 +while ! test -f "$HOME/release-hook"; do + attempt=$((attempt + 1)) + if test "$attempt" -ge 500; then exit 9; fi + sleep 0.01 +done +"#, + ); + fs::set_permissions(hook, fs::Permissions::from_mode(0o755)).unwrap(); + let server = + FakeServer::start(vec![write_turn(), ScriptTurn::text("done")], FakeOpts::default()).unwrap(); + let mut child = Command::new(env!("CARGO_BIN_EXE_oxa")) + .args(["work", "../tasks.json"]) + .current_dir(project(&env)) + .env_clear() + .envs(env.envs(&server.url())) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + until(|| env.home().join("hook-started").exists()); + child.kill().unwrap(); + child.wait().unwrap(); + let blocked = run(&env, vec![]); + assert!( + blocked["detail"].as_str().unwrap().contains("already running"), + "{blocked}" + ); + assert_eq!(load(&env)["step"], "saving"); + fs::write(&release.0, "release").unwrap(); + let path = project(&env).join(".git/oxa-work/test.lock"); + until(|| { + let file = fs::OpenOptions::new().read(true).write(true).open(&path).unwrap(); + file.try_lock().is_ok() + }); + let record = run(&env, vec![]); + assert_eq!(record["outcome"], "done", "{record}"); + assert_eq!(git(&env, tree(&record), &["show", "HEAD:answer.txt"]), "retained"); +} +#[test] +fn status_reports_an_unlocked_interrupted_phase_as_blocked_without_mutating_state() { + let env = TestEnv::new(); + setup(&env, "[]"); + run(&env, vec![ScriptTurn::text("done")]); + let mut state = load(&env); + state["step"] = "agent".into(); + state["outcome"] = "running".into(); + replace_state(&env, &state); + let out = Command::new(env!("CARGO_BIN_EXE_oxa")) + .args(["work", "--status"]) + .current_dir(project(&env)) + .env_clear() + .envs(env.envs("http://127.0.0.1:1")) + .output() + .unwrap(); + let text = String::from_utf8_lossy(&out.stdout); + assert!( + text.starts_with("test blocked ") && text.trim().ends_with("agent"), + "{text}" + ); + assert_eq!(load(&env), state); +} diff --git a/oxa/oxa-work/src/execute.rs b/oxa/oxa-work/src/execute.rs index 115ec7317..a6e08fd12 100644 --- a/oxa/oxa-work/src/execute.rs +++ b/oxa/oxa-work/src/execute.rs @@ -101,23 +101,25 @@ pub(crate) fn execute( } } /// Save all staged, unstaged and untracked files. Conflicts are kept in progress for the agent. -pub(crate) fn save_work(repo: &Path, tree: &Path, task: &Task) -> Result<()> { +pub(crate) fn save_work(repo: &Path, tree: &Path, task: &Task, lock: &File) -> Result<()> { if !tree.exists() { return Ok(()); } tree::validate(repo, tree, task)?; - if tree::merging(tree)? || !checked(git(tree, &["diff", "--name-only", "--diff-filter=U"]))?.is_empty() { + let run = |args: &[&str]| -> Result { + let mut command = git(tree, args); + command.stdin(lock.try_clone().map_err(|e| e.to_string())?); + super::checked_output(command.output().map_err(|e| e.to_string())?) + }; + if tree::merging(tree)? || !run(&["diff", "--name-only", "--diff-filter=U"])?.is_empty() { return Err("unresolved merge retained in progress; files were not discarded".into()); } - if checked(git(tree, &["status", "--porcelain"]))?.is_empty() { + if run(&["status", "--porcelain"])?.is_empty() { return Ok(()); } - checked(git(tree, &["add", "--all"]))?; - checked(git( - tree, - &["commit", "-m", &format!("oxa work: save task {}", task.id)], - ))?; - if !checked(git(tree, &["status", "--porcelain"]))?.is_empty() { + run(&["add", "--all"])?; + run(&["commit", "-m", &format!("oxa work: save task {}", task.id)])?; + if !run(&["status", "--porcelain"])?.is_empty() { return Err("uncommitted changes remain after saving; retained for inspection".into()); } Ok(()) diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index 0ddd10e97..1883032d2 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -250,7 +250,7 @@ pub fn run_task_with_options( state.record.detail = error; } state::save(&receipt, &mut state, Step::Saving)?; - if let Err(error) = execute::save_work(&root, &state.record.worktree, task) { + if let Err(error) = execute::save_work(&root, &state.record.worktree, task, &lock) { state.record.outcome = Outcome::Blocked; if !state.record.detail.is_empty() { state.record.detail.push('\n'); @@ -289,7 +289,32 @@ pub fn status(repo: &Path) -> Result> { for entry in fs::read_dir(directory).map_err(|e| e.to_string())? { let path = entry.map_err(|e| e.to_string())?.path(); if path.extension().is_some_and(|s| s == "json") { - records.push(state::read(&path)?.record); + let mut value = state::read(&path)?; + if value.record.step != Step::Finished { + let file = fs::OpenOptions::new() + .read(true) + .write(true) + .open(path.with_extension("lock")); + let busy = match file { + Ok(file) => match file.try_lock() { + Ok(()) => { + value = state::read(&path)?; + false + } + Err(std::fs::TryLockError::WouldBlock) => true, + Err(error) => return Err(error.to_string()), + }, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => false, + Err(error) => return Err(error.to_string()), + }; + if value.record.step != Step::Finished { + value.record.outcome = if busy { Outcome::Running } else { Outcome::Blocked }; + if !busy { + value.record.detail = "task was interrupted; rerun its task file to resume".into(); + } + } + } + records.push(value.record); } } records.sort_by(|a, b| a.id.cmp(&b.id)); From 232eb0efc75474912093db5f9c4734195fa8eb62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:11:09 +0000 Subject: [PATCH 09/52] Publish accepted work and file scrubbed duplicate-checked problem notes (#333) --- oxa/docs/work.md | 27 ++- oxa/oxa-cli/src/args.rs | 2 +- oxa/oxa-cli/src/dispatch.rs | 4 +- oxa/oxa-cli/src/lib.rs | 1 + oxa/oxa-cli/src/work_cmd.rs | 56 ++++- oxa/oxa-cli/src/work_reports.rs | 380 ++++++++++++++++++++++++++++++ oxa/oxa-cli/tests/work_publish.rs | 256 ++++++++++++++++++++ oxa/oxa-work/src/execute.rs | 19 +- oxa/oxa-work/src/lib.rs | 43 +++- oxa/oxa-work/src/notes.rs | 51 ++++ oxa/oxa-work/src/publish.rs | 221 +++++++++++++++++ oxa/oxa-work/src/state.rs | 3 + 12 files changed, 1047 insertions(+), 16 deletions(-) create mode 100644 oxa/oxa-cli/src/work_reports.rs create mode 100644 oxa/oxa-cli/tests/work_publish.rs create mode 100644 oxa/oxa-work/src/notes.rs create mode 100644 oxa/oxa-work/src/publish.rs diff --git a/oxa/docs/work.md b/oxa/docs/work.md index d6089db74..14a62278c 100644 --- a/oxa/docs/work.md +++ b/oxa/docs/work.md @@ -46,7 +46,7 @@ Uncommitted task work, including red work, is saved as a commit with the reposit configured identity. Commit hooks run normally. A missing identity or rejected commit blocks the task and retains every file. Unresolved conflicts remain in progress for the agent; they are never staged as a fabricated resolution. There is no automatic -cleanup, push or PR creation yet. +cleanup. Publication is explicitly opt-in as described below. To continue an interrupted, red or blocked task, run the same file again. Use `--rerun` to continue a completed task after its base changes. The retained receipt @@ -76,3 +76,28 @@ step avoid replaying it when only acceptance or saving remains. Status probes the task lock. An interrupted phase with no surviving worker is shown as blocked; the checkpoint stays intact for the next task-file invocation. + + +`oxa work tasks.json --publish` pushes the accepted, saved commit to `origin` and +creates or updates its PR through `gh`. Red acceptance creates a draft; green +acceptance makes the PR ready. Nothing-to-do and blocked tasks do not create PRs. +The runner checks existing PR queue membership before every push and refuses a +queued PR; use a new task branch for follow-up work. It refuses branch changes +made after acceptance until `--rerun` checks them. PR bodies include task ID and +acceptance state, without raw prompts, transcripts or failed-check output. +A publication failure keeps a publication checkpoint; another `--publish` retries +without replaying the agent. Offline mode refuses network publication. + +`--report-problems` independently authorizes filing the task's own structured +problem notes using the `report-issue` workflow. This flag must be launched by the +user outside an agent tool. Normal `oxa report-issue submit` still requires its +terminal confirmation. Reports use the user setting `oxa.reportIssue.repo`; +project settings cannot choose a destination. Notes, titles and queries pass +through the existing report scrubber. The runner performs two or three symptom +queries against open issues and PRs and reads the top matches before writing. +It comments only new notes on a matching issue, links an existing fixing PR, or +files a new report with related links. Search failure, offline mode or missing +report configuration keeps a local scrubbed draft and records its path/detail. +A later explicit `--report-problems` retries local reports without replaying the +agent. No notes are filed unless this flag is present; model output and acceptance +logs are never treated as report bodies. diff --git a/oxa/oxa-cli/src/args.rs b/oxa/oxa-cli/src/args.rs index ec2fe79aa..345b5700c 100644 --- a/oxa/oxa-cli/src/args.rs +++ b/oxa/oxa-cli/src/args.rs @@ -177,7 +177,7 @@ const COMMANDS: &[(&str, &str)] = &[ "Draft a scrubbed ox-llm issue; submit posts it only after you type yes", ), ( - "oxa work [--rerun] | --issue N --model MODEL | --status", + "oxa work [--rerun] [--publish] [--report-problems] | --status", "Run tasks in isolated worktrees with external acceptance checks", ), ( diff --git a/oxa/oxa-cli/src/dispatch.rs b/oxa/oxa-cli/src/dispatch.rs index 4ef83d386..052ba1b9a 100644 --- a/oxa/oxa-cli/src/dispatch.rs +++ b/oxa/oxa-cli/src/dispatch.rs @@ -125,7 +125,9 @@ pub fn run(argv: &[String], env: &Env, io: Stdio) -> i32 { Err(e) => return report(&mut *stderr, &e), }; let result = match sub { - Subcommand::Work(words) => return crate::work_cmd::run(&words, &cwd, &mut *stdout, &mut *stderr), + Subcommand::Work(words) => { + return crate::work_cmd::run(&words, &cwd, &dirs, env, &mut *stdout, &mut *stderr) + } Subcommand::Mcp(words) => run_mcp(&words, &dirs, &cwd, &mut *stdout, &mut *stderr), Subcommand::Pkg(verb, words) => run_pkg(&verb, &words, &dirs, &cwd, &mut *stdout, &mut *stderr), Subcommand::Config(words) => run_config(&words, &dirs, &cwd, &mut *stdout), diff --git a/oxa/oxa-cli/src/lib.rs b/oxa/oxa-cli/src/lib.rs index f5cc30f16..afd53316a 100644 --- a/oxa/oxa-cli/src/lib.rs +++ b/oxa/oxa-cli/src/lib.rs @@ -32,6 +32,7 @@ mod sandbox_layout; mod themes; mod wipe_cmd; mod work_cmd; +mod work_reports; pub use args::{parse_args, usage_text, ApiKey, Args, ModeArg, Subcommand}; pub use config_cmd::run_config; diff --git a/oxa/oxa-cli/src/work_cmd.rs b/oxa/oxa-cli/src/work_cmd.rs index 6978aa9f6..0ac619a6b 100644 --- a/oxa/oxa-cli/src/work_cmd.rs +++ b/oxa/oxa-cli/src/work_cmd.rs @@ -1,7 +1,14 @@ use std::io::Write; use std::path::{Path, PathBuf}; -pub(crate) fn run(args: &[String], cwd: &Path, out: &mut dyn Write, err: &mut dyn Write) -> i32 { +pub(crate) fn run( + args: &[String], + cwd: &Path, + dirs: &oxa_config::Dirs, + env: &oxa_config::Env, + out: &mut dyn Write, + err: &mut dyn Write, +) -> i32 { let result = (|| -> Result { if args == ["--status"] { for record in oxa_work::status(cwd)? { @@ -19,8 +26,21 @@ pub(crate) fn run(args: &[String], cwd: &Path, out: &mut dyn Write, err: &mut dy } return Ok(0); } - let rerun = args.last().is_some_and(|s| s == "--rerun"); - let args = if rerun { &args[..args.len() - 1] } else { args }; + let mut end = args.len(); + let (mut rerun, mut publish, mut reports) = (false, false, false); + while let Some(flag) = args.get(end.wrapping_sub(1)) { + match flag.as_str() { + "--rerun" => rerun = true, + "--publish" => publish = true, + "--report-problems" => reports = true, + _ => break, + } + end -= 1; + } + let args = &args[..end]; + if (publish || reports) && env.get("OXA_CODING_AGENT").is_some() { + return Err("publishing and problem reporting must be explicitly launched by the user outside an agent tool".into()); + } let tasks = match args { [file] if !file.starts_with('-') => oxa_work::read_tasks(&cwd.join(file))?, [flag, number, model_flag, model] if flag == "--issue" && model_flag == "--model" => { @@ -35,7 +55,7 @@ pub(crate) fn run(args: &[String], cwd: &Path, out: &mut dyn Write, err: &mut dy } _ => { return Err( - "usage: oxa work [--rerun] | --issue N --model MODEL [--rerun] | --status" + "usage: oxa work | --issue N --model MODEL | --status; task runs accept --rerun, --publish, --report-problems" .into(), ) } @@ -43,8 +63,32 @@ pub(crate) fn run(args: &[String], cwd: &Path, out: &mut dyn Write, err: &mut dy let binary: PathBuf = std::env::current_exe().map_err(|e| e.to_string())?; let mut code = 0; for task in tasks { - let record = oxa_work::run_task_with_options(cwd, &binary, &task, rerun, oxa_proto::offline()) - .unwrap_or_else(|detail| oxa_work::blocked_record(&task, detail)); + let mut record = + oxa_work::run_task_with_options(cwd, &binary, &task, rerun, oxa_proto::offline()) + .unwrap_or_else(|detail| oxa_work::blocked_record(&task, detail)); + if publish && !record.worktree.as_os_str().is_empty() { + record = oxa_work::publish(cwd, &task, oxa_proto::offline()) + .unwrap_or_else(|detail| oxa_work::blocked_record(&task, detail)); + } + if reports && !record.worktree.as_os_str().is_empty() { + let settings = oxa_config::Settings::load(dirs, cwd, false); + let ctx = crate::report_cmd::ScrubCtx::from_env(env, dirs); + record = + oxa_work::report_notes(cwd, &task, oxa_proto::offline(), &mut |note, lock, offline| { + let mut runner = |program: &str, args: &[String]| { + crate::work_reports::process(program, args, cwd, lock) + }; + crate::work_reports::report( + note, + dirs, + &settings, + &ctx, + offline || oxa_proto::offline(), + &mut runner, + ) + }) + .unwrap_or_else(|detail| oxa_work::blocked_record(&task, detail)); + } if matches!( record.outcome, oxa_work::Outcome::Red | oxa_work::Outcome::Blocked diff --git a/oxa/oxa-cli/src/work_reports.rs b/oxa/oxa-cli/src/work_reports.rs new file mode 100644 index 000000000..d5cd7e763 --- /dev/null +++ b/oxa/oxa-cli/src/work_reports.rs @@ -0,0 +1,380 @@ +//! The report-issue workflow for an explicit user-launched --report-problems run. +//! Normal `report-issue submit` retains its controlling-terminal confirmation. +use crate::report_cmd::{clean_notes, report_repo, save_draft, scrub, Runner, ScrubCtx}; +use oxa_config::{Dirs, Settings}; +use oxa_work::{ProblemNote, ReportRecord}; +use serde_json::Value; +use std::{ + fs::File, + io::Read, + path::Path, + process::{Command, Stdio}, + time::{Duration, Instant}, +}; + +pub(crate) fn process(program: &str, args: &[String], cwd: &Path, lock: &File) -> Option<(i32, String)> { + let mut child = Command::new(program) + .args(args) + .current_dir(cwd) + .stdin(Stdio::from(lock.try_clone().ok()?)) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .ok()?; + let mut stdout = child.stdout.take()?; + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let mut out = String::new(); + let result = (&mut stdout).take(1_048_576).read_to_string(&mut out); + let _ = tx.send(result.ok().map(|_| out)); + }); + let start = Instant::now(); + loop { + match child.try_wait() { + Ok(Some(status)) => { + return Some(( + status.code().unwrap_or(1), + rx.recv_timeout(Duration::from_secs(2)).ok()??, + )) + } + Ok(None) if start.elapsed() < Duration::from_secs(30) => { + std::thread::sleep(Duration::from_millis(20)) + } + _ => { + let _ = child.kill(); + let _ = child.wait(); + return None; + } + } + } +} +fn call(runner: Runner<'_>, args: &[String]) -> Result { + match runner("gh", args) { + Some((0, text)) => Ok(text), + _ => Err("GitHub search or write failed; the local draft is retained".into()), + } +} +fn words(items: &[&str]) -> Vec { + items.iter().map(|s| s.to_string()).collect() +} +fn matches(note: &ProblemNote, candidate: &Value) -> bool { + let title = candidate["title"].as_str().unwrap_or_default().to_lowercase(); + if title == note.title.to_lowercase() { + return true; + } + let all = format!( + "{title} {}", + candidate["body"].as_str().unwrap_or_default().to_lowercase() + ); + let terms: std::collections::HashSet<_> = note + .keywords + .iter() + .flat_map(|q| q.split_whitespace()) + .filter(|word| word.len() >= 2) + .map(str::to_lowercase) + .collect(); + terms.len() >= 3 && terms.iter().all(|word| all.contains(word)) +} +pub(crate) fn report( + note: &ProblemNote, + dirs: &Dirs, + settings: &Settings, + ctx: &ScrubCtx, + offline: bool, + runner: Runner<'_>, +) -> Result { + let (draft, title, body) = + save_draft(¬e.title, ¬e.notes, dirs, ctx, runner).map_err(|e| e.to_string())?; + let mut result = ReportRecord { + disposition: "local".into(), + draft, + url: None, + detail: String::new(), + }; + if offline { + result.detail = "offline: report retained locally".into(); + return Ok(result); + } + let Some(repo) = report_repo(settings) else { + result.detail = "issue reporting is not configured; draft retained locally".into(); + return Ok(result); + }; + let note = ProblemNote { + title, + notes: clean_notes(¬e.notes, ctx), + keywords: note.keywords.iter().map(|q| scrub(q, ctx)).collect(), + }; + if note.notes.is_empty() + || !(2..=3).contains(¬e.keywords.len()) + || note.keywords.iter().any(|q| q.trim().is_empty()) + { + result.detail = "report has no safe symptom text or queries; draft retained locally".into(); + return Ok(result); + } + let attempt = (|| -> Result<(), String> { + let mut candidates = vec![]; + for query in ¬e.keywords { + for kind in ["issue", "pr"] { + let text = call( + runner, + &words(&[ + kind, + "list", + "--repo", + &repo, + "--state", + "open", + "--search", + query, + "--limit", + "5", + "--json", + "number,title,url", + ]), + )?; + let rows: Vec = + serde_json::from_str(&text).map_err(|_| "invalid GitHub search response".to_string())?; + for row in rows { + let number = row["number"].as_u64().ok_or("search match has no number")?; + if !candidates.contains(&(kind, number)) { + candidates.push((kind, number)); + } + } + } + } + // Read all top matches before any write, including PRs which may already fix it. + let mut read = vec![]; + for (kind, number) in candidates { + let fields = if kind == "issue" { + "title,body,url,comments" + } else { + "title,body,url" + }; + let text = call( + runner, + &words(&[ + kind, + "view", + &number.to_string(), + "--repo", + &repo, + "--json", + fields, + ]), + )?; + let value: Value = + serde_json::from_str(&text).map_err(|_| "invalid GitHub match response".to_string())?; + if !value.is_object() + || value["title"].as_str().is_none() + || value["body"].as_str().is_none() + || value["url"].as_str().is_none() + { + return Err("incomplete GitHub match; draft retained locally".into()); + } + read.push((kind, number, value)); + } + if let Some((_, _, value)) = read + .iter() + .find(|(kind, _, v)| *kind == "pr" && matches(¬e, v)) + { + result.disposition = "existing_pr".into(); + result.url = value["url"].as_str().map(str::to_string); + return Ok(()); + } + let body_path = result.draft.with_extension("work-body.md"); + if let Some((_, number, value)) = read + .iter() + .find(|(kind, _, v)| *kind == "issue" && matches(¬e, v)) + { + let already = value["body"].as_str().unwrap_or_default().contains(¬e.notes) + || value["comments"].as_array().is_some_and(|rows| { + rows.iter() + .any(|c| c["body"].as_str().unwrap_or_default().contains(¬e.notes)) + }); + if !already { + oxa_resources::write_atomic(&body_path, note.notes.as_bytes()).map_err(|e| e.to_string())?; + call( + runner, + &words(&[ + "issue", + "comment", + &number.to_string(), + "--repo", + &repo, + "--body-file", + &body_path.to_string_lossy(), + ]), + )?; + } + result.disposition = "existing_issue".into(); + result.url = value["url"].as_str().map(str::to_string); + } else { + let links = read + .iter() + .filter_map(|(_, _, v)| v["url"].as_str()) + .collect::>() + .join("\n"); + let body = if links.is_empty() { + body.clone() + } else { + format!("{body}\n\nRelated open reports (different symptoms):\n{links}\n") + }; + let body = scrub(&body, ctx); + oxa_resources::write_atomic(&body_path, body.as_bytes()).map_err(|e| e.to_string())?; + let url = call( + runner, + &words(&[ + "issue", + "create", + "--repo", + &repo, + "--title", + ¬e.title, + "--body-file", + &body_path.to_string_lossy(), + ]), + )?; + result.disposition = "filed".into(); + result.url = Some(url.trim().into()); + } + let _ = std::fs::remove_file(body_path); + Ok(()) + })(); + if let Err(error) = attempt { + result.detail = error; + } + Ok(result) +} + +#[cfg(test)] +mod tests { + use super::*; + use oxa_fake::TestEnv; + fn note() -> ProblemNote { + ProblemNote { + title: "oxa work commit is read only".into(), + notes: "Another synthetic reproduction of a read only git commit.".into(), + keywords: vec!["oxa work".into(), "read only git".into()], + } + } + fn settings() -> Settings { + Settings::from_values( + serde_json::json!({"oxa":{"reportIssue":{"repo":"example/project"}}}), + serde_json::json!({}), + ) + } + #[test] + fn matching_issue_is_commented_after_all_queries_and_reads() { + let env = TestEnv::new(); + let dirs = Dirs::under(env.root()); + let mut log = vec![]; + let mut runner = |program: &str, args: &[String]| { + if program != "gh" { + return Some((0, "{}".into())); + } + log.push(args.to_vec()); + match (args[0].as_str(), args[1].as_str()) { + ("issue", "list") => Some((0, r#"[{"number":7,"title":"oxa work commit is read only"}]"#.into())), + ("pr", "list") => Some((0, "[]".into())), + ("issue", "view") => Some((0, r#"{"title":"oxa work commit is read only","body":"Original symptom","url":"https://github.com/example/project/issues/7","comments":[]}"#.into())), + ("issue", "comment") => { let body = std::fs::read_to_string(args.last().unwrap()).unwrap(); assert_eq!(body, note().notes); Some((0, "commented".into())) }, + _ => panic!("unexpected write: {args:?}"), + } + }; + let result = report( + ¬e(), + &dirs, + &settings(), + &ScrubCtx::default(), + false, + &mut runner, + ) + .unwrap(); + assert_eq!(result.disposition, "existing_issue"); + assert_eq!(log.iter().filter(|a| a[1] == "list").count(), 4); + assert_eq!(log.last().unwrap()[1], "comment"); + } + #[test] + fn failed_search_and_offline_never_file_blind() { + for offline in [false, true] { + let env = TestEnv::new(); + let dirs = Dirs::under(env.root()); + let mut gh_calls = 0; + let mut runner = |program: &str, _: &[String]| { + if program == "gh" { + gh_calls += 1; + None + } else { + Some((0, "{}".into())) + } + }; + let result = report( + ¬e(), + &dirs, + &settings(), + &ScrubCtx::default(), + offline, + &mut runner, + ) + .unwrap(); + assert_eq!(result.disposition, "local"); + assert!(result.draft.exists()); + assert!(!result.detail.is_empty()); + assert_eq!(gh_calls, if offline { 0 } else { 1 }); + } + } + #[test] + fn an_open_fixing_pr_prevents_issue_or_comment_writes() { + let env = TestEnv::new(); + let dirs = Dirs::under(env.root()); + let mut runner = |program: &str, args: &[String]| { + if program != "gh" { + return Some((0, "{}".into())); + } + if args[1] == "list" { + return Some(( + 0, + if args[0] == "pr" { + r#"[{"number":9}]"# + } else { + "[]" + } + .into(), + )); + } + assert_eq!(args[1], "view"); + Some((0, r#"{"title":"oxa work commit is read only","body":"Fixed by this PR","url":"https://github.com/example/project/pull/9"}"#.into())) + }; + let result = report( + ¬e(), + &dirs, + &settings(), + &ScrubCtx::default(), + false, + &mut runner, + ) + .unwrap(); + assert_eq!(result.disposition, "existing_pr"); + } + #[test] + fn new_report_is_scrubbed_and_related_reports_are_linked() { + let env = TestEnv::new(); + let dirs = Dirs::under(env.root()); + let mut n = note(); + n.notes = "Synthetic reproduction token=example at /home/synthetic/project with user@example.invalid" + .into(); + let mut runner = |program: &str, args: &[String]| { + if program != "gh" { + return Some((0, "{}".into())); + } + match args[1].as_str() { + "list" => Some((0, if args[0] == "issue" { r#"[{"number":8}]"# } else { "[]" }.into())), + "view" => Some((0, r#"{"title":"Different symptom","body":"Unrelated reproduction","url":"https://github.com/example/project/issues/8","comments":[]}"#.into())), + "create" => { let body = std::fs::read_to_string(args.last().unwrap()).unwrap(); assert!(!body.contains("token=example") && !body.contains("/home/synthetic") && !body.contains("user@example.invalid")); assert!(body.contains("issues/8")); Some((0, "https://github.com/example/project/issues/10".into())) }, + _ => panic!("unexpected command"), + } + }; + let result = report(&n, &dirs, &settings(), &ScrubCtx::default(), false, &mut runner).unwrap(); + assert_eq!(result.disposition, "filed"); + } +} diff --git a/oxa/oxa-cli/tests/work_publish.rs b/oxa/oxa-cli/tests/work_publish.rs new file mode 100644 index 000000000..ff0c2ae78 --- /dev/null +++ b/oxa/oxa-cli/tests/work_publish.rs @@ -0,0 +1,256 @@ +//! Local bare pushes and fake gh, including draft updates and merge-queue protection. +#[path = "common/work.rs"] +mod work; +use oxa_fake::{ScriptTurn, TestEnv}; +use std::{fs, path::PathBuf, process::Command}; +use work::{git, run, run_args, setup}; +fn configure(env: &TestEnv) -> String { + use std::os::unix::fs::PermissionsExt; + env.write( + "home/.oxa/agent/settings.json", + r#"{"defaultProjectTrust":"always","oxa":{"reportIssue":{"repo":"example/project"}}}"#, + ); + let gh = env.write("tmp/gh", r##"#!/bin/sh +printf '%s\n' "$*" >> "$HOME/gh-calls" +case "$1 $2" in + 'repo view') printf '%s' '{"nameWithOwner":"example/project"}' ;; + 'pr list') if test -f "$HOME/pr-state.json"; then printf '['; cat "$HOME/pr-state.json"; printf ']'; else printf '[]'; fi ;; + 'api graphql') if test -f "$HOME/queued"; then printf '%s' '{"data":{"repository":{"pullRequest":{"mergeQueueEntry":{"id":"queue"}}}}}'; else printf '%s' '{"data":{"repository":{"pullRequest":{"mergeQueueEntry":null}}}}'; fi ;; + 'pr create') + draft=false + body_next=false + for arg in "$@"; do + if test "$body_next" = true; then cat "$arg" > "$HOME/published-body"; body_next=false; fi + if test "$arg" = '--draft'; then draft=true; fi + if test "$arg" = '--body-file'; then body_next=true; fi + done + printf '{"number":1,"url":"https://github.com/example/project/pull/1","isDraft":%s}' "$draft" > "$HOME/pr-state.json" + printf '%s' 'https://github.com/example/project/pull/1' ;; + 'pr edit') : ;; + 'pr ready') + draft=false + for arg in "$@"; do if test "$arg" = '--undo'; then draft=true; fi; done + printf '{"number":1,"url":"https://github.com/example/project/pull/1","isDraft":%s}' "$draft" > "$HOME/pr-state.json" ;; + *) exit 7 ;; +esac +"##); + let git_wrapper = env.write("tmp/git", "#!/bin/sh\nif test \"$1\" = push; then printf 'push\\n' >> \"$HOME/push-calls\"; fi\nexec /usr/bin/git \"$@\"\n"); + let controller = env.write("tmp/oxc", "#!/bin/sh\nprintf '{}'\n"); + for file in [gh, git_wrapper, controller] { + fs::set_permissions(file, fs::Permissions::from_mode(0o755)).unwrap(); + } + format!("{}:{}", env.tmp().display(), std::env::var("PATH").unwrap()) +} +fn edits() -> Vec { + vec![ + ScriptTurn::tool("write", "write", r#"{"path":"answer.txt","content":"published"}"#), + ScriptTurn::text("done"), + ] +} +#[test] +fn publishes_local_branch_and_updates_same_pr_on_restart() { + let env = TestEnv::new(); + setup(&env, "[]"); + let path = configure(&env); + let first = run_args( + &env, + edits(), + &["work", "../tasks.json", "--publish"], + Some(path.clone()), + ); + assert_eq!(first["outcome"], "done", "{first}"); + assert_eq!(first["draft"], false); + assert_eq!(first["pull_request"], "https://github.com/example/project/pull/1"); + assert_eq!( + git( + &env, + &env.work(), + &[ + "--git-dir", + "remote.git", + "show", + "refs/heads/agent/test:answer.txt" + ] + ), + "published" + ); + let again = run_args(&env, vec![], &["work", "../tasks.json", "--publish"], Some(path)); + assert_eq!(again["outcome"], "done", "{again}"); + let calls = fs::read_to_string(env.home().join("gh-calls")).unwrap(); + assert_eq!(calls.lines().filter(|l| l.starts_with("pr create")).count(), 1); + assert_eq!(calls.lines().filter(|l| l.starts_with("pr edit")).count(), 1); + let body = fs::read_to_string(env.home().join("published-body")).unwrap(); + assert!(!body.contains("Finish task")); +} +#[test] +fn red_checks_create_a_draft_pr() { + let env = TestEnv::new(); + setup(&env, r#"["printf red > failed-check; exit 1"]"#); + let path = configure(&env); + let record = run_args( + &env, + vec![ScriptTurn::text("done"), ScriptTurn::text("still red")], + &["work", "../tasks.json", "--publish"], + Some(path), + ); + assert_eq!(record["outcome"], "red", "{record}"); + assert_eq!(record["draft"], true); + assert!(record["pull_request"].is_string()); +} +#[test] +fn queued_pr_is_never_pushed_and_publication_retries_without_agent_replay() { + let env = TestEnv::new(); + setup(&env, "[]"); + let path = configure(&env); + run_args( + &env, + edits(), + &["work", "../tasks.json", "--publish"], + Some(path.clone()), + ); + env.write("home/queued", "yes"); + let blocked = run_args( + &env, + vec![], + &["work", "../tasks.json", "--publish"], + Some(path.clone()), + ); + assert_eq!(blocked["outcome"], "blocked", "{blocked}"); + assert!(blocked["detail"].as_str().unwrap().contains("merge queue")); + assert_eq!( + fs::read_to_string(env.home().join("push-calls")) + .unwrap() + .lines() + .count(), + 1 + ); + fs::remove_file(env.home().join("queued")).unwrap(); + let retried = run_args(&env, vec![], &["work", "../tasks.json", "--publish"], Some(path)); + assert_eq!(retried["outcome"], "done", "{retried}"); +} +#[test] +fn offline_publication_never_starts_gh_or_push() { + let env = TestEnv::new(); + setup(&env, "[]"); + let path = configure(&env); + let record = run_args( + &env, + edits(), + &["--offline", "work", "../tasks.json", "--publish"], + Some(path), + ); + assert_eq!(record["outcome"], "blocked", "{record}"); + assert!(record["detail"].as_str().unwrap().contains("offline")); + assert!(!env.home().join("gh-calls").exists()); + assert!(!env.home().join("push-calls").exists()); +} +#[test] +fn branch_drift_after_acceptance_cannot_publish_unchecked_commits() { + let env = TestEnv::new(); + setup(&env, "[]"); + let path = configure(&env); + let done = run(&env, edits()); + let tree = PathBuf::from(done["worktree"].as_str().unwrap()); + fs::write(tree.join("later.txt"), "unchecked").unwrap(); + git(&env, &tree, &["add", "."]); + git(&env, &tree, &["commit", "-m", "later commit"]); + let record = run_args(&env, vec![], &["work", "../tasks.json", "--publish"], Some(path)); + assert_eq!(record["outcome"], "blocked", "{record}"); + assert!(record["detail"] + .as_str() + .unwrap() + .contains("changed after acceptance")); + assert!(!env.home().join("push-calls").exists()); +} +#[test] +fn own_problem_notes_are_captured_and_failed_search_keeps_a_local_draft() { + let env = TestEnv::new(); + setup(&env, "[]"); + let path = configure(&env); + let text = "done [{\"title\":\"oxa work symptom\",\"notes\":\"Synthetic reproduction token=example\",\"keywords\":[\"oxa work\",\"synthetic symptom\"]}]"; + let record = run_args( + &env, + vec![ScriptTurn::text(text)], + &["work", "../tasks.json", "--report-problems"], + Some(path), + ); + assert_eq!(record["problem_notes"].as_array().unwrap().len(), 1); + assert_eq!(record["reports"][0]["disposition"], "local"); + let draft = fs::read_to_string(record["reports"][0]["draft"].as_str().unwrap()).unwrap(); + assert!(!draft.contains("token=example")); + let calls = fs::read_to_string(env.home().join("gh-calls")).unwrap(); + assert!(!calls.contains("issue create")); +} +#[test] +fn a_model_tool_cannot_opt_itself_into_reporting_or_publishing() { + let env = TestEnv::new(); + setup(&env, "[]"); + let path = configure(&env); + let out = Command::new(env!("CARGO_BIN_EXE_oxa")) + .args(["work", "../tasks.json", "--report-problems"]) + .current_dir(env.work().join("project")) + .env_clear() + .envs(env.envs("http://127.0.0.1:1")) + .env("OXA_CODING_AGENT", "1") + .env("PATH", path) + .output() + .unwrap(); + assert_eq!(out.status.code(), Some(2)); + assert!(!env.home().join("gh-calls").exists()); +} + +#[test] +fn green_rerun_marks_an_existing_red_draft_ready() { + let env = TestEnv::new(); + setup( + &env, + r#"["test -f fixed || { printf red > failed-check; exit 1; }"]"#, + ); + let path = configure(&env); + let red = run_args( + &env, + vec![ScriptTurn::text("done"), ScriptTurn::text("still red")], + &["work", "../tasks.json", "--publish"], + Some(path.clone()), + ); + assert_eq!(red["draft"], true, "{red}"); + let green = run_args( + &env, + vec![ + ScriptTurn::tool("fix", "write", r#"{"path":"fixed","content":"fixed"}"#), + ScriptTurn::text("done"), + ], + &["work", "../tasks.json", "--rerun", "--publish"], + Some(path), + ); + assert_eq!(green["outcome"], "done", "{green}"); + assert_eq!(green["draft"], false); + assert!(fs::read_to_string(env.home().join("gh-calls")) + .unwrap() + .contains("pr ready")); +} +#[test] +fn a_local_report_retries_without_replaying_the_completed_task() { + let env = TestEnv::new(); + setup(&env, "[]"); + let path = configure(&env); + let text = "[{\"title\":\"oxa work symptom\",\"notes\":\"Synthetic reproduction\",\"keywords\":[\"oxa work\",\"synthetic symptom\"]}]"; + let first = run_args( + &env, + vec![ScriptTurn::text(text)], + &["work", "../tasks.json", "--report-problems"], + Some(path.clone()), + ); + assert_eq!(first["reports"][0]["disposition"], "local"); + let gh = env.tmp().join("gh"); + let script = fs::read_to_string(&gh).unwrap().replace(" *) exit 7 ;;", " 'issue list') printf '[]' ;;\n 'issue create') printf '%s' 'https://github.com/example/project/issues/2' ;;\n *) exit 7 ;;"); + fs::write(gh, script).unwrap(); + let retry = run_args( + &env, + vec![], + &["work", "../tasks.json", "--report-problems"], + Some(path), + ); + assert_eq!(retry["reports"][0]["disposition"], "filed", "{retry}"); + assert_eq!(retry["step"], "finished"); +} diff --git a/oxa/oxa-work/src/execute.rs b/oxa/oxa-work/src/execute.rs index a6e08fd12..a62807e77 100644 --- a/oxa/oxa-work/src/execute.rs +++ b/oxa/oxa-work/src/execute.rs @@ -34,7 +34,7 @@ pub(crate) fn execute( state.checkpoint = Some(Checkpoint { initial: prepared.initial, base: prepared.base, - prompt: prepared.prompt, + prompt: prepared.prompt + super::notes::INSTRUCTIONS, commands, agent_error: None, failures: String::new(), @@ -54,12 +54,25 @@ pub(crate) fn execute( point.prompt.clone() }; let result = agent(binary, &tree, task, &prompt, lock, state.offline); + let error = match result { + Ok(reply) => { + for note in reply.notes { + if state.record.problem_notes.len() < 5 + && !state.record.problem_notes.contains(¬e) + { + state.record.problem_notes.push(note); + } + } + reply.error + } + Err(error) => Some(error), + }; tree::ensure_merged(&tree, &point.base, &task.branch)?; state .checkpoint .as_mut() .ok_or("missing task checkpoint")? - .agent_error = result.err(); + .agent_error = error; state::save( path, state, @@ -96,7 +109,7 @@ pub(crate) fn execute( state::save(path, state, Step::Saving)?; } } - Step::Saving | Step::Finished => return Ok(()), + Step::Saving | Step::Finished | Step::Publishing | Step::Reporting => return Ok(()), } } } diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index 1883032d2..b29965cac 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -1,5 +1,9 @@ //! Opt-in, sequential unattended task execution. Worktrees are retained on every outcome. mod execute; +mod notes; +mod publish; +pub use notes::{ProblemNote, ReportRecord}; +pub use publish::{publish, report_notes}; mod state; mod tree; use serde::{Deserialize, Serialize}; @@ -43,6 +47,11 @@ pub struct Record { pub detail: String, pub step: Step, pub progress_at: u64, + pub head: Option, + pub pull_request: Option, + pub draft: bool, + pub problem_notes: Vec, + pub reports: Vec, } fn output(mut command: Command) -> Result { @@ -157,7 +166,7 @@ fn agent( prompt: &str, lock: &fs::File, offline: bool, -) -> Result<()> { +) -> Result { let mut c = Command::new(binary); c.current_dir(root) .args(["-p", "--model", &task.model, "--no-session"]); @@ -168,9 +177,18 @@ fn agent( // and fd 0 keeps the shared kernel lock alive if the runner dies before its child. c.args(["--", prompt]) .stdin(lock.try_clone().map_err(|e| e.to_string())?); - checked_output(c.output().map_err(|e| e.to_string())?).map(|_| ()) + let output = c.output().map_err(|e| e.to_string())?; + let notes = notes::parse(&String::from_utf8_lossy(&output.stdout)); + Ok(AgentReply { + notes, + error: checked_output(output).err(), + }) } +struct AgentReply { + notes: Vec, + error: Option, +} pub fn blocked_record(task: &Task, detail: String) -> Record { Record { id: task.id.clone(), @@ -180,6 +198,11 @@ pub fn blocked_record(task: &Task, detail: String) -> Record { detail, step: Step::Finished, progress_at: state::now(), + head: None, + pull_request: None, + draft: false, + problem_notes: vec![], + reports: vec![], } } pub fn run_task_with_policy(repo: &Path, binary: &Path, task: &Task, offline: bool) -> Result { @@ -232,7 +255,10 @@ pub fn run_task_with_options( { return Err("task specification changed; use a new task id".into()); } - if old.record.step != Step::Finished { + if matches!(old.record.step, Step::Publishing | Step::Reporting) && !rerun { + return Ok(old.record); + } + if old.record.step != Step::Finished && !rerun { old } else if !rerun && matches!(old.record.outcome, Outcome::Done | Outcome::NothingToDo) { return Ok(old.record); @@ -242,7 +268,7 @@ pub fn run_task_with_options( } else { new_state(task, tree, offline) }; - state.offline |= offline; + state.offline = offline; let step = state.record.step; state::save(&receipt, &mut state, step)?; if let Err(error) = execute::execute(&root, binary, task, &receipt, &mut state, &lock) { @@ -257,6 +283,9 @@ pub fn run_task_with_options( } state.record.detail.push_str(&format!("save: {error}")); } + if state.record.worktree.exists() && state.record.outcome != Outcome::Blocked { + state.record.head = Some(checked(git(&state.record.worktree, &["rev-parse", "HEAD"]))?); + } state::save(&receipt, &mut state, Step::Finished)?; Ok(state.record) } @@ -270,10 +299,16 @@ fn new_state(task: &Task, worktree: PathBuf, offline: bool) -> state::State { detail: String::new(), step: Step::Preparing, progress_at: state::now(), + head: None, + pull_request: None, + draft: false, + problem_notes: vec![], + reports: vec![], }, task: task.clone(), checkpoint: None, offline, + acceptance_outcome: None, } } pub fn status(repo: &Path) -> Result> { diff --git a/oxa/oxa-work/src/notes.rs b/oxa/oxa-work/src/notes.rs new file mode 100644 index 000000000..7e09e9beb --- /dev/null +++ b/oxa/oxa-work/src/notes.rs @@ -0,0 +1,51 @@ +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +pub struct ProblemNote { + pub title: String, + pub notes: String, + pub keywords: Vec, +} +#[derive(Debug, Deserialize, Serialize)] +pub struct ReportRecord { + pub disposition: String, + pub draft: PathBuf, + pub url: Option, + pub detail: String, +} +pub(crate) const INSTRUCTIONS: &str = "\n\nIf you encountered a problem with oxa, oxc or the engine, follow the report-issue skill's privacy rules. Do not post it yourself. End your final reply with [{\"title\":\"component and symptom\",\"notes\":\"your own brief reproduction, expected and actual behavior\",\"keywords\":[\"component symptom\",\"command error\"]}]. Supply two or three distinct symptom keyword queries. Never include prompts, transcripts, secrets, personal data or file contents. Omit the block when there is no problem to report. The runner only files notes when its user explicitly requests --report-problems."; +pub(crate) fn parse(output: &str) -> Vec { + let Some((_, rest)) = output.rsplit_once("") else { + return vec![]; + }; + let Some((json, _)) = rest.split_once("") else { + return vec![]; + }; + if json.len() > 32_000 { + return vec![]; + } + let Ok(notes) = serde_json::from_str::>(json) else { + return vec![]; + }; + notes + .into_iter() + .take(5) + .filter(|n| { + !n.title.trim().is_empty() + && n.title.chars().count() <= 200 + && !n.notes.trim().is_empty() + && n.notes.chars().count() <= 2000 + && (2..=3).contains(&n.keywords.len()) + && n.keywords + .iter() + .all(|k| !k.trim().is_empty() && k.chars().count() <= 100) + && n.keywords + .iter() + .map(|q| q.trim().to_lowercase()) + .collect::>() + .len() + == n.keywords.len() + }) + .collect() +} diff --git a/oxa/oxa-work/src/publish.rs b/oxa/oxa-work/src/publish.rs new file mode 100644 index 000000000..ee0fe06ba --- /dev/null +++ b/oxa/oxa-work/src/publish.rs @@ -0,0 +1,221 @@ +//! Explicit publishing of the accepted commit; existing queued PRs are never pushed. +use super::{checked, git, state, tree, Outcome, Record, Result, Task}; +use state::Step; +use std::{ + fs::File, + path::{Path, PathBuf}, + process::Command, +}; + +fn context(repo: &Path, task: &Task) -> Result<(PathBuf, File, state::State)> { + let common = PathBuf::from(checked(git( + repo, + &["rev-parse", "--path-format=absolute", "--git-common-dir"], + ))?); + let path = common.join("oxa-work").join(format!("{}.json", task.id)); + let lock = state::lock(&path.with_extension("lock"))?; + let value = state::read(&path)?; + if value.task != *task { + return Err("task specification changed".into()); + } + Ok((path, lock, value)) +} +fn gh(repo: &Path, lock: &File, args: &[&str]) -> Result { + let mut command = Command::new("gh"); + command.current_dir(repo).args(args); + command.stdin(lock.try_clone().map_err(|e| e.to_string())?); + super::checked_output(command.output().map_err(|e| e.to_string())?) +} +fn json(text: &str) -> Result { + serde_json::from_str(text).map_err(|e| e.to_string()) +} +fn pr_base(repo: &Path, task: &Task) -> Result { + let reference = checked(git(repo, &["rev-parse", "--symbolic-full-name", &task.base]))?; + reference + .strip_prefix("refs/heads/") + .or_else(|| reference.strip_prefix("refs/remotes/origin/")) + .filter(|s| !s.is_empty()) + .map(str::to_string) + .ok_or_else(|| "publishing requires a named local or origin base branch".into()) +} +pub fn publish(repo: &Path, task: &Task, offline: bool) -> Result { + let (path, lock, mut value) = context(repo, task)?; + let accepted = value.acceptance_outcome.unwrap_or(value.record.outcome); + if !matches!(accepted, Outcome::Done | Outcome::Red) { + return Ok(value.record); + } + value.acceptance_outcome = Some(accepted); + value.offline = offline; + state::save(&path, &mut value, Step::Publishing)?; + let result = (|| -> Result<(String, bool)> { + if offline { + return Err("publishing is unavailable in offline mode".into()); + } + let tree = &value.record.worktree; + tree::validate(repo, tree, task)?; + if !checked(git(tree, &["status", "--porcelain"]))?.is_empty() { + return Err("uncommitted work must be saved before publishing".into()); + } + let head = checked(git(tree, &["rev-parse", "HEAD"]))?; + if value.record.head.as_deref() != Some(head.as_str()) { + return Err("task branch changed after acceptance; use --rerun before publishing".into()); + } + let base = pr_base(tree, task)?; + let repository = json(&gh(tree, &lock, &["repo", "view", "--json", "nameWithOwner"])?)?; + let name = repository["nameWithOwner"] + .as_str() + .ok_or("repository name missing")?; + let (owner, repo_name) = name.split_once('/').ok_or("invalid repository name")?; + let prs = json(&gh( + tree, + &lock, + &[ + "pr", + "list", + "--repo", + name, + "--state", + "open", + "--head", + &task.branch, + "--json", + "number,url,isDraft", + ], + )?)?; + let prs = prs.as_array().ok_or("invalid PR list")?; + if prs.len() > 1 { + return Err("multiple open PRs match the task branch".into()); + } + if let Some(pr) = prs.first() { + let number = pr["number"].as_u64().ok_or("PR number missing")?.to_string(); + let query = "query($owner:String!,$name:String!,$number:Int!){repository(owner:$owner,name:$name){pullRequest(number:$number){mergeQueueEntry{id}}}}"; + let data = json(&gh( + tree, + &lock, + &[ + "api", + "graphql", + "-f", + &format!("query={query}"), + "-F", + &format!("owner={owner}"), + "-F", + &format!("name={repo_name}"), + "-F", + &format!("number={number}"), + ], + )?)?; + let pull = &data["data"]["repository"]["pullRequest"]; + if !pull.is_object() || pull.get("mergeQueueEntry").is_none() { + return Err("cannot determine whether PR is in the merge queue".into()); + } + if !pull["mergeQueueEntry"].is_null() { + return Err("PR is in the merge queue; use a new task branch for follow-up work".into()); + } + } + let mut push = git( + repo, + &["push", "origin", &format!("{head}:refs/heads/{}", task.branch)], + ); + push.env("GIT_TERMINAL_PROMPT", "0") + .stdin(lock.try_clone().map_err(|e| e.to_string())?); + super::checked_output(push.output().map_err(|e| e.to_string())?)?; + let draft = accepted == Outcome::Red; + let body = path.with_extension("pr-body.md"); + let text = format!("Task `{}` executed by `oxa work`.\n\nAcceptance result: `{}`.\n\nThe runner retained the task worktree and saved its changes.\n", task.id, if draft { "red" } else { "done" }); + std::fs::write(&body, text).map_err(|e| e.to_string())?; + let body_arg = body.to_str().ok_or("non-UTF8 PR body path")?; + let result = if let Some(pr) = prs.first() { + let number = pr["number"].as_u64().ok_or("PR number missing")?.to_string(); + gh( + tree, + &lock, + &["pr", "edit", &number, "--repo", name, "--body-file", body_arg], + )?; + let was_draft = pr["isDraft"].as_bool().ok_or("PR draft state missing")?; + if was_draft != draft { + let mut args = vec!["pr", "ready", &number, "--repo", name]; + if draft { + args.push("--undo"); + } + gh(tree, &lock, &args)?; + } + pr["url"].as_str().ok_or("PR URL missing")?.to_string() + } else { + let title = format!("oxa work: {}", task.id); + let mut args = vec![ + "pr", + "create", + "--repo", + name, + "--head", + &task.branch, + "--base", + &base, + "--title", + &title, + "--body-file", + body_arg, + ]; + if draft { + args.push("--draft"); + } + gh(tree, &lock, &args)? + }; + let _ = std::fs::remove_file(body); + Ok((result.trim().to_string(), draft)) + })(); + match result { + Ok((url, draft)) => { + value.record.pull_request = Some(url); + value.record.draft = draft; + value.record.outcome = accepted; + value.record.detail.clear(); + state::save(&path, &mut value, Step::Finished)?; + } + Err(error) => { + value.record.outcome = Outcome::Blocked; + value.record.detail = format!("publishing: {error}"); + state::save(&path, &mut value, Step::Publishing)?; + } + } + Ok(value.record) +} +pub fn report_notes( + repo: &Path, + task: &Task, + offline: bool, + reporter: &mut dyn FnMut(&super::ProblemNote, &File, bool) -> Result, +) -> Result { + let (path, lock, mut value) = context(repo, task)?; + value.offline = offline; + let prior_step = value.record.step; + for index in 0..value.record.problem_notes.len() { + if value + .record + .reports + .get(index) + .is_some_and(|r| r.disposition != "local") + { + continue; + } + state::save(&path, &mut value, Step::Reporting)?; + let report = reporter(&value.record.problem_notes[index], &lock, value.offline)?; + if index < value.record.reports.len() { + value.record.reports[index] = report; + } else { + value.record.reports.push(report); + } + state::save(&path, &mut value, Step::Reporting)?; + } + state::save( + &path, + &mut value, + if prior_step == Step::Publishing { + Step::Publishing + } else { + Step::Finished + }, + )?; + Ok(value.record) +} diff --git a/oxa/oxa-work/src/state.rs b/oxa/oxa-work/src/state.rs index ff0b433e9..81a86825d 100644 --- a/oxa/oxa-work/src/state.rs +++ b/oxa/oxa-work/src/state.rs @@ -17,6 +17,8 @@ pub enum Step { Recheck, Saving, Finished, + Publishing, + Reporting, } #[derive(Debug, Deserialize, Serialize)] @@ -35,6 +37,7 @@ pub(crate) struct State { pub task: Task, pub checkpoint: Option, pub offline: bool, + pub acceptance_outcome: Option, } pub(crate) fn now() -> u64 { SystemTime::now() From 8bf1f38bbcd28502a7327ed4deadd18698a6ff3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:22:36 +0000 Subject: [PATCH 10/52] Preserve execution privacy and finish failed reporting checkpoints --- oxa/docs/work.md | 7 +- oxa/oxa-cli/src/work_cmd.rs | 5 +- oxa/oxa-cli/tests/work_publish.rs | 115 ++++++++++++++++++++++++++++++ oxa/oxa-work/src/lib.rs | 14 +++- oxa/oxa-work/src/publish.rs | 13 ++-- oxa/oxa-work/src/state.rs | 1 + 6 files changed, 144 insertions(+), 11 deletions(-) diff --git a/oxa/docs/work.md b/oxa/docs/work.md index 14a62278c..90786b4ab 100644 --- a/oxa/docs/work.md +++ b/oxa/docs/work.md @@ -85,6 +85,8 @@ The runner checks existing PR queue membership before every push and refuses a queued PR; use a new task branch for follow-up work. It refuses branch changes made after acceptance until `--rerun` checks them. PR bodies include task ID and acceptance state, without raw prompts, transcripts or failed-check output. +Execution preserves a saved offline restriction across restarts; later explicit +publication uses the current network policy without changing that restriction. A publication failure keeps a publication checkpoint; another `--publish` retries without replaying the agent. Offline mode refuses network publication. @@ -99,5 +101,6 @@ It comments only new notes on a matching issue, links an existing fixing PR, or files a new report with related links. Search failure, offline mode or missing report configuration keeps a local scrubbed draft and records its path/detail. A later explicit `--report-problems` retries local reports without replaying the -agent. No notes are filed unless this flag is present; model output and acceptance -logs are never treated as report bodies. +agent. A draft-write failure records a failed report and finishes the task checkpoint; +`--report-problems` can retry it. Reports contain only scrubbed structured symptom +summaries, without raw assistant output or acceptance logs. diff --git a/oxa/oxa-cli/src/work_cmd.rs b/oxa/oxa-cli/src/work_cmd.rs index 0ac619a6b..8f6306b9b 100644 --- a/oxa/oxa-cli/src/work_cmd.rs +++ b/oxa/oxa-cli/src/work_cmd.rs @@ -89,10 +89,7 @@ pub(crate) fn run( }) .unwrap_or_else(|detail| oxa_work::blocked_record(&task, detail)); } - if matches!( - record.outcome, - oxa_work::Outcome::Red | oxa_work::Outcome::Blocked - ) { + if record.failed() { code = 1; } serde_json::to_writer(&mut *out, &record).map_err(|e| e.to_string())?; diff --git a/oxa/oxa-cli/tests/work_publish.rs b/oxa/oxa-cli/tests/work_publish.rs index ff0c2ae78..c1ae16f20 100644 --- a/oxa/oxa-cli/tests/work_publish.rs +++ b/oxa/oxa-cli/tests/work_publish.rs @@ -254,3 +254,118 @@ fn a_local_report_retries_without_replaying_the_completed_task() { assert_eq!(retry["reports"][0]["disposition"], "filed", "{retry}"); assert_eq!(retry["step"], "finished"); } + +#[test] +fn unfinished_execution_keeps_its_saved_offline_restriction() { + use oxa_fake::{FakeModel, FakeOpts, FakeServer}; + let env = TestEnv::new(); + setup(&env, "[]"); + run_args( + &env, + vec![ScriptTurn::text("done")], + &["--offline", "work", "../tasks.json"], + None, + ); + let state_path = env.work().join("project/.git/oxa-work/test.json"); + let mut state: serde_json::Value = serde_json::from_slice(&fs::read(&state_path).unwrap()).unwrap(); + state["step"] = "agent".into(); + state["outcome"] = "running".into(); + fs::write(&state_path, serde_json::to_vec(&state).unwrap()).unwrap(); + let server = FakeServer::start( + vec![ScriptTurn::text("network was allowed")], + FakeOpts { + remote: Some("openrouter"), + models: vec![FakeModel { + alias: "fake-model".into(), + cap: 64000, + max_output: Some(4096), + caps: None, + }], + ..FakeOpts::default() + }, + ) + .unwrap(); + let out = Command::new(env!("CARGO_BIN_EXE_oxa")) + .args(["work", "../tasks.json"]) + .current_dir(env.work().join("project")) + .env_clear() + .envs(env.envs(&server.url())) + .output() + .unwrap(); + let record: serde_json::Value = serde_json::from_slice(&out.stdout).unwrap(); + assert_eq!(record["outcome"], "blocked", "{record}"); + assert!(record["detail"].as_str().unwrap().contains("offline")); + assert!(server.requests().is_empty()); +} +#[test] +fn later_publication_and_reporting_do_not_erase_execution_policy() { + let env = TestEnv::new(); + setup(&env, "[]"); + let path = configure(&env); + run_args( + &env, + edits(), + &["--offline", "work", "../tasks.json"], + Some(path.clone()), + ); + let state_path = env.work().join("project/.git/oxa-work/test.json"); + for flag in ["--publish", "--report-problems"] { + let record = run_args(&env, vec![], &["work", "../tasks.json", flag], Some(path.clone())); + assert_eq!(record["outcome"], "done", "{record}"); + let state: serde_json::Value = serde_json::from_slice(&fs::read(&state_path).unwrap()).unwrap(); + assert_eq!(state["offline"], true, "{flag} erased execution policy"); + } +} +#[test] +fn optional_publication_metadata_defaults_to_inactive() { + let env = TestEnv::new(); + setup(&env, "[]"); + run(&env, vec![ScriptTurn::text("done")]); + let state_path = env.work().join("project/.git/oxa-work/test.json"); + let mut state: serde_json::Value = serde_json::from_slice(&fs::read(&state_path).unwrap()).unwrap(); + for key in [ + "head", + "pull_request", + "draft", + "problem_notes", + "reports", + "acceptance_outcome", + ] { + state.as_object_mut().unwrap().remove(key); + } + fs::write(&state_path, serde_json::to_vec(&state).unwrap()).unwrap(); + let record = run(&env, vec![]); + assert_eq!(record["outcome"], "nothing_to_do", "{record}"); + assert_eq!(record["draft"], false); + assert_eq!(record["reports"], serde_json::json!([])); +} +#[test] +fn failed_draft_creation_does_not_strand_reporting_checkpoint() { + let env = TestEnv::new(); + setup(&env, "[]"); + let path = configure(&env); + env.write( + "home/.oxa/agent/issue-drafts", + "a file blocks the draft directory", + ); + let text="[{\"title\":\"oxa work symptom\",\"notes\":\"Synthetic reproduction\",\"keywords\":[\"oxa work\",\"synthetic symptom\"]}]"; + let first = run_args( + &env, + vec![ScriptTurn::text(text)], + &["work", "../tasks.json", "--report-problems"], + Some(path.clone()), + ); + assert_eq!(first["step"], "finished", "{first}"); + assert_eq!(first["reports"][0]["disposition"], "failed"); + let plain = run_args(&env, vec![], &["work", "../tasks.json"], Some(path.clone())); + assert_eq!(plain["step"], "finished"); + fs::remove_file(env.home().join(".oxa/agent/issue-drafts")).unwrap(); + let retry = run_args( + &env, + vec![], + &["work", "../tasks.json", "--report-problems"], + Some(path), + ); + assert_eq!(retry["reports"][0]["disposition"], "local"); + assert_eq!(retry["step"], "finished"); +} diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index b29965cac..794d485f3 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -47,13 +47,25 @@ pub struct Record { pub detail: String, pub step: Step, pub progress_at: u64, + #[serde(default)] pub head: Option, + #[serde(default)] pub pull_request: Option, + #[serde(default)] pub draft: bool, + #[serde(default)] pub problem_notes: Vec, + #[serde(default)] pub reports: Vec, } +impl Record { + pub fn failed(&self) -> bool { + matches!(self.outcome, Outcome::Red | Outcome::Blocked) + || self.reports.iter().any(|r| r.disposition == "failed") + } +} + fn output(mut command: Command) -> Result { command.stdin(Stdio::null()).output().map_err(|e| e.to_string()) } @@ -268,7 +280,7 @@ pub fn run_task_with_options( } else { new_state(task, tree, offline) }; - state.offline = offline; + state.offline |= offline; let step = state.record.step; state::save(&receipt, &mut state, step)?; if let Err(error) = execute::execute(&root, binary, task, &receipt, &mut state, &lock) { diff --git a/oxa/oxa-work/src/publish.rs b/oxa/oxa-work/src/publish.rs index ee0fe06ba..1e8eecf6f 100644 --- a/oxa/oxa-work/src/publish.rs +++ b/oxa/oxa-work/src/publish.rs @@ -45,7 +45,6 @@ pub fn publish(repo: &Path, task: &Task, offline: bool) -> Result { return Ok(value.record); } value.acceptance_outcome = Some(accepted); - value.offline = offline; state::save(&path, &mut value, Step::Publishing)?; let result = (|| -> Result<(String, bool)> { if offline { @@ -188,19 +187,25 @@ pub fn report_notes( reporter: &mut dyn FnMut(&super::ProblemNote, &File, bool) -> Result, ) -> Result { let (path, lock, mut value) = context(repo, task)?; - value.offline = offline; let prior_step = value.record.step; for index in 0..value.record.problem_notes.len() { if value .record .reports .get(index) - .is_some_and(|r| r.disposition != "local") + .is_some_and(|r| !matches!(r.disposition.as_str(), "local" | "failed")) { continue; } state::save(&path, &mut value, Step::Reporting)?; - let report = reporter(&value.record.problem_notes[index], &lock, value.offline)?; + let report = reporter(&value.record.problem_notes[index], &lock, offline).unwrap_or_else(|detail| { + super::ReportRecord { + disposition: "failed".into(), + draft: PathBuf::new(), + url: None, + detail, + } + }); if index < value.record.reports.len() { value.record.reports[index] = report; } else { diff --git a/oxa/oxa-work/src/state.rs b/oxa/oxa-work/src/state.rs index 81a86825d..fae423fc1 100644 --- a/oxa/oxa-work/src/state.rs +++ b/oxa/oxa-work/src/state.rs @@ -37,6 +37,7 @@ pub(crate) struct State { pub task: Task, pub checkpoint: Option, pub offline: bool, + #[serde(default)] pub acceptance_outcome: Option, } pub(crate) fn now() -> u64 { From 5c227ed702d0c2121bc17608f5c9cd151347bd1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:44:12 +0000 Subject: [PATCH 11/52] Run oldest queued tasks with bounded workers, disk checks and monitor state (#333) --- oxa/docs/work.md | 30 ++ oxa/oxa-cli/src/args.rs | 2 +- oxa/oxa-cli/src/work_cmd.rs | 159 +++++++--- oxa/oxa-cli/src/work_reports.rs | 12 + oxa/oxa-cli/tests/common/work.rs | 39 +-- oxa/oxa-cli/tests/common/work_repo.rs | 36 +++ oxa/oxa-cli/tests/work_queue.rs | 405 ++++++++++++++++++++++++++ oxa/oxa-work/src/execute.rs | 2 +- oxa/oxa-work/src/lib.rs | 28 +- oxa/oxa-work/src/queue.rs | 320 ++++++++++++++++++++ oxa/oxa-work/src/state.rs | 9 +- oxa/oxa-work/src/tree.rs | 17 +- 12 files changed, 964 insertions(+), 95 deletions(-) create mode 100644 oxa/oxa-cli/tests/common/work_repo.rs create mode 100644 oxa/oxa-cli/tests/work_queue.rs create mode 100644 oxa/oxa-work/src/queue.rs diff --git a/oxa/docs/work.md b/oxa/docs/work.md index 90786b4ab..a08a07615 100644 --- a/oxa/docs/work.md +++ b/oxa/docs/work.md @@ -104,3 +104,33 @@ A later explicit `--report-problems` retries local reports without replaying the agent. A draft-write failure records a failed report and finishes the task checkpoint; `--report-problems` can retry it. Reports contain only scrubbed structured symptom summaries, without raw assistant output or acceptance logs. + + +Queue mode watches a directory of regular, non-hidden `*.json` task files. +After adding task files, run +`oxa work --queue ./tasks --max 2 --min-free-mib 1024`. + +Files are admitted oldest modification time first (filename breaks ties), with +array order preserved inside a file. At most `--max` tasks run; surviving workers +from an interrupted controller still occupy slots until their locks release. +Task IDs must be unique across the queue. Completed task state prevents agent +replay on restart. Files remain available for inspection and manual archiving. +Write new files with an atomic rename so the runner reads a complete task file. + +The default free-space floor is 512 MiB. The runner checks the repository, common +git directory, worktree storage and queue filesystems before each admission. A low +or unreadable free-space value pauses new work while active jobs finish. Invalid +JSON files report an error and do not skip other work. `--once` drains available +work and exits; when disk space blocks pending tasks it exits 1 with their files +retained. Without `--once`, the directory stays under watch for new files. + +Queue mode supports `--publish`, `--report-problems` and `--rerun` with the same +explicit policy as single-file mode. Its worker threads preserve the parent's +offline policy. Standard output emits JSON lines tagged `state`, `task` or `error`. +The latest state line is also replaced atomically at +`/.oxa-queue-status.json` for a monitor. It contains a heartbeat timestamp, +active task IDs, pending/finished/failed counts, free bytes and disk-pause details. +A controller lock prevents two queue controllers from exceeding the worker cap. + +Problem-report workflows hold a per-user kernel lease across drafting, duplicate +search, match reads and writing, so concurrent workers share one reporting lane. diff --git a/oxa/oxa-cli/src/args.rs b/oxa/oxa-cli/src/args.rs index 345b5700c..72de10363 100644 --- a/oxa/oxa-cli/src/args.rs +++ b/oxa/oxa-cli/src/args.rs @@ -177,7 +177,7 @@ const COMMANDS: &[(&str, &str)] = &[ "Draft a scrubbed ox-llm issue; submit posts it only after you type yes", ), ( - "oxa work [--rerun] [--publish] [--report-problems] | --status", + "oxa work [options] | --queue DIR --max N | --status", "Run tasks in isolated worktrees with external acceptance checks", ), ( diff --git a/oxa/oxa-cli/src/work_cmd.rs b/oxa/oxa-cli/src/work_cmd.rs index 8f6306b9b..1ba4c87ce 100644 --- a/oxa/oxa-cli/src/work_cmd.rs +++ b/oxa/oxa-cli/src/work_cmd.rs @@ -26,21 +26,52 @@ pub(crate) fn run( } return Ok(0); } - let mut end = args.len(); let (mut rerun, mut publish, mut reports) = (false, false, false); - while let Some(flag) = args.get(end.wrapping_sub(1)) { - match flag.as_str() { - "--rerun" => rerun = true, - "--publish" => publish = true, - "--report-problems" => reports = true, - _ => break, - } - end -= 1; - } - let args = &args[..end]; + let args: Vec = args + .iter() + .filter(|word| match word.as_str() { + "--rerun" => { + rerun = true; + false + } + "--publish" => { + publish = true; + false + } + "--report-problems" => { + reports = true; + false + } + _ => true, + }) + .cloned() + .collect(); + let args = args.as_slice(); if (publish || reports) && env.get("OXA_CODING_AGENT").is_some() { return Err("publishing and problem reporting must be explicitly launched by the user outside an agent tool".into()); } + let context = std::sync::Arc::new(Context { + cwd: cwd.to_path_buf(), + dirs: dirs.clone(), + env: env.clone(), + binary: std::env::current_exe().map_err(|e| e.to_string())?, + offline: oxa_proto::offline(), + rerun, + publish, + reports, + }); + if args.first().is_some_and(|s| s == "--queue") { + let (directory, options) = queue_options(args, cwd)?; + let worker = { + let context = context.clone(); + std::sync::Arc::new(move |task: &oxa_work::Task| context.task(task)) + }; + let summary = oxa_work::run_queue(cwd, &directory, &options, worker, &mut |event| { + serde_json::to_writer(&mut *out, &event).map_err(|e| e.to_string())?; + writeln!(out).and_then(|_| out.flush()).map_err(|e| e.to_string()) + })?; + return Ok(i32::from(summary.failed > 0 || summary.pending > 0)); + } let tasks = match args { [file] if !file.starts_with('-') => oxa_work::read_tasks(&cwd.join(file))?, [flag, number, model_flag, model] if flag == "--issue" && model_flag == "--model" => { @@ -55,40 +86,14 @@ pub(crate) fn run( } _ => { return Err( - "usage: oxa work | --issue N --model MODEL | --status; task runs accept --rerun, --publish, --report-problems" + "usage: oxa work | --issue N --model MODEL | --queue DIR --max N [--once] [--min-free-mib N] | --status; task runs accept --rerun, --publish, --report-problems" .into(), ) } }; - let binary: PathBuf = std::env::current_exe().map_err(|e| e.to_string())?; let mut code = 0; for task in tasks { - let mut record = - oxa_work::run_task_with_options(cwd, &binary, &task, rerun, oxa_proto::offline()) - .unwrap_or_else(|detail| oxa_work::blocked_record(&task, detail)); - if publish && !record.worktree.as_os_str().is_empty() { - record = oxa_work::publish(cwd, &task, oxa_proto::offline()) - .unwrap_or_else(|detail| oxa_work::blocked_record(&task, detail)); - } - if reports && !record.worktree.as_os_str().is_empty() { - let settings = oxa_config::Settings::load(dirs, cwd, false); - let ctx = crate::report_cmd::ScrubCtx::from_env(env, dirs); - record = - oxa_work::report_notes(cwd, &task, oxa_proto::offline(), &mut |note, lock, offline| { - let mut runner = |program: &str, args: &[String]| { - crate::work_reports::process(program, args, cwd, lock) - }; - crate::work_reports::report( - note, - dirs, - &settings, - &ctx, - offline || oxa_proto::offline(), - &mut runner, - ) - }) - .unwrap_or_else(|detail| oxa_work::blocked_record(&task, detail)); - } + let record = context.task(&task); if record.failed() { code = 1; } @@ -105,3 +110,79 @@ pub(crate) fn run( } } } + +struct Context { + cwd: PathBuf, + dirs: oxa_config::Dirs, + env: oxa_config::Env, + binary: PathBuf, + offline: bool, + rerun: bool, + publish: bool, + reports: bool, +} +impl Context { + fn task(&self, task: &oxa_work::Task) -> oxa_work::Record { + let mut record = + oxa_work::run_task_with_options(&self.cwd, &self.binary, task, self.rerun, self.offline) + .unwrap_or_else(|detail| oxa_work::blocked_record(task, detail)); + if self.publish && !record.worktree.as_os_str().is_empty() { + record = oxa_work::publish(&self.cwd, task, self.offline) + .unwrap_or_else(|detail| oxa_work::blocked_record(task, detail)); + } + if self.reports && !record.worktree.as_os_str().is_empty() { + let settings = oxa_config::Settings::load(&self.dirs, &self.cwd, false); + let ctx = crate::report_cmd::ScrubCtx::from_env(&self.env, &self.dirs); + record = oxa_work::report_notes(&self.cwd, task, self.offline, &mut |note, lock, offline| { + let mut runner = |program: &str, args: &[String]| { + crate::work_reports::process(program, args, &self.cwd, lock) + }; + crate::work_reports::report( + note, + &self.dirs, + &settings, + &ctx, + offline || self.offline, + &mut runner, + ) + }) + .unwrap_or_else(|detail| oxa_work::blocked_record(task, detail)); + } + record + } +} +fn queue_options(args: &[String], cwd: &Path) -> Result<(PathBuf, oxa_work::QueueOptions), String> { + let directory = cwd.join(args.get(1).ok_or("--queue needs a directory")?); + let mut options = oxa_work::QueueOptions::default(); + let mut i = 2; + while i < args.len() { + match args[i].as_str() { + "--max" => { + i += 1; + options.max = args + .get(i) + .ok_or("--max needs a value")? + .parse() + .map_err(|_| "invalid --max")?; + } + "--min-free-mib" => { + i += 1; + let mib: u64 = args + .get(i) + .ok_or("--min-free-mib needs a value")? + .parse() + .map_err(|_| "invalid free-space minimum")?; + options.min_free_bytes = mib + .checked_mul(1024 * 1024) + .ok_or("free-space minimum is too large")?; + } + "--once" => options.once = true, + word => return Err(format!("unknown queue option: {word}")), + } + i += 1; + } + if options.max == 0 { + return Err("--max must be positive".into()); + } + Ok((directory, options)) +} diff --git a/oxa/oxa-cli/src/work_reports.rs b/oxa/oxa-cli/src/work_reports.rs index d5cd7e763..57fdd069c 100644 --- a/oxa/oxa-cli/src/work_reports.rs +++ b/oxa/oxa-cli/src/work_reports.rs @@ -83,6 +83,18 @@ pub(crate) fn report( offline: bool, runner: Runner<'_>, ) -> Result { + // Queue workers share one per-user reporting lease across draft/search/read/write. + use std::os::unix::fs::OpenOptionsExt; + std::fs::create_dir_all(&dirs.agent_dir).map_err(|e| e.to_string())?; + let gate = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .mode(0o600) + .open(dirs.agent_dir.join("work-report.lock")) + .map_err(|e| e.to_string())?; + gate.lock().map_err(|e| e.to_string())?; let (draft, title, body) = save_draft(¬e.title, ¬e.notes, dirs, ctx, runner).map_err(|e| e.to_string())?; let mut result = ReportRecord { diff --git a/oxa/oxa-cli/tests/common/work.rs b/oxa/oxa-cli/tests/common/work.rs index 44062a99f..a14c6cdd7 100644 --- a/oxa/oxa-cli/tests/common/work.rs +++ b/oxa/oxa-cli/tests/common/work.rs @@ -1,39 +1,8 @@ use oxa_fake::{FakeOpts, FakeServer, ScriptTurn, TestEnv}; -use std::{path::Path, process::Command}; -pub fn git(env: &TestEnv, dir: &Path, args: &[&str]) -> String { - let out = Command::new("git") - .args(args) - .current_dir(dir) - .env_clear() - .envs(env.envs("http://127.0.0.1:1")) - .env("GIT_AUTHOR_NAME", "Test") - .env("GIT_AUTHOR_EMAIL", "test@example.invalid") - .env("GIT_COMMITTER_NAME", "Test") - .env("GIT_COMMITTER_EMAIL", "test@example.invalid") - .output() - .unwrap(); - assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); - String::from_utf8_lossy(&out.stdout).trim().into() -} -pub fn setup(env: &TestEnv, done: &str) { - git(env, &env.work(), &["init", "--bare", "remote.git"]); - git(env, &env.work(), &["init", "-b", "main", "project"]); - let project = env.work().join("project"); - git(env, &project, &["config", "user.name", "Test"]); - git(env, &project, &["config", "user.email", "test@example.invalid"]); - env.write( - "work/project/.oxa/settings.json", - &format!(r#"{{"done":{done}}}"#), - ); - git(env, &project, &["add", "."]); - git(env, &project, &["commit", "-m", "fixture"]); - git(env, &project, &["remote", "add", "origin", "../remote.git"]); - git(env, &project, &["push", "origin", "main"]); - env.write( - "work/tasks.json", - r#"[{"id":"test","branch":"agent/test","model":"fake","prompt":"Finish task"}]"#, - ); -} +use std::process::Command; +#[path = "work_repo.rs"] +mod repo; +pub use repo::{git, setup}; pub fn run(env: &TestEnv, turns: Vec) -> serde_json::Value { run_args(env, turns, &["work", "../tasks.json"], None) } diff --git a/oxa/oxa-cli/tests/common/work_repo.rs b/oxa/oxa-cli/tests/common/work_repo.rs new file mode 100644 index 000000000..a75103727 --- /dev/null +++ b/oxa/oxa-cli/tests/common/work_repo.rs @@ -0,0 +1,36 @@ +use oxa_fake::TestEnv; +use std::{path::Path, process::Command}; +pub fn git(env: &TestEnv, dir: &Path, args: &[&str]) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .env_clear() + .envs(env.envs("http://127.0.0.1:1")) + .env("GIT_AUTHOR_NAME", "Test") + .env("GIT_AUTHOR_EMAIL", "test@example.invalid") + .env("GIT_COMMITTER_NAME", "Test") + .env("GIT_COMMITTER_EMAIL", "test@example.invalid") + .output() + .unwrap(); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + String::from_utf8_lossy(&out.stdout).trim().into() +} +pub fn setup(env: &TestEnv, done: &str) { + git(env, &env.work(), &["init", "--bare", "remote.git"]); + git(env, &env.work(), &["init", "-b", "main", "project"]); + let project = env.work().join("project"); + git(env, &project, &["config", "user.name", "Test"]); + git(env, &project, &["config", "user.email", "test@example.invalid"]); + env.write( + "work/project/.oxa/settings.json", + &format!(r#"{{"done":{done}}}"#), + ); + git(env, &project, &["add", "."]); + git(env, &project, &["commit", "-m", "fixture"]); + git(env, &project, &["remote", "add", "origin", "../remote.git"]); + git(env, &project, &["push", "origin", "main"]); + env.write( + "work/tasks.json", + r#"[{"id":"test","branch":"agent/test","model":"fake","prompt":"Finish task"}]"#, + ); +} diff --git a/oxa/oxa-cli/tests/work_queue.rs b/oxa/oxa-cli/tests/work_queue.rs new file mode 100644 index 000000000..5d4388b4b --- /dev/null +++ b/oxa/oxa-cli/tests/work_queue.rs @@ -0,0 +1,405 @@ +//! Queue behavior against real worktrees and a fake engine, with no external network. +#[path = "common/work_repo.rs"] +mod work; +use oxa_fake::{FakeOpts, FakeServer, ScriptTurn, TestEnv}; +use std::{ + fs, + process::{Command, Output}, + time::{Duration, Instant, SystemTime}, +}; +use work::{git, setup}; +fn task_file(env: &TestEnv, file: &str, id: &str, age: u64) { + let path = env.write(&format!("work/queue/{file}.json"), &serde_json::json!([{"id":id,"model":"fake","branch":format!("agent/{id}"),"prompt":format!("{id} task")}]).to_string()); + fs::OpenOptions::new() + .write(true) + .open(path) + .unwrap() + .set_modified(SystemTime::now() - Duration::from_secs(age)) + .unwrap(); +} +fn command(env: &TestEnv, url: &str, max: &str) -> Command { + let mut c = Command::new(env!("CARGO_BIN_EXE_oxa")); + c.args([ + "work", + "--queue", + "../queue", + "--max", + max, + "--min-free-mib", + "0", + "--once", + ]) + .current_dir(env.work().join("project")) + .env_clear() + .envs(env.envs(url)); + c +} +fn finish(mut c: Command) -> Output { + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send(c.output().unwrap()); + }); + rx.recv_timeout(Duration::from_secs(30)).expect("queue finished") +} +fn rows(out: &Output) -> Vec { + assert!(out.status.code().is_some(), "queue terminated"); + String::from_utf8_lossy(&out.stdout) + .lines() + .map(|line| { + serde_json::from_str(line) + .unwrap_or_else(|e| panic!("{e}: {line} / {}", String::from_utf8_lossy(&out.stderr))) + }) + .collect() +} +fn until(mut f: impl FnMut() -> bool) { + let deadline = Instant::now() + Duration::from_secs(15); + while !f() { + assert!(Instant::now() < deadline); + std::thread::sleep(Duration::from_millis(10)); + } +} +#[test] +fn oldest_file_runs_first_and_restart_skips_completed_agents() { + let env = TestEnv::new(); + setup(&env, "[]"); + task_file(&env, "a-new", "new", 10); + task_file(&env, "z-old", "old", 20); + let server = FakeServer::start( + vec![ScriptTurn::text("done"), ScriptTurn::text("done")], + FakeOpts::default(), + ) + .unwrap(); + let out = finish(command(&env, &server.url(), "1")); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + let messages = server + .requests() + .into_iter() + .filter(|r| r.max_tokens > 0) + .collect::>(); + assert_eq!(messages.len(), 2); + assert!(messages[0] + .messages + .iter() + .any(|m| m.content.as_deref().is_some_and(|s| s.contains("old task")))); + let events = rows(&out); + let ids: Vec<_> = events + .iter() + .filter(|r| r["event"] == "task") + .map(|r| r["id"].as_str().unwrap()) + .collect(); + assert_eq!(ids, ["old", "new"]); + let cached = finish(command(&env, &server.url(), "1")); + assert!(cached.status.success()); + assert_eq!( + server.requests().into_iter().filter(|r| r.max_tokens > 0).count(), + 2 + ); +} +#[test] +fn concurrency_is_bounded_and_monitor_snapshot_is_a_complete_line() { + let env = TestEnv::new(); + setup(&env, "[]"); + for (index, id) in ["a", "b", "c"].iter().enumerate() { + task_file(&env, id, id, index as u64); + } + let server = FakeServer::start( + vec![ScriptTurn::text("done"); 3], + FakeOpts { + hold_headers_ms: 300, + ..FakeOpts::default() + }, + ) + .unwrap(); + let out = finish(command(&env, &server.url(), "2")); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + let events = rows(&out); + let active: Vec<_> = events + .iter() + .filter(|r| r["event"] == "state") + .map(|r| r["active"].as_array().unwrap().len()) + .collect(); + assert!(active.contains(&2)); + assert!(active.iter().all(|n| *n <= 2)); + let line = fs::read_to_string(env.work().join("queue/.oxa-queue-status.json")).unwrap(); + assert_eq!(line.lines().count(), 1); + let status: serde_json::Value = serde_json::from_str(&line).unwrap(); + assert_eq!(status["finished"], 3); + assert_eq!(status["active"], serde_json::json!([])); +} +#[test] +fn low_disk_never_starts_a_task_and_leaves_the_file_pending() { + use std::os::unix::fs::PermissionsExt; + let env = TestEnv::new(); + setup(&env, "[]"); + task_file(&env, "task", "low", 1); + let df = env.write("tmp/df", "#!/bin/sh\nprintf 'Avail\\n0\\n'\n"); + fs::set_permissions(df, fs::Permissions::from_mode(0o755)).unwrap(); + let server = FakeServer::start(vec![], FakeOpts::default()).unwrap(); + let mut cmd = command(&env, &server.url(), "2"); + cmd.args(["--min-free-mib", "1"]).env( + "PATH", + format!("{}:{}", env.tmp().display(), std::env::var("PATH").unwrap()), + ); + let out = finish(cmd); + assert_eq!(out.status.code(), Some(1)); + assert!(server.requests().is_empty()); + let events = rows(&out); + let status = events.last().unwrap(); + assert_eq!(status["disk_blocked"], true); + assert_eq!(status["pending"], 1); + assert!(!env.work().join("project/.git/oxa-work/low.json").exists()); + assert!(env.work().join("queue/task.json").exists()); +} +#[test] +fn invalid_file_does_not_skip_later_valid_work() { + let env = TestEnv::new(); + setup(&env, "[]"); + env.write("work/queue/bad.json", "invalid"); + task_file(&env, "good", "good", 1); + let server = FakeServer::start(vec![ScriptTurn::text("done")], FakeOpts::default()).unwrap(); + let out = finish(command(&env, &server.url(), "2")); + assert_eq!(out.status.code(), Some(1)); + let events = rows(&out); + assert!(events.iter().any(|r| r["event"] == "error")); + assert!(events.iter().any(|r| r["event"] == "task" && r["id"] == "good")); +} +#[test] +fn a_surviving_worker_counts_against_max_until_its_lock_releases() { + let env = TestEnv::new(); + setup(&env, "[]"); + task_file(&env, "held", "held", 20); + task_file(&env, "next", "next", 10); + env.write("work/project/.git/oxa-work/held.lock", ""); + let file = fs::OpenOptions::new() + .read(true) + .write(true) + .open(env.work().join("project/.git/oxa-work/held.lock")) + .unwrap(); + file.try_lock().unwrap(); + let server = FakeServer::start(vec![ScriptTurn::text("done"); 2], FakeOpts::default()).unwrap(); + let mut child = command(&env, &server.url(), "1") + .stdout(std::process::Stdio::null()) + .spawn() + .unwrap(); + until(|| { + fs::read(env.work().join("queue/.oxa-queue-status.json")) + .ok() + .and_then(|b| serde_json::from_slice::(&b).ok()) + .is_some_and(|v| v["active"] == serde_json::json!(["held"])) + }); + assert!(server.requests().is_empty()); + drop(file); + assert!(child.wait().unwrap().success()); + git( + &env, + &env.work().join("project"), + &["show-ref", "--verify", "refs/heads/agent/next"], + ); +} +#[test] +fn files_arriving_while_a_worker_runs_are_picked_up_before_once_exits() { + let env = TestEnv::new(); + setup(&env, "[]"); + task_file(&env, "first", "first", 1); + let server = FakeServer::start( + vec![ScriptTurn::text("done"); 2], + FakeOpts { + hold_headers_ms: 400, + ..FakeOpts::default() + }, + ) + .unwrap(); + let mut child = command(&env, &server.url(), "1") + .stdout(std::process::Stdio::null()) + .spawn() + .unwrap(); + until(|| !server.requests().is_empty()); + task_file(&env, "arrived", "arrived", 1); + assert!(child.wait().unwrap().success()); + let path = env.work().join("project/.git/oxa-work/arrived.json"); + assert!(path.exists()); +} + +#[test] +fn invalid_max_is_rejected_without_starting_workers() { + let env = TestEnv::new(); + setup(&env, "[]"); + task_file(&env, "one", "one", 1); + let server = FakeServer::start(vec![], FakeOpts::default()).unwrap(); + let out = finish(command(&env, &server.url(), "0")); + assert_eq!(out.status.code(), Some(2)); + assert!(server.requests().is_empty()); + assert!(!env.work().join("queue/.oxa-queue-status.json").exists()); +} +#[test] +fn queued_workers_keep_the_parent_command_line_offline_policy() { + let env = TestEnv::new(); + setup(&env, "[]"); + task_file(&env, "remote", "remote", 1); + let server = FakeServer::start( + vec![ScriptTurn::text("should not be requested")], + FakeOpts { + remote: Some("openrouter"), + models: vec![oxa_fake::FakeModel { + alias: "fake-model".into(), + cap: 64000, + max_output: Some(4096), + caps: None, + }], + ..FakeOpts::default() + }, + ) + .unwrap(); + let mut c = Command::new(env!("CARGO_BIN_EXE_oxa")); + c.args([ + "--offline", + "work", + "--queue", + "../queue", + "--max", + "2", + "--once", + "--min-free-mib", + "0", + ]) + .current_dir(env.work().join("project")) + .env_clear() + .envs(env.envs(&server.url())); + let out = finish(c); + assert_eq!(out.status.code(), Some(1)); + assert!(server.requests().is_empty()); + let events = rows(&out); + assert!(events.iter().any(|r| r["event"] == "task" + && r["outcome"] == "blocked" + && r["detail"].as_str().is_some_and(|s| s.contains("offline")))); +} + +#[test] +fn preparing_git_keeps_ownership_after_a_queue_controller_is_killed() { + use std::os::unix::fs::PermissionsExt; + struct Release { + release: std::path::PathBuf, + done: std::path::PathBuf, + } + impl Drop for Release { + fn drop(&mut self) { + let _ = fs::write(&self.release, "release"); + let end = Instant::now() + Duration::from_secs(2); + while !self.done.exists() && Instant::now() < end { + std::thread::sleep(Duration::from_millis(10)); + } + } + } + let env = TestEnv::new(); + setup(&env, "[]"); + task_file(&env, "preparing", "preparing", 1); + let release = Release { + release: env.home().join("release-prep"), + done: env.home().join("prep-done"), + }; + let hook = env.write( + "work/project/.git/hooks/post-checkout", + r#"#!/bin/sh +touch "$HOME/prep-started" +attempt=0 +while ! test -f "$HOME/release-prep"; do + attempt=$((attempt + 1)) + if test "$attempt" -ge 500; then exit 9; fi + sleep 0.01 +done +touch "$HOME/prep-done" +"#, + ); + fs::set_permissions(hook, fs::Permissions::from_mode(0o755)).unwrap(); + let server = FakeServer::start(vec![ScriptTurn::text("resumed")], FakeOpts::default()).unwrap(); + let mut child = command(&env, &server.url(), "1") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .unwrap(); + until(|| env.home().join("prep-started").exists()); + child.kill().unwrap(); + child.wait().unwrap(); + let lock_path = env.work().join("project/.git/oxa-work/preparing.lock"); + let probe = fs::OpenOptions::new() + .read(true) + .write(true) + .open(&lock_path) + .unwrap(); + assert!( + matches!(probe.try_lock(), Err(fs::TryLockError::WouldBlock)), + "surviving checkout lost task ownership" + ); + drop(probe); + fs::write(&release.release, "release").unwrap(); + until(|| { + let file = fs::OpenOptions::new() + .read(true) + .write(true) + .open(&lock_path) + .unwrap(); + file.try_lock().is_ok() + }); + let out = finish(command(&env, &server.url(), "1")); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + assert!(rows(&out) + .iter() + .any(|r| r["event"] == "task" && r["outcome"] == "nothing_to_do")); +} + +#[test] +fn concurrent_same_problem_reports_search_and_file_serially() { + use std::os::unix::fs::PermissionsExt; + let env = TestEnv::new(); + setup(&env, "[]"); + task_file(&env, "a", "a", 2); + task_file(&env, "b", "b", 1); + env.write( + "home/.oxa/agent/settings.json", + r#"{"oxa":{"reportIssue":{"repo":"example/project"}}}"#, + ); + let gh=env.write("tmp/gh",r##"#!/bin/sh +case "$1 $2" in + 'issue list') if test -f "$HOME/reported"; then printf '[{"number":1}]'; else printf '[]'; fi ;; + 'pr list') printf '[]' ;; + 'issue view') printf '%s' '{"title":"oxa queue problem","body":"Same synthetic reproduction","url":"https://github.com/example/project/issues/1","comments":[]}' ;; + 'issue create') sleep 0.2; touch "$HOME/reported"; printf 'created\n' >> "$HOME/report-writes"; printf '%s' 'https://github.com/example/project/issues/1' ;; + *) exit 7 ;; +esac +"##); + let oxc = env.write("tmp/oxc", "#!/bin/sh\nprintf '{}'\n"); + for file in [gh, oxc] { + fs::set_permissions(file, fs::Permissions::from_mode(0o755)).unwrap(); + } + let text="[{\"title\":\"oxa queue problem\",\"notes\":\"Same synthetic reproduction\",\"keywords\":[\"oxa queue\",\"same reproduction\"]}]"; + let server = FakeServer::start(vec![ScriptTurn::text(text); 2], FakeOpts::default()).unwrap(); + let mut c = command(&env, &server.url(), "2"); + c.arg("--report-problems").env( + "PATH", + format!("{}:{}", env.tmp().display(), std::env::var("PATH").unwrap()), + ); + let out = finish(c); + assert!( + out.status.success(), + "{} / {}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + assert_eq!( + fs::read_to_string(env.home().join("report-writes")) + .unwrap() + .lines() + .count(), + 1 + ); + let events = rows(&out); + let dispositions: std::collections::HashSet<_> = events + .iter() + .filter(|r| r["event"] == "task") + .map(|r| r["reports"][0]["disposition"].as_str().unwrap()) + .collect(); + assert!( + dispositions.contains("filed") && dispositions.contains("existing_issue"), + "{dispositions:?}" + ); +} diff --git a/oxa/oxa-work/src/execute.rs b/oxa/oxa-work/src/execute.rs index a62807e77..d6539758f 100644 --- a/oxa/oxa-work/src/execute.rs +++ b/oxa/oxa-work/src/execute.rs @@ -29,7 +29,7 @@ pub(crate) fn execute( loop { match state.record.step { Step::Preparing => { - let prepared = tree::prepare(repo, &tree, task)?; + let prepared = tree::prepare(repo, &tree, task, lock)?; let commands = tree::done_from_base(&tree, &prepared.base)?; state.checkpoint = Some(Checkpoint { initial: prepared.initial, diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index 794d485f3..09335a5c9 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -2,8 +2,10 @@ mod execute; mod notes; mod publish; +mod queue; pub use notes::{ProblemNote, ReportRecord}; pub use publish::{publish, report_notes}; +pub use queue::{run_queue, QueueEvent, QueueOptions, QueueStatus}; mod state; mod tree; use serde::{Deserialize, Serialize}; @@ -201,6 +203,19 @@ struct AgentReply { notes: Vec, error: Option, } +pub(crate) fn worktree_root(common: &Path) -> Result { + let repo_dir = if common.file_name().is_some_and(|name| name == ".git") { + common.parent().ok_or("git directory has no parent")? + } else { + common + }; + let parent = repo_dir + .parent() + .ok_or("repository has no parent for worktrees")?; + let name = repo_dir.file_name().ok_or("repository has no directory name")?; + Ok(parent.join(".oxa-worktrees").join(name)) +} + pub fn blocked_record(task: &Task, detail: String) -> Record { Record { id: task.id.clone(), @@ -244,18 +259,7 @@ pub fn run_task_with_options( let lock_path = storage.join(format!("{}.lock", task.id)); let lock = state::lock(&lock_path)?; let receipt = storage.join(format!("{}.json", task.id)); - // The common git directory must not be an ancestor of the child cwd: the sandbox - // deliberately refuses writable git binds with that layout. - let repo_dir = if common.file_name().is_some_and(|name| name == ".git") { - common.parent().ok_or("git directory has no parent")? - } else { - &common - }; - let parent = repo_dir - .parent() - .ok_or("repository has no parent for worktrees")?; - let name = repo_dir.file_name().ok_or("repository has no directory name")?; - let trees = parent.join(".oxa-worktrees").join(name); + let trees = worktree_root(&common)?; fs::create_dir_all(&trees).map_err(|e| e.to_string())?; let tree = trees.join(&task.id); let mut state = if receipt.exists() { diff --git a/oxa/oxa-work/src/queue.rs b/oxa/oxa-work/src/queue.rs new file mode 100644 index 000000000..5d5872a98 --- /dev/null +++ b/oxa/oxa-work/src/queue.rs @@ -0,0 +1,320 @@ +//! Directory queue: oldest regular JSON files first, with bounded independent workers. +use super::{blocked_record, checked, git, read_tasks, state, Record, Result, Task}; +use serde::Serialize; +use std::{ + collections::{HashMap, HashSet}, + fs, + path::{Path, PathBuf}, + process::Command, + sync::{mpsc, Arc}, + thread, + time::{Duration, SystemTime}, +}; + +#[derive(Clone, Debug)] +pub struct QueueOptions { + pub max: usize, + pub min_free_bytes: u64, + pub once: bool, +} +impl Default for QueueOptions { + fn default() -> Self { + Self { + max: 1, + min_free_bytes: 512 * 1024 * 1024, + once: false, + } + } +} +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +pub struct QueueStatus { + pub time: u64, + pub active: Vec, + pub pending: usize, + pub finished: usize, + pub failed: usize, + pub free_bytes: Option, + pub disk_blocked: bool, + pub disk_error: Option, +} +#[derive(Serialize)] +#[serde(tag = "event", rename_all = "snake_case")] +pub enum QueueEvent { + State { + #[serde(flatten)] + status: QueueStatus, + }, + Task { + #[serde(flatten)] + record: Record, + }, + Error { + file: PathBuf, + detail: String, + }, +} +#[derive(Clone)] +struct Job { + task: Task, + modified: SystemTime, + file: PathBuf, + index: usize, +} +fn busy(storage: &Path, id: &str) -> Result { + match fs::OpenOptions::new() + .read(true) + .write(true) + .open(storage.join(format!("{id}.lock"))) + { + Ok(file) => match file.try_lock() { + Ok(()) => Ok(false), + Err(fs::TryLockError::WouldBlock) => Ok(true), + Err(error) => Err(error.to_string()), + }, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error.to_string()), + } +} +fn available(path: &Path) -> Result { + let mut command = Command::new("df"); + command.args(["--output=avail", "-B1", "--"]).arg(path); + checked(command)? + .lines() + .last() + .ok_or("df returned no free-space value")? + .trim() + .parse() + .map_err(|_| "df returned an invalid free-space value".into()) +} +fn free_space(repo: &Path, common: &Path, queue: &Path) -> Result { + let trees = super::worktree_root(common)?; + let tree_volume = trees + .ancestors() + .find(|p| p.is_dir()) + .ok_or("worktree storage has no existing parent")?; + Ok(available(repo)? + .min(available(common)?) + .min(available(queue)?) + .min(available(tree_volume)?)) +} +fn scan( + directory: &Path, + pending: &mut Vec, + seen_files: &mut HashSet<(PathBuf, SystemTime, u64)>, + seen_ids: &mut HashSet, + failed: &mut usize, + emit: &mut dyn FnMut(QueueEvent) -> Result<()>, +) -> Result<()> { + let mut files = vec![]; + for entry in fs::read_dir(directory).map_err(|e| e.to_string())? { + let entry = entry.map_err(|e| e.to_string())?; + let path = entry.path(); + if !entry.file_type().map_err(|e| e.to_string())?.is_file() + || path + .file_name() + .is_some_and(|s| s.to_string_lossy().starts_with('.')) + || path.extension().is_none_or(|s| s != "json") + { + continue; + } + let meta = entry.metadata().map_err(|e| e.to_string())?; + files.push((path, meta.modified().map_err(|e| e.to_string())?, meta.len())); + } + files.sort_by(|a, b| (&a.1, &a.0).cmp(&(&b.1, &b.0))); + for (file, modified, len) in files { + if !seen_files.insert((file.clone(), modified, len)) { + continue; + } + match read_tasks(&file) { + Ok(tasks) => { + for (index, task) in tasks.into_iter().enumerate() { + if !seen_ids.insert(task.id.clone()) { + *failed += 1; + emit(QueueEvent::Error { + file: file.clone(), + detail: format!( + "task {} was already queued; use a new id for changed work", + task.id + ), + })?; + } else { + pending.push(Job { + task, + file: file.clone(), + modified, + index, + }); + } + } + } + Err(detail) => { + *failed += 1; + emit(QueueEvent::Error { file, detail })?; + } + } + } + Ok(()) +} +pub fn run_queue( + repo: &Path, + directory: &Path, + options: &QueueOptions, + worker: Arc Record + Send + Sync>, + emit: &mut dyn FnMut(QueueEvent) -> Result<()>, +) -> Result { + if options.max == 0 { + return Err("--max must be positive".into()); + } + let directory = directory.canonicalize().map_err(|e| e.to_string())?; + if !directory.is_dir() { + return Err("queue path is not a directory".into()); + } + let _controller = state::lock(&directory.join(".oxa-queue.lock"))?; + let common = PathBuf::from(checked(git( + repo, + &["rev-parse", "--path-format=absolute", "--git-common-dir"], + ))?); + let storage = common.join("oxa-work"); + let (tx, rx) = mpsc::channel(); + let mut active: HashMap> = HashMap::new(); + let (mut pending, mut external, mut seen_files, mut seen_ids) = + (vec![], vec![], HashSet::new(), HashSet::new()); + let (mut finished, mut failed, mut last) = (0, 0, None); + loop { + scan( + &directory, + &mut pending, + &mut seen_files, + &mut seen_ids, + &mut failed, + emit, + )?; + // Surviving workers from a dead controller still count toward the concurrency cap. + let mut waiting = vec![]; + for job in external.drain(..) { + let job: Job = job; + if busy(&storage, &job.task.id)? { + waiting.push(job); + } else { + pending.push(job); + } + } + external = waiting; + let mut ready = vec![]; + for job in pending.drain(..) { + if busy(&storage, &job.task.id)? { + external.push(job); + } else { + ready.push(job); + } + } + pending = ready; + pending.sort_by(|a, b| (&a.modified, &a.file, a.index).cmp(&(&b.modified, &b.file, b.index))); + let disk = free_space(repo, &common, &directory); + let (mut free_bytes, mut disk_error) = match disk { + Ok(free) => (Some(free), None), + Err(error) => (None, Some(error)), + }; + let mut disk_blocked = free_bytes.is_none_or(|free| free < options.min_free_bytes); + while !disk_blocked && active.len() + external.len() < options.max && !pending.is_empty() { + // Check again for each admission: earlier workers may already be writing. + match free_space(repo, &common, &directory) { + Ok(free) => { + free_bytes = Some(free); + disk_error = None; + disk_blocked = free < options.min_free_bytes; + } + Err(error) => { + free_bytes = None; + disk_error = Some(error); + disk_blocked = true; + } + } + if disk_blocked { + break; + } + let job = pending.remove(0); + let id = job.task.id.clone(); + let worker = worker.clone(); + let tx = tx.clone(); + let input = job.clone(); + match thread::Builder::new() + .name(format!("oxa-work-{id}")) + .spawn(move || { + let record = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| worker(&input.task))) + .unwrap_or_else(|_| { + blocked_record( + &input.task, + "queue worker panicked; retained state can be resumed".into(), + ) + }); + let _ = tx.send((input, record)); + }) { + Ok(handle) => { + active.insert(id, handle); + } + Err(error) => { + finished += 1; + failed += 1; + emit(QueueEvent::Task { + record: blocked_record(&job.task, error.to_string()), + })?; + } + } + } + let mut ids: Vec<_> = active + .keys() + .cloned() + .chain(external.iter().map(|j| j.task.id.clone())) + .collect(); + ids.sort(); + let status = QueueStatus { + time: state::now(), + active: ids, + pending: pending.len(), + finished, + failed, + free_bytes, + disk_blocked: disk_blocked && !pending.is_empty(), + disk_error, + }; + if last.as_ref() != Some(&status) { + let mut bytes = serde_json::to_vec(&QueueEvent::State { + status: status.clone(), + }) + .map_err(|e| e.to_string())?; + bytes.push(b'\n'); + state::atomic_write(&directory.join(".oxa-queue-status.json"), &bytes)?; + emit(QueueEvent::State { + status: status.clone(), + })?; + last = Some(status.clone()); + } + if options.once && active.is_empty() && external.is_empty() && (pending.is_empty() || disk_blocked) { + return Ok(status); + } + if active.is_empty() { + thread::sleep(Duration::from_millis(250)); + continue; + } + match rx.recv_timeout(Duration::from_millis(250)) { + Ok((job, record)) => { + if let Some(handle) = active.remove(&job.task.id) { + let _ = handle.join(); + } + if busy(&storage, &job.task.id)? { + external.push(job); + continue; + } + finished += 1; + if record.failed() { + failed += 1; + } + emit(QueueEvent::Task { record })?; + } + Err(mpsc::RecvTimeoutError::Timeout) => {} + Err(error) => return Err(error.to_string()), + } + } +} diff --git a/oxa/oxa-work/src/state.rs b/oxa/oxa-work/src/state.rs index fae423fc1..f51b6a64a 100644 --- a/oxa/oxa-work/src/state.rs +++ b/oxa/oxa-work/src/state.rs @@ -68,6 +68,12 @@ pub(crate) fn read(path: &Path) -> Result { pub(crate) fn save(path: &Path, state: &mut State, step: Step) -> Result<()> { state.record.step = step; state.record.progress_at = now(); + atomic_write( + path, + &serde_json::to_vec_pretty(state).map_err(|e| e.to_string())?, + ) +} +pub(crate) fn atomic_write(path: &Path, bytes: &[u8]) -> Result<()> { let temporary = path.with_extension(format!( "tmp-{}-{}", std::process::id(), @@ -83,8 +89,7 @@ pub(crate) fn save(path: &Path, state: &mut State, step: Step) -> Result<()> { .mode(0o600) .open(&temporary) .map_err(|e| e.to_string())?; - file.write_all(&serde_json::to_vec_pretty(state).map_err(|e| e.to_string())?) - .map_err(|e| e.to_string())?; + file.write_all(bytes).map_err(|e| e.to_string())?; file.sync_all().map_err(|e| e.to_string())?; fs::rename(&temporary, path).map_err(|e| e.to_string())?; File::open(path.parent().ok_or("state has no parent")?) diff --git a/oxa/oxa-work/src/tree.rs b/oxa/oxa-work/src/tree.rs index 341282286..52d2b530f 100644 --- a/oxa/oxa-work/src/tree.rs +++ b/oxa/oxa-work/src/tree.rs @@ -1,6 +1,7 @@ //! Worktree continuation uses ordinary merges; no reset, stash or force checkout. use super::{checked, git, output, Result, Task}; use std::path::{Path, PathBuf}; +use std::{fs::File, process::Command}; pub(crate) struct Prepared { pub initial: String, @@ -22,21 +23,25 @@ fn merge_prompt(tree: &Path, task: &Task, base: &str, output: &str) -> Result Result { +pub(crate) fn prepare(repo: &Path, tree: &Path, task: &Task, lock: &File) -> Result { let base = checked(git( repo, &["rev-parse", "--verify", &format!("{}^{{commit}}", task.base)], ))?; + let run = |mut command: Command| -> Result { + command.stdin(lock.try_clone().map_err(|e| e.to_string())?); + super::checked_output(command.output().map_err(|e| e.to_string())?) + }; if !tree.exists() { let branch = format!("refs/heads/{}", task.branch); let exists = output(git(repo, &["show-ref", "--verify", "--quiet", &branch]))?; let tree_arg = tree.to_str().ok_or("non-UTF8 worktree path")?; match exists.status.code() { Some(0) => { - checked(git(repo, &["worktree", "add", tree_arg, &task.branch]))?; + run(git(repo, &["worktree", "add", tree_arg, &task.branch]))?; } Some(1) => { - checked(git( + run(git( repo, &["worktree", "add", "-b", &task.branch, tree_arg, &base], ))?; @@ -58,10 +63,12 @@ pub(crate) fn prepare(repo: &Path, tree: &Path, task: &Task) -> Result prompt: merge_prompt(tree, task, &base, "Resuming the retained merge.")?, }); } - if !checked(git(tree, &["status", "--porcelain"]))?.is_empty() { + if !run(git(tree, &["status", "--porcelain"]))?.is_empty() { return Err("retained worktree has uncommitted changes; commit them before continuing".into()); } - let merged = output(git(tree, &["merge", "--no-edit", &base]))?; + let mut merge = git(tree, &["merge", "--no-edit", &base]); + merge.stdin(lock.try_clone().map_err(|e| e.to_string())?); + let merged = merge.output().map_err(|e| e.to_string())?; if !merged.status.success() { if !merging(tree)? { return Err(format!( From d7ab5cfa31ae22b3ef4da2bc606e5cd2c151c74f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:54:33 +0000 Subject: [PATCH 12/52] Keep prompt parsing stable and retain agent failure causes (#333) --- oxa/oxa-cli/src/args.rs | 9 +---- oxa/oxa-cli/tests/c12_argument_parsing.rs | 47 +++++++++++++++++++++++ oxa/oxa-cli/tests/work_cli.rs | 15 ++++++++ oxa/oxa-work/src/lib.rs | 3 ++ 4 files changed, 66 insertions(+), 8 deletions(-) diff --git a/oxa/oxa-cli/src/args.rs b/oxa/oxa-cli/src/args.rs index b4a6942ed..0ca62ae99 100644 --- a/oxa/oxa-cli/src/args.rs +++ b/oxa/oxa-cli/src/args.rs @@ -463,14 +463,7 @@ pub fn parse_args(argv: &[String], ext_flags: &[FlagSpec]) -> Result Date: Sun, 15 Nov 2026 18:03:19 +0000 Subject: [PATCH 13/52] Require terminal approval for public work actions and retain interrupted checkpoints (#333) --- oxa/docs/work.md | 10 ++- oxa/oxa-cli/src/report_cmd.rs | 5 +- oxa/oxa-cli/src/work_cmd.rs | 10 ++- oxa/oxa-cli/src/work_reports.rs | 31 +++++++++- oxa/oxa-cli/tests/common/work.rs | 4 +- oxa/oxa-cli/tests/common/work_confirm.rs | 66 ++++++++++++++++++++ oxa/oxa-cli/tests/work_publish.rs | 77 +++++++++++++++++++++++- oxa/oxa-work/src/lib.rs | 5 +- oxa/oxa-work/src/publish.rs | 13 +++- 9 files changed, 209 insertions(+), 12 deletions(-) create mode 100644 oxa/oxa-cli/tests/common/work_confirm.rs diff --git a/oxa/docs/work.md b/oxa/docs/work.md index 90786b4ab..797a3d991 100644 --- a/oxa/docs/work.md +++ b/oxa/docs/work.md @@ -49,7 +49,8 @@ the agent; they are never staged as a fabricated resolution. There is no automat cleanup. Publication is explicitly opt-in as described below. To continue an interrupted, red or blocked task, run the same file again. Use -`--rerun` to continue a completed task after its base changes. The retained receipt +`--rerun` to continue a completed task after its base changes. Interrupted execution +always resumes its captured base, checks and progress, even with `--rerun`. The retained receipt must match its ID, branch and worktree. A clean worktree is reused; a branch checked out elsewhere is blocked. Existing dirty work is saved when possible before a later retry. Merge conflicts are passed to the agent with the merge still in progress, the @@ -89,11 +90,14 @@ Execution preserves a saved offline restriction across restarts; later explicit publication uses the current network policy without changing that restriction. A publication failure keeps a publication checkpoint; another `--publish` retries without replaying the agent. Offline mode refuses network publication. +Before each push, the runner asks through the controlling terminal for approval +of the accepted commit, branch and PR destination. Piped input cannot approve it. `--report-problems` independently authorizes filing the task's own structured problem notes using the `report-issue` workflow. This flag must be launched by the -user outside an agent tool. Normal `oxa report-issue submit` still requires its -terminal confirmation. Reports use the user setting `oxa.reportIssue.repo`; +user outside an agent tool. Each new report or comment shows its exact cleaned +text and requires controlling-terminal confirmation, as `oxa report-issue submit` +does. Without confirmation it stays a local draft. Reports use the user setting `oxa.reportIssue.repo`; project settings cannot choose a destination. Notes, titles and queries pass through the existing report scrubber. The runner performs two or three symptom queries against open issues and PRs and reads the top matches before writing. diff --git a/oxa/oxa-cli/src/report_cmd.rs b/oxa/oxa-cli/src/report_cmd.rs index 77ca53fee..3c48281f9 100644 --- a/oxa/oxa-cli/src/report_cmd.rs +++ b/oxa/oxa-cli/src/report_cmd.rs @@ -404,11 +404,14 @@ pub fn submit( /// The real confirmation: the question and the answer go through `/dev/tty` (never stdin, so /// piped input cannot confirm). `None` without a controlling terminal, or when oxa's own tools /// run this (`OXA_CODING_AGENT` is set: the model must not confirm for the user). -fn tty_confirm(env: &Env) -> impl FnMut(&str) -> Option + '_ { +pub(crate) fn tty_confirm(env: &Env) -> impl FnMut(&str) -> Option + '_ { move |question: &str| { if env.get("OXA_CODING_AGENT").is_some() { return None; } + // Keep concurrent workers' questions and answers together. + static TERMINAL: std::sync::Mutex<()> = std::sync::Mutex::new(()); + let _terminal = TERMINAL.lock().ok()?; let mut tty = std::fs::OpenOptions::new() .read(true) .write(true) diff --git a/oxa/oxa-cli/src/work_cmd.rs b/oxa/oxa-cli/src/work_cmd.rs index 8f6306b9b..48d9706bf 100644 --- a/oxa/oxa-cli/src/work_cmd.rs +++ b/oxa/oxa-cli/src/work_cmd.rs @@ -67,8 +67,13 @@ pub(crate) fn run( oxa_work::run_task_with_options(cwd, &binary, &task, rerun, oxa_proto::offline()) .unwrap_or_else(|detail| oxa_work::blocked_record(&task, detail)); if publish && !record.worktree.as_os_str().is_empty() { - record = oxa_work::publish(cwd, &task, oxa_proto::offline()) - .unwrap_or_else(|detail| oxa_work::blocked_record(&task, detail)); + record = oxa_work::publish( + cwd, + &task, + oxa_proto::offline(), + &mut crate::report_cmd::tty_confirm(env), + ) + .unwrap_or_else(|detail| oxa_work::blocked_record(&task, detail)); } if reports && !record.worktree.as_os_str().is_empty() { let settings = oxa_config::Settings::load(dirs, cwd, false); @@ -84,6 +89,7 @@ pub(crate) fn run( &settings, &ctx, offline || oxa_proto::offline(), + &mut crate::report_cmd::tty_confirm(env), &mut runner, ) }) diff --git a/oxa/oxa-cli/src/work_reports.rs b/oxa/oxa-cli/src/work_reports.rs index d5cd7e763..d7fca26b2 100644 --- a/oxa/oxa-cli/src/work_reports.rs +++ b/oxa/oxa-cli/src/work_reports.rs @@ -1,6 +1,6 @@ //! The report-issue workflow for an explicit user-launched --report-problems run. //! Normal `report-issue submit` retains its controlling-terminal confirmation. -use crate::report_cmd::{clean_notes, report_repo, save_draft, scrub, Runner, ScrubCtx}; +use crate::report_cmd::{clean_notes, report_repo, save_draft, scrub, Confirm, Runner, ScrubCtx}; use oxa_config::{Dirs, Settings}; use oxa_work::{ProblemNote, ReportRecord}; use serde_json::Value; @@ -81,6 +81,7 @@ pub(crate) fn report( settings: &Settings, ctx: &ScrubCtx, offline: bool, + confirm: Confirm<'_>, runner: Runner<'_>, ) -> Result { let (draft, title, body) = @@ -192,6 +193,13 @@ pub(crate) fn report( .any(|c| c["body"].as_str().unwrap_or_default().contains(¬e.notes)) }); if !already { + if confirm(&format!( + "Comment on {repo}#{number}:\n\n{}\n\nType yes to post these notes: ", + note.notes + )) != Some(true) + { + return Err("problem report requires confirmation through the controlling terminal; draft retained locally".into()); + } oxa_resources::write_atomic(&body_path, note.notes.as_bytes()).map_err(|e| e.to_string())?; call( runner, @@ -220,6 +228,13 @@ pub(crate) fn report( format!("{body}\n\nRelated open reports (different symptoms):\n{links}\n") }; let body = scrub(&body, ctx); + if confirm(&format!( + "Create issue in {repo}:\nTitle: {}\n\n{body}\n\nType yes to post this report: ", + note.title + )) != Some(true) + { + return Err("problem report requires confirmation through the controlling terminal; draft retained locally".into()); + } oxa_resources::write_atomic(&body_path, body.as_bytes()).map_err(|e| e.to_string())?; let url = call( runner, @@ -287,6 +302,7 @@ mod tests { &settings(), &ScrubCtx::default(), false, + &mut |_| Some(true), &mut runner, ) .unwrap(); @@ -314,6 +330,7 @@ mod tests { &settings(), &ScrubCtx::default(), offline, + &mut |_| Some(true), &mut runner, ) .unwrap(); @@ -351,6 +368,7 @@ mod tests { &settings(), &ScrubCtx::default(), false, + &mut |_| Some(true), &mut runner, ) .unwrap(); @@ -374,7 +392,16 @@ mod tests { _ => panic!("unexpected command"), } }; - let result = report(&n, &dirs, &settings(), &ScrubCtx::default(), false, &mut runner).unwrap(); + let result = report( + &n, + &dirs, + &settings(), + &ScrubCtx::default(), + false, + &mut |_| Some(true), + &mut runner, + ) + .unwrap(); assert_eq!(result.disposition, "filed"); } } diff --git a/oxa/oxa-cli/tests/common/work.rs b/oxa/oxa-cli/tests/common/work.rs index 44062a99f..ebc82f69d 100644 --- a/oxa/oxa-cli/tests/common/work.rs +++ b/oxa/oxa-cli/tests/common/work.rs @@ -1,4 +1,6 @@ use oxa_fake::{FakeOpts, FakeServer, ScriptTurn, TestEnv}; +#[path = "work_confirm.rs"] +mod confirm; use std::{path::Path, process::Command}; pub fn git(env: &TestEnv, dir: &Path, args: &[&str]) -> String { let out = Command::new("git") @@ -63,7 +65,7 @@ pub fn run_observed( } let (tx, rx) = std::sync::mpsc::channel(); std::thread::spawn(move || { - tx.send(command.output().unwrap()).unwrap(); + tx.send(confirm::output(command)).unwrap(); }); let out = rx .recv_timeout(std::time::Duration::from_secs(90)) diff --git a/oxa/oxa-cli/tests/common/work_confirm.rs b/oxa/oxa-cli/tests/common/work_confirm.rs new file mode 100644 index 000000000..6e2b78fa2 --- /dev/null +++ b/oxa/oxa-cli/tests/common/work_confirm.rs @@ -0,0 +1,66 @@ +//! Real controlling-terminal approvals for publication integration tests. +use std::{ + io::Write, + process::{Command, Output, Stdio}, + sync::atomic::{AtomicU64, Ordering}, +}; + +pub fn output(mut command: Command) -> Output { + if !command + .get_args() + .any(|a| a == "--publish" || a == "--report-problems") + { + return command.output().unwrap(); + } + static NEXT: AtomicU64 = AtomicU64::new(0); + let tmp = command + .get_envs() + .find_map(|(k, v)| (k == "TMPDIR").then_some(v).flatten()) + .unwrap(); + let prefix = std::path::Path::new(tmp).join(format!( + "approval-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let stdout = prefix.with_extension("out"); + let stderr = prefix.with_extension("err"); + let quote = |s: &std::ffi::OsStr| format!("'{}'", s.to_string_lossy().replace('\'', "'\\''")); + let words = std::iter::once(command.get_program()) + .chain(command.get_args()) + .map(quote) + .collect::>() + .join(" "); + let shell = format!( + "{words} >{} 2>{}", + quote(stdout.as_os_str()), + quote(stderr.as_os_str()) + ); + let mut terminal = Command::new("script"); + terminal.args(["-qefc", &shell, "/dev/null"]).env_clear(); + for (key, value) in command.get_envs() { + if let Some(value) = value { + terminal.env(key, value); + } + } + if let Some(cwd) = command.get_current_dir() { + terminal.current_dir(cwd); + } + let mut child = terminal + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .expect("script provides a real controlling terminal"); + child + .stdin + .take() + .unwrap() + .write_all("yes\n".repeat(32).as_bytes()) + .unwrap(); + let mut out = child.wait_with_output().unwrap(); + out.stdout = std::fs::read(&stdout).unwrap_or_default(); + out.stderr.extend(std::fs::read(&stderr).unwrap_or_default()); + let _ = std::fs::remove_file(stdout); + let _ = std::fs::remove_file(stderr); + out +} diff --git a/oxa/oxa-cli/tests/work_publish.rs b/oxa/oxa-cli/tests/work_publish.rs index c1ae16f20..cd38c3091 100644 --- a/oxa/oxa-cli/tests/work_publish.rs +++ b/oxa/oxa-cli/tests/work_publish.rs @@ -3,7 +3,7 @@ mod work; use oxa_fake::{ScriptTurn, TestEnv}; use std::{fs, path::PathBuf, process::Command}; -use work::{git, run, run_args, setup}; +use work::{git, run, run_args, run_observed, setup}; fn configure(env: &TestEnv) -> String { use std::os::unix::fs::PermissionsExt; env.write( @@ -369,3 +369,78 @@ fn failed_draft_creation_does_not_strand_reporting_checkpoint() { assert_eq!(retry["reports"][0]["disposition"], "local"); assert_eq!(retry["step"], "finished"); } + +#[test] +fn unsetting_the_agent_marker_and_piping_yes_cannot_authorize_public_writes() { + use std::io::Write; + use std::process::Stdio; + let mut unauthorized = vec![]; + for flag in ["--publish", "--report-problems"] { + let env = TestEnv::new(); + setup(&env, "[]"); + let path = configure(&env); + env.write("tmp/gh", r##"#!/bin/sh +case "$1 $2" in + 'repo view') printf '%s' '{"nameWithOwner":"example/project"}' ;; + 'pr list'|'issue list') printf '[]' ;; + 'pr create'|'issue create'|'issue comment') touch "$HOME/public-write"; printf '%s' 'https://github.com/example/project/issues/1' ;; + *) exit 7 ;; +esac +"##); + let text = "[{\"title\":\"oxa work symptom\",\"notes\":\"Synthetic reproduction\",\"keywords\":[\"oxa work\",\"synthetic symptom\"]}]"; + let mut turns = edits(); + turns.pop(); + turns.push(ScriptTurn::text(text)); + assert_eq!(run(&env, turns)["outcome"], "done"); + // A fresh session has no controlling terminal. Removing the marker and + // supplying yes on stdin must not stand in for a person's confirmation. + let mut child = Command::new("setsid") + .env_clear() + .envs(env.envs("http://127.0.0.1:1")) + .env("PATH", path) + .env_remove("OXA_CODING_AGENT") + .args(["--wait", env!("CARGO_BIN_EXE_oxa"), "work", "../tasks.json", flag]) + .current_dir(env.work().join("project")) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + child.stdin.take().unwrap().write_all(b"yes\n").unwrap(); + let out = child.wait_with_output().unwrap(); + assert!(out.status.code().is_some()); + if env.home().join("public-write").exists() || env.home().join("push-calls").exists() { + unauthorized.push(flag); + } + } + assert!( + unauthorized.is_empty(), + "public writes without confirmation: {unauthorized:?}" + ); +} + +#[test] +fn rerun_resumes_interrupted_execution_without_discarding_its_checkpoint() { + for step in ["acceptance", "recheck", "saving"] { + let env = TestEnv::new(); + setup(&env, r#"["printf checked >> checkpoint-checks"]"#); + let first = run(&env, edits()); + let state_path = env.work().join("project/.git/oxa-work/test.json"); + let mut state: serde_json::Value = serde_json::from_slice(&fs::read(&state_path).unwrap()).unwrap(); + state["step"] = step.into(); + state["outcome"] = if step == "saving" { "done" } else { "running" }.into(); + state["pull_request"] = "https://github.com/example/project/pull/1".into(); + state["problem_notes"] = serde_json::json!([{"title":"Synthetic problem","notes":"Keep these notes","keywords":["synthetic problem","keep notes"]}]); + state["reports"] = serde_json::json!([{"draft":"report.md","disposition":"existing_issue","url":"https://github.com/example/project/issues/1","detail":""}]); + fs::write(&state_path, serde_json::to_vec(&state).unwrap()).unwrap(); + let (record, requests) = run_observed(&env, vec![], &["work", "../tasks.json", "--rerun"], None); + assert_eq!(record["outcome"], "done", "{step}: {record}"); + assert!(requests.is_empty(), "{step}: agent was replayed"); + for key in ["pull_request", "problem_notes", "reports"] { + assert_eq!(record[key], state[key], "{step}: {key}"); + } + let after: serde_json::Value = serde_json::from_slice(&fs::read(&state_path).unwrap()).unwrap(); + assert_eq!(after["checkpoint"], state["checkpoint"], "{step}"); + assert_eq!(record["worktree"], first["worktree"]); + } +} diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index 794d485f3..730439711 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -270,7 +270,10 @@ pub fn run_task_with_options( if matches!(old.record.step, Step::Publishing | Step::Reporting) && !rerun { return Ok(old.record); } - if old.record.step != Step::Finished && !rerun { + if !matches!( + old.record.step, + Step::Finished | Step::Publishing | Step::Reporting + ) { old } else if !rerun && matches!(old.record.outcome, Outcome::Done | Outcome::NothingToDo) { return Ok(old.record); diff --git a/oxa/oxa-work/src/publish.rs b/oxa/oxa-work/src/publish.rs index 1e8eecf6f..04fdefec6 100644 --- a/oxa/oxa-work/src/publish.rs +++ b/oxa/oxa-work/src/publish.rs @@ -38,7 +38,12 @@ fn pr_base(repo: &Path, task: &Task) -> Result { .map(str::to_string) .ok_or_else(|| "publishing requires a named local or origin base branch".into()) } -pub fn publish(repo: &Path, task: &Task, offline: bool) -> Result { +pub fn publish( + repo: &Path, + task: &Task, + offline: bool, + confirm: &mut dyn FnMut(&str) -> Option, +) -> Result { let (path, lock, mut value) = context(repo, task)?; let accepted = value.acceptance_outcome.unwrap_or(value.record.outcome); if !matches!(accepted, Outcome::Done | Outcome::Red) { @@ -112,6 +117,12 @@ pub fn publish(repo: &Path, task: &Task, offline: bool) -> Result { return Err("PR is in the merge queue; use a new task branch for follow-up work".into()); } } + if confirm(&format!( + "Push accepted commit {head} as {} to origin and create/update its PR in {name} (base {base})? Type yes to publish: ", + task.branch + )) != Some(true) { + return Err("publication requires confirmation through the controlling terminal".into()); + } let mut push = git( repo, &["push", "origin", &format!("{head}:refs/heads/{}", task.branch)], From ad1db00d0de1da8ca9f4206b0c80e79fd58b029b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Sun, 15 Nov 2026 18:09:30 +0000 Subject: [PATCH 14/52] Drain queue workers on errors and reconcile changed inputs (#333) --- oxa/docs/work.md | 5 + oxa/oxa-cli/tests/work_queue.rs | 115 +++++++++++++++++++++- oxa/oxa-work/src/lib.rs | 6 +- oxa/oxa-work/src/queue.rs | 167 +++++++++++++++++++++++--------- 4 files changed, 244 insertions(+), 49 deletions(-) diff --git a/oxa/docs/work.md b/oxa/docs/work.md index 99bfff37d..1654d08ac 100644 --- a/oxa/docs/work.md +++ b/oxa/docs/work.md @@ -127,6 +127,11 @@ or unreadable free-space value pauses new work while active jobs finish. Invalid JSON files report an error and do not skip other work. `--once` drains available work and exits; when disk space blocks pending tasks it exits 1 with their files retained. Without `--once`, the directory stays under watch for new files. +Appending tasks to an existing file queues only new IDs. Discovered task definitions +are immutable; changing one requires a new ID. Correcting or archiving an invalid +input clears its current queue errors. Archiving inputs does not interrupt admitted +work. A controller or monitor error stops admission and drains active workers +before returning an error. Queue mode supports `--publish`, `--report-problems` and `--rerun` with the same explicit policy as single-file mode. Its worker threads preserve the parent's diff --git a/oxa/oxa-cli/tests/work_queue.rs b/oxa/oxa-cli/tests/work_queue.rs index 6107a0e07..d6ce02bff 100644 --- a/oxa/oxa-cli/tests/work_queue.rs +++ b/oxa/oxa-cli/tests/work_queue.rs @@ -36,7 +36,7 @@ fn command(env: &TestEnv, url: &str, max: &str) -> Command { .envs(env.envs(url)); c } -fn finish(mut c: Command) -> Output { +fn finish(c: Command) -> Output { let (tx, rx) = std::sync::mpsc::channel(); std::thread::spawn(move || { let _ = tx.send(confirm::output(c)); @@ -405,3 +405,116 @@ esac "{dispositions:?}" ); } + +fn has_active(env: &TestEnv, count: usize) -> bool { + fs::read(env.work().join("queue/.oxa-queue-status.json")) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .is_some_and(|state| state["active"].as_array().is_some_and(|ids| ids.len() == count)) +} + +#[test] +fn appended_tasks_and_archived_inputs_do_not_fail_or_interrupt_active_work() { + use std::process::Stdio; + let env = TestEnv::new(); + setup(&env, "[]"); + task_file(&env, "input", "first", 1); + let server = FakeServer::start( + vec![ScriptTurn::text("done"); 2], + FakeOpts { + hold_headers_ms: 700, + ..FakeOpts::default() + }, + ) + .unwrap(); + let child = command(&env, &server.url(), "2") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + until(|| has_active(&env, 1)); + let file = env.work().join("queue/input.json"); + let mut tasks: Vec = serde_json::from_slice(&fs::read(&file).unwrap()).unwrap(); + tasks.push( + serde_json::json!({"id":"second","model":"fake","branch":"agent/second","prompt":"second task"}), + ); + fs::write(&file, serde_json::to_vec(&tasks).unwrap()).unwrap(); + until(|| has_active(&env, 2)); + fs::create_dir(env.work().join("archive")).unwrap(); + fs::rename(&file, env.work().join("archive/input.json")).unwrap(); + let out = child.wait_with_output().unwrap(); + assert!( + out.status.success(), + "{} / {}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + let events = rows(&out); + assert_eq!(events.iter().filter(|v| v["event"] == "task").count(), 2); + assert_eq!(events.last().unwrap()["failed"], 0); +} + +#[test] +fn correcting_a_malformed_file_clears_its_queue_error() { + use std::process::Stdio; + let env = TestEnv::new(); + setup(&env, "[]"); + env.write("work/queue/bad.json", "{"); + task_file(&env, "active", "active", 1); + let server = FakeServer::start( + vec![ScriptTurn::text("done"); 2], + FakeOpts { + hold_headers_ms: 700, + ..FakeOpts::default() + }, + ) + .unwrap(); + let child = command(&env, &server.url(), "2") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + until(|| has_active(&env, 1)); + task_file(&env, "bad", "corrected", 0); + let out = child.wait_with_output().unwrap(); + assert!( + out.status.success(), + "{} / {}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + assert_eq!(rows(&out).last().unwrap()["failed"], 0); +} + +#[test] +fn a_monitor_error_drains_admitted_workers_before_returning() { + use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }; + let env = TestEnv::new(); + setup(&env, "[]"); + task_file(&env, "input", "active", 1); + let finished = Arc::new(AtomicBool::new(false)); + let observed = finished.clone(); + let result = oxa_work::run_queue( + &env.work().join("project"), + &env.work().join("queue"), + &oxa_work::QueueOptions { + max: 1, + min_free_bytes: 0, + once: true, + }, + Arc::new(move |task| { + std::thread::sleep(Duration::from_millis(100)); + observed.store(true, Ordering::Release); + oxa_work::blocked_record(task, "synthetic worker result".into()) + }), + &mut |_| Err("synthetic monitor failure".into()), + ); + assert!(result.unwrap_err().contains("synthetic monitor failure")); + assert!( + finished.load(Ordering::Acquire), + "controller returned before its admitted worker finished" + ); +} diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index 72b67e50a..d3edfe4df 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -87,8 +87,10 @@ fn git(repo: &Path, args: &[&str]) -> Command { } pub fn read_tasks(path: &Path) -> Result> { - let tasks: Vec = serde_json::from_slice(&fs::read(path).map_err(|e| e.to_string())?) - .map_err(|e| format!("task file: {e}"))?; + parse_tasks(&fs::read(path).map_err(|e| e.to_string())?) +} +pub(crate) fn parse_tasks(bytes: &[u8]) -> Result> { + let tasks: Vec = serde_json::from_slice(bytes).map_err(|e| format!("task file: {e}"))?; if tasks.is_empty() { return Err("task file is empty".into()); } diff --git a/oxa/oxa-work/src/queue.rs b/oxa/oxa-work/src/queue.rs index 5d5872a98..c4b66ddb0 100644 --- a/oxa/oxa-work/src/queue.rs +++ b/oxa/oxa-work/src/queue.rs @@ -1,5 +1,5 @@ //! Directory queue: oldest regular JSON files first, with bounded independent workers. -use super::{blocked_record, checked, git, read_tasks, state, Record, Result, Task}; +use super::{blocked_record, checked, git, state, Record, Result, Task}; use serde::Serialize; use std::{ collections::{HashMap, HashSet}, @@ -97,47 +97,79 @@ fn free_space(repo: &Path, common: &Path, queue: &Path) -> Result { .min(available(queue)?) .min(available(tree_volume)?)) } +fn input_file(entry: fs::DirEntry) -> Result> { + let path = entry.path(); + if path + .file_name() + .is_some_and(|s| s.to_string_lossy().starts_with('.')) + || path.extension().is_none_or(|s| s != "json") + { + return Ok(None); + } + // A user may archive an input after read_dir produced this entry. + let meta = match entry.metadata() { + Ok(meta) => meta, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.to_string()), + }; + if !meta.is_file() { + return Ok(None); + } + Ok(Some(( + path, + meta.modified().map_err(|e| e.to_string())?, + meta.len(), + ))) +} +#[derive(Default)] +struct Inputs { + files: HashMap, + tasks: HashMap, + errors: HashMap>, +} fn scan( directory: &Path, pending: &mut Vec, - seen_files: &mut HashSet<(PathBuf, SystemTime, u64)>, - seen_ids: &mut HashSet, - failed: &mut usize, + inputs: &mut Inputs, emit: &mut dyn FnMut(QueueEvent) -> Result<()>, ) -> Result<()> { let mut files = vec![]; for entry in fs::read_dir(directory).map_err(|e| e.to_string())? { - let entry = entry.map_err(|e| e.to_string())?; - let path = entry.path(); - if !entry.file_type().map_err(|e| e.to_string())?.is_file() - || path - .file_name() - .is_some_and(|s| s.to_string_lossy().starts_with('.')) - || path.extension().is_none_or(|s| s != "json") - { - continue; + let entry = match entry { + Ok(entry) => entry, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => return Err(error.to_string()), + }; + if let Some(file) = input_file(entry)? { + files.push(file); } - let meta = entry.metadata().map_err(|e| e.to_string())?; - files.push((path, meta.modified().map_err(|e| e.to_string())?, meta.len())); } + let paths: HashSet<_> = files.iter().map(|(p, _, _)| p.clone()).collect(); + inputs.files.retain(|p, _| paths.contains(p)); + inputs.errors.retain(|p, _| paths.contains(p)); files.sort_by(|a, b| (&a.1, &a.0).cmp(&(&b.1, &b.0))); for (file, modified, len) in files { - if !seen_files.insert((file.clone(), modified, len)) { + if inputs.files.get(&file) == Some(&(modified, len)) && !inputs.errors.contains_key(&file) { continue; } - match read_tasks(&file) { + let tasks = match fs::read(&file) { + Ok(bytes) => super::parse_tasks(&bytes), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => Err(error.to_string()), + }; + let previous = inputs.errors.remove(&file); + let mut problems = vec![]; + match tasks { Ok(tasks) => { for (index, task) in tasks.into_iter().enumerate() { - if !seen_ids.insert(task.id.clone()) { - *failed += 1; - emit(QueueEvent::Error { - file: file.clone(), - detail: format!( - "task {} was already queued; use a new id for changed work", - task.id - ), - })?; + if let Some((source, known)) = inputs.tasks.get_mut(&task.id) { + if *known == task && (*source == file || !paths.contains(source)) { + *source = file.clone(); + continue; + } + problems.push(format!("task {} was already queued; use a new id for changed work or remove the duplicate input", task.id)); } else { + inputs.tasks.insert(task.id.clone(), (file.clone(), task.clone())); pending.push(Job { task, file: file.clone(), @@ -147,14 +179,36 @@ fn scan( } } } - Err(detail) => { - *failed += 1; - emit(QueueEvent::Error { file, detail })?; + Err(detail) => problems.push(detail), + } + inputs.files.insert(file.clone(), (modified, len)); + if !problems.is_empty() { + if previous.as_ref() != Some(&problems) { + for detail in &problems { + emit(QueueEvent::Error { + file: file.clone(), + detail: detail.clone(), + })?; + } } + inputs.errors.insert(file, problems); } } Ok(()) } +#[derive(Default)] +struct Workers { + handles: HashMap>, +} +impl Drop for Workers { + fn drop(&mut self) { + // Every error path drains admitted work before the CLI can exit. + for (_, handle) in self.handles.drain() { + let _ = handle.join(); + } + } +} + pub fn run_queue( repo: &Path, directory: &Path, @@ -176,19 +230,11 @@ pub fn run_queue( ))?); let storage = common.join("oxa-work"); let (tx, rx) = mpsc::channel(); - let mut active: HashMap> = HashMap::new(); - let (mut pending, mut external, mut seen_files, mut seen_ids) = - (vec![], vec![], HashSet::new(), HashSet::new()); + let mut active = Workers::default(); + let (mut pending, mut external, mut inputs) = (vec![], vec![], Inputs::default()); let (mut finished, mut failed, mut last) = (0, 0, None); loop { - scan( - &directory, - &mut pending, - &mut seen_files, - &mut seen_ids, - &mut failed, - emit, - )?; + scan(&directory, &mut pending, &mut inputs, emit)?; // Surviving workers from a dead controller still count toward the concurrency cap. let mut waiting = vec![]; for job in external.drain(..) { @@ -216,7 +262,7 @@ pub fn run_queue( Err(error) => (None, Some(error)), }; let mut disk_blocked = free_bytes.is_none_or(|free| free < options.min_free_bytes); - while !disk_blocked && active.len() + external.len() < options.max && !pending.is_empty() { + while !disk_blocked && active.handles.len() + external.len() < options.max && !pending.is_empty() { // Check again for each admission: earlier workers may already be writing. match free_space(repo, &common, &directory) { Ok(free) => { @@ -252,7 +298,7 @@ pub fn run_queue( let _ = tx.send((input, record)); }) { Ok(handle) => { - active.insert(id, handle); + active.handles.insert(id, handle); } Err(error) => { finished += 1; @@ -264,6 +310,7 @@ pub fn run_queue( } } let mut ids: Vec<_> = active + .handles .keys() .cloned() .chain(external.iter().map(|j| j.task.id.clone())) @@ -274,7 +321,7 @@ pub fn run_queue( active: ids, pending: pending.len(), finished, - failed, + failed: failed + inputs.errors.values().map(Vec::len).sum::(), free_bytes, disk_blocked: disk_blocked && !pending.is_empty(), disk_error, @@ -291,16 +338,20 @@ pub fn run_queue( })?; last = Some(status.clone()); } - if options.once && active.is_empty() && external.is_empty() && (pending.is_empty() || disk_blocked) { + if options.once + && active.handles.is_empty() + && external.is_empty() + && (pending.is_empty() || disk_blocked) + { return Ok(status); } - if active.is_empty() { + if active.handles.is_empty() { thread::sleep(Duration::from_millis(250)); continue; } match rx.recv_timeout(Duration::from_millis(250)) { Ok((job, record)) => { - if let Some(handle) = active.remove(&job.task.id) { + if let Some(handle) = active.handles.remove(&job.task.id) { let _ = handle.join(); } if busy(&storage, &job.task.id)? { @@ -318,3 +369,27 @@ pub fn run_queue( } } } + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn archiving_between_directory_read_and_metadata_is_not_a_controller_error() { + let dir = std::env::temp_dir().join(format!( + "oxa-queue-scan-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir(&dir).unwrap(); + let path = dir.join("task.json"); + fs::write(&path, "[]").unwrap(); + let entry = fs::read_dir(&dir).unwrap().next().unwrap().unwrap(); + fs::rename(path, dir.join("archived")).unwrap(); + let result = input_file(entry); + fs::remove_dir_all(dir).unwrap(); + assert!(result.unwrap().is_none()); + } +} From 06585e84acd71b7ed6e92d4c45b0ccc446e07ce8 Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 15:50:57 +0100 Subject: [PATCH 15/52] Keep the leading subcommand dispatch in parse_args instead of rewriting it `oxa --offline work ..` has to reach the work runner, and the previous change met that by replacing the whole leading subcommand block with an in-loop check. That deleted existing parser source for no behavioural gain and left the dispatch condition harder to read. Restore the block and add the one case the runner needs after it, scoped to `work`, so a command word after any other option stays an ordinary prompt word. This change removes nothing: the only other edit to existing source is the wider visibility of `report_cmd::tty_confirm`, which the work report workflow calls. No `Removes:` statement is needed. Tests: c12_argument_parsing pins that command words stay prompts after prompt words and options and that only a leading `--offline` dispatches `work`. fmt, clippy and the crate tests run in CI (#333). --- oxa/oxa-cli/src/args.rs | 26 +++++++++++++++++------ oxa/oxa-cli/tests/c12_argument_parsing.rs | 5 +++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/oxa/oxa-cli/src/args.rs b/oxa/oxa-cli/src/args.rs index 36e62773d..a895af4d1 100644 --- a/oxa/oxa-cli/src/args.rs +++ b/oxa/oxa-cli/src/args.rs @@ -458,17 +458,31 @@ fn positional(a: &mut Args, word: &str) { /// `unsupported_in_oxa`. pub fn parse_args(argv: &[String], ext_flags: &[FlagSpec]) -> Result { let mut a = Args::default(); + if let Some(first) = argv.first() { + if !first.starts_with('-') && !first.starts_with('@') { + if let Some(sub) = subcommand(first, &argv[1..])? { + a.subcommand = Some(sub); + return Ok(a); + } + } + } + // `oxa --offline work ..`: the global offline switch is the one option that may come + // before the work subcommand, because the runner passes the policy on to every child. + // Any other option first keeps the old meaning: `work` is then an ordinary prompt word. + let leading_offline = argv + .iter() + .take_while(|word| word.as_str() == "--offline") + .count(); + if leading_offline > 0 && argv.get(leading_offline).map(String::as_str) == Some("work") { + a.offline = true; + a.subcommand = Some(Subcommand::Work(argv[leading_offline + 1..].to_vec())); + return Ok(a); + } let mut i = 0; let mut stop = false; while i < argv.len() { let arg = argv[i].as_str(); i += 1; - if !stop && (i == 1 || (arg == "work" && argv[..i - 1].iter().all(|word| word == "--offline"))) { - if let Some(sub) = subcommand(arg, &argv[i..])? { - a.subcommand = Some(sub); - break; - } - } if stop || !arg.starts_with('-') { positional(&mut a, arg); continue; diff --git a/oxa/oxa-cli/tests/c12_argument_parsing.rs b/oxa/oxa-cli/tests/c12_argument_parsing.rs index c51fe7c6f..65b1f6f8b 100644 --- a/oxa/oxa-cli/tests/c12_argument_parsing.rs +++ b/oxa/oxa-cli/tests/c12_argument_parsing.rs @@ -418,6 +418,11 @@ fn agent_options_keep_command_words_as_prompts() { assert_eq!(a.subcommand, None, "{word}"); assert_eq!(a.messages, [word, "example"]); } + // Only option words may precede the work command; a prompt word before it keeps + // `work` a prompt word in place. + let a = parse(&["summarise", word, "example"]); + assert_eq!(a.subcommand, None, "{word}"); + assert_eq!(a.messages, ["summarise", word, "example"]); } } From a8e2efe27e65e75eb8bc2a3c31754a847bb1a808 Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 16:59:03 +0100 Subject: [PATCH 16/52] fix(work): preserve action and save retries on current checkpoints --- oxa/oxa-cli/src/work_cmd.rs | 7 +- oxa/oxa-cli/src/work_reports.rs | 33 ++++++++++ oxa/oxa-cli/tests/work_publish.rs | 105 +++++++----------------------- oxa/oxa-work/src/lib.rs | 32 ++++++++- 4 files changed, 94 insertions(+), 83 deletions(-) diff --git a/oxa/oxa-cli/src/work_cmd.rs b/oxa/oxa-cli/src/work_cmd.rs index b46a41a23..5747cefcb 100644 --- a/oxa/oxa-cli/src/work_cmd.rs +++ b/oxa/oxa-cli/src/work_cmd.rs @@ -185,7 +185,12 @@ pub(crate) fn run( .into_iter() .map(|task| { if is_trusted { - oxa_work::plan_task_for_run(&root, task.clone(), &policy, rerun) + let planner = if publish || reports { + oxa_work::plan_task_for_actions + } else { + oxa_work::plan_task_for_run + }; + planner(&root, task.clone(), &policy, rerun) .map_err(|detail| Box::new(oxa_work::blocked_record(&task, detail))) } else { Err(Box::new(oxa_work::blocked_record( diff --git a/oxa/oxa-cli/src/work_reports.rs b/oxa/oxa-cli/src/work_reports.rs index 7c5929632..6fff5ec63 100644 --- a/oxa/oxa-cli/src/work_reports.rs +++ b/oxa/oxa-cli/src/work_reports.rs @@ -298,6 +298,39 @@ mod tests { ) } #[test] + fn declining_visible_queries_prevents_any_github_request() { + let env = TestEnv::new(); + let dirs = Dirs::under(env.root()); + let mut value = note(); + value.keywords[0] = "component\u{1b}[2Jsymptom\u{200f}".into(); + let mut calls = 0; + let mut approved = false; + let result = report( + &value, + &dirs, + &settings(), + &ScrubCtx::default(), + false, + &mut |prompt| { + assert!(!prompt.contains('\u{1b}') && !prompt.contains('\u{200f}')); + assert!(prompt.contains("\\u{1b}") && prompt.contains("\\u{200f}")); + approved = true; + Some(false) + }, + &mut |program, _| { + if program == "gh" { + calls += 1; + } + Some((0, "[]".into())) + }, + ) + .unwrap(); + assert!(approved); + assert_eq!(calls, 0); + assert_eq!(result.disposition, "local"); + assert!(result.draft.exists()); + } + #[test] fn matching_issue_is_commented_after_all_queries_and_reads() { let env = TestEnv::new(); let dirs = Dirs::under(env.root()); diff --git a/oxa/oxa-cli/tests/work_publish.rs b/oxa/oxa-cli/tests/work_publish.rs index cd38c3091..b7ef61657 100644 --- a/oxa/oxa-cli/tests/work_publish.rs +++ b/oxa/oxa-cli/tests/work_publish.rs @@ -2,8 +2,17 @@ #[path = "common/work.rs"] mod work; use oxa_fake::{ScriptTurn, TestEnv}; +use sha2::{Digest, Sha256}; use std::{fs, path::PathBuf, process::Command}; use work::{git, run, run_args, run_observed, setup}; +fn state_path(env: &TestEnv) -> PathBuf { + let common = env.work().join("project/.git").canonicalize().unwrap(); + let key: String = Sha256::digest(common.as_os_str().as_encoded_bytes()) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + env.agent_dir().join("work").join(key).join("test.json") +} fn configure(env: &TestEnv) -> String { use std::os::unix::fs::PermissionsExt; env.write( @@ -24,17 +33,17 @@ case "$1 $2" in if test "$arg" = '--draft'; then draft=true; fi if test "$arg" = '--body-file'; then body_next=true; fi done - printf '{"number":1,"url":"https://github.com/example/project/pull/1","isDraft":%s}' "$draft" > "$HOME/pr-state.json" + printf '{"number":1,"url":"https://github.com/example/project/pull/1","isDraft":%s,"headRepositoryOwner":{"login":"example"}}' "$draft" > "$HOME/pr-state.json" printf '%s' 'https://github.com/example/project/pull/1' ;; 'pr edit') : ;; 'pr ready') draft=false for arg in "$@"; do if test "$arg" = '--undo'; then draft=true; fi; done - printf '{"number":1,"url":"https://github.com/example/project/pull/1","isDraft":%s}' "$draft" > "$HOME/pr-state.json" ;; + printf '{"number":1,"url":"https://github.com/example/project/pull/1","isDraft":%s,"headRepositoryOwner":{"login":"example"}}' "$draft" > "$HOME/pr-state.json" ;; *) exit 7 ;; esac "##); - let git_wrapper = env.write("tmp/git", "#!/bin/sh\nif test \"$1\" = push; then printf 'push\\n' >> \"$HOME/push-calls\"; fi\nexec /usr/bin/git \"$@\"\n"); + let git_wrapper = env.write("tmp/git", "#!/bin/sh\nfor arg do if test \"$arg\" = push; then printf 'push\\n' >> \"$HOME/push-calls\"; fi; done\nexec /usr/bin/git \"$@\"\n"); let controller = env.write("tmp/oxc", "#!/bin/sh\nprintf '{}'\n"); for file in [gh, git_wrapper, controller] { fs::set_permissions(file, fs::Permissions::from_mode(0o755)).unwrap(); @@ -83,7 +92,7 @@ fn publishes_local_branch_and_updates_same_pr_on_restart() { assert!(!body.contains("Finish task")); } #[test] -fn red_checks_create_a_draft_pr() { +fn red_checks_never_push_or_create_a_pr() { let env = TestEnv::new(); setup(&env, r#"["printf red > failed-check; exit 1"]"#); let path = configure(&env); @@ -94,8 +103,10 @@ fn red_checks_create_a_draft_pr() { Some(path), ); assert_eq!(record["outcome"], "red", "{record}"); - assert_eq!(record["draft"], true); - assert!(record["pull_request"].is_string()); + assert_eq!(record["draft"], false); + assert!(record["pull_request"].is_null()); + assert!(!env.home().join("gh-calls").exists()); + assert!(!env.home().join("push-calls").exists()); } #[test] fn queued_pr_is_never_pushed_and_publication_retries_without_agent_replay() { @@ -200,7 +211,7 @@ fn a_model_tool_cannot_opt_itself_into_reporting_or_publishing() { } #[test] -fn green_rerun_marks_an_existing_red_draft_ready() { +fn green_rerun_can_publish_after_red_remained_local() { let env = TestEnv::new(); setup( &env, @@ -213,7 +224,8 @@ fn green_rerun_marks_an_existing_red_draft_ready() { &["work", "../tasks.json", "--publish"], Some(path.clone()), ); - assert_eq!(red["draft"], true, "{red}"); + assert_eq!(red["draft"], false, "{red}"); + assert!(red["pull_request"].is_null()); let green = run_args( &env, vec![ @@ -227,7 +239,7 @@ fn green_rerun_marks_an_existing_red_draft_ready() { assert_eq!(green["draft"], false); assert!(fs::read_to_string(env.home().join("gh-calls")) .unwrap() - .contains("pr ready")); + .contains("pr create")); } #[test] fn a_local_report_retries_without_replaying_the_completed_task() { @@ -255,48 +267,6 @@ fn a_local_report_retries_without_replaying_the_completed_task() { assert_eq!(retry["step"], "finished"); } -#[test] -fn unfinished_execution_keeps_its_saved_offline_restriction() { - use oxa_fake::{FakeModel, FakeOpts, FakeServer}; - let env = TestEnv::new(); - setup(&env, "[]"); - run_args( - &env, - vec![ScriptTurn::text("done")], - &["--offline", "work", "../tasks.json"], - None, - ); - let state_path = env.work().join("project/.git/oxa-work/test.json"); - let mut state: serde_json::Value = serde_json::from_slice(&fs::read(&state_path).unwrap()).unwrap(); - state["step"] = "agent".into(); - state["outcome"] = "running".into(); - fs::write(&state_path, serde_json::to_vec(&state).unwrap()).unwrap(); - let server = FakeServer::start( - vec![ScriptTurn::text("network was allowed")], - FakeOpts { - remote: Some("openrouter"), - models: vec![FakeModel { - alias: "fake-model".into(), - cap: 64000, - max_output: Some(4096), - caps: None, - }], - ..FakeOpts::default() - }, - ) - .unwrap(); - let out = Command::new(env!("CARGO_BIN_EXE_oxa")) - .args(["work", "../tasks.json"]) - .current_dir(env.work().join("project")) - .env_clear() - .envs(env.envs(&server.url())) - .output() - .unwrap(); - let record: serde_json::Value = serde_json::from_slice(&out.stdout).unwrap(); - assert_eq!(record["outcome"], "blocked", "{record}"); - assert!(record["detail"].as_str().unwrap().contains("offline")); - assert!(server.requests().is_empty()); -} #[test] fn later_publication_and_reporting_do_not_erase_execution_policy() { let env = TestEnv::new(); @@ -308,7 +278,7 @@ fn later_publication_and_reporting_do_not_erase_execution_policy() { &["--offline", "work", "../tasks.json"], Some(path.clone()), ); - let state_path = env.work().join("project/.git/oxa-work/test.json"); + let state_path = state_path(&env); for flag in ["--publish", "--report-problems"] { let record = run_args(&env, vec![], &["work", "../tasks.json", flag], Some(path.clone())); assert_eq!(record["outcome"], "done", "{record}"); @@ -321,17 +291,16 @@ fn optional_publication_metadata_defaults_to_inactive() { let env = TestEnv::new(); setup(&env, "[]"); run(&env, vec![ScriptTurn::text("done")]); - let state_path = env.work().join("project/.git/oxa-work/test.json"); + let state_path = state_path(&env); let mut state: serde_json::Value = serde_json::from_slice(&fs::read(&state_path).unwrap()).unwrap(); for key in [ - "head", "pull_request", "draft", "problem_notes", "reports", "acceptance_outcome", ] { - state.as_object_mut().unwrap().remove(key); + state["record"].as_object_mut().unwrap().remove(key); } fs::write(&state_path, serde_json::to_vec(&state).unwrap()).unwrap(); let record = run(&env, vec![]); @@ -418,29 +387,3 @@ esac "public writes without confirmation: {unauthorized:?}" ); } - -#[test] -fn rerun_resumes_interrupted_execution_without_discarding_its_checkpoint() { - for step in ["acceptance", "recheck", "saving"] { - let env = TestEnv::new(); - setup(&env, r#"["printf checked >> checkpoint-checks"]"#); - let first = run(&env, edits()); - let state_path = env.work().join("project/.git/oxa-work/test.json"); - let mut state: serde_json::Value = serde_json::from_slice(&fs::read(&state_path).unwrap()).unwrap(); - state["step"] = step.into(); - state["outcome"] = if step == "saving" { "done" } else { "running" }.into(); - state["pull_request"] = "https://github.com/example/project/pull/1".into(); - state["problem_notes"] = serde_json::json!([{"title":"Synthetic problem","notes":"Keep these notes","keywords":["synthetic problem","keep notes"]}]); - state["reports"] = serde_json::json!([{"draft":"report.md","disposition":"existing_issue","url":"https://github.com/example/project/issues/1","detail":""}]); - fs::write(&state_path, serde_json::to_vec(&state).unwrap()).unwrap(); - let (record, requests) = run_observed(&env, vec![], &["work", "../tasks.json", "--rerun"], None); - assert_eq!(record["outcome"], "done", "{step}: {record}"); - assert!(requests.is_empty(), "{step}: agent was replayed"); - for key in ["pull_request", "problem_notes", "reports"] { - assert_eq!(record[key], state[key], "{step}: {key}"); - } - let after: serde_json::Value = serde_json::from_slice(&fs::read(&state_path).unwrap()).unwrap(); - assert_eq!(after["checkpoint"], state["checkpoint"], "{step}"); - assert_eq!(record["worktree"], first["worktree"]); - } -} diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index cdf682c72..9e9496294 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -131,6 +131,27 @@ pub fn plan_task(repo: &Path, task: Task, policy: &RunPolicy) -> Result Result { + plan_task_mode(repo, task, policy, rerun, false) +} + +/// Explicit actions reuse completed execution inputs, including red results, +/// without treating a report/publication retry as another model attempt. +pub fn plan_task_for_actions( + repo: &Path, + task: Task, + policy: &RunPolicy, + rerun: bool, +) -> Result { + plan_task_mode(repo, task, policy, rerun, true) +} + +fn plan_task_mode( + repo: &Path, + task: Task, + policy: &RunPolicy, + rerun: bool, + actions: bool, +) -> Result { validate(&task)?; let path = policy.storage.join(format!("{}.json", task.id)); match fs::symlink_metadata(&path) { @@ -146,7 +167,7 @@ pub fn plan_task_for_run(repo: &Path, task: Task, policy: &RunPolicy, rerun: boo return Err("task specification changed; use a new task id".into()); } if saved.record.step != Step::Finished - || (!rerun && matches!(saved.record.outcome, Outcome::Done | Outcome::NothingToDo)) + || (!rerun && (actions || matches!(saved.record.outcome, Outcome::Done | Outcome::NothingToDo))) { let execution = saved.execution.ok_or("missing task execution checkpoint")?; if saved.record.base_sha.as_deref() != Some(&execution.base) { @@ -652,6 +673,10 @@ pub fn run_task_with_observer( state.record.detail = error; } } + state + .record + .acceptance_outcome + .get_or_insert(state.record.outcome); writer .save(&mut state, Step::Saving, observer) .map_err(RunError::Checkpoint)?; @@ -663,7 +688,12 @@ pub fn run_task_with_observer( state.record.detail.push('\n'); } state.record.detail.push_str(&format!("save: {error}")); + writer + .save(&mut state, Step::Saving, observer) + .map_err(RunError::Checkpoint)?; + return Ok(state.record); } + state.record.outcome = state.record.acceptance_outcome.unwrap_or(state.record.outcome); writer .save(&mut state, Step::Finished, observer) .map_err(RunError::Checkpoint)?; From f1b5ec32da65d85ab49d2a8d7d895598914ef0ea Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:00:35 +0100 Subject: [PATCH 17/52] test(work): reject dispatch of an archived pending input --- oxa/oxa-work/src/queue.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/oxa/oxa-work/src/queue.rs b/oxa/oxa-work/src/queue.rs index c4b66ddb0..6cea11af4 100644 --- a/oxa/oxa-work/src/queue.rs +++ b/oxa/oxa-work/src/queue.rs @@ -374,6 +374,33 @@ pub fn run_queue( mod tests { use super::*; #[test] + fn archiving_an_unadmitted_file_cancels_its_pending_job() { + let dir = std::env::temp_dir().join(format!( + "oxa-queue-pending-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir(&dir).unwrap(); + let path = dir.join("task.json"); + fs::write( + &path, + r#"[{"id":"queued","branch":"agent/queued","model":"fake","prompt":"Synthetic task"}]"#, + ) + .unwrap(); + let mut pending = vec![]; + let mut inputs = Inputs::default(); + scan(&dir, &mut pending, &mut inputs, &mut |_| Ok(())).unwrap(); + assert_eq!(pending.len(), 1); + fs::rename(&path, dir.join("archived")).unwrap(); + scan(&dir, &mut pending, &mut inputs, &mut |_| Ok(())).unwrap(); + fs::remove_dir_all(&dir).unwrap(); + assert!(pending.is_empty(), "archived input must not be dispatched"); + assert!(!inputs.tasks.contains_key("queued")); + } + #[test] fn archiving_between_directory_read_and_metadata_is_not_a_controller_error() { let dir = std::env::temp_dir().join(format!( "oxa-queue-scan-{}-{}", From 494cfff64081f2279aab57fc203371f0f6892a79 Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:14:35 +0100 Subject: [PATCH 18/52] fix(work): clear completed save outcome before later execution --- oxa/oxa-cli/tests/work_publish.rs | 7 +++++-- oxa/oxa-work/src/lib.rs | 17 +++++++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/oxa/oxa-cli/tests/work_publish.rs b/oxa/oxa-cli/tests/work_publish.rs index b7ef61657..d3aa1d52c 100644 --- a/oxa/oxa-cli/tests/work_publish.rs +++ b/oxa/oxa-cli/tests/work_publish.rs @@ -4,7 +4,7 @@ mod work; use oxa_fake::{ScriptTurn, TestEnv}; use sha2::{Digest, Sha256}; use std::{fs, path::PathBuf, process::Command}; -use work::{git, run, run_args, run_observed, setup}; +use work::{git, run, run_args, setup}; fn state_path(env: &TestEnv) -> PathBuf { let common = env.work().join("project/.git").canonicalize().unwrap(); let key: String = Sha256::digest(common.as_os_str().as_encoded_bytes()) @@ -127,7 +127,10 @@ fn queued_pr_is_never_pushed_and_publication_retries_without_agent_replay() { Some(path.clone()), ); assert_eq!(blocked["outcome"], "blocked", "{blocked}"); - assert!(blocked["detail"].as_str().unwrap().contains("merge queue")); + assert!( + blocked["detail"].as_str().unwrap().contains("merge queue"), + "{blocked}" + ); assert_eq!( fs::read_to_string(env.home().join("push-calls")) .unwrap() diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index 9e9496294..2b4b7f4a3 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -688,12 +688,25 @@ pub fn run_task_with_observer( state.record.detail.push('\n'); } state.record.detail.push_str(&format!("save: {error}")); + let retry = if matches!( + state.record.acceptance_outcome, + Some(Outcome::Done | Outcome::NothingToDo | Outcome::Red) + ) { + Step::Saving + } else { + // An unfinished merge/worker still needs another execution attempt. + Step::Finished + }; writer - .save(&mut state, Step::Saving, observer) + .save(&mut state, retry, observer) .map_err(RunError::Checkpoint)?; return Ok(state.record); } - state.record.outcome = state.record.acceptance_outcome.unwrap_or(state.record.outcome); + state.record.outcome = state + .record + .acceptance_outcome + .take() + .unwrap_or(state.record.outcome); writer .save(&mut state, Step::Finished, observer) .map_err(RunError::Checkpoint)?; From 153804acc918b45ce7ed6dd3a73b1a9c55637681 Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:14:37 +0100 Subject: [PATCH 19/52] test(work): retain queue drain control with trusted runner inputs --- oxa/oxa-cli/tests/work_queue.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/oxa/oxa-cli/tests/work_queue.rs b/oxa/oxa-cli/tests/work_queue.rs index d6ce02bff..326685b84 100644 --- a/oxa/oxa-cli/tests/work_queue.rs +++ b/oxa/oxa-cli/tests/work_queue.rs @@ -497,6 +497,13 @@ fn a_monitor_error_drains_admitted_workers_before_returning() { task_file(&env, "input", "active", 1); let finished = Arc::new(AtomicBool::new(false)); let observed = finished.clone(); + let policy = oxa_work::RunPolicy { + git_binary: "/usr/bin/git".into(), + host_path: "/usr/bin:/bin".into(), + terminate_binary: None, + offline: true, + storage: env.agent_dir().join("work/monitor-fixture"), + }; let result = oxa_work::run_queue( &env.work().join("project"), &env.work().join("queue"), @@ -505,10 +512,12 @@ fn a_monitor_error_drains_admitted_workers_before_returning() { min_free_bytes: 0, once: true, }, + &policy, + std::path::Path::new("/usr/bin/df"), Arc::new(move |task| { std::thread::sleep(Duration::from_millis(100)); observed.store(true, Ordering::Release); - oxa_work::blocked_record(task, "synthetic worker result".into()) + Ok(oxa_work::blocked_record(task, "synthetic worker result".into())) }), &mut |_| Err("synthetic monitor failure".into()), ); From 02d68898879bf17733586eef580b177541299a1b Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:26:56 +0100 Subject: [PATCH 20/52] fix(work): authorize only confirmed publication transports --- oxa/oxa-cli/tests/work_publish.rs | 92 +++++++++++++++++++++++++++++++ oxa/oxa-work/src/publish.rs | 5 ++ 2 files changed, 97 insertions(+) diff --git a/oxa/oxa-cli/tests/work_publish.rs b/oxa/oxa-cli/tests/work_publish.rs index d3aa1d52c..f37576a30 100644 --- a/oxa/oxa-cli/tests/work_publish.rs +++ b/oxa/oxa-cli/tests/work_publish.rs @@ -57,10 +57,101 @@ fn edits() -> Vec { ] } #[test] +fn a_real_save_failure_retries_without_another_worker() { + const CHILD: &str = "OXA_SAVE_RECOVERY_TEST_ROOT"; + if let Some(root) = std::env::var_os(CHILD) { + use std::sync::{ + atomic::{AtomicBool, Ordering}, + Mutex, + }; + let root = PathBuf::from(root); + let project = root.join("work/project"); + let task = oxa_work::read_tasks(&root.join("work/tasks.json")) + .unwrap() + .remove(0); + let policy = oxa_work::RunPolicy { + git_binary: "/usr/bin/git".into(), + host_path: std::env::var_os("PATH").unwrap(), + terminate_binary: Some("/usr/bin/kill".into()), + offline: false, + storage: root.join("home/.oxa/agent/work/save-recovery"), + }; + let binary = std::path::Path::new(env!("CARGO_BIN_EXE_oxa")); + let injected = AtomicBool::new(false); + let lock_path = Mutex::new(None); + let observe = |record: &oxa_work::Record| { + if record.step == oxa_work::Step::Saving && !injected.swap(true, Ordering::SeqCst) { + let out = Command::new("/usr/bin/git") + .current_dir(&record.worktree) + .args(["rev-parse", "--path-format=absolute", "--git-path", "index"]) + .output() + .map_err(|error| error.to_string())?; + if !out.status.success() { + return Err("cannot locate synthetic index".into()); + } + let path = PathBuf::from(format!("{}.lock", String::from_utf8_lossy(&out.stdout).trim())); + fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + .map_err(|error| error.to_string())?; + *lock_path.lock().unwrap() = Some(path); + } + Ok(()) + }; + let plan = oxa_work::plan_task(&project, task.clone(), &policy).unwrap(); + let first = + oxa_work::run_task_with_observer(&project, binary, &plan, &policy, false, &observe).unwrap(); + assert_eq!(first.outcome, oxa_work::Outcome::Blocked, "{first:?}"); + assert_eq!(first.step, oxa_work::Step::Saving, "{first:?}"); + assert!(matches!( + first.worker.as_ref().unwrap().outcome, + oxa_work::WorkerOutcome::Done + )); + fs::remove_file(lock_path.into_inner().unwrap().unwrap()).unwrap(); + let plan = oxa_work::plan_task_for_run(&project, task, &policy, false).unwrap(); + let retry = oxa_work::run_task_with_policy(&project, binary, &plan, &policy).unwrap(); + assert_eq!(retry.outcome, oxa_work::Outcome::Done, "{retry:?}"); + assert_eq!(retry.step, oxa_work::Step::Finished); + assert_eq!(retry.session_id, first.session_id); + let saved = Command::new("/usr/bin/git") + .current_dir(&retry.worktree) + .args(["show", "HEAD:answer.txt"]) + .output() + .unwrap(); + assert!(saved.status.success()); + assert_eq!(String::from_utf8(saved.stdout).unwrap(), "published"); + return; + } + let env = TestEnv::new(); + setup(&env, "[]"); + let server = oxa_fake::FakeServer::start(edits(), oxa_fake::FakeOpts::default()).unwrap(); + let out = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "a_real_save_failure_retries_without_another_worker"]) + .env_clear() + .envs(env.envs(&server.url())) + .env(CHILD, env.root()) + .output() + .unwrap(); + assert!( + out.status.success(), + "{} / {}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + assert_eq!(server.requests().len(), 2, "save retry must not call a model"); +} +#[test] fn publishes_local_branch_and_updates_same_pr_on_restart() { + use std::os::unix::fs::PermissionsExt; let env = TestEnv::new(); setup(&env, "[]"); let path = configure(&env); + let hook = env.write( + "work/project/.git/hooks/pre-push", + "#!/bin/sh\ntouch \"$HOME/publish-hook-ran\"\n", + ); + fs::set_permissions(hook, fs::Permissions::from_mode(0o755)).unwrap(); let first = run_args( &env, edits(), @@ -68,6 +159,7 @@ fn publishes_local_branch_and_updates_same_pr_on_restart() { Some(path.clone()), ); assert_eq!(first["outcome"], "done", "{first}"); + assert!(!env.home().join("publish-hook-ran").exists()); assert_eq!(first["draft"], false); assert_eq!(first["pull_request"], "https://github.com/example/project/pull/1"); assert_eq!( diff --git a/oxa/oxa-work/src/publish.rs b/oxa/oxa-work/src/publish.rs index 46adc889a..2b6635700 100644 --- a/oxa/oxa-work/src/publish.rs +++ b/oxa/oxa-work/src/publish.rs @@ -186,6 +186,8 @@ pub fn publish( let mut push = git( repo, &[ + "-c", + "core.hooksPath=/dev/null", "-c", "protocol.https.allow=always", "-c", @@ -198,6 +200,9 @@ pub fn publish( ], ); push.env("GIT_TERMINAL_PROMPT", "0") + // Metadata probes deny every transport. Only this explicit, + // confirmed publication may use the normal GitHub/file transports. + .env("GIT_ALLOW_PROTOCOL", "https:ssh:file") .stdin(lock.try_clone().map_err(|e| e.to_string())?); let pushed = push.output().map_err(|e| e.to_string())?; if !pushed.status.success() { From 8e654696d6bba36a1711c5d5004642d3b7dee696 Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:26:59 +0100 Subject: [PATCH 21/52] fix(work): withdraw archived inputs before queue admission --- oxa/oxa-cli/tests/work_queue.rs | 37 +++++++++++++++++++++++---------- oxa/oxa-work/src/queue.rs | 8 +++++++ 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/oxa/oxa-cli/tests/work_queue.rs b/oxa/oxa-cli/tests/work_queue.rs index 326685b84..d292ddbf2 100644 --- a/oxa/oxa-cli/tests/work_queue.rs +++ b/oxa/oxa-cli/tests/work_queue.rs @@ -4,12 +4,21 @@ mod confirm; #[path = "common/work_repo.rs"] mod work; use oxa_fake::{FakeOpts, FakeServer, ScriptTurn, TestEnv}; +use sha2::{Digest, Sha256}; use std::{ fs, process::{Command, Output}, time::{Duration, Instant, SystemTime}, }; use work::{git, setup}; +fn storage(env: &TestEnv) -> std::path::PathBuf { + let common = env.work().join("project/.git").canonicalize().unwrap(); + let key: String = Sha256::digest(common.as_os_str().as_encoded_bytes()) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + env.agent_dir().join("work").join(key) +} fn task_file(env: &TestEnv, file: &str, id: &str, age: u64) { let path = env.write(&format!("work/queue/{file}.json"), &serde_json::json!([{"id":id,"model":"fake","branch":format!("agent/{id}"),"prompt":format!("{id} task")}]).to_string()); fs::OpenOptions::new() @@ -149,7 +158,7 @@ fn low_disk_never_starts_a_task_and_leaves_the_file_pending() { let status = events.last().unwrap(); assert_eq!(status["disk_blocked"], true); assert_eq!(status["pending"], 1); - assert!(!env.work().join("project/.git/oxa-work/low.json").exists()); + assert!(!storage(&env).join("low.json").exists()); assert!(env.work().join("queue/task.json").exists()); } #[test] @@ -171,11 +180,12 @@ fn a_surviving_worker_counts_against_max_until_its_lock_releases() { setup(&env, "[]"); task_file(&env, "held", "held", 20); task_file(&env, "next", "next", 10); - env.write("work/project/.git/oxa-work/held.lock", ""); + fs::create_dir_all(storage(&env)).unwrap(); + fs::write(storage(&env).join("held.lock"), "").unwrap(); let file = fs::OpenOptions::new() .read(true) .write(true) - .open(env.work().join("project/.git/oxa-work/held.lock")) + .open(storage(&env).join("held.lock")) .unwrap(); file.try_lock().unwrap(); let server = FakeServer::start(vec![ScriptTurn::text("done"); 2], FakeOpts::default()).unwrap(); @@ -218,7 +228,7 @@ fn files_arriving_while_a_worker_runs_are_picked_up_before_once_exits() { until(|| !server.requests().is_empty()); task_file(&env, "arrived", "arrived", 1); assert!(child.wait().unwrap().success()); - let path = env.work().join("project/.git/oxa-work/arrived.json"); + let path = storage(&env).join("arrived.json"); assert!(path.exists()); } @@ -295,21 +305,26 @@ fn preparing_git_keeps_ownership_after_a_queue_controller_is_killed() { let env = TestEnv::new(); setup(&env, "[]"); task_file(&env, "preparing", "preparing", 1); + let tree = env.work().join(".oxa-worktrees/project/preparing"); + env.write( + "work/project/.git/info/exclude", + ".prep-started\n.release-prep\n.prep-done\n", + ); let release = Release { - release: env.home().join("release-prep"), - done: env.home().join("prep-done"), + release: tree.join(".release-prep"), + done: tree.join(".prep-done"), }; let hook = env.write( "work/project/.git/hooks/post-checkout", r#"#!/bin/sh -touch "$HOME/prep-started" +touch .prep-started attempt=0 -while ! test -f "$HOME/release-prep"; do +while ! test -f .release-prep; do attempt=$((attempt + 1)) if test "$attempt" -ge 500; then exit 9; fi sleep 0.01 done -touch "$HOME/prep-done" +touch .prep-done "#, ); fs::set_permissions(hook, fs::Permissions::from_mode(0o755)).unwrap(); @@ -319,10 +334,10 @@ touch "$HOME/prep-done" .stderr(std::process::Stdio::null()) .spawn() .unwrap(); - until(|| env.home().join("prep-started").exists()); + until(|| tree.join(".prep-started").exists()); child.kill().unwrap(); child.wait().unwrap(); - let lock_path = env.work().join("project/.git/oxa-work/preparing.lock"); + let lock_path = storage(&env).join("preparing.lock"); let probe = fs::OpenOptions::new() .read(true) .write(true) diff --git a/oxa/oxa-work/src/queue.rs b/oxa/oxa-work/src/queue.rs index 6f998f5a9..5dc6f0994 100644 --- a/oxa/oxa-work/src/queue.rs +++ b/oxa/oxa-work/src/queue.rs @@ -148,6 +148,14 @@ fn scan( } } let paths: HashSet<_> = files.iter().map(|(p, _, _)| p.clone()).collect(); + let withdrawn: HashSet<_> = pending + .iter() + .filter(|job| !paths.contains(&job.file)) + .map(|job| job.task.id.clone()) + .collect(); + pending.retain(|job| paths.contains(&job.file)); + // Active/external workers are not pending and retain their identities. + inputs.tasks.retain(|id, _| !withdrawn.contains(id)); inputs.files.retain(|p, _| paths.contains(p)); inputs.errors.retain(|p, _| paths.contains(p)); files.sort_by(|a, b| (&a.1, &a.0).cmp(&(&b.1, &b.0))); From 1b9e38a19a1fd9cfff289018b5a799c1a264f81c Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:03:28 +0100 Subject: [PATCH 22/52] test(work): expose note privacy and publication metadata regressions --- oxa/oxa-cli/tests/work_publish.rs | 90 +++++++++++++++++++++++++++++-- 1 file changed, 85 insertions(+), 5 deletions(-) diff --git a/oxa/oxa-cli/tests/work_publish.rs b/oxa/oxa-cli/tests/work_publish.rs index f37576a30..f0badf211 100644 --- a/oxa/oxa-cli/tests/work_publish.rs +++ b/oxa/oxa-cli/tests/work_publish.rs @@ -21,8 +21,10 @@ fn configure(env: &TestEnv) -> String { ); let gh = env.write("tmp/gh", r##"#!/bin/sh printf '%s\n' "$*" >> "$HOME/gh-calls" +host=github.com +if test -f "$HOME/enterprise-host"; then host=code.example.invalid; fi case "$1 $2" in - 'repo view') printf '%s' '{"nameWithOwner":"example/project"}' ;; + 'repo view') printf '{"nameWithOwner":"example/project","url":"https://%s/example/project"}' "$host" ;; 'pr list') if test -f "$HOME/pr-state.json"; then printf '['; cat "$HOME/pr-state.json"; printf ']'; else printf '[]'; fi ;; 'api graphql') if test -f "$HOME/queued"; then printf '%s' '{"data":{"repository":{"pullRequest":{"mergeQueueEntry":{"id":"queue"}}}}}'; else printf '%s' '{"data":{"repository":{"pullRequest":{"mergeQueueEntry":null}}}}'; fi ;; 'pr create') @@ -33,13 +35,19 @@ case "$1 $2" in if test "$arg" = '--draft'; then draft=true; fi if test "$arg" = '--body-file'; then body_next=true; fi done - printf '{"number":1,"url":"https://github.com/example/project/pull/1","isDraft":%s,"headRepositoryOwner":{"login":"example"}}' "$draft" > "$HOME/pr-state.json" - printf '%s' 'https://github.com/example/project/pull/1' ;; - 'pr edit') : ;; + printf '{"number":1,"url":"https://%s/example/project/pull/1","isDraft":%s,"headRepositoryOwner":{"login":"example"}}' "$host" "$draft" > "$HOME/pr-state.json" + printf 'https://%s/example/project/pull/1' "$host" ;; + 'pr view') jq -Rs '{body:.}' "$HOME/published-body" ;; + 'pr edit') + body_next=false + for arg in "$@"; do + if test "$body_next" = true; then cat "$arg" > "$HOME/published-body"; body_next=false; fi + if test "$arg" = '--body-file'; then body_next=true; fi + done ;; 'pr ready') draft=false for arg in "$@"; do if test "$arg" = '--undo'; then draft=true; fi; done - printf '{"number":1,"url":"https://github.com/example/project/pull/1","isDraft":%s,"headRepositoryOwner":{"login":"example"}}' "$draft" > "$HOME/pr-state.json" ;; + printf '{"number":1,"url":"https://%s/example/project/pull/1","isDraft":%s,"headRepositoryOwner":{"login":"example"}}' "$host" "$draft" > "$HOME/pr-state.json" ;; *) exit 7 ;; esac "##); @@ -184,6 +192,68 @@ fn publishes_local_branch_and_updates_same_pr_on_restart() { assert!(!body.contains("Finish task")); } #[test] +fn publication_keeps_the_origin_host_for_prs_and_graphql() { + let env = TestEnv::new(); + setup(&env, "[]"); + let path = configure(&env); + env.write("home/enterprise-host", "synthetic"); + let first = run_args( + &env, + edits(), + &["work", "../tasks.json", "--publish"], + Some(path.clone()), + ); + assert_eq!(first["outcome"], "done", "{first}"); + let again = run_args(&env, vec![], &["work", "../tasks.json", "--publish"], Some(path)); + assert_eq!(again["outcome"], "done", "{again}"); + let calls = fs::read_to_string(env.home().join("gh-calls")).unwrap(); + assert!( + calls.contains("pr list --repo code.example.invalid/example/project"), + "{calls}" + ); + assert!( + calls.contains("pr create --repo code.example.invalid/example/project"), + "{calls}" + ); + assert!( + calls.contains("api graphql --hostname code.example.invalid"), + "{calls}" + ); +} + +#[test] +fn rerun_updates_managed_metadata_without_erasing_owner_body() { + let env = TestEnv::new(); + setup(&env, "[]"); + let path = configure(&env); + let first = run_args( + &env, + edits(), + &["work", "../tasks.json", "--publish"], + Some(path.clone()), + ); + assert_eq!(first["outcome"], "done", "{first}"); + let body_path = env.home().join("published-body"); + let body = + fs::read_to_string(&body_path).unwrap() + "\nOwner what/why and validated controls stay here.\n"; + fs::write(&body_path, body).unwrap(); + let next = run_args( + &env, + vec![ + ScriptTurn::tool("update", "write", r#"{"path":"answer.txt","content":"updated"}"#), + ScriptTurn::text("done"), + ], + &["work", "../tasks.json", "--rerun", "--publish"], + Some(path), + ); + assert_eq!(next["outcome"], "done", "{next}"); + assert_ne!(next["head_sha"], first["head_sha"]); + let body = fs::read_to_string(body_path).unwrap(); + assert!(body.contains("Owner what/why and validated controls stay here.")); + assert!(body.contains(next["head_sha"].as_str().unwrap()), "{body}"); + assert!(!body.contains(first["head_sha"].as_str().unwrap()), "{body}"); +} +#[test] fn red_checks_never_push_or_create_a_pr() { let env = TestEnv::new(); setup(&env, r#"["printf red > failed-check; exit 1"]"#); @@ -281,6 +351,16 @@ fn own_problem_notes_are_captured_and_failed_search_keeps_a_local_draft() { Some(path), ); assert_eq!(record["problem_notes"].as_array().unwrap().len(), 1); + assert!( + !serde_json::to_string(&record).unwrap().contains("token=example"), + "unscrubbed record output" + ); + assert!( + !fs::read_to_string(state_path(&env)) + .unwrap() + .contains("token=example"), + "unscrubbed checkpoint" + ); assert_eq!(record["reports"][0]["disposition"], "local"); let draft = fs::read_to_string(record["reports"][0]["draft"].as_str().unwrap()).unwrap(); assert!(!draft.contains("token=example")); From a7db5ecb4995c32b764e28f2fe27338bd6d12373 Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:14:44 +0100 Subject: [PATCH 23/52] fix(work): sanitize notes and retain publication host and owner body --- oxa/oxa-cli/src/work_cmd.rs | 4 +- oxa/oxa-cli/src/work_reports.rs | 22 +++++++ oxa/oxa-cli/tests/work_publish.rs | 2 +- oxa/oxa-work/src/execute.rs | 7 ++- oxa/oxa-work/src/lib.rs | 16 +++++ oxa/oxa-work/src/publish.rs | 97 +++++++++++++++++++++++++++---- 6 files changed, 133 insertions(+), 15 deletions(-) diff --git a/oxa/oxa-cli/src/work_cmd.rs b/oxa/oxa-cli/src/work_cmd.rs index 5747cefcb..7aabc060b 100644 --- a/oxa/oxa-cli/src/work_cmd.rs +++ b/oxa/oxa-cli/src/work_cmd.rs @@ -212,13 +212,15 @@ pub(crate) fn run( let mut record = if let Some(record) = cached { record } else { - match oxa_work::run_task_with_observer( + let ctx = crate::report_cmd::ScrubCtx::from_env(env, dirs); + match oxa_work::run_task_with_notes( &root, &binary, &plan, &policy, rerun, &|_: &oxa_work::Record| Ok(()), + &|note| crate::work_reports::scrub_note(note, &ctx), ) { Ok(record) => record, Err(oxa_work::RunError::Blocked(detail)) => { diff --git a/oxa/oxa-cli/src/work_reports.rs b/oxa/oxa-cli/src/work_reports.rs index 6fff5ec63..6259e38d4 100644 --- a/oxa/oxa-cli/src/work_reports.rs +++ b/oxa/oxa-cli/src/work_reports.rs @@ -64,6 +64,28 @@ fn visible(text: &str) -> String { { ch.escape_unicode().to_string() } else { ch.to_string() } }).collect() } +pub(crate) fn scrub_note(note: &ProblemNote, ctx: &ScrubCtx) -> Option { + let note = ProblemNote { + title: visible(&scrub(¬e.title, ctx)).chars().take(200).collect(), + notes: visible(&clean_notes(¬e.notes, ctx)) + .chars() + .take(2000) + .collect(), + keywords: note + .keywords + .iter() + .map(|query| visible(&scrub(query, ctx)).chars().take(100).collect()) + .collect(), + }; + (!note.title.trim().is_empty() + && !note.notes.trim().is_empty() + && (2..=3).contains(¬e.keywords.len()) + && note + .keywords + .iter() + .all(|query: &String| !query.trim().is_empty())) + .then_some(note) +} fn matches(note: &ProblemNote, candidate: &Value) -> bool { let title = candidate["title"].as_str().unwrap_or_default().to_lowercase(); if title == note.title.to_lowercase() { diff --git a/oxa/oxa-cli/tests/work_publish.rs b/oxa/oxa-cli/tests/work_publish.rs index f0badf211..9f4acb85d 100644 --- a/oxa/oxa-cli/tests/work_publish.rs +++ b/oxa/oxa-cli/tests/work_publish.rs @@ -525,7 +525,7 @@ fn unsetting_the_agent_marker_and_piping_yes_cannot_authorize_public_writes() { let path = configure(&env); env.write("tmp/gh", r##"#!/bin/sh case "$1 $2" in - 'repo view') printf '%s' '{"nameWithOwner":"example/project"}' ;; + 'repo view') printf '%s' '{"nameWithOwner":"example/project","url":"https://github.com/example/project"}' ;; 'pr list'|'issue list') printf '[]' ;; 'pr create'|'issue create'|'issue comment') touch "$HOME/public-write"; printf '%s' 'https://github.com/example/project/issues/1' ;; *) exit 7 ;; diff --git a/oxa/oxa-work/src/execute.rs b/oxa/oxa-work/src/execute.rs index 3921a11da..5e440a38e 100644 --- a/oxa/oxa-work/src/execute.rs +++ b/oxa/oxa-work/src/execute.rs @@ -12,6 +12,7 @@ pub(crate) struct Runner<'a> { pub lock: &'a File, pub writer: &'a mut Writer, pub observer: &'a dyn CheckpointObserver, + pub scrub_note: &'a (dyn Fn(&crate::ProblemNote) -> Option + Sync), } impl Runner<'_> { fn advance(&mut self, state: &mut State, step: Step) -> std::result::Result<(), RunError> { @@ -103,11 +104,11 @@ impl Runner<'_> { state.record.interruption = worker.result.interruption; } state.record.worker = Some(worker.result.clone()); - for note in &worker.notes { + for note in worker.notes.iter().filter_map(self.scrub_note) { if state.record.problem_notes.len() < 5 - && !state.record.problem_notes.contains(note) + && !state.record.problem_notes.contains(¬e) { - state.record.problem_notes.push(note.clone()); + state.record.problem_notes.push(note); } } state.record.session_id = worker.session.clone(); diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index 2b4b7f4a3..4946658d3 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -550,6 +550,21 @@ pub fn run_task_with_observer( policy: &RunPolicy, rerun: bool, observer: &dyn CheckpointObserver, +) -> std::result::Result { + run_task_with_notes(repo, binary, planned, policy, rerun, observer, &|_| None) +} + +/// The trusted frontend supplies its privacy scrubber before any worker note +/// enters a checkpoint, observation or emitted record. Other callers discard +/// notes by default rather than accidentally publishing model-supplied text. +pub fn run_task_with_notes( + repo: &Path, + binary: &Path, + planned: &PlannedTask, + policy: &RunPolicy, + rerun: bool, + observer: &dyn CheckpointObserver, + scrub_note: &(dyn Fn(&ProblemNote) -> Option + Sync), ) -> std::result::Result { let task = &planned.task; validate(task)?; @@ -653,6 +668,7 @@ pub fn run_task_with_observer( lock: &lock_file, writer: &mut writer, observer, + scrub_note, }) .drive(&mut state) { diff --git a/oxa/oxa-work/src/publish.rs b/oxa/oxa-work/src/publish.rs index 2b6635700..fddec2c25 100644 --- a/oxa/oxa-work/src/publish.rs +++ b/oxa/oxa-work/src/publish.rs @@ -50,6 +50,61 @@ fn gh(binary: &Path, policy: &RunPolicy, repo: &Path, lock: &File, args: &[&str] fn json(text: &str) -> Result { serde_json::from_str(text).map_err(|e| e.to_string()) } +struct Repository { + qualified: String, + host: String, + owner: String, + name: String, +} +fn repository(value: &serde_json::Value) -> Result { + let identity = value["nameWithOwner"] + .as_str() + .ok_or("repository identity missing")?; + let (owner, name) = identity.split_once('/').ok_or("invalid repository identity")?; + let url = value["url"].as_str().ok_or("repository URL missing")?; + let (host, path) = url + .strip_prefix("https://") + .and_then(|url| url.split_once('/')) + .ok_or("repository URL must identify its HTTPS host")?; + if owner.is_empty() + || name.is_empty() + || name.contains('/') + || path.trim_end_matches('/') != identity + || host.is_empty() + || !host + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b".-:".contains(&byte)) + { + return Err("repository URL disagrees with its identity".into()); + } + Ok(Repository { + qualified: format!("{host}/{identity}"), + host: host.into(), + owner: owner.into(), + name: name.into(), + }) +} +const BODY_BEGIN: &str = ""; +const BODY_END: &str = ""; +fn body(task: &Task, head: &str, existing: Option<&str>) -> Result { + let section = format!("{BODY_BEGIN}\nAccepted commit: `{head}`\nLocal acceptance: passed\nCI: not yet qualified for this commit\n{BODY_END}"); + let default = format!("T2 — task `{}` executed by `oxa work`.\n\nWhat/why: explicitly requested task; review its changes against the base.\n\nChanges: accepted worktree retained.\n\nRemoves: inspect the diff before promotion; no deletion clearance is inferred.\n\nValidation: frozen local acceptance passed; CI qualification is separate.\n", task.id); + let original = existing.unwrap_or(&default); + match ( + original.matches(BODY_BEGIN).count(), + original.matches(BODY_END).count(), + ) { + (0, 0) => Ok(format!("{original}\n\n{section}\n")), + (1, 1) => { + let (before, rest) = original + .split_once(BODY_BEGIN) + .ok_or("invalid managed PR section")?; + let (_, after) = rest.split_once(BODY_END).ok_or("invalid managed PR section")?; + Ok(format!("{before}{section}{after}")) + } + _ => Err("ambiguous managed PR section; preserve the owner description".into()), + } +} fn pr_base(repo: &Path, task: &Task, policy: &RunPolicy) -> Result { let reference = checked(super::git( policy, @@ -102,15 +157,14 @@ pub fn publish( if remote.lines().count() != 1 { return Err("publishing requires one origin push URL".into()); } - let repository = json(&gh( + let repository = repository(&json(&gh( tree, &lock, - &["repo", "view", &remote, "--json", "nameWithOwner"], - )?)?; - let name = repository["nameWithOwner"] - .as_str() - .ok_or("repository name missing")?; - let (owner, repo_name) = name.split_once('/').ok_or("invalid repository name")?; + &["repo", "view", &remote, "--json", "nameWithOwner,url"], + )?)?)?; + let name = repository.qualified.as_str(); + let owner = repository.owner.as_str(); + let repo_name = repository.name.as_str(); let prs = json(&gh( tree, &lock, @@ -143,6 +197,18 @@ pub fn publish( { return Err("nothing to publish".into()); } + let existing_body = if let Some(pr) = prs.first() { + let number = pr["number"].as_u64().ok_or("PR number missing")?.to_string(); + let value = json(&gh( + tree, + &lock, + &["pr", "view", &number, "--repo", name, "--json", "body"], + )?)?; + Some(value["body"].as_str().ok_or("PR body missing")?.to_owned()) + } else { + None + }; + let text = body(task, &head, existing_body.as_deref())?; if confirm(&format!( "Push accepted commit {head} as {} to origin and create/update its PR in {name} (base {base})? Type yes to publish: ", task.branch @@ -165,6 +231,8 @@ pub fn publish( &[ "api", "graphql", + "--hostname", + &repository.host, "-f", &format!("query={query}"), "-F", @@ -210,15 +278,24 @@ pub fn publish( } let draft = false; let body = policy.storage.join(format!("{}.pr-body.md", task.id)); - let text = format!("T2 — task `{}` executed by `oxa work`.\n\nWhat/why: explicitly requested task; review its changes against the base.\n\nChanges: accepted commit `{head}`; worktree retained.\n\nRemoves: inspect the diff before promotion; no deletion clearance is inferred.\n\nValidation: frozen local acceptance passed. CI: not yet qualified.\n", task.id); - std::fs::write(&body, text).map_err(|e| e.to_string())?; + crate::checkpoint::write_atomic(&body, text.as_bytes(), body.exists())?; let body_arg = body.to_str().ok_or("non-UTF8 PR body path")?; let result = if let Some(pr) = prs.first() { let number = pr["number"].as_u64().ok_or("PR number missing")?.to_string(); gh( tree, &lock, - &["pr", "edit", &number, "--repo", name, "--base", &base], + &[ + "pr", + "edit", + &number, + "--repo", + name, + "--base", + &base, + "--body-file", + body_arg, + ], )?; let was_draft = pr["isDraft"].as_bool().ok_or("PR draft state missing")?; if was_draft != draft { From ed83dc37ede77446bdfacd9cc195f4aac3986d47 Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:19:55 +0100 Subject: [PATCH 24/52] test(work): expose global queue admission boundary failures --- oxa/oxa-cli/tests/work_queue.rs | 94 +++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/oxa/oxa-cli/tests/work_queue.rs b/oxa/oxa-cli/tests/work_queue.rs index d292ddbf2..db4702254 100644 --- a/oxa/oxa-cli/tests/work_queue.rs +++ b/oxa/oxa-cli/tests/work_queue.rs @@ -542,3 +542,97 @@ fn a_monitor_error_drains_admitted_workers_before_returning() { "controller returned before its admitted worker finished" ); } + +#[test] +fn duplicate_branches_in_separate_files_are_rejected_before_admission() { + let env = TestEnv::new(); + setup(&env, "[]"); + task_file(&env, "first", "first", 2); + env.write( + "work/queue/second.json", + r#"[{"id":"second","model":"fake","branch":"agent/first","prompt":"Synthetic duplicate branch"}]"#, + ); + let server = FakeServer::start(vec![ScriptTurn::text("done"); 2], FakeOpts::default()).unwrap(); + let out = finish(command(&env, &server.url(), "1")); + let events = rows(&out); + assert!( + events.iter().any(|row| row["event"] == "error" + && row["detail"] + .as_str() + .is_some_and(|detail| detail.contains("branch"))), + "{events:?}" + ); + assert!(!events + .iter() + .any(|row| row["event"] == "task" && row["id"] == "second")); + assert_eq!(server.requests().len(), 1); +} + +#[test] +fn a_full_common_git_volume_blocks_admission_independently() { + use std::os::unix::fs::PermissionsExt; + let env = TestEnv::new(); + setup(&env, "[]"); + task_file(&env, "input", "input", 1); + let df = env.write("tmp/df", "#!/bin/sh\nfor arg do checked=$arg; done\nif test \"$checked\" = \"$OXA_TEST_COMMON_GIT\"; then printf '0\\n'; else printf '1000000000000\\n'; fi\n"); + fs::set_permissions(df, fs::Permissions::from_mode(0o755)).unwrap(); + let server = FakeServer::start(vec![ScriptTurn::text("done")], FakeOpts::default()).unwrap(); + let mut cmd = command(&env, &server.url(), "1"); + cmd.args(["--min-free-mib", "512"]) + .env( + "OXA_TEST_COMMON_GIT", + env.work().join("project/.git").canonicalize().unwrap(), + ) + .env( + "PATH", + format!("{}:{}", env.tmp().display(), std::env::var("PATH").unwrap()), + ); + let out = finish(cmd); + assert_eq!(out.status.code(), Some(1)); + assert_eq!(rows(&out).last().unwrap()["disk_blocked"], true); + assert!(server.requests().is_empty()); +} + +#[test] +fn a_retained_busy_lease_without_an_input_counts_against_max() { + use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt}; + let env = TestEnv::new(); + setup(&env, "[]"); + task_file(&env, "next", "next", 1); + fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(storage(&env)) + .unwrap(); + let lease = fs::OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .mode(0o600) + .open(storage(&env).join("retained.lock")) + .unwrap(); + lease.try_lock().unwrap(); + let server = FakeServer::start( + vec![ScriptTurn::text("done")], + FakeOpts { + hold_headers_ms: 700, + ..FakeOpts::default() + }, + ) + .unwrap(); + let mut child = command(&env, &server.url(), "1") + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + let status = env.work().join("queue/.oxa-queue-status.json"); + until(|| status.exists()); + let before: serde_json::Value = serde_json::from_slice(&fs::read(status).unwrap()).unwrap(); + let requests = server.requests().len(); + drop(lease); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + assert_eq!(before["active"], serde_json::json!(["retained"]), "{before}"); + assert_eq!(requests, 0, "a retained lease must block another admission"); + assert_eq!(server.requests().len(), 1); +} From ff82d247a1a5587e252a5f63716a6e06c32a3504 Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:38:58 +0100 Subject: [PATCH 25/52] test(work): reporting must retain a pending accepted save --- oxa/oxa-cli/tests/work_publish.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/oxa/oxa-cli/tests/work_publish.rs b/oxa/oxa-cli/tests/work_publish.rs index 9f4acb85d..8096b4588 100644 --- a/oxa/oxa-cli/tests/work_publish.rs +++ b/oxa/oxa-cli/tests/work_publish.rs @@ -116,6 +116,13 @@ fn a_real_save_failure_retries_without_another_worker() { first.worker.as_ref().unwrap().outcome, oxa_work::WorkerOutcome::Done )); + let reported = + oxa_work::report_notes(&task, &policy, &|_: &oxa_work::Record| Ok(()), &mut |_, _, _| { + panic!("no notes were emitted by this worker") + }) + .unwrap(); + assert_eq!(reported.step, oxa_work::Step::Saving, "{reported:?}"); + assert_eq!(reported.session_id, first.session_id); fs::remove_file(lock_path.into_inner().unwrap().unwrap()).unwrap(); let plan = oxa_work::plan_task_for_run(&project, task, &policy, false).unwrap(); let retry = oxa_work::run_task_with_policy(&project, binary, &plan, &policy).unwrap(); From 2b6a47b349b1b6446ec4d66d180d88e573bfcf30 Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:47:43 +0100 Subject: [PATCH 26/52] fix(work): preserve pending save through report checkpoints --- oxa/oxa-work/src/publish.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/oxa/oxa-work/src/publish.rs b/oxa/oxa-work/src/publish.rs index fddec2c25..138af3225 100644 --- a/oxa/oxa-work/src/publish.rs +++ b/oxa/oxa-work/src/publish.rs @@ -354,6 +354,12 @@ pub fn report_notes( ) -> Result { let (mut writer, lock, mut value) = context(task, policy, observer)?; let prior_step = value.record.step; + // Reporting is ancillary: a failed save/publication remains recoverable even + // if reporting is interrupted between durable report receipts. + let reporting_step = match prior_step { + Step::Saving | Step::Publishing => prior_step, + _ => Step::Reporting, + }; for index in 0..value.record.problem_notes.len() { if value .record @@ -363,7 +369,7 @@ pub fn report_notes( { continue; } - writer.save(&mut value, Step::Reporting, observer)?; + writer.save(&mut value, reporting_step, observer)?; let report = reporter(&value.record.problem_notes[index], &lock, policy.offline).unwrap_or_else(|detail| { super::ReportRecord { @@ -378,12 +384,12 @@ pub fn report_notes( } else { value.record.reports.push(report); } - writer.save(&mut value, Step::Reporting, observer)?; + writer.save(&mut value, reporting_step, observer)?; } writer.save( &mut value, - if prior_step == Step::Publishing { - Step::Publishing + if matches!(prior_step, Step::Saving | Step::Publishing) { + prior_step } else { Step::Finished }, From 6936ee2efdca287980cfdb8ecdb764a8a03682e5 Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:19:39 +0100 Subject: [PATCH 27/52] test(work): reject publication recovery acceptance bypasses --- oxa/oxa-cli/tests/work_publish.rs | 64 +++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/oxa/oxa-cli/tests/work_publish.rs b/oxa/oxa-cli/tests/work_publish.rs index 8096b4588..ca9eeebb2 100644 --- a/oxa/oxa-cli/tests/work_publish.rs +++ b/oxa/oxa-cli/tests/work_publish.rs @@ -327,6 +327,70 @@ fn offline_publication_never_starts_gh_or_push() { assert!(!env.home().join("gh-calls").exists()); assert!(!env.home().join("push-calls").exists()); } +#[test] +fn publication_drift_requires_fresh_checks_on_explicit_rerun() { + let env = TestEnv::new(); + setup(&env, r#"["test ! -f later.txt"]"#); + let path = configure(&env); + let done = run(&env, edits()); + let tree = PathBuf::from(done["worktree"].as_str().unwrap()); + fs::write(tree.join("later.txt"), "unchecked\n").unwrap(); + git(&env, &tree, &["add", "."]); + git( + &env, + &tree, + &["commit", "-m", "Synthetic acceptance-breaking drift"], + ); + let blocked = run_args( + &env, + vec![], + &["work", "../tasks.json", "--publish"], + Some(path.clone()), + ); + assert_eq!(blocked["step"], "publishing", "{blocked}"); + let (retry, requests) = work::run_observed( + &env, + vec![ScriptTurn::text("done"), ScriptTurn::text("done")], + &["work", "../tasks.json", "--rerun", "--publish"], + Some(path), + ); + assert_eq!(retry["outcome"], "red", "{retry}"); + assert_eq!( + requests.len(), + 2, + "fresh execution plus the existing repair attempt" + ); + assert!(!env.home().join("push-calls").exists()); +} + +#[test] +fn ordinary_run_cannot_reseal_a_failed_publication_checkpoint() { + let env = TestEnv::new(); + setup(&env, "[]"); + let path = configure(&env); + let done = run(&env, edits()); + let tree = PathBuf::from(done["worktree"].as_str().unwrap()); + fs::write(tree.join("later.txt"), "unchecked\n").unwrap(); + git(&env, &tree, &["add", "."]); + git(&env, &tree, &["commit", "-m", "Synthetic publication drift"]); + let blocked = run_args( + &env, + vec![], + &["work", "../tasks.json", "--publish"], + Some(path.clone()), + ); + assert_eq!(blocked["step"], "publishing", "{blocked}"); + let (retry, requests) = work::run_observed(&env, vec![], &["work", "../tasks.json"], Some(path.clone())); + assert_eq!( + retry["head_sha"], done["head_sha"], + "a normal run cannot certify a new head" + ); + assert!(requests.is_empty()); + let retry = run_args(&env, vec![], &["work", "../tasks.json", "--publish"], Some(path)); + assert_eq!(retry["outcome"], "blocked", "{retry}"); + assert!(!env.home().join("push-calls").exists()); +} + #[test] fn branch_drift_after_acceptance_cannot_publish_unchecked_commits() { let env = TestEnv::new(); From 95c55cba9ed3f422ed055d7b364bd625ea2133a6 Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:30:33 +0100 Subject: [PATCH 28/52] fix(work): revalidate terminal action checkpoints on explicit rerun --- oxa/oxa-work/src/lib.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index f1a0b1834..476d479f5 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -152,6 +152,10 @@ pub fn plan_task_for_actions( plan_task_mode(repo, task, policy, rerun, true) } +fn completed_execution(step: Step) -> bool { + matches!(step, Step::Finished | Step::Publishing | Step::Reporting) +} + fn plan_task_mode( repo: &Path, task: Task, @@ -173,8 +177,9 @@ fn plan_task_mode( if saved.task != task { return Err("task specification changed; use a new task id".into()); } - if saved.record.step != Step::Finished - || (!rerun && (actions || matches!(saved.record.outcome, Outcome::Done | Outcome::NothingToDo))) + if !(rerun && completed_execution(saved.record.step)) + && (saved.record.step != Step::Finished + || (!rerun && (actions || matches!(saved.record.outcome, Outcome::Done | Outcome::NothingToDo)))) { let execution = saved.execution.ok_or("missing task execution checkpoint")?; if saved.record.base_sha.as_deref() != Some(&execution.base) { @@ -645,7 +650,12 @@ pub fn run_task_with_notes( return Err("task specification changed; use a new task id".into()); } writer.replay(&old, observer).map_err(RunError::Checkpoint)?; - if old.record.step == Step::Finished { + // Action recovery cannot turn a changed HEAD into fresh acceptance. + // Only an explicit rerun starts a new execution for these checkpoints. + if !rerun && matches!(old.record.step, Step::Publishing | Step::Reporting) { + return Ok(old.record); + } + if completed_execution(old.record.step) { if !rerun && matches!(old.record.outcome, Outcome::Done | Outcome::NothingToDo) { return Ok(old.record); } From 6e3f801c30a53d6ac883ce3d0af8d68f4fdd9426 Mon Sep 17 00:00:00 2001 From: oxa-agent Date: Tue, 17 Nov 2026 20:58:45 +0100 Subject: [PATCH 29/52] feat(work): reserve idle helper claims under the shared lock (#1017) --- oxa/oxa-work/src/board/claims.rs | 5 ++ oxa/oxa-work/src/board/claims/controls.rs | 29 +++++++ oxa/oxa-work/tests/board_controls.rs | 94 +++++++++++++++++++++++ 3 files changed, 128 insertions(+) diff --git a/oxa/oxa-work/src/board/claims.rs b/oxa/oxa-work/src/board/claims.rs index 686395df7..99a9ab30e 100644 --- a/oxa/oxa-work/src/board/claims.rs +++ b/oxa/oxa-work/src/board/claims.rs @@ -160,6 +160,11 @@ impl Claims { Self::open(&self.root)?; let lock = io(File::open(&self.root))?; io(lock.lock())?; + self.rows_locked() + } + + /// Caller holds the directory lock used by every reservation and control update. + fn rows_locked(&self) -> Result> { let mut states = BTreeMap::new(); for entry in io(fs::read_dir(&self.root))? { let name = io(entry)?.file_name(); diff --git a/oxa/oxa-work/src/board/claims/controls.rs b/oxa/oxa-work/src/board/claims/controls.rs index eda2a454d..1ad3802a2 100644 --- a/oxa/oxa-work/src/board/claims/controls.rs +++ b/oxa/oxa-work/src/board/claims/controls.rs @@ -163,4 +163,33 @@ impl Claims { Ok(controls.allows(worker, entry.action)) }) } + + /// Helper admission checks idleness and reserves under the SAME directory lock. + /// Every outstanding target/worker lease, including expired work, requires fenced completion. + /// Worker identity and roles must come from the trusted frontend, as with `claim_for`. + /// Generic manual reservations retain their separate `assign`/`claim_for` semantics. + pub fn claim_idle_for( + &self, + entry: &Entry, + policy: &Policy, + worker: &Worker, + now: u64, + ttl: u64, + ) -> Result> { + self.claim_checked(entry, policy, &worker.id, (now, ttl), |row| { + if row.state.lease.is_some() + || self.rows_locked()?.values().any(|saved| { + saved + .state + .lease + .as_ref() + .is_some_and(|lease| lease.holder == worker.id) + }) + { + return Ok(false); + } + let controls = row.data.policy.get_or_insert_with(AssignmentPolicy::default); + Ok(controls.allows(worker, entry.action)) + }) + } } diff --git a/oxa/oxa-work/tests/board_controls.rs b/oxa/oxa-work/tests/board_controls.rs index 0c52d6a78..6266756e3 100644 --- a/oxa/oxa-work/tests/board_controls.rs +++ b/oxa/oxa-work/tests/board_controls.rs @@ -216,6 +216,10 @@ fn typed_issue_intake_uses_one_snapshot_and_builder_authority() { worker.roles = [Role::Reviewer].into(); let denied = store.claim_for(issue, &policy(), &worker, 10, 10).unwrap(); assert!(denied.is_none()); + assert!(store + .claim_idle_for(issue, &policy(), &worker, 10, 10) + .unwrap() + .is_none()); let legacy = store.claim(issue, &policy(), &worker.id, 10, 10).unwrap(); assert!(legacy.is_none()); worker.roles = [Role::Builder].into(); @@ -294,6 +298,10 @@ fn import_resumes_without_resetting_attempts_or_admitting_preserved_jobs() { .claim_for(&issue, &policy(), &worker, 21, 10) .unwrap() .is_none()); + assert!(store + .claim_idle_for(&issue, &policy(), &worker, 21, 10) + .unwrap() + .is_none()); assert!(store .claim(&entry(), &policy(), &worker.id, 21, 10) .unwrap() @@ -528,3 +536,89 @@ fn records_without_release_metadata_remain_readable() { assert_eq!(rows[&3].state.attempts, 1); assert!(rows[&3].data.last_release.is_none()); } + +#[test] +fn idle_claims_preserve_expired_work_until_fenced_completion() { + let fixture = Fixture::new(); + let store = fixture.store(); + let worker = worker("slot-1"); + let first = entry(); + let mut next = first.clone(); + next.number = 2; + assert!(store + .claim_idle_for(&next, &Policy::default(), &worker, 10, 10) + .unwrap() + .is_none()); + assert_eq!(fs::read_dir(&fixture.0).unwrap().count(), 0); + let lease = store + .claim_for(&first, &policy(), &worker, 10, 10) + .unwrap() + .unwrap(); + // Expiry is not evidence that the existing helper actually finished. + assert!(store + .claim_idle_for(&next, &policy(), &worker, 20, 10) + .unwrap() + .is_none()); + assert_eq!(store.snapshot().unwrap()[&2].attempts, 0); + let other = crate::worker("slot-2"); + assert!(store + .claim_idle_for(&first, &policy(), &other, 20, 10) + .unwrap() + .is_none()); + let mut independent = first.clone(); + independent.number = 1; + assert!(store + .claim_idle_for(&independent, &policy(), &other, 20, 10) + .unwrap() + .is_some()); + assert!(store + .release_for(3, &worker.id, &lease, 20, "verified completion") + .unwrap()); + let next_lease = store + .claim_idle_for(&next, &policy(), &worker, 20, 10) + .unwrap() + .unwrap(); + let rows = fixture.store().assignments().unwrap(); + assert_eq!(rows[&3].state.attempts, 1); + assert_eq!(rows[&2].state.attempts, 1); + assert_eq!(rows[&2].state.lease, Some(next_lease)); + assert!(rows[&3].state.lease.is_none()); + // Losing another worker's established state must fail the complete idle check closed. + fs::remove_file(fixture.0.join("1.board-claim.json")).unwrap(); + next.number = 6; + assert!(store + .claim_idle_for(&next, &policy(), &crate::worker("slot-3"), 21, 10) + .is_err()); + assert!(!fixture.0.join("6.board-claim.json").exists()); +} + +#[test] +fn concurrent_idle_reservations_share_one_worker_fence_across_refs() { + let fixture = Fixture::new(); + let barrier = std::sync::Barrier::new(6); + let winners = std::thread::scope(|scope| { + let mut pending = vec![]; + for number in 1..=6 { + let store = fixture.store(); + let worker = worker("shared-slot"); + let mut candidate = entry(); + candidate.number = number; + let barrier = &barrier; + pending.push(scope.spawn(move || { + barrier.wait(); + store + .claim_idle_for(&candidate, &policy(), &worker, 10, 10) + .unwrap() + })); + } + pending + .into_iter() + .map(|thread| thread.join().unwrap()) + .filter(Option::is_some) + .count() + }); + assert_eq!(winners, 1); + let rows = fixture.store().assignments().unwrap(); + assert_eq!(rows.values().map(|row| row.state.attempts).sum::(), 1); + assert_eq!(rows.values().filter(|row| row.state.lease.is_some()).count(), 1); +} From 812c0890b399e68450a50d025db793e50244edec Mon Sep 17 00:00:00 2001 From: oxa-agent Date: Tue, 17 Nov 2026 21:04:27 +0100 Subject: [PATCH 30/52] test(work): reject transient markers as helper bootstrap state (#1017) --- oxa/oxa-work/tests/board_controls.rs | 50 ++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/oxa/oxa-work/tests/board_controls.rs b/oxa/oxa-work/tests/board_controls.rs index 6266756e3..027344795 100644 --- a/oxa/oxa-work/tests/board_controls.rs +++ b/oxa/oxa-work/tests/board_controls.rs @@ -18,6 +18,23 @@ impl Fixture { fn store(&self) -> Claims { Claims::open(&self.0).unwrap() } + fn establish(&self, numbers: impl IntoIterator) { + let incoming = numbers + .into_iter() + .map(|number| { + ( + number, + ImportState { + state: AttemptState::default(), + excluded: false, + }, + ) + }) + .collect(); + self.store() + .import_snapshot(Some(ManualAuthority::Owner), &incoming, 10) + .unwrap(); + } } impl Drop for Fixture { fn drop(&mut self) { @@ -550,6 +567,7 @@ fn idle_claims_preserve_expired_work_until_fenced_completion() { .unwrap() .is_none()); assert_eq!(fs::read_dir(&fixture.0).unwrap().count(), 0); + fixture.establish([1, 2, 6]); let lease = store .claim_for(&first, &policy(), &worker, 10, 10) .unwrap() @@ -595,6 +613,7 @@ fn idle_claims_preserve_expired_work_until_fenced_completion() { #[test] fn concurrent_idle_reservations_share_one_worker_fence_across_refs() { let fixture = Fixture::new(); + fixture.establish(1..=6); let barrier = std::sync::Barrier::new(6); let winners = std::thread::scope(|scope| { let mut pending = vec![]; @@ -622,3 +641,34 @@ fn concurrent_idle_reservations_share_one_worker_fence_across_refs() { assert_eq!(rows.values().map(|row| row.state.attempts).sum::(), 1); assert_eq!(rows.values().filter(|row| row.state.lease.is_some()).count(), 1); } + +#[test] +fn idle_claims_require_saved_state_not_a_denied_claim_marker() { + let fixture = Fixture::new(); + let store = fixture.store(); + let mut candidate = entry(); + candidate.number = 44; + candidate.action = Action::BuildIssue; + let builder = worker("slot-1"); + let reviewer = Worker { + roles: [Role::Reviewer].into(), + ..builder.clone() + }; + assert!(store + .claim_for(&candidate, &policy(), &reviewer, 10, 10) + .unwrap() + .is_none()); + assert!(fixture.0.join("44.board-claim.lock").exists()); + assert!(!fixture.0.join("44.board-claim.json").exists()); + assert!(store + .claim_idle_for(&candidate, &policy(), &builder, 10, 10) + .unwrap() + .is_none()); + assert!(!fixture.0.join("44.board-claim.json").exists()); + fixture.establish([44]); + assert!(store + .claim_idle_for(&candidate, &policy(), &builder, 10, 10) + .unwrap() + .is_some()); + assert_eq!(fixture.store().snapshot().unwrap()[&44].attempts, 1); +} From e4ef6897dfcdf4a0b436e47e7d312559c77cc5b2 Mon Sep 17 00:00:00 2001 From: oxa-agent Date: Tue, 17 Nov 2026 21:13:30 +0100 Subject: [PATCH 31/52] fix(work): require saved bootstrap state for idle helper admission (#1017) --- oxa/oxa-work/src/board/claims/controls.rs | 4 +++- oxa/oxa-work/tests/board_controls.rs | 8 +++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/oxa/oxa-work/src/board/claims/controls.rs b/oxa/oxa-work/src/board/claims/controls.rs index 1ad3802a2..bf15856e1 100644 --- a/oxa/oxa-work/src/board/claims/controls.rs +++ b/oxa/oxa-work/src/board/claims/controls.rs @@ -165,6 +165,7 @@ impl Claims { } /// Helper admission checks idleness and reserves under the SAME directory lock. + /// The target must have validated Saved state; transient lock markers are not bootstrap facts. /// Every outstanding target/worker lease, including expired work, requires fenced completion. /// Worker identity and roles must come from the trusted frontend, as with `claim_for`. /// Generic manual reservations retain their separate `assign`/`claim_for` semantics. @@ -177,7 +178,8 @@ impl Claims { ttl: u64, ) -> Result> { self.claim_checked(entry, policy, &worker.id, (now, ttl), |row| { - if row.state.lease.is_some() + if !self.established(entry.number)? + || row.state.lease.is_some() || self.rows_locked()?.values().any(|saved| { saved .state diff --git a/oxa/oxa-work/tests/board_controls.rs b/oxa/oxa-work/tests/board_controls.rs index 027344795..ed1d10c3c 100644 --- a/oxa/oxa-work/tests/board_controls.rs +++ b/oxa/oxa-work/tests/board_controls.rs @@ -604,10 +604,11 @@ fn idle_claims_preserve_expired_work_until_fenced_completion() { // Losing another worker's established state must fail the complete idle check closed. fs::remove_file(fixture.0.join("1.board-claim.json")).unwrap(); next.number = 6; + let untouched = fs::read(fixture.0.join("6.board-claim.json")).unwrap(); assert!(store .claim_idle_for(&next, &policy(), &crate::worker("slot-3"), 21, 10) .is_err()); - assert!(!fixture.0.join("6.board-claim.json").exists()); + assert_eq!(fs::read(fixture.0.join("6.board-claim.json")).unwrap(), untouched); } #[test] @@ -666,6 +667,11 @@ fn idle_claims_require_saved_state_not_a_denied_claim_marker() { .is_none()); assert!(!fixture.0.join("44.board-claim.json").exists()); fixture.establish([44]); + assert!(store + .claim_idle_for(&candidate, &policy(), &reviewer, 10, 10) + .unwrap() + .is_none()); + assert_eq!(store.snapshot().unwrap()[&44].attempts, 0); assert!(store .claim_idle_for(&candidate, &policy(), &builder, 10, 10) .unwrap() From 08c2c9b2803efb15487dda862056494037158202 Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 22:10:41 +0100 Subject: [PATCH 32/52] test(work): expose stale save and publication verification failures --- oxa/oxa-cli/src/work_reports.rs | 9 +++ oxa/oxa-cli/tests/work_publish.rs | 95 ++++++++++++++++++++++++++++++- 2 files changed, 102 insertions(+), 2 deletions(-) diff --git a/oxa/oxa-cli/src/work_reports.rs b/oxa/oxa-cli/src/work_reports.rs index 6259e38d4..f3c33831c 100644 --- a/oxa/oxa-cli/src/work_reports.rs +++ b/oxa/oxa-cli/src/work_reports.rs @@ -306,6 +306,15 @@ pub(crate) fn report( mod tests { use super::*; use oxa_fake::TestEnv; + + #[test] + fn locked_report_command_refuses_oversized_stdout() { + let env = TestEnv::new(); + let lock = File::open("/dev/null").unwrap(); + let args = words(&["-c", "1048577", "/dev/zero"]); + assert!(process("/usr/bin/head", &args, &env.work(), &lock).is_none()); + } + fn note() -> ProblemNote { ProblemNote { title: "oxa work commit is read only".into(), diff --git a/oxa/oxa-cli/tests/work_publish.rs b/oxa/oxa-cli/tests/work_publish.rs index ca9eeebb2..6a6eac141 100644 --- a/oxa/oxa-cli/tests/work_publish.rs +++ b/oxa/oxa-cli/tests/work_publish.rs @@ -66,6 +66,36 @@ fn edits() -> Vec { } #[test] fn a_real_save_failure_retries_without_another_worker() { + exercise_save_recovery( + "a_real_save_failure_retries_without_another_worker", + SaveRecovery::Unchanged, + ); +} + +#[test] +fn publication_does_not_replace_a_pending_save_checkpoint() { + exercise_save_recovery( + "publication_does_not_replace_a_pending_save_checkpoint", + SaveRecovery::PendingPublish, + ); +} + +#[test] +fn an_edit_after_acceptance_cannot_reuse_the_pending_save_verdict() { + exercise_save_recovery( + "an_edit_after_acceptance_cannot_reuse_the_pending_save_verdict", + SaveRecovery::Edited, + ); +} + +#[derive(PartialEq, Eq)] +enum SaveRecovery { + Unchanged, + PendingPublish, + Edited, +} + +fn exercise_save_recovery(test_name: &str, case: SaveRecovery) { const CHILD: &str = "OXA_SAVE_RECOVERY_TEST_ROOT"; if let Some(root) = std::env::var_os(CHILD) { use std::sync::{ @@ -123,9 +153,37 @@ fn a_real_save_failure_retries_without_another_worker() { .unwrap(); assert_eq!(reported.step, oxa_work::Step::Saving, "{reported:?}"); assert_eq!(reported.session_id, first.session_id); + if case == SaveRecovery::PendingPublish { + let published = oxa_work::publish( + &project, + binary, + &task, + &policy, + std::path::Path::new("/unavailable/gh"), + &|_: &oxa_work::Record| Ok(()), + &mut |_| panic!("a pending save cannot request publication approval"), + ) + .unwrap(); + assert_eq!(published.step, oxa_work::Step::Saving, "{published:?}"); + assert_eq!(published.checkpoint_revision, reported.checkpoint_revision); + } fs::remove_file(lock_path.into_inner().unwrap().unwrap()).unwrap(); + if case == SaveRecovery::Edited { + fs::write(first.worktree.join("answer.txt"), "edited after acceptance").unwrap(); + } let plan = oxa_work::plan_task_for_run(&project, task, &policy, false).unwrap(); let retry = oxa_work::run_task_with_policy(&project, binary, &plan, &policy).unwrap(); + if case == SaveRecovery::Edited { + assert!( + matches!(retry.outcome, oxa_work::Outcome::Red | oxa_work::Outcome::Blocked), + "{retry:?}" + ); + assert_eq!( + fs::read_to_string(first.worktree.join("answer.txt")).unwrap(), + "edited after acceptance" + ); + return; + } assert_eq!(retry.outcome, oxa_work::Outcome::Done, "{retry:?}"); assert_eq!(retry.step, oxa_work::Step::Finished); assert_eq!(retry.session_id, first.session_id); @@ -139,10 +197,15 @@ fn a_real_save_failure_retries_without_another_worker() { return; } let env = TestEnv::new(); - setup(&env, "[]"); + let checks = if case == SaveRecovery::Edited { + r#"["grep -Fx published answer.txt"]"# + } else { + "[]" + }; + setup(&env, checks); let server = oxa_fake::FakeServer::start(edits(), oxa_fake::FakeOpts::default()).unwrap(); let out = Command::new(std::env::current_exe().unwrap()) - .args(["--exact", "a_real_save_failure_retries_without_another_worker"]) + .args(["--exact", test_name]) .env_clear() .envs(env.envs(&server.url())) .env(CHILD, env.root()) @@ -156,6 +219,34 @@ fn a_real_save_failure_retries_without_another_worker() { ); assert_eq!(server.requests().len(), 2, "save retry must not call a model"); } + +#[test] +fn publishing_requires_acceptance_without_ignored_helpers() { + let env = TestEnv::new(); + setup(&env, r#"["sh helper.sh"]"#); + let repo = env.work().join("project"); + env.write("work/project/.gitignore", "helper.sh\n"); + git(&env, &repo, &["add", ".gitignore"]); + git(&env, &repo, &["commit", "-m", "Ignore synthetic local helper"]); + let mut turns = edits(); + turns.insert( + 0, + ScriptTurn::tool("helper", "write", r#"{"path":"helper.sh","content":"exit 0\n"}"#), + ); + let saved = run_args( + &env, + turns, + &["work", "../tasks.json", "--publish"], + Some(configure(&env)), + ); + assert_eq!(saved["outcome"], "blocked", "{saved}"); + assert!(saved["detail"] + .as_str() + .unwrap() + .contains("clean committed acceptance")); + assert!(!env.home().join("push-calls").exists()); + assert!(!env.home().join("gh-calls").exists()); +} #[test] fn publishes_local_branch_and_updates_same_pr_on_restart() { use std::os::unix::fs::PermissionsExt; From e58b910b294cf14728259fef30c61621c26d0f8e Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 22:21:11 +0100 Subject: [PATCH 33/52] Test queue input limits and inherited reporting lease --- oxa/oxa-cli/tests/work_queue.rs | 12 ++++++++- oxa/oxa-work/src/queue.rs | 48 +++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/oxa/oxa-cli/tests/work_queue.rs b/oxa/oxa-cli/tests/work_queue.rs index 0ba0d0049..d0045fdfc 100644 --- a/oxa/oxa-cli/tests/work_queue.rs +++ b/oxa/oxa-cli/tests/work_queue.rs @@ -380,7 +380,13 @@ case "$1 $2" in 'issue list') if test -f "$HOME/reported"; then printf '[{"number":1}]'; else printf '[]'; fi ;; 'pr list') printf '[]' ;; 'issue view') printf '%s' '{"title":"oxa queue problem","body":"Same synthetic reproduction","url":"https://github.com/example/project/issues/1","comments":[]}' ;; - 'issue create') sleep 0.2; touch "$HOME/reported"; printf 'created\n' >> "$HOME/report-writes"; printf '%s' 'https://github.com/example/project/issues/1' ;; + 'issue create') + for fd in /proc/self/fd/*; do + if test "$fd" -ef "$HOME/.oxa/agent/work-report.lock"; then + touch "$HOME/report-lease-inherited" + fi + done + sleep 0.2; touch "$HOME/reported"; printf 'created\n' >> "$HOME/report-writes"; printf '%s' 'https://github.com/example/project/issues/1' ;; *) exit 7 ;; esac "##); @@ -410,6 +416,10 @@ esac 1 ); let events = rows(&out); + assert!( + env.home().join("report-lease-inherited").exists(), + "a surviving GitHub writer must retain the reporting lease" + ); let dispositions: std::collections::HashSet<_> = events .iter() .filter(|r| r["event"] == "task") diff --git a/oxa/oxa-work/src/queue.rs b/oxa/oxa-work/src/queue.rs index 478e1a7f0..fc1914f9c 100644 --- a/oxa/oxa-work/src/queue.rs +++ b/oxa/oxa-work/src/queue.rs @@ -431,6 +431,54 @@ pub fn run_queue( #[cfg(test)] mod tests { use super::*; + #[test] + fn queue_refuses_oversized_inputs_without_admitting_their_tasks() { + let dir = std::env::temp_dir().join(format!( + "oxa-queue-limits-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir(&dir).unwrap(); + let valid = r#"[{"id":"large","branch":"agent/large","model":"fake","prompt":"Synthetic task"}]"#; + let mut oversized = valid.to_owned(); + oversized.push_str(&" ".repeat(1024 * 1024)); + fs::write(dir.join("bytes.json"), oversized).unwrap(); + let many: Vec<_> = (0..129) + .map(|n| { + serde_json::json!({ + "id": format!("task-{n}"), + "branch": format!("agent/task-{n}"), + "model": "fake", + "prompt": "Synthetic task" + }) + }) + .collect(); + fs::write(dir.join("count.json"), serde_json::to_vec(&many).unwrap()).unwrap(); + fs::write(dir.join("outside"), valid.replace("large", "symlink")).unwrap(); + std::os::unix::fs::symlink(dir.join("outside"), dir.join("link.json")).unwrap(); + + let mut pending = vec![]; + let mut inputs = Inputs::default(); + let mut errors = vec![]; + let result = scan(&dir, &mut pending, &mut inputs, &mut |event| { + if let QueueEvent::Error { file, .. } = event { + errors.push(file); + } + Ok(()) + }); + fs::remove_dir_all(&dir).unwrap(); + result.unwrap(); + assert!( + pending.is_empty(), + "oversized and linked inputs must never be admitted" + ); + assert!(errors.contains(&dir.join("bytes.json"))); + assert!(errors.contains(&dir.join("count.json"))); + } + #[test] fn archiving_an_unadmitted_file_cancels_its_pending_job() { let dir = std::env::temp_dir().join(format!( From 8ac1cc9754adefef06bc0094c12f6fd55f153222 Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 22:21:36 +0100 Subject: [PATCH 34/52] Verify committed publication and recovered saves --- oxa/oxa-cli/src/report_cmd.rs | 9 +++++-- oxa/oxa-cli/src/work_reports.rs | 43 +++++------------------------ oxa/oxa-work/src/committed.rs | 48 +++++++++++++++++++++------------ oxa/oxa-work/src/lib.rs | 24 +++++++++++++++++ oxa/oxa-work/src/publish.rs | 11 +++++++- 5 files changed, 79 insertions(+), 56 deletions(-) diff --git a/oxa/oxa-cli/src/report_cmd.rs b/oxa/oxa-cli/src/report_cmd.rs index 30ab4339d..a8df98b8e 100644 --- a/oxa/oxa-cli/src/report_cmd.rs +++ b/oxa/oxa-cli/src/report_cmd.rs @@ -456,12 +456,17 @@ system-wide or put it in a trusted PATH entry outside the project (for example ~ ) } -pub(crate) fn command_runner( +pub(crate) fn command_runner(command: std::process::Command, limit: Option) -> Option<(i32, String)> { + command_runner_with_stdin(command, limit, std::process::Stdio::null()) +} + +pub(crate) fn command_runner_with_stdin( mut command: std::process::Command, limit: Option, + stdin: std::process::Stdio, ) -> Option<(i32, String)> { let mut child = command - .stdin(std::process::Stdio::null()) + .stdin(stdin) .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::null()) .spawn() diff --git a/oxa/oxa-cli/src/work_reports.rs b/oxa/oxa-cli/src/work_reports.rs index f3c33831c..38b50e275 100644 --- a/oxa/oxa-cli/src/work_reports.rs +++ b/oxa/oxa-cli/src/work_reports.rs @@ -6,47 +6,18 @@ use oxa_work::{ProblemNote, ReportRecord}; use serde_json::Value; use std::{ fs::File, - io::Read, path::Path, process::{Command, Stdio}, - time::{Duration, Instant}, }; pub(crate) fn process(program: &str, args: &[String], cwd: &Path, lock: &File) -> Option<(i32, String)> { - let mut child = Command::new(program) - .args(args) - .current_dir(cwd) - .stdin(Stdio::from(lock.try_clone().ok()?)) - .stdout(Stdio::piped()) - .stderr(Stdio::null()) - .spawn() - .ok()?; - let mut stdout = child.stdout.take()?; - let (tx, rx) = std::sync::mpsc::channel(); - std::thread::spawn(move || { - let mut out = String::new(); - let result = (&mut stdout).take(1_048_576).read_to_string(&mut out); - let _ = tx.send(result.ok().map(|_| out)); - }); - let start = Instant::now(); - loop { - match child.try_wait() { - Ok(Some(status)) => { - return Some(( - status.code().unwrap_or(1), - rx.recv_timeout(Duration::from_secs(2)).ok()??, - )) - } - Ok(None) if start.elapsed() < Duration::from_secs(30) => { - std::thread::sleep(Duration::from_millis(20)) - } - _ => { - let _ = child.kill(); - let _ = child.wait(); - return None; - } - } - } + let mut command = Command::new(program); + command.args(args).current_dir(cwd); + crate::report_cmd::command_runner_with_stdin( + command, + Some(1_048_576), + Stdio::from(lock.try_clone().ok()?), + ) } fn call(runner: Runner<'_>, args: &[String]) -> Result { match runner("gh", args) { diff --git a/oxa/oxa-work/src/committed.rs b/oxa/oxa-work/src/committed.rs index e9200cc6a..c30c54718 100644 --- a/oxa/oxa-work/src/committed.rs +++ b/oxa/oxa-work/src/committed.rs @@ -57,13 +57,31 @@ pub fn verify_saved_task(repo: &Path, binary: &Path, policy: &RunPolicy, id: &st .map_err(|_| "cannot open saved task lock")?; lock.try_lock().map_err(|_| "task is already running")?; let saved = state::load(&policy.storage.join(format!("{id}.json")))?; - let record = saved.record; + let record = &saved.record; if saved.task.id != id || record.step != Step::Finished || !matches!(record.outcome, Outcome::Done | Outcome::NothingToDo) { return Err("only finished successful work can be verified for publication".into()); } + let export = verify_commit(repo, binary, policy, &saved, &lock)?; + Ok(VerifiedTask { + record: saved.record, + export, + _lock: lock, + checked_at_ms: crate::state::now_ms(), + }) +} + +/// Check the saved commit while the caller holds the task lock. +pub(crate) fn verify_commit( + repo: &Path, + binary: &Path, + policy: &RunPolicy, + saved: &state::State, + lock: &File, +) -> Result { + let record = &saved.record; // An explicit later online publication never relaxes the saved check policy. let verification_policy = RunPolicy { offline: saved.offline || policy.offline, @@ -72,6 +90,7 @@ pub fn verify_saved_task(repo: &Path, binary: &Path, policy: &RunPolicy, id: &st let policy = &verification_policy; let execution = saved .execution + .as_ref() .ok_or("saved acceptance commands are unavailable")?; let head = record.head_sha.as_deref().ok_or("saved head is unavailable")?; let tree_id = record.tree_sha.as_deref().ok_or("saved tree is unavailable")?; @@ -82,7 +101,7 @@ pub fn verify_saved_task(repo: &Path, binary: &Path, policy: &RunPolicy, id: &st tree::validate(policy, repo, &record.worktree, &saved.task)?; tree::ensure_merged(policy, &record.worktree, base, &saved.task.branch)?; same_saved_commit(policy, &record.worktree, head, tree_id)?; - if dirty_worktree(binary, &record.worktree, repo, policy, &lock)? { + if dirty_worktree(binary, &record.worktree, repo, policy, lock)? { return Err("saved task has uncommitted changes".into()); } let export = Export::create(policy, &record.worktree, head, tree_id, base)?; @@ -99,18 +118,18 @@ pub fn verify_saved_task(repo: &Path, binary: &Path, policy: &RunPolicy, id: &st SupervisorRequest { source: repo.into(), operation: SupervisorOperation::Checkout { - branch: saved.task.branch, + branch: saved.task.branch.clone(), create: false, }, }, policy, - &lock, + lock, )?; if !checkout.status.success() { return Err("clean acceptance checkout failed; saved work is retained".into()); } same_saved_commit(policy, &export.tree, head, tree_id)?; - if dirty_worktree(binary, &export.tree, repo, policy, &lock)? { + if dirty_worktree(binary, &export.tree, repo, policy, lock)? { return Err("clean acceptance checkout is dirty".into()); } // This fresh index comes from the verified commit and lives outside both @@ -143,18 +162,18 @@ pub fn verify_saved_task(repo: &Path, binary: &Path, policy: &RunPolicy, id: &st SupervisorRequest { source: repo.into(), operation: SupervisorOperation::CommittedChecks { - commands: execution.commands, + commands: execution.commands.clone(), index: index.clone(), }, }, policy, - &lock, + lock, )?)?; if !failed.is_empty() { return Err(format!("clean committed acceptance failed: {failed}")); } same_saved_commit(policy, &export.tree, head, tree_id)?; - if dirty_worktree(binary, &export.tree, repo, policy, &lock)? { + if dirty_worktree(binary, &export.tree, repo, policy, lock)? { return Err("acceptance changed committed source files".into()); } let inspected = supervise( @@ -165,22 +184,17 @@ pub fn verify_saved_task(repo: &Path, binary: &Path, policy: &RunPolicy, id: &st operation: SupervisorOperation::InspectCommitted { index }, }, policy, - &lock, + lock, )?; if !inspected.status.success() { return Err("acceptance changed committed source files or could not be inspected".into()); } export.verify(policy, head, base)?; same_saved_commit(policy, &record.worktree, head, tree_id)?; - if dirty_worktree(binary, &record.worktree, repo, policy, &lock)? { + if dirty_worktree(binary, &record.worktree, repo, policy, lock)? { return Err("saved work changed during committed acceptance".into()); } - Ok(VerifiedTask { - record, - export, - _lock: lock, - checked_at_ms: crate::state::now_ms(), - }) + Ok(export) } fn same_saved_commit(policy: &RunPolicy, tree: &Path, head: &str, tree_id: &str) -> Result<()> { @@ -365,7 +379,7 @@ impl Export { Ok(export) } - fn verify(&self, policy: &RunPolicy, head: &str, base: &str) -> Result<()> { + pub(crate) fn verify(&self, policy: &RunPolicy, head: &str, base: &str) -> Result<()> { match (fs::read(self.store.join("shallow")), &self.shallow) { (Ok(actual), Some(expected)) if actual == *expected => {} (Err(error), None) if error.kind() == std::io::ErrorKind::NotFound => {} diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index 8c7eaf673..506f8e490 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -694,6 +694,11 @@ pub fn run_task_with_notes( }; // Replaying the prior checkpoint precedes this first new checkpoint or action. let step = state.record.step; + let recover_accepted_save = step == Step::Saving + && matches!( + state.record.acceptance_outcome.unwrap_or(state.record.outcome), + Outcome::Done | Outcome::NothingToDo + ); writer .save(&mut state, step, observer) .map_err(RunError::Checkpoint)?; @@ -759,6 +764,25 @@ pub fn run_task_with_notes( .acceptance_outcome .take() .unwrap_or(state.record.outcome); + if recover_accepted_save { + // A failed or interrupted save can leave a worktree that changed since + // acceptance. Qualify the exact recovered commit before restoring success. + match committed::verify_commit(&root, binary, &policy, &state, &lock_file) { + Ok(_) => { + let initial = state.execution.as_ref().and_then(|point| point.initial.as_ref()); + state.record.outcome = if state.record.head_sha.as_ref() == initial { + Outcome::NothingToDo + } else { + Outcome::Done + }; + state.record.detail.clear(); + } + Err(error) => { + state.record.outcome = Outcome::Blocked; + state.record.detail = format!("save recovery: {error}"); + } + } + } writer .save(&mut state, Step::Finished, observer) .map_err(RunError::Checkpoint)?; diff --git a/oxa/oxa-work/src/publish.rs b/oxa/oxa-work/src/publish.rs index 138af3225..7afe9c460 100644 --- a/oxa/oxa-work/src/publish.rs +++ b/oxa/oxa-work/src/publish.rs @@ -129,7 +129,9 @@ pub fn publish( ) -> Result { let (mut writer, lock, mut value) = context(task, policy, observer)?; let accepted = value.record.acceptance_outcome.unwrap_or(value.record.outcome); - if !matches!(accepted, Outcome::Done | Outcome::NothingToDo) { + if !super::completed_execution(value.record.step) + || !matches!(accepted, Outcome::Done | Outcome::NothingToDo) + { return Ok(value.record); } value.record.acceptance_outcome = Some(accepted); @@ -149,6 +151,12 @@ pub fn publish( if value.record.head_sha.as_deref() != Some(head.as_str()) { return Err("task branch changed after acceptance; use --rerun before publishing".into()); } + let proof = super::committed::verify_commit(repo, binary, policy, &value, &lock)?; + let accepted_base = value + .record + .base_sha + .as_deref() + .ok_or("saved base is unavailable")?; let base = pr_base(tree, task, policy)?; if task.branch == base { return Err("task branch must differ from its base branch".into()); @@ -220,6 +228,7 @@ pub fn publish( { return Err("publication destination or task head changed during confirmation".into()); } + proof.verify(policy, &head, accepted_base)?; // Confirmation may remain open indefinitely: queue admission is checked // only after it returns, immediately before the push. if let Some(pr) = prs.first() { From 46f497bf2323cea2012f09557f51d162f5f00aab Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 22:24:32 +0100 Subject: [PATCH 35/52] Bound queue inputs and preserve the reporting lease --- oxa/docs/work.md | 2 + oxa/oxa-cli/src/work_cmd.rs | 6 +-- oxa/oxa-cli/src/work_reports.rs | 18 +++++-- oxa/oxa-cli/tests/common/work.rs | 42 ++--------------- oxa/oxa-work/src/lib.rs | 10 ++-- oxa/oxa-work/src/publish.rs | 21 ++++++++- oxa/oxa-work/src/queue.rs | 81 +++++++++++++++++++++++++++++--- 7 files changed, 123 insertions(+), 57 deletions(-) diff --git a/oxa/docs/work.md b/oxa/docs/work.md index abc67449c..4f6f89e47 100644 --- a/oxa/docs/work.md +++ b/oxa/docs/work.md @@ -218,6 +218,8 @@ summaries, without raw assistant output or acceptance logs. Queue mode watches a directory of regular, non-hidden `*.json` task files. +Each file may contain up to 128 tasks and occupy up to 1 MiB. Split larger +batches across files. After adding task files, run `oxa work --queue ./tasks --max 2 --min-free-mib 1024`. diff --git a/oxa/oxa-cli/src/work_cmd.rs b/oxa/oxa-cli/src/work_cmd.rs index ad2d8b3f8..1132183b6 100644 --- a/oxa/oxa-cli/src/work_cmd.rs +++ b/oxa/oxa-cli/src/work_cmd.rs @@ -419,14 +419,14 @@ impl Context { &policy, &|_: &oxa_work::Record| Ok(()), &mut |note, lock, offline| { - let mut runner = |program: &str, args: &[String]| { + let runner = |program: &str, args: &[String], gate: &std::fs::File| { let executable = crate::netguard::trusted_executable_for( program, Some(policy.host_path.as_os_str()), Some(&root), Some(&dirs.home), )?; - crate::work_reports::process(&executable.to_string_lossy(), args, &root, lock) + crate::work_reports::process(&executable.to_string_lossy(), args, &root, lock, gate) }; crate::work_reports::report( note, @@ -435,7 +435,7 @@ impl Context { &ctx, offline, &mut crate::report_cmd::tty_confirm(env), - &mut runner, + runner, ) }, )?; diff --git a/oxa/oxa-cli/src/work_reports.rs b/oxa/oxa-cli/src/work_reports.rs index 66f82f764..7c025e346 100644 --- a/oxa/oxa-cli/src/work_reports.rs +++ b/oxa/oxa-cli/src/work_reports.rs @@ -12,10 +12,17 @@ use std::{ time::{Duration, Instant}, }; -pub(crate) fn process(program: &str, args: &[String], cwd: &Path, lock: &File) -> Option<(i32, String)> { - let mut child = Command::new(program) - .args(args) - .current_dir(cwd) +pub(crate) fn process( + program: &str, + args: &[String], + cwd: &Path, + lock: &File, + gate: &File, +) -> Option<(i32, String)> { + let mut command = Command::new(program); + command.args(args).current_dir(cwd); + oxa_work::inherit_report_lease(&mut command, gate).ok()?; + let mut child = command .stdin(Stdio::from(lock.try_clone().ok()?)) .stdout(Stdio::piped()) .stderr(Stdio::null()) @@ -111,7 +118,7 @@ pub(crate) fn report( ctx: &ScrubCtx, offline: bool, confirm: Confirm<'_>, - runner: Runner<'_>, + mut run: impl FnMut(&str, &[String], &File) -> Option<(i32, String)>, ) -> Result { // Retain the queue's per-user reporting lease across every effect. use std::os::unix::fs::OpenOptionsExt; @@ -125,6 +132,7 @@ pub(crate) fn report( .open(dirs.agent_dir.join("work-report.lock")) .map_err(|e| e.to_string())?; gate.lock().map_err(|e| e.to_string())?; + let runner: Runner<'_> = &mut |program, args| run(program, args, &gate); let note = &ProblemNote { title: visible(¬e.title), notes: visible(¬e.notes), diff --git a/oxa/oxa-cli/tests/common/work.rs b/oxa/oxa-cli/tests/common/work.rs index 6465f48c5..1b889bcab 100644 --- a/oxa/oxa-cli/tests/common/work.rs +++ b/oxa/oxa-cli/tests/common/work.rs @@ -1,44 +1,10 @@ use oxa_fake::{FakeOpts, FakeServer, ScriptTurn, TestEnv}; #[path = "work_confirm.rs"] mod confirm; -use std::{path::Path, process::Command}; -pub fn git(env: &TestEnv, dir: &Path, args: &[&str]) -> String { - let out = Command::new("git") - .args(args) - .current_dir(dir) - .env_clear() - .envs(env.envs("http://127.0.0.1:1")) - .env("GIT_AUTHOR_NAME", "Test") - .env("GIT_AUTHOR_EMAIL", "test@example.invalid") - .env("GIT_COMMITTER_NAME", "Test") - .env("GIT_COMMITTER_EMAIL", "test@example.invalid") - .output() - .unwrap(); - assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); - String::from_utf8_lossy(&out.stdout).trim().into() -} -pub fn setup(env: &TestEnv, done: &str) { - git(env, &env.work(), &["init", "--bare", "remote.git"]); - git(env, &env.work(), &["init", "-b", "main", "project"]); - let project = env.work().join("project"); - git(env, &project, &["config", "user.name", "Test"]); - git(env, &project, &["config", "user.email", "test@example.invalid"]); - env.write( - "work/project/.oxa/settings.json", - &format!(r#"{{"done":{done}}}"#), - ); - git(env, &project, &["add", "."]); - git(env, &project, &["commit", "-m", "fixture"]); - git(env, &project, &["remote", "add", "origin", "../remote.git"]); - git(env, &project, &["push", "origin", "main"]); - oxa_perm::TrustStore::default() - .save(&env.agent_dir().join("trust.json"), &project, true) - .unwrap(); - env.write( - "work/tasks.json", - r#"[{"id":"test","branch":"agent/test","model":"fake","prompt":"Finish task"}]"#, - ); -} +#[path = "work_repo.rs"] +mod work_repo; +use std::process::Command; +pub use work_repo::{git, setup}; pub fn run(env: &TestEnv, turns: Vec) -> serde_json::Value { run_args(env, turns, &["work", "../tasks.json"], None) } diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index bdb18885a..f237a2445 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -15,7 +15,7 @@ mod notes; mod publish; mod queue; pub use notes::{ProblemNote, ReportRecord}; -pub use publish::{publish, report_notes}; +pub use publish::{inherit_report_lease, publish, report_notes}; pub use queue::{run_queue, QueueEvent, QueueOptions, QueueStatus}; pub mod mirrors; pub mod shadow; @@ -269,10 +269,12 @@ pub fn metadata_git(binary: &Path, path: &std::ffi::OsStr, repo: &Path, args: &[ } pub fn read_tasks(path: &Path) -> Result> { - parse_tasks(&fs::read(path).map_err(|e| e.to_string())?) + let bytes = fs::read(path).map_err(|e| e.to_string())?; + let tasks = serde_json::from_slice(&bytes).map_err(|e| format!("task file: {e}"))?; + validate_tasks(tasks) } -pub(crate) fn parse_tasks(bytes: &[u8]) -> Result> { - let tasks: Vec = serde_json::from_slice(bytes).map_err(|e| format!("task file: {e}"))?; + +fn validate_tasks(tasks: Vec) -> Result> { if tasks.is_empty() { return Err("task file is empty".into()); } diff --git a/oxa/oxa-work/src/publish.rs b/oxa/oxa-work/src/publish.rs index 138af3225..ab388ff28 100644 --- a/oxa/oxa-work/src/publish.rs +++ b/oxa/oxa-work/src/publish.rs @@ -1,7 +1,10 @@ //! Explicit publishing of the accepted commit; existing queued PRs are never pushed. use super::{checked, state, tree, CheckpointObserver, Outcome, Record, Result, RunPolicy, Task}; use state::Step; -use std::os::unix::fs::OpenOptionsExt; +use std::os::{ + fd::AsRawFd, + unix::{fs::OpenOptionsExt, process::CommandExt}, +}; use std::{ fs::File, path::{Path, PathBuf}, @@ -346,6 +349,22 @@ pub fn publish( } Ok(value.record) } +/// Keep a report's per-user lease held if the controller dies during a host command. +pub fn inherit_report_lease(command: &mut Command, gate: &File) -> std::io::Result<()> { + let gate = gate.try_clone()?; + // SAFETY: the closure owns the descriptor, and only changes its flags + // in this child. fcntl is async-signal-safe. + unsafe { + command.pre_exec(move || { + if libc::fcntl(gate.as_raw_fd(), libc::F_SETFD, 0) == -1 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + Ok(()) +} + pub fn report_notes( task: &Task, policy: &RunPolicy, diff --git a/oxa/oxa-work/src/queue.rs b/oxa/oxa-work/src/queue.rs index fc1914f9c..94cb4aecf 100644 --- a/oxa/oxa-work/src/queue.rs +++ b/oxa/oxa-work/src/queue.rs @@ -1,10 +1,14 @@ //! Directory queue: oldest regular JSON files first, with bounded independent workers. use super::{blocked_record, checked, state, Record, Result, RunPolicy, Task}; -use serde::Serialize; -use std::os::unix::fs::DirBuilderExt; +use serde::{ + de::{Error, IgnoredAny, SeqAccess, Visitor}, + Deserialize, Deserializer, Serialize, +}; +use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt}; use std::{ collections::{HashMap, HashSet}, fs, + io::{self, Read}, path::{Path, PathBuf}, process::Command, sync::{mpsc, Arc}, @@ -101,6 +105,71 @@ fn free_space(repo: &Path, common: &Path, queue: &Path, df: &Path, policy: &RunP .min(available(queue, df, policy)?) .min(available(tree_volume, df, policy)?)) } +const MAX_INPUT_BYTES: u64 = 1024 * 1024; +const MAX_INPUT_TASKS: usize = 128; + +struct QueueTasks(Vec); +impl<'de> Deserialize<'de> for QueueTasks { + fn deserialize>(deserializer: D) -> std::result::Result { + struct TasksVisitor; + impl<'de> Visitor<'de> for TasksVisitor { + type Value = QueueTasks; + + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "an array of at most {MAX_INPUT_TASKS} tasks") + } + + fn visit_seq>(self, mut seq: A) -> std::result::Result { + let mut tasks = Vec::new(); + while tasks.len() < MAX_INPUT_TASKS { + let Some(task) = seq.next_element()? else { + return Ok(QueueTasks(tasks)); + }; + tasks.push(task); + } + if seq.next_element::()?.is_some() { + return Err(A::Error::custom(format!( + "task file exceeds {MAX_INPUT_TASKS} tasks" + ))); + } + Ok(QueueTasks(tasks)) + } + } + deserializer.deserialize_seq(TasksVisitor) + } +} + +fn read_input(path: &Path) -> Result>> { + // A queued file may be replaced after the directory scan. Open the final + // component without following links or blocking on a substituted FIFO. + let file = match fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) + .open(path) + { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.to_string()), + }; + let meta = file.metadata().map_err(|e| e.to_string())?; + if !meta.is_file() { + return Err("queue input must be a regular file".into()); + } + let too_large = || format!("task file exceeds {MAX_INPUT_BYTES} bytes"); + if meta.len() > MAX_INPUT_BYTES { + return Err(too_large()); + } + let mut bytes = Vec::new(); + file.take(MAX_INPUT_BYTES + 1) + .read_to_end(&mut bytes) + .map_err(|e| e.to_string())?; + if bytes.len() as u64 > MAX_INPUT_BYTES { + return Err(too_large()); + } + let QueueTasks(tasks) = serde_json::from_slice(&bytes).map_err(|e| format!("task file: {e}"))?; + super::validate_tasks(tasks).map(Some) +} + fn input_file(entry: fs::DirEntry) -> Result> { let path = entry.path(); if path @@ -164,10 +233,10 @@ fn scan( if inputs.files.get(&file) == Some(&(modified, len)) && !inputs.errors.contains_key(&file) { continue; } - let tasks = match fs::read(&file) { - Ok(bytes) => super::parse_tasks(&bytes), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, - Err(error) => Err(error.to_string()), + let tasks = match read_input(&file) { + Ok(Some(tasks)) => Ok(tasks), + Ok(None) => continue, + Err(error) => Err(error), }; let previous = inputs.errors.remove(&file); let mut problems = vec![]; From 10a13925eee9f71ae343506aa62b167525e0c31a Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 22:46:21 +0100 Subject: [PATCH 36/52] Retain queue admission receipts in the original claim row --- oxa/oxa-work/src/board.rs | 2 +- oxa/oxa-work/src/board/claims.rs | 11 +++- oxa/oxa-work/src/board/claims/controls.rs | 33 +++++++++- .../src/board/claims/controls/leases.rs | 28 +++++++++ oxa/oxa-work/tests/board_controls.rs | 62 +++++++++++++++++++ 5 files changed, 130 insertions(+), 6 deletions(-) diff --git a/oxa/oxa-work/src/board.rs b/oxa/oxa-work/src/board.rs index 922c6b966..3a084911d 100644 --- a/oxa/oxa-work/src/board.rs +++ b/oxa/oxa-work/src/board.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet}; mod claims; -pub use claims::{Assignment, AssignmentBoard, AssignmentData, Claims, ReleaseRecord}; +pub use claims::{Assignment, AssignmentBoard, AssignmentData, Claims, QueueAdmission, ReleaseRecord}; pub mod assignment; mod intake; pub mod landing; diff --git a/oxa/oxa-work/src/board/claims.rs b/oxa/oxa-work/src/board/claims.rs index 7dff3c653..4d6581714 100644 --- a/oxa/oxa-work/src/board/claims.rs +++ b/oxa/oxa-work/src/board/claims.rs @@ -9,7 +9,7 @@ use std::os::unix::fs::{MetadataExt, OpenOptionsExt}; use std::path::{Path, PathBuf}; mod controls; -pub use controls::{Assignment, AssignmentBoard, AssignmentData, ReleaseRecord}; +pub use controls::{Assignment, AssignmentBoard, AssignmentData, QueueAdmission, ReleaseRecord}; mod imports; mod reviews; @@ -126,6 +126,11 @@ impl Claims { let saved: Saved = serde_json::from_slice(&bytes).map_err(|_| "invalid board claim state")?; if saved.version != 1 || saved.number != number + || saved + .data + .queue_admission + .as_ref() + .is_some_and(|admission| !admission.matches_lease(saved.state.lease.as_ref())) || saved.review_requests.iter().any(|key| !key.valid()) || saved.state.lease.as_ref().is_some_and(|l| { saved.state.attempts == 0 @@ -249,7 +254,7 @@ impl Claims { if self.established(entry.number)? { self.mark(entry.number)?; } - if row.data.import_hold { + if row.data.import_hold || row.data.queue_admission.is_some() { return Ok(None); } if !authorize(&mut row)? { @@ -316,7 +321,7 @@ impl Claims { } let _lock = self.lock(number)?; let mut row = self.load(number)?; - if row.data.policy.is_some() { + if row.data.policy.is_some() || row.data.queue_admission.is_some() { return Err("configured assignments require authenticated release with a reason".into()); } if row.state.lease.as_ref() != Some(lease) { diff --git a/oxa/oxa-work/src/board/claims/controls.rs b/oxa/oxa-work/src/board/claims/controls.rs index bf15856e1..5d9f87f71 100644 --- a/oxa/oxa-work/src/board/claims/controls.rs +++ b/oxa/oxa-work/src/board/claims/controls.rs @@ -15,6 +15,31 @@ pub struct AssignmentData { pub import_hold: bool, #[serde(default)] pub last_release: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub queue_admission: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(tag = "state", rename_all = "snake_case", deny_unknown_fields)] +pub enum QueueAdmission { + Reserved, + Retained { key: String, lease: Lease }, +} + +fn valid_queue_key(key: &str) -> bool { + key.len() == 64 + && key + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +impl QueueAdmission { + pub(super) fn matches_lease(&self, current: Option<&Lease>) -> bool { + match self { + Self::Reserved => current.is_some(), + Self::Retained { key, lease } => valid_queue_key(key) && current == Some(lease), + } + } } #[derive(Clone, Deserialize, Serialize)] @@ -95,7 +120,7 @@ impl Claims { for entry in &mut board.entries { if assignments .get(&entry.number) - .is_some_and(|row| row.data.import_hold) + .is_some_and(|row| row.data.import_hold || row.data.queue_admission.is_some()) { entry.eligible = false; } @@ -191,7 +216,11 @@ impl Claims { return Ok(false); } let controls = row.data.policy.get_or_insert_with(AssignmentPolicy::default); - Ok(controls.allows(worker, entry.action)) + if !controls.allows(worker, entry.action) { + return Ok(false); + } + row.data.queue_admission = Some(QueueAdmission::Reserved); + Ok(true) }) } } diff --git a/oxa/oxa-work/src/board/claims/controls/leases.rs b/oxa/oxa-work/src/board/claims/controls/leases.rs index 9b04394c4..bd8b3d63b 100644 --- a/oxa/oxa-work/src/board/claims/controls/leases.rs +++ b/oxa/oxa-work/src/board/claims/controls/leases.rs @@ -36,6 +36,33 @@ impl Claims { }) } + /// Retain the original queue receipt after idle reservation and before publication. + /// Holds only the Claims directory lock through the exact-lease comparison and + /// durable save. The caller may hold the spool lock for the full publication. + pub fn retain_queue_receipt(&self, number: u64, worker_id: &str, key: &str, lease: &Lease) -> Result<()> { + if !valid_queue_key(key) || !valid_worker_id(worker_id) || worker_id != lease.holder { + return Err("invalid board queue receipt or authenticated worker".into()); + } + let _lock = self.lock(number)?; + let mut row = self.load(number)?; + if row.state.lease.as_ref() != Some(lease) { + return Err("board queue lease changed before receipt retention".into()); + } + match row.data.queue_admission.as_ref() { + Some(QueueAdmission::Reserved) => {} + Some(QueueAdmission::Retained { + key: retained_key, + lease: retained_lease, + }) if retained_key == key && retained_lease == lease => return Ok(()), + _ => return Err("board queue admission did not reserve this receipt".into()), + } + row.data.queue_admission = Some(QueueAdmission::Retained { + key: key.into(), + lease: lease.clone(), + }); + self.save(number, row, true) + } + /// The trusted worker frontend supplies identity; knowing a lease is not authority to release it. pub fn release_for( &self, @@ -94,6 +121,7 @@ impl Claims { if lease.claimed_at.is_none_or(|at| at > now) { return Err("board release clock moved backwards".into()); } + row.data.queue_admission = None; row.data.last_release = Some(ReleaseRecord { holder: lease.holder, released_at: now, diff --git a/oxa/oxa-work/tests/board_controls.rs b/oxa/oxa-work/tests/board_controls.rs index 6fb95ccf8..deabc4367 100644 --- a/oxa/oxa-work/tests/board_controls.rs +++ b/oxa/oxa-work/tests/board_controls.rs @@ -801,3 +801,65 @@ fn idle_claims_require_saved_state_not_a_denied_claim_marker() { .is_some()); assert_eq!(fixture.store().snapshot().unwrap()[&44].attempts, 1); } + +#[test] +fn queue_receipt_survives_reopen_and_cannot_be_replaced_or_released_by_an_old_lease() { + let fixture = Fixture::new(); + fixture.establish([3]); + let store = fixture.store(); + let task = entry(); + let first = worker("slot-1"); + let second = worker("slot-2"); + let key = "a".repeat(64); + let lease = store + .claim_idle_for(&task, &policy(), &first, 10, 10) + .unwrap() + .unwrap(); + + assert!(store.retain_queue_receipt(3, &second.id, &key, &lease).is_err()); + assert!(store + .retain_queue_receipt(3, &first.id, "invalid", &lease) + .is_err()); + let reserved = fixture.store().assignments().unwrap().remove(&3).unwrap(); + assert_eq!(reserved.data.queue_admission, Some(QueueAdmission::Reserved)); + assert_eq!(reserved.state.lease.as_ref(), Some(&lease)); + assert!(store + .claim_for(&task, &policy(), &second, 20, 10) + .unwrap() + .is_none()); + + store.retain_queue_receipt(3, &first.id, &key, &lease).unwrap(); + let reopened = fixture.store(); + let retained = reopened.assignments().unwrap().remove(&3).unwrap(); + assert_eq!( + retained.data.queue_admission, + Some(QueueAdmission::Retained { + key: key.clone(), + lease: lease.clone(), + }) + ); + reopened.retain_queue_receipt(3, &first.id, &key, &lease).unwrap(); + assert!(reopened + .retain_queue_receipt(3, &first.id, &"b".repeat(64), &lease) + .is_err()); + assert_eq!( + reopened.assignments().unwrap()[&3].data.revision, + retained.data.revision + ); + assert!(reopened + .release_for(3, &first.id, &lease, 20, "reconciled") + .unwrap()); + + let replacement = reopened + .claim_idle_for(&task, &policy(), &first, 21, 10) + .unwrap() + .unwrap(); + assert_ne!(replacement, lease); + assert!(reopened.retain_queue_receipt(3, &first.id, &key, &lease).is_err()); + assert!(!reopened + .release_for(3, &first.id, &lease, 22, "stale completion") + .unwrap()); + let assignment = reopened.assignments().unwrap().remove(&3).unwrap(); + assert_eq!(assignment.data.queue_admission, Some(QueueAdmission::Reserved)); + assert_eq!(assignment.state.lease, Some(replacement)); +} From 63122e722ca0529edc10ffe896fd3b91b03a43c1 Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 22:47:16 +0100 Subject: [PATCH 37/52] Revert "Retain queue admission receipts in the original claim row" This reverts commit 10a13925eee9f71ae343506aa62b167525e0c31a. --- oxa/oxa-work/src/board.rs | 2 +- oxa/oxa-work/src/board/claims.rs | 11 +--- oxa/oxa-work/src/board/claims/controls.rs | 33 +--------- .../src/board/claims/controls/leases.rs | 28 --------- oxa/oxa-work/tests/board_controls.rs | 62 ------------------- 5 files changed, 6 insertions(+), 130 deletions(-) diff --git a/oxa/oxa-work/src/board.rs b/oxa/oxa-work/src/board.rs index 3a084911d..922c6b966 100644 --- a/oxa/oxa-work/src/board.rs +++ b/oxa/oxa-work/src/board.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet}; mod claims; -pub use claims::{Assignment, AssignmentBoard, AssignmentData, Claims, QueueAdmission, ReleaseRecord}; +pub use claims::{Assignment, AssignmentBoard, AssignmentData, Claims, ReleaseRecord}; pub mod assignment; mod intake; pub mod landing; diff --git a/oxa/oxa-work/src/board/claims.rs b/oxa/oxa-work/src/board/claims.rs index 4d6581714..7dff3c653 100644 --- a/oxa/oxa-work/src/board/claims.rs +++ b/oxa/oxa-work/src/board/claims.rs @@ -9,7 +9,7 @@ use std::os::unix::fs::{MetadataExt, OpenOptionsExt}; use std::path::{Path, PathBuf}; mod controls; -pub use controls::{Assignment, AssignmentBoard, AssignmentData, QueueAdmission, ReleaseRecord}; +pub use controls::{Assignment, AssignmentBoard, AssignmentData, ReleaseRecord}; mod imports; mod reviews; @@ -126,11 +126,6 @@ impl Claims { let saved: Saved = serde_json::from_slice(&bytes).map_err(|_| "invalid board claim state")?; if saved.version != 1 || saved.number != number - || saved - .data - .queue_admission - .as_ref() - .is_some_and(|admission| !admission.matches_lease(saved.state.lease.as_ref())) || saved.review_requests.iter().any(|key| !key.valid()) || saved.state.lease.as_ref().is_some_and(|l| { saved.state.attempts == 0 @@ -254,7 +249,7 @@ impl Claims { if self.established(entry.number)? { self.mark(entry.number)?; } - if row.data.import_hold || row.data.queue_admission.is_some() { + if row.data.import_hold { return Ok(None); } if !authorize(&mut row)? { @@ -321,7 +316,7 @@ impl Claims { } let _lock = self.lock(number)?; let mut row = self.load(number)?; - if row.data.policy.is_some() || row.data.queue_admission.is_some() { + if row.data.policy.is_some() { return Err("configured assignments require authenticated release with a reason".into()); } if row.state.lease.as_ref() != Some(lease) { diff --git a/oxa/oxa-work/src/board/claims/controls.rs b/oxa/oxa-work/src/board/claims/controls.rs index 5d9f87f71..bf15856e1 100644 --- a/oxa/oxa-work/src/board/claims/controls.rs +++ b/oxa/oxa-work/src/board/claims/controls.rs @@ -15,31 +15,6 @@ pub struct AssignmentData { pub import_hold: bool, #[serde(default)] pub last_release: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub queue_admission: Option, -} - -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] -#[serde(tag = "state", rename_all = "snake_case", deny_unknown_fields)] -pub enum QueueAdmission { - Reserved, - Retained { key: String, lease: Lease }, -} - -fn valid_queue_key(key: &str) -> bool { - key.len() == 64 - && key - .bytes() - .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) -} - -impl QueueAdmission { - pub(super) fn matches_lease(&self, current: Option<&Lease>) -> bool { - match self { - Self::Reserved => current.is_some(), - Self::Retained { key, lease } => valid_queue_key(key) && current == Some(lease), - } - } } #[derive(Clone, Deserialize, Serialize)] @@ -120,7 +95,7 @@ impl Claims { for entry in &mut board.entries { if assignments .get(&entry.number) - .is_some_and(|row| row.data.import_hold || row.data.queue_admission.is_some()) + .is_some_and(|row| row.data.import_hold) { entry.eligible = false; } @@ -216,11 +191,7 @@ impl Claims { return Ok(false); } let controls = row.data.policy.get_or_insert_with(AssignmentPolicy::default); - if !controls.allows(worker, entry.action) { - return Ok(false); - } - row.data.queue_admission = Some(QueueAdmission::Reserved); - Ok(true) + Ok(controls.allows(worker, entry.action)) }) } } diff --git a/oxa/oxa-work/src/board/claims/controls/leases.rs b/oxa/oxa-work/src/board/claims/controls/leases.rs index bd8b3d63b..9b04394c4 100644 --- a/oxa/oxa-work/src/board/claims/controls/leases.rs +++ b/oxa/oxa-work/src/board/claims/controls/leases.rs @@ -36,33 +36,6 @@ impl Claims { }) } - /// Retain the original queue receipt after idle reservation and before publication. - /// Holds only the Claims directory lock through the exact-lease comparison and - /// durable save. The caller may hold the spool lock for the full publication. - pub fn retain_queue_receipt(&self, number: u64, worker_id: &str, key: &str, lease: &Lease) -> Result<()> { - if !valid_queue_key(key) || !valid_worker_id(worker_id) || worker_id != lease.holder { - return Err("invalid board queue receipt or authenticated worker".into()); - } - let _lock = self.lock(number)?; - let mut row = self.load(number)?; - if row.state.lease.as_ref() != Some(lease) { - return Err("board queue lease changed before receipt retention".into()); - } - match row.data.queue_admission.as_ref() { - Some(QueueAdmission::Reserved) => {} - Some(QueueAdmission::Retained { - key: retained_key, - lease: retained_lease, - }) if retained_key == key && retained_lease == lease => return Ok(()), - _ => return Err("board queue admission did not reserve this receipt".into()), - } - row.data.queue_admission = Some(QueueAdmission::Retained { - key: key.into(), - lease: lease.clone(), - }); - self.save(number, row, true) - } - /// The trusted worker frontend supplies identity; knowing a lease is not authority to release it. pub fn release_for( &self, @@ -121,7 +94,6 @@ impl Claims { if lease.claimed_at.is_none_or(|at| at > now) { return Err("board release clock moved backwards".into()); } - row.data.queue_admission = None; row.data.last_release = Some(ReleaseRecord { holder: lease.holder, released_at: now, diff --git a/oxa/oxa-work/tests/board_controls.rs b/oxa/oxa-work/tests/board_controls.rs index deabc4367..6fb95ccf8 100644 --- a/oxa/oxa-work/tests/board_controls.rs +++ b/oxa/oxa-work/tests/board_controls.rs @@ -801,65 +801,3 @@ fn idle_claims_require_saved_state_not_a_denied_claim_marker() { .is_some()); assert_eq!(fixture.store().snapshot().unwrap()[&44].attempts, 1); } - -#[test] -fn queue_receipt_survives_reopen_and_cannot_be_replaced_or_released_by_an_old_lease() { - let fixture = Fixture::new(); - fixture.establish([3]); - let store = fixture.store(); - let task = entry(); - let first = worker("slot-1"); - let second = worker("slot-2"); - let key = "a".repeat(64); - let lease = store - .claim_idle_for(&task, &policy(), &first, 10, 10) - .unwrap() - .unwrap(); - - assert!(store.retain_queue_receipt(3, &second.id, &key, &lease).is_err()); - assert!(store - .retain_queue_receipt(3, &first.id, "invalid", &lease) - .is_err()); - let reserved = fixture.store().assignments().unwrap().remove(&3).unwrap(); - assert_eq!(reserved.data.queue_admission, Some(QueueAdmission::Reserved)); - assert_eq!(reserved.state.lease.as_ref(), Some(&lease)); - assert!(store - .claim_for(&task, &policy(), &second, 20, 10) - .unwrap() - .is_none()); - - store.retain_queue_receipt(3, &first.id, &key, &lease).unwrap(); - let reopened = fixture.store(); - let retained = reopened.assignments().unwrap().remove(&3).unwrap(); - assert_eq!( - retained.data.queue_admission, - Some(QueueAdmission::Retained { - key: key.clone(), - lease: lease.clone(), - }) - ); - reopened.retain_queue_receipt(3, &first.id, &key, &lease).unwrap(); - assert!(reopened - .retain_queue_receipt(3, &first.id, &"b".repeat(64), &lease) - .is_err()); - assert_eq!( - reopened.assignments().unwrap()[&3].data.revision, - retained.data.revision - ); - assert!(reopened - .release_for(3, &first.id, &lease, 20, "reconciled") - .unwrap()); - - let replacement = reopened - .claim_idle_for(&task, &policy(), &first, 21, 10) - .unwrap() - .unwrap(); - assert_ne!(replacement, lease); - assert!(reopened.retain_queue_receipt(3, &first.id, &key, &lease).is_err()); - assert!(!reopened - .release_for(3, &first.id, &lease, 22, "stale completion") - .unwrap()); - let assignment = reopened.assignments().unwrap().remove(&3).unwrap(); - assert_eq!(assignment.data.queue_admission, Some(QueueAdmission::Reserved)); - assert_eq!(assignment.state.lease, Some(replacement)); -} From 9f4fb848d6366b3eb0649a6f8ad3d6d86d1316a1 Mon Sep 17 00:00:00 2001 From: oxa-agent Date: Tue, 17 Nov 2026 23:15:45 +0100 Subject: [PATCH 38/52] Retain native attempt bindings in board claim rows (#1017) --- oxa/oxa-work/src/board.rs | 2 +- oxa/oxa-work/src/board/claims.rs | 13 ++ oxa/oxa-work/src/board/claims/controls.rs | 2 + oxa/oxa-work/src/board/claims/native.rs | 108 ++++++++++ oxa/oxa-work/tests/board_native_binding.rs | 219 +++++++++++++++++++++ 5 files changed, 343 insertions(+), 1 deletion(-) create mode 100644 oxa/oxa-work/src/board/claims/native.rs create mode 100644 oxa/oxa-work/tests/board_native_binding.rs diff --git a/oxa/oxa-work/src/board.rs b/oxa/oxa-work/src/board.rs index c39eb5322..02dcb3cca 100644 --- a/oxa/oxa-work/src/board.rs +++ b/oxa/oxa-work/src/board.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet}; mod claims; -pub use claims::{Assignment, AssignmentBoard, AssignmentData, Claims, ReleaseRecord}; +pub use claims::{Assignment, AssignmentBoard, AssignmentData, Claims, NativeBinding, ReleaseRecord}; pub mod assignment; mod intake; pub mod landing; diff --git a/oxa/oxa-work/src/board/claims.rs b/oxa/oxa-work/src/board/claims.rs index 7dff3c653..84a2aea76 100644 --- a/oxa/oxa-work/src/board/claims.rs +++ b/oxa/oxa-work/src/board/claims.rs @@ -11,6 +11,8 @@ use std::path::{Path, PathBuf}; mod controls; pub use controls::{Assignment, AssignmentBoard, AssignmentData, ReleaseRecord}; mod imports; +mod native; +pub use native::NativeBinding; mod reviews; #[derive(Deserialize, Serialize)] @@ -126,6 +128,16 @@ impl Claims { let saved: Saved = serde_json::from_slice(&bytes).map_err(|_| "invalid board claim state")?; if saved.version != 1 || saved.number != number + || saved.data.native_binding.as_ref().is_some_and(|binding| { + !binding.valid() + || binding.number != number + || binding.lease.generation != (saved.state.utc_day, saved.state.attempts) + || saved + .state + .lease + .as_ref() + .is_some_and(|lease| lease != &binding.lease) + }) || saved.review_requests.iter().any(|key| !key.valid()) || saved.state.lease.as_ref().is_some_and(|l| { saved.state.attempts == 0 @@ -281,6 +293,7 @@ impl Claims { generation: (day, state.attempts), }; state.lease = Some(lease.clone()); + row.data.native_binding = None; let replacing = self.established(entry.number)?; self.save(entry.number, row, replacing)?; self.mark(entry.number)?; diff --git a/oxa/oxa-work/src/board/claims/controls.rs b/oxa/oxa-work/src/board/claims/controls.rs index bf15856e1..2b0f65910 100644 --- a/oxa/oxa-work/src/board/claims/controls.rs +++ b/oxa/oxa-work/src/board/claims/controls.rs @@ -15,6 +15,8 @@ pub struct AssignmentData { pub import_hold: bool, #[serde(default)] pub last_release: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub native_binding: Option, } #[derive(Clone, Deserialize, Serialize)] diff --git a/oxa/oxa-work/src/board/claims/native.rs b/oxa/oxa-work/src/board/claims/native.rs new file mode 100644 index 000000000..3e8c46463 --- /dev/null +++ b/oxa/oxa-work/src/board/claims/native.rs @@ -0,0 +1,108 @@ +//! Immutable native attempt receipts in the existing protected assignment row. +use super::*; + +#[derive(Clone, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct NativeBinding { + pub number: u64, + pub key: String, + pub lease: Lease, + pub native_task_id: String, + pub branch: String, +} + +impl NativeBinding { + /// Inputs come from the trusted prepared packet and the original claim receipt. + pub fn new(number: u64, key: &str, lease: &Lease, branch: &str) -> Result { + let binding = Self { + number, + key: key.into(), + lease: lease.clone(), + native_task_id: format!("board-{key}-{}-{}", lease.generation.0, lease.generation.1), + branch: branch.into(), + }; + if !binding.valid() { + return Err("invalid native attempt binding".into()); + } + Ok(binding) + } + + pub(super) fn valid(&self) -> bool { + self.number > 0 + && self.key.len() == 64 + && self + .key + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + && valid_worker_id(&self.lease.holder) + && self.lease.generation.1 > 0 + && self.lease.claimed_at.is_some_and(|at| at < self.lease.expires_at) + && self.native_task_id + == format!( + "board-{}-{}-{}", + self.key, self.lease.generation.0, self.lease.generation.1 + ) + && crate::valid_task_id(&self.native_task_id) + && !self.branch.is_empty() + && self.branch.len() <= 256 + && !self.branch.starts_with(['-', '/']) + && !self.branch.ends_with(['/', '.']) + && !self.branch.contains([' ', '~', '^', ':', '?', '*', '[', '\\']) + && !self.branch.contains("..") + && !self.branch.contains("@{") + && !self.branch.chars().any(char::is_control) + && self + .branch + .split('/') + .all(|part| !part.is_empty() && !part.starts_with('.') && !part.ends_with(".lock")) + } +} + +impl Claims { + /// Persist before queue publication. Identity is authenticated by the trusted frontend. + /// Failure leaves the original reservation charged and must prevent the ready hardlink. + pub fn bind_native(&self, worker_id: &str, binding: &NativeBinding) -> Result { + if !binding.valid() || worker_id != binding.lease.holder { + return Err("authenticated native attempt binding required".into()); + } + let _lock = self.lock(binding.number)?; + let mut row = self.load(binding.number)?; + if row.state.lease.as_ref() != Some(&binding.lease) { + return Ok(false); + } + if let Some(existing) = &row.data.native_binding { + return Ok(existing == binding); + } + for (number, other) in self.rows_locked()? { + if number != binding.number + && other.data.native_binding.as_ref().is_some_and(|existing| { + existing.native_task_id == binding.native_task_id + || (other.state.lease.is_some() && existing.branch == binding.branch) + }) + { + return Ok(false); + } + } + row.data.native_binding = Some(binding.clone()); + self.save(binding.number, row, true)?; + self.mark(binding.number)?; + Ok(true) + } + + /// Match the admitted native identity, retaining the original full lease after release. + /// This read is not completion authority; release still requires trusted terminal evidence. + pub fn native_binding(&self, native_task_id: &str, branch: &str) -> Result> { + let mut found = None; + for row in self.rows()?.into_values() { + if let Some(binding) = row.data.native_binding { + if binding.native_task_id == native_task_id && binding.branch == branch { + if found.is_some() { + return Err("ambiguous native attempt binding".into()); + } + found = Some(binding); + } + } + } + Ok(found) + } +} diff --git a/oxa/oxa-work/tests/board_native_binding.rs b/oxa/oxa-work/tests/board_native_binding.rs new file mode 100644 index 000000000..9b4656852 --- /dev/null +++ b/oxa/oxa-work/tests/board_native_binding.rs @@ -0,0 +1,219 @@ +#[path = "support/private_dir.rs"] +mod test_dir; +use oxa_work::board::{assignment::*, *}; +use std::{collections::BTreeMap, fs}; +use test_dir::PrivateDir; + +fn setup() -> (PrivateDir, Claims, Lease) { + let dir = PrivateDir::new("native-binding"); + let store = Claims::open(dir.path()).unwrap(); + store + .import_snapshot( + Some(ManualAuthority::Owner), + &BTreeMap::from([( + 1, + ImportState { + state: AttemptState::default(), + excluded: false, + }, + )]), + 10, + ) + .unwrap(); + let lease = store + .claim_idle_for(&entry(), &policy(), &worker(), 10, 20) + .unwrap() + .unwrap(); + (dir, store, lease) +} +fn worker() -> Worker { + Worker { + id: "codex:node/slot-1".into(), + kind: WorkerKind::Codex, + skills: Default::default(), + roles: [Role::Builder].into(), + } +} +fn policy() -> Policy { + Policy { + enabled: true, + ..Policy::default() + } +} +fn entry() -> Entry { + Entry { + number: 1, + head: "synthetic-head".into(), + release_pinned: false, + action: Action::FixFindings, + lease: None, + attempts_today: 0, + eligible: true, + } +} +fn binding(lease: &Lease) -> NativeBinding { + NativeBinding::new(1, &"a".repeat(64), lease, "agent/test").unwrap() +} + +#[test] +fn binding_is_durable_idempotent_and_immutable_for_the_original_lease() { + let (dir, store, lease) = setup(); + let binding = binding(&lease); + assert_eq!(binding.native_task_id, format!("board-{}-0-1", "a".repeat(64))); + assert!(store.bind_native(&worker().id, &binding).unwrap()); + let path = dir.path().join("1.board-claim.json"); + let bytes = fs::read(&path).unwrap(); + let reopened = Claims::open(dir.path()).unwrap(); + assert!(reopened.bind_native(&worker().id, &binding).unwrap()); + assert_eq!(fs::read(&path).unwrap(), bytes); + let mut changed = binding.clone(); + changed.branch = "agent/other".into(); + assert!(!reopened.bind_native(&worker().id, &changed).unwrap()); + changed = NativeBinding::new(1, &"b".repeat(64), &lease, &binding.branch).unwrap(); + assert!(!reopened.bind_native(&worker().id, &changed).unwrap()); + assert!( + reopened + .native_binding(&binding.native_task_id, &binding.branch) + .unwrap() + == Some(binding) + ); + assert_eq!(fs::read(&path).unwrap(), bytes); +} + +#[test] +fn every_original_lease_component_and_holder_authentication_are_fenced() { + let (dir, store, lease) = setup(); + let before = fs::read(dir.path().join("1.board-claim.json")).unwrap(); + assert!(store.bind_native("codex:node/other", &binding(&lease)).is_err()); + for field in 0..5 { + let mut stale = lease.clone(); + match field { + 0 => stale.holder = "codex:node/other".into(), + 1 => stale.expires_at += 1, + 2 => stale.claimed_at = Some(11), + 3 => stale.generation.0 += 1, + _ => stale.generation.1 += 1, + } + assert!(!store.bind_native(&stale.holder, &binding(&stale)).unwrap()); + } + assert_eq!(fs::read(dir.path().join("1.board-claim.json")).unwrap(), before); +} + +#[test] +fn completion_lookup_never_substitutes_a_new_lease_or_a_different_branch() { + let (_dir, store, lease) = setup(); + let original = binding(&lease); + assert!(store.bind_native(&worker().id, &original).unwrap()); + assert!(store + .native_binding(&original.native_task_id, "agent/wrong") + .unwrap() + .is_none()); + assert!(store + .native_binding("wrong-task", &original.branch) + .unwrap() + .is_none()); + assert!(store + .release_for(1, &worker().id, &lease, 20, "finished") + .unwrap()); + assert!( + store + .native_binding(&original.native_task_id, &original.branch) + .unwrap() + == Some(original.clone()) + ); + let replacement = store + .claim_for(&entry(), &policy(), &worker(), 21, 20) + .unwrap() + .unwrap(); + assert!(store + .native_binding(&original.native_task_id, &original.branch) + .unwrap() + .is_none()); + assert!(!store.bind_native(&worker().id, &original).unwrap()); + assert!(!store + .release_for(1, &worker().id, &lease, 22, "stale completion") + .unwrap()); + assert!(store.bind_native(&worker().id, &binding(&replacement)).unwrap()); + assert_eq!(store.snapshot().unwrap()[&1].attempts, 2); +} + +#[test] +fn old_rows_remain_readable_and_malformed_saved_bindings_fail_closed() { + let (dir, store, lease) = setup(); + let path = dir.path().join("1.board-claim.json"); + let old: serde_json::Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + assert!(old["data"].get("native_binding").is_none()); + assert!(store.assignments().unwrap()[&1].data.native_binding.is_none()); + assert!(store.bind_native(&worker().id, &binding(&lease)).unwrap()); + let saved: serde_json::Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + for field in ["number", "key", "native_task_id", "branch", "lease"] { + let mut corrupt = saved.clone(); + corrupt["data"]["native_binding"][field] = serde_json::json!("invalid"); + fs::write(&path, serde_json::to_vec(&corrupt).unwrap()).unwrap(); + assert!(Claims::open(dir.path()).unwrap().assignments().is_err()); + } +} + +#[test] +fn constructor_rejects_invalid_identity_without_touching_the_store() { + let (_dir, _store, lease) = setup(); + for branch in ["", "-bad", "a..b", "a//b", "a.lock", "a\nb", "a/.bad", "a@{b"] { + assert!(NativeBinding::new(1, &"a".repeat(64), &lease, branch).is_err()); + } + assert!(NativeBinding::new(0, &"a".repeat(64), &lease, "agent/test").is_err()); + assert!(NativeBinding::new(1, &"A".repeat(64), &lease, "agent/test").is_err()); + assert!(NativeBinding::new(1, "short", &lease, "agent/test").is_err()); +} + +#[test] +fn concurrent_conflicting_bindings_have_one_durable_winner() { + let (_dir, store, lease) = setup(); + let first = binding(&lease); + let second = NativeBinding::new(1, &"b".repeat(64), &lease, "agent/other").unwrap(); + let barrier = std::sync::Barrier::new(2); + let results = std::thread::scope(|scope| { + let a = scope.spawn(|| { + barrier.wait(); + store.bind_native(&worker().id, &first).unwrap() + }); + let b = scope.spawn(|| { + barrier.wait(); + store.bind_native(&worker().id, &second).unwrap() + }); + (a.join().unwrap(), b.join().unwrap()) + }); + assert_ne!(results.0, results.1); + let winner = store + .assignments() + .unwrap() + .remove(&1) + .unwrap() + .data + .native_binding + .unwrap(); + assert!(winner == if results.0 { first } else { second }); + assert_eq!(store.snapshot().unwrap()[&1].attempts, 1); +} + +#[test] +fn a_live_or_expired_binding_cannot_be_reused_for_another_item() { + let (_dir, store, lease) = setup(); + let original = binding(&lease); + assert!(store.bind_native(&worker().id, &original).unwrap()); + let mut other_entry = entry(); + other_entry.number = 2; + // Generic claim_for remains available, including after an existing lease expires. + let other_lease = store + .claim_for(&other_entry, &policy(), &worker(), 10, 20) + .unwrap() + .unwrap(); + let same_task = NativeBinding::new(2, &original.key, &other_lease, "agent/other").unwrap(); + assert!(!store.bind_native(&worker().id, &same_task).unwrap()); + let same_branch = NativeBinding::new(2, &"b".repeat(64), &other_lease, &original.branch).unwrap(); + assert!(!store.bind_native(&worker().id, &same_branch).unwrap()); + assert!(store + .release_for(1, &worker().id, &lease, 40, "reconciled") + .unwrap()); + assert!(store.bind_native(&worker().id, &same_branch).unwrap()); + assert!(!store.bind_native(&worker().id, &original).unwrap()); +} From 7bd662d80968683e5f016dff0cac38623fe57ad7 Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 23:17:42 +0100 Subject: [PATCH 39/52] Keep task ownership through publication and reporting --- oxa/oxa-cli/src/work_cmd.rs | 83 +++++++++++++-------------- oxa/oxa-work/src/lib.rs | 108 +++++++++++++++++++++++++----------- oxa/oxa-work/src/publish.rs | 103 ++++++++++++++++++++-------------- 3 files changed, 179 insertions(+), 115 deletions(-) diff --git a/oxa/oxa-cli/src/work_cmd.rs b/oxa/oxa-cli/src/work_cmd.rs index 26a276717..8afc0187e 100644 --- a/oxa/oxa-cli/src/work_cmd.rs +++ b/oxa/oxa-cli/src/work_cmd.rs @@ -368,8 +368,17 @@ impl Context { let dirs = &self.dirs; let env = &self.env; let (rerun, publish, reports) = (self.rerun, self.publish, self.reports); + let blocked = |detail| { + let mut record = oxa_work::blocked_record(plan.task(), detail); + record.base_sha = Some(plan.base_sha().to_owned()); + record + }; + let mut locked = match oxa_work::LockedTask::acquire(plan.task(), policy) { + Ok(locked) => locked, + Err(error) => return Ok(blocked(error)), + }; let cached = if (publish || reports) && !rerun { - oxa_work::action_record(&policy, plan.task())? + oxa_work::action_record(policy, plan.task())? } else { None }; @@ -377,21 +386,16 @@ impl Context { record } else { let ctx = crate::report_cmd::ScrubCtx::from_env(env, dirs); - match oxa_work::run_task_with_notes( - &root, - &binary, - &plan, - &policy, + match locked.run_with_notes( + root, + binary, + plan, rerun, &|_: &oxa_work::Record| Ok(()), &|note| crate::work_reports::scrub_note(note, &ctx), ) { Ok(record) => record, - Err(oxa_work::RunError::Blocked(detail)) => { - let mut record = oxa_work::blocked_record(plan.task(), detail); - record.base_sha = Some(plan.base_sha().to_owned()); - record - } + Err(oxa_work::RunError::Blocked(detail)) => blocked(detail), Err(oxa_work::RunError::Checkpoint(error)) => return Err(error), } }; @@ -399,48 +403,41 @@ impl Context { let gh = crate::netguard::trusted_executable_for( "gh", Some(policy.host_path.as_os_str()), - Some(&root), + Some(root), Some(&dirs.home), ) .unwrap_or_else(|| PathBuf::from("/unavailable/gh")); - record = oxa_work::publish( - &root, - &binary, - plan.task(), - &policy, + record = locked.publish( + root, + binary, &gh, &|_: &oxa_work::Record| Ok(()), &mut crate::report_cmd::tty_confirm(env), )?; } if reports && !record.worktree.as_os_str().is_empty() { - let settings = oxa_config::Settings::load(dirs, &root, false); + let settings = oxa_config::Settings::load(dirs, root, false); let ctx = crate::report_cmd::ScrubCtx::from_env(env, dirs); - record = oxa_work::report_notes( - plan.task(), - &policy, - &|_: &oxa_work::Record| Ok(()), - &mut |note, lock, offline| { - let runner = |program: &str, args: &[String], gate: &std::fs::File| { - let executable = crate::netguard::trusted_executable_for( - program, - Some(policy.host_path.as_os_str()), - Some(&root), - Some(&dirs.home), - )?; - crate::work_reports::process(&executable.to_string_lossy(), args, &root, lock, gate) - }; - crate::work_reports::report( - note, - dirs, - &settings, - &ctx, - offline, - &mut crate::report_cmd::tty_confirm(env), - runner, - ) - }, - )?; + record = locked.report_notes(&|_: &oxa_work::Record| Ok(()), &mut |note, lock, offline| { + let runner = |program: &str, args: &[String], gate: &std::fs::File| { + let executable = crate::netguard::trusted_executable_for( + program, + Some(policy.host_path.as_os_str()), + Some(root), + Some(&dirs.home), + )?; + crate::work_reports::process(&executable.to_string_lossy(), args, root, lock, gate) + }; + crate::work_reports::report( + note, + dirs, + &settings, + &ctx, + offline, + &mut crate::report_cmd::tty_confirm(env), + runner, + ) + })?; } Ok(record) } diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index a586f7421..9ff5cee23 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -808,40 +808,86 @@ pub fn run_task_with_notes( observer: &dyn CheckpointObserver, scrub_note: &(dyn Fn(&ProblemNote) -> Option + Sync), ) -> std::result::Result { - let task = &planned.task; - validate(task)?; + LockedTask::acquire(planned.task(), policy)? + .run_with_notes(repo, binary, planned, rerun, observer, scrub_note) +} + +/// Own the existing task lock across execution and its explicit terminal actions. +pub struct LockedTask<'a> { + task: &'a Task, + policy: &'a RunPolicy, + lock: fs::File, +} + +impl<'a> LockedTask<'a> { + pub fn acquire(task: &'a Task, policy: &'a RunPolicy) -> Result { + Self::open(task, policy, true) + } + + fn open(task: &'a Task, policy: &'a RunPolicy, create: bool) -> Result { + validate(task)?; + if create { + fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(&policy.storage) + .map_err(|error| format!("task storage: {error}"))?; + } + verify_task_storage(&policy.storage).map_err(|error| format!("task storage: {error}"))?; + let lock_path = policy.storage.join(format!("{}.lock", task.id)); + let lock = fs::OpenOptions::new() + .read(true) + .write(true) + .mode(0o600) + .custom_flags(shadow::OWNER_FILE_FLAGS) + .create(create) + .truncate(false) + .open(&lock_path) + .map_err(|e| format!("cannot lock task {}: {e}", task.id))?; + let lock_metadata = lock + .metadata() + .map_err(|e| format!("cannot inspect task lock: {e}"))?; + // SAFETY: geteuid has no preconditions and returns this process's effective owner. + if !shadow::regular_file_owned_by(&lock_metadata, unsafe { libc::geteuid() }) { + return Err("cannot lock task: lock must be an owner-controlled regular file".into()); + } + lock.try_lock() + .map_err(|e| format!("task {} is already running: {e}", task.id))?; + Ok(Self { task, policy, lock }) + } + + pub fn run_with_notes( + &mut self, + repo: &Path, + binary: &Path, + planned: &PlannedTask, + rerun: bool, + observer: &dyn CheckpointObserver, + scrub_note: &(dyn Fn(&ProblemNote) -> Option + Sync), + ) -> std::result::Result { + run_locked(repo, binary, planned, self, rerun, observer, scrub_note) + } +} + +fn run_locked( + repo: &Path, + binary: &Path, + planned: &PlannedTask, + locked: &LockedTask<'_>, + rerun: bool, + observer: &dyn CheckpointObserver, + scrub_note: &(dyn Fn(&ProblemNote) -> Option + Sync), +) -> std::result::Result { + let (task, policy, lock_file) = (locked.task, locked.policy, &locked.lock); + if task != &planned.task { + return Err("task specification changed".into()); + } let root = PathBuf::from(checked(git(policy, repo, &["rev-parse", "--show-toplevel"]))?); let common = PathBuf::from(checked(git( policy, &root, &["rev-parse", "--path-format=absolute", "--git-common-dir"], ))?); - fs::DirBuilder::new() - .recursive(true) - .mode(0o700) - .create(&policy.storage) - .and_then(|()| verify_task_storage(&policy.storage)) - .map_err(|error| format!("task storage: {error}"))?; - let lock_path = policy.storage.join(format!("{}.lock", task.id)); - let lock_file = fs::OpenOptions::new() - .read(true) - .write(true) - .mode(0o600) - .custom_flags(shadow::OWNER_FILE_FLAGS) - .create(true) - .truncate(false) - .open(&lock_path) - .map_err(|e| format!("cannot lock task {}: {e}", task.id))?; - let lock_metadata = lock_file - .metadata() - .map_err(|e| format!("cannot inspect task lock: {e}"))?; - // SAFETY: geteuid has no preconditions and returns this process's effective owner. - if !shadow::regular_file_owned_by(&lock_metadata, unsafe { libc::geteuid() }) { - return Err("cannot lock task: lock must be an owner-controlled regular file".into()); - } - lock_file - .try_lock() - .map_err(|e| format!("task {} is already running: {e}", task.id))?; // The inherited lock retains its inode; status never tries to acquire it. let receipt = policy.storage.join(format!("{}.json", task.id)); let replacing = receipt.exists(); @@ -926,7 +972,7 @@ pub fn run_task_with_notes( repo: &root, binary, policy: &policy, - lock: &lock_file, + lock: lock_file, writer: &mut writer, observer, scrub_note, @@ -957,7 +1003,7 @@ pub fn run_task_with_notes( writer .save(&mut state, Step::Saving, observer) .map_err(RunError::Checkpoint)?; - if let Err(error) = execute::save_work(&root, binary, task, &policy, &lock_file, &mut state.record) { + if let Err(error) = execute::save_work(&root, binary, task, &policy, lock_file, &mut state.record) { if state.record.outcome != Outcome::Interrupted { state.record.outcome = Outcome::Blocked; } @@ -987,7 +1033,7 @@ pub fn run_task_with_notes( if recover_accepted_save { // A failed or interrupted save can leave a worktree that changed since // acceptance. Qualify the exact recovered commit before restoring success. - match committed::verify_commit(&root, binary, &policy, &state, &lock_file) { + match committed::verify_commit(&root, binary, &policy, &state, lock_file) { Ok(_) => { let initial = state.execution.as_ref().and_then(|point| point.initial.as_ref()); state.record.outcome = if state.record.head_sha.as_ref() == initial { diff --git a/oxa/oxa-work/src/publish.rs b/oxa/oxa-work/src/publish.rs index 298d3b73f..4206def9b 100644 --- a/oxa/oxa-work/src/publish.rs +++ b/oxa/oxa-work/src/publish.rs @@ -1,46 +1,25 @@ //! Explicit publishing of the accepted commit; existing queued PRs are never pushed. -use super::{checked, state, tree, CheckpointObserver, Outcome, Record, Result, RunPolicy, Task}; +use super::{checked, state, tree, CheckpointObserver, LockedTask, Outcome, Record, Result, RunPolicy, Task}; use state::Step; -use std::os::{ - fd::AsRawFd, - unix::{fs::OpenOptionsExt, process::CommandExt}, -}; +use std::os::{fd::AsRawFd, unix::process::CommandExt}; use std::{ fs::File, path::{Path, PathBuf}, process::Command, }; -fn context( - task: &Task, - policy: &RunPolicy, +fn action_context( + locked: &LockedTask<'_>, observer: &dyn CheckpointObserver, -) -> Result<(state::Writer, File, state::State)> { - super::validate(task)?; - super::verify_task_storage(&policy.storage).map_err(|e| format!("task storage: {e}"))?; - let path = policy.storage.join(format!("{}.json", task.id)); - let lock = std::fs::OpenOptions::new() - .read(true) - .write(true) - .custom_flags(super::shadow::OWNER_FILE_FLAGS) - .open(path.with_extension("lock")) - .map_err(|e| e.to_string())?; - let metadata = lock - .metadata() - .map_err(|e| format!("cannot inspect task lock: {e}"))?; - // SAFETY: geteuid has no arguments or preconditions. - if !super::shadow::regular_file_owned_by(&metadata, unsafe { libc::geteuid() }) { - return Err("cannot lock task: lock must be an owner-controlled regular file".into()); - } - lock.try_lock() - .map_err(|e| format!("task is already running: {e}"))?; +) -> Result<(state::Writer, state::State)> { + let path = locked.policy.storage.join(format!("{}.json", locked.task.id)); let value = state::load(&path)?; - if value.task != *task { + if value.task != *locked.task { return Err("task specification changed".into()); } let mut writer = state::Writer::new(path, true); writer.replay(&value, observer)?; - Ok((writer, lock, value)) + Ok((writer, value)) } fn gh(binary: &Path, policy: &RunPolicy, repo: &Path, lock: &File, args: &[&str]) -> Result { let mut command = Command::new(binary); @@ -138,7 +117,19 @@ pub fn publish( observer: &dyn CheckpointObserver, confirm: &mut dyn FnMut(&str) -> Option, ) -> Result { - let (mut writer, lock, mut value) = context(task, policy, observer)?; + LockedTask::open(task, policy, false)?.publish(repo, binary, gh_binary, observer, confirm) +} + +fn publish_locked( + repo: &Path, + binary: &Path, + locked: &LockedTask<'_>, + gh_binary: &Path, + observer: &dyn CheckpointObserver, + confirm: &mut dyn FnMut(&str) -> Option, +) -> Result { + let (mut writer, mut value) = action_context(locked, observer)?; + let (task, policy, lock) = (locked.task, locked.policy, &locked.lock); let accepted = value.record.acceptance_outcome.unwrap_or(value.record.outcome); if !super::completed_execution(value.record.step) || !matches!(accepted, Outcome::Done | Outcome::NothingToDo) @@ -155,14 +146,14 @@ pub fn publish( } let tree = &value.record.worktree; tree::validate(policy, repo, tree, task)?; - if super::dirty_worktree(binary, tree, repo, policy, &lock)? { + if super::dirty_worktree(binary, tree, repo, policy, lock)? { return Err("uncommitted work must be saved before publishing".into()); } let head = checked(git(tree, &["rev-parse", "HEAD"]))?; if value.record.head_sha.as_deref() != Some(head.as_str()) { return Err("task branch changed after acceptance; use --rerun before publishing".into()); } - let proof = super::committed::verify_commit(repo, binary, policy, &value, &lock)?; + let proof = super::committed::verify_commit(repo, binary, policy, &value, lock)?; let accepted_base = value .record .base_sha @@ -178,7 +169,7 @@ pub fn publish( } let repository = repository(&json(&gh( tree, - &lock, + lock, &["repo", "view", &remote, "--json", "nameWithOwner,url"], )?)?)?; let name = repository.qualified.as_str(); @@ -186,7 +177,7 @@ pub fn publish( let repo_name = repository.name.as_str(); let prs = json(&gh( tree, - &lock, + lock, &[ "pr", "list", @@ -220,7 +211,7 @@ pub fn publish( let number = pr["number"].as_u64().ok_or("PR number missing")?.to_string(); let value = json(&gh( tree, - &lock, + lock, &["pr", "view", &number, "--repo", name, "--json", "body"], )?)?; Some(value["body"].as_str().ok_or("PR body missing")?.to_owned()) @@ -247,7 +238,7 @@ pub fn publish( let query = "query($owner:String!,$name:String!,$number:Int!){repository(owner:$owner,name:$name){pullRequest(number:$number){mergeQueueEntry{id}}}}"; let data = json(&gh( tree, - &lock, + lock, &[ "api", "graphql", @@ -304,7 +295,7 @@ pub fn publish( let number = pr["number"].as_u64().ok_or("PR number missing")?.to_string(); gh( tree, - &lock, + lock, &[ "pr", "edit", @@ -323,7 +314,7 @@ pub fn publish( if draft { args.push("--undo"); } - gh(tree, &lock, &args)?; + gh(tree, lock, &args)?; } pr["url"].as_str().ok_or("PR URL missing")?.to_string() } else { @@ -345,7 +336,7 @@ pub fn publish( if draft { args.push("--draft"); } - gh(tree, &lock, &args)? + gh(tree, lock, &args)? }; let _ = std::fs::remove_file(body); Ok((result.trim().to_string(), draft)) @@ -388,7 +379,16 @@ pub fn report_notes( observer: &dyn CheckpointObserver, reporter: &mut dyn FnMut(&super::ProblemNote, &File, bool) -> Result, ) -> Result { - let (mut writer, lock, mut value) = context(task, policy, observer)?; + LockedTask::open(task, policy, false)?.report_notes(observer, reporter) +} + +fn report_locked( + locked: &LockedTask<'_>, + observer: &dyn CheckpointObserver, + reporter: &mut dyn FnMut(&super::ProblemNote, &File, bool) -> Result, +) -> Result { + let (mut writer, mut value) = action_context(locked, observer)?; + let (policy, lock) = (locked.policy, &locked.lock); let prior_step = value.record.step; // Reporting is ancillary: a failed save/publication remains recoverable even // if reporting is interrupted between durable report receipts. @@ -407,7 +407,7 @@ pub fn report_notes( } writer.save(&mut value, reporting_step, observer)?; let report = - reporter(&value.record.problem_notes[index], &lock, policy.offline).unwrap_or_else(|detail| { + reporter(&value.record.problem_notes[index], lock, policy.offline).unwrap_or_else(|detail| { super::ReportRecord { disposition: "failed".into(), draft: PathBuf::new(), @@ -433,3 +433,24 @@ pub fn report_notes( )?; Ok(value.record) } + +impl LockedTask<'_> { + pub fn publish( + &mut self, + repo: &Path, + binary: &Path, + gh_binary: &Path, + observer: &dyn CheckpointObserver, + confirm: &mut dyn FnMut(&str) -> Option, + ) -> Result { + publish_locked(repo, binary, self, gh_binary, observer, confirm) + } + + pub fn report_notes( + &mut self, + observer: &dyn CheckpointObserver, + reporter: &mut dyn FnMut(&super::ProblemNote, &File, bool) -> Result, + ) -> Result { + report_locked(self, observer, reporter) + } +} From bb47e728b5b60f9738b14981618be9c229da8e20 Mon Sep 17 00:00:00 2001 From: oxa-agent Date: Tue, 17 Nov 2026 23:19:11 +0100 Subject: [PATCH 40/52] Exercise exhausted binding writes without refunding the lease (#1017) --- oxa/oxa-work/tests/board_native_binding.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/oxa/oxa-work/tests/board_native_binding.rs b/oxa/oxa-work/tests/board_native_binding.rs index 9b4656852..1b83d6645 100644 --- a/oxa/oxa-work/tests/board_native_binding.rs +++ b/oxa/oxa-work/tests/board_native_binding.rs @@ -152,6 +152,10 @@ fn old_rows_remain_readable_and_malformed_saved_bindings_fail_closed() { fs::write(&path, serde_json::to_vec(&corrupt).unwrap()).unwrap(); assert!(Claims::open(dir.path()).unwrap().assignments().is_err()); } + let mut mismatched = saved; + mismatched["data"]["native_binding"]["lease"]["expires_at"] = serde_json::json!(31); + fs::write(&path, serde_json::to_vec(&mismatched).unwrap()).unwrap(); + assert!(store.assignments().is_err()); } #[test] @@ -217,3 +221,21 @@ fn a_live_or_expired_binding_cannot_be_reused_for_another_item() { assert!(store.bind_native(&worker().id, &same_branch).unwrap()); assert!(!store.bind_native(&worker().id, &original).unwrap()); } + +#[test] +fn a_failed_binding_write_preserves_the_reservation_without_refund() { + let (dir, store, lease) = setup(); + let path = dir.path().join("1.board-claim.json"); + let mut saved: serde_json::Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + saved["data"]["revision"] = serde_json::json!(u64::MAX); + let before = serde_json::to_vec(&saved).unwrap(); + fs::write(&path, &before).unwrap(); + assert!(store.bind_native(&worker().id, &binding(&lease)).is_err()); + assert_eq!(fs::read(&path).unwrap(), before); + assert_eq!(store.snapshot().unwrap()[&1].lease, Some(lease)); + assert_eq!(store.snapshot().unwrap()[&1].attempts, 1); + assert!(store + .claim_idle_for(&entry(), &policy(), &worker(), 40, 20) + .unwrap() + .is_none()); +} From e7866a56cc47cc55375d7460db5e434499b32f67 Mon Sep 17 00:00:00 2001 From: oxa-agent Date: Tue, 17 Nov 2026 23:30:44 +0100 Subject: [PATCH 41/52] Use an invalid Git branch in the malformed binding fixture (#1017) --- oxa/oxa-work/tests/board_native_binding.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/oxa/oxa-work/tests/board_native_binding.rs b/oxa/oxa-work/tests/board_native_binding.rs index 1b83d6645..88fa16f90 100644 --- a/oxa/oxa-work/tests/board_native_binding.rs +++ b/oxa/oxa-work/tests/board_native_binding.rs @@ -148,9 +148,16 @@ fn old_rows_remain_readable_and_malformed_saved_bindings_fail_closed() { let saved: serde_json::Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); for field in ["number", "key", "native_task_id", "branch", "lease"] { let mut corrupt = saved.clone(); - corrupt["data"]["native_binding"][field] = serde_json::json!("invalid"); + corrupt["data"]["native_binding"][field] = serde_json::json!(if field == "branch" { + "bad..branch" + } else { + "invalid" + }); fs::write(&path, serde_json::to_vec(&corrupt).unwrap()).unwrap(); - assert!(Claims::open(dir.path()).unwrap().assignments().is_err()); + assert!( + Claims::open(dir.path()).unwrap().assignments().is_err(), + "field: {field}" + ); } let mut mismatched = saved; mismatched["data"]["native_binding"]["lease"]["expires_at"] = serde_json::json!(31); From 08b4094d395f8a7237cd862649ed2d2a1c6bcb3b Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 23:35:48 +0100 Subject: [PATCH 42/52] Fence native queue completion with the original task binding --- oxa/oxa-work/src/lib.rs | 2 +- oxa/oxa-work/src/queue.rs | 129 ++++++++++++- oxa/oxa-work/src/queue/native.rs | 312 +++++++++++++++++++++++++++++++ 3 files changed, 433 insertions(+), 10 deletions(-) create mode 100644 oxa/oxa-work/src/queue/native.rs diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index 9ff5cee23..61daa0511 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -17,7 +17,7 @@ mod publish; mod queue; pub use notes::{ProblemNote, ReportRecord}; pub use publish::{inherit_report_lease, publish, report_notes}; -pub use queue::{run_queue, QueueEvent, QueueOptions, QueueStatus}; +pub use queue::{native_task, run_queue, NativeQueueOwner, QueueEvent, QueueOptions, QueueRun, QueueStatus}; pub mod mirrors; pub mod shadow; mod state; diff --git a/oxa/oxa-work/src/queue.rs b/oxa/oxa-work/src/queue.rs index 94cb4aecf..1a75c03d1 100644 --- a/oxa/oxa-work/src/queue.rs +++ b/oxa/oxa-work/src/queue.rs @@ -1,5 +1,5 @@ //! Directory queue: oldest regular JSON files first, with bounded independent workers. -use super::{blocked_record, checked, state, Record, Result, RunPolicy, Task}; +use super::{blocked_record, board, checked, state, LockedTask, Record, Result, RunPolicy, Task}; use serde::{ de::{Error, IgnoredAny, SeqAccess, Visitor}, Deserialize, Deserializer, Serialize, @@ -16,6 +16,9 @@ use std::{ time::{Duration, SystemTime}, }; +mod native; +pub use native::{native_task, NativeQueueOwner}; + #[derive(Clone, Debug)] pub struct QueueOptions { pub max: usize, @@ -64,6 +67,39 @@ struct Job { modified: SystemTime, file: PathBuf, index: usize, + binding: Option, +} + +type WorkerFn = dyn Fn(&Task) -> Result + Send + Sync; +type NativeWorkerFn = dyn Fn(&Task, &mut LockedTask<'_>) -> Result + Send + Sync; + +#[derive(Clone)] +enum QueueWorker { + Manual(Arc), + Native(Arc), +} + +struct NativeWorker { + claims: board::Claims, + worker_id: String, + policy: RunPolicy, + run: Arc, +} + +impl QueueWorker { + fn run(&self, job: &mut Job) -> Result { + let worker = match self { + Self::Manual(run) => return run(&job.task), + Self::Native(worker) => worker, + }; + let mut locked = LockedTask::acquire(&job.task, &worker.policy)?; + let owner = NativeQueueOwner { + claims: &worker.claims, + worker_id: &worker.worker_id, + }; + job.binding = Some(owner.admit(&job.task)?); + (worker.run)(&job.task, &mut locked) + } } fn busy(storage: &Path, id: &str) -> Result { match fs::OpenOptions::new() @@ -267,6 +303,7 @@ fn scan( file: file.clone(), modified, index, + binding: None, }); } } @@ -326,9 +363,62 @@ pub fn run_queue( options: &QueueOptions, policy: &RunPolicy, df: &Path, - worker: Arc Result + Send + Sync>, + worker: Arc, emit: &mut dyn FnMut(QueueEvent) -> Result<()>, ) -> Result { + run_queue_inner( + QueueRun { + repo, + directory, + options, + policy, + df, + }, + None, + QueueWorker::Manual(worker), + emit, + ) +} + +/// The same execution queue with trusted owner context for a native admitted attempt. +pub struct QueueRun<'a> { + pub repo: &'a Path, + pub directory: &'a Path, + pub options: &'a QueueOptions, + pub policy: &'a RunPolicy, + pub df: &'a Path, +} + +impl QueueRun<'_> { + pub fn run_native( + self, + owner: &NativeQueueOwner<'_>, + worker: Arc, + emit: &mut dyn FnMut(QueueEvent) -> Result<()>, + ) -> Result { + let worker = QueueWorker::Native(Arc::new(NativeWorker { + claims: owner.claims.clone(), + worker_id: owner.worker_id.into(), + policy: self.policy.clone(), + run: worker, + })); + run_queue_inner(self, Some(owner), worker, emit) + } +} + +fn run_queue_inner( + queue: QueueRun<'_>, + native: Option<&NativeQueueOwner<'_>>, + worker: QueueWorker, + emit: &mut dyn FnMut(QueueEvent) -> Result<()>, +) -> Result { + let QueueRun { + repo, + directory, + options, + policy, + df, + } = queue; if options.max == 0 { return Err("--max must be positive".into()); } @@ -406,12 +496,12 @@ pub fn run_queue( let id = job.task.id.clone(); let worker = worker.clone(); let tx = tx.clone(); - let input = job.clone(); + let mut input = job.clone(); match thread::Builder::new() .name(format!("oxa-work-{id}")) .spawn(move || { let record = - std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| worker(&input.task))) + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| worker.run(&mut input))) .unwrap_or_else(|_| { Ok(blocked_record( &input.task, @@ -480,11 +570,32 @@ pub fn run_queue( if let Some(handle) = active.handles.remove(&job.task.id) { let _ = handle.join(); } - if busy(&storage, &job.task.id)? { - external.push(job); - continue; - } - let record = record?; + let record = if let Some(native) = native { + match record.and_then(|record| { + let binding = job + .binding + .as_ref() + .ok_or("native admission was not established")?; + native.finish(policy, &job.task, binding, &record)?; + Ok(record) + }) { + Ok(record) => record, + Err(detail) => { + failed += 1; + emit(QueueEvent::Error { + file: job.file, + detail, + })?; + continue; + } + } + } else { + if busy(storage, &job.task.id)? { + external.push(job); + continue; + } + record? + }; finished += 1; if record.failed() { failed += 1; diff --git a/oxa/oxa-work/src/queue/native.rs b/oxa/oxa-work/src/queue/native.rs new file mode 100644 index 000000000..bdf7f9eee --- /dev/null +++ b/oxa/oxa-work/src/queue/native.rs @@ -0,0 +1,312 @@ +//! The native queue consumes the original claim receipt and settles after joining the worker. +use super::super::{board, load_task_checkpoint, validate, CheckpointKind, LockedTask, Outcome, Step}; +use super::{Record, Result, RunPolicy, Task}; + +/// Supplied by the trusted queue caller after authenticating its worker. +pub struct NativeQueueOwner<'a> { + pub claims: &'a board::Claims, + pub worker_id: &'a str, +} + +/// Materialize a prepared task with the canonical attempt identity and existing branch. +pub fn native_task(binding: &board::NativeBinding, mut task: Task) -> Result { + let canonical = board::NativeBinding::new(binding.number, &binding.key, &binding.lease, &binding.branch)?; + if binding != &canonical || task.branch != binding.branch { + return Err("prepared task does not own the bound native attempt".into()); + } + task.id.clone_from(&binding.native_task_id); + if !task.prompt.starts_with(crate::ISSUE_RUNNER_GUIDANCE) { + task.prompt.insert_str(0, crate::ISSUE_RUNNER_GUIDANCE); + } + validate(&task)?; + Ok(task) +} + +impl NativeQueueOwner<'_> { + pub(super) fn admit(&self, task: &Task) -> Result { + let binding = self + .claims + .native_binding(&task.id, &task.branch)? + .ok_or("native task has no retained admission binding")?; + if self.worker_id != binding.lease.holder || !task.runner_owns_github() { + return Err("native task lacks the authenticated worker or runner-owned GitHub policy".into()); + } + let assignment = self.claims.assignments()?; + if !assignment.get(&binding.number).is_some_and(|row| { + row.state.lease.as_ref() == Some(&binding.lease) + && row.data.native_binding.as_ref() == Some(&binding) + }) { + return Err("native admission no longer owns the original lease".into()); + } + Ok(binding) + } + + pub(super) fn finish( + &self, + policy: &RunPolicy, + task: &Task, + binding: &board::NativeBinding, + observed: &Record, + ) -> Result<()> { + if task.id != binding.native_task_id + || task.branch != binding.branch + || observed.id != task.id + || observed.branch != task.branch + || !finished(observed) + { + return Err("native completion lacks a whole-task finish observation".into()); + } + + // The queue has joined the worker. A surviving child retains this same + // lock and prevents reconciliation; retain it through checkpoint read and CAS. + let _locked = LockedTask::open(task, policy, false)?; + let saved = + load_task_checkpoint(policy, &task.id)?.ok_or("native completion checkpoint is unavailable")?; + if saved.task != *task + || !finished(&saved.record) + || saved.record.checkpoint_revision != observed.checkpoint_revision + || saved.record.checkpoint_at_ms != observed.checkpoint_at_ms + || saved.record.outcome != observed.outcome + || binding + .lease + .claimed_at + .and_then(|at| at.checked_mul(1000)) + .is_none_or(|at| at > saved.record.checkpoint_at_ms) + { + return Err("native completion does not match the current durable checkpoint".into()); + } + if !self.claims.release_for( + binding.number, + self.worker_id, + &binding.lease, + super::state::now_ms() / 1000, + "native task finished and its process exited", + )? { + return Err("native completion no longer owns the original lease".into()); + } + Ok(()) + } +} + +fn finished(record: &Record) -> bool { + record.step == Step::Finished + && record.checkpoint_kind == CheckpointKind::TaskTransition + && record.checkpoint_revision > 0 + && matches!( + record.outcome, + Outcome::Done | Outcome::NothingToDo | Outcome::Red + ) +} + +#[cfg(test)] +#[path = "../../tests/support/private_dir.rs"] +mod test_dir; + +#[cfg(test)] +mod tests { + use super::test_dir::PrivateDir; + use super::*; + use crate::{blocked_record, state, QueueEvent, QueueOptions, QueueRun}; + use board::assignment::{ManualAuthority, Role, Worker, WorkerKind}; + use board::{Action, AttemptState, Claims, Entry, ImportState, Policy}; + use std::collections::BTreeMap; + use std::fs; + use std::path::Path; + use std::process::Command; + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }; + + fn setup() -> (PrivateDir, Claims, board::NativeBinding, Task, RunPolicy) { + let directory = PrivateDir::new("native-completion"); + let claims = Claims::open(directory.path()).unwrap(); + claims + .import_snapshot( + Some(ManualAuthority::Owner), + &BTreeMap::from([( + 1, + ImportState { + state: AttemptState::default(), + excluded: false, + }, + )]), + 10, + ) + .unwrap(); + let worker = Worker { + id: "codex:fixture/worker".into(), + kind: WorkerKind::Codex, + skills: Default::default(), + roles: [Role::Builder].into(), + }; + let entry = Entry { + number: 1, + head: "fixture-head".into(), + release_pinned: false, + action: Action::FixFindings, + lease: None, + attempts_today: 0, + eligible: true, + }; + let lease = claims + .claim_idle_for( + &entry, + &Policy { + enabled: true, + ..Policy::default() + }, + &worker, + 10, + 20, + ) + .unwrap() + .unwrap(); + let binding = board::NativeBinding::new(1, &"a".repeat(64), &lease, "agent/fixture").unwrap(); + assert!(claims.bind_native(&worker.id, &binding).unwrap()); + let prepared = Task { + id: "c1".into(), + model: "fake".into(), + branch: binding.branch.clone(), + base: "main".into(), + prompt: "Synthetic task".into(), + max_time: 30, + }; + let policy = RunPolicy { + git_binary: "/usr/bin/git".into(), + host_path: "/usr/bin".into(), + terminate_binary: None, + offline: true, + storage: directory.path().join("runner"), + }; + (directory, claims, binding, prepared, policy) + } + + #[test] + fn only_the_exact_durable_whole_task_after_lock_release_settles_the_original_claim() { + let (_directory, claims, binding, prepared, policy) = setup(); + let owner = NativeQueueOwner { + claims: &claims, + worker_id: &binding.lease.holder, + }; + assert!(owner.admit(&prepared).is_err()); + let task = native_task(&binding, prepared).unwrap(); + assert!(owner.admit(&task).unwrap() == binding); + let locked = LockedTask::acquire(&task, &policy).unwrap(); + let synthetic = blocked_record(&task, "synthetic record".into()); + assert!(owner.finish(&policy, &task, &binding, &synthetic).is_err()); + + let mut saved = state::State::new(task.clone(), synthetic, true); + let mut writer = state::Writer::new(policy.storage.join(format!("{}.json", task.id)), false); + let observe = |_: &Record| Ok(()); + saved.record.outcome = Outcome::Done; + writer + .save_kind( + &mut saved, + Step::Acceptance, + CheckpointKind::WorkerFinished, + &observe, + ) + .unwrap(); + assert!(owner.finish(&policy, &task, &binding, &saved.record).is_err()); + writer.save(&mut saved, Step::Reporting, &observe).unwrap(); + assert!(owner.finish(&policy, &task, &binding, &saved.record).is_err()); + writer.save(&mut saved, Step::Finished, &observe).unwrap(); + + // A child surviving its controller owns the original file description. + let surviving_child = locked.lock.try_clone().unwrap(); + drop(locked); + assert!(owner.finish(&policy, &task, &binding, &saved.record).is_err()); + assert_eq!( + claims.snapshot().unwrap()[&1].lease.as_ref(), + Some(&binding.lease) + ); + drop(surviving_child); + + let mut stale = saved.record.clone(); + stale.checkpoint_revision -= 1; + assert!(owner.finish(&policy, &task, &binding, &stale).is_err()); + owner.finish(&policy, &task, &binding, &saved.record).unwrap(); + assert!(claims.snapshot().unwrap()[&1].lease.is_none()); + assert!(owner.admit(&task).is_err()); + } + + #[test] + fn native_queue_dispatches_one_canonical_attempt_and_settles_before_the_event() { + let (directory, claims, binding, prepared, policy) = setup(); + let owner = NativeQueueOwner { + claims: &claims, + worker_id: &binding.lease.holder, + }; + let task = native_task(&binding, prepared).unwrap(); + let repo = directory.path().join("repo"); + let queue = directory.path().join("queue"); + fs::create_dir(&repo).unwrap(); + fs::create_dir(&queue).unwrap(); + assert!(Command::new(&policy.git_binary) + .current_dir(&repo) + .env_clear() + .env("HOME", directory.path()) + .args(["init", "--quiet"]) + .status() + .unwrap() + .success()); + fs::write( + queue.join("task.json"), + serde_json::to_vec(&[task.clone()]).unwrap(), + ) + .unwrap(); + + let starts = Arc::new(AtomicUsize::new(0)); + let started = starts.clone(); + let receipt = binding.clone(); + let worker_claims = claims.clone(); + let worker: Arc = Arc::new(move |found, locked| { + assert_eq!(found.id, receipt.native_task_id); + assert_eq!(found.branch, receipt.branch); + assert_eq!( + worker_claims.snapshot().unwrap()[&receipt.number].lease.as_ref(), + Some(&receipt.lease) + ); + let second = LockedTask::acquire(found, locked.policy); + assert!(second.is_err(), "worker must receive the held task lock"); + started.fetch_add(1, Ordering::Relaxed); + let mut saved = state::State::new(found.clone(), blocked_record(found, String::new()), true); + saved.record.outcome = Outcome::NothingToDo; + let mut writer = + state::Writer::new(locked.policy.storage.join(format!("{}.json", found.id)), false); + writer + .save(&mut saved, Step::Finished, &|_: &Record| Ok(())) + .unwrap(); + Ok(saved.record) + }); + let options = QueueOptions { + min_free_bytes: 0, + once: true, + ..QueueOptions::default() + }; + let run = || QueueRun { + repo: &repo, + directory: &queue, + options: &options, + policy: &policy, + df: Path::new("/usr/bin/df"), + }; + let mut completed = 0; + let status = run() + .run_native(&owner, worker.clone(), &mut |event| { + if let QueueEvent::Task { record } = event { + assert_eq!(record.id, task.id); + assert!(claims.snapshot().unwrap()[&binding.number].lease.is_none()); + completed += 1; + } + Ok(()) + }) + .unwrap(); + assert_eq!((status.finished, status.failed, completed), (1, 0, 1)); + + let again = run().run_native(&owner, worker, &mut |_| Ok(())).unwrap(); + assert_eq!((again.finished, again.failed), (0, 1)); + assert_eq!(starts.load(Ordering::Relaxed), 1); + } +} From 72025f394a764e7318c1902620f220dcf3809afc Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 23:58:23 +0100 Subject: [PATCH 43/52] test(work): cover publication subprocess and destination guards --- oxa/oxa-cli/tests/common/work.rs | 10 +++++ oxa/oxa-cli/tests/work_publish.rs | 64 ++++++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/oxa/oxa-cli/tests/common/work.rs b/oxa/oxa-cli/tests/common/work.rs index 6465f48c5..30e5ef9db 100644 --- a/oxa/oxa-cli/tests/common/work.rs +++ b/oxa/oxa-cli/tests/common/work.rs @@ -55,6 +55,15 @@ pub fn run_observed( turns: Vec, args: &[&str], path: Option, +) -> (serde_json::Value, Vec) { + run_observed_with(env, turns, args, path, |_| {}) +} +pub fn run_observed_with( + env: &TestEnv, + turns: Vec, + args: &[&str], + path: Option, + configure: impl FnOnce(&mut Command), ) -> (serde_json::Value, Vec) { let server = FakeServer::start(turns, FakeOpts::default()).unwrap(); let mut command = Command::new(env!("CARGO_BIN_EXE_oxa")); @@ -66,6 +75,7 @@ pub fn run_observed( if let Some(path) = path { command.env("PATH", path); } + configure(&mut command); let (tx, rx) = std::sync::mpsc::channel(); std::thread::spawn(move || { tx.send(confirm::output(command)).unwrap(); diff --git a/oxa/oxa-cli/tests/work_publish.rs b/oxa/oxa-cli/tests/work_publish.rs index 6a6eac141..ce9e2df80 100644 --- a/oxa/oxa-cli/tests/work_publish.rs +++ b/oxa/oxa-cli/tests/work_publish.rs @@ -24,7 +24,12 @@ printf '%s\n' "$*" >> "$HOME/gh-calls" host=github.com if test -f "$HOME/enterprise-host"; then host=code.example.invalid; fi case "$1 $2" in - 'repo view') printf '{"nameWithOwner":"example/project","url":"https://%s/example/project"}' "$host" ;; + 'repo view') + if test "${GH_ENTERPRISE_TOKEN+x}" = x || test "${GITHUB_ENTERPRISE_TOKEN+x}" = x; then + touch "$HOME/ambient-publication-auth" + fi + printf '{"nameWithOwner":"example/project","url":"https://%s/example/project"}' "$host" + if test -f "$HOME/oversized-repo-response"; then head -c 1048576 /dev/zero | tr '\000' ' '; fi ;; 'pr list') if test -f "$HOME/pr-state.json"; then printf '['; cat "$HOME/pr-state.json"; printf ']'; else printf '[]'; fi ;; 'api graphql') if test -f "$HOME/queued"; then printf '%s' '{"data":{"repository":{"pullRequest":{"mergeQueueEntry":{"id":"queue"}}}}}'; else printf '%s' '{"data":{"repository":{"pullRequest":{"mergeQueueEntry":null}}}}'; fi ;; 'pr create') @@ -247,6 +252,63 @@ fn publishing_requires_acceptance_without_ignored_helpers() { assert!(!env.home().join("push-calls").exists()); assert!(!env.home().join("gh-calls").exists()); } +#[test] +fn publication_refuses_oversized_repository_metadata_before_push() { + let env = TestEnv::new(); + setup(&env, "[]"); + env.write("home/oversized-repo-response", ""); + let record = run_args( + &env, + edits(), + &["work", "../tasks.json", "--publish"], + Some(configure(&env)), + ); + assert_eq!(record["outcome"], "blocked", "{record}"); + assert!(env.home().join("gh-calls").exists()); + assert!(!env.home().join("push-calls").exists()); +} + +#[test] +fn publication_does_not_forward_ambient_enterprise_credentials_to_origin() { + let env = TestEnv::new(); + setup(&env, "[]"); + env.write("home/enterprise-host", ""); + let (record, _) = work::run_observed_with( + &env, + edits(), + &["work", "../tasks.json", "--publish"], + Some(configure(&env)), + |command| { + command.env("GH_ENTERPRISE_TOKEN", "synthetic-fixture"); + command.env("GITHUB_ENTERPRISE_TOKEN", "synthetic-fixture"); + }, + ); + assert_eq!(record["outcome"], "done", "{record}"); + assert!(env.home().join("push-calls").exists()); + assert!(!env.home().join("ambient-publication-auth").exists()); +} + +#[test] +fn publication_refuses_a_same_owner_fork_before_push() { + let env = TestEnv::new(); + setup(&env, "[]"); + env.write( + "home/pr-state.json", + r#"{"number":1,"url":"https://github.com/example/project/pull/1","isDraft":false,"headRepositoryOwner":{"login":"example"},"isCrossRepository":true}"#, + ); + env.write("home/published-body", "Synthetic fork"); + let record = run_args( + &env, + edits(), + &["work", "../tasks.json", "--publish"], + Some(configure(&env)), + ); + assert_eq!(record["outcome"], "blocked", "{record}"); + let calls = fs::read_to_string(env.home().join("gh-calls")).unwrap(); + assert!(calls.contains("pr list")); + assert!(!env.home().join("push-calls").exists()); +} + #[test] fn publishes_local_branch_and_updates_same_pr_on_restart() { use std::os::unix::fs::PermissionsExt; From d449d022fa4791bab4a7a328df9e81258ae365c5 Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Wed, 18 Nov 2026 00:00:11 +0100 Subject: [PATCH 44/52] test(work): cover whole queue input limits --- oxa/oxa-work/src/queue.rs | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/oxa/oxa-work/src/queue.rs b/oxa/oxa-work/src/queue.rs index 1a75c03d1..f1709c3b3 100644 --- a/oxa/oxa-work/src/queue.rs +++ b/oxa/oxa-work/src/queue.rs @@ -659,6 +659,42 @@ mod tests { assert!(errors.contains(&dir.join("count.json"))); } + #[test] + fn queue_rejects_an_overfull_directory_before_admission() { + let dir = crate::test_dir::PrivateDir::new("queue-directory-limit"); + for i in 0..257 { + fs::write( + dir.path().join(format!("{i}.json")), + format!(r#"[{{"id":"task-{i}","branch":"agent/task-{i}","model":"fake","prompt":"Synthetic task"}}]"#), + ) + .unwrap(); + } + let mut pending = vec![]; + let mut inputs = Inputs::default(); + let result = scan(dir.path(), &mut pending, &mut inputs, &mut |_| Ok(())); + assert!(result.unwrap_err().contains("256 task files")); + assert!(pending.is_empty()); + assert!(inputs.tasks.is_empty()); + } + + #[test] + fn queue_rejects_aggregate_input_bytes_before_admission() { + let dir = crate::test_dir::PrivateDir::new("queue-input-budget"); + for i in 0..17 { + let mut task = format!( + r#"[{{"id":"task-{i}","branch":"agent/task-{i}","model":"fake","prompt":"Synthetic task"}}]"# + ); + task.push_str(&" ".repeat(1024 * 1024 - task.len())); + fs::write(dir.path().join(format!("{i}.json")), task).unwrap(); + } + let mut pending = vec![]; + let mut inputs = Inputs::default(); + let result = scan(dir.path(), &mut pending, &mut inputs, &mut |_| Ok(())); + assert!(result.unwrap_err().contains("16 MiB")); + assert!(pending.is_empty()); + assert!(inputs.tasks.is_empty()); + } + #[test] fn archiving_an_unadmitted_file_cancels_its_pending_job() { let dir = std::env::temp_dir().join(format!( From 72a5d6e380f405609dbc6321ff9e377314d93fbd Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Wed, 18 Nov 2026 00:02:00 +0100 Subject: [PATCH 45/52] test(work): assert the confirmed report host is used --- oxa/oxa-cli/src/work_reports.rs | 40 +++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/oxa/oxa-cli/src/work_reports.rs b/oxa/oxa-cli/src/work_reports.rs index 8a914d26e..6029cb170 100644 --- a/oxa/oxa-cli/src/work_reports.rs +++ b/oxa/oxa-cli/src/work_reports.rs @@ -333,6 +333,46 @@ mod tests { assert_eq!(result.disposition, "local"); assert!(result.draft.exists()); } + #[test] + fn report_confirms_and_targets_the_same_qualified_repository() { + let env = TestEnv::new(); + let dirs = Dirs::under(env.root()); + let mut prompts = Vec::new(); + let mut calls = Vec::new(); + let result = report( + ¬e(), + &dirs, + &settings(), + &ScrubCtx::default(), + false, + &mut |prompt| { + prompts.push(prompt.to_owned()); + Some(true) + }, + &mut |program, args| { + if program != "gh" { + return Some((0, "{}".into())); + } + calls.push(args.to_vec()); + match args[1].as_str() { + "list" => Some((0, "[]".into())), + "create" => Some((0, "https://github.com/example/project/issues/10".into())), + _ => panic!("unexpected report request: {args:?}"), + } + }, + ) + .unwrap(); + assert_eq!(result.disposition, "filed"); + assert_eq!(prompts.len(), 2); + assert!(prompts + .iter() + .all(|prompt| prompt.contains("github.com/example/project"))); + assert_eq!(calls.len(), 5); + assert!(calls.iter().all(|args| args + .windows(2) + .any(|pair| { pair[0] == "--repo" && pair[1] == "github.com/example/project" }))); + } + #[test] fn matching_issue_is_commented_after_all_queries_and_reads() { let env = TestEnv::new(); From 4ab6488bad7c41ca88a1b48ef22d89a8cda9e95b Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Wed, 18 Nov 2026 00:03:38 +0100 Subject: [PATCH 46/52] Bound queue admission scans and retained task identities --- oxa/docs/work.md | 6 +++-- oxa/oxa-work/src/lib.rs | 11 +-------- oxa/oxa-work/src/queue.rs | 52 +++++++++++++++++++++++++++++++++------ 3 files changed, 50 insertions(+), 19 deletions(-) diff --git a/oxa/docs/work.md b/oxa/docs/work.md index 20dccfc7b..a43e4df3d 100644 --- a/oxa/docs/work.md +++ b/oxa/docs/work.md @@ -231,8 +231,10 @@ summaries, without raw assistant output or acceptance logs. Queue mode watches a directory of regular, non-hidden `*.json` task files. -Each file may contain up to 128 tasks and occupy up to 1 MiB. Split larger -batches across files. +Each file may contain up to 128 tasks and occupy up to 1 MiB. A queue accepts +up to 256 task files and 16 MiB of input per scan. The controller retains at +most 4,096 task identities and 16 MiB of task text across scans. Archive +completed inputs and restart the controller if it reaches the retention limit. After adding task files, run `oxa work --queue ./tasks --max 2 --min-free-mib 1024`. diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index 3e269392a..88691bfd8 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -956,16 +956,7 @@ fn run_locked( if planned.checkpoint_revision.is_some() && !replacing { return Err("task checkpoint disappeared after planning; retry the task file".into()); } - let repo_dir = if common.file_name().is_some_and(|name| name == ".git") { - common.parent().ok_or("git directory has no parent")? - } else { - &common - }; - let parent = repo_dir - .parent() - .ok_or("repository has no parent for worktrees")?; - let name = repo_dir.file_name().ok_or("repository has no directory name")?; - let trees = parent.join(".oxa-worktrees").join(name); + let trees = worktree_root(&common)?; let tree = trees.join(&task.id); let fresh = |revision| { let mut record = blocked_record(task, String::new()); diff --git a/oxa/oxa-work/src/queue.rs b/oxa/oxa-work/src/queue.rs index f1709c3b3..6d2e6fc99 100644 --- a/oxa/oxa-work/src/queue.rs +++ b/oxa/oxa-work/src/queue.rs @@ -143,6 +143,13 @@ fn free_space(repo: &Path, common: &Path, queue: &Path, df: &Path, policy: &RunP } const MAX_INPUT_BYTES: u64 = 1024 * 1024; const MAX_INPUT_TASKS: usize = 128; +const MAX_QUEUE_FILES: usize = 256; +const MAX_QUEUE_BYTES: u64 = 16 * 1024 * 1024; +const MAX_RETAINED_TASKS: usize = 4096; + +fn task_bytes(task: &Task) -> u64 { + (task.id.len() + task.branch.len() + task.base.len() + task.model.len() + task.prompt.len()) as u64 +} struct QueueTasks(Vec); impl<'de> Deserialize<'de> for QueueTasks { @@ -235,6 +242,8 @@ struct Inputs { files: HashMap, tasks: HashMap, errors: HashMap>, + branches: HashSet, + task_bytes: u64, } fn scan( directory: &Path, @@ -243,6 +252,7 @@ fn scan( emit: &mut dyn FnMut(QueueEvent) -> Result<()>, ) -> Result<()> { let mut files = vec![]; + let mut bytes = 0; for entry in fs::read_dir(directory).map_err(|e| e.to_string())? { let entry = match entry { Ok(entry) => entry, @@ -250,6 +260,16 @@ fn scan( Err(error) => return Err(error.to_string()), }; if let Some(file) = input_file(entry)? { + if files.len() == MAX_QUEUE_FILES { + return Err(format!( + "queue exceeds {MAX_QUEUE_FILES} task files; archive old inputs" + )); + } + // Oversized files are reported individually by read_input. + bytes += file.2.min(MAX_INPUT_BYTES); + if bytes > MAX_QUEUE_BYTES { + return Err("queue exceeds 16 MiB of task inputs; archive old inputs".into()); + } files.push(file); } } @@ -261,7 +281,12 @@ fn scan( .collect(); pending.retain(|job| paths.contains(&job.file)); // Active/external workers are not pending and retain their identities. - inputs.tasks.retain(|id, _| !withdrawn.contains(id)); + for id in withdrawn { + if let Some((_, task)) = inputs.tasks.remove(&id) { + inputs.branches.remove(&task.branch); + inputs.task_bytes -= task_bytes(&task); + } + } inputs.files.retain(|p, _| paths.contains(p)); inputs.errors.retain(|p, _| paths.contains(p)); files.sort_by(|a, b| (&a.1, &a.0).cmp(&(&b.1, &b.0))); @@ -286,17 +311,22 @@ fn scan( } problems.push(format!("task {} was already queued; use a new id for changed work or remove the duplicate input", task.id)); } else { - if inputs - .tasks - .values() - .any(|(_, known)| known.branch == task.branch) - { + if inputs.branches.contains(&task.branch) { problems.push(format!( "branch {} was already queued; use a distinct branch", task.branch )); continue; } + let bytes = task_bytes(&task); + if inputs.tasks.len() == MAX_RETAINED_TASKS + || inputs.task_bytes + bytes > MAX_QUEUE_BYTES + { + problems.push("queue task identity capacity reached; archive completed inputs and restart the controller".into()); + break; + } + inputs.branches.insert(task.branch.clone()); + inputs.task_bytes += bytes; inputs.tasks.insert(task.id.clone(), (file.clone(), task.clone())); pending.push(Job { task, @@ -718,9 +748,17 @@ mod tests { assert_eq!(pending.len(), 1); fs::rename(&path, dir.join("archived")).unwrap(); scan(&dir, &mut pending, &mut inputs, &mut |_| Ok(())).unwrap(); - fs::remove_dir_all(&dir).unwrap(); assert!(pending.is_empty(), "archived input must not be dispatched"); assert!(!inputs.tasks.contains_key("queued")); + fs::write( + &path, + r#"[{"id":"replacement","branch":"agent/queued","model":"fake","prompt":"Synthetic replacement"}]"#, + ) + .unwrap(); + scan(&dir, &mut pending, &mut inputs, &mut |_| Ok(())).unwrap(); + fs::remove_dir_all(&dir).unwrap(); + assert_eq!(pending.len(), 1); + assert_eq!(pending[0].task.id, "replacement"); } #[test] fn archiving_between_directory_read_and_metadata_is_not_a_controller_error() { From 373b8bdfb9f149fc798c46ec02e29b984f19d1de Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Wed, 18 Nov 2026 00:03:38 +0100 Subject: [PATCH 47/52] Bound publication commands and bind confirmed destinations --- oxa/docs/work.md | 3 +- oxa/oxa-cli/src/work_reports.rs | 1 + oxa/oxa-cli/tests/work_publish.rs | 4 +- oxa/oxa-work/src/publish.rs | 62 ++++++++++++++++++++++--------- 4 files changed, 50 insertions(+), 20 deletions(-) diff --git a/oxa/docs/work.md b/oxa/docs/work.md index 95ea8facb..7ee628202 100644 --- a/oxa/docs/work.md +++ b/oxa/docs/work.md @@ -200,7 +200,8 @@ The [checkpoint integration contract](work-state.md) defines the runner's public event hook. Event/situation/standing stores and their CLI adapters are a separate integration; this slice does not infer facts from worker prose or raw logs. `oxa work tasks.json --publish` pushes the accepted, saved commit to `origin` and -creates or updates its PR through `gh`. Red acceptance never pushes; green +creates or updates its PR through `gh`. Sign in with `gh auth login --hostname HOST` +for the host in `origin`. Publication uses that host's saved login. Red acceptance never pushes; green acceptance makes the PR ready. Blocked tasks do not create PRs. Drift-only accepted reruns may update an existing PR. The runner checks existing PR queue membership before every push and refuses a queued PR; use a new task branch for follow-up work. It refuses branch changes diff --git a/oxa/oxa-cli/src/work_reports.rs b/oxa/oxa-cli/src/work_reports.rs index 6029cb170..cb10fe81b 100644 --- a/oxa/oxa-cli/src/work_reports.rs +++ b/oxa/oxa-cli/src/work_reports.rs @@ -106,6 +106,7 @@ pub(crate) fn report( result.detail = "issue reporting is not configured; draft retained locally".into(); return Ok(result); }; + let repo = format!("github.com/{repo}"); let note = ProblemNote { title, notes: clean_notes(¬e.notes, ctx), diff --git a/oxa/oxa-cli/tests/work_publish.rs b/oxa/oxa-cli/tests/work_publish.rs index ce9e2df80..512d5d871 100644 --- a/oxa/oxa-cli/tests/work_publish.rs +++ b/oxa/oxa-cli/tests/work_publish.rs @@ -40,7 +40,7 @@ case "$1 $2" in if test "$arg" = '--draft'; then draft=true; fi if test "$arg" = '--body-file'; then body_next=true; fi done - printf '{"number":1,"url":"https://%s/example/project/pull/1","isDraft":%s,"headRepositoryOwner":{"login":"example"}}' "$host" "$draft" > "$HOME/pr-state.json" + printf '{"number":1,"url":"https://%s/example/project/pull/1","isDraft":%s,"headRepositoryOwner":{"login":"example"},"isCrossRepository":false}' "$host" "$draft" > "$HOME/pr-state.json" printf 'https://%s/example/project/pull/1' "$host" ;; 'pr view') jq -Rs '{body:.}' "$HOME/published-body" ;; 'pr edit') @@ -52,7 +52,7 @@ case "$1 $2" in 'pr ready') draft=false for arg in "$@"; do if test "$arg" = '--undo'; then draft=true; fi; done - printf '{"number":1,"url":"https://%s/example/project/pull/1","isDraft":%s,"headRepositoryOwner":{"login":"example"}}' "$host" "$draft" > "$HOME/pr-state.json" ;; + printf '{"number":1,"url":"https://%s/example/project/pull/1","isDraft":%s,"headRepositoryOwner":{"login":"example"},"isCrossRepository":false}' "$host" "$draft" > "$HOME/pr-state.json" ;; *) exit 7 ;; esac "##); diff --git a/oxa/oxa-work/src/publish.rs b/oxa/oxa-work/src/publish.rs index a490e58c7..5d230cc97 100644 --- a/oxa/oxa-work/src/publish.rs +++ b/oxa/oxa-work/src/publish.rs @@ -5,7 +5,7 @@ use std::os::unix::fs::OpenOptionsExt; use std::{ fs::File, path::{Path, PathBuf}, - process::Command, + process::{Command, Stdio}, }; fn context( @@ -39,21 +39,41 @@ fn context( writer.replay(&value, observer)?; Ok((writer, lock, value)) } +const MAX_PUBLICATION_RESPONSE_BYTES: usize = 1024 * 1024; + +fn isolate_github_environment(command: &mut Command) { + // The destination comes from origin. Let the CLI choose its stored + // login for that host, independently of host-agnostic shell overrides. + for key in [ + "GH_HOST", + "GH_REPO", + "GH_TOKEN", + "GITHUB_TOKEN", + "GH_ENTERPRISE_TOKEN", + "GITHUB_ENTERPRISE_TOKEN", + ] { + command.env_remove(key); + } +} + fn gh(binary: &Path, policy: &RunPolicy, repo: &Path, lock: &File, args: &[&str]) -> Result { let mut command = Command::new(binary); super::isolate_git_environment(&mut command); + isolate_github_environment(&mut command); command .env("PATH", &policy.host_path) - .env_remove("GH_REPO") - .env_remove("GH_HOST") + .env("GH_PROMPT_DISABLED", "1") .current_dir(repo) .args(args); - command.stdin(lock.try_clone().map_err(|e| e.to_string())?); - let output = command.output().map_err(|e| e.to_string())?; - if !output.status.success() { - return Err("GitHub publication request failed".into()); + match super::command_runner_with_stdin( + command, + Some(MAX_PUBLICATION_RESPONSE_BYTES), + Stdio::from(lock.try_clone().map_err(|e| e.to_string())?), + super::COMMAND_DEADLINE, + ) { + Some((0, text)) => Ok(text.trim().into()), + _ => Err("GitHub publication request failed".into()), } - Ok(String::from_utf8_lossy(&output.stdout).trim().into()) } fn json(text: &str) -> Result { serde_json::from_str(text).map_err(|e| e.to_string()) @@ -194,17 +214,17 @@ pub fn publish( "--head", &task.branch, "--json", - "number,url,isDraft,headRepositoryOwner", + "number,url,isDraft,headRepositoryOwner,isCrossRepository", ], )?)?; let prs = prs.as_array().ok_or("invalid PR list")?; if prs.len() > 1 { return Err("multiple open PRs match the task branch".into()); } - if prs - .first() - .is_some_and(|pr| pr["headRepositoryOwner"]["login"].as_str() != Some(owner)) - { + if prs.first().is_some_and(|pr| { + pr["headRepositoryOwner"]["login"].as_str() != Some(owner) + || pr["isCrossRepository"].as_bool() != Some(false) + }) { return Err("PR head repository does not match origin".into()); } if accepted == Outcome::NothingToDo @@ -284,13 +304,21 @@ pub fn publish( &format!("{head}:refs/heads/{}", task.branch), ], ); + isolate_github_environment(&mut push); push.env("GIT_TERMINAL_PROMPT", "0") + .env("GH_PROMPT_DISABLED", "1") // Metadata probes deny every transport. Only this explicit, // confirmed publication may use the normal GitHub/file transports. - .env("GIT_ALLOW_PROTOCOL", "https:ssh:file") - .stdin(lock.try_clone().map_err(|e| e.to_string())?); - let pushed = push.output().map_err(|e| e.to_string())?; - if !pushed.status.success() { + .env("GIT_ALLOW_PROTOCOL", "https:ssh:file"); + if !matches!( + super::command_runner_with_stdin( + push, + Some(MAX_PUBLICATION_RESPONSE_BYTES), + Stdio::from(lock.try_clone().map_err(|e| e.to_string())?), + super::COMMAND_DEADLINE, + ), + Some((0, _)) + ) { return Err("accepted commit push failed".into()); } let draft = false; From b67cc48126ad96a3930de42c403087739743a028 Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Wed, 18 Nov 2026 00:05:17 +0100 Subject: [PATCH 48/52] Use the queue report gate in the host regression --- oxa/oxa-cli/src/work_reports.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oxa/oxa-cli/src/work_reports.rs b/oxa/oxa-cli/src/work_reports.rs index d58d7ff0f..116a4605c 100644 --- a/oxa/oxa-cli/src/work_reports.rs +++ b/oxa/oxa-cli/src/work_reports.rs @@ -370,7 +370,7 @@ mod tests { prompts.push(prompt.to_owned()); Some(true) }, - &mut |program, args| { + |program, args: &[String], _: &File| { if program != "gh" { return Some((0, "{}".into())); } From 7275f1023c7422e9d85c979eab7c2734fd27c2ee Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Wed, 18 Nov 2026 00:12:53 +0100 Subject: [PATCH 49/52] Run an admitted native board packet through the held task lock --- oxa/docs/work.md | 8 ++ oxa/oxa-cli/src/lead_input.rs | 2 +- oxa/oxa-cli/src/work_cmd.rs | 44 +++++-- oxa/oxa-cli/src/work_cmd/native_queue.rs | 146 +++++++++++++++++++++++ oxa/oxa-work/src/helper_queue.rs | 71 ++++++++++- oxa/oxa-work/src/lib.rs | 27 +++-- oxa/oxa-work/src/queue.rs | 50 +++++++- oxa/oxa-work/src/queue/native.rs | 32 +++++ 8 files changed, 356 insertions(+), 24 deletions(-) create mode 100644 oxa/oxa-cli/src/work_cmd/native_queue.rs diff --git a/oxa/docs/work.md b/oxa/docs/work.md index c74e2794a..30a389046 100644 --- a/oxa/docs/work.md +++ b/oxa/docs/work.md @@ -266,5 +266,13 @@ The latest state line is also replaced atomically at active task IDs, pending/finished/failed counts, free bytes and disk-pause details. A controller lock prevents two queue controllers from exceeding the worker cap. +The native board adapter runs one previously admitted packet from that board's +existing spool. Use `oxa --offline work --native-queue --config PATH` from the +source repository. The private board config must explicitly enable +`nativeExecution` and identify the canonical source path in `nativeRoot`. The +adapter moves the matching packet into `started` and resumes from the existing +task checkpoint after interruption. It releases the original claim only after +the whole task finishes and its worker process exits. + Problem-report workflows hold a per-user kernel lease across drafting, duplicate search, match reads and writing, so concurrent workers share one reporting lane. diff --git a/oxa/oxa-cli/src/lead_input.rs b/oxa/oxa-cli/src/lead_input.rs index e81f4e1a6..1a44ecd59 100644 --- a/oxa/oxa-cli/src/lead_input.rs +++ b/oxa/oxa-cli/src/lead_input.rs @@ -16,7 +16,7 @@ type Result = std::result::Result; /// The caller supplies the owner's uid from its user configuration, never project metadata. /// Sources must have private parent directories; producers may atomically replace files. -fn open(path: &Path, uid: u32) -> Result { +pub(crate) fn open(path: &Path, uid: u32) -> Result { let parent = path.parent().ok_or("missing input directory")?; let directory = fs::symlink_metadata(parent).map_err(|_| "input directory unavailable")?; if !directory.is_dir() || directory.uid() != uid || directory.mode() & 0o077 != 0 { diff --git a/oxa/oxa-cli/src/work_cmd.rs b/oxa/oxa-cli/src/work_cmd.rs index 8afc0187e..0a6636cd1 100644 --- a/oxa/oxa-cli/src/work_cmd.rs +++ b/oxa/oxa-cli/src/work_cmd.rs @@ -3,6 +3,9 @@ use sha2::{Digest, Sha256}; use std::io::Write; use std::path::{Path, PathBuf}; +#[path = "work_cmd/native_queue.rs"] +mod native_queue; + pub(crate) fn trusted(cwd: &Path, env: &Env, dirs: &Dirs) -> Result { crate::dispatch::project_trusted( &crate::Args::default(), @@ -236,6 +239,18 @@ pub(crate) fn run( publish, reports, }); + if task_args.first().is_some_and(|word| word == "--native-queue") { + if !is_trusted { + return Err("source project is not trusted; approve it before unattended work".into()); + } + let [_, flag, config] = task_args else { + return Err("usage: oxa work --native-queue --config PATH".into()); + }; + if flag != "--config" || rerun || publish { + return Err("invalid native queue options".into()); + } + return native_queue::run(context, &common, &cwd.join(config), out); + } if task_args.first().is_some_and(|word| word == "--queue") { if !is_trusted { return Err("source project is not trusted; approve it before unattended work".into()); @@ -362,21 +377,24 @@ struct Context { } impl Context { fn planned(&self, plan: &oxa_work::PlannedTask) -> Result { + let mut locked = match oxa_work::LockedTask::acquire(plan.task(), &self.policy) { + Ok(locked) => locked, + Err(error) => return Ok(blocked_plan(plan, error)), + }; + self.planned_locked(plan, &mut locked) + } + + fn planned_locked( + &self, + plan: &oxa_work::PlannedTask, + locked: &mut oxa_work::LockedTask<'_>, + ) -> Result { let root = &self.cwd; let binary = &self.binary; let policy = &self.policy; let dirs = &self.dirs; let env = &self.env; let (rerun, publish, reports) = (self.rerun, self.publish, self.reports); - let blocked = |detail| { - let mut record = oxa_work::blocked_record(plan.task(), detail); - record.base_sha = Some(plan.base_sha().to_owned()); - record - }; - let mut locked = match oxa_work::LockedTask::acquire(plan.task(), policy) { - Ok(locked) => locked, - Err(error) => return Ok(blocked(error)), - }; let cached = if (publish || reports) && !rerun { oxa_work::action_record(policy, plan.task())? } else { @@ -395,7 +413,7 @@ impl Context { &|note| crate::work_reports::scrub_note(note, &ctx), ) { Ok(record) => record, - Err(oxa_work::RunError::Blocked(detail)) => blocked(detail), + Err(oxa_work::RunError::Blocked(detail)) => blocked_plan(plan, detail), Err(oxa_work::RunError::Checkpoint(error)) => return Err(error), } }; @@ -442,6 +460,12 @@ impl Context { Ok(record) } } +fn blocked_plan(plan: &oxa_work::PlannedTask, detail: String) -> oxa_work::Record { + let mut record = oxa_work::blocked_record(plan.task(), detail); + record.base_sha = Some(plan.base_sha().to_owned()); + record +} + fn queue_options(args: &[String], cwd: &Path) -> Result<(PathBuf, oxa_work::QueueOptions), String> { let directory = cwd.join(args.get(1).ok_or("--queue needs a directory")?); let mut options = oxa_work::QueueOptions::default(); diff --git a/oxa/oxa-cli/src/work_cmd/native_queue.rs b/oxa/oxa-cli/src/work_cmd/native_queue.rs new file mode 100644 index 000000000..7bd921911 --- /dev/null +++ b/oxa/oxa-cli/src/work_cmd/native_queue.rs @@ -0,0 +1,146 @@ +//! Explicit one-shot execution of an existing, fully bound native board packet. +use super::{protected_storage, Context}; +use oxa_work::{ + board::{assignment::Worker, Claims}, + LockedTask, NativeQueueOwner, QueueOptions, QueueRun, Task, +}; +use serde_json::Value; +use std::{ + fs, + io::{Read, Write}, + os::unix::fs::MetadataExt, + path::{Path, PathBuf}, + sync::Arc, +}; + +const MAX_CONFIG_BYTES: u64 = 64 * 1024; + +fn paused(path: &Path) -> Result { + match fs::symlink_metadata(path) { + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(_) => Err("cannot establish board pause state".into()), + } +} + +pub(super) fn run( + context: Arc, + common: &Path, + config: &Path, + out: &mut dyn Write, +) -> Result { + if context.env.pairs().iter().any(|(name, _)| { + matches!( + name.as_str(), + "OXA_CODING_AGENT" | "AI_AGENT" | "OXA_SUBAGENT_DEPTH" + ) + }) { + return Err("native queue execution requires the owner outside an agent tool".into()); + } + let root = &context.cwd; + let config = config.canonicalize().map_err(|_| "board config unavailable")?; + let owner = context + .dirs + .agent_dir + .canonicalize() + .map_err(|_| "owner controls unavailable")?; + if !config.starts_with(&owner) { + return Err("board config must be in protected owner controls".into()); + } + protected_storage(&config, root, common)?; + let uid = fs::metadata(&context.dirs.home) + .map_err(|_| "owner home unavailable")? + .uid(); + let mut bytes = Vec::new(); + crate::lead_input::open(&config, uid)? + .take(MAX_CONFIG_BYTES + 1) + .read_to_end(&mut bytes) + .map_err(|_| "cannot read native board configuration")?; + if bytes.len() as u64 > MAX_CONFIG_BYTES { + return Err("native board configuration exceeds capacity".into()); + } + let value: Value = serde_json::from_slice(&bytes).map_err(|_| "invalid native board configuration")?; + if value["enabled"] != true || value["nativeExecution"] != true { + return Ok(0); + } + let owner_path = |key: &str| -> Result { + let path = PathBuf::from(value[key].as_str().ok_or("missing native board path")?); + protected_storage(&path, root, common)?; + Ok(path) + }; + // The legacy check supervisor does not yet inherit the model worker's + // runner-owned GitHub policy. Offline mode also masks credentials and + // fences deterministic checkout, acceptance and save subprocesses. + if !context.policy.offline { + return Err("native queue execution currently requires --offline".into()); + } + let source = value["nativeRoot"].as_str().ok_or("missing native source root")?; + if Path::new(source) != root { + return Err("native board configuration belongs to a different source root".into()); + } + let pause = owner_path("pause")?; + if paused(&pause)? { + return Ok(0); + } + let directory = owner_path("queue")?; + let claims = Claims::open(&owner_path("claims")?)?; + let worker: Worker = + serde_json::from_value(value["worker"].clone()).map_err(|_| "invalid authenticated worker export")?; + if !oxa_work::board::assignment::valid_worker_id(&worker.id) { + return Err("invalid native worker identity".into()); + } + let rows = claims.assignments()?; + let mut owned = rows.into_values().filter(|row| { + row.state + .lease + .as_ref() + .is_some_and(|lease| lease.holder == worker.id) + }); + let Some(row) = owned.next() else { return Ok(0) }; + if owned.next().is_some() { + return Err("native worker has multiple active assignments".into()); + } + let binding = row + .data + .native_binding + .filter(|binding| row.state.lease.as_ref() == Some(&binding.lease)) + .ok_or("native worker lacks an exact retained binding")?; + let owner = NativeQueueOwner { + claims: &claims, + worker_id: &worker.id, + }; + let policy = &context.policy; + let df = crate::netguard::trusted_executable_for( + "df", + Some(&policy.host_path), + Some(root), + Some(&context.dirs.home), + ) + .ok_or("no trusted disk-space executable outside the source project")?; + let worker_context = Arc::clone(&context); + let run = Arc::new(move |task: &Task, lock: &mut LockedTask<'_>| { + if paused(&pause)? { + return Err("native board execution paused".into()); + } + match lock.plan_native(&worker_context.cwd, worker_context.reports) { + Ok(plan) => worker_context.planned_locked(&plan, lock), + Err(error) => Ok(oxa_work::blocked_record(task, error)), + } + }); + let options = QueueOptions { + once: true, + ..QueueOptions::default() + }; + let summary = QueueRun { + repo: root, + directory: &directory, + options: &options, + policy, + df: &df, + } + .run_native_packet(&owner, &binding, run, &mut |event| { + serde_json::to_writer(&mut *out, &event).map_err(|e| e.to_string())?; + writeln!(out).and_then(|_| out.flush()).map_err(|e| e.to_string()) + })?; + Ok(i32::from(summary.failed > 0 || summary.pending > 0)) +} diff --git a/oxa/oxa-work/src/helper_queue.rs b/oxa/oxa-work/src/helper_queue.rs index 970f4a200..3f8047249 100644 --- a/oxa/oxa-work/src/helper_queue.rs +++ b/oxa/oxa-work/src/helper_queue.rs @@ -1,11 +1,11 @@ //! Atomic publication only. The caller must reserve a core claim and authorize the target first. use crate::{board::Policy, Result}; use std::fs::{self, File, OpenOptions}; -use std::io::Write; +use std::io::{Read, Write}; use std::os::unix::fs::{DirBuilderExt, MetadataExt, OpenOptionsExt}; use std::path::{Path, PathBuf}; -const MAX_TASK_BYTES: usize = 64 * 1024; +pub(crate) const MAX_TASK_BYTES: usize = 64 * 1024; fn io(value: std::io::Result) -> Result { value.map_err(|e| format!("board queue storage: {:?}", e.kind())) @@ -83,7 +83,11 @@ pub fn publish_reserved( for prefix in ["board-", "p1-board-"] { let name = format!("{prefix}{key}.txt"); // Check pending before started: the consumer's one-way rename cannot hide both. - for path in [queue.join(&name), started.join(&name)] { + for path in [ + queue.join(&name), + started.join(&name), + started.join(format!("native-{name}")), + ] { match fs::symlink_metadata(path) { Ok(metadata) => { if !metadata.is_file() || metadata.mode() & 0o077 != 0 { @@ -121,3 +125,64 @@ pub fn publish_reserved( io(lock.sync_all())?; Ok(true) } + +/// Take the admitted packet into the existing started directory. The native name +/// keeps retries distinguishable from packets already started by another consumer. +pub fn take_native_packet(queue: &Path, binding: &crate::board::NativeBinding) -> Result { + if crate::board::NativeBinding::new(binding.number, &binding.key, &binding.lease, &binding.branch)? + != *binding + { + return Err("invalid native queue binding".into()); + } + directory(queue)?; + let started = queue.join("started"); + directory(&started)?; + let lock = io(File::open(queue))?; + io(lock.lock())?; + let mut found = None; + for prefix in ["board-", "p1-board-"] { + let name = format!("{prefix}{}.txt", binding.key); + let ready = queue.join(&name); + let active = started.join(format!("native-{name}")); + // A started packet belonging to another consumer is never adopted. + match fs::symlink_metadata(started.join(&name)) { + Ok(_) => return Err("board packet is already started by another consumer".into()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => (), + Err(error) => return io(Err(error)), + } + for (path, renamed) in [(&ready, false), (&active, true)] { + let file = match OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) + .open(path) + { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => return io(Err(error)), + }; + let metadata = io(file.metadata())?; + if !metadata.is_file() + || metadata.uid() != io(lock.metadata())?.uid() + || metadata.nlink() != 1 + || metadata.mode() & 0o077 != 0 + || metadata.len() > MAX_TASK_BYTES as u64 + || found.is_some() + { + return Err("unsafe or ambiguous native queue packet".into()); + } + found = Some((file, ready.clone(), active.clone(), renamed)); + } + } + let (file, ready, active, renamed) = found.ok_or("native queue packet unavailable")?; + let mut packet = String::new(); + io(file.take(MAX_TASK_BYTES as u64 + 1).read_to_string(&mut packet))?; + if packet.len() > MAX_TASK_BYTES { + return Err("native queue packet exceeds capacity".into()); + } + if !renamed { + io(fs::rename(ready, active))?; + io(io(File::open(started))?.sync_all())?; + io(lock.sync_all())?; + } + Ok(packet) +} diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index 88691bfd8..59d2816c7 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -143,6 +143,10 @@ impl PlannedTask { } pub fn plan_task(repo: &Path, task: Task, policy: &RunPolicy) -> Result { + plan_new_task(repo, task, policy, false) +} + +fn plan_new_task(repo: &Path, task: Task, policy: &RunPolicy, native: bool) -> Result { validate(&task)?; let branch = checked(git(policy, repo, &["check-ref-format", "--branch", &task.branch]))?; if branch != task.branch { @@ -159,9 +163,9 @@ pub fn plan_task(repo: &Path, task: Task, policy: &RunPolicy) -> Result Result Result { - plan_task_mode(repo, task, policy, rerun, false) + plan_task_mode(repo, task, policy, rerun, false, false) } /// Explicit actions reuse completed execution inputs, including red results, @@ -186,7 +190,7 @@ pub fn plan_task_for_actions( policy: &RunPolicy, rerun: bool, ) -> Result { - plan_task_mode(repo, task, policy, rerun, true) + plan_task_mode(repo, task, policy, rerun, true, false) } fn completed_execution(step: Step) -> bool { @@ -199,10 +203,11 @@ fn plan_task_mode( policy: &RunPolicy, rerun: bool, actions: bool, + native: bool, ) -> Result { validate(&task)?; let Some(saved) = load_task_checkpoint(policy, &task.id)? else { - return plan_task(repo, task, policy); + return plan_new_task(repo, task, policy, native); }; if saved.task != task { return Err("task specification changed; use a new task id".into()); @@ -222,7 +227,7 @@ fn plan_task_mode( checkpoint_revision: Some(saved.record.checkpoint_revision), }); } - plan_task(repo, task, policy) + plan_new_task(repo, task, policy, native) } fn load_task_checkpoint(policy: &RunPolicy, id: &str) -> Result> { @@ -918,6 +923,14 @@ impl<'a> LockedTask<'a> { Ok(Self { task, policy, lock }) } + /// Plan the task admitted by the native queue while retaining its lock. + pub fn plan_native(&self, repo: &Path, actions: bool) -> Result { + if !self.task.id.starts_with("board-") || !self.task.runner_owns_github() { + return Err("native planning requires an admitted board task".into()); + } + plan_task_mode(repo, self.task.clone(), self.policy, false, actions, true) + } + pub fn run_with_notes( &mut self, repo: &Path, diff --git a/oxa/oxa-work/src/queue.rs b/oxa/oxa-work/src/queue.rs index 6d2e6fc99..28a64cd9e 100644 --- a/oxa/oxa-work/src/queue.rs +++ b/oxa/oxa-work/src/queue.rs @@ -405,6 +405,7 @@ pub fn run_queue( df, }, None, + None, QueueWorker::Manual(worker), emit, ) @@ -425,6 +426,35 @@ impl QueueRun<'_> { owner: &NativeQueueOwner<'_>, worker: Arc, emit: &mut dyn FnMut(QueueEvent) -> Result<()>, + ) -> Result { + self.run_native_with(owner, None, worker, emit) + } + + /// Consume one protected spool packet in memory using the existing queue and worker lifecycle. + pub fn run_native_packet( + self, + owner: &NativeQueueOwner<'_>, + binding: &board::NativeBinding, + worker: Arc, + emit: &mut dyn FnMut(QueueEvent) -> Result<()>, + ) -> Result { + if !self.options.once { + return Err("native packet execution requires a one-shot queue".into()); + } + if binding.lease.holder != owner.worker_id { + return Err("native packet belongs to a different worker".into()); + } + let packet = crate::helper_queue::take_native_packet(self.directory, binding)?; + let task = native::from_packet(binding, &packet)?; + self.run_native_with(owner, Some(task), worker, emit) + } + + fn run_native_with( + self, + owner: &NativeQueueOwner<'_>, + prepared: Option, + worker: Arc, + emit: &mut dyn FnMut(QueueEvent) -> Result<()>, ) -> Result { let worker = QueueWorker::Native(Arc::new(NativeWorker { claims: owner.claims.clone(), @@ -432,13 +462,14 @@ impl QueueRun<'_> { policy: self.policy.clone(), run: worker, })); - run_queue_inner(self, Some(owner), worker, emit) + run_queue_inner(self, Some(owner), prepared, worker, emit) } } fn run_queue_inner( queue: QueueRun<'_>, native: Option<&NativeQueueOwner<'_>>, + prepared: Option, worker: QueueWorker, emit: &mut dyn FnMut(QueueEvent) -> Result<()>, ) -> Result { @@ -470,10 +501,23 @@ fn run_queue_inner( .map_err(|e| e.to_string())?; let (tx, rx) = mpsc::channel(); let mut active = Workers::default(); - let (mut pending, mut external, mut inputs) = (vec![], vec![], Inputs::default()); + let read_directory = prepared.is_none(); + let mut pending: Vec<_> = prepared + .into_iter() + .map(|task| Job { + task, + modified: SystemTime::UNIX_EPOCH, + file: directory.clone(), + index: 0, + binding: None, + }) + .collect(); + let (mut external, mut inputs) = (vec![], Inputs::default()); let (mut finished, mut failed, mut last) = (0, 0, None); loop { - scan(&directory, &mut pending, &mut inputs, emit)?; + if read_directory { + scan(&directory, &mut pending, &mut inputs, emit)?; + } // Surviving workers from a dead controller still count toward the concurrency cap. let mut waiting = vec![]; for job in external.drain(..) { diff --git a/oxa/oxa-work/src/queue/native.rs b/oxa/oxa-work/src/queue/native.rs index bdf7f9eee..d1143d514 100644 --- a/oxa/oxa-work/src/queue/native.rs +++ b/oxa/oxa-work/src/queue/native.rs @@ -22,6 +22,38 @@ pub fn native_task(binding: &board::NativeBinding, mut task: Task) -> Result Result { + let (header, body) = packet.split_once('\n').ok_or("native task packet has no body")?; + let mut fields = header.split_ascii_whitespace(); + let id = fields.next().ok_or("native task packet has no source id")?; + let model = fields.next().ok_or("native task packet has no model")?; + let branch = fields.next().ok_or("native task packet has no branch")?; + let base = fields.next().unwrap_or("main"); + let issue = id == format!("i{}", binding.number); + let change = id == format!("c{}", binding.number); + if fields.next().is_some() + || branch != binding.branch + || !(issue && base == "main" || change && base == branch) + || packet.len() > crate::helper_queue::MAX_TASK_BYTES + || packet.contains('\0') + || body.trim().is_empty() + || body.lines().any(|line| line.trim() == "---") + { + return Err("native task packet disagrees with its bound source".into()); + } + native_task( + binding, + Task { + id: id.into(), + model: model.into(), + branch: branch.into(), + base: base.into(), + prompt: body.into(), + max_time: crate::default_max_time(), + }, + ) +} + impl NativeQueueOwner<'_> { pub(super) fn admit(&self, task: &Task) -> Result { let binding = self From 80cea9af8b2de5b17921c8a13515066b78d4a4fc Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Wed, 18 Nov 2026 00:16:58 +0100 Subject: [PATCH 50/52] Preserve the native queue command and apply isolation internally --- oxa/docs/work.md | 12 ++++++------ oxa/oxa-cli/src/work_cmd.rs | 4 ++-- oxa/oxa-cli/src/work_cmd/native_queue.rs | 17 ++++++----------- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/oxa/docs/work.md b/oxa/docs/work.md index 30a389046..14e22ebde 100644 --- a/oxa/docs/work.md +++ b/oxa/docs/work.md @@ -267,12 +267,12 @@ active task IDs, pending/finished/failed counts, free bytes and disk-pause detai A controller lock prevents two queue controllers from exceeding the worker cap. The native board adapter runs one previously admitted packet from that board's -existing spool. Use `oxa --offline work --native-queue --config PATH` from the -source repository. The private board config must explicitly enable -`nativeExecution` and identify the canonical source path in `nativeRoot`. The -adapter moves the matching packet into `started` and resumes from the existing -task checkpoint after interruption. It releases the original claim only after -the whole task finishes and its worker process exits. +existing spool. Use `oxa work --native-queue --config PATH` from the source +repository with the same private config used by refill and `nativeExecution` +explicitly enabled. This execution path uses the offline policy. It moves the +matching packet into `started` and resumes from the task checkpoint after +interruption. The original claim releases after the whole task finishes and +its worker process exits. Problem-report workflows hold a per-user kernel lease across drafting, duplicate search, match reads and writing, so concurrent workers share one reporting lane. diff --git a/oxa/oxa-cli/src/work_cmd.rs b/oxa/oxa-cli/src/work_cmd.rs index 0a6636cd1..f197f74b6 100644 --- a/oxa/oxa-cli/src/work_cmd.rs +++ b/oxa/oxa-cli/src/work_cmd.rs @@ -229,7 +229,7 @@ pub(crate) fn run( } let task_args = &args[..end]; let is_trusted = trusted(&root, env, dirs)?; - let context = std::sync::Arc::new(Context { + let context = Context { cwd: root.clone(), binary: binary.clone(), policy: policy.clone(), @@ -238,7 +238,7 @@ pub(crate) fn run( rerun, publish, reports, - }); + }; if task_args.first().is_some_and(|word| word == "--native-queue") { if !is_trusted { return Err("source project is not trusted; approve it before unattended work".into()); diff --git a/oxa/oxa-cli/src/work_cmd/native_queue.rs b/oxa/oxa-cli/src/work_cmd/native_queue.rs index 7bd921911..7edd8abe5 100644 --- a/oxa/oxa-cli/src/work_cmd/native_queue.rs +++ b/oxa/oxa-cli/src/work_cmd/native_queue.rs @@ -24,7 +24,7 @@ fn paused(path: &Path) -> Result { } pub(super) fn run( - context: Arc, + mut context: Context, common: &Path, config: &Path, out: &mut dyn Write, @@ -37,6 +37,11 @@ pub(super) fn run( }) { return Err("native queue execution requires the owner outside an agent tool".into()); } + // The existing supervisor receives the runner's offline switch. Keep + // credential mounts and network off for deterministic steps until the + // supervisor also inherits the worker's runner-owned GitHub policy. + context.policy.offline = true; + let context = Arc::new(context); let root = &context.cwd; let config = config.canonicalize().map_err(|_| "board config unavailable")?; let owner = context @@ -68,16 +73,6 @@ pub(super) fn run( protected_storage(&path, root, common)?; Ok(path) }; - // The legacy check supervisor does not yet inherit the model worker's - // runner-owned GitHub policy. Offline mode also masks credentials and - // fences deterministic checkout, acceptance and save subprocesses. - if !context.policy.offline { - return Err("native queue execution currently requires --offline".into()); - } - let source = value["nativeRoot"].as_str().ok_or("missing native source root")?; - if Path::new(source) != root { - return Err("native board configuration belongs to a different source root".into()); - } let pause = owner_path("pause")?; if paused(&pause)? { return Ok(0); From 0872759532440a3a5e9804f61c8ab3703e0a4b6f Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Wed, 18 Nov 2026 00:21:33 +0100 Subject: [PATCH 51/52] Fence started spool packets to the original lease generation --- oxa/oxa-cli/tests/helper_queue.rs | 28 ++++++++++++++++++++ oxa/oxa-work/src/helper_queue.rs | 44 ++++++++++++++++++++++++++----- 2 files changed, 65 insertions(+), 7 deletions(-) diff --git a/oxa/oxa-cli/tests/helper_queue.rs b/oxa/oxa-cli/tests/helper_queue.rs index bdca93ef9..bb88a9a23 100644 --- a/oxa/oxa-cli/tests/helper_queue.rs +++ b/oxa/oxa-cli/tests/helper_queue.rs @@ -53,6 +53,34 @@ fn complete_private_publication_dedups_started_and_priority_changes() { ); } +#[test] +fn native_started_packet_resumes_only_its_original_lease_generation() { + use oxa_work::{ + board::{Lease, NativeBinding}, + helper_queue::take_native_packet, + }; + + let env = TestEnv::new(); + let queue = env.tmp().join("queue"); + fs::create_dir(&queue).unwrap(); + let key = "a".repeat(64); + let mut lease = Lease { + holder: "oxa:fixture".into(), + claimed_at: Some(10), + expires_at: 20, + generation: (0, 1), + }; + let original = NativeBinding::new(42, &key, &lease, "agent/42-fix").unwrap(); + assert!(publish(&queue, &key, TASK, true, &enabled()).unwrap()); + assert_eq!(take_native_packet(&queue, &original).unwrap(), TASK); + + lease.generation = (0, 2); + let newer = NativeBinding::new(42, &key, &lease, "agent/42-fix").unwrap(); + assert!(take_native_packet(&queue, &newer).is_err()); + assert!(!publish(&queue, &key, TASK, false, &enabled()).unwrap()); + assert_eq!(take_native_packet(&queue, &original).unwrap(), TASK); +} + #[test] fn concurrent_publishers_create_exactly_one_complete_task() { if let Some(queue) = std::env::var_os("OXA_TEST_BOARD_QUEUE") { diff --git a/oxa/oxa-work/src/helper_queue.rs b/oxa/oxa-work/src/helper_queue.rs index 3f8047249..43cf6b7a3 100644 --- a/oxa/oxa-work/src/helper_queue.rs +++ b/oxa/oxa-work/src/helper_queue.rs @@ -83,11 +83,7 @@ pub fn publish_reserved( for prefix in ["board-", "p1-board-"] { let name = format!("{prefix}{key}.txt"); // Check pending before started: the consumer's one-way rename cannot hide both. - for path in [ - queue.join(&name), - started.join(&name), - started.join(format!("native-{name}")), - ] { + for path in [queue.join(&name), started.join(&name)] { match fs::symlink_metadata(path) { Ok(metadata) => { if !metadata.is_file() || metadata.mode() & 0o077 != 0 { @@ -99,6 +95,15 @@ pub fn publish_reserved( Err(e) => return io(Err(e)), } } + let native = started.join(format!("native-{prefix}{key}")); + match fs::symlink_metadata(&native) { + Ok(_) => { + directory(&native)?; + return Ok(false); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => (), + Err(error) => return io(Err(error)), + } } if !reserve()? { return Ok(false); @@ -143,7 +148,24 @@ pub fn take_native_packet(queue: &Path, binding: &crate::board::NativeBinding) - for prefix in ["board-", "p1-board-"] { let name = format!("{prefix}{}.txt", binding.key); let ready = queue.join(&name); - let active = started.join(format!("native-{name}")); + let active_dir = started.join(format!("native-{prefix}{}", binding.key)); + let active_name = format!("{}.txt", binding.native_task_id); + let active = active_dir.join(&active_name); + match fs::symlink_metadata(&active_dir) { + Ok(_) => { + directory(&active_dir)?; + let mut entries = io(fs::read_dir(&active_dir))?; + if let Some(entry) = io(entries.next().transpose())? { + if entry.file_name() != std::ffi::OsStr::new(&active_name) + || io(entries.next().transpose())?.is_some() + { + return Err("native started packet belongs to another lease generation".into()); + } + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => (), + Err(error) => return io(Err(error)), + } // A started packet belonging to another consumer is never adopted. match fs::symlink_metadata(started.join(&name)) { Ok(_) => return Err("board packet is already started by another consumer".into()), @@ -180,7 +202,15 @@ pub fn take_native_packet(queue: &Path, binding: &crate::board::NativeBinding) - return Err("native queue packet exceeds capacity".into()); } if !renamed { - io(fs::rename(ready, active))?; + let active_dir = active.parent().ok_or("missing native started directory")?; + match fs::DirBuilder::new().mode(0o700).create(active_dir) { + Ok(()) => (), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => (), + Err(error) => return io(Err(error)), + } + directory(active_dir)?; + io(fs::rename(ready, &active))?; + io(io(File::open(active_dir))?.sync_all())?; io(io(File::open(started))?.sync_all())?; io(lock.sync_all())?; } From 49ed0fce026a930f1d6838990981f89e87fedb7a Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Wed, 18 Nov 2026 00:33:17 +0100 Subject: [PATCH 52/52] Use #1090 guidance to materialize native board tasks --- oxa/oxa-work/src/queue/native.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/oxa/oxa-work/src/queue/native.rs b/oxa/oxa-work/src/queue/native.rs index d1143d514..84797cbe5 100644 --- a/oxa/oxa-work/src/queue/native.rs +++ b/oxa/oxa-work/src/queue/native.rs @@ -15,8 +15,13 @@ pub fn native_task(binding: &board::NativeBinding, mut task: Task) -> Result