diff --git a/oxa/docs/work.md b/oxa/docs/work.md index e3b57c7a..05299710 100644 --- a/oxa/docs/work.md +++ b/oxa/docs/work.md @@ -255,3 +255,50 @@ A later explicit `--report-problems` retries local reports without replaying the agent. A draft-write failure records a failed report and finishes the task checkpoint; `--report-problems` can retry it. Reports contain only scrubbed structured symptom summaries, without raw assistant output or acceptance logs. + + +Queue mode watches a directory of regular, non-hidden `*.json` task files. +Each file may contain up to 128 tasks and occupy up to 1 MiB. A queue accepts +up to 256 task files and 16 MiB of input per scan. The controller retains at +most 4,096 task identities and 16 MiB of task text across scans. Archive +completed inputs and restart the controller if it reaches the retention limit. +After adding task files, run +`oxa work --queue ./tasks --max 2 --min-free-mib 1024`. + +Files are admitted oldest modification time first (filename breaks ties), with +array order preserved inside a file. At most `--max` tasks run; surviving workers +from an interrupted controller still occupy slots until their locks release. +Task IDs must be unique across the queue. Completed task state prevents agent +replay on restart. Files remain available for inspection and manual archiving. +Write new files with an atomic rename so the runner reads a complete task file. + +The default free-space floor is 512 MiB. The runner checks the repository, common +git directory, worktree storage and queue filesystems before each admission. A low +or unreadable free-space value pauses new work while active jobs finish. Invalid +JSON files report an error and do not skip other work. `--once` drains available +work and exits; when disk space blocks pending tasks it exits 1 with their files +retained. Without `--once`, the directory stays under watch for new files. +Appending tasks to an existing file queues only new IDs. Discovered task definitions +are immutable; changing one requires a new ID. Correcting or archiving an invalid +input clears its current queue errors. Archiving inputs does not interrupt admitted +work. A controller or monitor error stops admission and drains active workers +before returning an error. + +Queue mode supports `--publish`, `--report-problems` and `--rerun` with the same +explicit policy as single-file mode. Its worker threads preserve the parent's +offline policy. Standard output emits JSON lines tagged `state`, `task` or `error`. +The latest state line is also replaced atomically at +`/.oxa-queue-status.json` for a monitor. It contains a heartbeat timestamp, +active task IDs, pending/finished/failed counts, free bytes and disk-pause details. +A controller lock prevents two queue controllers from exceeding the worker cap. + +The native board adapter runs one previously admitted packet from that board's +existing spool. Use `oxa work --native-queue --config PATH` from the source +repository with the same private config used by refill and `nativeExecution` +explicitly enabled. This execution path uses the offline policy. It moves the +matching packet into `started` and resumes from the task checkpoint after +interruption. The original claim releases after the whole task finishes and +its worker process exits. + +Problem-report workflows hold a per-user kernel lease across drafting, duplicate +search, match reads and writing, so concurrent workers share one reporting lane. diff --git a/oxa/oxa-cli/src/work_cmd.rs b/oxa/oxa-cli/src/work_cmd.rs index 8a5d94bd..7edfe1fe 100644 --- a/oxa/oxa-cli/src/work_cmd.rs +++ b/oxa/oxa-cli/src/work_cmd.rs @@ -3,6 +3,9 @@ use sha2::{Digest, Sha256}; use std::io::Write; use std::path::{Path, PathBuf}; +#[path = "work_cmd/native_queue.rs"] +mod native_queue; + pub(crate) fn trusted(cwd: &Path, env: &Env, dirs: &Dirs) -> Result { crate::dispatch::project_trusted( &crate::Args::default(), @@ -227,6 +230,63 @@ pub(crate) fn run( } let task_args = &args[..end]; let is_trusted = trusted(&root, env, dirs)?; + let context = Context { + cwd: root.clone(), + binary: binary.clone(), + policy: policy.clone(), + dirs: dirs.clone(), + env: env.clone(), + rerun, + publish, + reports, + }; + if task_args.first().is_some_and(|word| word == "--native-queue") { + if !is_trusted { + return Err("source project is not trusted; approve it before unattended work".into()); + } + let [_, flag, config] = task_args else { + return Err("usage: oxa work --native-queue --config PATH".into()); + }; + if flag != "--config" || rerun || publish { + return Err("invalid native queue options".into()); + } + return native_queue::run(context, &common, &cwd.join(config), out); + } + if task_args.first().is_some_and(|word| word == "--queue") { + if !is_trusted { + return Err("source project is not trusted; approve it before unattended work".into()); + } + let (directory, options) = queue_options(task_args, cwd)?; + let df = crate::netguard::trusted_executable_for( + "df", + Some(policy.host_path.as_os_str()), + Some(&root), + Some(&dirs.home), + ) + .ok_or("no trusted disk-space executable outside the source project")?; + let worker = std::sync::Arc::new(move |task: &oxa_work::Task| { + let plan = if context.publish || context.reports { + oxa_work::plan_task_for_actions( + &context.cwd, + task.clone(), + &context.policy, + context.rerun, + ) + } else { + oxa_work::plan_task_for_run(&context.cwd, task.clone(), &context.policy, context.rerun) + }; + match plan { + Ok(plan) => context.planned(&plan), + Err(error) => Ok(oxa_work::blocked_record(task, error)), + } + }); + let summary = + oxa_work::run_queue(&root, &directory, &options, &policy, &df, worker, &mut |event| { + serde_json::to_writer(&mut *out, &event).map_err(|e| e.to_string())?; + writeln!(out).and_then(|_| out.flush()).map_err(|e| e.to_string()) + })?; + return Ok(i32::from(summary.failed > 0 || summary.pending > 0)); + } let tasks = match task_args { [file] if !file.starts_with('-') => oxa_work::read_tasks(&cwd.join(file))?, [flag, number, model_flag, model] if flag == "--issue" && model_flag == "--model" => { @@ -298,88 +358,7 @@ pub(crate) fn run( let mut code = 0; for task in planned { let record = match task { - Ok(plan) => { - let cached = if (publish || reports) && !rerun { - oxa_work::action_record(&policy, plan.task())? - } else { - None - }; - let mut record = if let Some(record) = cached { - record - } else { - let ctx = crate::report_cmd::ScrubCtx::from_env(env, dirs); - match oxa_work::run_task_with_notes( - &root, - &binary, - &plan, - &policy, - rerun, - &|_: &oxa_work::Record| Ok(()), - &|note| crate::work_reports::scrub_note(note, &ctx), - ) { - Ok(record) => record, - Err(oxa_work::RunError::Blocked(detail)) => { - let mut record = oxa_work::blocked_record(plan.task(), detail); - record.base_sha = Some(plan.base_sha().to_owned()); - record - } - Err(oxa_work::RunError::Checkpoint(error)) => return Err(error), - } - }; - if publish && !record.worktree.as_os_str().is_empty() { - let gh = crate::netguard::trusted_executable_for( - "gh", - Some(policy.host_path.as_os_str()), - Some(&root), - Some(&dirs.home), - ) - .unwrap_or_else(|| PathBuf::from("/unavailable/gh")); - record = oxa_work::publish( - &root, - &binary, - plan.task(), - &policy, - &gh, - &|_: &oxa_work::Record| Ok(()), - &mut crate::report_cmd::tty_confirm(env), - )?; - } - if reports && !record.worktree.as_os_str().is_empty() { - let settings = oxa_config::Settings::load(dirs, &root, false); - let ctx = crate::report_cmd::ScrubCtx::from_env(env, dirs); - record = oxa_work::report_notes( - plan.task(), - &policy, - &|_: &oxa_work::Record| Ok(()), - &mut |note, lock, offline| { - let mut runner = |program: &str, args: &[String]| { - let executable = crate::netguard::trusted_executable_for( - program, - Some(policy.host_path.as_os_str()), - Some(&root), - Some(&dirs.home), - )?; - crate::work_reports::process( - &executable.to_string_lossy(), - args, - &root, - lock, - ) - }; - crate::work_reports::report( - note, - dirs, - &settings, - &ctx, - offline, - &mut crate::report_cmd::tty_confirm(env), - &mut runner, - ) - }, - )?; - } - record - } + Ok(plan) => context.planned(&plan)?, Err(record) => *record, }; if matches!( @@ -402,3 +381,140 @@ pub(crate) fn run( } } } + +struct Context { + cwd: PathBuf, + binary: PathBuf, + policy: oxa_work::RunPolicy, + dirs: Dirs, + env: Env, + rerun: bool, + publish: bool, + reports: bool, +} +impl Context { + fn planned(&self, plan: &oxa_work::PlannedTask) -> Result { + let mut locked = match oxa_work::LockedTask::acquire(plan.task(), &self.policy) { + Ok(locked) => locked, + Err(error) => return Ok(blocked_plan(plan, error)), + }; + self.planned_locked(plan, &mut locked) + } + + fn planned_locked( + &self, + plan: &oxa_work::PlannedTask, + locked: &mut oxa_work::LockedTask<'_>, + ) -> Result { + let root = &self.cwd; + let binary = &self.binary; + let policy = &self.policy; + let dirs = &self.dirs; + let env = &self.env; + let (rerun, publish, reports) = (self.rerun, self.publish, self.reports); + let cached = if (publish || reports) && !rerun { + oxa_work::action_record(policy, plan.task())? + } else { + None + }; + let mut record = if let Some(record) = cached { + record + } else { + let ctx = crate::report_cmd::ScrubCtx::from_env(env, dirs); + match locked.run_with_notes( + root, + binary, + plan, + rerun, + &|_: &oxa_work::Record| Ok(()), + &|note| crate::work_reports::scrub_note(note, &ctx), + ) { + Ok(record) => record, + Err(oxa_work::RunError::Blocked(detail)) => blocked_plan(plan, detail), + Err(oxa_work::RunError::Checkpoint(error)) => return Err(error), + } + }; + if publish && !record.worktree.as_os_str().is_empty() { + let gh = crate::netguard::trusted_executable_for( + "gh", + Some(policy.host_path.as_os_str()), + Some(root), + Some(&dirs.home), + ) + .unwrap_or_else(|| PathBuf::from("/unavailable/gh")); + record = locked.publish( + root, + binary, + &gh, + &|_: &oxa_work::Record| Ok(()), + &mut crate::report_cmd::tty_confirm(env), + )?; + } + if reports && !record.worktree.as_os_str().is_empty() { + let settings = oxa_config::Settings::load(dirs, root, false); + let ctx = crate::report_cmd::ScrubCtx::from_env(env, dirs); + record = locked.report_notes(&|_: &oxa_work::Record| Ok(()), &mut |note, lock, offline| { + let runner = |program: &str, args: &[String], gate: &std::fs::File| { + let executable = crate::netguard::trusted_executable_for( + program, + Some(policy.host_path.as_os_str()), + Some(root), + Some(&dirs.home), + )?; + crate::work_reports::process(&executable.to_string_lossy(), args, root, lock, gate) + }; + crate::work_reports::report( + note, + dirs, + &settings, + &ctx, + offline, + &mut crate::report_cmd::tty_confirm(env), + runner, + ) + })?; + } + Ok(record) + } +} +fn blocked_plan(plan: &oxa_work::PlannedTask, detail: String) -> oxa_work::Record { + let mut record = oxa_work::blocked_record(plan.task(), detail); + record.base_sha = Some(plan.base_sha().to_owned()); + record +} + +fn queue_options(args: &[String], cwd: &Path) -> Result<(PathBuf, oxa_work::QueueOptions), String> { + let directory = cwd.join(args.get(1).ok_or("--queue needs a directory")?); + let mut options = oxa_work::QueueOptions::default(); + let mut i = 2; + while i < args.len() { + match args[i].as_str() { + "--max" => { + i += 1; + options.max = args + .get(i) + .ok_or("--max needs a value")? + .parse() + .map_err(|_| "invalid --max")?; + } + "--min-free-mib" => { + i += 1; + let mib: u64 = args + .get(i) + .ok_or("--min-free-mib needs a value")? + .parse() + .map_err(|_| "invalid free-space minimum")?; + options.min_free_bytes = mib + .checked_mul(1024 * 1024) + .ok_or("free-space minimum is too large")?; + } + "--once" => options.once = true, + word => return Err(format!("unknown queue option: {word}")), + } + i += 1; + } + if options.max == 0 { + return Err("--max must be positive".into()); + } + Ok((directory, options)) +} diff --git a/oxa/oxa-cli/src/work_cmd/native_queue.rs b/oxa/oxa-cli/src/work_cmd/native_queue.rs new file mode 100644 index 00000000..7edd8abe --- /dev/null +++ b/oxa/oxa-cli/src/work_cmd/native_queue.rs @@ -0,0 +1,141 @@ +//! Explicit one-shot execution of an existing, fully bound native board packet. +use super::{protected_storage, Context}; +use oxa_work::{ + board::{assignment::Worker, Claims}, + LockedTask, NativeQueueOwner, QueueOptions, QueueRun, Task, +}; +use serde_json::Value; +use std::{ + fs, + io::{Read, Write}, + os::unix::fs::MetadataExt, + path::{Path, PathBuf}, + sync::Arc, +}; + +const MAX_CONFIG_BYTES: u64 = 64 * 1024; + +fn paused(path: &Path) -> Result { + match fs::symlink_metadata(path) { + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(_) => Err("cannot establish board pause state".into()), + } +} + +pub(super) fn run( + mut context: Context, + common: &Path, + config: &Path, + out: &mut dyn Write, +) -> Result { + if context.env.pairs().iter().any(|(name, _)| { + matches!( + name.as_str(), + "OXA_CODING_AGENT" | "AI_AGENT" | "OXA_SUBAGENT_DEPTH" + ) + }) { + return Err("native queue execution requires the owner outside an agent tool".into()); + } + // The existing supervisor receives the runner's offline switch. Keep + // credential mounts and network off for deterministic steps until the + // supervisor also inherits the worker's runner-owned GitHub policy. + context.policy.offline = true; + let context = Arc::new(context); + let root = &context.cwd; + let config = config.canonicalize().map_err(|_| "board config unavailable")?; + let owner = context + .dirs + .agent_dir + .canonicalize() + .map_err(|_| "owner controls unavailable")?; + if !config.starts_with(&owner) { + return Err("board config must be in protected owner controls".into()); + } + protected_storage(&config, root, common)?; + let uid = fs::metadata(&context.dirs.home) + .map_err(|_| "owner home unavailable")? + .uid(); + let mut bytes = Vec::new(); + crate::lead_input::open(&config, uid)? + .take(MAX_CONFIG_BYTES + 1) + .read_to_end(&mut bytes) + .map_err(|_| "cannot read native board configuration")?; + if bytes.len() as u64 > MAX_CONFIG_BYTES { + return Err("native board configuration exceeds capacity".into()); + } + let value: Value = serde_json::from_slice(&bytes).map_err(|_| "invalid native board configuration")?; + if value["enabled"] != true || value["nativeExecution"] != true { + return Ok(0); + } + let owner_path = |key: &str| -> Result { + let path = PathBuf::from(value[key].as_str().ok_or("missing native board path")?); + protected_storage(&path, root, common)?; + Ok(path) + }; + let pause = owner_path("pause")?; + if paused(&pause)? { + return Ok(0); + } + let directory = owner_path("queue")?; + let claims = Claims::open(&owner_path("claims")?)?; + let worker: Worker = + serde_json::from_value(value["worker"].clone()).map_err(|_| "invalid authenticated worker export")?; + if !oxa_work::board::assignment::valid_worker_id(&worker.id) { + return Err("invalid native worker identity".into()); + } + let rows = claims.assignments()?; + let mut owned = rows.into_values().filter(|row| { + row.state + .lease + .as_ref() + .is_some_and(|lease| lease.holder == worker.id) + }); + let Some(row) = owned.next() else { return Ok(0) }; + if owned.next().is_some() { + return Err("native worker has multiple active assignments".into()); + } + let binding = row + .data + .native_binding + .filter(|binding| row.state.lease.as_ref() == Some(&binding.lease)) + .ok_or("native worker lacks an exact retained binding")?; + let owner = NativeQueueOwner { + claims: &claims, + worker_id: &worker.id, + }; + let policy = &context.policy; + let df = crate::netguard::trusted_executable_for( + "df", + Some(&policy.host_path), + Some(root), + Some(&context.dirs.home), + ) + .ok_or("no trusted disk-space executable outside the source project")?; + let worker_context = Arc::clone(&context); + let run = Arc::new(move |task: &Task, lock: &mut LockedTask<'_>| { + if paused(&pause)? { + return Err("native board execution paused".into()); + } + match lock.plan_native(&worker_context.cwd, worker_context.reports) { + Ok(plan) => worker_context.planned_locked(&plan, lock), + Err(error) => Ok(oxa_work::blocked_record(task, error)), + } + }); + let options = QueueOptions { + once: true, + ..QueueOptions::default() + }; + let summary = QueueRun { + repo: root, + directory: &directory, + options: &options, + policy, + df: &df, + } + .run_native_packet(&owner, &binding, run, &mut |event| { + serde_json::to_writer(&mut *out, &event).map_err(|e| e.to_string())?; + writeln!(out).and_then(|_| out.flush()).map_err(|e| e.to_string()) + })?; + Ok(i32::from(summary.failed > 0 || summary.pending > 0)) +} diff --git a/oxa/oxa-cli/src/work_reports.rs b/oxa/oxa-cli/src/work_reports.rs index cb10fe81..116a4605 100644 --- a/oxa/oxa-cli/src/work_reports.rs +++ b/oxa/oxa-cli/src/work_reports.rs @@ -10,9 +10,16 @@ use std::{ process::{Command, Stdio}, }; -pub(crate) fn process(program: &str, args: &[String], cwd: &Path, lock: &File) -> Option<(i32, String)> { +pub(crate) fn process( + program: &str, + args: &[String], + cwd: &Path, + lock: &File, + gate: &File, +) -> Option<(i32, String)> { let mut command = Command::new(program); command.args(args).current_dir(cwd); + oxa_work::inherit_report_lease(&mut command, gate).ok()?; oxa_work::command_runner_with_stdin( command, Some(1_048_576), @@ -83,8 +90,21 @@ pub(crate) fn report( ctx: &ScrubCtx, offline: bool, confirm: Confirm<'_>, - runner: Runner<'_>, + mut run: impl FnMut(&str, &[String], &File) -> Option<(i32, String)>, ) -> Result { + // Retain the queue's per-user reporting lease across every effect. + use std::os::unix::fs::OpenOptionsExt; + std::fs::create_dir_all(&dirs.agent_dir).map_err(|e| e.to_string())?; + let gate = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .mode(0o600) + .open(dirs.agent_dir.join("work-report.lock")) + .map_err(|e| e.to_string())?; + gate.lock().map_err(|e| e.to_string())?; + let runner: Runner<'_> = &mut |program, args| run(program, args, &gate); let note = &ProblemNote { title: visible(¬e.title), notes: visible(¬e.notes), @@ -285,7 +305,7 @@ mod tests { let env = TestEnv::new(); let lock = File::open("/dev/null").unwrap(); let args = words(&["-c", "1048577", "/dev/zero"]); - assert!(process("/usr/bin/head", &args, &env.work(), &lock).is_none()); + assert!(process("/usr/bin/head", &args, &env.work(), &lock, &lock).is_none()); } fn note() -> ProblemNote { @@ -321,7 +341,7 @@ mod tests { approved = true; Some(false) }, - &mut |program, _| { + |program, _, _| { if program == "gh" { calls += 1; } @@ -350,7 +370,7 @@ mod tests { prompts.push(prompt.to_owned()); Some(true) }, - &mut |program, args| { + |program, args: &[String], _: &File| { if program != "gh" { return Some((0, "{}".into())); } @@ -379,7 +399,7 @@ mod tests { let env = TestEnv::new(); let dirs = Dirs::under(env.root()); let mut log = vec![]; - let mut runner = |program: &str, args: &[String]| { + let runner = |program: &str, args: &[String], _: &File| { if program != "gh" { return Some((0, "{}".into())); } @@ -399,7 +419,7 @@ mod tests { &ScrubCtx::default(), false, &mut |_| Some(true), - &mut runner, + runner, ) .unwrap(); assert_eq!(result.disposition, "existing_issue"); @@ -412,7 +432,7 @@ mod tests { let env = TestEnv::new(); let dirs = Dirs::under(env.root()); let mut gh_calls = 0; - let mut runner = |program: &str, _: &[String]| { + let runner = |program: &str, _: &[String], _: &File| { if program == "gh" { gh_calls += 1; None @@ -427,7 +447,7 @@ mod tests { &ScrubCtx::default(), offline, &mut |_| Some(true), - &mut runner, + runner, ) .unwrap(); assert_eq!(result.disposition, "local"); @@ -440,7 +460,7 @@ mod tests { fn an_open_fixing_pr_prevents_issue_or_comment_writes() { let env = TestEnv::new(); let dirs = Dirs::under(env.root()); - let mut runner = |program: &str, args: &[String]| { + let runner = |program: &str, args: &[String], _: &File| { if program != "gh" { return Some((0, "{}".into())); } @@ -465,7 +485,7 @@ mod tests { &ScrubCtx::default(), false, &mut |_| Some(true), - &mut runner, + runner, ) .unwrap(); assert_eq!(result.disposition, "existing_pr"); @@ -477,7 +497,7 @@ mod tests { let mut n = note(); n.notes = "Synthetic reproduction token=example at /home/synthetic/project with user@example.invalid" .into(); - let mut runner = |program: &str, args: &[String]| { + let runner = |program: &str, args: &[String], _: &File| { if program != "gh" { return Some((0, "{}".into())); } @@ -495,7 +515,7 @@ mod tests { &ScrubCtx::default(), false, &mut |_| Some(true), - &mut runner, + runner, ) .unwrap(); assert_eq!(result.disposition, "filed"); diff --git a/oxa/oxa-cli/tests/c12_argument_parsing.rs b/oxa/oxa-cli/tests/c12_argument_parsing.rs index cdf9f2c2..7a58d2d4 100644 --- a/oxa/oxa-cli/tests/c12_argument_parsing.rs +++ b/oxa/oxa-cli/tests/c12_argument_parsing.rs @@ -479,6 +479,11 @@ fn agent_options_keep_command_words_as_prompts() { assert_eq!(a.subcommand, None, "{word}"); assert_eq!(a.messages, [word, "example"]); } + // Only option words may precede the work command; a prompt word before it keeps + // `work` a prompt word in place. + let a = parse(&["summarise", word, "example"]); + assert_eq!(a.subcommand, None, "{word}"); + assert_eq!(a.messages, ["summarise", word, "example"]); } } diff --git a/oxa/oxa-cli/tests/common/work.rs b/oxa/oxa-cli/tests/common/work.rs index 790ec03b..ab503296 100644 --- a/oxa/oxa-cli/tests/common/work.rs +++ b/oxa/oxa-cli/tests/common/work.rs @@ -1,44 +1,10 @@ use oxa_fake::{FakeOpts, FakeServer, ScriptTurn, TestEnv}; #[path = "work_confirm.rs"] mod confirm; -use std::{path::Path, process::Command}; -pub fn git(env: &TestEnv, dir: &Path, args: &[&str]) -> String { - let out = Command::new("git") - .args(args) - .current_dir(dir) - .env_clear() - .envs(env.envs("http://127.0.0.1:1")) - .env("GIT_AUTHOR_NAME", "Test") - .env("GIT_AUTHOR_EMAIL", "test@example.invalid") - .env("GIT_COMMITTER_NAME", "Test") - .env("GIT_COMMITTER_EMAIL", "test@example.invalid") - .output() - .unwrap(); - assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); - String::from_utf8_lossy(&out.stdout).trim().into() -} -pub fn setup(env: &TestEnv, done: &str) { - git(env, &env.work(), &["init", "--bare", "remote.git"]); - git(env, &env.work(), &["init", "-b", "main", "project"]); - let project = env.work().join("project"); - git(env, &project, &["config", "user.name", "Test"]); - git(env, &project, &["config", "user.email", "test@example.invalid"]); - env.write( - "work/project/.oxa/settings.json", - &format!(r#"{{"done":{done}}}"#), - ); - git(env, &project, &["add", "."]); - git(env, &project, &["commit", "-m", "fixture"]); - git(env, &project, &["remote", "add", "origin", "../remote.git"]); - git(env, &project, &["push", "origin", "main"]); - oxa_perm::TrustStore::default() - .save(&env.agent_dir().join("trust.json"), &project, true) - .unwrap(); - env.write( - "work/tasks.json", - r#"[{"id":"test","branch":"agent/test","model":"fake","prompt":"Finish task"}]"#, - ); -} +#[path = "work_repo.rs"] +mod work_repo; +use std::process::Command; +pub use work_repo::{git, setup}; pub fn run(env: &TestEnv, turns: Vec) -> serde_json::Value { run_args(env, turns, &["work", "../tasks.json"], None) } diff --git a/oxa/oxa-cli/tests/common/work_repo.rs b/oxa/oxa-cli/tests/common/work_repo.rs new file mode 100644 index 00000000..14343124 --- /dev/null +++ b/oxa/oxa-cli/tests/common/work_repo.rs @@ -0,0 +1,39 @@ +use oxa_fake::TestEnv; +use std::{path::Path, process::Command}; +pub fn git(env: &TestEnv, dir: &Path, args: &[&str]) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .env_clear() + .envs(env.envs("http://127.0.0.1:1")) + .env("GIT_AUTHOR_NAME", "Test") + .env("GIT_AUTHOR_EMAIL", "test@example.invalid") + .env("GIT_COMMITTER_NAME", "Test") + .env("GIT_COMMITTER_EMAIL", "test@example.invalid") + .output() + .unwrap(); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + String::from_utf8_lossy(&out.stdout).trim().into() +} +pub fn setup(env: &TestEnv, done: &str) { + git(env, &env.work(), &["init", "--bare", "remote.git"]); + git(env, &env.work(), &["init", "-b", "main", "project"]); + let project = env.work().join("project"); + git(env, &project, &["config", "user.name", "Test"]); + git(env, &project, &["config", "user.email", "test@example.invalid"]); + env.write( + "work/project/.oxa/settings.json", + &format!(r#"{{"done":{done}}}"#), + ); + git(env, &project, &["add", "."]); + git(env, &project, &["commit", "-m", "fixture"]); + git(env, &project, &["remote", "add", "origin", "../remote.git"]); + git(env, &project, &["push", "origin", "main"]); + oxa_perm::TrustStore::default() + .save(&env.agent_dir().join("trust.json"), &project, true) + .unwrap(); + env.write( + "work/tasks.json", + r#"[{"id":"test","branch":"agent/test","model":"fake","prompt":"Finish task"}]"#, + ); +} diff --git a/oxa/oxa-cli/tests/helper_queue.rs b/oxa/oxa-cli/tests/helper_queue.rs index bdca93ef..bb88a9a2 100644 --- a/oxa/oxa-cli/tests/helper_queue.rs +++ b/oxa/oxa-cli/tests/helper_queue.rs @@ -53,6 +53,34 @@ fn complete_private_publication_dedups_started_and_priority_changes() { ); } +#[test] +fn native_started_packet_resumes_only_its_original_lease_generation() { + use oxa_work::{ + board::{Lease, NativeBinding}, + helper_queue::take_native_packet, + }; + + let env = TestEnv::new(); + let queue = env.tmp().join("queue"); + fs::create_dir(&queue).unwrap(); + let key = "a".repeat(64); + let mut lease = Lease { + holder: "oxa:fixture".into(), + claimed_at: Some(10), + expires_at: 20, + generation: (0, 1), + }; + let original = NativeBinding::new(42, &key, &lease, "agent/42-fix").unwrap(); + assert!(publish(&queue, &key, TASK, true, &enabled()).unwrap()); + assert_eq!(take_native_packet(&queue, &original).unwrap(), TASK); + + lease.generation = (0, 2); + let newer = NativeBinding::new(42, &key, &lease, "agent/42-fix").unwrap(); + assert!(take_native_packet(&queue, &newer).is_err()); + assert!(!publish(&queue, &key, TASK, false, &enabled()).unwrap()); + assert_eq!(take_native_packet(&queue, &original).unwrap(), TASK); +} + #[test] fn concurrent_publishers_create_exactly_one_complete_task() { if let Some(queue) = std::env::var_os("OXA_TEST_BOARD_QUEUE") { diff --git a/oxa/oxa-cli/tests/work_queue.rs b/oxa/oxa-cli/tests/work_queue.rs new file mode 100644 index 00000000..3f5e77c1 --- /dev/null +++ b/oxa/oxa-cli/tests/work_queue.rs @@ -0,0 +1,649 @@ +//! Queue behavior against real worktrees and a fake engine, with no external network. +#[path = "common/work_confirm.rs"] +mod confirm; +#[path = "common/work_repo.rs"] +mod work; +use oxa_fake::{FakeOpts, FakeServer, ScriptTurn, TestEnv}; +use sha2::{Digest, Sha256}; +use std::{ + fs, + process::{Command, Output}, + time::{Duration, Instant, SystemTime}, +}; +use work::{git, setup}; +fn storage(env: &TestEnv) -> std::path::PathBuf { + let common = env.work().join("project/.git").canonicalize().unwrap(); + let key: String = Sha256::digest(common.as_os_str().as_encoded_bytes()) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + env.agent_dir().join("work").join(key) +} +fn task_file(env: &TestEnv, file: &str, id: &str, age: u64) { + let path = env.write(&format!("work/queue/{file}.json"), &serde_json::json!([{"id":id,"model":"fake","branch":format!("agent/{id}"),"prompt":format!("{id} task")}]).to_string()); + fs::OpenOptions::new() + .write(true) + .open(path) + .unwrap() + .set_modified(SystemTime::now() - Duration::from_secs(age)) + .unwrap(); +} +fn command(env: &TestEnv, url: &str, max: &str) -> Command { + let mut c = Command::new(env!("CARGO_BIN_EXE_oxa")); + c.args([ + "work", + "--queue", + "../queue", + "--max", + max, + "--min-free-mib", + "0", + "--once", + ]) + .current_dir(env.work().join("project")) + .env_clear() + .envs(env.envs(url)); + c +} +fn finish(c: Command) -> Output { + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send(confirm::output(c)); + }); + rx.recv_timeout(Duration::from_secs(30)).expect("queue finished") +} +fn rows(out: &Output) -> Vec { + assert!(out.status.code().is_some(), "queue terminated"); + String::from_utf8_lossy(&out.stdout) + .lines() + .map(|line| { + serde_json::from_str(line) + .unwrap_or_else(|e| panic!("{e}: {line} / {}", String::from_utf8_lossy(&out.stderr))) + }) + .collect() +} +fn until(mut f: impl FnMut() -> bool) { + let deadline = Instant::now() + Duration::from_secs(15); + while !f() { + assert!(Instant::now() < deadline); + std::thread::sleep(Duration::from_millis(10)); + } +} +#[test] +fn oldest_file_runs_first_and_restart_skips_completed_agents() { + let env = TestEnv::new(); + setup(&env, "[]"); + task_file(&env, "a-new", "new", 10); + task_file(&env, "z-old", "old", 20); + let server = FakeServer::start( + vec![ScriptTurn::text("done"), ScriptTurn::text("done")], + FakeOpts::default(), + ) + .unwrap(); + let out = finish(command(&env, &server.url(), "1")); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + let messages = server + .requests() + .into_iter() + .filter(|r| r.max_tokens > 0) + .collect::>(); + assert_eq!(messages.len(), 2); + assert!(messages[0] + .messages + .iter() + .any(|m| m.content.as_deref().is_some_and(|s| s.contains("old task")))); + let events = rows(&out); + let ids: Vec<_> = events + .iter() + .filter(|r| r["event"] == "task") + .map(|r| r["id"].as_str().unwrap()) + .collect(); + assert_eq!(ids, ["old", "new"]); + let cached = finish(command(&env, &server.url(), "1")); + assert!(cached.status.success()); + assert_eq!( + server.requests().into_iter().filter(|r| r.max_tokens > 0).count(), + 2 + ); +} +#[test] +fn concurrency_is_bounded_and_monitor_snapshot_is_a_complete_line() { + let env = TestEnv::new(); + setup(&env, "[]"); + for (index, id) in ["a", "b", "c"].iter().enumerate() { + task_file(&env, id, id, index as u64); + } + let server = FakeServer::start( + vec![ScriptTurn::text("done"); 3], + FakeOpts { + hold_headers_ms: 300, + ..FakeOpts::default() + }, + ) + .unwrap(); + let out = finish(command(&env, &server.url(), "2")); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + let events = rows(&out); + let active: Vec<_> = events + .iter() + .filter(|r| r["event"] == "state") + .map(|r| r["active"].as_array().unwrap().len()) + .collect(); + assert!(active.contains(&2)); + assert!(active.iter().all(|n| *n <= 2)); + let line = fs::read_to_string(env.work().join("queue/.oxa-queue-status.json")).unwrap(); + assert_eq!(line.lines().count(), 1); + let status: serde_json::Value = serde_json::from_str(&line).unwrap(); + assert_eq!(status["finished"], 3); + assert_eq!(status["active"], serde_json::json!([])); +} +#[test] +fn low_disk_never_starts_a_task_and_leaves_the_file_pending() { + use std::os::unix::fs::PermissionsExt; + let env = TestEnv::new(); + setup(&env, "[]"); + task_file(&env, "task", "low", 1); + let df = env.write("tmp/df", "#!/bin/sh\nprintf 'Avail\\n0\\n'\n"); + fs::set_permissions(df, fs::Permissions::from_mode(0o755)).unwrap(); + let server = FakeServer::start(vec![], FakeOpts::default()).unwrap(); + let mut cmd = command(&env, &server.url(), "2"); + cmd.args(["--min-free-mib", "1"]).env( + "PATH", + format!("{}:{}", env.tmp().display(), std::env::var("PATH").unwrap()), + ); + let out = finish(cmd); + assert_eq!(out.status.code(), Some(1)); + assert!(server.requests().is_empty()); + let events = rows(&out); + let status = events.last().unwrap(); + assert_eq!(status["disk_blocked"], true); + assert_eq!(status["pending"], 1); + assert!(!storage(&env).join("low.json").exists()); + assert!(env.work().join("queue/task.json").exists()); +} +#[test] +fn invalid_file_does_not_skip_later_valid_work() { + let env = TestEnv::new(); + setup(&env, "[]"); + env.write("work/queue/bad.json", "invalid"); + task_file(&env, "good", "good", 1); + let server = FakeServer::start(vec![ScriptTurn::text("done")], FakeOpts::default()).unwrap(); + let out = finish(command(&env, &server.url(), "2")); + assert_eq!(out.status.code(), Some(1)); + let events = rows(&out); + assert!(events.iter().any(|r| r["event"] == "error")); + assert!(events.iter().any(|r| r["event"] == "task" && r["id"] == "good")); +} +#[test] +fn a_surviving_worker_counts_against_max_until_its_lock_releases() { + let env = TestEnv::new(); + setup(&env, "[]"); + task_file(&env, "held", "held", 20); + task_file(&env, "next", "next", 10); + fs::create_dir_all(storage(&env)).unwrap(); + fs::write(storage(&env).join("held.lock"), "").unwrap(); + let file = fs::OpenOptions::new() + .read(true) + .write(true) + .open(storage(&env).join("held.lock")) + .unwrap(); + file.try_lock().unwrap(); + let server = FakeServer::start(vec![ScriptTurn::text("done"); 2], FakeOpts::default()).unwrap(); + let mut child = command(&env, &server.url(), "1") + .stdout(std::process::Stdio::null()) + .spawn() + .unwrap(); + until(|| { + fs::read(env.work().join("queue/.oxa-queue-status.json")) + .ok() + .and_then(|b| serde_json::from_slice::(&b).ok()) + .is_some_and(|v| v["active"] == serde_json::json!(["held"])) + }); + assert!(server.requests().is_empty()); + drop(file); + assert!(child.wait().unwrap().success()); + git( + &env, + &env.work().join("project"), + &["show-ref", "--verify", "refs/heads/agent/next"], + ); +} +#[test] +fn files_arriving_while_a_worker_runs_are_picked_up_before_once_exits() { + let env = TestEnv::new(); + setup(&env, "[]"); + task_file(&env, "first", "first", 1); + let server = FakeServer::start( + vec![ScriptTurn::text("done"); 2], + FakeOpts { + hold_headers_ms: 400, + ..FakeOpts::default() + }, + ) + .unwrap(); + let mut child = command(&env, &server.url(), "1") + .stdout(std::process::Stdio::null()) + .spawn() + .unwrap(); + until(|| !server.requests().is_empty()); + task_file(&env, "arrived", "arrived", 1); + assert!(child.wait().unwrap().success()); + let path = storage(&env).join("arrived.json"); + assert!(path.exists()); +} + +#[test] +fn invalid_max_is_rejected_without_starting_workers() { + let env = TestEnv::new(); + setup(&env, "[]"); + task_file(&env, "one", "one", 1); + let server = FakeServer::start(vec![], FakeOpts::default()).unwrap(); + let out = finish(command(&env, &server.url(), "0")); + assert_eq!(out.status.code(), Some(2)); + assert!(server.requests().is_empty()); + assert!(!env.work().join("queue/.oxa-queue-status.json").exists()); +} +#[test] +fn queued_workers_keep_the_parent_command_line_offline_policy() { + let env = TestEnv::new(); + setup(&env, "[]"); + task_file(&env, "remote", "remote", 1); + let server = FakeServer::start( + vec![ScriptTurn::text("should not be requested")], + FakeOpts { + remote: Some("openrouter"), + models: vec![oxa_fake::FakeModel { + alias: "fake-model".into(), + cap: 64000, + max_output: Some(4096), + caps: None, + }], + ..FakeOpts::default() + }, + ) + .unwrap(); + let mut c = Command::new(env!("CARGO_BIN_EXE_oxa")); + c.env_clear() + .args([ + "--offline", + "work", + "--queue", + "../queue", + "--max", + "2", + "--once", + "--min-free-mib", + "0", + ]) + .current_dir(env.work().join("project")) + .envs(env.envs(&server.url())); + let out = finish(c); + assert_eq!(out.status.code(), Some(1)); + assert!(server.requests().is_empty()); + let events = rows(&out); + assert!(events.iter().any(|r| r["event"] == "task" + && r["outcome"] == "blocked" + && r["detail"].as_str().is_some_and(|s| s.contains("offline")))); +} + +#[test] +fn preparing_git_keeps_ownership_after_a_queue_controller_is_killed() { + use std::os::unix::fs::PermissionsExt; + struct Release { + release: std::path::PathBuf, + done: std::path::PathBuf, + } + impl Drop for Release { + fn drop(&mut self) { + let _ = fs::write(&self.release, "release"); + let end = Instant::now() + Duration::from_secs(2); + while !self.done.exists() && Instant::now() < end { + std::thread::sleep(Duration::from_millis(10)); + } + } + } + let env = TestEnv::new(); + setup(&env, "[]"); + task_file(&env, "preparing", "preparing", 1); + let tree = env.work().join(".oxa-worktrees/project/preparing"); + env.write( + "work/project/.git/info/exclude", + ".prep-started\n.release-prep\n.prep-done\n", + ); + let release = Release { + release: tree.join(".release-prep"), + done: tree.join(".prep-done"), + }; + let hook = env.write( + "work/project/.git/hooks/post-checkout", + r#"#!/bin/sh +touch .prep-started +attempt=0 +while ! test -f .release-prep; do + attempt=$((attempt + 1)) + if test "$attempt" -ge 500; then exit 9; fi + sleep 0.01 +done +touch .prep-done +"#, + ); + fs::set_permissions(hook, fs::Permissions::from_mode(0o755)).unwrap(); + let server = FakeServer::start(vec![ScriptTurn::text("resumed")], FakeOpts::default()).unwrap(); + let mut child = command(&env, &server.url(), "1") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .unwrap(); + until(|| tree.join(".prep-started").exists()); + child.kill().unwrap(); + child.wait().unwrap(); + let lock_path = storage(&env).join("preparing.lock"); + let probe = fs::OpenOptions::new() + .read(true) + .write(true) + .open(&lock_path) + .unwrap(); + assert!( + matches!(probe.try_lock(), Err(fs::TryLockError::WouldBlock)), + "surviving checkout lost task ownership" + ); + drop(probe); + fs::write(&release.release, "release").unwrap(); + until(|| { + let file = fs::OpenOptions::new() + .read(true) + .write(true) + .open(&lock_path) + .unwrap(); + file.try_lock().is_ok() + }); + let out = finish(command(&env, &server.url(), "1")); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + assert!(rows(&out) + .iter() + .any(|r| r["event"] == "task" && r["outcome"] == "nothing_to_do")); +} + +#[test] +fn concurrent_same_problem_reports_search_and_file_serially() { + use std::os::unix::fs::PermissionsExt; + let env = TestEnv::new(); + setup(&env, "[]"); + task_file(&env, "a", "a", 2); + task_file(&env, "b", "b", 1); + env.write( + "home/.oxa/agent/settings.json", + r#"{"oxa":{"reportIssue":{"repo":"example/project"}}}"#, + ); + let gh=env.write("tmp/gh",r##"#!/bin/sh +case "$1 $2" in + 'issue list') if test -f "$HOME/reported"; then printf '[{"number":1}]'; else printf '[]'; fi ;; + 'pr list') printf '[]' ;; + 'issue view') printf '%s' '{"title":"oxa queue problem","body":"Same synthetic reproduction","url":"https://github.com/example/project/issues/1","comments":[]}' ;; + 'issue create') + for fd in /proc/self/fd/*; do + if test "$fd" -ef "$HOME/.oxa/agent/work-report.lock"; then + touch "$HOME/report-lease-inherited" + fi + done + sleep 0.2; touch "$HOME/reported"; printf 'created\n' >> "$HOME/report-writes"; printf '%s' 'https://github.com/example/project/issues/1' ;; + *) exit 7 ;; +esac +"##); + let oxc = env.write("tmp/oxc", "#!/bin/sh\nprintf '{}'\n"); + for file in [gh, oxc] { + fs::set_permissions(file, fs::Permissions::from_mode(0o755)).unwrap(); + } + let text="[{\"title\":\"oxa queue problem\",\"notes\":\"Same synthetic reproduction\",\"keywords\":[\"oxa queue\",\"same reproduction\"]}]"; + let server = FakeServer::start(vec![ScriptTurn::text(text); 2], FakeOpts::default()).unwrap(); + let mut c = command(&env, &server.url(), "2"); + c.arg("--report-problems").env( + "PATH", + format!("{}:{}", env.tmp().display(), std::env::var("PATH").unwrap()), + ); + let out = finish(c); + assert!( + out.status.success(), + "{} / {}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + assert_eq!( + fs::read_to_string(env.home().join("report-writes")) + .unwrap() + .lines() + .count(), + 1 + ); + let events = rows(&out); + assert!( + env.home().join("report-lease-inherited").exists(), + "a surviving GitHub writer must retain the reporting lease" + ); + let dispositions: std::collections::HashSet<_> = events + .iter() + .filter(|r| r["event"] == "task") + .map(|r| r["reports"][0]["disposition"].as_str().unwrap()) + .collect(); + assert!( + dispositions.contains("filed") && dispositions.contains("existing_issue"), + "{dispositions:?}" + ); +} + +fn has_active(env: &TestEnv, count: usize) -> bool { + fs::read(env.work().join("queue/.oxa-queue-status.json")) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .is_some_and(|state| state["active"].as_array().is_some_and(|ids| ids.len() == count)) +} + +#[test] +fn appended_tasks_and_archived_inputs_do_not_fail_or_interrupt_active_work() { + use std::process::Stdio; + let env = TestEnv::new(); + setup(&env, "[]"); + task_file(&env, "input", "first", 1); + let server = FakeServer::start( + vec![ScriptTurn::text("done"); 2], + FakeOpts { + hold_headers_ms: 700, + ..FakeOpts::default() + }, + ) + .unwrap(); + let child = command(&env, &server.url(), "2") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + until(|| has_active(&env, 1)); + let file = env.work().join("queue/input.json"); + let mut tasks: Vec = serde_json::from_slice(&fs::read(&file).unwrap()).unwrap(); + tasks.push( + serde_json::json!({"id":"second","model":"fake","branch":"agent/second","prompt":"second task"}), + ); + fs::write(&file, serde_json::to_vec(&tasks).unwrap()).unwrap(); + until(|| has_active(&env, 2)); + fs::create_dir(env.work().join("archive")).unwrap(); + fs::rename(&file, env.work().join("archive/input.json")).unwrap(); + let out = child.wait_with_output().unwrap(); + assert!( + out.status.success(), + "{} / {}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + let events = rows(&out); + assert_eq!(events.iter().filter(|v| v["event"] == "task").count(), 2); + assert_eq!(events.last().unwrap()["failed"], 0); +} + +#[test] +fn correcting_a_malformed_file_clears_its_queue_error() { + use std::process::Stdio; + let env = TestEnv::new(); + setup(&env, "[]"); + env.write("work/queue/bad.json", "{"); + task_file(&env, "active", "active", 1); + let server = FakeServer::start( + vec![ScriptTurn::text("done"); 2], + FakeOpts { + hold_headers_ms: 700, + ..FakeOpts::default() + }, + ) + .unwrap(); + let child = command(&env, &server.url(), "2") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + until(|| has_active(&env, 1)); + task_file(&env, "bad", "corrected", 0); + let out = child.wait_with_output().unwrap(); + assert!( + out.status.success(), + "{} / {}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + assert_eq!(rows(&out).last().unwrap()["failed"], 0); +} + +#[test] +fn a_monitor_error_drains_admitted_workers_before_returning() { + use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }; + let env = TestEnv::new(); + setup(&env, "[]"); + task_file(&env, "input", "active", 1); + let finished = Arc::new(AtomicBool::new(false)); + let observed = finished.clone(); + let policy = oxa_work::RunPolicy { + git_binary: "/usr/bin/git".into(), + host_path: "/usr/bin:/bin".into(), + terminate_binary: None, + offline: true, + runner_owns_github: false, + storage: env.agent_dir().join("work/monitor-fixture"), + }; + let result = oxa_work::run_queue( + &env.work().join("project"), + &env.work().join("queue"), + &oxa_work::QueueOptions { + max: 1, + min_free_bytes: 0, + once: true, + }, + &policy, + std::path::Path::new("/usr/bin/df"), + Arc::new(move |task| { + std::thread::sleep(Duration::from_millis(100)); + observed.store(true, Ordering::Release); + Ok(oxa_work::blocked_record(task, "synthetic worker result".into())) + }), + &mut |_| Err("synthetic monitor failure".into()), + ); + assert!(result.unwrap_err().contains("synthetic monitor failure")); + assert!( + finished.load(Ordering::Acquire), + "controller returned before its admitted worker finished" + ); +} + +#[test] +fn duplicate_branches_in_separate_files_are_rejected_before_admission() { + let env = TestEnv::new(); + setup(&env, "[]"); + task_file(&env, "first", "first", 2); + env.write( + "work/queue/second.json", + r#"[{"id":"second","model":"fake","branch":"agent/first","prompt":"Synthetic duplicate branch"}]"#, + ); + let server = FakeServer::start(vec![ScriptTurn::text("done"); 2], FakeOpts::default()).unwrap(); + let out = finish(command(&env, &server.url(), "1")); + let events = rows(&out); + assert!( + events.iter().any(|row| row["event"] == "error" + && row["detail"] + .as_str() + .is_some_and(|detail| detail.contains("branch"))), + "{events:?}" + ); + assert!(!events + .iter() + .any(|row| row["event"] == "task" && row["id"] == "second")); + assert_eq!(server.requests().len(), 1); +} + +#[test] +fn a_full_common_git_volume_blocks_admission_independently() { + use std::os::unix::fs::PermissionsExt; + let env = TestEnv::new(); + setup(&env, "[]"); + task_file(&env, "input", "input", 1); + let df = env.write("tmp/df", "#!/bin/sh\nfor arg do checked=$arg; done\nif test \"$checked\" = \"$OXA_TEST_COMMON_GIT\"; then printf '0\\n'; else printf '1000000000000\\n'; fi\n"); + fs::set_permissions(df, fs::Permissions::from_mode(0o755)).unwrap(); + let server = FakeServer::start(vec![ScriptTurn::text("done")], FakeOpts::default()).unwrap(); + let mut cmd = command(&env, &server.url(), "1"); + cmd.args(["--min-free-mib", "512"]) + .env( + "OXA_TEST_COMMON_GIT", + env.work().join("project/.git").canonicalize().unwrap(), + ) + .env( + "PATH", + format!("{}:{}", env.tmp().display(), std::env::var("PATH").unwrap()), + ); + let out = finish(cmd); + assert_eq!(out.status.code(), Some(1)); + assert_eq!(rows(&out).last().unwrap()["disk_blocked"], true); + assert!(server.requests().is_empty()); +} + +#[test] +fn a_retained_busy_lease_without_an_input_counts_against_max() { + use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt}; + let env = TestEnv::new(); + setup(&env, "[]"); + task_file(&env, "next", "next", 1); + fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(storage(&env)) + .unwrap(); + let lease = fs::OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .mode(0o600) + .open(storage(&env).join("retained.lock")) + .unwrap(); + lease.try_lock().unwrap(); + let server = FakeServer::start( + vec![ScriptTurn::text("done")], + FakeOpts { + hold_headers_ms: 700, + ..FakeOpts::default() + }, + ) + .unwrap(); + let child = command(&env, &server.url(), "1") + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + let status = env.work().join("queue/.oxa-queue-status.json"); + until(|| status.exists()); + let before: serde_json::Value = serde_json::from_slice(&fs::read(status).unwrap()).unwrap(); + let requests = server.requests().len(); + drop(lease); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + assert_eq!(before["active"], serde_json::json!(["retained"]), "{before}"); + assert_eq!(requests, 0, "a retained lease must block another admission"); + assert_eq!(server.requests().len(), 1); +} diff --git a/oxa/oxa-work/src/helper_queue.rs b/oxa/oxa-work/src/helper_queue.rs index 970f4a20..43cf6b7a 100644 --- a/oxa/oxa-work/src/helper_queue.rs +++ b/oxa/oxa-work/src/helper_queue.rs @@ -1,11 +1,11 @@ //! Atomic publication only. The caller must reserve a core claim and authorize the target first. use crate::{board::Policy, Result}; use std::fs::{self, File, OpenOptions}; -use std::io::Write; +use std::io::{Read, Write}; use std::os::unix::fs::{DirBuilderExt, MetadataExt, OpenOptionsExt}; use std::path::{Path, PathBuf}; -const MAX_TASK_BYTES: usize = 64 * 1024; +pub(crate) const MAX_TASK_BYTES: usize = 64 * 1024; fn io(value: std::io::Result) -> Result { value.map_err(|e| format!("board queue storage: {:?}", e.kind())) @@ -95,6 +95,15 @@ pub fn publish_reserved( Err(e) => return io(Err(e)), } } + let native = started.join(format!("native-{prefix}{key}")); + match fs::symlink_metadata(&native) { + Ok(_) => { + directory(&native)?; + return Ok(false); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => (), + Err(error) => return io(Err(error)), + } } if !reserve()? { return Ok(false); @@ -121,3 +130,89 @@ pub fn publish_reserved( io(lock.sync_all())?; Ok(true) } + +/// Take the admitted packet into the existing started directory. The native name +/// keeps retries distinguishable from packets already started by another consumer. +pub fn take_native_packet(queue: &Path, binding: &crate::board::NativeBinding) -> Result { + if crate::board::NativeBinding::new(binding.number, &binding.key, &binding.lease, &binding.branch)? + != *binding + { + return Err("invalid native queue binding".into()); + } + directory(queue)?; + let started = queue.join("started"); + directory(&started)?; + let lock = io(File::open(queue))?; + io(lock.lock())?; + let mut found = None; + for prefix in ["board-", "p1-board-"] { + let name = format!("{prefix}{}.txt", binding.key); + let ready = queue.join(&name); + let active_dir = started.join(format!("native-{prefix}{}", binding.key)); + let active_name = format!("{}.txt", binding.native_task_id); + let active = active_dir.join(&active_name); + match fs::symlink_metadata(&active_dir) { + Ok(_) => { + directory(&active_dir)?; + let mut entries = io(fs::read_dir(&active_dir))?; + if let Some(entry) = io(entries.next().transpose())? { + if entry.file_name() != std::ffi::OsStr::new(&active_name) + || io(entries.next().transpose())?.is_some() + { + return Err("native started packet belongs to another lease generation".into()); + } + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => (), + Err(error) => return io(Err(error)), + } + // A started packet belonging to another consumer is never adopted. + match fs::symlink_metadata(started.join(&name)) { + Ok(_) => return Err("board packet is already started by another consumer".into()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => (), + Err(error) => return io(Err(error)), + } + for (path, renamed) in [(&ready, false), (&active, true)] { + let file = match OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) + .open(path) + { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => return io(Err(error)), + }; + let metadata = io(file.metadata())?; + if !metadata.is_file() + || metadata.uid() != io(lock.metadata())?.uid() + || metadata.nlink() != 1 + || metadata.mode() & 0o077 != 0 + || metadata.len() > MAX_TASK_BYTES as u64 + || found.is_some() + { + return Err("unsafe or ambiguous native queue packet".into()); + } + found = Some((file, ready.clone(), active.clone(), renamed)); + } + } + let (file, ready, active, renamed) = found.ok_or("native queue packet unavailable")?; + let mut packet = String::new(); + io(file.take(MAX_TASK_BYTES as u64 + 1).read_to_string(&mut packet))?; + if packet.len() > MAX_TASK_BYTES { + return Err("native queue packet exceeds capacity".into()); + } + if !renamed { + let active_dir = active.parent().ok_or("missing native started directory")?; + match fs::DirBuilder::new().mode(0o700).create(active_dir) { + Ok(()) => (), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => (), + Err(error) => return io(Err(error)), + } + directory(active_dir)?; + io(fs::rename(ready, &active))?; + io(io(File::open(active_dir))?.sync_all())?; + io(io(File::open(started))?.sync_all())?; + io(lock.sync_all())?; + } + Ok(packet) +} diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index 4655b380..3b8991ed 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -18,8 +18,10 @@ mod execute; pub mod helper_queue; mod notes; mod publish; +mod queue; pub use notes::{ProblemNote, ReportRecord}; -pub use publish::{publish, report_notes}; +pub use publish::{inherit_report_lease, publish, report_notes}; +pub use queue::{native_task, run_queue, NativeQueueOwner, QueueEvent, QueueOptions, QueueRun, QueueStatus}; pub mod mirrors; pub mod shadow; mod state; @@ -105,6 +107,27 @@ pub struct Record { #[serde(default)] pub acceptance_outcome: Option, } +impl Record { + pub fn failed(&self) -> bool { + matches!( + self.outcome, + Outcome::Red | Outcome::Blocked | Outcome::Interrupted + ) || self.reports.iter().any(|report| report.disposition == "failed") + } +} + +pub(crate) fn worktree_root(common: &Path) -> Result { + let repo_dir = if common.file_name().is_some_and(|name| name == ".git") { + common.parent().ok_or("git directory has no parent")? + } else { + common + }; + let parent = repo_dir + .parent() + .ok_or("repository has no parent for worktrees")?; + let name = repo_dir.file_name().ok_or("repository has no directory name")?; + Ok(parent.join(".oxa-worktrees").join(name)) +} /// An immutable task/base/check snapshot, built before any task in its batch runs. pub struct PlannedTask { @@ -124,6 +147,10 @@ impl PlannedTask { } pub fn plan_task(repo: &Path, task: Task, policy: &RunPolicy) -> Result { + plan_new_task(repo, task, policy, false) +} + +fn plan_new_task(repo: &Path, task: Task, policy: &RunPolicy, native: bool) -> Result { validate(&task)?; let branch = checked(git(policy, repo, &["check-ref-format", "--branch", &task.branch]))?; if branch != task.branch { @@ -140,9 +167,9 @@ pub fn plan_task(repo: &Path, task: Task, policy: &RunPolicy) -> Result Result Result { - plan_task_mode(repo, task, policy, rerun, false) + plan_task_mode(repo, task, policy, rerun, false, false) } /// Explicit actions reuse completed execution inputs, including red results, @@ -169,7 +196,7 @@ pub fn plan_task_for_actions( policy: &RunPolicy, rerun: bool, ) -> Result { - plan_task_mode(repo, task, policy, rerun, true) + plan_task_mode(repo, task, policy, rerun, true, false) } fn completed_execution(step: Step) -> bool { @@ -182,10 +209,11 @@ fn plan_task_mode( policy: &RunPolicy, rerun: bool, actions: bool, + native: bool, ) -> Result { validate(&task)?; let Some(saved) = load_task_checkpoint(policy, &task.id)? else { - return plan_task(repo, task, policy); + return plan_new_task(repo, task, policy, native); }; if saved.task != task { return Err("task specification changed; use a new task id".into()); @@ -206,7 +234,7 @@ fn plan_task_mode( base_alias: None, }); } - plan_task(repo, task, policy) + plan_new_task(repo, task, policy, native) } /// Freeze a batch before callbacks, reconciling a literal local base with its @@ -238,7 +266,7 @@ fn plan_tasks( ) -> Vec> { let mut plans: Vec<_> = tasks .iter() - .map(|task| plan_task_mode(repo, task.clone(), policy, rerun, actions)) + .map(|task| plan_task_mode(repo, task.clone(), policy, rerun, actions, false)) .collect(); let mut targets = std::collections::BTreeMap::>>::new(); @@ -518,8 +546,12 @@ pub fn metadata_git(binary: &Path, path: &std::ffi::OsStr, repo: &Path, args: &[ } pub fn read_tasks(path: &Path) -> Result> { - let tasks: Vec = serde_json::from_slice(&fs::read(path).map_err(|e| e.to_string())?) - .map_err(|e| format!("task file: {e}"))?; + let bytes = fs::read(path).map_err(|e| e.to_string())?; + let tasks = serde_json::from_slice(&bytes).map_err(|e| format!("task file: {e}"))?; + validate_tasks(tasks) +} + +fn validate_tasks(tasks: Vec) -> Result> { if tasks.is_empty() { return Err("task file is empty".into()); } @@ -1011,37 +1043,101 @@ pub fn run_task_with_notes( observer: &dyn CheckpointObserver, scrub_note: &(dyn Fn(&ProblemNote) -> Option + Sync), ) -> std::result::Result { - let task = &planned.task; - validate(task)?; + LockedTask::acquire(planned.task(), policy)? + .run_with_notes(repo, binary, planned, rerun, observer, scrub_note) +} + +/// Own the existing task lock across execution and its explicit terminal actions. +pub struct LockedTask<'a> { + task: &'a Task, + policy: &'a RunPolicy, + lock: fs::File, +} + +impl<'a> LockedTask<'a> { + pub fn acquire(task: &'a Task, policy: &'a RunPolicy) -> Result { + Self::open(task, policy, true) + } + + fn open(task: &'a Task, policy: &'a RunPolicy, create: bool) -> Result { + validate(task)?; + if create { + fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(&policy.storage) + .map_err(|error| format!("task storage: {error}"))?; + } + verify_task_storage(&policy.storage).map_err(|error| format!("task storage: {error}"))?; + let lock_path = policy.storage.join(format!("{}.lock", task.id)); + let lock = fs::OpenOptions::new() + .read(true) + .write(true) + .mode(0o600) + .custom_flags(shadow::OWNER_FILE_FLAGS) + .create(create) + .truncate(false) + .open(&lock_path) + .map_err(|e| format!("cannot lock task {}: {e}", task.id))?; + let lock_metadata = lock + .metadata() + .map_err(|e| format!("cannot inspect task lock: {e}"))?; + // SAFETY: geteuid has no preconditions and returns this process's effective owner. + if !shadow::regular_file_owned_by(&lock_metadata, unsafe { libc::geteuid() }) { + return Err("cannot lock task: lock must be an owner-controlled regular file".into()); + } + lock.try_lock() + .map_err(|e| format!("task {} is already running: {e}", task.id))?; + Ok(Self { task, policy, lock }) + } + + /// Plan the task admitted by the native queue while retaining its lock. + pub fn plan_native(&self, repo: &Path, actions: bool) -> Result { + if !self.task.id.starts_with("board-") || !self.task.runner_owns_github() { + return Err("native planning requires an admitted board task".into()); + } + plan_task_mode(repo, self.task.clone(), self.policy, false, actions, true) + } + + pub fn run_with_notes( + &mut self, + repo: &Path, + binary: &Path, + planned: &PlannedTask, + rerun: bool, + observer: &dyn CheckpointObserver, + scrub_note: &(dyn Fn(&ProblemNote) -> Option + Sync), + ) -> std::result::Result { + run_locked(repo, binary, planned, self, rerun, observer, scrub_note) + } +} + +fn run_locked( + repo: &Path, + binary: &Path, + planned: &PlannedTask, + locked: &LockedTask<'_>, + rerun: bool, + observer: &dyn CheckpointObserver, + scrub_note: &(dyn Fn(&ProblemNote) -> Option + Sync), +) -> std::result::Result { + let (task, policy, lock_file) = (locked.task, locked.policy, &locked.lock); + if task != &planned.task { + return Err("task specification changed".into()); + } let root = PathBuf::from(checked(git(policy, repo, &["rev-parse", "--show-toplevel"]))?); let common = PathBuf::from(checked(git( policy, &root, &["rev-parse", "--path-format=absolute", "--git-common-dir"], ))?); - fs::DirBuilder::new() - .recursive(true) - .mode(0o700) - .create(&policy.storage) - .and_then(|()| verify_task_storage(&policy.storage)) - .map_err(|error| format!("task storage: {error}"))?; - let lock_file = lock_task(policy, &task.id)?; // The inherited lock retains its inode; status never tries to acquire it. let receipt = policy.storage.join(format!("{}.json", task.id)); let replacing = receipt.exists(); if planned.checkpoint_revision.is_some() && !replacing { return Err("task checkpoint disappeared after planning; retry the task file".into()); } - let repo_dir = if common.file_name().is_some_and(|name| name == ".git") { - common.parent().ok_or("git directory has no parent")? - } else { - &common - }; - let parent = repo_dir - .parent() - .ok_or("repository has no parent for worktrees")?; - let name = repo_dir.file_name().ok_or("repository has no directory name")?; - let trees = parent.join(".oxa-worktrees").join(name); + let trees = worktree_root(&common)?; let tree = trees.join(&task.id); let fresh = |revision| { let mut record = blocked_record(task, String::new()); @@ -1111,7 +1207,7 @@ pub fn run_task_with_notes( repo: &root, binary, policy: &policy, - lock: &lock_file, + lock: lock_file, writer: &mut writer, observer, scrub_note, @@ -1143,7 +1239,7 @@ pub fn run_task_with_notes( writer .save(&mut state, Step::Saving, observer) .map_err(RunError::Checkpoint)?; - if let Err(error) = execute::save_work(&root, binary, task, &policy, &lock_file, &mut state.record) { + if let Err(error) = execute::save_work(&root, binary, task, &policy, lock_file, &mut state.record) { if state.record.outcome != Outcome::Interrupted { state.record.outcome = Outcome::Blocked; } @@ -1173,7 +1269,7 @@ pub fn run_task_with_notes( if recover_accepted_save { // A failed or interrupted save can leave a worktree that changed since // acceptance. Qualify the exact recovered commit before restoring success. - match committed::verify_commit(&root, binary, &policy, &state, &lock_file) { + match committed::verify_commit(&root, binary, &policy, &state, lock_file) { Ok(_) => { let initial = state.execution.as_ref().and_then(|point| point.initial.as_ref()); state.record.outcome = if state.record.head_sha.as_ref() == initial { diff --git a/oxa/oxa-work/src/publish.rs b/oxa/oxa-work/src/publish.rs index 5d230cc9..256718b9 100644 --- a/oxa/oxa-work/src/publish.rs +++ b/oxa/oxa-work/src/publish.rs @@ -1,43 +1,25 @@ //! Explicit publishing of the accepted commit; existing queued PRs are never pushed. -use super::{checked, state, tree, CheckpointObserver, Outcome, Record, Result, RunPolicy, Task}; +use super::{checked, state, tree, CheckpointObserver, LockedTask, Outcome, Record, Result, RunPolicy, Task}; use state::Step; -use std::os::unix::fs::OpenOptionsExt; +use std::os::{fd::AsRawFd, unix::process::CommandExt}; use std::{ fs::File, path::{Path, PathBuf}, process::{Command, Stdio}, }; -fn context( - task: &Task, - policy: &RunPolicy, +fn action_context( + locked: &LockedTask<'_>, observer: &dyn CheckpointObserver, -) -> Result<(state::Writer, File, state::State)> { - super::validate(task)?; - super::verify_task_storage(&policy.storage).map_err(|e| format!("task storage: {e}"))?; - let path = policy.storage.join(format!("{}.json", task.id)); - let lock = std::fs::OpenOptions::new() - .read(true) - .write(true) - .custom_flags(super::shadow::OWNER_FILE_FLAGS) - .open(path.with_extension("lock")) - .map_err(|e| e.to_string())?; - let metadata = lock - .metadata() - .map_err(|e| format!("cannot inspect task lock: {e}"))?; - // SAFETY: geteuid has no arguments or preconditions. - if !super::shadow::regular_file_owned_by(&metadata, unsafe { libc::geteuid() }) { - return Err("cannot lock task: lock must be an owner-controlled regular file".into()); - } - lock.try_lock() - .map_err(|e| format!("task is already running: {e}"))?; +) -> Result<(state::Writer, state::State)> { + let path = locked.policy.storage.join(format!("{}.json", locked.task.id)); let value = state::load(&path)?; - if value.task != *task { + if value.task != *locked.task { return Err("task specification changed".into()); } let mut writer = state::Writer::new(path, true); writer.replay(&value, observer)?; - Ok((writer, lock, value)) + Ok((writer, value)) } const MAX_PUBLICATION_RESPONSE_BYTES: usize = 1024 * 1024; @@ -155,7 +137,19 @@ pub fn publish( observer: &dyn CheckpointObserver, confirm: &mut dyn FnMut(&str) -> Option, ) -> Result { - let (mut writer, lock, mut value) = context(task, policy, observer)?; + LockedTask::open(task, policy, false)?.publish(repo, binary, gh_binary, observer, confirm) +} + +fn publish_locked( + repo: &Path, + binary: &Path, + locked: &LockedTask<'_>, + gh_binary: &Path, + observer: &dyn CheckpointObserver, + confirm: &mut dyn FnMut(&str) -> Option, +) -> Result { + let (mut writer, mut value) = action_context(locked, observer)?; + let (task, policy, lock) = (locked.task, locked.policy, &locked.lock); let accepted = value.record.acceptance_outcome.unwrap_or(value.record.outcome); if !super::completed_execution(value.record.step) || !matches!(accepted, Outcome::Done | Outcome::NothingToDo) @@ -172,14 +166,14 @@ pub fn publish( } let tree = &value.record.worktree; tree::validate(policy, repo, tree, task)?; - if super::dirty_worktree(binary, tree, repo, policy, &lock)? { + if super::dirty_worktree(binary, tree, repo, policy, lock)? { return Err("uncommitted work must be saved before publishing".into()); } let head = checked(git(tree, &["rev-parse", "HEAD"]))?; if value.record.head_sha.as_deref() != Some(head.as_str()) { return Err("task branch changed after acceptance; use --rerun before publishing".into()); } - let proof = super::committed::verify_commit(repo, binary, policy, &value, &lock)?; + let proof = super::committed::verify_commit(repo, binary, policy, &value, lock)?; let accepted_base = value .record .base_sha @@ -195,7 +189,7 @@ pub fn publish( } let repository = repository(&json(&gh( tree, - &lock, + lock, &["repo", "view", &remote, "--json", "nameWithOwner,url"], )?)?)?; let name = repository.qualified.as_str(); @@ -203,7 +197,7 @@ pub fn publish( let repo_name = repository.name.as_str(); let prs = json(&gh( tree, - &lock, + lock, &[ "pr", "list", @@ -237,7 +231,7 @@ pub fn publish( let number = pr["number"].as_u64().ok_or("PR number missing")?.to_string(); let value = json(&gh( tree, - &lock, + lock, &["pr", "view", &number, "--repo", name, "--json", "body"], )?)?; Some(value["body"].as_str().ok_or("PR body missing")?.to_owned()) @@ -264,7 +258,7 @@ pub fn publish( let query = "query($owner:String!,$name:String!,$number:Int!){repository(owner:$owner,name:$name){pullRequest(number:$number){mergeQueueEntry{id}}}}"; let data = json(&gh( tree, - &lock, + lock, &[ "api", "graphql", @@ -329,7 +323,7 @@ pub fn publish( let number = pr["number"].as_u64().ok_or("PR number missing")?.to_string(); gh( tree, - &lock, + lock, &[ "pr", "edit", @@ -348,7 +342,7 @@ pub fn publish( if draft { args.push("--undo"); } - gh(tree, &lock, &args)?; + gh(tree, lock, &args)?; } pr["url"].as_str().ok_or("PR URL missing")?.to_string() } else { @@ -370,7 +364,7 @@ pub fn publish( if draft { args.push("--draft"); } - gh(tree, &lock, &args)? + gh(tree, lock, &args)? }; let _ = std::fs::remove_file(body); Ok((result.trim().to_string(), draft)) @@ -391,13 +385,38 @@ pub fn publish( } Ok(value.record) } +/// Keep a report's per-user lease held if the controller dies during a host command. +pub fn inherit_report_lease(command: &mut Command, gate: &File) -> std::io::Result<()> { + let gate = gate.try_clone()?; + // SAFETY: the closure owns the descriptor, and only changes its flags + // in this child. fcntl is async-signal-safe. + unsafe { + command.pre_exec(move || { + if libc::fcntl(gate.as_raw_fd(), libc::F_SETFD, 0) == -1 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + Ok(()) +} + pub fn report_notes( task: &Task, policy: &RunPolicy, observer: &dyn CheckpointObserver, reporter: &mut dyn FnMut(&super::ProblemNote, &File, bool) -> Result, ) -> Result { - let (mut writer, lock, mut value) = context(task, policy, observer)?; + LockedTask::open(task, policy, false)?.report_notes(observer, reporter) +} + +fn report_locked( + locked: &LockedTask<'_>, + observer: &dyn CheckpointObserver, + reporter: &mut dyn FnMut(&super::ProblemNote, &File, bool) -> Result, +) -> Result { + let (mut writer, mut value) = action_context(locked, observer)?; + let (policy, lock) = (locked.policy, &locked.lock); let prior_step = value.record.step; // Reporting is ancillary: a failed save/publication remains recoverable even // if reporting is interrupted between durable report receipts. @@ -416,7 +435,7 @@ pub fn report_notes( } writer.save(&mut value, reporting_step, observer)?; let report = - reporter(&value.record.problem_notes[index], &lock, policy.offline).unwrap_or_else(|detail| { + reporter(&value.record.problem_notes[index], lock, policy.offline).unwrap_or_else(|detail| { super::ReportRecord { disposition: "failed".into(), draft: PathBuf::new(), @@ -442,3 +461,24 @@ pub fn report_notes( )?; Ok(value.record) } + +impl LockedTask<'_> { + pub fn publish( + &mut self, + repo: &Path, + binary: &Path, + gh_binary: &Path, + observer: &dyn CheckpointObserver, + confirm: &mut dyn FnMut(&str) -> Option, + ) -> Result { + publish_locked(repo, binary, self, gh_binary, observer, confirm) + } + + pub fn report_notes( + &mut self, + observer: &dyn CheckpointObserver, + reporter: &mut dyn FnMut(&super::ProblemNote, &File, bool) -> Result, + ) -> Result { + report_locked(self, observer, reporter) + } +} diff --git a/oxa/oxa-work/src/queue.rs b/oxa/oxa-work/src/queue.rs new file mode 100644 index 00000000..28a64cd9 --- /dev/null +++ b/oxa/oxa-work/src/queue.rs @@ -0,0 +1,826 @@ +//! Directory queue: oldest regular JSON files first, with bounded independent workers. +use super::{blocked_record, board, checked, state, LockedTask, Record, Result, RunPolicy, Task}; +use serde::{ + de::{Error, IgnoredAny, SeqAccess, Visitor}, + Deserialize, Deserializer, Serialize, +}; +use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt}; +use std::{ + collections::{HashMap, HashSet}, + fs, + io::{self, Read}, + path::{Path, PathBuf}, + process::Command, + sync::{mpsc, Arc}, + thread, + time::{Duration, SystemTime}, +}; + +mod native; +pub use native::{native_task, NativeQueueOwner}; + +#[derive(Clone, Debug)] +pub struct QueueOptions { + pub max: usize, + pub min_free_bytes: u64, + pub once: bool, +} +impl Default for QueueOptions { + fn default() -> Self { + Self { + max: 1, + min_free_bytes: 512 * 1024 * 1024, + once: false, + } + } +} +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +pub struct QueueStatus { + pub time: u64, + pub active: Vec, + pub pending: usize, + pub finished: usize, + pub failed: usize, + pub free_bytes: Option, + pub disk_blocked: bool, + pub disk_error: Option, +} +#[derive(Serialize)] +#[serde(tag = "event", rename_all = "snake_case")] +pub enum QueueEvent { + State { + #[serde(flatten)] + status: QueueStatus, + }, + Task { + #[serde(flatten)] + record: Record, + }, + Error { + file: PathBuf, + detail: String, + }, +} +#[derive(Clone)] +struct Job { + task: Task, + modified: SystemTime, + file: PathBuf, + index: usize, + binding: Option, +} + +type WorkerFn = dyn Fn(&Task) -> Result + Send + Sync; +type NativeWorkerFn = dyn Fn(&Task, &mut LockedTask<'_>) -> Result + Send + Sync; + +#[derive(Clone)] +enum QueueWorker { + Manual(Arc), + Native(Arc), +} + +struct NativeWorker { + claims: board::Claims, + worker_id: String, + policy: RunPolicy, + run: Arc, +} + +impl QueueWorker { + fn run(&self, job: &mut Job) -> Result { + let worker = match self { + Self::Manual(run) => return run(&job.task), + Self::Native(worker) => worker, + }; + let mut locked = LockedTask::acquire(&job.task, &worker.policy)?; + let owner = NativeQueueOwner { + claims: &worker.claims, + worker_id: &worker.worker_id, + }; + job.binding = Some(owner.admit(&job.task)?); + (worker.run)(&job.task, &mut locked) + } +} +fn busy(storage: &Path, id: &str) -> Result { + match fs::OpenOptions::new() + .read(true) + .write(true) + .open(storage.join(format!("{id}.lock"))) + { + Ok(file) => match file.try_lock() { + Ok(()) => Ok(false), + Err(fs::TryLockError::WouldBlock) => Ok(true), + Err(error) => Err(error.to_string()), + }, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error.to_string()), + } +} +fn available(path: &Path, df: &Path, policy: &RunPolicy) -> Result { + let mut command = Command::new(df); + super::isolate_git_environment(&mut command); + command.env("PATH", &policy.host_path); + command.args(["--output=avail", "-B1", "--"]).arg(path); + checked(command)? + .lines() + .last() + .ok_or("df returned no free-space value")? + .trim() + .parse() + .map_err(|_| "df returned an invalid free-space value".into()) +} +fn free_space(repo: &Path, common: &Path, queue: &Path, df: &Path, policy: &RunPolicy) -> Result { + let trees = super::worktree_root(common)?; + let tree_volume = trees + .ancestors() + .find(|p| p.is_dir()) + .ok_or("worktree storage has no existing parent")?; + Ok(available(repo, df, policy)? + .min(available(common, df, policy)?) + .min(available(&policy.storage, df, policy)?) + .min(available(queue, df, policy)?) + .min(available(tree_volume, df, policy)?)) +} +const MAX_INPUT_BYTES: u64 = 1024 * 1024; +const MAX_INPUT_TASKS: usize = 128; +const MAX_QUEUE_FILES: usize = 256; +const MAX_QUEUE_BYTES: u64 = 16 * 1024 * 1024; +const MAX_RETAINED_TASKS: usize = 4096; + +fn task_bytes(task: &Task) -> u64 { + (task.id.len() + task.branch.len() + task.base.len() + task.model.len() + task.prompt.len()) as u64 +} + +struct QueueTasks(Vec); +impl<'de> Deserialize<'de> for QueueTasks { + fn deserialize>(deserializer: D) -> std::result::Result { + struct TasksVisitor; + impl<'de> Visitor<'de> for TasksVisitor { + type Value = QueueTasks; + + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "an array of at most {MAX_INPUT_TASKS} tasks") + } + + fn visit_seq>(self, mut seq: A) -> std::result::Result { + let mut tasks = Vec::new(); + while tasks.len() < MAX_INPUT_TASKS { + let Some(task) = seq.next_element()? else { + return Ok(QueueTasks(tasks)); + }; + tasks.push(task); + } + if seq.next_element::()?.is_some() { + return Err(A::Error::custom(format!( + "task file exceeds {MAX_INPUT_TASKS} tasks" + ))); + } + Ok(QueueTasks(tasks)) + } + } + deserializer.deserialize_seq(TasksVisitor) + } +} + +fn read_input(path: &Path) -> Result>> { + // A queued file may be replaced after the directory scan. Open the final + // component without following links or blocking on a substituted FIFO. + let file = match fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) + .open(path) + { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.to_string()), + }; + let meta = file.metadata().map_err(|e| e.to_string())?; + if !meta.is_file() { + return Err("queue input must be a regular file".into()); + } + let too_large = || format!("task file exceeds {MAX_INPUT_BYTES} bytes"); + if meta.len() > MAX_INPUT_BYTES { + return Err(too_large()); + } + let mut bytes = Vec::new(); + file.take(MAX_INPUT_BYTES + 1) + .read_to_end(&mut bytes) + .map_err(|e| e.to_string())?; + if bytes.len() as u64 > MAX_INPUT_BYTES { + return Err(too_large()); + } + let QueueTasks(tasks) = serde_json::from_slice(&bytes).map_err(|e| format!("task file: {e}"))?; + super::validate_tasks(tasks).map(Some) +} + +fn input_file(entry: fs::DirEntry) -> Result> { + let path = entry.path(); + if path + .file_name() + .is_some_and(|s| s.to_string_lossy().starts_with('.')) + || path.extension().is_none_or(|s| s != "json") + { + return Ok(None); + } + // A user may archive an input after read_dir produced this entry. + let meta = match entry.metadata() { + Ok(meta) => meta, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.to_string()), + }; + if !meta.is_file() { + return Ok(None); + } + Ok(Some(( + path, + meta.modified().map_err(|e| e.to_string())?, + meta.len(), + ))) +} +#[derive(Default)] +struct Inputs { + files: HashMap, + tasks: HashMap, + errors: HashMap>, + branches: HashSet, + task_bytes: u64, +} +fn scan( + directory: &Path, + pending: &mut Vec, + inputs: &mut Inputs, + emit: &mut dyn FnMut(QueueEvent) -> Result<()>, +) -> Result<()> { + let mut files = vec![]; + let mut bytes = 0; + for entry in fs::read_dir(directory).map_err(|e| e.to_string())? { + let entry = match entry { + Ok(entry) => entry, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => return Err(error.to_string()), + }; + if let Some(file) = input_file(entry)? { + if files.len() == MAX_QUEUE_FILES { + return Err(format!( + "queue exceeds {MAX_QUEUE_FILES} task files; archive old inputs" + )); + } + // Oversized files are reported individually by read_input. + bytes += file.2.min(MAX_INPUT_BYTES); + if bytes > MAX_QUEUE_BYTES { + return Err("queue exceeds 16 MiB of task inputs; archive old inputs".into()); + } + files.push(file); + } + } + let paths: HashSet<_> = files.iter().map(|(p, _, _)| p.clone()).collect(); + let withdrawn: HashSet<_> = pending + .iter() + .filter(|job| !paths.contains(&job.file)) + .map(|job| job.task.id.clone()) + .collect(); + pending.retain(|job| paths.contains(&job.file)); + // Active/external workers are not pending and retain their identities. + for id in withdrawn { + if let Some((_, task)) = inputs.tasks.remove(&id) { + inputs.branches.remove(&task.branch); + inputs.task_bytes -= task_bytes(&task); + } + } + inputs.files.retain(|p, _| paths.contains(p)); + inputs.errors.retain(|p, _| paths.contains(p)); + files.sort_by(|a, b| (&a.1, &a.0).cmp(&(&b.1, &b.0))); + for (file, modified, len) in files { + if inputs.files.get(&file) == Some(&(modified, len)) && !inputs.errors.contains_key(&file) { + continue; + } + let tasks = match read_input(&file) { + Ok(Some(tasks)) => Ok(tasks), + Ok(None) => continue, + Err(error) => Err(error), + }; + let previous = inputs.errors.remove(&file); + let mut problems = vec![]; + match tasks { + Ok(tasks) => { + for (index, task) in tasks.into_iter().enumerate() { + if let Some((source, known)) = inputs.tasks.get_mut(&task.id) { + if *known == task && (*source == file || !paths.contains(source)) { + *source = file.clone(); + continue; + } + problems.push(format!("task {} was already queued; use a new id for changed work or remove the duplicate input", task.id)); + } else { + if inputs.branches.contains(&task.branch) { + problems.push(format!( + "branch {} was already queued; use a distinct branch", + task.branch + )); + continue; + } + let bytes = task_bytes(&task); + if inputs.tasks.len() == MAX_RETAINED_TASKS + || inputs.task_bytes + bytes > MAX_QUEUE_BYTES + { + problems.push("queue task identity capacity reached; archive completed inputs and restart the controller".into()); + break; + } + inputs.branches.insert(task.branch.clone()); + inputs.task_bytes += bytes; + inputs.tasks.insert(task.id.clone(), (file.clone(), task.clone())); + pending.push(Job { + task, + file: file.clone(), + modified, + index, + binding: None, + }); + } + } + } + Err(detail) => problems.push(detail), + } + inputs.files.insert(file.clone(), (modified, len)); + if !problems.is_empty() { + if previous.as_ref() != Some(&problems) { + for detail in &problems { + emit(QueueEvent::Error { + file: file.clone(), + detail: detail.clone(), + })?; + } + } + inputs.errors.insert(file, problems); + } + } + Ok(()) +} +#[derive(Default)] +struct Workers { + handles: HashMap>, +} +impl Drop for Workers { + fn drop(&mut self) { + // Every error path drains admitted work before the CLI can exit. + for (_, handle) in self.handles.drain() { + let _ = handle.join(); + } + } +} + +fn orphan_leases(storage: &Path, active: &Workers, external: &[Job]) -> Result> { + let mut ids = vec![]; + for entry in fs::read_dir(storage).map_err(|e| e.to_string())? { + let entry = entry.map_err(|e| e.to_string())?; + let name = entry.file_name(); + let Some(id) = name.to_str().and_then(|name| name.strip_suffix(".lock")) else { + continue; + }; + if super::valid_task_id(id) + && !active.handles.contains_key(id) + && !external.iter().any(|job| job.task.id == id) + && busy(storage, id)? + { + ids.push(id.to_owned()); + } + } + Ok(ids) +} + +pub fn run_queue( + repo: &Path, + directory: &Path, + options: &QueueOptions, + policy: &RunPolicy, + df: &Path, + worker: Arc, + emit: &mut dyn FnMut(QueueEvent) -> Result<()>, +) -> Result { + run_queue_inner( + QueueRun { + repo, + directory, + options, + policy, + df, + }, + None, + None, + QueueWorker::Manual(worker), + emit, + ) +} + +/// The same execution queue with trusted owner context for a native admitted attempt. +pub struct QueueRun<'a> { + pub repo: &'a Path, + pub directory: &'a Path, + pub options: &'a QueueOptions, + pub policy: &'a RunPolicy, + pub df: &'a Path, +} + +impl QueueRun<'_> { + pub fn run_native( + self, + owner: &NativeQueueOwner<'_>, + worker: Arc, + emit: &mut dyn FnMut(QueueEvent) -> Result<()>, + ) -> Result { + self.run_native_with(owner, None, worker, emit) + } + + /// Consume one protected spool packet in memory using the existing queue and worker lifecycle. + pub fn run_native_packet( + self, + owner: &NativeQueueOwner<'_>, + binding: &board::NativeBinding, + worker: Arc, + emit: &mut dyn FnMut(QueueEvent) -> Result<()>, + ) -> Result { + if !self.options.once { + return Err("native packet execution requires a one-shot queue".into()); + } + if binding.lease.holder != owner.worker_id { + return Err("native packet belongs to a different worker".into()); + } + let packet = crate::helper_queue::take_native_packet(self.directory, binding)?; + let task = native::from_packet(binding, &packet)?; + self.run_native_with(owner, Some(task), worker, emit) + } + + fn run_native_with( + self, + owner: &NativeQueueOwner<'_>, + prepared: Option, + worker: Arc, + emit: &mut dyn FnMut(QueueEvent) -> Result<()>, + ) -> Result { + let worker = QueueWorker::Native(Arc::new(NativeWorker { + claims: owner.claims.clone(), + worker_id: owner.worker_id.into(), + policy: self.policy.clone(), + run: worker, + })); + run_queue_inner(self, Some(owner), prepared, worker, emit) + } +} + +fn run_queue_inner( + queue: QueueRun<'_>, + native: Option<&NativeQueueOwner<'_>>, + prepared: Option, + worker: QueueWorker, + emit: &mut dyn FnMut(QueueEvent) -> Result<()>, +) -> Result { + let QueueRun { + repo, + directory, + options, + policy, + df, + } = queue; + if options.max == 0 { + return Err("--max must be positive".into()); + } + let directory = directory.canonicalize().map_err(|e| e.to_string())?; + if !directory.is_dir() { + return Err("queue path is not a directory".into()); + } + let _controller = state::lock(&directory.join(".oxa-queue.lock"))?; + let common = PathBuf::from(checked(super::git( + policy, + repo, + &["rev-parse", "--path-format=absolute", "--git-common-dir"], + ))?); + let storage = &policy.storage; + fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(storage) + .map_err(|e| e.to_string())?; + let (tx, rx) = mpsc::channel(); + let mut active = Workers::default(); + let read_directory = prepared.is_none(); + let mut pending: Vec<_> = prepared + .into_iter() + .map(|task| Job { + task, + modified: SystemTime::UNIX_EPOCH, + file: directory.clone(), + index: 0, + binding: None, + }) + .collect(); + let (mut external, mut inputs) = (vec![], Inputs::default()); + let (mut finished, mut failed, mut last) = (0, 0, None); + loop { + if read_directory { + scan(&directory, &mut pending, &mut inputs, emit)?; + } + // Surviving workers from a dead controller still count toward the concurrency cap. + let mut waiting = vec![]; + for job in external.drain(..) { + let job: Job = job; + if busy(&storage, &job.task.id)? { + waiting.push(job); + } else { + pending.push(job); + } + } + external = waiting; + let mut ready = vec![]; + for job in pending.drain(..) { + if busy(&storage, &job.task.id)? { + external.push(job); + } else { + ready.push(job); + } + } + pending = ready; + let orphans = orphan_leases(storage, &active, &external)?; + pending.sort_by(|a, b| (&a.modified, &a.file, a.index).cmp(&(&b.modified, &b.file, b.index))); + let disk = free_space(repo, &common, &directory, df, policy); + let (mut free_bytes, mut disk_error) = match disk { + Ok(free) => (Some(free), None), + Err(error) => (None, Some(error)), + }; + let mut disk_blocked = free_bytes.is_none_or(|free| free < options.min_free_bytes); + while !disk_blocked + && active.handles.len() + external.len() + orphans.len() < options.max + && !pending.is_empty() + { + // Check again for each admission: earlier workers may already be writing. + match free_space(repo, &common, &directory, df, policy) { + Ok(free) => { + free_bytes = Some(free); + disk_error = None; + disk_blocked = free < options.min_free_bytes; + } + Err(error) => { + free_bytes = None; + disk_error = Some(error); + disk_blocked = true; + } + } + if disk_blocked { + break; + } + let job = pending.remove(0); + let id = job.task.id.clone(); + let worker = worker.clone(); + let tx = tx.clone(); + let mut input = job.clone(); + match thread::Builder::new() + .name(format!("oxa-work-{id}")) + .spawn(move || { + let record = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| worker.run(&mut input))) + .unwrap_or_else(|_| { + Ok(blocked_record( + &input.task, + "queue worker panicked; retained state can be resumed".into(), + )) + }); + let _ = tx.send((input, record)); + }) { + Ok(handle) => { + active.handles.insert(id, handle); + } + Err(error) => { + finished += 1; + failed += 1; + emit(QueueEvent::Task { + record: blocked_record(&job.task, error.to_string()), + })?; + } + } + } + let mut ids: Vec<_> = active + .handles + .keys() + .cloned() + .chain(external.iter().map(|j| j.task.id.clone())) + .chain(orphans.iter().cloned()) + .collect(); + ids.sort(); + let status = QueueStatus { + time: state::now_ms() / 1000, + active: ids, + pending: pending.len(), + finished, + failed: failed + inputs.errors.values().map(Vec::len).sum::(), + free_bytes, + disk_blocked: disk_blocked && !pending.is_empty(), + disk_error, + }; + if last.as_ref() != Some(&status) { + let mut bytes = serde_json::to_vec(&QueueEvent::State { + status: status.clone(), + }) + .map_err(|e| e.to_string())?; + bytes.push(b'\n'); + let status_path = directory.join(".oxa-queue-status.json"); + super::checkpoint::write_atomic(&status_path, &bytes, status_path.exists())?; + emit(QueueEvent::State { + status: status.clone(), + })?; + last = Some(status.clone()); + } + if options.once + && active.handles.is_empty() + && external.is_empty() + && orphans.is_empty() + && (pending.is_empty() || disk_blocked) + { + return Ok(status); + } + if active.handles.is_empty() { + thread::sleep(Duration::from_millis(250)); + continue; + } + match rx.recv_timeout(Duration::from_millis(250)) { + Ok((job, record)) => { + if let Some(handle) = active.handles.remove(&job.task.id) { + let _ = handle.join(); + } + let record = if let Some(native) = native { + match record.and_then(|record| { + let binding = job + .binding + .as_ref() + .ok_or("native admission was not established")?; + native.finish(policy, &job.task, binding, &record)?; + Ok(record) + }) { + Ok(record) => record, + Err(detail) => { + failed += 1; + emit(QueueEvent::Error { + file: job.file, + detail, + })?; + continue; + } + } + } else { + if busy(storage, &job.task.id)? { + external.push(job); + continue; + } + record? + }; + finished += 1; + if record.failed() { + failed += 1; + } + emit(QueueEvent::Task { record })?; + } + Err(mpsc::RecvTimeoutError::Timeout) => {} + Err(error) => return Err(error.to_string()), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn queue_refuses_oversized_inputs_without_admitting_their_tasks() { + let dir = std::env::temp_dir().join(format!( + "oxa-queue-limits-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir(&dir).unwrap(); + let valid = r#"[{"id":"large","branch":"agent/large","model":"fake","prompt":"Synthetic task"}]"#; + let mut oversized = valid.to_owned(); + oversized.push_str(&" ".repeat(1024 * 1024)); + fs::write(dir.join("bytes.json"), oversized).unwrap(); + let many: Vec<_> = (0..129) + .map(|n| { + serde_json::json!({ + "id": format!("task-{n}"), + "branch": format!("agent/task-{n}"), + "model": "fake", + "prompt": "Synthetic task" + }) + }) + .collect(); + fs::write(dir.join("count.json"), serde_json::to_vec(&many).unwrap()).unwrap(); + fs::write(dir.join("outside"), valid.replace("large", "symlink")).unwrap(); + std::os::unix::fs::symlink(dir.join("outside"), dir.join("link.json")).unwrap(); + + let mut pending = vec![]; + let mut inputs = Inputs::default(); + let mut errors = vec![]; + let result = scan(&dir, &mut pending, &mut inputs, &mut |event| { + if let QueueEvent::Error { file, .. } = event { + errors.push(file); + } + Ok(()) + }); + fs::remove_dir_all(&dir).unwrap(); + result.unwrap(); + assert!( + pending.is_empty(), + "oversized and linked inputs must never be admitted" + ); + assert!(errors.contains(&dir.join("bytes.json"))); + assert!(errors.contains(&dir.join("count.json"))); + } + + #[test] + fn queue_rejects_an_overfull_directory_before_admission() { + let dir = crate::test_dir::PrivateDir::new("queue-directory-limit"); + for i in 0..257 { + fs::write( + dir.path().join(format!("{i}.json")), + format!(r#"[{{"id":"task-{i}","branch":"agent/task-{i}","model":"fake","prompt":"Synthetic task"}}]"#), + ) + .unwrap(); + } + let mut pending = vec![]; + let mut inputs = Inputs::default(); + let result = scan(dir.path(), &mut pending, &mut inputs, &mut |_| Ok(())); + assert!(result.unwrap_err().contains("256 task files")); + assert!(pending.is_empty()); + assert!(inputs.tasks.is_empty()); + } + + #[test] + fn queue_rejects_aggregate_input_bytes_before_admission() { + let dir = crate::test_dir::PrivateDir::new("queue-input-budget"); + for i in 0..17 { + let mut task = format!( + r#"[{{"id":"task-{i}","branch":"agent/task-{i}","model":"fake","prompt":"Synthetic task"}}]"# + ); + task.push_str(&" ".repeat(1024 * 1024 - task.len())); + fs::write(dir.path().join(format!("{i}.json")), task).unwrap(); + } + let mut pending = vec![]; + let mut inputs = Inputs::default(); + let result = scan(dir.path(), &mut pending, &mut inputs, &mut |_| Ok(())); + assert!(result.unwrap_err().contains("16 MiB")); + assert!(pending.is_empty()); + assert!(inputs.tasks.is_empty()); + } + + #[test] + fn archiving_an_unadmitted_file_cancels_its_pending_job() { + let dir = std::env::temp_dir().join(format!( + "oxa-queue-pending-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir(&dir).unwrap(); + let path = dir.join("task.json"); + fs::write( + &path, + r#"[{"id":"queued","branch":"agent/queued","model":"fake","prompt":"Synthetic task"}]"#, + ) + .unwrap(); + let mut pending = vec![]; + let mut inputs = Inputs::default(); + scan(&dir, &mut pending, &mut inputs, &mut |_| Ok(())).unwrap(); + assert_eq!(pending.len(), 1); + fs::rename(&path, dir.join("archived")).unwrap(); + scan(&dir, &mut pending, &mut inputs, &mut |_| Ok(())).unwrap(); + assert!(pending.is_empty(), "archived input must not be dispatched"); + assert!(!inputs.tasks.contains_key("queued")); + fs::write( + &path, + r#"[{"id":"replacement","branch":"agent/queued","model":"fake","prompt":"Synthetic replacement"}]"#, + ) + .unwrap(); + scan(&dir, &mut pending, &mut inputs, &mut |_| Ok(())).unwrap(); + fs::remove_dir_all(&dir).unwrap(); + assert_eq!(pending.len(), 1); + assert_eq!(pending[0].task.id, "replacement"); + } + #[test] + fn archiving_between_directory_read_and_metadata_is_not_a_controller_error() { + let dir = std::env::temp_dir().join(format!( + "oxa-queue-scan-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir(&dir).unwrap(); + let path = dir.join("task.json"); + fs::write(&path, "[]").unwrap(); + let entry = fs::read_dir(&dir).unwrap().next().unwrap().unwrap(); + fs::rename(path, dir.join("archived")).unwrap(); + let result = input_file(entry); + fs::remove_dir_all(dir).unwrap(); + assert!(result.unwrap().is_none()); + } +} diff --git a/oxa/oxa-work/src/queue/native.rs b/oxa/oxa-work/src/queue/native.rs new file mode 100644 index 00000000..59a7ed23 --- /dev/null +++ b/oxa/oxa-work/src/queue/native.rs @@ -0,0 +1,350 @@ +//! The native queue consumes the original claim receipt and settles after joining the worker. +use super::super::{board, load_task_checkpoint, validate, CheckpointKind, LockedTask, Outcome, Step}; +use super::{Record, Result, RunPolicy, Task}; + +/// Supplied by the trusted queue caller after authenticating its worker. +pub struct NativeQueueOwner<'a> { + pub claims: &'a board::Claims, + pub worker_id: &'a str, +} + +/// Materialize a prepared task with the canonical attempt identity and existing branch. +pub fn native_task(binding: &board::NativeBinding, mut task: Task) -> Result { + let canonical = board::NativeBinding::new(binding.number, &binding.key, &binding.lease, &binding.branch)?; + if binding != &canonical || task.branch != binding.branch { + return Err("prepared task does not own the bound native attempt".into()); + } + task.id.clone_from(&binding.native_task_id); + if !task.runner_owns_github() { + let kind = if task.base == task.branch { + "pull request" + } else { + "issue" + }; + task.prompt.insert_str(0, &crate::github_runner_guidance(kind)); + } + validate(&task)?; + Ok(task) +} + +pub(super) fn from_packet(binding: &board::NativeBinding, packet: &str) -> Result { + let (header, body) = packet.split_once('\n').ok_or("native task packet has no body")?; + let mut fields = header.split_ascii_whitespace(); + let id = fields.next().ok_or("native task packet has no source id")?; + let model = fields.next().ok_or("native task packet has no model")?; + let branch = fields.next().ok_or("native task packet has no branch")?; + let base = fields.next().unwrap_or("main"); + let issue = id == format!("i{}", binding.number); + let change = id == format!("c{}", binding.number); + if fields.next().is_some() + || branch != binding.branch + || !(issue && base == "main" || change && base == branch) + || packet.len() > crate::helper_queue::MAX_TASK_BYTES + || packet.contains('\0') + || body.trim().is_empty() + || body.lines().any(|line| line.trim() == "---") + { + return Err("native task packet disagrees with its bound source".into()); + } + native_task( + binding, + Task { + id: id.into(), + model: model.into(), + branch: branch.into(), + base: base.into(), + prompt: body.into(), + max_time: crate::default_max_time(), + }, + ) +} + +impl NativeQueueOwner<'_> { + pub(super) fn admit(&self, task: &Task) -> Result { + let binding = self + .claims + .native_binding(&task.id, &task.branch)? + .ok_or("native task has no retained admission binding")?; + if self.worker_id != binding.lease.holder || !task.runner_owns_github() { + return Err("native task lacks the authenticated worker or runner-owned GitHub policy".into()); + } + let assignment = self.claims.assignments()?; + if !assignment.get(&binding.number).is_some_and(|row| { + row.state.lease.as_ref() == Some(&binding.lease) + && row.data.native_binding.as_ref() == Some(&binding) + }) { + return Err("native admission no longer owns the original lease".into()); + } + Ok(binding) + } + + pub(super) fn finish( + &self, + policy: &RunPolicy, + task: &Task, + binding: &board::NativeBinding, + observed: &Record, + ) -> Result<()> { + if task.id != binding.native_task_id + || task.branch != binding.branch + || observed.id != task.id + || observed.branch != task.branch + || !finished(observed) + { + return Err("native completion lacks a whole-task finish observation".into()); + } + + // The queue has joined the worker. A surviving child retains this same + // lock and prevents reconciliation; retain it through checkpoint read and CAS. + let _locked = LockedTask::open(task, policy, false)?; + let saved = + load_task_checkpoint(policy, &task.id)?.ok_or("native completion checkpoint is unavailable")?; + if saved.task != *task + || !finished(&saved.record) + || saved.record.checkpoint_revision != observed.checkpoint_revision + || saved.record.checkpoint_at_ms != observed.checkpoint_at_ms + || saved.record.outcome != observed.outcome + || binding + .lease + .claimed_at + .and_then(|at| at.checked_mul(1000)) + .is_none_or(|at| at > saved.record.checkpoint_at_ms) + { + return Err("native completion does not match the current durable checkpoint".into()); + } + if !self.claims.release_for( + binding.number, + self.worker_id, + &binding.lease, + super::state::now_ms() / 1000, + "native task finished and its process exited", + )? { + return Err("native completion no longer owns the original lease".into()); + } + Ok(()) + } +} + +fn finished(record: &Record) -> bool { + record.step == Step::Finished + && record.checkpoint_kind == CheckpointKind::TaskTransition + && record.checkpoint_revision > 0 + && matches!( + record.outcome, + Outcome::Done | Outcome::NothingToDo | Outcome::Red + ) +} + +#[cfg(test)] +#[path = "../../tests/support/private_dir.rs"] +mod test_dir; + +#[cfg(test)] +mod tests { + use super::test_dir::PrivateDir; + use super::*; + use crate::{blocked_record, state, QueueEvent, QueueOptions, QueueRun}; + use board::assignment::{ManualAuthority, Role, Worker, WorkerKind}; + use board::{Action, AttemptState, Claims, Entry, ImportState, Policy}; + use std::collections::BTreeMap; + use std::fs; + use std::path::Path; + use std::process::Command; + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }; + + fn setup() -> (PrivateDir, Claims, board::NativeBinding, Task, RunPolicy) { + let directory = PrivateDir::new("native-completion"); + let claims = Claims::open(directory.path()).unwrap(); + claims + .import_snapshot( + Some(ManualAuthority::Owner), + &BTreeMap::from([( + 1, + ImportState { + state: AttemptState::default(), + excluded: false, + }, + )]), + 10, + ) + .unwrap(); + let worker = Worker { + id: "codex:fixture/worker".into(), + kind: WorkerKind::Codex, + skills: Default::default(), + roles: [Role::Builder].into(), + }; + let entry = Entry { + number: 1, + head: "fixture-head".into(), + release_pinned: false, + action: Action::FixFindings, + lease: None, + attempts_today: 0, + eligible: true, + }; + let lease = claims + .claim_idle_for( + &entry, + &Policy { + enabled: true, + ..Policy::default() + }, + &worker, + 10, + 20, + ) + .unwrap() + .unwrap(); + let binding = board::NativeBinding::new(1, &"a".repeat(64), &lease, "agent/fixture").unwrap(); + assert!(claims.bind_native(&worker.id, &binding).unwrap()); + let prepared = Task { + id: "c1".into(), + model: "fake".into(), + branch: binding.branch.clone(), + base: "main".into(), + prompt: "Synthetic task".into(), + max_time: 30, + }; + let policy = RunPolicy { + git_binary: "/usr/bin/git".into(), + host_path: "/usr/bin".into(), + terminate_binary: None, + offline: true, + runner_owns_github: false, + storage: directory.path().join("runner"), + }; + (directory, claims, binding, prepared, policy) + } + + #[test] + fn only_the_exact_durable_whole_task_after_lock_release_settles_the_original_claim() { + let (_directory, claims, binding, prepared, policy) = setup(); + let owner = NativeQueueOwner { + claims: &claims, + worker_id: &binding.lease.holder, + }; + assert!(owner.admit(&prepared).is_err()); + let task = native_task(&binding, prepared).unwrap(); + assert!(owner.admit(&task).unwrap() == binding); + let locked = LockedTask::acquire(&task, &policy).unwrap(); + let synthetic = blocked_record(&task, "synthetic record".into()); + assert!(owner.finish(&policy, &task, &binding, &synthetic).is_err()); + + let mut saved = state::State::new(task.clone(), synthetic, true); + let mut writer = state::Writer::new(policy.storage.join(format!("{}.json", task.id)), false); + let observe = |_: &Record| Ok(()); + saved.record.outcome = Outcome::Done; + writer + .save_kind( + &mut saved, + Step::Acceptance, + CheckpointKind::WorkerFinished, + &observe, + ) + .unwrap(); + assert!(owner.finish(&policy, &task, &binding, &saved.record).is_err()); + writer.save(&mut saved, Step::Reporting, &observe).unwrap(); + assert!(owner.finish(&policy, &task, &binding, &saved.record).is_err()); + writer.save(&mut saved, Step::Finished, &observe).unwrap(); + + // A child surviving its controller owns the original file description. + let surviving_child = locked.lock.try_clone().unwrap(); + drop(locked); + assert!(owner.finish(&policy, &task, &binding, &saved.record).is_err()); + assert_eq!( + claims.snapshot().unwrap()[&1].lease.as_ref(), + Some(&binding.lease) + ); + drop(surviving_child); + + let mut stale = saved.record.clone(); + stale.checkpoint_revision -= 1; + assert!(owner.finish(&policy, &task, &binding, &stale).is_err()); + owner.finish(&policy, &task, &binding, &saved.record).unwrap(); + assert!(claims.snapshot().unwrap()[&1].lease.is_none()); + assert!(owner.admit(&task).is_err()); + } + + #[test] + fn native_queue_dispatches_one_canonical_attempt_and_settles_before_the_event() { + let (directory, claims, binding, prepared, policy) = setup(); + let owner = NativeQueueOwner { + claims: &claims, + worker_id: &binding.lease.holder, + }; + let task = native_task(&binding, prepared).unwrap(); + let repo = directory.path().join("repo"); + let queue = directory.path().join("queue"); + fs::create_dir(&repo).unwrap(); + fs::create_dir(&queue).unwrap(); + assert!(Command::new(&policy.git_binary) + .current_dir(&repo) + .env_clear() + .env("HOME", directory.path()) + .args(["init", "--quiet"]) + .status() + .unwrap() + .success()); + fs::write( + queue.join("task.json"), + serde_json::to_vec(&[task.clone()]).unwrap(), + ) + .unwrap(); + + let starts = Arc::new(AtomicUsize::new(0)); + let started = starts.clone(); + let receipt = binding.clone(); + let worker_claims = claims.clone(); + let worker: Arc = Arc::new(move |found, locked| { + assert_eq!(found.id, receipt.native_task_id); + assert_eq!(found.branch, receipt.branch); + assert_eq!( + worker_claims.snapshot().unwrap()[&receipt.number].lease.as_ref(), + Some(&receipt.lease) + ); + let second = LockedTask::acquire(found, locked.policy); + assert!(second.is_err(), "worker must receive the held task lock"); + started.fetch_add(1, Ordering::Relaxed); + let mut saved = state::State::new(found.clone(), blocked_record(found, String::new()), true); + saved.record.outcome = Outcome::NothingToDo; + let mut writer = + state::Writer::new(locked.policy.storage.join(format!("{}.json", found.id)), false); + writer + .save(&mut saved, Step::Finished, &|_: &Record| Ok(())) + .unwrap(); + Ok(saved.record) + }); + let options = QueueOptions { + min_free_bytes: 0, + once: true, + ..QueueOptions::default() + }; + let run = || QueueRun { + repo: &repo, + directory: &queue, + options: &options, + policy: &policy, + df: Path::new("/usr/bin/df"), + }; + let mut completed = 0; + let status = run() + .run_native(&owner, worker.clone(), &mut |event| { + if let QueueEvent::Task { record } = event { + assert_eq!(record.id, task.id); + assert!(claims.snapshot().unwrap()[&binding.number].lease.is_none()); + completed += 1; + } + Ok(()) + }) + .unwrap(); + assert_eq!((status.finished, status.failed, completed), (1, 0, 1)); + + let again = run().run_native(&owner, worker, &mut |_| Ok(())).unwrap(); + assert_eq!((again.finished, again.failed), (0, 1)); + assert_eq!(starts.load(Ordering::Relaxed), 1); + } +} diff --git a/oxa/oxa-work/src/state.rs b/oxa/oxa-work/src/state.rs index 61e38495..d3a7b0d4 100644 --- a/oxa/oxa-work/src/state.rs +++ b/oxa/oxa-work/src/state.rs @@ -9,6 +9,21 @@ use std::path::{Path, PathBuf}; const VERSION: u64 = 1; const MAX_STATE_BYTES: u64 = 4 * 1024 * 1024; +pub(crate) fn lock(path: &Path) -> Result { + use std::os::unix::fs::OpenOptionsExt; + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .mode(0o600) + .open(path) + .map_err(|error| error.to_string())?; + file.try_lock() + .map_err(|error| format!("queue is already running: {error}"))?; + Ok(file) +} + #[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum Step {