diff --git a/oxa/oxa-cli/src/work_cmd.rs b/oxa/oxa-cli/src/work_cmd.rs index 3f6c67bb9..53b7b3c96 100644 --- a/oxa/oxa-cli/src/work_cmd.rs +++ b/oxa/oxa-cli/src/work_cmd.rs @@ -5,6 +5,7 @@ use std::path::{Path, PathBuf}; #[path = "work_cmd/native_queue.rs"] mod native_queue; +pub(crate) use native_queue::authenticate_native_claim; pub(crate) fn trusted(cwd: &Path, env: &Env, dirs: &Dirs) -> Result { crate::dispatch::project_trusted( diff --git a/oxa/oxa-cli/src/work_cmd/native_queue.rs b/oxa/oxa-cli/src/work_cmd/native_queue.rs index 638414a98..2765e824c 100644 --- a/oxa/oxa-cli/src/work_cmd/native_queue.rs +++ b/oxa/oxa-cli/src/work_cmd/native_queue.rs @@ -1,7 +1,7 @@ //! Explicit one-shot execution of an existing, fully bound native board packet. use super::{protected_storage, registered_native_worker, Context}; use oxa_work::{ - board::{Claims, ItemLease, LeaseSignal}, + board::{Assignment, Claims, ItemLease, ItemOwner, LeaseSignal, NativeBinding}, CheckpointKind, LockedTask, NativeQueueOwner, QueueOptions, QueueRun, Record, Task, }; use serde_json::Value; @@ -23,6 +23,49 @@ fn paused(path: &Path) -> Result { } } +pub(crate) fn authenticate_native_claim<'a>( + row: &'a Assignment, + host: &str, +) -> Result<(&'a NativeBinding, &'a ItemOwner), String> { + if row.data.import_hold { + return Err("native claim is held for import".into()); + } + let owner = row + .data + .item_owner + .as_ref() + .filter(|owner| owner.host_id == host) + .ok_or("native claim does not belong to the authenticated host")?; + let binding = row + .data + .native_binding + .as_ref() + .filter(|binding| binding.number == row.number && row.state.lease.as_ref() == Some(&binding.lease)) + .ok_or("native worker lacks an exact retained binding")?; + binding.validate()?; + Ok((binding, owner)) +} + +fn runnable_claim<'a>( + rows: impl Iterator, + worker: &str, + host: &str, +) -> Result, String> { + let mut runnable = None; + for row in rows.filter(|row| { + row.state + .lease + .as_ref() + .is_some_and(|lease| lease.holder == worker) + }) { + let (binding, owner) = authenticate_native_claim(row, host)?; + if row.data.dependency.is_none() && runnable.replace((binding, owner)).is_some() { + return Err("native worker has multiple runnable assignments".into()); + } + } + Ok(runnable) +} + pub(super) fn run( mut context: Context, common: &Path, @@ -101,27 +144,13 @@ pub(super) fn run( let claims_path = owner_path("claims")?; let claims = Claims::open(&claims_path)?; let (worker, host) = registered_native_worker(&value)?; - let rows = claims.assignments()?; - let mut owned = rows.into_values().filter(|row| { - row.state - .lease - .as_ref() - .is_some_and(|lease| lease.holder == worker.id) - }); - let Some(row) = owned.next() else { return Ok(0) }; - if owned.next().is_some() { - return Err("native worker has multiple active assignments".into()); - } - let item_owner = row - .data - .item_owner - .filter(|owner| owner.host_id == host) - .ok_or("native claim does not belong to the authenticated host")?; - let binding = row - .data - .native_binding - .filter(|binding| row.state.lease.as_ref() == Some(&binding.lease)) - .ok_or("native worker lacks an exact retained binding")?; + let (binding, item_owner) = { + let rows = claims.assignments()?; + let Some((binding, item_owner)) = runnable_claim(rows.values(), &worker.id, host)? else { + return Ok(0); + }; + (binding.clone(), item_owner.clone()) + }; if steering.is_some_and(|(attempt, _)| attempt != binding.native_task_id) { return Err("native steering objective belongs to a different attempt".into()); } @@ -168,10 +197,20 @@ pub(super) fn run( let steering_objective = steering.map(|(_, objective)| objective.to_owned()); #[cfg(all(target_os = "linux", feature = "managed-steering"))] let steering_binding = binding.clone(); + let admission_claims = claims.clone(); + let admission_binding = binding.clone(); + let admission_host = host.to_owned(); let run = Arc::new(move |task: &Task, lock: &mut LockedTask<'_>| { if paused(&pause)? { return Err("native board execution paused".into()); } + { + let rows = admission_claims.assignments()?; + let current = runnable_claim(rows.values(), &admission_binding.lease.holder, &admission_host)?; + if !current.is_some_and(|(binding, _)| binding == &admission_binding) { + return Err("native claim is no longer runnable".into()); + } + } #[cfg(all(target_os = "linux", feature = "managed-steering"))] if let Some(objective) = &steering_objective { lock.bind_managed_steering(&claims_path, &steering_binding, objective)?; diff --git a/oxa/oxa-cli/src/work_refill.rs b/oxa/oxa-cli/src/work_refill.rs index b55c34cdc..65989af9a 100644 --- a/oxa/oxa-cli/src/work_refill.rs +++ b/oxa/oxa-cli/src/work_refill.rs @@ -140,12 +140,17 @@ impl Consumer<'_> { } let rows = self.claims.assignments()?; // Recheck under the shared queue publication lock, including other refs. + // A bound hosted wait can yield; claim_idle_item_for still checks idleness atomically. if !rows.contains_key(&number) || rows.values().any(|row| { - row.state - .lease - .as_ref() - .is_some_and(|l| l.holder == self.worker.id) + row.state.lease.as_ref().is_some_and(|lease| { + lease.holder == self.worker.id + && (row.data.import_hold + || row.data.dependency.is_none() + || host.as_deref().is_none_or(|host| { + crate::work_cmd::authenticate_native_claim(row, host).is_err() + })) + }) }) || rows[&number].state.lease.is_some() || (host.is_none() && !source.revalidate(entry)?) diff --git a/oxa/oxa-cli/tests/work_native_lifecycle.rs b/oxa/oxa-cli/tests/work_native_lifecycle.rs index b8e508c5d..9226592df 100644 --- a/oxa/oxa-cli/tests/work_native_lifecycle.rs +++ b/oxa/oxa-cli/tests/work_native_lifecycle.rs @@ -10,7 +10,7 @@ use sha2::{Digest, Sha256}; use std::{ collections::BTreeMap, fs, - io::Read, + io::{Read, Write}, os::unix::fs::PermissionsExt, path::{Path, PathBuf}, process::{Child, Command, Output, Stdio}, @@ -134,7 +134,7 @@ impl Fixture { } fn invoke(&self, server: &FakeServer, args: &[&str]) -> (Child, Output) { - let mut child = Command::new(env!("CARGO_BIN_EXE_oxa")) + let child = Command::new(env!("CARGO_BIN_EXE_oxa")) .args(args) .current_dir(self.env.work().join("project")) .env_clear() @@ -143,6 +143,10 @@ impl Fixture { .stderr(Stdio::piped()) .spawn() .unwrap(); + self.joined(child) + } + + fn joined(&self, mut child: Child) -> (Child, Output) { let deadline = Instant::now() + Duration::from_secs(90); let status = loop { if let Some(status) = child.try_wait().unwrap() { @@ -172,6 +176,102 @@ impl Fixture { fn reopened(&self) -> Claims { Claims::open(self.config.parent().unwrap().join("claims").as_path()).unwrap() } + + fn park(&self) -> DependencyWait { + renew(self); + let rows = self.reopened().assignments().unwrap(); + let row = &rows[&42]; + let receipt = ItemLease { + lease: self.original.lease.clone(), + host_id: "host-a".into(), + expires_at: row.data.item_owner.as_ref().unwrap().expires_at, + }; + let wait = DependencyWait { + dependency: Dependency::RequiredCi(99), + deadline: self.now + 300, + }; + // This fixture has not launched a process or acquired a task lock. + assert!(self + .claims + .wait_for_dependency( + &self.original.lease.holder, + "host-a", + &self.binding, + &receipt, + self.now, + row.data.revision, + &wait, + |observed| Ok(observed == &wait), + ) + .unwrap()); + wait + } + + fn refill_next(&self) -> Output { + self.claims + .import_snapshot( + Some(ManualAuthority::Owner), + &BTreeMap::from([( + 44, + ImportState { + state: AttemptState::default(), + excluded: false, + }, + )]), + self.now, + ) + .unwrap(); + let controls = self.config.parent().unwrap(); + let taskfile = controls.join("next.txt"); + fs::write(&taskfile, "i44 fake agent/44 main\nSynthetic next task.\n").unwrap(); + fs::set_permissions(&taskfile, fs::Permissions::from_mode(0o600)).unwrap(); + let mut config: serde_json::Value = serde_json::from_slice(&fs::read(&self.config).unwrap()).unwrap(); + config.as_object_mut().unwrap().remove("nativeSteering"); + config["intake"] = json!(true); + config["partition"] = json!({"modulus":22,"residues":[0]}); + config["log"] = json!(controls.join("audit.jsonl")); + config["packets"] = json!([{ + "number":44,"revision":"revision","action":"build_issue", + "branch":"agent/44","taskfile":taskfile + }]); + fs::write(&self.config, config.to_string()).unwrap(); + + let tools = self.env.tmp().join("parked-tools"); + fs::create_dir(&tools).unwrap(); + fs::set_permissions(&tools, fs::Permissions::from_mode(0o700)).unwrap(); + let gh = tools.join("gh"); + fs::write(&gh, "#!/bin/sh\n[ \"$GH_TOKEN\" = synthetic ] || exit 93\nprintf 'HTTP/2.0 200 OK\\r\\n\\r\\n'\ncat \"$0.reply\"\n").unwrap(); + fs::set_permissions(&gh, fs::Permissions::from_mode(0o700)).unwrap(); + fs::write(tools.join("gh.reply"), json!({"data":{"repository":{ + "mergeQueue":null, + "pullRequests":{"totalCount":0,"nodes":[],"pageInfo":{"hasNextPage":false,"endCursor":"last"}}, + "i44":{"number":44,"state":"OPEN","updatedAt":"revision"},"b44":null + }}}).to_string()).unwrap(); + let path = format!("{}:{}", tools.display(), std::env::var("PATH").unwrap()); + let mut child = Command::new(env!("CARGO_BIN_EXE_oxa")) + .args([ + "work", + "board", + "--refill", + "--repo", + "example/project", + "--once", + "--token-stdin", + "--config", + ]) + .arg(&self.config) + .current_dir(self.env.work().join("project")) + .env_clear() + .envs(self.env.envs("http://127.0.0.1:1")) + .env("PATH", path) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + child.stdin.take().unwrap().write_all(b"synthetic\n").unwrap(); + self.joined(child).1 + } } fn files(root: &Path) -> BTreeMap> { @@ -367,6 +467,68 @@ fn checkpoint(fixture: &Fixture) -> serde_json::Value { .unwrap() } +#[test] +fn all_parked_native_work_keeps_its_packet_and_claim_without_a_model_call() { + let fixture = Fixture::new(); + fixture.park(); + let claim = serde_json::to_value(&fixture.reopened().assignments().unwrap()[&42]).unwrap(); + let queue = files(&fixture.queue); + let server = FakeServer::start(vec![], FakeOpts::default()).unwrap(); + let (_, output) = fixture.run(&server); + assert_eq!(files(&fixture.queue), queue, "a parked packet must not be taken"); + assert!(output.status.success(), "{output:?}"); + assert!(server.requests().is_empty()); + assert!(!fixture + .storage + .join(format!("{}.json", fixture.binding.native_task_id)) + .exists()); + assert_eq!( + serde_json::to_value(&fixture.reopened().assignments().unwrap()[&42]).unwrap(), + claim + ); +} + +#[test] +fn parked_native_work_yields_to_one_refilled_task_and_retains_its_original_receipt() { + let fixture = Fixture::new(); + fixture.park(); + let parked = serde_json::to_value(&fixture.reopened().assignments().unwrap()[&42]).unwrap(); + let parked_packet = fixture.queue.join(format!("board-{}.txt", fixture.binding.key)); + let parked_bytes = fs::read(&parked_packet).unwrap(); + let refilled = fixture.refill_next(); + assert!(refilled.status.success(), "{refilled:?}"); + let next = fixture.reopened().assignments().unwrap()[&44] + .data + .native_binding + .clone() + .expect("a verified parked lease must permit the real refill"); + assert!(fixture.queue.join(format!("board-{}.txt", next.key)).is_file()); + let server = FakeServer::start(vec![ScriptTurn::text("finished")], FakeOpts::default()).unwrap(); + let (_, output) = fixture.run(&server); + assert!(output.status.success(), "{output:?}"); + assert_eq!(server.requests().len(), 1); + let rows = fixture.reopened().assignments().unwrap(); + assert_eq!(serde_json::to_value(&rows[&42]).unwrap(), parked); + assert_eq!(fs::read(&parked_packet).unwrap(), parked_bytes); + assert!(rows[&44].state.lease.is_none()); + assert_eq!(rows[&44].state.attempts, 1); + assert!(rows[&44].data.native_binding.as_ref() == Some(&next)); + let completed: serde_json::Value = serde_json::from_slice( + &fs::read(fixture.storage.join(format!("{}.json", next.native_task_id))).unwrap(), + ) + .unwrap(); + assert_eq!(completed["record"]["id"], next.native_task_id); + assert_eq!(completed["record"]["step"], "finished"); + let (_, idle) = fixture.run(&server); + assert!(idle.status.success(), "{idle:?}"); + assert_eq!(server.requests().len(), 1, "all remaining work is still parked"); + assert_eq!( + serde_json::to_value(&fixture.reopened().assignments().unwrap()[&42]).unwrap(), + parked + ); + assert_eq!(fs::read(parked_packet).unwrap(), parked_bytes); +} + #[test] fn activity_requires_opt_in_and_finished_replay_does_not_advance_it() { for enabled in [false, true] {