Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions oxa/oxa-cli/src/work_cmd.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ use std::path::{Path, PathBuf};

#[path = "work_cmd/native_queue.rs"]
mod native_queue;
pub(crate) use native_queue::authenticate_native_claim;

pub(crate) fn trusted(cwd: &Path, env: &Env, dirs: &Dirs) -> Result<bool, String> {
crate::dispatch::project_trusted(
Expand Down
83 changes: 61 additions & 22 deletions oxa/oxa-cli/src/work_cmd/native_queue.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
//! Explicit one-shot execution of an existing, fully bound native board packet.
use super::{protected_storage, registered_native_worker, Context};
use oxa_work::{
board::{Claims, ItemLease, LeaseSignal},
board::{Assignment, Claims, ItemLease, ItemOwner, LeaseSignal, NativeBinding},
CheckpointKind, LockedTask, NativeQueueOwner, QueueOptions, QueueRun, Record, Task,
};
use serde_json::Value;
Expand All @@ -23,6 +23,49 @@ fn paused(path: &Path) -> Result<bool, String> {
}
}

pub(crate) fn authenticate_native_claim<'a>(
row: &'a Assignment,
host: &str,
) -> Result<(&'a NativeBinding, &'a ItemOwner), String> {
if row.data.import_hold {
return Err("native claim is held for import".into());
}
let owner = row
.data
.item_owner
.as_ref()
.filter(|owner| owner.host_id == host)
.ok_or("native claim does not belong to the authenticated host")?;
let binding = row
.data
.native_binding
.as_ref()
.filter(|binding| binding.number == row.number && row.state.lease.as_ref() == Some(&binding.lease))
.ok_or("native worker lacks an exact retained binding")?;
binding.validate()?;
Ok((binding, owner))
}

fn runnable_claim<'a>(
rows: impl Iterator<Item = &'a Assignment>,
worker: &str,
host: &str,
) -> Result<Option<(&'a NativeBinding, &'a ItemOwner)>, String> {
let mut runnable = None;
for row in rows.filter(|row| {
row.state
.lease
.as_ref()
.is_some_and(|lease| lease.holder == worker)
}) {
let (binding, owner) = authenticate_native_claim(row, host)?;
if row.data.dependency.is_none() && runnable.replace((binding, owner)).is_some() {
return Err("native worker has multiple runnable assignments".into());
}
}
Ok(runnable)
}

pub(super) fn run(
mut context: Context,
common: &Path,
Expand Down Expand Up @@ -101,27 +144,13 @@ pub(super) fn run(
let claims_path = owner_path("claims")?;
let claims = Claims::open(&claims_path)?;
let (worker, host) = registered_native_worker(&value)?;
let rows = claims.assignments()?;
let mut owned = rows.into_values().filter(|row| {
row.state
.lease
.as_ref()
.is_some_and(|lease| lease.holder == worker.id)
});
let Some(row) = owned.next() else { return Ok(0) };
if owned.next().is_some() {
return Err("native worker has multiple active assignments".into());
}
let item_owner = row
.data
.item_owner
.filter(|owner| owner.host_id == host)
.ok_or("native claim does not belong to the authenticated host")?;
let binding = row
.data
.native_binding
.filter(|binding| row.state.lease.as_ref() == Some(&binding.lease))
.ok_or("native worker lacks an exact retained binding")?;
let (binding, item_owner) = {
let rows = claims.assignments()?;
let Some((binding, item_owner)) = runnable_claim(rows.values(), &worker.id, host)? else {
return Ok(0);
};
(binding.clone(), item_owner.clone())
};
if steering.is_some_and(|(attempt, _)| attempt != binding.native_task_id) {
return Err("native steering objective belongs to a different attempt".into());
}
Expand Down Expand Up @@ -168,10 +197,20 @@ pub(super) fn run(
let steering_objective = steering.map(|(_, objective)| objective.to_owned());
#[cfg(all(target_os = "linux", feature = "managed-steering"))]
let steering_binding = binding.clone();
let admission_claims = claims.clone();
let admission_binding = binding.clone();
let admission_host = host.to_owned();
let run = Arc::new(move |task: &Task, lock: &mut LockedTask<'_>| {
if paused(&pause)? {
return Err("native board execution paused".into());
}
{
let rows = admission_claims.assignments()?;
let current = runnable_claim(rows.values(), &admission_binding.lease.holder, &admission_host)?;
if !current.is_some_and(|(binding, _)| binding == &admission_binding) {
return Err("native claim is no longer runnable".into());
}
}
#[cfg(all(target_os = "linux", feature = "managed-steering"))]
if let Some(objective) = &steering_objective {
lock.bind_managed_steering(&claims_path, &steering_binding, objective)?;
Expand Down
13 changes: 9 additions & 4 deletions oxa/oxa-cli/src/work_refill.rs
Original file line number Diff line number Diff line change
Expand Up @@ -140,12 +140,17 @@ impl Consumer<'_> {
}
let rows = self.claims.assignments()?;
// Recheck under the shared queue publication lock, including other refs.
// A bound hosted wait can yield; claim_idle_item_for still checks idleness atomically.
if !rows.contains_key(&number)
|| rows.values().any(|row| {
row.state
.lease
.as_ref()
.is_some_and(|l| l.holder == self.worker.id)
row.state.lease.as_ref().is_some_and(|lease| {
lease.holder == self.worker.id
&& (row.data.import_hold
|| row.data.dependency.is_none()
|| host.as_deref().is_none_or(|host| {
crate::work_cmd::authenticate_native_claim(row, host).is_err()
}))
})
})
|| rows[&number].state.lease.is_some()
|| (host.is_none() && !source.revalidate(entry)?)
Expand Down
166 changes: 164 additions & 2 deletions oxa/oxa-cli/tests/work_native_lifecycle.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ use sha2::{Digest, Sha256};
use std::{
collections::BTreeMap,
fs,
io::Read,
io::{Read, Write},
os::unix::fs::PermissionsExt,
path::{Path, PathBuf},
process::{Child, Command, Output, Stdio},
Expand Down Expand Up @@ -134,7 +134,7 @@ impl Fixture {
}

fn invoke(&self, server: &FakeServer, args: &[&str]) -> (Child, Output) {
let mut child = Command::new(env!("CARGO_BIN_EXE_oxa"))
let child = Command::new(env!("CARGO_BIN_EXE_oxa"))
.args(args)
.current_dir(self.env.work().join("project"))
.env_clear()
Expand All @@ -143,6 +143,10 @@ impl Fixture {
.stderr(Stdio::piped())
.spawn()
.unwrap();
self.joined(child)
}

fn joined(&self, mut child: Child) -> (Child, Output) {
let deadline = Instant::now() + Duration::from_secs(90);
let status = loop {
if let Some(status) = child.try_wait().unwrap() {
Expand Down Expand Up @@ -172,6 +176,102 @@ impl Fixture {
fn reopened(&self) -> Claims {
Claims::open(self.config.parent().unwrap().join("claims").as_path()).unwrap()
}

fn park(&self) -> DependencyWait {
renew(self);
let rows = self.reopened().assignments().unwrap();
let row = &rows[&42];
let receipt = ItemLease {
lease: self.original.lease.clone(),
host_id: "host-a".into(),
expires_at: row.data.item_owner.as_ref().unwrap().expires_at,
};
let wait = DependencyWait {
dependency: Dependency::RequiredCi(99),
deadline: self.now + 300,
};
// This fixture has not launched a process or acquired a task lock.
assert!(self
.claims
.wait_for_dependency(
&self.original.lease.holder,
"host-a",
&self.binding,
&receipt,
self.now,
row.data.revision,
&wait,
|observed| Ok(observed == &wait),
)
.unwrap());
wait
}

fn refill_next(&self) -> Output {
self.claims
.import_snapshot(
Some(ManualAuthority::Owner),
&BTreeMap::from([(
44,
ImportState {
state: AttemptState::default(),
excluded: false,
},
)]),
self.now,
)
.unwrap();
let controls = self.config.parent().unwrap();
let taskfile = controls.join("next.txt");
fs::write(&taskfile, "i44 fake agent/44 main\nSynthetic next task.\n").unwrap();
fs::set_permissions(&taskfile, fs::Permissions::from_mode(0o600)).unwrap();
let mut config: serde_json::Value = serde_json::from_slice(&fs::read(&self.config).unwrap()).unwrap();
config.as_object_mut().unwrap().remove("nativeSteering");
config["intake"] = json!(true);
config["partition"] = json!({"modulus":22,"residues":[0]});
config["log"] = json!(controls.join("audit.jsonl"));
config["packets"] = json!([{
"number":44,"revision":"revision","action":"build_issue",
"branch":"agent/44","taskfile":taskfile
}]);
fs::write(&self.config, config.to_string()).unwrap();

let tools = self.env.tmp().join("parked-tools");
fs::create_dir(&tools).unwrap();
fs::set_permissions(&tools, fs::Permissions::from_mode(0o700)).unwrap();
let gh = tools.join("gh");
fs::write(&gh, "#!/bin/sh\n[ \"$GH_TOKEN\" = synthetic ] || exit 93\nprintf 'HTTP/2.0 200 OK\\r\\n\\r\\n'\ncat \"$0.reply\"\n").unwrap();
fs::set_permissions(&gh, fs::Permissions::from_mode(0o700)).unwrap();
fs::write(tools.join("gh.reply"), json!({"data":{"repository":{
"mergeQueue":null,
"pullRequests":{"totalCount":0,"nodes":[],"pageInfo":{"hasNextPage":false,"endCursor":"last"}},
"i44":{"number":44,"state":"OPEN","updatedAt":"revision"},"b44":null
}}}).to_string()).unwrap();
let path = format!("{}:{}", tools.display(), std::env::var("PATH").unwrap());
let mut child = Command::new(env!("CARGO_BIN_EXE_oxa"))
.args([
"work",
"board",
"--refill",
"--repo",
"example/project",
"--once",
"--token-stdin",
"--config",
])
.arg(&self.config)
.current_dir(self.env.work().join("project"))
.env_clear()
.envs(self.env.envs("http://127.0.0.1:1"))
.env("PATH", path)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.unwrap();
child.stdin.take().unwrap().write_all(b"synthetic\n").unwrap();
self.joined(child).1
}
}

fn files(root: &Path) -> BTreeMap<PathBuf, Vec<u8>> {
Expand Down Expand Up @@ -367,6 +467,68 @@ fn checkpoint(fixture: &Fixture) -> serde_json::Value {
.unwrap()
}

#[test]
fn all_parked_native_work_keeps_its_packet_and_claim_without_a_model_call() {
let fixture = Fixture::new();
fixture.park();
let claim = serde_json::to_value(&fixture.reopened().assignments().unwrap()[&42]).unwrap();
let queue = files(&fixture.queue);
let server = FakeServer::start(vec![], FakeOpts::default()).unwrap();
let (_, output) = fixture.run(&server);
assert_eq!(files(&fixture.queue), queue, "a parked packet must not be taken");
assert!(output.status.success(), "{output:?}");
assert!(server.requests().is_empty());
assert!(!fixture
.storage
.join(format!("{}.json", fixture.binding.native_task_id))
.exists());
assert_eq!(
serde_json::to_value(&fixture.reopened().assignments().unwrap()[&42]).unwrap(),
claim
);
}

#[test]
fn parked_native_work_yields_to_one_refilled_task_and_retains_its_original_receipt() {
let fixture = Fixture::new();
fixture.park();
let parked = serde_json::to_value(&fixture.reopened().assignments().unwrap()[&42]).unwrap();
let parked_packet = fixture.queue.join(format!("board-{}.txt", fixture.binding.key));
let parked_bytes = fs::read(&parked_packet).unwrap();
let refilled = fixture.refill_next();
assert!(refilled.status.success(), "{refilled:?}");
let next = fixture.reopened().assignments().unwrap()[&44]
.data
.native_binding
.clone()
.expect("a verified parked lease must permit the real refill");
assert!(fixture.queue.join(format!("board-{}.txt", next.key)).is_file());
let server = FakeServer::start(vec![ScriptTurn::text("finished")], FakeOpts::default()).unwrap();
let (_, output) = fixture.run(&server);
assert!(output.status.success(), "{output:?}");
assert_eq!(server.requests().len(), 1);
let rows = fixture.reopened().assignments().unwrap();
assert_eq!(serde_json::to_value(&rows[&42]).unwrap(), parked);
assert_eq!(fs::read(&parked_packet).unwrap(), parked_bytes);
assert!(rows[&44].state.lease.is_none());
assert_eq!(rows[&44].state.attempts, 1);
assert!(rows[&44].data.native_binding.as_ref() == Some(&next));
let completed: serde_json::Value = serde_json::from_slice(
&fs::read(fixture.storage.join(format!("{}.json", next.native_task_id))).unwrap(),
)
.unwrap();
assert_eq!(completed["record"]["id"], next.native_task_id);
assert_eq!(completed["record"]["step"], "finished");
let (_, idle) = fixture.run(&server);
assert!(idle.status.success(), "{idle:?}");
assert_eq!(server.requests().len(), 1, "all remaining work is still parked");
assert_eq!(
serde_json::to_value(&fixture.reopened().assignments().unwrap()[&42]).unwrap(),
parked
);
assert_eq!(fs::read(parked_packet).unwrap(), parked_bytes);
}

#[test]
fn activity_requires_opt_in_and_finished_replay_does_not_advance_it() {
for enabled in [false, true] {
Expand Down
Loading