From 3ab703fbfbf3922aeb9f9ace7de74d4552a638a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 09:56:56 +0000 Subject: [PATCH 1/9] Describe each durable runner step as a typed transition --- oxa/docs/work-facts.md | 9 +++++ oxa/oxa-proto/src/work.rs | 26 ++++++++++++- oxa/oxa-proto/tests/work_facts.rs | 61 +++++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 1 deletion(-) diff --git a/oxa/docs/work-facts.md b/oxa/docs/work-facts.md index abb44ba87..bdcc80eb4 100644 --- a/oxa/docs/work-facts.md +++ b/oxa/docs/work-facts.md @@ -17,6 +17,15 @@ external acceptance bind a full PR head. A head-independent `Subject::PullReques available for blocks, asks and future standing holds. Task, host, repository and inbox subjects also exist, including tasks that do not yet have a PR. +`TaskTransition { task, step, session_id, reason }` reports every durable runner step: +`preparing → worker → acceptance → repair/recheck → saving → finished`. Each observation gets +its own checkpoint revision; worker completion and local acceptance may have separate revisions. +`TaskStep` is closed, `session_id` is optional until known, and `StepReason` is a closed code +such as `checks_failed`, `session_unavailable` or `deadline`, not diagnostic prose. A finished +step requires a reason and says only that this local attempt ended. It does not imply acceptance, +publication or merge. These observations are routine; a separate block/ask/incident carries any +attention condition. Running situation rows expose their current step and cannot be `finished`. + `Snapshot` is a materialized read view: `as_of_seq`, observation time, standing revision, optional goal, running tasks (host, branch, worktree and optional session), locally accepted work, publications, landed PRs, blocks, asks and incidents. It is not authoritative task state. diff --git a/oxa/oxa-proto/src/work.rs b/oxa/oxa-proto/src/work.rs index 36dfc99b5..8813ec3bf 100644 --- a/oxa/oxa-proto/src/work.rs +++ b/oxa/oxa-proto/src/work.rs @@ -211,6 +211,18 @@ choice!(WorkerOutcome { Done => "done", NothingToDo => "nothing_to_do", Blocked => "blocked", Failed => "failed", Interrupted => "interrupted", }); +choice!(TaskStep { + Preparing => "preparing", Worker => "worker", Acceptance => "acceptance", + Repair => "repair", Recheck => "recheck", Saving => "saving", Finished => "finished", +}); +choice!(StepReason { + Resumed => "resumed", Prepared => "prepared", WorkerDone => "worker_done", + WorkerNothingToDo => "worker_nothing_to_do", WorkerBlocked => "worker_blocked", + WorkerFailed => "worker_failed", ChecksPassed => "checks_passed", ChecksFailed => "checks_failed", + RepairDone => "repair_done", RepairFailed => "repair_failed", Saved => "saved", SaveFailed => "save_failed", + Interrupted => "interrupted", Deadline => "deadline", SessionUnavailable => "session_unavailable", + SetupFailed => "setup_failed", +}); choice!(EvidenceKind { LocalCheck => "local_check", Gate => "gate", Review => "review", Queue => "queue", Status => "status", Standing => "standing", @@ -305,14 +317,25 @@ record!( | { atom(&s.id, 256) } ); record!(Running { - task: String, host: String, branch: String, worktree: String, session_id: Option, + task: String, host: String, branch: String, worktree: String, session_id: Option, step: TaskStep, } |s| { atom(&s.task, 128)?; atom(&s.host, 128)?; require(!s.branch.starts_with('-') && s.branch.len() <= 256, "invalid task branch")?; if let Some(id) = &s.session_id { atom(id, 128)?; } + require(s.step != TaskStep::Finished, "finished task cannot be running")?; Ok(()) }); +record!( + /// One durable runner step, with no task-state contents or implied publication/landing. + /// Finished means this local attempt ended; only the separate acceptance/merge facts + /// provide those respective claims. A terminal step requires an explicit fixed reason. + TaskTransition { task: String, step: TaskStep, session_id: Option, reason: Option } |s| { + atom(&s.task, 128)?; + if let Some(id) = &s.session_id { atom(id, 128)?; } + require(s.step != TaskStep::Finished || s.reason.is_some(), "finished step requires a reason") + } +); record!(WorkerFinished { task: String, session_id: Option, outcome: WorkerOutcome, error_class: Option, } |s| { @@ -402,6 +425,7 @@ tagged!( /// No variant grants permission or updates runner state. Only PrMerged is evidence of landing. Fact { TaskStarted(Running) => "task_started", + TaskTransition(TaskTransition) => "task_transition", WorkerFinished(WorkerFinished) => "worker_finished", LocalAcceptance(LocalAcceptance) => "local_acceptance", Published(Publication) => "published", diff --git a/oxa/oxa-proto/tests/work_facts.rs b/oxa/oxa-proto/tests/work_facts.rs index 8a21350c7..e43eaaf69 100644 --- a/oxa/oxa-proto/tests/work_facts.rs +++ b/oxa/oxa-proto/tests/work_facts.rs @@ -20,6 +20,7 @@ fn running() -> Running { branch: "agent/17-work".into(), worktree: "~/project/task-17".into(), session_id: Some("session-17".into()), + step: TaskStep::Worker, } } fn ask() -> Ask { @@ -355,3 +356,63 @@ fn outcomes_and_evidence_are_explicit_and_head_bound() { zero_seq.seq = 0; assert!(zero_seq.to_json().is_err()); } + +#[test] +fn every_durable_runner_step_has_its_own_observation_and_replay_identity() { + let steps = [ + TaskStep::Preparing, + TaskStep::Worker, + TaskStep::Acceptance, + TaskStep::Repair, + TaskStep::Recheck, + TaskStep::Saving, + TaskStep::Finished, + ]; + for (index, step) in steps.into_iter().enumerate() { + let observation = TaskTransition { + task: "task-17".into(), + step, + session_id: (index > 0).then(|| "session-17".into()), + reason: (step == TaskStep::Finished).then_some(StepReason::Saved), + }; + let event = event(index as u64 + 1, Fact::TaskTransition(observation)); + assert_eq!(event.fact.class(), EventClass::Routine); + assert_eq!(Event::from_json(&event.to_json().unwrap()).unwrap(), event); + assert_eq!(event.checkpoint.revision, index as u64 + 1); + assert!(!matches!(event.fact, Fact::PrMerged(_))); + } + let mut terminal = TaskTransition { + task: "task-17".into(), + step: TaskStep::Finished, + session_id: None, + reason: None, + }; + assert!(event(1, Fact::TaskTransition(terminal.clone())) + .to_json() + .is_err()); + for reason in [ + StepReason::WorkerBlocked, + StepReason::WorkerFailed, + StepReason::ChecksFailed, + StepReason::RepairFailed, + StepReason::SaveFailed, + StepReason::Interrupted, + StepReason::Deadline, + StepReason::SessionUnavailable, + StepReason::SetupFailed, + ] { + terminal.reason = Some(reason); + let observation = event(1, Fact::TaskTransition(terminal.clone())); + assert_eq!( + Event::from_json(&observation.to_json().unwrap()).unwrap(), + observation + ); + } + let mut bad_running = running(); + bad_running.step = TaskStep::Finished; + assert!(event(1, Fact::TaskStarted(bad_running)).to_json().is_err()); + let mut encoded: Value = + serde_json::from_slice(&event(1, Fact::TaskTransition(terminal)).to_json().unwrap()).unwrap(); + encoded["data"]["fact"]["data"]["reason"] = json!("free-form failure prose"); + assert!(Event::from_json(&serde_json::to_vec(&encoded).unwrap()).is_err()); +} From ba060e6773ed12c0057febad5c05149f8f93bb3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 10:44:51 +0000 Subject: [PATCH 2/9] Persist bounded lead facts with replay and stall incidents --- oxa/Cargo.lock | 2 + oxa/docs/lead-facts-store.md | 75 +++++ oxa/oxa-proto/src/work.rs | 3 +- oxa/oxa-work/Cargo.toml | 2 + oxa/oxa-work/src/events.rs | 321 ++++++++++++++++++++ oxa/oxa-work/src/events/tests.rs | 483 +++++++++++++++++++++++++++++++ oxa/oxa-work/src/facts_io.rs | 203 +++++++++++++ oxa/oxa-work/src/incidents.rs | 194 +++++++++++++ oxa/oxa-work/src/lib.rs | 6 + oxa/oxa-work/src/situation.rs | 132 +++++++++ 10 files changed, 1420 insertions(+), 1 deletion(-) create mode 100644 oxa/docs/lead-facts-store.md create mode 100644 oxa/oxa-work/src/events.rs create mode 100644 oxa/oxa-work/src/events/tests.rs create mode 100644 oxa/oxa-work/src/facts_io.rs create mode 100644 oxa/oxa-work/src/incidents.rs create mode 100644 oxa/oxa-work/src/situation.rs diff --git a/oxa/Cargo.lock b/oxa/Cargo.lock index f11c3ce6a..68acb522d 100644 --- a/oxa/Cargo.lock +++ b/oxa/Cargo.lock @@ -1199,6 +1199,8 @@ dependencies = [ name = "oxa-work" version = "0.2.0" dependencies = [ + "libc", + "oxa-proto", "serde", "serde_json", ] diff --git a/oxa/docs/lead-facts-store.md b/oxa/docs/lead-facts-store.md new file mode 100644 index 000000000..4912aa26e --- /dev/null +++ b/oxa/docs/lead-facts-store.md @@ -0,0 +1,75 @@ +# Runner facts storage + +`oxa_work::events::Facts` is the L1 storage module for #629. The runner remains the sole task +state writer. These modules never open its task files, invoke a model or start a background +process. The lead consumes the public read APIs rather than logs or private runner records. + +The runner supplies exactly `/facts`, outside writable workspace +and Git binds. `Facts::initialize(root)` explicitly creates a fresh store; `Facts::open(root)` +opens/replays an existing one. Missing, corrupt, unsafe or partial initialization is an error, +never an empty situation. Directories are private (0700), files private (0600), and files and +path components are checked without following symlinks. It opens only its fixed storage leaves. + +## Append after a durable checkpoint + +Call `append(Checkpoint { source, revision }, at_ms, fact)` only after the corresponding +observation is durable. Stop before another action if append fails. The runner's source is +`runner`; its checkpoint revision is monotonic per repository. A short repository checkpoint +lock serializes allocation, durable task rename, callback and acknowledgement. The runner +recovers a pending task/revision before allocating again and does not re-emit older acknowledged +task records. Reserved checkpoint revisions may have gaps; event sequence numbers do not. + +Other producers use their own stable emitter domain and monotonic revisions, not a fresh source +per task. The bounded source table holds the latest observation for up to 16 emitter domains; +it retains no unbounded per-task tombstones. Retrying the latest source/revision with the same +fact returns its original event and timestamp, even after journal rotation. Different facts +under that identity and stale revisions are errors. Sequence exhaustion is an error. + +Each append validates the projected situation before reserving the event. It then saves a +pending observation in `facts.json`, appends and syncs the journal line, commits the source +cursor and atomically replaces `situation.json`. Atomic replacements sync the file and directory. +Recovery replays that reserved observation across a torn final line, interrupted rotation, +lost acknowledgement or lagging situation. Complete malformed lines, unreserved torn tails, +conflicting contents and unrecoverable gaps fail closed. + +`TaskStarted` supplies the initial preparing location (task, host label, branch and worktree). +Later `TaskTransition` observations update its step/session; `finished` removes that running +attempt. A transition without a known running location is an error. Worker completion, local +acceptance, publication, external acceptance and merge stay distinct. Only `PrMerged` enters +the landed list. Blocks and asks persist until explicit `Unblocked` and `AskClosed` facts; +updating a PR head cannot silently clear either. + +## Bounded reads + +`situation()` returns a validated `oxa_proto::work::Snapshot` with its exact `as_of_seq`. +It is capped at 64 KiB and 128 total rows. Running work, local acceptance awaiting publication, +published work, blocks and asks are never silently dropped to fit. Capacity failure stops the +next action. Landed/incident history retains at most 16 rows each and may shrink to fit; +`history_truncated` records that loss. An accepted/published pair is retired only by the matching +head's merge observation. Standing revision/goal remain unknown until the standing integration. + +`events_after(after, limit)` takes an exclusive repository cursor and a limit from 1 to 64. +The two journals (`events.jsonl` and `events.previous.jsonl`) rotate at 256 KiB each. Pages +report `first_available`, `next` and an explicit `gap`; a consumer seeing a gap reloads the +situation rather than inventing old transitions. A cursor ahead of the journal is an error. + +`incidents::recent(&facts)` returns the bounded incident projection from that same situation; +the journal is its durable source, not a competing incident/task-state writer. Reads recover +only derived facts metadata/projections under the facts lock and never alter task state. + +## Stall incidents + +`incidents::detect_stall(status_bytes, expected_session_id, expected_pid, now_ms)` consumes a +bounded status snapshot supplied by the runner. It validates identity, phase, progress, timeout +and deadline; stale invocation identity or malformed data is an error. It returns a typed +`Stall` only when the conservative deadline has elapsed. Sessions without an active deadline and finished sessions do not +stall. The caller may append `Stall::incident(task)` through its durable event boundary. + +Stalls are incidents only: this module never wraps, cancels, kills or polls a worker. The +runner passes `--stall-timeout 900` for the decided 15-minute threshold and separately owns +the per-task wall-time limit and escalation. No second watchdog is introduced. + +The runner owns `oxa work situation`, `oxa work events --after N`, checkpoint adapters and +action wiring. The lead owns the status/ask cards. The common schema is described in +[work-facts.md](work-facts.md); storage/recovery tests are in +[events/tests.rs](../oxa-work/src/events/tests.rs). diff --git a/oxa/oxa-proto/src/work.rs b/oxa/oxa-proto/src/work.rs index 8813ec3bf..dade3c648 100644 --- a/oxa/oxa-proto/src/work.rs +++ b/oxa/oxa-proto/src/work.rs @@ -412,7 +412,7 @@ record!( } ); record!(AskClosed { id: String } | s | { atom(&s.id, 128) }); -record!(Incident { id: String, subject: Subject, kind: IncidentKind, evidence: Option } |s| { atom(&s.id, 128) }); +record!(Incident { id: String, subject: Subject, kind: IncidentKind, evidence: Option } |s| { atom(&s.id, 256) }); record!( Mechanical { subject: Subject, @@ -433,6 +433,7 @@ tagged!( Enqueued(PullRequest) => "enqueued", PrMerged(Landed) => "pr_merged", Blocked(Blocked) => "blocked", + Unblocked(Subject) => "unblocked", AskOpened(Ask) => "ask_opened", AskClosed(AskClosed) => "ask_closed", Incident(Incident) => "incident", diff --git a/oxa/oxa-work/Cargo.toml b/oxa/oxa-work/Cargo.toml index 6097461ce..bb9e07e5d 100644 --- a/oxa/oxa-work/Cargo.toml +++ b/oxa/oxa-work/Cargo.toml @@ -7,6 +7,8 @@ license.workspace = true publish.workspace = true [dependencies] +libc.workspace = true +oxa-proto.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/oxa/oxa-work/src/events.rs b/oxa/oxa-work/src/events.rs new file mode 100644 index 000000000..e70e267b0 --- /dev/null +++ b/oxa/oxa-work/src/events.rs @@ -0,0 +1,321 @@ +//! Durable facts journal and its bounded read view. Never opens runner task-state files. +use crate::facts_io::Root; +use crate::{situation, Result}; +use oxa_proto::work::{Checkpoint, Event, EventPage, Fact, Snapshot, MAX_EVENT_BYTES, MAX_SNAPSHOT_BYTES}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::path::Path; + +const CURRENT: &str = "events.jsonl"; +const PREVIOUS: &str = "events.previous.jsonl"; +const STATE: &str = "facts.json"; +const SITUATION: &str = "situation.json"; +const LOCK: &str = ".facts.lock"; +const JOURNAL_BYTES: usize = 256 * 1024; +const STATE_BYTES: usize = 256 * 1024; +const MAX_SOURCES: usize = 16; + +#[derive(Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct State { + version: u32, + last_seq: u64, + last_by_source: Vec, + // Write-ahead observation: a crash can replay it even if the journal line was torn. + pending: Option, +} +impl State { + fn sources(&self) -> Result> { + if self.version != 1 || self.last_by_source.len() > MAX_SOURCES { + return Err("unsupported facts metadata or source capacity".into()); + } + let events: Vec = self + .last_by_source + .iter() + .map(read_event_value) + .collect::>()?; + let mut sources = std::collections::HashSet::new(); + let mut sequences = std::collections::HashSet::new(); + for event in &events { + if event.seq > self.last_seq + || !sources.insert(&event.checkpoint.source) + || !sequences.insert(event.seq) + { + return Err("inconsistent facts source metadata".into()); + } + } + if events.iter().map(|event| event.seq).max().unwrap_or(0) != self.last_seq { + return Err("facts source metadata has no latest observation".into()); + } + Ok(events) + } + fn committed(&mut self, event: &Event) -> Result<()> { + let mut sources = self.sources()?; + sources.retain(|old| old.checkpoint.source != event.checkpoint.source); + sources.push(event.clone()); + if sources.len() > MAX_SOURCES { + return Err("facts source capacity exceeded".into()); + } + self.last_seq = event.seq; + self.last_by_source = sources.iter().map(event_value).collect::>()?; + self.pending = None; + Ok(()) + } + fn bytes(&self) -> Result> { + let bytes = serde_json::to_vec(self).map_err(|_| "cannot encode facts metadata")?; + if bytes.len() > STATE_BYTES { + return Err("facts metadata capacity exceeded".into()); + } + Ok(bytes) + } +} +fn event_value(event: &Event) -> Result { + serde_json::from_slice(&event.to_json().map_err(|error| error.to_string())?) + .map_err(|_| "cannot encode fact".into()) +} +fn read_event_value(value: &Value) -> Result { + Event::from_json(&serde_json::to_vec(value).map_err(|_| "invalid fact metadata")?) + .map_err(|error| error.to_string()) +} + +struct Journal { + events: Vec, + current_bytes: usize, + torn_at: Option, +} +impl Journal { + fn last_seq(&self) -> u64 { + self.events.last().map_or(0, |event| event.seq) + } +} + +#[derive(Clone, Debug)] +pub struct Facts { + root: Root, + journal_bytes: usize, +} +impl Facts { + /// Explicit first initialization. Existing or partial artifacts are never reset to empty. + /// Supply /facts, outside writable workspace/Git binds. + pub fn initialize(root: &Path) -> Result { + let root = Root::open(root, true)?; + let _lock = root.lock(LOCK)?; + for leaf in [CURRENT, PREVIOUS, STATE, SITUATION] { + if root.exists(leaf)? { + return Err("facts store already exists or initialization is incomplete".into()); + } + } + root.create(CURRENT, b"")?; + root.create( + SITUATION, + &situation::empty().to_json().map_err(|error| error.to_string())?, + )?; + root.create( + STATE, + &State { + version: 1, + ..State::default() + } + .bytes()?, + )?; + Ok(Self { + root, + journal_bytes: JOURNAL_BYTES, + }) + } + pub fn open(root: &Path) -> Result { + let facts = Self { + root: Root::open(root, false)?, + journal_bytes: JOURNAL_BYTES, + }; + facts.situation()?; + Ok(facts) + } + /// One event per durable source/revision. A retry of the latest revision returns its + /// original event; older revisions and different contents under that identity are errors. + /// Stop before the next action if this call fails. Replaying the same checkpoint repairs + /// a lost acknowledgement, a torn last line or an interrupted snapshot replacement. + pub fn append(&self, checkpoint: Checkpoint, at_ms: u64, fact: Fact) -> Result { + let _lock = self.root.lock(LOCK)?; + let (mut state, mut snapshot, _) = self.recover()?; + let sources = state.sources()?; + if let Some(previous) = sources + .iter() + .find(|event| event.checkpoint.source == checkpoint.source) + { + if checkpoint.revision == previous.checkpoint.revision { + return if previous.fact == fact { + Ok(previous.clone()) + } else { + Err("checkpoint identity has conflicting facts".into()) + }; + } + if checkpoint.revision < previous.checkpoint.revision { + return Err("stale facts checkpoint".into()); + } + } else if sources.len() == MAX_SOURCES { + return Err("facts source capacity exceeded".into()); + } + let event = Event { + seq: state.last_seq.checked_add(1).ok_or("facts sequence exhausted")?, + at_ms, + checkpoint, + fact, + }; + if event.to_json().map_err(|error| error.to_string())?.len() + 1 > self.journal_bytes { + return Err("event exceeds journal capacity".into()); + } + // Fail capacity/semantic checks before durably reserving an observation. + situation::apply(&mut snapshot, &event)?; + state.pending = Some(event_value(&event)?); + self.root.atomic(STATE, &state.bytes()?)?; + self.recover()?; + Ok(event) + } + pub fn situation(&self) -> Result { + let _lock = self.root.lock(LOCK)?; + self.recover().map(|(_, snapshot, _)| snapshot) + } + pub fn events_after(&self, after: u64, limit: usize) -> Result { + if !(1..=64).contains(&limit) { + return Err("event page limit must be between 1 and 64".into()); + } + let _lock = self.root.lock(LOCK)?; + let (state, _, journal) = self.recover()?; + if after > state.last_seq { + return Err("event cursor is ahead of the journal".into()); + } + let first_available = journal.events.first().map_or(1, |event| event.seq); + let events: Vec<_> = journal + .events + .into_iter() + .filter(|event| event.seq > after) + .take(limit) + .collect(); + let next = events + .last() + .map_or(after.max(first_available - 1), |event| event.seq); + let page = EventPage { + after, + events, + next, + first_available, + gap: after.saturating_add(1) < first_available, + }; + page.to_json().map_err(|error| error.to_string())?; + Ok(page) + } + fn journal(&self, pending: bool) -> Result { + // The only allowed missing current file is an interrupted, durably reserved rotation. + if !self.root.exists(CURRENT)? { + if !pending { + return Err("facts journal is missing".into()); + } + self.root.create(CURRENT, b"")?; + } + let mut events = Vec::new(); + if self.root.exists(PREVIOUS)? { + let bytes = self.root.read(PREVIOUS, self.journal_bytes)?; + parse_lines(&bytes, false, &mut events)?; + } + let bytes = self.root.read(CURRENT, self.journal_bytes)?; + let torn_at = parse_lines(&bytes, pending, &mut events)?; + for pair in events.windows(2) { + if pair[0].seq.checked_add(1) != Some(pair[1].seq) { + return Err("facts journal has a sequence gap".into()); + } + } + Ok(Journal { + events, + current_bytes: bytes.len(), + torn_at, + }) + } + fn recover(&self) -> Result<(State, Snapshot, Journal)> { + let mut state: State = serde_json::from_slice(&self.root.read(STATE, STATE_BYTES)?) + .map_err(|_| "invalid facts metadata")?; + let sources = state.sources()?; + let pending = state.pending.as_ref().map(read_event_value).transpose()?; + let mut journal = self.journal(pending.is_some())?; + let mut snapshot = Snapshot::from_json(&self.root.read(SITUATION, MAX_SNAPSHOT_BYTES)?) + .map_err(|error| error.to_string())?; + let original_seq = snapshot.as_of_seq; + if snapshot.as_of_seq > state.last_seq { + return Err("situation is ahead of committed facts".into()); + } + for event in journal + .events + .iter() + .filter(|event| event.seq > original_seq && event.seq <= state.last_seq) + { + situation::apply(&mut snapshot, event)?; + } + if snapshot.as_of_seq != state.last_seq { + return Err("retained journal cannot recover the situation".into()); + } + if let Some(event) = pending { + if state.last_seq.checked_add(1) != Some(event.seq) + || sources.iter().any(|old| { + old.checkpoint.source == event.checkpoint.source + && old.checkpoint.revision >= event.checkpoint.revision + }) + || !sources + .iter() + .any(|old| old.checkpoint.source == event.checkpoint.source) + && sources.len() >= MAX_SOURCES + { + return Err("invalid pending facts checkpoint".into()); + } + situation::apply(&mut snapshot, &event)?; + if journal.last_seq() == state.last_seq { + if let Some(length) = journal.torn_at { + self.root.truncate(CURRENT, length)?; + journal.current_bytes = length as usize; + } + let mut bytes = event.to_json().map_err(|error| error.to_string())?; + bytes.push(b'\n'); + if bytes.len() > self.journal_bytes { + return Err("event exceeds journal capacity".into()); + } + if journal.current_bytes + bytes.len() > self.journal_bytes { + self.root.rotate(CURRENT, PREVIOUS)?; + } + self.root.append(CURRENT, &bytes)?; + } else if journal.events.last() != Some(&event) || journal.torn_at.is_some() { + return Err("pending fact disagrees with the journal".into()); + } + state.committed(&event)?; + self.root.atomic(STATE, &state.bytes()?)?; + journal = self.journal(false)?; + } else if journal.last_seq() != state.last_seq { + return Err("journal disagrees with committed facts".into()); + } + if snapshot.as_of_seq != original_seq { + self.root + .atomic(SITUATION, &snapshot.to_json().map_err(|error| error.to_string())?)?; + } + Ok((state, snapshot, journal)) + } +} + +fn parse_lines(bytes: &[u8], allow_torn: bool, events: &mut Vec) -> Result> { + let mut offset = 0usize; + for line in bytes.split_inclusive(|byte| *byte == b'\n') { + if !line.ends_with(b"\n") { + return if allow_torn { + Ok(Some(offset as u64)) + } else { + Err("facts journal has an unreserved torn line".into()) + }; + } + if line.len() > MAX_EVENT_BYTES + 1 { + return Err("facts journal line capacity exceeded".into()); + } + events.push(Event::from_json(&line[..line.len() - 1]).map_err(|error| error.to_string())?); + offset += line.len(); + } + Ok(None) +} + +#[cfg(test)] +mod tests; diff --git a/oxa/oxa-work/src/events/tests.rs b/oxa/oxa-work/src/events/tests.rs new file mode 100644 index 000000000..387848616 --- /dev/null +++ b/oxa/oxa-work/src/events/tests.rs @@ -0,0 +1,483 @@ +use super::*; +use oxa_proto::work::*; +use std::fs; +use std::io::Write; +use std::os::unix::fs::{DirBuilderExt, PermissionsExt}; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; + +struct Fixture(PathBuf); +impl Fixture { + fn new() -> Self { + static NEXT: AtomicU64 = AtomicU64::new(0); + loop { + let root = std::env::temp_dir().join(format!( + "oxa-facts-test-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + match fs::DirBuilder::new().mode(0o700).create(&root) { + Ok(()) => return Self(root), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => panic!("create isolated fixture: {error}"), + } + } + } + fn root(&self) -> PathBuf { + self.0.join("facts") + } + fn facts(&self) -> Facts { + Facts::initialize(&self.root()).unwrap() + } +} +impl Drop for Fixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} +fn checkpoint(source: &str, revision: u64) -> Checkpoint { + Checkpoint { + source: source.into(), + revision, + } +} +fn mechanical() -> Fact { + Fact::Mechanical(Mechanical { + subject: Subject::Repository, + kind: MechanicalKind::TransientRetry, + }) +} +fn task(id: usize) -> Running { + Running { + task: format!("task-{id}"), + host: "helper".into(), + branch: format!("agent/task-{id}"), + worktree: format!("~/project/task-{id}"), + session_id: None, + step: TaskStep::Preparing, + } +} +fn reserve(facts: &Facts, fact: Fact) -> Event { + let mut state: State = serde_json::from_slice(&facts.root.read(STATE, STATE_BYTES).unwrap()).unwrap(); + let event = Event { + seq: state.last_seq + 1, + at_ms: 20, + checkpoint: checkpoint("runner", state.last_seq + 1), + fact, + }; + state.pending = Some(event_value(&event).unwrap()); + facts.root.atomic(STATE, &state.bytes().unwrap()).unwrap(); + event +} + +#[test] +fn lifecycle_facts_do_not_turn_worker_success_into_landed_work() { + let env = Fixture::new(); + let facts = env.facts(); + let marker = env.0.join("runner-state.json"); + fs::write(&marker, b"owned only by runner").unwrap(); + let pr = PullRequest { + number: 123, + head: "a".repeat(40), + }; + let observations = [ + Fact::TaskStarted(task(1)), + Fact::TaskTransition(TaskTransition { + task: "task-1".into(), + step: TaskStep::Worker, + session_id: Some("saved".into()), + reason: Some(StepReason::Prepared), + }), + Fact::WorkerFinished(WorkerFinished { + task: "task-1".into(), + session_id: Some("saved".into()), + outcome: WorkerOutcome::Done, + error_class: None, + }), + Fact::LocalAcceptance(LocalAcceptance { + task: "task-1".into(), + commit: pr.head.clone(), + evidence: Evidence { + kind: EvidenceKind::LocalCheck, + id: "checks-1".into(), + }, + }), + Fact::Published(Publication { + task: "task-1".into(), + pr: pr.clone(), + }), + Fact::ExternalAcceptance(ExternalAcceptance { + pr: pr.clone(), + evidence: Evidence { + kind: EvidenceKind::Gate, + id: "pr-run-1".into(), + }, + }), + ]; + for (index, fact) in observations.into_iter().enumerate() { + facts + .append(checkpoint("runner", index as u64 + 1), index as u64, fact) + .unwrap(); + assert!(facts.situation().unwrap().landed.is_empty()); + } + assert_eq!( + facts.situation().unwrap().running[0].session_id.as_deref(), + Some("saved") + ); + facts + .append( + checkpoint("runner", 7), + 7, + Fact::PrMerged(Landed { + pr, + merge_commit: "b".repeat(40), + merged_at_ms: 7, + }), + ) + .unwrap(); + let snapshot = facts.situation().unwrap(); + assert_eq!(snapshot.landed.len(), 1); + assert!(snapshot.published.is_empty()); + assert!(snapshot.local_accepted.is_empty()); + facts + .append( + checkpoint("runner", 8), + 8, + Fact::TaskTransition(TaskTransition { + task: "task-1".into(), + step: TaskStep::Finished, + session_id: Some("saved".into()), + reason: Some(StepReason::Saved), + }), + ) + .unwrap(); + assert!(facts.situation().unwrap().running.is_empty()); + assert_eq!(fs::read(marker).unwrap(), b"owned only by runner"); +} + +#[test] +fn lost_acknowledgements_deduplicate_across_restart_and_changed_clocks() { + let env = Fixture::new(); + let facts = env.facts(); + let original = facts.append(checkpoint("runner", 1), 10, mechanical()).unwrap(); + let reopened = Facts::open(&env.root()).unwrap(); + assert_eq!( + reopened + .append(checkpoint("runner", 1), 999, mechanical()) + .unwrap(), + original + ); + assert!(reopened + .append( + checkpoint("runner", 1), + 999, + Fact::Enqueued(PullRequest { + number: 123, + head: "a".repeat(40) + }) + ) + .is_err()); + reopened + .append(checkpoint("runner", 2), 20, mechanical()) + .unwrap(); + assert!(reopened + .append(checkpoint("runner", 1), 10, mechanical()) + .is_err()); + assert_eq!(reopened.events_after(0, 64).unwrap().events.len(), 2); +} + +#[test] +fn repository_revision_gaps_do_not_create_event_gaps_or_per_task_tombstones() { + let env = Fixture::new(); + let facts = env.facts(); + for revision in [3, 8, 100] { + facts + .append(checkpoint("runner", revision), revision, mechanical()) + .unwrap(); + } + let page = facts.events_after(0, 64).unwrap(); + assert_eq!( + page.events.iter().map(|event| event.seq).collect::>(), + [1, 2, 3] + ); + let state: State = serde_json::from_slice(&facts.root.read(STATE, STATE_BYTES).unwrap()).unwrap(); + assert_eq!(state.last_by_source.len(), 1); +} + +#[test] +fn a_reserved_torn_line_is_replayed_once_and_the_situation_catches_up() { + let env = Fixture::new(); + let facts = env.facts(); + facts.append(checkpoint("runner", 1), 10, mechanical()).unwrap(); + let pending = reserve(&facts, mechanical()); + let mut file = fs::OpenOptions::new() + .append(true) + .open(env.root().join(CURRENT)) + .unwrap(); + file.write_all(b"{\"type\":\"oxa_work_event\"").unwrap(); + file.sync_all().unwrap(); + let recovered = Facts::open(&env.root()).unwrap(); + assert_eq!(recovered.situation().unwrap().as_of_seq, 2); + assert_eq!( + recovered + .append(pending.checkpoint.clone(), 999, pending.fact.clone()) + .unwrap(), + pending + ); + assert_eq!(recovered.events_after(0, 64).unwrap().events.len(), 2); + assert!(fs::read(env.root().join(CURRENT)).unwrap().ends_with(b"\n")); +} + +#[test] +fn an_appended_pending_fact_and_a_lagging_snapshot_recover_without_another_line() { + for committed in [false, true] { + let env = Fixture::new(); + let facts = env.facts(); + facts.append(checkpoint("runner", 1), 10, mechanical()).unwrap(); + let pending = reserve(&facts, mechanical()); + let mut bytes = pending.to_json().unwrap(); + bytes.push(b'\n'); + facts.root.append(CURRENT, &bytes).unwrap(); + if committed { + let mut state: State = + serde_json::from_slice(&facts.root.read(STATE, STATE_BYTES).unwrap()).unwrap(); + state.committed(&pending).unwrap(); + facts.root.atomic(STATE, &state.bytes().unwrap()).unwrap(); + } + let recovered = Facts::open(&env.root()).unwrap(); + assert_eq!(recovered.situation().unwrap().as_of_seq, 2); + assert_eq!(recovered.events_after(0, 64).unwrap().events.len(), 2); + } +} + +#[test] +fn interruption_between_rotation_and_new_current_file_recovers_the_reserved_event() { + let env = Fixture::new(); + let facts = env.facts(); + facts.append(checkpoint("runner", 1), 10, mechanical()).unwrap(); + reserve(&facts, mechanical()); + fs::rename(env.root().join(CURRENT), env.root().join(PREVIOUS)).unwrap(); + let recovered = Facts::open(&env.root()).unwrap(); + assert_eq!( + recovered + .events_after(0, 64) + .unwrap() + .events + .iter() + .map(|event| event.seq) + .collect::>(), + [1, 2] + ); +} + +#[test] +fn rotation_is_bounded_and_cursor_gaps_are_explicit_without_resetting_sequences() { + let env = Fixture::new(); + let mut facts = env.facts(); + facts.journal_bytes = 1024; + let quiet = facts.append(checkpoint("quiet", 1), 0, mechanical()).unwrap(); + for revision in 1..=40 { + facts + .append(checkpoint("busy", revision), revision, mechanical()) + .unwrap(); + } + for leaf in [CURRENT, PREVIOUS] { + assert!(fs::metadata(env.root().join(leaf)).unwrap().len() <= 1024); + } + let page = facts.events_after(0, 64).unwrap(); + assert!(page.gap && page.first_available > 1); + assert_eq!(page.events[0].seq, page.first_available); + assert_eq!(page.next, 41); + assert_eq!( + facts.append(checkpoint("quiet", 1), 999, mechanical()).unwrap(), + quiet + ); + assert_eq!(facts.situation().unwrap().as_of_seq, 41); + assert!(facts.events_after(42, 64).is_err()); + assert!(facts.events_after(0, 0).is_err()); +} + +#[test] +fn concurrent_sources_get_unique_contiguous_repository_sequences() { + let env = Fixture::new(); + let facts = env.facts(); + std::thread::scope(|scope| { + for source in ["one", "two", "three", "four"] { + let facts = &facts; + scope.spawn(move || { + for revision in 1..=8 { + facts + .append(checkpoint(source, revision), revision, mechanical()) + .unwrap(); + } + }); + } + }); + let page = facts.events_after(0, 64).unwrap(); + assert_eq!( + page.events.iter().map(|event| event.seq).collect::>(), + (1..=32).collect::>() + ); +} + +#[test] +fn active_capacity_is_an_error_and_only_recent_terminal_history_is_trimmed() { + let env = Fixture::new(); + let facts = env.facts(); + for id in 0..MAX_ITEMS { + facts + .append( + checkpoint("runner", id as u64 + 1), + 0, + Fact::TaskStarted(task(id)), + ) + .unwrap(); + } + assert!(facts + .append( + checkpoint("runner", MAX_ITEMS as u64 + 1), + 0, + Fact::TaskStarted(task(MAX_ITEMS)) + ) + .is_err()); + assert_eq!(facts.situation().unwrap().running.len(), MAX_ITEMS); + assert_eq!(facts.situation().unwrap().as_of_seq, MAX_ITEMS as u64); + let history = Fixture::new(); + let facts = history.facts(); + for id in 1..=20 { + facts + .append( + checkpoint("runner", id), + id, + Fact::Incident(Incident { + id: format!("incident-{id}"), + subject: Subject::Repository, + kind: IncidentKind::NoBuildRequeue, + evidence: None, + }), + ) + .unwrap(); + } + let snapshot = facts.situation().unwrap(); + assert!(snapshot.history_truncated); + assert_eq!(snapshot.incidents.len(), 16); + assert_eq!(crate::incidents::recent(&facts).unwrap()[0].id, "incident-5"); +} + +#[test] +fn head_changes_and_unrelated_facts_do_not_silently_clear_blocks_or_asks() { + let env = Fixture::new(); + let facts = env.facts(); + let subject = Subject::PullRequest(123); + facts + .append( + checkpoint("runner", 1), + 1, + Fact::Blocked(Blocked { + subject: subject.clone(), + code: "held".into(), + ask_id: Some("ask-1".into()), + }), + ) + .unwrap(); + facts + .append( + checkpoint("runner", 2), + 2, + Fact::AskOpened(Ask { + id: "ask-1".into(), + subject: subject.clone(), + question: "Release hold?".into(), + }), + ) + .unwrap(); + facts + .append( + checkpoint("runner", 3), + 3, + Fact::Published(Publication { + task: "task-1".into(), + pr: PullRequest { + number: 123, + head: "b".repeat(40), + }, + }), + ) + .unwrap(); + let snapshot = facts.situation().unwrap(); + assert_eq!(snapshot.blocked.len(), 1); + assert_eq!(snapshot.asks.len(), 1); + facts + .append(checkpoint("runner", 4), 4, Fact::Unblocked(subject)) + .unwrap(); + facts + .append( + checkpoint("runner", 5), + 5, + Fact::AskClosed(AskClosed { id: "ask-1".into() }), + ) + .unwrap(); + let snapshot = facts.situation().unwrap(); + assert!(snapshot.blocked.is_empty() && snapshot.asks.is_empty()); +} + +#[test] +fn missing_corrupt_and_unreserved_torn_files_fail_instead_of_becoming_empty() { + for leaf in [STATE, SITUATION, CURRENT] { + let env = Fixture::new(); + let _facts = env.facts(); + fs::remove_file(env.root().join(leaf)).unwrap(); + assert!(Facts::open(&env.root()).is_err()); + assert!(Facts::initialize(&env.root()).is_err()); + } + for leaf in [STATE, SITUATION, CURRENT] { + let env = Fixture::new(); + let _facts = env.facts(); + fs::write(env.root().join(leaf), b"PRIVATE_MARKER").unwrap(); + let error = Facts::open(&env.root()).unwrap_err(); + assert!(!error.contains("PRIVATE_MARKER")); + } +} + +#[test] +fn poisoned_leaves_and_root_symlinks_never_read_or_replace_the_target() { + for leaf in [STATE, SITUATION, CURRENT, LOCK] { + let env = Fixture::new(); + let _facts = env.facts(); + let target = env.0.join("outside"); + fs::write(&target, b"keep this target").unwrap(); + fs::remove_file(env.root().join(leaf)).unwrap(); + std::os::unix::fs::symlink(&target, env.root().join(leaf)).unwrap(); + assert!(Facts::open(&env.root()).is_err()); + assert_eq!(fs::read(target).unwrap(), b"keep this target"); + } + let env = Fixture::new(); + let target = env.0.join("outside"); + fs::create_dir(&target).unwrap(); + std::os::unix::fs::symlink(&target, env.root()).unwrap(); + assert!(Facts::initialize(&env.root()).is_err()); + assert!(fs::read_dir(target).unwrap().next().is_none()); +} + +#[test] +fn artifacts_are_private_and_source_capacity_does_not_consume_a_sequence() { + let env = Fixture::new(); + let facts = env.facts(); + assert_eq!( + fs::metadata(env.root()).unwrap().permissions().mode() & 0o777, + 0o700 + ); + for leaf in [STATE, SITUATION, CURRENT, LOCK] { + assert_eq!( + fs::metadata(env.root().join(leaf)).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } + for source in 0..MAX_SOURCES { + facts + .append(checkpoint(&format!("source-{source}"), 1), 0, mechanical()) + .unwrap(); + } + assert!(facts.append(checkpoint("too-many", 1), 0, mechanical()).is_err()); + assert_eq!(facts.situation().unwrap().as_of_seq, MAX_SOURCES as u64); +} diff --git a/oxa/oxa-work/src/facts_io.rs b/oxa/oxa-work/src/facts_io.rs new file mode 100644 index 000000000..f55f20855 --- /dev/null +++ b/oxa/oxa-work/src/facts_io.rs @@ -0,0 +1,203 @@ +//! Private files under the caller's protected repository root, outside writable Git binds. +use crate::Result; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +use std::os::unix::fs::{DirBuilderExt, MetadataExt, OpenOptionsExt}; +use std::path::{Component, Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +fn io(result: std::io::Result, what: &str) -> Result { + result.map_err(|error| format!("{what}: {:?}", error.kind())) +} +fn private(metadata: &fs::Metadata, directory: bool) -> Result<()> { + // SAFETY: geteuid has no arguments, pointers or failure mode. + let uid = unsafe { libc::geteuid() }; + if metadata.uid() != uid + || metadata.mode() & 0o077 != 0 + || if directory { + !metadata.is_dir() + } else { + !metadata.is_file() + } + { + return Err("facts storage must be private and owned by this user".into()); + } + Ok(()) +} + +#[derive(Clone, Debug)] +pub(crate) struct Root(PathBuf); +impl Root { + pub(crate) fn open(path: &Path, create: bool) -> Result { + if !path.is_absolute() { + return Err("facts root must be absolute".into()); + } + let mut parent = PathBuf::new(); + for component in path.components() { + if matches!(component, Component::ParentDir) { + return Err("facts root must be canonical".into()); + } + parent.push(component); + match fs::symlink_metadata(&parent) { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err("facts root contains a symlink".into()) + } + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound && parent == path && create => { + io( + fs::DirBuilder::new().mode(0o700).create(path), + "cannot create facts root", + )?; + } + Err(error) => return Err(format!("cannot inspect facts root: {:?}", error.kind())), + } + } + let metadata = io(fs::symlink_metadata(path), "cannot inspect facts root")?; + private(&metadata, true)?; + Ok(Self(path.to_path_buf())) + } + fn path(&self, leaf: &str) -> PathBuf { + debug_assert!(!leaf.contains('/') && leaf != "." && leaf != ".."); + self.0.join(leaf) + } + fn options(&self) -> OpenOptions { + let mut options = OpenOptions::new(); + options + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) + .mode(0o600); + options + } + pub(crate) fn exists(&self, leaf: &str) -> Result { + match fs::symlink_metadata(self.path(leaf)) { + Ok(metadata) => { + private(&metadata, false)?; + Ok(true) + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(format!("cannot inspect facts file: {:?}", error.kind())), + } + } + pub(crate) fn lock(&self, leaf: &str) -> Result { + let file = io( + self.options() + .read(true) + .write(true) + .create(true) + .open(self.path(leaf)), + "cannot open facts lock", + )?; + private(&io(file.metadata(), "cannot inspect facts lock")?, false)?; + io(file.lock(), "cannot lock facts storage")?; + Ok(file) + } + pub(crate) fn read(&self, leaf: &str, limit: usize) -> Result> { + let file = io( + self.options().read(true).open(self.path(leaf)), + "cannot open facts file", + )?; + let metadata = io(file.metadata(), "cannot inspect facts file")?; + private(&metadata, false)?; + if metadata.len() > limit as u64 { + return Err("facts file capacity exceeded".into()); + } + let mut bytes = Vec::new(); + io( + file.take(limit as u64 + 1).read_to_end(&mut bytes), + "cannot read facts file", + )?; + if bytes.len() > limit { + return Err("facts file capacity exceeded".into()); + } + Ok(bytes) + } + pub(crate) fn create(&self, leaf: &str, bytes: &[u8]) -> Result<()> { + let mut file = io( + self.options().write(true).create_new(true).open(self.path(leaf)), + "cannot create facts file", + )?; + io( + file.write_all(bytes).and_then(|()| file.sync_all()), + "cannot save facts file", + )?; + self.sync() + } + pub(crate) fn atomic(&self, leaf: &str, bytes: &[u8]) -> Result<()> { + // Inspect the existing leaf without following it; never replace poisoned enforcement data. + self.exists(leaf)?; + static NEXT: AtomicU64 = AtomicU64::new(0); + for _ in 0..16 { + let sequence = NEXT.fetch_add(1, Ordering::Relaxed); + let temporary = format!(".{leaf}.tmp-{}-{sequence}", std::process::id()); + let mut file = match self + .options() + .write(true) + .create_new(true) + .open(self.path(&temporary)) + { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(format!("cannot create facts temporary: {:?}", error.kind())), + }; + let result = io( + file.write_all(bytes).and_then(|()| file.sync_all()), + "cannot save facts temporary", + ) + .and_then(|()| { + io( + fs::rename(self.path(&temporary), self.path(leaf)), + "cannot replace facts file", + ) + }) + .and_then(|()| self.sync()); + if result.is_err() { + let _ = fs::remove_file(self.path(&temporary)); + } + return result; + } + Err("cannot reserve facts temporary".into()) + } + pub(crate) fn append(&self, leaf: &str, bytes: &[u8]) -> Result<()> { + let mut file = io( + self.options().append(true).open(self.path(leaf)), + "cannot open facts journal", + )?; + private(&io(file.metadata(), "cannot inspect facts journal")?, false)?; + io( + file.write_all(bytes).and_then(|()| file.sync_all()), + "cannot append facts journal", + ) + } + pub(crate) fn truncate(&self, leaf: &str, length: u64) -> Result<()> { + let file = io( + self.options().write(true).open(self.path(leaf)), + "cannot open torn facts journal", + )?; + private(&io(file.metadata(), "cannot inspect torn facts journal")?, false)?; + io( + file.set_len(length).and_then(|()| file.sync_all()), + "cannot recover torn facts journal", + ) + } + pub(crate) fn rotate(&self, from: &str, to: &str) -> Result<()> { + if !self.exists(from)? { + return Err("facts journal is missing".into()); + } + self.exists(to)?; + io( + fs::rename(self.path(from), self.path(to)), + "cannot rotate facts journal", + )?; + self.sync()?; + self.create(from, b"") + } + fn sync(&self) -> Result<()> { + let directory = io( + self.options() + .read(true) + .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW) + .open(&self.0), + "cannot open facts directory", + )?; + io(directory.sync_all(), "cannot sync facts directory") + } +} diff --git a/oxa/oxa-work/src/incidents.rs b/oxa/oxa-work/src/incidents.rs new file mode 100644 index 000000000..e97d1efaf --- /dev/null +++ b/oxa/oxa-work/src/incidents.rs @@ -0,0 +1,194 @@ +//! Bounded incident reads and pure stall detection. A stall produces an incident, never a kill. +use crate::{events::Facts, Result}; +use oxa_proto::work::{Evidence, EvidenceKind, Incident, IncidentKind, Subject}; +use serde_json::Value; + +pub fn recent(facts: &Facts) -> Result> { + facts.situation().map(|snapshot| snapshot.incidents) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Stall { + pub session_id: String, + pub pid: u32, + pub last_progress_ms: u64, + pub deadline_ms: u64, +} +impl Stall { + /// Build an incident for the runner to append. This never wraps, cancels or kills a worker. + pub fn incident(&self, task: &str) -> Incident { + Incident { + id: format!("stall:{}:{}:{}", self.session_id, self.pid, self.last_progress_ms), + subject: Subject::Task(task.into()), + kind: IncidentKind::WorkerStalled, + evidence: Some(Evidence { + kind: EvidenceKind::Status, + id: format!("session:{}", self.session_id), + }), + } + } +} +/// Consume a bounded status snapshot supplied by the runner, never a transcript or tool output. +pub fn detect_stall(bytes: &[u8], session_id: &str, pid: u32, now_ms: u64) -> Result> { + if pid == 0 + || session_id.is_empty() + || session_id.len() > 128 + || !session_id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"-_".contains(&byte)) + { + return Err("invalid expected worker identity".into()); + } + if bytes.len() > 4096 { + return Err("worker status capacity exceeded".into()); + } + let status: Value = serde_json::from_slice(bytes).map_err(|_| "invalid worker status JSON")?; + if status["type"] != "oxa_status" + || status["version"] != 1 + || status["session_id"] != session_id + || status["pid"].as_u64() != Some(pid.into()) + { + return Err("worker status identity or version mismatch".into()); + } + let phase = status["phase"].as_str().ok_or("worker status phase is missing")?; + let outcome = status.get("outcome").ok_or("worker status outcome is missing")?; + let deadline = status + .get("stall_deadline_ms") + .ok_or("worker status deadline is missing")?; + let timeout = status + .get("stall_timeout_ms") + .ok_or("worker status timeout is missing")?; + let timeout = if timeout.is_null() { + None + } else { + Some( + timeout + .as_u64() + .filter(|timeout| *timeout > 0) + .ok_or("invalid worker stall timeout")?, + ) + }; + if !matches!( + phase, + "idle" | "working" | "model" | "prefill" | "compaction" | "tool" | "closing" | "finished" + ) { + return Err("unknown worker status phase".into()); + } + if phase == "finished" { + if outcome["type"] != "oxa_result" + || !matches!( + outcome["outcome"].as_str(), + Some("done" | "nothing_to_do" | "blocked" | "failed" | "interrupted") + ) + || outcome["exit_code"].as_i64().is_none() + || !deadline.is_null() + { + return Err("finished worker status has no outcome".into()); + } + return Ok(None); + } + if !outcome.is_null() { + return Err("active worker status has a terminal outcome".into()); + } + let progress = status["last_progress_ms"] + .as_u64() + .ok_or("worker status progress is missing")?; + if deadline.is_null() { + return Ok(None); + } + let deadline = deadline.as_u64().ok_or("invalid worker status deadline")?; + let timeout = timeout.ok_or("worker deadline has no configured timeout")?; + let earliest = progress.saturating_add(timeout); + if deadline < earliest || deadline > earliest.saturating_add(1000) { + return Err("worker status deadline is inconsistent with its timeout".into()); + } + Ok((now_ms >= deadline).then(|| Stall { + session_id: session_id.into(), + pid, + last_progress_ms: progress, + deadline_ms: deadline, + })) +} + +#[cfg(test)] +mod tests { + use super::*; + use oxa_proto::work::{Checkpoint, Event, Fact}; + use serde_json::json; + fn status() -> Value { + json!({"type":"oxa_status", "version":1, "session_id":"saved", "pid":123, + "phase":"model", "last_progress_ms":100, "stall_timeout_ms":100, "stall_deadline_ms":200, "outcome":null}) + } + fn bytes(value: &Value) -> Vec { + serde_json::to_vec(value).unwrap() + } + + #[test] + fn deadline_is_conservative_and_detection_never_changes_the_input_or_takes_action() { + let input = bytes(&status()); + let original = input.clone(); + assert!(detect_stall(&input, "saved", 123, 199).unwrap().is_none()); + let stall = detect_stall(&input, "saved", 123, 200).unwrap().unwrap(); + assert_eq!(stall.last_progress_ms, 100); + assert_eq!(stall.deadline_ms, 200); + let event = Event { + seq: 1, + at_ms: 200, + checkpoint: Checkpoint { + source: "runner".into(), + revision: 1, + }, + fact: Fact::Incident(stall.incident("task-1")), + }; + assert_eq!(Event::from_json(&event.to_json().unwrap()).unwrap(), event); + assert_eq!(input, original); + let mut progress = status(); + progress["last_progress_ms"] = json!(190); + progress["stall_deadline_ms"] = json!(290); + assert!(detect_stall(&bytes(&progress), "saved", 123, 200) + .unwrap() + .is_none()); + } + + #[test] + fn stale_identity_missing_fields_and_invalid_status_are_errors() { + let input = bytes(&status()); + assert!(detect_stall(&input, "different", 123, 300).is_err()); + assert!(detect_stall(&input, "saved", 456, 300).is_err()); + for field in [ + "pid", + "phase", + "last_progress_ms", + "stall_deadline_ms", + "stall_timeout_ms", + "outcome", + ] { + let mut invalid = status(); + invalid.as_object_mut().unwrap().remove(field); + assert!( + detect_stall(&bytes(&invalid), "saved", 123, 300).is_err(), + "{field}" + ); + } + let mut invalid = status(); + invalid["stall_deadline_ms"] = json!(50); + assert!(detect_stall(&bytes(&invalid), "saved", 123, 300).is_err()); + assert!(detect_stall(&vec![b' '; 4097], "saved", 123, 300).is_err()); + assert!(detect_stall(b"{}", "saved", 123, 300).is_err()); + } + + #[test] + fn idle_disabled_and_finished_sessions_do_not_stall() { + let mut value = status(); + value["phase"] = json!("idle"); + value["stall_deadline_ms"] = Value::Null; + assert!(detect_stall(&bytes(&value), "saved", 123, 300).unwrap().is_none()); + value["phase"] = json!("model"); + assert!(detect_stall(&bytes(&value), "saved", 123, 300).unwrap().is_none()); + value["phase"] = json!("finished"); + value["outcome"] = json!({"type":"oxa_result", "outcome":"done", "exit_code":0}); + assert!(detect_stall(&bytes(&value), "saved", 123, 300).unwrap().is_none()); + value["outcome"] = Value::Null; + assert!(detect_stall(&bytes(&value), "saved", 123, 300).is_err()); + } +} diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index dc74e7942..51e0ec533 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -11,6 +11,12 @@ pub use worker::{WorkerOutcome, WorkerResult}; pub type Result = std::result::Result; +// Lead facts consume public observations and never open runner task-state files. +pub mod events; +mod facts_io; +pub mod incidents; +pub mod situation; + #[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub struct Task { diff --git a/oxa/oxa-work/src/situation.rs b/oxa/oxa-work/src/situation.rs new file mode 100644 index 000000000..3a84f03e1 --- /dev/null +++ b/oxa/oxa-work/src/situation.rs @@ -0,0 +1,132 @@ +//! Bounded projection of typed facts. This module never reads or writes runner task state. +use crate::Result; +use oxa_proto::work::{Event, Fact, Snapshot, TaskStep, MAX_ITEMS}; + +pub(crate) fn empty() -> Snapshot { + Snapshot { + as_of_seq: 0, + at_ms: 0, + standing_revision: 0, + goal: None, + running: vec![], + local_accepted: vec![], + published: vec![], + landed: vec![], + blocked: vec![], + asks: vec![], + incidents: vec![], + history_truncated: false, + } +} + +pub(crate) fn apply(snapshot: &mut Snapshot, event: &Event) -> Result<()> { + if snapshot.as_of_seq.checked_add(1) != Some(event.seq) { + return Err("situation replay has a sequence gap".into()); + } + match &event.fact { + Fact::TaskStarted(task) => { + snapshot.running.retain(|row| row.task != task.task); + snapshot.running.push(task.clone()); + } + Fact::TaskTransition(change) => { + if change.step == TaskStep::Finished { + snapshot.running.retain(|row| row.task != change.task); + } else { + let task = snapshot + .running + .iter_mut() + .find(|row| row.task == change.task) + .ok_or("task transition has no running location")?; + task.step = change.step; + task.session_id = change.session_id.clone(); + } + } + Fact::WorkerFinished(worker) => { + if let Some(task) = snapshot.running.iter_mut().find(|row| row.task == worker.task) { + task.session_id = worker.session_id.clone(); + } + } + Fact::LocalAcceptance(accepted) => { + snapshot.local_accepted.retain(|row| row.task != accepted.task); + snapshot.local_accepted.push(accepted.clone()); + } + Fact::Published(publication) => { + snapshot.published.retain(|row| row.task != publication.task); + snapshot.published.push(publication.clone()); + } + Fact::PrMerged(landed) => { + let tasks: Vec<_> = snapshot + .published + .iter() + .filter(|row| row.pr == landed.pr) + .map(|row| row.task.clone()) + .collect(); + snapshot + .local_accepted + .retain(|row| !(tasks.contains(&row.task) && row.commit == landed.pr.head)); + snapshot.published.retain(|row| row.pr != landed.pr); + snapshot.landed.retain(|row| row.pr.number != landed.pr.number); + snapshot.landed.push(landed.clone()); + } + Fact::Blocked(block) => { + snapshot.blocked.retain(|row| row.subject != block.subject); + snapshot.blocked.push(block.clone()); + } + Fact::Unblocked(subject) => snapshot.blocked.retain(|row| &row.subject != subject), + Fact::AskOpened(ask) => { + if let Some(existing) = snapshot.asks.iter().find(|row| row.id == ask.id) { + if existing != ask { + return Err("ask id has conflicting contents".into()); + } + } else { + snapshot.asks.push(ask.clone()); + } + } + Fact::AskClosed(ask) => snapshot.asks.retain(|row| row.id != ask.id), + Fact::Incident(incident) => { + if let Some(existing) = snapshot.incidents.iter().find(|row| row.id == incident.id) { + if existing != incident { + return Err("incident id has conflicting contents".into()); + } + } else { + snapshot.incidents.push(incident.clone()); + } + } + Fact::ExternalAcceptance(_) | Fact::Enqueued(_) | Fact::Mechanical(_) => {} + } + const HISTORY: usize = 16; + for removed in [ + snapshot + .landed + .drain(..snapshot.landed.len().saturating_sub(HISTORY)) + .count(), + snapshot + .incidents + .drain(..snapshot.incidents.len().saturating_sub(HISTORY)) + .count(), + ] { + snapshot.history_truncated |= removed > 0; + } + while snapshot.running.len() + + snapshot.local_accepted.len() + + snapshot.published.len() + + snapshot.landed.len() + + snapshot.blocked.len() + + snapshot.asks.len() + + snapshot.incidents.len() + > MAX_ITEMS + { + if !snapshot.landed.is_empty() { + snapshot.landed.remove(0); + } else if !snapshot.incidents.is_empty() { + snapshot.incidents.remove(0); + } else { + return Err("active situation capacity exceeded".into()); + } + snapshot.history_truncated = true; + } + snapshot.as_of_seq = event.seq; + snapshot.at_ms = event.at_ms; + snapshot.to_json().map_err(|error| error.to_string())?; + Ok(()) +} From f765d3b06a80e4af0a43cf6424a5cb235501d491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:07:55 +0000 Subject: [PATCH 3/9] Commit a prepared fact without replaying the journal twice --- oxa/oxa-work/src/events.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/oxa/oxa-work/src/events.rs b/oxa/oxa-work/src/events.rs index e70e267b0..20530c36d 100644 --- a/oxa/oxa-work/src/events.rs +++ b/oxa/oxa-work/src/events.rs @@ -137,7 +137,7 @@ impl Facts { /// a lost acknowledgement, a torn last line or an interrupted snapshot replacement. pub fn append(&self, checkpoint: Checkpoint, at_ms: u64, fact: Fact) -> Result { let _lock = self.root.lock(LOCK)?; - let (mut state, mut snapshot, _) = self.recover()?; + let (mut state, mut snapshot, journal) = self.recover()?; let sources = state.sources()?; if let Some(previous) = sources .iter() @@ -162,14 +162,23 @@ impl Facts { checkpoint, fact, }; - if event.to_json().map_err(|error| error.to_string())?.len() + 1 > self.journal_bytes { + let mut bytes = event.to_json().map_err(|error| error.to_string())?; + bytes.push(b'\n'); + if bytes.len() > self.journal_bytes { return Err("event exceeds journal capacity".into()); } // Fail capacity/semantic checks before durably reserving an observation. situation::apply(&mut snapshot, &event)?; state.pending = Some(event_value(&event)?); self.root.atomic(STATE, &state.bytes()?)?; - self.recover()?; + if journal.current_bytes + bytes.len() > self.journal_bytes { + self.root.rotate(CURRENT, PREVIOUS)?; + } + self.root.append(CURRENT, &bytes)?; + state.committed(&event)?; + self.root.atomic(STATE, &state.bytes()?)?; + self.root + .atomic(SITUATION, &snapshot.to_json().map_err(|error| error.to_string())?)?; Ok(event) } pub fn situation(&self) -> Result { From f0da0e0e68728e377e53de79b8dde95ca17dd04d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:59:05 +0000 Subject: [PATCH 4/9] test(work): reproduce facts byte retention and parent durability failures --- oxa/oxa-work/src/events/tests.rs | 76 ++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/oxa/oxa-work/src/events/tests.rs b/oxa/oxa-work/src/events/tests.rs index 387848616..54a7be3a0 100644 --- a/oxa/oxa-work/src/events/tests.rs +++ b/oxa/oxa-work/src/events/tests.rs @@ -364,6 +364,82 @@ fn active_capacity_is_an_error_and_only_recent_terminal_history_is_trimmed() { assert_eq!(crate::incidents::recent(&facts).unwrap()[0].id, "incident-5"); } +#[test] +fn byte_capacity_trims_history_before_refusing_active_asks() { + let env = Fixture::new(); + let facts = env.facts(); + for id in 0..16 { + facts + .append( + checkpoint("runner", id + 1), + 0, + Fact::Incident(Incident { + id: format!("{id:016}{}", "i".repeat(240)), + subject: Subject::Task("t".repeat(128)), + kind: IncidentKind::NoBuildRequeue, + evidence: Some(Evidence { + kind: EvidenceKind::Gate, + id: "e".repeat(256), + }), + }), + ) + .unwrap(); + } + for id in 0..95 { + facts + .append( + checkpoint("runner", id + 17), + 0, + Fact::AskOpened(Ask { + id: format!("ask-{id}"), + subject: Subject::Repository, + question: "q".repeat(512), + }), + ) + .unwrap(); + } + let snapshot = facts.situation().unwrap(); + assert_eq!(snapshot.as_of_seq, 111); + assert_eq!(snapshot.asks.len(), 95); + assert!(snapshot.history_truncated); + assert!(snapshot.incidents.len() < 16); + assert!(snapshot.to_json().unwrap().len() <= MAX_SNAPSHOT_BYTES); + assert_eq!(Facts::open(&env.root()).unwrap().situation().unwrap(), snapshot); +} + +#[test] +fn parent_directory_sync_error_blocks_initialization() { + const CHILD_ROOT: &str = "OXA_FACTS_TEST_UNREADABLE_PARENT"; + if let Some(root) = std::env::var_os(CHILD_ROOT) { + let error = Facts::initialize(Path::new(&root)).expect_err("parent cannot be synced"); + assert!(error.contains("directory"), "{error}"); + return; + } + let env = Fixture::new(); + let mut child = std::process::Command::new(std::env::current_exe().unwrap()); + child.env_clear().env(CHILD_ROOT, env.root()).args([ + "--exact", + "events::tests::parent_directory_sync_error_blocks_initialization", + ]); + // Root can read mode 0300. Exercise the same real permission failure as a normal user. + // SAFETY: geteuid has no arguments, pointers or failure mode. + if unsafe { libc::geteuid() } == 0 { + use std::os::unix::process::CommandExt; + std::os::unix::fs::chown(&env.0, Some(65534), Some(65534)).unwrap(); + child.uid(65534).gid(65534); + } + fs::set_permissions(&env.0, fs::Permissions::from_mode(0o300)).unwrap(); + let output = child.output(); + fs::set_permissions(&env.0, fs::Permissions::from_mode(0o700)).unwrap(); + let output = output.unwrap(); + assert!( + output.status.success(), + "child failed: {}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + #[test] fn head_changes_and_unrelated_facts_do_not_silently_clear_blocks_or_asks() { let env = Fixture::new(); From 65f463c876c1fc2325bacc4e2ad53c942b293333 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:00:32 +0000 Subject: [PATCH 5/9] fix(work): retain active facts within byte bounds and sync root parent --- oxa/oxa-work/src/events/tests.rs | 19 +++++++++++++++++++ oxa/oxa-work/src/facts_io.rs | 25 +++++++++++++++++-------- oxa/oxa-work/src/situation.rs | 25 ++++++++++++++++++------- 3 files changed, 54 insertions(+), 15 deletions(-) diff --git a/oxa/oxa-work/src/events/tests.rs b/oxa/oxa-work/src/events/tests.rs index 54a7be3a0..f754da8a5 100644 --- a/oxa/oxa-work/src/events/tests.rs +++ b/oxa/oxa-work/src/events/tests.rs @@ -405,6 +405,25 @@ fn byte_capacity_trims_history_before_refusing_active_asks() { assert!(snapshot.incidents.len() < 16); assert!(snapshot.to_json().unwrap().len() <= MAX_SNAPSHOT_BYTES); assert_eq!(Facts::open(&env.root()).unwrap().situation().unwrap(), snapshot); + for id in 95..MAX_ITEMS { + let before = facts.situation().unwrap(); + let result = facts.append( + checkpoint("runner", id as u64 + 17), + 0, + Fact::AskOpened(Ask { + id: format!("ask-{id}"), + subject: Subject::Repository, + question: "q".repeat(512), + }), + ); + if result.is_err() { + assert!(before.incidents.is_empty()); + assert_eq!(facts.situation().unwrap(), before); + return; + } + assert_eq!(facts.situation().unwrap().asks.len(), id + 1); + } + panic!("active asks alone must reach the byte limit before the row limit"); } #[test] diff --git a/oxa/oxa-work/src/facts_io.rs b/oxa/oxa-work/src/facts_io.rs index f55f20855..34029fd57 100644 --- a/oxa/oxa-work/src/facts_io.rs +++ b/oxa/oxa-work/src/facts_io.rs @@ -54,6 +54,11 @@ impl Root { } let metadata = io(fs::symlink_metadata(path), "cannot inspect facts root")?; private(&metadata, true)?; + if create { + // A synced child does not persist its entry in the containing directory. + // Also sync on an initialization retry after an earlier mkdir/sync failure. + sync_directory(path.parent().ok_or("facts root has no parent directory")?)?; + } Ok(Self(path.to_path_buf())) } fn path(&self, leaf: &str) -> PathBuf { @@ -191,13 +196,17 @@ impl Root { self.create(from, b"") } fn sync(&self) -> Result<()> { - let directory = io( - self.options() - .read(true) - .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW) - .open(&self.0), - "cannot open facts directory", - )?; - io(directory.sync_all(), "cannot sync facts directory") + sync_directory(&self.0) } } + +fn sync_directory(path: &Path) -> Result<()> { + let directory = io( + OpenOptions::new() + .read(true) + .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW) + .open(path), + "cannot open facts directory", + )?; + io(directory.sync_all(), "cannot sync facts directory") +} diff --git a/oxa/oxa-work/src/situation.rs b/oxa/oxa-work/src/situation.rs index 3a84f03e1..fd9f67fef 100644 --- a/oxa/oxa-work/src/situation.rs +++ b/oxa/oxa-work/src/situation.rs @@ -116,17 +116,28 @@ pub(crate) fn apply(snapshot: &mut Snapshot, event: &Event) -> Result<()> { + snapshot.incidents.len() > MAX_ITEMS { - if !snapshot.landed.is_empty() { - snapshot.landed.remove(0); - } else if !snapshot.incidents.is_empty() { - snapshot.incidents.remove(0); - } else { + if !discard_history(snapshot) { return Err("active situation capacity exceeded".into()); } - snapshot.history_truncated = true; } snapshot.as_of_seq = event.seq; snapshot.at_ms = event.at_ms; - snapshot.to_json().map_err(|error| error.to_string())?; + while let Err(error) = snapshot.to_json() { + if !discard_history(snapshot) { + return Err(error.to_string()); + } + } Ok(()) } + +fn discard_history(snapshot: &mut Snapshot) -> bool { + if !snapshot.landed.is_empty() { + snapshot.landed.remove(0); + } else if !snapshot.incidents.is_empty() { + snapshot.incidents.remove(0); + } else { + return false; + } + snapshot.history_truncated = true; + true +} From d325ab2bf7c832cf7dd0f661ffbe2e1e6ef1dc3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:15:34 +0000 Subject: [PATCH 6/9] test(work): reproduce acknowledging an unsynced pending journal line --- oxa/oxa-work/src/events/tests.rs | 49 ++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/oxa/oxa-work/src/events/tests.rs b/oxa/oxa-work/src/events/tests.rs index f754da8a5..420a1e650 100644 --- a/oxa/oxa-work/src/events/tests.rs +++ b/oxa/oxa-work/src/events/tests.rs @@ -250,6 +250,55 @@ fn an_appended_pending_fact_and_a_lagging_snapshot_recover_without_another_line( } } +#[test] +fn pending_recovery_cannot_commit_without_opening_the_journal_for_sync() { + // Root bypasses the real journal permission failure; run this case as an ordinary user. + // SAFETY: geteuid has no arguments, pointers or failure mode. + if unsafe { libc::geteuid() } == 0 { + use std::os::unix::process::CommandExt; + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .env_clear() + .args([ + "--exact", + "events::tests::pending_recovery_cannot_commit_without_opening_the_journal_for_sync", + ]) + .uid(65534) + .gid(65534) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + return; + } + let env = Fixture::new(); + let facts = env.facts(); + let pending = reserve(&facts, mechanical()); + let mut bytes = pending.to_json().unwrap(); + bytes.push(b'\n'); + // The complete line is visible without having been synced by its writer. + fs::OpenOptions::new() + .append(true) + .open(env.root().join(CURRENT)) + .unwrap() + .write_all(&bytes) + .unwrap(); + let before = facts.root.read(STATE, STATE_BYTES).unwrap(); + let snapshot = facts.root.read(SITUATION, MAX_SNAPSHOT_BYTES).unwrap(); + fs::set_permissions(env.root().join(CURRENT), fs::Permissions::from_mode(0o400)).unwrap(); + let recovered = Facts::open(&env.root()); + fs::set_permissions(env.root().join(CURRENT), fs::Permissions::from_mode(0o600)).unwrap(); + assert!(recovered.is_err(), "unsynced pending line was acknowledged"); + assert_eq!(facts.root.read(STATE, STATE_BYTES).unwrap(), before); + assert_eq!(facts.root.read(SITUATION, MAX_SNAPSHOT_BYTES).unwrap(), snapshot); + let recovered = Facts::open(&env.root()).unwrap(); + assert_eq!(recovered.situation().unwrap().as_of_seq, pending.seq); + assert_eq!(recovered.events_after(0, 64).unwrap().events, [pending]); +} + #[test] fn interruption_between_rotation_and_new_current_file_recovers_the_reserved_event() { let env = Fixture::new(); From 667f15ad525c57d97040b359e8eee071e2744a5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:16:25 +0000 Subject: [PATCH 7/9] fix(work): sync visible pending journal data before recovery acknowledgement --- oxa/oxa-work/src/events.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/oxa/oxa-work/src/events.rs b/oxa/oxa-work/src/events.rs index 20530c36d..6f45883a4 100644 --- a/oxa/oxa-work/src/events.rs +++ b/oxa/oxa-work/src/events.rs @@ -292,6 +292,14 @@ impl Facts { self.root.append(CURRENT, &bytes)?; } else if journal.events.last() != Some(&event) || journal.torn_at.is_some() { return Err("pending fact disagrees with the journal".into()); + } else { + // Visibility is not durability: the original append may have died before sync. + let leaf = if journal.current_bytes == 0 { + PREVIOUS + } else { + CURRENT + }; + self.root.append(leaf, b"")?; } state.committed(&event)?; self.root.atomic(STATE, &state.bytes()?)?; From f8753beb5e7460ee72f251410405df7f9cade876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:34:56 +0000 Subject: [PATCH 8/9] test(gateway): reserve fixture directories independently of wall-clock ticks --- oxa/oxc-gateway/tests/common/skeleton.rs | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/oxa/oxc-gateway/tests/common/skeleton.rs b/oxa/oxc-gateway/tests/common/skeleton.rs index a2acb8b28..f57f8f849 100644 --- a/oxa/oxc-gateway/tests/common/skeleton.rs +++ b/oxa/oxc-gateway/tests/common/skeleton.rs @@ -6,7 +6,7 @@ use std::{ io::Write, os::unix::fs::{DirBuilderExt, OpenOptionsExt}, path::PathBuf, - time::{SystemTime, UNIX_EPOCH}, + sync::atomic::{AtomicU64, Ordering}, }; pub const MODEL: &str = "synthetic-cloud"; @@ -23,12 +23,19 @@ impl Fixture { Self::for_endpoint(&mock.base_url()) } pub fn for_endpoint(endpoint: &str) -> Self { - let root = std::env::temp_dir().join(format!( - "oxc-skeleton-{}-{}", - std::process::id(), - SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos() - )); - DirBuilder::new().mode(0o700).create(&root).unwrap(); + static NEXT: AtomicU64 = AtomicU64::new(0); + let root = loop { + let path = std::env::temp_dir().join(format!( + "oxc-skeleton-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + match DirBuilder::new().mode(0o700).create(&path) { + Ok(()) => break path, + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => panic!("cannot create private synthetic fixture: {error}"), + } + }; let key_path = root.join("credentials.json"); let mut key = OpenOptions::new() .write(true) From ab952f4ac622a0932c12a2f6588d25a2cd2f9d9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:19:30 +0000 Subject: [PATCH 9/9] test(proto): supply explicit running steps in shared evening fixtures --- oxa/oxa-proto/tests/fixtures/work/evening.jsonl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/oxa/oxa-proto/tests/fixtures/work/evening.jsonl b/oxa/oxa-proto/tests/fixtures/work/evening.jsonl index d82c33d31..1a87ed294 100644 --- a/oxa/oxa-proto/tests/fixtures/work/evening.jsonl +++ b/oxa/oxa-proto/tests/fixtures/work/evening.jsonl @@ -1,4 +1,4 @@ -{"type":"oxa_work_event","version":1,"data":{"seq":1,"at_ms":1794847200000,"checkpoint":{"source":"fixture-runner","revision":1},"fact":{"kind":"task_started","data":{"task":"task-landed","host":"helper","branch":"agent/fixture-task-landed","worktree":"~/project/task-landed","session_id":"session-task-landed"}}}} +{"type":"oxa_work_event","version":1,"data":{"seq":1,"at_ms":1794847200000,"checkpoint":{"source":"fixture-runner","revision":1},"fact":{"kind":"task_started","data":{"task":"task-landed","host":"helper","branch":"agent/fixture-task-landed","worktree":"~/project/task-landed","session_id":"session-task-landed","step":"preparing"}}}} {"type":"oxa_work_event","version":1,"data":{"seq":2,"at_ms":1794847450000,"checkpoint":{"source":"fixture-runner","revision":2},"fact":{"kind":"worker_finished","data":{"task":"task-landed","session_id":"session-task-landed","outcome":"done","error_class":null}}}} {"type":"oxa_work_event","version":1,"data":{"seq":3,"at_ms":1794847700000,"checkpoint":{"source":"fixture-runner","revision":3},"fact":{"kind":"local_acceptance","data":{"task":"task-landed","commit":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","evidence":{"kind":"local_check","id":"local-landed"}}}}} {"type":"oxa_work_event","version":1,"data":{"seq":4,"at_ms":1794847950000,"checkpoint":{"source":"fixture-runner","revision":4},"fact":{"kind":"published","data":{"task":"task-landed","pr":{"number":101,"head":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}}}}} @@ -6,7 +6,7 @@ {"type":"oxa_work_event","version":1,"data":{"seq":6,"at_ms":1794848450000,"checkpoint":{"source":"fixture-runner","revision":6},"fact":{"kind":"enqueued","data":{"number":101,"head":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}}}} {"type":"oxa_work_event","version":1,"data":{"seq":7,"at_ms":1794848700000,"checkpoint":{"source":"fixture-runner","revision":7},"fact":{"kind":"pr_merged","data":{"pr":{"number":101,"head":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"merge_commit":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","merged_at_ms":1794848700000}}}} {"type":"oxa_work_situation","version":1,"data":{"as_of_seq":7,"at_ms":1794848700000,"standing_revision":1,"goal":"Finish the synthetic evening","running":[],"local_accepted":[{"task":"task-landed","commit":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","evidence":{"kind":"local_check","id":"local-landed"}}],"published":[{"task":"task-landed","pr":{"number":101,"head":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}}],"landed":[{"pr":{"number":101,"head":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"merge_commit":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","merged_at_ms":1794848700000}],"blocked":[],"asks":[],"incidents":[],"history_truncated":false}} -{"type":"oxa_work_event","version":1,"data":{"seq":8,"at_ms":1794849000000,"checkpoint":{"source":"fixture-runner","revision":8},"fact":{"kind":"task_started","data":{"task":"task-red","host":"helper","branch":"agent/fixture-task-red","worktree":"~/project/task-red","session_id":"session-task-red"}}}} +{"type":"oxa_work_event","version":1,"data":{"seq":8,"at_ms":1794849000000,"checkpoint":{"source":"fixture-runner","revision":8},"fact":{"kind":"task_started","data":{"task":"task-red","host":"helper","branch":"agent/fixture-task-red","worktree":"~/project/task-red","session_id":"session-task-red","step":"preparing"}}}} {"type":"oxa_work_event","version":1,"data":{"seq":9,"at_ms":1794849250000,"checkpoint":{"source":"fixture-runner","revision":9},"fact":{"kind":"worker_finished","data":{"task":"task-red","session_id":"session-task-red","outcome":"done","error_class":null}}}} {"type":"oxa_work_event","version":1,"data":{"seq":10,"at_ms":1794849500000,"checkpoint":{"source":"fixture-runner","revision":10},"fact":{"kind":"local_acceptance","data":{"task":"task-red","commit":"cccccccccccccccccccccccccccccccccccccccc","evidence":{"kind":"local_check","id":"local-red"}}}}} {"type":"oxa_work_event","version":1,"data":{"seq":11,"at_ms":1794849750000,"checkpoint":{"source":"fixture-runner","revision":11},"fact":{"kind":"published","data":{"task":"task-red","pr":{"number":102,"head":"cccccccccccccccccccccccccccccccccccccccc"}}}}} @@ -15,7 +15,7 @@ {"type":"oxa_work_event","version":1,"data":{"seq":14,"at_ms":1794850500000,"checkpoint":{"source":"fixture-runner","revision":14},"fact":{"kind":"external_acceptance","data":{"pr":{"number":102,"head":"cccccccccccccccccccccccccccccccccccccccc"},"evidence":{"kind":"queue","id":"merge-group/dddddddddddddddddddddddddddddddddddddddd"}}}}} {"type":"oxa_work_event","version":1,"data":{"seq":15,"at_ms":1794850800000,"checkpoint":{"source":"fixture-runner","revision":15},"fact":{"kind":"incident","data":{"id":"red-queue","subject":{"kind":"pull_request","data":102},"kind":"queue_rejected","evidence":{"kind":"gate","id":"gate-red"}}}}} {"type":"oxa_work_situation","version":1,"data":{"as_of_seq":15,"at_ms":1794850800000,"standing_revision":1,"goal":"Finish the synthetic evening","running":[],"local_accepted":[{"task":"task-landed","commit":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","evidence":{"kind":"local_check","id":"local-landed"}},{"task":"task-red","commit":"cccccccccccccccccccccccccccccccccccccccc","evidence":{"kind":"local_check","id":"local-red"}}],"published":[{"task":"task-landed","pr":{"number":101,"head":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}},{"task":"task-red","pr":{"number":102,"head":"cccccccccccccccccccccccccccccccccccccccc"}}],"landed":[{"pr":{"number":101,"head":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"merge_commit":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","merged_at_ms":1794848700000}],"blocked":[],"asks":[],"incidents":[{"id":"red-queue","subject":{"kind":"pull_request","data":102},"kind":"queue_rejected","evidence":{"kind":"gate","id":"gate-red"}}],"history_truncated":false}} -{"type":"oxa_work_event","version":1,"data":{"seq":16,"at_ms":1794851100000,"checkpoint":{"source":"fixture-runner","revision":16},"fact":{"kind":"task_started","data":{"task":"task-blocked","host":"helper","branch":"agent/fixture-task-blocked","worktree":"~/project/task-blocked","session_id":"session-task-blocked"}}}} +{"type":"oxa_work_event","version":1,"data":{"seq":16,"at_ms":1794851100000,"checkpoint":{"source":"fixture-runner","revision":16},"fact":{"kind":"task_started","data":{"task":"task-blocked","host":"helper","branch":"agent/fixture-task-blocked","worktree":"~/project/task-blocked","session_id":"session-task-blocked","step":"preparing"}}}} {"type":"oxa_work_event","version":1,"data":{"seq":17,"at_ms":1794851400000,"checkpoint":{"source":"fixture-runner","revision":17},"fact":{"kind":"worker_finished","data":{"task":"task-blocked","session_id":"session-task-blocked","outcome":"blocked","error_class":"needs-interface"}}}} {"type":"oxa_work_event","version":1,"data":{"seq":18,"at_ms":1794851700000,"checkpoint":{"source":"fixture-runner","revision":18},"fact":{"kind":"ask_opened","data":{"id":"ask-blocked","subject":{"kind":"task","data":"task-blocked"},"question":"Which synthetic interface should the task target?"}}}} {"type":"oxa_work_event","version":1,"data":{"seq":19,"at_ms":1794852000000,"checkpoint":{"source":"fixture-runner","revision":19},"fact":{"kind":"blocked","data":{"subject":{"kind":"task","data":"task-blocked"},"code":"needs-interface","ask_id":"ask-blocked"}}}} @@ -23,5 +23,5 @@ {"type":"oxa_work_event","version":1,"data":{"seq":21,"at_ms":1794855600000,"checkpoint":{"source":"fixture-runner","revision":21},"fact":{"kind":"incident","data":{"id":"helper-idle","subject":{"kind":"host","data":"helper"},"kind":"host_idle","evidence":{"kind":"status","id":"idle-since-1900"}}}}} {"type":"oxa_work_event","version":1,"data":{"seq":22,"at_ms":1794862800000,"checkpoint":{"source":"fixture-runner","revision":22},"fact":{"kind":"mechanical","data":{"subject":{"kind":"host","data":"helper"},"kind":"refill"}}}} {"type":"oxa_work_situation","version":1,"data":{"as_of_seq":22,"at_ms":1794862800000,"standing_revision":1,"goal":"Finish the synthetic evening","running":[],"local_accepted":[{"task":"task-landed","commit":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","evidence":{"kind":"local_check","id":"local-landed"}},{"task":"task-red","commit":"cccccccccccccccccccccccccccccccccccccccc","evidence":{"kind":"local_check","id":"local-red"}}],"published":[{"task":"task-landed","pr":{"number":101,"head":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}},{"task":"task-red","pr":{"number":102,"head":"cccccccccccccccccccccccccccccccccccccccc"}}],"landed":[{"pr":{"number":101,"head":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"merge_commit":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","merged_at_ms":1794848700000}],"blocked":[{"subject":{"kind":"task","data":"task-blocked"},"code":"needs-interface","ask_id":"ask-blocked"},{"subject":{"kind":"pull_request","data":102},"code":"held","ask_id":null}],"asks":[{"id":"ask-blocked","subject":{"kind":"task","data":"task-blocked"},"question":"Which synthetic interface should the task target?"}],"incidents":[{"id":"red-queue","subject":{"kind":"pull_request","data":102},"kind":"queue_rejected","evidence":{"kind":"gate","id":"gate-red"}},{"id":"helper-idle","subject":{"kind":"host","data":"helper"},"kind":"host_idle","evidence":{"kind":"status","id":"idle-since-1900"}}],"history_truncated":false}} -{"type":"oxa_work_events","version":1,"data":{"after":10,"events":[{"seq":14,"at_ms":1794850500000,"checkpoint":{"source":"fixture-runner","revision":14},"fact":{"kind":"external_acceptance","data":{"pr":{"number":102,"head":"cccccccccccccccccccccccccccccccccccccccc"},"evidence":{"kind":"queue","id":"merge-group/dddddddddddddddddddddddddddddddddddddddd"}}}},{"seq":15,"at_ms":1794850800000,"checkpoint":{"source":"fixture-runner","revision":15},"fact":{"kind":"incident","data":{"id":"red-queue","subject":{"kind":"pull_request","data":102},"kind":"queue_rejected","evidence":{"kind":"gate","id":"gate-red"}}}},{"seq":16,"at_ms":1794851100000,"checkpoint":{"source":"fixture-runner","revision":16},"fact":{"kind":"task_started","data":{"task":"task-blocked","host":"helper","branch":"agent/fixture-task-blocked","worktree":"~/project/task-blocked","session_id":"session-task-blocked"}}},{"seq":17,"at_ms":1794851400000,"checkpoint":{"source":"fixture-runner","revision":17},"fact":{"kind":"worker_finished","data":{"task":"task-blocked","session_id":"session-task-blocked","outcome":"blocked","error_class":"needs-interface"}}},{"seq":18,"at_ms":1794851700000,"checkpoint":{"source":"fixture-runner","revision":18},"fact":{"kind":"ask_opened","data":{"id":"ask-blocked","subject":{"kind":"task","data":"task-blocked"},"question":"Which synthetic interface should the task target?"}}},{"seq":19,"at_ms":1794852000000,"checkpoint":{"source":"fixture-runner","revision":19},"fact":{"kind":"blocked","data":{"subject":{"kind":"task","data":"task-blocked"},"code":"needs-interface","ask_id":"ask-blocked"}}},{"seq":20,"at_ms":1794852300000,"checkpoint":{"source":"fixture-runner","revision":20},"fact":{"kind":"blocked","data":{"subject":{"kind":"pull_request","data":102},"code":"held","ask_id":null}}},{"seq":21,"at_ms":1794855600000,"checkpoint":{"source":"fixture-runner","revision":21},"fact":{"kind":"incident","data":{"id":"helper-idle","subject":{"kind":"host","data":"helper"},"kind":"host_idle","evidence":{"kind":"status","id":"idle-since-1900"}}}},{"seq":22,"at_ms":1794862800000,"checkpoint":{"source":"fixture-runner","revision":22},"fact":{"kind":"mechanical","data":{"subject":{"kind":"host","data":"helper"},"kind":"refill"}}}],"next":22,"first_available":14,"gap":true}} +{"type":"oxa_work_events","version":1,"data":{"after":10,"events":[{"seq":14,"at_ms":1794850500000,"checkpoint":{"source":"fixture-runner","revision":14},"fact":{"kind":"external_acceptance","data":{"pr":{"number":102,"head":"cccccccccccccccccccccccccccccccccccccccc"},"evidence":{"kind":"queue","id":"merge-group/dddddddddddddddddddddddddddddddddddddddd"}}}},{"seq":15,"at_ms":1794850800000,"checkpoint":{"source":"fixture-runner","revision":15},"fact":{"kind":"incident","data":{"id":"red-queue","subject":{"kind":"pull_request","data":102},"kind":"queue_rejected","evidence":{"kind":"gate","id":"gate-red"}}}},{"seq":16,"at_ms":1794851100000,"checkpoint":{"source":"fixture-runner","revision":16},"fact":{"kind":"task_started","data":{"task":"task-blocked","host":"helper","branch":"agent/fixture-task-blocked","worktree":"~/project/task-blocked","session_id":"session-task-blocked","step":"preparing"}}},{"seq":17,"at_ms":1794851400000,"checkpoint":{"source":"fixture-runner","revision":17},"fact":{"kind":"worker_finished","data":{"task":"task-blocked","session_id":"session-task-blocked","outcome":"blocked","error_class":"needs-interface"}}},{"seq":18,"at_ms":1794851700000,"checkpoint":{"source":"fixture-runner","revision":18},"fact":{"kind":"ask_opened","data":{"id":"ask-blocked","subject":{"kind":"task","data":"task-blocked"},"question":"Which synthetic interface should the task target?"}}},{"seq":19,"at_ms":1794852000000,"checkpoint":{"source":"fixture-runner","revision":19},"fact":{"kind":"blocked","data":{"subject":{"kind":"task","data":"task-blocked"},"code":"needs-interface","ask_id":"ask-blocked"}}},{"seq":20,"at_ms":1794852300000,"checkpoint":{"source":"fixture-runner","revision":20},"fact":{"kind":"blocked","data":{"subject":{"kind":"pull_request","data":102},"code":"held","ask_id":null}}},{"seq":21,"at_ms":1794855600000,"checkpoint":{"source":"fixture-runner","revision":21},"fact":{"kind":"incident","data":{"id":"helper-idle","subject":{"kind":"host","data":"helper"},"kind":"host_idle","evidence":{"kind":"status","id":"idle-since-1900"}}}},{"seq":22,"at_ms":1794862800000,"checkpoint":{"source":"fixture-runner","revision":22},"fact":{"kind":"mechanical","data":{"subject":{"kind":"host","data":"helper"},"kind":"refill"}}}],"next":22,"first_available":14,"gap":true}} {"type":"oxa_work_event","version":1,"data":{"seq":23,"at_ms": \ No newline at end of file