From 3ab703fbfbf3922aeb9f9ace7de74d4552a638a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 09:56:56 +0000 Subject: [PATCH 01/27] Describe each durable runner step as a typed transition --- oxa/docs/work-facts.md | 9 +++++ oxa/oxa-proto/src/work.rs | 26 ++++++++++++- oxa/oxa-proto/tests/work_facts.rs | 61 +++++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 1 deletion(-) diff --git a/oxa/docs/work-facts.md b/oxa/docs/work-facts.md index abb44ba87..bdcc80eb4 100644 --- a/oxa/docs/work-facts.md +++ b/oxa/docs/work-facts.md @@ -17,6 +17,15 @@ external acceptance bind a full PR head. A head-independent `Subject::PullReques available for blocks, asks and future standing holds. Task, host, repository and inbox subjects also exist, including tasks that do not yet have a PR. +`TaskTransition { task, step, session_id, reason }` reports every durable runner step: +`preparing → worker → acceptance → repair/recheck → saving → finished`. Each observation gets +its own checkpoint revision; worker completion and local acceptance may have separate revisions. +`TaskStep` is closed, `session_id` is optional until known, and `StepReason` is a closed code +such as `checks_failed`, `session_unavailable` or `deadline`, not diagnostic prose. A finished +step requires a reason and says only that this local attempt ended. It does not imply acceptance, +publication or merge. These observations are routine; a separate block/ask/incident carries any +attention condition. Running situation rows expose their current step and cannot be `finished`. + `Snapshot` is a materialized read view: `as_of_seq`, observation time, standing revision, optional goal, running tasks (host, branch, worktree and optional session), locally accepted work, publications, landed PRs, blocks, asks and incidents. It is not authoritative task state. diff --git a/oxa/oxa-proto/src/work.rs b/oxa/oxa-proto/src/work.rs index 36dfc99b5..8813ec3bf 100644 --- a/oxa/oxa-proto/src/work.rs +++ b/oxa/oxa-proto/src/work.rs @@ -211,6 +211,18 @@ choice!(WorkerOutcome { Done => "done", NothingToDo => "nothing_to_do", Blocked => "blocked", Failed => "failed", Interrupted => "interrupted", }); +choice!(TaskStep { + Preparing => "preparing", Worker => "worker", Acceptance => "acceptance", + Repair => "repair", Recheck => "recheck", Saving => "saving", Finished => "finished", +}); +choice!(StepReason { + Resumed => "resumed", Prepared => "prepared", WorkerDone => "worker_done", + WorkerNothingToDo => "worker_nothing_to_do", WorkerBlocked => "worker_blocked", + WorkerFailed => "worker_failed", ChecksPassed => "checks_passed", ChecksFailed => "checks_failed", + RepairDone => "repair_done", RepairFailed => "repair_failed", Saved => "saved", SaveFailed => "save_failed", + Interrupted => "interrupted", Deadline => "deadline", SessionUnavailable => "session_unavailable", + SetupFailed => "setup_failed", +}); choice!(EvidenceKind { LocalCheck => "local_check", Gate => "gate", Review => "review", Queue => "queue", Status => "status", Standing => "standing", @@ -305,14 +317,25 @@ record!( | { atom(&s.id, 256) } ); record!(Running { - task: String, host: String, branch: String, worktree: String, session_id: Option, + task: String, host: String, branch: String, worktree: String, session_id: Option, step: TaskStep, } |s| { atom(&s.task, 128)?; atom(&s.host, 128)?; require(!s.branch.starts_with('-') && s.branch.len() <= 256, "invalid task branch")?; if let Some(id) = &s.session_id { atom(id, 128)?; } + require(s.step != TaskStep::Finished, "finished task cannot be running")?; Ok(()) }); +record!( + /// One durable runner step, with no task-state contents or implied publication/landing. + /// Finished means this local attempt ended; only the separate acceptance/merge facts + /// provide those respective claims. A terminal step requires an explicit fixed reason. + TaskTransition { task: String, step: TaskStep, session_id: Option, reason: Option } |s| { + atom(&s.task, 128)?; + if let Some(id) = &s.session_id { atom(id, 128)?; } + require(s.step != TaskStep::Finished || s.reason.is_some(), "finished step requires a reason") + } +); record!(WorkerFinished { task: String, session_id: Option, outcome: WorkerOutcome, error_class: Option, } |s| { @@ -402,6 +425,7 @@ tagged!( /// No variant grants permission or updates runner state. Only PrMerged is evidence of landing. Fact { TaskStarted(Running) => "task_started", + TaskTransition(TaskTransition) => "task_transition", WorkerFinished(WorkerFinished) => "worker_finished", LocalAcceptance(LocalAcceptance) => "local_acceptance", Published(Publication) => "published", diff --git a/oxa/oxa-proto/tests/work_facts.rs b/oxa/oxa-proto/tests/work_facts.rs index 8a21350c7..e43eaaf69 100644 --- a/oxa/oxa-proto/tests/work_facts.rs +++ b/oxa/oxa-proto/tests/work_facts.rs @@ -20,6 +20,7 @@ fn running() -> Running { branch: "agent/17-work".into(), worktree: "~/project/task-17".into(), session_id: Some("session-17".into()), + step: TaskStep::Worker, } } fn ask() -> Ask { @@ -355,3 +356,63 @@ fn outcomes_and_evidence_are_explicit_and_head_bound() { zero_seq.seq = 0; assert!(zero_seq.to_json().is_err()); } + +#[test] +fn every_durable_runner_step_has_its_own_observation_and_replay_identity() { + let steps = [ + TaskStep::Preparing, + TaskStep::Worker, + TaskStep::Acceptance, + TaskStep::Repair, + TaskStep::Recheck, + TaskStep::Saving, + TaskStep::Finished, + ]; + for (index, step) in steps.into_iter().enumerate() { + let observation = TaskTransition { + task: "task-17".into(), + step, + session_id: (index > 0).then(|| "session-17".into()), + reason: (step == TaskStep::Finished).then_some(StepReason::Saved), + }; + let event = event(index as u64 + 1, Fact::TaskTransition(observation)); + assert_eq!(event.fact.class(), EventClass::Routine); + assert_eq!(Event::from_json(&event.to_json().unwrap()).unwrap(), event); + assert_eq!(event.checkpoint.revision, index as u64 + 1); + assert!(!matches!(event.fact, Fact::PrMerged(_))); + } + let mut terminal = TaskTransition { + task: "task-17".into(), + step: TaskStep::Finished, + session_id: None, + reason: None, + }; + assert!(event(1, Fact::TaskTransition(terminal.clone())) + .to_json() + .is_err()); + for reason in [ + StepReason::WorkerBlocked, + StepReason::WorkerFailed, + StepReason::ChecksFailed, + StepReason::RepairFailed, + StepReason::SaveFailed, + StepReason::Interrupted, + StepReason::Deadline, + StepReason::SessionUnavailable, + StepReason::SetupFailed, + ] { + terminal.reason = Some(reason); + let observation = event(1, Fact::TaskTransition(terminal.clone())); + assert_eq!( + Event::from_json(&observation.to_json().unwrap()).unwrap(), + observation + ); + } + let mut bad_running = running(); + bad_running.step = TaskStep::Finished; + assert!(event(1, Fact::TaskStarted(bad_running)).to_json().is_err()); + let mut encoded: Value = + serde_json::from_slice(&event(1, Fact::TaskTransition(terminal)).to_json().unwrap()).unwrap(); + encoded["data"]["fact"]["data"]["reason"] = json!("free-form failure prose"); + assert!(Event::from_json(&serde_json::to_vec(&encoded).unwrap()).is_err()); +} From ba060e6773ed12c0057febad5c05149f8f93bb3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 10:44:51 +0000 Subject: [PATCH 02/27] Persist bounded lead facts with replay and stall incidents --- oxa/Cargo.lock | 2 + oxa/docs/lead-facts-store.md | 75 +++++ oxa/oxa-proto/src/work.rs | 3 +- oxa/oxa-work/Cargo.toml | 2 + oxa/oxa-work/src/events.rs | 321 ++++++++++++++++++++ oxa/oxa-work/src/events/tests.rs | 483 +++++++++++++++++++++++++++++++ oxa/oxa-work/src/facts_io.rs | 203 +++++++++++++ oxa/oxa-work/src/incidents.rs | 194 +++++++++++++ oxa/oxa-work/src/lib.rs | 6 + oxa/oxa-work/src/situation.rs | 132 +++++++++ 10 files changed, 1420 insertions(+), 1 deletion(-) create mode 100644 oxa/docs/lead-facts-store.md create mode 100644 oxa/oxa-work/src/events.rs create mode 100644 oxa/oxa-work/src/events/tests.rs create mode 100644 oxa/oxa-work/src/facts_io.rs create mode 100644 oxa/oxa-work/src/incidents.rs create mode 100644 oxa/oxa-work/src/situation.rs diff --git a/oxa/Cargo.lock b/oxa/Cargo.lock index f11c3ce6a..68acb522d 100644 --- a/oxa/Cargo.lock +++ b/oxa/Cargo.lock @@ -1199,6 +1199,8 @@ dependencies = [ name = "oxa-work" version = "0.2.0" dependencies = [ + "libc", + "oxa-proto", "serde", "serde_json", ] diff --git a/oxa/docs/lead-facts-store.md b/oxa/docs/lead-facts-store.md new file mode 100644 index 000000000..4912aa26e --- /dev/null +++ b/oxa/docs/lead-facts-store.md @@ -0,0 +1,75 @@ +# Runner facts storage + +`oxa_work::events::Facts` is the L1 storage module for #629. The runner remains the sole task +state writer. These modules never open its task files, invoke a model or start a background +process. The lead consumes the public read APIs rather than logs or private runner records. + +The runner supplies exactly `/facts`, outside writable workspace +and Git binds. `Facts::initialize(root)` explicitly creates a fresh store; `Facts::open(root)` +opens/replays an existing one. Missing, corrupt, unsafe or partial initialization is an error, +never an empty situation. Directories are private (0700), files private (0600), and files and +path components are checked without following symlinks. It opens only its fixed storage leaves. + +## Append after a durable checkpoint + +Call `append(Checkpoint { source, revision }, at_ms, fact)` only after the corresponding +observation is durable. Stop before another action if append fails. The runner's source is +`runner`; its checkpoint revision is monotonic per repository. A short repository checkpoint +lock serializes allocation, durable task rename, callback and acknowledgement. The runner +recovers a pending task/revision before allocating again and does not re-emit older acknowledged +task records. Reserved checkpoint revisions may have gaps; event sequence numbers do not. + +Other producers use their own stable emitter domain and monotonic revisions, not a fresh source +per task. The bounded source table holds the latest observation for up to 16 emitter domains; +it retains no unbounded per-task tombstones. Retrying the latest source/revision with the same +fact returns its original event and timestamp, even after journal rotation. Different facts +under that identity and stale revisions are errors. Sequence exhaustion is an error. + +Each append validates the projected situation before reserving the event. It then saves a +pending observation in `facts.json`, appends and syncs the journal line, commits the source +cursor and atomically replaces `situation.json`. Atomic replacements sync the file and directory. +Recovery replays that reserved observation across a torn final line, interrupted rotation, +lost acknowledgement or lagging situation. Complete malformed lines, unreserved torn tails, +conflicting contents and unrecoverable gaps fail closed. + +`TaskStarted` supplies the initial preparing location (task, host label, branch and worktree). +Later `TaskTransition` observations update its step/session; `finished` removes that running +attempt. A transition without a known running location is an error. Worker completion, local +acceptance, publication, external acceptance and merge stay distinct. Only `PrMerged` enters +the landed list. Blocks and asks persist until explicit `Unblocked` and `AskClosed` facts; +updating a PR head cannot silently clear either. + +## Bounded reads + +`situation()` returns a validated `oxa_proto::work::Snapshot` with its exact `as_of_seq`. +It is capped at 64 KiB and 128 total rows. Running work, local acceptance awaiting publication, +published work, blocks and asks are never silently dropped to fit. Capacity failure stops the +next action. Landed/incident history retains at most 16 rows each and may shrink to fit; +`history_truncated` records that loss. An accepted/published pair is retired only by the matching +head's merge observation. Standing revision/goal remain unknown until the standing integration. + +`events_after(after, limit)` takes an exclusive repository cursor and a limit from 1 to 64. +The two journals (`events.jsonl` and `events.previous.jsonl`) rotate at 256 KiB each. Pages +report `first_available`, `next` and an explicit `gap`; a consumer seeing a gap reloads the +situation rather than inventing old transitions. A cursor ahead of the journal is an error. + +`incidents::recent(&facts)` returns the bounded incident projection from that same situation; +the journal is its durable source, not a competing incident/task-state writer. Reads recover +only derived facts metadata/projections under the facts lock and never alter task state. + +## Stall incidents + +`incidents::detect_stall(status_bytes, expected_session_id, expected_pid, now_ms)` consumes a +bounded status snapshot supplied by the runner. It validates identity, phase, progress, timeout +and deadline; stale invocation identity or malformed data is an error. It returns a typed +`Stall` only when the conservative deadline has elapsed. Sessions without an active deadline and finished sessions do not +stall. The caller may append `Stall::incident(task)` through its durable event boundary. + +Stalls are incidents only: this module never wraps, cancels, kills or polls a worker. The +runner passes `--stall-timeout 900` for the decided 15-minute threshold and separately owns +the per-task wall-time limit and escalation. No second watchdog is introduced. + +The runner owns `oxa work situation`, `oxa work events --after N`, checkpoint adapters and +action wiring. The lead owns the status/ask cards. The common schema is described in +[work-facts.md](work-facts.md); storage/recovery tests are in +[events/tests.rs](../oxa-work/src/events/tests.rs). diff --git a/oxa/oxa-proto/src/work.rs b/oxa/oxa-proto/src/work.rs index 8813ec3bf..dade3c648 100644 --- a/oxa/oxa-proto/src/work.rs +++ b/oxa/oxa-proto/src/work.rs @@ -412,7 +412,7 @@ record!( } ); record!(AskClosed { id: String } | s | { atom(&s.id, 128) }); -record!(Incident { id: String, subject: Subject, kind: IncidentKind, evidence: Option } |s| { atom(&s.id, 128) }); +record!(Incident { id: String, subject: Subject, kind: IncidentKind, evidence: Option } |s| { atom(&s.id, 256) }); record!( Mechanical { subject: Subject, @@ -433,6 +433,7 @@ tagged!( Enqueued(PullRequest) => "enqueued", PrMerged(Landed) => "pr_merged", Blocked(Blocked) => "blocked", + Unblocked(Subject) => "unblocked", AskOpened(Ask) => "ask_opened", AskClosed(AskClosed) => "ask_closed", Incident(Incident) => "incident", diff --git a/oxa/oxa-work/Cargo.toml b/oxa/oxa-work/Cargo.toml index 6097461ce..bb9e07e5d 100644 --- a/oxa/oxa-work/Cargo.toml +++ b/oxa/oxa-work/Cargo.toml @@ -7,6 +7,8 @@ license.workspace = true publish.workspace = true [dependencies] +libc.workspace = true +oxa-proto.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/oxa/oxa-work/src/events.rs b/oxa/oxa-work/src/events.rs new file mode 100644 index 000000000..e70e267b0 --- /dev/null +++ b/oxa/oxa-work/src/events.rs @@ -0,0 +1,321 @@ +//! Durable facts journal and its bounded read view. Never opens runner task-state files. +use crate::facts_io::Root; +use crate::{situation, Result}; +use oxa_proto::work::{Checkpoint, Event, EventPage, Fact, Snapshot, MAX_EVENT_BYTES, MAX_SNAPSHOT_BYTES}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::path::Path; + +const CURRENT: &str = "events.jsonl"; +const PREVIOUS: &str = "events.previous.jsonl"; +const STATE: &str = "facts.json"; +const SITUATION: &str = "situation.json"; +const LOCK: &str = ".facts.lock"; +const JOURNAL_BYTES: usize = 256 * 1024; +const STATE_BYTES: usize = 256 * 1024; +const MAX_SOURCES: usize = 16; + +#[derive(Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct State { + version: u32, + last_seq: u64, + last_by_source: Vec, + // Write-ahead observation: a crash can replay it even if the journal line was torn. + pending: Option, +} +impl State { + fn sources(&self) -> Result> { + if self.version != 1 || self.last_by_source.len() > MAX_SOURCES { + return Err("unsupported facts metadata or source capacity".into()); + } + let events: Vec = self + .last_by_source + .iter() + .map(read_event_value) + .collect::>()?; + let mut sources = std::collections::HashSet::new(); + let mut sequences = std::collections::HashSet::new(); + for event in &events { + if event.seq > self.last_seq + || !sources.insert(&event.checkpoint.source) + || !sequences.insert(event.seq) + { + return Err("inconsistent facts source metadata".into()); + } + } + if events.iter().map(|event| event.seq).max().unwrap_or(0) != self.last_seq { + return Err("facts source metadata has no latest observation".into()); + } + Ok(events) + } + fn committed(&mut self, event: &Event) -> Result<()> { + let mut sources = self.sources()?; + sources.retain(|old| old.checkpoint.source != event.checkpoint.source); + sources.push(event.clone()); + if sources.len() > MAX_SOURCES { + return Err("facts source capacity exceeded".into()); + } + self.last_seq = event.seq; + self.last_by_source = sources.iter().map(event_value).collect::>()?; + self.pending = None; + Ok(()) + } + fn bytes(&self) -> Result> { + let bytes = serde_json::to_vec(self).map_err(|_| "cannot encode facts metadata")?; + if bytes.len() > STATE_BYTES { + return Err("facts metadata capacity exceeded".into()); + } + Ok(bytes) + } +} +fn event_value(event: &Event) -> Result { + serde_json::from_slice(&event.to_json().map_err(|error| error.to_string())?) + .map_err(|_| "cannot encode fact".into()) +} +fn read_event_value(value: &Value) -> Result { + Event::from_json(&serde_json::to_vec(value).map_err(|_| "invalid fact metadata")?) + .map_err(|error| error.to_string()) +} + +struct Journal { + events: Vec, + current_bytes: usize, + torn_at: Option, +} +impl Journal { + fn last_seq(&self) -> u64 { + self.events.last().map_or(0, |event| event.seq) + } +} + +#[derive(Clone, Debug)] +pub struct Facts { + root: Root, + journal_bytes: usize, +} +impl Facts { + /// Explicit first initialization. Existing or partial artifacts are never reset to empty. + /// Supply /facts, outside writable workspace/Git binds. + pub fn initialize(root: &Path) -> Result { + let root = Root::open(root, true)?; + let _lock = root.lock(LOCK)?; + for leaf in [CURRENT, PREVIOUS, STATE, SITUATION] { + if root.exists(leaf)? { + return Err("facts store already exists or initialization is incomplete".into()); + } + } + root.create(CURRENT, b"")?; + root.create( + SITUATION, + &situation::empty().to_json().map_err(|error| error.to_string())?, + )?; + root.create( + STATE, + &State { + version: 1, + ..State::default() + } + .bytes()?, + )?; + Ok(Self { + root, + journal_bytes: JOURNAL_BYTES, + }) + } + pub fn open(root: &Path) -> Result { + let facts = Self { + root: Root::open(root, false)?, + journal_bytes: JOURNAL_BYTES, + }; + facts.situation()?; + Ok(facts) + } + /// One event per durable source/revision. A retry of the latest revision returns its + /// original event; older revisions and different contents under that identity are errors. + /// Stop before the next action if this call fails. Replaying the same checkpoint repairs + /// a lost acknowledgement, a torn last line or an interrupted snapshot replacement. + pub fn append(&self, checkpoint: Checkpoint, at_ms: u64, fact: Fact) -> Result { + let _lock = self.root.lock(LOCK)?; + let (mut state, mut snapshot, _) = self.recover()?; + let sources = state.sources()?; + if let Some(previous) = sources + .iter() + .find(|event| event.checkpoint.source == checkpoint.source) + { + if checkpoint.revision == previous.checkpoint.revision { + return if previous.fact == fact { + Ok(previous.clone()) + } else { + Err("checkpoint identity has conflicting facts".into()) + }; + } + if checkpoint.revision < previous.checkpoint.revision { + return Err("stale facts checkpoint".into()); + } + } else if sources.len() == MAX_SOURCES { + return Err("facts source capacity exceeded".into()); + } + let event = Event { + seq: state.last_seq.checked_add(1).ok_or("facts sequence exhausted")?, + at_ms, + checkpoint, + fact, + }; + if event.to_json().map_err(|error| error.to_string())?.len() + 1 > self.journal_bytes { + return Err("event exceeds journal capacity".into()); + } + // Fail capacity/semantic checks before durably reserving an observation. + situation::apply(&mut snapshot, &event)?; + state.pending = Some(event_value(&event)?); + self.root.atomic(STATE, &state.bytes()?)?; + self.recover()?; + Ok(event) + } + pub fn situation(&self) -> Result { + let _lock = self.root.lock(LOCK)?; + self.recover().map(|(_, snapshot, _)| snapshot) + } + pub fn events_after(&self, after: u64, limit: usize) -> Result { + if !(1..=64).contains(&limit) { + return Err("event page limit must be between 1 and 64".into()); + } + let _lock = self.root.lock(LOCK)?; + let (state, _, journal) = self.recover()?; + if after > state.last_seq { + return Err("event cursor is ahead of the journal".into()); + } + let first_available = journal.events.first().map_or(1, |event| event.seq); + let events: Vec<_> = journal + .events + .into_iter() + .filter(|event| event.seq > after) + .take(limit) + .collect(); + let next = events + .last() + .map_or(after.max(first_available - 1), |event| event.seq); + let page = EventPage { + after, + events, + next, + first_available, + gap: after.saturating_add(1) < first_available, + }; + page.to_json().map_err(|error| error.to_string())?; + Ok(page) + } + fn journal(&self, pending: bool) -> Result { + // The only allowed missing current file is an interrupted, durably reserved rotation. + if !self.root.exists(CURRENT)? { + if !pending { + return Err("facts journal is missing".into()); + } + self.root.create(CURRENT, b"")?; + } + let mut events = Vec::new(); + if self.root.exists(PREVIOUS)? { + let bytes = self.root.read(PREVIOUS, self.journal_bytes)?; + parse_lines(&bytes, false, &mut events)?; + } + let bytes = self.root.read(CURRENT, self.journal_bytes)?; + let torn_at = parse_lines(&bytes, pending, &mut events)?; + for pair in events.windows(2) { + if pair[0].seq.checked_add(1) != Some(pair[1].seq) { + return Err("facts journal has a sequence gap".into()); + } + } + Ok(Journal { + events, + current_bytes: bytes.len(), + torn_at, + }) + } + fn recover(&self) -> Result<(State, Snapshot, Journal)> { + let mut state: State = serde_json::from_slice(&self.root.read(STATE, STATE_BYTES)?) + .map_err(|_| "invalid facts metadata")?; + let sources = state.sources()?; + let pending = state.pending.as_ref().map(read_event_value).transpose()?; + let mut journal = self.journal(pending.is_some())?; + let mut snapshot = Snapshot::from_json(&self.root.read(SITUATION, MAX_SNAPSHOT_BYTES)?) + .map_err(|error| error.to_string())?; + let original_seq = snapshot.as_of_seq; + if snapshot.as_of_seq > state.last_seq { + return Err("situation is ahead of committed facts".into()); + } + for event in journal + .events + .iter() + .filter(|event| event.seq > original_seq && event.seq <= state.last_seq) + { + situation::apply(&mut snapshot, event)?; + } + if snapshot.as_of_seq != state.last_seq { + return Err("retained journal cannot recover the situation".into()); + } + if let Some(event) = pending { + if state.last_seq.checked_add(1) != Some(event.seq) + || sources.iter().any(|old| { + old.checkpoint.source == event.checkpoint.source + && old.checkpoint.revision >= event.checkpoint.revision + }) + || !sources + .iter() + .any(|old| old.checkpoint.source == event.checkpoint.source) + && sources.len() >= MAX_SOURCES + { + return Err("invalid pending facts checkpoint".into()); + } + situation::apply(&mut snapshot, &event)?; + if journal.last_seq() == state.last_seq { + if let Some(length) = journal.torn_at { + self.root.truncate(CURRENT, length)?; + journal.current_bytes = length as usize; + } + let mut bytes = event.to_json().map_err(|error| error.to_string())?; + bytes.push(b'\n'); + if bytes.len() > self.journal_bytes { + return Err("event exceeds journal capacity".into()); + } + if journal.current_bytes + bytes.len() > self.journal_bytes { + self.root.rotate(CURRENT, PREVIOUS)?; + } + self.root.append(CURRENT, &bytes)?; + } else if journal.events.last() != Some(&event) || journal.torn_at.is_some() { + return Err("pending fact disagrees with the journal".into()); + } + state.committed(&event)?; + self.root.atomic(STATE, &state.bytes()?)?; + journal = self.journal(false)?; + } else if journal.last_seq() != state.last_seq { + return Err("journal disagrees with committed facts".into()); + } + if snapshot.as_of_seq != original_seq { + self.root + .atomic(SITUATION, &snapshot.to_json().map_err(|error| error.to_string())?)?; + } + Ok((state, snapshot, journal)) + } +} + +fn parse_lines(bytes: &[u8], allow_torn: bool, events: &mut Vec) -> Result> { + let mut offset = 0usize; + for line in bytes.split_inclusive(|byte| *byte == b'\n') { + if !line.ends_with(b"\n") { + return if allow_torn { + Ok(Some(offset as u64)) + } else { + Err("facts journal has an unreserved torn line".into()) + }; + } + if line.len() > MAX_EVENT_BYTES + 1 { + return Err("facts journal line capacity exceeded".into()); + } + events.push(Event::from_json(&line[..line.len() - 1]).map_err(|error| error.to_string())?); + offset += line.len(); + } + Ok(None) +} + +#[cfg(test)] +mod tests; diff --git a/oxa/oxa-work/src/events/tests.rs b/oxa/oxa-work/src/events/tests.rs new file mode 100644 index 000000000..387848616 --- /dev/null +++ b/oxa/oxa-work/src/events/tests.rs @@ -0,0 +1,483 @@ +use super::*; +use oxa_proto::work::*; +use std::fs; +use std::io::Write; +use std::os::unix::fs::{DirBuilderExt, PermissionsExt}; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; + +struct Fixture(PathBuf); +impl Fixture { + fn new() -> Self { + static NEXT: AtomicU64 = AtomicU64::new(0); + loop { + let root = std::env::temp_dir().join(format!( + "oxa-facts-test-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + match fs::DirBuilder::new().mode(0o700).create(&root) { + Ok(()) => return Self(root), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => panic!("create isolated fixture: {error}"), + } + } + } + fn root(&self) -> PathBuf { + self.0.join("facts") + } + fn facts(&self) -> Facts { + Facts::initialize(&self.root()).unwrap() + } +} +impl Drop for Fixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} +fn checkpoint(source: &str, revision: u64) -> Checkpoint { + Checkpoint { + source: source.into(), + revision, + } +} +fn mechanical() -> Fact { + Fact::Mechanical(Mechanical { + subject: Subject::Repository, + kind: MechanicalKind::TransientRetry, + }) +} +fn task(id: usize) -> Running { + Running { + task: format!("task-{id}"), + host: "helper".into(), + branch: format!("agent/task-{id}"), + worktree: format!("~/project/task-{id}"), + session_id: None, + step: TaskStep::Preparing, + } +} +fn reserve(facts: &Facts, fact: Fact) -> Event { + let mut state: State = serde_json::from_slice(&facts.root.read(STATE, STATE_BYTES).unwrap()).unwrap(); + let event = Event { + seq: state.last_seq + 1, + at_ms: 20, + checkpoint: checkpoint("runner", state.last_seq + 1), + fact, + }; + state.pending = Some(event_value(&event).unwrap()); + facts.root.atomic(STATE, &state.bytes().unwrap()).unwrap(); + event +} + +#[test] +fn lifecycle_facts_do_not_turn_worker_success_into_landed_work() { + let env = Fixture::new(); + let facts = env.facts(); + let marker = env.0.join("runner-state.json"); + fs::write(&marker, b"owned only by runner").unwrap(); + let pr = PullRequest { + number: 123, + head: "a".repeat(40), + }; + let observations = [ + Fact::TaskStarted(task(1)), + Fact::TaskTransition(TaskTransition { + task: "task-1".into(), + step: TaskStep::Worker, + session_id: Some("saved".into()), + reason: Some(StepReason::Prepared), + }), + Fact::WorkerFinished(WorkerFinished { + task: "task-1".into(), + session_id: Some("saved".into()), + outcome: WorkerOutcome::Done, + error_class: None, + }), + Fact::LocalAcceptance(LocalAcceptance { + task: "task-1".into(), + commit: pr.head.clone(), + evidence: Evidence { + kind: EvidenceKind::LocalCheck, + id: "checks-1".into(), + }, + }), + Fact::Published(Publication { + task: "task-1".into(), + pr: pr.clone(), + }), + Fact::ExternalAcceptance(ExternalAcceptance { + pr: pr.clone(), + evidence: Evidence { + kind: EvidenceKind::Gate, + id: "pr-run-1".into(), + }, + }), + ]; + for (index, fact) in observations.into_iter().enumerate() { + facts + .append(checkpoint("runner", index as u64 + 1), index as u64, fact) + .unwrap(); + assert!(facts.situation().unwrap().landed.is_empty()); + } + assert_eq!( + facts.situation().unwrap().running[0].session_id.as_deref(), + Some("saved") + ); + facts + .append( + checkpoint("runner", 7), + 7, + Fact::PrMerged(Landed { + pr, + merge_commit: "b".repeat(40), + merged_at_ms: 7, + }), + ) + .unwrap(); + let snapshot = facts.situation().unwrap(); + assert_eq!(snapshot.landed.len(), 1); + assert!(snapshot.published.is_empty()); + assert!(snapshot.local_accepted.is_empty()); + facts + .append( + checkpoint("runner", 8), + 8, + Fact::TaskTransition(TaskTransition { + task: "task-1".into(), + step: TaskStep::Finished, + session_id: Some("saved".into()), + reason: Some(StepReason::Saved), + }), + ) + .unwrap(); + assert!(facts.situation().unwrap().running.is_empty()); + assert_eq!(fs::read(marker).unwrap(), b"owned only by runner"); +} + +#[test] +fn lost_acknowledgements_deduplicate_across_restart_and_changed_clocks() { + let env = Fixture::new(); + let facts = env.facts(); + let original = facts.append(checkpoint("runner", 1), 10, mechanical()).unwrap(); + let reopened = Facts::open(&env.root()).unwrap(); + assert_eq!( + reopened + .append(checkpoint("runner", 1), 999, mechanical()) + .unwrap(), + original + ); + assert!(reopened + .append( + checkpoint("runner", 1), + 999, + Fact::Enqueued(PullRequest { + number: 123, + head: "a".repeat(40) + }) + ) + .is_err()); + reopened + .append(checkpoint("runner", 2), 20, mechanical()) + .unwrap(); + assert!(reopened + .append(checkpoint("runner", 1), 10, mechanical()) + .is_err()); + assert_eq!(reopened.events_after(0, 64).unwrap().events.len(), 2); +} + +#[test] +fn repository_revision_gaps_do_not_create_event_gaps_or_per_task_tombstones() { + let env = Fixture::new(); + let facts = env.facts(); + for revision in [3, 8, 100] { + facts + .append(checkpoint("runner", revision), revision, mechanical()) + .unwrap(); + } + let page = facts.events_after(0, 64).unwrap(); + assert_eq!( + page.events.iter().map(|event| event.seq).collect::>(), + [1, 2, 3] + ); + let state: State = serde_json::from_slice(&facts.root.read(STATE, STATE_BYTES).unwrap()).unwrap(); + assert_eq!(state.last_by_source.len(), 1); +} + +#[test] +fn a_reserved_torn_line_is_replayed_once_and_the_situation_catches_up() { + let env = Fixture::new(); + let facts = env.facts(); + facts.append(checkpoint("runner", 1), 10, mechanical()).unwrap(); + let pending = reserve(&facts, mechanical()); + let mut file = fs::OpenOptions::new() + .append(true) + .open(env.root().join(CURRENT)) + .unwrap(); + file.write_all(b"{\"type\":\"oxa_work_event\"").unwrap(); + file.sync_all().unwrap(); + let recovered = Facts::open(&env.root()).unwrap(); + assert_eq!(recovered.situation().unwrap().as_of_seq, 2); + assert_eq!( + recovered + .append(pending.checkpoint.clone(), 999, pending.fact.clone()) + .unwrap(), + pending + ); + assert_eq!(recovered.events_after(0, 64).unwrap().events.len(), 2); + assert!(fs::read(env.root().join(CURRENT)).unwrap().ends_with(b"\n")); +} + +#[test] +fn an_appended_pending_fact_and_a_lagging_snapshot_recover_without_another_line() { + for committed in [false, true] { + let env = Fixture::new(); + let facts = env.facts(); + facts.append(checkpoint("runner", 1), 10, mechanical()).unwrap(); + let pending = reserve(&facts, mechanical()); + let mut bytes = pending.to_json().unwrap(); + bytes.push(b'\n'); + facts.root.append(CURRENT, &bytes).unwrap(); + if committed { + let mut state: State = + serde_json::from_slice(&facts.root.read(STATE, STATE_BYTES).unwrap()).unwrap(); + state.committed(&pending).unwrap(); + facts.root.atomic(STATE, &state.bytes().unwrap()).unwrap(); + } + let recovered = Facts::open(&env.root()).unwrap(); + assert_eq!(recovered.situation().unwrap().as_of_seq, 2); + assert_eq!(recovered.events_after(0, 64).unwrap().events.len(), 2); + } +} + +#[test] +fn interruption_between_rotation_and_new_current_file_recovers_the_reserved_event() { + let env = Fixture::new(); + let facts = env.facts(); + facts.append(checkpoint("runner", 1), 10, mechanical()).unwrap(); + reserve(&facts, mechanical()); + fs::rename(env.root().join(CURRENT), env.root().join(PREVIOUS)).unwrap(); + let recovered = Facts::open(&env.root()).unwrap(); + assert_eq!( + recovered + .events_after(0, 64) + .unwrap() + .events + .iter() + .map(|event| event.seq) + .collect::>(), + [1, 2] + ); +} + +#[test] +fn rotation_is_bounded_and_cursor_gaps_are_explicit_without_resetting_sequences() { + let env = Fixture::new(); + let mut facts = env.facts(); + facts.journal_bytes = 1024; + let quiet = facts.append(checkpoint("quiet", 1), 0, mechanical()).unwrap(); + for revision in 1..=40 { + facts + .append(checkpoint("busy", revision), revision, mechanical()) + .unwrap(); + } + for leaf in [CURRENT, PREVIOUS] { + assert!(fs::metadata(env.root().join(leaf)).unwrap().len() <= 1024); + } + let page = facts.events_after(0, 64).unwrap(); + assert!(page.gap && page.first_available > 1); + assert_eq!(page.events[0].seq, page.first_available); + assert_eq!(page.next, 41); + assert_eq!( + facts.append(checkpoint("quiet", 1), 999, mechanical()).unwrap(), + quiet + ); + assert_eq!(facts.situation().unwrap().as_of_seq, 41); + assert!(facts.events_after(42, 64).is_err()); + assert!(facts.events_after(0, 0).is_err()); +} + +#[test] +fn concurrent_sources_get_unique_contiguous_repository_sequences() { + let env = Fixture::new(); + let facts = env.facts(); + std::thread::scope(|scope| { + for source in ["one", "two", "three", "four"] { + let facts = &facts; + scope.spawn(move || { + for revision in 1..=8 { + facts + .append(checkpoint(source, revision), revision, mechanical()) + .unwrap(); + } + }); + } + }); + let page = facts.events_after(0, 64).unwrap(); + assert_eq!( + page.events.iter().map(|event| event.seq).collect::>(), + (1..=32).collect::>() + ); +} + +#[test] +fn active_capacity_is_an_error_and_only_recent_terminal_history_is_trimmed() { + let env = Fixture::new(); + let facts = env.facts(); + for id in 0..MAX_ITEMS { + facts + .append( + checkpoint("runner", id as u64 + 1), + 0, + Fact::TaskStarted(task(id)), + ) + .unwrap(); + } + assert!(facts + .append( + checkpoint("runner", MAX_ITEMS as u64 + 1), + 0, + Fact::TaskStarted(task(MAX_ITEMS)) + ) + .is_err()); + assert_eq!(facts.situation().unwrap().running.len(), MAX_ITEMS); + assert_eq!(facts.situation().unwrap().as_of_seq, MAX_ITEMS as u64); + let history = Fixture::new(); + let facts = history.facts(); + for id in 1..=20 { + facts + .append( + checkpoint("runner", id), + id, + Fact::Incident(Incident { + id: format!("incident-{id}"), + subject: Subject::Repository, + kind: IncidentKind::NoBuildRequeue, + evidence: None, + }), + ) + .unwrap(); + } + let snapshot = facts.situation().unwrap(); + assert!(snapshot.history_truncated); + assert_eq!(snapshot.incidents.len(), 16); + assert_eq!(crate::incidents::recent(&facts).unwrap()[0].id, "incident-5"); +} + +#[test] +fn head_changes_and_unrelated_facts_do_not_silently_clear_blocks_or_asks() { + let env = Fixture::new(); + let facts = env.facts(); + let subject = Subject::PullRequest(123); + facts + .append( + checkpoint("runner", 1), + 1, + Fact::Blocked(Blocked { + subject: subject.clone(), + code: "held".into(), + ask_id: Some("ask-1".into()), + }), + ) + .unwrap(); + facts + .append( + checkpoint("runner", 2), + 2, + Fact::AskOpened(Ask { + id: "ask-1".into(), + subject: subject.clone(), + question: "Release hold?".into(), + }), + ) + .unwrap(); + facts + .append( + checkpoint("runner", 3), + 3, + Fact::Published(Publication { + task: "task-1".into(), + pr: PullRequest { + number: 123, + head: "b".repeat(40), + }, + }), + ) + .unwrap(); + let snapshot = facts.situation().unwrap(); + assert_eq!(snapshot.blocked.len(), 1); + assert_eq!(snapshot.asks.len(), 1); + facts + .append(checkpoint("runner", 4), 4, Fact::Unblocked(subject)) + .unwrap(); + facts + .append( + checkpoint("runner", 5), + 5, + Fact::AskClosed(AskClosed { id: "ask-1".into() }), + ) + .unwrap(); + let snapshot = facts.situation().unwrap(); + assert!(snapshot.blocked.is_empty() && snapshot.asks.is_empty()); +} + +#[test] +fn missing_corrupt_and_unreserved_torn_files_fail_instead_of_becoming_empty() { + for leaf in [STATE, SITUATION, CURRENT] { + let env = Fixture::new(); + let _facts = env.facts(); + fs::remove_file(env.root().join(leaf)).unwrap(); + assert!(Facts::open(&env.root()).is_err()); + assert!(Facts::initialize(&env.root()).is_err()); + } + for leaf in [STATE, SITUATION, CURRENT] { + let env = Fixture::new(); + let _facts = env.facts(); + fs::write(env.root().join(leaf), b"PRIVATE_MARKER").unwrap(); + let error = Facts::open(&env.root()).unwrap_err(); + assert!(!error.contains("PRIVATE_MARKER")); + } +} + +#[test] +fn poisoned_leaves_and_root_symlinks_never_read_or_replace_the_target() { + for leaf in [STATE, SITUATION, CURRENT, LOCK] { + let env = Fixture::new(); + let _facts = env.facts(); + let target = env.0.join("outside"); + fs::write(&target, b"keep this target").unwrap(); + fs::remove_file(env.root().join(leaf)).unwrap(); + std::os::unix::fs::symlink(&target, env.root().join(leaf)).unwrap(); + assert!(Facts::open(&env.root()).is_err()); + assert_eq!(fs::read(target).unwrap(), b"keep this target"); + } + let env = Fixture::new(); + let target = env.0.join("outside"); + fs::create_dir(&target).unwrap(); + std::os::unix::fs::symlink(&target, env.root()).unwrap(); + assert!(Facts::initialize(&env.root()).is_err()); + assert!(fs::read_dir(target).unwrap().next().is_none()); +} + +#[test] +fn artifacts_are_private_and_source_capacity_does_not_consume_a_sequence() { + let env = Fixture::new(); + let facts = env.facts(); + assert_eq!( + fs::metadata(env.root()).unwrap().permissions().mode() & 0o777, + 0o700 + ); + for leaf in [STATE, SITUATION, CURRENT, LOCK] { + assert_eq!( + fs::metadata(env.root().join(leaf)).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } + for source in 0..MAX_SOURCES { + facts + .append(checkpoint(&format!("source-{source}"), 1), 0, mechanical()) + .unwrap(); + } + assert!(facts.append(checkpoint("too-many", 1), 0, mechanical()).is_err()); + assert_eq!(facts.situation().unwrap().as_of_seq, MAX_SOURCES as u64); +} diff --git a/oxa/oxa-work/src/facts_io.rs b/oxa/oxa-work/src/facts_io.rs new file mode 100644 index 000000000..f55f20855 --- /dev/null +++ b/oxa/oxa-work/src/facts_io.rs @@ -0,0 +1,203 @@ +//! Private files under the caller's protected repository root, outside writable Git binds. +use crate::Result; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +use std::os::unix::fs::{DirBuilderExt, MetadataExt, OpenOptionsExt}; +use std::path::{Component, Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +fn io(result: std::io::Result, what: &str) -> Result { + result.map_err(|error| format!("{what}: {:?}", error.kind())) +} +fn private(metadata: &fs::Metadata, directory: bool) -> Result<()> { + // SAFETY: geteuid has no arguments, pointers or failure mode. + let uid = unsafe { libc::geteuid() }; + if metadata.uid() != uid + || metadata.mode() & 0o077 != 0 + || if directory { + !metadata.is_dir() + } else { + !metadata.is_file() + } + { + return Err("facts storage must be private and owned by this user".into()); + } + Ok(()) +} + +#[derive(Clone, Debug)] +pub(crate) struct Root(PathBuf); +impl Root { + pub(crate) fn open(path: &Path, create: bool) -> Result { + if !path.is_absolute() { + return Err("facts root must be absolute".into()); + } + let mut parent = PathBuf::new(); + for component in path.components() { + if matches!(component, Component::ParentDir) { + return Err("facts root must be canonical".into()); + } + parent.push(component); + match fs::symlink_metadata(&parent) { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err("facts root contains a symlink".into()) + } + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound && parent == path && create => { + io( + fs::DirBuilder::new().mode(0o700).create(path), + "cannot create facts root", + )?; + } + Err(error) => return Err(format!("cannot inspect facts root: {:?}", error.kind())), + } + } + let metadata = io(fs::symlink_metadata(path), "cannot inspect facts root")?; + private(&metadata, true)?; + Ok(Self(path.to_path_buf())) + } + fn path(&self, leaf: &str) -> PathBuf { + debug_assert!(!leaf.contains('/') && leaf != "." && leaf != ".."); + self.0.join(leaf) + } + fn options(&self) -> OpenOptions { + let mut options = OpenOptions::new(); + options + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) + .mode(0o600); + options + } + pub(crate) fn exists(&self, leaf: &str) -> Result { + match fs::symlink_metadata(self.path(leaf)) { + Ok(metadata) => { + private(&metadata, false)?; + Ok(true) + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(format!("cannot inspect facts file: {:?}", error.kind())), + } + } + pub(crate) fn lock(&self, leaf: &str) -> Result { + let file = io( + self.options() + .read(true) + .write(true) + .create(true) + .open(self.path(leaf)), + "cannot open facts lock", + )?; + private(&io(file.metadata(), "cannot inspect facts lock")?, false)?; + io(file.lock(), "cannot lock facts storage")?; + Ok(file) + } + pub(crate) fn read(&self, leaf: &str, limit: usize) -> Result> { + let file = io( + self.options().read(true).open(self.path(leaf)), + "cannot open facts file", + )?; + let metadata = io(file.metadata(), "cannot inspect facts file")?; + private(&metadata, false)?; + if metadata.len() > limit as u64 { + return Err("facts file capacity exceeded".into()); + } + let mut bytes = Vec::new(); + io( + file.take(limit as u64 + 1).read_to_end(&mut bytes), + "cannot read facts file", + )?; + if bytes.len() > limit { + return Err("facts file capacity exceeded".into()); + } + Ok(bytes) + } + pub(crate) fn create(&self, leaf: &str, bytes: &[u8]) -> Result<()> { + let mut file = io( + self.options().write(true).create_new(true).open(self.path(leaf)), + "cannot create facts file", + )?; + io( + file.write_all(bytes).and_then(|()| file.sync_all()), + "cannot save facts file", + )?; + self.sync() + } + pub(crate) fn atomic(&self, leaf: &str, bytes: &[u8]) -> Result<()> { + // Inspect the existing leaf without following it; never replace poisoned enforcement data. + self.exists(leaf)?; + static NEXT: AtomicU64 = AtomicU64::new(0); + for _ in 0..16 { + let sequence = NEXT.fetch_add(1, Ordering::Relaxed); + let temporary = format!(".{leaf}.tmp-{}-{sequence}", std::process::id()); + let mut file = match self + .options() + .write(true) + .create_new(true) + .open(self.path(&temporary)) + { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(format!("cannot create facts temporary: {:?}", error.kind())), + }; + let result = io( + file.write_all(bytes).and_then(|()| file.sync_all()), + "cannot save facts temporary", + ) + .and_then(|()| { + io( + fs::rename(self.path(&temporary), self.path(leaf)), + "cannot replace facts file", + ) + }) + .and_then(|()| self.sync()); + if result.is_err() { + let _ = fs::remove_file(self.path(&temporary)); + } + return result; + } + Err("cannot reserve facts temporary".into()) + } + pub(crate) fn append(&self, leaf: &str, bytes: &[u8]) -> Result<()> { + let mut file = io( + self.options().append(true).open(self.path(leaf)), + "cannot open facts journal", + )?; + private(&io(file.metadata(), "cannot inspect facts journal")?, false)?; + io( + file.write_all(bytes).and_then(|()| file.sync_all()), + "cannot append facts journal", + ) + } + pub(crate) fn truncate(&self, leaf: &str, length: u64) -> Result<()> { + let file = io( + self.options().write(true).open(self.path(leaf)), + "cannot open torn facts journal", + )?; + private(&io(file.metadata(), "cannot inspect torn facts journal")?, false)?; + io( + file.set_len(length).and_then(|()| file.sync_all()), + "cannot recover torn facts journal", + ) + } + pub(crate) fn rotate(&self, from: &str, to: &str) -> Result<()> { + if !self.exists(from)? { + return Err("facts journal is missing".into()); + } + self.exists(to)?; + io( + fs::rename(self.path(from), self.path(to)), + "cannot rotate facts journal", + )?; + self.sync()?; + self.create(from, b"") + } + fn sync(&self) -> Result<()> { + let directory = io( + self.options() + .read(true) + .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW) + .open(&self.0), + "cannot open facts directory", + )?; + io(directory.sync_all(), "cannot sync facts directory") + } +} diff --git a/oxa/oxa-work/src/incidents.rs b/oxa/oxa-work/src/incidents.rs new file mode 100644 index 000000000..e97d1efaf --- /dev/null +++ b/oxa/oxa-work/src/incidents.rs @@ -0,0 +1,194 @@ +//! Bounded incident reads and pure stall detection. A stall produces an incident, never a kill. +use crate::{events::Facts, Result}; +use oxa_proto::work::{Evidence, EvidenceKind, Incident, IncidentKind, Subject}; +use serde_json::Value; + +pub fn recent(facts: &Facts) -> Result> { + facts.situation().map(|snapshot| snapshot.incidents) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Stall { + pub session_id: String, + pub pid: u32, + pub last_progress_ms: u64, + pub deadline_ms: u64, +} +impl Stall { + /// Build an incident for the runner to append. This never wraps, cancels or kills a worker. + pub fn incident(&self, task: &str) -> Incident { + Incident { + id: format!("stall:{}:{}:{}", self.session_id, self.pid, self.last_progress_ms), + subject: Subject::Task(task.into()), + kind: IncidentKind::WorkerStalled, + evidence: Some(Evidence { + kind: EvidenceKind::Status, + id: format!("session:{}", self.session_id), + }), + } + } +} +/// Consume a bounded status snapshot supplied by the runner, never a transcript or tool output. +pub fn detect_stall(bytes: &[u8], session_id: &str, pid: u32, now_ms: u64) -> Result> { + if pid == 0 + || session_id.is_empty() + || session_id.len() > 128 + || !session_id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"-_".contains(&byte)) + { + return Err("invalid expected worker identity".into()); + } + if bytes.len() > 4096 { + return Err("worker status capacity exceeded".into()); + } + let status: Value = serde_json::from_slice(bytes).map_err(|_| "invalid worker status JSON")?; + if status["type"] != "oxa_status" + || status["version"] != 1 + || status["session_id"] != session_id + || status["pid"].as_u64() != Some(pid.into()) + { + return Err("worker status identity or version mismatch".into()); + } + let phase = status["phase"].as_str().ok_or("worker status phase is missing")?; + let outcome = status.get("outcome").ok_or("worker status outcome is missing")?; + let deadline = status + .get("stall_deadline_ms") + .ok_or("worker status deadline is missing")?; + let timeout = status + .get("stall_timeout_ms") + .ok_or("worker status timeout is missing")?; + let timeout = if timeout.is_null() { + None + } else { + Some( + timeout + .as_u64() + .filter(|timeout| *timeout > 0) + .ok_or("invalid worker stall timeout")?, + ) + }; + if !matches!( + phase, + "idle" | "working" | "model" | "prefill" | "compaction" | "tool" | "closing" | "finished" + ) { + return Err("unknown worker status phase".into()); + } + if phase == "finished" { + if outcome["type"] != "oxa_result" + || !matches!( + outcome["outcome"].as_str(), + Some("done" | "nothing_to_do" | "blocked" | "failed" | "interrupted") + ) + || outcome["exit_code"].as_i64().is_none() + || !deadline.is_null() + { + return Err("finished worker status has no outcome".into()); + } + return Ok(None); + } + if !outcome.is_null() { + return Err("active worker status has a terminal outcome".into()); + } + let progress = status["last_progress_ms"] + .as_u64() + .ok_or("worker status progress is missing")?; + if deadline.is_null() { + return Ok(None); + } + let deadline = deadline.as_u64().ok_or("invalid worker status deadline")?; + let timeout = timeout.ok_or("worker deadline has no configured timeout")?; + let earliest = progress.saturating_add(timeout); + if deadline < earliest || deadline > earliest.saturating_add(1000) { + return Err("worker status deadline is inconsistent with its timeout".into()); + } + Ok((now_ms >= deadline).then(|| Stall { + session_id: session_id.into(), + pid, + last_progress_ms: progress, + deadline_ms: deadline, + })) +} + +#[cfg(test)] +mod tests { + use super::*; + use oxa_proto::work::{Checkpoint, Event, Fact}; + use serde_json::json; + fn status() -> Value { + json!({"type":"oxa_status", "version":1, "session_id":"saved", "pid":123, + "phase":"model", "last_progress_ms":100, "stall_timeout_ms":100, "stall_deadline_ms":200, "outcome":null}) + } + fn bytes(value: &Value) -> Vec { + serde_json::to_vec(value).unwrap() + } + + #[test] + fn deadline_is_conservative_and_detection_never_changes_the_input_or_takes_action() { + let input = bytes(&status()); + let original = input.clone(); + assert!(detect_stall(&input, "saved", 123, 199).unwrap().is_none()); + let stall = detect_stall(&input, "saved", 123, 200).unwrap().unwrap(); + assert_eq!(stall.last_progress_ms, 100); + assert_eq!(stall.deadline_ms, 200); + let event = Event { + seq: 1, + at_ms: 200, + checkpoint: Checkpoint { + source: "runner".into(), + revision: 1, + }, + fact: Fact::Incident(stall.incident("task-1")), + }; + assert_eq!(Event::from_json(&event.to_json().unwrap()).unwrap(), event); + assert_eq!(input, original); + let mut progress = status(); + progress["last_progress_ms"] = json!(190); + progress["stall_deadline_ms"] = json!(290); + assert!(detect_stall(&bytes(&progress), "saved", 123, 200) + .unwrap() + .is_none()); + } + + #[test] + fn stale_identity_missing_fields_and_invalid_status_are_errors() { + let input = bytes(&status()); + assert!(detect_stall(&input, "different", 123, 300).is_err()); + assert!(detect_stall(&input, "saved", 456, 300).is_err()); + for field in [ + "pid", + "phase", + "last_progress_ms", + "stall_deadline_ms", + "stall_timeout_ms", + "outcome", + ] { + let mut invalid = status(); + invalid.as_object_mut().unwrap().remove(field); + assert!( + detect_stall(&bytes(&invalid), "saved", 123, 300).is_err(), + "{field}" + ); + } + let mut invalid = status(); + invalid["stall_deadline_ms"] = json!(50); + assert!(detect_stall(&bytes(&invalid), "saved", 123, 300).is_err()); + assert!(detect_stall(&vec![b' '; 4097], "saved", 123, 300).is_err()); + assert!(detect_stall(b"{}", "saved", 123, 300).is_err()); + } + + #[test] + fn idle_disabled_and_finished_sessions_do_not_stall() { + let mut value = status(); + value["phase"] = json!("idle"); + value["stall_deadline_ms"] = Value::Null; + assert!(detect_stall(&bytes(&value), "saved", 123, 300).unwrap().is_none()); + value["phase"] = json!("model"); + assert!(detect_stall(&bytes(&value), "saved", 123, 300).unwrap().is_none()); + value["phase"] = json!("finished"); + value["outcome"] = json!({"type":"oxa_result", "outcome":"done", "exit_code":0}); + assert!(detect_stall(&bytes(&value), "saved", 123, 300).unwrap().is_none()); + value["outcome"] = Value::Null; + assert!(detect_stall(&bytes(&value), "saved", 123, 300).is_err()); + } +} diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index dc74e7942..51e0ec533 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -11,6 +11,12 @@ pub use worker::{WorkerOutcome, WorkerResult}; pub type Result = std::result::Result; +// Lead facts consume public observations and never open runner task-state files. +pub mod events; +mod facts_io; +pub mod incidents; +pub mod situation; + #[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] pub struct Task { diff --git a/oxa/oxa-work/src/situation.rs b/oxa/oxa-work/src/situation.rs new file mode 100644 index 000000000..3a84f03e1 --- /dev/null +++ b/oxa/oxa-work/src/situation.rs @@ -0,0 +1,132 @@ +//! Bounded projection of typed facts. This module never reads or writes runner task state. +use crate::Result; +use oxa_proto::work::{Event, Fact, Snapshot, TaskStep, MAX_ITEMS}; + +pub(crate) fn empty() -> Snapshot { + Snapshot { + as_of_seq: 0, + at_ms: 0, + standing_revision: 0, + goal: None, + running: vec![], + local_accepted: vec![], + published: vec![], + landed: vec![], + blocked: vec![], + asks: vec![], + incidents: vec![], + history_truncated: false, + } +} + +pub(crate) fn apply(snapshot: &mut Snapshot, event: &Event) -> Result<()> { + if snapshot.as_of_seq.checked_add(1) != Some(event.seq) { + return Err("situation replay has a sequence gap".into()); + } + match &event.fact { + Fact::TaskStarted(task) => { + snapshot.running.retain(|row| row.task != task.task); + snapshot.running.push(task.clone()); + } + Fact::TaskTransition(change) => { + if change.step == TaskStep::Finished { + snapshot.running.retain(|row| row.task != change.task); + } else { + let task = snapshot + .running + .iter_mut() + .find(|row| row.task == change.task) + .ok_or("task transition has no running location")?; + task.step = change.step; + task.session_id = change.session_id.clone(); + } + } + Fact::WorkerFinished(worker) => { + if let Some(task) = snapshot.running.iter_mut().find(|row| row.task == worker.task) { + task.session_id = worker.session_id.clone(); + } + } + Fact::LocalAcceptance(accepted) => { + snapshot.local_accepted.retain(|row| row.task != accepted.task); + snapshot.local_accepted.push(accepted.clone()); + } + Fact::Published(publication) => { + snapshot.published.retain(|row| row.task != publication.task); + snapshot.published.push(publication.clone()); + } + Fact::PrMerged(landed) => { + let tasks: Vec<_> = snapshot + .published + .iter() + .filter(|row| row.pr == landed.pr) + .map(|row| row.task.clone()) + .collect(); + snapshot + .local_accepted + .retain(|row| !(tasks.contains(&row.task) && row.commit == landed.pr.head)); + snapshot.published.retain(|row| row.pr != landed.pr); + snapshot.landed.retain(|row| row.pr.number != landed.pr.number); + snapshot.landed.push(landed.clone()); + } + Fact::Blocked(block) => { + snapshot.blocked.retain(|row| row.subject != block.subject); + snapshot.blocked.push(block.clone()); + } + Fact::Unblocked(subject) => snapshot.blocked.retain(|row| &row.subject != subject), + Fact::AskOpened(ask) => { + if let Some(existing) = snapshot.asks.iter().find(|row| row.id == ask.id) { + if existing != ask { + return Err("ask id has conflicting contents".into()); + } + } else { + snapshot.asks.push(ask.clone()); + } + } + Fact::AskClosed(ask) => snapshot.asks.retain(|row| row.id != ask.id), + Fact::Incident(incident) => { + if let Some(existing) = snapshot.incidents.iter().find(|row| row.id == incident.id) { + if existing != incident { + return Err("incident id has conflicting contents".into()); + } + } else { + snapshot.incidents.push(incident.clone()); + } + } + Fact::ExternalAcceptance(_) | Fact::Enqueued(_) | Fact::Mechanical(_) => {} + } + const HISTORY: usize = 16; + for removed in [ + snapshot + .landed + .drain(..snapshot.landed.len().saturating_sub(HISTORY)) + .count(), + snapshot + .incidents + .drain(..snapshot.incidents.len().saturating_sub(HISTORY)) + .count(), + ] { + snapshot.history_truncated |= removed > 0; + } + while snapshot.running.len() + + snapshot.local_accepted.len() + + snapshot.published.len() + + snapshot.landed.len() + + snapshot.blocked.len() + + snapshot.asks.len() + + snapshot.incidents.len() + > MAX_ITEMS + { + if !snapshot.landed.is_empty() { + snapshot.landed.remove(0); + } else if !snapshot.incidents.is_empty() { + snapshot.incidents.remove(0); + } else { + return Err("active situation capacity exceeded".into()); + } + snapshot.history_truncated = true; + } + snapshot.as_of_seq = event.seq; + snapshot.at_ms = event.at_ms; + snapshot.to_json().map_err(|error| error.to_string())?; + Ok(()) +} From f765d3b06a80e4af0a43cf6424a5cb235501d491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:07:55 +0000 Subject: [PATCH 03/27] Commit a prepared fact without replaying the journal twice --- oxa/oxa-work/src/events.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/oxa/oxa-work/src/events.rs b/oxa/oxa-work/src/events.rs index e70e267b0..20530c36d 100644 --- a/oxa/oxa-work/src/events.rs +++ b/oxa/oxa-work/src/events.rs @@ -137,7 +137,7 @@ impl Facts { /// a lost acknowledgement, a torn last line or an interrupted snapshot replacement. pub fn append(&self, checkpoint: Checkpoint, at_ms: u64, fact: Fact) -> Result { let _lock = self.root.lock(LOCK)?; - let (mut state, mut snapshot, _) = self.recover()?; + let (mut state, mut snapshot, journal) = self.recover()?; let sources = state.sources()?; if let Some(previous) = sources .iter() @@ -162,14 +162,23 @@ impl Facts { checkpoint, fact, }; - if event.to_json().map_err(|error| error.to_string())?.len() + 1 > self.journal_bytes { + let mut bytes = event.to_json().map_err(|error| error.to_string())?; + bytes.push(b'\n'); + if bytes.len() > self.journal_bytes { return Err("event exceeds journal capacity".into()); } // Fail capacity/semantic checks before durably reserving an observation. situation::apply(&mut snapshot, &event)?; state.pending = Some(event_value(&event)?); self.root.atomic(STATE, &state.bytes()?)?; - self.recover()?; + if journal.current_bytes + bytes.len() > self.journal_bytes { + self.root.rotate(CURRENT, PREVIOUS)?; + } + self.root.append(CURRENT, &bytes)?; + state.committed(&event)?; + self.root.atomic(STATE, &state.bytes()?)?; + self.root + .atomic(SITUATION, &snapshot.to_json().map_err(|error| error.to_string())?)?; Ok(event) } pub fn situation(&self) -> Result { From f0da0e0e68728e377e53de79b8dde95ca17dd04d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:59:05 +0000 Subject: [PATCH 04/27] test(work): reproduce facts byte retention and parent durability failures --- oxa/oxa-work/src/events/tests.rs | 76 ++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/oxa/oxa-work/src/events/tests.rs b/oxa/oxa-work/src/events/tests.rs index 387848616..54a7be3a0 100644 --- a/oxa/oxa-work/src/events/tests.rs +++ b/oxa/oxa-work/src/events/tests.rs @@ -364,6 +364,82 @@ fn active_capacity_is_an_error_and_only_recent_terminal_history_is_trimmed() { assert_eq!(crate::incidents::recent(&facts).unwrap()[0].id, "incident-5"); } +#[test] +fn byte_capacity_trims_history_before_refusing_active_asks() { + let env = Fixture::new(); + let facts = env.facts(); + for id in 0..16 { + facts + .append( + checkpoint("runner", id + 1), + 0, + Fact::Incident(Incident { + id: format!("{id:016}{}", "i".repeat(240)), + subject: Subject::Task("t".repeat(128)), + kind: IncidentKind::NoBuildRequeue, + evidence: Some(Evidence { + kind: EvidenceKind::Gate, + id: "e".repeat(256), + }), + }), + ) + .unwrap(); + } + for id in 0..95 { + facts + .append( + checkpoint("runner", id + 17), + 0, + Fact::AskOpened(Ask { + id: format!("ask-{id}"), + subject: Subject::Repository, + question: "q".repeat(512), + }), + ) + .unwrap(); + } + let snapshot = facts.situation().unwrap(); + assert_eq!(snapshot.as_of_seq, 111); + assert_eq!(snapshot.asks.len(), 95); + assert!(snapshot.history_truncated); + assert!(snapshot.incidents.len() < 16); + assert!(snapshot.to_json().unwrap().len() <= MAX_SNAPSHOT_BYTES); + assert_eq!(Facts::open(&env.root()).unwrap().situation().unwrap(), snapshot); +} + +#[test] +fn parent_directory_sync_error_blocks_initialization() { + const CHILD_ROOT: &str = "OXA_FACTS_TEST_UNREADABLE_PARENT"; + if let Some(root) = std::env::var_os(CHILD_ROOT) { + let error = Facts::initialize(Path::new(&root)).expect_err("parent cannot be synced"); + assert!(error.contains("directory"), "{error}"); + return; + } + let env = Fixture::new(); + let mut child = std::process::Command::new(std::env::current_exe().unwrap()); + child.env_clear().env(CHILD_ROOT, env.root()).args([ + "--exact", + "events::tests::parent_directory_sync_error_blocks_initialization", + ]); + // Root can read mode 0300. Exercise the same real permission failure as a normal user. + // SAFETY: geteuid has no arguments, pointers or failure mode. + if unsafe { libc::geteuid() } == 0 { + use std::os::unix::process::CommandExt; + std::os::unix::fs::chown(&env.0, Some(65534), Some(65534)).unwrap(); + child.uid(65534).gid(65534); + } + fs::set_permissions(&env.0, fs::Permissions::from_mode(0o300)).unwrap(); + let output = child.output(); + fs::set_permissions(&env.0, fs::Permissions::from_mode(0o700)).unwrap(); + let output = output.unwrap(); + assert!( + output.status.success(), + "child failed: {}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + #[test] fn head_changes_and_unrelated_facts_do_not_silently_clear_blocks_or_asks() { let env = Fixture::new(); From 65f463c876c1fc2325bacc4e2ad53c942b293333 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:00:32 +0000 Subject: [PATCH 05/27] fix(work): retain active facts within byte bounds and sync root parent --- oxa/oxa-work/src/events/tests.rs | 19 +++++++++++++++++++ oxa/oxa-work/src/facts_io.rs | 25 +++++++++++++++++-------- oxa/oxa-work/src/situation.rs | 25 ++++++++++++++++++------- 3 files changed, 54 insertions(+), 15 deletions(-) diff --git a/oxa/oxa-work/src/events/tests.rs b/oxa/oxa-work/src/events/tests.rs index 54a7be3a0..f754da8a5 100644 --- a/oxa/oxa-work/src/events/tests.rs +++ b/oxa/oxa-work/src/events/tests.rs @@ -405,6 +405,25 @@ fn byte_capacity_trims_history_before_refusing_active_asks() { assert!(snapshot.incidents.len() < 16); assert!(snapshot.to_json().unwrap().len() <= MAX_SNAPSHOT_BYTES); assert_eq!(Facts::open(&env.root()).unwrap().situation().unwrap(), snapshot); + for id in 95..MAX_ITEMS { + let before = facts.situation().unwrap(); + let result = facts.append( + checkpoint("runner", id as u64 + 17), + 0, + Fact::AskOpened(Ask { + id: format!("ask-{id}"), + subject: Subject::Repository, + question: "q".repeat(512), + }), + ); + if result.is_err() { + assert!(before.incidents.is_empty()); + assert_eq!(facts.situation().unwrap(), before); + return; + } + assert_eq!(facts.situation().unwrap().asks.len(), id + 1); + } + panic!("active asks alone must reach the byte limit before the row limit"); } #[test] diff --git a/oxa/oxa-work/src/facts_io.rs b/oxa/oxa-work/src/facts_io.rs index f55f20855..34029fd57 100644 --- a/oxa/oxa-work/src/facts_io.rs +++ b/oxa/oxa-work/src/facts_io.rs @@ -54,6 +54,11 @@ impl Root { } let metadata = io(fs::symlink_metadata(path), "cannot inspect facts root")?; private(&metadata, true)?; + if create { + // A synced child does not persist its entry in the containing directory. + // Also sync on an initialization retry after an earlier mkdir/sync failure. + sync_directory(path.parent().ok_or("facts root has no parent directory")?)?; + } Ok(Self(path.to_path_buf())) } fn path(&self, leaf: &str) -> PathBuf { @@ -191,13 +196,17 @@ impl Root { self.create(from, b"") } fn sync(&self) -> Result<()> { - let directory = io( - self.options() - .read(true) - .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW) - .open(&self.0), - "cannot open facts directory", - )?; - io(directory.sync_all(), "cannot sync facts directory") + sync_directory(&self.0) } } + +fn sync_directory(path: &Path) -> Result<()> { + let directory = io( + OpenOptions::new() + .read(true) + .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW) + .open(path), + "cannot open facts directory", + )?; + io(directory.sync_all(), "cannot sync facts directory") +} diff --git a/oxa/oxa-work/src/situation.rs b/oxa/oxa-work/src/situation.rs index 3a84f03e1..fd9f67fef 100644 --- a/oxa/oxa-work/src/situation.rs +++ b/oxa/oxa-work/src/situation.rs @@ -116,17 +116,28 @@ pub(crate) fn apply(snapshot: &mut Snapshot, event: &Event) -> Result<()> { + snapshot.incidents.len() > MAX_ITEMS { - if !snapshot.landed.is_empty() { - snapshot.landed.remove(0); - } else if !snapshot.incidents.is_empty() { - snapshot.incidents.remove(0); - } else { + if !discard_history(snapshot) { return Err("active situation capacity exceeded".into()); } - snapshot.history_truncated = true; } snapshot.as_of_seq = event.seq; snapshot.at_ms = event.at_ms; - snapshot.to_json().map_err(|error| error.to_string())?; + while let Err(error) = snapshot.to_json() { + if !discard_history(snapshot) { + return Err(error.to_string()); + } + } Ok(()) } + +fn discard_history(snapshot: &mut Snapshot) -> bool { + if !snapshot.landed.is_empty() { + snapshot.landed.remove(0); + } else if !snapshot.incidents.is_empty() { + snapshot.incidents.remove(0); + } else { + return false; + } + snapshot.history_truncated = true; + true +} From d325ab2bf7c832cf7dd0f661ffbe2e1e6ef1dc3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:15:34 +0000 Subject: [PATCH 06/27] test(work): reproduce acknowledging an unsynced pending journal line --- oxa/oxa-work/src/events/tests.rs | 49 ++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/oxa/oxa-work/src/events/tests.rs b/oxa/oxa-work/src/events/tests.rs index f754da8a5..420a1e650 100644 --- a/oxa/oxa-work/src/events/tests.rs +++ b/oxa/oxa-work/src/events/tests.rs @@ -250,6 +250,55 @@ fn an_appended_pending_fact_and_a_lagging_snapshot_recover_without_another_line( } } +#[test] +fn pending_recovery_cannot_commit_without_opening_the_journal_for_sync() { + // Root bypasses the real journal permission failure; run this case as an ordinary user. + // SAFETY: geteuid has no arguments, pointers or failure mode. + if unsafe { libc::geteuid() } == 0 { + use std::os::unix::process::CommandExt; + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .env_clear() + .args([ + "--exact", + "events::tests::pending_recovery_cannot_commit_without_opening_the_journal_for_sync", + ]) + .uid(65534) + .gid(65534) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + return; + } + let env = Fixture::new(); + let facts = env.facts(); + let pending = reserve(&facts, mechanical()); + let mut bytes = pending.to_json().unwrap(); + bytes.push(b'\n'); + // The complete line is visible without having been synced by its writer. + fs::OpenOptions::new() + .append(true) + .open(env.root().join(CURRENT)) + .unwrap() + .write_all(&bytes) + .unwrap(); + let before = facts.root.read(STATE, STATE_BYTES).unwrap(); + let snapshot = facts.root.read(SITUATION, MAX_SNAPSHOT_BYTES).unwrap(); + fs::set_permissions(env.root().join(CURRENT), fs::Permissions::from_mode(0o400)).unwrap(); + let recovered = Facts::open(&env.root()); + fs::set_permissions(env.root().join(CURRENT), fs::Permissions::from_mode(0o600)).unwrap(); + assert!(recovered.is_err(), "unsynced pending line was acknowledged"); + assert_eq!(facts.root.read(STATE, STATE_BYTES).unwrap(), before); + assert_eq!(facts.root.read(SITUATION, MAX_SNAPSHOT_BYTES).unwrap(), snapshot); + let recovered = Facts::open(&env.root()).unwrap(); + assert_eq!(recovered.situation().unwrap().as_of_seq, pending.seq); + assert_eq!(recovered.events_after(0, 64).unwrap().events, [pending]); +} + #[test] fn interruption_between_rotation_and_new_current_file_recovers_the_reserved_event() { let env = Fixture::new(); From 667f15ad525c57d97040b359e8eee071e2744a5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:16:25 +0000 Subject: [PATCH 07/27] fix(work): sync visible pending journal data before recovery acknowledgement --- oxa/oxa-work/src/events.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/oxa/oxa-work/src/events.rs b/oxa/oxa-work/src/events.rs index 20530c36d..6f45883a4 100644 --- a/oxa/oxa-work/src/events.rs +++ b/oxa/oxa-work/src/events.rs @@ -292,6 +292,14 @@ impl Facts { self.root.append(CURRENT, &bytes)?; } else if journal.events.last() != Some(&event) || journal.torn_at.is_some() { return Err("pending fact disagrees with the journal".into()); + } else { + // Visibility is not durability: the original append may have died before sync. + let leaf = if journal.current_bytes == 0 { + PREVIOUS + } else { + CURRENT + }; + self.root.append(leaf, b"")?; } state.committed(&event)?; self.root.atomic(STATE, &state.bytes()?)?; From f8753beb5e7460ee72f251410405df7f9cade876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:34:56 +0000 Subject: [PATCH 08/27] test(gateway): reserve fixture directories independently of wall-clock ticks --- oxa/oxc-gateway/tests/common/skeleton.rs | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/oxa/oxc-gateway/tests/common/skeleton.rs b/oxa/oxc-gateway/tests/common/skeleton.rs index a2acb8b28..f57f8f849 100644 --- a/oxa/oxc-gateway/tests/common/skeleton.rs +++ b/oxa/oxc-gateway/tests/common/skeleton.rs @@ -6,7 +6,7 @@ use std::{ io::Write, os::unix::fs::{DirBuilderExt, OpenOptionsExt}, path::PathBuf, - time::{SystemTime, UNIX_EPOCH}, + sync::atomic::{AtomicU64, Ordering}, }; pub const MODEL: &str = "synthetic-cloud"; @@ -23,12 +23,19 @@ impl Fixture { Self::for_endpoint(&mock.base_url()) } pub fn for_endpoint(endpoint: &str) -> Self { - let root = std::env::temp_dir().join(format!( - "oxc-skeleton-{}-{}", - std::process::id(), - SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos() - )); - DirBuilder::new().mode(0o700).create(&root).unwrap(); + static NEXT: AtomicU64 = AtomicU64::new(0); + let root = loop { + let path = std::env::temp_dir().join(format!( + "oxc-skeleton-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + match DirBuilder::new().mode(0o700).create(&path) { + Ok(()) => break path, + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => panic!("cannot create private synthetic fixture: {error}"), + } + }; let key_path = root.join("credentials.json"); let mut key = OpenOptions::new() .write(true) From 9e92adb7c77c5251497bf2091971a7a91197a681 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:19:04 +0000 Subject: [PATCH 09/27] WIP: implement protected standing core and revision-bound asks --- oxa/oxa-proto/src/work.rs | 22 +- oxa/oxa-work/src/events.rs | 12 + oxa/oxa-work/src/facts_io.rs | 9 + oxa/oxa-work/src/lib.rs | 1 + oxa/oxa-work/src/situation.rs | 55 ++- oxa/oxa-work/src/standing.rs | 9 + oxa/oxa-work/src/standing/asks.rs | 132 ++++++ oxa/oxa-work/src/standing/guard.rs | 233 +++++++++ oxa/oxa-work/src/standing/store.rs | 578 +++++++++++++++++++++++ oxa/oxa-work/src/standing/store/tests.rs | 406 ++++++++++++++++ oxa/oxa-work/src/standing/types.rs | 191 ++++++++ 11 files changed, 1630 insertions(+), 18 deletions(-) create mode 100644 oxa/oxa-work/src/standing.rs create mode 100644 oxa/oxa-work/src/standing/asks.rs create mode 100644 oxa/oxa-work/src/standing/guard.rs create mode 100644 oxa/oxa-work/src/standing/store.rs create mode 100644 oxa/oxa-work/src/standing/store/tests.rs create mode 100644 oxa/oxa-work/src/standing/types.rs diff --git a/oxa/oxa-proto/src/work.rs b/oxa/oxa-proto/src/work.rs index dade3c648..d9bbbeb00 100644 --- a/oxa/oxa-proto/src/work.rs +++ b/oxa/oxa-proto/src/work.rs @@ -413,6 +413,16 @@ record!( ); record!(AskClosed { id: String } | s | { atom(&s.id, 128) }); record!(Incident { id: String, subject: Subject, kind: IncidentKind, evidence: Option } |s| { atom(&s.id, 256) }); +record!( + /// One durable standing revision. Ask details/answers stay in the protected standing file. + StandingChanged { + revision: u64, goal: Option, ask_opened: Option, + ask_closed: Option, incident: Option + } |s| { + require(s.revision > 0, "standing revision must be positive")?; + require(s.ask_opened.is_none() || s.ask_closed.is_none(), "standing mutation has two ask transitions") + } +); record!( Mechanical { subject: Subject, @@ -437,6 +447,7 @@ tagged!( AskOpened(Ask) => "ask_opened", AskClosed(AskClosed) => "ask_closed", Incident(Incident) => "incident", + StandingChanged(StandingChanged) => "standing_changed", Mechanical(Mechanical) => "mechanical", } ); @@ -447,6 +458,11 @@ impl Fact { Self::Mechanical(_) => EventClass::Mechanical, Self::Blocked(_) | Self::AskOpened(_) => EventClass::Attention, Self::Incident(incident) => incident.kind.class(), + Self::StandingChanged(change) if change.ask_opened.is_some() => EventClass::Attention, + Self::StandingChanged(change) => change + .incident + .as_ref() + .map_or(EventClass::Routine, |incident| incident.kind.class()), _ => EventClass::Routine, } } @@ -455,7 +471,11 @@ impl Fact { record!( /// Sequence is repository-local, starts at one and never resets on rotation. Event { seq: u64, at_ms: u64, checkpoint: Checkpoint, fact: Fact } |s| { - require(s.seq > 0, "event sequence must be positive") + require(s.seq > 0, "event sequence must be positive")?; + if let Fact::StandingChanged(change) = &s.fact { + require(s.checkpoint.source == "standing" && s.checkpoint.revision == change.revision, "standing checkpoint identity disagrees")?; + } + Ok(()) } ); record!( diff --git a/oxa/oxa-work/src/events.rs b/oxa/oxa-work/src/events.rs index 6f45883a4..d6d8e3454 100644 --- a/oxa/oxa-work/src/events.rs +++ b/oxa/oxa-work/src/events.rs @@ -136,6 +136,17 @@ impl Facts { /// Stop before the next action if this call fails. Replaying the same checkpoint repairs /// a lost acknowledgement, a torn last line or an interrupted snapshot replacement. pub fn append(&self, checkpoint: Checkpoint, at_ms: u64, fact: Fact) -> Result { + self.append_prepared(checkpoint, at_ms, fact, |_| Ok(())) + } + /// Validate capacity under the facts lock, then let another protected store reserve its + /// mutation before the facts reservation. Never calls preparation for an existing identity. + pub(crate) fn append_prepared( + &self, + checkpoint: Checkpoint, + at_ms: u64, + fact: Fact, + prepare: impl FnOnce(&Event) -> Result<()>, + ) -> Result { let _lock = self.root.lock(LOCK)?; let (mut state, mut snapshot, journal) = self.recover()?; let sources = state.sources()?; @@ -169,6 +180,7 @@ impl Facts { } // Fail capacity/semantic checks before durably reserving an observation. situation::apply(&mut snapshot, &event)?; + prepare(&event)?; state.pending = Some(event_value(&event)?); self.root.atomic(STATE, &state.bytes()?)?; if journal.current_bytes + bytes.len() > self.journal_bytes { diff --git a/oxa/oxa-work/src/facts_io.rs b/oxa/oxa-work/src/facts_io.rs index 34029fd57..6bc3da81b 100644 --- a/oxa/oxa-work/src/facts_io.rs +++ b/oxa/oxa-work/src/facts_io.rs @@ -195,6 +195,15 @@ impl Root { self.sync()?; self.create(from, b"") } + pub(crate) fn durable(&self, leaf: &str) -> Result<()> { + let file = io( + self.options().read(true).write(true).open(self.path(leaf)), + "cannot open protected metadata", + )?; + private(&io(file.metadata(), "cannot inspect protected metadata")?, false)?; + io(file.sync_all(), "cannot sync protected metadata")?; + self.sync() + } fn sync(&self) -> Result<()> { sync_directory(&self.0) } diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index a2d192ffd..112e19ed9 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -17,6 +17,7 @@ pub mod events; mod facts_io; pub mod incidents; pub mod situation; +pub mod standing; #[derive(Clone, Debug, Deserialize)] #[serde(deny_unknown_fields)] diff --git a/oxa/oxa-work/src/situation.rs b/oxa/oxa-work/src/situation.rs index fd9f67fef..bdae916e2 100644 --- a/oxa/oxa-work/src/situation.rs +++ b/oxa/oxa-work/src/situation.rs @@ -1,6 +1,6 @@ //! Bounded projection of typed facts. This module never reads or writes runner task state. use crate::Result; -use oxa_proto::work::{Event, Fact, Snapshot, TaskStep, MAX_ITEMS}; +use oxa_proto::work::{Ask, Event, Fact, Incident, Snapshot, TaskStep, MAX_ITEMS}; pub(crate) fn empty() -> Snapshot { Snapshot { @@ -73,23 +73,23 @@ pub(crate) fn apply(snapshot: &mut Snapshot, event: &Event) -> Result<()> { snapshot.blocked.push(block.clone()); } Fact::Unblocked(subject) => snapshot.blocked.retain(|row| &row.subject != subject), - Fact::AskOpened(ask) => { - if let Some(existing) = snapshot.asks.iter().find(|row| row.id == ask.id) { - if existing != ask { - return Err("ask id has conflicting contents".into()); - } - } else { - snapshot.asks.push(ask.clone()); - } - } + Fact::AskOpened(ask) => open_ask(snapshot, ask)?, Fact::AskClosed(ask) => snapshot.asks.retain(|row| row.id != ask.id), - Fact::Incident(incident) => { - if let Some(existing) = snapshot.incidents.iter().find(|row| row.id == incident.id) { - if existing != incident { - return Err("incident id has conflicting contents".into()); - } - } else { - snapshot.incidents.push(incident.clone()); + Fact::Incident(incident) => add_incident(snapshot, incident)?, + Fact::StandingChanged(change) => { + if change.revision <= snapshot.standing_revision { + return Err("stale standing projection revision".into()); + } + snapshot.standing_revision = change.revision; + snapshot.goal = change.goal.clone(); + if let Some(ask) = &change.ask_opened { + open_ask(snapshot, ask)?; + } + if let Some(ask) = &change.ask_closed { + snapshot.asks.retain(|row| row.id != ask.id); + } + if let Some(incident) = &change.incident { + add_incident(snapshot, incident)?; } } Fact::ExternalAcceptance(_) | Fact::Enqueued(_) | Fact::Mechanical(_) => {} @@ -130,6 +130,27 @@ pub(crate) fn apply(snapshot: &mut Snapshot, event: &Event) -> Result<()> { Ok(()) } +fn open_ask(snapshot: &mut Snapshot, ask: &Ask) -> Result<()> { + if let Some(existing) = snapshot.asks.iter().find(|row| row.id == ask.id) { + if existing != ask { + return Err("ask id has conflicting contents".into()); + } + } else { + snapshot.asks.push(ask.clone()); + } + Ok(()) +} +fn add_incident(snapshot: &mut Snapshot, incident: &Incident) -> Result<()> { + if let Some(existing) = snapshot.incidents.iter().find(|row| row.id == incident.id) { + if existing != incident { + return Err("incident id has conflicting contents".into()); + } + } else { + snapshot.incidents.push(incident.clone()); + } + Ok(()) +} + fn discard_history(snapshot: &mut Snapshot) -> bool { if !snapshot.landed.is_empty() { snapshot.landed.remove(0); diff --git a/oxa/oxa-work/src/standing.rs b/oxa/oxa-work/src/standing.rs new file mode 100644 index 000000000..ea71fa672 --- /dev/null +++ b/oxa/oxa-work/src/standing.rs @@ -0,0 +1,9 @@ +//! Owner standing data and action guards. Never reads or writes runner task state. +mod guard; +pub use guard::{Action, Refusal, RefusalReason, Scope, Subject}; +mod types; +pub use types::{Authority, Entry, EntryState, Kind, Rule, Source, Value}; +mod store; +pub use store::{GuardError, Guarded, Read, Standing, Verification}; +mod asks; +pub use asks::{Answer, AnswerRequest, Ask, Choice, Question, QuestionOption}; diff --git a/oxa/oxa-work/src/standing/asks.rs b/oxa/oxa-work/src/standing/asks.rs new file mode 100644 index 000000000..1c2296a21 --- /dev/null +++ b/oxa/oxa-work/src/standing/asks.rs @@ -0,0 +1,132 @@ +use super::{types, Scope, Source}; +use crate::Result; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct QuestionOption { + pub label: String, + pub detail: String, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Question { + pub scope: Scope, + pub question: String, + pub options: Vec, + pub recommended: String, + /// Unknown stays None; no default action or authority is inferred from prose here. + pub meanwhile: Option, + pub source: Source, +} +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde( + tag = "kind", + content = "value", + rename_all = "snake_case", + deny_unknown_fields +)] +pub enum Choice { + Option(String), + Text(String), +} +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Answer { + pub revision: u64, + pub question_revision: u64, + pub choice: Choice, + pub at: u64, + pub source: Source, +} +pub struct AnswerRequest { + pub ask_id: String, + pub question_revision: u64, + pub choice: Choice, + pub source: Source, +} +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Ask { + pub id: String, + pub revision: u64, + pub scope: Scope, + pub question: String, + pub options: Vec, + pub recommended: String, + pub meanwhile: Option, + pub at: u64, + pub source: Source, + pub answer: Option, +} +impl Question { + pub(super) fn validate(&self) -> Result<()> { + self.scope.validate()?; + self.source.validate()?; + types::text(&self.question, 512)?; + types::text(&self.recommended, 64)?; + if let Some(value) = &self.meanwhile { + types::text(value, 512)?; + } + if !(2..=4).contains(&self.options.len()) { + return Err("ask requires 2 to 4 options".into()); + } + let mut labels = std::collections::HashSet::new(); + for option in &self.options { + types::text(&option.label, 64)?; + types::text(&option.detail, 512)?; + if !labels.insert(&option.label) { + return Err("ask option labels must be unique".into()); + } + } + if !labels.contains(&self.recommended) { + return Err("ask recommendation must name an option".into()); + } + Ok(()) + } +} +impl Ask { + pub(super) fn validate_choice(&self, choice: &Choice) -> Result<()> { + match choice { + Choice::Option(label) if self.options.iter().any(|option| option.label == *label) => Ok(()), + Choice::Option(_) => Err("answer does not name an ask option".into()), + Choice::Text(value) => types::text(value, 1024), + } + } + pub(super) fn validate(&self) -> Result<()> { + types::id(&self.id)?; + if self.revision == 0 { + return Err("ask revision must be positive".into()); + } + Question { + scope: self.scope.clone(), + question: self.question.clone(), + options: self.options.clone(), + recommended: self.recommended.clone(), + meanwhile: self.meanwhile.clone(), + source: self.source.clone(), + } + .validate()?; + if let Some(answer) = &self.answer { + if answer.question_revision != self.revision { + return Err("answer names a different question revision".into()); + } + answer.source.validate()?; + self.validate_choice(&answer.choice)?; + } + Ok(()) + } + pub(super) fn fact(&self) -> oxa_proto::work::Ask { + let subject = match &self.scope { + Scope::PullRequest(pr) => oxa_proto::work::Subject::PullRequest(*pr), + Scope::Task(task) => oxa_proto::work::Subject::Task(task.clone()), + Scope::Host(host) => oxa_proto::work::Subject::Host(host.clone()), + Scope::Repository | Scope::Branch(_) => oxa_proto::work::Subject::Repository, + }; + oxa_proto::work::Ask { + id: self.id.clone(), + subject, + question: self.question.clone(), + } + } +} diff --git a/oxa/oxa-work/src/standing/guard.rs b/oxa/oxa-work/src/standing/guard.rs new file mode 100644 index 000000000..be4e479c2 --- /dev/null +++ b/oxa/oxa-work/src/standing/guard.rs @@ -0,0 +1,233 @@ +use serde::{Deserialize, Serialize}; + +/// One side effect, never a worker or batch lifetime. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Action { + Admit, + Push, + Approve, + Enqueue, + Update, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde( + tag = "kind", + content = "value", + rename_all = "snake_case", + deny_unknown_fields +)] +pub enum Scope { + Repository, + PullRequest(u64), + Task(String), + Branch(String), + Host(String), +} + +/// Identity resolved by the trusted frontend, not tool JSON. None means checked absence. +/// Supply the current PR whenever one exists; its head is deliberately not an identity field. +#[derive(Clone, Copy, Debug, Default)] +pub struct Subject<'a> { + pub pr: Option, + pub task: Option<&'a str>, + pub branch: Option<&'a str>, + pub host: Option<&'a str>, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RefusalReason { + Held, + Frozen, + InvalidSubject, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Refusal { + pub revision: u64, + pub rule_id: Option, + pub reason: RefusalReason, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum ConstraintKind { + Hold, + Freeze, +} + +pub(super) struct Constraint<'a> { + pub rule_id: &'a str, + pub scope: &'a Scope, + pub kind: ConstraintKind, +} + +impl Scope { + fn matches(&self, subject: Subject<'_>) -> bool { + match self { + Self::Repository => true, + Self::PullRequest(pr) => subject.pr == Some(*pr), + Self::Task(task) => subject.task == Some(task.as_str()), + Self::Branch(branch) => subject.branch == Some(branch.as_str()), + Self::Host(host) => subject.host == Some(host.as_str()), + } + } +} + +pub(super) fn check<'a>( + revision: u64, + constraints: impl IntoIterator>, + action: Action, + subject: Subject<'_>, +) -> Result<(), Refusal> { + let valid = subject.pr != Some(0) + && match action { + Action::Admit => { + subject.task.is_some_and(|id| !id.is_empty()) + && subject.branch.is_some_and(|branch| !branch.is_empty()) + } + Action::Push => subject.branch.is_some_and(|branch| !branch.is_empty()), + Action::Approve | Action::Enqueue | Action::Update => subject.pr.is_some(), + }; + if !valid { + return Err(Refusal { + revision, + rule_id: None, + reason: RefusalReason::InvalidSubject, + }); + } + for constraint in constraints { + let reason = match constraint.kind { + ConstraintKind::Hold => RefusalReason::Held, + ConstraintKind::Freeze if matches!(action, Action::Approve | Action::Enqueue) => { + RefusalReason::Frozen + } + ConstraintKind::Freeze => continue, + }; + if constraint.scope.matches(subject) { + return Err(Refusal { + revision, + rule_id: Some(constraint.rule_id.to_owned()), + reason, + }); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn subject() -> Subject<'static> { + Subject { + pr: Some(480), + task: Some("task"), + branch: Some("agent/task"), + host: Some("helper"), + } + } + + #[test] + fn a_pr_hold_refuses_every_effect_and_has_no_head_or_actor_bypass() { + let scope = Scope::PullRequest(480); + for action in [ + Action::Admit, + Action::Push, + Action::Approve, + Action::Enqueue, + Action::Update, + ] { + let result = check( + 7, + [Constraint { + rule_id: "R-7", + scope: &scope, + kind: ConstraintKind::Hold, + }], + action, + subject(), + ); + assert_eq!( + result, + Err(Refusal { + revision: 7, + rule_id: Some("R-7".into()), + reason: RefusalReason::Held + }) + ); + } + let different = Subject { + pr: Some(481), + ..subject() + }; + assert!(check( + 7, + [Constraint { + rule_id: "R-7", + scope: &scope, + kind: ConstraintKind::Hold + }], + Action::Update, + different + ) + .is_ok()); + } + + #[test] + fn freeze_refuses_only_approval_and_enqueue() { + let scope = Scope::Repository; + for action in [ + Action::Admit, + Action::Push, + Action::Approve, + Action::Enqueue, + Action::Update, + ] { + let result = check( + 3, + [Constraint { + rule_id: "R-3", + scope: &scope, + kind: ConstraintKind::Freeze, + }], + action, + subject(), + ); + assert_eq!( + result.is_err(), + matches!(action, Action::Approve | Action::Enqueue) + ); + } + } + + #[test] + fn pre_publication_admission_honors_task_branch_and_host_holds() { + let subject = Subject { + pr: None, + ..subject() + }; + for scope in [ + Scope::Task("task".into()), + Scope::Branch("agent/task".into()), + Scope::Host("helper".into()), + ] { + assert!(check( + 4, + [Constraint { + rule_id: "R-4", + scope: &scope, + kind: ConstraintKind::Hold + }], + Action::Admit, + subject + ) + .is_err()); + } + for action in [Action::Approve, Action::Enqueue, Action::Update] { + assert_eq!( + check(4, [], action, subject).unwrap_err().reason, + RefusalReason::InvalidSubject + ); + } + } +} diff --git a/oxa/oxa-work/src/standing/store.rs b/oxa/oxa-work/src/standing/store.rs new file mode 100644 index 000000000..9dd1cbe06 --- /dev/null +++ b/oxa/oxa-work/src/standing/store.rs @@ -0,0 +1,578 @@ +use super::guard::{self, Constraint, ConstraintKind}; +use super::types::{self, EntryState}; +use super::{ + Action, Answer, AnswerRequest, Ask, Authority, Entry, Kind, Question, Refusal, Rule, Scope, Subject, + Value, +}; +use crate::{events::Facts, facts_io::Root, Result}; +use oxa_proto::work::{Checkpoint, Event, Fact, Incident, IncidentKind, StandingChanged}; +use serde::{Deserialize, Serialize}; +use std::{fmt, path::Path}; + +const FILE: &str = "standing.json"; +const LOCK: &str = ".standing.lock"; +const MAX_FILE: usize = 1024 * 1024; +const MAX_VIEW: usize = 256 * 1024; +const MAX_ACTIVE: usize = 128; +const HISTORY: usize = 16; + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Read { + pub revision: u64, + pub at: u64, + pub entries: Vec, + pub asks: Vec, + pub history_truncated: bool, +} +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct Pending { + after: Read, + observation: serde_json::Value, +} +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct Document { + version: u32, + current: Read, + pending: Option, +} +impl Document { + fn bytes(&self) -> Result> { + let bytes = serde_json::to_vec(self).map_err(|_| "cannot encode standing data")?; + if bytes.len() > MAX_FILE { + return Err("standing file capacity exceeded".into()); + } + Ok(bytes) + } +} +impl Read { + fn validate(&self) -> Result<()> { + if self.entries.len() + self.asks.len() > MAX_ACTIVE + HISTORY { + return Err("standing entry capacity exceeded".into()); + } + let mut ids = std::collections::HashSet::new(); + let mut live = vec![]; + for entry in &self.entries { + entry.validate()?; + if !ids.insert(&entry.id) { + return Err("duplicate standing entry identity".into()); + } + if entry.live() { + if live + .iter() + .any(|(kind, scope)| *kind == entry.kind && *scope == &entry.scope) + { + return Err("conflicting live standing rules".into()); + } + live.push((entry.kind, &entry.scope)); + } + } + for ask in &self.asks { + ask.validate()?; + if !ids.insert(&ask.id) { + return Err("duplicate standing ask identity".into()); + } + if ask.revision > self.revision { + return Err("ask is ahead of standing revision".into()); + } + if ask + .answer + .as_ref() + .is_some_and(|answer| answer.revision <= ask.revision || answer.revision > self.revision) + { + return Err("answer is outside its standing revision".into()); + } + } + if live.len() + self.asks.iter().filter(|ask| ask.answer.is_none()).count() > MAX_ACTIVE { + return Err("active standing capacity exceeded".into()); + } + if serde_json::to_vec(self) + .map_err(|_| "cannot encode standing view")? + .len() + > MAX_VIEW + { + return Err("standing view byte capacity exceeded".into()); + } + Ok(()) + } + fn bounded(&mut self) -> Result<()> { + while self.entries.iter().filter(|entry| !entry.live()).count() + + self.asks.iter().filter(|ask| ask.answer.is_some()).count() + > HISTORY + { + self.discard_history()?; + } + while self.validate().is_err() { + self.discard_history()?; + } + self.validate() + } + fn discard_history(&mut self) -> Result<()> { + if let Some(index) = self.entries.iter().position(|entry| !entry.live()) { + self.entries.remove(index); + } else if let Some((index, _)) = self + .asks + .iter() + .enumerate() + .filter_map(|(index, ask)| ask.answer.as_ref().map(|answer| (index, answer.revision))) + .min_by_key(|(_, revision)| *revision) + { + self.asks.remove(index); + } else { + return Err("active standing capacity exceeded".into()); + } + self.history_truncated = true; + Ok(()) + } + fn next(&self, expected: u64, at: u64) -> Result { + if self.revision != expected { + return Err("standing revision changed; read it again".into()); + } + let mut next = self.clone(); + next.revision = self + .revision + .checked_add(1) + .ok_or("standing revision exhausted")?; + next.at = at; + Ok(next) + } + fn change(&self, incident: Option) -> StandingChanged { + let goal = self + .entries + .iter() + .find_map(|entry| match (&entry.kind, &entry.value) { + (Kind::Goal, Value::Text(value)) if entry.live() && entry.scope == Scope::Repository => { + Some(value.clone()) + } + _ => None, + }); + StandingChanged { + revision: self.revision, + goal, + ask_opened: None, + ask_closed: None, + incident, + } + } + fn difference(&self, before: &Read, incident: Option) -> Result { + let mut change = self.change(incident); + for ask in self + .asks + .iter() + .filter(|ask| ask.answer.is_none() && !before.asks.iter().any(|old| old.id == ask.id)) + { + if change.ask_opened.replace(ask.fact()).is_some() { + return Err("standing mutation opens multiple asks".into()); + } + } + for ask in before.asks.iter().filter(|ask| ask.answer.is_none()) { + if self + .asks + .iter() + .any(|new| new.id == ask.id && new.answer.is_some()) + { + if change + .ask_closed + .replace(oxa_proto::work::AskClosed { id: ask.id.clone() }) + .is_some() + { + return Err("standing mutation closes multiple asks".into()); + } + } else if !self.asks.iter().any(|new| new.id == ask.id) { + return Err("standing mutation drops an unanswered ask".into()); + } + } + Ok(change) + } +} + +#[derive(Clone, Debug)] +pub struct Standing { + root: Root, + facts: Facts, +} +#[derive(Debug)] +pub enum GuardError { + Storage(String), + Refused(Refusal), +} +impl fmt::Display for GuardError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Storage(error) => write!(f, "standing data unavailable: {error}"), + Self::Refused(refusal) => write!( + f, + "standing revision {} refuses this action: {:?}", + refusal.revision, refusal.reason + ), + } + } +} +impl std::error::Error for GuardError {} +impl From for GuardError { + fn from(error: String) -> Self { + Self::Storage(error) + } +} +pub struct Guarded { + pub revision: u64, + pub result: Result, +} +pub struct Verification { + pub revision: u64, + pub scope: Scope, + pub value: Value, + pub at: u64, + pub evidence: String, +} + +impl Standing { + /// Explicit owner initialization at a protected per-repository configuration root. + pub fn initialize(root: &Path, authority: Authority, facts: Facts) -> Result { + if authority != Authority::Owner { + return Err("owner initialization required".into()); + } + let root = Root::open(root, true)?; + let _lock = root.lock(LOCK)?; + if root.exists(FILE)? { + return Err("standing data already exists".into()); + } + if facts.situation()?.standing_revision != 0 { + return Err("standing facts already exist".into()); + } + let document = Document { + version: 1, + current: Read { + revision: 0, + at: 0, + entries: vec![], + asks: vec![], + history_truncated: false, + }, + pending: None, + }; + root.create(FILE, &document.bytes()?)?; + Ok(Self { root, facts }) + } + pub fn open(root: &Path, facts: Facts) -> Result { + let standing = Self { + root: Root::open(root, false)?, + facts, + }; + standing.read()?; + Ok(standing) + } + pub fn read(&self) -> Result { + let _lock = self.root.lock(LOCK)?; + self.load() + } + fn load(&self) -> Result { + let mut document: Document = serde_json::from_slice(&self.root.read(FILE, MAX_FILE)?) + .map_err(|_| "invalid standing document")?; + if document.version != 1 { + return Err("unsupported standing version".into()); + } + document.current.validate()?; + if let Some(pending) = &document.pending { + pending.after.validate()?; + if document.current.revision.checked_add(1) != Some(pending.after.revision) { + return Err("invalid pending standing revision".into()); + } + let event = Event::from_json( + &serde_json::to_vec(&pending.observation).map_err(|_| "invalid standing observation")?, + ) + .map_err(|error| error.to_string())?; + if event.checkpoint.source != "standing" + || event.checkpoint.revision != pending.after.revision + || event.at_ms != pending.after.at + { + return Err("pending standing identity disagrees".into()); + } + let Fact::StandingChanged(change) = &event.fact else { + return Err("invalid standing observation kind".into()); + }; + if change + != &pending + .after + .difference(&document.current, change.incident.clone())? + { + return Err("pending standing projection disagrees".into()); + } + self.facts.append(event.checkpoint, event.at_ms, event.fact)?; + document.current = pending.after.clone(); + document.pending = None; + self.root.atomic(FILE, &document.bytes()?)?; + } + // Establish durability after a prior post-rename sync failure before authorizing work. + self.root.durable(FILE)?; + let situation = self.facts.situation()?; + if situation.standing_revision != document.current.revision + || situation.goal != document.current.change(None).goal + { + return Err("standing file disagrees with facts revision".into()); + } + Ok(document.current) + } + fn save(&self, before: Read, mut after: Read, incident: Option) -> Result { + after.bounded()?; + let fact = Fact::StandingChanged(after.difference(&before, incident)?); + let mut prepared = false; + self.facts.append_prepared( + Checkpoint { + source: "standing".into(), + revision: after.revision, + }, + after.at, + fact, + |event| { + let observation = + serde_json::from_slice(&event.to_json().map_err(|error| error.to_string())?) + .map_err(|_| "cannot encode standing observation")?; + let document = Document { + version: 1, + current: before, + pending: Some(Pending { + after: after.clone(), + observation, + }), + }; + self.root.atomic(FILE, &document.bytes()?)?; + prepared = true; + Ok(()) + }, + )?; + if !prepared { + return Err("standing facts revision was already used".into()); + } + let document = Document { + version: 1, + current: after.clone(), + pending: None, + }; + self.root.atomic(FILE, &document.bytes()?)?; + Ok(after) + } + pub fn rule(&self, expected: u64, authority: Authority, at: u64, request: Rule) -> Result { + request.validate()?; + let _lock = self.root.lock(LOCK)?; + let before = self.load()?; + let mut after = before.next(expected, at)?; + let prior = after + .entries + .iter_mut() + .find(|entry| entry.live() && entry.kind == request.kind && entry.scope == request.scope); + let mut protection = authority; + if let Some(prior) = prior { + if request.supersedes.as_deref() != Some(&prior.id) { + return Err("live standing collision requires named supersession".into()); + } + let releasing = matches!( + request.value, + Value::Retired | Value::Constraint { active: false } + ); + if matches!(request.kind, Kind::Hold | Kind::Freeze) && prior.authority == Authority::Owner { + if releasing && authority != Authority::Owner { + return Err("only the owner releases this constraint".into()); + } + protection = Authority::Owner; + } + prior.state = EntryState::Superseded; + } else { + if request.supersedes.is_some() { + return Err("superseded standing entry is not live".into()); + } + if matches!( + request.value, + Value::Retired | Value::Constraint { active: false } + ) { + return Err("no live standing entry to release".into()); + } + } + let entry = Entry { + id: format!("R-{}", after.revision), + kind: request.kind, + scope: request.scope, + value: request.value, + said: request.said, + at, + source: request.source, + supersedes: request.supersedes, + state: EntryState::Stored, + reminders: 0, + authority: protection, + }; + after.entries.push(entry.clone()); + self.save(before, after, None)?; + Ok(entry) + } + pub fn remind(&self, expected: u64, entry_id: &str, at: u64) -> Result { + let _lock = self.root.lock(LOCK)?; + let before = self.load()?; + let mut after = before.next(expected, at)?; + let entry = after + .entries + .iter_mut() + .find(|entry| entry.id == entry_id && entry.live()) + .ok_or("standing entry is not live")?; + entry.reminders = entry + .reminders + .checked_add(1) + .ok_or("standing reminder count exhausted")?; + let result = entry.clone(); + let subject = match &entry.scope { + Scope::PullRequest(pr) => oxa_proto::work::Subject::PullRequest(*pr), + Scope::Task(task) => oxa_proto::work::Subject::Task(task.clone()), + Scope::Host(host) => oxa_proto::work::Subject::Host(host.clone()), + Scope::Repository | Scope::Branch(_) => oxa_proto::work::Subject::Repository, + }; + let incident = Incident { + id: format!("{}-reminded-{}", entry.id, after.revision), + subject, + kind: IncidentKind::RuleReminder, + evidence: None, + }; + self.save(before, after, Some(incident))?; + Ok(result) + } + /// A trusted frontend checker supplies actual runtime values/evidence; tool JSON cannot. + pub fn verify( + &self, + expected: u64, + entry_id: &str, + check: impl FnOnce(&Entry) -> Result, + ) -> Result { + let _lock = self.root.lock(LOCK)?; + let before = self.load()?; + if before.revision != expected { + return Err("standing revision changed; read it again".into()); + } + let mut after = before.clone(); + let entry = after + .entries + .iter_mut() + .find(|entry| entry.id == entry_id && entry.live()) + .ok_or("standing entry is not live")?; + let verified = check(entry)?; + types::text(&verified.evidence, 256)?; + if verified.revision != expected || verified.scope != entry.scope || verified.value != entry.value { + return Err("runtime evidence does not match the standing rule".into()); + } + if matches!(entry.state, EntryState::Applied { .. }) { + return Ok(entry.clone()); + } + entry.state = EntryState::Applied { + verified_at: verified.at, + evidence: verified.evidence, + }; + let result = entry.clone(); + after.revision = expected.checked_add(1).ok_or("standing revision exhausted")?; + after.at = verified.at; + self.save(before, after, None)?; + Ok(result) + } + pub fn ask(&self, expected: u64, at: u64, request: Question) -> Result { + request.validate()?; + let _lock = self.root.lock(LOCK)?; + let before = self.load()?; + let mut after = before.next(expected, at)?; + let ask = Ask { + id: format!("A-{}", after.revision), + revision: after.revision, + scope: request.scope, + question: request.question, + options: request.options, + recommended: request.recommended, + meanwhile: request.meanwhile, + at, + source: request.source, + answer: None, + }; + after.asks.push(ask.clone()); + self.save(before, after, None)?; + Ok(ask) + } + /// An answer records an owner decision; it never releases a hold or grants tool permission. + pub fn answer( + &self, + expected: u64, + authority: Authority, + at: u64, + request: AnswerRequest, + ) -> Result { + if authority != Authority::Owner { + return Err("only the owner answers standing asks".into()); + } + request.source.validate()?; + let _lock = self.root.lock(LOCK)?; + let before = self.load()?; + let mut after = before.next(expected, at)?; + let ask = after + .asks + .iter_mut() + .find(|ask| ask.id == request.ask_id) + .ok_or("standing ask was not found")?; + if ask.revision != request.question_revision || ask.answer.is_some() { + return Err("standing question changed or was already answered".into()); + } + ask.validate_choice(&request.choice)?; + ask.answer = Some(Answer { + revision: after.revision, + question_revision: request.question_revision, + choice: request.choice, + at, + source: request.source, + }); + let result = ask.clone(); + self.save(before, after, None)?; + Ok(result) + } + pub fn is_held(&self, pr: u64) -> Result> { + if pr == 0 { + return Err("PR number must be positive".into()); + } + let read = self.read()?; + Ok(read + .entries + .iter() + .find(|entry| { + entry.live() + && entry.kind == Kind::Hold + && (entry.scope == Scope::Repository || entry.scope == Scope::PullRequest(pr)) + }) + .map(|entry| entry.id.clone())) + } + pub fn with_guard( + &self, + action: Action, + subject: Subject<'_>, + effect: impl FnOnce(&Read) -> Result, + ) -> std::result::Result, GuardError> { + let _lock = self.root.lock(LOCK)?; + let read = self.load()?; + let constraints = read.entries.iter().filter_map(|entry| { + if !entry.live() { + return None; + } + let kind = match entry.kind { + Kind::Hold => ConstraintKind::Hold, + Kind::Freeze => ConstraintKind::Freeze, + _ => return None, + }; + Some(Constraint { + rule_id: &entry.id, + scope: &entry.scope, + kind, + }) + }); + guard::check(read.revision, constraints, action, subject).map_err(GuardError::Refused)?; + Ok(Guarded { + revision: read.revision, + result: effect(&read), + }) + } +} + +#[cfg(test)] +mod tests; diff --git a/oxa/oxa-work/src/standing/store/tests.rs b/oxa/oxa-work/src/standing/store/tests.rs new file mode 100644 index 000000000..5c927e625 --- /dev/null +++ b/oxa/oxa-work/src/standing/store/tests.rs @@ -0,0 +1,406 @@ +use super::*; +use std::{ + fs, + os::unix::fs::DirBuilderExt, + path::PathBuf, + sync::atomic::{AtomicU64, AtomicUsize, Ordering}, +}; + +struct Fixture(PathBuf); +impl Fixture { + fn new() -> Self { + static NEXT: AtomicU64 = AtomicU64::new(0); + loop { + let root = std::env::temp_dir().join(format!( + "oxa-standing-test-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + match fs::DirBuilder::new().mode(0o700).create(&root) { + Ok(()) => return Self(root), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => panic!("create private fixture: {error}"), + } + } + } + fn facts(&self) -> Facts { + Facts::initialize(&self.0.join("facts")).unwrap() + } + fn standing(&self, facts: Facts) -> Standing { + Standing::initialize(&self.0.join("standing"), Authority::Owner, facts).unwrap() + } +} +impl Drop for Fixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} +fn hold(active: bool, supersedes: Option<&str>) -> Rule { + Rule { + kind: Kind::Hold, + scope: Scope::PullRequest(480), + value: Value::Constraint { active }, + said: "Hold this PR until released".into(), + source: super::super::Source::Command { + command: "hold 480".into(), + }, + supersedes: supersedes.map(str::to_owned), + } +} +fn subject() -> Subject<'static> { + Subject { + pr: Some(480), + task: Some("task"), + branch: Some("agent/task"), + host: Some("helper"), + } +} + +#[test] +fn a_fresh_hold_blocks_each_effect_in_a_handle_opened_before_it() { + let fixture = Fixture::new(); + let facts = fixture.facts(); + let standing = fixture.standing(facts.clone()); + let old_handle = Standing::open(&fixture.0.join("standing"), facts.clone()).unwrap(); + let effects = AtomicUsize::new(0); + old_handle + .with_guard(Action::Update, subject(), |_| { + effects.fetch_add(1, Ordering::SeqCst); + Ok(()) + }) + .unwrap() + .result + .unwrap(); + let entry = standing.rule(0, Authority::Owner, 10, hold(true, None)).unwrap(); + assert_eq!(entry.state, EntryState::Stored); + for action in [ + Action::Admit, + Action::Push, + Action::Approve, + Action::Enqueue, + Action::Update, + ] { + let result = old_handle.with_guard(action, subject(), |_| { + effects.fetch_add(1, Ordering::SeqCst); + Ok(()) + }); + assert!(matches!( + result, + Err(GuardError::Refused(Refusal { + reason: super::super::RefusalReason::Held, + revision: 1, + .. + })) + )); + } + assert_eq!(effects.load(Ordering::SeqCst), 1); + assert_eq!(old_handle.is_held(480).unwrap(), Some(entry.id)); + assert_eq!(facts.situation().unwrap().standing_revision, 1); +} + +#[test] +fn supersession_cannot_launder_owner_hold_authority() { + let fixture = Fixture::new(); + let facts = fixture.facts(); + let standing = fixture.standing(facts); + let first = standing.rule(0, Authority::Owner, 1, hold(true, None)).unwrap(); + assert!(standing.rule(1, Authority::Lead, 2, hold(false, None)).is_err()); + let second = standing + .rule(1, Authority::Lead, 2, hold(true, Some(&first.id))) + .unwrap(); + assert!(standing + .rule(2, Authority::Lead, 3, hold(false, Some(&second.id))) + .is_err()); + let mut retired = hold(false, Some(&second.id)); + retired.value = Value::Retired; + assert!(standing.rule(2, Authority::Lead, 3, retired).is_err()); + assert_eq!(standing.read().unwrap().revision, 2); + standing + .rule(2, Authority::Owner, 3, hold(false, Some(&second.id))) + .unwrap(); + assert!(standing.is_held(480).unwrap().is_none()); + assert!(standing.rule(2, Authority::Owner, 4, hold(true, None)).is_err()); + standing + .with_guard(Action::Approve, subject(), |read| Ok(read.revision)) + .unwrap() + .result + .unwrap(); +} + +#[test] +fn missing_corrupt_or_symlinked_policy_never_authorizes_the_callback() { + for poison in ["missing", "corrupt", "symlink"] { + let fixture = Fixture::new(); + let facts = fixture.facts(); + let standing = fixture.standing(facts.clone()); + let path = fixture.0.join("standing/standing.json"); + fs::remove_file(&path).unwrap(); + let target = fixture.0.join("outside"); + fs::write(&target, b"keep outside").unwrap(); + match poison { + "missing" => {} + "corrupt" => fs::write(&path, b"PRIVATE_REJECTED_CONTENT").unwrap(), + "symlink" => std::os::unix::fs::symlink(&target, &path).unwrap(), + _ => unreachable!(), + } + let effects = AtomicUsize::new(0); + let result = standing.with_guard(Action::Update, subject(), |_| { + effects.fetch_add(1, Ordering::SeqCst); + Ok(()) + }); + assert!(matches!(result, Err(GuardError::Storage(_)))); + assert_eq!(effects.load(Ordering::SeqCst), 0); + assert_eq!(fs::read(&target).unwrap(), b"keep outside"); + assert!(Standing::open(&fixture.0.join("standing"), facts).is_err()); + } +} + +#[test] +fn pending_revisions_recover_before_or_after_the_fact_without_duplicate_delivery() { + for appended in [false, true] { + let fixture = Fixture::new(); + let facts = fixture.facts(); + let standing = fixture.standing(facts.clone()); + let before = standing.read().unwrap(); + let mut after = before.next(0, 9).unwrap(); + let request = hold(true, None); + after.entries.push(Entry { + id: "R-1".into(), + kind: request.kind, + scope: request.scope, + value: request.value, + said: request.said, + at: 9, + source: request.source, + supersedes: None, + state: EntryState::Stored, + reminders: 0, + authority: Authority::Owner, + }); + let event = Event { + seq: 1, + at_ms: 9, + checkpoint: Checkpoint { + source: "standing".into(), + revision: 1, + }, + fact: Fact::StandingChanged(after.change(None)), + }; + let observation = serde_json::from_slice(&event.to_json().unwrap()).unwrap(); + if appended { + facts.append(event.checkpoint, event.at_ms, event.fact).unwrap(); + } + let document = Document { + version: 1, + current: before, + pending: Some(Pending { after, observation }), + }; + standing.root.atomic(FILE, &document.bytes().unwrap()).unwrap(); + let reopened = Standing::open(&fixture.0.join("standing"), facts.clone()).unwrap(); + assert_eq!(reopened.read().unwrap().revision, 1); + assert_eq!(reopened.is_held(480).unwrap().as_deref(), Some("R-1")); + assert_eq!(facts.events_after(0, 64).unwrap().events.len(), 1); + assert_eq!(facts.situation().unwrap().standing_revision, 1); + } +} + +#[test] +fn applied_requires_matching_runtime_value_scope_and_revision() { + let fixture = Fixture::new(); + let facts = fixture.facts(); + let standing = fixture.standing(facts); + let request = Rule { + kind: Kind::Goal, + scope: Scope::Repository, + value: Value::Text("Finish Rust".into()), + said: "Finish Rust".into(), + source: super::super::Source::Command { + command: "goal Finish Rust".into(), + }, + supersedes: None, + }; + let entry = standing.rule(0, Authority::Owner, 1, request).unwrap(); + let observed = fixture.0.join("runtime.json"); + fs::write(&observed, br#"{"goal":"different"}"#).unwrap(); + let checker = |entry: &Entry| -> Result { + let actual: serde_json::Value = serde_json::from_slice(&fs::read(&observed).unwrap()).unwrap(); + Ok(Verification { + revision: 1, + scope: entry.scope.clone(), + value: Value::Text(actual["goal"].as_str().unwrap().into()), + at: 2, + evidence: "synthetic-runtime:1".into(), + }) + }; + assert!(standing.verify(1, &entry.id, checker).is_err()); + assert_eq!(standing.read().unwrap().entries[0].state, EntryState::Stored); + fs::write(&observed, br#"{"goal":"Finish Rust"}"#).unwrap(); + let applied = standing.verify(1, &entry.id, checker).unwrap(); + assert!(matches!( + applied.state, + EntryState::Applied { verified_at: 2, .. } + )); + assert_eq!(standing.read().unwrap().revision, 2); +} + +#[test] +fn reminders_create_incidents_and_offer_without_changing_advisory_rules() { + let fixture = Fixture::new(); + let facts = fixture.facts(); + let standing = fixture.standing(facts.clone()); + let request = Rule { + kind: Kind::Note, + scope: Scope::Repository, + value: Value::Text("Keep it brief".into()), + said: "Keep it brief".into(), + source: super::super::Source::Command { + command: "note Keep it brief".into(), + }, + supersedes: None, + }; + let note = standing.rule(0, Authority::Owner, 1, request).unwrap(); + assert!(!standing.remind(1, ¬e.id, 2).unwrap().needs_conversion_offer()); + assert!(standing.remind(2, ¬e.id, 3).unwrap().needs_conversion_offer()); + let read = standing.read().unwrap(); + assert_eq!(read.entries[0].kind, Kind::Note); + assert_eq!(read.entries[0].reminders, 2); + assert_eq!(facts.situation().unwrap().incidents.len(), 2); +} + +fn question() -> Question { + Question { + scope: Scope::PullRequest(480), + question: "Release this hold?".into(), + options: vec![ + super::super::QuestionOption { + label: "Keep".into(), + detail: "Keep the hold".into(), + }, + super::super::QuestionOption { + label: "Release".into(), + detail: "Request an explicit release".into(), + }, + ], + recommended: "Keep".into(), + meanwhile: None, + source: super::super::Source::Command { + command: "ask 480".into(), + }, + } +} +fn answer(ask: &Ask) -> super::super::AnswerRequest { + super::super::AnswerRequest { + ask_id: ask.id.clone(), + question_revision: ask.revision, + choice: super::super::Choice::Option("Release".into()), + source: super::super::Source::Command { + command: "decide Release".into(), + }, + } +} + +#[test] +fn answers_are_owner_bound_to_a_question_and_do_not_release_holds() { + let fixture = Fixture::new(); + let facts = fixture.facts(); + let standing = fixture.standing(facts.clone()); + standing.rule(0, Authority::Owner, 1, hold(true, None)).unwrap(); + let ask = standing.ask(1, 2, question()).unwrap(); + assert!(ask.meanwhile.is_none()); + assert_eq!(facts.situation().unwrap().asks[0].id, ask.id); + assert!(standing.answer(2, Authority::Lead, 3, answer(&ask)).is_err()); + let mut stale = answer(&ask); + stale.question_revision += 1; + assert!(standing.answer(2, Authority::Owner, 3, stale).is_err()); + let mut invalid = answer(&ask); + invalid.choice = super::super::Choice::Option("missing".into()); + assert!(standing.answer(2, Authority::Owner, 3, invalid).is_err()); + let result = standing.answer(2, Authority::Owner, 3, answer(&ask)).unwrap(); + assert!(result.answer.is_some()); + assert!(facts.situation().unwrap().asks.is_empty()); + assert!(standing.is_held(480).unwrap().is_some()); + assert!(standing + .with_guard(Action::Enqueue, subject(), |_| Ok(())) + .is_err()); + assert!(standing.answer(3, Authority::Owner, 4, answer(&ask)).is_err()); +} + +#[test] +fn an_ask_capacity_refusal_does_not_reserve_a_stuck_standing_revision() { + let fixture = Fixture::new(); + let facts = fixture.facts(); + let standing = fixture.standing(facts.clone()); + for index in 0..128 { + facts + .append( + Checkpoint { + source: "runner".into(), + revision: index + 1, + }, + 1, + Fact::TaskStarted(oxa_proto::work::Running { + task: format!("task-{index}"), + host: "helper".into(), + branch: format!("agent/task-{index}"), + worktree: "~/project".into(), + session_id: None, + step: oxa_proto::work::TaskStep::Preparing, + }), + ) + .unwrap(); + } + assert!(standing.ask(0, 2, question()).is_err()); + assert_eq!(standing.read().unwrap().revision, 0); + assert!(standing.read().unwrap().asks.is_empty()); + let document: Document = serde_json::from_slice(&standing.root.read(FILE, MAX_FILE).unwrap()).unwrap(); + assert!(document.pending.is_none()); + facts + .append( + Checkpoint { + source: "runner".into(), + revision: 129, + }, + 3, + Fact::TaskTransition(oxa_proto::work::TaskTransition { + task: "task-0".into(), + step: oxa_proto::work::TaskStep::Finished, + session_id: None, + reason: Some(oxa_proto::work::StepReason::Interrupted), + }), + ) + .unwrap(); + let ask = standing.ask(0, 4, question()).unwrap(); + assert_eq!(facts.situation().unwrap().asks.len(), 1); + standing.answer(1, Authority::Owner, 5, answer(&ask)).unwrap(); + assert!(facts.situation().unwrap().asks.is_empty()); +} + +#[test] +fn a_late_answer_is_retained_even_when_its_question_is_the_oldest() { + let fixture = Fixture::new(); + let facts = fixture.facts(); + let standing = fixture.standing(facts); + let oldest = standing.ask(0, 100, question()).unwrap(); + let mut revision = 1; + for _ in 0..16 { + let ask = standing.ask(revision, 100, question()).unwrap(); + revision += 1; + standing + .answer(revision, Authority::Owner, 100, answer(&ask)) + .unwrap(); + revision += 1; + } + standing + .answer(revision, Authority::Owner, 1, answer(&oldest)) + .unwrap(); + let read = standing.read().unwrap(); + assert!(read.history_truncated); + assert_eq!(read.asks.len(), 16); + assert!(read + .asks + .iter() + .find(|ask| ask.id == oldest.id) + .unwrap() + .answer + .is_some()); +} diff --git a/oxa/oxa-work/src/standing/types.rs b/oxa/oxa-work/src/standing/types.rs new file mode 100644 index 000000000..3a0a59cef --- /dev/null +++ b/oxa/oxa-work/src/standing/types.rs @@ -0,0 +1,191 @@ +use super::Scope; +use crate::Result; +use serde::{Deserialize, Serialize}; + +/// Supplied by a trusted frontend; this is not a field in a rule/tool request. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Authority { + Owner, + Lead, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Kind { + Goal, + Priority, + Freeze, + Hold, + Budget, + Model, + Placement, + Gate, + Eligible, + Decided, + Reserve, + Note, +} + +/// A source reference is selected/checked by the frontend, never inferred from tool success. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum Source { + Command { command: String }, + SessionEntry { session_id: String, entry_id: String }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde( + tag = "kind", + content = "value", + rename_all = "snake_case", + deny_unknown_fields +)] +pub enum Value { + Text(String), + /// An explicit release is a new entry which supersedes the hold/freeze it releases. + Constraint { + active: bool, + }, + Retired, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Rule { + pub kind: Kind, + pub scope: Scope, + pub value: Value, + pub said: String, + pub source: Source, + pub supersedes: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum EntryState { + Stored, + Applied { verified_at: u64, evidence: String }, + Superseded, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Entry { + pub id: String, + pub kind: Kind, + pub scope: Scope, + pub value: Value, + pub said: String, + pub at: u64, + pub source: Source, + pub supersedes: Option, + pub state: EntryState, + pub reminders: u32, + // Inherited by a strengthening replacement: a lead cannot launder an owner's live hold + // into its own hold and then release it. Request source remains attributed separately. + pub(super) authority: Authority, +} + +pub(super) fn text(value: &str, maximum: usize) -> Result<()> { + if value.trim().is_empty() + || value.len() > maximum + || value + .chars() + .any(|c| c.is_control() || matches!(c, '\u{2028}' | '\u{2029}')) + { + return Err("standing text must be bounded and single-line".into()); + } + Ok(()) +} +pub(super) fn id(value: &str) -> Result<()> { + if value.is_empty() + || value.len() > 128 + || !value + .bytes() + .all(|c| c.is_ascii_alphanumeric() || b"-_.:/".contains(&c)) + { + return Err("invalid standing identifier".into()); + } + Ok(()) +} +impl Scope { + pub(super) fn validate(&self) -> Result<()> { + match self { + Self::Repository => Ok(()), + Self::PullRequest(number) if *number > 0 => Ok(()), + Self::PullRequest(_) => Err("standing PR number must be positive".into()), + Self::Task(value) | Self::Host(value) => id(value), + Self::Branch(value) => { + text(value, 256)?; + if value.starts_with('-') { + return Err("invalid standing branch".into()); + } + Ok(()) + } + } + } +} +impl Source { + pub(super) fn validate(&self) -> Result<()> { + match self { + Self::Command { command } => text(command, 512), + Self::SessionEntry { session_id, entry_id } => { + id(session_id)?; + id(entry_id) + } + } + } +} +impl Rule { + pub(super) fn validate(&self) -> Result<()> { + self.scope.validate()?; + self.source.validate()?; + text(&self.said, 1024)?; + if let Some(prior) = &self.supersedes { + id(prior)?; + } + match (&self.kind, &self.value) { + (_, Value::Retired) => Ok(()), + (Kind::Hold | Kind::Freeze, Value::Constraint { .. }) => Ok(()), + ( + Kind::Goal + | Kind::Priority + | Kind::Model + | Kind::Placement + | Kind::Eligible + | Kind::Decided + | Kind::Note, + Value::Text(value), + ) => text(value, 1024), + _ => Err("standing kind and value schema disagree".into()), + } + } +} +impl Entry { + pub(super) fn live(&self) -> bool { + self.state != EntryState::Superseded + && !matches!(self.value, Value::Retired | Value::Constraint { active: false }) + } + pub fn needs_conversion_offer(&self) -> bool { + self.kind == Kind::Note && self.reminders >= 2 && self.state != EntryState::Superseded + } + pub(super) fn rule(&self) -> Rule { + Rule { + kind: self.kind, + scope: self.scope.clone(), + value: self.value.clone(), + said: self.said.clone(), + source: self.source.clone(), + supersedes: self.supersedes.clone(), + } + } + pub(super) fn validate(&self) -> Result<()> { + id(&self.id)?; + self.rule().validate()?; + if let EntryState::Applied { evidence, .. } = &self.state { + text(evidence, 256)?; + } + Ok(()) + } +} From 0e9289c84fe1e0829cb78bcbaea11eb0df64bf96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:19:26 +0000 Subject: [PATCH 10/27] test(work): reproduce unsynced prepared policy recovery --- oxa/oxa-work/src/standing/store/tests.rs | 76 ++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/oxa/oxa-work/src/standing/store/tests.rs b/oxa/oxa-work/src/standing/store/tests.rs index 5c927e625..a2a1b3006 100644 --- a/oxa/oxa-work/src/standing/store/tests.rs +++ b/oxa/oxa-work/src/standing/store/tests.rs @@ -404,3 +404,79 @@ fn a_late_answer_is_retained_even_when_its_question_is_the_oldest() { .answer .is_some()); } + +#[test] +fn recovery_syncs_the_prepared_policy_before_committing_its_fact() { + use std::os::unix::fs::PermissionsExt; + // SAFETY: geteuid has no arguments, pointers or failure mode. + if unsafe { libc::geteuid() } == 0 { + use std::os::unix::process::CommandExt; + let result = std::process::Command::new(std::env::current_exe().unwrap()) + .env_clear() + .args([ + "--exact", + "standing::store::tests::recovery_syncs_the_prepared_policy_before_committing_its_fact", + ]) + .uid(65534) + .gid(65534) + .output() + .unwrap(); + assert!( + result.status.success(), + "{}{}", + String::from_utf8_lossy(&result.stdout), + String::from_utf8_lossy(&result.stderr) + ); + return; + } + let fixture = Fixture::new(); + let facts = fixture.facts(); + let standing = fixture.standing(facts.clone()); + let before = standing.read().unwrap(); + let mut after = before.next(0, 9).unwrap(); + let request = hold(true, None); + after.entries.push(Entry { + id: "R-1".into(), + kind: request.kind, + scope: request.scope, + value: request.value, + said: request.said, + at: 9, + source: request.source, + supersedes: None, + state: EntryState::Stored, + reminders: 0, + authority: Authority::Owner, + }); + let event = Event { + seq: 1, + at_ms: 9, + checkpoint: Checkpoint { + source: "standing".into(), + revision: 1, + }, + fact: Fact::StandingChanged(after.change(None)), + }; + let document = Document { + version: 1, + current: before, + pending: Some(Pending { + after, + observation: serde_json::from_slice(&event.to_json().unwrap()).unwrap(), + }), + }; + standing.root.atomic(FILE, &document.bytes().unwrap()).unwrap(); + let path = fixture.0.join("standing/standing.json"); + let original = fs::read(&path).unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o400)).unwrap(); + let recovered = standing.read(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); + assert!( + recovered.is_err(), + "policy was acknowledged before its prepared data could be synced" + ); + assert_eq!(fs::read(&path).unwrap(), original); + assert_eq!(facts.situation().unwrap().standing_revision, 0); + assert_eq!(standing.read().unwrap().revision, 1); + assert_eq!(facts.events_after(0, 64).unwrap().events.len(), 1); +} From 551bb0028d0a1439f8f360eb02dfb1962f25623a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:57:06 +0000 Subject: [PATCH 11/27] Complete typed standing policy, bound proposals and checked release conditions --- oxa/docs/standing.md | 116 +++++++ oxa/oxa-proto/src/work.rs | 1 + oxa/oxa-work/src/standing.rs | 7 +- oxa/oxa-work/src/standing/asks.rs | 25 +- oxa/oxa-work/src/standing/guard.rs | 2 + oxa/oxa-work/src/standing/store.rs | 235 +++++++++++-- oxa/oxa-work/src/standing/store/tests.rs | 407 ++++++++++++++++++++++- oxa/oxa-work/src/standing/types.rs | 117 ++++++- 8 files changed, 880 insertions(+), 30 deletions(-) create mode 100644 oxa/docs/standing.md diff --git a/oxa/docs/standing.md b/oxa/docs/standing.md new file mode 100644 index 000000000..8eb8fa942 --- /dev/null +++ b/oxa/docs/standing.md @@ -0,0 +1,116 @@ +# Standing rules and owner decisions + +`oxa_work::standing` stores the owner's rules, holds and questions for one repository. It +never opens runner task state, starts a model, writes a task receipt or creates a background +thread. The runner owns command/action wiring; the lead owns cards and tool registration. + +## Protected root and revisions + +The trusted frontend supplies a fixed per-repository root in owner configuration, outside +workspace and writable Git binds, and the repository's `Facts` instance. Neither path comes +from tool JSON. `Standing::initialize(root, Authority::Owner, facts)` is explicit. `open` +never turns missing, partial, corrupt, unsafe or inconsistent enforcement data into empty rules. +The existing private directory/file/no-follow primitives are reused. + +`read()` returns the current revision, entries, asks and explicit history-truncation flag. +`live_rules()` excludes superseded/retired rules and released constraints; `open_asks()` +excludes answered questions. Active rules and unanswered asks share a 128-item limit. Up to +16 terminal records are retained; they may shrink to fit the 256 KiB read view. Pending +metadata is capped at 1 MiB. Active data is never dropped to make room. + +Every mutation supplies the document revision it read. A stale revision refuses without +changing the store. A live collision of kind/scope requires its exact `supersedes` ID. +IDs are derived from the checked revision, not from model text or time. + +## Authority, values and checked application + +`Authority` is supplied separately by the trusted interface. It is absent from `Rule`, +`Question` and `AnswerRequest`. A model cannot claim owner authority by adding a JSON field +or quoting a message. The frontend checks the source reference and binds owner operations to +actual owner input. `said` preserves those words; `said_by` attributes them separately from +inherited hold protection. Sources name a command or saved session entry. + +Kinds are goal, priority, freeze, hold, budget, model, placement, gate, eligible, decided, +reserve and note. Text policy values remain explicit data. Budgets use integer USD micros +per day; the frontend presents/parses ordinary USD amounts. Gates contain an ordered T1/T2/T3 +tier and a closed set of additional evidence requirements. They supplement existing policy +floors. Reserve values use the closed public/paid/credentials/destructive/desktop classes. + +Only the owner raises a budget, establishes an initial positive budget/gate, weakens a gate, +changes reserves, or releases an owner constraint. Lead reductions and conjunctive gate +strengthening are allowed. An owner hold keeps its protection through a lead replacement; +replacing it cannot turn it into a releasable lead hold or change its release condition. + +Storing a rule gives it `Stored` state. Gateway caps, model profiles, placement, eligibility, +priority and merge-policy application belong to their existing runner/runtime adapters. +They consume the locked read view; this store does not invent another policy engine. +`verify` accepts only a frontend-owned checker reporting the exact current revision, scope, +actual value and evidence reference. Mismatches leave the rule Stored. An accepted proposal +also starts Stored; it does not claim a runtime was reconfigured. + +Reminders require owner context and a checked source. Counts, the latest source/time and +incident observations persist. After two reminders a live note reports that a conversion +should be offered. It is never converted automatically. An explicit owner decision may +supersede that note with an enforced kind. + +## One action guard + +`with_guard(action, subject, callback)` reloads and establishes the current policy's durability +under its stable lock. Subject identity is resolved by the trusted frontend; a missing PR +means checked absence, and a known PR must always be supplied. Admission can precede a PR. +Approve/enqueue/update require a positive PR identity. There is no head-dependent hold key. + +A live hold refuses admit, push, approve, enqueue and update. Freeze refuses approve/enqueue. +Both apply immediately while Stored. `is_held(pr)` is a read for presentation; a preceding +read never authorizes an effect. Callback receives `&Read` with the exact guarded revision, +including the other policy values for their owning adapters, and runs only after guard success. +It must not recursively read/write this Standing instance. + +The lock covers one effect and its result, never an entire worker, loop or batch. The caller +owns bounded execution/cancellation, side-effect idempotency and its receipt containing the +guarded revision. It must invoke another guard before the next effect. The store's guard +does not override existing tool, sandbox, gateway or publishing policy. + +## Questions and proposals + +The existing ask tool's payload is reused: question, 2–4 unique label/detail options and an +exact-label recommendation. Stored asks add ID, question revision, scope, time/source, +optional meanwhile plan and an optional owner answer. Unknown meanwhile remains `None`. +The existing UI/journal adapter is not replaced by a second tool in this module. + +`ask` records the question. A recommendation never answers it or grants permission. +`answer` requires owner context, the current document revision and the exact question +revision. Option labels are checked; free text is stored as free text. Answers preserve the +owner's actual words/source. A generic answer labelled "Release" cannot release a hold. + +`propose` binds an exact typed Rule to an ask and an accept label. Cards **must render that +rule's actual kind, scope and value**, not only model-written labels/details. Scope must match +the question. Only selecting that label through the owner interface applies the bound rule; +it cannot substitute a new value. A changed/superseded target refuses without closing the ask. +Free text has no implicit accept meaning. Other answers close only the question. This changes +standing data, never performs a GitHub/paid/tool side effect. + +A named `PrMerged(N)` condition can release a hold/freeze without asking for a second decision +already authorized by the owner. `release_when` uses a frontend-owned checker and requires the +exact matching `PrMerged` Event retained in this same Facts store. The runner records that +verified merge first. Wrong PRs, invented events and unavailable observations refuse; prose, +a successful worker and a changed head are insufficient. The resulting release retains its +source words and records checked merge evidence. Only the separate merge fact is landed work. + +## Durable fact handoff + +One fixed `standing` emitter uses the document revision. `StandingChanged` projects revision, +goal, one ask transition and an optional incident. Full rule/option/answer data stays in this +protected file. It does not emit two different facts under one source/revision. + +Under Standing → Facts lock order, the facts module first validates capacity/semantics, then +the prepare callback durably saves the pending policy, before reserving/appending its fact. +Only after that fact succeeds is the current policy committed. A rejected projection leaves +no pending standing revision: the owner can still close an old ask to free capacity. Recovery +syncs a visible prepared policy before committing its fact, replays its identity once, commits +the policy and establishes its durability before another action. A failed observation stops +before an effect. Readers never rewrite runner task state. + +The runner owns `oxa work rule|hold|ask`, actual action guards and local command proofs. The +lead joins detailed asks by ID from `Read` with the small shared situation. These adapters +must be wired and proven before claiming the complete #629 L2 user flow is delivered. diff --git a/oxa/oxa-proto/src/work.rs b/oxa/oxa-proto/src/work.rs index d9bbbeb00..b216f2ae3 100644 --- a/oxa/oxa-proto/src/work.rs +++ b/oxa/oxa-proto/src/work.rs @@ -475,6 +475,7 @@ record!( if let Fact::StandingChanged(change) = &s.fact { require(s.checkpoint.source == "standing" && s.checkpoint.revision == change.revision, "standing checkpoint identity disagrees")?; } + require(s.checkpoint.source != "standing" || matches!(s.fact, Fact::StandingChanged(_)), "standing source is reserved for standing revisions")?; Ok(()) } ); diff --git a/oxa/oxa-work/src/standing.rs b/oxa/oxa-work/src/standing.rs index ea71fa672..6bb42f11a 100644 --- a/oxa/oxa-work/src/standing.rs +++ b/oxa/oxa-work/src/standing.rs @@ -2,8 +2,11 @@ mod guard; pub use guard::{Action, Refusal, RefusalReason, Scope, Subject}; mod types; -pub use types::{Authority, Entry, EntryState, Kind, Rule, Source, Value}; +pub use types::{ + Authority, Condition, Entry, EntryState, GateEvidence, Kind, Reminder, ReservedClass, ReviewTier, Rule, + Source, Value, +}; mod store; pub use store::{GuardError, Guarded, Read, Standing, Verification}; mod asks; -pub use asks::{Answer, AnswerRequest, Ask, Choice, Question, QuestionOption}; +pub use asks::{Answer, AnswerRequest, Ask, Choice, Proposal, Question, QuestionOption}; diff --git a/oxa/oxa-work/src/standing/asks.rs b/oxa/oxa-work/src/standing/asks.rs index 1c2296a21..ab8e1aa1a 100644 --- a/oxa/oxa-work/src/standing/asks.rs +++ b/oxa/oxa-work/src/standing/asks.rs @@ -1,4 +1,4 @@ -use super::{types, Scope, Source}; +use super::{types, Rule, Scope, Source}; use crate::Result; use serde::{Deserialize, Serialize}; @@ -38,12 +38,20 @@ pub struct Answer { pub choice: Choice, pub at: u64, pub source: Source, + pub said: String, } pub struct AnswerRequest { pub ask_id: String, pub question_revision: u64, pub choice: Choice, pub source: Source, + pub said: String, +} +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Proposal { + pub rule: Rule, + pub accept_label: String, } #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] #[serde(deny_unknown_fields)] @@ -58,6 +66,7 @@ pub struct Ask { pub at: u64, pub source: Source, pub answer: Option, + pub proposal: Option, } impl Question { pub(super) fn validate(&self) -> Result<()> { @@ -112,8 +121,20 @@ impl Ask { return Err("answer names a different question revision".into()); } answer.source.validate()?; + types::text(&answer.said, 1024)?; self.validate_choice(&answer.choice)?; } + if let Some(proposal) = &self.proposal { + proposal.rule.validate()?; + if proposal.rule.scope != self.scope + || !self + .options + .iter() + .any(|option| option.label == proposal.accept_label) + { + return Err("proposal is not bound to this question".into()); + } + } Ok(()) } pub(super) fn fact(&self) -> oxa_proto::work::Ask { @@ -121,7 +142,7 @@ impl Ask { Scope::PullRequest(pr) => oxa_proto::work::Subject::PullRequest(*pr), Scope::Task(task) => oxa_proto::work::Subject::Task(task.clone()), Scope::Host(host) => oxa_proto::work::Subject::Host(host.clone()), - Scope::Repository | Scope::Branch(_) => oxa_proto::work::Subject::Repository, + Scope::Repository | Scope::Branch(_) | Scope::Path(_) => oxa_proto::work::Subject::Repository, }; oxa_proto::work::Ask { id: self.id.clone(), diff --git a/oxa/oxa-work/src/standing/guard.rs b/oxa/oxa-work/src/standing/guard.rs index be4e479c2..1f9ee6827 100644 --- a/oxa/oxa-work/src/standing/guard.rs +++ b/oxa/oxa-work/src/standing/guard.rs @@ -23,6 +23,7 @@ pub enum Scope { Task(String), Branch(String), Host(String), + Path(String), } /// Identity resolved by the trusted frontend, not tool JSON. None means checked absence. @@ -69,6 +70,7 @@ impl Scope { Self::Task(task) => subject.task == Some(task.as_str()), Self::Branch(branch) => subject.branch == Some(branch.as_str()), Self::Host(host) => subject.host == Some(host.as_str()), + Self::Path(_) => false, // Path selectors are policy data, never hold identities. } } } diff --git a/oxa/oxa-work/src/standing/store.rs b/oxa/oxa-work/src/standing/store.rs index 9dd1cbe06..1c19279d9 100644 --- a/oxa/oxa-work/src/standing/store.rs +++ b/oxa/oxa-work/src/standing/store.rs @@ -1,8 +1,8 @@ use super::guard::{self, Constraint, ConstraintKind}; use super::types::{self, EntryState}; use super::{ - Action, Answer, AnswerRequest, Ask, Authority, Entry, Kind, Question, Refusal, Rule, Scope, Subject, - Value, + Action, Answer, AnswerRequest, Ask, Authority, Choice, Entry, Kind, Proposal, Question, Refusal, Rule, + Scope, Subject, Value, }; use crate::{events::Facts, facts_io::Root, Result}; use oxa_proto::work::{Checkpoint, Event, Fact, Incident, IncidentKind, StandingChanged}; @@ -48,6 +48,12 @@ impl Document { } } impl Read { + pub fn live_rules(&self) -> impl Iterator { + self.entries.iter().filter(|entry| entry.is_live()) + } + pub fn open_asks(&self) -> impl Iterator { + self.asks.iter().filter(|ask| ask.answer.is_none()) + } fn validate(&self) -> Result<()> { if self.entries.len() + self.asks.len() > MAX_ACTIVE + HISTORY { return Err("standing entry capacity exceeded".into()); @@ -59,7 +65,7 @@ impl Read { if !ids.insert(&entry.id) { return Err("duplicate standing entry identity".into()); } - if entry.live() { + if entry.is_live() { if live .iter() .any(|(kind, scope)| *kind == entry.kind && *scope == &entry.scope) @@ -98,7 +104,7 @@ impl Read { Ok(()) } fn bounded(&mut self) -> Result<()> { - while self.entries.iter().filter(|entry| !entry.live()).count() + while self.entries.iter().filter(|entry| !entry.is_live()).count() + self.asks.iter().filter(|ask| ask.answer.is_some()).count() > HISTORY { @@ -110,7 +116,7 @@ impl Read { self.validate() } fn discard_history(&mut self) -> Result<()> { - if let Some(index) = self.entries.iter().position(|entry| !entry.live()) { + if let Some(index) = self.entries.iter().position(|entry| !entry.is_live()) { self.entries.remove(index); } else if let Some((index, _)) = self .asks @@ -143,7 +149,7 @@ impl Read { .entries .iter() .find_map(|entry| match (&entry.kind, &entry.value) { - (Kind::Goal, Value::Text(value)) if entry.live() && entry.scope == Scope::Repository => { + (Kind::Goal, Value::Text(value)) if entry.is_live() && entry.scope == Scope::Repository => { Some(value.clone()) } _ => None, @@ -300,6 +306,9 @@ impl Standing { { return Err("pending standing projection disagrees".into()); } + // A prior rename may be visible after directory sync failed. Establish the + // prepared policy's durability before committing its separately stored fact. + self.root.durable(FILE)?; self.facts.append(event.checkpoint, event.at_ms, event.fact)?; document.current = pending.after.clone(); document.pending = None; @@ -359,10 +368,70 @@ impl Standing { let _lock = self.root.lock(LOCK)?; let before = self.load()?; let mut after = before.next(expected, at)?; - let prior = after + let entry = Self::stage_rule(&mut after, authority, at, request)?; + self.save(before, after, None)?; + Ok(entry) + } + fn stage_rule(after: &mut Read, authority: Authority, at: u64, request: Rule) -> Result { + request.validate()?; + let target = after .entries - .iter_mut() - .find(|entry| entry.live() && entry.kind == request.kind && entry.scope == request.scope); + .iter() + .position(|entry| entry.is_live() && entry.kind == request.kind && entry.scope == request.scope); + let conversion = if target.is_none() && request.kind != Kind::Note { + after.entries.iter().position(|entry| { + entry.is_live() + && entry.kind == Kind::Note + && entry.scope == request.scope + && request.supersedes.as_deref() == Some(&entry.id) + }) + } else { + None + }; + if conversion.is_some() && authority != Authority::Owner { + return Err("owner confirmation required to convert an advisory note".into()); + } + let prior = target.or(conversion).map(|index| &mut after.entries[index]); + if authority != Authority::Owner { + let owner_required = match (&request.kind, &request.value, prior.as_deref()) { + (Kind::Reserve, _, _) => true, + ( + Kind::Budget, + Value::Budget { + daily_usd_micros: new, + }, + Some(Entry { + value: + Value::Budget { + daily_usd_micros: old, + }, + .. + }), + ) => new > old, + (Kind::Budget, Value::Budget { daily_usd_micros: 0 }, None) => false, + (Kind::Budget, _, _) => true, + ( + Kind::Gate, + Value::Gate { + tier: new_tier, + evidence: new, + }, + Some(Entry { + value: + Value::Gate { + tier: old_tier, + evidence: old, + }, + .. + }), + ) => new_tier < old_tier || !old.iter().all(|required| new.contains(required)), + (Kind::Gate, _, _) => true, + _ => false, + }; + if owner_required { + return Err("this standing change requires owner authority; propose it instead".into()); + } + } let mut protection = authority; if let Some(prior) = prior { if request.supersedes.as_deref() != Some(&prior.id) { @@ -370,9 +439,16 @@ impl Standing { } let releasing = matches!( request.value, - Value::Retired | Value::Constraint { active: false } + Value::Retired + | Value::Constraint { + active: false, + until: None + } ); if matches!(request.kind, Kind::Hold | Kind::Freeze) && prior.authority == Authority::Owner { + if authority != Authority::Owner && prior.value != request.value { + return Err("only the owner changes this release condition".into()); + } if releasing && authority != Authority::Owner { return Err("only the owner releases this constraint".into()); } @@ -385,7 +461,11 @@ impl Standing { } if matches!( request.value, - Value::Retired | Value::Constraint { active: false } + Value::Retired + | Value::Constraint { + active: false, + until: None + } ) { return Err("no live standing entry to release".into()); } @@ -397,35 +477,48 @@ impl Standing { value: request.value, said: request.said, at, + said_by: authority, source: request.source, supersedes: request.supersedes, state: EntryState::Stored, reminders: 0, + last_reminder: None, authority: protection, }; after.entries.push(entry.clone()); - self.save(before, after, None)?; Ok(entry) } - pub fn remind(&self, expected: u64, entry_id: &str, at: u64) -> Result { + pub fn remind( + &self, + expected: u64, + authority: Authority, + entry_id: &str, + at: u64, + source: super::Source, + ) -> Result { + if authority != Authority::Owner { + return Err("owner reminder context required".into()); + } + source.validate()?; let _lock = self.root.lock(LOCK)?; let before = self.load()?; let mut after = before.next(expected, at)?; let entry = after .entries .iter_mut() - .find(|entry| entry.id == entry_id && entry.live()) + .find(|entry| entry.id == entry_id && entry.is_live()) .ok_or("standing entry is not live")?; entry.reminders = entry .reminders .checked_add(1) .ok_or("standing reminder count exhausted")?; + entry.last_reminder = Some(super::Reminder { at, source }); let result = entry.clone(); let subject = match &entry.scope { Scope::PullRequest(pr) => oxa_proto::work::Subject::PullRequest(*pr), Scope::Task(task) => oxa_proto::work::Subject::Task(task.clone()), Scope::Host(host) => oxa_proto::work::Subject::Host(host.clone()), - Scope::Repository | Scope::Branch(_) => oxa_proto::work::Subject::Repository, + Scope::Repository | Scope::Branch(_) | Scope::Path(_) => oxa_proto::work::Subject::Repository, }; let incident = Incident { id: format!("{}-reminded-{}", entry.id, after.revision), @@ -452,7 +545,7 @@ impl Standing { let entry = after .entries .iter_mut() - .find(|entry| entry.id == entry_id && entry.live()) + .find(|entry| entry.id == entry_id && entry.state != EntryState::Superseded) .ok_or("standing entry is not live")?; let verified = check(entry)?; types::text(&verified.evidence, 256)?; @@ -473,6 +566,26 @@ impl Standing { Ok(result) } pub fn ask(&self, expected: u64, at: u64, request: Question) -> Result { + self.open_question(expected, at, request, None) + } + pub fn propose( + &self, + expected: u64, + at: u64, + rule: Rule, + question: Question, + accept_label: String, + ) -> Result { + rule.validate()?; + self.open_question(expected, at, question, Some(Proposal { rule, accept_label })) + } + fn open_question( + &self, + expected: u64, + at: u64, + request: Question, + proposal: Option, + ) -> Result { request.validate()?; let _lock = self.root.lock(LOCK)?; let before = self.load()?; @@ -488,12 +601,15 @@ impl Standing { at, source: request.source, answer: None, + proposal, }; + ask.validate()?; after.asks.push(ask.clone()); self.save(before, after, None)?; Ok(ask) } - /// An answer records an owner decision; it never releases a hold or grants tool permission. + /// Only acceptance of the exact bound proposal changes a rule. Generic answers grant no + /// permission, and storing an accepted rule never claims it is applied to a runtime. pub fn answer( &self, expected: u64, @@ -505,6 +621,7 @@ impl Standing { return Err("only the owner answers standing asks".into()); } request.source.validate()?; + types::text(&request.said, 1024)?; let _lock = self.root.lock(LOCK)?; let before = self.load()?; let mut after = before.next(expected, at)?; @@ -517,14 +634,28 @@ impl Standing { return Err("standing question changed or was already answered".into()); } ask.validate_choice(&request.choice)?; + let accepted = match (&ask.proposal, &request.choice) { + (Some(proposal), Choice::Option(label)) if *label == proposal.accept_label => { + Some(proposal.rule.clone()) + } + _ => None, + }; + let owner_source = request.source.clone(); + let owner_words = request.said.clone(); ask.answer = Some(Answer { revision: after.revision, question_revision: request.question_revision, choice: request.choice, at, source: request.source, + said: request.said, }); let result = ask.clone(); + if let Some(mut rule) = accepted { + rule.source = owner_source; + rule.said = owner_words; + Self::stage_rule(&mut after, Authority::Owner, at, rule)?; + } self.save(before, after, None)?; Ok(result) } @@ -537,12 +668,78 @@ impl Standing { .entries .iter() .find(|entry| { - entry.live() + entry.is_live() && entry.kind == Kind::Hold && (entry.scope == Scope::Repository || entry.scope == Scope::PullRequest(pr)) }) .map(|entry| entry.id.clone())) } + /// Verify a named immutable merge through a frontend-owned checker. The original condition + /// authorizes release; model prose and changed PR heads never supply this proof. + pub fn release_when( + &self, + expected: u64, + entry_id: &str, + at: u64, + check: impl FnOnce(&super::Condition) -> Result>, + ) -> Result> { + let _lock = self.root.lock(LOCK)?; + let before = self.load()?; + if before.revision != expected { + return Err("standing revision changed; read it again".into()); + } + let entry = before + .entries + .iter() + .find(|entry| entry.id == entry_id && entry.is_live()) + .ok_or("standing entry is not live")?; + let Value::Constraint { + active: true, + until: Some(condition), + } = &entry.value + else { + return Err("standing entry has no release condition".into()); + }; + let Some(observation) = check(condition)? else { + return Ok(None); + }; + let Fact::PrMerged(proof) = &observation.fact else { + return Err("condition requires a merge observation".into()); + }; + let super::Condition::PrMerged(pr) = condition; + if *pr != proof.pr.number { + return Err("merge proof names a different condition".into()); + } + let previous = observation + .seq + .checked_sub(1) + .ok_or("invalid merge observation sequence")?; + let page = self.facts.events_after(previous, 1)?; + if page.gap || page.events.first() != Some(&observation) { + return Err("merge observation is not retained in this facts store".into()); + } + let mut after = before.next(expected, at)?; + let rule = Rule { + kind: entry.kind, + scope: entry.scope.clone(), + value: Value::Constraint { + active: false, + until: None, + }, + said: entry.said.clone(), + source: entry.source.clone(), + supersedes: Some(entry.id.clone()), + }; + let mut released = Self::stage_rule(&mut after, entry.authority, at, rule)?; + released.said_by = entry.said_by; + released.state = EntryState::Applied { + verified_at: at, + evidence: format!("merged-pr:{pr}:{}", proof.merge_commit), + }; + *after.entries.last_mut().expect("new release entry") = released.clone(); + self.save(before, after, None)?; + Ok(Some(released)) + } pub fn with_guard( &self, action: Action, @@ -552,7 +749,7 @@ impl Standing { let _lock = self.root.lock(LOCK)?; let read = self.load()?; let constraints = read.entries.iter().filter_map(|entry| { - if !entry.live() { + if !entry.is_live() { return None; } let kind = match entry.kind { diff --git a/oxa/oxa-work/src/standing/store/tests.rs b/oxa/oxa-work/src/standing/store/tests.rs index a2a1b3006..927a8acd0 100644 --- a/oxa/oxa-work/src/standing/store/tests.rs +++ b/oxa/oxa-work/src/standing/store/tests.rs @@ -39,7 +39,7 @@ fn hold(active: bool, supersedes: Option<&str>) -> Rule { Rule { kind: Kind::Hold, scope: Scope::PullRequest(480), - value: Value::Constraint { active }, + value: Value::Constraint { active, until: None }, said: "Hold this PR until released".into(), source: super::super::Source::Command { command: "hold 480".into(), @@ -171,10 +171,12 @@ fn pending_revisions_recover_before_or_after_the_fact_without_duplicate_delivery value: request.value, said: request.said, at: 9, + said_by: Authority::Owner, source: request.source, supersedes: None, state: EntryState::Stored, reminders: 0, + last_reminder: None, authority: Authority::Owner, }); let event = Event { @@ -259,8 +261,30 @@ fn reminders_create_incidents_and_offer_without_changing_advisory_rules() { supersedes: None, }; let note = standing.rule(0, Authority::Owner, 1, request).unwrap(); - assert!(!standing.remind(1, ¬e.id, 2).unwrap().needs_conversion_offer()); - assert!(standing.remind(2, ¬e.id, 3).unwrap().needs_conversion_offer()); + assert!(!standing + .remind( + 1, + Authority::Owner, + ¬e.id, + 2, + super::super::Source::Command { + command: "remind".into() + } + ) + .unwrap() + .needs_conversion_offer()); + assert!(standing + .remind( + 2, + Authority::Owner, + ¬e.id, + 3, + super::super::Source::Command { + command: "remind".into() + } + ) + .unwrap() + .needs_conversion_offer()); let read = standing.read().unwrap(); assert_eq!(read.entries[0].kind, Kind::Note); assert_eq!(read.entries[0].reminders, 2); @@ -296,6 +320,7 @@ fn answer(ask: &Ask) -> super::super::AnswerRequest { source: super::super::Source::Command { command: "decide Release".into(), }, + said: "Release".into(), } } @@ -442,10 +467,12 @@ fn recovery_syncs_the_prepared_policy_before_committing_its_fact() { value: request.value, said: request.said, at: 9, + said_by: Authority::Owner, source: request.source, supersedes: None, state: EntryState::Stored, reminders: 0, + last_reminder: None, authority: Authority::Owner, }); let event = Event { @@ -480,3 +507,377 @@ fn recovery_syncs_the_prepared_policy_before_committing_its_fact() { assert_eq!(standing.read().unwrap().revision, 1); assert_eq!(facts.events_after(0, 64).unwrap().events.len(), 1); } + +#[test] +fn a_concurrent_owner_hold_waits_for_one_effect_then_blocks_the_next() { + const CHILD_ROOT: &str = "OXA_STANDING_GUARD_CHILD"; + if let Some(root) = std::env::var_os(CHILD_ROOT) { + let root = PathBuf::from(root); + let facts = Facts::open(&root.join("facts")).unwrap(); + fs::write(root.join("child-ready"), b"ready").unwrap(); + let standing = Standing::open(&root.join("standing"), facts).unwrap(); + standing.rule(0, Authority::Owner, 1, hold(true, None)).unwrap(); + return; + } + struct Child(std::process::Child); + impl Drop for Child { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } + let fixture = Fixture::new(); + let facts = fixture.facts(); + let standing = fixture.standing(facts); + let mut child = None; + let completed = standing.with_guard(Action::Update, subject(), |read| { + let process = std::process::Command::new(std::env::current_exe().unwrap()) + .env_clear().env(CHILD_ROOT, &fixture.0) + .args(["--exact", "standing::store::tests::a_concurrent_owner_hold_waits_for_one_effect_then_blocks_the_next"]) + .stdout(std::process::Stdio::null()).stderr(std::process::Stdio::piped()).spawn().unwrap(); + child = Some(Child(process)); + let until = std::time::Instant::now() + std::time::Duration::from_secs(10); + while !fixture.0.join("child-ready").exists() { + assert!(std::time::Instant::now() < until, "child failed to start"); + assert!(child.as_mut().unwrap().0.try_wait().unwrap().is_none()); + std::thread::sleep(std::time::Duration::from_millis(1)); + } + std::thread::sleep(std::time::Duration::from_millis(50)); + assert!(child.as_mut().unwrap().0.try_wait().unwrap().is_none(), "hold writer passed the live effect guard"); + let document: Document = serde_json::from_slice(&standing.root.read(FILE, MAX_FILE).unwrap()).unwrap(); + assert_eq!(document.current.revision, read.revision); + Ok(read.revision) + }).unwrap(); + assert_eq!(completed.result.unwrap(), 0); + let child = child.as_mut().unwrap(); + let until = std::time::Instant::now() + std::time::Duration::from_secs(10); + loop { + if let Some(status) = child.0.try_wait().unwrap() { + assert!(status.success()); + break; + } + assert!( + std::time::Instant::now() < until, + "hold writer did not acquire the released guard" + ); + std::thread::sleep(std::time::Duration::from_millis(1)); + } + assert!(standing + .with_guard(Action::Update, subject(), |_| Ok(())) + .is_err()); +} + +fn policy(kind: Kind, scope: Scope, value: Value, supersedes: Option<&str>) -> Rule { + Rule { + kind, + scope, + value, + said: "Synthetic owner rule".into(), + source: super::super::Source::Command { + command: "rule".into(), + }, + supersedes: supersedes.map(str::to_owned), + } +} + +#[test] +fn budget_increases_gate_weakening_and_reserves_require_owner_authority() { + use super::super::{GateEvidence as E, ReservedClass, ReviewTier as T}; + let fixture = Fixture::new(); + let facts = fixture.facts(); + let standing = fixture.standing(facts); + let host = Scope::Host("helper".into()); + let first = standing + .rule( + 0, + Authority::Owner, + 1, + policy( + Kind::Budget, + host.clone(), + Value::Budget { + daily_usd_micros: 200_000_000, + }, + None, + ), + ) + .unwrap(); + let lower = standing + .rule( + 1, + Authority::Lead, + 2, + policy( + Kind::Budget, + host.clone(), + Value::Budget { + daily_usd_micros: 100_000_000, + }, + Some(&first.id), + ), + ) + .unwrap(); + assert!(standing + .rule( + 2, + Authority::Lead, + 3, + policy( + Kind::Budget, + host.clone(), + Value::Budget { + daily_usd_micros: 150_000_000 + }, + Some(&lower.id) + ) + ) + .is_err()); + assert!(standing + .rule( + 2, + Authority::Lead, + 3, + policy(Kind::Budget, host, Value::Retired, Some(&lower.id)) + ) + .is_err()); + let path = Scope::Path("engine/**".into()); + let gate = standing + .rule( + 2, + Authority::Owner, + 3, + policy( + Kind::Gate, + path.clone(), + Value::Gate { + tier: T::T2, + evidence: vec![E::LocalChecks, E::PrBuild], + }, + None, + ), + ) + .unwrap(); + let stronger = standing + .rule( + 3, + Authority::Lead, + 4, + policy( + Kind::Gate, + path.clone(), + Value::Gate { + tier: T::T3, + evidence: vec![E::LocalChecks, E::PrBuild, E::HardwareMeasurement], + }, + Some(&gate.id), + ), + ) + .unwrap(); + for value in [ + Value::Gate { + tier: T::T2, + evidence: vec![E::LocalChecks, E::PrBuild, E::HardwareMeasurement], + }, + Value::Gate { + tier: T::T3, + evidence: vec![E::LocalChecks, E::PrBuild], + }, + Value::Retired, + ] { + assert!(standing + .rule( + 4, + Authority::Lead, + 5, + policy(Kind::Gate, path.clone(), value, Some(&stronger.id)) + ) + .is_err()); + } + let reserve = policy( + Kind::Reserve, + Scope::Repository, + Value::Reserve { + classes: vec![ReservedClass::Paid], + }, + None, + ); + assert!(standing.rule(4, Authority::Lead, 5, reserve.clone()).is_err()); + let reserve = standing.rule(4, Authority::Owner, 5, reserve).unwrap(); + assert!(standing + .rule( + 5, + Authority::Lead, + 6, + policy( + Kind::Reserve, + Scope::Repository, + Value::Retired, + Some(&reserve.id) + ) + ) + .is_err()); + assert_eq!(standing.read().unwrap().revision, 5); +} + +#[test] +fn only_accepting_the_bound_proposal_applies_its_exact_rule() { + let fixture = Fixture::new(); + let facts = fixture.facts(); + let standing = fixture.standing(facts.clone()); + let prior = standing.rule(0, Authority::Owner, 1, hold(true, None)).unwrap(); + let ask = standing + .propose(1, 2, hold(false, Some(&prior.id)), question(), "Release".into()) + .unwrap(); + assert!(standing.is_held(480).unwrap().is_some()); + assert!(standing.answer(2, Authority::Lead, 3, answer(&ask)).is_err()); + let mut accepted = answer(&ask); + accepted.said = "Yes, release this hold".into(); + standing.answer(2, Authority::Owner, 3, accepted).unwrap(); + let read = standing.read().unwrap(); + let released = read.entries.iter().find(|entry| entry.id == "R-3").unwrap(); + assert_eq!(released.said, "Yes, release this hold"); + assert_eq!( + released.value, + Value::Constraint { + active: false, + until: None + } + ); + assert_eq!(released.state, EntryState::Stored); + assert!(standing.is_held(480).unwrap().is_none()); + assert_eq!(facts.situation().unwrap().standing_revision, 3); + assert!(facts.situation().unwrap().asks.is_empty()); +} + +#[test] +fn a_proposal_cannot_replace_a_rule_changed_after_the_question() { + let fixture = Fixture::new(); + let facts = fixture.facts(); + let standing = fixture.standing(facts); + let prior = standing.rule(0, Authority::Owner, 1, hold(true, None)).unwrap(); + let ask = standing + .propose(1, 2, hold(false, Some(&prior.id)), question(), "Release".into()) + .unwrap(); + let current = standing + .rule(2, Authority::Owner, 3, hold(true, Some(&prior.id))) + .unwrap(); + assert!(standing.answer(3, Authority::Owner, 4, answer(&ask)).is_err()); + let read = standing.read().unwrap(); + assert_eq!(read.revision, 3); + assert!(read.asks[0].answer.is_none()); + assert_eq!(standing.is_held(480).unwrap(), Some(current.id)); +} + +#[test] +fn an_advisory_conversion_requires_an_explicit_owner_decision() { + let fixture = Fixture::new(); + let facts = fixture.facts(); + let standing = fixture.standing(facts); + let note = standing + .rule( + 0, + Authority::Owner, + 1, + policy( + Kind::Note, + Scope::PullRequest(480), + Value::Text("Pause this PR".into()), + None, + ), + ) + .unwrap(); + let source = super::super::Source::Command { + command: "remind".into(), + }; + assert!(standing + .remind(1, Authority::Lead, ¬e.id, 2, source.clone()) + .is_err()); + standing + .remind(1, Authority::Owner, ¬e.id, 2, source.clone()) + .unwrap(); + let reminded = standing + .remind(2, Authority::Owner, ¬e.id, 3, source.clone()) + .unwrap(); + assert!(reminded.needs_conversion_offer()); + assert_eq!(reminded.last_reminder.unwrap().source, source); + assert!(standing.is_held(480).unwrap().is_none()); + let conversion = hold(true, Some(¬e.id)); + assert!(standing.rule(3, Authority::Lead, 4, conversion.clone()).is_err()); + let mut prompt = question(); + prompt.question = "Enforce the advisory pause?".into(); + prompt.options[0].label = "Keep advisory".into(); + prompt.options[0].detail = "Leave the note advisory".into(); + prompt.options[1].label = "Enforce".into(); + prompt.options[1].detail = "Create the exact PR hold shown in this proposal".into(); + prompt.recommended = "Keep advisory".into(); + let ask = standing + .propose(3, 4, conversion, prompt, "Enforce".into()) + .unwrap(); + let mut accepted = answer(&ask); + accepted.choice = super::super::Choice::Option("Enforce".into()); + accepted.said = "Enforce".into(); + standing.answer(4, Authority::Owner, 5, accepted).unwrap(); + let read = standing.read().unwrap(); + assert!(!read.live_rules().any(|entry| entry.kind == Kind::Note)); + assert!(standing.is_held(480).unwrap().is_some()); +} + +#[test] +fn a_named_release_needs_the_matching_durable_merge_observation() { + let fixture = Fixture::new(); + let facts = fixture.facts(); + let standing = fixture.standing(facts.clone()); + let mut request = hold(true, None); + request.said = "Hold until PR 478 is merged".into(); + request.value = Value::Constraint { + active: true, + until: Some(super::super::Condition::PrMerged(478)), + }; + let entry = standing.rule(0, Authority::Owner, 1, request).unwrap(); + let mut changed = hold(true, Some(&entry.id)); + assert!(standing.rule(1, Authority::Lead, 2, changed.clone()).is_err()); + changed.value = Value::Constraint { + active: true, + until: Some(super::super::Condition::PrMerged(1)), + }; + assert!(standing.rule(1, Authority::Lead, 2, changed).is_err()); + assert!(standing + .release_when(1, &entry.id, 2, |_| Ok(None)) + .unwrap() + .is_none()); + let proof = Event { + seq: 2, + at_ms: 3, + checkpoint: Checkpoint { + source: "github".into(), + revision: 1, + }, + fact: Fact::PrMerged(oxa_proto::work::Landed { + pr: oxa_proto::work::PullRequest { + number: 478, + head: "a".repeat(40), + }, + merge_commit: "b".repeat(40), + merged_at_ms: 3, + }), + }; + assert!(standing + .release_when(1, &entry.id, 3, |_| Ok(Some(proof.clone()))) + .is_err()); + assert!(standing.is_held(480).unwrap().is_some()); + let observed = facts + .append(proof.checkpoint.clone(), proof.at_ms, proof.fact.clone()) + .unwrap(); + assert_eq!(observed, proof); + let release = standing + .release_when(1, &entry.id, 4, |condition| { + assert_eq!(*condition, super::super::Condition::PrMerged(478)); + Ok(Some(observed.clone())) + }) + .unwrap() + .unwrap(); + assert!(matches!(release.state, EntryState::Applied { .. })); + assert_eq!(release.said, "Hold until PR 478 is merged"); + assert!(standing.is_held(480).unwrap().is_none()); + assert_eq!(facts.situation().unwrap().landed[0].pr.number, 478); +} diff --git a/oxa/oxa-work/src/standing/types.rs b/oxa/oxa-work/src/standing/types.rs index 3a0a59cef..df29e8aa2 100644 --- a/oxa/oxa-work/src/standing/types.rs +++ b/oxa/oxa-work/src/standing/types.rs @@ -47,11 +47,59 @@ pub enum Value { /// An explicit release is a new entry which supersedes the hold/freeze it releases. Constraint { active: bool, + until: Option, }, Retired, + Budget { + daily_usd_micros: u64, + }, + Gate { + tier: ReviewTier, + evidence: Vec, + }, + Reserve { + classes: Vec, + }, +} +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde( + tag = "kind", + content = "value", + rename_all = "snake_case", + deny_unknown_fields +)] +pub enum Condition { + PrMerged(u64), } -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ReviewTier { + T1, + T2, + T3, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum GateEvidence { + LocalChecks, + PrBuild, + Review, + Queue, + HardwareMeasurement, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ReservedClass { + Public, + Paid, + Credentials, + Destructive, + Desktop, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct Rule { pub kind: Kind, pub scope: Scope, @@ -78,14 +126,22 @@ pub struct Entry { pub value: Value, pub said: String, pub at: u64, + pub said_by: Authority, pub source: Source, pub supersedes: Option, pub state: EntryState, pub reminders: u32, + pub last_reminder: Option, // Inherited by a strengthening replacement: a lead cannot launder an owner's live hold // into its own hold and then release it. Request source remains attributed separately. pub(super) authority: Authority, } +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Reminder { + pub at: u64, + pub source: Source, +} pub(super) fn text(value: &str, maximum: usize) -> Result<()> { if value.trim().is_empty() @@ -123,6 +179,13 @@ impl Scope { } Ok(()) } + Self::Path(value) => { + text(value, 256)?; + if value.starts_with('/') || value.split('/').any(|part| matches!(part, "" | "." | "..")) { + return Err("standing path selector must be repository-relative".into()); + } + Ok(()) + } } } } @@ -145,9 +208,46 @@ impl Rule { if let Some(prior) = &self.supersedes { id(prior)?; } + if let Value::Constraint { + active, + until: Some(Condition::PrMerged(pr)), + } = &self.value + { + if !active || *pr == 0 { + return Err("invalid standing release condition".into()); + } + } match (&self.kind, &self.value) { (_, Value::Retired) => Ok(()), - (Kind::Hold | Kind::Freeze, Value::Constraint { .. }) => Ok(()), + (Kind::Hold | Kind::Freeze, Value::Constraint { .. }) + if !matches!(self.scope, Scope::Path(_)) => + { + Ok(()) + } + (Kind::Budget, Value::Budget { .. }) => Ok(()), + (Kind::Gate, Value::Gate { evidence, .. }) => { + if evidence.len() > 5 + || evidence + .iter() + .enumerate() + .any(|(i, value)| evidence[..i].contains(value)) + { + return Err("invalid gate requirements".into()); + } + Ok(()) + } + (Kind::Reserve, Value::Reserve { classes }) => { + if classes.is_empty() + || classes.len() > 5 + || classes + .iter() + .enumerate() + .any(|(i, value)| classes[..i].contains(value)) + { + return Err("invalid reserved classes".into()); + } + Ok(()) + } ( Kind::Goal | Kind::Priority @@ -163,9 +263,12 @@ impl Rule { } } impl Entry { - pub(super) fn live(&self) -> bool { + pub fn is_live(&self) -> bool { self.state != EntryState::Superseded - && !matches!(self.value, Value::Retired | Value::Constraint { active: false }) + && !matches!( + self.value, + Value::Retired | Value::Constraint { active: false, .. } + ) } pub fn needs_conversion_offer(&self) -> bool { self.kind == Kind::Note && self.reminders >= 2 && self.state != EntryState::Superseded @@ -183,6 +286,12 @@ impl Entry { pub(super) fn validate(&self) -> Result<()> { id(&self.id)?; self.rule().validate()?; + if (self.reminders == 0) != self.last_reminder.is_none() { + return Err("standing reminder metadata disagrees".into()); + } + if let Some(reminder) = &self.last_reminder { + reminder.source.validate()?; + } if let EntryState::Applied { evidence, .. } = &self.state { text(evidence, 256)?; } From ecacd23d49127c21f4a360df6cb5b351ff9f47ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:07:23 +0000 Subject: [PATCH 12/27] Return revision-consistent standing and facts views for cards --- oxa/docs/standing.md | 2 ++ oxa/oxa-work/src/standing/store.rs | 10 +++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/oxa/docs/standing.md b/oxa/docs/standing.md index 8eb8fa942..a187e7ac8 100644 --- a/oxa/docs/standing.md +++ b/oxa/docs/standing.md @@ -13,6 +13,8 @@ never turns missing, partial, corrupt, unsafe or inconsistent enforcement data i The existing private directory/file/no-follow primitives are reused. `read()` returns the current revision, entries, asks and explicit history-truncation flag. +`read_facts()` returns that view paired with a shared situation checked at the same standing +revision. Cards use this pair rather than joining independently read versions. `live_rules()` excludes superseded/retired rules and released constraints; `open_asks()` excludes answered questions. Active rules and unanswered asks share a 128-item limit. Up to 16 terminal records are retained; they may shrink to fit the 256 KiB read view. Pending diff --git a/oxa/oxa-work/src/standing/store.rs b/oxa/oxa-work/src/standing/store.rs index 1c19279d9..41123db0d 100644 --- a/oxa/oxa-work/src/standing/store.rs +++ b/oxa/oxa-work/src/standing/store.rs @@ -274,7 +274,15 @@ impl Standing { let _lock = self.root.lock(LOCK)?; self.load() } + /// One revision-consistent view for cards; never join independently read revisions. + pub fn read_facts(&self) -> Result<(Read, oxa_proto::work::Snapshot)> { + let _lock = self.root.lock(LOCK)?; + self.load_facts() + } fn load(&self) -> Result { + self.load_facts().map(|(read, _)| read) + } + fn load_facts(&self) -> Result<(Read, oxa_proto::work::Snapshot)> { let mut document: Document = serde_json::from_slice(&self.root.read(FILE, MAX_FILE)?) .map_err(|_| "invalid standing document")?; if document.version != 1 { @@ -322,7 +330,7 @@ impl Standing { { return Err("standing file disagrees with facts revision".into()); } - Ok(document.current) + Ok((document.current, situation)) } fn save(&self, before: Read, mut after: Read, incident: Option) -> Result { after.bounded()?; From ab952f4ac622a0932c12a2f6588d25a2cd2f9d9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:19:30 +0000 Subject: [PATCH 13/27] test(proto): supply explicit running steps in shared evening fixtures --- oxa/oxa-proto/tests/fixtures/work/evening.jsonl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/oxa/oxa-proto/tests/fixtures/work/evening.jsonl b/oxa/oxa-proto/tests/fixtures/work/evening.jsonl index d82c33d31..1a87ed294 100644 --- a/oxa/oxa-proto/tests/fixtures/work/evening.jsonl +++ b/oxa/oxa-proto/tests/fixtures/work/evening.jsonl @@ -1,4 +1,4 @@ -{"type":"oxa_work_event","version":1,"data":{"seq":1,"at_ms":1794847200000,"checkpoint":{"source":"fixture-runner","revision":1},"fact":{"kind":"task_started","data":{"task":"task-landed","host":"helper","branch":"agent/fixture-task-landed","worktree":"~/project/task-landed","session_id":"session-task-landed"}}}} +{"type":"oxa_work_event","version":1,"data":{"seq":1,"at_ms":1794847200000,"checkpoint":{"source":"fixture-runner","revision":1},"fact":{"kind":"task_started","data":{"task":"task-landed","host":"helper","branch":"agent/fixture-task-landed","worktree":"~/project/task-landed","session_id":"session-task-landed","step":"preparing"}}}} {"type":"oxa_work_event","version":1,"data":{"seq":2,"at_ms":1794847450000,"checkpoint":{"source":"fixture-runner","revision":2},"fact":{"kind":"worker_finished","data":{"task":"task-landed","session_id":"session-task-landed","outcome":"done","error_class":null}}}} {"type":"oxa_work_event","version":1,"data":{"seq":3,"at_ms":1794847700000,"checkpoint":{"source":"fixture-runner","revision":3},"fact":{"kind":"local_acceptance","data":{"task":"task-landed","commit":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","evidence":{"kind":"local_check","id":"local-landed"}}}}} {"type":"oxa_work_event","version":1,"data":{"seq":4,"at_ms":1794847950000,"checkpoint":{"source":"fixture-runner","revision":4},"fact":{"kind":"published","data":{"task":"task-landed","pr":{"number":101,"head":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}}}}} @@ -6,7 +6,7 @@ {"type":"oxa_work_event","version":1,"data":{"seq":6,"at_ms":1794848450000,"checkpoint":{"source":"fixture-runner","revision":6},"fact":{"kind":"enqueued","data":{"number":101,"head":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}}}} {"type":"oxa_work_event","version":1,"data":{"seq":7,"at_ms":1794848700000,"checkpoint":{"source":"fixture-runner","revision":7},"fact":{"kind":"pr_merged","data":{"pr":{"number":101,"head":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"merge_commit":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","merged_at_ms":1794848700000}}}} {"type":"oxa_work_situation","version":1,"data":{"as_of_seq":7,"at_ms":1794848700000,"standing_revision":1,"goal":"Finish the synthetic evening","running":[],"local_accepted":[{"task":"task-landed","commit":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","evidence":{"kind":"local_check","id":"local-landed"}}],"published":[{"task":"task-landed","pr":{"number":101,"head":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}}],"landed":[{"pr":{"number":101,"head":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"merge_commit":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","merged_at_ms":1794848700000}],"blocked":[],"asks":[],"incidents":[],"history_truncated":false}} -{"type":"oxa_work_event","version":1,"data":{"seq":8,"at_ms":1794849000000,"checkpoint":{"source":"fixture-runner","revision":8},"fact":{"kind":"task_started","data":{"task":"task-red","host":"helper","branch":"agent/fixture-task-red","worktree":"~/project/task-red","session_id":"session-task-red"}}}} +{"type":"oxa_work_event","version":1,"data":{"seq":8,"at_ms":1794849000000,"checkpoint":{"source":"fixture-runner","revision":8},"fact":{"kind":"task_started","data":{"task":"task-red","host":"helper","branch":"agent/fixture-task-red","worktree":"~/project/task-red","session_id":"session-task-red","step":"preparing"}}}} {"type":"oxa_work_event","version":1,"data":{"seq":9,"at_ms":1794849250000,"checkpoint":{"source":"fixture-runner","revision":9},"fact":{"kind":"worker_finished","data":{"task":"task-red","session_id":"session-task-red","outcome":"done","error_class":null}}}} {"type":"oxa_work_event","version":1,"data":{"seq":10,"at_ms":1794849500000,"checkpoint":{"source":"fixture-runner","revision":10},"fact":{"kind":"local_acceptance","data":{"task":"task-red","commit":"cccccccccccccccccccccccccccccccccccccccc","evidence":{"kind":"local_check","id":"local-red"}}}}} {"type":"oxa_work_event","version":1,"data":{"seq":11,"at_ms":1794849750000,"checkpoint":{"source":"fixture-runner","revision":11},"fact":{"kind":"published","data":{"task":"task-red","pr":{"number":102,"head":"cccccccccccccccccccccccccccccccccccccccc"}}}}} @@ -15,7 +15,7 @@ {"type":"oxa_work_event","version":1,"data":{"seq":14,"at_ms":1794850500000,"checkpoint":{"source":"fixture-runner","revision":14},"fact":{"kind":"external_acceptance","data":{"pr":{"number":102,"head":"cccccccccccccccccccccccccccccccccccccccc"},"evidence":{"kind":"queue","id":"merge-group/dddddddddddddddddddddddddddddddddddddddd"}}}}} {"type":"oxa_work_event","version":1,"data":{"seq":15,"at_ms":1794850800000,"checkpoint":{"source":"fixture-runner","revision":15},"fact":{"kind":"incident","data":{"id":"red-queue","subject":{"kind":"pull_request","data":102},"kind":"queue_rejected","evidence":{"kind":"gate","id":"gate-red"}}}}} {"type":"oxa_work_situation","version":1,"data":{"as_of_seq":15,"at_ms":1794850800000,"standing_revision":1,"goal":"Finish the synthetic evening","running":[],"local_accepted":[{"task":"task-landed","commit":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","evidence":{"kind":"local_check","id":"local-landed"}},{"task":"task-red","commit":"cccccccccccccccccccccccccccccccccccccccc","evidence":{"kind":"local_check","id":"local-red"}}],"published":[{"task":"task-landed","pr":{"number":101,"head":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}},{"task":"task-red","pr":{"number":102,"head":"cccccccccccccccccccccccccccccccccccccccc"}}],"landed":[{"pr":{"number":101,"head":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"merge_commit":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","merged_at_ms":1794848700000}],"blocked":[],"asks":[],"incidents":[{"id":"red-queue","subject":{"kind":"pull_request","data":102},"kind":"queue_rejected","evidence":{"kind":"gate","id":"gate-red"}}],"history_truncated":false}} -{"type":"oxa_work_event","version":1,"data":{"seq":16,"at_ms":1794851100000,"checkpoint":{"source":"fixture-runner","revision":16},"fact":{"kind":"task_started","data":{"task":"task-blocked","host":"helper","branch":"agent/fixture-task-blocked","worktree":"~/project/task-blocked","session_id":"session-task-blocked"}}}} +{"type":"oxa_work_event","version":1,"data":{"seq":16,"at_ms":1794851100000,"checkpoint":{"source":"fixture-runner","revision":16},"fact":{"kind":"task_started","data":{"task":"task-blocked","host":"helper","branch":"agent/fixture-task-blocked","worktree":"~/project/task-blocked","session_id":"session-task-blocked","step":"preparing"}}}} {"type":"oxa_work_event","version":1,"data":{"seq":17,"at_ms":1794851400000,"checkpoint":{"source":"fixture-runner","revision":17},"fact":{"kind":"worker_finished","data":{"task":"task-blocked","session_id":"session-task-blocked","outcome":"blocked","error_class":"needs-interface"}}}} {"type":"oxa_work_event","version":1,"data":{"seq":18,"at_ms":1794851700000,"checkpoint":{"source":"fixture-runner","revision":18},"fact":{"kind":"ask_opened","data":{"id":"ask-blocked","subject":{"kind":"task","data":"task-blocked"},"question":"Which synthetic interface should the task target?"}}}} {"type":"oxa_work_event","version":1,"data":{"seq":19,"at_ms":1794852000000,"checkpoint":{"source":"fixture-runner","revision":19},"fact":{"kind":"blocked","data":{"subject":{"kind":"task","data":"task-blocked"},"code":"needs-interface","ask_id":"ask-blocked"}}}} @@ -23,5 +23,5 @@ {"type":"oxa_work_event","version":1,"data":{"seq":21,"at_ms":1794855600000,"checkpoint":{"source":"fixture-runner","revision":21},"fact":{"kind":"incident","data":{"id":"helper-idle","subject":{"kind":"host","data":"helper"},"kind":"host_idle","evidence":{"kind":"status","id":"idle-since-1900"}}}}} {"type":"oxa_work_event","version":1,"data":{"seq":22,"at_ms":1794862800000,"checkpoint":{"source":"fixture-runner","revision":22},"fact":{"kind":"mechanical","data":{"subject":{"kind":"host","data":"helper"},"kind":"refill"}}}} {"type":"oxa_work_situation","version":1,"data":{"as_of_seq":22,"at_ms":1794862800000,"standing_revision":1,"goal":"Finish the synthetic evening","running":[],"local_accepted":[{"task":"task-landed","commit":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","evidence":{"kind":"local_check","id":"local-landed"}},{"task":"task-red","commit":"cccccccccccccccccccccccccccccccccccccccc","evidence":{"kind":"local_check","id":"local-red"}}],"published":[{"task":"task-landed","pr":{"number":101,"head":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}},{"task":"task-red","pr":{"number":102,"head":"cccccccccccccccccccccccccccccccccccccccc"}}],"landed":[{"pr":{"number":101,"head":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"merge_commit":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","merged_at_ms":1794848700000}],"blocked":[{"subject":{"kind":"task","data":"task-blocked"},"code":"needs-interface","ask_id":"ask-blocked"},{"subject":{"kind":"pull_request","data":102},"code":"held","ask_id":null}],"asks":[{"id":"ask-blocked","subject":{"kind":"task","data":"task-blocked"},"question":"Which synthetic interface should the task target?"}],"incidents":[{"id":"red-queue","subject":{"kind":"pull_request","data":102},"kind":"queue_rejected","evidence":{"kind":"gate","id":"gate-red"}},{"id":"helper-idle","subject":{"kind":"host","data":"helper"},"kind":"host_idle","evidence":{"kind":"status","id":"idle-since-1900"}}],"history_truncated":false}} -{"type":"oxa_work_events","version":1,"data":{"after":10,"events":[{"seq":14,"at_ms":1794850500000,"checkpoint":{"source":"fixture-runner","revision":14},"fact":{"kind":"external_acceptance","data":{"pr":{"number":102,"head":"cccccccccccccccccccccccccccccccccccccccc"},"evidence":{"kind":"queue","id":"merge-group/dddddddddddddddddddddddddddddddddddddddd"}}}},{"seq":15,"at_ms":1794850800000,"checkpoint":{"source":"fixture-runner","revision":15},"fact":{"kind":"incident","data":{"id":"red-queue","subject":{"kind":"pull_request","data":102},"kind":"queue_rejected","evidence":{"kind":"gate","id":"gate-red"}}}},{"seq":16,"at_ms":1794851100000,"checkpoint":{"source":"fixture-runner","revision":16},"fact":{"kind":"task_started","data":{"task":"task-blocked","host":"helper","branch":"agent/fixture-task-blocked","worktree":"~/project/task-blocked","session_id":"session-task-blocked"}}},{"seq":17,"at_ms":1794851400000,"checkpoint":{"source":"fixture-runner","revision":17},"fact":{"kind":"worker_finished","data":{"task":"task-blocked","session_id":"session-task-blocked","outcome":"blocked","error_class":"needs-interface"}}},{"seq":18,"at_ms":1794851700000,"checkpoint":{"source":"fixture-runner","revision":18},"fact":{"kind":"ask_opened","data":{"id":"ask-blocked","subject":{"kind":"task","data":"task-blocked"},"question":"Which synthetic interface should the task target?"}}},{"seq":19,"at_ms":1794852000000,"checkpoint":{"source":"fixture-runner","revision":19},"fact":{"kind":"blocked","data":{"subject":{"kind":"task","data":"task-blocked"},"code":"needs-interface","ask_id":"ask-blocked"}}},{"seq":20,"at_ms":1794852300000,"checkpoint":{"source":"fixture-runner","revision":20},"fact":{"kind":"blocked","data":{"subject":{"kind":"pull_request","data":102},"code":"held","ask_id":null}}},{"seq":21,"at_ms":1794855600000,"checkpoint":{"source":"fixture-runner","revision":21},"fact":{"kind":"incident","data":{"id":"helper-idle","subject":{"kind":"host","data":"helper"},"kind":"host_idle","evidence":{"kind":"status","id":"idle-since-1900"}}}},{"seq":22,"at_ms":1794862800000,"checkpoint":{"source":"fixture-runner","revision":22},"fact":{"kind":"mechanical","data":{"subject":{"kind":"host","data":"helper"},"kind":"refill"}}}],"next":22,"first_available":14,"gap":true}} +{"type":"oxa_work_events","version":1,"data":{"after":10,"events":[{"seq":14,"at_ms":1794850500000,"checkpoint":{"source":"fixture-runner","revision":14},"fact":{"kind":"external_acceptance","data":{"pr":{"number":102,"head":"cccccccccccccccccccccccccccccccccccccccc"},"evidence":{"kind":"queue","id":"merge-group/dddddddddddddddddddddddddddddddddddddddd"}}}},{"seq":15,"at_ms":1794850800000,"checkpoint":{"source":"fixture-runner","revision":15},"fact":{"kind":"incident","data":{"id":"red-queue","subject":{"kind":"pull_request","data":102},"kind":"queue_rejected","evidence":{"kind":"gate","id":"gate-red"}}}},{"seq":16,"at_ms":1794851100000,"checkpoint":{"source":"fixture-runner","revision":16},"fact":{"kind":"task_started","data":{"task":"task-blocked","host":"helper","branch":"agent/fixture-task-blocked","worktree":"~/project/task-blocked","session_id":"session-task-blocked","step":"preparing"}}},{"seq":17,"at_ms":1794851400000,"checkpoint":{"source":"fixture-runner","revision":17},"fact":{"kind":"worker_finished","data":{"task":"task-blocked","session_id":"session-task-blocked","outcome":"blocked","error_class":"needs-interface"}}},{"seq":18,"at_ms":1794851700000,"checkpoint":{"source":"fixture-runner","revision":18},"fact":{"kind":"ask_opened","data":{"id":"ask-blocked","subject":{"kind":"task","data":"task-blocked"},"question":"Which synthetic interface should the task target?"}}},{"seq":19,"at_ms":1794852000000,"checkpoint":{"source":"fixture-runner","revision":19},"fact":{"kind":"blocked","data":{"subject":{"kind":"task","data":"task-blocked"},"code":"needs-interface","ask_id":"ask-blocked"}}},{"seq":20,"at_ms":1794852300000,"checkpoint":{"source":"fixture-runner","revision":20},"fact":{"kind":"blocked","data":{"subject":{"kind":"pull_request","data":102},"code":"held","ask_id":null}}},{"seq":21,"at_ms":1794855600000,"checkpoint":{"source":"fixture-runner","revision":21},"fact":{"kind":"incident","data":{"id":"helper-idle","subject":{"kind":"host","data":"helper"},"kind":"host_idle","evidence":{"kind":"status","id":"idle-since-1900"}}}},{"seq":22,"at_ms":1794862800000,"checkpoint":{"source":"fixture-runner","revision":22},"fact":{"kind":"mechanical","data":{"subject":{"kind":"host","data":"helper"},"kind":"refill"}}}],"next":22,"first_available":14,"gap":true}} {"type":"oxa_work_event","version":1,"data":{"seq":23,"at_ms": \ No newline at end of file From 92e8fe9ebd480d6c6e1cb866ce4c819f702c800e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:58:56 +0000 Subject: [PATCH 14/27] test(work): reproduce stranded standing recovery after failed fact reservation --- oxa/oxa-work/src/standing/store/tests.rs | 86 ++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/oxa/oxa-work/src/standing/store/tests.rs b/oxa/oxa-work/src/standing/store/tests.rs index 927a8acd0..55bb0ca83 100644 --- a/oxa/oxa-work/src/standing/store/tests.rs +++ b/oxa/oxa-work/src/standing/store/tests.rs @@ -430,6 +430,92 @@ fn a_late_answer_is_retained_even_when_its_question_is_the_oldest() { .is_some()); } +#[test] +fn a_failed_fact_reservation_cannot_strand_policy_at_projection_capacity() { + use std::os::unix::fs::PermissionsExt; + // SAFETY: geteuid takes no arguments and cannot fail. + if unsafe { libc::geteuid() } == 0 { + use std::os::unix::process::CommandExt; + let result = std::process::Command::new(std::env::current_exe().unwrap()) + .env_clear() + .args([ + "--exact", + "standing::store::tests::a_failed_fact_reservation_cannot_strand_policy_at_projection_capacity", + ]) + .uid(65534) + .gid(65534) + .output() + .unwrap(); + assert!( + result.status.success(), + "{}{}", + String::from_utf8_lossy(&result.stdout), + String::from_utf8_lossy(&result.stderr) + ); + return; + } + let fixture = Fixture::new(); + let facts = fixture.facts(); + let standing = fixture.standing(facts.clone()); + standing.rule(0, Authority::Owner, 1, hold(true, None)).unwrap(); + let existing = standing.ask(1, 2, question()).unwrap(); + for revision in 1..=126 { + facts + .append( + Checkpoint { + source: "runner".into(), + revision, + }, + revision + 10, + Fact::TaskStarted(oxa_proto::work::Running { + task: format!("busy-{revision}"), + host: "local".into(), + branch: format!("agent/busy-{revision}"), + worktree: "/tmp/synthetic".into(), + session_id: None, + step: oxa_proto::work::TaskStep::Preparing, + }), + ) + .unwrap(); + } + let root = fixture.0.join("facts"); + fs::set_permissions(&root, fs::Permissions::from_mode(0o500)).unwrap(); + let failed = standing.ask(2, 500, question()); + fs::set_permissions(&root, fs::Permissions::from_mode(0o700)).unwrap(); + assert!(failed.is_err(), "facts reservation unexpectedly succeeded"); + facts + .append( + Checkpoint { + source: "runner".into(), + revision: 127, + }, + 600, + Fact::TaskStarted(oxa_proto::work::Running { + task: "last-slot".into(), + host: "local".into(), + branch: "agent/last-slot".into(), + worktree: "/tmp/synthetic".into(), + session_id: None, + step: oxa_proto::work::TaskStep::Preparing, + }), + ) + .unwrap(); + let read = standing + .read() + .expect("a failed reservation stranded every standing operation"); + assert_eq!(read.revision, 2); + assert_eq!(read.open_asks().count(), 1); + standing + .answer(2, Authority::Owner, 700, answer(&existing)) + .unwrap(); + assert!(standing.read().unwrap().open_asks().next().is_none()); + assert_eq!(facts.situation().unwrap().running.len(), 127); + assert!(matches!( + standing.with_guard(Action::Update, subject(), |_| Ok(())), + Err(GuardError::Refused(_)) + )); +} + #[test] fn recovery_syncs_the_prepared_policy_before_committing_its_fact() { use std::os::unix::fs::PermissionsExt; From b084de5c05d7f6758c9630b422ab75b7d9b8543b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 16:11:39 +0000 Subject: [PATCH 15/27] fix(work): reserve standing projection capacity before preparing policy --- oxa/docs/standing.md | 24 ++- oxa/oxa-work/src/events.rs | 180 ++++++++++++++++------- oxa/oxa-work/src/events/tests.rs | 91 ++++++++++++ oxa/oxa-work/src/standing/store.rs | 13 +- oxa/oxa-work/src/standing/store/tests.rs | 51 +++++++ 5 files changed, 293 insertions(+), 66 deletions(-) diff --git a/oxa/docs/standing.md b/oxa/docs/standing.md index a187e7ac8..768b65020 100644 --- a/oxa/docs/standing.md +++ b/oxa/docs/standing.md @@ -105,13 +105,23 @@ One fixed `standing` emitter uses the document revision. `StandingChanged` proje goal, one ask transition and an optional incident. Full rule/option/answer data stays in this protected file. It does not emit two different facts under one source/revision. -Under Standing → Facts lock order, the facts module first validates capacity/semantics, then -the prepare callback durably saves the pending policy, before reserving/appending its fact. -Only after that fact succeeds is the current policy committed. A rejected projection leaves -no pending standing revision: the owner can still close an old ask to free capacity. Recovery -syncs a visible prepared policy before committing its fact, replays its identity once, commits -the policy and establishes its durability before another action. A failed observation stops -before an effect. Readers never rewrite runner task state. +Under Standing → Facts lock order, the facts module validates capacity/semantics and durably +reserves an unpublished observation **before** the callback saves the pending policy. Other +appenders cannot consume its projection capacity, even if the process dies in the callback. +Only after the policy is durable is the reservation made replayable and its fact appended; +the current policy is committed last. Ordinary facts keep their existing append path. + +Recovery syncs a visible prepared policy, completes its reserved fact once, commits the policy +and establishes durability before another action. If the durable policy has no pending +mutation, a matching unpublished next-revision reservation has no prepared data: recovery +discards only that reservation, preserving the current rules, revision and committed facts. +A rejected initial projection leaves no policy mutation. A failed fact reservation therefore +cannot strand an ask; the owner can still answer an existing ask to free capacity. + +Recover standing before replaying runner callbacks that append facts. A facts-only read reports +the last committed sequence; it never publishes an unfinished standing preparation. The +standing read/guard APIs perform this recovery under their stable lock. No background worker +or task-state writer is added, and a failed observation stops before an effect. The runner owns `oxa work rule|hold|ask`, actual action guards and local command proofs. The lead joins detailed asks by ID from `Read` with the small shared situation. These adapters diff --git a/oxa/oxa-work/src/events.rs b/oxa/oxa-work/src/events.rs index d6d8e3454..e99e8f8fe 100644 --- a/oxa/oxa-work/src/events.rs +++ b/oxa/oxa-work/src/events.rs @@ -23,12 +23,21 @@ struct State { last_by_source: Vec, // Write-ahead observation: a crash can replay it even if the journal line was torn. pending: Option, + // A cross-store reservation is unpublished until its owner syncs prepared data. + #[serde(default, skip_serializing_if = "not_preparing")] + preparing: bool, +} +fn not_preparing(value: &bool) -> bool { + !value } impl State { fn sources(&self) -> Result> { if self.version != 1 || self.last_by_source.len() > MAX_SOURCES { return Err("unsupported facts metadata or source capacity".into()); } + if self.preparing && self.pending.is_none() { + return Err("facts preparation has no reserved observation".into()); + } let events: Vec = self .last_by_source .iter() @@ -59,6 +68,7 @@ impl State { self.last_seq = event.seq; self.last_by_source = sources.iter().map(event_value).collect::>()?; self.pending = None; + self.preparing = false; Ok(()) } fn bytes(&self) -> Result> { @@ -136,10 +146,11 @@ impl Facts { /// Stop before the next action if this call fails. Replaying the same checkpoint repairs /// a lost acknowledgement, a torn last line or an interrupted snapshot replacement. pub fn append(&self, checkpoint: Checkpoint, at_ms: u64, fact: Fact) -> Result { - self.append_prepared(checkpoint, at_ms, fact, |_| Ok(())) + self.append_with(checkpoint, at_ms, fact, None:: Result<()>>) } - /// Validate capacity under the facts lock, then let another protected store reserve its - /// mutation before the facts reservation. Never calls preparation for an existing identity. + /// Durably reserve capacity before the protected store prepares its mutation. A failed + /// preparation blocks other appenders until that store commits or cancels the reservation. + /// Never calls preparation for an already committed identity. pub(crate) fn append_prepared( &self, checkpoint: Checkpoint, @@ -147,31 +158,56 @@ impl Facts { fact: Fact, prepare: impl FnOnce(&Event) -> Result<()>, ) -> Result { + self.append_with(checkpoint, at_ms, fact, Some(prepare)) + } + fn append_with( + &self, + checkpoint: Checkpoint, + at_ms: u64, + fact: Fact, + prepare: Option Result<()>>, + ) -> Result { + if prepare.is_some() && !matches!(fact, Fact::StandingChanged(_)) { + return Err("only standing observations use protected preparation".into()); + } let _lock = self.root.lock(LOCK)?; let (mut state, mut snapshot, journal) = self.recover()?; let sources = state.sources()?; - if let Some(previous) = sources - .iter() - .find(|event| event.checkpoint.source == checkpoint.source) - { - if checkpoint.revision == previous.checkpoint.revision { - return if previous.fact == fact { - Ok(previous.clone()) - } else { - Err("checkpoint identity has conflicting facts".into()) - }; + let reserved = state.preparing; + let event = if reserved { + if prepare.is_none() { + return Err("standing preparation must recover before another fact".into()); } - if checkpoint.revision < previous.checkpoint.revision { - return Err("stale facts checkpoint".into()); + let event = read_event_value(state.pending.as_ref().ok_or("missing reserved preparation")?)?; + if event.checkpoint != checkpoint || event.at_ms != at_ms || event.fact != fact { + return Err("another standing preparation is reserved".into()); + } + self.root.durable(STATE)?; + event + } else { + if let Some(previous) = sources + .iter() + .find(|event| event.checkpoint.source == checkpoint.source) + { + if checkpoint.revision == previous.checkpoint.revision { + return if previous.fact == fact { + Ok(previous.clone()) + } else { + Err("checkpoint identity has conflicting facts".into()) + }; + } + if checkpoint.revision < previous.checkpoint.revision { + return Err("stale facts checkpoint".into()); + } + } else if sources.len() == MAX_SOURCES { + return Err("facts source capacity exceeded".into()); + } + Event { + seq: state.last_seq.checked_add(1).ok_or("facts sequence exhausted")?, + at_ms, + checkpoint, + fact, } - } else if sources.len() == MAX_SOURCES { - return Err("facts source capacity exceeded".into()); - } - let event = Event { - seq: state.last_seq.checked_add(1).ok_or("facts sequence exhausted")?, - at_ms, - checkpoint, - fact, }; let mut bytes = event.to_json().map_err(|error| error.to_string())?; bytes.push(b'\n'); @@ -180,9 +216,16 @@ impl Facts { } // Fail capacity/semantic checks before durably reserving an observation. situation::apply(&mut snapshot, &event)?; - prepare(&event)?; - state.pending = Some(event_value(&event)?); - self.root.atomic(STATE, &state.bytes()?)?; + if !reserved { + state.pending = Some(event_value(&event)?); + state.preparing = prepare.is_some(); + self.root.atomic(STATE, &state.bytes()?)?; + } + if let Some(prepare) = prepare { + prepare(&event)?; + state.preparing = false; + self.root.atomic(STATE, &state.bytes()?)?; + } if journal.current_bytes + bytes.len() > self.journal_bytes { self.root.rotate(CURRENT, PREVIOUS)?; } @@ -193,6 +236,27 @@ impl Facts { .atomic(SITUATION, &snapshot.to_json().map_err(|error| error.to_string())?)?; Ok(event) } + /// The standing lock is held and its durable document has no pending mutation. A matching + /// next-revision reservation therefore has no prepared data and can be discarded safely. + pub(crate) fn cancel_unprepared_standing(&self, revision: u64, goal: Option<&str>) -> Result { + let _lock = self.root.lock(LOCK)?; + let (mut state, snapshot, _) = self.recover()?; + if snapshot.standing_revision != revision || snapshot.goal.as_deref() != goal { + return Err("standing file disagrees with facts revision".into()); + } + if state.preparing { + let event = read_event_value(state.pending.as_ref().ok_or("missing reserved preparation")?)?; + if event.checkpoint.source != "standing" + || revision.checked_add(1) != Some(event.checkpoint.revision) + { + return Err("standing preparation disagrees with current policy".into()); + } + state.pending = None; + state.preparing = false; + self.root.atomic(STATE, &state.bytes()?)?; + } + Ok(snapshot) + } pub fn situation(&self) -> Result { let _lock = self.root.lock(LOCK)?; self.recover().map(|(_, snapshot, _)| snapshot) @@ -257,7 +321,7 @@ impl Facts { .map_err(|_| "invalid facts metadata")?; let sources = state.sources()?; let pending = state.pending.as_ref().map(read_event_value).transpose()?; - let mut journal = self.journal(pending.is_some())?; + let mut journal = self.journal(pending.is_some() && !state.preparing)?; let mut snapshot = Snapshot::from_json(&self.root.read(SITUATION, MAX_SNAPSHOT_BYTES)?) .map_err(|error| error.to_string())?; let original_seq = snapshot.as_of_seq; @@ -287,35 +351,47 @@ impl Facts { { return Err("invalid pending facts checkpoint".into()); } - situation::apply(&mut snapshot, &event)?; - if journal.last_seq() == state.last_seq { - if let Some(length) = journal.torn_at { - self.root.truncate(CURRENT, length)?; - journal.current_bytes = length as usize; + if state.preparing { + if !matches!(event.fact, Fact::StandingChanged(_)) + || journal.last_seq() != state.last_seq + || journal.torn_at.is_some() + { + return Err("invalid unpublished standing preparation".into()); } - let mut bytes = event.to_json().map_err(|error| error.to_string())?; - bytes.push(b'\n'); - if bytes.len() > self.journal_bytes { - return Err("event exceeds journal capacity".into()); - } - if journal.current_bytes + bytes.len() > self.journal_bytes { - self.root.rotate(CURRENT, PREVIOUS)?; - } - self.root.append(CURRENT, &bytes)?; - } else if journal.events.last() != Some(&event) || journal.torn_at.is_some() { - return Err("pending fact disagrees with the journal".into()); + // Reads report committed facts only. Append cannot consume this capacity; + // the owning standing store must first commit or cancel its preparation. + situation::apply(&mut snapshot.clone(), &event)?; } else { - // Visibility is not durability: the original append may have died before sync. - let leaf = if journal.current_bytes == 0 { - PREVIOUS + situation::apply(&mut snapshot, &event)?; + if journal.last_seq() == state.last_seq { + if let Some(length) = journal.torn_at { + self.root.truncate(CURRENT, length)?; + journal.current_bytes = length as usize; + } + let mut bytes = event.to_json().map_err(|error| error.to_string())?; + bytes.push(b'\n'); + if bytes.len() > self.journal_bytes { + return Err("event exceeds journal capacity".into()); + } + if journal.current_bytes + bytes.len() > self.journal_bytes { + self.root.rotate(CURRENT, PREVIOUS)?; + } + self.root.append(CURRENT, &bytes)?; + } else if journal.events.last() != Some(&event) || journal.torn_at.is_some() { + return Err("pending fact disagrees with the journal".into()); } else { - CURRENT - }; - self.root.append(leaf, b"")?; + // Visibility is not durability: the original append may have died before sync. + let leaf = if journal.current_bytes == 0 { + PREVIOUS + } else { + CURRENT + }; + self.root.append(leaf, b"")?; + } + state.committed(&event)?; + self.root.atomic(STATE, &state.bytes()?)?; + journal = self.journal(false)?; } - state.committed(&event)?; - self.root.atomic(STATE, &state.bytes()?)?; - journal = self.journal(false)?; } else if journal.last_seq() != state.last_seq { return Err("journal disagrees with committed facts".into()); } diff --git a/oxa/oxa-work/src/events/tests.rs b/oxa/oxa-work/src/events/tests.rs index 420a1e650..2302ab8ab 100644 --- a/oxa/oxa-work/src/events/tests.rs +++ b/oxa/oxa-work/src/events/tests.rs @@ -70,6 +70,97 @@ fn reserve(facts: &Facts, fact: Fact) -> Event { event } +#[test] +fn a_crashed_preparation_keeps_its_projection_capacity_and_commits_once() { + const CHILD: &str = "OXA_FACTS_PREPARATION_CHILD"; + let observation = || { + Fact::StandingChanged(StandingChanged { + revision: 1, + goal: None, + ask_opened: Some(Ask { + id: "prepared-ask".into(), + subject: Subject::Repository, + question: "Choose an option".into(), + }), + ask_closed: None, + incident: None, + }) + }; + if let Some(root) = std::env::var_os(CHILD) { + let facts = Facts::open(Path::new(&root).join("facts").as_path()).unwrap(); + let policy = Root::open(Path::new(&root), false).unwrap(); + facts + .append_prepared(checkpoint("standing", 1), 500, observation(), |event| { + policy.atomic("prepared.json", &event.to_json().unwrap()).unwrap(); + std::process::exit(73); + }) + .unwrap(); + panic!("preparation did not reach its crash boundary"); + } + let fixture = Fixture::new(); + let facts = fixture.facts(); + for revision in 1..=127 { + facts + .append( + checkpoint("runner", revision), + revision, + Fact::TaskStarted(task(revision as usize)), + ) + .unwrap(); + } + let child = std::process::Command::new(std::env::current_exe().unwrap()) + .env_clear() + .env(CHILD, &fixture.0) + .args([ + "--exact", + "events::tests::a_crashed_preparation_keeps_its_projection_capacity_and_commits_once", + ]) + .output() + .unwrap(); + assert_eq!( + child.status.code(), + Some(73), + "{}", + String::from_utf8_lossy(&child.stderr) + ); + let recovered = Facts::open(&fixture.root()).unwrap(); + assert_eq!(recovered.situation().unwrap().as_of_seq, 127); + assert!(recovered + .append(checkpoint("runner", 128), 600, Fact::TaskStarted(task(128))) + .is_err()); + assert_eq!(recovered.events_after(127, 64).unwrap().events.len(), 0); + let committed = recovered + .append_prepared(checkpoint("standing", 1), 500, observation(), |event| { + assert_eq!( + fs::read(fixture.0.join("prepared.json")).unwrap(), + event.to_json().unwrap() + ); + Ok(()) + }) + .unwrap(); + assert_eq!(committed.seq, 128); + assert_eq!(recovered.situation().unwrap().asks.len(), 1); + let again = recovered + .append_prepared(checkpoint("standing", 1), 500, observation(), |_| { + panic!("prepared twice") + }) + .unwrap(); + assert_eq!(again, committed); + recovered + .append( + checkpoint("cleanup", 1), + 700, + Fact::AskClosed(AskClosed { + id: "prepared-ask".into(), + }), + ) + .unwrap(); + recovered + .append(checkpoint("runner", 128), 800, Fact::TaskStarted(task(128))) + .unwrap(); + assert_eq!(recovered.situation().unwrap().running.len(), 128); +} + #[test] fn lifecycle_facts_do_not_turn_worker_success_into_landed_work() { let env = Fixture::new(); diff --git a/oxa/oxa-work/src/standing/store.rs b/oxa/oxa-work/src/standing/store.rs index 41123db0d..120c59c53 100644 --- a/oxa/oxa-work/src/standing/store.rs +++ b/oxa/oxa-work/src/standing/store.rs @@ -317,19 +317,18 @@ impl Standing { // A prior rename may be visible after directory sync failed. Establish the // prepared policy's durability before committing its separately stored fact. self.root.durable(FILE)?; - self.facts.append(event.checkpoint, event.at_ms, event.fact)?; + self.facts + .append_prepared(event.checkpoint, event.at_ms, event.fact, |_| Ok(()))?; document.current = pending.after.clone(); document.pending = None; self.root.atomic(FILE, &document.bytes()?)?; } // Establish durability after a prior post-rename sync failure before authorizing work. self.root.durable(FILE)?; - let situation = self.facts.situation()?; - if situation.standing_revision != document.current.revision - || situation.goal != document.current.change(None).goal - { - return Err("standing file disagrees with facts revision".into()); - } + let situation = self.facts.cancel_unprepared_standing( + document.current.revision, + document.current.change(None).goal.as_deref(), + )?; Ok((document.current, situation)) } fn save(&self, before: Read, mut after: Read, incident: Option) -> Result { diff --git a/oxa/oxa-work/src/standing/store/tests.rs b/oxa/oxa-work/src/standing/store/tests.rs index 55bb0ca83..2fe0f5149 100644 --- a/oxa/oxa-work/src/standing/store/tests.rs +++ b/oxa/oxa-work/src/standing/store/tests.rs @@ -516,6 +516,57 @@ fn a_failed_fact_reservation_cannot_strand_policy_at_projection_capacity() { )); } +#[test] +fn failed_policy_preparation_cancels_only_the_unpublished_reservation() { + use std::os::unix::fs::PermissionsExt; + // SAFETY: geteuid takes no arguments and cannot fail. + if unsafe { libc::geteuid() } == 0 { + use std::os::unix::process::CommandExt; + let result = std::process::Command::new(std::env::current_exe().unwrap()) + .env_clear() + .args([ + "--exact", + "standing::store::tests::failed_policy_preparation_cancels_only_the_unpublished_reservation", + ]) + .uid(65534) + .gid(65534) + .output() + .unwrap(); + assert!( + result.status.success(), + "{}{}", + String::from_utf8_lossy(&result.stdout), + String::from_utf8_lossy(&result.stderr) + ); + return; + } + let fixture = Fixture::new(); + let facts = fixture.facts(); + let standing = fixture.standing(facts.clone()); + let held = standing.rule(0, Authority::Owner, 1, hold(true, None)).unwrap(); + let root = fixture.0.join("standing"); + fs::set_permissions(&root, fs::Permissions::from_mode(0o500)).unwrap(); + let failed = standing.rule(1, Authority::Owner, 2, hold(false, Some(&held.id))); + fs::set_permissions(&root, fs::Permissions::from_mode(0o700)).unwrap(); + assert!(failed.is_err()); + assert_eq!( + facts.situation().unwrap().standing_revision, + 1, + "unprepared policy was published" + ); + let reopened = Standing::open(&root, Facts::open(&fixture.0.join("facts")).unwrap()).unwrap(); + assert_eq!(reopened.read().unwrap().revision, 1); + assert!(matches!( + reopened.with_guard(Action::Update, subject(), |_| Ok(())), + Err(GuardError::Refused(_)) + )); + reopened + .rule(1, Authority::Owner, 3, hold(false, Some(&held.id))) + .unwrap(); + assert!(reopened.is_held(480).unwrap().is_none()); + assert_eq!(facts.situation().unwrap().standing_revision, 2); +} + #[test] fn recovery_syncs_the_prepared_policy_before_committing_its_fact() { use std::os::unix::fs::PermissionsExt; From c255237f6aa2229c270079da73058d3dcdbffb1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 16:48:00 +0000 Subject: [PATCH 16/27] test(work): reproduce oversized event pages that cannot advance --- oxa/oxa-work/src/events/tests.rs | 69 ++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/oxa/oxa-work/src/events/tests.rs b/oxa/oxa-work/src/events/tests.rs index 2302ab8ab..e492eedec 100644 --- a/oxa/oxa-work/src/events/tests.rs +++ b/oxa/oxa-work/src/events/tests.rs @@ -437,6 +437,75 @@ fn rotation_is_bounded_and_cursor_gaps_are_explicit_without_resetting_sequences( assert!(facts.events_after(0, 0).is_err()); } +#[test] +fn large_valid_events_page_by_bytes_and_keep_advancing_across_retention_gaps() { + let fixture = Fixture::new(); + let facts = fixture.facts(); + let observation = |revision| { + Fact::StandingChanged(StandingChanged { + revision, + goal: Some("\"".repeat(1024)), + ask_opened: Some(Ask { + id: "ask".repeat(32), + subject: Subject::Repository, + question: "\"".repeat(512), + }), + ask_closed: None, + incident: Some(Incident { + id: "incident".repeat(32), + subject: Subject::Task("t".repeat(128)), + kind: IncidentKind::RuleReminder, + evidence: Some(Evidence { + kind: EvidenceKind::LocalCheck, + id: "e".repeat(256), + }), + }), + }) + }; + for revision in 1..=64 { + let event = facts + .append(checkpoint("standing", revision), revision, observation(revision)) + .unwrap(); + assert!(event.to_json().unwrap().len() <= MAX_EVENT_BYTES); + } + let first = facts + .events_after(0, 64) + .expect("healthy large events must return a usable bounded page"); + assert!(!first.gap); + assert!(!first.events.is_empty() && first.events.len() < 64); + let mut cursor = 0; + let mut seen = Vec::new(); + while cursor < 64 { + let page = facts.events_after(cursor, 64).unwrap(); + assert!(page.to_json().unwrap().len() <= oxa_proto::work::MAX_PAGE_BYTES); + assert_eq!(page.next, page.events.last().unwrap().seq); + assert!(page.next > cursor); + seen.extend(page.events.iter().map(|event| event.seq)); + cursor = page.next; + } + assert_eq!(seen, (1..=64).collect::>()); + for revision in 65..=180 { + facts + .append(checkpoint("standing", revision), revision, observation(revision)) + .unwrap(); + } + let page = facts.events_after(0, 64).unwrap(); + assert!(page.gap && page.first_available > 1); + let first_retained = page.first_available; + cursor = first_retained - 1; + seen.clear(); + while cursor < 180 { + let page = facts.events_after(cursor, 64).unwrap(); + assert!(page.to_json().unwrap().len() <= oxa_proto::work::MAX_PAGE_BYTES); + assert!(page.next > cursor); + assert_eq!(page.next, page.events.last().unwrap().seq); + seen.extend(page.events.iter().map(|event| event.seq)); + cursor = page.next; + } + assert_eq!(seen, (first_retained..=180).collect::>()); + assert!(facts.events_after(180, 64).unwrap().events.is_empty()); +} + #[test] fn concurrent_sources_get_unique_contiguous_repository_sequences() { let env = Fixture::new(); From 01ea760574a7016168b80877f9257823e6a53ca6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 16:49:55 +0000 Subject: [PATCH 17/27] fix(work): bound event pages by wire size and preserve cursor progress --- oxa/docs/standing.md | 4 ++++ oxa/oxa-work/src/events.rs | 45 +++++++++++++++++++++++++++----------- 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/oxa/docs/standing.md b/oxa/docs/standing.md index 768b65020..d76629113 100644 --- a/oxa/docs/standing.md +++ b/oxa/docs/standing.md @@ -105,6 +105,10 @@ One fixed `standing` emitter uses the document revision. `StandingChanged` proje goal, one ask transition and an optional incident. Full rule/option/answer data stays in this protected file. It does not emit two different facts under one source/revision. +Event pages honor both the requested count (at most 64) and the serialized byte limit. +Large valid observations return smaller pages with `next` at the last included event; +readers keep advancing through retained history and receive explicit retention gaps. + Under Standing → Facts lock order, the facts module validates capacity/semantics and durably reserves an unpublished observation **before** the callback saves the pending policy. Other appenders cannot consume its projection capacity, even if the process dies in the callback. diff --git a/oxa/oxa-work/src/events.rs b/oxa/oxa-work/src/events.rs index e99e8f8fe..f5dbcae86 100644 --- a/oxa/oxa-work/src/events.rs +++ b/oxa/oxa-work/src/events.rs @@ -1,7 +1,9 @@ //! Durable facts journal and its bounded read view. Never opens runner task-state files. use crate::facts_io::Root; use crate::{situation, Result}; -use oxa_proto::work::{Checkpoint, Event, EventPage, Fact, Snapshot, MAX_EVENT_BYTES, MAX_SNAPSHOT_BYTES}; +use oxa_proto::work::{ + Checkpoint, Event, EventPage, Fact, Snapshot, MAX_EVENT_BYTES, MAX_PAGE_BYTES, MAX_SNAPSHOT_BYTES, +}; use serde::{Deserialize, Serialize}; use serde_json::Value; use std::path::Path; @@ -271,22 +273,39 @@ impl Facts { return Err("event cursor is ahead of the journal".into()); } let first_available = journal.events.first().map_or(1, |event| event.seq); - let events: Vec<_> = journal - .events - .into_iter() - .filter(|event| event.seq > after) - .take(limit) - .collect(); - let next = events - .last() - .map_or(after.max(first_available - 1), |event| event.seq); - let page = EventPage { + let mut page = EventPage { after, - events, - next, + events: Vec::new(), + next: after.max(first_available - 1), first_available, gap: after.saturating_add(1) < first_available, }; + // The envelope's next cursor can grow to last_seq. An event's standalone wire + // encoding bounds its nested encoding; counting one comma per event is conservative. + // Encode each candidate once instead of repeatedly serializing a growing page. + let mut bytes = page.to_json().map_err(|error| error.to_string())?.len() + + state + .last_seq + .to_string() + .len() + .saturating_sub(page.next.to_string().len()); + for event in journal + .events + .into_iter() + .filter(|event| event.seq > after) + .take(limit) + { + let event_bytes = event.to_json().map_err(|error| error.to_string())?.len() + 1; + if bytes + event_bytes > MAX_PAGE_BYTES { + if page.events.is_empty() { + return Err("event exceeds bounded page capacity".into()); + } + break; + } + bytes += event_bytes; + page.next = event.seq; + page.events.push(event); + } page.to_json().map_err(|error| error.to_string())?; Ok(page) } From 1372010f7001f515e1800702e8371234294b76b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 17:05:13 +0000 Subject: [PATCH 18/27] test(work): reproduce answered-ask trimming blocking owner closure --- oxa/oxa-work/src/standing/store/tests.rs | 102 +++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/oxa/oxa-work/src/standing/store/tests.rs b/oxa/oxa-work/src/standing/store/tests.rs index 2fe0f5149..bd43e3c57 100644 --- a/oxa/oxa-work/src/standing/store/tests.rs +++ b/oxa/oxa-work/src/standing/store/tests.rs @@ -350,6 +350,108 @@ fn answers_are_owner_bound_to_a_question_and_do_not_release_holds() { assert!(standing.answer(3, Authority::Owner, 4, answer(&ask)).is_err()); } +fn fill_view_with_open_asks(standing: &Standing) -> Vec { + standing.rule(0, Authority::Owner, 1, hold(true, None)).unwrap(); + let initial = standing.read().unwrap(); + let mut request = question(); + request.question = "\"".repeat(512); + request.meanwhile = Some("\"".repeat(512)); + request.source = super::super::Source::Command { + command: "\"".repeat(512), + }; + request.options.push(super::super::QuestionOption { + label: "Later".into(), + detail: String::new(), + }); + request.options.push(super::super::QuestionOption { + label: "Reconsider".into(), + detail: String::new(), + }); + for width in (1..=512).rev() { + for option in &mut request.options { + option.detail = "\"".repeat(width); + } + let mut projected = initial.clone(); + projected.revision = 41; + projected.at = 41; + projected.asks = (2..=41) + .map(|revision| Ask { + id: format!("A-{revision}"), + revision, + scope: request.scope.clone(), + question: request.question.clone(), + options: request.options.clone(), + recommended: request.recommended.clone(), + meanwhile: request.meanwhile.clone(), + at: revision, + source: request.source.clone(), + answer: None, + proposal: None, + }) + .collect(); + if serde_json::to_vec(&projected).unwrap().len() <= MAX_VIEW - 1024 { + break; + } + } + let asks: Vec<_> = (2..=41) + .map(|revision| standing.ask(revision - 1, revision, request.clone()).unwrap()) + .collect(); + let before = standing.read().unwrap(); + let mut projected = before.next(41, 42).unwrap(); + let mut rule = Rule { + kind: Kind::Note, + scope: Scope::Repository, + value: Value::Text("p".into()), + said: "Keep the owner instructions".into(), + source: super::super::Source::Command { + command: "note".into(), + }, + supersedes: None, + }; + Standing::stage_rule(&mut projected, Authority::Owner, 42, rule.clone()).unwrap(); + let padding = MAX_VIEW - 16 - serde_json::to_vec(&projected).unwrap().len(); + assert!(padding < 1024, "remaining padding: {padding}"); + rule.value = Value::Text("p".repeat(padding + 1)); + standing.rule(41, Authority::Owner, 42, rule).unwrap(); + let read = standing.read().unwrap(); + assert_eq!(serde_json::to_vec(&read).unwrap().len(), MAX_VIEW - 16); + assert!(!read.history_truncated); + assert_eq!(read.open_asks().count(), 40); + asks +} + +#[test] +fn answering_near_the_byte_limit_closes_the_ask_even_when_its_history_is_trimmed() { + let fixture = Fixture::new(); + let facts = fixture.facts(); + let standing = fixture.standing(facts.clone()); + let asks = fill_view_with_open_asks(&standing); + let cursor = facts.situation().unwrap().as_of_seq; + let answered = standing + .answer(42, Authority::Owner, 43, answer(&asks[0])) + .expect("the owner must be able to close an ask to free capacity"); + assert!(answered.answer.is_some()); + let reopened = Standing::open(&fixture.0.join("standing"), facts.clone()).unwrap(); + let (read, situation) = reopened.read_facts().unwrap(); + assert_eq!(read.revision, 43); + assert!(read.history_truncated); + assert_eq!(read.open_asks().count(), 39); + assert!(!read.asks.iter().any(|ask| ask.id == asks[0].id)); + assert_eq!(situation.standing_revision, 43); + assert_eq!(situation.asks.len(), 39); + assert!(!situation.asks.iter().any(|ask| ask.id == asks[0].id)); + let events = facts.events_after(cursor, 64).unwrap().events; + assert_eq!(events.len(), 1); + let Fact::StandingChanged(change) = &events[0].fact else { + panic!("standing change") + }; + assert_eq!(change.ask_closed.as_ref().unwrap().id, asks[0].id); + assert!(reopened.is_held(480).unwrap().is_some()); + assert!(reopened + .with_guard(Action::Update, subject(), |_| Ok(())) + .is_err()); +} + #[test] fn an_ask_capacity_refusal_does_not_reserve_a_stuck_standing_revision() { let fixture = Fixture::new(); From 7eda27f6fd2ee5382aa9950fa788ba2e22046d7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Nordstr=C3=B6m?= <13300531+n0rpan@users.noreply.github.com> Date: Mon, 16 Nov 2026 17:08:42 +0000 Subject: [PATCH 19/27] fix(work): preserve ask-close evidence across bounded history trimming --- oxa/docs/standing.md | 6 ++ oxa/oxa-work/src/standing/store.rs | 43 +++++++-- oxa/oxa-work/src/standing/store/tests.rs | 117 ++++++++++++++++++++++- 3 files changed, 159 insertions(+), 7 deletions(-) diff --git a/oxa/docs/standing.md b/oxa/docs/standing.md index d76629113..f92c478d6 100644 --- a/oxa/docs/standing.md +++ b/oxa/docs/standing.md @@ -20,6 +20,12 @@ excludes answered questions. Active rules and unanswered asks share a 128-item l 16 terminal records are retained; they may shrink to fit the 256 KiB read view. Pending metadata is capped at 1 MiB. Active data is never dropped to make room. +An owner can close a question even when its answer would exceed the view's byte budget. +The close observation is captured before answered history is trimmed. If that new answer +is immediately trimmed, pending metadata retains that one exact answered question until +the fact commits. Recovery validates its original question and answer revision; absent or +changed evidence cannot turn a missing unanswered question into a silent close. + Every mutation supplies the document revision it read. A stale revision refuses without changing the store. A live collision of kind/scope requires its exact `supersedes` ID. IDs are derived from the checked revision, not from model text or time. diff --git a/oxa/oxa-work/src/standing/store.rs b/oxa/oxa-work/src/standing/store.rs index 120c59c53..3d07e65b2 100644 --- a/oxa/oxa-work/src/standing/store.rs +++ b/oxa/oxa-work/src/standing/store.rs @@ -30,6 +30,9 @@ pub struct Read { struct Pending { after: Read, observation: serde_json::Value, + // Keep evidence of a close until its fact commits, even if bounded history drops it. + #[serde(default, skip_serializing_if = "Option::is_none")] + trimmed_answer: Option>, } #[derive(Deserialize, Serialize)] #[serde(deny_unknown_fields)] @@ -47,6 +50,28 @@ impl Document { Ok(bytes) } } +impl Pending { + fn difference(&self, before: &Read, incident: Option) -> Result { + let Some(closed) = self.trimmed_answer.as_deref() else { + return self.after.difference(before, incident); + }; + closed.validate()?; + let answer = closed.answer.as_ref().ok_or("trimmed ask has no owner answer")?; + let mut original = closed.clone(); + original.answer = None; + if !self.after.history_truncated + || self.after.asks.iter().any(|ask| ask.id == closed.id) + || !before.asks.contains(&original) + || answer.revision != self.after.revision + || answer.at != self.after.at + { + return Err("trimmed answer disagrees with the standing transition".into()); + } + let mut complete = self.after.clone(); + complete.asks.push(closed.clone()); + complete.difference(before, incident) + } +} impl Read { pub fn live_rules(&self) -> impl Iterator { self.entries.iter().filter(|entry| entry.is_live()) @@ -307,11 +332,7 @@ impl Standing { let Fact::StandingChanged(change) = &event.fact else { return Err("invalid standing observation kind".into()); }; - if change - != &pending - .after - .difference(&document.current, change.incident.clone())? - { + if change != &pending.difference(&document.current, change.incident.clone())? { return Err("pending standing projection disagrees".into()); } // A prior rename may be visible after directory sync failed. Establish the @@ -332,8 +353,17 @@ impl Standing { Ok((document.current, situation)) } fn save(&self, before: Read, mut after: Read, incident: Option) -> Result { + let change = after.difference(&before, incident)?; + let closed = change + .ask_closed + .as_ref() + .and_then(|closed| after.asks.iter().find(|ask| ask.id == closed.id)) + .cloned(); after.bounded()?; - let fact = Fact::StandingChanged(after.difference(&before, incident)?); + let trimmed_answer = closed + .filter(|closed| !after.asks.iter().any(|ask| ask.id == closed.id)) + .map(Box::new); + let fact = Fact::StandingChanged(change); let mut prepared = false; self.facts.append_prepared( Checkpoint { @@ -352,6 +382,7 @@ impl Standing { pending: Some(Pending { after: after.clone(), observation, + trimmed_answer, }), }; self.root.atomic(FILE, &document.bytes()?)?; diff --git a/oxa/oxa-work/src/standing/store/tests.rs b/oxa/oxa-work/src/standing/store/tests.rs index bd43e3c57..3bc2d201b 100644 --- a/oxa/oxa-work/src/standing/store/tests.rs +++ b/oxa/oxa-work/src/standing/store/tests.rs @@ -195,7 +195,11 @@ fn pending_revisions_recover_before_or_after_the_fact_without_duplicate_delivery let document = Document { version: 1, current: before, - pending: Some(Pending { after, observation }), + pending: Some(Pending { + after, + observation, + trimmed_answer: None, + }), }; standing.root.atomic(FILE, &document.bytes().unwrap()).unwrap(); let reopened = Standing::open(&fixture.0.join("standing"), facts.clone()).unwrap(); @@ -452,6 +456,116 @@ fn answering_near_the_byte_limit_closes_the_ask_even_when_its_history_is_trimmed .is_err()); } +#[test] +fn pending_trimmed_answers_recover_once_and_require_the_exact_prior_question() { + for phase in [ + "prepared", + "fact_committed", + "missing", + "changed", + "wrong_revision", + ] { + let fixture = Fixture::new(); + let facts = fixture.facts(); + let standing = fixture.standing(facts.clone()); + let asks = fill_view_with_open_asks(&standing); + let before = standing.read().unwrap(); + let cursor = facts.situation().unwrap().as_of_seq; + let mut after = before.next(42, 43).unwrap(); + let request = answer(&asks[0]); + after.asks[0].answer = Some(Answer { + revision: 43, + question_revision: request.question_revision, + choice: request.choice, + at: 43, + source: request.source, + said: request.said, + }); + let closed = after.asks[0].clone(); + let fact = Fact::StandingChanged(after.difference(&before, None).unwrap()); + after.bounded().unwrap(); + assert!(!after.asks.iter().any(|ask| ask.id == closed.id)); + // Persist the real reserved/prepared boundary, then interrupt before publication. + let interrupted = facts.append_prepared( + Checkpoint { + source: "standing".into(), + revision: 43, + }, + 43, + fact.clone(), + |event| { + let pending = Pending { + after: after.clone(), + observation: serde_json::from_slice(&event.to_json().unwrap()).unwrap(), + trimmed_answer: Some(Box::new(closed.clone())), + }; + let document = Document { + version: 1, + current: before.clone(), + pending: Some(pending), + }; + standing.root.atomic(FILE, &document.bytes()?)?; + Err("interrupted after preparing the owner answer".into()) + }, + ); + assert!(interrupted.is_err()); + assert_eq!(facts.situation().unwrap().as_of_seq, cursor); + if phase == "fact_committed" { + facts + .append_prepared( + Checkpoint { + source: "standing".into(), + revision: 43, + }, + 43, + fact, + |_| Ok(()), + ) + .unwrap(); + } + let valid_pending = standing.root.read(FILE, MAX_FILE).unwrap(); + if matches!(phase, "missing" | "changed" | "wrong_revision") { + let mut document: Document = serde_json::from_slice(&valid_pending).unwrap(); + let pending = document.pending.as_mut().unwrap(); + match phase { + "missing" => pending.trimmed_answer = None, + "changed" => pending.trimmed_answer.as_mut().unwrap().question = "different question".into(), + "wrong_revision" => { + pending + .trimmed_answer + .as_mut() + .unwrap() + .answer + .as_mut() + .unwrap() + .revision += 1 + } + _ => unreachable!(), + } + standing.root.atomic(FILE, &document.bytes().unwrap()).unwrap(); + assert!(Standing::open(&fixture.0.join("standing"), facts.clone()).is_err()); + assert!(matches!( + standing.with_guard(Action::Update, subject(), |_| Ok(())), + Err(GuardError::Storage(_)) + )); + assert_eq!(facts.situation().unwrap().as_of_seq, cursor); + standing.root.atomic(FILE, &valid_pending).unwrap(); + } + let reopened = Standing::open(&fixture.0.join("standing"), facts.clone()).unwrap(); + let (read, situation) = reopened.read_facts().unwrap(); + assert_eq!((read.revision, situation.standing_revision), (43, 43)); + assert_eq!((read.open_asks().count(), situation.asks.len()), (39, 39)); + assert_eq!(facts.events_after(cursor, 64).unwrap().events.len(), 1); + assert!(reopened.is_held(480).unwrap().is_some()); + assert_eq!(reopened.read().unwrap().revision, 43); + assert_eq!(facts.events_after(cursor, 64).unwrap().events.len(), 1); + reopened + .answer(43, Authority::Owner, 44, answer(&asks[1])) + .unwrap(); + assert_eq!(reopened.read().unwrap().open_asks().count(), 38); + } +} + #[test] fn an_ask_capacity_refusal_does_not_reserve_a_stuck_standing_revision() { let fixture = Fixture::new(); @@ -729,6 +843,7 @@ fn recovery_syncs_the_prepared_policy_before_committing_its_fact() { pending: Some(Pending { after, observation: serde_json::from_slice(&event.to_json().unwrap()).unwrap(), + trimmed_answer: None, }), }; standing.root.atomic(FILE, &document.bytes().unwrap()).unwrap(); From 727c282019f14b4966153fbf56a36c4c31eb2ca5 Mon Sep 17 00:00:00 2001 From: oxstack-agent Date: Tue, 17 Nov 2026 17:32:25 +0100 Subject: [PATCH 20/27] fix(work): refuse to verify released or retired standing rules `verify` selected an entry by `state != Superseded` alone. A release is a new entry that supersedes the constraint it releases, so every earlier record carries `state = Superseded`; the call therefore never matched somebody else's rule. But `is_live()` also excludes `Retired` and an inactive constraint in `Stored` state: - `rule(false, supersedes=Some(id))` stores the release in `Stored` state, and a second `rule(_, Owner, Value::Retired, supersedes=Some(id))` stores a retired note the same way; - both were verify-able, so a frontend checker that reports the already released value and scope would move the record to `Applied` and advance the standing revision; - the record is not live, `rule()` can never supersede it again (stale supersession refuses), and `is_held`/`with_guard` ignore it, so "applied" evidence would describe a rule that already released itself. With_guard, remind and answer already filter on liveness. `verify` now uses the same `is_live()` predicate. The test fails before the change. Test: `verify_refuses_released_and_retired_rules` in `oxa/oxa-work/src/standing/store/tests.rs`. --- oxa/docs/standing.md | 3 +- oxa/oxa-work/src/standing/store.rs | 3 +- oxa/oxa-work/src/standing/store/tests.rs | 63 ++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 2 deletions(-) diff --git a/oxa/docs/standing.md b/oxa/docs/standing.md index f92c478d6..3168417fc 100644 --- a/oxa/docs/standing.md +++ b/oxa/docs/standing.md @@ -28,7 +28,8 @@ changed evidence cannot turn a missing unanswered question into a silent close. Every mutation supplies the document revision it read. A stale revision refuses without changing the store. A live collision of kind/scope requires its exact `supersedes` ID. -IDs are derived from the checked revision, not from model text or time. +IDs are derived from the checked revision, not from model text or time. Released and retired +records are not live: only a live entry can be verified as applied, answered or reminded. ## Authority, values and checked application diff --git a/oxa/oxa-work/src/standing/store.rs b/oxa/oxa-work/src/standing/store.rs index 3d07e65b2..bd416fbd7 100644 --- a/oxa/oxa-work/src/standing/store.rs +++ b/oxa/oxa-work/src/standing/store.rs @@ -583,7 +583,8 @@ impl Standing { let entry = after .entries .iter_mut() - .find(|entry| entry.id == entry_id && entry.state != EntryState::Superseded) + // A released or retired rule stays Stored; never claim it was applied. + .find(|entry| entry.id == entry_id && entry.is_live()) .ok_or("standing entry is not live")?; let verified = check(entry)?; types::text(&verified.evidence, 256)?; diff --git a/oxa/oxa-work/src/standing/store/tests.rs b/oxa/oxa-work/src/standing/store/tests.rs index 3bc2d201b..54d4df6b6 100644 --- a/oxa/oxa-work/src/standing/store/tests.rs +++ b/oxa/oxa-work/src/standing/store/tests.rs @@ -249,6 +249,69 @@ fn applied_requires_matching_runtime_value_scope_and_revision() { assert_eq!(standing.read().unwrap().revision, 2); } +#[test] +fn verify_refuses_released_and_retired_rules() { + let fixture = Fixture::new(); + let facts = fixture.facts(); + let standing = fixture.standing(facts); + let held = standing.rule(0, Authority::Owner, 1, hold(true, None)).unwrap(); + let released = standing + .rule(1, Authority::Owner, 2, hold(false, Some(&held.id))) + .unwrap(); + let note = standing + .rule( + 2, + Authority::Owner, + 3, + Rule { + kind: Kind::Note, + scope: Scope::Repository, + value: Value::Text("Keep notes brief".into()), + said: "Keep notes brief".into(), + source: super::super::Source::Command { + command: "note".into(), + }, + supersedes: None, + }, + ) + .unwrap(); + let retired = standing + .rule( + 3, + Authority::Owner, + 4, + Rule { + kind: Kind::Note, + scope: Scope::Repository, + value: Value::Retired, + said: "Retire the note".into(), + source: super::super::Source::Command { + command: "retire".into(), + }, + supersedes: Some(note.id.clone()), + }, + ) + .unwrap(); + assert!(!released.is_live() && !retired.is_live()); + for entry in [&released, &retired] { + let result = standing.verify(4, &entry.id, |entry: &Entry| { + Ok(Verification { + revision: 4, + scope: entry.scope.clone(), + value: entry.value.clone(), + at: 5, + evidence: "synthetic-runtime:4".into(), + }) + }); + assert!(result.is_err(), "verified a non-live rule: {}", entry.id); + } + let read = standing.read().unwrap(); + assert_eq!(read.revision, 4); + for entry in read.entries.iter().filter(|entry| !entry.is_live()) { + assert_eq!(entry.state, EntryState::Stored); + } +} + #[test] fn reminders_create_incidents_and_offer_without_changing_advisory_rules() { let fixture = Fixture::new(); From f7972917dd78b7dfe18426ef064e3976134daee7 Mon Sep 17 00:00:00 2001 From: oxstack-agent Date: Tue, 17 Nov 2026 17:33:10 +0100 Subject: [PATCH 21/27] test(work): keep the new standing test within the formatter width --- oxa/oxa-work/src/standing/store/tests.rs | 56 ++++++++++-------------- 1 file changed, 22 insertions(+), 34 deletions(-) diff --git a/oxa/oxa-work/src/standing/store/tests.rs b/oxa/oxa-work/src/standing/store/tests.rs index 54d4df6b6..cd54314c3 100644 --- a/oxa/oxa-work/src/standing/store/tests.rs +++ b/oxa/oxa-work/src/standing/store/tests.rs @@ -258,40 +258,28 @@ fn verify_refuses_released_and_retired_rules() { let released = standing .rule(1, Authority::Owner, 2, hold(false, Some(&held.id))) .unwrap(); - let note = standing - .rule( - 2, - Authority::Owner, - 3, - Rule { - kind: Kind::Note, - scope: Scope::Repository, - value: Value::Text("Keep notes brief".into()), - said: "Keep notes brief".into(), - source: super::super::Source::Command { - command: "note".into(), - }, - supersedes: None, - }, - ) - .unwrap(); - let retired = standing - .rule( - 3, - Authority::Owner, - 4, - Rule { - kind: Kind::Note, - scope: Scope::Repository, - value: Value::Retired, - said: "Retire the note".into(), - source: super::super::Source::Command { - command: "retire".into(), - }, - supersedes: Some(note.id.clone()), - }, - ) - .unwrap(); + let request = Rule { + kind: Kind::Note, + scope: Scope::Repository, + value: Value::Text("Keep notes brief".into()), + said: "Keep notes brief".into(), + source: super::super::Source::Command { + command: "note".into(), + }, + supersedes: None, + }; + let note = standing.rule(2, Authority::Owner, 3, request).unwrap(); + let request = Rule { + kind: Kind::Note, + scope: Scope::Repository, + value: Value::Retired, + said: "Retire the note".into(), + source: super::super::Source::Command { + command: "retire".into(), + }, + supersedes: Some(note.id.clone()), + }; + let retired = standing.rule(3, Authority::Owner, 4, request).unwrap(); assert!(!released.is_live() && !retired.is_live()); for entry in [&released, &retired] { let result = standing.verify(4, &entry.id, |entry: &Entry| { From ca7eb1c35993736ccbb1d47c25cbce8c6b1a9846 Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:52:11 +0100 Subject: [PATCH 22/27] test(work): assert released and retired rules stay stored The new verification test asserted that every non-live entry is `Stored`. A release or retirement supersedes the record it releases, so that record is `Superseded` by design; the test failed on the superseded hold rather than on the behaviour it covers. Assert the two refused records (the release and the retirement) directly, and that the refused verification changed no state. Test: `verify_refuses_released_and_retired_rules` in `oxa/oxa-work/src/standing/store/tests.rs`. --- oxa/oxa-work/src/standing/store/tests.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/oxa/oxa-work/src/standing/store/tests.rs b/oxa/oxa-work/src/standing/store/tests.rs index cd54314c3..cbc0d1571 100644 --- a/oxa/oxa-work/src/standing/store/tests.rs +++ b/oxa/oxa-work/src/standing/store/tests.rs @@ -295,7 +295,14 @@ fn verify_refuses_released_and_retired_rules() { } let read = standing.read().unwrap(); assert_eq!(read.revision, 4); - for entry in read.entries.iter().filter(|entry| !entry.is_live()) { + // The refused verification leaves the released and retired records as stored; only the + // records they supersede carry `Superseded`. + for refused in [&released.id, &retired.id] { + let entry = read + .entries + .iter() + .find(|entry| &entry.id == refused) + .expect("refused entry is still recorded"); assert_eq!(entry.state, EntryState::Stored); } } From dbce0646b4dd3f63e24992bcea957bf0bb31a46f Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:52:17 +0100 Subject: [PATCH 23/27] test(work): expose standing privacy and cross-source boundary failures --- oxa/oxa-work/src/standing/store/tests.rs | 144 +++++++++++++++++++++++ 1 file changed, 144 insertions(+) diff --git a/oxa/oxa-work/src/standing/store/tests.rs b/oxa/oxa-work/src/standing/store/tests.rs index cd54314c3..79b9fe6c4 100644 --- a/oxa/oxa-work/src/standing/store/tests.rs +++ b/oxa/oxa-work/src/standing/store/tests.rs @@ -56,6 +56,150 @@ fn subject() -> Subject<'static> { } } +#[test] +fn debug_never_formats_private_standing_text_or_nested_payloads() { + let fixture = Fixture::new(); + let standing = fixture.standing(fixture.facts()); + let marker = "synthetic-private-standing-value"; + let mut rule = hold(true, None); + rule.said = marker.into(); + rule.source = super::super::Source::Command { + command: marker.into(), + }; + let entry = standing.rule(0, Authority::Owner, 1, rule.clone()).unwrap(); + let mut request = question(); + request.question = marker.into(); + request.meanwhile = Some(marker.into()); + let ask = standing.ask(1, 2, request.clone()).unwrap(); + let answered = standing.answer(2, Authority::Owner, 3, answer(&ask)).unwrap(); + let read = standing.read().unwrap(); + for output in [ + format!("{read:?}"), + format!("{entry:?}"), + format!("{rule:?}"), + format!("{:?}", rule.source), + format!("{request:?}"), + format!("{answered:?}"), + format!("{:?}", Value::Text(marker.into())), + format!("{:?}", Scope::Branch(marker.into())), + format!("{:?}", super::super::Choice::Text(marker.into())), + format!( + "{:?}", + super::super::QuestionOption { + label: marker.into(), + detail: marker.into() + } + ), + ] { + assert!(!output.contains(marker), "Debug exposed private standing data"); + } + assert_eq!(read.entries[0].said, marker); + assert!(serde_json::to_string(&read).unwrap().contains(marker)); +} + +#[test] +fn ordinary_facts_cannot_publish_standing_revisions() { + let fixture = Fixture::new(); + let facts = fixture.facts(); + let standing = fixture.standing(facts.clone()); + let change = Fact::StandingChanged(StandingChanged { + revision: 1, + goal: None, + ask_opened: None, + ask_closed: None, + incident: None, + }); + assert!(facts + .append( + Checkpoint { + source: "other".into(), + revision: 1 + }, + 1, + change + ) + .is_err()); + assert_eq!(facts.situation().unwrap().standing_revision, 0); + standing.rule(0, Authority::Owner, 2, hold(true, None)).unwrap(); + assert_eq!(standing.read().unwrap().revision, 1); +} + +#[test] +fn standing_asks_do_not_collide_with_another_producers_common_id() { + let fixture = Fixture::new(); + let facts = fixture.facts(); + let standing = fixture.standing(facts.clone()); + let external = oxa_proto::work::Ask { + id: "A-1".into(), + subject: oxa_proto::work::Subject::Repository, + question: "A separate producer question".into(), + }; + facts + .append( + Checkpoint { + source: "runner".into(), + revision: 1, + }, + 1, + Fact::AskOpened(external.clone()), + ) + .unwrap(); + let ask = standing.ask(0, 2, question()).unwrap(); + assert_ne!(ask.id, external.id); + standing.answer(1, Authority::Owner, 3, answer(&ask)).unwrap(); + assert_eq!(facts.situation().unwrap().asks, [external]); +} + +fn fill_fact_sources(facts: &Facts, count: usize) { + for index in 0..count { + facts + .append( + Checkpoint { + source: format!("producer-{index}"), + revision: 1, + }, + index as u64, + Fact::Unblocked(oxa_proto::work::Subject::Repository), + ) + .unwrap(); + } +} + +#[test] +fn initialization_reserves_source_capacity_before_the_first_rule() { + let fixture = Fixture::new(); + let facts = fixture.facts(); + fill_fact_sources(&facts, 15); + let standing = fixture.standing(facts.clone()); + let late = facts.append( + Checkpoint { + source: "late-producer".into(), + revision: 1, + }, + 16, + Fact::Unblocked(oxa_proto::work::Subject::Repository), + ); + assert!( + late.is_err(), + "ordinary producer consumed the reserved standing slot" + ); + assert_eq!(facts.situation().unwrap().as_of_seq, 15); + standing.rule(0, Authority::Owner, 17, hold(true, None)).unwrap(); + assert_eq!(facts.situation().unwrap().as_of_seq, 16); + assert_eq!(standing.read().unwrap().revision, 1); +} + +#[test] +fn initialization_refuses_an_already_full_nonstanding_source_table() { + let fixture = Fixture::new(); + let facts = fixture.facts(); + fill_fact_sources(&facts, 16); + let root = fixture.0.join("standing"); + assert!(Standing::initialize(&root, Authority::Owner, facts.clone()).is_err()); + assert!(!root.join("standing.json").exists()); + assert_eq!(facts.situation().unwrap().as_of_seq, 16); +} + #[test] fn a_fresh_hold_blocks_each_effect_in_a_handle_opened_before_it() { let fixture = Fixture::new(); From 93ffb2284c19131d5bb90d2b52c9067322c27cb6 Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:31:59 +0100 Subject: [PATCH 24/27] fix(work): protect standing privacy and reserved fact identities --- oxa/docs/lead-facts-store.md | 5 ++++ oxa/oxa-work/src/events.rs | 37 ++++++++++++++++++++++++++++-- oxa/oxa-work/src/events/tests.rs | 14 +++++++++-- oxa/oxa-work/src/standing.rs | 10 ++++++++ oxa/oxa-work/src/standing/asks.rs | 14 ++++++----- oxa/oxa-work/src/standing/guard.rs | 8 ++++--- oxa/oxa-work/src/standing/store.rs | 10 ++++---- oxa/oxa-work/src/standing/types.rs | 16 +++++++------ 8 files changed, 90 insertions(+), 24 deletions(-) diff --git a/oxa/docs/lead-facts-store.md b/oxa/docs/lead-facts-store.md index 4912aa26e..02f259171 100644 --- a/oxa/docs/lead-facts-store.md +++ b/oxa/docs/lead-facts-store.md @@ -25,6 +25,11 @@ it retains no unbounded per-task tombstones. Retrying the latest source/revision fact returns its original event and timestamp, even after journal rotation. Different facts under that identity and stale revisions are errors. Sequence exhaustion is an error. +Standing initialization reserves one of those 16 slots before its first mutation; +ordinary producers cannot consume that reservation. The `standing` checkpoint +source, `StandingChanged` observations and `standing:` ask-ID namespace belong +to the standing store's protected prepared-append path, not public `Facts::append`. + Each append validates the projected situation before reserving the event. It then saves a pending observation in `facts.json`, appends and syncs the journal line, commits the source cursor and atomically replaces `situation.json`. Atomic replacements sync the file and directory. diff --git a/oxa/oxa-work/src/events.rs b/oxa/oxa-work/src/events.rs index f5dbcae86..c8d2d623e 100644 --- a/oxa/oxa-work/src/events.rs +++ b/oxa/oxa-work/src/events.rs @@ -28,6 +28,8 @@ struct State { // A cross-store reservation is unpublished until its owner syncs prepared data. #[serde(default, skip_serializing_if = "not_preparing")] preparing: bool, + #[serde(default, skip_serializing_if = "not_preparing")] + standing_reserved: bool, } fn not_preparing(value: &bool) -> bool { !value @@ -148,8 +150,33 @@ impl Facts { /// Stop before the next action if this call fails. Replaying the same checkpoint repairs /// a lost acknowledgement, a torn last line or an interrupted snapshot replacement. pub fn append(&self, checkpoint: Checkpoint, at_ms: u64, fact: Fact) -> Result { + let reserved_ask = match &fact { + Fact::AskOpened(ask) => ask.id.starts_with("standing:"), + Fact::AskClosed(ask) => ask.id.starts_with("standing:"), + _ => false, + }; + if checkpoint.source == "standing" || matches!(fact, Fact::StandingChanged(_)) || reserved_ask { + return Err("standing observations require the protected prepared path".into()); + } self.append_with(checkpoint, at_ms, fact, None:: Result<()>>) } + /// Reserve capacity under the facts lock without inventing a standing event. + /// A failed initialization can retry the same reservation; ordinary stores + /// retain their full source capacity until standing explicitly claims a slot. + pub(crate) fn reserve_standing(&self) -> Result<()> { + let _lock = self.root.lock(LOCK)?; + let (mut state, _, _) = self.recover()?; + let sources = state.sources()?; + if !sources.iter().any(|event| event.checkpoint.source == "standing") && sources.len() == MAX_SOURCES + { + return Err("facts source capacity unavailable for standing".into()); + } + if !state.standing_reserved { + state.standing_reserved = true; + self.root.atomic(STATE, &state.bytes()?)?; + } + Ok(()) + } /// Durably reserve capacity before the protected store prepares its mutation. A failed /// preparation blocks other appenders until that store commits or cancels the reservation. /// Never calls preparation for an already committed identity. @@ -169,7 +196,8 @@ impl Facts { fact: Fact, prepare: Option Result<()>>, ) -> Result { - if prepare.is_some() && !matches!(fact, Fact::StandingChanged(_)) { + if prepare.is_some() && (!matches!(fact, Fact::StandingChanged(_)) || checkpoint.source != "standing") + { return Err("only standing observations use protected preparation".into()); } let _lock = self.root.lock(LOCK)?; @@ -201,7 +229,12 @@ impl Facts { if checkpoint.revision < previous.checkpoint.revision { return Err("stale facts checkpoint".into()); } - } else if sources.len() == MAX_SOURCES { + } else if sources.len() == MAX_SOURCES + || (state.standing_reserved + && checkpoint.source != "standing" + && !sources.iter().any(|event| event.checkpoint.source == "standing") + && sources.len() == MAX_SOURCES - 1) + { return Err("facts source capacity exceeded".into()); } Event { diff --git a/oxa/oxa-work/src/events/tests.rs b/oxa/oxa-work/src/events/tests.rs index e492eedec..6d2882250 100644 --- a/oxa/oxa-work/src/events/tests.rs +++ b/oxa/oxa-work/src/events/tests.rs @@ -464,7 +464,12 @@ fn large_valid_events_page_by_bytes_and_keep_advancing_across_retention_gaps() { }; for revision in 1..=64 { let event = facts - .append(checkpoint("standing", revision), revision, observation(revision)) + .append_prepared( + checkpoint("standing", revision), + revision, + observation(revision), + |_| Ok(()), + ) .unwrap(); assert!(event.to_json().unwrap().len() <= MAX_EVENT_BYTES); } @@ -486,7 +491,12 @@ fn large_valid_events_page_by_bytes_and_keep_advancing_across_retention_gaps() { assert_eq!(seen, (1..=64).collect::>()); for revision in 65..=180 { facts - .append(checkpoint("standing", revision), revision, observation(revision)) + .append_prepared( + checkpoint("standing", revision), + revision, + observation(revision), + |_| Ok(()), + ) .unwrap(); } let page = facts.events_after(0, 64).unwrap(); diff --git a/oxa/oxa-work/src/standing.rs b/oxa/oxa-work/src/standing.rs index 6bb42f11a..3b9991c2a 100644 --- a/oxa/oxa-work/src/standing.rs +++ b/oxa/oxa-work/src/standing.rs @@ -1,4 +1,14 @@ //! Owner standing data and action guards. Never reads or writes runner task state. +// Public Debug remains available without recursively exposing protected payloads. +macro_rules! private_debug { + ($($value:ty),+ $(,)?) => {$( + impl std::fmt::Debug for $value { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(concat!(stringify!($value), "()")) + } + } + )+}; +} mod guard; pub use guard::{Action, Refusal, RefusalReason, Scope, Subject}; mod types; diff --git a/oxa/oxa-work/src/standing/asks.rs b/oxa/oxa-work/src/standing/asks.rs index ab8e1aa1a..d221db52b 100644 --- a/oxa/oxa-work/src/standing/asks.rs +++ b/oxa/oxa-work/src/standing/asks.rs @@ -2,14 +2,14 @@ use super::{types, Rule, Scope, Source}; use crate::Result; use serde::{Deserialize, Serialize}; -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[derive(Clone, PartialEq, Eq, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct QuestionOption { pub label: String, pub detail: String, } -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, PartialEq, Eq)] pub struct Question { pub scope: Scope, pub question: String, @@ -19,7 +19,7 @@ pub struct Question { pub meanwhile: Option, pub source: Source, } -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[derive(Clone, PartialEq, Eq, Deserialize, Serialize)] #[serde( tag = "kind", content = "value", @@ -30,7 +30,7 @@ pub enum Choice { Option(String), Text(String), } -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[derive(Clone, PartialEq, Eq, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct Answer { pub revision: u64, @@ -47,13 +47,13 @@ pub struct AnswerRequest { pub source: Source, pub said: String, } -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[derive(Clone, PartialEq, Eq, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct Proposal { pub rule: Rule, pub accept_label: String, } -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[derive(Clone, PartialEq, Eq, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct Ask { pub id: String, @@ -151,3 +151,5 @@ impl Ask { } } } + +private_debug!(QuestionOption, Question, Choice, Answer, Proposal, Ask); diff --git a/oxa/oxa-work/src/standing/guard.rs b/oxa/oxa-work/src/standing/guard.rs index 1f9ee6827..6f1dd1de9 100644 --- a/oxa/oxa-work/src/standing/guard.rs +++ b/oxa/oxa-work/src/standing/guard.rs @@ -10,7 +10,7 @@ pub enum Action { Update, } -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[derive(Clone, PartialEq, Eq, Deserialize, Serialize)] #[serde( tag = "kind", content = "value", @@ -28,7 +28,7 @@ pub enum Scope { /// Identity resolved by the trusted frontend, not tool JSON. None means checked absence. /// Supply the current PR whenever one exists; its head is deliberately not an identity field. -#[derive(Clone, Copy, Debug, Default)] +#[derive(Clone, Copy, Default)] pub struct Subject<'a> { pub pr: Option, pub task: Option<&'a str>, @@ -43,7 +43,7 @@ pub enum RefusalReason { InvalidSubject, } -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, PartialEq, Eq)] pub struct Refusal { pub revision: u64, pub rule_id: Option, @@ -233,3 +233,5 @@ mod tests { } } } + +private_debug!(Scope, Subject<'_>, Refusal); diff --git a/oxa/oxa-work/src/standing/store.rs b/oxa/oxa-work/src/standing/store.rs index bd416fbd7..1fd625cc2 100644 --- a/oxa/oxa-work/src/standing/store.rs +++ b/oxa/oxa-work/src/standing/store.rs @@ -16,7 +16,7 @@ const MAX_VIEW: usize = 256 * 1024; const MAX_ACTIVE: usize = 128; const HISTORY: usize = 16; -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive(Clone, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct Read { pub revision: u64, @@ -219,12 +219,11 @@ impl Read { } } -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct Standing { root: Root, facts: Facts, } -#[derive(Debug)] pub enum GuardError { Storage(String), Refused(Refusal), @@ -258,6 +257,7 @@ pub struct Verification { pub at: u64, pub evidence: String, } +private_debug!(Read, Standing, GuardError); impl Standing { /// Explicit owner initialization at a protected per-repository configuration root. @@ -273,6 +273,7 @@ impl Standing { if facts.situation()?.standing_revision != 0 { return Err("standing facts already exist".into()); } + facts.reserve_standing()?; let document = Document { version: 1, current: Read { @@ -293,6 +294,7 @@ impl Standing { facts, }; standing.read()?; + standing.facts.reserve_standing()?; Ok(standing) } pub fn read(&self) -> Result { @@ -630,7 +632,7 @@ impl Standing { let before = self.load()?; let mut after = before.next(expected, at)?; let ask = Ask { - id: format!("A-{}", after.revision), + id: format!("standing:A-{}", after.revision), revision: after.revision, scope: request.scope, question: request.question, diff --git a/oxa/oxa-work/src/standing/types.rs b/oxa/oxa-work/src/standing/types.rs index df29e8aa2..e0d4b15c3 100644 --- a/oxa/oxa-work/src/standing/types.rs +++ b/oxa/oxa-work/src/standing/types.rs @@ -28,14 +28,14 @@ pub enum Kind { } /// A source reference is selected/checked by the frontend, never inferred from tool success. -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[derive(Clone, PartialEq, Eq, Deserialize, Serialize)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum Source { Command { command: String }, SessionEntry { session_id: String, entry_id: String }, } -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[derive(Clone, PartialEq, Eq, Deserialize, Serialize)] #[serde( tag = "kind", content = "value", @@ -61,7 +61,7 @@ pub enum Value { classes: Vec, }, } -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[derive(Clone, PartialEq, Eq, Deserialize, Serialize)] #[serde( tag = "kind", content = "value", @@ -98,7 +98,7 @@ pub enum ReservedClass { Desktop, } -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[derive(Clone, PartialEq, Eq, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct Rule { pub kind: Kind, @@ -109,7 +109,7 @@ pub struct Rule { pub supersedes: Option, } -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[derive(Clone, PartialEq, Eq, Deserialize, Serialize)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum EntryState { Stored, @@ -117,7 +117,7 @@ pub enum EntryState { Superseded, } -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[derive(Clone, PartialEq, Eq, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct Entry { pub id: String, @@ -136,7 +136,7 @@ pub struct Entry { // into its own hold and then release it. Request source remains attributed separately. pub(super) authority: Authority, } -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[derive(Clone, PartialEq, Eq, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct Reminder { pub at: u64, @@ -298,3 +298,5 @@ impl Entry { Ok(()) } } + +private_debug!(Source, Value, Condition, Rule, EntryState, Entry, Reminder); From a312ba7fdc603f51966ffe4dddafef7b8ed47c5a Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:48:33 +0100 Subject: [PATCH 25/27] test(work): recover standing revisions through protected append --- oxa/oxa-work/src/standing/store/tests.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/oxa/oxa-work/src/standing/store/tests.rs b/oxa/oxa-work/src/standing/store/tests.rs index 932993946..3a1d18f90 100644 --- a/oxa/oxa-work/src/standing/store/tests.rs +++ b/oxa/oxa-work/src/standing/store/tests.rs @@ -334,7 +334,9 @@ fn pending_revisions_recover_before_or_after_the_fact_without_duplicate_delivery }; let observation = serde_json::from_slice(&event.to_json().unwrap()).unwrap(); if appended { - facts.append(event.checkpoint, event.at_ms, event.fact).unwrap(); + facts + .append_prepared(event.checkpoint, event.at_ms, event.fact, |_| Ok(())) + .unwrap(); } let document = Document { version: 1, From 651f732b7e09a65075fe9887b4c18daff90e9347 Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 20:49:15 +0100 Subject: [PATCH 26/27] test(work): reproduce standing reservation sync retry failure --- oxa/oxa-work/src/events/tests.rs | 40 ++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/oxa/oxa-work/src/events/tests.rs b/oxa/oxa-work/src/events/tests.rs index 6d2882250..2aa8befd5 100644 --- a/oxa/oxa-work/src/events/tests.rs +++ b/oxa/oxa-work/src/events/tests.rs @@ -678,6 +678,46 @@ fn parent_directory_sync_error_blocks_initialization() { ); } +#[test] +fn retry_must_sync_a_visible_standing_reservation() { + const CHILD_ROOT: &str = "OXA_FACTS_TEST_STANDING_RESERVATION"; + if let Some(root) = std::env::var_os(CHILD_ROOT) { + let root = Path::new(&root); + let facts = Facts::initialize(root).unwrap(); + fs::set_permissions(root, fs::Permissions::from_mode(0o300)).unwrap(); + let first = facts.reserve_standing().expect_err("directory sync must fail"); + assert!(first.contains("directory"), "{first}"); + let visible: State = + serde_json::from_slice(&facts.root.read(STATE, STATE_BYTES).unwrap()).unwrap(); + assert!(visible.standing_reserved, "rename must precede the sync failure"); + let retry = facts.reserve_standing(); + fs::set_permissions(root, fs::Permissions::from_mode(0o700)).unwrap(); + let retry = retry.expect_err("retry must finish syncing the reservation"); + assert!(retry.contains("directory"), "{retry}"); + facts.reserve_standing().unwrap(); + return; + } + let env = Fixture::new(); + let mut child = std::process::Command::new(std::env::current_exe().unwrap()); + child.env_clear().env(CHILD_ROOT, env.root()).args([ + "--exact", + "events::tests::retry_must_sync_a_visible_standing_reservation", + ]); + // SAFETY: geteuid has no arguments, pointers or failure mode. + if unsafe { libc::geteuid() } == 0 { + use std::os::unix::process::CommandExt; + std::os::unix::fs::chown(&env.0, Some(65534), Some(65534)).unwrap(); + child.uid(65534).gid(65534); + } + let output = child.output().unwrap(); + assert!( + output.status.success(), + "child failed: {}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + #[test] fn head_changes_and_unrelated_facts_do_not_silently_clear_blocks_or_asks() { let env = Fixture::new(); From df5997e5373acffa1b1cdd3e3554a412deac1f3b Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Tue, 17 Nov 2026 20:59:05 +0100 Subject: [PATCH 27/27] fix(work): sync visible standing reservations on retry --- oxa/oxa-work/src/events.rs | 5 ++++- oxa/oxa-work/src/events/tests.rs | 3 +-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/oxa/oxa-work/src/events.rs b/oxa/oxa-work/src/events.rs index c8d2d623e..8d480a0d9 100644 --- a/oxa/oxa-work/src/events.rs +++ b/oxa/oxa-work/src/events.rs @@ -171,7 +171,10 @@ impl Facts { { return Err("facts source capacity unavailable for standing".into()); } - if !state.standing_reserved { + if state.standing_reserved { + // A previous rename may be visible even though its directory sync failed. + self.root.durable(STATE)?; + } else { state.standing_reserved = true; self.root.atomic(STATE, &state.bytes()?)?; } diff --git a/oxa/oxa-work/src/events/tests.rs b/oxa/oxa-work/src/events/tests.rs index 2aa8befd5..145e9afd9 100644 --- a/oxa/oxa-work/src/events/tests.rs +++ b/oxa/oxa-work/src/events/tests.rs @@ -687,8 +687,7 @@ fn retry_must_sync_a_visible_standing_reservation() { fs::set_permissions(root, fs::Permissions::from_mode(0o300)).unwrap(); let first = facts.reserve_standing().expect_err("directory sync must fail"); assert!(first.contains("directory"), "{first}"); - let visible: State = - serde_json::from_slice(&facts.root.read(STATE, STATE_BYTES).unwrap()).unwrap(); + let visible: State = serde_json::from_slice(&facts.root.read(STATE, STATE_BYTES).unwrap()).unwrap(); assert!(visible.standing_reserved, "rename must precede the sync failure"); let retry = facts.reserve_standing(); fs::set_permissions(root, fs::Permissions::from_mode(0o700)).unwrap();