diff --git a/oxa/Cargo.lock b/oxa/Cargo.lock index 5c8752f6..353a5c1f 100644 --- a/oxa/Cargo.lock +++ b/oxa/Cargo.lock @@ -1467,6 +1467,7 @@ version = "0.2.0" dependencies = [ "libc", "oxa-proto", + "oxa-session", "serde", "serde_json", ] diff --git a/oxa/oxa-work/Cargo.toml b/oxa/oxa-work/Cargo.toml index bb9e07e5..aae7c869 100644 --- a/oxa/oxa-work/Cargo.toml +++ b/oxa/oxa-work/Cargo.toml @@ -9,6 +9,7 @@ publish.workspace = true [dependencies] libc.workspace = true oxa-proto.workspace = true +oxa-session.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/oxa/oxa-work/src/board/queue.rs b/oxa/oxa-work/src/board/queue.rs index 4119ce66..425949d3 100644 --- a/oxa/oxa-work/src/board/queue.rs +++ b/oxa/oxa-work/src/board/queue.rs @@ -20,6 +20,24 @@ pub struct RejectionEvidence { failed_required_workflows: BTreeMap, } +pub(crate) struct Removal<'a> { + pub number: u64, + pub event: &'a str, + pub group: &'a str, + pub removed_at: &'a str, +} + +impl RejectionEvidence { + pub(crate) fn admission_identity(&self) -> Removal<'_> { + Removal { + number: self.number, + event: &self.removal_event, + group: &self.merge_group_head, + removed_at: &self.removed_at, + } + } +} + #[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)] struct WorkflowIdentity { check_suite_id: u64, diff --git a/oxa/oxa-work/src/shadow.rs b/oxa/oxa-work/src/shadow.rs index e0f84c0c..d902de88 100644 --- a/oxa/oxa-work/src/shadow.rs +++ b/oxa/oxa-work/src/shadow.rs @@ -1,7 +1,7 @@ //! Read-only sampling: this runner has no executor or GitHub mutation interface. use crate::board::{landing::Next, queue::RejectionEvidence, review, Board, Entry}; use crate::Result; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::fs::{File, Metadata, OpenOptions}; use std::io::{Read, Seek, SeekFrom, Write}; @@ -9,6 +9,9 @@ use std::os::unix::fs::{MetadataExt, OpenOptionsExt, PermissionsExt}; use std::path::Path; use std::time::{Duration, Instant}; +mod landing_history; +use landing_history::{AdmissionEvidence, Admissions}; + pub const INTERVAL: Duration = Duration::from_secs(300); pub const MAX_LOG_BYTES: u64 = 64 * 1024 * 1024; @@ -23,6 +26,8 @@ struct Sample<'a> { #[serde(skip_serializing_if = "<[RejectionEvidence]>::is_empty")] queue_rejections: &'a [RejectionEvidence], #[serde(skip_serializing_if = "BTreeMap::is_empty")] + matched_queue_admissions: &'a BTreeMap, + #[serde(skip_serializing_if = "BTreeMap::is_empty")] landing: &'a BTreeMap, #[serde(skip_serializing_if = "Option::is_none")] landing_main: Option<&'a str>, @@ -143,6 +148,61 @@ mod tests { assert!(refused); } + fn snapshot_bytes() -> Vec { + serde_json::to_vec(&serde_json::json!({ + "schema_version": 1, + "observed_at": 0, + "board": Board::default() + })) + .unwrap() + } + + #[test] + fn admission_replay_rejects_and_open_log_preserves_interrupted_tail_bytes() { + let full = snapshot_bytes(); + for fragment in [full.as_slice(), &full[..full.len() / 2]] { + let dir = path("interrupted-admission"); + std::fs::create_dir(&dir).unwrap(); + let journal = dir.join("owner.jsonl"); + let mut log = open_log(&journal).unwrap(); + log.write_all(&full).unwrap(); + log.write_all(b"\n").unwrap(); + log.write_all(fragment).unwrap(); + assert!(Admissions::load(&mut log).is_err()); + drop(log); + + let mut log = open_log(&journal).unwrap(); + assert!(Admissions::load(&mut log).is_ok()); + drop(log); + assert_eq!(std::fs::read(&journal).unwrap(), [&full[..], b"\n"].concat()); + let tail = std::fs::read_dir(&dir) + .unwrap() + .map(|entry| entry.unwrap().path()) + .find(|path| path != &journal) + .unwrap(); + assert_eq!(std::fs::read(&tail).unwrap(), fragment); + std::fs::remove_file(tail).unwrap(); + std::fs::remove_file(journal).unwrap(); + std::fs::remove_dir(dir).unwrap(); + } + } + + #[test] + fn admission_replay_refuses_multiple_objects_on_one_complete_line() { + let journal = path("corrupt-admission"); + let snapshot = snapshot_bytes(); + let line = [&snapshot[..], &snapshot[..], b"\n"].concat(); + let mut log = open_log(&journal).unwrap(); + log.write_all(&line).unwrap(); + drop(log); + + let mut log = open_log(&journal).unwrap(); + assert!(Admissions::load(&mut log).is_err()); + drop(log); + assert_eq!(std::fs::read(&journal).unwrap(), line); + std::fs::remove_file(journal).unwrap(); + } + #[test] fn a_full_log_is_not_extended() { let path = path("limit"); @@ -225,7 +285,7 @@ pub struct LandingDecision { pub queue: Option, } -#[derive(Serialize)] +#[derive(Deserialize, Serialize)] pub struct QueueObservation { pub enqueued_at: u64, #[serde(skip_serializing_if = "Option::is_none")] @@ -249,6 +309,7 @@ pub fn run_with_landing( mut snapshot: impl FnMut() -> Result, mut wait: impl FnMut(Duration), ) -> Result<()> { + let mut history = None; loop { let started = Instant::now(); let LandingSample { @@ -258,6 +319,18 @@ pub fn run_with_landing( landing, landing_main, } = snapshot()?; + let matched_queue_admissions = if landing.is_empty() { + BTreeMap::new() + } else { + if history.is_none() { + history = Some(Admissions::load(log)?); + } + history.as_mut().expect("owner journal was loaded").matches( + &board, + landing_main.as_deref(), + queue_rejections.iter().map(RejectionEvidence::admission_identity), + ) + }; let sample = Sample { schema_version: 1, observed_at: board.observed_at, @@ -265,6 +338,7 @@ pub fn run_with_landing( board: &board, reviews: &reviews, queue_rejections: &queue_rejections, + matched_queue_admissions: &matched_queue_admissions, landing: &landing, landing_main: landing_main.as_deref(), }; @@ -276,6 +350,14 @@ pub fn run_with_landing( } log.write_all(&line).map_err(|_| "cannot append shadow board")?; log.sync_data().map_err(|_| "cannot sync shadow board")?; + if let Some(history) = &mut history { + history.observe( + &board, + landing + .iter() + .map(|(number, decision)| (*number, decision.queue.as_ref())), + ); + } if once { return Ok(()); } diff --git a/oxa/oxa-work/src/shadow/landing_history.rs b/oxa/oxa-work/src/shadow/landing_history.rs new file mode 100644 index 00000000..c507e2b5 --- /dev/null +++ b/oxa/oxa-work/src/shadow/landing_history.rs @@ -0,0 +1,231 @@ +//! Correlate a verified rejection with an earlier owner-observed queue admission. +use super::{Board, File, QueueObservation, Read, Seek, SeekFrom, MAX_LOG_BYTES}; +use crate::board::queue::{is_object_id, Removal}; +use crate::Result; +use serde::{Deserialize, Serialize}; +use std::{ + collections::BTreeMap, + io::{BufRead, BufReader}, +}; + +#[derive(Deserialize)] +struct PriorSample { + schema_version: u32, + observed_at: u64, + board: Board, + #[serde(default)] + landing: BTreeMap, + #[serde(default)] + queue_rejections: Vec, + landing_main: Option, +} + +#[derive(Deserialize)] +struct PriorRejection { + number: u64, + removal_event: String, + merge_group_head: String, + removed_at: Option, +} + +#[derive(Deserialize)] +struct PriorLanding { + queue: Option, +} + +struct Admission { + head: String, + group: String, + enqueued_at: u64, + observed_at: u64, +} + +#[derive(Clone, Serialize)] +pub(super) struct AdmissionEvidence { + removal_event: String, + pr_head: String, + merge_group_head: String, + enqueued_at: u64, + admission_observed_at: u64, + #[serde(skip)] + removed_at: u64, + #[serde(skip_serializing_if = "Option::is_none")] + rejection_baseline: Option, +} + +/// Actual main observed during the first matching rejection snapshot. +#[derive(Clone, Serialize)] +struct RejectionBaseline { + main: String, + observed_at: u64, +} + +#[derive(Default)] +pub(super) struct Admissions { + queued: BTreeMap, + matched: BTreeMap, +} + +impl Admissions { + pub(super) fn load(log: &mut File) -> Result { + log.seek(SeekFrom::Start(0)) + .map_err(|_| "cannot seek owner shadow journal")?; + let mut history = Self::default(); + let mut reader = BufReader::new(log.take(MAX_LOG_BYTES + 1)); + let mut line = Vec::new(); + let mut offset = 0; + loop { + line.clear(); + let bytes = reader + .read_until(b'\n', &mut line) + .map_err(|_| "cannot read owner shadow journal")?; + if bytes == 0 { + break; + } + offset += bytes as u64; + if offset > MAX_LOG_BYTES || line.last() != Some(&b'\n') { + return Err("owner shadow journal exceeds its limit or contains an incomplete record".into()); + } + let prior: PriorSample = + serde_json::from_slice(&line).map_err(|_| "cannot read owner shadow journal")?; + if prior.schema_version != 1 || prior.observed_at != prior.board.observed_at { + return Err("invalid owner shadow journal sample".into()); + } + if prior.landing.is_empty() { + continue; + } + history.matches( + &prior.board, + prior.landing_main.as_deref(), + prior.queue_rejections.iter().filter_map(|r| { + Some(Removal { + number: r.number, + event: &r.removal_event, + group: &r.merge_group_head, + removed_at: r.removed_at.as_deref()?, + }) + }), + ); + history.observe( + &prior.board, + prior.landing.iter().map(|(n, row)| (*n, row.queue.as_ref())), + ); + } + Ok(history) + } + + pub(super) fn observe<'a>( + &mut self, + board: &Board, + landing: impl Iterator)>, + ) { + let landing: BTreeMap<_, _> = landing.collect(); + if landing.is_empty() { + return; + } + let active: BTreeMap<_, _> = board + .entries + .iter() + .map(|entry| (entry.number, entry.head.as_str())) + .collect(); + self.queued + .retain(|number, admission| active.get(number).copied() == Some(admission.head.as_str())); + for (number, queued) in landing { + let Some(head) = active.get(&number).copied().filter(|head| is_object_id(head)) else { + continue; + }; + let Some(queue) = queued.filter(|queue| queue.enqueued_at <= board.observed_at) else { + continue; + }; + let Some(group) = queue.group_head.as_deref().filter(|group| is_object_id(group)) else { + continue; + }; + if self + .queued + .get(&number) + .is_some_and(|prior| prior.group == group && prior.enqueued_at == queue.enqueued_at) + { + continue; + } + self.queued.insert( + number, + Admission { + head: head.to_owned(), + group: group.to_owned(), + enqueued_at: queue.enqueued_at, + observed_at: board.observed_at, + }, + ); + } + } + + pub(super) fn matches<'a>( + &mut self, + board: &Board, + main: Option<&str>, + rejections: impl Iterator>, + ) -> BTreeMap { + let active: BTreeMap<_, _> = board + .entries + .iter() + .map(|entry| (entry.number, entry.head.as_str())) + .collect(); + let main = main.filter(|head| is_object_id(head)); + self.matched.retain(|number, known| { + active.get(number).copied() == Some(known.pr_head.as_str()) + && self.queued.get(number).is_some_and(|prior| { + prior.group == known.merge_group_head + && prior.enqueued_at == known.enqueued_at + && prior.observed_at == known.admission_observed_at + }) + }); + rejections + .filter_map(|removal| { + let Removal { + number, + event, + group, + removed_at, + } = removal; + let prior = self.queued.get(&number)?; + let removed_at = oxa_session::parse_utc_seconds(removed_at)?; + if active.get(&number).copied() != Some(prior.head.as_str()) + || prior.group != group + || removed_at <= prior.observed_at + || removed_at > board.observed_at + { + return None; + } + let previous = self.matched.get(&number); + if previous.is_some_and(|known| { + removed_at < known.removed_at + || (known.removal_event == event) != (known.removed_at == removed_at) + }) { + return None; + } + let baseline = previous + .filter(|known| known.removal_event == event) + .and_then(|known| known.rejection_baseline.as_ref()) + .filter(|baseline| baseline.observed_at <= board.observed_at) + .cloned() + .or_else(|| { + main.map(|main| RejectionBaseline { + main: main.to_owned(), + observed_at: board.observed_at, + }) + }); + let observed = AdmissionEvidence { + removal_event: event.to_owned(), + pr_head: prior.head.clone(), + merge_group_head: prior.group.clone(), + enqueued_at: prior.enqueued_at, + admission_observed_at: prior.observed_at, + removed_at, + rejection_baseline: baseline, + }; + self.matched.insert(number, observed.clone()); + Some((number, observed)) + }) + .collect() + } +} diff --git a/oxa/oxa-work/tests/board_queue.rs b/oxa/oxa-work/tests/board_queue.rs index d6742855..84510499 100644 --- a/oxa/oxa-work/tests/board_queue.rs +++ b/oxa/oxa-work/tests/board_queue.rs @@ -138,6 +138,143 @@ fn changed_pr_head_refuses_the_snapshot() { assert!(queue::read(&board(), |_| Ok((data.clone(), 4096))).is_err()); } +#[test] +fn the_owner_journal_matches_only_the_previously_queued_head_and_group() { + use oxa_work::board::landing::Next; + use shadow::{LandingDecision, LandingSample, QueueObservation}; + use std::collections::BTreeMap; + + let path = std::env::temp_dir().join(format!( + "oxa-queue-admission-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let mut prior = board(); + prior.observed_at = 1_767_225_600; + for observed_at in [prior.observed_at, prior.observed_at + 1] { + let mut log = shadow::open_log(&path).unwrap(); + shadow::run_with_landing( + &mut log, + true, + || { + let mut queued_board = prior.clone(); + queued_board.observed_at = observed_at; + Ok(LandingSample { + board: queued_board, + landing: BTreeMap::from([( + 42, + LandingDecision { + next: Next::AlreadyQueued, + queue_alarm: false, + queue: Some(QueueObservation { + enqueued_at: prior.observed_at - 5, + group_head: Some(GROUP.into()), + }), + }, + )]), + ..Default::default() + }) + }, + |_| unreachable!(), + ) + .unwrap(); + } + + let mut current = prior.clone(); + current.observed_at += 3; + let other = "cccccccccccccccccccccccccccccccccccccccc"; + let first_main = "dddddddddddddddddddddddddddddddddddddddd"; + let later_main = "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"; + for (head, group, main, newly_queued) in [ + (HEAD, GROUP, first_main, false), + (HEAD, other, later_main, false), + (HEAD, GROUP, later_main, false), + (HEAD, GROUP, later_main, true), + (HEAD, other, later_main, false), + (other, GROUP, later_main, false), + ] { + let mut log = shadow::open_log(&path).unwrap(); + current.entries[0].head = head.into(); + let mut evidence = fixture("FAILURE"); + evidence["p42"]["headRefOid"] = head.into(); + let group_event = &mut evidence["p42"]["timelineItems"]["nodes"][0]["beforeCommit"]; + group_event["oid"] = group.into(); + group_event["statusCheckRollup"]["commit"]["oid"] = group.into(); + let rejections = queue::read(¤t, |_| Ok((evidence.clone(), 4096))).unwrap(); + shadow::run_with_landing( + &mut log, + true, + || { + let queued = newly_queued.then(|| QueueObservation { + enqueued_at: current.observed_at, + group_head: Some(other.into()), + }); + Ok(LandingSample { + board: current.clone(), + landing_main: Some(main.into()), + queue_rejections: rejections.clone(), + landing: BTreeMap::from([( + 42, + LandingDecision { + next: if queued.is_some() { + Next::AlreadyQueued + } else { + Next::WaitQueue + }, + queue_alarm: false, + queue: queued, + }, + )]), + ..Default::default() + }) + }, + |_| unreachable!(), + ) + .unwrap(); + current.observed_at += 1; + } + let saved = fs::read_to_string(&path).unwrap(); + fs::remove_file(path).unwrap(); + let samples: Vec = saved + .lines() + .map(|row| serde_json::from_str(row).unwrap()) + .collect(); + assert!(samples[0].get("matched_queue_admissions").is_none()); + let rows = &samples[1..]; + assert!(rows[0].get("matched_queue_admissions").is_none()); + assert_eq!( + rows[1]["matched_queue_admissions"]["42"], + json!({ + "removal_event":"removal-1","pr_head":HEAD,"merge_group_head":GROUP, + "enqueued_at":prior.observed_at-5,"admission_observed_at":prior.observed_at, + "rejection_baseline":{"main":first_main,"observed_at":prior.observed_at+3} + }) + ); + assert_eq!(rows[3]["landing_main"], later_main); + assert_eq!( + rows[3]["matched_queue_admissions"], + rows[1]["matched_queue_admissions"] + ); + assert_eq!(rows[2]["queue_rejections"][0]["merge_group_head"], other); + assert_eq!( + rows[4]["matched_queue_admissions"], + rows[1]["matched_queue_admissions"] + ); + assert_eq!(rows[4]["landing"]["42"]["next"], "already_queued"); + assert_eq!(rows[5]["queue_rejections"][0]["merge_group_head"], other); + assert_eq!( + rows[5]["queue_rejections"][0]["removed_at"], + "2026-01-01T00:00:01Z" + ); + assert_eq!(rows[6]["queue_rejections"][0]["merge_group_head"], GROUP); + for row in [&rows[2], &rows[5], &rows[6]] { + assert!(row.get("matched_queue_admissions").is_none()); + } +} + #[test] fn failed_required_workflows_belong_to_the_removed_commit() { let mut data = fixture("FAILURE");