From e788bbdaf24234f8368a31fc40879f49576a7013 Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Wed, 18 Nov 2026 07:57:48 +0100 Subject: [PATCH] Compose exact required-check recovery with the #307 rerun journal --- oxa/oxa-work/src/publish/rerun.rs | 46 +++++- oxa/oxa-work/src/shadow/rerun.rs | 51 ++++++- oxa/oxa-work/src/shadow/rerun/current.rs | 73 +++++++++ oxa/oxa-work/tests/rerun_conformance.rs | 180 +++++++++++++++++++++++ 4 files changed, 345 insertions(+), 5 deletions(-) create mode 100644 oxa/oxa-work/src/shadow/rerun/current.rs create mode 100644 oxa/oxa-work/tests/rerun_conformance.rs diff --git a/oxa/oxa-work/src/publish/rerun.rs b/oxa/oxa-work/src/publish/rerun.rs index a9b1dcfd9..5f0b223d3 100644 --- a/oxa/oxa-work/src/publish/rerun.rs +++ b/oxa/oxa-work/src/publish/rerun.rs @@ -1,5 +1,8 @@ use super::{github_capture, LandingPublisher, Repository}; -use crate::board::queue::{is_node_id, is_object_id}; +use crate::board::{ + check_recovery::{JobAttempt, JobLineage}, + queue::{is_node_id, is_object_id}, +}; use crate::{github_http_response, Result}; use serde::{Deserialize, Serialize}; @@ -23,7 +26,48 @@ pub struct RerunIntent { pub run_attempt: u64, } +impl From> for RerunIntent { + fn from(attempt: JobAttempt<'_>) -> Self { + let lineage = attempt.lineage; + Self { + key: RerunKey { + repository_id: lineage.repository_id.to_owned(), + head: lineage.head.to_owned(), + workflow_run_id: lineage.workflow_run_id, + logical_job: lineage.required_name.to_owned(), + }, + repository: lineage.repository.to_owned(), + workflow_path: lineage.workflow_path.to_owned(), + check_suite_id: lineage.check_suite_id, + check_run_id: attempt.check_run_id, + job_id: attempt.job_id, + run_attempt: attempt.run_attempt, + } + } +} + impl RerunIntent { + pub(crate) fn replaces(&self, before: &Self) -> bool { + self.valid() && self.as_attempt().replaces(&before.as_attempt()) + } + + fn as_attempt(&self) -> JobAttempt<'_> { + JobAttempt { + lineage: JobLineage { + repository: &self.repository, + repository_id: &self.key.repository_id, + head: &self.key.head, + workflow_path: &self.workflow_path, + required_name: &self.key.logical_job, + check_suite_id: self.check_suite_id, + workflow_run_id: self.key.workflow_run_id, + }, + check_run_id: self.check_run_id, + job_id: self.job_id, + run_attempt: self.run_attempt, + } + } + pub(crate) fn valid(&self) -> bool { is_node_id(&self.key.repository_id) && is_object_id(&self.key.head) diff --git a/oxa/oxa-work/src/shadow/rerun.rs b/oxa/oxa-work/src/shadow/rerun.rs index 661df11f7..12e5710e2 100644 --- a/oxa/oxa-work/src/shadow/rerun.rs +++ b/oxa/oxa-work/src/shadow/rerun.rs @@ -4,6 +4,9 @@ use crate::{LandingPublisher, RerunAcknowledgment, RerunIntent, RerunKey, Result use serde::{Deserialize, Serialize}; use std::{collections::BTreeMap, path::Path}; +mod current; +pub use current::{run_current_once, CurrentSample}; + #[derive(Default)] pub struct Policy { pub enabled: bool, @@ -15,6 +18,10 @@ pub enum Status { Skipped, Reserved, Acknowledged(RerunAcknowledgment), + Reconciled { + acknowledgment: RerunAcknowledgment, + replacement: RerunIntent, + }, } #[derive(Clone, Deserialize, Serialize)] @@ -22,6 +29,10 @@ pub enum Status { pub(super) enum Record { Reserved(RerunIntent), Acknowledged(RerunAcknowledgment), + Reconciled { + acknowledgment: RerunAcknowledgment, + replacement: RerunIntent, + }, } #[derive(Default)] @@ -32,10 +43,18 @@ impl History { let intent = match &record { Record::Reserved(intent) => intent, Record::Acknowledged(ack) => &ack.intent, + Record::Reconciled { acknowledgment, .. } => &acknowledgment.intent, }; let valid_transition = match (&record, self.0.get(&intent.key)) { (Record::Reserved(_), None) => true, (Record::Acknowledged(_), Some(Record::Reserved(before))) => before == intent, + ( + Record::Reconciled { + acknowledgment, + replacement, + }, + Some(Record::Acknowledged(ack)), + ) => acknowledgment == ack && replacement.replaces(intent), _ => false, }; if !intent.valid() || !valid_transition { @@ -49,6 +68,13 @@ impl History { self.0.get(key).map(|record| match record { Record::Reserved(_) => Status::Reserved, Record::Acknowledged(ack) => Status::Acknowledged(ack.clone()), + Record::Reconciled { + acknowledgment, + replacement, + } => Status::Reconciled { + acknowledgment: acknowledgment.clone(), + replacement: replacement.clone(), + }, }) } } @@ -67,6 +93,14 @@ pub fn run_once( if !policy.enabled || run.offline { return Ok(Status::Skipped); } + let (mut log, history) = locked_history(journal_path)?; + if let Some(status) = history.reruns.status(&intent.key) { + return Ok(status); + } + publish(&mut log, policy, repo, run, gh_binary, intent, read) +} + +fn locked_history(journal_path: &Path) -> Result<(File, Admissions)> { let mut log = open_log_with_create(journal_path, false)?; if log .metadata() @@ -77,15 +111,24 @@ pub fn run_once( return Err("rerun history is missing from the owner journal".into()); } let history = Admissions::load(&mut log)?; - if let Some(status) = history.reruns.status(&intent.key) { - return Ok(status); - } + Ok((log, history)) +} + +fn publish( + log: &mut File, + policy: &Policy, + repo: &Path, + run: &RunPolicy, + gh_binary: &Path, + intent: RerunIntent, + read: impl FnOnce() -> Result<(Board, Option)>, +) -> Result { let inherited = log .try_clone() .map_err(|_| "cannot retain the owner journal lease")?; let publisher = LandingPublisher::new(repo, run, gh_binary, &inherited); let target = publisher.rerun_target(&intent, &policy.publisher_id)?; - dispatch(&mut log, intent, read, |intent| { + dispatch(log, intent, read, |intent| { publisher.rerun_reserved(target, intent) }) } diff --git a/oxa/oxa-work/src/shadow/rerun/current.rs b/oxa/oxa-work/src/shadow/rerun/current.rs new file mode 100644 index 000000000..fe7e6aac0 --- /dev/null +++ b/oxa/oxa-work/src/shadow/rerun/current.rs @@ -0,0 +1,73 @@ +use super::{append_publication, locked_history, publish, Board, Policy, Record, Result, RunPolicy, Status}; +use crate::board::check_recovery::{decide, Current, Decision, History}; +use crate::RerunIntent; +use serde_json::Value; +use std::path::Path; + +/// A selected check and its snapshot from the protected Board reader. +pub struct CurrentSample<'a> { + pub board: &'a Board, + pub target: Current<'a>, +} + +/// Use both read callbacks from the same bounded Board Reader. This entry point +/// has no production caller. The publisher and the journal stay unchanged when off. +pub fn run_current_once( + policy: &Policy, + journal_path: &Path, + repo: &Path, + run: &RunPolicy, + gh_binary: &Path, + sample: CurrentSample<'_>, + reader: ( + impl FnMut(&str) -> Result, + impl FnMut(&str) -> Result, + ), +) -> Result { + if !policy.enabled || run.offline { + return Ok(Status::Skipped); + } + let (mut log, history) = locked_history(journal_path)?; + let (mut repository_read, mut actions_json) = reader; + let mut observe = || { + sample + .target + .observe(&mut repository_read, &mut actions_json, |verified| { + verified.map(|verified| { + let intent = RerunIntent::from(verified.attempt); + let next = decide( + true, + &verified.observation, + sample.board.observed_at, + History::Unused, + ); + let eligible = next == Decision::RerunJob(intent.job_id); + (intent, eligible) + }) + }) + }; + let Some((intent, eligible)) = observe()? else { + return Ok(Status::Skipped); + }; + match history.reruns.status(&intent.key) { + Some(Status::Acknowledged(acknowledgment)) if intent.replaces(&acknowledgment.intent) => { + let record = Record::Reconciled { + acknowledgment: acknowledgment.clone(), + replacement: intent.clone(), + }; + append_publication(&mut log, sample.board, None, Some(&record))?; + Ok(Status::Reconciled { + acknowledgment, + replacement: intent, + }) + } + Some(status) => Ok(status), + None if !eligible => Ok(Status::Skipped), + None => publish(&mut log, policy, repo, run, gh_binary, intent, || { + let fresh = observe()? + .filter(|(_, eligible)| *eligible) + .map(|(intent, _)| intent); + Ok((sample.board.clone(), fresh)) + }), + } +} diff --git a/oxa/oxa-work/tests/rerun_conformance.rs b/oxa/oxa-work/tests/rerun_conformance.rs new file mode 100644 index 000000000..3fc3e07aa --- /dev/null +++ b/oxa/oxa-work/tests/rerun_conformance.rs @@ -0,0 +1,180 @@ +use oxa_work::{ + board::{ + check_recovery::{Current, Trigger}, + Board, + }, + shadow::{ + self, + rerun::{run_current_once, CurrentSample, Policy, Status}, + }, + RunPolicy, +}; +use serde_json::{json, Value}; +use std::{cell::Cell, fs, os::unix::fs::PermissionsExt}; + +#[path = "support/private_dir.rs"] +mod private_dir; + +const HEAD: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const OTHER_HEAD: &str = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +const WORKFLOW: &str = ".github/workflows/checks.yml"; + +fn repository(attempt: u64, status: &str) -> Value { + json!({ + "id":"R_repo", "nameWithOwner":"sample/repo", + "pr":{"number":42,"state":"OPEN","headRefOid":HEAD}, + "checked":{"oid":HEAD,"statusCheckRollup":{"commit":{"oid":HEAD}, + "contexts":{"totalCount":1,"nodes":[{ + "__typename":"CheckRun", "databaseId":30+attempt, "isRequired":true, + "name":"gate","status":status,"conclusion":"SUCCESS", + "completedAt":"1970-01-02T00:00:00Z", + "checkSuite":{"databaseId":19,"commit":{"oid":HEAD}, + "workflowRun":{"databaseId":12,"runAttempt":attempt,"event":"pull_request"}} + }]} + }} + }) +} + +#[test] +fn current_reader_reconciles_only_an_acknowledged_job_and_never_reissues_a_pending_send() { + for acknowledged in [true, false] { + let temp = private_dir::PrivateDir::new("rerun-reader-conformance"); + let repo = temp.path(); + let journal = repo.join("owner.jsonl"); + let git = repo.join("git"); + let gh = repo.join("gh"); + fs::write( + &git, + "#!/bin/sh\nprintf '%s\\n' https://example.invalid/sample/repo\n", + ) + .unwrap(); + fs::write( + &gh, + r#"#!/bin/sh +case "$*" in + 'repo view https://example.invalid/sample/repo --json id,nameWithOwner,url') + printf '%s\n' '{"id":"R_repo","nameWithOwner":"sample/repo","url":"https://example.invalid/sample/repo"}' ;; + 'api graphql --hostname example.invalid -f query=query{viewer{id}}') + printf '%s\n' '{"data":{"viewer":{"id":"U_publisher"}}}' ;; + 'api --hostname example.invalid --include --method POST repos/sample/repo/actions/jobs/23/rerun') + printf '%s\n' 23 >> calls + if test -f accepted; then printf 'HTTP/2 201\r\ncontent-length: 0\r\n\r\n'; else exit 1; fi ;; + *) exit 1 ;; +esac +"#, + ) + .unwrap(); + for executable in [&git, &gh] { + fs::set_permissions(executable, fs::Permissions::from_mode(0o700)).unwrap(); + } + if acknowledged { + fs::write(repo.join("accepted"), "").unwrap(); + } + let board = Board { + observed_at: 87_002, + ..Default::default() + }; + { + let mut log = shadow::open_log(&journal).unwrap(); + shadow::run(&mut log, true, || Ok(board.clone()), |_| unreachable!()).unwrap(); + } + let policy = Policy { + enabled: true, + publisher_id: "U_publisher".into(), + }; + let run = RunPolicy { + git_binary: git, + host_path: Default::default(), + terminate_binary: None, + offline: false, + runner_owns_github: false, + storage: repo.to_owned(), + }; + + let apply = |policy: &Policy, attempt: u64, run_id, job_head: &str, changed_job| { + let attempt_reads = Cell::new(0); + run_current_once( + policy, + &journal, + repo, + &run, + &gh, + CurrentSample { + board: &board, + target: Current { + repository: "sample/repo", + repository_id: "R_repo", + number: 42, + pr_head: HEAD, + head: HEAD, + workflow_path: WORKFLOW, + required_name: "gate", + trigger: Trigger::PullRequest, + }, + }, + ( + |fields: &str| { + assert!(fields.contains("pullRequest(number:42)")); + assert!( + shadow::open_log(&journal).is_err(), + "Reader needs the journal lease" + ); + Ok(repository( + attempt, + if attempt == 2 { "IN_PROGRESS" } else { "COMPLETED" }, + )) + }, + |address: &str| { + let prefix = format!("runs/12/attempts/{attempt}"); + if address == prefix { + Ok(json!({ + "id":run_id,"check_suite_id":19,"head_sha":HEAD,"run_attempt":attempt, + "event":"pull_request","path":WORKFLOW + })) + } else { + assert_eq!(address, format!("{prefix}/jobs?per_page=100")); + let prior = attempt_reads.replace(attempt_reads.get() + 1); + let job = 21 + attempt + u64::from(changed_job && prior > 0); + Ok(json!({"total_count":1,"jobs":[{ + "id":job,"run_id":12,"run_attempt":attempt,"head_sha":job_head,"name":"gate", + "check_run_url":format!("https://api.github.com/repos/sample/repo/check-runs/{}",30+attempt) + }]})) + } + }, + ), + ) + }; + + assert_eq!( + apply(&Policy::default(), 2, 12, HEAD, false).unwrap(), + Status::Skipped + ); + for (run_id, head, change) in [(13, HEAD, false), (12, OTHER_HEAD, false), (12, HEAD, true)] { + assert_eq!(apply(&policy, 2, run_id, head, change).unwrap(), Status::Skipped); + assert!(!repo.join("calls").exists()); + } + let dispatched = apply(&policy, 2, 12, HEAD, false); + if acknowledged { + assert!(matches!(dispatched, Ok(Status::Acknowledged(_)))); + assert_eq!(apply(&policy, 2, 12, HEAD, false).unwrap(), dispatched.unwrap()); + let reconciled = apply(&policy, 3, 12, HEAD, false).unwrap(); + let Status::Reconciled { + acknowledgment, + replacement, + } = &reconciled + else { + panic!("a later verified attempt should reconcile, got {reconciled:?}"); + }; + assert_eq!(acknowledgment.intent.job_id, 23); + assert_eq!(replacement.job_id, 24); + assert_eq!(replacement.check_run_id, 33); + assert_eq!(replacement.run_attempt, 3); + assert_eq!(apply(&policy, 3, 12, HEAD, false).unwrap(), reconciled); + } else { + assert!(dispatched.is_err()); + assert_eq!(apply(&policy, 2, 12, HEAD, false).unwrap(), Status::Reserved); + assert_eq!(apply(&policy, 3, 12, HEAD, false).unwrap(), Status::Reserved); + } + assert_eq!(fs::read_to_string(repo.join("calls")).unwrap(), "23\n"); + } +}