diff --git a/oxa/oxa-cli/src/work_board.rs b/oxa/oxa-cli/src/work_board.rs index a611b907a..c9d097ac7 100644 --- a/oxa/oxa-cli/src/work_board.rs +++ b/oxa/oxa-cli/src/work_board.rs @@ -262,19 +262,8 @@ impl<'a> Reader<'a> { } fn response<'b>(&self, text: &'b str, resource: GitHubResource) -> Result<(u16, &'b str), QueryError> { - let invalid = QueryError::Other; - let (headers, body) = text - .split_once("\r\n\r\n") - .or_else(|| text.split_once("\n\n")) - .ok_or(invalid("GitHub command failed without an HTTP response"))?; - let status = headers - .lines() - .next() - .filter(|line| line.starts_with("HTTP/")) - .and_then(|line| line.split_whitespace().nth(1)) - .and_then(|value| value.parse::().ok()) - .filter(|value| (100..600).contains(value)) - .ok_or(invalid("invalid GitHub HTTP status"))?; + let (status, headers, body) = + oxa_work::github_http_response_with_headers(text).map_err(QueryError::Other)?; if let Some(quota) = &self.app_quota { let headers = if (200..300).contains(&status) { headers } else { "" }; quota.borrow_mut().observe(GitHubIdentity::App, resource, headers); diff --git a/oxa/oxa-work/src/lib.rs b/oxa/oxa-work/src/lib.rs index 10ee2919b..33a451033 100644 --- a/oxa/oxa-work/src/lib.rs +++ b/oxa/oxa-work/src/lib.rs @@ -1645,3 +1645,28 @@ mod comment_packet_tests { assert!(thread.starts_with("--- comment by a ---\nx")); } } + +/// Decode the response captured from GitHub CLI with headers included. +pub fn github_http_response(text: &str) -> std::result::Result<(u16, &str), &'static str> { + let (status, _, body) = github_http_response_with_headers(text)?; + Ok((status, body)) +} + +/// Decode the same GitHub CLI response while retaining the borrowed header block. +pub fn github_http_response_with_headers(text: &str) -> std::result::Result<(u16, &str, &str), &'static str> { + let (headers, body) = text + .split_once("\r\n\r\n") + .or_else(|| text.split_once("\n\n")) + .ok_or("GitHub command failed without an HTTP response")?; + let status = headers + .lines() + .next() + .filter(|line| line.starts_with("HTTP/")) + .and_then(|line| line.split_whitespace().nth(1)) + .and_then(|value| value.parse::().ok()) + .filter(|value| (100..600).contains(value)) + .ok_or("invalid GitHub HTTP status")?; + Ok((status, headers, body)) +} + +pub use publish::{RerunAcknowledgment, RerunIntent, RerunKey}; diff --git a/oxa/oxa-work/src/publish.rs b/oxa/oxa-work/src/publish.rs index d7aa53157..e0daf6396 100644 --- a/oxa/oxa-work/src/publish.rs +++ b/oxa/oxa-work/src/publish.rs @@ -2,10 +2,12 @@ use super::{checked, state, tree, CheckpointObserver, LockedTask, Outcome, Record, Result, RunPolicy, Task}; use state::Step; mod landing; +pub(crate) mod rerun; pub use landing::{ Admission, AdmissionAcknowledgment, AdmissionReceipt, DeliveryReceipt, LandingPublisher, NoticeIntent, RejectedLanding, RetryIntent, }; +pub use rerun::{RerunAcknowledgment, RerunIntent, RerunKey}; use std::os::{fd::AsRawFd, unix::process::CommandExt}; use std::{ fs::File, @@ -44,6 +46,16 @@ fn isolate_github_environment(command: &mut Command) { } fn gh(binary: &Path, policy: &RunPolicy, repo: &Path, lock: &File, args: &[&str]) -> Result { + Ok(github_capture(binary, policy, repo, lock, args)?.trim().into()) +} + +fn github_capture( + binary: &Path, + policy: &RunPolicy, + repo: &Path, + lock: &File, + args: &[&str], +) -> Result { let mut command = Command::new(binary); super::isolate_git_environment(&mut command); isolate_github_environment(&mut command); @@ -58,7 +70,7 @@ fn gh(binary: &Path, policy: &RunPolicy, repo: &Path, lock: &File, args: &[&str] Stdio::from(lock.try_clone().map_err(|e| e.to_string())?), super::COMMAND_DEADLINE, ) { - Some((0, text)) => Ok(text.trim().into()), + Some((0, text)) => Ok(text), _ => Err("GitHub publication request failed".into()), } } diff --git a/oxa/oxa-work/src/publish/landing.rs b/oxa/oxa-work/src/publish/landing.rs index be9a85894..d415312ce 100644 --- a/oxa/oxa-work/src/publish/landing.rs +++ b/oxa/oxa-work/src/publish/landing.rs @@ -160,10 +160,10 @@ impl AdmissionAcknowledgment { /// hold the owner journal lock, reserve the intent, and revalidate current /// Reader evidence before calling. An ambiguous outcome is never a retry signal. pub struct LandingPublisher<'a> { - repo: &'a Path, - policy: &'a RunPolicy, - gh_binary: &'a Path, - journal: &'a File, + pub(super) repo: &'a Path, + pub(super) policy: &'a RunPolicy, + pub(super) gh_binary: &'a Path, + pub(super) journal: &'a File, } impl<'a> LandingPublisher<'a> { @@ -190,7 +190,7 @@ impl<'a> LandingPublisher<'a> { data(self.call(&args)?) } - fn host(&self, intent: &NoticeIntent) -> Result { + pub(super) fn location(&self, repository_id: &str, publisher_id: &str) -> Result { if self.policy.offline { return Err("publishing is unavailable in offline mode".into()); } @@ -203,22 +203,22 @@ impl<'a> LandingPublisher<'a> { return Err("publishing requires one origin push URL".into()); } let origin = self.call(&["repo", "view", &remote, "--json", "id,nameWithOwner,url"])?; - if origin["id"].as_str() != Some(&intent.rejection.repository_id) { + if origin["id"].as_str() != Some(repository_id) { return Err("landing reservation does not match publication origin".into()); } - let host = repository(&origin)?.host; - let viewer = self.graphql(&host, "query{viewer{id}}", &[])?; - if viewer["viewer"]["id"].as_str() != Some(&intent.publisher_id) { + let location = repository(&origin)?; + let viewer = self.graphql(&location.host, "query{viewer{id}}", &[])?; + if viewer["viewer"]["id"].as_str() != Some(publisher_id) { return Err("landing reservation does not match the publisher identity".into()); } - Ok(host) + Ok(location) } pub fn notice(&self, intent: &NoticeIntent, failed_tests: &[&str]) -> Result { let body = intent.body(failed_tests)?; - let host = self.host(intent)?; + let location = self.location(&intent.rejection.repository_id, &intent.publisher_id)?; let result = self.graphql( - &host, + &location.host, ADD_COMMENT, &[ &format!("pr={}", intent.rejection.pull_request_id), @@ -233,9 +233,9 @@ impl<'a> LandingPublisher<'a> { pub fn enqueue(&self, intent: &RetryIntent) -> Result { intent.check()?; let rejection = &intent.delivery.intent.rejection; - let host = self.host(&intent.delivery.intent)?; + let location = self.location(&rejection.repository_id, &intent.delivery.intent.publisher_id)?; let result = self.graphql( - &host, + &location.host, ENQUEUE, &[ &format!("pr={}", rejection.pull_request_id), @@ -351,7 +351,7 @@ mod tests { } #[test] - fn notice_uses_the_canonical_transport_on_a_mock_executable() { + fn notice_and_rerun_use_the_canonical_transport_on_a_mock_executable() { use std::{fs, os::unix::fs::PermissionsExt}; let temp = crate::test_dir::PrivateDir::new("landing-transport"); let git = temp.path().join("git"); @@ -374,6 +374,8 @@ case "$*" in "headRefOid":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, "author":{"__typename":"User","id":"U_publisher"},"createdAt":"1970-01-01T01:00:00Z"}}}}}' ;; + 'api --hostname example.invalid --include --method POST repos/sample/repo/actions/jobs/9/rerun') + printf 'HTTP/2 201\r\ncontent-length: 0\r\n\r\n' ;; *) exit 1 ;; esac "#).unwrap(); @@ -392,6 +394,12 @@ esac let publisher = LandingPublisher::new(temp.path(), &policy, &gh, &log); let receipt = publisher.notice(&test_intent(), &["synthetic::test"]).unwrap(); assert_eq!(receipt.comment_id, "IC_comment"); + let intent = crate::publish::rerun::test_intent(); + let target = publisher.rerun_target(&intent, "U_publisher").unwrap(); + assert_eq!( + publisher.rerun_reserved(target, intent.clone()).unwrap().intent, + intent + ); } #[test] diff --git a/oxa/oxa-work/src/publish/rerun.rs b/oxa/oxa-work/src/publish/rerun.rs new file mode 100644 index 000000000..a9b1dcfd9 --- /dev/null +++ b/oxa/oxa-work/src/publish/rerun.rs @@ -0,0 +1,119 @@ +use super::{github_capture, LandingPublisher, Repository}; +use crate::board::queue::{is_node_id, is_object_id}; +use crate::{github_http_response, Result}; +use serde::{Deserialize, Serialize}; + +/// One allowance per logical required job on this exact repository, head and run. +#[derive(Clone, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] +pub struct RerunKey { + pub repository_id: String, + pub head: String, + pub workflow_run_id: u64, + pub logical_job: String, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct RerunIntent { + pub key: RerunKey, + pub repository: String, + pub workflow_path: String, + pub check_suite_id: u64, + pub check_run_id: u64, + pub job_id: u64, + pub run_attempt: u64, +} + +impl RerunIntent { + pub(crate) fn valid(&self) -> bool { + is_node_id(&self.key.repository_id) + && is_object_id(&self.key.head) + && self.key.workflow_run_id > 0 + && !self.key.logical_job.trim().is_empty() + && self.key.logical_job.len() <= 512 + && !self.key.logical_job.chars().any(char::is_control) + && !self.workflow_path.is_empty() + && self.check_suite_id > 0 + && self.check_run_id > 0 + && self.job_id > 0 + && self.run_attempt > 0 + } +} + +/// HTTP 201 proves that GitHub accepted this request. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct RerunAcknowledgment { + pub intent: RerunIntent, +} + +pub(crate) struct RerunTarget(Repository); + +impl LandingPublisher<'_> { + pub(crate) fn rerun_target(&self, intent: &RerunIntent, publisher_id: &str) -> Result { + if !intent.valid() || !is_node_id(publisher_id) { + return Err("invalid GitHub rerun binding".into()); + } + Ok(RerunTarget( + self.location(&intent.key.repository_id, publisher_id)?, + )) + } + + pub(crate) fn rerun_reserved( + &self, + target: RerunTarget, + intent: RerunIntent, + ) -> Result { + let Repository { + owner, name, host, .. + } = target.0; + let path = format!("repos/{owner}/{name}/actions/jobs/{}/rerun", intent.job_id); + let response = github_capture( + self.gh_binary, + self.policy, + self.repo, + self.journal, + &["api", "--hostname", &host, "--include", "--method", "POST", &path], + )?; + acknowledgment(intent, &response) + } +} + +fn acknowledgment(intent: RerunIntent, response: &str) -> Result { + let (status, _) = github_http_response(response)?; + if status != 201 { + return Err("GitHub job rerun lacks a 201 acknowledgment".into()); + } + Ok(RerunAcknowledgment { intent }) +} + +#[cfg(test)] +pub(crate) fn test_intent() -> RerunIntent { + RerunIntent { + key: RerunKey { + repository_id: "R_repo".into(), + head: "a".repeat(40), + workflow_run_id: 4, + logical_job: "required build".into(), + }, + repository: "sample/repo".into(), + workflow_path: ".github/workflows/checks.yml".into(), + check_suite_id: 7, + check_run_id: 8, + job_id: 9, + run_attempt: 2, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn acknowledgment_requires_a_201_envelope_and_preserves_the_exact_attempt() { + let intent = test_intent(); + for response in ["", "HTTP/2 201", "HTTP/2 200\n\n201", "HTTP/2 202\n\n"] { + assert!(acknowledgment(intent.clone(), response).is_err()); + } + let ack = acknowledgment(intent.clone(), "HTTP/2 201\r\ncontent-length: 0\r\n\r\n").unwrap(); + assert_eq!(ack.intent, intent); + } +} diff --git a/oxa/oxa-work/src/shadow.rs b/oxa/oxa-work/src/shadow.rs index 103cad546..6979483af 100644 --- a/oxa/oxa-work/src/shadow.rs +++ b/oxa/oxa-work/src/shadow.rs @@ -16,6 +16,7 @@ use std::time::{Duration, Instant}; mod landing_history; pub mod publication; +pub mod rerun; use landing_history::{AdmissionEvidence, Admissions}; pub const INTERVAL: Duration = Duration::from_secs(300); @@ -72,6 +73,8 @@ struct Sample<'a> { app_quota: Option, #[serde(skip_serializing_if = "Option::is_none")] landing_publication: Option<&'a publication::Publication>, + #[serde(skip_serializing_if = "Option::is_none")] + rerun_publication: Option<&'a rerun::Record>, } pub(crate) const OWNER_FILE_FLAGS: libc::c_int = libc::O_NOFOLLOW | libc::O_NONBLOCK | libc::O_CLOEXEC; @@ -82,15 +85,19 @@ pub(crate) fn regular_file_owned_by(metadata: &Metadata, owner: u32) -> bool { /// Refuse links, shared files and concurrent writers before consulting GitHub. pub fn open_log(path: &Path) -> Result { + open_log_with_create(path, true) +} + +fn open_log_with_create(path: &Path, create: bool) -> Result { // SAFETY: geteuid has no preconditions and returns this process's effective owner. - open_log_for_owner(path, unsafe { libc::geteuid() }) + open_log_for_owner(path, unsafe { libc::geteuid() }, create) } -fn open_log_for_owner(path: &Path, owner: u32) -> Result { +fn open_log_for_owner(path: &Path, owner: u32, create: bool) -> Result { let mut file = OpenOptions::new() .read(true) .append(true) - .create(true) + .create(create) .mode(0o600) .custom_flags(OWNER_FILE_FLAGS) .open(path) @@ -184,7 +191,7 @@ mod tests { let path = path("owner"); drop(open_log(&path).unwrap()); let other = std::fs::metadata(&path).unwrap().uid().wrapping_add(1); - let refused = open_log_for_owner(&path, other).is_err(); + let refused = open_log_for_owner(&path, other, true).is_err(); std::fs::remove_file(path).unwrap(); assert!(refused); } @@ -415,6 +422,7 @@ pub fn run_with_landing( landing_main: landing_main.as_deref(), app_quota, landing_publication: None, + rerun_publication: None, }; append(log, sample)?; if let Some(history) = &mut history { @@ -524,6 +532,31 @@ fn plan_matched_rejections( } } +fn append_publication( + log: &mut File, + board: &Board, + landing: Option<&publication::Publication>, + rerun: Option<&rerun::Record>, +) -> Result<()> { + append( + log, + Sample { + schema_version: 1, + observed_at: board.observed_at, + board, + would_take: Vec::new(), + reviews: &BTreeMap::new(), + queue_rejections: &[], + matched_queue_admissions: &BTreeMap::new(), + landing: &BTreeMap::new(), + landing_main: None, + app_quota: None, + landing_publication: landing, + rerun_publication: rerun, + }, + ) +} + fn append(log: &mut File, mut sample: Sample<'_>) -> Result<()> { let mut line = serde_json::to_vec(&sample).map_err(|_| "cannot encode shadow board")?; if sample.app_quota.is_some() && line.len() as u64 > MAX_LEAD_SAMPLE_BYTES { diff --git a/oxa/oxa-work/src/shadow/landing_history.rs b/oxa/oxa-work/src/shadow/landing_history.rs index 056acf93f..593551422 100644 --- a/oxa/oxa-work/src/shadow/landing_history.rs +++ b/oxa/oxa-work/src/shadow/landing_history.rs @@ -22,6 +22,7 @@ struct PriorSample { queue_rejections: Vec, landing_main: Option, landing_publication: Option, + rerun_publication: Option, } #[derive(Deserialize)] @@ -77,6 +78,7 @@ impl AdmissionEvidence { #[derive(Default)] pub(super) struct Admissions { pub(super) publications: Publications, + pub(super) reruns: super::rerun::History, queued: BTreeMap, matched: BTreeMap, } @@ -109,6 +111,9 @@ impl Admissions { if let Some(publication) = prior.landing_publication { history.publications.observe(publication)?; } + if let Some(record) = prior.rerun_publication { + history.reruns.observe(record)?; + } if prior.landing.is_empty() { continue; } diff --git a/oxa/oxa-work/src/shadow/publication.rs b/oxa/oxa-work/src/shadow/publication.rs index 43c74f786..9239dda4e 100644 --- a/oxa/oxa-work/src/shadow/publication.rs +++ b/oxa/oxa-work/src/shadow/publication.rs @@ -1,5 +1,5 @@ //! Opt-in landing dispatch through the canonical publisher under the owner journal lock. -use super::{append, open_log, Admissions, Board, File, LandingSample, Sample}; +use super::{append_publication, open_log, Admissions, Board, File, LandingSample}; use crate::board::{landing, queue::RejectionEvidence, Action}; use crate::{ Admission, AdmissionAcknowledgment, AdmissionReceipt, DeliveryReceipt, LandingPublisher, NoticeIntent, @@ -151,22 +151,7 @@ impl Publications { } fn record(log: &mut File, board: &Board, publication: &Publication) -> Result<()> { - append( - log, - Sample { - schema_version: 1, - observed_at: board.observed_at, - board, - would_take: Vec::new(), - reviews: &BTreeMap::new(), - queue_rejections: &[], - matched_queue_admissions: &BTreeMap::new(), - landing: &BTreeMap::new(), - landing_main: None, - app_quota: None, - landing_publication: Some(publication), - }, - ) + append_publication(log, board, Some(publication), None) } /// Apply one fresh Reader snapshot under the existing owner-journal lock. diff --git a/oxa/oxa-work/src/shadow/rerun.rs b/oxa/oxa-work/src/shadow/rerun.rs new file mode 100644 index 000000000..661df11f7 --- /dev/null +++ b/oxa/oxa-work/src/shadow/rerun.rs @@ -0,0 +1,172 @@ +//! Reserve one exact job rerun in the existing owner journal before publishing. +use super::{append_publication, open_log_with_create, Admissions, Board, File}; +use crate::{LandingPublisher, RerunAcknowledgment, RerunIntent, RerunKey, Result, RunPolicy}; +use serde::{Deserialize, Serialize}; +use std::{collections::BTreeMap, path::Path}; + +#[derive(Default)] +pub struct Policy { + pub enabled: bool, + pub publisher_id: String, +} + +#[derive(Debug, Eq, PartialEq)] +pub enum Status { + Skipped, + Reserved, + Acknowledged(RerunAcknowledgment), +} + +#[derive(Clone, Deserialize, Serialize)] +#[serde(tag = "state", content = "binding", rename_all = "snake_case")] +pub(super) enum Record { + Reserved(RerunIntent), + Acknowledged(RerunAcknowledgment), +} + +#[derive(Default)] +pub(super) struct History(BTreeMap); + +impl History { + pub(super) fn observe(&mut self, record: Record) -> Result<()> { + let intent = match &record { + Record::Reserved(intent) => intent, + Record::Acknowledged(ack) => &ack.intent, + }; + let valid_transition = match (&record, self.0.get(&intent.key)) { + (Record::Reserved(_), None) => true, + (Record::Acknowledged(_), Some(Record::Reserved(before))) => before == intent, + _ => false, + }; + if !intent.valid() || !valid_transition { + return Err("invalid rerun journal transition".into()); + } + self.0.insert(intent.key.clone(), record); + Ok(()) + } + + fn status(&self, key: &RerunKey) -> Option { + self.0.get(key).map(|record| match record { + Record::Reserved(_) => Status::Reserved, + Record::Acknowledged(ack) => Status::Acknowledged(ack.clone()), + }) + } +} + +/// The caller retains eligibility policy and supplies the exact intent again +/// from the existing Reader under the owner journal lock. +pub fn run_once( + policy: &Policy, + journal_path: &Path, + repo: &Path, + run: &RunPolicy, + gh_binary: &Path, + intent: RerunIntent, + read: impl FnOnce() -> Result<(Board, Option)>, +) -> Result { + if !policy.enabled || run.offline { + return Ok(Status::Skipped); + } + let mut log = open_log_with_create(journal_path, false)?; + if log + .metadata() + .map_err(|_| "cannot inspect the owner journal")? + .len() + == 0 + { + return Err("rerun history is missing from the owner journal".into()); + } + let history = Admissions::load(&mut log)?; + if let Some(status) = history.reruns.status(&intent.key) { + return Ok(status); + } + let inherited = log + .try_clone() + .map_err(|_| "cannot retain the owner journal lease")?; + let publisher = LandingPublisher::new(repo, run, gh_binary, &inherited); + let target = publisher.rerun_target(&intent, &policy.publisher_id)?; + dispatch(&mut log, intent, read, |intent| { + publisher.rerun_reserved(target, intent) + }) +} + +fn dispatch( + log: &mut File, + intent: RerunIntent, + read: impl FnOnce() -> Result<(Board, Option)>, + send: impl FnOnce(RerunIntent) -> Result, +) -> Result { + let (board, fresh) = read()?; + if fresh.as_ref() != Some(&intent) { + return Ok(Status::Skipped); + } + append_publication(log, &board, None, Some(&Record::Reserved(intent.clone())))?; + let ack = send(intent.clone())?; + if ack.intent != intent { + return Err("rerun acknowledgment changed the reserved binding".into()); + } + append_publication(log, &board, None, Some(&Record::Acknowledged(ack.clone())))?; + Ok(Status::Acknowledged(ack)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn moved_attempt_refuses_dispatch_and_ambiguous_request_stays_reserved_after_restart() { + let temp = crate::test_dir::PrivateDir::new("rerun-journal"); + let path = temp.path().join("owner.jsonl"); + let intent = crate::publish::rerun::test_intent(); + let run = RunPolicy { + git_binary: temp.path().join("git"), + host_path: Default::default(), + terminate_binary: None, + offline: false, + runner_owns_github: false, + storage: temp.path().to_owned(), + }; + let launch = |policy: &Policy| { + run_once( + policy, + &path, + temp.path(), + &run, + &temp.path().join("gh"), + intent.clone(), + || panic!("disabled or previously reserved request must not read"), + ) + }; + assert_eq!(launch(&Policy::default()).unwrap(), Status::Skipped); + assert!(!path.exists()); + { + let mut log = super::super::open_log(&path).unwrap(); + let moved = RerunIntent { + run_attempt: 3, + ..intent.clone() + }; + assert_eq!( + dispatch( + &mut log, + intent.clone(), + || Ok((Board::default(), Some(moved))), + |_| panic!("changed attempt must not publish") + ) + .unwrap(), + Status::Skipped + ); + assert!(dispatch( + &mut log, + intent.clone(), + || Ok((Board::default(), Some(intent.clone()))), + |_| Err("connection lost".into()) + ) + .is_err()); + } + let policy = Policy { + enabled: true, + publisher_id: "U_publisher".into(), + }; + assert_eq!(launch(&policy).unwrap(), Status::Reserved); + } +}