Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 2 additions & 13 deletions oxa/oxa-cli/src/work_board.rs
Original file line number Diff line number Diff line change
Expand Up @@ -262,19 +262,8 @@ impl<'a> Reader<'a> {
}

fn response<'b>(&self, text: &'b str, resource: GitHubResource) -> Result<(u16, &'b str), QueryError> {
let invalid = QueryError::Other;
let (headers, body) = text
.split_once("\r\n\r\n")
.or_else(|| text.split_once("\n\n"))
.ok_or(invalid("GitHub command failed without an HTTP response"))?;
let status = headers
.lines()
.next()
.filter(|line| line.starts_with("HTTP/"))
.and_then(|line| line.split_whitespace().nth(1))
.and_then(|value| value.parse::<u16>().ok())
.filter(|value| (100..600).contains(value))
.ok_or(invalid("invalid GitHub HTTP status"))?;
let (status, headers, body) =
oxa_work::github_http_response_with_headers(text).map_err(QueryError::Other)?;
if let Some(quota) = &self.app_quota {
let headers = if (200..300).contains(&status) { headers } else { "" };
quota.borrow_mut().observe(GitHubIdentity::App, resource, headers);
Expand Down
25 changes: 25 additions & 0 deletions oxa/oxa-work/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1645,3 +1645,28 @@ mod comment_packet_tests {
assert!(thread.starts_with("--- comment by a ---\nx"));
}
}

/// Decode the response captured from GitHub CLI with headers included.
pub fn github_http_response(text: &str) -> std::result::Result<(u16, &str), &'static str> {
let (status, _, body) = github_http_response_with_headers(text)?;
Ok((status, body))
}

/// Decode the same GitHub CLI response while retaining the borrowed header block.
pub fn github_http_response_with_headers(text: &str) -> std::result::Result<(u16, &str, &str), &'static str> {
let (headers, body) = text
.split_once("\r\n\r\n")
.or_else(|| text.split_once("\n\n"))
.ok_or("GitHub command failed without an HTTP response")?;
let status = headers
.lines()
.next()
.filter(|line| line.starts_with("HTTP/"))
.and_then(|line| line.split_whitespace().nth(1))
.and_then(|value| value.parse::<u16>().ok())
.filter(|value| (100..600).contains(value))
.ok_or("invalid GitHub HTTP status")?;
Ok((status, headers, body))
}

pub use publish::{RerunAcknowledgment, RerunIntent, RerunKey};
14 changes: 13 additions & 1 deletion oxa/oxa-work/src/publish.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@
use super::{checked, state, tree, CheckpointObserver, LockedTask, Outcome, Record, Result, RunPolicy, Task};
use state::Step;
mod landing;
pub(crate) mod rerun;
pub use landing::{
Admission, AdmissionAcknowledgment, AdmissionReceipt, DeliveryReceipt, LandingPublisher, NoticeIntent,
RejectedLanding, RetryIntent,
};
pub use rerun::{RerunAcknowledgment, RerunIntent, RerunKey};
use std::os::{fd::AsRawFd, unix::process::CommandExt};
use std::{
fs::File,
Expand Down Expand Up @@ -44,6 +46,16 @@ fn isolate_github_environment(command: &mut Command) {
}

fn gh(binary: &Path, policy: &RunPolicy, repo: &Path, lock: &File, args: &[&str]) -> Result<String> {
Ok(github_capture(binary, policy, repo, lock, args)?.trim().into())
}

fn github_capture(
binary: &Path,
policy: &RunPolicy,
repo: &Path,
lock: &File,
args: &[&str],
) -> Result<String> {
let mut command = Command::new(binary);
super::isolate_git_environment(&mut command);
isolate_github_environment(&mut command);
Expand All @@ -58,7 +70,7 @@ fn gh(binary: &Path, policy: &RunPolicy, repo: &Path, lock: &File, args: &[&str]
Stdio::from(lock.try_clone().map_err(|e| e.to_string())?),
super::COMMAND_DEADLINE,
) {
Some((0, text)) => Ok(text.trim().into()),
Some((0, text)) => Ok(text),
_ => Err("GitHub publication request failed".into()),
}
}
Expand Down
38 changes: 23 additions & 15 deletions oxa/oxa-work/src/publish/landing.rs
Original file line number Diff line number Diff line change
Expand Up @@ -160,10 +160,10 @@ impl AdmissionAcknowledgment {
/// hold the owner journal lock, reserve the intent, and revalidate current
/// Reader evidence before calling. An ambiguous outcome is never a retry signal.
pub struct LandingPublisher<'a> {
repo: &'a Path,
policy: &'a RunPolicy,
gh_binary: &'a Path,
journal: &'a File,
pub(super) repo: &'a Path,
pub(super) policy: &'a RunPolicy,
pub(super) gh_binary: &'a Path,
pub(super) journal: &'a File,
}

impl<'a> LandingPublisher<'a> {
Expand All @@ -190,7 +190,7 @@ impl<'a> LandingPublisher<'a> {
data(self.call(&args)?)
}

fn host(&self, intent: &NoticeIntent) -> Result<String> {
pub(super) fn location(&self, repository_id: &str, publisher_id: &str) -> Result<super::Repository> {
if self.policy.offline {
return Err("publishing is unavailable in offline mode".into());
}
Expand All @@ -203,22 +203,22 @@ impl<'a> LandingPublisher<'a> {
return Err("publishing requires one origin push URL".into());
}
let origin = self.call(&["repo", "view", &remote, "--json", "id,nameWithOwner,url"])?;
if origin["id"].as_str() != Some(&intent.rejection.repository_id) {
if origin["id"].as_str() != Some(repository_id) {
return Err("landing reservation does not match publication origin".into());
}
let host = repository(&origin)?.host;
let viewer = self.graphql(&host, "query{viewer{id}}", &[])?;
if viewer["viewer"]["id"].as_str() != Some(&intent.publisher_id) {
let location = repository(&origin)?;
let viewer = self.graphql(&location.host, "query{viewer{id}}", &[])?;
if viewer["viewer"]["id"].as_str() != Some(publisher_id) {
return Err("landing reservation does not match the publisher identity".into());
}
Ok(host)
Ok(location)
}

pub fn notice(&self, intent: &NoticeIntent, failed_tests: &[&str]) -> Result<DeliveryReceipt> {
let body = intent.body(failed_tests)?;
let host = self.host(intent)?;
let location = self.location(&intent.rejection.repository_id, &intent.publisher_id)?;
let result = self.graphql(
&host,
&location.host,
ADD_COMMENT,
&[
&format!("pr={}", intent.rejection.pull_request_id),
Expand All @@ -233,9 +233,9 @@ impl<'a> LandingPublisher<'a> {
pub fn enqueue(&self, intent: &RetryIntent) -> Result<Admission> {
intent.check()?;
let rejection = &intent.delivery.intent.rejection;
let host = self.host(&intent.delivery.intent)?;
let location = self.location(&rejection.repository_id, &intent.delivery.intent.publisher_id)?;
let result = self.graphql(
&host,
&location.host,
ENQUEUE,
&[
&format!("pr={}", rejection.pull_request_id),
Expand Down Expand Up @@ -351,7 +351,7 @@ mod tests {
}

#[test]
fn notice_uses_the_canonical_transport_on_a_mock_executable() {
fn notice_and_rerun_use_the_canonical_transport_on_a_mock_executable() {
use std::{fs, os::unix::fs::PermissionsExt};
let temp = crate::test_dir::PrivateDir::new("landing-transport");
let git = temp.path().join("git");
Expand All @@ -374,6 +374,8 @@ case "$*" in
"headRefOid":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
"author":{"__typename":"User","id":"U_publisher"},"createdAt":"1970-01-01T01:00:00Z"}}}}}'
;;
'api --hostname example.invalid --include --method POST repos/sample/repo/actions/jobs/9/rerun')
printf 'HTTP/2 201\r\ncontent-length: 0\r\n\r\n' ;;
*) exit 1 ;;
esac
"#).unwrap();
Expand All @@ -392,6 +394,12 @@ esac
let publisher = LandingPublisher::new(temp.path(), &policy, &gh, &log);
let receipt = publisher.notice(&test_intent(), &["synthetic::test"]).unwrap();
assert_eq!(receipt.comment_id, "IC_comment");
let intent = crate::publish::rerun::test_intent();
let target = publisher.rerun_target(&intent, "U_publisher").unwrap();
assert_eq!(
publisher.rerun_reserved(target, intent.clone()).unwrap().intent,
intent
);
}

#[test]
Expand Down
119 changes: 119 additions & 0 deletions oxa/oxa-work/src/publish/rerun.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
use super::{github_capture, LandingPublisher, Repository};
use crate::board::queue::{is_node_id, is_object_id};
use crate::{github_http_response, Result};
use serde::{Deserialize, Serialize};

/// One allowance per logical required job on this exact repository, head and run.
#[derive(Clone, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)]
pub struct RerunKey {
pub repository_id: String,
pub head: String,
pub workflow_run_id: u64,
pub logical_job: String,
}

#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub struct RerunIntent {
pub key: RerunKey,
pub repository: String,
pub workflow_path: String,
pub check_suite_id: u64,
pub check_run_id: u64,
pub job_id: u64,
pub run_attempt: u64,
}

impl RerunIntent {
pub(crate) fn valid(&self) -> bool {
is_node_id(&self.key.repository_id)
&& is_object_id(&self.key.head)
&& self.key.workflow_run_id > 0
&& !self.key.logical_job.trim().is_empty()
&& self.key.logical_job.len() <= 512
&& !self.key.logical_job.chars().any(char::is_control)
&& !self.workflow_path.is_empty()
&& self.check_suite_id > 0
&& self.check_run_id > 0
&& self.job_id > 0
&& self.run_attempt > 0
}
}

/// HTTP 201 proves that GitHub accepted this request.
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub struct RerunAcknowledgment {
pub intent: RerunIntent,
}

pub(crate) struct RerunTarget(Repository);

impl LandingPublisher<'_> {
pub(crate) fn rerun_target(&self, intent: &RerunIntent, publisher_id: &str) -> Result<RerunTarget> {
if !intent.valid() || !is_node_id(publisher_id) {
return Err("invalid GitHub rerun binding".into());
}
Ok(RerunTarget(
self.location(&intent.key.repository_id, publisher_id)?,
))
}

pub(crate) fn rerun_reserved(
&self,
target: RerunTarget,
intent: RerunIntent,
) -> Result<RerunAcknowledgment> {
let Repository {
owner, name, host, ..
} = target.0;
let path = format!("repos/{owner}/{name}/actions/jobs/{}/rerun", intent.job_id);
let response = github_capture(
self.gh_binary,
self.policy,
self.repo,
self.journal,
&["api", "--hostname", &host, "--include", "--method", "POST", &path],
)?;
acknowledgment(intent, &response)
}
}

fn acknowledgment(intent: RerunIntent, response: &str) -> Result<RerunAcknowledgment> {
let (status, _) = github_http_response(response)?;
if status != 201 {
return Err("GitHub job rerun lacks a 201 acknowledgment".into());
}
Ok(RerunAcknowledgment { intent })
}

#[cfg(test)]
pub(crate) fn test_intent() -> RerunIntent {
RerunIntent {
key: RerunKey {
repository_id: "R_repo".into(),
head: "a".repeat(40),
workflow_run_id: 4,
logical_job: "required build".into(),
},
repository: "sample/repo".into(),
workflow_path: ".github/workflows/checks.yml".into(),
check_suite_id: 7,
check_run_id: 8,
job_id: 9,
run_attempt: 2,
}
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn acknowledgment_requires_a_201_envelope_and_preserves_the_exact_attempt() {
let intent = test_intent();
for response in ["", "HTTP/2 201", "HTTP/2 200\n\n201", "HTTP/2 202\n\n"] {
assert!(acknowledgment(intent.clone(), response).is_err());
}
let ack = acknowledgment(intent.clone(), "HTTP/2 201\r\ncontent-length: 0\r\n\r\n").unwrap();
assert_eq!(ack.intent, intent);
}
}
41 changes: 37 additions & 4 deletions oxa/oxa-work/src/shadow.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ use std::time::{Duration, Instant};

mod landing_history;
pub mod publication;
pub mod rerun;
use landing_history::{AdmissionEvidence, Admissions};

pub const INTERVAL: Duration = Duration::from_secs(300);
Expand Down Expand Up @@ -72,6 +73,8 @@ struct Sample<'a> {
app_quota: Option<AppQuota>,
#[serde(skip_serializing_if = "Option::is_none")]
landing_publication: Option<&'a publication::Publication>,
#[serde(skip_serializing_if = "Option::is_none")]
rerun_publication: Option<&'a rerun::Record>,
}

pub(crate) const OWNER_FILE_FLAGS: libc::c_int = libc::O_NOFOLLOW | libc::O_NONBLOCK | libc::O_CLOEXEC;
Expand All @@ -82,15 +85,19 @@ pub(crate) fn regular_file_owned_by(metadata: &Metadata, owner: u32) -> bool {

/// Refuse links, shared files and concurrent writers before consulting GitHub.
pub fn open_log(path: &Path) -> Result<File> {
open_log_with_create(path, true)
}

fn open_log_with_create(path: &Path, create: bool) -> Result<File> {
// SAFETY: geteuid has no preconditions and returns this process's effective owner.
open_log_for_owner(path, unsafe { libc::geteuid() })
open_log_for_owner(path, unsafe { libc::geteuid() }, create)
}

fn open_log_for_owner(path: &Path, owner: u32) -> Result<File> {
fn open_log_for_owner(path: &Path, owner: u32, create: bool) -> Result<File> {
let mut file = OpenOptions::new()
.read(true)
.append(true)
.create(true)
.create(create)
.mode(0o600)
.custom_flags(OWNER_FILE_FLAGS)
.open(path)
Expand Down Expand Up @@ -184,7 +191,7 @@ mod tests {
let path = path("owner");
drop(open_log(&path).unwrap());
let other = std::fs::metadata(&path).unwrap().uid().wrapping_add(1);
let refused = open_log_for_owner(&path, other).is_err();
let refused = open_log_for_owner(&path, other, true).is_err();
std::fs::remove_file(path).unwrap();
assert!(refused);
}
Expand Down Expand Up @@ -415,6 +422,7 @@ pub fn run_with_landing(
landing_main: landing_main.as_deref(),
app_quota,
landing_publication: None,
rerun_publication: None,
};
append(log, sample)?;
if let Some(history) = &mut history {
Expand Down Expand Up @@ -524,6 +532,31 @@ fn plan_matched_rejections(
}
}

fn append_publication(
log: &mut File,
board: &Board,
landing: Option<&publication::Publication>,
rerun: Option<&rerun::Record>,
) -> Result<()> {
append(
log,
Sample {
schema_version: 1,
observed_at: board.observed_at,
board,
would_take: Vec::new(),
reviews: &BTreeMap::new(),
queue_rejections: &[],
matched_queue_admissions: &BTreeMap::new(),
landing: &BTreeMap::new(),
landing_main: None,
app_quota: None,
landing_publication: landing,
rerun_publication: rerun,
},
)
}

fn append(log: &mut File, mut sample: Sample<'_>) -> Result<()> {
let mut line = serde_json::to_vec(&sample).map_err(|_| "cannot encode shadow board")?;
if sample.app_quota.is_some() && line.len() as u64 > MAX_LEAD_SAMPLE_BYTES {
Expand Down
Loading