From 7fba2effb6c8567a2a71e129c4a4f099eb410f7c Mon Sep 17 00:00:00 2001 From: Oscar Nordstrom <13300531+n0rpan@users.noreply.github.com> Date: Wed, 18 Nov 2026 03:32:26 +0100 Subject: [PATCH] Decode failed JUnit tests from verified queue reports --- oxa/Cargo.lock | 10 + oxa/docs/work.md | 4 +- oxa/oxa-cli/Cargo.toml | 3 +- oxa/oxa-cli/src/work_board/landing/report.rs | 65 ++++-- .../src/work_board/landing/report/junit.rs | 194 ++++++++++++++++++ oxa/oxa-cli/tests/work_shadow.rs | 14 +- oxa/oxa-work/src/board/queue/report.rs | 46 ++++- 7 files changed, 317 insertions(+), 19 deletions(-) create mode 100644 oxa/oxa-cli/src/work_board/landing/report/junit.rs diff --git a/oxa/Cargo.lock b/oxa/Cargo.lock index 794a50c44..8cc30f088 100644 --- a/oxa/Cargo.lock +++ b/oxa/Cargo.lock @@ -1200,6 +1200,7 @@ dependencies = [ "oxc-gateway", "oxc-local", "pulldown-cmark", + "quick-xml", "regex", "serde_json", "sha2", @@ -1644,6 +1645,15 @@ dependencies = [ "unicase", ] +[[package]] +name = "quick-xml" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" +dependencies = [ + "memchr", +] + [[package]] name = "quinn" version = "0.11.12" diff --git a/oxa/docs/work.md b/oxa/docs/work.md index 6b5315ccf..b7e4ed403 100644 --- a/oxa/docs/work.md +++ b/oxa/docs/work.md @@ -31,8 +31,8 @@ of required checks that consume those reports, and every JUnit producer and arti ``` That option also enables queue evidence. The shadow journal records the checked -run, attempt, required check IDs, and SHA-256 digests after verifying every -configured archive for that attempt. +run, attempt, required check IDs, SHA-256 digests, and failed test identities after +verifying and reading every configured JUnit archive for that attempt. Limit: 100 PRs/items per connection. Reported checks are matched against the base's classic protection and active repository/organization rulesets; missing required diff --git a/oxa/oxa-cli/Cargo.toml b/oxa/oxa-cli/Cargo.toml index 1ae35c97f..4e0f2a32d 100644 --- a/oxa/oxa-cli/Cargo.toml +++ b/oxa/oxa-cli/Cargo.toml @@ -44,14 +44,15 @@ oxa-interactive.workspace = true oxa-rpc.workspace = true oxa-export.workspace = true serde_json.workspace = true +quick-xml = "0.42" regex.workspace = true sha2.workspace = true +zip = { version = "9.6.1", default-features = false, features = ["deflate-flate2-zlib-rs"] } signal-hook.workspace = true pulldown-cmark = { version = "0.13.6", default-features = false } [dev-dependencies] oxa-fake.workspace = true -zip = { version = "9.6.1", default-features = false, features = ["deflate-flate2-zlib-rs"] } [target.'cfg(target_os = "linux")'.dev-dependencies] oxa-orchestrator = { path = "../oxa-orchestrator", features = ["a2a"] } diff --git a/oxa/oxa-cli/src/work_board/landing/report.rs b/oxa/oxa-cli/src/work_board/landing/report.rs index d5b4e591e..b570aca2e 100644 --- a/oxa/oxa-cli/src/work_board/landing/report.rs +++ b/oxa/oxa-cli/src/work_board/landing/report.rs @@ -2,7 +2,8 @@ use crate::work_board::{QueryError, Reader}; use oxa_work::board::queue::{ report::{ - attempt_for, select, ActionMetadata, ReportArchive, ReportSource, VerifiedArchive, VerifiedReports, + attempt_for, select, ActionMetadata, ReportArchive, ReportAttemptIdentity, ReportSource, + VerifiedArchive, VerifiedReports, }, RejectionEvidence, }; @@ -14,18 +15,25 @@ use std::{ process::Command, }; +mod junit; + struct AttemptRead { workflow: Value, jobs: Value, artifacts: Value, - verified_archives: BTreeSet<(u64, u64, String)>, + verified_archives: BTreeMap<(u64, u64, String), Vec>, +} + +struct VerifiedAttempt { + identity: ReportAttemptIdentity, + reports: VerifiedReports, } pub(in crate::work_board) struct Settings { workflow_path: String, required_check_names: BTreeSet, reports: Vec<(String, String)>, - verified: BTreeMap<(String, u64, u64), VerifiedReports>, + verified: BTreeMap<(String, u64, u64), VerifiedAttempt>, } impl Settings { @@ -107,6 +115,7 @@ impl Reader<'_> { ) -> Result<(), String> { let mut previous = std::mem::take(&mut settings.verified); let mut attempts_read = BTreeMap::new(); + let mut decoded_bytes = 0; for rejection in rejections { let mut attempts = BTreeMap::<_, Vec<_>>::new(); for id in rejection.workflow_checks() { @@ -115,9 +124,18 @@ impl Reader<'_> { } } for ((run, attempt), checks) in attempts { + let Some(identity) = rejection.report_attempt_identity(&checks) else { + continue; + }; let key = (rejection.removal_event().to_owned(), run, attempt); let proof = match previous.remove(&key) { - Some(saved) if saved.failed_check_runs == checks => Some(saved), + Some(saved) if saved.identity == identity => { + for name in &saved.reports.failed_tests { + crate::work_board::charge(&mut decoded_bytes, name.len()) + .map_err(|_| "decoded test names exceed sample limit")?; + } + Some(saved.reports) + } _ => { let sources = match attempts_read.entry((run, attempt)) { Entry::Occupied(entry) => entry.into_mut(), @@ -126,12 +144,25 @@ impl Reader<'_> { } }; let Some(sources) = sources else { continue }; - self.verify_attempt_reports(rejection, checks, (run, attempt), settings, sources)? + self.verify_attempt_reports( + rejection, + checks, + (run, attempt), + settings, + sources, + &mut decoded_bytes, + )? } }; if let Some(proof) = proof { rejection.record_verified_reports(proof.clone()); - settings.verified.insert(key, proof); + settings.verified.insert( + key, + VerifiedAttempt { + identity, + reports: proof, + }, + ); } } } @@ -152,7 +183,7 @@ impl Reader<'_> { workflow, jobs: self.actions_json(&format!("runs/{run}/attempts/{attempt}/jobs?per_page=100"))?, artifacts: self.actions_json(&format!("runs/{run}/artifacts?per_page=100"))?, - verified_archives: BTreeSet::new(), + verified_archives: BTreeMap::new(), })) } @@ -163,6 +194,7 @@ impl Reader<'_> { (run, attempt): (u64, u64), settings: &Settings, sources: &mut AttemptRead, + decoded_bytes: &mut usize, ) -> Result, String> { let metadata = ActionMetadata { attempt: &sources.workflow, @@ -188,11 +220,19 @@ impl Reader<'_> { archives.push(first); } let mut verified = Vec::new(); + let mut failed_tests = BTreeSet::new(); for report in archives { let key = (report.artifact_id, report.size_bytes, report.sha256.to_owned()); - if !sources.verified_archives.contains(&key) { - self.verify_archive(&report)?; - sources.verified_archives.insert(key); + let names = match sources.verified_archives.entry(key) { + Entry::Occupied(entry) => entry.into_mut(), + Entry::Vacant(entry) => entry.insert(junit::failed_tests(&self.verify_archive(&report)?)?), + }; + for name in names { + if !failed_tests.contains(name) { + crate::work_board::charge(decoded_bytes, name.len()) + .map_err(|_| "decoded test names exceed sample limit")?; + failed_tests.insert(name.clone()); + } } verified.push(VerifiedArchive { id: report.artifact_id, @@ -204,10 +244,11 @@ impl Reader<'_> { workflow_run: run, run_attempt: attempt, archives: verified, + failed_tests: failed_tests.into_iter().collect(), })) } - fn verify_archive(&self, selected: &ReportArchive<'_>) -> Result<(), String> { + fn verify_archive(&self, selected: &ReportArchive<'_>) -> Result, String> { let size = usize::try_from(selected.size_bytes).map_err(|_| "invalid GitHub archive size")?; let (remaining, duration) = self.read_budget().map_err(QueryError::message)?; if size > remaining { @@ -226,6 +267,6 @@ impl Reader<'_> { if format!("{:x}", Sha256::digest(&bytes)) != selected.sha256 { return Err("GitHub test archive digest mismatch".into()); } - Ok(()) + Ok(bytes) } } diff --git a/oxa/oxa-cli/src/work_board/landing/report/junit.rs b/oxa/oxa-cli/src/work_board/landing/report/junit.rs new file mode 100644 index 000000000..ee1402fa1 --- /dev/null +++ b/oxa/oxa-cli/src/work_board/landing/report/junit.rs @@ -0,0 +1,194 @@ +//! Decode failed test identities from a verified single-report archive. +use quick_xml::{ + events::{BytesStart, Event}, + Reader, XmlVersion, +}; +use std::{ + collections::BTreeSet, + io::{Cursor, Read}, +}; +use zip::ZipArchive; + +const MAX_XML_BYTES: u64 = 16 * 1024 * 1024; +const MAX_ELEMENTS_DEEP: usize = 64; +const MAX_NAME_BYTES: usize = 512; + +pub(super) fn failed_tests(bytes: &[u8]) -> Result, String> { + let mut archive = ZipArchive::new(Cursor::new(bytes)).map_err(|_| "invalid CI report archive")?; + if archive.len() != 1 { + return Err("CI archive must contain one JUnit report".into()); + } + let entry = archive.by_index(0).map_err(|_| "CI report entry unavailable")?; + if !entry.is_file() + || entry.is_symlink() + || entry.encrypted() + || entry.enclosed_name().is_none() + || !entry.name().is_ok_and(|name| name.ends_with(".xml")) + || entry.size() == 0 + || entry.size() > MAX_XML_BYTES + { + return Err("invalid CI report entry".into()); + } + let expected = entry.size(); + let mut xml = Vec::new(); + entry + .take(MAX_XML_BYTES + 1) + .read_to_end(&mut xml) + .map_err(|_| "cannot read CI report")?; + if xml.len() as u64 != expected { + return Err("CI report size mismatch".into()); + } + parse(&xml) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Element { + Suites, + Suite, + Test, + Other, +} + +struct TestCase { + qualified_name: String, + failed: bool, +} + +fn test_case(element: &BytesStart<'_>) -> Result { + let mut class = None; + let mut name = None; + for attr in element.attributes() { + let attr = attr.map_err(|_| "invalid JUnit test attributes")?; + let slot = match attr.key.as_ref() { + "classname" => &mut class, + "name" => &mut name, + _ => continue, + }; + let text = attr + .normalized_value(XmlVersion::Implicit1_0) + .map_err(|_| "invalid JUnit test name")?; + if text.is_empty() || text.len() > MAX_NAME_BYTES || text.chars().any(char::is_control) { + return Err("invalid JUnit test name".into()); + } + *slot = Some(text.into_owned()); + } + let class = class.ok_or("JUnit test is missing a class name")?; + let name = name.ok_or("JUnit test is missing a name")?; + Ok(TestCase { + qualified_name: format!("{class}::{name}"), + failed: false, + }) +} + +fn parse(xml: &[u8]) -> Result, String> { + let mut reader = Reader::from_reader(xml); + reader.config_mut().expand_empty_elements = true; + let mut stack = Vec::new(); + let mut root_seen = false; + let mut test: Option = None; + let mut failed = BTreeSet::new(); + loop { + match reader.read_event().map_err(|_| "invalid JUnit XML")? { + Event::Start(tag) => { + let parent = stack.last().copied(); + let element = match (parent, tag.name().as_ref()) { + (None, "testsuites") => Element::Suites, + (None | Some(Element::Suites | Element::Suite), "testsuite") => Element::Suite, + (Some(Element::Suite), "testcase") => { + test = Some(test_case(&tag)?); + Element::Test + } + (Some(Element::Test), "failure" | "error") => { + test.as_mut().ok_or("missing JUnit test case")?.failed = true; + Element::Other + } + (None, _) | (_, "testcase" | "testsuite" | "testsuites") => { + return Err("unexpected JUnit element".into()); + } + _ => Element::Other, + }; + if parent.is_none() && std::mem::replace(&mut root_seen, true) { + return Err("multiple JUnit roots".into()); + } + if stack.len() == MAX_ELEMENTS_DEEP { + return Err("JUnit report exceeds nesting limit".into()); + } + stack.push(element); + } + Event::End(_) => { + if stack.pop().ok_or("unexpected JUnit closing tag")? == Element::Test { + let test = test.take().ok_or("missing JUnit test case")?; + if test.failed { + failed.insert(test.qualified_name); + } + } + } + Event::DocType(_) => return Err("JUnit document type declarations are unsupported".into()), + Event::Text(text) + if stack.is_empty() && !text.as_ref().bytes().all(|byte| byte.is_ascii_whitespace()) => + { + return Err("unexpected text outside JUnit root".into()); + } + Event::CData(_) | Event::GeneralRef(_) if stack.is_empty() => { + return Err("unexpected text outside JUnit root".into()); + } + Event::Eof => break, + _ => {} + } + } + if !root_seen || !stack.is_empty() { + return Err("incomplete JUnit report".into()); + } + Ok(failed.into_iter().collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + use zip::{write::SimpleFileOptions, CompressionMethod, ZipWriter}; + + fn archive(documents: &[(&str, &str)]) -> Vec { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let options = SimpleFileOptions::default().compression_method(CompressionMethod::Deflated); + for (path, xml) in documents { + writer.start_file(*path, options).unwrap(); + writer.write_all(xml.as_bytes()).unwrap(); + } + writer.finish().unwrap().into_inner() + } + + #[test] + fn finds_failed_and_errored_identities_without_reading_test_output_as_metadata() { + let xml = r#" + + arbitrary output + + <failure/> + +"#; + let bytes = archive(&[("junit.xml", xml)]); + assert_eq!( + failed_tests(&bytes).unwrap(), + ["crate::suite::errored", "crate::suite::failed"] + ); + } + + #[test] + fn refuses_untrusted_report_shapes() { + let good = ""; + for xml in [ + "", + "", + "&", + "", + "", + "", + ] { + assert!(failed_tests(&archive(&[("junit.xml", xml)])).is_err()); + } + assert!(failed_tests(&archive(&[("one.xml", good), ("two.xml", good)])).is_err()); + assert!(failed_tests(&archive(&[("../junit.xml", good)])).is_err()); + assert!(failed_tests(b"not an archive").is_err()); + } +} diff --git a/oxa/oxa-cli/tests/work_shadow.rs b/oxa/oxa-cli/tests/work_shadow.rs index 0d15e5bb4..9e2b86642 100644 --- a/oxa/oxa-cli/tests/work_shadow.rs +++ b/oxa/oxa-cli/tests/work_shadow.rs @@ -1119,7 +1119,7 @@ exit 94 row["verified_test_reports"], json!([{ "failed_check_runs":[11,12],"workflow_run":61,"run_attempt":1, - "archives":[{"id":71,"sha256":digest}] + "archives":[{"id":71,"sha256":digest}],"failed_tests":["example::failed"] }]) ); } @@ -1183,7 +1183,7 @@ exit 94 "{}", String::from_utf8_lossy(&result.stderr) ); - let text = fs::read_to_string(log).unwrap(); + let text = fs::read_to_string(&log).unwrap(); let sample: Value = serde_json::from_str(text.lines().last().unwrap()).unwrap(); assert_eq!( sample["queue_rejections"][0]["verified_test_reports"][0]["failed_check_runs"], @@ -1198,7 +1198,7 @@ exit 94 "read\nread\nread\n" ); - let mut wrong_suite = valid_queue; + let mut wrong_suite = valid_queue.clone(); let checks = &mut wrong_suite["data"]["repository"]["p43"]["timelineItems"]["nodes"][0]["beforeCommit"] ["statusCheckRollup"]["contexts"]["nodes"]; for check in checks.as_array_mut().unwrap() { @@ -1226,6 +1226,14 @@ exit 94 fs::read_to_string(env.work().join("report-reads")).unwrap(), "read\nread\nread\nread\n" ); + let invalid_archive = vec![0; bytes.len()]; + let invalid_digest = format!("{:x}", Sha256::digest(&invalid_archive)); + fs::write(env.work().join("report.zip"), invalid_archive).unwrap(); + write("report-artifacts.json", listing(&invalid_digest)); + let result = invoke(&env, &args); + assert_eq!(result.status.code(), Some(2)); + assert!(String::from_utf8_lossy(&result.stderr).contains("invalid CI report archive")); + assert_eq!(fs::read_to_string(log_for(&env, "example/repo")).unwrap(), text); assert!(!env.work().join("unexpected-github-command").exists()); } diff --git a/oxa/oxa-work/src/board/queue/report.rs b/oxa/oxa-work/src/board/queue/report.rs index 5cc69d2cb..5b813700f 100644 --- a/oxa/oxa-work/src/board/queue/report.rs +++ b/oxa/oxa-work/src/board/queue/report.rs @@ -1,5 +1,5 @@ //! Resolve a CI report from the rejected group's authenticated GitHub metadata. -use super::{github_utc, positive_id, RejectionEvidence}; +use super::{github_utc, positive_id, RejectionEvidence, WorkflowIdentity}; use serde::Serialize; use serde_json::Value; use std::collections::BTreeSet; @@ -40,6 +40,17 @@ pub struct VerifiedReports { pub workflow_run: u64, pub run_attempt: u64, pub archives: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub failed_tests: Vec, +} + +/// The removal and exact job identities an earlier archive verification used. +#[derive(PartialEq, Eq)] +pub struct ReportAttemptIdentity { + number: u64, + group: String, + removed_at: String, + checks: Vec<(u64, WorkflowIdentity)>, } impl RejectionEvidence { @@ -51,6 +62,24 @@ impl RejectionEvidence { self.failed_required_workflows.keys().copied() } + pub fn report_attempt_identity(&self, checks: &[u64]) -> Option { + let checks = checks + .iter() + .map(|id| { + self.failed_required_workflows + .get(id) + .copied() + .map(|workflow| (*id, workflow)) + }) + .collect::>()?; + Some(ReportAttemptIdentity { + number: self.number, + group: self.merge_group_head.clone(), + removed_at: self.removed_at.clone(), + checks, + }) + } + pub fn record_verified_reports(&mut self, report: VerifiedReports) { if !report.failed_check_runs.is_empty() && report.failed_check_runs.windows(2).all(|ids| ids[0] < ids[1]) @@ -191,6 +220,21 @@ mod tests { artifact_name: "test-report-2", max_archive_bytes: 1024, }; + let identity = evidence.report_attempt_identity(&[42]).unwrap(); + assert!(evidence.report_attempt_identity(&[9]).is_none()); + let mut shifted = evidence.clone(); + shifted.merge_group_head = DIGEST.into(); + assert!(shifted.report_attempt_identity(&[42]).unwrap() != identity); + shifted = evidence.clone(); + shifted.removed_at = "2026-01-01T00:08:58Z".into(); + assert!(shifted.report_attempt_identity(&[42]).unwrap() != identity); + shifted = evidence.clone(); + shifted + .failed_required_workflows + .get_mut(&42) + .unwrap() + .check_suite_id = 52; + assert!(shifted.report_attempt_identity(&[42]).unwrap() != identity); let failed = json!({"id":42,"run_id":61,"run_attempt":2,"head_sha":HEAD, "status":"completed","name":"required aggregate", "conclusion":"failure","completed_at":"2026-01-01T00:08:59Z"});