From 74bc58316613aee4635a3dc5796ad89a9390179b Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:27:07 +0100 Subject: [PATCH 01/14] test(browser): add og-browser, a Rust headless-Chrome driver for the browser checks og-browser starts the Chrome on the machine with --remote-debugging-pipe and drives one page over the DevTools protocol: go, eval, wait and the page's errors. It also serves a folder with a Content-Security-Policy and scans a site's files for outside URLs, inline scripts, on* attributes and eval. Without Chrome a check skips, and fails in CI. The Guard job rust runs it in cargo test, then two red controls: a page error planted in the clean page fails its test, and CHROME=/nonexistent fails in CI and skips outside it. Bead: D1 --- .github/workflows/guard.yml | 30 ++ rust/Cargo.lock | 9 + rust/Cargo.toml | 2 +- rust/crates/og-browser/Cargo.toml | 20 ++ rust/crates/og-browser/src/assets.rs | 134 ++++++++ rust/crates/og-browser/src/cdp.rs | 345 ++++++++++++++++++++ rust/crates/og-browser/src/lib.rs | 187 +++++++++++ rust/crates/og-browser/src/serve.rs | 225 +++++++++++++ rust/crates/og-browser/tests/cdp.rs | 114 +++++++ rust/crates/og-browser/tests/live_assets.rs | 42 +++ rust/crates/og-browser/tests/live_replay.rs | 194 +++++++++++ tests/browser/README.md | 55 ++++ 12 files changed, 1356 insertions(+), 1 deletion(-) create mode 100644 rust/crates/og-browser/Cargo.toml create mode 100644 rust/crates/og-browser/src/assets.rs create mode 100644 rust/crates/og-browser/src/cdp.rs create mode 100644 rust/crates/og-browser/src/lib.rs create mode 100644 rust/crates/og-browser/src/serve.rs create mode 100644 rust/crates/og-browser/tests/cdp.rs create mode 100644 rust/crates/og-browser/tests/live_assets.rs create mode 100644 rust/crates/og-browser/tests/live_replay.rs create mode 100644 tests/browser/README.md diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index d203d4668..32d0bffbc 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -480,6 +480,36 @@ jobs: - name: cargo test run: cargo test --workspace --locked --no-fail-fast + # The browser checks of og-browser (tests/browser/README.md) ran in + # cargo test above, in the runner's Chrome. Two controls: a page error + # planted in the clean page must fail its test, and with no Chrome the + # tests must fail in CI and only skip outside it. + - name: The browser checks fail on a planted page error, and without Chrome in CI + if: ${{ !cancelled() }} + run: | + log="${RUNNER_TEMP}/og-browser-red.log" + if cargo test --locked --no-fail-fast -p og-browser --features planted-page-error --test cdp > "$log" 2>&1; then + cat "$log" + echo "ERROR: the clean page's test passed with a planted page error" + exit 1 + fi + cat "$log" + grep -E 'test a_clean_page_loads_with_no_page_error \.\.\. FAILED' "$log" + grep -F "Cannot read properties of null (reading 'planted')" "$log" + echo "Control: a planted page error failed the clean page's test." + if CHROME=/nonexistent cargo test --locked --no-fail-fast -p og-browser --test cdp > "$log" 2>&1; then + cat "$log" + echo "ERROR: the browser checks passed in CI without Chrome" + exit 1 + fi + cat "$log" + grep -F "[FAIL] no Chrome or Chromium to load the test page" "$log" + echo "Control: without Chrome the browser checks fail in CI." + CHROME=/nonexistent CI= cargo test --locked -p og-browser --test cdp -- --nocapture > "$log" 2>&1 || { cat "$log"; exit 1; } + cat "$log" + test "$(grep -c -F '[SKIP] no Chrome or Chromium to load the test page' "$log")" -eq 2 + echo "Outside CI, without Chrome the browser checks skip." + # The planned-verb test (og/tests/port_scaffold.rs) must fail when a # later merge drops a verb's arm. - name: The planned-verb test fails when a verb is dropped diff --git a/rust/Cargo.lock b/rust/Cargo.lock index ec8e92fcc..a79d82ca0 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -671,6 +671,15 @@ dependencies = [ "pycompat", ] +[[package]] +name = "og-browser" +version = "0.1.0" +dependencies = [ + "indexmap", + "pycompat", + "regex", +] + [[package]] name = "og-chip" version = "0.1.0" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 416ee70c9..386ca20d6 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "3" -members = ["crates/pycompat", "crates/og-io", "crates/og-vcf", "crates/og-bam", "crates/og-map", "crates/og-tsv", "crates/og-intervals", "crates/og-dev", "crates/og-qc", "crates/og-clinvar", "crates/og-pgx", "crates/og-filters", "crates/og-nf", "crates/og-runlog", "crates/og-steps", "crates/og-setup", "crates/og-chip", "crates/og-cn", "crates/og-bloodtype", "crates/og-ancestry", "crates/og-pedigree", "crates/og-reports", "crates/og-summary", "crates/og-lookup", "crates/og-fresh", "crates/og-recheck", "crates/og-render", "crates/og-live", "crates/og-couple", "crates/og-admix", "og"] +members = ["crates/pycompat", "crates/og-io", "crates/og-vcf", "crates/og-bam", "crates/og-map", "crates/og-tsv", "crates/og-intervals", "crates/og-dev", "crates/og-qc", "crates/og-clinvar", "crates/og-pgx", "crates/og-filters", "crates/og-nf", "crates/og-runlog", "crates/og-steps", "crates/og-setup", "crates/og-chip", "crates/og-cn", "crates/og-bloodtype", "crates/og-ancestry", "crates/og-pedigree", "crates/og-reports", "crates/og-summary", "crates/og-lookup", "crates/og-fresh", "crates/og-recheck", "crates/og-render", "crates/og-live", "crates/og-couple", "crates/og-admix", "crates/og-browser", "og"] [workspace.package] version = "0.1.0" diff --git a/rust/crates/og-browser/Cargo.toml b/rust/crates/og-browser/Cargo.toml new file mode 100644 index 000000000..a8c3633a5 --- /dev/null +++ b/rust/crates/og-browser/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "og-browser" +description = "Test support: headless Chrome over the DevTools protocol, a static file server and the asset scan, for the browser checks" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +[features] +# A planted page error: the clean page of tests/cdp.rs throws, and that test +# must fail (the red control in .github/workflows/guard.yml). +planted-page-error = [] + +[dependencies] +pycompat.workspace = true +indexmap.workspace = true +regex.workspace = true + +[lints] +workspace = true diff --git a/rust/crates/og-browser/src/assets.rs b/rust/crates/og-browser/src/assets.rs new file mode 100644 index 000000000..55103e55f --- /dev/null +++ b/rust/crates/og-browser/src/assets.rs @@ -0,0 +1,134 @@ +//! The scan of a site's files for anything that would load or run code +//! from outside the files themselves. + +use regex::Regex; +use std::collections::BTreeMap; +use std::path::Path; +use std::sync::LazyLock; + +static URL: LazyLock = + LazyLock::new(|| Regex::new(r#"(?:https?:)?//[A-Za-z0-9.-]+\.[a-z]{2,}[^\s'"]*"#).unwrap()); +static SCRIPT_TAG: LazyLock = LazyLock::new(|| Regex::new(r"(?i)]*>").unwrap()); +static ON_ATTR: LazyLock = LazyLock::new(|| Regex::new(r"(?i)\son[a-z]+\s*=").unwrap()); +static EVAL: LazyLock = + LazyLock::new(|| Regex::new(r"\beval\s*\(|new\s+Function\s*\(").unwrap()); + +/// The SVG namespace name: a URL in the text, never fetched. +const SVG_NS: &str = "http://www.w3.org/2000/svg"; + +/// (file, problem) for each outside URL in any file, each inline +/// `" + ), + [("a.html".into(), "" + ) + .is_empty() + ); + assert!(one("a.js", "fetch('/state.json'); var u = 'a//b';").is_empty()); + } +} diff --git a/rust/crates/og-browser/src/cdp.rs b/rust/crates/og-browser/src/cdp.rs new file mode 100644 index 000000000..13fea93dd --- /dev/null +++ b/rust/crates/og-browser/src/cdp.rs @@ -0,0 +1,345 @@ +//! One headless Chrome and one page, driven over the DevTools protocol. +//! +//! Chrome runs with `--remote-debugging-pipe`: it reads commands on fd 3 +//! and writes answers and events on fd 4, each one JSON message ended by a +//! NUL byte. `/bin/sh` moves our two pipes to those numbers before it +//! starts Chrome, so no port is opened and no unsafe code is needed. + +use indexmap::IndexMap; +use pycompat::json::{self, Dump, Value}; +use std::io::{BufRead, BufReader, Write}; +use std::path::Path; +use std::process::{Child, ChildStdin, Command, Stdio}; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError}; +use std::thread; +use std::time::{Duration, Instant}; + +/// How long one call may take before it is an error. +const CALL_TIMEOUT: Duration = Duration::from_secs(30); + +/// A headless Chrome with one page attached. Dropping it closes Chrome. +pub struct Browser { + child: Child, + to_chrome: ChildStdin, + from_chrome: Receiver, + next_id: i128, + session: String, + errors: Vec, +} + +impl Browser { + /// Start `chrome` with its profile in `profile` and attach to a blank + /// page. A headless Chrome on a busy CI runner sometimes never answers + /// its first call, so a fresh one (profile `-2`, `-3`) is + /// started, up to three times. + pub fn launch(chrome: &Path, profile: &Path) -> Result { + let mut last = String::new(); + for attempt in 1..=3 { + let dir = if attempt == 1 { + profile.to_path_buf() + } else { + let mut p = profile.as_os_str().to_owned(); + p.push(format!("-{attempt}")); + p.into() + }; + match Browser::start(chrome, &dir) { + Ok(b) => return Ok(b), + Err(e) => { + if attempt < 3 { + eprintln!( + "Chrome did not start ({e}); trying again ({} of 3)", + attempt + 1 + ); + } + last = e; + } + } + } + Err(last) + } + + fn start(chrome: &Path, profile: &Path) -> Result { + let mut user_data = std::ffi::OsString::from("--user-data-dir="); + user_data.push(profile); + let mut child = Command::new("/bin/sh") + .arg("-c") + .arg(r#"exec "$0" "$@" 3<&0 4>&1 /dev/null"#) + .arg(chrome) + .args([ + "--headless=new", + "--disable-gpu", + "--no-sandbox", + "--no-first-run", + "--no-default-browser-check", + ]) + .arg(user_data) + .args(["--remote-debugging-pipe", "about:blank"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .map_err(|e| format!("cannot start {}: {e}", chrome.display()))?; + let to_chrome = child.stdin.take().expect("stdin is piped"); + let out = child.stdout.take().expect("stdout is piped"); + let (tx, from_chrome) = mpsc::channel(); + thread::spawn(move || { + let mut r = BufReader::new(out); + let mut buf = Vec::new(); + loop { + buf.clear(); + match r.read_until(0, &mut buf) { + Ok(0) | Err(_) => break, + Ok(_) => {} + } + if buf.last() == Some(&0) { + buf.pop(); + } + if let Ok(v) = json::loads(&String::from_utf8_lossy(&buf)) + && tx.send(v).is_err() + { + break; + } + } + }); + let mut b = Browser { + child, + to_chrome, + from_chrome, + next_id: 0, + session: String::new(), + errors: Vec::new(), + }; + let attached = (|| { + let t = b.send( + "Target.createTarget", + obj([("url", s("about:blank"))]), + None, + CALL_TIMEOUT, + )?; + let target = field(&t, "targetId"); + let a = b.send( + "Target.attachToTarget", + obj([("targetId", target), ("flatten", Value::Bool(true))]), + None, + CALL_TIMEOUT, + )?; + b.session = match field(&a, "sessionId") { + Value::Str(id) => id, + v => return Err(format!("Target.attachToTarget: no sessionId in {v:?}")), + }; + b.call("Runtime.enable", obj([]))?; + b.call("Page.enable", obj([])) + })(); + match attached { + Ok(_) => Ok(b), + Err(e) => { + b.kill(); + Err(e) + } + } + } + + /// Call `method` on the page with `params` (an object) and return its + /// result, or the protocol's error. + pub fn call(&mut self, method: &str, params: Value) -> Result { + let session = self.session.clone(); + self.send(method, params, Some(&session), CALL_TIMEOUT) + } + + fn send( + &mut self, + method: &str, + params: Value, + session: Option<&str>, + timeout: Duration, + ) -> Result { + self.next_id += 1; + let id = self.next_id; + let mut msg = IndexMap::new(); + msg.insert("id".to_string(), Value::Int(id)); + msg.insert("method".to_string(), s(method)); + msg.insert("params".to_string(), params); + if let Some(sid) = session { + msg.insert("sessionId".to_string(), s(sid)); + } + let mut line = json::dumps(&Value::Object(msg), &Dump::default()) + .map_err(|e| format!("{method}: {e:?}"))?; + line.push('\0'); + self.to_chrome + .write_all(line.as_bytes()) + .and_then(|()| self.to_chrome.flush()) + .map_err(|e| format!("{method}: Chrome closed the pipe ({e})"))?; + let end = Instant::now() + timeout; + loop { + let left = end.saturating_duration_since(Instant::now()); + let m = match self.from_chrome.recv_timeout(left) { + Ok(m) => m, + Err(RecvTimeoutError::Timeout) => { + return Err(format!("{method}: no answer in {} s", timeout.as_secs())); + } + Err(RecvTimeoutError::Disconnected) => { + return Err(format!("{method}: Chrome closed the pipe")); + } + }; + if field(&m, "id") != Value::Int(id) { + self.note(&m); + continue; + } + if let Value::Object(e) = field(&m, "error") { + return Err(format!( + "{method}: {}", + field(&Value::Object(e), "message").as_text() + )); + } + return Ok(field(&m, "result")); + } + } + + /// Keep an event that reports a page error: an uncaught exception or a + /// `console.error` call. + fn note(&mut self, m: &Value) { + let p = field(m, "params"); + match field(m, "method").as_text().as_str() { + "Runtime.exceptionThrown" => { + let d = field(&p, "exceptionDetails"); + let desc = field(&field(&d, "exception"), "description").as_text(); + self.errors.push(if desc.is_empty() { + field(&d, "text").as_text() + } else { + desc + }); + } + "Runtime.consoleAPICalled" if field(&p, "type") == s("error") => { + let args = match field(&p, "args") { + Value::Array(a) => a, + _ => Vec::new(), + }; + let words: Vec = args + .iter() + .map(|a| match field(a, "value") { + Value::Null => field(a, "description").as_text(), + v => v.as_text(), + }) + .collect(); + self.errors + .push(format!("console.error: {}", words.join(" "))); + } + _ => {} + } + } + + /// Open `url` in the page. It returns once Chrome has started the + /// navigation; [`Browser::wait`] for what the page must show. + pub fn go(&mut self, url: &str) -> Result<(), String> { + self.call("Page.navigate", obj([("url", s(url))])).map(drop) + } + + /// The value of the JavaScript expression `expr` in the page (`Null` + /// for `undefined`), or the exception it threw. + pub fn eval(&mut self, expr: &str) -> Result { + let r = self.call( + "Runtime.evaluate", + obj([ + ("expression", s(expr)), + ("returnByValue", Value::Bool(true)), + ]), + )?; + if let Value::Object(d) = field(&r, "exceptionDetails") { + let d = Value::Object(d); + let desc = field(&field(&d, "exception"), "description").as_text(); + return Err(if desc.is_empty() { + field(&d, "text").as_text() + } else { + desc + }); + } + Ok(field(&field(&r, "result"), "value")) + } + + /// Evaluate `expr` every 0.2 s until its value is truthy (as in + /// JavaScript and Python), and return that value; `None` after + /// `timeout`. An expression that throws counts as not yet. + pub fn wait(&mut self, expr: &str, timeout: Duration) -> Option { + let end = Instant::now() + timeout; + loop { + if let Ok(v) = self.eval(expr) + && pycompat::value::truthy(&v) + { + return Some(v); + } + if Instant::now() >= end { + return None; + } + thread::sleep(Duration::from_millis(200)); + } + } + + /// The page's uncaught exceptions and `console.error` calls so far, in + /// the order they came. Errors of [`Browser::eval`]'s own expressions + /// are not among them. + pub fn page_errors(&mut self) -> Vec { + while let Ok(m) = self.from_chrome.try_recv() { + self.note(&m); + } + self.errors.clone() + } + + fn kill(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +impl Drop for Browser { + /// Ask Chrome to close, give it 10 s, then kill it. + fn drop(&mut self) { + let _ = self.send("Browser.close", obj([]), None, Duration::from_secs(5)); + let end = Instant::now() + Duration::from_secs(10); + while Instant::now() < end { + if matches!(self.child.try_wait(), Ok(Some(_))) { + return; + } + thread::sleep(Duration::from_millis(100)); + } + self.kill(); + } +} + +/// `s` as a JavaScript string literal, to build an expression for +/// [`Browser::eval`] (Python's `json.dumps(s)`). +pub fn js_str(text: &str) -> String { + let mut out = String::new(); + json::quote(&mut out, text, true); + out +} + +fn s(text: &str) -> Value { + Value::Str(text.to_string()) +} + +fn obj(pairs: [(&str, Value); N]) -> Value { + Value::Object(pairs.into_iter().map(|(k, v)| (k.to_string(), v)).collect()) +} + +/// `v[key]`, `Null` when `v` is not an object or has no such key. +fn field(v: &Value, key: &str) -> Value { + match v { + Value::Object(m) => m.get(key).cloned().unwrap_or(Value::Null), + _ => Value::Null, + } +} + +trait Text { + fn as_text(&self) -> String; +} + +impl Text for Value { + /// A string as it is, anything else as JSON. + fn as_text(&self) -> String { + match self { + Value::Str(t) => t.clone(), + Value::Null => String::new(), + v => json::dumps(v, &Dump::default()).unwrap_or_default(), + } + } +} diff --git a/rust/crates/og-browser/src/lib.rs b/rust/crates/og-browser/src/lib.rs new file mode 100644 index 000000000..a23c1e4d0 --- /dev/null +++ b/rust/crates/og-browser/src/lib.rs @@ -0,0 +1,187 @@ +//! Test support for OwnGenome's browser checks: a headless Chrome driven +//! over the DevTools protocol, a static file server for the pages and the +//! scan of the live site's assets. No product code depends on it. +//! +//! - [`find_chrome`] and [`chrome_or_skip`]: which Chrome to run, and what a +//! test does without one (skip on a laptop, fail in CI). +//! - [`Browser`]: one headless Chrome and one page: [`Browser::go`], +//! [`Browser::eval`], [`Browser::wait`] and [`Browser::page_errors`]. +//! - [`StaticServer`]: a directory on 127.0.0.1 with the headers a test +//! asks for (a Content-Security-Policy). +//! - [`asset_problems`]: outside URLs, inline scripts, `on*` attributes and +//! `eval` in a site's files. +//! +//! How the checks run: `tests/browser/README.md`. + +mod assets; +mod cdp; +mod serve; + +pub use assets::{asset_problems, read_assets}; +pub use cdp::{Browser, js_str}; +pub use pycompat::json::Value; +pub use serve::StaticServer; + +use std::ffi::OsStr; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; + +/// The names tried on `PATH` when `CHROME` is not set, then the macOS app. +const NAMES: [&str; 4] = [ + "google-chrome", + "google-chrome-stable", + "chromium", + "chromium-browser", +]; +const MAC_APP: &str = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"; + +/// The Chrome to run: `CHROME` when it is set, else the first of +/// `google-chrome`, `google-chrome-stable`, `chromium`, `chromium-browser` +/// on `PATH`, else Google Chrome's macOS app. `None` when there is none. +pub fn find_chrome() -> Option { + find_chrome_in( + std::env::var_os("CHROME").as_deref(), + std::env::var_os("PATH").as_deref(), + ) +} + +/// [`find_chrome`] with `CHROME` and `PATH` given. A `CHROME` that is set +/// but names no executable gives `None`: a wrong path fails loudly instead +/// of falling back to another Chrome. +pub fn find_chrome_in(chrome: Option<&OsStr>, path: Option<&OsStr>) -> Option { + if let Some(c) = chrome.filter(|c| !c.is_empty()) { + return resolve(Path::new(c), path); + } + NAMES + .iter() + .find_map(|n| resolve(Path::new(n), path)) + .or_else(|| resolve(Path::new(MAC_APP), None)) +} + +/// A path with a slash as it is, a bare name looked up on `path`; only an +/// executable file counts. +fn resolve(name: &Path, path: Option<&OsStr>) -> Option { + let exe = |p: &Path| { + p.metadata() + .is_ok_and(|m| m.is_file() && m.permissions().mode() & 0o111 != 0) + }; + if name.components().count() > 1 { + return exe(name).then(|| name.to_path_buf()); + } + std::env::split_paths(path?) + .map(|d| d.join(name)) + .find(|p| exe(p)) +} + +/// The Chrome for a test of `what`, or `None` after a `[SKIP]` line when +/// there is none. In CI (`CI=true`) a missing Chrome panics, so a runner +/// without one can never pass the browser checks by skipping them. +pub fn chrome_or_skip(what: &str) -> Option { + let ci = std::env::var("CI").is_ok_and(|v| v == "true"); + match without_chrome(find_chrome(), ci, what) { + Ok(c) => c, + Err(e) => panic!("{e}"), + } +} + +/// The decision of [`chrome_or_skip`]: the Chrome found, `Ok(None)` after a +/// `[SKIP]` line, or the failure in CI. +fn without_chrome(found: Option, ci: bool, what: &str) -> Result, String> { + match found { + Some(c) => Ok(Some(c)), + None if ci => Err(format!( + "[FAIL] no Chrome or Chromium to {what}: CI must run the browser checks (set CHROME)" + )), + None => { + println!("[SKIP] no Chrome or Chromium to {what}"); + Ok(None) + } + } +} + +/// The repository's root: three levels above this crate. +pub fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .ancestors() + .nth(3) + .expect("rust/crates/og-browser sits three levels below the root") + .to_path_buf() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + /// A fresh scratch folder for one test. + fn scratch(name: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("og-browser-{}-{name}", std::process::id())); + let _ = fs::remove_dir_all(&d); + fs::create_dir_all(&d).unwrap(); + d + } + + fn executable(p: &Path, mode: u32) { + fs::write(p, "#!/bin/sh\n").unwrap(); + fs::set_permissions(p, fs::Permissions::from_mode(mode)).unwrap(); + } + + #[test] + fn chrome_names_an_executable_or_nothing() { + let d = scratch("chrome-env"); + let c = d.join("my-chrome"); + executable(&c, 0o755); + assert_eq!(find_chrome_in(Some(c.as_os_str()), None), Some(c.clone())); + // Set but wrong: no fallback to a Chrome on PATH. + let bin = d.join("bin"); + fs::create_dir(&bin).unwrap(); + executable(&bin.join("chromium"), 0o755); + let missing = d.join("nothing-here"); + assert_eq!( + find_chrome_in(Some(missing.as_os_str()), Some(bin.as_os_str())), + None + ); + // Not executable. + let plain = d.join("plain"); + executable(&plain, 0o644); + assert_eq!(find_chrome_in(Some(plain.as_os_str()), None), None); + fs::remove_dir_all(&d).unwrap(); + } + + #[test] + fn without_chrome_set_a_name_on_path_is_found() { + let d = scratch("path"); + let (a, b) = (d.join("a"), d.join("b")); + fs::create_dir(&a).unwrap(); + fs::create_dir(&b).unwrap(); + executable(&b.join("chromium"), 0o755); + executable(&a.join("google-chrome-stable"), 0o644); // not executable: skipped + let path = std::env::join_paths([&a, &b]).unwrap(); + assert_eq!(find_chrome_in(None, Some(&path)), Some(b.join("chromium"))); + assert_eq!( + find_chrome_in(Some(OsStr::new("")), Some(&path)), + Some(b.join("chromium")) + ); + // Nothing on PATH: only the macOS app can still be found. + let found = find_chrome_in(None, Some(d.join("empty").as_os_str())); + assert!( + found.is_none() || found == Some(PathBuf::from(MAC_APP)), + "{found:?}" + ); + fs::remove_dir_all(&d).unwrap(); + } + + #[test] + fn a_missing_chrome_skips_locally_and_fails_in_ci() { + assert_eq!(without_chrome(None, false, "load the page"), Ok(None)); + let e = without_chrome(None, true, "load the page").unwrap_err(); + assert!(e.contains("no Chrome or Chromium to load the page"), "{e}"); + let c = PathBuf::from("/x/chrome"); + assert_eq!(without_chrome(Some(c.clone()), true, "x"), Ok(Some(c))); + } + + #[test] + fn the_root_holds_the_live_assets() { + assert!(repo_root().join("assets/live/live.js").is_file()); + } +} diff --git a/rust/crates/og-browser/src/serve.rs b/rust/crates/og-browser/src/serve.rs new file mode 100644 index 000000000..0a7919357 --- /dev/null +++ b/rust/crates/og-browser/src/serve.rs @@ -0,0 +1,225 @@ +//! A directory served on 127.0.0.1 for a page under test: GET only, one +//! request per connection, the headers the test asks for on every answer. + +use std::fs; +use std::io::{BufRead, BufReader, Write}; +use std::net::{Shutdown, TcpListener, TcpStream}; +use std::path::{Component, Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::thread::{self, JoinHandle}; +use std::time::Duration; + +/// A static file server on a free port of 127.0.0.1. It stops when dropped. +pub struct StaticServer { + port: u16, + stop: Arc, + accept: Option>, +} + +impl StaticServer { + /// Serve the files under `dir`, with `headers` (name, value) added to + /// every answer. A path ending in `/` serves its `index.html`. + pub fn start(dir: &Path, headers: &[(&str, &str)]) -> std::io::Result { + let listener = TcpListener::bind("127.0.0.1:0")?; + let port = listener.local_addr()?.port(); + let stop = Arc::new(AtomicBool::new(false)); + let (root, extra) = ( + Arc::new(dir.to_path_buf()), + Arc::new( + headers + .iter() + .map(|(k, v)| format!("{k}: {v}\r\n")) + .collect::(), + ), + ); + let flag = Arc::clone(&stop); + let accept = thread::spawn(move || { + for conn in listener.incoming() { + if flag.load(Ordering::SeqCst) { + break; + } + if let Ok(c) = conn { + let (root, extra) = (Arc::clone(&root), Arc::clone(&extra)); + thread::spawn(move || answer(c, &root, &extra)); + } + } + }); + Ok(StaticServer { + port, + stop, + accept: Some(accept), + }) + } + + pub fn port(&self) -> u16 { + self.port + } + + /// `http://127.0.0.1:/`. + pub fn url(&self, path: &str) -> String { + format!( + "http://127.0.0.1:{}/{}", + self.port, + path.trim_start_matches('/') + ) + } +} + +impl Drop for StaticServer { + fn drop(&mut self) { + self.stop.store(true, Ordering::SeqCst); + // Wake the accept loop so it sees the flag. + let _ = TcpStream::connect(("127.0.0.1", self.port)); + if let Some(h) = self.accept.take() { + let _ = h.join(); + } + } +} + +fn answer(mut c: TcpStream, root: &Path, extra: &str) { + let _ = c.set_read_timeout(Some(Duration::from_secs(10))); + let mut r = BufReader::new(&c); + let mut first = String::new(); + if r.read_line(&mut first).is_err() { + return; + } + // Read the rest of the head; a GET has no body. + let mut line = String::new(); + while r.read_line(&mut line).is_ok_and(|n| n > 2) { + line.clear(); + } + let mut parts = first.split_whitespace(); + let (method, target) = (parts.next().unwrap_or(""), parts.next().unwrap_or("")); + let (status, ctype, body) = if method != "GET" { + ( + "405 Method Not Allowed", + "text/plain", + b"method not allowed\n".to_vec(), + ) + } else { + match file_of(root, target).and_then(|p| fs::read(&p).ok().map(|b| (p, b))) { + Some((p, b)) => ("200 OK", content_type(&p), b), + None => ("404 Not Found", "text/plain", b"not found\n".to_vec()), + } + }; + let head = format!( + "HTTP/1.1 {status}\r\nContent-Type: {ctype}\r\nContent-Length: {}\r\n{extra}Connection: close\r\n\r\n", + body.len() + ); + let _ = c + .write_all(head.as_bytes()) + .and_then(|()| c.write_all(&body)); + let _ = c.shutdown(Shutdown::Write); +} + +/// The file under `root` that the request target names, or `None` for a +/// path that leaves `root`. +fn file_of(root: &Path, target: &str) -> Option { + let path = target.split(['?', '#']).next()?; + let mut rel = percent_decode(path.strip_prefix('/')?)?; + if rel.is_empty() || rel.ends_with('/') { + rel.push_str("index.html"); + } + let rel = Path::new(&rel); + if !rel.components().all(|c| matches!(c, Component::Normal(_))) { + return None; + } + Some(root.join(rel)) +} + +fn percent_decode(s: &str) -> Option { + let b = s.as_bytes(); + let mut out = Vec::with_capacity(b.len()); + let mut i = 0; + while i < b.len() { + if b[i] == b'%' { + let hex = std::str::from_utf8(b.get(i + 1..i + 3)?).ok()?; + out.push(u8::from_str_radix(hex, 16).ok()?); + i += 3; + } else { + out.push(b[i]); + i += 1; + } + } + String::from_utf8(out).ok() +} + +fn content_type(p: &Path) -> &'static str { + match p.extension().and_then(|e| e.to_str()) { + Some("html") => "text/html; charset=utf-8", + Some("js") => "text/javascript; charset=utf-8", + Some("css") => "text/css; charset=utf-8", + Some("json") => "application/json", + Some("svg") => "image/svg+xml", + Some("png") => "image/png", + _ => "application/octet-stream", + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_target_maps_into_the_root_only() { + let root = Path::new("/srv"); + assert_eq!(file_of(root, "/"), Some(PathBuf::from("/srv/index.html"))); + assert_eq!( + file_of(root, "/a/b.js?x=1"), + Some(PathBuf::from("/srv/a/b.js")) + ); + assert_eq!( + file_of(root, "/my%20page.html"), + Some(PathBuf::from("/srv/my page.html")) + ); + assert_eq!( + file_of(root, "/running/"), + Some(PathBuf::from("/srv/running/index.html")) + ); + assert_eq!(file_of(root, "/../etc/passwd"), None); + assert_eq!(file_of(root, "/a/%2e%2e/%2e%2e/x"), None); + assert_eq!(file_of(root, "/%zz"), None); + assert_eq!(file_of(root, "no-slash"), None); + } + + #[test] + fn it_serves_a_file_with_the_headers_and_404_otherwise() { + let d = std::env::temp_dir().join(format!("og-browser-serve-{}", std::process::id())); + let _ = fs::remove_dir_all(&d); + fs::create_dir_all(&d).unwrap(); + fs::write(d.join("a.json"), "{}").unwrap(); + let srv = + StaticServer::start(&d, &[("Content-Security-Policy", "default-src 'none'")]).unwrap(); + let get = |path: &str| { + let mut c = TcpStream::connect(("127.0.0.1", srv.port())).unwrap(); + write!(c, "GET {path} HTTP/1.1\r\nHost: x\r\n\r\n").unwrap(); + let mut s = String::new(); + std::io::Read::read_to_string(&mut c, &mut s).unwrap(); + s + }; + let ok = get("/a.json"); + assert!(ok.starts_with("HTTP/1.1 200 OK\r\n"), "{ok}"); + assert!( + ok.contains("\r\nContent-Type: application/json\r\n"), + "{ok}" + ); + assert!( + ok.contains("\r\nContent-Security-Policy: default-src 'none'\r\n"), + "{ok}" + ); + assert!(ok.ends_with("\r\n\r\n{}"), "{ok}"); + let missing = get("/b.json"); + assert!( + missing.starts_with("HTTP/1.1 404 Not Found\r\n"), + "{missing}" + ); + assert!(missing.contains("Content-Security-Policy"), "{missing}"); + assert_eq!( + srv.url("/x/y.html"), + format!("http://127.0.0.1:{}/x/y.html", srv.port()) + ); + drop(srv); + fs::remove_dir_all(&d).unwrap(); + } +} diff --git a/rust/crates/og-browser/tests/cdp.rs b/rust/crates/og-browser/tests/cdp.rs new file mode 100644 index 000000000..c3c7ec3f6 --- /dev/null +++ b/rust/crates/og-browser/tests/cdp.rs @@ -0,0 +1,114 @@ +//! The driver itself, on two tiny pages: a clean one must load with no page +//! error, and one with a planted exception and `console.error` must report +//! both. With the `planted-page-error` feature the clean page throws too, +//! and its test must fail (the red control in guard.yml). Without Chrome +//! both say SKIP, and fail in CI. + +use og_browser::{Browser, StaticServer, Value, chrome_or_skip, js_str}; +use std::fs; +use std::path::PathBuf; +use std::time::Duration; + +const CSP: &str = "default-src 'none'; script-src 'self'"; + +const PAGE: &str = "\n"; + +#[cfg(not(feature = "planted-page-error"))] +const CLEAN_JS: &str = "document.body.dataset.ok = 'yes'; window.answer = 6 * 7;\n"; +#[cfg(feature = "planted-page-error")] +const CLEAN_JS: &str = "document.body.dataset.ok = 'yes'; window.answer = 6 * 7; null.planted;\n"; + +const ERRORS_JS: &str = "document.body.dataset.ok = 'yes'; console.error('planted console error', 7); \ + setTimeout(function () { throw new Error('planted page error'); }, 0);\n"; + +/// A fresh folder for one test, under cargo's scratch folder. +fn scratch(name: &str) -> PathBuf { + let d = PathBuf::from(env!("CARGO_TARGET_TMPDIR")) + .join(format!("og-browser-{name}-{}", std::process::id())); + let _ = fs::remove_dir_all(&d); + fs::create_dir_all(d.join("site")).unwrap(); + d +} + +/// Serve PAGE with `js` as its script, open it and wait until it ran. +fn open(name: &str, js: &str) -> Option<(Browser, StaticServer, PathBuf)> { + let chrome = chrome_or_skip("load the test page")?; + let d = scratch(name); + fs::write(d.join("site/index.html"), PAGE).unwrap(); + fs::write(d.join("site/app.js"), js).unwrap(); + let srv = StaticServer::start(&d.join("site"), &[("Content-Security-Policy", CSP)]).unwrap(); + let mut b = Browser::launch(&chrome, &d.join("chrome")).unwrap(); + b.go(&srv.url("/")).unwrap(); + let ok = b.wait( + "document.body && document.body.dataset.ok", + Duration::from_secs(20), + ); + assert_eq!( + ok, + Some(Value::Str("yes".into())), + "the page did not run its script" + ); + Some((b, srv, d)) +} + +#[test] +fn a_clean_page_loads_with_no_page_error() { + let Some((mut b, srv, d)) = open("clean", CLEAN_JS) else { + return; + }; + assert_eq!(b.eval("window.answer").unwrap(), Value::Int(42)); + assert_eq!( + b.eval(&format!("{} + '!'", js_str("it's \"quoted\""))) + .unwrap(), + Value::Str("it's \"quoted\"!".into()) + ); + assert_eq!(b.eval("undefined").unwrap(), Value::Null); + // An exception in our own expression is an Err, not a page error. + assert!( + b.eval("nothing.here") + .unwrap_err() + .contains("nothing is not defined") + ); + // wait gives up on a value that never turns truthy. + assert_eq!( + b.wait("document.body.dataset.never", Duration::from_millis(500)), + None + ); + std::thread::sleep(Duration::from_millis(300)); + assert_eq!( + b.page_errors(), + Vec::::new(), + "the clean page reported errors" + ); + drop((b, srv)); + let _ = fs::remove_dir_all(d); +} + +#[test] +fn a_planted_exception_and_console_error_are_reported() { + let Some((mut b, srv, d)) = open("errors", ERRORS_JS) else { + return; + }; + let mut errors = Vec::new(); + for _ in 0..50 { + errors = b.page_errors(); + if errors.len() >= 2 { + break; + } + std::thread::sleep(Duration::from_millis(100)); + } + assert!( + errors + .iter() + .any(|e| e == "console.error: planted console error 7"), + "{errors:?}" + ); + assert!( + errors + .iter() + .any(|e| e.contains("Error: planted page error")), + "{errors:?}" + ); + drop((b, srv)); + let _ = fs::remove_dir_all(d); +} diff --git a/rust/crates/og-browser/tests/live_assets.rs b/rust/crates/og-browser/tests/live_assets.rs new file mode 100644 index 000000000..792ccac2e --- /dev/null +++ b/rust/crates/og-browser/tests/live_assets.rs @@ -0,0 +1,42 @@ +//! assets/live, its plugins included, holds no outside URL, inline script, +//! on* attribute or eval. Moved here from tests/test_owngenome_live.py +//! (check 7), with its two planted mistakes, which the scan must catch. + +use og_browser::{asset_problems, read_assets, repo_root}; + +#[test] +fn live_assets_load_nothing_from_outside() { + let texts = read_assets(&repo_root().join("assets/live")).unwrap(); + for f in ["index.html", "live.js", "plugins/charts.js"] { + assert!( + texts.contains_key(f), + "{f} was not read: {:?}", + texts.keys() + ); + } + assert_eq!(asset_problems(&texts), Vec::<(String, String)>::new()); + + let mut planted = texts.clone(); + let page = planted.get_mut("index.html").unwrap(); + assert!(page.contains("")); + *page = page.replace("", ""); + assert_eq!( + asset_problems(&planted), + [("index.html".to_string(), "") - control("an inline script in index.html", not asset_problems(planted)) - planted = dict(texts, **{"live.js": texts["live.js"] + "\nfetch('https://cdn.example.com/x.js');\n"}) - control("an outside URL in live.js", not asset_problems(planted)) # 8. plugins code, _, body = request(port, "GET", f"/fixture/{RUN}", headers=auth) From 9f8478cb76fb02409c7f85fb12b64de11cda6b40 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:30:07 +0100 Subject: [PATCH 03/14] refactor(lookup): share the summary names rule and the query reader with the search index og report index builds the search index with the names rule og lookup uses for "in the reports" (names_of, flat, the rsID test) and reads a query with og lookup's parser, so the search and a lookup cannot disagree. Expose them; nothing else changes. Bead: D1 --- rust/crates/og-lookup/src/lib.rs | 6 ++++++ rust/crates/og-lookup/src/summary.rs | 14 +++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/rust/crates/og-lookup/src/lib.rs b/rust/crates/og-lookup/src/lib.rs index 5562d540d..1fbadd818 100644 --- a/rust/crates/og-lookup/src/lib.rs +++ b/rust/crates/og-lookup/src/lib.rs @@ -25,6 +25,12 @@ mod resolve; mod summary; mod vcf; +/// The summary names rule the search index shares (`og report index`). +pub mod names { + pub use crate::summary::{flat, is_rsid, names_of}; +} +pub use query::{Query, parse as parse_query}; + use std::io::{self, Write}; use vcf::{Region, Span}; diff --git a/rust/crates/og-lookup/src/summary.rs b/rust/crates/og-lookup/src/summary.rs index a7d161e65..f17504819 100644 --- a/rust/crates/og-lookup/src/summary.rs +++ b/rust/crates/og-lookup/src/summary.rs @@ -1,7 +1,7 @@ -//! What the reports (`summary.json`) say about a gene or an rsID: the same -//! names rule `bin/results_index.py` builds the search index with -//! (`names_of` in `bin/genome_lookup.py`), so a lookup and the search find -//! the same mentions. +//! What the reports (`summary.json`) say about a gene or an rsID. The +//! search index (`og report index`, `og_render::index`) is built with this +//! same names rule ([`names_of`], [`flat`], [`is_rsid`]), so a lookup and +//! the search find the same mentions. use crate::Error; use pycompat::json::{JsonError, Value}; @@ -42,7 +42,7 @@ fn decode_error(doc: &str, msg: &str, byte: usize) -> String { } /// `^rs\d+$` in any case, `$` also before a final `\n` (Python's `re`). -fn is_rsid(s: &str) -> bool { +pub fn is_rsid(s: &str) -> bool { let s = s.strip_suffix('\n').unwrap_or(s); let mut cs = s.chars(); matches!((cs.next(), cs.next()), (Some('r' | 'R'), Some('s' | 'S'))) @@ -54,7 +54,7 @@ fn is_rsid(s: &str) -> bool { /// gene key's string is split on `, ; & / ( )` and spaces and keeps the /// words with an upper-case letter ("VDBP (GC)" gives VDBP and GC, "SMN /// exon 7-8 deletion" gives SMN); a list gives its strings, a dict its keys. -fn names_of(obj: &Value) -> (BTreeSet, BTreeSet) { +pub fn names_of(obj: &Value) -> (BTreeSet, BTreeSet) { let mut genes = BTreeSet::new(); let mut rsids = BTreeSet::new(); let Value::Object(o) = obj else { @@ -142,7 +142,7 @@ fn s(v: &Value) -> String { } /// One entry as `key=value` pairs, cut at `limit` characters. -fn flat(d: &Value, limit: usize) -> String { +pub fn flat(d: &Value, limit: usize) -> String { let Value::Object(o) = d else { return String::new(); }; From 0b70e821f9de2371e6c710054ae4963cd20ed2f7 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:30:07 +0100 Subject: [PATCH 04/14] perf(search): build the results search index in og with the rule the browser uses og report index build writes _search_index.json from summary.json and the step 23 and 31 tables, as bin/results_index.py does: the same terms, items, anchors, counts and mentions, byte for byte on the summary-collect goldens and the synthetic sample. og report index query answers one query with the rule search.js applies in the browser; a cargo test runs search.js's match() under node against it. The veil tag needs the findings a child's page keeps closed, which the live results build and which are not ported yet: build() takes them and the rule is ported and tested, but the command line passes none, so it writes no veil field until then. No step calls the verb yet. Bead: D1 --- rust/Cargo.lock | 1 + rust/crates/og-render/Cargo.toml | 1 + rust/crates/og-render/src/index.rs | 983 +++++++++++++++++++- rust/crates/og-render/src/lib.rs | 4 +- rust/crates/og-render/tests/index_search.rs | 775 +++++++++++++++ 5 files changed, 1755 insertions(+), 9 deletions(-) create mode 100644 rust/crates/og-render/tests/index_search.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index ec8e92fcc..98201c3ea 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -850,6 +850,7 @@ version = "0.1.0" dependencies = [ "indexmap", "og-io", + "og-lookup", "og-summary", "pycompat", ] diff --git a/rust/crates/og-render/Cargo.toml b/rust/crates/og-render/Cargo.toml index 5112b56a7..13d9a0436 100644 --- a/rust/crates/og-render/Cargo.toml +++ b/rust/crates/og-render/Cargo.toml @@ -17,6 +17,7 @@ fault-escape-apos-39 = [] [dependencies] pycompat.workspace = true og-io.workspace = true +og-lookup.workspace = true og-summary.workspace = true indexmap.workspace = true diff --git a/rust/crates/og-render/src/index.rs b/rust/crates/og-render/src/index.rs index 4a9025aab..43f29c0c0 100644 --- a/rust/crates/og-render/src/index.rs +++ b/rust/crates/og-render/src/index.rs @@ -1,13 +1,982 @@ //! `og report index`: the search index of one run, every gene symbol and -//! rsID the summary names, which `bin/results_index.py` writes today. +//! rsID its results name, and the query rule the live site's search box +//! (`assets/live/plugins/search.js`) applies to it in the browser. //! -//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so -//! on stderr and exits 2, before reading any argument. +//! ```text +//! og report index build --summary summary.json [--sample-dir DIR] -o INDEX.json +//! og report index query --index INDEX.json QUERY +//! ``` +//! +//! What it reads: +//! - every section of summary.json in state ok or stale: each dict in its +//! data that names a gene or an rsID, by `og lookup`'s names rule +//! ([`og_lookup::names::names_of`]), so the search and a lookup cannot +//! disagree. A dict inside one that already named a name adds only the +//! names its parents did not; +//! - the gene column of step 23 (clinical filter) and step 31 (slivar), +//! whose sections keep counts only: from the summary table on the bash +//! path (`GENE`; `SYMBOL` and `Existing_variation`), else from the VCF the +//! Nextflow report stages (INFO/PGP_GENE, else the genes of the CSQ +//! entries `csq_rank` picks; the rsIDs of every CSQ entry). +//! +//! A section that is missing, failed or unreadable gives no entry, and a +//! table that cannot be read gives none either: one warning line on stderr, +//! and the index is still written. +//! +//! The index: +//! `{schema_version: 1, sample, generated_utc, terms: {TERM: [{section, +//! anchor, title, step, state, source, count, mentions: [{text, locus}], +//! veil}]}}`. TERM is a gene symbol in upper case or an rsID in lower case. +//! One item per section that names the term; count is how many variants (a +//! mention with a locus counts once per locus) or other mentions it has +//! there, mentions the first [`MENTIONS_KEPT`] of them. anchor is the +//! section's id in the HTML report, or null for a key no section shows. +//! veil, only on an item that names a result a child's page keeps closed +//! (its gene, or a locus inside it), is that result's id ([`Closable`]). +//! +//! A query ([`query`], and the same rule in search.js, which a cargo test +//! runs under node) is an rsID, a gene symbol (a prefix also lists the terms +//! it starts: BRCA gives BRCA1 and BRCA2) or a position, which the index +//! cannot answer: a position is a genotype question, for +//! `scripts/owngenome-lookup.sh`. +//! +//! The veil needs the findings a child's page keeps closed, which the live +//! results build. Until they are ported, the command line passes none, so +//! `og report index build` writes no veil field; [`build`] takes them. +use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::io::Write; -/// `og report index ARGS`: not ported yet, so one line on stderr and exit 2. -pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { - let _ = writeln!(err, "og report index: not ported yet"); - 2 +use indexmap::IndexMap; +use og_lookup::names::{flat, is_rsid, names_of}; +use og_lookup::{Query, parse_query}; +use og_summary::collect::acmg::csq_rank; +use og_summary::collect::text::{ReadError, cell, cell_or, each_line, info_map, rows}; +use pycompat::json::{Dump, Value, dumps, loads}; +use pycompat::text; +use pycompat::value::{py_str, truthy}; + +use crate::chapters::section_of; + +pub const SCHEMA_VERSION: i128 = 1; +/// Mentions kept per term and section; count says how many there were. +pub const MENTIONS_KEPT: usize = 20; +/// Terms listed for a prefix query. +pub const SUGGESTIONS: usize = 20; +/// What a position query gets instead of items. +const POSITION_HINT: &str = "The search finds gene symbols and rsIDs in these results. A position is a \ +question about your genotype: run scripts/owngenome-lookup.sh {q}, which reads it from the gVCF on \ +this computer."; + +const USAGE: &str = "usage: og report index build --summary JSON [--sample-dir DIR] -o INDEX.json + og report index query --index INDEX.json QUERY"; + +/// A result a child's page keeps closed: its id, its gene (upper case) and +/// its locus (`chrN`, start, end) when it has one. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Closable { + pub id: String, + pub gene: Option, + pub locus: Option<(String, i128, i128)>, +} + +/// One section's entry for one term while the index is built. +struct Item { + section: String, + anchor: Value, + title: Value, + step: Value, + state: Value, + source: Value, + count: i128, + mentions: Vec<(String, Option)>, + /// One mention per variant (locus) or per text: the ClinVar list, the + /// secondary findings and a step's table can name the same variant. + seen: HashSet<(bool, String)>, +} + +/// A mention of a step's table: its text, its locus and the names in it. +type Record = (String, String, BTreeSet, BTreeSet); + +/// Every dict of `obj` that names a gene or rsID no dict above it named, +/// with those new names. +fn walk<'a>( + obj: &'a Value, + blocked: &BTreeSet, + out: &mut Vec<(&'a Value, BTreeSet)>, +) { + match obj { + Value::Object(o) => { + let (genes, rsids) = names_of(obj); + let names: BTreeSet = genes.into_iter().chain(rsids).collect(); + let new: BTreeSet = names.difference(blocked).cloned().collect(); + if !new.is_empty() { + out.push((obj, new)); + } + let inner: BTreeSet = blocked.union(&names).cloned().collect(); + for v in o.values() { + walk(v, &inner, out); + } + } + Value::Array(a) => { + for v in a { + walk(v, blocked, out); + } + } + _ => {} + } +} + +/// `true` for a character of Python's `\w`. +fn is_word(c: char) -> bool { + c.is_alphanumeric() || c == '_' +} + +/// `chrom:pos` of a mention that carries them (`chrom` and `pos` keys, else +/// a `variant` that starts with `chrom:pos`), else `None`. +fn locus_of(d: &Value) -> Option { + let Value::Object(o) = d else { + return None; + }; + if let Some(Value::Str(c)) = o.get("chrom") + && !c.is_empty() + { + match o.get("pos") { + Some(Value::Int(p)) => return Some(format!("{c}:{p}")), + Some(Value::Str(p)) if !p.is_empty() && p.chars().all(text::is_decimal) => { + return Some(format!("{c}:{p}")); + } + _ => {} + } + } + let variant = match o.get("variant") { + Some(v) if truthy(v) => py_str(v).unwrap_or_default(), + _ => return None, + }; + // ^((?:chr)?[0-9XYMT]{1,2}):([0-9]+)\b + let rest = variant.strip_prefix("chr").unwrap_or(&variant); + let chrom_len = rest + .bytes() + .take(2) + .take_while(|b| b.is_ascii_digit() || b"XYMT".contains(b)) + .count(); + if chrom_len == 0 || rest.as_bytes().get(chrom_len) != Some(&b':') { + return None; + } + let after = &rest[chrom_len + 1..]; + let digits = after.bytes().take_while(u8::is_ascii_digit).count(); + if digits == 0 || after[digits..].chars().next().is_some_and(is_word) { + return None; + } + let head = &variant[..variant.len() - rest.len() + chrom_len]; + Some(format!("{head}:{}", &after[..digits])) +} + +/// The rsIDs of an Existing_variation value (`rs123&COSV1` gives `rs123`). +fn rs_ids(value: &str) -> BTreeSet { + value + .split(['&', ',']) + .filter(|x| is_rsid(x)) + .map(text::lower) + .collect() +} + +fn variant_text(chrom: &str, pos: &str, r: &str, a: &str) -> String { + format!("{chrom}:{pos} {r}>{a}") +} + +/// `{x.upper()} - {"", "."}`. +fn one_gene(g: &str) -> BTreeSet { + let g = text::upper(g); + if g.is_empty() || g == "." { + BTreeSet::new() + } else { + BTreeSet::from([g]) + } +} + +/// The variants of step 23 or 31's summary table: the gene column (`GENE` +/// or `SYMBOL`), and for step 31 the rsIDs of `Existing_variation`. +fn table_records(path: &str, gene_col: &str, with_rsids: bool) -> Result, ReadError> { + Ok(rows(path)? + .iter() + .map(|r| { + let c = |k: &str| cell(r, k).unwrap_or("None").to_owned(); + let (chrom, pos) = (c("CHROM"), c("POS")); + let rsids = if with_rsids { + rs_ids(cell_or(r, "Existing_variation")) + } else { + BTreeSet::new() + }; + ( + variant_text(&chrom, &pos, &c("REF"), &c("ALT")), + format!("{chrom}:{pos}"), + one_gene(cell_or(r, gene_col)), + rsids, + ) + }) + .collect()) +} + +/// The variants of step 23 or 31's VCF: the gene of `gene_tag` when the +/// record has one, else the genes of the CSQ entries `csq_rank` picks; with +/// `with_rsids`, the rsIDs of every CSQ entry's Existing_variation. +fn vcf_records( + path: &str, + gene_tag: Option<&str>, + with_rsids: bool, +) -> Result, ReadError> { + let mut fmt: Option> = None; + let mut out = Vec::new(); + each_line(path, |line| { + if line.starts_with("##INFO= = line.split('\t').collect(); + if r.len() < 8 { + return; + } + let info = info_map(r[7]); + let csq = info.get("CSQ").copied().unwrap_or(""); + let tagged = gene_tag + .and_then(|t| info.get(t)) + .filter(|g| !g.is_empty() && **g != "."); + let genes = match (tagged, &fmt) { + (Some(g), _) => BTreeSet::from([text::upper(g)]), + (None, Some(f)) => csq_rank(f, csq) + .1 + .iter() + .filter(|e| !e.gene.is_empty()) + .map(|e| text::upper(&e.gene)) + .collect(), + (None, None) => BTreeSet::new(), + }; + let mut rsids = BTreeSet::new(); + if with_rsids + && let Some(f) = &fmt + && let Some(i) = f.iter().position(|k| k == "Existing_variation") + { + for t in csq.split(',') { + let v: Vec<&str> = t.split('|').collect(); + if v.len() == f.len() { + rsids.extend(rs_ids(v[i])); + } + } + } + out.push(( + variant_text(r[0], r[1], r[3], r[4]), + format!("{}:{}", r[0], r[1]), + genes, + rsids, + )); + })?; + Ok(out) +} + +/// The variants of a section whose genes are in its step's table, not in +/// summary.json: step 23 (`clinical`) and step 31 (`slivar`). `None` for +/// any other section. +fn table_of(key: &str, path: &str) -> Option, ReadError>> { + let tsv = path.ends_with(".tsv"); + Some(match (key, tsv) { + ("clinical", true) => table_records(path, "GENE", false), + ("clinical", false) => vcf_records(path, Some("PGP_GENE"), false), + ("slivar", true) => table_records(path, "SYMBOL", true), + ("slivar", false) => vcf_records(path, None, true), + _ => return None, + }) +} + +/// `os.path.isabs(src)` and `os.path.join(sample_dir, src)`. +fn table_path(sample_dir: &str, src: &str) -> String { + if src.starts_with('/') || sample_dir.is_empty() { + src.to_owned() + } else if sample_dir.ends_with('/') { + format!("{sample_dir}{src}") + } else { + format!("{sample_dir}/{src}") + } +} + +/// Every `chrom:pos` or `chrom:start-end` in `text`, as (`chrN`, low, +/// high): a chromosome `1`-`99`, X, Y, M or MT, with an optional `chr`, +/// not right after a letter or digit; commas in the numbers are dropped. +fn loci(s: &str) -> Vec<(String, i128, i128)> { + let b = s.as_bytes(); + let mut out = Vec::new(); + let mut i = 0; + while i < b.len() { + let free = i == 0 || !b[i - 1].is_ascii_alphanumeric(); + match free.then(|| locus_at(b, i)).flatten() { + Some((end, found)) => { + out.push(found); + i = end; + } + None => i += 1, + } + } + out +} + +/// The number of a `[0-9][0-9,]*` run at `b[i..]`, and where it ends. +fn number_at(b: &[u8], i: usize) -> Option<(usize, i128)> { + if !b.get(i).is_some_and(u8::is_ascii_digit) { + return None; + } + let end = i + b[i..] + .iter() + .take_while(|c| c.is_ascii_digit() || **c == b',') + .count(); + let n = b[i..end] + .iter() + .filter(|c| c.is_ascii_digit()) + .fold(0i128, |n, c| { + n.saturating_mul(10).saturating_add(i128::from(c - b'0')) + }); + Some((end, n)) +} + +/// The locus that starts at `b[i]`, if one does, and where it ends. +fn locus_at(b: &[u8], i: usize) -> Option<(usize, (String, i128, i128))> { + let starts = if b[i..].starts_with(b"chr") { + vec![i + 3, i] + } else { + vec![i] + }; + for j in starts { + let two = b.get(j..j + 2).filter(|d| d.iter().all(u8::is_ascii_digit)); + let one = b.get(j..j + 1).filter(|d| d[0].is_ascii_digit()); + let names: [Option<&[u8]>; 6] = [two, one, Some(b"X"), Some(b"Y"), Some(b"M"), Some(b"MT")]; + for name in names.into_iter().flatten() { + let k = j + name.len(); + if !b[j..].starts_with(name) || b.get(k) != Some(&b':') { + continue; + } + let Some((end, lo)) = number_at(b, k + 1) else { + continue; + }; + let (end, hi) = match b.get(end) { + Some(b'-') => number_at(b, end + 1).unwrap_or((end, lo)), + _ => (end, lo), + }; + let chrom = format!("chr{}", String::from_utf8_lossy(name)); + return Some((end, (chrom, lo.min(hi), lo.max(hi)))); + } + } + None +} + +/// The words of `text` in upper case (`[A-Z0-9][A-Z0-9._-]*`), each also +/// without its trailing `.`, `_` or `-`: "in KCNH2." names KCNH2, and HLA-A +/// stays whole. +fn words(s: &str) -> HashSet { + let up = text::upper(s); + let mut out = HashSet::new(); + for w in up + .split(|c: char| { + !(c.is_ascii_uppercase() || c.is_ascii_digit() || matches!(c, '.' | '_' | '-')) + }) + .filter(|w| !w.is_empty()) + { + // A word starts at its first letter or digit. + let start = w.find(|c: char| c.is_ascii_uppercase() || c.is_ascii_digit()); + if let Some(st) = start { + let w = &w[st..]; + out.insert(w.to_owned()); + out.insert(w.trim_end_matches(['.', '_', '-']).to_owned()); + } + } + out +} + +/// The id of the closed result a row belongs to, or `None`: the result +/// whose locus the row's text names exactly comes first, else the first +/// result whose gene the text names or whose span holds a locus of it. +pub fn veil_of(spans: &[Closable], text: &str) -> Option { + if spans.is_empty() { + return None; + } + let locs = loci(text); + if let Some(c) = spans + .iter() + .find(|c| c.locus.as_ref().is_some_and(|l| locs.contains(l))) + { + return Some(c.id.clone()); + } + let ws = words(text); + spans + .iter() + .find(|c| { + c.gene + .as_ref() + .is_some_and(|g| !g.is_empty() && ws.contains(g)) + || c.locus.as_ref().is_some_and(|(lc, a, b)| { + locs.iter().any(|(c2, s, e)| c2 == lc && s <= b && e >= a) + }) + }) + .map(|c| c.id.clone()) +} + +/// The index of `summary` (a summary.json value). `sample_dir`, when given, +/// is where the step 23 and 31 tables are read from (the paths summary.json +/// records); `spans` are the results a child's page keeps closed; a table +/// that cannot be read is said on `warn`. +pub fn build( + summary: &Value, + sample_dir: Option<&str>, + spans: &[Closable], + warn: &mut dyn Write, +) -> Result { + let Value::Object(top) = summary else { + return Err("summary.json is not an object".to_owned()); + }; + let sections = match top.get("sections") { + Some(Value::Object(m)) => Some(m), + Some(v) if truthy(v) => return Err("summary.sections is not an object".to_owned()), + _ => None, + }; + let mut terms: BTreeMap> = BTreeMap::new(); + for (key, sec) in sections.into_iter().flatten() { + let Value::Object(s) = sec else { + return Err(format!("summary.sections.{key} is not an object")); + }; + if !matches!(s.get("state"), Some(Value::Str(st)) if st == "ok" || st == "stale") { + continue; + } + let field = |k: &str| s.get(k).cloned().unwrap_or(Value::Null); + let mut add = |term: &str, txt: &str, locus: Option<&str>| { + let items = terms.entry(term.to_owned()).or_default(); + let at = match items.iter().position(|x| x.section == *key) { + Some(at) => at, + None => { + items.push(Item { + section: key.clone(), + anchor: section_of(key) + .map_or(Value::Null, |x| Value::Str(x.id.to_owned())), + title: s + .get("title") + .filter(|t| truthy(t)) + .cloned() + .unwrap_or_else(|| Value::Str(key.clone())), + step: field("step"), + state: field("state"), + source: field("source"), + count: 0, + mentions: Vec::new(), + seen: HashSet::new(), + }); + items.len() - 1 + } + }; + let item = &mut items[at]; + let mark = match locus { + Some(l) => (true, l.to_owned()), + None => (false, txt.to_owned()), + }; + if !item.seen.insert(mark) { + return; + } + item.count += 1; + if item.mentions.len() < MENTIONS_KEPT { + item.mentions + .push((txt.to_owned(), locus.map(str::to_owned))); + } + }; + let mut found = Vec::new(); + if let Some(d) = s.get("data").filter(|d| truthy(d)) { + walk(d, &BTreeSet::new(), &mut found); + } + for (d, names) in found { + let (txt, locus) = (flat(d, 160), locus_of(d)); + for n in &names { + add(n, &txt, locus.as_deref()); + } + } + let src = match s.get("source") { + Some(Value::Str(x)) if !x.is_empty() => x, + _ => continue, + }; + let Some(dir) = sample_dir else { continue }; + let path = table_path(dir, src); + if !std::path::Path::new(&path).is_file() { + continue; + } + let records = match table_of(key, &path) { + None => continue, + Some(Ok(r)) => r, + Some(Err(e)) => { + let _ = writeln!( + warn, + "[WARN] {key}: cannot read {src} for the search index: {}", + e.text() + ); + Vec::new() + } + }; + for (txt, locus, genes, rsids) in records { + for n in genes.union(&rsids) { + add(n, &txt, Some(&locus)); + } + } + } + let mut out = IndexMap::new(); + for (term, items) in terms { + let shown: Vec = std::iter::once(term.clone()) + .chain( + items + .iter() + .flat_map(|it| it.mentions.iter()) + .map(|(t, l)| format!("{t} {}", l.as_deref().unwrap_or(""))), + ) + .collect(); + let veil = veil_of(spans, &shown.join(" ")); + let list = items + .into_iter() + .map(|it| { + let mut o = IndexMap::new(); + o.insert("section".to_owned(), Value::Str(it.section)); + o.insert("anchor".to_owned(), it.anchor); + o.insert("title".to_owned(), it.title); + o.insert("step".to_owned(), it.step); + o.insert("state".to_owned(), it.state); + o.insert("source".to_owned(), it.source); + o.insert("count".to_owned(), Value::Int(it.count)); + let mentions = it + .mentions + .into_iter() + .map(|(t, l)| { + let mut m = IndexMap::new(); + m.insert("text".to_owned(), Value::Str(t)); + m.insert("locus".to_owned(), l.map_or(Value::Null, Value::Str)); + Value::Object(m) + }) + .collect(); + o.insert("mentions".to_owned(), Value::Array(mentions)); + if let Some(id) = &veil { + o.insert("veil".to_owned(), Value::Str(id.clone())); + } + Value::Object(o) + }) + .collect(); + out.insert(term, Value::Array(list)); + } + let mut idx = IndexMap::new(); + idx.insert("schema_version".to_owned(), Value::Int(SCHEMA_VERSION)); + idx.insert( + "sample".to_owned(), + top.get("sample").cloned().unwrap_or(Value::Null), + ); + idx.insert( + "generated_utc".to_owned(), + top.get("generated_utc").cloned().unwrap_or(Value::Null), + ); + idx.insert("terms".to_owned(), Value::Object(out)); + Ok(Value::Object(idx)) +} + +/// What a query asks. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Kind { + Rsid, + Gene, + Site, + Error, +} + +impl Kind { + /// The word search.js uses for it. + pub fn word(self) -> &'static str { + match self { + Kind::Rsid => "rsid", + Kind::Gene => "gene", + Kind::Site => "site", + Kind::Error => "error", + } + } +} + +/// The answer to one query. +#[derive(Debug, Clone, PartialEq)] +pub struct Answer { + pub kind: Kind, + /// The term looked up: an rsID in lower case, a gene in upper case. + pub term: Option, + /// The term's items in the index. + pub items: Vec, + /// For a gene: up to [`SUGGESTIONS`] other terms that start with it, in + /// order. + pub suggestions: Vec, + /// For a position or a query that is none of these: what to do instead. + pub hint: Option, +} + +/// The answer to query `q` on `index`: an rsID, a gene symbol (whose +/// suggestions are the other terms that start with it) or a position, +/// which the index cannot answer. search.js's `match()` is the same rule. +pub fn query(index: &Value, q: &str) -> Answer { + let none = |kind, hint| Answer { + kind, + term: None, + items: Vec::new(), + suggestions: Vec::new(), + hint: Some(hint), + }; + let parsed = match parse_query(q) { + Ok(p) => p, + Err(e) => return none(Kind::Error, e.msg), + }; + let terms = match index { + Value::Object(o) => match o.get("terms") { + Some(Value::Object(t)) => Some(t), + _ => None, + }, + _ => None, + }; + let (kind, term) = match parsed { + Query::Site(..) => { + return none( + Kind::Site, + POSITION_HINT.replace("{q}", text::strip(q, None)), + ); + } + Query::Rsid(r) => (Kind::Rsid, text::lower(&r)), + Query::Gene(g) => (Kind::Gene, text::upper(&g)), + }; + let items = match terms.and_then(|t| t.get(&term)) { + Some(Value::Array(a)) => a.clone(), + _ => Vec::new(), + }; + let mut suggestions = Vec::new(); + if kind == Kind::Gene { + let mut keys: Vec<&String> = terms.into_iter().flat_map(|t| t.keys()).collect(); + keys.sort(); + suggestions = keys + .into_iter() + .filter(|k| **k != term && k.starts_with(term.as_str())) + .take(SUGGESTIONS) + .cloned() + .collect(); + } + Answer { + kind, + term: Some(term), + items, + suggestions, + hint: None, + } +} + +/// `str(v)` of a field of an item, `None` when it is missing. +fn field(item: &Value, key: &str) -> String { + match item { + Value::Object(o) => o + .get(key) + .map_or_else(|| "None".to_owned(), |v| py_str(v).unwrap_or_default()), + _ => "None".to_owned(), + } +} + +/// The lines `og report index query` prints for an answer. +pub fn query_lines(a: &Answer) -> Vec { + if matches!(a.kind, Kind::Error | Kind::Site) { + return vec![a.hint.clone().unwrap_or_default()]; + } + let term = a.term.as_deref().unwrap_or_default(); + let mut out = Vec::new(); + if a.items.is_empty() { + out.push(format!("none of these results mentions {term}")); + } else { + out.push(format!("{term} is in {} section(s):", a.items.len())); + for it in &a.items { + let stale = if field(it, "state") == "stale" { + " (stale: from an earlier run)" + } else { + "" + }; + out.push(format!( + " {} (step {}){stale}: {} mention(s)", + field(it, "title"), + field(it, "step"), + field(it, "count") + )); + let mentions = match it { + Value::Object(o) => match o.get("mentions") { + Some(Value::Array(m)) => m.as_slice(), + _ => &[], + }, + _ => &[], + }; + for m in mentions { + out.push(format!(" - {}", field(m, "text"))); + } + let count = match it { + Value::Object(o) => match o.get("count") { + Some(Value::Int(n)) => *n, + _ => 0, + }, + _ => 0, + }; + let shown = i128::try_from(mentions.len()).unwrap_or(i128::MAX); + if count > shown { + out.push(format!( + " - ... and {} more in {}", + count - shown, + field(it, "source") + )); + } + } + } + if !a.suggestions.is_empty() { + out.push(format!( + "terms that start with it: {}", + a.suggestions.join(", ") + )); + } + out +} + +/// The options of one command: `--name value` pairs and positionals. +fn options( + args: &[String], + names: &[&str], +) -> Result<(BTreeMap, Vec), String> { + let mut opts = BTreeMap::new(); + let mut pos = Vec::new(); + let mut it = args.iter(); + while let Some(arg) = it.next() { + if arg == "--" { + pos.extend(it.by_ref().cloned()); + break; + } + if !arg.starts_with('-') || arg == "-" { + pos.push(arg.clone()); + continue; + } + let (name, inline) = match arg.split_once('=') { + Some((n, v)) if n.starts_with("--") => (n.to_owned(), Some(v.to_owned())), + _ => (arg.clone(), None), + }; + let name = if name == "--out" { + "-o".to_owned() + } else { + name + }; + if !names.contains(&name.as_str()) { + return Err(format!("unrecognized arguments: {arg}")); + } + let value = match inline { + Some(v) => v, + None => it + .next() + .cloned() + .ok_or_else(|| format!("argument {name}: expected one argument"))?, + }; + opts.insert(name, value); + } + Ok((opts, pos)) +} + +fn read_json(path: &str) -> Result { + let t = std::fs::read_to_string(path).map_err(|e| format!("{path}: {e}"))?; + loads(&t).map_err(|e| format!("{path}: {e}")) +} + +fn write(path: &str, body: &str) -> std::io::Result<()> { + let mut f = og_io::AtomicFile::create(path)?; + f.write_all(body.as_bytes())?; + f.commit() +} + +/// `og report index build|query ARGS`: 0 when the index is written or a +/// query names an rsID or a gene, 2 for a position, a query that is none of +/// them or a command line it cannot read, 1 when a file cannot be read or +/// written. +pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + if args.is_empty() { + let _ = writeln!(err, "{USAGE}\nog report index: error: give build or query"); + return 2; + } + if args[0] == "-h" || args[0] == "--help" { + let _ = writeln!(out, "{USAGE}"); + return 0; + } + let usage = |err: &mut dyn Write, why: &str| { + let _ = writeln!(err, "{USAGE}\nog report index: error: {why}"); + 2 + }; + let fail = |err: &mut dyn Write, why: String| { + let _ = writeln!(err, "og report index: {why}"); + 1 + }; + let rest = &args[1..]; + if rest.iter().any(|a| a == "-h" || a == "--help") { + let _ = writeln!(out, "{USAGE}"); + return 0; + } + match args[0].as_str() { + "build" => { + let (o, pos) = match options(rest, &["--summary", "--sample-dir", "-o"]) { + Ok(x) => x, + Err(why) => return usage(err, &why), + }; + if let Some(p) = pos.first() { + return usage(err, &format!("unrecognized arguments: {p}")); + } + let (Some(summary), Some(dest)) = (o.get("--summary"), o.get("-o")) else { + return usage(err, "give --summary and -o"); + }; + let s = match read_json(summary) { + Ok(s) => s, + Err(why) => return fail(err, why), + }; + let idx = match build(&s, o.get("--sample-dir").map(String::as_str), &[], err) { + Ok(i) => i, + Err(why) => return fail(err, format!("{summary}: {why}")), + }; + let opts = Dump { + indent: Some(1), + ..Dump::default() + }; + let body = match dumps(&idx, &opts) { + Ok(b) => b + "\n", + Err(e) => return fail(err, format!("{dest}: {e}")), + }; + if let Err(e) = write(dest, &body) { + return fail(err, format!("{dest}: {e}")); + } + let n = match &idx { + Value::Object(o) => match o.get("terms") { + Some(Value::Object(t)) => t.len(), + _ => 0, + }, + _ => 0, + }; + let _ = writeln!(out, "{n} terms"); + 0 + } + "query" => { + let (o, pos) = match options(rest, &["--index"]) { + Ok(x) => x, + Err(why) => return usage(err, &why), + }; + let (Some(path), [q]) = (o.get("--index"), pos.as_slice()) else { + return usage(err, "give --index and one QUERY"); + }; + let idx = match read_json(path) { + Ok(i) => i, + Err(why) => return fail(err, why), + }; + let a = query(&idx, q); + for line in query_lines(&a) { + let _ = writeln!(out, "{line}"); + } + if matches!(a.kind, Kind::Rsid | Kind::Gene) { + 0 + } else { + 2 + } + } + other => usage( + err, + &format!("argument cmd: invalid choice: '{other}' (choose from 'build', 'query')"), + ), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn v(t: &str) -> Value { + loads(t).unwrap() + } + + #[test] + fn a_locus_comes_from_chrom_and_pos_or_the_variant() { + assert_eq!( + locus_of(&v(r#"{"chrom": "chr7", "pos": 117559590}"#)).as_deref(), + Some("chr7:117559590") + ); + assert_eq!( + locus_of(&v(r#"{"chrom": "chr7", "pos": "12"}"#)).as_deref(), + Some("chr7:12") + ); + assert_eq!( + locus_of(&v(r#"{"variant": "chrM:11778 G>A"}"#)).as_deref(), + Some("chrM:11778") + ); + assert_eq!( + locus_of(&v(r#"{"variant": "13:2000_A"}"#)), + None, + "a word right after the position" + ); + assert_eq!(locus_of(&v(r#"{"variant": "chr123:5"}"#)), None); + assert_eq!(locus_of(&v(r#"{"variant": "NM_1:c.5"}"#)), None); + assert_eq!( + locus_of(&v(r#"{"chrom": "", "pos": 5, "variant": "X:5-9"}"#)).as_deref(), + Some("X:5") + ); + } + + #[test] + fn existing_variation_keeps_the_rsids() { + assert_eq!( + rs_ids("rs123&COSV1,RS9").into_iter().collect::>(), + ["rs123", "rs9"] + ); + assert!(rs_ids("").is_empty()); + } + + #[test] + fn loci_and_words_read_a_row_like_the_live_veil() { + assert_eq!( + loci("CFTR chr7:1,000-900 x13:5 (MT:3) chrX:7"), + [ + ("chr7".to_owned(), 900, 1000), + ("chrMT".to_owned(), 3, 3), + ("chrX".to_owned(), 7, 7) + ] + ); + let w = words("in kcnh2. HLA-A"); + assert!( + w.contains("KCNH2") && w.contains("KCNH2.") && w.contains("HLA-A"), + "{w:?}" + ); + } + + #[test] + fn the_veil_takes_the_exact_locus_first_then_gene_or_overlap() { + let spans = [ + Closable { + id: "a".into(), + gene: Some("BRCA2".into()), + locus: None, + }, + Closable { + id: "b".into(), + gene: None, + locus: Some(("chr13".into(), 2000, 2000)), + }, + Closable { + id: "c".into(), + gene: None, + locus: Some(("chr7".into(), 100, 500)), + }, + ]; + assert_eq!( + veil_of(&spans, "BRCA2 chr13:2000 G>A").as_deref(), + Some("b") + ); + assert_eq!(veil_of(&spans, "BRCA2 gene=BRCA2").as_deref(), Some("a")); + assert_eq!(veil_of(&spans, "TP53 chr7:300").as_deref(), Some("c")); + assert_eq!(veil_of(&spans, "TP53 chr7:501"), None); + assert_eq!(veil_of(&[], "BRCA2"), None); + } } diff --git a/rust/crates/og-render/src/lib.rs b/rust/crates/og-render/src/lib.rs index 839f2be52..a3bee69c7 100644 --- a/rust/crates/og-render/src/lib.rs +++ b/rust/crates/og-render/src/lib.rs @@ -11,8 +11,8 @@ //! //! Ported so far: the frame (chapters, contents, section frames, the run //! facts, what the report leaves out) and the quality-control and variant -//! sections. A section not ported yet is left out of both reports; the index -//! and charts verbs say they are not ported and exit 2. +//! sections, and the search index. A section not ported yet is left out of +//! both reports; the charts verb says it is not ported and exits 2. pub mod chapters; pub mod charts; diff --git a/rust/crates/og-render/tests/index_search.rs b/rust/crates/og-render/tests/index_search.rs new file mode 100644 index 000000000..3a5ea9e77 --- /dev/null +++ b/rust/crates/og-render/tests/index_search.rs @@ -0,0 +1,775 @@ +//! `og report index` on the synthetic sample of +//! `tests/fixtures/results_index/SYNTH` (invented): ClinVar hits in CFTR and +//! BRCA2, a CPIC table (CYP2C19, CYP2D6), and the tables of steps 23 and 31 +//! in both forms, the summary TSVs the bash steps write and the VCFs the +//! Nextflow report stages. The summary is collected by `og summary +//! collect`'s own code. +//! +//! Checks: the terms and their items; that `og lookup` finds every term the +//! index finds in summary.json (one names rule); a prose gene field and a +//! `locus` key; the TSV and the VCF form give the same index; stale, failed +//! and unreadable sections; a table that cannot be read; the query rule; the +//! command line; the veil; and search.js's `match()`, run under node, +//! answering each query as [`query`] does (node is required on CI and the +//! check is skipped with a note without it). + +use std::io::Write; +use std::num::NonZero; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use og_render::index::{Closable, Kind, build, cli, query, query_lines}; +use og_summary::collect::{Group, Input}; +use pycompat::json::{Dump, Value, dumps, loads}; + +/// The repository's root. +fn repo() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +/// A scratch folder, removed when dropped. +struct Dir(PathBuf); + +impl Dir { + fn new(tag: &str) -> Dir { + let p = std::env::temp_dir().join(format!( + "og-render-index-{tag}-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + let _ = std::fs::remove_dir_all(&p); + std::fs::create_dir_all(&p).unwrap(); + Dir(p) + } + + fn path(&self, name: &str) -> String { + self.0.join(name).display().to_string() + } +} + +impl Drop for Dir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +fn copy_tree(from: &Path, to: &Path) { + std::fs::create_dir_all(to).unwrap(); + for e in std::fs::read_dir(from).unwrap() { + let e = e.unwrap(); + let dest = to.join(e.file_name()); + if e.file_type().unwrap().is_dir() { + copy_tree(&e.path(), &dest); + } else { + std::fs::copy(e.path(), dest).unwrap(); + } + } +} + +fn gzip(path: &Path) { + let body = std::fs::read(path).unwrap(); + let mut gz = PathBuf::from(path).into_os_string(); + gz.push(".gz"); + let f = std::fs::File::create(&gz).unwrap(); + let mut w = og_io::bgzf::Writer::new(f, NonZero::new(1).unwrap()).unwrap(); + w.write_all(&body).unwrap(); + w.finish().unwrap(); + std::fs::remove_file(path).unwrap(); +} + +/// A copy of the fixture under `work/form/SYNTH` with the step 23 and 31 +/// tables in one form: "tsv" (bash: the summary TSVs, the VCFs too) or +/// "vcf" (Nextflow: the VCFs only). +fn sample(work: &Dir, form: &str) -> String { + let d = work.0.join(form).join("SYNTH"); + copy_tree(&repo().join("tests/fixtures/results_index/SYNTH"), &d); + gzip(&d.join("clinical/SYNTH_clinical.vcf")); + gzip(&d.join("slivar/SYNTH_prioritized.vcf")); + if form == "vcf" { + std::fs::remove_file(d.join("clinical/SYNTH_clinical_summary.tsv")).unwrap(); + std::fs::remove_file(d.join("slivar/SYNTH_slivar_summary.tsv")).unwrap(); + } + d.display().to_string() +} + +/// summary.json of a sample folder, as `og summary collect` writes it. +fn collect(dir: &str) -> Value { + let input = Input { + sample: "SYNTH".to_owned(), + sample_dir: dir.to_owned(), + declared_sex: None, + assets: repo().join("assets").display().to_string(), + group: Group::All, + tables: repo().join("bin").display().to_string(), + }; + og_summary::collect::post::collect(&input).expect("the summary") +} + +fn index(summary: &Value, dir: Option<&str>) -> Value { + build(summary, dir, &[], &mut std::io::sink()).expect("the index") +} + +fn get<'a>(v: &'a Value, path: &[&str]) -> &'a Value { + path.iter().fold(v, |v, k| match v { + Value::Object(o) => o.get(*k).unwrap_or(&Value::Null), + _ => &Value::Null, + }) +} + +fn terms(idx: &Value) -> Vec { + match get(idx, &["terms"]) { + Value::Object(o) => o.keys().cloned().collect(), + _ => Vec::new(), + } +} + +fn items<'a>(idx: &'a Value, term: &str) -> &'a [Value] { + match get(idx, &["terms", term]) { + Value::Array(a) => a, + _ => &[], + } +} + +fn s(v: &Value) -> String { + match v { + Value::Str(x) => x.clone(), + Value::Int(n) => n.to_string(), + Value::Null => "null".to_owned(), + other => format!("{other:?}"), + } +} + +/// (section, anchor, count) of each item of `term`. +fn sections(idx: &Value, term: &str) -> Vec<(String, String, String)> { + items(idx, term) + .iter() + .map(|it| { + ( + s(get(it, &["section"])), + s(get(it, &["anchor"])), + s(get(it, &["count"])), + ) + }) + .collect() +} + +/// {term: [mention text]} of the items of section `key`. +fn table_entries(idx: &Value, key: &str) -> Vec<(String, Vec)> { + terms(idx) + .into_iter() + .filter_map(|t| { + let texts: Vec = items(idx, &t) + .iter() + .filter(|it| s(get(it, &["section"])) == key) + .flat_map(|it| match get(it, &["mentions"]) { + Value::Array(m) => m.iter().map(|x| s(get(x, &["text"]))).collect(), + _ => Vec::new(), + }) + .collect(); + (!texts.is_empty()).then_some((t, texts)) + }) + .collect() +} + +fn set(v: &mut Value, path: &[&str], to: Value) { + let mut cur = v; + for k in &path[..path.len() - 1] { + let Value::Object(o) = cur else { panic!("{k}") }; + cur = o.get_mut(*k).expect("the key"); + } + let Value::Object(o) = cur else { + panic!("not an object") + }; + o.insert(path[path.len() - 1].to_owned(), to); +} + +fn json(t: &str) -> Value { + loads(t).unwrap() +} + +fn write_json(path: &str, v: &Value) { + std::fs::write(path, dumps(v, &Dump::default()).unwrap()).unwrap(); +} + +/// What `og lookup report` prints under "in the reports" for one gene or +/// rsID, on an empty gVCF. +fn lookup_says(work: &Dir, summary: &Value, term: &str) -> String { + let spath = work.path("lookup-summary.json"); + write_json(&spath, summary); + let gvcf = work.path("empty.g.vcf"); + std::fs::write( + &gvcf, + "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n", + ) + .unwrap(); + let kind = if term.starts_with("rs") { + "site" + } else { + "gene" + }; + let args: Vec = [ + "report", + "--query", + term, + "--kind", + kind, + "--chrom", + "chr1", + "--start", + "1", + "--end", + "1", + "--gvcf", + &gvcf, + "--summary", + &spath, + ] + .iter() + .map(|x| (*x).to_owned()) + .collect(); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = og_lookup::cli(&args, &mut out, &mut err); + let out = String::from_utf8(out).unwrap(); + match out.split_once(" in the reports:\n") { + Some((_, rest)) if code == 0 => rest.to_owned(), + _ => format!( + "og lookup report failed: {code} {out} {}", + String::from_utf8_lossy(&err) + ), + } +} + +fn run(args: &[&str]) -> (u8, String, String) { + let args: Vec = args.iter().map(|x| (*x).to_owned()).collect(); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = cli(&args, &mut out, &mut err); + ( + code, + String::from_utf8(out).unwrap(), + String::from_utf8(err).unwrap(), + ) +} + +#[test] +fn the_index_holds_every_gene_and_rsid_with_its_sections() { + let work = Dir::new("terms"); + let tsv = sample(&work, "tsv"); + let sum = collect(&tsv); + assert_eq!( + ["clinical", "slivar"].map(|k| s(get(&sum, &["sections", k, "source"]))), + [ + "clinical/SYNTH_clinical_summary.tsv", + "slivar/SYNTH_slivar_summary.tsv" + ] + ); + let idx = index(&sum, Some(&tsv)); + assert_eq!( + terms(&idx), + [ + "BRCA1", + "BRCA2", + "CFTR", + "CYP2C19", + "CYP2D6", + "rs1000002", + "rs123" + ] + ); + assert_eq!(get(&idx, &["schema_version"]), &Value::Int(1)); + assert_eq!(s(get(&idx, &["sample"])), "SYNTH"); + // One item per section with its report anchor, one mention per variant: + // the secondary findings and the table both name chr13:2000. + let want = [ + ("clinvar", "clinvar"), + ("carrier", "carrier-list"), + ("clinical", "clinical-filter"), + ("slivar", "slivar"), + ] + .map(|(a, b)| (a.to_owned(), b.to_owned(), "1".to_owned())); + assert_eq!(sections(&idx, "BRCA2"), want); + // Existing_variation 'rs123&COSV1' gives rs123 only. + assert!(!terms(&idx).iter().any(|t| t.eq_ignore_ascii_case("cosv1"))); +} + +#[test] +fn og_lookup_finds_every_term_the_index_finds_in_the_summary() { + let work = Dir::new("lookup"); + let tsv = sample(&work, "tsv"); + let sum = collect(&tsv); + let idx = index(&sum, Some(&tsv)); + let from_summary: Vec = terms(&idx) + .into_iter() + .filter(|t| { + items(&idx, t) + .iter() + .any(|it| !["clinical", "slivar"].contains(&s(get(it, &["section"])).as_str())) + }) + .collect(); + assert_eq!(from_summary, ["BRCA2", "CFTR", "CYP2C19", "CYP2D6"]); + for t in &from_summary { + let said = lookup_says(&work, &sum, t); + assert!( + !said.contains("none of the reports mentions"), + "{t}: {said}" + ); + for it in items(&idx, t) { + let key = s(get(it, &["section"])); + if key == "clinical" || key == "slivar" { + continue; + } + let title = s(get(it, &["title"])); + assert!(said.contains(&title), "{t}: {title} missing from {said}"); + } + } + // A ClinVar hit's rsids: the index and the lookup both find it there, + // and without them neither does (so the check above can fail). + let mut rs = sum.clone(); + let Value::Array(hits) = get(&sum, &["sections", "clinvar", "data", "hits"]).clone() else { + panic!("clinvar hits") + }; + let mut hits = hits; + let Value::Object(h0) = &mut hits[0] else { + panic!("a hit") + }; + h0.insert("rsids".to_owned(), json(r#"["rs7000001"]"#)); + set( + &mut rs, + &["sections", "clinvar", "data", "hits"], + Value::Array(hits), + ); + let ri = index(&rs, Some(&tsv)); + assert_eq!(sections(&ri, "rs7000001")[0].0, "clinvar"); + assert!(lookup_says(&work, &rs, "rs7000001").contains("ClinVar screen (step 06)")); + assert!( + lookup_says(&work, &sum, "rs7000001").contains("none of the reports mentions rs7000001") + ); + assert!(items(&index(&sum, Some(&tsv)), "rs7000001").is_empty()); +} + +#[test] +fn a_prose_gene_field_and_a_locus_key_give_their_symbols() { + let work = Dir::new("prose"); + let tsv = sample(&work, "tsv"); + let sum = collect(&tsv); + let mut prose = sum.clone(); + set( + &mut prose, + &["sections", "mito", "state"], + Value::Str("ok".into()), + ); + set( + &mut prose, + &["sections", "mito", "data"], + json(r#"{"variants": [{"gene": "SMN exon 7-8 deletion"}, {"gene": "VDBP (GC)"}]}"#), + ); + let pt = terms(&index(&prose, Some(&tsv))); + for t in ["SMN", "VDBP", "GC"] { + assert!(pt.contains(&t.to_owned()), "{t} not in {pt:?}"); + } + for t in ["EXON", "DELETION", "7-8", "(GC)", "(GC"] { + assert!(!pt.contains(&t.to_owned()), "{t} in {pt:?}"); + } + assert!(!lookup_says(&work, &prose, "GC").contains("none of the reports mentions")); + + let mut loc = sum.clone(); + for (k, data) in [ + ( + "expansions", + r#"{"flagged": [{"locus": "HTT", "status": "pathogenic"}], "key_loci": [{"locus": "FMR1", "repeat_count": "."}], + "readings": [{"locus": "UGT1A1", "repeat": "promoter TA repeat"}]}"#, + ), + ( + "locityper", + r#"{"loci": [{"locus": "FCGR", "region": "chr1:161505430-161678654", "status": "called"}, + {"locus": "HLA-DRB1", "region": "chr6:32484348-32593538", "status": "called"}]}"#, + ), + ] { + set(&mut loc, &["sections", k, "state"], Value::Str("ok".into())); + set(&mut loc, &["sections", k, "data"], json(data)); + } + let li = index(&loc, Some(&tsv)); + for (t, sec) in [ + ("HTT", "expansions"), + ("FMR1", "expansions"), + ("UGT1A1", "expansions"), + ("FCGR", "locityper"), + ("HLA-DRB1", "locityper"), + ] { + assert!( + sections(&li, t).iter().any(|x| x.0 == sec), + "{t}: {:?}", + sections(&li, t) + ); + } + assert!( + !terms(&li).iter().any(|t| t.starts_with("CHR")), + "a region is no term" + ); + assert!(!lookup_says(&work, &loc, "HTT").contains("none of the reports mentions")); +} + +#[test] +fn the_tsv_and_the_vcf_form_give_the_same_index() { + let work = Dir::new("forms"); + let (tsv, vcf) = (sample(&work, "tsv"), sample(&work, "vcf")); + let (st, sv) = (collect(&tsv), collect(&vcf)); + assert_eq!( + ["clinical", "slivar"].map(|k| s(get(&sv, &["sections", k, "source"]))), + [ + "clinical/SYNTH_clinical.vcf.gz", + "slivar/SYNTH_prioritized.vcf.gz" + ] + ); + let (a, b) = (index(&st, Some(&tsv)), index(&sv, Some(&vcf))); + for key in ["clinical", "slivar"] { + assert!(!table_entries(&a, key).is_empty(), "{key}"); + assert_eq!(table_entries(&a, key), table_entries(&b, key), "{key}"); + } + let names = |k| { + table_entries(&a, k) + .into_iter() + .map(|x| x.0) + .collect::>() + }; + assert_eq!(names("clinical"), ["BRCA1", "BRCA2"]); + assert_eq!(names("slivar"), ["BRCA2", "CFTR", "rs1000002", "rs123"]); + // Without the sample folder no table is read (red for the check above). + assert!(table_entries(&index(&sv, None), "slivar").is_empty()); + // The whole index is the same, the file names and the time aside. + let strip = |mut v: Value| { + set(&mut v, &["generated_utc"], Value::Null); + if let Value::Object(o) = &mut v + && let Some(Value::Object(t)) = o.get_mut("terms") + { + for list in t.values_mut() { + if let Value::Array(items) = list { + for it in items { + set(it, &["source"], Value::Null); + } + } + } + } + v + }; + assert_eq!(strip(a), strip(b)); +} + +#[test] +fn only_ok_and_stale_sections_count_and_a_bad_table_adds_nothing() { + let work = Dir::new("states"); + let tsv = sample(&work, "tsv"); + let sum = collect(&tsv); + let mut st = sum.clone(); + set( + &mut st, + &["sections", "cpic", "state"], + Value::Str("stale".into()), + ); + for (k, state, data) in [ + ( + "mito", + "failed", + r#"{"variants": [{"gene": "MT-ND4", "variant": "chrM:11778 G>A"}]}"#, + ), + ( + "lpa", + "missing", + r#"{"variants": [{"rsid": "rs10455872", "gene": "LPA"}]}"#, + ), + ("haplogroup", "unreadable", r#"{"gene": "MT-CYB"}"#), + ] { + set(&mut st, &["sections", k, "state"], Value::Str(state.into())); + set(&mut st, &["sections", k, "data"], json(data)); + } + let si = index(&st, Some(&tsv)); + let states: Vec = items(&si, "CYP2D6") + .iter() + .map(|it| s(get(it, &["state"]))) + .collect(); + assert_eq!(states, ["stale"]); + for t in ["MT-ND4", "LPA", "rs10455872", "MT-CYB"] { + assert!(!terms(&si).contains(&t.to_owned()), "{t}"); + } + let mut ok = st.clone(); + set( + &mut ok, + &["sections", "mito", "state"], + Value::Str("ok".into()), + ); + let oi = index(&ok, Some(&tsv)); + assert_eq!( + sections(&oi, "MT-ND4")[0].0, + "mito", + "red: the same data in an ok section" + ); + let it = &items(&oi, "MT-ND4")[0]; + let anchor = og_render::chapters::section_of("mito").map_or("null", |x| x.id); + assert_eq!(s(get(it, &["anchor"])), anchor); + assert_eq!(s(get(it, &["count"])), "1"); + assert_eq!( + get(it, &["mentions"]), + &json(r#"[{"text": "gene=MT-ND4, variant=chrM:11778 G>A", "locus": "chrM:11778"}]"#) + ); + + // The summary of a readable run, the tables of a damaged copy: the + // slivar VCF cut short after step 31 ran. + let vcf = sample(&work, "vcf"); + let sv = collect(&vcf); + let gz = Path::new(&vcf).join("slivar/SYNTH_prioritized.vcf.gz"); + let head = std::fs::read(&gz).unwrap()[..40].to_vec(); + std::fs::write(&gz, head).unwrap(); + let mut warn = Vec::new(); + let bi = build(&sv, Some(&vcf), &[], &mut warn).expect("the index is still written"); + let warn = String::from_utf8(warn).unwrap(); + assert!( + warn.starts_with( + "[WARN] slivar: cannot read slivar/SYNTH_prioritized.vcf.gz for the search index: " + ), + "{warn}" + ); + assert!(table_entries(&bi, "slivar").is_empty()); + assert!(!table_entries(&bi, "clinical").is_empty()); +} + +#[test] +fn a_query_is_an_rsid_a_gene_or_its_prefix_or_a_position() { + let work = Dir::new("query"); + let tsv = sample(&work, "tsv"); + let idx = index(&collect(&tsv), Some(&tsv)); + let a = query(&idx, "RS123"); + assert_eq!((a.kind, a.term.as_deref()), (Kind::Rsid, Some("rs123"))); + assert_eq!(a.items.len(), 1); + assert_eq!(s(get(&a.items[0], &["section"])), "slivar"); + let Value::Array(m) = get(&a.items[0], &["mentions"]) else { + panic!("mentions") + }; + assert_eq!(s(get(&m[0], &["locus"])), "chr7:3000"); + let a = query(&idx, "brca"); + assert_eq!((a.kind, a.items.len()), (Kind::Gene, 0)); + assert_eq!(a.suggestions, ["BRCA1", "BRCA2"]); + let a = query(&idx, "brca2"); + let secs: Vec = a.items.iter().map(|it| s(get(it, &["section"]))).collect(); + assert_eq!(secs, ["clinvar", "carrier", "clinical", "slivar"]); + assert!(a.suggestions.is_empty()); + let a = query(&idx, "chr7:1,000"); + assert_eq!(a.kind, Kind::Site); + assert!( + a.items.is_empty() + && a.hint + .unwrap() + .contains("owngenome-lookup.sh chr7:1,000") + ); + let a = query(&idx, "TP53"); + assert_eq!((a.kind, a.items.len()), (Kind::Gene, 0)); + assert_eq!(query_lines(&a), ["none of these results mentions TP53"]); + let a = query(&idx, "two words"); + assert_eq!(a.kind, Kind::Error); + assert_eq!( + a.hint.as_deref(), + Some( + "not an rsID (rs123), a position (chr7:117559590) or a gene symbol (CFTR): 'two words'" + ) + ); + assert_eq!(query(&idx, "chr1:0").kind, Kind::Error); +} + +#[test] +fn the_command_line_builds_and_queries_the_index() { + let work = Dir::new("cli"); + let tsv = sample(&work, "tsv"); + let sum = collect(&tsv); + let spath = work.path("summary.json"); + write_json(&spath, &sum); + let out = work.path("idx.json"); + let (code, stdout, stderr) = run(&[ + "build", + "--summary", + &spath, + "--sample-dir", + &tsv, + "-o", + &out, + ]); + assert_eq!( + (code, stdout.as_str(), stderr.as_str()), + (0, "7 terms\n", "") + ); + let text = std::fs::read_to_string(&out).unwrap(); + assert!( + text.starts_with("{\n \"schema_version\": 1,\n \"sample\": \"SYNTH\""), + "{text}" + ); + assert!(text.ends_with("}\n")); + assert_eq!(loads(&text).unwrap(), index(&sum, Some(&tsv))); + + let (code, stdout, _) = run(&["query", "--index", &out, "rs123"]); + assert_eq!(code, 0); + assert!( + stdout.starts_with("rs123 is in 1 section(s):\n Variant prioritization"), + "{stdout}" + ); + assert!(stdout.contains(" - chr7:3000 T>C\n"), "{stdout}"); + let (code, stdout, _) = run(&["query", "--index", &out, "brc"]); + assert_eq!(code, 0); + assert!( + stdout.ends_with("terms that start with it: BRCA1, BRCA2\n"), + "{stdout}" + ); + let (code, stdout, _) = run(&["query", "--index", &out, "7:117559590"]); + assert_eq!(code, 2); + assert!(stdout.contains("owngenome-lookup.sh"), "{stdout}"); + let (code, _, stderr) = run(&["build", "--summary", &spath]); + assert_eq!(code, 2, "{stderr}"); + let (code, _, stderr) = run(&["build", "--summary", &work.path("absent.json"), "-o", &out]); + assert_eq!(code, 1, "{stderr}"); + let (code, _, stderr) = run(&["serve"]); + assert!( + code == 2 && stderr.contains("invalid choice: 'serve'"), + "{stderr}" + ); +} + +#[test] +fn the_veil_tags_the_items_that_name_a_closed_result() { + let work = Dir::new("veil"); + let tsv = sample(&work, "tsv"); + let sum = collect(&tsv); + let spans = [ + Closable { + id: "sf-BRCA2".to_owned(), + gene: Some("BRCA2".to_owned()), + locus: None, + }, + Closable { + id: "clinvar-chr7-1000".to_owned(), + gene: None, + locus: Some(("chr7".to_owned(), 1000, 1000)), + }, + ]; + let idx = build(&sum, Some(&tsv), &spans, &mut std::io::sink()).unwrap(); + let veil = |t: &str| -> Vec { + items(&idx, t) + .iter() + .map(|it| s(get(it, &["veil"]))) + .collect() + }; + assert_eq!(veil("BRCA2"), ["sf-BRCA2"; 4]); + // CFTR's ClinVar hit is chr7:1000 itself. + assert!( + veil("CFTR").iter().all(|v| v == "clinvar-chr7-1000"), + "{:?}", + veil("CFTR") + ); + assert_eq!(veil("CYP2D6"), ["null"]); + // No closed result, no field. + assert!( + !dumps(&index(&sum, Some(&tsv)), &Dump::default()) + .unwrap() + .contains("\"veil\"") + ); +} + +#[test] +fn search_js_answers_each_query_as_the_rust_rule_does() { + let js_path = repo().join("assets/live/plugins/search.js"); + let js = std::fs::read_to_string(&js_path).unwrap(); + // The page asks for nothing but the index. + assert_eq!(js.matches("fetch(").count(), 1); + for bad in [ + "XMLHttpRequest", + "sendBeacon", + "WebSocket", + "eval(", + "innerHTML", + ] { + assert!(!js.contains(bad), "search.js uses {bad}"); + } + let node = Command::new("node").arg("--version").output(); + if node.is_err() { + assert!( + std::env::var_os("CI").is_none(), + "node is not on the PATH, and CI needs it for this check" + ); + eprintln!("note: node is not on the PATH, so search.js is not run against the Rust rule"); + return; + } + let work = Dir::new("node"); + let tsv = sample(&work, "tsv"); + let idx = index(&collect(&tsv), Some(&tsv)); + let ipath = work.path("idx.json"); + write_json(&ipath, &idx); + let queries = [ + "RS123", + "rs1000002", + "brca", + "BRCA2", + "cyp2", + "C", + "TP53", + "chr7:1,000", + "13-2000", + "chr1:0", + "chrMT_5", + "two words", + "", + " cftr ", + "HLA-DRB1", + "rs", + "rs12x", + "-x", + ]; + let prog = "const m=require(process.argv[1]).match;\ + const idx=JSON.parse(require('fs').readFileSync(process.argv[2]));\ + const qs=JSON.parse(process.argv[3]);\ + console.log(JSON.stringify(qs.map(q=>{const r=m(idx,q);\ + return [r.kind,r.term,r.items.map(i=>i.section),r.suggestions]})))"; + let qs = Value::Array( + queries + .iter() + .map(|q| Value::Str((*q).to_owned())) + .collect(), + ); + let out = Command::new("node") + .args(["-e", prog, &js_path.display().to_string(), &ipath]) + .arg(dumps(&qs, &Dump::default()).unwrap()) + .output() + .expect("node runs"); + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + let got = loads(String::from_utf8(out.stdout).unwrap().trim()).unwrap(); + let want = Value::Array( + queries + .iter() + .map(|q| { + let a = query(&idx, q); + Value::Array(vec![ + Value::Str(a.kind.word().to_owned()), + a.term.map_or(Value::Null, Value::Str), + Value::Array( + a.items + .iter() + .map(|it| get(it, &["section"]).clone()) + .collect(), + ), + Value::Array(a.suggestions.into_iter().map(Value::Str).collect()), + ]) + }) + .collect(), + ); + let Value::Array(g) = &got else { + panic!("{got:?}") + }; + let Value::Array(w) = &want else { + unreachable!() + }; + for ((q, g), w) in queries.iter().zip(g).zip(w) { + assert_eq!( + g, w, + "query {q:?}: search.js (left) and the Rust rule (right) differ" + ); + } + assert_eq!(g.len(), queries.len()); +} From 88fb283bb03fcc953a8c1edaf861c5786d39f080 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:33:53 +0100 Subject: [PATCH 05/14] docs(search): name og report index as the rule search.js mirrors Bead: D1 --- CHANGELOG.md | 1 + assets/live/plugins/search.js | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05c3388e1..d72fb389a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- `og report index build --summary summary.json [--sample-dir DIR] -o _search_index.json` writes the run's search index in Rust. `og report index query --index FILE QUERY` answers one query with the rule of the live site's search box (`assets/live/plugins/search.js`); a cargo test runs that script under node against the Rust rule. The file is the one `bin/results_index.py` writes: the same terms, sections, anchors, counts and mentions, and the same answer to each query. No step calls the verb yet: step 24 and the Nextflow report still write the index with `bin/results_index.py`. One field is not written yet: `veil`, which tags an item that names a result a child's page keeps closed. It needs the live site's findings, which are not ported yet. - The self-test of `scripts/ci/check-images.sh` copies the whole tree again once the list of tracked paths passes 128 KiB. It piped that list through `xargs` to `tar`, and GNU `xargs` split a list that long over two `tar` runs; the reading `tar` stopped after the first archive, so Guard's `images` job failed on a broken pipe (exit 123). One `tar` now reads the list itself (`tar --null -T -`). Nothing the check reports changes. - The phased family VCF is indexed by the Rust `og` binary. WhatsHap's step (step 38 and the Nextflow WHATSHAP_PED process) wrote `family.phased.vcf.gz.tbi` with a Python one-liner, `pysam.tabix_index`. It now runs `og vcf index -f`. The `.tbi` keeps its name and format, and its bytes, once inflated, are the ones htslib 1.24 writes (the htslib of the pysam and bcftools the pipeline pins): the same bins in the same order, the same chunks and the same linear index. Only the deflate stream can differ, as two deflate libraries differ. On a 5-million-record three-sample VCF it took 0.75 to 0.91 s and 31 MB at peak, where htslib's own `tabix` took 0.81 to 0.93 s and 19 MB. - As in htslib 1.24, a record ends at the largest of INFO/END, the REF length, the SVLEN of a ``, ``, `` or `` ALT, and a sample's LEN on a gVCF block. A line with no position to read, unsorted records, a sequence that starts again, a record past 2^29 and an empty file stop the index with a message, and nothing is written. Without `-f` it never writes over an index, even one that appears while it runs. diff --git a/assets/live/plugins/search.js b/assets/live/plugins/search.js index 5c7719d34..e9b4fc8d6 100644 --- a/assets/live/plugins/search.js +++ b/assets/live/plugins/search.js @@ -4,10 +4,12 @@ // filters that index here in the browser: no request per keystroke, so the query is in no // request and no server log. A hit lists each section that names the term, with a link to it in // the report and, for a mention with a locus, to the reads in the genome viewer. The query rule -// is bin/results_index.py's query(): an rsID, a gene symbol (a prefix also lists the terms that -// start with it), or a position, which the index cannot answer (scripts/owngenome-lookup.sh can). +// is og report index's query() (rust/crates/og-render/src/index.rs) and bin/results_index.py's: +// an rsID, a gene symbol (a prefix also lists the terms that start with it, in sorted order), or +// a position, which the index cannot answer (scripts/owngenome-lookup.sh can). // The box sits in the page header; on a person's page a hit opens its card (#s/
). -// tests/test_results_index.py runs match() under node against the Python rule. +// rust/crates/og-render/tests/index_search.rs runs match() under node against the Rust rule, and +// tests/test_results_index.py against the Python one. (function (root) { 'use strict'; var SUGGESTIONS = 20; From bb03567e56168ce4c84a258e006171c502cd013a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:34:26 +0100 Subject: [PATCH 06/14] feat(y): write Yleaf's table for a pileup with no read in og Yleaf stops on an empty pileup and writes a header with no row. og y yleaf-no-reads writes the row it gives for too few markers (Hg NA, the BAM's mapped reads from samtools idxstats), the part of bin/yleaf_run.py predict that Yleaf's own command line cannot do. Bead: port-yleaf-wrap --- rust/crates/og-reports/src/lib.rs | 7 +- rust/crates/og-reports/src/y.rs | 9 +- rust/crates/og-reports/src/yleaf.rs | 125 +++++++++++++++++++++++++--- 3 files changed, 124 insertions(+), 17 deletions(-) diff --git a/rust/crates/og-reports/src/lib.rs b/rust/crates/og-reports/src/lib.rs index 89bc67fd6..744bbd112 100644 --- a/rust/crates/og-reports/src/lib.rs +++ b/rust/crates/og-reports/src/lib.rs @@ -9,9 +9,10 @@ //! - [`viral`] (`og viral report`, step 47): viral DNA in the unmapped reads. //! - [`ystr`] (`og y str-bed`, `og y str-report`, step 45): Y-STR counts. //! - [`ytree`], [`yprivate`] and [`yleaf`] (`og y tree-positions`, -//! `tree-place`, `private-snps`, `yleaf-positions`, step 37): the -//! placement on the current Y tree, the private SNPs and Yleaf's marker -//! positions; [`y`] routes every `og y` verb. +//! `tree-place`, `private-snps`, `yleaf-positions`, `yleaf-no-reads`, +//! step 37): the placement on the current Y tree, the private SNPs, +//! Yleaf's marker positions and its table for an empty pileup; [`y`] +//! routes every `og y` verb. //! - [`exomiser`] (`og exomiser prepare`, `og exomiser report`, step 48). //! //! Each one writes what the Python it replaced wrote, cell for cell diff --git a/rust/crates/og-reports/src/y.rs b/rust/crates/og-reports/src/y.rs index ce419cd73..fa7d85d8c 100644 --- a/rust/crates/og-reports/src/y.rs +++ b/rust/crates/og-reports/src/y.rs @@ -5,12 +5,14 @@ //! placement on the current YFull YTree. //! - `private-snps` ([`crate::yprivate`], step 37): the chrY SNVs no public //! SNP table names. -//! - `yleaf-positions` ([`crate::yleaf`], step 37): Yleaf's own marker -//! positions, for the pileup made beside it. +//! - `yleaf-positions`, `yleaf-no-reads` ([`crate::yleaf`], step 37): +//! Yleaf's own marker positions, for the pileup made beside it, and its +//! prediction table when that pileup is empty. use std::io::Write; -const VERBS: &str = "str-bed|str-report|tree-positions|tree-place|private-snps|yleaf-positions"; +const VERBS: &str = + "str-bed|str-report|tree-positions|tree-place|private-snps|yleaf-positions|yleaf-no-reads"; /// Routes `og y VERB ARGS`; the exit code. pub fn cli(verb: Option<&str>, rest: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { @@ -20,6 +22,7 @@ pub fn cli(verb: Option<&str>, rest: &[String], out: &mut dyn Write, err: &mut d Some("tree-place") => crate::ytree::place_cli(rest, out, err), Some("private-snps") => crate::yprivate::cli(rest, out, err), Some("yleaf-positions") => crate::yleaf::cli(rest, out, err), + Some("yleaf-no-reads") => crate::yleaf::no_reads_cli(rest, out, err), other => crate::unknown_verb("y", VERBS, other, err), } } diff --git a/rust/crates/og-reports/src/yleaf.rs b/rust/crates/og-reports/src/yleaf.rs index 855c2ed39..1a8eed56a 100644 --- a/rust/crates/og-reports/src/yleaf.rs +++ b/rust/crates/og-reports/src/yleaf.rs @@ -1,20 +1,28 @@ -//! `og y yleaf-positions` (step 37): Yleaf's own GRCh38 marker positions, -//! for the pileup step 37 makes beside Yleaf. +//! Step 37's two steps around Yleaf's own command line. //! //! ```text //! og y yleaf-positions --data DATA OUT +//! og y yleaf-no-reads --bam BAM --idxstats IDXSTATS OUT //! ``` //! -//! Writes `chrYpos` for each distinct position in column 4 of -//! `DATA/hg38/new_positions.txt`, in file order, for `samtools mpileup -l`, -//! and checks that `DATA/hg_prediction_tables/tree.json` is there: the two -//! files Yleaf 4.1's `yleaf_constants` names (`HG38`, `NEW_POSITION_FILE`, -//! `TREE_FILE`). Running Yleaf on that pileup stays in `bin/yleaf_run.py -//! predict`, which is glue inside Yleaf's own Python. +//! `yleaf-positions` writes `chrYpos` for each distinct position in +//! column 4 of `DATA/hg38/new_positions.txt`, in file order, for +//! `samtools mpileup -l`, and checks that +//! `DATA/hg_prediction_tables/tree.json` is there: the two files Yleaf 4.1's +//! `yleaf_constants` names (`HG38`, `NEW_POSITION_FILE`, `TREE_FILE`). +//! Step 37 then hands that pileup to Yleaf (`Yleaf -rp`). //! -//! What changes on purpose: the output is written whole or not at all, and -//! a position is ASCII digits only (the Python's `isdigit` also took other -//! Unicode digits). +//! `yleaf-no-reads` writes Yleaf's prediction table for a pileup with no +//! line: Yleaf stops on an empty pileup ("No columns to parse") and writes +//! a header with no row, so the step writes the row Yleaf gives when too +//! few markers had reads: `Hg` NA, `Valid_markers` 0, and `Total_reads`, +//! the mapped reads in `samtools idxstats` (column 3, summed). The sample +//! name is the BAM's file name without its last extension, as Yleaf names +//! it. +//! +//! What changes on purpose from the Python these replace: the output is +//! written whole or not at all, and a position or a read count is ASCII +//! digits only (the Python's `isdigit` also took other Unicode digits). use crate::y::{is_file, path_join}; use crate::{Error, Outcome}; @@ -90,6 +98,57 @@ pub(crate) fn cli(rest: &[String], out: &mut dyn Write, err: &mut dyn Write) -> }) } +/// The header of Yleaf 4's `hg_prediction.hg`. +pub const HG_HEADER: &str = + "Sample_name\tHg\tHg_marker\tTotal_reads\tValid_markers\tQC-score\tQC-1\tQC-2\tQC-3\n"; + +/// `Path(bam).name.rsplit('.', 1)[0]`: Yleaf's name for the sample. +fn sample_name(bam: &str) -> &str { + let name = bam.trim_end_matches('/').rsplit('/').next().unwrap_or(""); + name.rsplit_once('.').map_or(name, |(stem, _)| stem) +} + +fn no_reads(bam: &str, idxstats: &str, out: &str) -> Outcome { + let fail = |e| (e, 1); + let mut reads: u128 = 0; + for line in crate::lines(idxstats, Utf8::Strict).map_err(fail)? { + let line = line.map_err(fail)?; + if let Some(c) = line.split('\t').nth(2) { + let c = crate::strip(c); + if crate::digits(c) { + let n = crate::int(c).and_then(|n| u128::try_from(n).ok()); + reads = n.and_then(|n| reads.checked_add(n)).ok_or_else(|| { + fail(Error(format!("read count {c} too large in {idxstats}"))) + })?; + } + } + } + let body = format!( + "{HG_HEADER}{}\tNA\t\t{reads}\t0\tNA\tNA\tNA\tNA\n", + sample_name(bam) + ); + crate::write_atomic(out, body.as_bytes()).map_err(fail)?; + Ok("No read at any Y marker: Hg NA (insufficient markers)\n".to_owned()) +} + +const NO_READS_SPEC: Spec = Spec { + prog: "og y yleaf-no-reads", + usage: "usage: og y yleaf-no-reads [-h] --bam BAM --idxstats IDXSTATS out\n", + options: &["--bam", "--idxstats"], + positionals: &["out"], + required: &["--bam", "--idxstats"], +}; + +pub(crate) fn no_reads_cli(rest: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + crate::run(&NO_READS_SPEC, rest, out, err, |a| { + Ok(no_reads( + crate::opt(a, "--bam"), + crate::opt(a, "--idxstats"), + &a.pos[0], + )) + }) +} + #[cfg(test)] mod tests { use super::*; @@ -120,4 +179,48 @@ mod tests { ) ); } + + #[test] + fn no_reads_writes_the_na_row_with_the_mapped_reads() { + let d = Dir::new("yleaf-noreads"); + // idxstats: name, length, mapped, unmapped; the last line is `*`. + let idx = d.put( + "idx.txt", + "chr1\t248956422\t35933\t331\nchrY\t57227415\t1200\t7\nshort\t1\n*\t0\t0\t2048\n", + ); + let out = d.path("hg_prediction.hg"); + let line = no_reads("/x/aligned/S1_sorted.bam", &idx, &out).unwrap(); + assert_eq!( + line, + "No read at any Y marker: Hg NA (insufficient markers)\n" + ); + assert_eq!( + d.read("hg_prediction.hg"), + format!("{HG_HEADER}S1_sorted\tNA\t\t37133\t0\tNA\tNA\tNA\tNA\n") + ); + assert_eq!(sample_name("a.b.bam"), "a.b"); + assert_eq!(sample_name("noext"), "noext"); + assert_eq!(sample_name("dir/x.bam"), "x"); + let e = no_reads("x.bam", &d.path("missing.txt"), &d.path("o")).unwrap_err(); + assert_eq!(e.1, 1); + assert!(std::fs::metadata(d.path("o")).is_err()); + } + + #[test] + fn no_reads_cli_routes_through_og_y() { + let d = Dir::new("yleaf-noreads-cli"); + let idx = d.put("idx.txt", "chrY\t57227415\t5\t0\n"); + let out = d.path("t.hg"); + let args: Vec = ["yleaf-no-reads", "--bam", "s.bam", "--idxstats", &idx, &out] + .iter() + .map(|s| (*s).to_owned()) + .collect(); + let (mut o, mut e) = (Vec::new(), Vec::new()); + let code = crate::y::cli(Some("yleaf-no-reads"), &args[1..], &mut o, &mut e); + assert_eq!(code, 0, "{}", String::from_utf8_lossy(&e)); + assert_eq!( + d.read("t.hg"), + format!("{HG_HEADER}s\tNA\t\t5\t0\tNA\tNA\tNA\tNA\n") + ); + } } From 8e509bef0a8a0b54ea2d49604f24781d002df5f0 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:34:26 +0100 Subject: [PATCH 07/14] refactor(y): run Yleaf through its own CLI instead of a Python wrapper that patches it Yleaf 4.1.7 has the options bin/yleaf_run.py stood in for: -rp reads the pileup step 37 makes beside the chrY BAM, --ref-fasta names the reference so it downloads none, --no-update-check keeps it offline. Step 37 and Y_HAPLOGROUP call Yleaf directly, and og y yleaf-no-reads covers the empty pileup. The image test checks the options, the pileup file name, the samtools flags and that the image's marker table and tree match the pinned archive, in shell instead of Python. The wrapper is deleted and dropped from the e2e paths table. Bead: port-yleaf-wrap --- CHANGELOG.md | 1 + bin/yleaf_run.py | 139 ----------------------------- docs/37-y-haplogroup.md | 6 +- modules/local/y_haplogroup/main.nf | 40 ++++++--- scripts/37-y-haplogroup.sh | 50 +++++++---- scripts/ci/e2e-case-paths.tsv | 10 +-- tests/smoke/yleaf.sh | 73 ++++++++------- 7 files changed, 113 insertions(+), 206 deletions(-) delete mode 100755 bin/yleaf_run.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 05c3388e1..b7a3ef87e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Step 37 and the Nextflow `Y_HAPLOGROUP` process run Yleaf through its own command line. `bin/yleaf_run.py`, our Python launcher that patched Yleaf at run time (its data folder, its reference, its samtools calls and its process pools), is gone. Yleaf 4.1.7 has the options the launcher stood in for: `-rp` reads the pileup the step makes beside the chrY BAM, `--ref-fasta` names the reference so it downloads none, and `--no-update-check` keeps it offline. Yleaf now predicts with the marker table and tree its image ships; the image test checks they are the files of the archive `setup.sh --yleaf-data` installs, whose positions the pileup is made at, and that Yleaf still has those options, that pileup file and the samtools flags the pileup uses. When no read falls on any marker, Yleaf stops before it predicts, so `og y yleaf-no-reads` writes the row it gives for too few markers (`Hg` NA, the BAM's mapped reads), as the launcher did. On the fixture's chrY slice the prediction table and every file Yleaf writes are the same, byte for byte, as with the launcher, and so is the empty-pileup row. No option, output file or column changes. A reference whose name does not end in `.fa`, `.fasta` or `.fna` is now refused by Yleaf; `setup.sh` installs `.fasta`. - The self-test of `scripts/ci/check-images.sh` copies the whole tree again once the list of tracked paths passes 128 KiB. It piped that list through `xargs` to `tar`, and GNU `xargs` split a list that long over two `tar` runs; the reading `tar` stopped after the first archive, so Guard's `images` job failed on a broken pipe (exit 123). One `tar` now reads the list itself (`tar --null -T -`). Nothing the check reports changes. - The phased family VCF is indexed by the Rust `og` binary. WhatsHap's step (step 38 and the Nextflow WHATSHAP_PED process) wrote `family.phased.vcf.gz.tbi` with a Python one-liner, `pysam.tabix_index`. It now runs `og vcf index -f`. The `.tbi` keeps its name and format, and its bytes, once inflated, are the ones htslib 1.24 writes (the htslib of the pysam and bcftools the pipeline pins): the same bins in the same order, the same chunks and the same linear index. Only the deflate stream can differ, as two deflate libraries differ. On a 5-million-record three-sample VCF it took 0.75 to 0.91 s and 31 MB at peak, where htslib's own `tabix` took 0.81 to 0.93 s and 19 MB. - As in htslib 1.24, a record ends at the largest of INFO/END, the REF length, the SVLEN of a ``, ``, `` or `` ALT, and a sample's LEN on a gVCF block. A line with no position to read, unsorted records, a sequence that starts again, a record past 2^29 and an empty file stop the index with a message, and nothing is written. Without `-f` it never writes over an index, even one that appears while it runs. diff --git a/bin/yleaf_run.py b/bin/yleaf_run.py deleted file mode 100755 index 614372024..000000000 --- a/bin/yleaf_run.py +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env python3 -"""yleaf_run.py: run Yleaf 4.1.6 (YLEAF_IMAGE) offline, on a pileup made beside it. - -Step 37 makes Yleaf's pileup and the chrY BAM in one SAMTOOLS_IMAGE run, -and the current tree's pileup (og y tree-place) in a second one; the -Nextflow Y_PILEUP task makes all three in one task. Yleaf then reads its pileup from a file instead of running -samtools itself, with its data folder (marker positions, haplogroup trees) -set to the copy `setup.sh --yleaf-data` installs from the release archive as -reference/yleaf-/data, the folder that also holds the current tree. -Yleaf also downloads the whole hg38 FASTA when its read-only config names -none, and checks GitHub for a newer release. So it runs in three parts: - - og y yleaf-positions --data DATA OUT - (YLEAF_IMAGE) write Yleaf's GRCh38 marker positions as the "chrYpos" - list `samtools mpileup -l` reads (rust/crates/og-reports/src/yleaf.rs; - it reads DATA/hg38/new_positions.txt, the file named here by - yleaf_constants, so a Yleaf release that renames it stops that step) - samtools idxstats BAM > IDXSTATS; samtools mpileup -l OUT -ABQ20q1 BAM > PILEUP - (SAMTOOLS_IMAGE) the two commands Yleaf would run, with its default - quality threshold of 20 - yleaf_run.py predict --data DATA --bam BAM --reference FASTA --idxstats IDXSTATS --pileup PILEUP --out DIR - (YLEAF_IMAGE) Yleaf's own flow on its default tree (YFull v14.01), - with its data folder set to DATA, its reference constant set to FASTA - (a BAM never needs the sequence), no update check, each samtools call - served from those two files (a pileup made with other flags than the - ones Yleaf asks for stops the run), and its multiprocessing pools - replaced by a serial map, so every call runs in this process, where - the served samtools calls are. - -DIR/hg_prediction.hg is Yleaf's prediction table (Hg is NA when too few -markers had reads). Standard library plus what Yleaf itself imports. -""" -import multiprocessing -import shutil -import sys -from pathlib import Path - -QUALITY = 20 # Yleaf's -q default; the mpileup above uses the same -FLAGS = f"-ABQ{QUALITY}q1" # what Yleaf 4 gives samtools mpileup at that quality - - -class SerialPool: - """multiprocessing.Pool in this process: the initializer runs here, once.""" - - def __init__(self, processes=None, initializer=None, initargs=(), **kwargs): - if initializer is not None: - initializer(*initargs) - - def __enter__(self): - return self - - def __exit__(self, *args): - return False - - def map(self, fn, items): - return list(map(fn, items)) - - def imap(self, fn, items): - return map(fn, items) - - imap_unordered = imap - - -def use_data(data): - """Point Yleaf's constants at DATA (the yleaf/data folder of its release).""" - from yleaf import yleaf_constants as c - # Absolute: Yleaf opens DATA_FOLDER / HG_PREDICTION_FOLDER / tree.json. - # An absolute HG_PREDICTION_FOLDER replaces DATA_FOLDER in that join; a - # relative one gives data/data/... (a Nextflow task stages it as "data"). - data = Path(data).absolute() - for need in (data / c.HG38 / c.NEW_POSITION_FILE, data / "hg_prediction_tables" / c.TREE_FILE): - if not need.is_file(): - sys.exit(f"ERROR: {need} not found: install Yleaf's data with scripts/setup.sh --yleaf-data ") - c.DATA_FOLDER = data - c.HG_PREDICTION_FOLDER = data / "hg_prediction_tables" - return c - - -def predict(argv): - import argparse - ap = argparse.ArgumentParser(prog="yleaf_run.py predict") - for a in ("--data", "--bam", "--reference", "--idxstats", "--pileup", "--out"): - ap.add_argument(a, required=True) - a = ap.parse_args(argv) - multiprocessing.Pool = SerialPool - yleaf_constants = use_data(a.data) - yleaf_constants.HG38_FULL_GENOME = Path(a.reference) - if Path(a.pileup).stat().st_size == 0: - # No read at any marker: Yleaf's pandas read of an empty pileup fails - # before it predicts, so write the prediction it gives for too few - # markers (Hg NA) here, with the BAM's mapped reads from idxstats. - reads = 0 - with open(a.idxstats) as f: - for line in f: - cols = line.split("\t") - if len(cols) > 2 and cols[2].strip().isdigit(): - reads += int(cols[2]) - Path(a.out).mkdir(parents=True, exist_ok=True) - (Path(a.out) / "hg_prediction.hg").write_text( - "Sample_name\tHg\tHg_marker\tTotal_reads\tValid_markers\tQC-score\tQC-1\tQC-2\tQC-3\n" - f"{Path(a.bam).name.rsplit('.', 1)[0]}\tNA\t\t{reads}\t0\tNA\tNA\tNA\tNA\n") - print("No read at any Y marker: Hg NA (insufficient markers)") - return - from yleaf import Yleaf - - def serve(cmd, stdout_location=None): - if cmd.startswith("samtools idxstats "): - with open(a.idxstats) as f: - shutil.copyfileobj(f, stdout_location) - stdout_location.flush() - elif cmd.startswith("samtools mpileup ") and " > " in cmd: - if f" {FLAGS} " not in f" {cmd} ": - sys.exit(f"ERROR: Yleaf asks for samtools mpileup with other flags than {FLAGS}, the ones the " - f"pileup was made with: {cmd}") - shutil.copyfile(a.pileup, cmd.rsplit(" > ", 1)[1].strip().strip('"')) - elif cmd.startswith("samtools index "): - pass # the BAM is indexed; Yleaf only asks when it finds no .bai - else: - sys.exit(f"ERROR: Yleaf asked for a command this launcher does not serve: {cmd}") - - Yleaf.call_command = serve - sys.argv = ["Yleaf", "-bam", a.bam, "-o", a.out, "-rg", "hg38", "-tree", "yfull", "-force", "-t", "1", - "-q", str(QUALITY), "--no-update-check"] - Yleaf.main() - pred = Path(a.out) / "hg_prediction.hg" - if not pred.is_file() or len(pred.read_text().splitlines()) < 2: - # Yleaf 4 logs a failed sample as "Skipping" and goes on: its log says why. - sys.exit(f"ERROR: Yleaf wrote no prediction ({pred}); its log is in {a.out}") - - -def main(argv): - if argv[:1] == ["predict"]: - predict(argv[1:]) - else: - sys.exit(__doc__) - - -if __name__ == "__main__": - main(sys.argv[1:]) diff --git a/docs/37-y-haplogroup.md b/docs/37-y-haplogroup.md index 6b3cadbb0..0505f95b7 100644 --- a/docs/37-y-haplogroup.md +++ b/docs/37-y-haplogroup.md @@ -10,7 +10,7 @@ The mitochondrial haplogroup (step 12) traces the maternal line only. The Y chro ## Tool -- **Yleaf** 4.1.6 (Ralf et al., Mol Biol Evol 2018; GPL-3.0), Erasmus MC. It reads the BAM's pileup at its Y markers and predicts the haplogroup they support on its default tree, YFull v14.01 (about 183,000 marker positions on GRCh38). +- **Yleaf** 4.1.7 (Ralf et al., Mol Biol Evol 2018; GPL-3.0), Erasmus MC. It reads the BAM's pileup at its Y markers and predicts the haplogroup they support on its default tree, YFull v14.01 (about 183,000 marker positions on GRCh38). - **The current YFull YTree** (release 14.06.0 of 22 September 2026, CC BY 4.0, [github.com/YFullTeam/YTree](https://github.com/YFullTeam/YTree)), walked by `og y tree-place` (`rust/crates/og-reports/src/ytree.rs`). It is newer than Yleaf's v14.01, and the second position table below places SNPs Yleaf's own table does not have, so the branch it reports can sit below the one Yleaf reports. It also writes the path from the root with the calls of every level. The SNPs on YFull's tree have names only; their GRCh38 positions and bases come from two tables of Yleaf 4.1.6 (GPL-3.0): `ftdna_positions_hg38.txt` (made for FamilyTreeDNA's tree, the widest of its tables) first, then `new_positions.txt` (the one Yleaf 4 uses for its YFull v14 tree). SNP names are shared between the trees, so a name is looked up as it stands. Together they place 85% of the tree's 459,468 SNPs (76% from the first table alone), counted on 2026-10-25. @@ -20,7 +20,7 @@ The step reports both results, each with the tree it comes from. Both reports (s - `YLEAF_IMAGE`, and `SAMTOOLS_IMAGE` for the pileup -The step runs in three parts (`bin/yleaf_run.py` says why): Yleaf's marker positions, written by `og y yleaf-positions` in its image, `samtools idxstats` and `samtools mpileup -l -ABQ20q1` (Yleaf 4's own flags at its default quality 20) in `SAMTOOLS_IMAGE`, in the same run as the chrY BAM (step 37 makes the current tree's pileup in a second run; Nextflow's `Y_PILEUP` task makes all three in one task), then Yleaf on that pileup. If a later Yleaf asks samtools for other flags, the launcher stops instead of handing it a pileup made another way. +The step runs in three parts: Yleaf's marker positions, written by `og y yleaf-positions` in its image, `samtools idxstats` and `samtools mpileup -l -ABQ20q1` (Yleaf 4's own flags at its default quality 20) in `SAMTOOLS_IMAGE`, in the same run as the chrY BAM (step 37 makes the current tree's pileup in a second run; Nextflow's `Y_PILEUP` task makes all three in one task), then Yleaf's own command line on that pileup. The step puts the pileup where Yleaf keeps its own, `//temp_haplogroup_pileup.pu`, and runs `Yleaf -bam -rg hg38 --ref-fasta -tree yfull -q 20 -t 1 -rp -force --no-update-check`; `-rp` makes Yleaf read that file instead of running samtools. The image test (the `YLEAF_IMAGE` row of `tests/smoke/commands.tsv`) checks that the image still has these options, that pileup file name and those samtools flags, and that its marker table and tree are the ones in the archive `setup.sh --yleaf-data` installs. Yleaf stops on an empty pileup (no read at any marker) and writes no row; the step then writes the row Yleaf gives for too few markers (`Hg` NA) with `og y yleaf-no-reads`. Pinned in `versions.env`; [Image versions](versions.md) lists the current tag. @@ -43,7 +43,7 @@ In the Nextflow pipeline (`y_haplogroup` in `--tools`) the process runs for the With `run-all.sh`: `TOOLS=...,y_haplogroup` (it is opt-in, so a default run lists it as skipped; without the data it is skipped with the reason). -Yleaf downloads the whole hg38 FASTA on its first run unless its config file names one, and the image's config is read-only. The launcher points Yleaf at the pipeline's reference before it starts (a BAM never needs the sequence itself) and turns off its check for a newer release, so nothing is downloaded and every container runs without a network. +Yleaf downloads the whole hg38 FASTA on its first run unless it is given one. The step passes the pipeline's reference with `--ref-fasta` (Yleaf checks its chrY length against hg38 in the `.fai`; a BAM never needs the sequence itself) and `--no-update-check` turns off its check for a newer release, so nothing is downloaded and every container runs without a network. `--ref-fasta` takes a file ending in `.fa`, `.fasta` or `.fna`, as `setup.sh` installs it. ## Output diff --git a/modules/local/y_haplogroup/main.nf b/modules/local/y_haplogroup/main.nf index 8cf601bfb..0f56b7fb8 100644 --- a/modules/local/y_haplogroup/main.nf +++ b/modules/local/y_haplogroup/main.nf @@ -6,17 +6,19 @@ workflows/bam_analysis.nf passes the BAMs whose sex is male, the sex INDEXCOV has checked against the reads. - Yleaf (4.1.6, its own tree YFull v14.01) reads its marker tables from - --yleaf_data (setup.sh --yleaf-data), and its pileup is made in the - samtools pass that also writes the chrY BAM, so the work is three - processes, as in scripts/37-y-haplogroup.sh (bin/yleaf_run.py says why): + Yleaf (its own tree YFull v14.01) reads a pileup at its marker positions + (from --yleaf_data, setup.sh --yleaf-data) made in the samtools pass that + also writes the chrY BAM, so the work is three processes, as in + scripts/37-y-haplogroup.sh (its comments say why): Y_POSITIONS Yleaf's GRCh38 marker positions (og y yleaf-positions, in the Yleaf image, once) Y_PILEUP samtools idxstats and the pileup at those positions, and _chrY.bam (+ .bai), the chrY reads only: the file YFull and the Y-DNA projects read. Nothing is uploaded. - Y_HAPLOGROUP Yleaf on that pileup, offline; Hg NA means too few Y - markers had reads to call a haplogroup + Y_HAPLOGROUP Yleaf's own command line on that pileup (-rp), offline; + Hg NA means too few Y markers had reads to call a + haplogroup (og y yleaf-no-reads writes that table when + the pileup is empty, where Yleaf stops) Each process also does the same for the pinned YFull YTree release in /ytree (og y tree-positions and og y tree-place): its SNP positions, a second pileup at them, and the placement, published as _y_tree.txt and @@ -98,8 +100,9 @@ process Y_PILEUP { script: // -ABQ20q1: the flags Yleaf 4 gives samtools mpileup at its default quality 20 - // (bin/yleaf_run.py stops if Yleaf asks for others). The current tree's - // pileup keeps -AQ20q1, the flags og y tree-place was checked with. + // (the YLEAF_IMAGE row of tests/smoke/commands.tsv checks they are still + // Yleaf's). The current tree's pileup keeps -AQ20q1, the flags og y + // tree-place was checked with. """ samtools idxstats ${bam} > ${meta.id}_idxstats.txt samtools mpileup -l ${positions} -ABQ20q1 ${bam} > ${meta.id}_y_pileup.txt @@ -149,8 +152,25 @@ process Y_HAPLOGROUP { script: """ - yleaf_run.py predict --data ${yleaf_data} --bam ${bam} --reference "\$(readlink -f ${reference})" \\ - --idxstats ${idxstats} --pileup ${pileup} --out yleaf + # Yleaf names the sample, and its folder, after the BAM without its + # extension, and reads the pileup it finds there (-rp). --ref-fasta names + # the reference so it downloads none; it never reads a BAM's sequence. + stem=\$(basename ${bam}); stem=\${stem%.*} + if [ -s ${pileup} ]; then + mkdir -p "yleaf/\$stem" + cp ${pileup} "yleaf/\$stem/temp_haplogroup_pileup.pu" + Yleaf -bam ${bam} -o yleaf -rg hg38 --ref-fasta "\$(readlink -f ${reference})" -tree yfull \\ + -q 20 -t 1 -rp -force --no-update-check + rm -f "yleaf/\$stem/temp_haplogroup_pileup.pu" + else + mkdir -p yleaf + og y yleaf-no-reads --bam ${bam} --idxstats ${idxstats} yleaf/hg_prediction.hg + fi + if [ "\$(grep -c . yleaf/hg_prediction.hg 2>/dev/null || true)" -lt 2 ]; then + # Yleaf logs a failed sample as "Skipping" and goes on: its log says why. + echo "ERROR: Yleaf wrote no prediction (yleaf/hg_prediction.hg); its log is yleaf/run.log" >&2 + exit 1 + fi cp yleaf/hg_prediction.hg ${meta.id}_y_haplogroup.txt if [ -s ${yleaf_data}/ytree/version.txt ]; then og y tree-place --tree-dir ${yleaf_data}/ytree --pileup ${tree_pileup} \\ diff --git a/scripts/37-y-haplogroup.sh b/scripts/37-y-haplogroup.sh index d8b70859e..c05d4763f 100755 --- a/scripts/37-y-haplogroup.sh +++ b/scripts/37-y-haplogroup.sh @@ -2,11 +2,12 @@ # 37-y-haplogroup.sh — Y-chromosome haplogroup (paternal line) with Yleaf # Usage: ./scripts/37-y-haplogroup.sh # -# Opt-in. Yleaf (YLEAF_IMAGE, 4.1.6) reads the BAM's pileup at its -# Y-chromosome markers and predicts the haplogroup they support on its own -# tree, YFull v14.01. Its marker tables come from setup.sh --yleaf-data; the -# pileup is made in the same SAMTOOLS_IMAGE run as the chrY BAM (the current -# tree's pileup in a second one), and bin/yleaf_run.py runs Yleaf on it. Runs only when step 16 (indexcov) infers male from +# Opt-in. Yleaf (YLEAF_IMAGE) reads the BAM's pileup at its Y-chromosome +# markers and predicts the haplogroup they support on its own tree, YFull +# v14.01. The marker positions come from setup.sh --yleaf-data; the pileup +# is made in the same SAMTOOLS_IMAGE run as the chrY BAM (the current tree's +# pileup in a second one), and Yleaf's own command line reads it (-rp, reuse +# the pileup). Runs only when step 16 (indexcov) infers male from # the reads: the sex comes from indexcov/indexcov-indexcov.ped, not from an # argument. A female sample, or one with no Y to read, is skipped with one line. # @@ -69,10 +70,13 @@ rm -rf "${OUTDIR}/yleaf" "$OUT" "${OUTDIR}/${SAMPLE}_y_tree.txt" "${OUTDIR}/${SA "${OUTDIR}/${SAMPLE}_y_private_snps.tsv" "$CHRY" "${CHRY}.bai" TREE=false [ -s "${YTREE}/version.txt" ] && TREE=true -# Yleaf downloads the whole hg38 FASTA unless its read-only config names one: -# og y yleaf-positions writes Yleaf's marker positions, SAMTOOLS_IMAGE makes the -# pileup Yleaf would make, and Yleaf then runs on it, offline -# (bin/yleaf_run.py says why and how). +# og y yleaf-positions writes Yleaf's marker positions and SAMTOOLS_IMAGE +# makes the pileup Yleaf would make (its flags at its default quality 20), in +# the run that also writes the chrY BAM. Yleaf then reads that pileup where it +# keeps its own (-rp), offline: --ref-fasta names the reference so it +# downloads none (it never reads a BAM's sequence), --no-update-check skips +# GitHub. It stops on an empty pileup, so with no read at any marker +# og y yleaf-no-reads writes the table Yleaf gives for too few markers. O=$(cpath "$OUTDIR") echo "[1/3] Yleaf's Y marker positions..." run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 2g "$YLEAF_IMAGE" \ @@ -92,10 +96,27 @@ if $TREE; then fi echo "[3/3] Yleaf..." echo "Reference: ${REF_FASTA}" -run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 4g "$YLEAF_IMAGE" \ - python3 /pgp-bin/yleaf_run.py predict --data "$(cpath "$YDATA")" --bam "$(cpath "$BAM")" --reference "$REF_FASTA_C" \ - --idxstats "${O}/idxstats.txt" --pileup "${O}/pileup.txt" --out "${O}/yleaf" +# Yleaf names the sample, and its folder, after the BAM without its extension. +STEM=$(basename "$BAM"); STEM=${STEM%.*} +if [ -s "${OUTDIR}/pileup.txt" ]; then + mkdir -p "${OUTDIR}/yleaf/${STEM}" + mv "${OUTDIR}/pileup.txt" "${OUTDIR}/yleaf/${STEM}/temp_haplogroup_pileup.pu" + run_in --cpus 1 --memory 4g "$YLEAF_IMAGE" \ + Yleaf -bam "$(cpath "$BAM")" -o "${O}/yleaf" -rg hg38 --ref-fasta "$REF_FASTA_C" -tree yfull \ + -q 20 -t 1 -rp -force --no-update-check + rm -f "${OUTDIR}/yleaf/${STEM}/temp_haplogroup_pileup.pu" +else + mkdir -p "${OUTDIR}/yleaf" + run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 2g "$YLEAF_IMAGE" \ + /pgp-bin/og y yleaf-no-reads --bam "$(cpath "$BAM")" --idxstats "${O}/idxstats.txt" "${O}/yleaf/hg_prediction.hg" +fi rm -f "${OUTDIR}/idxstats.txt" "${OUTDIR}/pileup.txt" +PRED="${OUTDIR}/yleaf/hg_prediction.hg" +if [ ! -s "$PRED" ] || [ "$(grep -c . "$PRED")" -lt 2 ]; then + # Yleaf logs a failed sample as "Skipping" and goes on: its log says why. + echo "ERROR: Yleaf wrote no prediction (${PRED}); its log is ${OUTDIR}/yleaf/run.log" >&2 + exit 1 +fi TREE_LINE="Current tree: not installed (${YTREE}); run scripts/setup.sh --yleaf-data ${GENOME_DIR} to add it" if $TREE; then TREE_LINE=$(run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 4g "$YLEAF_IMAGE" \ @@ -121,11 +142,6 @@ if $TREE; then fi fi -PRED="${OUTDIR}/yleaf/hg_prediction.hg" -if [ ! -s "$PRED" ] || [ "$(grep -c . "$PRED")" -lt 2 ]; then - echo "ERROR: Yleaf wrote no prediction (${PRED})." >&2 - exit 1 -fi cp "$PRED" "$OUT" read -r HG MARKERS QC < <(awk -F'\t' 'NR == 1 { for (i = 1; i <= NF; i++) c[$i] = i; next } NR == 2 { print $c["Hg"], $c["Valid_markers"], $c["QC-score"] }' "$OUT") diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index 5b2f85dc2..0c9171533 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -102,7 +102,7 @@ genotype-rules-1-phase assets/genotype_rules.tsv assets/traits.tsv modules/local genotype-rules-2-g2 assets/genotype_rules.tsv bin/collect_summary.py bin/og genotype-rules-2-lpa-null bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py modules/local/lpa/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/generate-report.sh genotype-rules-2-smn1-marker bin/collect_summary.py bin/og bin/render_report.py modules/local/parascopy/* scripts/24-html-report.sh scripts/35-paralogs.sh scripts/generate-report.sh -haplogroup-explore-1-files assets/callability_genes.tsv assets/live/index.html assets/mito_confirmed.tsv assets/viewer/index.html bin/collect_summary.py bin/og bin/yleaf_run.py modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/vcf_precheck/* modules/local/y_haplogroup/* scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh +haplogroup-explore-1-files assets/callability_genes.tsv assets/live/index.html assets/mito_confirmed.tsv assets/viewer/index.html bin/collect_summary.py bin/og modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/vcf_precheck/* modules/local/y_haplogroup/* scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh hba-short-read-accuracy-1-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/export_consumer/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz hla-second-caller-1-bash assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/og bin/render_report.py scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh hla-second-caller-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/hla_typing/* modules/local/html_report/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh @@ -117,7 +117,7 @@ nextflow-from-fastq-1-bash assets/callability_genes.tsv assets/construct_variant nextflow-from-fastq-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgs_scores.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/viewer/index.html bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/cyrius/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/prs/* modules/local/roh/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/validate-setup.sh nextflow-from-fastq-3-sexcheck assets/callability_genes.tsv assets/live/index.html assets/viewer/index.html bin/collect_summary.py modules/local/indexcov/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh nextflow-hardening-1-manta assets/par_grch38.bed assets/sv_rule_regions.tsv bin/collect_summary.py bin/family_report.py bin/og scripts/04-manta.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh -nextflow-hardening-2-nf assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv assets/viewer/index.html bin/cn_loci.tsv bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py bin/yleaf_run.py modules/local/clinvar_screen/* modules/local/delly/* modules/local/html_report/* modules/local/indexcov/* modules/local/manta/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/survivor_merge/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* modules/local/y_haplogroup/* scripts/04-manta.sh scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/10-telomere-hunter.sh scripts/12-mito-haplogroup.sh scripts/15-duphold.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/22-survivor-merge.sh scripts/24-html-report.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh +nextflow-hardening-2-nf assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv assets/viewer/index.html bin/cn_loci.tsv bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/delly/* modules/local/html_report/* modules/local/indexcov/* modules/local/manta/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/survivor_merge/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* modules/local/y_haplogroup/* scripts/04-manta.sh scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/10-telomere-hunter.sh scripts/12-mito-haplogroup.sh scripts/15-duphold.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/22-survivor-merge.sh scripts/24-html-report.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh nextflow-report-parity-1-sections assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv bin/cn_loci.tsv bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/annotsv/* modules/local/clinvar_screen/* modules/local/expansion_hunter/* modules/local/export_consumer/* modules/local/html_report/* modules/local/mito_variants/* modules/local/pypgx/* modules/local/slivar/* modules/local/stranger/* modules/local/vcf_precheck/* modules/local/vep/* scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/13-vep-annotation.sh scripts/15-duphold.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/31-slivar.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz nf-wiring-1-report-tables assets/acmg_sf.tsv assets/test_samplesheet.csv assets/stub/* bin/collect_summary.py bin/cpsr_sample_id bin/og bin/render_report.py bin/results_index.py conf/test_all.config conf/test.config modules/local/align_minimap2/* modules/local/clinical_filter/* modules/local/clinvar_screen/* modules/local/cpsr/* modules/local/html_report/* modules/local/slivar/* modules/local/vep/* scripts/24-html-report.sh tests/fixtures/consumer_export/sites.tsv.gz nf-wiring-2-sv-str-tools bin/family_report.py bin/og modules/local/cnvpytor/* modules/local/duphold/* modules/local/expansion_hunter/* modules/local/indexcov/* modules/local/manta/* modules/local/vcf_precheck/* @@ -163,7 +163,7 @@ str-catalog-1-strchive assets/str_extra_loci.json assets/stub/db/catalog.json as str-extra-repeats-1-readings assets/str_extra_loci.json bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/expansion_hunter/* modules/local/html_report/* scripts/09-expansion-hunter.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh rust/crates/og-dev/src/fixture.rs sv-mito-telomere-steps-1-sv-merge scripts/22-survivor-merge.sh scripts/ci/fake-docker/lib.sh tests/fixtures/sv/cnvpytor.vcf.in tests/fixtures/sv/delly.vcf.in tests/fixtures/sv/manta.vcf.in sv-mito-telomere-steps-2-mito assets/callability_genes.tsv bin/collect_summary.py scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh -sv-mito-telomere-steps-3-y-haplogroup assets/live/index.html assets/viewer/index.html bin/og bin/yleaf_run.py scripts/16-indexcov.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh +sv-mito-telomere-steps-3-y-haplogroup assets/live/index.html assets/viewer/index.html bin/og scripts/16-indexcov.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh sv-mito-telomere-steps-4-clinvar-tier bin/og scripts/23-clinical-filter.sh scripts/ci/fake-docker/lib.sh sv-mito-telomere-steps-5-giab scripts/benchmark-variants.sh scripts/ci/fake-docker/lib.sh sv-mito-telomere-steps-6-scatter scripts/03a-gatk-haplotypecaller.sh scripts/03b-freebayes.sh scripts/ci/fake-docker/lib.sh @@ -176,9 +176,9 @@ vendor-vcf-intake-4-nf-header-clinvar assets/acmg_sf.tsv assets/callability_gene vendor-vcf-intake-5-docs scripts/ci/fake-docker/lib.sh viral-reads-1 bin/og modules/local/vcf_precheck/* modules/local/viral/* rust/crates/og-dev/src/fixture.rs scripts/47-viral.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh vntyper-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/vcf_precheck/* modules/local/vntyper/* scripts/24-html-report.sh scripts/52-vntyper.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -y-private-report-1-private-and-tree assets/live/index.html assets/viewer/index.html bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/yleaf_run.py modules/local/html_report/* modules/local/y_haplogroup/* rust/crates/og-dev/src/fixture.rs scripts/16-indexcov.sh scripts/24-html-report.sh scripts/37-y-haplogroup.sh scripts/generate-report.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh +y-private-report-1-private-and-tree assets/live/index.html assets/viewer/index.html bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/y_haplogroup/* rust/crates/og-dev/src/fixture.rs scripts/16-indexcov.sh scripts/24-html-report.sh scripts/37-y-haplogroup.sh scripts/generate-report.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh y-str-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/live/index.html assets/mito_confirmed.tsv assets/stub/sample.vcf.gz assets/viewer/index.html assets/ystr_panel.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/indexcov/* modules/local/vcf_precheck/* modules/local/y_str/* rust/crates/og-dev/src/fixture.rs scripts/16-indexcov.sh scripts/24-html-report.sh scripts/45-y-str.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -y-tree-current-1-placement assets/live/index.html assets/viewer/index.html bin/og bin/yleaf_run.py scripts/16-indexcov.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh +y-tree-current-1-placement assets/live/index.html assets/viewer/index.html bin/og scripts/16-indexcov.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh rust-z2e-readers bin/og tests/e2e/lib.sh scripts/ci/image-smoke.sh scripts/ci/nf-trace-requests.tsv rust-c2b-traits assets/traits.tsv assets/genotype_rules.tsv bin/og modules/local/traits/* scripts/42-traits.sh qc-3-cram-checksum modules/local/cram_archive/* scripts/34-cram-archive.sh scripts/lib/common.sh diff --git a/tests/smoke/yleaf.sh b/tests/smoke/yleaf.sh index b6e9641fe..a723b80bf 100644 --- a/tests/smoke/yleaf.sh +++ b/tests/smoke/yleaf.sh @@ -1,34 +1,43 @@ # shellcheck shell=sh +# shellcheck disable=SC2154 # YLEAF_DATA_VERSION and YLEAF_DATA_SHA256 come from versions.env, set by image-smoke.sh # YLEAF_IMAGE row. Steps 37 and Y_HAPLOGROUP make Yleaf's pileup in -# SAMTOOLS_IMAGE and run Yleaf on it with bin/yleaf_run.py and the data -# folder setup.sh --yleaf-data installs (the e2e case -# sv-mito-telomere-steps-3-y-haplogroup runs that on the fixture's chrY -# slice). This row fetches that archive as setup.sh does (tag v), -# checks its sha256 (YLEAF_DATA_SHA256), and checks what the launcher relies -# on: the marker table, the tree, and the functions, flags and calls it -# replaces or serves. -set -e -python3 - <<'PY' -import hashlib, inspect, io, os, tarfile, urllib.request -from yleaf import Yleaf, predict_haplogroup, yleaf_constants as c -v, want = os.environ["YLEAF_DATA_VERSION"], os.environ["YLEAF_DATA_SHA256"] -blob = urllib.request.urlopen(f"https://github.com/genid/Yleaf/archive/refs/tags/v{v}.tar.gz", timeout=120).read() -got = hashlib.sha256(blob).hexdigest() -print(f"archive sha256 {got}, versions.env {want}") -tar = tarfile.open(fileobj=io.BytesIO(blob)) -pos = tar.extractfile(f"Yleaf-{v}/yleaf/data/hg38/{c.NEW_POSITION_FILE}").read().decode() -tree = tar.extractfile(f"Yleaf-{v}/yleaf/data/hg_prediction_tables/{c.TREE_FILE}").read() -n = sum(1 for line in pos.splitlines() if line.strip()) -print(f"{n} marker positions, tree {len(tree)} bytes") -open("/out/sha_ok.txt", "w").write("ok\n" if got == want else f"got {got}\n") -open("/out/positions_count.txt", "w").write(f"{n}\n") -src = inspect.getsource(Yleaf) -missing = [s for s in ("def call_command(", "samtools idxstats", "samtools mpileup", "-ABQ{quality_thresh}q1", - "multiprocessing.Pool(", "def check_reference(", "HG38_FULL_GENOME", "DATA_FOLDER", - "--no-update-check", "LOG.warning(f\"Skipping") - if s not in src] -missing += [s for s in ("multiprocessing.Pool(", "initializer=", "imap_unordered(", "HG_PREDICTION_FOLDER") - if s not in inspect.getsource(predict_haplogroup)] -print("missing:", missing) -open("/out/launcher_contract.txt", "w").write("ok\n" if not missing else "missing: " + ", ".join(missing) + "\n") -PY +# SAMTOOLS_IMAGE at the marker positions of the data folder setup.sh +# --yleaf-data installs, and run Yleaf's own command line on it (-rp); the +# e2e case sv-mito-telomere-steps-3-y-haplogroup runs that on the fixture's +# chrY slice. This row fetches that archive (tag v), checks its +# sha256 (YLEAF_DATA_SHA256), and checks what the steps rely on: the marker +# table and the tree in the archive are the ones this Yleaf predicts with, +# and this Yleaf still has the options, the pileup file and the samtools +# flags the steps use. +set -eu +v=$YLEAF_DATA_VERSION +t=$(mktemp -d) +# codeload is where github.com/archive/refs/tags/v.tar.gz (setup.sh's +# URL) redirects; the image's busybox wget stops on github.com's long response +# headers. The sha256 below shows the bytes are the ones setup.sh checks. +wget -q -T 120 -O "$t/a.tgz" "https://codeload.github.com/genid/Yleaf/tar.gz/refs/tags/v${v}" +got=$(sha256sum "$t/a.tgz" | cut -d' ' -f1) +echo "archive sha256 ${got}, versions.env ${YLEAF_DATA_SHA256}" +if [ "$got" = "$YLEAF_DATA_SHA256" ]; then echo ok > /out/sha_ok.txt; else echo "got $got" > /out/sha_ok.txt; fi +d="Yleaf-${v}/yleaf/data" +tar -xzf "$t/a.tgz" -C "$t" "${d}/hg38/new_positions.txt" "${d}/hg_prediction_tables/tree.json" +n=$(grep -c . "$t/${d}/hg38/new_positions.txt") +echo "${n} marker positions, tree $(wc -c < "$t/${d}/hg_prediction_tables/tree.json") bytes" +echo "$n" > /out/positions_count.txt + +# The installed Yleaf's own package folder (its data and Yleaf.py). +pkg=$(dirname "$(find / -path '*/site-packages/yleaf/Yleaf.py' 2>/dev/null | head -n 1)") +missing="" +for f in hg38/new_positions.txt hg_prediction_tables/tree.json; do + cmp -s "$t/${d}/$f" "${pkg}/data/$f" || missing="${missing} data/${f} (differs from the archive)" +done +help=$(Yleaf -h 2>&1 || true) +for o in -rp --reuse_pileup --ref-fasta --no-update-check -rg -tree -force; do + printf '%s\n' "$help" | grep -q -e " ${o}[ ,]" -e " ${o}\$" || missing="${missing} option ${o}" +done +for s in 'temp_haplogroup_pileup.pu' '-ABQ{quality_thresh}q1' 'Reusing existing pileup file' 'LOG.warning(f"Skipping'; do + grep -qF -e "$s" "${pkg}/Yleaf.py" || missing="${missing} Yleaf.py: ${s}" +done +echo "missing:${missing:- none}" +if [ -z "$missing" ]; then echo ok > /out/launcher_contract.txt; else echo "missing:${missing}" > /out/launcher_contract.txt; fi +rm -rf "$t" From 9432051f517d554d68912d73d88f13433a72b6ba Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:37:54 +0100 Subject: [PATCH 08/14] ci(guard): quote the empty CI value in the og-browser control (SC1007) Bead: D1 --- .github/workflows/guard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index 32d0bffbc..d6a2905d6 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -505,7 +505,7 @@ jobs: cat "$log" grep -F "[FAIL] no Chrome or Chromium to load the test page" "$log" echo "Control: without Chrome the browser checks fail in CI." - CHROME=/nonexistent CI= cargo test --locked -p og-browser --test cdp -- --nocapture > "$log" 2>&1 || { cat "$log"; exit 1; } + CHROME=/nonexistent CI='' cargo test --locked -p og-browser --test cdp -- --nocapture > "$log" 2>&1 || { cat "$log"; exit 1; } cat "$log" test "$(grep -c -F '[SKIP] no Chrome or Chromium to load the test page' "$log")" -eq 2 echo "Outside CI, without Chrome the browser checks skip." From 03e3b85734f49808b855d13ac1e0efd76d6767b0 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 21:36:06 +0100 Subject: [PATCH 09/14] fix(y): stop with a clear message on a missing prediction or a compressed reference Y_HAPLOGROUP's no-prediction guard now fires when Yleaf writes no hg_prediction.hg at all, as step 37's does. Step 37 refuses a REF_FASTA whose name Yleaf's --ref-fasta does not take, before Yleaf raises a traceback on it. --- CHANGELOG.md | 2 +- docs/37-y-haplogroup.md | 2 +- modules/local/y_haplogroup/main.nf | 2 +- scripts/37-y-haplogroup.sh | 6 ++++++ 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7a3ef87e..c37218284 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- Step 37 and the Nextflow `Y_HAPLOGROUP` process run Yleaf through its own command line. `bin/yleaf_run.py`, our Python launcher that patched Yleaf at run time (its data folder, its reference, its samtools calls and its process pools), is gone. Yleaf 4.1.7 has the options the launcher stood in for: `-rp` reads the pileup the step makes beside the chrY BAM, `--ref-fasta` names the reference so it downloads none, and `--no-update-check` keeps it offline. Yleaf now predicts with the marker table and tree its image ships; the image test checks they are the files of the archive `setup.sh --yleaf-data` installs, whose positions the pileup is made at, and that Yleaf still has those options, that pileup file and the samtools flags the pileup uses. When no read falls on any marker, Yleaf stops before it predicts, so `og y yleaf-no-reads` writes the row it gives for too few markers (`Hg` NA, the BAM's mapped reads), as the launcher did. On the fixture's chrY slice the prediction table and every file Yleaf writes are the same, byte for byte, as with the launcher, and so is the empty-pileup row. No option, output file or column changes. A reference whose name does not end in `.fa`, `.fasta` or `.fna` is now refused by Yleaf; `setup.sh` installs `.fasta`. +- Step 37 and the Nextflow `Y_HAPLOGROUP` process run Yleaf through its own command line. `bin/yleaf_run.py`, our Python launcher that patched Yleaf at run time (its data folder, its reference, its samtools calls and its process pools), is gone. Yleaf 4.1.7 has the options the launcher stood in for: `-rp` reads the pileup the step makes beside the chrY BAM, `--ref-fasta` names the reference so it downloads none, and `--no-update-check` keeps it offline. Yleaf now predicts with the marker table and tree its image ships; the image test checks they are the files of the archive `setup.sh --yleaf-data` installs, whose positions the pileup is made at, and that Yleaf still has those options, that pileup file and the samtools flags the pileup uses. When no read falls on any marker, Yleaf stops before it predicts, so `og y yleaf-no-reads` writes the row it gives for too few markers (`Hg` NA, the BAM's mapped reads), as the launcher did. On the fixture's chrY slice the prediction table and every file Yleaf writes are the same, byte for byte, as with the launcher, and so is the empty-pileup row. No option, output file or column changes. Step 37 now stops with a clear message when `REF_FASTA` does not end in `.fa`, `.fasta` or `.fna` (a compressed FASTA, say): Yleaf's `--ref-fasta` takes no other name, and the Nextflow `--reference` already asks for a plain FASTA. `setup.sh` installs `.fasta`. - The self-test of `scripts/ci/check-images.sh` copies the whole tree again once the list of tracked paths passes 128 KiB. It piped that list through `xargs` to `tar`, and GNU `xargs` split a list that long over two `tar` runs; the reading `tar` stopped after the first archive, so Guard's `images` job failed on a broken pipe (exit 123). One `tar` now reads the list itself (`tar --null -T -`). Nothing the check reports changes. - The phased family VCF is indexed by the Rust `og` binary. WhatsHap's step (step 38 and the Nextflow WHATSHAP_PED process) wrote `family.phased.vcf.gz.tbi` with a Python one-liner, `pysam.tabix_index`. It now runs `og vcf index -f`. The `.tbi` keeps its name and format, and its bytes, once inflated, are the ones htslib 1.24 writes (the htslib of the pysam and bcftools the pipeline pins): the same bins in the same order, the same chunks and the same linear index. Only the deflate stream can differ, as two deflate libraries differ. On a 5-million-record three-sample VCF it took 0.75 to 0.91 s and 31 MB at peak, where htslib's own `tabix` took 0.81 to 0.93 s and 19 MB. - As in htslib 1.24, a record ends at the largest of INFO/END, the REF length, the SVLEN of a ``, ``, `` or `` ALT, and a sample's LEN on a gVCF block. A line with no position to read, unsorted records, a sequence that starts again, a record past 2^29 and an empty file stop the index with a message, and nothing is written. Without `-f` it never writes over an index, even one that appears while it runs. diff --git a/docs/37-y-haplogroup.md b/docs/37-y-haplogroup.md index 0505f95b7..55a65dee7 100644 --- a/docs/37-y-haplogroup.md +++ b/docs/37-y-haplogroup.md @@ -43,7 +43,7 @@ In the Nextflow pipeline (`y_haplogroup` in `--tools`) the process runs for the With `run-all.sh`: `TOOLS=...,y_haplogroup` (it is opt-in, so a default run lists it as skipped; without the data it is skipped with the reason). -Yleaf downloads the whole hg38 FASTA on its first run unless it is given one. The step passes the pipeline's reference with `--ref-fasta` (Yleaf checks its chrY length against hg38 in the `.fai`; a BAM never needs the sequence itself) and `--no-update-check` turns off its check for a newer release, so nothing is downloaded and every container runs without a network. `--ref-fasta` takes a file ending in `.fa`, `.fasta` or `.fna`, as `setup.sh` installs it. +Yleaf downloads the whole hg38 FASTA on its first run unless it is given one. The step passes the pipeline's reference with `--ref-fasta` (Yleaf checks its chrY length against hg38 in the `.fai`; a BAM never needs the sequence itself) and `--no-update-check` turns off its check for a newer release, so nothing is downloaded and every container runs without a network. `--ref-fasta` takes only a file ending in `.fa`, `.fasta` or `.fna`, as `setup.sh` installs it, so step 37 stops with a message on any other `REF_FASTA` (a compressed FASTA, say). ## Output diff --git a/modules/local/y_haplogroup/main.nf b/modules/local/y_haplogroup/main.nf index 0f56b7fb8..d23074029 100644 --- a/modules/local/y_haplogroup/main.nf +++ b/modules/local/y_haplogroup/main.nf @@ -166,7 +166,7 @@ process Y_HAPLOGROUP { mkdir -p yleaf og y yleaf-no-reads --bam ${bam} --idxstats ${idxstats} yleaf/hg_prediction.hg fi - if [ "\$(grep -c . yleaf/hg_prediction.hg 2>/dev/null || true)" -lt 2 ]; then + if [ ! -s yleaf/hg_prediction.hg ] || [ "\$(grep -c . yleaf/hg_prediction.hg)" -lt 2 ]; then # Yleaf logs a failed sample as "Skipping" and goes on: its log says why. echo "ERROR: Yleaf wrote no prediction (yleaf/hg_prediction.hg); its log is yleaf/run.log" >&2 exit 1 diff --git a/scripts/37-y-haplogroup.sh b/scripts/37-y-haplogroup.sh index c05d4763f..b76dbc74b 100755 --- a/scripts/37-y-haplogroup.sh +++ b/scripts/37-y-haplogroup.sh @@ -65,6 +65,12 @@ if [ "$SEX" != male ]; then exit 0 fi +# Yleaf's --ref-fasta takes only these names (a compressed FASTA stops it +# with a traceback), as the Nextflow pipeline's --reference does. +case "$REF_FASTA" in + *.fa | *.fasta | *.fna) ;; + *) echo "ERROR: Yleaf needs a plain reference FASTA ending in .fa, .fasta or .fna (REF_FASTA is ${REF_FASTA})" >&2; exit 1 ;; +esac mkdir -p "$OUTDIR" rm -rf "${OUTDIR}/yleaf" "$OUT" "${OUTDIR}/${SAMPLE}_y_tree.txt" "${OUTDIR}/${SAMPLE}_y_tree_path.tsv" \ "${OUTDIR}/${SAMPLE}_y_private_snps.tsv" "$CHRY" "${CHRY}.bai" From 9cebacd03f718ce667b94eab590f77c8765da5b3 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 23:20:04 +0100 Subject: [PATCH 10/14] fix(search): veil each index item by its own mentions, as results_index.py does build() worked out one veil per term from the mentions of all its items, so an item took the veil of a locus only another section names. Each item now gets the veil of the term plus its own mentions. --- rust/crates/og-render/src/index.rs | 23 +++++++++++---------- rust/crates/og-render/tests/index_search.rs | 19 ++++++++++++----- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/rust/crates/og-render/src/index.rs b/rust/crates/og-render/src/index.rs index 43f29c0c0..5f7797a74 100644 --- a/rust/crates/og-render/src/index.rs +++ b/rust/crates/og-render/src/index.rs @@ -528,18 +528,19 @@ pub fn build( } let mut out = IndexMap::new(); for (term, items) in terms { - let shown: Vec = std::iter::once(term.clone()) - .chain( - items - .iter() - .flat_map(|it| it.mentions.iter()) - .map(|(t, l)| format!("{t} {}", l.as_deref().unwrap_or(""))), - ) - .collect(); - let veil = veil_of(spans, &shown.join(" ")); let list = items .into_iter() .map(|it| { + // The veil of each item, from the term and that item's own + // mentions: a locus another section names does not veil it. + let shown: Vec = std::iter::once(term.clone()) + .chain( + it.mentions + .iter() + .map(|(t, l)| format!("{t} {}", l.as_deref().unwrap_or(""))), + ) + .collect(); + let veil = veil_of(spans, &shown.join(" ")); let mut o = IndexMap::new(); o.insert("section".to_owned(), Value::Str(it.section)); o.insert("anchor".to_owned(), it.anchor); @@ -559,8 +560,8 @@ pub fn build( }) .collect(); o.insert("mentions".to_owned(), Value::Array(mentions)); - if let Some(id) = &veil { - o.insert("veil".to_owned(), Value::Str(id.clone())); + if let Some(id) = veil { + o.insert("veil".to_owned(), Value::Str(id)); } Value::Object(o) }) diff --git a/rust/crates/og-render/tests/index_search.rs b/rust/crates/og-render/tests/index_search.rs index 3a5ea9e77..13c7218ac 100644 --- a/rust/crates/og-render/tests/index_search.rs +++ b/rust/crates/og-render/tests/index_search.rs @@ -654,11 +654,20 @@ fn the_veil_tags_the_items_that_name_a_closed_result() { .collect() }; assert_eq!(veil("BRCA2"), ["sf-BRCA2"; 4]); - // CFTR's ClinVar hit is chr7:1000 itself. - assert!( - veil("CFTR").iter().all(|v| v == "clinvar-chr7-1000"), - "{:?}", - veil("CFTR") + // Each item by its own mentions: CFTR's ClinVar hit and its carrier row + // name chr7:1000 itself, while its slivar row at chr7:3000 names no + // closed result, so the locus the other sections name does not veil it. + let cftr: Vec<(String, String)> = items(&idx, "CFTR") + .iter() + .map(|it| (s(get(it, &["section"])), s(get(it, &["veil"])))) + .collect(); + assert_eq!( + cftr, + [ + ("clinvar".to_owned(), "clinvar-chr7-1000".to_owned()), + ("carrier".to_owned(), "clinvar-chr7-1000".to_owned()), + ("slivar".to_owned(), "null".to_owned()), + ] ); assert_eq!(veil("CYP2D6"), ["null"]); // No closed result, no field. From 909efcff3a5b1de4a92e9d43f1481f58d0ce2a00 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 23:20:39 +0100 Subject: [PATCH 11/14] docs(browser): say the [SKIP] line shows only with --nocapture --- tests/browser/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/browser/README.md b/tests/browser/README.md index 02d848e98..cf0d0a300 100644 --- a/tests/browser/README.md +++ b/tests/browser/README.md @@ -12,7 +12,7 @@ CHROME=/path/to/chrome cargo test -p og-browser # a Chrome that is not on PATH Which Chrome runs: `CHROME` when it is set, else the first of `google-chrome`, `google-chrome-stable`, `chromium` and `chromium-browser` on `PATH`, else `/Applications/Google Chrome.app`. A `CHROME` that names no executable means no Chrome: the driver does not fall back to another one. -Without a Chrome a check prints `[SKIP] no Chrome or Chromium to ...` and passes. In CI (`CI=true`) it fails instead, so a runner without Chrome cannot pass by skipping. The checks run in the Guard job `rust`, inside `cargo test --workspace`, on the runner's Chrome. Two red controls follow there: a page error planted in the clean test page (`--features planted-page-error`) must fail its test, and `CHROME=/nonexistent` must fail in CI and skip with `CI=` unset. +Without a Chrome a check prints `[SKIP] no Chrome or Chromium to ...` and passes. The test harness hides the output of a passing test, so the line shows only with `cargo test -p og-browser -- --nocapture`; without it a skipped check reads as a plain `ok`. In CI (`CI=true`) it fails instead, so a runner without Chrome cannot pass by skipping. The checks run in the Guard job `rust`, inside `cargo test --workspace`, on the runner's Chrome. Two red controls follow there: a page error planted in the clean test page (`--features planted-page-error`) must fail its test, and `CHROME=/nonexistent` must fail in CI and skip with `CI=` unset. ## What is checked so far From 623acb0ee5d902067110919864e61c8915ee15ad Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 18 Nov 2026 10:31:57 +0100 Subject: [PATCH 12/14] fix(browser): bound every DevTools wait, confine the test server and prove the replay control A stalled pipe write no longer blocks a call past its timeout (a writer thread owns Chrome's stdin), wait never runs past its deadline, a failed navigation and a value JSON cannot hold are errors, and an unreadable DevTools message is a page error instead of being dropped. StaticServer serves only files inside its folder after links are followed and closes its open connections when dropped. The asset scan reads hosts and src in any case and refuses a link to a folder. The replay control now needs the 404 for frames.json, the replay policy must equal the live site's CSP, test folders are removed on a panic (Scratch), and the Guard controls check each named test and the planted identifier instead of V8's wording. --- .github/workflows/guard.yml | 4 +- rust/crates/og-browser/src/assets.rs | 69 +++++++++++- rust/crates/og-browser/src/cdp.rs | 111 +++++++++++++++----- rust/crates/og-browser/src/lib.rs | 58 ++++++++-- rust/crates/og-browser/src/serve.rs | 81 +++++++++++++- rust/crates/og-browser/tests/cdp.rs | 61 ++++++----- rust/crates/og-browser/tests/live_replay.rs | 49 ++++++--- tests/browser/README.md | 15 +-- 8 files changed, 362 insertions(+), 86 deletions(-) diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index d6a2905d6..55d40cb8b 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -495,7 +495,7 @@ jobs: fi cat "$log" grep -E 'test a_clean_page_loads_with_no_page_error \.\.\. FAILED' "$log" - grep -F "Cannot read properties of null (reading 'planted')" "$log" + grep -E "TypeError: .*\bplanted\b" "$log" echo "Control: a planted page error failed the clean page's test." if CHROME=/nonexistent cargo test --locked --no-fail-fast -p og-browser --test cdp > "$log" 2>&1; then cat "$log" @@ -504,6 +504,8 @@ jobs: fi cat "$log" grep -F "[FAIL] no Chrome or Chromium to load the test page" "$log" + grep -E 'test a_clean_page_loads_with_no_page_error \.\.\. FAILED' "$log" + grep -E 'test a_planted_exception_and_console_error_are_reported \.\.\. FAILED' "$log" echo "Control: without Chrome the browser checks fail in CI." CHROME=/nonexistent CI='' cargo test --locked -p og-browser --test cdp -- --nocapture > "$log" 2>&1 || { cat "$log"; exit 1; } cat "$log" diff --git a/rust/crates/og-browser/src/assets.rs b/rust/crates/og-browser/src/assets.rs index 55103e55f..3c94ec4c6 100644 --- a/rust/crates/og-browser/src/assets.rs +++ b/rust/crates/og-browser/src/assets.rs @@ -7,8 +7,10 @@ use std::path::Path; use std::sync::LazyLock; static URL: LazyLock = - LazyLock::new(|| Regex::new(r#"(?:https?:)?//[A-Za-z0-9.-]+\.[a-z]{2,}[^\s'"]*"#).unwrap()); + LazyLock::new(|| Regex::new(r#"(?i)(?:https?:)?//[a-z0-9.-]+\.[a-z]{2,}[^\s'"]*"#).unwrap()); static SCRIPT_TAG: LazyLock = LazyLock::new(|| Regex::new(r"(?i)]*>").unwrap()); +/// A `src` attribute of a tag (not `data-src`). +static SRC_ATTR: LazyLock = LazyLock::new(|| Regex::new(r"(?i)\ssrc\s*=").unwrap()); static ON_ATTR: LazyLock = LazyLock::new(|| Regex::new(r"(?i)\son[a-z]+\s*=").unwrap()); static EVAL: LazyLock = LazyLock::new(|| Regex::new(r"\beval\s*\(|new\s+Function\s*\(").unwrap()); @@ -33,7 +35,7 @@ pub fn asset_problems(texts: &BTreeMap) -> Vec<(String, String)> } if name.ends_with(".html") { for tag in SCRIPT_TAG.find_iter(t).map(|m| m.as_str()) { - if !tag.contains("src=") { + if !SRC_ATTR.is_match(tag) { add(tag); } } @@ -50,20 +52,35 @@ pub fn asset_problems(texts: &BTreeMap) -> Vec<(String, String)> /// The text of every file in `dir` and its subfolders, by its path relative /// to `dir` (`plugins/charts.js`); names that start with `.` are skipped. +/// A link to a file is read; a link to a folder, a broken link or an entry +/// that cannot be looked at is an error, so no file is left out unseen. pub fn read_assets(dir: &Path) -> std::io::Result> { let mut out = BTreeMap::new(); let mut todo = vec![dir.to_path_buf()]; while let Some(d) = todo.pop() { for e in std::fs::read_dir(&d)? { - let p = e?.path(); + let e = e?; + let p = e.path(); if p.file_name() .is_some_and(|n| n.to_string_lossy().starts_with('.')) { continue; } - if p.is_dir() { + let kind = e.file_type()?; + let real = if kind.is_symlink() { + std::fs::metadata(&p)?.file_type() + } else { + kind + }; + if real.is_dir() && kind.is_symlink() { + return Err(std::io::Error::other(format!( + "{}: a link to a folder is not scanned", + p.display() + ))); + } + if real.is_dir() { todo.push(p); - } else if p.is_file() { + } else if real.is_file() { let rel = p .strip_prefix(dir) .expect("under dir") @@ -131,4 +148,46 @@ mod tests { ); assert!(one("a.js", "fetch('/state.json'); var u = 'a//b';").is_empty()); } + + #[test] + fn hosts_and_src_are_read_in_any_case() { + assert_eq!( + one("a.js", "fetch('HTTPS://CDN.Example.COM/x.js');"), + [("a.js".into(), "HTTPS://CDN.Example.COM/x.js".into())] + ); + assert_eq!(one("a.css", "url(//cdn.example.com/f.woff)").len(), 1); + // data-src is not src: the script is inline. + assert_eq!( + one("a.html", ""), + [("a.html".into(), "").is_empty()); + } + + #[test] + fn read_assets_reads_subfolders_and_refuses_a_linked_folder() { + let d = crate::Scratch::new(&std::env::temp_dir(), "assets"); + let site = d.path().join("site"); + std::fs::create_dir_all(site.join("plugins")).unwrap(); + std::fs::write(site.join("a.js"), "1").unwrap(); + std::fs::write(site.join("plugins/b.js"), "2").unwrap(); + std::fs::write(site.join(".hidden"), "3").unwrap(); + std::fs::write(d.path().join("outside.js"), "4").unwrap(); + std::os::unix::fs::symlink(d.path().join("outside.js"), site.join("linked.js")).unwrap(); + let got = read_assets(&site).unwrap(); + assert_eq!( + got.into_iter().collect::>(), + [ + ("a.js".to_string(), "1".to_string()), + ("linked.js".into(), "4".into()), + ("plugins/b.js".into(), "2".into()) + ] + ); + std::os::unix::fs::symlink(d.path(), site.join("up")).unwrap(); + let e = read_assets(&site).unwrap_err().to_string(); + assert!(e.contains("a link to a folder is not scanned"), "{e}"); + std::fs::remove_file(site.join("up")).unwrap(); + std::os::unix::fs::symlink(d.path().join("gone.js"), site.join("broken.js")).unwrap(); + assert!(read_assets(&site).is_err()); + } } diff --git a/rust/crates/og-browser/src/cdp.rs b/rust/crates/og-browser/src/cdp.rs index 13fea93dd..bc897a4d1 100644 --- a/rust/crates/og-browser/src/cdp.rs +++ b/rust/crates/og-browser/src/cdp.rs @@ -9,18 +9,24 @@ use indexmap::IndexMap; use pycompat::json::{self, Dump, Value}; use std::io::{BufRead, BufReader, Write}; use std::path::Path; -use std::process::{Child, ChildStdin, Command, Stdio}; -use std::sync::mpsc::{self, Receiver, RecvTimeoutError}; +use std::process::{Child, Command, Stdio}; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError, Sender}; use std::thread; use std::time::{Duration, Instant}; /// How long one call may take before it is an error. const CALL_TIMEOUT: Duration = Duration::from_secs(30); +/// The least time [`Browser::wait`] gives its last evaluation. +const MIN_EVAL: Duration = Duration::from_millis(100); +/// The method name a message from Chrome that could not be read gets. +const UNREADABLE: &str = "og-browser.unreadable"; /// A headless Chrome with one page attached. Dropping it closes Chrome. pub struct Browser { child: Child, - to_chrome: ChildStdin, + /// Frames for the writer thread, which owns Chrome's stdin: a pipe + /// write that stalls never blocks a call past its timeout. + to_chrome: Sender>, from_chrome: Receiver, next_id: i128, session: String, @@ -79,8 +85,16 @@ impl Browser { .stderr(Stdio::null()) .spawn() .map_err(|e| format!("cannot start {}: {e}", chrome.display()))?; - let to_chrome = child.stdin.take().expect("stdin is piped"); + let mut stdin = child.stdin.take().expect("stdin is piped"); let out = child.stdout.take().expect("stdout is piped"); + let (to_chrome, frames) = mpsc::channel::>(); + thread::spawn(move || { + for f in frames { + if stdin.write_all(&f).and_then(|()| stdin.flush()).is_err() { + break; + } + } + }); let (tx, from_chrome) = mpsc::channel(); thread::spawn(move || { let mut r = BufReader::new(out); @@ -94,9 +108,25 @@ impl Browser { if buf.last() == Some(&0) { buf.pop(); } - if let Ok(v) = json::loads(&String::from_utf8_lossy(&buf)) - && tx.send(v).is_err() - { + let text = String::from_utf8_lossy(&buf); + // A message this JSON port cannot read (a lone surrogate in a + // string) is kept as a page error, never dropped. + let v = json::loads(&text).unwrap_or_else(|e| { + obj([ + ("method", s(UNREADABLE)), + ( + "params", + obj([( + "text", + s(&format!( + "{e:?}: {}", + text.chars().take(200).collect::() + )), + )]), + ), + ]) + }); + if tx.send(v).is_err() { break; } } @@ -162,14 +192,13 @@ impl Browser { if let Some(sid) = session { msg.insert("sessionId".to_string(), s(sid)); } + let end = Instant::now() + timeout; let mut line = json::dumps(&Value::Object(msg), &Dump::default()) .map_err(|e| format!("{method}: {e:?}"))?; line.push('\0'); self.to_chrome - .write_all(line.as_bytes()) - .and_then(|()| self.to_chrome.flush()) - .map_err(|e| format!("{method}: Chrome closed the pipe ({e})"))?; - let end = Instant::now() + timeout; + .send(line.into_bytes()) + .map_err(|_| format!("{method}: Chrome closed the pipe"))?; loop { let left = end.saturating_duration_since(Instant::now()); let m = match self.from_chrome.recv_timeout(left) { @@ -195,11 +224,15 @@ impl Browser { } } - /// Keep an event that reports a page error: an uncaught exception or a - /// `console.error` call. + /// Keep an event that reports a page error: an uncaught exception, a + /// `console.error` call, or a message from Chrome that could not be read. fn note(&mut self, m: &Value) { let p = field(m, "params"); match field(m, "method").as_text().as_str() { + UNREADABLE => self.errors.push(format!( + "unreadable DevTools message: {}", + field(&p, "text").as_text() + )), "Runtime.exceptionThrown" => { let d = field(&p, "exceptionDetails"); let desc = field(&field(&d, "exception"), "description").as_text(); @@ -229,20 +262,35 @@ impl Browser { } /// Open `url` in the page. It returns once Chrome has started the - /// navigation; [`Browser::wait`] for what the page must show. + /// navigation; [`Browser::wait`] for what the page must show. A + /// navigation that fails at once (no server: `net::ERR_CONNECTION_REFUSED`) + /// is an error; an HTTP error page is not. pub fn go(&mut self, url: &str) -> Result<(), String> { - self.call("Page.navigate", obj([("url", s(url))])).map(drop) + let r = self.call("Page.navigate", obj([("url", s(url))]))?; + match field(&r, "errorText") { + Value::Str(e) if !e.is_empty() => Err(format!("Page.navigate {url}: {e}")), + _ => Ok(()), + } } /// The value of the JavaScript expression `expr` in the page (`Null` - /// for `undefined`), or the exception it threw. + /// for `undefined`; `Float` for `Infinity`, `-Infinity`, `NaN` and + /// `-0`), or the exception it threw. A value JSON cannot hold (a + /// `BigInt`) is an error. pub fn eval(&mut self, expr: &str) -> Result { - let r = self.call( + self.eval_within(expr, CALL_TIMEOUT) + } + + fn eval_within(&mut self, expr: &str, timeout: Duration) -> Result { + let session = self.session.clone(); + let r = self.send( "Runtime.evaluate", obj([ ("expression", s(expr)), ("returnByValue", Value::Bool(true)), ]), + Some(&session), + timeout, )?; if let Value::Object(d) = field(&r, "exceptionDetails") { let d = Value::Object(d); @@ -253,24 +301,39 @@ impl Browser { desc }); } - Ok(field(&field(&r, "result"), "value")) + let res = field(&r, "result"); + if let Value::Str(u) = field(&res, "unserializableValue") { + return match u.as_str() { + "Infinity" => Ok(Value::Float(f64::INFINITY)), + "-Infinity" => Ok(Value::Float(f64::NEG_INFINITY)), + "NaN" => Ok(Value::Float(f64::NAN)), + "-0" => Ok(Value::Float(-0.0)), + other => Err(format!("{expr}: the value {other} has no JSON form")), + }; + } + Ok(field(&res, "value")) } - /// Evaluate `expr` every 0.2 s until its value is truthy (as in - /// JavaScript and Python), and return that value; `None` after - /// `timeout`. An expression that throws counts as not yet. + /// Evaluate `expr` every 0.2 s until its value is truthy, and return + /// that value; `None` after `timeout`. Truthy is Python's rule, as in + /// the Python driver the checks come from: `null`, `false`, `0`, `""`, + /// `[]` and `{}` all mean not yet (JavaScript would take `[]` and `{}` + /// as true). An expression that throws counts as not yet. No + /// evaluation or pause runs past `timeout`. pub fn wait(&mut self, expr: &str, timeout: Duration) -> Option { let end = Instant::now() + timeout; loop { - if let Ok(v) = self.eval(expr) + let left = end.saturating_duration_since(Instant::now()); + if let Ok(v) = self.eval_within(expr, left.clamp(MIN_EVAL, CALL_TIMEOUT)) && pycompat::value::truthy(&v) { return Some(v); } - if Instant::now() >= end { + let left = end.saturating_duration_since(Instant::now()); + if left.is_zero() { return None; } - thread::sleep(Duration::from_millis(200)); + thread::sleep(left.min(Duration::from_millis(200))); } } diff --git a/rust/crates/og-browser/src/lib.rs b/rust/crates/og-browser/src/lib.rs index a23c1e4d0..cd51ae17b 100644 --- a/rust/crates/og-browser/src/lib.rs +++ b/rust/crates/og-browser/src/lib.rs @@ -10,6 +10,7 @@ //! asks for (a Content-Security-Policy). //! - [`asset_problems`]: outside URLs, inline scripts, `on*` attributes and //! `eval` in a site's files. +//! - [`Scratch`]: a test's own folder, removed when the test ends or panics. //! //! How the checks run: `tests/browser/README.md`. @@ -35,9 +36,10 @@ const NAMES: [&str; 4] = [ ]; const MAC_APP: &str = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"; -/// The Chrome to run: `CHROME` when it is set, else the first of -/// `google-chrome`, `google-chrome-stable`, `chromium`, `chromium-browser` -/// on `PATH`, else Google Chrome's macOS app. `None` when there is none. +/// The Chrome to run: `CHROME` when it is set and not empty, else the +/// first of `google-chrome`, `google-chrome-stable`, `chromium`, +/// `chromium-browser` on `PATH`, else Google Chrome's macOS app. `None` +/// when there is none. An empty `CHROME` counts as unset. pub fn find_chrome() -> Option { find_chrome_in( std::env::var_os("CHROME").as_deref(), @@ -45,9 +47,10 @@ pub fn find_chrome() -> Option { ) } -/// [`find_chrome`] with `CHROME` and `PATH` given. A `CHROME` that is set -/// but names no executable gives `None`: a wrong path fails loudly instead -/// of falling back to another Chrome. +/// [`find_chrome`] with `CHROME` and `PATH` given. A `CHROME` that is not +/// empty but names no executable gives `None`: a wrong path fails loudly +/// instead of falling back to another Chrome. An empty one searches as if +/// it were unset. pub fn find_chrome_in(chrome: Option<&OsStr>, path: Option<&OsStr>) -> Option { if let Some(c) = chrome.filter(|c| !c.is_empty()) { return resolve(Path::new(c), path); @@ -99,6 +102,32 @@ fn without_chrome(found: Option, ci: bool, what: &str) -> Result