From a8f3b4e701e3006e6b9798ce1938242858fe39f8 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:36:45 +0100 Subject: [PATCH 1/7] feat(report): write the ancestry, haplogroup and PRS sections in Rust og report render now writes the fine-grained ancestry, mitochondrial haplogroup, Y chromosome (Yleaf, the second placement, Y-STR), archaic, ancient and polygenic score sections, byte for byte as bin/render_report.py does. A section lays out its tables and pictures in a Raw line, escaping each text it puts in. The report-render parity cases gain 14 summaries made from case c-full (every state, a planted escape in the haplogroup links, the painting and its legend, the raw and sex-specific scores, numbers where the summary writes text), and the oracle filter drops the charts, which og report charts will write. Bead: D4 Bead: D5 Bead: D8 --- rust/crates/og-render/src/chapters.rs | 15 +- rust/crates/og-render/src/html.rs | 1 + rust/crates/og-render/src/lib.rs | 6 +- rust/crates/og-render/src/model.rs | 10 +- .../crates/og-render/src/sections/ancestry.rs | 176 +++++++++- .../og-render/src/sections/ancestry_fine.rs | 286 +++++++++++++++- rust/crates/og-render/src/sections/ancient.rs | 132 ++++++- rust/crates/og-render/src/sections/archaic.rs | 91 ++++- .../og-render/src/sections/haplogroups.rs | 201 ++++++++++- rust/crates/og-render/src/sections/prs.rs | 324 +++++++++++++++++- rust/crates/og-render/src/sections/y_tree.rs | 142 +++++++- rust/crates/og-render/src/sections/ystr.rs | 102 +++++- tests/parity/cases.tsv | 16 + tests/parity/gen/report-corpus.sh | 80 ++++- 14 files changed, 1555 insertions(+), 27 deletions(-) diff --git a/rust/crates/og-render/src/chapters.rs b/rust/crates/og-render/src/chapters.rs index 92ace64e9..5367904e9 100644 --- a/rust/crates/og-render/src/chapters.rs +++ b/rust/crates/og-render/src/chapters.rs @@ -207,21 +207,24 @@ pub fn writer(sid: &str) -> Option<(TextFn, HtmlFn)> { "locityper" => None, - "ancestry-fine" => None, + "ancestry-fine" => Some((sections::ancestry_fine::text, sections::ancestry_fine::html)), - "mt-haplogroup" => None, + "mt-haplogroup" => Some((sections::haplogroups::text, sections::haplogroups::html)), - "y-chromosome" => None, + "y-chromosome" => Some(( + sections::haplogroups::y_chromosome_text, + sections::haplogroups::y_chromosome_html, + )), - "archaic" => None, + "archaic" => Some((sections::archaic::text, sections::archaic::html)), - "ancient" => None, + "ancient" => Some((sections::ancient::text, sections::ancient::html)), "roh" => None, "traits-panel" => None, - "prs" => None, + "prs" => Some((sections::prs::text, sections::prs::html)), "bloodtype" => None, diff --git a/rust/crates/og-render/src/html.rs b/rust/crates/og-render/src/html.rs index 43e75f02b..34883da28 100644 --- a/rust/crates/og-render/src/html.rs +++ b/rust/crates/og-render/src/html.rs @@ -70,6 +70,7 @@ fn line(out: &mut Vec, l: &Line) { out.extend(items.iter().map(|t| format!("
  • {}
  • ", e(t)))); out.push(" ".to_owned()); } + Line::Raw(t) => out.push(t.clone()), } } diff --git a/rust/crates/og-render/src/lib.rs b/rust/crates/og-render/src/lib.rs index 839f2be52..83e46ab58 100644 --- a/rust/crates/og-render/src/lib.rs +++ b/rust/crates/og-render/src/lib.rs @@ -10,9 +10,9 @@ //! [`css`] the page's style. //! //! Ported so far: the frame (chapters, contents, section frames, the run -//! facts, what the report leaves out) and the quality-control and variant -//! sections. A section not ported yet is left out of both reports; the index -//! and charts verbs say they are not ported and exit 2. +//! facts, what the report leaves out) and the sections +//! [`chapters::writer`] names. A section not ported yet is left out of both +//! reports; the index and charts verbs say they are not ported and exit 2. pub mod chapters; pub mod charts; diff --git a/rust/crates/og-render/src/model.rs b/rust/crates/og-render/src/model.rs index 54ea06364..71093922f 100644 --- a/rust/crates/og-render/src/model.rs +++ b/rust/crates/og-render/src/model.rs @@ -3,8 +3,10 @@ //! `==`), and the document the section modules build from it, which the //! text and HTML writers lay out. //! -//! A section module never writes markup. It returns cards of typed lines; -//! every text in them is escaped once, by the HTML writer. +//! A section module returns cards of typed lines; every text in them is +//! escaped once, by the HTML writer. The one exception is [`Line::Raw`]: +//! a table or picture the section lays out itself, escaping each text it +//! puts in. use std::fmt; @@ -325,6 +327,10 @@ pub enum Line { Para(String), /// A plain bulleted list. List(Vec), + /// Markup a section built itself, each text in it escaped with + /// [`crate::escape::e`]: a table, a picture or a link whose layout the + /// report fixes byte for byte. + Raw(String), } /// One card: a title, the caveats shown under it, the notes saying its diff --git a/rust/crates/og-render/src/sections/ancestry.rs b/rust/crates/og-render/src/sections/ancestry.rs index f882faadb..f091cf9e9 100644 --- a/rust/crates/og-render/src/sections/ancestry.rs +++ b/rust/crates/og-render/src/sections/ancestry.rs @@ -1,2 +1,174 @@ -//! The report section `ancestry`. Empty until its unit of the 1.0.0 plan -//! lands. +//! What the ancestry chapter's sections share: reading a row of the +//! summary the way the Python report did (`r["key"]`, `f"{v:<35}"`, a list +//! walked with `for`), the percentages of the fine-grained ancestry, and the +//! links where a haplogroup can be explored further (steps 12 and 37). +//! +//! Used by the fine-grained ancestry, haplogroup, Y chromosome, archaic, +//! ancient and polygenic score sections. + +use pycompat::fmt::{fixed, ljust, rjust}; +use pycompat::json::Value; + +use crate::escape::e; +use crate::html::offsite_link; +use crate::model::{Line, Malformed, get, need, num, py, truthy}; +use crate::notes::EXPLORE_NOTE; + +/// Every explore address but the docs one carries the haplogroup in it. +pub const EXPLORE_DOCS: &str = "https://github.com/GeiserX/OwnGenome/blob/main/docs/"; + +/// The mark of an address that leaves this computer with the haplogroup in it. +pub const OFFSITE_HAPLOGROUP: &str = "opens a website; the address carries the haplogroup"; + +/// `str(v)` of an optional value: `None` when it is absent. +pub fn py_or(v: Option<&Value>, default: &str) -> Result { + v.map_or(Ok(default.to_owned()), py) +} + +/// `d.get(key, default)` as text: the default only when the key is absent. +pub fn get_text(d: &Value, key: &str, default: &str, path: &str) -> Result { + py_or(get(d, key, path)?, default) +} + +/// `str(d[key])`. +pub fn need_text(d: &Value, key: &str, path: &str) -> Result { + py(need(d, key, path)?) +} + +/// The items `for x in v` walks: a list's items, a string's characters, a +/// dict's keys. `None` and numbers cannot be walked. +pub fn items(v: &Value, path: &str) -> Result, Malformed> { + match v { + Value::Array(a) => Ok(a.clone()), + Value::Str(s) => Ok(s.chars().map(|c| Value::Str(c.to_string())).collect()), + Value::Object(o) => Ok(o.keys().map(|k| Value::Str(k.clone())).collect()), + _ => Err(Malformed(format!("{path} is not a list"))), + } +} + +/// `d.get(key, [])` walked with `for`: nothing when the key is absent. +pub fn list(d: &Value, key: &str, path: &str) -> Result, Malformed> { + match get(d, key, path)? { + None => Ok(Vec::new()), + Some(v) => items(v, &format!("{path}.{key}")), + } +} + +/// `len(v)`. +pub fn py_len(v: &Value, path: &str) -> Result { + match v { + Value::Array(a) => Ok(a.len()), + Value::Str(s) => Ok(s.chars().count()), + Value::Object(o) => Ok(o.len()), + _ => Err(Malformed(format!("{path} has no length"))), + } +} + +/// Which side `f"{v:N}"` (right) pads. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Align { + Left, + Right, +} + +/// `f"{v:N}"`: a string or a number padded to `width` +/// characters. A bool formats as the number it is, as in Python; `None`, a +/// list or a dict cannot take a width. +pub fn align(v: &Value, width: usize, side: Align, path: &str) -> Result { + let text = match v { + Value::Str(s) => s.clone(), + Value::Int(i) => i.to_string(), + Value::Float(_) => py(v)?, + Value::Bool(b) => u8::from(*b).to_string(), + _ => return Err(Malformed(format!("{path} cannot be padded"))), + }; + Ok(match side { + Align::Left => ljust(&text, width), + Align::Right => rjust(&text, width), + }) +} + +/// `f"{d[key]:N}"`. +pub fn need_align( + d: &Value, + key: &str, + width: usize, + side: Align, + path: &str, +) -> Result { + align(need(d, key, path)?, width, side, &format!("{path}.{key}")) +} + +/// A number as Python compares it with `>=`; anything else is an error. +pub fn number(v: &Value, path: &str) -> Result { + num(Some(v)).ok_or_else(|| Malformed(format!("{path} is not a number"))) +} + +/// A fraction as a whole percentage (`95%`), `.` for none. +pub fn share(v: Option<&Value>, path: &str) -> Result { + match v { + None | Some(Value::Null) => Ok(".".to_owned()), + Some(x) => Ok(format!("{}%", fixed(100.0 * number(x, path)?, 0))), + } +} + +/// A stat line whose value is markup built here. +pub fn stat_raw(label: &str, value: &str) -> Line { + Line::Raw(format!( + "
    {}{value}
    ", + e(label) + )) +} + +/// One explore entry of a haplogroup section: its name and address. +fn explore(d: &Value, path: &str) -> Result, Malformed> { + let at = format!("{path}.explore[]"); + list(d, "explore", path)? + .iter() + .map(|x| Ok((need_text(x, "name", &at)?, need(x, "url", &at)?.clone()))) + .collect() +} + +/// The text report's explore lines: `name: url` for each link, then the +/// note that nothing is uploaded; nothing when there is no link. +pub fn explore_lines(d: &Value, path: &str) -> Result, Malformed> { + let mut out = Vec::new(); + for (name, url) in explore(d, path)? { + out.push(format!("{name}: {}", py(&url)?)); + } + if !out.is_empty() { + out.push(EXPLORE_NOTE.to_owned()); + } + Ok(out) +} + +/// The HTML report's explore lines: one stat per link, each marked as +/// leaving this computer, and whether the card shows the explore caveat. +/// The address is escaped, so a branch name cannot open a tag or end the +/// attribute. +pub fn explore_html(d: &Value, path: &str) -> Result<(Vec, bool), Malformed> { + let mut out = Vec::new(); + for (name, url) in explore(d, path)? { + let Value::Str(url) = url else { + return Err(Malformed(format!("{path}.explore[].url is not a text"))); + }; + let why = if url.starts_with(EXPLORE_DOCS) { + crate::html::OFFSITE + } else { + OFFSITE_HAPLOGROUP + }; + out.push(stat_raw(&name, &offsite_link(&url, &url, why))); + } + let note = !out.is_empty(); + Ok((out, note)) +} + +/// `d.get(key, ".") not in (".", "")`: a value worth a line. +pub fn shown<'a>(d: &'a Value, key: &str, path: &str) -> Result, Malformed> { + Ok(get(d, key, path)?.filter(|v| !matches!(v, Value::Str(s) if s.is_empty() || s == "."))) +} + +/// `bool(d.get(key))`. +pub fn has(d: &Value, key: &str, path: &str) -> Result { + Ok(get(d, key, path)?.is_some_and(truthy)) +} diff --git a/rust/crates/og-render/src/sections/ancestry_fine.rs b/rust/crates/og-render/src/sections/ancestry_fine.rs index d87e67915..67b9c2134 100644 --- a/rust/crates/og-render/src/sections/ancestry_fine.rs +++ b/rust/crates/og-render/src/sections/ancestry_fine.rs @@ -1,2 +1,284 @@ -//! The report section `ancestry_fine`. Empty until its unit of the 1.0.0 -//! plan lands. +//! Fine-grained ancestry (step 26b): the continental and regional groups +//! with their bootstrap ranges, the groups the model leaves out, the share +//! of the genome each group paints, and the chromosome painting with its +//! legend. + +use pycompat::json::Value; + +use crate::escape::e; +use crate::model::{ + Card, HtmlPart, Inline, Line, Malformed, State, Summary, get, need, py, truthy, +}; +use crate::notes::ANCESTRY_FINE_NOTE; +use crate::sections::ancestry::{get_text, has, list, need_text, number, share}; +use crate::sections::haplogroups::shows_body; +use crate::{html, text}; + +const PATH: &str = "sections.ancestry_fine.data"; + +/// The two levels the model reports, with their headings. +pub const LEVELS: [(&str, &str); 2] = [ + ("continent", "Continental groups"), + ("region", "Regional groups"), +]; + +/// The groups of one level worth a line: every continental group of 1% or +/// more, a regional one of 1% or more or whose bootstrap range reaches 5%. +pub fn rows(d: &Value, level: &str) -> Result, Malformed> { + let at = format!("{PATH}.{level}[]"); + let mut out = Vec::new(); + for r in list(d, level, PATH)? { + let f = need(&r, "fraction", &at)?; + if matches!(f, Value::Null) { + continue; + } + let keep = number(f, &format!("{at}.fraction"))? >= 0.01 + || (level == "region" + && match get(&r, "high", &at)?.filter(|v| truthy(v)) { + Some(h) => number(h, &format!("{at}.high"))? >= 0.05, + None => false, + }); + if keep { + out.push(r); + } + } + Ok(out) +} + +/// ` (93% to 97%)`: the bootstrap range of a row, when it has one. +pub fn spread(r: &Value, at: &str) -> Result { + match get(r, "low", at)? { + None | Some(Value::Null) => Ok(String::new()), + Some(low) => Ok(format!( + " ({} to {})", + share(Some(low), at)?, + share(Some(need(r, "high", at)?), at)? + )), + } +} + +/// `(.*) \((.*) group\)` matched against the whole name: the group and its +/// kind (`Oceania (continental group)`), or the name alone. +fn name_and_kind(group: &str) -> (&str, &str) { + let Some(inner) = group.strip_suffix(" group)") else { + return (group, ""); + }; + match inner.rfind(" (") { + Some(i) if !inner.contains('\n') => (&inner[..i], &inner[i + 2..]), + _ => (group, ""), + } +} + +/// The groups the model leaves out, as one sentence, or "". +pub fn left_out(d: &Value) -> Result { + if !has(d, "left_out", PATH)? { + return Ok(String::new()); + } + let at = format!("{PATH}.left_out[]"); + // One row per level ("Oceania (continental group)"); a group both + // levels leave out reads as one name. + let mut kinds: Vec<(String, Vec)> = Vec::new(); + for r in list(d, "left_out", PATH)? { + let Value::Str(group) = need(&r, "group", &at)? else { + return Err(Malformed(format!("{at}.group is not a text"))); + }; + let (name, kind) = name_and_kind(group); + let i = match kinds.iter().position(|(n, _)| n == name) { + Some(i) => i, + None => { + kinds.push((name.to_owned(), Vec::new())); + kinds.len() - 1 + } + }; + if !kind.is_empty() && !kinds[i].1.iter().any(|k| k == kind) { + kinds[i].1.push(kind.to_owned()); + } + } + let names: Vec = kinds + .iter() + .map(|(n, k)| { + if k.is_empty() { + n.clone() + } else { + format!("{n} ({} group)", k.join(" and ")) + } + }) + .collect(); + Ok(format!( + "Left out of the model: {}. The panel has fewer than {} unrelated reference people for \ + it, so its ancestry is spread over the groups shown.", + names.join(", "), + get_text(d, "min_references", "10", PATH)? + )) +} + +/// The painted shares, or why there are none, and the chromosomes the +/// painting failed on. +pub fn painted(d: &Value) -> Result { + let codes = list(d, "not_painted", PATH)? + .iter() + .map( + |r| match need(r, "code", &format!("{PATH}.not_painted[]"))? { + Value::Str(c) => Ok(c.clone()), + _ => Err(Malformed(format!( + "{PATH}.not_painted[].code is not a text" + ))), + }, + ) + .collect::, _>>()?; + let failed = codes.join(", "); + let mut text = if has(d, "painting", PATH)? { + let at = format!("{PATH}.painting[]"); + list(d, "painting", PATH)? + .iter() + .map(|r| { + Ok(format!( + "{} {}", + need_text(r, "group", &at)?, + share(Some(need(r, "fraction", &at)?), &at)? + )) + }) + .collect::, Malformed>>()? + .join(", ") + } else if failed.is_empty() { + "none (no chromosome long enough)".to_owned() + } else { + "none".to_owned() + }; + if !failed.is_empty() { + text += &format!("; not painted: RFMix failed on {failed}"); + } + Ok(text) +} + +/// The text report's block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head( + s, + "ancestry_fine", + Some("Fine-Grained Ancestry (ADMIXTURE, RFMix)"), + )? + else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if !head.ran { + return Ok(w); + } + let d = s.section("ancestry_fine").data; + for (level, title) in LEVELS { + let rs = rows(d, level)?; + if rs.is_empty() { + continue; + } + w.push(format!(" {title} (fraction, bootstrap range):")); + let at = format!("{PATH}.{level}[]"); + for r in &rs { + let group = crate::sections::ancestry::need_align( + r, + "group", + 50, + crate::sections::ancestry::Align::Left, + &at, + )?; + let frac = share(Some(need(r, "fraction", &at)?), &at)?; + w.push(format!( + " {group} {}{}", + pycompat::fmt::rjust(&frac, 5), + spread(r, &at)? + )); + } + } + let lo = left_out(d)?; + if !lo.is_empty() { + w.push(format!(" {lo}")); + } + let lead = if has(d, "painting", PATH)? { + " Chromosome painting, share of the painted genome: " + } else { + " Chromosome painting: " + }; + w.push(format!("{lead}{}", painted(d)?)); + w.push(format!(" {ANCESTRY_FINE_NOTE}")); + w.push(String::new()); + Ok(w) +} + +/// The HTML report's card, when step 26b has a result: one table per +/// level, the painted shares, and the painting with its legend. +pub fn html(s: &Summary) -> Result { + let sec = s.section("ancestry_fine"); + if sec.state == State::Missing { + return Ok(HtmlPart::default()); + } + let mut c = Card { + title: "Fine-Grained Ancestry".to_owned(), + full: true, + ..Card::default() + }; + if shows_body(&sec) { + let d = sec.data; + for (level, title) in LEVELS { + let at = format!("{PATH}.{level}[]"); + let mut rows_html = String::new(); + for r in rows(d, level)? { + let range = spread(&r, &at)?; + let range = range.trim_matches([' ', '(', ')']); + rows_html += &format!( + "{}{}{}\n", + e(&need_text(&r, "group", &at)?), + e(&share(Some(need(&r, "fraction", &at)?), &at)?), + e(if range.is_empty() { "." } else { range }) + ); + } + if !rows_html.is_empty() { + c.body.extend([ + Line::Raw(format!("

    {}

    ", e(title))), + Line::Raw(" ".to_owned()), + Line::Raw( + " " + .to_owned(), + ), + Line::Raw(rows_html.trim_end_matches('\n').to_owned()), + Line::Raw("
    GroupFractionBootstrap range
    ".to_owned()), + ]); + } + } + let lo = left_out(d)?; + if !lo.is_empty() { + c.body.push(Line::Para(lo)); + } + c.body.push(Line::Stat( + "Painted genome".to_owned(), + Inline::Text(painted(d)?), + )); + if let Some(png) = get(d, "painting_png", PATH)?.filter(|v| truthy(v)) { + c.body.push(Line::Raw(format!( + " \"Chromosome", + e(&py(png)?) + ))); + let at = format!("{PATH}.legend[]"); + let keys = list(d, "legend", PATH)? + .iter() + .map(|x| { + Ok(format!( + "■ {}", + e(&need_text(x, "colour", &at)?), + e(&need_text(x, "group", &at)?) + )) + }) + .collect::, Malformed>>()?; + c.body.push(Line::Raw(format!( + "

    {} ■ not painted

    ", + keys.join(" ") + ))); + } + c.caveats.push(ANCESTRY_FINE_NOTE); + } + Ok(HtmlPart { + cards: vec![html::card(&sec, c)?], + files: Vec::new(), + }) +} diff --git a/rust/crates/og-render/src/sections/ancient.rs b/rust/crates/og-render/src/sections/ancient.rs index 0bcd3409a..d6000869a 100644 --- a/rust/crates/og-render/src/sections/ancient.rs +++ b/rust/crates/og-render/src/sections/ancient.rs @@ -1,2 +1,130 @@ -//! The report section `ancient`. Empty until its unit of the 1.0.0 plan -//! lands. +//! Ancient populations (step 44): where the genome falls on a PCA of +//! present-day West Eurasians, and the ancient groups of the AADR nearest +//! to it there, with the plot. + +use pycompat::json::Value; + +use crate::escape::e; +use crate::model::{Card, HtmlPart, Line, Malformed, State, Summary, get, py, truthy}; +use crate::notes::ANCIENT_NOTE; +use crate::sections::ancestry::{Align, get_text, need_align, need_text}; +use crate::{html, text}; + +const PATH: &str = "sections.ancient.data"; +const ROW: &str = "sections.ancient.data.nearest[]"; + +/// `(d.get("nearest") or [])[:10]`: the ten nearest groups. +fn nearest(d: &Value) -> Result, Malformed> { + match get(d, "nearest", PATH)?.filter(|v| truthy(v)) { + None => Ok(Vec::new()), + Some(Value::Array(a)) => Ok(a.iter().take(10).collect()), + Some(_) => Err(Malformed(format!("{PATH}.nearest is not a list"))), + } +} + +/// Step 44's lines, the same in both reports: what the PCA was built from, +/// then the nearest groups. +pub fn lines(d: &Value) -> Result, Malformed> { + let g = |k: &str| get_text(d, k, ".", PATH); + let mut out = vec![ + format!( + "AADR {}: {} of {} sites used; axes from {} present-day West Eurasians, {} ancient \ + people in {} groups projected.", + g("release")?, + g("sites_used")?, + g("panel_sites")?, + g("present_day_people")?, + g("ancient_people")?, + g("ancient_groups")? + ), + "Nearest ancient groups:".to_owned(), + ]; + for r in nearest(d)? { + let n = |k: &str| need_text(r, k, ROW); + out.push(format!( + " {}. {} ({}, {}; {} people)", + need_align(r, "rank", 2, Align::Right, ROW)?, + n("group")?, + n("dates")?, + n("country")?, + n("n")? + )); + } + Ok(out) +} + +/// The text report's block, with the plot's file name. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head( + s, + "ancient", + Some("Ancient Populations (AADR PCA, step 44)"), + )? + else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if head.ran { + w.extend( + lines(s.section("ancient").data)? + .iter() + .map(|l| format!(" {l}")), + ); + w.push(format!( + " {ANCIENT_NOTE} Plot: ancient/{}_ancient_pca.svg", + py(s.need("sample")?)? + )); + w.push(String::new()); + } + Ok(w) +} + +/// The HTML report's card: the nearest groups as a table, and the plot as +/// an image (shown, never run as part of the page). +pub fn html(s: &Summary) -> Result { + let sec = s.section("ancient"); + if sec.state == State::Missing { + return Ok(HtmlPart::default()); + } + let mut c = Card { + title: "Ancient Populations".to_owned(), + full: true, + ..Card::default() + }; + if crate::sections::haplogroups::shows_body(&sec) { + let d = sec.data; + let first = lines(d)?.swap_remove(0); + let mut rows = String::new(); + for r in nearest(d)? { + rows += ""; + for k in ["rank", "group", "dates", "country", "locality", "n"] { + rows += &format!("{}", e(&need_text(r, k, ROW)?)); + } + rows += "\n"; + } + c.body = vec![ + Line::Para(first), + Line::Raw(" ".to_owned()), + Line::Raw( + " \ + " + .to_owned(), + ), + Line::Raw(rows.trim_end_matches('\n').to_owned()), + Line::Raw("
    #Ancient groupDatesCountryPlacePeople
    ".to_owned()), + ]; + if let Some(svg) = get(d, "plot_svg", PATH)?.filter(|v| truthy(v)) { + c.body.push(Line::Raw(format!( + " \"PC1", + e(&py(svg)?) + ))); + } + c.caveats.push(ANCIENT_NOTE); + } + Ok(HtmlPart { + cards: vec![html::card(&sec, c)?], + files: Vec::new(), + }) +} diff --git a/rust/crates/og-render/src/sections/archaic.rs b/rust/crates/og-render/src/sections/archaic.rs index 4295ec431..42cc364a2 100644 --- a/rust/crates/og-render/src/sections/archaic.rs +++ b/rust/crates/og-render/src/sections/archaic.rs @@ -1,2 +1,89 @@ -//! The report section `archaic`. Empty until its unit of the 1.0.0 plan -//! lands. +//! Archaic ancestry (step 43, IBDmix): the share of the analysed autosomes +//! in Neanderthal and Denisovan segments, and the longest segments. + +use pycompat::fmt::fixed; +use pycompat::json::Value; + +use crate::model::{Card, Inline, Line, Malformed, State, Summary, need, py}; +use crate::notes::ARCHAIC_NOTE; +use crate::sections::ancestry::{get_text, has, items, need_text, number}; +use crate::{html, text}; + +const PATH: &str = "sections.archaic.data"; + +/// A length in base pairs as megabases (`50.0 Mb`). +fn mb(d: &Value, key: &str) -> Result { + let bp = number(need(d, key, PATH)?, &format!("{PATH}.{key}"))?; + Ok(format!("{} Mb", fixed(bp / 1e6, 1))) +} + +/// Step 43's lines, the same in both reports. +pub fn lines(d: &Value) -> Result, Malformed> { + let n = |k: &str| need_text(d, k, PATH); + let mut out = Vec::new(); + for (name, key) in [("Neanderthal", "neanderthal"), ("Denisovan", "denisovan")] { + out.push(format!( + "{name}: {}% ({} in {} segments)", + n(&format!("{key}_percent"))?, + mb(d, &format!("{key}_bp"))?, + n(&format!("{key}_segments"))? + )); + } + out.push(format!( + "Analysed: {} of the autosomes; modern frequencies: 1000 Genomes {}", + mb(d, "analysed_bp")?, + get_text(d, "population", ".", PATH)? + )); + if has(d, "longest", PATH)? { + out.push("Longest segments:".to_owned()); + for x in items(need(d, "longest", PATH)?, &format!("{PATH}.longest"))? { + out.push(format!(" {}", py(&x)?)); + } + } + Ok(out) +} + +/// The text report's block. It ends without a blank line, as the Python +/// report's did. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "archaic", Some("Archaic Ancestry (IBDmix, step 43)"))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if head.ran { + let d = s.section("archaic").data; + w.extend(lines(d)?.iter().map(|l| format!(" {l}"))); + w.push(format!(" {ARCHAIC_NOTE}")); + } + Ok(w) +} + +/// The HTML report's card, when step 43 has a result. +pub fn html(s: &Summary) -> Result { + let sec = s.section("archaic"); + if sec.state == State::Missing { + return Ok(crate::model::HtmlPart::default()); + } + let mut c = Card { + title: "Archaic Ancestry (Neanderthal, Denisovan)".to_owned(), + ..Card::default() + }; + if sec.state.ran() { + let d = sec.data; + for (label, key) in [ + ("Neanderthal", "neanderthal_percent"), + ("Denisovan", "denisovan_percent"), + ] { + c.body.push(Line::Stat( + label.to_owned(), + Inline::Text(format!("{}%", need_text(d, key, PATH)?)), + )); + } + c.body.extend(lines(d)?.into_iter().map(Line::Para)); + c.caveats.push(ARCHAIC_NOTE); + } + Ok(crate::model::HtmlPart { + cards: vec![html::card(&sec, c)?], + files: Vec::new(), + }) +} diff --git a/rust/crates/og-render/src/sections/haplogroups.rs b/rust/crates/og-render/src/sections/haplogroups.rs index 8f28f8f32..94fdad73e 100644 --- a/rust/crates/og-render/src/sections/haplogroups.rs +++ b/rust/crates/og-render/src/sections/haplogroups.rs @@ -1,2 +1,199 @@ -//! The report section `haplogroups`. Empty until its unit of the 1.0.0 plan -//! lands. +//! The haplogroups: the mitochondrial one with haplocheck's contamination +//! verdict (step 12), and the Y chromosome section, which shows Yleaf's +//! haplogroup (step 37), the second placement on the current public tree +//! ([`crate::sections::y_tree`]) and the Y-STR markers +//! ([`crate::sections::ystr`]). Each haplogroup carries the links where it +//! can be explored further. + +use pycompat::json::Value; + +use crate::model::{Card, Colour, HtmlPart, Inline, Line, Malformed, Section, State, Summary, py}; +use crate::notes::EXPLORE_NOTE; +use crate::sections::ancestry::{explore_html, explore_lines, get_text, need_text}; +use crate::sections::{y_tree, ystr}; +use crate::{html, text}; + +/// The Status badge of a section that ran: Stale for an earlier run's +/// result, else Complete. +pub fn done_badge(sec: &Section<'_>) -> Inline { + if sec.state == State::Stale { + Inline::Badge("Stale".to_owned(), Colour::Yellow) + } else { + Inline::Badge("Complete".to_owned(), Colour::Green) + } +} + +/// Whether a card's lines are shown: the card says Not run, Failed or +/// Unreadable in their place otherwise. +pub fn shows_body(sec: &Section<'_>) -> bool { + !matches!( + sec.state, + State::Missing | State::Failed | State::Unreadable + ) +} + +fn path(sec: &Section<'_>) -> String { + format!("sections.{}.data", sec.key) +} + +/// haplocheck's verdict: whether the reads hold two mtDNA haplogroups. +pub fn contamination_text(sec: &Section<'_>) -> Result { + let Some(status) = sec.get("contamination_status")? else { + return Ok("not checked (needs step 20's Mutect2 chrM calls)".to_owned()); + }; + let word = match status { + Value::Str(s) => match s.as_str() { + "YES" => "YES, two mtDNA haplogroups in the reads (another person's DNA?)".to_owned(), + "NO" => "no".to_owned(), + "ND" => "not determined".to_owned(), + _ => s.clone(), + }, + Value::Array(_) | Value::Object(_) => { + return Err(Malformed(format!( + "{}.contamination_status is not a value", + path(sec) + ))); + } + v => py(v)?, + }; + let level = get_text(sec.data, "contamination_level", ".", &path(sec))?; + Ok(format!("{word} (level {level})")) +} + +/// The text report's mitochondrial block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "haplogroup", Some("Mitochondrial Haplogroup"))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if head.ran { + let sec = s.section("haplogroup"); + let p = path(&sec); + w.push(format!( + " Haplogroup: {}", + need_text(sec.data, "haplogroup", &p)? + )); + w.push(format!( + " Contamination (haplocheck): {}", + contamination_text(&sec)? + )); + w.extend( + explore_lines(sec.data, &p)? + .iter() + .map(|l| format!(" {l}")), + ); + w.push(String::new()); + } + Ok(w) +} + +/// The HTML report's mitochondrial card. It is always there, so a run +/// without step 12 says so. +pub fn html(s: &Summary) -> Result { + let sec = s.section("haplogroup"); + let mut c = Card { + title: "Mitochondrial Haplogroup".to_owned(), + ..Card::default() + }; + if shows_body(&sec) { + let p = path(&sec); + c.body = vec![ + Line::Stat("Status".to_owned(), done_badge(&sec)), + Line::Stat( + "Haplogroup".to_owned(), + Inline::Text(get_text(sec.data, "haplogroup", ".", &p)?), + ), + Line::Stat( + "Contamination (haplocheck)".to_owned(), + Inline::Text(contamination_text(&sec)?), + ), + ]; + let (links, note) = explore_html(sec.data, &p)?; + c.body.extend(links); + if note { + c.caveats.push(EXPLORE_NOTE); + } + } + Ok(HtmlPart { + cards: vec![html::card(&sec, c)?], + files: Vec::new(), + }) +} + +/// The text report's block of Yleaf's haplogroup. +fn y_text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "y_haplogroup", Some("Y-Chromosome Haplogroup (Yleaf)"))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if head.ran { + let sec = s.section("y_haplogroup"); + let p = path(&sec); + let n = |k: &str| need_text(sec.data, k, &p); + w.push(format!( + " Haplogroup: {} ({} markers, QC-score {})", + n("haplogroup")?, + n("valid_markers")?, + n("qc_score")? + )); + w.extend( + explore_lines(sec.data, &p)? + .iter() + .map(|l| format!(" {l}")), + ); + w.push(String::new()); + } + Ok(w) +} + +/// The HTML report's card of Yleaf's haplogroup, when step 37 has a result. +fn y_card(s: &Summary) -> Result, Malformed> { + let sec = s.section("y_haplogroup"); + if sec.state == State::Missing { + return Ok(None); + } + let mut c = Card { + title: "Y-Chromosome Haplogroup".to_owned(), + ..Card::default() + }; + if shows_body(&sec) { + let p = path(&sec); + for (label, key) in [ + ("Haplogroup", "haplogroup"), + ("Markers", "valid_markers"), + ("QC-score", "qc_score"), + ] { + c.body.push(Line::Stat( + label.to_owned(), + Inline::Text(get_text(sec.data, key, ".", &p)?), + )); + } + let (links, note) = explore_html(sec.data, &p)?; + c.body.extend(links); + if note { + c.caveats.push(EXPLORE_NOTE); + } + } + Ok(Some(html::card(&sec, c)?)) +} + +/// The text report's Y chromosome blocks: Yleaf, the second placement, +/// the Y-STR markers. +pub fn y_chromosome_text(s: &Summary) -> Result, Malformed> { + let mut w = y_text(s)?; + w.extend(y_tree::text(s)?); + w.extend(ystr::text(s)?); + Ok(w) +} + +/// The HTML report's Y chromosome cards, each when its step has a result. +pub fn y_chromosome_html(s: &Summary) -> Result { + let cards = [y_card(s)?, y_tree::card(s)?, ystr::card(s)?] + .into_iter() + .flatten() + .collect(); + Ok(HtmlPart { + cards, + files: Vec::new(), + }) +} diff --git a/rust/crates/og-render/src/sections/prs.rs b/rust/crates/og-render/src/sections/prs.rs index 3d832baa8..c9faba465 100644 --- a/rust/crates/og-render/src/sections/prs.rs +++ b/rust/crates/og-render/src/sections/prs.rs @@ -1 +1,323 @@ -//! The report section `prs`. Empty until its unit of the 1.0.0 plan lands. +//! Polygenic risk scores (step 25, pgsc_calc): each score with the variants +//! it matched, its percentile among the most similar reference group when +//! the ancestry panel is installed, the effect a published evaluation +//! measured in that group, and the line that says what the numbers can be +//! compared with: the ancestry caveat, the chrX rows left out and the +//! sex-specific scores when no sex was declared. +//! +//! The scores are shown as the summary lists them. + +use pycompat::json::Value; + +use crate::escape::e; +use crate::model::{Card, HtmlPart, Line, Malformed, Summary, get, is_str, need, py, truthy}; +use crate::notes::PRS_ANCESTRY_NOTE; +use crate::sections::ancestry::{Align, items, need_align, need_text, py_or}; +use crate::{html, text}; + +const PATH: &str = "sections.prs.data"; +const ROW: &str = "sections.prs.data.scores[]"; + +/// What a score of the other sex says in place of a percentile. +pub const NOT_APPLICABLE: &str = "not applicable to the declared sex"; + +/// Part of the line under the scores, so not a caveat of its own. +pub const EFFECT_TEXT: &str = "Effect: the odds ratio (OR) or hazard ratio (HR) a published \ + evaluation of the score measured in people of the same reference group, in that \ + evaluation's unit (its PGS Catalog PPM page says which). It describes the score in that \ + group, not your own risk, and is shown only for a group the score was evaluated in."; + +fn field<'a>(r: &'a Value, key: &str) -> Result, Malformed> { + get(r, key, ROW) +} + +/// `r.get(key)` is true. +fn on<'a>(r: &'a Value, key: &str) -> Result, Malformed> { + Ok(field(r, key)?.filter(|v| truthy(v))) +} + +fn other_sex(r: &Value) -> Result { + Ok(is_str(field(r, "sex_check")?, NOT_APPLICABLE)) +} + +/// The scores of the summary. +fn scores(d: &Value) -> Result, Malformed> { + items(need(d, "scores", PATH)?, &format!("{PATH}.scores")) +} + +/// The percentile cell: the percentile with its group, the note for a score +/// of the other sex, or "" when there is none. +pub fn percentile(r: &Value) -> Result { + if other_sex(r)? { + let sex = py_or(on(r, "applies_to")?, "sex")?; + return Ok(format!("{NOT_APPLICABLE} ({sex}-specific score)")); + } + match on(r, "percentile")? { + Some(p) => Ok(format!( + "{} ({})", + py(p)?, + py_or(on(r, "group")?, "group unknown")? + )), + None => Ok(String::new()), + } +} + +/// `", ".join(v)`: the items of a list of texts, or a text's characters. +fn join(v: &Value, at: &str) -> Result { + let parts = items(v, at)? + .into_iter() + .map(|x| match x { + Value::Str(s) => Ok(s), + _ => Err(Malformed(format!("{at} holds an item that is not a text"))), + }) + .collect::, _>>()?; + Ok(parts.join(", ")) +} + +/// The effect cell: the odds or hazard ratio the PGS Catalog records for +/// the score in the sample's group, a line saying the score has none for +/// that group, or "" when the sample has no group, the score does not apply +/// to the declared sex, or the table names no evaluations. The bool says +/// whether the cell is the ratio itself. +pub fn effect(r: &Value) -> Result<(String, bool), Malformed> { + let evaluated = match field(r, "evaluated_in")? { + None | Some(Value::Null) => return Ok((String::new(), false)), + Some(v) => v, + }; + if on(r, "percentile")?.is_some() && on(r, "group")?.is_some() && !other_sex(r)? { + if let Some(x) = on(r, "effect")? { + return Ok((py(x)?, true)); + } + let group = py(need(r, "group", ROW)?)?; + let place = if truthy(evaluated) { + format!( + "; evaluated in {}", + join(evaluated, &format!("{ROW}.evaluated_in"))? + ) + } else { + String::new() + }; + return Ok((format!("no published OR or HR for {group}{place}"), false)); + } + Ok((String::new(), false)) +} + +/// The chrX rows left out of the scores because no sex was given. +fn chrx_note(rs: &[Value]) -> Result { + let mut left = Vec::new(); + for r in rs { + let Some(c) = on(r, "chrx")? else { continue }; + let Value::Str(c) = c else { + return Err(Malformed(format!("{ROW}.chrx is not a text"))); + }; + if c.ends_with("left out") { + let first = pycompat::text::split(c.as_str(), None, None) + .first() + .copied() + .unwrap_or_default() + .to_owned(); + left.push(format!("{} ({first})", need_text(r, "pgs_id", ROW)?)); + } + } + if left.is_empty() { + return Ok(String::new()); + } + Ok(format!( + " Rows on chrX were left out, because the sample's sex was not given: {}. Give the sex \ + to score them (docs/25-prs.md).", + left.join(", ") + )) +} + +/// The sex-specific scores shown for any sex, when no sex was declared. +fn sex_note(rs: &[Value]) -> Result { + let mut named = Vec::new(); + for r in rs { + if is_str(field(r, "sex_check")?, "no sex declared") + && let Some(a) = on(r, "applies_to")? + { + named.push(format!("{} ({})", need_text(r, "pgs_id", ROW)?, py(a)?)); + } + } + if named.is_empty() { + return Ok(String::new()); + } + Ok(format!( + " No sex was declared, so these sex-specific scores are shown for any sex: {}. Give the \ + sex to mark a score of the other sex not applicable (docs/25-prs.md).", + named.join(", ") + )) +} + +/// Whether the summary's table names evaluations (its Evaluated_In column). +fn names_evaluations(rs: &[Value]) -> Result { + for r in rs { + if !matches!(field(r, "evaluated_in")?, None | Some(Value::Null)) { + return Ok(true); + } + } + Ok(false) +} + +/// The line under the scores, in both reports: what the numbers can be +/// compared with. +pub fn note(d: &Value) -> Result { + let rs = scores(d)?; + let tail = chrx_note(&rs)? + &sex_note(&rs)?; + if !get(d, "adjusted", PATH)?.is_some_and(truthy) { + return Ok(format!( + "Raw score only: no ancestry reference panel was installed, so there is no percentile \ + and the sum cannot be compared with anyone (scripts/setup.sh --ancestry-panel adds \ + it). {PRS_ANCESTRY_NOTE} See docs/25-prs.md.{tail}" + )); + } + let empty = Value::Object(indexmap::IndexMap::new()); + let anc = get(d, "ancestry", PATH)? + .filter(|v| truthy(v)) + .unwrap_or(&empty); + let at = format!("{PATH}.ancestry"); + let mut group = match get(anc, "population", &at)?.filter(|v| truthy(v)) { + Some(p) => py(p)?, + None => { + let mut g = String::new(); + for r in &rs { + if let Some(x) = on(r, "group")? { + g = py(x)?; + break; + } + } + g + } + }; + if group.is_empty() { + group = "most similar".to_owned(); + } + let panel = match get(anc, "panel", &at)?.filter(|v| truthy(v)) { + Some(p) => format!(" of the {} reference panel", py(p)?), + None => " of the reference panel".to_owned(), + }; + let low = if is_str(get(anc, "low_confidence", &at)?, "True") { + " The ancestry match is low-confidence: read the percentile with care." + } else { + "" + }; + let eff = if names_evaluations(&rs)? { + format!(" {EFFECT_TEXT}") + } else { + String::new() + }; + Ok(format!( + "Percentile: where the score falls among the {group} samples{panel}, the group whose \ + genetic ancestry is most similar to this sample's (pgsc_calc).{low} A percentile is not \ + a risk.{eff} {PRS_ANCESTRY_NOTE} See docs/25-prs.md.{tail}" + )) +} + +/// The text report's block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "prs", Some("Polygenic Risk Scores"))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if !head.ran { + return Ok(w); + } + let d = s.section("prs").data; + for r in scores(d)? { + let condition = need_align(&r, "condition", 35, Align::Left, ROW)?; + let n = |k: &str| need_text(&r, k, ROW); + let line = format!( + " {condition} {} ({}/{} variants matched) {}", + n("score")?, + n("matched")?, + n("total")?, + n("pgs_id")? + ); + let pct = percentile(&r)?; + let pct = if other_sex(&r)? { + format!(" {pct}") + } else if pct.is_empty() { + String::new() + } else { + format!(" percentile {pct}") + }; + let (e, is_ratio) = effect(&r)?; + let eff = if e.is_empty() { + String::new() + } else if is_ratio { + format!(" effect {e}") + } else { + format!(" {e}") + }; + w.push(format!("{line}{pct}{eff}")); + } + w.push(format!(" {}", note(d)?)); + w.push(String::new()); + Ok(w) +} + +/// The HTML report's card, when step 25 has a result: one row per score, +/// with the effect column when the table names evaluations. +pub fn html(s: &Summary) -> Result { + let sec = s.section("prs"); + if !sec.state.ran() { + return Ok(HtmlPart::default()); + } + let d = sec.data; + let rs = scores(d)?; + let adjusted = get(d, "adjusted", PATH)?.is_some_and(truthy); + let show_effect = adjusted && names_evaluations(&rs)?; + let mut rows = String::new(); + for x in &rs { + let n = |k: &str| need_text(x, k, ROW).map(|t| e(&t).into_owned()); + let pct = percentile(x)?; + rows += &format!( + "{}{}{}{}/{}{}", + n("condition")?, + n("pgs_id")?, + n("score")?, + n("matched")?, + n("total")?, + e(if pct.is_empty() { + "raw score only" + } else { + &pct + }) + ); + if show_effect { + rows += &format!("{}", e(&effect(x)?.0)); + } + rows += "\n"; + } + let effect_th = if show_effect { + "Effect in your group" + } else { + "" + }; + let card = Card { + title: format!( + "Polygenic Risk Scores{}", + if adjusted { + "" + } else { + " (raw, not percentiles)" + } + ), + full: true, + body: vec![ + Line::Raw(" ".to_owned()), + Line::Raw(format!( + " \ + {effect_th}" + )), + Line::Raw(rows.trim_end_matches('\n').to_owned()), + Line::Raw("
    ConditionPGSScoreVariants matchedPercentile
    ".to_owned()), + Line::Para(note(d)?), + ], + ..Card::default() + }; + Ok(HtmlPart { + cards: vec![html::card(&sec, card)?], + files: Vec::new(), + }) +} diff --git a/rust/crates/og-render/src/sections/y_tree.rs b/rust/crates/og-render/src/sections/y_tree.rs index 892584a1c..da59d9f0e 100644 --- a/rust/crates/og-render/src/sections/y_tree.rs +++ b/rust/crates/og-render/src/sections/y_tree.rs @@ -1,2 +1,140 @@ -//! The report section `y_tree`. Empty until its unit of the 1.0.0 plan -//! lands. +//! The second Y placement (step 37, `og y tree-place`): the branch on a +//! public tree release newer than Yleaf's, the SNP counts on the path to +//! it, and one row per level of the walk. + +use crate::escape::e; +use crate::model::{Card, Inline, Line, Malformed, State, Summary}; +use crate::notes::{EXPLORE_NOTE, Y_TREE_NOTE}; +use crate::sections::ancestry::{ + Align, explore_html, get_text, list, need_align, need_text, py_or, shown, +}; +use crate::{html, text}; + +const PATH: &str = "sections.y_tree.data"; +const ROW: &str = "sections.y_tree.data.path[]"; + +/// The two lists of ancestral calls a reader should see, with their labels. +const ANCESTRAL: [(&str, &str); 2] = [ + ( + "Ancestral inside the terminal branch", + "ancestral_in_terminal", + ), + ( + "Ancestral on the path, outvoted by the SNPs below it", + "outvoted_ancestral", + ), +]; + +/// The text report's block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "y_tree", Some("Y Placement on the Current Public Tree"))? + else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if !head.ran { + return Ok(w); + } + let d = s.section("y_tree").data; + let n = |k: &str| need_text(d, k, PATH); + w.push(format!( + " Branch: {} (level {}, {})", + n("haplogroup")?, + n("level")?, + n("tree")? + )); + w.push(format!( + " Path SNPs: {} derived, {} ancestral, {} no call; {} SNPs called at depth {} or more", + n("path_derived")?, + n("path_ancestral")?, + n("path_no_call")?, + n("snps_called")?, + n("min_depth")? + )); + for (label, key) in ANCESTRAL { + if shown(d, key, PATH)?.is_some() { + w.push(format!(" {label}: {}", n(key)?)); + } + } + for x in list(d, "path", PATH)? { + let a = |k: &str, width: usize, side: Align| need_align(&x, k, width, side, ROW); + w.push(format!( + " {} {} {}D {}A {}N", + a("Level", 3, Align::Right)?, + a("Branch", 22, Align::Left)?, + a("Derived", 3, Align::Right)?, + a("Ancestral", 3, Align::Right)?, + a("No_call", 4, Align::Right)? + )); + } + w.push(format!(" {Y_TREE_NOTE}")); + w.push(String::new()); + Ok(w) +} + +/// The HTML report's card, when step 37 placed the sample on the tree. +pub fn card(s: &Summary) -> Result, Malformed> { + let sec = s.section("y_tree"); + if sec.state == State::Missing { + return Ok(None); + } + let mut c = Card { + title: "Y Placement on the Current Public Tree".to_owned(), + full: true, + ..Card::default() + }; + if sec.state.ran() { + let d = sec.data; + let g = |k: &str| get_text(d, k, ".", PATH); + for (label, key) in [ + ("Branch", "haplogroup"), + ("Level", "level"), + ("Tree", "tree"), + ] { + c.body + .push(Line::Stat(label.to_owned(), Inline::Text(g(key)?))); + } + c.body.push(Line::Para(format!( + "Path SNPs: {} derived, {} ancestral, {} no call; {} SNPs called at depth {} or more", + g("path_derived")?, + g("path_ancestral")?, + g("path_no_call")?, + g("snps_called")?, + g("min_depth")? + ))); + for (label, key) in ANCESTRAL { + if let Some(v) = shown(d, key, PATH)? { + c.body + .push(Line::Para(format!("{label}: {}", py_or(Some(v), "")?))); + } + } + let mut rows = String::new(); + for x in list(d, "path", PATH)? { + rows += ""; + for k in [ + "Level", + "Branch", + "Derived", + "Ancestral", + "No_call", + "Ancestral_SNPs", + ] { + rows += &format!("{}", e(&need_text(&x, k, ROW)?)); + } + rows += ""; + } + if !rows.is_empty() { + c.body.push(Line::Raw(format!( + " \ + {rows}
    LevelBranchDerivedAncestralNo callAncestral SNPs
    " + ))); + } + let (links, note) = explore_html(d, PATH)?; + c.body.extend(links); + if note { + c.caveats.push(EXPLORE_NOTE); + } + c.caveats.push(Y_TREE_NOTE); + } + Ok(Some(html::card(&sec, c)?)) +} diff --git a/rust/crates/og-render/src/sections/ystr.rs b/rust/crates/og-render/src/sections/ystr.rs index 7e7922e39..91019f76c 100644 --- a/rust/crates/og-render/src/sections/ystr.rs +++ b/rust/crates/og-render/src/sections/ystr.rs @@ -1 +1,101 @@ -//! The report section `ystr`. Empty until its unit of the 1.0.0 plan lands. +//! The Y-STR markers (step 45, HipSTR): the repeat count of each marker +//! called, in FamilyTreeDNA's Y-111 order and panels, so a count lines up +//! with FamilyTreeDNA results and surname-project tables. + +use pycompat::json::Value; + +use crate::escape::e; +use crate::model::{Card, Line, Malformed, State, Summary, need, truthy}; +use crate::notes::YSTR_NOTE; +use crate::sections::ancestry::{Align, items, need_align, need_text, py_len}; +use crate::{html, text}; + +const PATH: &str = "sections.y_str.data"; +const ROW: &str = "sections.y_str.data.markers[]"; + +/// How many markers were called, and with what confidence. +pub fn counts(d: &Value) -> Result { + let n = |k: &str| need_text(d, k, PATH); + let high = n("high")?; + let low = n("low")?; + let all = py_len(need(d, "markers", PATH)?, &format!("{PATH}.markers"))?; + Ok(format!( + "Markers called: {high} high and {low} low confidence of {all}; {} no call, {} not \ + callable from short reads", + n("no_call")?, + n("not_callable")? + )) +} + +/// The markers with a count, in the order the summary lists them (the +/// panel's order). +fn called(markers: &Value) -> Result, Malformed> { + let mut out = Vec::new(); + for x in items(markers, &format!("{PATH}.markers"))? { + if truthy(need(&x, "count", ROW)?) { + out.push(x); + } + } + Ok(out) +} + +/// The text report's block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "y_str", Some("Y-STR Markers (HipSTR)"))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if !head.ran { + return Ok(w); + } + let d = s.section("y_str").data; + w.push(format!(" {}", counts(d)?)); + for x in called(need(d, "markers", PATH)?)? { + w.push(format!( + " {} {} {} ({})", + need_align(&x, "panel", 6, Align::Left, ROW)?, + need_align(&x, "marker", 13, Align::Left, ROW)?, + need_align(&x, "count", 6, Align::Left, ROW)?, + need_text(&x, "confidence", ROW)? + )); + } + w.push(format!(" {YSTR_NOTE}")); + w.push(String::new()); + Ok(w) +} + +/// The HTML report's card, when step 45 has a result. +pub fn card(s: &Summary) -> Result, Malformed> { + let sec = s.section("y_str"); + if sec.state == State::Missing { + return Ok(None); + } + let mut c = Card { + title: "Y-STR Markers (HipSTR)".to_owned(), + full: true, + ..Card::default() + }; + if sec.state.ran() { + let d = sec.data; + let markers = crate::model::get(d, "markers", PATH)?; + let mut rows = String::new(); + if let Some(m) = markers { + for x in called(m)? { + rows += ""; + for k in ["panel", "marker", "count", "confidence"] { + rows += &format!("{}", e(&need_text(&x, k, ROW)?)); + } + rows += ""; + } + } + c.body = vec![ + Line::Para(counts(d)?), + Line::Raw(format!( + " \ + {rows}
    PanelMarkerRepeatsConfidence
    " + )), + ]; + c.caveats.push(YSTR_NOTE); + } + Ok(Some(html::card(&sec, c)?)) +} diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index 3e7e16214..eb96712b2 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -1255,3 +1255,19 @@ report-render m-variants-missing gen:report-corpus.sh --summary m-variants-missi report-render m-variants-ok gen:report-corpus.sh --summary m-variants-ok.json -o m-variants-ok.txt -o m-variants-ok.html m-variants-ok.txt,m-variants-ok.html bytes PYTHON_IMAGE report-render m-variants-stale gen:report-corpus.sh --summary m-variants-stale.json -o m-variants-stale.txt -o m-variants-stale.html m-variants-stale.txt,m-variants-stale.html bytes PYTHON_IMAGE report-render m-variants-unreadable gen:report-corpus.sh --summary m-variants-unreadable.json -o m-variants-unreadable.txt -o m-variants-unreadable.html m-variants-unreadable.txt,m-variants-unreadable.html bytes PYTHON_IMAGE +# report-render, the ancestry chapter and the PRS card: the a-* summaries of gen/report-corpus.sh. a-numbers is text-only: +# the Python's HTML report cannot render a number where the summary writes text. +report-render a-full gen:report-corpus.sh --summary a-full.json -o a-full.txt -o a-full.html a-full.txt,a-full.html bytes PYTHON_IMAGE +report-render a-stale gen:report-corpus.sh --summary a-stale.json -o a-stale.txt -o a-stale.html a-stale.txt,a-stale.html bytes PYTHON_IMAGE +report-render a-failed gen:report-corpus.sh --summary a-failed.json -o a-failed.txt -o a-failed.html a-failed.txt,a-failed.html bytes PYTHON_IMAGE +report-render a-unreadable gen:report-corpus.sh --summary a-unreadable.json -o a-unreadable.txt -o a-unreadable.html a-unreadable.txt,a-unreadable.html bytes PYTHON_IMAGE +report-render a-missing gen:report-corpus.sh --summary a-missing.json -o a-missing.txt -o a-missing.html a-missing.txt,a-missing.html bytes PYTHON_IMAGE +report-render a-partial gen:report-corpus.sh --summary a-partial.json -o a-partial.txt -o a-partial.html a-partial.txt,a-partial.html bytes PYTHON_IMAGE +report-render a-escape gen:report-corpus.sh --summary a-escape.json -o a-escape.txt -o a-escape.html a-escape.txt,a-escape.html bytes PYTHON_IMAGE +report-render a-painting gen:report-corpus.sh --summary a-painting.json -o a-painting.txt -o a-painting.html a-painting.txt,a-painting.html bytes PYTHON_IMAGE +report-render a-no-painting gen:report-corpus.sh --summary a-no-painting.json -o a-no-painting.txt -o a-no-painting.html a-no-painting.txt,a-no-painting.html bytes PYTHON_IMAGE +report-render a-prs-raw gen:report-corpus.sh --summary a-prs-raw.json -o a-prs-raw.txt -o a-prs-raw.html a-prs-raw.txt,a-prs-raw.html bytes PYTHON_IMAGE +report-render a-prs-sex gen:report-corpus.sh --summary a-prs-sex.json -o a-prs-sex.txt -o a-prs-sex.html a-prs-sex.txt,a-prs-sex.html bytes PYTHON_IMAGE +report-render a-y-calls gen:report-corpus.sh --summary a-y-calls.json -o a-y-calls.txt -o a-y-calls.html a-y-calls.txt,a-y-calls.html bytes PYTHON_IMAGE +report-render a-since gen:report-corpus.sh --summary a-since.json -o a-since.txt -o a-since.html a-since.txt,a-since.html bytes PYTHON_IMAGE +report-render a-numbers gen:report-corpus.sh --summary a-numbers.json -o a-numbers.txt a-numbers.txt bytes PYTHON_IMAGE diff --git a/tests/parity/gen/report-corpus.sh b/tests/parity/gen/report-corpus.sh index d16728bc9..bcddd670e 100644 --- a/tests/parity/gen/report-corpus.sh +++ b/tests/parity/gen/report-corpus.sh @@ -12,6 +12,8 @@ # re-check, the manifest with and without a release, a # sample name with characters HTML escapes and characters # outside ASCII, counts missing or of another type. +# a-.json case c-full's summary changed one way each, for the +# ancestry chapter and the PRS card (see that block below). # py/ holds the modules render_report.py imports, as they were at the # oracle's commit (tests/parity/oracle.ref, report-render). # @@ -116,6 +118,73 @@ mut counts '.sections.variants.state = "ok" | .sections.variants.data = {"total" mut counts-float '.sections.coverage.state = "ok" | .sections.coverage.data.mean_depth = 30.05' rm -f "$base" +# --- the ancestry chapter and the PRS card, from case c-full ----------------------- +# a-.json: c-full without its one PRS row that has no numbers (the +# Python's HTML report cannot render it), changed one way each: every state, +# a planted escape in the haplogroup links and names, a painting with its +# legend and the groups left out, the raw and the sex-specific scores, the Y +# placement's ancestral calls, new_since, and numbers where the summary +# writes text (a text-only case: the Python's HTML report cannot render it). +[ -s c-full.json ] || { echo "report-corpus.sh: og summary collect wrote no c-full.json" >&2; exit 1; } +abase=abase.tmp +jq -c '.sections.prs.data.scores |= map(select(.score != null))' c-full.json > "$abase" +amut() { jq -c "$2" "$abase" > "a-$1.json"; } +akeys='("ancestry_fine", "haplogroup", "y_haplogroup", "y_tree", "y_str", "archaic", "ancient", "prs")' +amut full '.' +for st in stale failed unreadable missing partial; do + amut "$st" "reduce $akeys as \$k (.; .sections[\$k].state = \"$st\" | .sections[\$k].note = \"why: &'\\\"\")" +done +amut escape '.sections.haplogroup.data.haplogroup = "H&\"x'"'"'" + | .sections.haplogroup.data.explore[0].url = "https://discover.familytreedna.com/mtdna/\">" + | .sections.haplogroup.data.explore[1].name = "YFull & co" + | .sections.haplogroup.data.contamination_status = "YES" | .sections.haplogroup.data.contamination_level = "0.4<" + | .sections.y_haplogroup.data.explore[1].url = "javascript:alert(\"x\")" + | .sections.y_tree.data.path[0].Branch = "" | .sections.y_tree.data.path[0].Ancestral_SNPs = "M1&M2" + | .sections.y_str.data.markers[0].marker = "DYS<393>" + | .sections.ancient.data.nearest[0].group = "Spain_" | .sections.ancient.data.nearest[0].locality = "A & B" + | .sections.ancestry_fine.data.continent[0].group = "West " + | .sections.ancestry_fine.data.legend[0].colour = "red\">" | .sections.ancestry_fine.data.legend[1].group = "W" + | .sections.archaic.data.longest = ["chr1:1-2 "] + | .sections.prs.data.scores[0].condition = "CD & \"x\"" | .sections.prs.data.scores[0].effect = "HR <1.5>"' +amut painting '.sections.ancestry_fine.data.painting = [{"code": "AFR", "group": "Africa", "fraction": 0.3}, + {"code": "WEA", "group": "West Eurasia", "fraction": 0.7}] + | .sections.ancestry_fine.data.not_painted = [] + | .sections.ancestry_fine.data.left_out = [{"group": "Oceania (continental group)"}, {"group": "Oceania (regional group)"}, + {"group": "Papua (regional group)"}, {"group": "Odd (name) (x group)"}, {"group": "Plain"}] + | .sections.ancestry_fine.data.min_references = 12 + | .sections.ancestry_fine.data.continent += [{"group": "Small", "fraction": 0.009, "low": 0.001, "high": 0.02}] + | .sections.ancestry_fine.data.region += [{"group": "Wide", "fraction": 0.005, "low": 0.0, "high": 0.06}, + {"group": "Narrow", "fraction": 0.005, "low": 0.0, "high": 0.02}, {"group": "Unfit", "fraction": null, "low": null, "high": null}]' +amut no-painting '.sections.ancestry_fine.data.painting = [] | .sections.ancestry_fine.data.not_painted = [] + | .sections.ancestry_fine.data.left_out = [] | .sections.ancestry_fine.data.painting_png = ""' +amut prs-raw '.sections.prs.data.adjusted = false | del(.sections.prs.data.ancestry) + | .sections.prs.data.scores |= map(.percentile = "" | .evaluated_in = null)' +amut prs-sex '.sections.prs.data.ancestry = {"population": "", "panel": "", "low_confidence": "True"} + | .sections.prs.data.scores = [ + {"condition": "Prostate", "pgs_id": "PGS2", "score": "1", "matched": "5", "total": "6", "chrx": "0", + "applies_to": "male", "sex_check": "not applicable to the declared sex", "percentile": "", "group": "", + "evaluated_in": ["EUR"], "effect": ""}, + {"condition": "Breast", "pgs_id": "PGS3", "score": "2", "matched": "7", "total": "9", "chrx": "3 left out", + "applies_to": "female", "sex_check": "no sex declared", "percentile": "40", "group": "AFR", + "evaluated_in": [], "effect": ""}, + {"condition": "Height", "pgs_id": "PGS4", "score": "3", "matched": "1", "total": "2", "chrx": "", + "applies_to": "", "sex_check": "", "percentile": "60", "group": "AFR", "evaluated_in": ["EUR", "AFR"], + "effect": ""}]' +amut y-calls '.sections.y_tree.data.ancestral_in_terminal = "M1,M2" | .sections.y_tree.data.outvoted_ancestral = null + | .sections.y_tree.data.path = [] | .sections.y_str.data.markers |= map(.count = "") + | .sections.ancient.data.nearest = [] | .sections.ancient.data.plot_svg = "" + | .sections.archaic.data.longest = [] + | .sections.haplogroup.data.explore = [] | del(.sections.haplogroup.data.contamination_status) + | .sections.y_haplogroup.data.explore = []' +amut since '.new_since = {"previous_utc": "2026-01-01T00:00:00Z", "sections": ["y_str", "prs", "haplogroup"]}' +amut numbers '.sections.y_str.data.markers[0].count = 13 | .sections.y_str.data.markers[1].panel = true + | .sections.ancient.data.nearest[0].rank = 1 | .sections.ancient.data.nearest[1].n = 5.5 + | .sections.y_tree.data.path[0].Level = 1 | .sections.y_tree.data.path[1].Derived = 2.0 + | .sections.archaic.data.neanderthal_bp = 51234567.0 | .sections.archaic.data.denisovan_bp = true + | .sections.prs.data.scores[0].condition = 7 | .sections.prs.data.scores[1].score = 0.25 + | .sections.ancestry_fine.data.continent[1].fraction = 1' +rm -f "$abase" + # --- the oracle ----------------------------------------------------------------------- cat > oracle.sh << 'ORACLE' #!/bin/sh @@ -132,6 +201,7 @@ PYTHONPATH=$d python3 "$d/render_report.py" "$@" rm -rf "$d" # The sections og report render writes; the others are left out. PORTED=" qc variants not-run not-assessed " +PORTED="${PORTED}ancestry-fine mt-haplogroup y-chromosome archaic ancient prs " while [ $# -gt 0 ]; do case "$1" in -o) @@ -148,8 +218,9 @@ done ORACLE # html.awk: drops each section not in keep (its line in the contents and its -# frame), then each chapter with no section left (its contents entry and its -# body), and the gene and rsID appendix. +# frame) and every chart (og report charts will write them), then each +# chapter with no section left (its contents entry and its body), and the +# gene and rsID appendix. cat > html.awk << 'AWK' function wanted(line, id) { if (!match(line, /id="[^"]*"/) && !match(line, /href="#[^"]*"/)) return 1 @@ -164,6 +235,7 @@ END { for (i = 1; i <= n; i++) { l = lines[i] if (l ~ /^
  • /) continue if (l ~ /^
    ") i++ continue @@ -194,6 +266,10 @@ cat > text.awk << 'AWK' BEGIN { rule = sprintf("%80s", ""); gsub(/ /, "#", rule) title["## Quality control"] = "qc"; title["## Variant Calling (DeepVariant)"] = "variants" + title["## Fine-Grained Ancestry (ADMIXTURE, RFMix)"] = "ancestry-fine"; title["## Mitochondrial Haplogroup"] = "mt-haplogroup" + title["## Y-Chromosome Haplogroup (Yleaf)"] = "y-chromosome"; title["## Y Placement on the Current Public Tree"] = "y-chromosome" + title["## Y-STR Markers (HipSTR)"] = "y-chromosome"; title["## Archaic Ancestry (IBDmix, step 43)"] = "archaic" + title["## Ancient Populations (AADR PCA, step 44)"] = "ancient"; title["## Polygenic Risk Scores"] = "prs" } { lines[++n] = $0 } END { From e7e69a3a35b49618a41ccb67d8a61956405d9f59 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:08:33 +0100 Subject: [PATCH 2/7] fix(report): a PRS row with no counts reads "not available" og summary collect writes null for the score, matched and total of a short row of pgsc_calc's table. The PRS card now shows "not available" in those cells in both reports. render_report.py wrote "None" in the text report and stopped the whole HTML report on that row (html.escape(None)). ancestry_prs_null.rs renders the golden summary of case summary-collect/all, which has such a row (Short, PGS1). The report-render oracle filter writes the text row as og does, so the parity cases with that row still compare. Bead: D8 --- rust/crates/og-render/src/sections/prs.rs | 23 +- .../og-render/tests/ancestry_prs_null.rs | 48 ++ tests/parity/gen/report-corpus.sh | 7 +- tests/test_ancient_report.py | 287 --------- tests/test_archaic_report.py | 592 ------------------ tests/test_haplogroup_explore.py | 98 --- tests/test_ystr_report.py | 360 ----------- 7 files changed, 71 insertions(+), 1344 deletions(-) create mode 100644 rust/crates/og-render/tests/ancestry_prs_null.rs delete mode 100644 tests/test_ancient_report.py delete mode 100644 tests/test_archaic_report.py delete mode 100755 tests/test_haplogroup_explore.py delete mode 100644 tests/test_ystr_report.py diff --git a/rust/crates/og-render/src/sections/prs.rs b/rust/crates/og-render/src/sections/prs.rs index c9faba465..83f06bca9 100644 --- a/rust/crates/og-render/src/sections/prs.rs +++ b/rust/crates/og-render/src/sections/prs.rs @@ -27,6 +27,18 @@ pub const EFFECT_TEXT: &str = "Effect: the odds ratio (OR) or hazard ratio (HR) evaluation's unit (its PGS Catalog PPM page says which). It describes the score in that \ group, not your own risk, and is shown only for a group the score was evaluated in."; +/// What a score, matched or total cell says when the summary has none: og +/// summary collect writes null for a short row of pgsc_calc's table. +pub const NOT_AVAILABLE: &str = "not available"; + +/// `str(r[key])`, or [`NOT_AVAILABLE`] for null. +fn count(r: &Value, key: &str) -> Result { + match need(r, key, ROW)? { + Value::Null => Ok(NOT_AVAILABLE.to_owned()), + v => py(v), + } +} + fn field<'a>(r: &'a Value, key: &str) -> Result, Malformed> { get(r, key, ROW) } @@ -225,13 +237,13 @@ pub fn text(s: &Summary) -> Result, Malformed> { let d = s.section("prs").data; for r in scores(d)? { let condition = need_align(&r, "condition", 35, Align::Left, ROW)?; - let n = |k: &str| need_text(&r, k, ROW); + let n = |k: &str| count(&r, k); let line = format!( " {condition} {} ({}/{} variants matched) {}", n("score")?, n("matched")?, n("total")?, - n("pgs_id")? + need_text(&r, "pgs_id", ROW)? ); let pct = percentile(&r)?; let pct = if other_sex(&r)? { @@ -270,14 +282,15 @@ pub fn html(s: &Summary) -> Result { let mut rows = String::new(); for x in &rs { let n = |k: &str| need_text(x, k, ROW).map(|t| e(&t).into_owned()); + let c = |k: &str| count(x, k).map(|t| e(&t).into_owned()); let pct = percentile(x)?; rows += &format!( "{}{}{}{}/{}{}", n("condition")?, n("pgs_id")?, - n("score")?, - n("matched")?, - n("total")?, + c("score")?, + c("matched")?, + c("total")?, e(if pct.is_empty() { "raw score only" } else { diff --git a/rust/crates/og-render/tests/ancestry_prs_null.rs b/rust/crates/og-render/tests/ancestry_prs_null.rs new file mode 100644 index 000000000..2c50f1118 --- /dev/null +++ b/rust/crates/og-render/tests/ancestry_prs_null.rs @@ -0,0 +1,48 @@ +//! A short row of pgsc_calc's table: og summary collect writes its score, +//! matched and total as null (the golden summary +//! tests/parity/golden/summary-collect/all/all.json has one, Short/PGS1). +//! Both reports render it, and the row reads "not available" in those +//! cells. bin/render_report.py stops the whole HTML report on that row +//! (`html.escape(None)`) and writes "None" in the text report, so this is a +//! known difference from the Python, outside the parity corpus. + +use std::path::Path; + +use og_render::model::Summary; +use og_render::{html, text}; + +fn golden() -> Summary { + let p = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../../tests/parity/golden/summary-collect/all/all.json"); + Summary::parse(&std::fs::read_to_string(p).unwrap()).unwrap() +} + +#[test] +fn a_null_score_reads_not_available_in_both_reports() { + let s = golden(); + let page = html::report(&s).unwrap(); + assert!( + page.contains( + "ShortPGS1not availablenot available/not \ + availableraw score only" + ), + "the HTML row" + ); + assert!( + page.contains("CADPGS0000180.5100/120"), + "the other rows are unchanged" + ); + let txt = text::report(&s).unwrap(); + let row = txt + .lines() + .find(|l| l.starts_with(" Short ")) + .expect("the text row"); + assert_eq!( + row.trim_end(), + format!( + " {:<35} not available (not available/not available variants matched) PGS1", + "Short" + ) + ); + assert!(!txt.contains("None (None/None"), "{txt}"); +} diff --git a/tests/parity/gen/report-corpus.sh b/tests/parity/gen/report-corpus.sh index bcddd670e..13e16811d 100644 --- a/tests/parity/gen/report-corpus.sh +++ b/tests/parity/gen/report-corpus.sh @@ -261,7 +261,10 @@ AWK # text.awk: drops each section block not in keep, then each chapter left # with no block. A block starts at its "## " title; the titles of the kept -# sections are listed here, as render_report.py writes them. +# sections are listed here, as render_report.py writes them. A PRS row +# whose counts the summary has as null reads "not available", as og writes +# it; render_report.py writes "None" there (and stops its HTML report on +# that row). cat > text.awk << 'AWK' BEGIN { rule = sprintf("%80s", ""); gsub(/ /, "#", rule) @@ -271,7 +274,7 @@ BEGIN { title["## Y-STR Markers (HipSTR)"] = "y-chromosome"; title["## Archaic Ancestry (IBDmix, step 43)"] = "archaic" title["## Ancient Populations (AADR PCA, step 44)"] = "ancient"; title["## Polygenic Risk Scores"] = "prs" } -{ lines[++n] = $0 } +{ sub(/ None \(None\/None variants matched\) /, " not available (not available/not available variants matched) "); lines[++n] = $0 } END { # The end part (steps not run onward) and the head (before the first # banner) stay; chapters are banner, title, banner, blank, then blocks. diff --git a/tests/test_ancient_report.py b/tests/test_ancient_report.py deleted file mode 100644 index aacc2535b..000000000 --- a/tests/test_ancient_report.py +++ /dev/null @@ -1,287 +0,0 @@ -#!/usr/bin/env python3 -# og-leg: OG_REQUIRED -"""og ancient (step 44) on synthetic files, run as the pipeline runs it. - -Checks: - 1. ranking: on a synthetic .evec the nearest ancient group is the one whose - mean position is closest on the weighted PCs, a group far on PC1 but - with a small eigenvalue on PC3 is ranked by the weighted distance, and - each row carries its dates (oldest first, AADR style BCE), country and - member count; - 2. axes: a run whose PC2 has the opposite sign is turned back before the - person is placed, and a run whose present-day coordinates differ stops - with the reason instead of placing the person on other axes; - 3. liftover: lift keeps a site inside a block, a site on a minus-strand - chain with its alleles complemented, and drops a site in a gap, one - whose alleles miss the GRCh38 base, chrX and a monomorphic marker; - 4. genotypes: person-geno appends one column, 9 where the person has no - usable genotype, and stops on a genotype file shorter than the .snp file; - 5. select: the present-day reference people come from MODERN_GROUPS, the - ancient ones need a date, a passing assessment, enough sites and a place - in West Eurasia, one genetic id per individual; - 6. the report command writes the table, the summary, the text and an SVG - with the person marked; - 7. bin/render_report.py shows them: an HTML card with the table and the - plot, and a section of the text report. - -The Rust unit tests (rust/crates/og-ancestry, og-intervals chain) check the -date labels, the dosages and the single-position lift in detail, and -tests/parity checks each command against the Python it replaced. Without an -og binary the test is skipped with a note; with OG_REQUIRED set the skip is a -failure. - -Run: OG=bin/og python3 tests/test_ancient_report.py -""" -import os -import shutil -import subprocess -import sys -import tempfile - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -OG = os.environ.get("OG", "") -FAILS = 0 -CAVEAT = "Closeness on a PCA is genetic similarity, not descent" - -if not OG: - print("skip: OG is not set (OG=bin/og)") - sys.exit(1 if os.environ.get("OG_REQUIRED") else 0) - - -def check(desc, ok, detail=""): - global FAILS - print(f"[{'PASS' if ok else 'FAIL'}] {desc}{'' if ok else ' -- ' + str(detail)[:600]}") - if not ok: - FAILS += 1 - - -def write(path, text): - with open(path, "w") as f: - f.write(text) - return path - - -def og(*args): - return subprocess.run([OG, "ancient", *args], capture_output=True, text=True) - - -# --- 1. ranking ------------------------------------------------------------------- -# Present-day people M0..M19 build the axes; three ancient groups. -EVALS = [9.0, 4.0, 0.01, 0.01] - - -def evec(rows, evals=EVALS): - out = ["#eigvals: " + " ".join(f"{v:.3f}" for v in evals)] - for i, (label, pcs) in rows.items(): - out.append(f"{i} " + " ".join(f"{v:.6f}" for v in pcs) + f" {label}") - return "\n".join(out) + "\n" - - -MODERNS = {f"M{i}": ("French" if i % 2 else "Sardinian", - [i * 0.01 - 0.1, (i % 5) * 0.02 - 0.04, (i % 3) * 0.01 - 0.01, (i % 4) * 0.01 - 0.015]) - for i in range(20)} -PANEL = dict(MODERNS) -PANEL.update({ - "A1": ("Ancient", [0.10, 0.10, 0.0, 0.0]), "A2": ("Ancient", [0.12, 0.10, 0.0, 0.0]), # Steppe - "B1": ("Ancient", [-0.20, 0.00, 0.0, 0.0]), # Farmer - "C1": ("Ancient", [0.00, 0.00, 0.9, 0.0]), # close on PC1/PC2, far on PC3 (eigenvalue 0.01) -}) -ANCIENTS = { - "A1": {"id": "A1", "group": "Steppe_EBA", "date_bp": "4800", "country": "Russia", "locality": "Samara"}, - "A2": {"id": "A2", "group": "Steppe_EBA", "date_bp": "4600", "country": "Russia", "locality": "Samara"}, - "B1": {"id": "B1", "group": "Anatolia_N", "date_bp": "8200", "country": "Turkey", "locality": "Barcin"}, - "C1": {"id": "C1", "group": "Odd_C", "date_bp": "3000", "country": "Italy", "locality": "Somewhere"}, -} -# The person's run: the present-day people with PC2 flipped, as smartpca may -# write it, and the person. -RUN = {k: (lab, [pcs[0], -pcs[1], pcs[2], pcs[3]]) for k, (lab, pcs) in MODERNS.items()} -RUN["S"] = ("Person", [0.08, -0.09, 0.0, 0.0]) - -tmp = tempfile.mkdtemp() -try: - stats = write(os.path.join(tmp, "stats.tsv"), "panel_sites\t4\nsites_genotyped\t3\nother_allele\t1\nsites_used\t2\n") - at = os.path.join(tmp, "anc.tsv") - with open(at, "w") as f: - f.write("id\tgroup\tdate_bp\tdate_sd\tfull_date\tlocality\tcountry\tlat\tlon\tsnps\n") - for r in ANCIENTS.values(): - f.write(f"{r['id']}\t{r['group']}\t{r['date_bp']}\t50\tx\t{r['locality']}\t{r['country']}\t50\t40\t500000\n") - outs = {k: os.path.join(tmp, f"S_{k}") for k in ("ancient.tsv", "info.tsv", "ancient.txt", "pca.svg")} - - def report(panel_evec, run_evec, with_stats=True): - return og("report", "--panel-evec", panel_evec, "--run-evec", run_evec, "--ancients", at, - "--sample", "S", "--release", "test", *(["--stats", stats] if with_stats else []), - "--out-tsv", outs["ancient.tsv"], "--out-info", outs["info.tsv"], - "--out-txt", outs["ancient.txt"], "--out-svg", outs["pca.svg"]) - - def table(): - with open(outs["ancient.tsv"]) as f: - head, *rows = [line.split("\t") for line in f.read().splitlines()] - return [dict(zip(head, r)) for r in rows] - - pe = write(os.path.join(tmp, "panel.evec"), evec(PANEL)) - re_ = write(os.path.join(tmp, "run.evec"), evec(RUN)) - p = report(pe, re_) - check("report exits 0", p.returncode == 0, p.stderr) - rows = table() if p.returncode == 0 else [] - order = [r["group"] for r in rows] - check("three ancient groups ranked", len(rows) == 3, rows) - check("the nearest group is first (Steppe_EBA), from the flipped run turned back", - order[:1] == ["Steppe_EBA"], order) - check("a group far only on a PC with a tiny eigenvalue is second, not last (Odd_C)", order[1:2] == ["Odd_C"], order) - check("the farthest group is last (Anatolia_N)", order[-1:] == ["Anatolia_N"], order) - st = rows[0] if rows else {} - check("the row has the member count, dates and place", st.get("n") == "2" and st.get("dates") == "2850-2650 BCE" - and st.get("country") == "Russia" and st.get("locality") == "Samara", st) - check("ranks are 1..n", [r["rank"] for r in rows] == ["1", "2", "3"], rows) - flat_pe = write(os.path.join(tmp, "flat_panel.evec"), evec(PANEL, [1.0, 1.0, 1.0, 1.0])) - flat_re = write(os.path.join(tmp, "flat_run.evec"), evec(RUN, [1.0, 1.0, 1.0, 1.0])) - p = report(flat_pe, flat_re) - check("control: without the eigenvalue weights Odd_C falls behind Anatolia_N", - p.returncode == 0 and [r["group"] for r in table()] == ["Steppe_EBA", "Anatolia_N", "Odd_C"], p.stderr) - - # --- 2. axes ---------------------------------------------------------------------- - p = og("align", "--panel-evec", pe, "--run-evec", re_, "--pcs", "2") - check("align: a flipped PC2 is found", p.returncode == 0 and "PC1+ PC2-" in p.stdout, (p.stdout, p.stderr)) - bad = {k: (lab, [pcs[0] * (1 if int(k[1:]) % 3 else -1)] + pcs[1:]) for k, (lab, pcs) in MODERNS.items()} - bad_re = write(os.path.join(tmp, "bad.evec"), evec(bad)) - p = og("align", "--panel-evec", pe, "--run-evec", bad_re, "--pcs", "2") - check("align: other axes stop, naming the PC", p.returncode == 1 and "PC1" in p.stderr, (p.returncode, p.stderr)) - - # --- 3. liftover -------------------------------------------------------------------- - chain = write(os.path.join(tmp, "test.chain"), "\n".join([ - # chr1 source 1000-1100 -> target 2000-2100 (+), gap of 50, then 1150-1200 -> 2150-2200 - "chain 100 chr1 100000 + 1000 1200 chr1 100000 + 2000 2200 1", - "100 50 50", "50", "", - # chr2 source 500-600 -> target on the minus strand of a 10,000 bp chr2, reverse start 300 - "chain 90 chr2 50000 + 500 600 chr2 10000 - 300 400 2", - "100", "", - # chr3: two chains over 100-200 - "chain 80 chr3 50000 + 100 200 chr3 50000 + 100 200 3", "100", "", - "chain 70 chr3 50000 + 150 250 chr3 50000 + 5150 5250 4", "100", ""]) + "\n") - seq1 = ["N"] * 3000 - seq1[2050] = "G" # 1-based 2051 - seq1[2159] = "T" # 1-based 2160 - seq2 = ["N"] * 10000 - seq2[9689] = "C" # 1-based 9690: the complement of the source G - fa = os.path.join(tmp, "ref.fa") - with open(fa, "w") as f, open(fa + ".fai", "w") as fai: - off = 0 - for name, seq in (("chr1", seq1), ("chr2", seq2)): - head = f">{name}\n" - f.write(head) - off += len(head) - s = "".join(seq) - for i in range(0, len(s), 60): - f.write(s[i:i + 60] + "\n") - fai.write(f"{name}\t{len(s)}\t{off}\t60\t61\n") - off += len(s) + (len(s) + 59) // 60 - snp = write(os.path.join(tmp, "src.snp"), "\n".join([ - "rs1 1 0.0 1051 G A", # ref G at target: kept - "rs2 1 0.0 1160 C T", # second allele is the target base: kept - "rs3 1 0.0 1120 A G", # gap: dropped - "rs4 2 0.0 511 G T", # minus strand: alleles C/A, target C: kept - "rs5 1 0.0 1051 A C", # neither allele is G: dropped - "rs6 23 0.0 1051 G A", # X: dropped - "rs7 1 0.0 1051 G X", # monomorphic marker: dropped - ]) + "\n") - out = os.path.join(tmp, "sites.tsv") - p = og("lift", "--snp", snp, "--chain", chain, "--fasta", fa, "--out", out) - got = open(out).read().splitlines() if os.path.exists(out) else [] - check("lift keeps rs1, rs2 and the complemented rs4, in GRCh38 order", - p.returncode == 0 and got == ["snp\tchrom\tpos\tallele1\tallele2", "rs1\tchr1\t2051\tG\tA", - "rs2\tchr1\t2160\tC\tT", "rs4\tchr2\t9690\tC\tA"], (p.returncode, got, p.stderr)) - - # --- 4. genotypes ----------------------------------------------------------------- - msnp = write(os.path.join(tmp, "moderns.snp"), - "rs1 1 0.0 1051 G A\nrs2 1 0.0 1160 C T\nrsX 23 0.0 5 A G\nrs4 2 0.0 511 G T\n") - mgeno = write(os.path.join(tmp, "moderns.geno"), "012\n220\n999\n101\n") - gts = write(os.path.join(tmp, "gts.tsv"), "chr1\t2051\tG\tA\tG/A\nchr1\t2160\tT\t.\tT/T\nchr2\t9690\tC\tG\tC/G\n") - merged = os.path.join(tmp, "merged.geno") - pstats = os.path.join(tmp, "pstats.tsv") - p = og("person-geno", "--sites", out, "--genotypes", gts, "--snp", msnp, "--moderns-geno", mgeno, - "--out", merged, "--stats", pstats) - got = open(merged).read().splitlines() if os.path.exists(merged) else [] - check("person-geno appends one column: 1, 0, 9 for X, 9 for an allele outside the pair", - p.returncode == 0 and got == ["0121", "2200", "9999", "1019"], (p.returncode, got, p.stderr)) - st = dict(line.split("\t") for line in open(pstats).read().splitlines()) if os.path.exists(pstats) else {} - check("its counts", st == {"panel_sites": "4", "sites_genotyped": "3", "other_allele": "1", "sites_used": "2"}, st) - short = write(os.path.join(tmp, "short.geno"), "012\n220\n") - p = og("person-geno", "--sites", out, "--genotypes", gts, "--snp", msnp, "--moderns-geno", short, - "--out", merged, "--stats", pstats) - check("a genotype file shorter than the .snp file stops", p.returncode == 1, (p.returncode, p.stderr)) - - # --- 5. select ------------------------------------------------------------------------ - head = ["Genetic ID (long text)", "Persistent Genetic ID", "Individual ID", '"Date mean in BP in years before 1950"', - "Date standard deviation in BP [x]", "Full Date One of two formats", "Group ID", "Locality", - "Political Entity", "Latitude", "Longitude", "Data type", - "SNPs hit on autosomal targets (Computed using easystats on 1240k snpset)", - "SNPs hit on autosomal targets (Computed using easystats on HO snpset)", "ASSESSMENT", - "ASSESSMENT WARNINGS (long)"] - - def arow(gid, ind, bp, group, lat, lon, dtype, snps, ass): - return [gid, gid, ind, str(bp), "50", "x", group, "Loc", "Land", str(lat), str(lon), dtype, str(snps), "0", ass, ""] - - rows_ = [arow(f"F{i}.DG", f"F{i}", 0, "French", 46, 2, "Shotgun.diploid", 1100000, "PROVISIONAL_PASS") - for i in range(60)] - rows_ += [ - arow("F0.SG", "F0", 0, "French", 46, 2, "Shotgun.diploid", 900000, "PASS"), # same individual, fewer sites - arow("Y1.DG", "Y1", 0, "Yakut", 62, 129, "Shotgun.diploid", 1100000, "PASS"), # not a reference group - arow("H1.HO", "H1", 0, "French", 46, 2, "HumanOrigins", 600000, "PASS"), # not shotgun diploid - arow("A1.AG", "A1", 4800, "Russia_Steppe", 53, 50, "1240k", 500000, "PASS"), - arow("A2.AG", "A2", 4800, "Russia_Steppe", 53, 50, "1240k", 50000, "PASS"), # too few sites - arow("A3.AG", "A3", 4800, "Russia_Steppe", 53, 50, "1240k", 500000, "Questionable"), - arow("A4.AG", "A4", 4800, "China_N", 40, 116, "1240k", 500000, "PASS"), # outside West Eurasia - arow("A5.AG", "A5", 4800, "Unknown", "..", "..", "1240k", 500000, "PASS"), # no place - ] - anno = write(os.path.join(tmp, "x.anno"), "\n".join("\t".join(r) for r in [head] + rows_) + "\n") - ind = write(os.path.join(tmp, "x.ind"), "".join(f"{r[0]} U {r[6]}\n" for r in rows_)) - oind, plist, anc = (os.path.join(tmp, n) for n in ("panel.ind", "poplist.txt", "ancients.tsv")) - p = og("select", "--anno", anno, "--ind", ind, "--out-ind", oind, "--poplist", plist, "--ancients", anc) - rc = p.returncode - labels = dict(line.split("\t")[0::2] for line in open(oind).read().splitlines()) if rc == 0 else {} - check("select: the 60 French shotgun genomes are the reference, one id per person (F0.DG)", - rc == 0 and sum(1 for v in labels.values() if v == "French") == 60 and labels.get("F0.SG") == "Ignore", - (labels, p.stderr)) - check("select: Yakut and a Human Origins genome are ignored", - labels.get("Y1.DG") == "Ignore" and labels.get("H1.HO") == "Ignore", labels) - check("select: only A1 is projected", [k for k, v in labels.items() if v == "Ancient"] == ["A1.AG"], labels) - check("select: the population list", open(plist).read() == "French\n" if rc == 0 else False) - check("select: ancients.tsv holds A1 with its group", - rc == 0 and open(anc).read().splitlines()[1].startswith("A1.AG\tRussia_Steppe\t4800"), - open(anc).read() if rc == 0 else "") - - # --- 6. the report command ---------------------------------------------------------- - p = report(pe, re_) - check("report exits 0", p.returncode == 0, p.stderr) - info = dict(line.split("\t") for line in open(outs["info.tsv"]).read().splitlines()[1:]) if p.returncode == 0 else {} - check("report: the person's PC2 in the panel's sign", info.get("pc2") == "0.09000", info) - check("report: the sites used", info.get("sites_used") == "2", info) - svg = open(outs["pca.svg"]).read() if p.returncode == 0 else "" - check("report: the SVG is an SVG with a dark background and the person's label", - svg.startswith("S" in svg and svg.rstrip().endswith(""), svg[:200]) - txt = open(outs["ancient.txt"]).read() if p.returncode == 0 else "" - check("report: the text carries the caveat and says qpAdm is not run", CAVEAT in txt and "qpAdm" in txt, txt) - # --- 7. both reports read the outputs ---------------------------------------------- - sd = os.path.join(tmp, "S") - os.makedirs(os.path.join(sd, "ancient")) - for k, f in outs.items(): - shutil.copy(f, os.path.join(sd, "ancient", f"S_{k.replace('pca.svg', 'ancient_pca.svg').replace('info.tsv', 'ancient_info.tsv')}")) - html_out, txt_out = os.path.join(tmp, "S.html"), os.path.join(tmp, "S.txt") - p = subprocess.run([sys.executable, os.path.join(REPO, "bin", "render_report.py"), "--sample", "S", "--sample-dir", sd, - "--json", os.path.join(tmp, "summary.json"), "-o", html_out, "-o", txt_out], - capture_output=True, text=True) - check("render_report exits 0", p.returncode == 0, p.stderr) - page = open(html_out).read() if os.path.exists(html_out) else "" - check("the HTML report has the card, its first group and the plot as an image", - "

    Ancient Populations

    " in page and "Steppe_EBA" in page and "data:image/svg+xml;base64," in page, page[-400:]) - text = open(txt_out).read() if os.path.exists(txt_out) else "" - check("the text report has the section with the nearest group and the caveat", - "Ancient Populations (AADR PCA, step 44)" in text and "1. Steppe_EBA (" in text and "not descent" in text, text[-600:]) - - p = report(pe, pe, with_stats=False) - check("report: a run without the person stops", p.returncode == 1 and "no row for S" in p.stderr, p.stderr) -finally: - shutil.rmtree(tmp) - -print(f"\n{FAILS} failure(s)") -sys.exit(1 if FAILS else 0) diff --git a/tests/test_archaic_report.py b/tests/test_archaic_report.py deleted file mode 100644 index 030f74e97..000000000 --- a/tests/test_archaic_report.py +++ /dev/null @@ -1,592 +0,0 @@ -#!/usr/bin/env python3 -# og-leg: OG_REQUIRED -"""og archaic (step 43) on a synthetic archaic panel, with the real IBDmix image. - -Checks: - 1. table: a 1000 Genomes site keeps its frequencies and gets each archaic - genome's ALT count, 10 plus it outside that genome's mask or inside an - excluded region; an archaic record with another ALT or another REF is - unknown (9); a site only an archaic genome varies at gets frequency 0; a - site no archaic genome was called at, or only called as reference with - no modern variation, is left out; a position given twice by 1000 Genomes - is left out; - 1b. strict: a lifted row outside the GRCh38 strict mask is kept once per - position with every archaic call marked masked (9 stays 9), not dropped; - 2. genotypes: a masked archaic call (10 to 12) goes to IBDmix as its ALT - count and its position into that genome's mask file; the stand-in - columns carry the population's ALT count; your column is your ALT - count, 9 for a non-PASS record, and where your VCF has no record 0 - inside a reference block of the gVCF and 9 (unknown) outside every - one; the span runs over the sites read; a site where your VCF has another ALT is left out; a - chromosome your VCF has no record on is not analysed; in a split VCF an - SNV keeps its call beside an insertion, a multi-allelic record or the - 0/0 half of a split site, in either order, a usable record of the same - SNV wins over a filtered one, and two carried SNVs at one position are - unknown (9); - 3. report: Altai and Vindija segments merge into Neanderthal ones, a - Denisova segment that overlaps one is not counted as Denisovan, segments - under 50 kb or LOD 4 are dropped, the share is of the analysed span, and - the genes of the longest segments are named; og summary collect - reads the summary and stops on a damaged one, and bin/render_report.py - prints both shares; - 4. with IBDmix itself (IBDMIX_IMAGE, docker): a planted 600 kb stretch where - the synthetic person carries the Neanderthal haplotype comes out as one - Neanderthal segment over it, and a planted Denisovan one as Denisovan; - the same person without the planted stretches gets no segment at all; - where the person contradicts both Neanderthals inside the planted - stretch and their calls there are masked, the segment stops at the - masked mismatches, while the same calls written as unknown (9, the - table before the fix) let it run through; a stretch where the person - has no call matches archaic genomes that carry the common reference - allele, and gives a segment only when a gVCF reference block says the - stretch was read (the step before that fix read it as the reference - either way). - Set ARCHAIC_TEST_NO_DOCKER=1 to skip part 4 on a machine without docker; - CI never sets it. - -The Rust unit tests (rust/crates/og-ancestry) check pick_call's symmetry and -which genes touch a segment's ends, and tests/parity checks each command -against the Python it replaced. Without an og binary the test is skipped with -a note; with OG_REQUIRED set the skip is a failure. - -Run: OG=bin/og python3 tests/test_archaic_report.py -""" -import gzip -import os -import random -import shutil -import subprocess -import sys -import tempfile - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) -OG = os.environ.get("OG", "") -FAILS = 0 -TABLE_HEADER = ["chrom", "pos", "ref", "alt", "Altai", "Vindija", "Denisova", - "AF_ALL", "AF_AFR", "AF_AMR", "AF_EAS", "AF_EUR", "AF_SAS"] -SEGMENT_HEADER = ["chrom", "start", "end", "length_bp", "ancestry", "matched", "lod", "genes"] -MASKED = 10 -ARCHAICS = ("Altai", "Vindija", "Denisova") - -if not OG: - print("skip: OG is not set (OG=bin/og)") - sys.exit(1 if os.environ.get("OG_REQUIRED") else 0) - - -def check(desc, ok, detail=""): - global FAILS - print(f"[{'PASS' if ok else 'FAIL'}] {desc}{'' if ok else ' -- ' + str(detail)[:600]}") - if not ok: - FAILS += 1 - - -def write(path, text): - with open(path, "w") as f: - f.write(text) - - -def run(*args): - return subprocess.run([OG, "archaic", *args], capture_output=True, text=True) - - -def image(var): - for line in open(os.path.join(REPO, "versions.env")): - if line.startswith(f"{var}="): - return line.split("=", 1)[1].split("#")[0].strip().strip('"') - raise SystemExit(f"{var} not in versions.env") - - -TMP = tempfile.mkdtemp() - -# --- 1. table ------------------------------------------------------------------ -afs = "0.1\t0.2\t0.05\t0\t0.15\t0.1" -write(f"{TMP}/modern.tsv", - f"100\tA\tG\t{afs}\n" # every archaic genome called - f"200\tC\tT\t{afs}\n" # Vindija has another ALT, Denisova another REF - f"300\tG\tA\t{afs}\n" # no archaic record - f"400\tT\tC\t{afs}\n400\tT\tG\t{afs}\n") # a split multi-allelic site -write(f"{TMP}/altai.tsv", "100\tA\tG\t1/1\n200\tC\t.\t0/0\n500\tG\tC\t0/1\n600\tA\t.\t0/0\n") -write(f"{TMP}/vindija.tsv", "100\tA\t.\t0/0\n200\tC\tA\t0/1\n500\tG\t.\t0/0\n600\tA\t.\t0/0\n") -write(f"{TMP}/denisova.tsv", "100\tA\tG\t0|1\n200\tT\tC\t0/1\n400\tT\tC\t1/1\n500\tG\tT\t1/1\n") -r = run("table", "--chrom", "22", "--modern", f"{TMP}/modern.tsv", "--archaic", f"Altai={TMP}/altai.tsv", - "--archaic", f"Vindija={TMP}/vindija.tsv", "--archaic", f"Denisova={TMP}/denisova.tsv", "--out", f"{TMP}/t.vcf") -check("table exits 0", r.returncode == 0, r.stderr) -recs = {int(l.split("\t")[1]): l.rstrip("\n").split("\t") for l in open(f"{TMP}/t.vcf") if not l.startswith("#")} -info = {p: dict(kv.split("=") for kv in c[7].split(";")) for p, c in recs.items()} -check("rows kept: the modern sites with an archaic call and the archaic-only site", sorted(recs) == [100, 200, 500], sorted(recs)) -check("a 1000 Genomes site: each genome's ALT count", (info[100]["Altai"], info[100]["Vindija"], info[100]["Denisova"]) == ("2", "0", "1"), info.get(100)) -check("its frequencies are kept", info[100]["AF_ALL"] == "0.1" and info[100]["AF_EUR"] == "0.15", info.get(100)) -check("another ALT or another REF is unknown (9)", (info[200]["Altai"], info[200]["Vindija"], info[200]["Denisova"]) == ("0", "9", "9"), info.get(200)) -check("an archaic-only site takes the first non-reference ALT and frequency 0", - recs[500][4] == "C" and info[500]["Altai"] == "1" and info[500]["Vindija"] == "0" and info[500]["Denisova"] == "9" - and info[500]["AF_ALL"] == "0", (recs.get(500), info.get(500))) -check("chrom gets its chr prefix for the liftover", recs[100][0] == "chr22", recs[100][0]) - -# The masks: each archaic genome is read only inside its own FilterBed. The -# first run is the control: without --mask the same inputs keep every call, -# so the codes below come from the mask and nothing else. -write(f"{TMP}/altai.bed", "22\t50\t150\n22\t450\t550\n") # 1-based 51-150 and 451-550: 100 and 500, not 200 -write(f"{TMP}/vindija.bed", "22\t150\t250\n") # holds 200, not 100 -write(f"{TMP}/denisova.bed.gz", "") -with gzip.open(f"{TMP}/denisova.bed.gz", "wt") as f: - f.write("chr22\t99\t100\n") # 1-based 100 only, and named chr22 -common = ["table", "--chrom", "22", "--modern", f"{TMP}/modern.tsv", "--archaic", f"Altai={TMP}/altai.tsv", - "--archaic", f"Vindija={TMP}/vindija.tsv", "--archaic", f"Denisova={TMP}/denisova.tsv"] - - -def table_info(out, extra=()): - r = run(*common, *extra, "--out", out) - check(f"table exits 0 ({'masked' if extra else 'no mask'})", r.returncode == 0, r.stderr) - rows = {int(l.split("\t")[1]): l.rstrip("\n").split("\t") for l in open(out) if not l.startswith("#")} - return rows, {p: dict(kv.split("=") for kv in c[7].split(";")) for p, c in rows.items()} - - -nm_rows, nm = table_info(f"{TMP}/nomask.vcf") -check("control, no mask: the Altai call at 100 is read, and 200 and 500 are rows", - nm[100]["Altai"] == "2" and sorted(nm_rows) == [100, 200, 500], (sorted(nm_rows), nm.get(100))) -m_rows, mi = table_info(f"{TMP}/masked.vcf", ("--mask", f"Altai={TMP}/altai.bed", - "--mask", f"Vindija={TMP}/vindija.bed", - "--mask", f"Denisova={TMP}/denisova.bed.gz")) -check("inside its mask a genome keeps its call", mi[100]["Altai"] == "2" and mi[100]["Denisova"] == "1", mi.get(100)) -# Outside its mask a call is kept as masked (10 plus its ALT count), as Chen -# et al. keep the site in IBDmix's input and mask it there; the old table -# coded it 9 (unknown), which let IBDmix's segments run through it. -check("outside its mask a genome's call is masked (10 for 0/0), where the control read 0", - mi[100]["Vindija"] == "10" and nm[100]["Vindija"] == "0", (mi.get(100), nm.get(100))) -check("a row whose only call is masked stays, with that call masked (200: Altai 0/0 is 10)", - sorted(m_rows) == [100, 200, 500] and mi[200]["Altai"] == "10" and nm[200]["Altai"] == "0", - (sorted(m_rows), mi.get(200))) -check("an archaic-only site inside its mask keeps that genome's ALT, and the other genome's call is masked", - m_rows[500][4] == "C" and mi[500]["Altai"] == "1" and mi[500]["Vindija"] == "10", (m_rows.get(500), mi.get(500))) -check("unknown stays 9 inside or outside a mask", - mi.get(200, {}).get("Vindija") == "9" and mi.get(500, {}).get("Denisova") == "9", (mi.get(200), mi.get(500))) -check("a mask file is read with and without the chr prefix", mi[100]["Denisova"] == "1", mi.get(100)) -# The excluded regions: every call at a site inside one is masked, whatever -# the genome's own mask says; the sites outside are untouched. -write(f"{TMP}/excl.bed", "22\t90\t120\tsegdup\n") -ex_rows, ex = table_info(f"{TMP}/excl.vcf", ("--exclude", f"{TMP}/excl.bed")) -check("every call at a site inside an excluded region is masked (100: 12, 10, 11), where the control read 2, 0, 1", - (ex[100]["Altai"], ex[100]["Vindija"], ex[100]["Denisova"]) == ("12", "10", "11") - and (nm[100]["Altai"], nm[100]["Vindija"], nm[100]["Denisova"]) == ("2", "0", "1"), (ex.get(100), nm.get(100))) -check("and the sites outside it keep their calls", sorted(ex_rows) == [100, 200, 500] - and (ex[200]["Altai"], ex[500]["Altai"]) == ("0", "1"), (sorted(ex_rows), ex.get(200), ex.get(500))) -write(f"{TMP}/excl_other.bed", "7\t90\t120\tsegdup\n") -r = run(*common, "--exclude", f"{TMP}/excl_other.bed", "--out", f"{TMP}/x.vcf") -check("an exclusion list with no interval on the chromosome stops the build", - r.returncode == 1 and "no interval" in r.stderr, r.stderr) -r = run(*common, "--mask", f"Altai={TMP}/altai.bed", "--mask", f"Altai={TMP}/altai.bed", "--out", f"{TMP}/x.vcf") -check("--mask twice for one genome stops the build", r.returncode == 2 and "twice" in r.stderr, r.stderr) -r = run(*common, "--mask", f"Nobody={TMP}/altai.bed", "--out", f"{TMP}/x.vcf") -check("--mask with an unknown genome stops the build", r.returncode == 2, r.stderr) -write(f"{TMP}/other.bed", "7\t50\t150\n") -r = run(*common, "--mask", f"Altai={TMP}/other.bed", "--out", f"{TMP}/x.vcf") -check("a mask with no interval on the chromosome stops the build, instead of emptying the table", - r.returncode == 1 and "no interval" in r.stderr, r.stderr) - -# --- 1b. strict: rows outside the GRCh38 strict mask are marked, not dropped ------- -# Lifted rows (13 columns, no header), sorted by position as Picard writes -# them. Strict mask 1-based 101-300 on chr22: 100 is outside, 150 and 300 -# inside, 301 outside. A duplicate position keeps the first row. Codes 0, 1, 2 -# outside become 10, 11, 12; 9 stays 9; a code already masked is not marked -# twice. -rows = ["chr22\t100\tA\tG\t2\t0\t9\t0.5\t0\t0\t0\t0\t0", - "chr22\t150\tA\tG\t1\t1\t1\t0.5\t0\t0\t0\t0\t0", - "chr22\t300\tG\tA\t0\t0\t0\t0.5\t0\t0\t0\t0\t0", - "chr22\t301\tC\tT\t0\t10\t12\t0.5\t0\t0\t0\t0\t0", - "chr22\t301\tC\tT\t2\t2\t2\t0.5\t0\t0\t0\t0\t0"] -write(f"{TMP}/lifted.tsv", "\n".join(rows) + "\n") -write(f"{TMP}/strict.bed", "chr22\t100\t300\n") -r = run("strict", "--chrom", "22", "--strict", f"{TMP}/strict.bed", "--rows", f"{TMP}/lifted.tsv", "--out", f"{TMP}/marked.tsv") -check("strict exits 0", r.returncode == 0, r.stderr) -got = [l.rstrip("\n").split("\t") for l in open(f"{TMP}/marked.tsv")] -by = {int(c[1]): c[4:7] for c in got} -check("every position is kept, once", [int(c[1]) for c in got] == [100, 150, 300, 301], [c[1] for c in got]) -check("outside the strict mask 2, 0 and unknown become 12, 10 and 9", by[100] == ["12", "10", "9"], by.get(100)) -check("inside the strict mask the calls are untouched (last base 300 too)", by[150] == ["1", "1", "1"] and by[300] == ["0", "0", "0"], (by.get(150), by.get(300))) -check("a masked call stays as it is, and the first of a duplicate wins", by[301] == ["10", "10", "12"], by.get(301)) -write(f"{TMP}/unsorted.tsv", "\n".join([rows[0], rows[3], rows[2]]) + "\n") -r = run("strict", "--chrom", "22", "--strict", f"{TMP}/strict.bed", "--rows", f"{TMP}/unsorted.tsv", "--out", f"{TMP}/marked3.tsv") -check("rows out of position order stop the step (a duplicate is found only next to its twin)", - r.returncode == 2 and "sorted" in r.stderr and not os.path.exists(f"{TMP}/marked3.tsv"), (r.returncode, r.stderr)) -write(f"{TMP}/strict_other.bed", "chr21\t0\t10\n") -r = run("strict", "--chrom", "22", "--strict", f"{TMP}/strict_other.bed", "--rows", f"{TMP}/lifted.tsv", "--out", f"{TMP}/marked2.tsv") -check("a strict mask with no interval on the chromosome stops the step", r.returncode != 0 and "no interval" in r.stderr, r.stderr) - -# --- 2. genotypes ------------------------------------------------------------------ -tdir = f"{TMP}/table" -os.makedirs(tdir) -hdr = "\t".join(TABLE_HEADER) + "\n" -rows = ["chr22\t1000\tA\tG\t2\t2\t0\t0.5\t0.5\t0.5\t0.5\t0.5\t0.5", - "chr22\t2000\tC\tT\t0\t0\t0\t0.01\t0\t0\t0\t0.02\t0", - "chr22\t3000\tG\tA\t1\t1\t2\t0\t0\t0\t0\t0\t0", - "chr22\t4000\tT\tC\t2\t9\t0\t0.2\t0\t0\t0\t0\t0", - "chr22\t5000\tA\tC\t0\t0\t0\t0.2\t0\t0\t0\t0\t0"] -with gzip.open(f"{tdir}/chr22.tsv.gz", "wt") as f: - f.write(hdr + "\n".join(rows) + "\n") -with gzip.open(f"{tdir}/chr21.tsv.gz", "wt") as f: - f.write(hdr + "chr21\t1000\tA\tG\t2\t2\t0\t0.5\t0.5\t0.5\t0.5\t0.5\t0.5\n") -vcf = ("##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" - "chr22\t1000\t.\tA\tG\t50\tPASS\t.\tGT:GQ\t0/1:30\n" - "chr22\t3000\t.\tG\tA\t50\tRefCall\t.\tGT\t1/1\n" - "chr22\t5000\t.\tA\tT\t50\tPASS\t.\tGT\t1/1\n") -write(f"{TMP}/s.vcf", vcf) -GVCF_HEAD = "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" - - -def ref_block(chrom, start, end): - return f"{chrom}\t{start}\t.\tN\t<*>\t0\t.\tEND={end}\tGT:GQ:MIN_DP\t0/0:50:30\n" - - -# A reference block over 1-2500: 2000 was read as the reference; 4000 was not read. -write(f"{TMP}/s.g.vcf", GVCF_HEAD + ref_block("chr22", 1, 2500)) -r = run("genotypes", "--table-dir", tdir, "--vcf", f"{TMP}/s.vcf", "--gvcf", f"{TMP}/s.g.vcf", - "--pop", "EUR", "--pseudo", "10", "--out-dir", f"{TMP}/gt") -check("genotypes exits 0", r.returncode == 0, r.stderr) -lines = gzip.open(f"{TMP}/gt/chr22.gt.gz", "rt").read().splitlines() -g = {int(l.split("\t")[1]): l.split("\t") for l in lines[1:]} -check("header: four columns, three archaic, SAMPLE, ten stand-ins", - lines[0].split("\t")[:8] == ["chrom", "pos", "ref", "alt", "Altai", "Vindija", "Denisova", "SAMPLE"] - and len(lines[0].split("\t")) == 18, lines[0]) -check("the chromosome is an integer for IBDmix", g[1000][0] == "22", g[1000][0]) -check("your heterozygous call is 1", g[1000][7] == "1", g[1000]) -check("no record inside a gVCF reference block is 0 (reference)", g[2000][7] == "0", g[2000]) -check("no record outside every reference block is 9 (unknown)", g[4000][7] == "9", g[4000]) -check("a non-PASS record is 9", g[3000][7] == "9", g[3000]) -check("a site where you carry another ALT is left out", 5000 not in g, sorted(g)) -check("stand-ins carry EUR's ALT count (0.5 of 20 alleles = 10)", sum(int(x) for x in g[1000][8:]) == 10, g[1000][8:]) -check("and 0.02 of 20 rounds to 0", sum(int(x) for x in g[2000][8:]) == 0, g[2000][8:]) -check("each stand-in column is one character", all(len(x) == 1 for x in g[1000][4:]), g[1000]) -check("a chromosome the VCF has no record on is not analysed", not os.path.exists(f"{TMP}/gt/chr21.gt.gz")) -spans = open(f"{TMP}/gt/spans.tsv").read().splitlines() -check("spans: the sites read, chr22 from 1000 to 2000, two sites", spans[1].split("\t")[:4] == ["chr22", "1000", "2000", "2"], spans) -samples = open(f"{TMP}/gt/samples.txt").read().split() -check("samples.txt: SAMPLE and the stand-ins", samples[0] == "SAMPLE" and len(samples) == 11, samples) -write(f"{TMP}/two.vcf", "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tA\tB\n") -r = run("genotypes", "--table-dir", tdir, "--vcf", f"{TMP}/two.vcf", "--out-dir", f"{TMP}/gt2") -check("a two-sample VCF stops the step", r.returncode != 0 and "one-sample" in r.stderr, r.stderr) -# One position twice, as bcftools norm -m- writes it. Rows: REF>ALT of the table. -sdir2 = f"{TMP}/table_split" -os.makedirs(sdir2) -split_rows = {1000: "A\tG", 2000: "C\tT", 3000: "G\tA", 4000: "T\tC", 5000: "A\tG", 6000: "C\tT", - 7000: "G\tA", 8000: "T\tC", 9000: "A\tC"} -with gzip.open(f"{sdir2}/chr22.tsv.gz", "wt") as f: - f.write(hdr + "".join(f"chr22\t{p}\t{ra}\t0\t0\t0\t0.2\t0\t0\t0\t0\t0\n" for p, ra in split_rows.items())) -write(f"{TMP}/split.vcf", "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" - "chr22\t1000\t.\tA\tAT\t50\tPASS\t.\tGT\t0/1\n" # insertion, then the SNV - "chr22\t1000\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\n" - "chr22\t2000\t.\tC\tT\t50\tPASS\t.\tGT\t1/1\n" # the SNV, then an insertion - "chr22\t2000\t.\tC\tCA\t50\tPASS\t.\tGT\t0/1\n" - "chr22\t3000\t.\tG\tA,GT\t50\tPASS\t.\tGT\t1/2\n" # a multi-allelic record, then the SNV - "chr22\t3000\t.\tG\tA\t50\tPASS\t.\tGT\t0/1\n" - "chr22\t4000\t.\tT\tC\t50\tPASS\t.\tGT\t0/1\n" # two SNVs, both carried - "chr22\t4000\t.\tT\tG\t50\tPASS\t.\tGT\t0/1\n" - "chr22\t4000\t.\tT\tC\t50\tPASS\t.\tGT\t0/1\n" - "chr22\t5000\t.\tA\tT\t50\tPASS\t.\tGT\t0/0\n" # a split 0/1 at A>G,T: 0/0, then 0/1 - "chr22\t5000\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\n" - "chr22\t6000\t.\tC\tT\t50\tPASS\t.\tGT\t0/1\n" # 0/1, then the 0/0 half - "chr22\t6000\t.\tC\tG\t50\tPASS\t.\tGT\t0/0\n" - "chr22\t7000\t.\tG\tA\t50\tLowQual\t.\tGT\t0/1\n" # the same SNV, filtered, then usable - "chr22\t7000\t.\tG\tA\t50\tPASS\t.\tGT\t1/1\n" - "chr22\t8000\t.\tT\tA\t50\tPASS\t.\tGT\t0/0\n" # two 0/0 halves: the reference - "chr22\t8000\t.\tT\tG\t50\tPASS\t.\tGT\t0/0\n") -r = run("genotypes", "--table-dir", sdir2, "--vcf", f"{TMP}/split.vcf", "--pseudo", "10", "--out-dir", f"{TMP}/gt3") -check("genotypes on a split VCF exits 0", r.returncode == 0, r.stderr) -g3 = {int(l.split("\t")[1]): l.split("\t") for l in gzip.open(f"{TMP}/gt3/chr22.gt.gz", "rt").read().splitlines()[1:]} -code3 = lambda p: g3.get(p, ["left out"] * 8)[7] -check("an insertion before the SNV at one position keeps the SNV call", code3(1000) == "1", g3.get(1000)) -check("an insertion after the SNV at one position keeps the SNV call", code3(2000) == "2", g3.get(2000)) -check("a multi-allelic record beside the SNV keeps the SNV call", code3(3000) == "1", g3.get(3000)) -check("two carried SNVs at one position are unknown (9), also when a third record follows", code3(4000) == "9", g3.get(4000)) -check("a 0/0 half of a split site before the carried SNV keeps the SNV call", code3(5000) == "1", g3.get(5000)) -check("a 0/0 half after the carried SNV keeps the SNV call", code3(6000) == "1", g3.get(6000)) -check("a usable record of the same SNV wins over a filtered one", code3(7000) == "2", g3.get(7000)) -check("two 0/0 halves are the reference (0)", code3(8000) == "0", g3.get(8000)) - -# Masked calls: IBDmix gets the call itself and a mask per genome of the -# positions where it is masked, the runs of written rows closed by the next -# unmasked one. Codes per row: Altai, Vindija, Denisova. -mdir = f"{TMP}/table_masked" -os.makedirs(mdir) -mrows = {100: ("A\tG", "12\t0\t9"), 200: ("C\tT", "10\t11\t0"), 300: ("G\tA", "2\t0\t0"), - 400: ("T\tC", "10\t0\t0"), 500: ("A\tC", "0\t0\t0"), 600: ("C\tT", "11\t0\t0")} -with gzip.open(f"{mdir}/chr22.tsv.gz", "wt") as f: - f.write(hdr + "".join(f"chr22\t{p}\t{ra}\t{c}\t0.2\t0\t0\t0\t0\t0\n" for p, (ra, c) in mrows.items())) -write(f"{TMP}/m.vcf", "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" - "chr22\t500\t.\tA\tT\t50\tPASS\t.\tGT\t0/1\n") # another ALT: row 500 is left out -write(f"{TMP}/m.g.vcf", GVCF_HEAD + ref_block("chr22", 1, 1000)) -r = run("genotypes", "--table-dir", mdir, "--vcf", f"{TMP}/m.vcf", "--gvcf", f"{TMP}/m.g.vcf", "--pseudo", "10", - "--out-dir", f"{TMP}/gtm") -check("genotypes on a table with masked calls exits 0", r.returncode == 0, r.stderr) -gm = {int(l.split("\t")[1]): l.split("\t")[4:7] for l in gzip.open(f"{TMP}/gtm/chr22.gt.gz", "rt").read().splitlines()[1:]} -check("IBDmix gets each masked call as its ALT count (12, 10, 11 are 2, 0, 1)", - gm.get(100) == ["2", "0", "9"] and gm.get(200) == ["0", "1", "0"] and gm.get(600) == ["1", "0", "0"], gm) -masks = {g: open(f"{TMP}/gtm/chr22.{g}.mask.bed").read() if os.path.exists(f"{TMP}/gtm/chr22.{g}.mask.bed") else None - for g in ARCHAICS} -check("Altai's mask: 100-200, then 400-600 (the left-out row 500 does not close it)", - masks["Altai"] == "22\t99\t200\n22\t399\t600\n", masks["Altai"]) -check("Vindija's mask: 200 alone", masks["Vindija"] == "22\t199\t200\n", masks["Vindija"]) -check("Denisova's mask is written and empty", masks["Denisova"] == "", masks["Denisova"]) -check("an unmasked table writes empty masks", open(f"{TMP}/gt/chr22.Altai.mask.bed").read() == "") -bdir = f"{TMP}/table_bad" -os.makedirs(bdir) -with gzip.open(f"{bdir}/chr22.tsv.gz", "wt") as f: - f.write(hdr + "chr22\t100\tA\tG\t7\t0\t0\t0.2\t0\t0\t0\t0\t0\n") -r = run("genotypes", "--table-dir", bdir, "--vcf", f"{TMP}/m.vcf", "--out-dir", f"{TMP}/gtb") -check("a code the table never holds (7) stops the step", r.returncode == 1 and "code '7'" in r.stderr, r.stderr) - -# A streaming join must refuse backwards positions, including records past -# the table's end and records whose genotype it would otherwise ignore. -vcf_header = "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" -for label, records in ( - ("within-table", [(3000, "G", "A", "GT"), (1000, "A", "G", "GT")]), - ("past-table", [(9000, "A", "G", "GT"), (8000, "A", "G", "GT")]), - ("ignored-call", [(3000, "G", "A", "GT"), (1000, "A", ".", "GQ")])): - bad_vcf = f"{TMP}/{label}.vcf" - write(bad_vcf, vcf_header + "".join( - f"chr22\t{p}\t.\t{ref}\t{alt}\t.\tPASS\t.\t{fmt}\t0/1\n" - for p, ref, alt, fmt in records)) - out = f"{TMP}/order-{label}" - os.makedirs(out) - protected = ["chr22.gt.gz", *(f"chr22.{g}.mask.bed" for g in ARCHAICS)] - for name in protected: - write(f"{out}/{name}", "previous output\n") - r = run("genotypes", "--table-dir", tdir, "--vcf", bad_vcf, "--out-dir", out) - check(f"unsorted VCF {label} exits 2", r.returncode == 2 and "not sorted" in r.stderr, r.stderr) - check(f"unsorted VCF {label} keeps prior genotype and masks", - all(open(f"{out}/{name}", "rb").read() == b"previous output\n" for name in protected)) - -unordered_table = f"{TMP}/table_unordered" -os.makedirs(unordered_table) -with gzip.open(f"{unordered_table}/chr22.tsv.gz", "wt") as f: - f.write(hdr + "\n".join([rows[1], rows[0], *rows[2:]]) + "\n") -r = run("genotypes", "--table-dir", unordered_table, "--vcf", f"{TMP}/s.vcf", - "--out-dir", f"{TMP}/gt_unordered") -check("an unsorted table exits 2", r.returncode == 2 and "not sorted" in r.stderr, r.stderr) -check("an unsorted table leaves no partial genotype or mask", - not any(name.endswith((".gt.gz", ".mask.bed")) for name in os.listdir(f"{TMP}/gt_unordered"))) - -# --- 3. report --------------------------------------------------------------------- -idir = f"{TMP}/ibd" -os.makedirs(idir) -head = "ID\tchrom\tstart\tend\tslod\n" -write(f"{idir}/chr22.Altai.txt", head + "SAMPLE\t22\t1000000\t1300000\t40\nP001\t22\t5000000\t9000000\t99\n" - "SAMPLE\t22\t2000000\t2040000\t30\nSAMPLE\t22\t3000000\t3100000\t3.5\n") -write(f"{idir}/chr22.Vindija.txt", head + "SAMPLE\t22\t1200000\t1500000\t50\n") -write(f"{idir}/chr22.Denisova.txt", head + "SAMPLE\t22\t1400000\t1450000\t12\nSAMPLE\t22\t6000000\t6200000\t20\n") -write(f"{TMP}/spans.tsv", "chrom\tfirst\tlast\tsites\nchr22\t1\t10000000\t5000\n") -write(f"{TMP}/genes.gtf", - 'chr22\tHAVANA\tgene\t1100000\t1110000\t.\t+\t.\tgene_id "G1"; gene_type "protein_coding"; gene_name "GENEA";\n' - 'chr22\tHAVANA\tgene\t1120000\t1130000\t.\t+\t.\tgene_id "G2"; gene_type "lncRNA"; gene_name "LNC1";\n' - 'chr22\tHAVANA\tgene\t6100000\t6300000\t.\t-\t.\tgene_id "G3"; gene_type "protein_coding"; gene_name "GENEB";\n') -r = run("report", "--sample", "S1", "--ibdmix-dir", idir, "--spans", f"{TMP}/spans.tsv", "--genes", f"{TMP}/genes.gtf", - "--out-dir", f"{TMP}/out") -check("report exits 0", r.returncode == 0, r.stderr) -segs = [l.split("\t") for l in open(f"{TMP}/out/S1_archaic_segments.tsv").read().splitlines()] -check("segments header", segs[0] == SEGMENT_HEADER, segs[0]) -body = segs[1:] -check("two segments: the merged Neanderthal one and the Denisovan one", len(body) == 2, body) -check("Altai 1.0-1.3 and Vindija 1.2-1.5 Mb merge into Neanderthal 1.0-1.5 Mb", - body and body[0][:6] == ["chr22", "1000000", "1500000", "500000", "Neanderthal", "Altai,Vindija,Denisova"], body[:1]) -check("the merged segment keeps the higher LOD", body and body[0][6] == "50.0", body[:1]) -check("a Denisova stretch inside a Neanderthal one is not Denisovan; one apart is", len(body) > 1 and body[1][:5] == ["chr22", "6000000", "6200000", "200000", "Denisovan"], body) -check("protein-coding genes only", body and body[0][7] == "GENEA", body[:1]) -check("a stand-in's segment is not yours, nor one under 50 kb or LOD 4", all(r[1] not in ("5000000", "2000000", "3000000") for r in body), body) -summ = dict(l.split("\t", 1) for l in open(f"{TMP}/out/S1_archaic.tsv").read().splitlines()[1:]) -check("Neanderthal share of the 10 Mb span: 5.00%", summ.get("neanderthal_percent") == "5.00", summ) -check("Denisovan share: 2.00%", summ.get("denisovan_percent") == "2.00", summ) -check("segment counts", summ.get("neanderthal_segments") == "1" and summ.get("denisovan_segments") == "1", summ) -check("the longest segment is listed first with its genes", summ.get("longest_1", "").startswith("chr22:1000000-1500000 Neanderthal") - and "GENEA" in summ.get("longest_1", ""), summ.get("longest_1")) -txt = open(f"{TMP}/out/S1_archaic.txt").read() -check("the text names the caveat", "An estimate, not a measurement." in txt, txt[-300:]) -r = run("report", "--sample", "S1", "--ibdmix-dir", f"{TMP}/nothing", "--spans", f"{TMP}/spans.tsv", "--out-dir", f"{TMP}/out2") -check("a missing IBDmix output stops the report", r.returncode != 0 and "no IBDmix output" in r.stderr, r.stderr) - -# --- 3b. the reports read it ------------------------------------------------------ -import collect_summary as cs # noqa: E402 -import render_report as rr # noqa: E402 - - -def archaic(d): - """The archaic section og summary collect writes for folder D.""" - return cs.collect("S1", d)["sections"]["archaic"] - - -sdir = f"{TMP}/sample" -os.makedirs(f"{sdir}/archaic") -shutil.copy(f"{TMP}/out/S1_archaic.tsv", f"{sdir}/archaic/S1_archaic.tsv") -sec = archaic(sdir) -data = sec["data"] -check("og summary collect reads the summary", sec["state"] == "ok" and data.get("neanderthal_percent") == "5.00" - and data.get("denisovan_segments") == 1 and len(data.get("longest", [])) == 2, sec) -check("the archaic section is in SECTIONS as step 43", any(x[0] == "archaic" and x[2] == "43" for x in cs.SECTIONS) - and sec["step"] == "43") -lines = rr.archaic_lines(data) -check("the report lines give both shares", lines[0].startswith("Neanderthal: 5.00% (0.5 Mb in 1 segments)") - and lines[1].startswith("Denisovan: 2.00%"), lines) -write(f"{sdir}/archaic/S1_archaic.tsv", "key\tvalue\nneanderthal_percent\tlots\n") -check("a damaged summary is unreadable, not a share", archaic(sdir)["state"] == "unreadable", archaic(sdir)) -write(f"{sdir}/archaic/S1_archaic.tsv", "key\tvalue\nneanderthal_segments\n") -check("a row cut after its key is unreadable, not a crash", archaic(sdir)["state"] == "unreadable", archaic(sdir)) -nowhere = archaic(f"{TMP}/nowhere") -check("no file: the section is missing", (nowhere["state"], nowhere["data"]) == ("missing", {}), nowhere) - -# --- 4. the real IBDmix ------------------------------------------------------------- -NEA = (2000000, 2600000) -DEN = (3200000, 3600000) - - -MID = (2200000, 2400000) - - -def synthetic(d, plant, mid=None): - """A 4 Mb chr22 table and one person. A third of the sites carry an allele - the Neanderthals share and modern people rarely do, a third one only the - Denisovan carries, a third are common. The person draws both copies from - the frequencies, except inside the planted stretches, where one copy - carries the archaic allele. - - mid: the middle of the planted Neanderthal stretch (MID) holds sites the - person contradicts (0/0 where both Neanderthals are 1/1), and the two - Neanderthals' calls there are outside their masks. "masked" writes them - as this table does (10 plus the call); "unknown" as the table before the - fix did (9).""" - rnd = random.Random(11) - os.makedirs(f"{d}/table") - rows, calls = [], [] - for i, pos in enumerate(range(1000000, 5000000, 1000)): - kind = i % 3 - if kind == 0: - f, al, vi, de = rnd.uniform(0.01, 0.04), 2, 2, 0 - elif kind == 1: - f, al, vi, de = rnd.uniform(0.01, 0.04), 0, 0, 2 - else: - f = rnd.uniform(0.1, 0.9) - al = vi = de = sum(rnd.random() < f for _ in range(2)) - ta, tv = al, vi # the codes the table writes - g = [rnd.random() < f, rnd.random() < f] - if plant and NEA[0] <= pos < NEA[1]: - g[0] = al == 2 - if mid and MID[0] <= pos < MID[1]: - if kind == 0: - g = [False, False] - ta, tv = (al + MASKED, vi + MASKED) if mid == "masked" else (9, 9) - rows.append(f"chr22\t{pos}\tA\tG\t{ta}\t{tv}\t{de}\t" + "\t".join([f"{f:.4f}"] * 6)) - if plant and DEN[0] <= pos < DEN[1]: - g[0] = de == 2 - if any(g): - calls.append(f"chr22\t{pos}\t.\tA\tG\t50\tPASS\t.\tGT\t{'1/1' if all(g) else '0/1'}") - with gzip.open(f"{d}/table/chr22.tsv.gz", "wt") as f: - f.write("\t".join(TABLE_HEADER) + "\n" + "\n".join(rows) + "\n") - write(f"{d}/s.vcf", "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" + "\n".join(calls) + "\n") - - -def ibdmix_run(d, gvcf=None): - if gvcf is None: - gvcf = GVCF_HEAD + ref_block("chr22", 1, 6000000) # every site read - write(f"{d}/s.g.vcf", gvcf) - r = run("genotypes", "--table-dir", f"{d}/table", "--vcf", f"{d}/s.vcf", "--gvcf", f"{d}/s.g.vcf", - "--out-dir", f"{d}/gt") - if r.returncode: - return r - os.makedirs(f"{d}/ibd") - # The flags of scripts/43-archaic.sh. - sh = ("set -e; gzip -dc /w/gt/chr22.gt.gz > /w/gt/chr22.gt; for a in Altai Vindija Denisova; do " - "ibdmix -g /w/gt/chr22.gt -s /w/gt/samples.txt -n $a -r /w/gt/chr22.$a.mask.bed -o /w/ibd/chr22.$a.txt " - "-d 4 -m 1 -a 0.01 -e 0.002 -c 2; done") - r = subprocess.run(["docker", "run", "--rm", "--network", "none", "-u", f"{os.getuid()}:{os.getgid()}", - "-v", f"{d}:/w", image("IBDMIX_IMAGE"), "sh", "-c", sh], capture_output=True, text=True) - if r.returncode: - return r - return run("report", "--sample", "S", "--ibdmix-dir", f"{d}/ibd", "--spans", f"{d}/gt/spans.tsv", "--out-dir", f"{d}/out") - - -if os.environ.get("ARCHAIC_TEST_NO_DOCKER") == "1": - print("[SKIP] part 4: ARCHAIC_TEST_NO_DOCKER=1") -else: - for plant in (True, False): - d = f"{TMP}/real_{plant}" - os.makedirs(d) - os.chmod(d, 0o777) - synthetic(d, plant) - r = ibdmix_run(d) - check(f"IBDmix chain exits 0 ({'planted' if plant else 'control'})", r.returncode == 0, r.stderr + r.stdout) - if r.returncode: - continue - body = [l.split("\t") for l in open(f"{d}/out/S_archaic_segments.tsv").read().splitlines()[1:]] - print(f" segments ({'planted' if plant else 'control'}): {[(b[1], b[2], b[4]) for b in body]}") - if plant: - nea = [b for b in body if b[4] == "Neanderthal"] - den = [b for b in body if b[4] == "Denisovan"] - cover = lambda segs, lo, hi: sum(max(0, min(int(b[2]), hi) - max(int(b[1]), lo)) for b in segs) - check("the planted Neanderthal stretch: one Neanderthal segment covering 90% of it", - len(nea) == 1 and cover(nea, *NEA) >= 0.9 * (NEA[1] - NEA[0]), nea) - check("and no more than 100 kb beyond it", nea and int(nea[0][3]) <= NEA[1] - NEA[0] + 100000, nea) - check("the planted Denisovan stretch: one Denisovan segment covering 90% of it", - len(den) == 1 and cover(den, *DEN) >= 0.9 * (DEN[1] - DEN[0]), den) - else: - check("without the planted stretches: no segment", body == [], body) - - # The masked middle. Chen et al. keep a masked site in IBDmix's input and - # mask it there, so a homozygous mismatch still counts against a segment; - # the table before the fix wrote such calls as unknown (9), and a segment - # ran through them unopposed. "unknown" is that control: the planted - # segment crosses the contradicted middle. "masked" is this table: it - # must not. - cover_mid = {} - for mid in ("unknown", "masked"): - d = f"{TMP}/real_mid_{mid}" - os.makedirs(d) - os.chmod(d, 0o777) - synthetic(d, True, mid) - r = ibdmix_run(d) - check(f"IBDmix chain exits 0 (middle {mid})", r.returncode == 0, r.stderr + r.stdout) - if r.returncode: - continue - body = [l.split("\t") for l in open(f"{d}/out/S_archaic_segments.tsv").read().splitlines()[1:]] - print(f" segments (middle {mid}): {[(b[1], b[2], b[4]) for b in body]}") - cover_mid[mid] = sum(max(0, min(int(b[2]), MID[1]) - max(int(b[1]), MID[0])) - for b in body if b[4] == "Neanderthal") - span = MID[1] - MID[0] - check("control, calls written as unknown (9): the Neanderthal segment runs through the contradicted middle", - cover_mid.get("unknown", 0) >= 0.9 * span, cover_mid) - check("calls written as masked: IBDmix's mask keeps the mismatches, and no segment covers the middle", - "masked" in cover_mid and cover_mid["masked"] <= 0.1 * span, cover_mid) - - # A stretch with no call (lin-01 of the 1.0.0 review). 150 sites where - # all three archaic genomes carry the reference allele and modern people - # carry the ALT at 0.9, and a person with one call far away: read as the - # reference, the stretch matches the archaic genomes. Control: with a - # reference block over it (the person was read and matches the - # reference) IBDmix calls a segment there. With no block over it, - # nothing was read, and no segment may come out. - gap_lo, gap_hi = 200000, 349000 - gap_segment = {} - for covered in (True, False): - d = f"{TMP}/real_gap_{covered}" - os.makedirs(f"{d}/table") - os.chmod(d, 0o777) - with gzip.open(f"{d}/table/chr22.tsv.gz", "wt") as f: - f.write("\t".join(TABLE_HEADER) + "\n" + "".join( - f"chr22\t{p}\tC\tT\t0\t0\t0\t" + "\t".join(["0.9"] * 6) + "\n" - for p in range(gap_lo, gap_hi + 1, 1000))) - write(f"{d}/s.vcf", GVCF_HEAD + "chr22\t10000000\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\n") - # Without the block the person has one read site left, at the start. - block = ref_block("chr22", 1, 400000) if covered else ref_block("chr22", 1, gap_lo) - r = ibdmix_run(d, GVCF_HEAD + block) - check(f"IBDmix chain exits 0 (stretch {'read' if covered else 'with no call'})", r.returncode == 0, - r.stderr + r.stdout) - if r.returncode: - continue - body = [l.split("\t") for l in open(f"{d}/out/S_archaic_segments.tsv").read().splitlines()[1:]] - print(f" segments (stretch {'read' if covered else 'with no call'}): {[(b[1], b[2], b[4]) for b in body]}") - gap_segment[covered] = sum(int(b[3]) for b in body if b[4] == "Neanderthal") - check("control: a stretch read as the reference gives a Neanderthal segment of 100 kb or more", - gap_segment.get(True, 0) >= 100000, gap_segment) - check("a stretch with no call and no reference block gives no segment (it scored 149 kb when read as the reference)", - gap_segment.get(False, 1) == 0, gap_segment) - -shutil.rmtree(TMP, ignore_errors=True) -print(f"{FAILS} check(s) failed" if FAILS else "all checks passed") -sys.exit(1 if FAILS else 0) diff --git a/tests/test_haplogroup_explore.py b/tests/test_haplogroup_explore.py deleted file mode 100755 index 635392884..000000000 --- a/tests/test_haplogroup_explore.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python3 -"""Both reports point to where a haplogroup can be explored further (steps 12 and 37). - -Each case writes the haplogroup files of one made-up sample S into a bash -output folder, collects the summary and renders both reports: - - 1. a male sample (Yleaf table with R-M269, haplogrep3 with H5a7): the Y and - the mtDNA links are built from those names, in the text and the HTML - report, with the note that nothing is uploaded - 2. Yleaf's paragroup form R-M269*(xR-L23): the links use R-M269 - 3. a female sample (no Yleaf table): no Y links; the mtDNA links stay - 4. no call (Hg NA, haplogroup '.'): no links at all - 5. a branch name with characters a URL cannot carry raw is escaped - -The branch names are the fixture's and public examples, not anyone's result. - -Run: python3 tests/test_haplogroup_explore.py -""" -import os -import sys -import tempfile - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) -import collect_summary # noqa: E402 -import render_report # noqa: E402 - -FAILS = [] -Y_HEAD = "Sample_name\tHg\tHg_marker\tTotal_reads\tValid_markers\tQC-score\tQC-1\tQC-2\tQC-3\n" - - -def check(desc, ok, detail=""): - print(f"{'PASS' if ok else 'FAIL'} {desc}" + ("" if ok else f": {detail}")) - if not ok: - FAILS.append(desc) - - -def put(path, text): - os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, "w") as f: - f.write(text) - - -def render(d): - summ = collect_summary.collect("S", d) - return summ, render_report.text_report(summ), render_report.html_report(summ) - - -def sample(tmp, name, y_hg=None, mt_hg="H5a7"): - d = os.path.join(tmp, name) - put(f"{d}/mito/S_haplogroup.txt", f'"SampleID"\t"Haplogroup"\t"Rank"\n"S"\t"{mt_hg}"\t"0.98"\n') - if y_hg is not None: - put(f"{d}/y_haplogroup/S_y_haplogroup.txt", Y_HEAD + f"S_sorted\t{y_hg}\tM269\t100\t42\t0.97\t1\t1\t0.97\n") - return d - - -def main(): - with tempfile.TemporaryDirectory() as tmp: - # 1. male - summ, txt, html = render(sample(tmp, "male", y_hg="R-M269")) - for url in ("https://discover.familytreedna.com/y-dna/R-M269", "https://www.yfull.com/tree/R-M269/", - "https://discover.familytreedna.com/mtdna/H5a7", "https://www.yfull.com/mtree/H5a7/"): - check(f"male: {url} in the text report", url in txt, txt) - check(f"male: {url} as a link in the HTML report", f'
    ' in html, "") - check("male: the docs section for the routes is linked", - "docs/37-y-haplogroup.md#explore-your-branch-further" in txt - and "docs/12-mito-haplogroup.md#explore-your-branch-further" in txt, txt) - check("male: both reports say the pipeline uploads nothing", - "Nothing is uploaded by this pipeline" in txt and "Nothing is uploaded by this pipeline" in html, txt) - - # 2. paragroup - summ, txt, html = render(sample(tmp, "para", y_hg="R-M269*(xR-L23)")) - check("paragroup: the link is built from R-M269", - "https://www.yfull.com/tree/R-M269/" in txt and "y-dna/R-M269*" not in txt and "%2A" not in txt, txt) - - # 3. female: no Yleaf table, so no Y section and no Y link - summ, txt, html = render(sample(tmp, "female")) - check("female: no Y link", "y-dna/" not in txt and "yfull.com/tree/" not in txt - and "y-dna/" not in html and "yfull.com/tree/" not in html, txt) - check("female: the mtDNA links stay", "https://discover.familytreedna.com/mtdna/H5a7" in txt - and "https://www.yfull.com/mtree/H5a7/" in html, txt) - - # 4. no call - summ, txt, html = render(sample(tmp, "nocall", y_hg="NA", mt_hg="")) - check("no call: no explore links", "discover.familytreedna.com" not in txt and "yfull.com" not in txt - and "discover.familytreedna.com" not in html, txt) - check("no call: the Y section still says insufficient markers", "insufficient markers" in txt, txt) - - # 5. escaping (a made-up PhyloTree-style name) - summ, txt, html = render(sample(tmp, "esc", mt_hg="L3e2b+152")) - check("escaped: '+' is encoded in the mtDNA links", "mtdna/L3e2b%2B152" in txt and "mtree/L3e2b%2B152/" in txt, txt) - - print(f"\n{len(FAILS)} failed" if FAILS else "\nall passed") - return 1 if FAILS else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/test_ystr_report.py b/tests/test_ystr_report.py deleted file mode 100644 index d8608e342..000000000 --- a/tests/test_ystr_report.py +++ /dev/null @@ -1,360 +0,0 @@ -#!/usr/bin/env python3 -# og-leg: OG_REQUIRED -"""og y str-bed and og y str-report: Y-STR repeat counts from HipSTR's calls (step 45). - -Each case writes a synthetic HipSTR VCF (one sample, only the records it -plants), runs og y str-report on assets/ystr_panel.tsv, and checks the table: - - 1. the panel: 111 markers in FamilyTreeDNA's order (the Y-12, Y-25 and - Y-37 heads checked by name), every multi-copy marker kind multicopy, - the BED holds every region of the single and parts markers once, and - one region over the whole locus for each span marker - 2. counts: reference copies plus GB / motif length; a partial repeat as - N.r; a negative GB; the sum of a marker's parts (DYS449); a span - marker's parts' copies plus its one GB (DYS510, DYS389II at 250 bp); - an offset from a panel that gives one - 3. confidence: high; low for a posterior under 0.9, for over 15 % of - reads with stutter or with a flank indel, and for under 10 reads; no - call for a region without a record or without a genotype, for two - lengths on the haploid Y, and for a marker whose other part is missing; - a region without a record carries the reason HipSTR's log gives for it - 4. not callable: multi-copy markers (DYS464, DYF395S1), markers the - marker set has no region for (DYS439), and a region longer than the - reads can span (DYS452 at 150 bp reads, callable at 250 bp; the spans - of DYS389II and DYS448) - 5. the CSV: one row per marker in the panel's order, the count and the - confidence - 6. a damaged input: a VCF with two samples, a panel out of order - 7. the reports: bin/collect_summary.py reads the table (counts per - confidence) and refuses one with an unknown confidence; the HTML card of - bin/render_report.py lists the called markers only, with the note, and - the HTML report shows an unreadable table as Unreadable - -Without OG (the og binary) the test is skipped with a note; with -OG_REQUIRED set too, it fails (Guard's rust job sets both). - -Run: OG=bin/og python3 tests/test_ystr_report.py -""" -import atexit -import csv -import gzip -import os -import shutil -import subprocess -import sys -import tempfile - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -OG = os.environ.get("OG", "") -PANEL = os.path.join(REPO, "assets", "ystr_panel.tsv") -sys.path.insert(0, os.path.join(REPO, "bin")) -if not OG: - print("skip: OG is not set (OG=bin/og)") - sys.exit(1 if os.environ.get("OG_REQUIRED") else 0) - -FAILS = [] - - -def check(name, got, want): - if got == want: - print(f"[PASS] {name}") - else: - print(f"[FAIL] {name}: got {got!r}, want {want!r}") - FAILS.append(name) - - -def read_panel(path): - """The panel's rows as dicts, each region as (chrom, start, end, period, copies, name); a span - marker has one region over its whole locus, its parts' copies rounded half up and summed.""" - with open(path, encoding="utf-8") as f: - lines = [line.rstrip("\n").split("\t") for line in f if line.strip() and not line.startswith("#")] - rows = [dict(zip(lines[0], c)) for c in lines[1:]] - for r in rows: - r["regions"] = [] if r["regions"] == "." else [ - (where.split(":")[0], int(where.split(":")[1].split("-")[0]), int(where.split(":")[1].split("-")[1]), - int(period), float(copies), name) - for where, period, copies, name in (x.split("/", 3) for x in r["regions"].split(";"))] - if r["kind"] == "span": # one region over the whole locus, named after the marker - regs = r["regions"] - r["regions"] = [(regs[0][0], regs[0][1], regs[-1][2], regs[0][3], - float(sum(int(x[4] + 0.5) for x in regs)), r["marker"])] - return rows - - -TMP = tempfile.mkdtemp(prefix="ystr-test-") -atexit.register(shutil.rmtree, TMP, ignore_errors=True) -FORMAT = "GT:GB:Q:PQ:DP:DSNP:PSNP:PDP:GLDIFF:DSTUTTER:DFLANKINDEL" - - -def vcf(records, samples=("S1",), gz=True): - """records: {HipSTR name: (GT, GB, Q, DP, DSTUTTER, DFLANKINDEL)}.""" - panel = read_panel(PANEL) - where = {n: (c, s) for r in panel for c, s, _e, _p, _k, n in r["regions"]} - lines = ["##fileformat=VCFv4.1", "##source=HipSTR", - "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t" + "\t".join(samples)] - for name, (gt, gb, q, dp, st, fl) in records.items(): - c, s = where.get(name, ("chrY", 1)) - cell = f"{gt}:{gb}:{q}:{q}:{dp}:0:0|0:{dp}|0:.:{st}:{fl}" - lines.append("\t".join([c, str(s - 1), name, "A", ".", ".", "PASS", "PERIOD=4", FORMAT] - + [cell] * len(samples))) - p = os.path.join(TMP, f"calls{len(os.listdir(TMP))}.vcf" + (".gz" if gz else "")) - data = ("\n".join(lines) + "\n").encode() - with (gzip.open(p, "wb") if gz else open(p, "wb")) as f: - f.write(data) - return p - - -def run(records=None, panel=PANEL, read_length=None, samples=("S1",), want_rc=0, log=None): - v = vcf(records or {}, samples) - tsv, out_csv = v + ".tsv", v + ".csv" - cmd = [OG, "y", "str-report", "--panel", panel, "--vcf", v, "--tsv", tsv, "--csv", out_csv] - if read_length: - cmd += ["--read-length", str(read_length)] - if log is not None: - lp = v + ".log" - with open(lp, "w") as f: - f.write(log) - cmd += ["--log", lp] - p = subprocess.run(cmd, capture_output=True, text=True) - if p.returncode != want_rc: - print(p.stdout, p.stderr) - check(f"exit code {want_rc} ({' '.join(cmd[2:3])})", p.returncode, want_rc) - if want_rc: - return p - with open(tsv, encoding="utf-8") as f: - rows = list(csv.DictReader(f, delimiter="\t")) - with open(out_csv, encoding="utf-8") as f: - crow = list(csv.reader(f)) - return {r["marker"]: r for r in rows}, rows, crow, p.stdout - - -GOOD = ("0", "0", "0.99", "20", "0", "0") - - -def good(gb="0", dp="20"): - return ("0" if gb == "0" else "1", gb, "0.99", dp, "0", "0") - - -# --- 1. the panel -------------------------------------------------------------------- -panel = read_panel(PANEL) -names = [r["marker"] for r in panel] -check("1 111 markers", len(panel), 111) -check("1 Y-12 in FamilyTreeDNA's order", names[:12], - ["DYS393", "DYS390", "DYS19", "DYS391", "DYS385a", "DYS385b", "DYS426", "DYS388", "DYS439", - "DYS389I", "DYS392", "DYS389II"]) -check("1 Y-25 (13-25)", names[12:25], - ["DYS458", "DYS459a", "DYS459b", "DYS455", "DYS454", "DYS447", "DYS437", "DYS448", "DYS449", - "DYS464a", "DYS464b", "DYS464c", "DYS464d"]) -check("1 Y-37 (26-37)", names[25:37], - ["DYS460", "Y-GATA-H4", "YCAIIa", "YCAIIb", "DYS456", "DYS607", "DYS576", "DYS570", "CDYa", "CDYb", - "DYS442", "DYS438"]) -check("1 the last marker is DYS435", names[-1], "DYS435") -check("1 panel sizes", [sum(1 for r in panel if r["panel"] == p) for p in ("Y-12", "Y-25", "Y-37", "Y-67", "Y-111")], - [12, 13, 12, 30, 44]) -multi = {r["marker"] for r in panel if r["kind"] == "multicopy"} -check("1 the multi-copy markers", multi, - {"DYS385a", "DYS385b", "DYS459a", "DYS459b", "DYS464a", "DYS464b", "DYS464c", "DYS464d", "YCAIIa", "YCAIIb", - "CDYa", "CDYb", "DYF395S1a", "DYF395S1b", "DYS413a", "DYS413b"}) -bed = os.path.join(TMP, "regions.bed") -check("1 bed exits 0", subprocess.run([OG, "y", "str-bed", "--panel", PANEL, "--out", bed]).returncode, 0) -with open(bed) as f: - bl = [line.rstrip("\n").split("\t") for line in f] -want_regions = {n for r in panel for *_x, n in r["regions"]} -check("1 the BED has every region once", sorted(x[5] for x in bl), sorted(want_regions)) -check("1 the BED has six columns, 1-based start below the end", all(len(x) == 6 and 0 < int(x[1]) < int(x[2]) for x in bl), - True) -check("1 the BED is sorted", [int(x[1]) for x in bl], sorted(int(x[1]) for x in bl)) -check("1 DYS393's region", [x for x in bl if x[5] == "DYS393"], [["chrY", "3263111", "3263158", "4", "12", "DYS393"]]) -spans = {r["marker"] for r in panel if r["kind"] == "span"} -check("1 the span markers", spans, {"DYS389II", "DYS448", "DYS450", "DYS504", "DYS510", "DYS532"}) -check("1 the BED: 86 regions (76 single, 6 spans, the 4 parts of DYS449 and DYS552)", len(bl), 86) -check("1 DYS450 is one region from its first part's start to its last part's end, 9 + 5 copies", - [x for x in bl if x[5].startswith("DYS450")], [["chrY", "8258259", "8258341", "5", "14", "DYS450"]]) -check("1 DYS510's three parts are one region, 17 + 5 + 3 copies (4.5 halves up)", - [x for x in bl if x[5].startswith("DYS510")], [["chrY", "15188021", "15188138", "4", "25", "DYS510"]]) -check("1 no part of a span marker is in the BED", [x[5] for x in bl if x[5].split(".")[0] in spans and "." in x[5]], []) -check("1 DYS449 keeps its two parts", sorted(x[5] for x in bl if x[5].startswith("DYS449")), ["DYS449.1", "DYS449.2"]) - -# --- 2. counts ------------------------------------------------------------------------- -by, rows, crow, out = run({ - "DYS393": good("4"), # one repeat longer than the reference's 12 - "DYS390": good("-8"), # two shorter than 24 - "DYS391": good("2"), # a partial repeat: 11.2 - "DYS392": good("-1"), # three-base motif, one base short: 12.2 - "DYS449.1": good("4"), "DYS449.2": good("-4"), # 15 (15.25 rounded) + 1, plus 21 - 1 - "DYS510": good("4"), # one region: 17 + 5 (4.5 halves up) + 3, plus 1 - "DYS389II": good("4"), "DYS448": good("6"), # spans longer than 150 bp reads can cover - "DYS19/DYS394": GOOD, -}) -check("2 a reference call keeps the reference copies (DYS19: 16)", by["DYS19"]["count"], "16") -check("2 GB +4 at a 4-base motif: one repeat more (DYS393 13)", by["DYS393"]["count"], "13") -check("2 GB -8: two repeats fewer (DYS390 22)", by["DYS390"]["count"], "22") -check("2 GB +2: a partial repeat (DYS391 11.2)", by["DYS391"]["count"], "11.2") -check("2 GB -1 at a 3-base motif (DYS392 12.2)", by["DYS392"]["count"], "12.2") -check("2 DYS449 is the sum of its two parts (16 + 20)", by["DYS449"]["count"], "36") -check("2 DYS510: its parts' copies (17 + 5 + 3, 4.5 halves up) plus its one GB", by["DYS510"]["count"], "26") -check("2 DYS510 reads one region: one read count, no part names in the note", - (by["DYS510"]["confidence"], by["DYS510"]["reads"], by["DYS510"]["note"]), ("high", "20", ".")) -for m, bp in (("DYS389II", 164), ("DYS448", 156)): - check(f"2 {m}: the span is {bp} bp, not callable from 150 bp reads", - (by[m]["confidence"], by[m]["count"], f"a {bp} bp repeat" in by[m]["note"]), ("not callable", "", True)) -by250 = run({"DYS389II": good("4")}, read_length=250)[0] -check("2 DYS389II at 250 bp reads: 17 + 12 copies, plus 1", by250["DYS389II"]["count"], "30") -check("2 the DYS389II note says the count covers DYS389I", "covers DYS389I" in by250["DYS389II"]["note"], True) -check("2 the summary line counts the confidences", out.strip().startswith("Y-STR markers: "), True) -off = os.path.join(TMP, "offset_panel.tsv") -with open(PANEL) as f, open(off, "w") as g: - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) == 8 and c[2] == "DYS393": - c[6] = "-1" - g.write("\t".join(c) + "\n") -oby = run({"DYS393": good("4")}, panel=off)[0] -check("2 a panel offset is added (13 - 1)", oby["DYS393"]["count"], "12") - -# --- 3. confidence --------------------------------------------------------------------- -by = run({ - "DYS393": GOOD, - "DYS390": ("0", "0", "0.6", "20", "0", "0"), # posterior below 0.9 - "DYS391": ("0", "0", "0.99", "20", "4", "0"), # 20 % stutter - "DYS426": ("0", "0", "0.99", "20", "0", "4"), # 20 % flank indels - "DYS388": ("0", "0", "0.99", "6", "0", "0"), # six reads - "DYS392": (".", ".", ".", "0", "0", "0"), # no genotype - "DYS458": ("0/1", "0/4", "0.99", "20", "0", "0"), # two lengths - "DYS449.1": GOOD, # DYS449.2 missing - "DYS456": ("0", "0", "0.99", "20", "3", "3"), # exactly 15 %: still high -})[0] -check("3 a clean call is high", (by["DYS393"]["confidence"], by["DYS393"]["reads"], by["DYS393"]["q"]), - ("high", "20", "0.99")) -check("3 posterior 0.6: low", by["DYS390"]["confidence"], "low") -check("3 and the note says why", "posterior 0.6 below 0.9" in by["DYS390"]["note"], True) -check("3 a low call keeps its count", by["DYS390"]["count"], "24") -check("3 4 of 20 reads with stutter: low", (by["DYS391"]["confidence"], by["DYS391"]["stutter_reads"]), ("low", "4")) -check("3 4 of 20 reads with a flank indel: low", by["DYS426"]["confidence"], "low") -check("3 6 reads: low", (by["DYS388"]["confidence"], "6 reads, fewer than 10" in by["DYS388"]["note"]), ("low", True)) -check("3 3 of 20 (15 %) is not above the limit: high", by["DYS456"]["confidence"], "high") -check("3 no genotype: no call, no count", (by["DYS392"]["confidence"], by["DYS392"]["count"]), ("no call", "")) -check("3 two lengths on the Y: no call", (by["DYS458"]["confidence"], "two lengths" in by["DYS458"]["note"]), - ("no call", True)) -check("3 a missing part: no call", (by["DYS449"]["confidence"], "DYS449.2" in by["DYS449"]["note"]), - ("no call", True)) -check("3 a region without a record: no call", by["DYS19"]["confidence"], "no call") -check("3 without HipSTR's log, the note points to it", "see HipSTR's log" in by["DYS19"]["note"], True) -# HipSTR's log: a "Processing region" line per region (0-based start), then why it stopped. -LOG = """Processing region chrY 9684379 9684443 -Aligning reads to each candidate haplotype -Aborting genotyping of the locus as the sequence downstream of the repeat is too repetitive for accurate genotyping -\tFlanking sequence = AAACACTATATATATATAACACTATATATATAATAC -Processing region chrY 18041621 18041651 -Skipping locus with too few reads: TOTAL=3, MIN=5 -Processing region chrY 3263110 3263158 -Locus timing: -Processing region chrY 8258258 8258341 -Skipping locus with too few informative reads for stutter training: TOTAL=2, MIN=5 -Processing region chrY 12500447 12500611 -Skipping region as the reference allele length exceeds the threshold (164 vs 150) -""" -by = run({"DYS390": GOOD}, log=LOG, read_length=250)[0] -check("3 the log's reason for DYS19 (its flank)", by["DYS19"]["note"], - "HipSTR wrote no record for DYS19/DYS394; its log: Aborting genotyping of the locus as the sequence " - "downstream of the repeat is too repetitive for accurate genotyping") -check("3 the log's reason for DYS425 (too few reads)", by["DYS425"]["note"], - "HipSTR wrote no record for DYS425; its log: Skipping locus with too few reads: TOTAL=3, MIN=5") -check("3 a span's reason, found by its one region (DYS450)", - "too few informative reads for stutter training" in by["DYS450"]["note"], True) -check("3 a span longer than --max-str-len (DYS389II at 250 bp reads)", - "exceeds the threshold (164 vs 150)" in by["DYS389II"]["note"], True) -check("3 a region the log names without a reason: the note says so", by["DYS393"]["note"], - "HipSTR wrote no record for DYS393, and its log gives no reason") -check("3 a region the log does not name: the note says so", by["DYS391"]["note"], - "HipSTR wrote no record for DYS391, and its log gives no reason") -check("3 a called marker carries no log line", by["DYS390"]["note"], ".") - -# --- 4. not callable --------------------------------------------------------------------- -by = run({"DYS452": GOOD})[0] -for m in ("DYS464a", "DYS464d", "DYF395S1a", "DYS385b", "CDYa"): - check(f"4 {m}: not callable, multi-copy", (by[m]["confidence"], "multi-copy" in by[m]["note"], by[m]["count"]), - ("not callable", True, "")) -check("4 DYS439 (no region in the marker set): not callable", (by["DYS439"]["confidence"], "no region" in by["DYS439"]["note"]), - ("not callable", True)) -check("4 DYS452 (155 bp) at 150 bp reads: not callable even with a call", - (by["DYS452"]["confidence"], by["DYS452"]["count"]), ("not callable", "")) -by = run({"DYS452": GOOD}, read_length=250)[0] -check("4 DYS452 at 250 bp reads: called", (by["DYS452"]["confidence"], by["DYS452"]["count"]), ("high", "31")) - -# --- 5. the CSV ---------------------------------------------------------------------------- -_by, rows, crow, _o = run({"DYS393": good("4")}) -check("5 CSV header", crow[0], ["marker", "value", "confidence"]) -check("5 CSV: a row per marker in the panel's order", [r[0] for r in crow[1:]], names) -check("5 CSV: DYS393's row", crow[1], ["DYS393", "13", "high"]) -check("5 CSV: a multi-copy row has no value", crow[5], ["DYS385a", "", "not callable"]) -check("5 TSV: rows in the panel's order", [r["marker"] for r in rows], names) - -# --- 6. damaged input -------------------------------------------------------------------- -p = run({"DYS393": GOOD}, samples=("S1", "S2"), want_rc=1) -check("6 two samples: the error says so", "2 samples" in p.stderr, True) -bad = os.path.join(TMP, "bad_panel.tsv") -with open(PANEL) as f, open(bad, "w") as g: - lines = f.readlines() - head = [x for x in lines if x.startswith("#") or x.startswith("order\t")] - body = [x for x in lines if x not in head] - g.writelines(head + [body[1], body[0]] + body[2:]) -p = run({}, panel=bad, want_rc=1) -check("6 a panel out of order stops", "order column" in p.stderr, True) - -# --- 7. the reports ----------------------------------------------------------------------- -import collect_summary as cs # noqa: E402 -import render_report as rr # noqa: E402 - -class Unreadable(Exception): - """A section og summary collect could not read (its state is unreadable).""" - - -def sec(d, s, key): - """(source path, data) of section KEY of the summary og summary collect writes - for folder D: (None, {}) without its file; Unreadable when it cannot be read.""" - x = cs.collect(s, d)["sections"][key] - if x["state"] == "unreadable": - raise Unreadable(x["note"]) - return (os.path.join(d, x["source"]) if x["source"] else None), x["data"] - - - -sd = os.path.join(TMP, "sample") -os.makedirs(os.path.join(sd, "y_str")) -v = vcf({"DYS393": good("4"), "DYS390": ("0", "0", "0.6", "20", "0", "0")}) -subprocess.run([OG, "y", "str-report", "--panel", PANEL, "--vcf", v, "--tsv", - os.path.join(sd, "y_str", "S1_y_str.tsv"), "--csv", os.path.join(sd, "y_str", "S1_y_str.csv")], - check=True, capture_output=True) -path, data = sec(sd, "S1", "y_str") -check("7 collect_summary finds the table", os.path.basename(path or ""), "S1_y_str.tsv") -check("7 counts per confidence", (data["high"], data["low"], data["not_callable"], data["no_call"]), - (1, 1, sum(1 for r in panel if r["kind"] in ("multicopy", "absent")) + 3, # + DYS452, DYS389II, DYS448: too long - 111 - 2 - sum(1 for r in panel if r["kind"] in ("multicopy", "absent")) - 3)) -check("7 the markers keep their order", [x["marker"] for x in data["markers"]], names) -check("7 a sample without the step has no section", sec(TMP, "nobody", "y_str"), (None, {})) -html = "\n".join(rr.ystr_body(data)) -check("7 the card lists DYS393 at 13", "DYS39313high" in html, True) -check("7 the card lists the low call too", "DYS39024low" in html, True) -check("7 and leaves out markers without a count", "DYS464a" in html, False) -check("7 the card carries the convention note", "fixed number of repeats" in html, True) -with open(os.path.join(sd, "y_str", "S1_y_str.tsv")) as f: - t = f.read().replace("\thigh\t", "\tgreat\t", 1) -with open(os.path.join(sd, "y_str", "S1_y_str.tsv"), "w") as f: - f.write(t) -try: - sec(sd, "S1", "y_str") - check("7 an unknown confidence is unreadable", "read", "Unreadable") -except Unreadable: - check("7 an unknown confidence is unreadable", "Unreadable", "Unreadable") -try: - page = rr.html_report(cs.collect("S1", sd)) -except (KeyError, TypeError) as e: - page = f"{type(e).__name__}: {e}" -check("7 the HTML report renders an unreadable Y-STR table as such", - "

    Y-STR Markers (HipSTR)

    " in page and "Unreadable" in page, True) - -if FAILS: - print(f"{len(FAILS)} check(s) failed") - sys.exit(1) -print("all checks passed") From 09ba83bbc9d07c96786d96d8a6e6b44098c60bef Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:08:34 +0100 Subject: [PATCH 3/7] test(report): move the ancestry chapter's tests from Python to cargo The tests of the Y-STR, haplogroup explore links, ancient and archaic steps and the fine-grained ancestry reports now run in cargo test, in og-render/tests/ancestry_*.rs: each runs the step's own verb, then og summary collect and both reports, as the Python tests did through bin/og and render_report.py. tests/test_ancient_report.py, test_archaic_report.py, test_haplogroup_explore.py and test_ystr_report.py are deleted; test_ancestry_fine_report.py keeps the og ancestry-fine checks and drops its render part, so remnants.tsv no longer lists the four as importers. The archaic test's IBDmix runs need docker and an amd64 image: they compile on x86_64 only and run in CI (or with OG_DOCKER_TESTS=1). Their fixture is drawn with Python's Mersenne Twister, seeded as before, so it is the same data; a test checks the generator against Python's first draws. A planted haplogroup address that closes the link's attribute and opens a script tag stays escaped in the HTML report. port_report_core's chapter order test now expects the mitochondrial card, which the report shows even when step 12 did not run. Bead: D1 Bead: D5 Bead: D14 --- rust/Cargo.lock | 3 + rust/crates/og-render/Cargo.toml | 7 + .../og-render/tests/ancestry_ancient.rs | 718 ++++++++++ .../og-render/tests/ancestry_archaic.rs | 1275 +++++++++++++++++ rust/crates/og-render/tests/ancestry_fine.rs | 117 ++ .../og-render/tests/ancestry_haplogroups.rs | 169 +++ rust/crates/og-render/tests/ancestry_ystr.rs | 811 +++++++++++ .../og-render/tests/port_report_core.rs | 10 +- tests/parity/remnants.tsv | 2 +- tests/test_ancestry_fine_report.py | 20 +- 10 files changed, 3111 insertions(+), 21 deletions(-) create mode 100644 rust/crates/og-render/tests/ancestry_ancient.rs create mode 100644 rust/crates/og-render/tests/ancestry_archaic.rs create mode 100644 rust/crates/og-render/tests/ancestry_fine.rs create mode 100644 rust/crates/og-render/tests/ancestry_haplogroups.rs create mode 100644 rust/crates/og-render/tests/ancestry_ystr.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index ec8e92fcc..960526ee3 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -848,8 +848,11 @@ dependencies = [ name = "og-render" version = "0.1.0" dependencies = [ + "flate2", "indexmap", + "og-ancestry", "og-io", + "og-reports", "og-summary", "pycompat", ] diff --git a/rust/crates/og-render/Cargo.toml b/rust/crates/og-render/Cargo.toml index 5112b56a7..08c642327 100644 --- a/rust/crates/og-render/Cargo.toml +++ b/rust/crates/og-render/Cargo.toml @@ -20,5 +20,12 @@ og-io.workspace = true og-summary.workspace = true indexmap.workspace = true +# The ancestry chapter's tests (tests/ancestry_*.rs) run each step's own +# verb, then collect and render its result, as the run does. +[dev-dependencies] +og-ancestry.workspace = true +og-reports.workspace = true +flate2.workspace = true + [lints] workspace = true diff --git a/rust/crates/og-render/tests/ancestry_ancient.rs b/rust/crates/og-render/tests/ancestry_ancient.rs new file mode 100644 index 000000000..97973c0ee --- /dev/null +++ b/rust/crates/og-render/tests/ancestry_ancient.rs @@ -0,0 +1,718 @@ +//! `og ancient` (step 44) on synthetic files, run as the pipeline runs it, +//! then the summary and both reports. It replaces +//! `tests/test_ancient_report.py`. +//! +//! 1. ranking: on a synthetic .evec the nearest ancient group is the one +//! whose mean position is closest on the weighted PCs, a group far on PC1 +//! but with a small eigenvalue on PC3 is ranked by the weighted distance, +//! and each row carries its dates (oldest first, AADR style BCE), country +//! and member count; +//! 2. axes: a run whose PC2 has the opposite sign is turned back before the +//! person is placed, and a run whose present-day coordinates differ +//! stops with the reason instead of placing the person on other axes; +//! 3. liftover: lift keeps a site inside a block, a site on a minus-strand +//! chain with its alleles complemented, and drops a site in a gap, one +//! whose alleles miss the GRCh38 base, chrX and a monomorphic marker; +//! 4. genotypes: person-geno appends one column, 9 where the person has no +//! usable genotype, and stops on a genotype file shorter than the .snp +//! file; +//! 5. select: the present-day reference people come from MODERN_GROUPS, the +//! ancient ones need a date, a passing assessment, enough sites and a +//! place in West Eurasia, one genetic id per individual; +//! 6. the report command writes the table, the summary, the text and an SVG +//! with the person marked; +//! 7. both reports show them: an HTML card with the table and the plot, and +//! a section of the text report. +//! +//! The unit tests in og-ancestry and og-intervals check the date labels, +//! the dosages and the single-position lift in detail. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use og_ancestry::ancient::cli; +use og_render::model::Summary; +use og_render::{html, text}; +use pycompat::json::{Dump, dumps}; + +const CAVEAT: &str = "Closeness on a PCA is genetic similarity, not descent"; +const EVALS: [f64; 4] = [9.0, 4.0, 0.01, 0.01]; + +fn repo() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +fn tmp(tag: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("og-ancestry-ancient-{}-{tag}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d +} + +fn write(d: &Path, name: &str, text: &str) -> String { + let p = d.join(name); + std::fs::write(&p, text).unwrap(); + p.display().to_string() +} + +fn read(p: &str) -> String { + std::fs::read_to_string(p).unwrap_or_default() +} + +fn og(args: &[&str]) -> (u8, String, String) { + let args: Vec = args.iter().map(|a| (*a).to_owned()).collect(); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = cli(&args, &mut out, &mut err); + ( + code, + String::from_utf8(out).unwrap(), + String::from_utf8(err).unwrap(), + ) +} + +/// One person of an .evec: id, label, four PCs. +type Pc = (String, &'static str, [f64; 4]); + +fn evec(rows: &[Pc], evals: [f64; 4]) -> String { + let mut out = vec![format!( + "#eigvals: {}", + evals + .iter() + .map(|v| format!("{v:.3}")) + .collect::>() + .join(" ") + )]; + for (i, label, pcs) in rows { + out.push(format!( + "{i} {} {label}", + pcs.iter() + .map(|v| format!("{v:.6}")) + .collect::>() + .join(" ") + )); + } + out.join("\n") + "\n" +} + +/// Present-day people M0..M19: they build the axes. +fn moderns() -> Vec { + (0..20) + .map(|i| { + let f = f64::from(i); + ( + format!("M{i}"), + if i % 2 == 1 { "French" } else { "Sardinian" }, + [ + f * 0.01 - 0.1, + f64::from(i % 5) * 0.02 - 0.04, + f64::from(i % 3) * 0.01 - 0.01, + f64::from(i % 4) * 0.01 - 0.015, + ], + ) + }) + .collect() +} + +/// The panel: the present-day people and three ancient groups. +fn panel() -> Vec { + let mut p = moderns(); + p.extend([ + ("A1".to_owned(), "Ancient", [0.10, 0.10, 0.0, 0.0]), // Steppe + ("A2".to_owned(), "Ancient", [0.12, 0.10, 0.0, 0.0]), + ("B1".to_owned(), "Ancient", [-0.20, 0.00, 0.0, 0.0]), // Farmer + // Close on PC1 and PC2, far on PC3 (eigenvalue 0.01). + ("C1".to_owned(), "Ancient", [0.00, 0.00, 0.9, 0.0]), + ]); + p +} + +/// The person's run: the present-day people with PC2 flipped, as smartpca +/// may write it, and the person. +fn run_rows() -> Vec { + let mut r: Vec = moderns() + .into_iter() + .map(|(k, l, p)| (k, l, [p[0], -p[1], p[2], p[3]])) + .collect(); + r.push(("S".to_owned(), "Person", [0.08, -0.09, 0.0, 0.0])); + r +} + +/// The files of one report run. +struct Report { + tsv: String, + info: String, + txt: String, + svg: String, + ancients: String, + stats: String, +} + +impl Report { + fn new(d: &Path) -> Report { + let mut anc = "id\tgroup\tdate_bp\tdate_sd\tfull_date\tlocality\tcountry\tlat\tlon\tsnps\n" + .to_owned(); + for (id, group, bp, country, locality) in [ + ("A1", "Steppe_EBA", "4800", "Russia", "Samara"), + ("A2", "Steppe_EBA", "4600", "Russia", "Samara"), + ("B1", "Anatolia_N", "8200", "Turkey", "Barcin"), + ("C1", "Odd_C", "3000", "Italy", "Somewhere"), + ] { + anc += &format!("{id}\t{group}\t{bp}\t50\tx\t{locality}\t{country}\t50\t40\t500000\n"); + } + let o = |k: &str| d.join(format!("S_{k}")).display().to_string(); + Report { + tsv: o("ancient.tsv"), + info: o("info.tsv"), + txt: o("ancient.txt"), + svg: o("pca.svg"), + ancients: write(d, "anc.tsv", &anc), + stats: write( + d, + "stats.tsv", + "panel_sites\t4\nsites_genotyped\t3\nother_allele\t1\nsites_used\t2\n", + ), + } + } + + fn run(&self, panel_evec: &str, run_evec: &str, with_stats: bool) -> (u8, String, String) { + let mut a = vec![ + "report", + "--panel-evec", + panel_evec, + "--run-evec", + run_evec, + "--ancients", + &self.ancients, + "--sample", + "S", + "--release", + "test", + ]; + if with_stats { + a.extend(["--stats", &self.stats]); + } + a.extend([ + "--out-tsv", + &self.tsv, + "--out-info", + &self.info, + "--out-txt", + &self.txt, + "--out-svg", + &self.svg, + ]); + og(&a) + } + + fn table(&self) -> Vec> { + let t = read(&self.tsv); + let mut lines = t.lines(); + let head: Vec<&str> = lines.next().unwrap_or_default().split('\t').collect(); + lines + .map(|l| { + head.iter() + .zip(l.split('\t')) + .map(|(h, v)| ((*h).to_owned(), v.to_owned())) + .collect() + }) + .collect() + } + + fn groups(&self) -> Vec { + self.table() + .into_iter() + .map(|r| r["group"].clone()) + .collect() + } +} + +#[test] +fn the_nearest_group_on_the_weighted_pcs_comes_first() { + let d = tmp("rank"); + let r = Report::new(&d); + let pe = write(&d, "panel.evec", &evec(&panel(), EVALS)); + let re = write(&d, "run.evec", &evec(&run_rows(), EVALS)); + let (code, _, err) = r.run(&pe, &re, true); + assert_eq!(code, 0, "{err}"); + assert_eq!( + r.groups(), + ["Steppe_EBA", "Odd_C", "Anatolia_N"], + "Steppe_EBA first from the flipped run turned back; Odd_C, far only on a PC with a tiny \ + eigenvalue, second" + ); + let rows = r.table(); + let st = &rows[0]; + assert_eq!( + [&st["n"], &st["dates"], &st["country"], &st["locality"]], + ["2", "2850-2650 BCE", "Russia", "Samara"] + ); + let ranks: Vec<&str> = rows.iter().map(|x| x["rank"].as_str()).collect(); + assert_eq!(ranks, ["1", "2", "3"]); + + // Control: without the eigenvalue weights Odd_C falls behind Anatolia_N. + let fe = write(&d, "flat_panel.evec", &evec(&panel(), [1.0; 4])); + let fr = write(&d, "flat_run.evec", &evec(&run_rows(), [1.0; 4])); + let (code, _, err) = r.run(&fe, &fr, true); + assert_eq!(code, 0, "{err}"); + assert_eq!(r.groups(), ["Steppe_EBA", "Anatolia_N", "Odd_C"]); +} + +#[test] +fn a_flipped_axis_is_turned_back_and_other_axes_stop() { + let d = tmp("align"); + let pe = write(&d, "panel.evec", &evec(&panel(), EVALS)); + let re = write(&d, "run.evec", &evec(&run_rows(), EVALS)); + let (code, out, err) = og(&[ + "align", + "--panel-evec", + &pe, + "--run-evec", + &re, + "--pcs", + "2", + ]); + assert_eq!(code, 0, "{err}"); + assert!(out.contains("PC1+ PC2-"), "{out}"); + let bad: Vec = moderns() + .into_iter() + .map(|(k, l, p)| { + let n: u32 = k[1..].parse().unwrap(); + let s = if n.is_multiple_of(3) { -1.0 } else { 1.0 }; + (k, l, [p[0] * s, p[1], p[2], p[3]]) + }) + .collect(); + let be = write(&d, "bad.evec", &evec(&bad, EVALS)); + let (code, _, err) = og(&[ + "align", + "--panel-evec", + &pe, + "--run-evec", + &be, + "--pcs", + "2", + ]); + assert_eq!(code, 1, "{err}"); + assert!(err.contains("PC1"), "{err}"); +} + +#[test] +fn lift_keeps_sites_in_blocks_and_complements_the_minus_strand() { + let d = tmp("lift"); + let chain = write( + &d, + "test.chain", + &([ + // chr1 source 1000-1100 -> target 2000-2100 (+), gap of 50, then + // 1150-1200 -> 2150-2200. + "chain 100 chr1 100000 + 1000 1200 chr1 100000 + 2000 2200 1", + "100 50 50", + "50", + "", + // chr2 source 500-600 -> target on the minus strand of a 10,000 bp + // chr2, reverse start 300. + "chain 90 chr2 50000 + 500 600 chr2 10000 - 300 400 2", + "100", + "", + // chr3: two chains over 100-200. + "chain 80 chr3 50000 + 100 200 chr3 50000 + 100 200 3", + "100", + "", + "chain 70 chr3 50000 + 150 250 chr3 50000 + 5150 5250 4", + "100", + "", + ] + .join("\n") + + "\n"), + ); + let mut seq1 = vec![b'N'; 3000]; + seq1[2050] = b'G'; // 1-based 2051 + seq1[2159] = b'T'; // 1-based 2160 + let mut seq2 = vec![b'N'; 10000]; + seq2[9689] = b'C'; // 1-based 9690: the complement of the source G + let (mut fa, mut fai, mut off) = (String::new(), String::new(), 0usize); + for (name, seq) in [("chr1", &seq1), ("chr2", &seq2)] { + let head = format!(">{name}\n"); + fa += &head; + off += head.len(); + for c in seq.chunks(60) { + fa += std::str::from_utf8(c).unwrap(); + fa.push('\n'); + } + fai += &format!("{name}\t{}\t{off}\t60\t61\n", seq.len()); + off += seq.len() + seq.len().div_ceil(60); + } + let fa = write(&d, "ref.fa", &fa); + write(&d, "ref.fa.fai", &fai); + let snp = write( + &d, + "src.snp", + &([ + "rs1 1 0.0 1051 G A", // ref G at target: kept + "rs2 1 0.0 1160 C T", // second allele is the target base: kept + "rs3 1 0.0 1120 A G", // gap: dropped + "rs4 2 0.0 511 G T", // minus strand: alleles C/A, target C: kept + "rs5 1 0.0 1051 A C", // neither allele is G: dropped + "rs6 23 0.0 1051 G A", // X: dropped + "rs7 1 0.0 1051 G X", // monomorphic marker: dropped + ] + .join("\n") + + "\n"), + ); + let out = d.join("sites.tsv").display().to_string(); + let (code, _, err) = og(&[ + "lift", "--snp", &snp, "--chain", &chain, "--fasta", &fa, "--out", &out, + ]); + assert_eq!(code, 0, "{err}"); + assert_eq!( + read(&out).lines().collect::>(), + [ + "snp\tchrom\tpos\tallele1\tallele2", + "rs1\tchr1\t2051\tG\tA", + "rs2\tchr1\t2160\tC\tT", + "rs4\tchr2\t9690\tC\tA" + ] + ); + + // person-geno on those sites. + let msnp = write( + &d, + "moderns.snp", + "rs1 1 0.0 1051 G A\nrs2 1 0.0 1160 C T\nrsX 23 0.0 5 A G\nrs4 2 0.0 511 G T\n", + ); + let mgeno = write(&d, "moderns.geno", "012\n220\n999\n101\n"); + let gts = write( + &d, + "gts.tsv", + "chr1\t2051\tG\tA\tG/A\nchr1\t2160\tT\t.\tT/T\nchr2\t9690\tC\tG\tC/G\n", + ); + let merged = d.join("merged.geno").display().to_string(); + let pstats = d.join("pstats.tsv").display().to_string(); + let geno = |m: &str| { + og(&[ + "person-geno", + "--sites", + &out, + "--genotypes", + >s, + "--snp", + &msnp, + "--moderns-geno", + m, + "--out", + &merged, + "--stats", + &pstats, + ]) + }; + let (code, _, err) = geno(&mgeno); + assert_eq!(code, 0, "{err}"); + assert_eq!( + read(&merged).lines().collect::>(), + ["0121", "2200", "9999", "1019"], + "one column: 1, 0, 9 for X, 9 for an allele outside the pair" + ); + let st: BTreeMap = read(&pstats) + .lines() + .map(|l| { + let (k, v) = l.split_once('\t').unwrap(); + (k.to_owned(), v.to_owned()) + }) + .collect(); + let want: BTreeMap = [ + ("panel_sites", "4"), + ("sites_genotyped", "3"), + ("other_allele", "1"), + ("sites_used", "2"), + ] + .map(|(k, v)| (k.to_owned(), v.to_owned())) + .into(); + assert_eq!(st, want); + let short = write(&d, "short.geno", "012\n220\n"); + let (code, _, err) = geno(&short); + assert_eq!( + code, 1, + "a genotype file shorter than the .snp file stops: {err}" + ); +} + +#[test] +fn select_takes_reference_and_ancient_people_by_the_rules() { + let d = tmp("select"); + let head = [ + "Genetic ID (long text)", + "Persistent Genetic ID", + "Individual ID", + "\"Date mean in BP in years before 1950\"", + "Date standard deviation in BP [x]", + "Full Date One of two formats", + "Group ID", + "Locality", + "Political Entity", + "Latitude", + "Longitude", + "Data type", + "SNPs hit on autosomal targets (Computed using easystats on 1240k snpset)", + "SNPs hit on autosomal targets (Computed using easystats on HO snpset)", + "ASSESSMENT", + "ASSESSMENT WARNINGS (long)", + ] + .map(str::to_owned) + .to_vec(); + #[allow(clippy::too_many_arguments)] + fn arow( + gid: &str, + ind: &str, + bp: u32, + group: &str, + lat: &str, + lon: &str, + dtype: &str, + snps: u32, + ass: &str, + ) -> Vec { + [ + gid, + gid, + ind, + &bp.to_string(), + "50", + "x", + group, + "Loc", + "Land", + lat, + lon, + dtype, + &snps.to_string(), + "0", + ass, + "", + ] + .map(str::to_owned) + .to_vec() + } + let mut rows: Vec> = (0..60) + .map(|i| { + arow( + &format!("F{i}.DG"), + &format!("F{i}"), + 0, + "French", + "46", + "2", + "Shotgun.diploid", + 1_100_000, + "PROVISIONAL_PASS", + ) + }) + .collect(); + rows.extend([ + // The same individual with fewer sites. + arow( + "F0.SG", + "F0", + 0, + "French", + "46", + "2", + "Shotgun.diploid", + 900_000, + "PASS", + ), + // Not a reference group. + arow( + "Y1.DG", + "Y1", + 0, + "Yakut", + "62", + "129", + "Shotgun.diploid", + 1_100_000, + "PASS", + ), + // Not shotgun diploid. + arow( + "H1.HO", + "H1", + 0, + "French", + "46", + "2", + "HumanOrigins", + 600_000, + "PASS", + ), + arow( + "A1.AG", + "A1", + 4800, + "Russia_Steppe", + "53", + "50", + "1240k", + 500_000, + "PASS", + ), + // Too few sites. + arow( + "A2.AG", + "A2", + 4800, + "Russia_Steppe", + "53", + "50", + "1240k", + 50_000, + "PASS", + ), + arow( + "A3.AG", + "A3", + 4800, + "Russia_Steppe", + "53", + "50", + "1240k", + 500_000, + "Questionable", + ), + // Outside West Eurasia. + arow( + "A4.AG", "A4", 4800, "China_N", "40", "116", "1240k", 500_000, "PASS", + ), + // No place. + arow( + "A5.AG", "A5", 4800, "Unknown", "..", "..", "1240k", 500_000, "PASS", + ), + ]); + let anno_text: String = std::iter::once(&head) + .chain(rows.iter()) + .map(|r| r.join("\t") + "\n") + .collect(); + let anno = write(&d, "x.anno", &anno_text); + let ind_text: String = rows + .iter() + .map(|r| format!("{} U {}\n", r[0], r[6])) + .collect(); + let ind = write(&d, "x.ind", &ind_text); + let o = |n: &str| d.join(n).display().to_string(); + let (oind, plist, anc) = (o("panel.ind"), o("poplist.txt"), o("ancients.tsv")); + let (code, _, err) = og(&[ + "select", + "--anno", + &anno, + "--ind", + &ind, + "--out-ind", + &oind, + "--poplist", + &plist, + "--ancients", + &anc, + ]); + assert_eq!(code, 0, "{err}"); + let labels: Vec<(String, String)> = read(&oind) + .lines() + .map(|l| { + let c: Vec<&str> = l.split('\t').collect(); + (c[0].to_owned(), c[2].to_owned()) + }) + .collect(); + let label = |k: &str| labels.iter().find(|(i, _)| i == k).map(|(_, v)| v.as_str()); + assert_eq!( + labels.iter().filter(|(_, v)| v == "French").count(), + 60, + "the 60 French shotgun genomes are the reference" + ); + assert_eq!(label("F0.SG"), Some("Ignore"), "one id per person (F0.DG)"); + assert_eq!(label("Y1.DG"), Some("Ignore")); + assert_eq!(label("H1.HO"), Some("Ignore")); + let projected: Vec<&str> = labels + .iter() + .filter(|(_, v)| v == "Ancient") + .map(|(k, _)| k.as_str()) + .collect(); + assert_eq!(projected, ["A1.AG"]); + assert_eq!(read(&plist), "French\n"); + assert!( + read(&anc) + .lines() + .nth(1) + .unwrap() + .starts_with("A1.AG\tRussia_Steppe\t4800") + ); +} + +/// The summary og summary collect writes for S, as text. +fn collect(sd: &Path) -> String { + let input = og_summary::collect::Input { + sample: "S".to_owned(), + sample_dir: sd.display().to_string(), + declared_sex: None, + assets: repo().join("assets").display().to_string(), + group: og_summary::collect::Group::All, + tables: repo().join("bin").display().to_string(), + }; + dumps( + &og_summary::collect::post::collect(&input).unwrap(), + &Dump::default(), + ) + .unwrap() +} + +#[test] +fn the_report_writes_its_files_and_both_reports_show_them() { + let d = tmp("report"); + let r = Report::new(&d); + let pe = write(&d, "panel.evec", &evec(&panel(), EVALS)); + let re = write(&d, "run.evec", &evec(&run_rows(), EVALS)); + let (code, _, err) = r.run(&pe, &re, true); + assert_eq!(code, 0, "{err}"); + let info: BTreeMap = read(&r.info) + .lines() + .skip(1) + .map(|l| { + let (k, v) = l.split_once('\t').unwrap(); + (k.to_owned(), v.to_owned()) + }) + .collect(); + assert_eq!( + info["pc2"], "0.09000", + "the person's PC2 in the panel's sign" + ); + assert_eq!(info["sites_used"], "2"); + let svg = read(&r.svg); + assert!(svg.starts_with("S"), "the person's label"); + assert!(svg.trim_end().ends_with("")); + let txt = read(&r.txt); + assert!(txt.contains(CAVEAT) && txt.contains("qpAdm"), "{txt}"); + + // Both reports read the outputs. + let sd = d.join("S"); + std::fs::create_dir_all(sd.join("ancient")).unwrap(); + for (from, to) in [ + (&r.tsv, "S_ancient.tsv"), + (&r.info, "S_ancient_info.tsv"), + (&r.txt, "S_ancient.txt"), + (&r.svg, "S_ancient_pca.svg"), + ] { + std::fs::copy(from, sd.join("ancient").join(to)).unwrap(); + } + let s = Summary::parse(&collect(&sd)).unwrap(); + let page = html::report(&s).unwrap(); + assert!(page.contains("

    Ancient Populations

    ")); + assert!(page.contains("1Steppe_EBA2850-2650 BCERussia")); + assert!(page.contains("data:image/svg+xml;base64,"), "the plot"); + let report = text::report(&s).unwrap(); + assert!(report.contains("Ancient Populations (AADR PCA, step 44)")); + assert!(report.contains("1. Steppe_EBA ("), "{report}"); + assert!(report.contains("not descent")); + assert!(report.contains("Plot: ancient/S_ancient_pca.svg")); + + let (code, _, err) = r.run(&pe, &pe, false); + assert_eq!(code, 1, "a run without the person stops"); + assert!(err.contains("no row for S"), "{err}"); +} diff --git a/rust/crates/og-render/tests/ancestry_archaic.rs b/rust/crates/og-render/tests/ancestry_archaic.rs new file mode 100644 index 000000000..6763b71ae --- /dev/null +++ b/rust/crates/og-render/tests/ancestry_archaic.rs @@ -0,0 +1,1275 @@ +//! `og archaic` (step 43) on a synthetic archaic panel, then the summary and +//! the report lines; on amd64 in CI also with the real IBDmix image. It +//! replaces `tests/test_archaic_report.py`. +//! +//! 1. table: a 1000 Genomes site keeps its frequencies and gets each archaic +//! genome's ALT count, 10 plus it outside that genome's mask or inside an +//! excluded region; an archaic record with another ALT or another REF is +//! unknown (9); a site only an archaic genome varies at gets frequency 0; +//! a site no archaic genome was called at is left out; a position given +//! twice by 1000 Genomes is left out; +//! 2. strict: a lifted row outside the GRCh38 strict mask is kept once per +//! position with every archaic call marked masked (9 stays 9); +//! 3. genotypes: a masked archaic call goes to IBDmix as its ALT count and +//! its position into that genome's mask file; the stand-in columns carry +//! the population's ALT count; your column is your ALT count, 9 for a +//! non-PASS record, and where your VCF has no record 0 inside a reference +//! block of the gVCF and 9 outside every one; a site where your VCF has +//! another ALT is left out; a chromosome your VCF has no record on is not +//! analysed; the split-VCF rules; unsorted input stops and keeps the +//! previous outputs; +//! 4. report: Altai and Vindija segments merge into Neanderthal ones, a +//! Denisova segment that overlaps one is not Denisovan, segments under +//! 50 kb or LOD 4 are dropped, the share is of the analysed span, and the +//! genes of the longest segments are named; og summary collect reads the +//! summary and marks a damaged one unreadable, and the report lines give +//! both shares; +//! 5. with IBDmix itself (`IBDMIX_IMAGE`, docker; amd64 only, run in CI or +//! with `OG_DOCKER_TESTS=1`): a planted Neanderthal and a planted +//! Denisovan stretch come out as one segment each, the same person +//! without them gets none; masked mismatches stop a segment where the +//! same calls written as unknown let it run through; a stretch with no +//! call gives a segment only when a gVCF reference block says it was read. + +use std::collections::BTreeMap; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; + +use flate2::Compression; +use flate2::read::GzDecoder; +use flate2::write::GzEncoder; +use og_ancestry::archaic::cli; +use pycompat::json::Value; + +const TABLE_HEADER: &str = "chrom\tpos\tref\talt\tAltai\tVindija\tDenisova\tAF_ALL\tAF_AFR\tAF_AMR\tAF_EAS\tAF_EUR\tAF_SAS"; +const VCF_HEAD: &str = + "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n"; +const ARCHAICS: [&str; 3] = ["Altai", "Vindija", "Denisova"]; + +fn repo() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +fn tmp(tag: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("og-ancestry-archaic-{}-{tag}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d +} + +fn p(d: &Path, name: &str) -> String { + d.join(name).display().to_string() +} + +fn write(path: &str, text: &str) { + std::fs::write(path, text).unwrap(); +} + +fn gzip(path: &str, text: &str) { + let mut z = GzEncoder::new(std::fs::File::create(path).unwrap(), Compression::default()); + z.write_all(text.as_bytes()).unwrap(); + z.finish().unwrap(); +} + +fn gunzip(path: &str) -> String { + let mut s = String::new(); + GzDecoder::new(std::fs::File::open(path).unwrap()) + .read_to_string(&mut s) + .unwrap(); + s +} + +fn og(args: &[&str]) -> (u8, String) { + let args: Vec = args.iter().map(|a| (*a).to_owned()).collect(); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = cli(&args, &mut out, &mut err); + (code, String::from_utf8(err).unwrap()) +} + +fn ref_block(chrom: &str, start: u32, end: u32) -> String { + format!("{chrom}\t{start}\t.\tN\t<*>\t0\t.\tEND={end}\tGT:GQ:MIN_DP\t0/0:50:30\n") +} + +/// The records of a table VCF by position, and each one's INFO. +type Rows = BTreeMap>; +type Info = BTreeMap>; + +fn vcf_rows(path: &str) -> (Rows, Info) { + let rows: Rows = std::fs::read_to_string(path) + .unwrap() + .lines() + .filter(|l| !l.starts_with('#')) + .map(|l| { + let c: Vec = l.split('\t').map(str::to_owned).collect(); + (c[1].parse().unwrap(), c) + }) + .collect(); + let info = rows + .iter() + .map(|(k, c)| { + ( + *k, + c[7].split(';') + .map(|kv| { + let (a, b) = kv.split_once('=').unwrap(); + (a.to_owned(), b.to_owned()) + }) + .collect(), + ) + }) + .collect(); + (rows, info) +} + +fn codes(i: &Info, pos: u32) -> [&str; 3] { + ARCHAICS.map(|g| i[&pos][g].as_str()) +} + +/// The inputs of part 1 in `d`. +fn table_inputs(d: &Path) -> Vec { + let afs = "0.1\t0.2\t0.05\t0\t0.15\t0.1"; + write( + &p(d, "modern.tsv"), + &format!( + // every archaic genome called; Vindija has another ALT and + // Denisova another REF; no archaic record; a split multi-allelic + // site. + "100\tA\tG\t{afs}\n200\tC\tT\t{afs}\n300\tG\tA\t{afs}\n400\tT\tC\t{afs}\n400\tT\tG\t{afs}\n" + ), + ); + write( + &p(d, "altai.tsv"), + "100\tA\tG\t1/1\n200\tC\t.\t0/0\n500\tG\tC\t0/1\n600\tA\t.\t0/0\n", + ); + write( + &p(d, "vindija.tsv"), + "100\tA\t.\t0/0\n200\tC\tA\t0/1\n500\tG\t.\t0/0\n600\tA\t.\t0/0\n", + ); + write( + &p(d, "denisova.tsv"), + "100\tA\tG\t0|1\n200\tT\tC\t0/1\n400\tT\tC\t1/1\n500\tG\tT\t1/1\n", + ); + [ + "table".to_owned(), + "--chrom".to_owned(), + "22".to_owned(), + "--modern".to_owned(), + p(d, "modern.tsv"), + "--archaic".to_owned(), + format!("Altai={}", p(d, "altai.tsv")), + "--archaic".to_owned(), + format!("Vindija={}", p(d, "vindija.tsv")), + "--archaic".to_owned(), + format!("Denisova={}", p(d, "denisova.tsv")), + ] + .to_vec() +} + +fn table(common: &[String], extra: &[&str], out: &str) -> (u8, String) { + let mut a: Vec<&str> = common.iter().map(String::as_str).collect(); + a.extend(extra); + a.extend(["--out", out]); + og(&a) +} + +#[test] +fn the_table_codes_each_archaic_call() { + let d = tmp("table"); + let common = table_inputs(&d); + let (code, err) = table(&common, &[], &p(&d, "t.vcf")); + assert_eq!(code, 0, "{err}"); + let (recs, info) = vcf_rows(&p(&d, "t.vcf")); + assert_eq!( + recs.keys().copied().collect::>(), + [100, 200, 500], + "the modern sites with an archaic call and the archaic-only site" + ); + assert_eq!( + codes(&info, 100), + ["2", "0", "1"], + "each genome's ALT count" + ); + assert_eq!( + [&info[&100]["AF_ALL"], &info[&100]["AF_EUR"]], + ["0.1", "0.15"] + ); + assert_eq!( + codes(&info, 200), + ["0", "9", "9"], + "another ALT or another REF is unknown" + ); + assert_eq!(recs[&500][4], "C", "the first non-reference ALT"); + assert_eq!(codes(&info, 500), ["1", "0", "9"]); + assert_eq!(info[&500]["AF_ALL"], "0"); + assert_eq!(recs[&100][0], "chr22", "the chr prefix for the liftover"); +} + +#[test] +fn a_call_outside_its_mask_or_inside_an_excluded_region_is_masked() { + let d = tmp("mask"); + let common = table_inputs(&d); + // 1-based 51-150 and 451-550: 100 and 500, not 200. + write(&p(&d, "altai.bed"), "22\t50\t150\n22\t450\t550\n"); + write(&p(&d, "vindija.bed"), "22\t150\t250\n"); // holds 200, not 100 + gzip(&p(&d, "denisova.bed.gz"), "chr22\t99\t100\n"); // 100 only, named chr22 + + // Control: without --mask the same inputs keep every call. + let (code, err) = table(&common, &[], &p(&d, "nomask.vcf")); + assert_eq!(code, 0, "{err}"); + let (nm_rows, nm) = vcf_rows(&p(&d, "nomask.vcf")); + assert_eq!(nm[&100]["Altai"], "2"); + assert_eq!(nm_rows.keys().copied().collect::>(), [100, 200, 500]); + + let masks = [ + format!("Altai={}", p(&d, "altai.bed")), + format!("Vindija={}", p(&d, "vindija.bed")), + format!("Denisova={}", p(&d, "denisova.bed.gz")), + ]; + let (code, err) = table( + &common, + &[ + "--mask", &masks[0], "--mask", &masks[1], "--mask", &masks[2], + ], + &p(&d, "masked.vcf"), + ); + assert_eq!(code, 0, "{err}"); + let (m_rows, mi) = vcf_rows(&p(&d, "masked.vcf")); + assert_eq!( + [&mi[&100]["Altai"], &mi[&100]["Denisova"]], + ["2", "1"], + "inside its mask a genome keeps its call; a mask is read with and without chr" + ); + assert_eq!( + [&mi[&100]["Vindija"], &nm[&100]["Vindija"]], + ["10", "0"], + "outside its mask a call is masked (10 for 0/0)" + ); + assert_eq!(m_rows.keys().copied().collect::>(), [100, 200, 500]); + assert_eq!( + mi[&200]["Altai"], "10", + "a row whose only call is masked stays" + ); + assert_eq!(m_rows[&500][4], "C"); + assert_eq!([&mi[&500]["Altai"], &mi[&500]["Vindija"]], ["1", "10"]); + assert_eq!( + [&mi[&200]["Vindija"], &mi[&500]["Denisova"]], + ["9", "9"], + "unknown stays 9" + ); + + // The excluded regions mask every call at a site inside one. + write(&p(&d, "excl.bed"), "22\t90\t120\tsegdup\n"); + let excl = p(&d, "excl.bed"); + let (code, err) = table(&common, &["--exclude", &excl], &p(&d, "excl.vcf")); + assert_eq!(code, 0, "{err}"); + let (ex_rows, ex) = vcf_rows(&p(&d, "excl.vcf")); + assert_eq!(codes(&ex, 100), ["12", "10", "11"]); + assert_eq!(codes(&nm, 100), ["2", "0", "1"]); + assert_eq!(ex_rows.keys().copied().collect::>(), [100, 200, 500]); + assert_eq!([&ex[&200]["Altai"], &ex[&500]["Altai"]], ["0", "1"]); + + write(&p(&d, "excl_other.bed"), "7\t90\t120\tsegdup\n"); + let other = p(&d, "excl_other.bed"); + let (code, err) = table(&common, &["--exclude", &other], &p(&d, "x.vcf")); + assert!(code == 1 && err.contains("no interval"), "{code} {err}"); + let (code, err) = table( + &common, + &["--mask", &masks[0], "--mask", &masks[0]], + &p(&d, "x.vcf"), + ); + assert!(code == 2 && err.contains("twice"), "{code} {err}"); + let nobody = format!("Nobody={}", p(&d, "altai.bed")); + let (code, err) = table(&common, &["--mask", &nobody], &p(&d, "x.vcf")); + assert_eq!(code, 2, "{err}"); + write(&p(&d, "other.bed"), "7\t50\t150\n"); + let elsewhere = format!("Altai={}", p(&d, "other.bed")); + let (code, err) = table(&common, &["--mask", &elsewhere], &p(&d, "x.vcf")); + assert!( + code == 1 && err.contains("no interval"), + "a mask with no interval on the chromosome stops: {code} {err}" + ); +} + +#[test] +fn strict_marks_rows_outside_the_mask_and_keeps_each_position_once() { + let d = tmp("strict"); + // Strict mask 1-based 101-300 on chr22. + let rows = [ + "chr22\t100\tA\tG\t2\t0\t9\t0.5\t0\t0\t0\t0\t0", + "chr22\t150\tA\tG\t1\t1\t1\t0.5\t0\t0\t0\t0\t0", + "chr22\t300\tG\tA\t0\t0\t0\t0.5\t0\t0\t0\t0\t0", + "chr22\t301\tC\tT\t0\t10\t12\t0.5\t0\t0\t0\t0\t0", + "chr22\t301\tC\tT\t2\t2\t2\t0.5\t0\t0\t0\t0\t0", + ]; + write(&p(&d, "lifted.tsv"), &(rows.join("\n") + "\n")); + write(&p(&d, "strict.bed"), "chr22\t100\t300\n"); + let strict = |mask: &str, rows: &str, out: &str| { + og(&[ + "strict", "--chrom", "22", "--strict", mask, "--rows", rows, "--out", out, + ]) + }; + let (code, err) = strict( + &p(&d, "strict.bed"), + &p(&d, "lifted.tsv"), + &p(&d, "marked.tsv"), + ); + assert_eq!(code, 0, "{err}"); + let got: Vec> = std::fs::read_to_string(p(&d, "marked.tsv")) + .unwrap() + .lines() + .map(|l| l.split('\t').map(str::to_owned).collect()) + .collect(); + assert_eq!( + got.iter().map(|c| c[1].as_str()).collect::>(), + ["100", "150", "300", "301"] + ); + let by = |pos: &str| -> Vec<&str> { + got.iter().find(|c| c[1] == pos).unwrap()[4..7] + .iter() + .map(String::as_str) + .collect() + }; + assert_eq!(by("100"), ["12", "10", "9"], "outside: 2, 0, unknown"); + assert_eq!(by("150"), ["1", "1", "1"], "inside: untouched"); + assert_eq!(by("300"), ["0", "0", "0"], "the last base inside too"); + assert_eq!( + by("301"), + ["10", "10", "12"], + "a masked call stays, the first of a duplicate wins" + ); + write( + &p(&d, "unsorted.tsv"), + &([rows[0], rows[3], rows[2]].join("\n") + "\n"), + ); + let (code, err) = strict( + &p(&d, "strict.bed"), + &p(&d, "unsorted.tsv"), + &p(&d, "marked3.tsv"), + ); + assert!(code == 2 && err.contains("sorted"), "{code} {err}"); + assert!(!d.join("marked3.tsv").exists()); + write(&p(&d, "strict_other.bed"), "chr21\t0\t10\n"); + let (code, err) = strict( + &p(&d, "strict_other.bed"), + &p(&d, "lifted.tsv"), + &p(&d, "marked2.tsv"), + ); + assert!(code != 0 && err.contains("no interval"), "{code} {err}"); +} + +/// A genotype file by position. +fn gt_rows(path: &str) -> BTreeMap> { + gunzip(path) + .lines() + .skip(1) + .map(|l| { + let c: Vec = l.split('\t').map(str::to_owned).collect(); + (c[1].parse().unwrap(), c) + }) + .collect() +} + +const ROWS: [&str; 5] = [ + "chr22\t1000\tA\tG\t2\t2\t0\t0.5\t0.5\t0.5\t0.5\t0.5\t0.5", + "chr22\t2000\tC\tT\t0\t0\t0\t0.01\t0\t0\t0\t0.02\t0", + "chr22\t3000\tG\tA\t1\t1\t2\t0\t0\t0\t0\t0\t0", + "chr22\t4000\tT\tC\t2\t9\t0\t0.2\t0\t0\t0\t0\t0", + "chr22\t5000\tA\tC\t0\t0\t0\t0.2\t0\t0\t0\t0\t0", +]; + +/// The table of part 2 and the person's VCF. +fn genotype_inputs(d: &Path) -> String { + let tdir = p(d, "table"); + std::fs::create_dir_all(&tdir).unwrap(); + gzip( + &format!("{tdir}/chr22.tsv.gz"), + &format!("{TABLE_HEADER}\n{}\n", ROWS.join("\n")), + ); + gzip( + &format!("{tdir}/chr21.tsv.gz"), + &format!("{TABLE_HEADER}\nchr21\t1000\tA\tG\t2\t2\t0\t0.5\t0.5\t0.5\t0.5\t0.5\t0.5\n"), + ); + write( + &p(d, "s.vcf"), + &format!( + "{VCF_HEAD}chr22\t1000\t.\tA\tG\t50\tPASS\t.\tGT:GQ\t0/1:30\n\ + chr22\t3000\t.\tG\tA\t50\tRefCall\t.\tGT\t1/1\n\ + chr22\t5000\t.\tA\tT\t50\tPASS\t.\tGT\t1/1\n" + ), + ); + tdir +} + +#[test] +fn genotypes_code_your_calls_and_the_stand_ins() { + let d = tmp("genotypes"); + let tdir = genotype_inputs(&d); + // A reference block over 1-2500: 2000 was read; 4000 was not. + write( + &p(&d, "s.g.vcf"), + &format!("{VCF_HEAD}{}", ref_block("chr22", 1, 2500)), + ); + let (code, err) = og(&[ + "genotypes", + "--table-dir", + &tdir, + "--vcf", + &p(&d, "s.vcf"), + "--gvcf", + &p(&d, "s.g.vcf"), + "--pop", + "EUR", + "--pseudo", + "10", + "--out-dir", + &p(&d, "gt"), + ]); + assert_eq!(code, 0, "{err}"); + let text = gunzip(&p(&d, "gt/chr22.gt.gz")); + let head: Vec<&str> = text.lines().next().unwrap().split('\t').collect(); + assert_eq!( + head[..8], + [ + "chrom", "pos", "ref", "alt", "Altai", "Vindija", "Denisova", "SAMPLE" + ] + ); + assert_eq!(head.len(), 18, "ten stand-ins"); + let g = gt_rows(&p(&d, "gt/chr22.gt.gz")); + assert_eq!(g[&1000][0], "22", "an integer chromosome for IBDmix"); + assert_eq!(g[&1000][7], "1", "a heterozygous call"); + assert_eq!(g[&2000][7], "0", "no record inside a reference block"); + assert_eq!(g[&4000][7], "9", "no record outside every block"); + assert_eq!(g[&3000][7], "9", "a non-PASS record"); + assert!(!g.contains_key(&5000), "another ALT is left out"); + let sum = |pos: u32| -> u32 { g[&pos][8..].iter().map(|x| x.parse::().unwrap()).sum() }; + assert_eq!(sum(1000), 10, "0.5 of 20 alleles"); + assert_eq!(sum(2000), 0, "0.02 of 20 rounds to 0"); + assert!(g[&1000][4..].iter().all(|x| x.len() == 1)); + assert!( + !d.join("gt/chr21.gt.gz").exists(), + "a chromosome the VCF has no record on is not analysed" + ); + let spans = std::fs::read_to_string(p(&d, "gt/spans.tsv")).unwrap(); + let span: Vec<&str> = spans.lines().nth(1).unwrap().split('\t').collect(); + assert_eq!(span[..4], ["chr22", "1000", "2000", "2"]); + let samples = std::fs::read_to_string(p(&d, "gt/samples.txt")).unwrap(); + let samples: Vec<&str> = samples.split_whitespace().collect(); + assert_eq!((samples[0], samples.len()), ("SAMPLE", 11)); + assert_eq!( + std::fs::read_to_string(p(&d, "gt/chr22.Altai.mask.bed")).unwrap(), + "", + "an unmasked table writes empty masks" + ); + + write( + &p(&d, "two.vcf"), + "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tA\tB\n", + ); + let (code, err) = og(&[ + "genotypes", + "--table-dir", + &tdir, + "--vcf", + &p(&d, "two.vcf"), + "--out-dir", + &p(&d, "gt2"), + ]); + assert!(code != 0 && err.contains("one-sample"), "{code} {err}"); +} + +#[test] +fn a_split_vcf_keeps_each_snv_call() { + let d = tmp("split"); + let sdir = p(&d, "table_split"); + std::fs::create_dir_all(&sdir).unwrap(); + let rows: String = [ + (1000, "A\tG"), + (2000, "C\tT"), + (3000, "G\tA"), + (4000, "T\tC"), + (5000, "A\tG"), + (6000, "C\tT"), + (7000, "G\tA"), + (8000, "T\tC"), + (9000, "A\tC"), + ] + .iter() + .map(|(pos, ra)| format!("chr22\t{pos}\t{ra}\t0\t0\t0\t0.2\t0\t0\t0\t0\t0\n")) + .collect(); + gzip( + &format!("{sdir}/chr22.tsv.gz"), + &format!("{TABLE_HEADER}\n{rows}"), + ); + let recs = [ + "1000\tA\tAT\tPASS\t0/1", // an insertion, then the SNV + "1000\tA\tG\tPASS\t0/1", + "2000\tC\tT\tPASS\t1/1", // the SNV, then an insertion + "2000\tC\tCA\tPASS\t0/1", + "3000\tG\tA,GT\tPASS\t1/2", // a multi-allelic record, then the SNV + "3000\tG\tA\tPASS\t0/1", + "4000\tT\tC\tPASS\t0/1", // two SNVs, both carried + "4000\tT\tG\tPASS\t0/1", + "4000\tT\tC\tPASS\t0/1", + "5000\tA\tT\tPASS\t0/0", // a split 0/1 at A>G,T: 0/0, then 0/1 + "5000\tA\tG\tPASS\t0/1", + "6000\tC\tT\tPASS\t0/1", // 0/1, then the 0/0 half + "6000\tC\tG\tPASS\t0/0", + "7000\tG\tA\tLowQual\t0/1", // filtered, then usable + "7000\tG\tA\tPASS\t1/1", + "8000\tT\tA\tPASS\t0/0", // two 0/0 halves: the reference + "8000\tT\tG\tPASS\t0/0", + ]; + let body: String = recs + .iter() + .map(|r| { + let c: Vec<&str> = r.split('\t').collect(); + format!( + "chr22\t{}\t.\t{}\t{}\t50\t{}\t.\tGT\t{}\n", + c[0], c[1], c[2], c[3], c[4] + ) + }) + .collect(); + write(&p(&d, "split.vcf"), &format!("{VCF_HEAD}{body}")); + let (code, err) = og(&[ + "genotypes", + "--table-dir", + &sdir, + "--vcf", + &p(&d, "split.vcf"), + "--pseudo", + "10", + "--out-dir", + &p(&d, "gt3"), + ]); + assert_eq!(code, 0, "{err}"); + let g = gt_rows(&p(&d, "gt3/chr22.gt.gz")); + let code = |pos: u32| g.get(&pos).map_or("left out", |c| c[7].as_str()); + assert_eq!(code(1000), "1", "an insertion before the SNV"); + assert_eq!(code(2000), "2", "an insertion after the SNV"); + assert_eq!(code(3000), "1", "a multi-allelic record beside the SNV"); + assert_eq!(code(4000), "9", "two carried SNVs at one position"); + assert_eq!(code(5000), "1", "a 0/0 half before the carried SNV"); + assert_eq!(code(6000), "1", "a 0/0 half after the carried SNV"); + assert_eq!(code(7000), "2", "a usable record wins over a filtered one"); + assert_eq!(code(8000), "0", "two 0/0 halves are the reference"); +} + +#[test] +fn masked_calls_go_to_ibdmix_with_a_mask_per_genome() { + let d = tmp("masked"); + let mdir = p(&d, "table_masked"); + std::fs::create_dir_all(&mdir).unwrap(); + let rows: String = [ + (100, "A\tG", "12\t0\t9"), + (200, "C\tT", "10\t11\t0"), + (300, "G\tA", "2\t0\t0"), + (400, "T\tC", "10\t0\t0"), + (500, "A\tC", "0\t0\t0"), + (600, "C\tT", "11\t0\t0"), + ] + .iter() + .map(|(pos, ra, c)| format!("chr22\t{pos}\t{ra}\t{c}\t0.2\t0\t0\t0\t0\t0\n")) + .collect(); + gzip( + &format!("{mdir}/chr22.tsv.gz"), + &format!("{TABLE_HEADER}\n{rows}"), + ); + // Another ALT: row 500 is left out. + write( + &p(&d, "m.vcf"), + &format!("{VCF_HEAD}chr22\t500\t.\tA\tT\t50\tPASS\t.\tGT\t0/1\n"), + ); + write( + &p(&d, "m.g.vcf"), + &format!("{VCF_HEAD}{}", ref_block("chr22", 1, 1000)), + ); + let (code, err) = og(&[ + "genotypes", + "--table-dir", + &mdir, + "--vcf", + &p(&d, "m.vcf"), + "--gvcf", + &p(&d, "m.g.vcf"), + "--pseudo", + "10", + "--out-dir", + &p(&d, "gtm"), + ]); + assert_eq!(code, 0, "{err}"); + let g = gt_rows(&p(&d, "gtm/chr22.gt.gz")); + let calls = |pos: u32| g[&pos][4..7].to_vec(); + assert_eq!(calls(100), ["2", "0", "9"], "12 is 2"); + assert_eq!(calls(200), ["0", "1", "0"], "10 and 11 are 0 and 1"); + assert_eq!(calls(600), ["1", "0", "0"]); + let mask = + |a: &str| std::fs::read_to_string(p(&d, &format!("gtm/chr22.{a}.mask.bed"))).unwrap(); + assert_eq!( + mask("Altai"), + "22\t99\t200\n22\t399\t600\n", + "the left-out row 500 does not close the run" + ); + assert_eq!(mask("Vindija"), "22\t199\t200\n"); + assert_eq!(mask("Denisova"), "", "written and empty"); + + let bdir = p(&d, "table_bad"); + std::fs::create_dir_all(&bdir).unwrap(); + gzip( + &format!("{bdir}/chr22.tsv.gz"), + &format!("{TABLE_HEADER}\nchr22\t100\tA\tG\t7\t0\t0\t0.2\t0\t0\t0\t0\t0\n"), + ); + let (code, err) = og(&[ + "genotypes", + "--table-dir", + &bdir, + "--vcf", + &p(&d, "m.vcf"), + "--out-dir", + &p(&d, "gtb"), + ]); + assert!(code == 1 && err.contains("code '7'"), "{code} {err}"); +} + +#[test] +fn unsorted_input_stops_and_keeps_the_previous_outputs() { + let d = tmp("order"); + let tdir = genotype_inputs(&d); + for (label, records) in [ + ( + "within-table", + [(3000, "G", "A", "GT"), (1000, "A", "G", "GT")], + ), + ( + "past-table", + [(9000, "A", "G", "GT"), (8000, "A", "G", "GT")], + ), + ( + "ignored-call", + [(3000, "G", "A", "GT"), (1000, "A", ".", "GQ")], + ), + ] { + let bad = p(&d, &format!("{label}.vcf")); + let body: String = records + .iter() + .map(|(pos, r, a, f)| format!("chr22\t{pos}\t.\t{r}\t{a}\t.\tPASS\t.\t{f}\t0/1\n")) + .collect(); + write(&bad, &format!("{VCF_HEAD}{body}")); + let out = p(&d, &format!("order-{label}")); + std::fs::create_dir_all(&out).unwrap(); + let protected: Vec = std::iter::once("chr22.gt.gz".to_owned()) + .chain(ARCHAICS.iter().map(|g| format!("chr22.{g}.mask.bed"))) + .collect(); + for name in &protected { + write(&format!("{out}/{name}"), "previous output\n"); + } + let (code, err) = og(&[ + "genotypes", + "--table-dir", + &tdir, + "--vcf", + &bad, + "--out-dir", + &out, + ]); + assert!( + code == 2 && err.contains("not sorted"), + "{label}: {code} {err}" + ); + for name in &protected { + assert_eq!( + std::fs::read(format!("{out}/{name}")).unwrap(), + b"previous output\n", + "{label} keeps {name}" + ); + } + } + let udir = p(&d, "table_unordered"); + std::fs::create_dir_all(&udir).unwrap(); + gzip( + &format!("{udir}/chr22.tsv.gz"), + &format!( + "{TABLE_HEADER}\n{}\n", + [ROWS[1], ROWS[0], ROWS[2], ROWS[3], ROWS[4]].join("\n") + ), + ); + let out = p(&d, "gt_unordered"); + let (code, err) = og(&[ + "genotypes", + "--table-dir", + &udir, + "--vcf", + &p(&d, "s.vcf"), + "--out-dir", + &out, + ]); + assert!(code == 2 && err.contains("not sorted"), "{code} {err}"); + let left: Vec = std::fs::read_dir(&out) + .map(|r| { + r.map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect() + }) + .unwrap_or_default(); + assert!( + !left + .iter() + .any(|n| n.ends_with(".gt.gz") || n.ends_with(".mask.bed")), + "no partial genotype or mask: {left:?}" + ); +} + +/// The archaic section og summary collect writes for S1 in `dir`. +fn archaic_section(dir: &Path) -> Value { + let input = og_summary::collect::Input { + sample: "S1".to_owned(), + sample_dir: dir.display().to_string(), + declared_sex: None, + assets: repo().join("assets").display().to_string(), + group: og_summary::collect::Group::All, + tables: repo().join("bin").display().to_string(), + }; + let v = og_summary::collect::post::collect(&input).unwrap(); + let get = |v: &Value, k: &str| pycompat::value::get(v, k).unwrap().unwrap().clone(); + get(&get(&v, "sections"), "archaic") +} + +fn field<'a>(v: &'a Value, k: &str) -> &'a Value { + pycompat::value::get(v, k).unwrap().unwrap() +} + +#[test] +fn the_report_merges_segments_and_both_reports_read_it() { + let d = tmp("report"); + let idir = p(&d, "ibd"); + std::fs::create_dir_all(&idir).unwrap(); + let head = "ID\tchrom\tstart\tend\tslod\n"; + write( + &format!("{idir}/chr22.Altai.txt"), + &format!( + "{head}SAMPLE\t22\t1000000\t1300000\t40\nP001\t22\t5000000\t9000000\t99\n\ + SAMPLE\t22\t2000000\t2040000\t30\nSAMPLE\t22\t3000000\t3100000\t3.5\n" + ), + ); + write( + &format!("{idir}/chr22.Vindija.txt"), + &format!("{head}SAMPLE\t22\t1200000\t1500000\t50\n"), + ); + write( + &format!("{idir}/chr22.Denisova.txt"), + &format!("{head}SAMPLE\t22\t1400000\t1450000\t12\nSAMPLE\t22\t6000000\t6200000\t20\n"), + ); + write( + &p(&d, "spans.tsv"), + "chrom\tfirst\tlast\tsites\nchr22\t1\t10000000\t5000\n", + ); + write( + &p(&d, "genes.gtf"), + "chr22\tHAVANA\tgene\t1100000\t1110000\t.\t+\t.\tgene_id \"G1\"; gene_type \"protein_coding\"; gene_name \"GENEA\";\n\ + chr22\tHAVANA\tgene\t1120000\t1130000\t.\t+\t.\tgene_id \"G2\"; gene_type \"lncRNA\"; gene_name \"LNC1\";\n\ + chr22\tHAVANA\tgene\t6100000\t6300000\t.\t-\t.\tgene_id \"G3\"; gene_type \"protein_coding\"; gene_name \"GENEB\";\n", + ); + let (code, err) = og(&[ + "report", + "--sample", + "S1", + "--ibdmix-dir", + &idir, + "--spans", + &p(&d, "spans.tsv"), + "--genes", + &p(&d, "genes.gtf"), + "--out-dir", + &p(&d, "out"), + ]); + assert_eq!(code, 0, "{err}"); + let segs = std::fs::read_to_string(p(&d, "out/S1_archaic_segments.tsv")).unwrap(); + let segs: Vec> = segs.lines().map(|l| l.split('\t').collect()).collect(); + assert_eq!( + segs[0], + [ + "chrom", + "start", + "end", + "length_bp", + "ancestry", + "matched", + "lod", + "genes" + ] + ); + let body = &segs[1..]; + assert_eq!(body.len(), 2, "{body:?}"); + assert_eq!( + body[0][..6], + [ + "chr22", + "1000000", + "1500000", + "500000", + "Neanderthal", + "Altai,Vindija,Denisova" + ], + "Altai 1.0-1.3 and Vindija 1.2-1.5 Mb merge" + ); + assert_eq!(body[0][6], "50.0", "the higher LOD"); + assert_eq!( + body[1][..5], + ["chr22", "6000000", "6200000", "200000", "Denisovan"], + "a Denisova stretch inside a Neanderthal one is not Denisovan; one apart is" + ); + assert_eq!(body[0][7], "GENEA", "protein-coding genes only"); + let summ: BTreeMap = std::fs::read_to_string(p(&d, "out/S1_archaic.tsv")) + .unwrap() + .lines() + .skip(1) + .map(|l| { + let (k, v) = l.split_once('\t').unwrap(); + (k.to_owned(), v.to_owned()) + }) + .collect(); + assert_eq!(summ["neanderthal_percent"], "5.00"); + assert_eq!(summ["denisovan_percent"], "2.00"); + assert_eq!( + [&summ["neanderthal_segments"], &summ["denisovan_segments"]], + ["1", "1"] + ); + assert!(summ["longest_1"].starts_with("chr22:1000000-1500000 Neanderthal")); + assert!(summ["longest_1"].contains("GENEA")); + let txt = std::fs::read_to_string(p(&d, "out/S1_archaic.txt")).unwrap(); + assert!(txt.contains("An estimate, not a measurement."), "{txt}"); + let (code, err) = og(&[ + "report", + "--sample", + "S1", + "--ibdmix-dir", + &p(&d, "nothing"), + "--spans", + &p(&d, "spans.tsv"), + "--out-dir", + &p(&d, "out2"), + ]); + assert!( + code != 0 && err.contains("no IBDmix output"), + "{code} {err}" + ); + + // The summary and the report lines read it. + let sd = d.join("sample"); + std::fs::create_dir_all(sd.join("archaic")).unwrap(); + let tsv = sd.join("archaic/S1_archaic.tsv"); + std::fs::copy(p(&d, "out/S1_archaic.tsv"), &tsv).unwrap(); + let sec = archaic_section(&sd); + assert_eq!(field(&sec, "state"), &Value::Str("ok".into())); + assert_eq!(field(&sec, "step"), &Value::Str("43".into())); + let data = field(&sec, "data"); + assert_eq!( + field(data, "neanderthal_percent"), + &Value::Str("5.00".into()) + ); + assert_eq!(field(data, "denisovan_segments"), &Value::Int(1)); + let Value::Array(longest) = field(data, "longest") else { + panic!("longest is not a list") + }; + assert_eq!(longest.len(), 2); + let lines = og_render::sections::archaic::lines(data).unwrap(); + assert!( + lines[0].starts_with("Neanderthal: 5.00% (0.5 Mb in 1 segments)"), + "{lines:?}" + ); + assert!(lines[1].starts_with("Denisovan: 2.00%"), "{lines:?}"); + for damaged in [ + "key\tvalue\nneanderthal_percent\tlots\n", + "key\tvalue\nneanderthal_segments\n", + ] { + std::fs::write(&tsv, damaged).unwrap(); + assert_eq!( + field(&archaic_section(&sd), "state"), + &Value::Str("unreadable".into()), + "{damaged:?}" + ); + } + let nowhere = archaic_section(&d.join("nowhere")); + assert_eq!(field(&nowhere, "state"), &Value::Str("missing".into())); + assert_eq!(field(&nowhere, "data"), &Value::Object(Default::default())); +} + +/// Python's `random.Random(seed)` for a small seed: MT19937 seeded by +/// `init_by_array([seed])`, and `random()` from two draws. The IBDmix +/// fixture below is the one the Python test drew. +struct PyRandom { + mt: [u32; 624], + i: usize, +} + +impl PyRandom { + fn new(seed: u32) -> PyRandom { + let mut mt = [0u32; 624]; + mt[0] = 19_650_218; + for i in 1..624 { + mt[i] = 1_812_433_253u32 + .wrapping_mul(mt[i - 1] ^ (mt[i - 1] >> 30)) + .wrapping_add(i as u32); + } + let key = [seed]; + let (mut i, mut j) = (1usize, 0usize); + for _ in 0..624 { + mt[i] = (mt[i] ^ (mt[i - 1] ^ (mt[i - 1] >> 30)).wrapping_mul(1_664_525)) + .wrapping_add(key[j]) + .wrapping_add(j as u32); + i += 1; + j += 1; + if i >= 624 { + mt[0] = mt[623]; + i = 1; + } + if j >= key.len() { + j = 0; + } + } + for _ in 0..623 { + mt[i] = (mt[i] ^ (mt[i - 1] ^ (mt[i - 1] >> 30)).wrapping_mul(1_566_083_941)) + .wrapping_sub(i as u32); + i += 1; + if i >= 624 { + mt[0] = mt[623]; + i = 1; + } + } + mt[0] = 0x8000_0000; + PyRandom { mt, i: 624 } + } + + fn next_u32(&mut self) -> u32 { + if self.i >= 624 { + for k in 0..624 { + let y = (self.mt[k] & 0x8000_0000) | (self.mt[(k + 1) % 624] & 0x7fff_ffff); + let mut v = self.mt[(k + 397) % 624] ^ (y >> 1); + if y & 1 == 1 { + v ^= 0x9908_b0df; + } + self.mt[k] = v; + } + self.i = 0; + } + let mut y = self.mt[self.i]; + self.i += 1; + y ^= y >> 11; + y ^= (y << 7) & 0x9d2c_5680; + y ^= (y << 15) & 0xefc6_0000; + y ^ (y >> 18) + } + + fn random(&mut self) -> f64 { + let a = f64::from(self.next_u32() >> 5); + let b = f64::from(self.next_u32() >> 6); + (a * 67_108_864.0 + b) / 9_007_199_254_740_992.0 + } + + fn uniform(&mut self, a: f64, b: f64) -> f64 { + a + (b - a) * self.random() + } +} + +#[test] +fn the_fixture_generator_draws_what_python_drew() { + // Python 3: random.Random(11), three random() then uniform(0.01, 0.04). + let mut r = PyRandom::new(11); + let got = [r.random(), r.random(), r.random(), r.uniform(0.01, 0.04)]; + assert_eq!( + got, + [ + 0.452_379_553_509_818_6, + 0.559_772_386_080_496, + 0.924_210_584_023_729_4, + 0.023_969_502_102_993_2 + ] + ); +} + +#[cfg(target_arch = "x86_64")] +mod ibdmix { + //! Part 4: the chain with IBDmix itself. + + use super::*; + use std::process::Command; + + const NEA: (u32, u32) = (2_000_000, 2_600_000); + const DEN: (u32, u32) = (3_200_000, 3_600_000); + const MID: (u32, u32) = (2_200_000, 2_400_000); + const MASKED: u32 = 10; + + /// Whether the docker tests run here: always in CI. + fn docker_tests(test: &str) -> bool { + let on = std::env::var_os("CI").is_some_and(|v| !v.is_empty()) + || std::env::var("OG_DOCKER_TESTS").as_deref() == Ok("1"); + if !on { + eprintln!( + "skip: {test} runs IBDmix in docker; set OG_DOCKER_TESTS=1 (CI always runs it)" + ); + } + on + } + + fn image() -> String { + let text = std::fs::read_to_string(repo().join("versions.env")).unwrap(); + let line = text + .lines() + .find(|l| l.starts_with("IBDMIX_IMAGE=")) + .expect("IBDMIX_IMAGE in versions.env"); + let v = &line["IBDMIX_IMAGE=".len()..]; + v.split('#') + .next() + .unwrap() + .trim() + .trim_matches('"') + .to_owned() + } + + fn id(flag: &str) -> String { + let o = Command::new("id").arg(flag).output().unwrap(); + String::from_utf8(o.stdout).unwrap().trim().to_owned() + } + + fn open_dir(name: &str) -> PathBuf { + let d = tmp(name); + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&d, std::fs::Permissions::from_mode(0o777)).unwrap(); + d + } + + /// A 4 Mb chr22 table and one person, drawn as the Python test drew + /// them. A third of the sites carry an allele the Neanderthals share + /// and modern people rarely do, a third one only the Denisovan carries, + /// a third are common. Inside a planted stretch one copy carries the + /// archaic allele. `mid`: the middle of the planted Neanderthal stretch + /// holds sites the person contradicts, with the Neanderthals' calls + /// written masked (10 plus the call) or unknown (9). + fn synthetic(d: &Path, plant: bool, mid: Option<&str>) { + let mut rnd = PyRandom::new(11); + std::fs::create_dir_all(d.join("table")).unwrap(); + let (mut rows, mut calls) = (Vec::new(), Vec::new()); + for (i, pos) in (1_000_000u32..5_000_000).step_by(1000).enumerate() { + let (f, al, vi, de) = match i % 3 { + 0 => (rnd.uniform(0.01, 0.04), 2, 2, 0), + 1 => (rnd.uniform(0.01, 0.04), 0, 0, 2), + _ => { + let f = rnd.uniform(0.1, 0.9); + let n = u32::from(rnd.random() < f) + u32::from(rnd.random() < f); + (f, n, n, n) + } + }; + let (mut ta, mut tv) = (al, vi); + let mut g = [rnd.random() < f, rnd.random() < f]; + if plant && NEA.0 <= pos && pos < NEA.1 { + g[0] = al == 2; + } + if let Some(m) = mid + && MID.0 <= pos + && pos < MID.1 + { + if i.is_multiple_of(3) { + g = [false, false]; + } + (ta, tv) = if m == "masked" { + (al + MASKED, vi + MASKED) + } else { + (9, 9) + }; + } + let fs = format!("{f:.4}"); + rows.push(format!( + "chr22\t{pos}\tA\tG\t{ta}\t{tv}\t{de}\t{}", + [fs.as_str(); 6].join("\t") + )); + if plant && DEN.0 <= pos && pos < DEN.1 { + g[0] = de == 2; + } + if g.iter().any(|x| *x) { + let gt = if g.iter().all(|x| *x) { "1/1" } else { "0/1" }; + calls.push(format!("chr22\t{pos}\t.\tA\tG\t50\tPASS\t.\tGT\t{gt}")); + } + } + gzip( + &p(d, "table/chr22.tsv.gz"), + &format!("{TABLE_HEADER}\n{}\n", rows.join("\n")), + ); + write(&p(d, "s.vcf"), &format!("{VCF_HEAD}{}\n", calls.join("\n"))); + } + + /// genotypes, IBDmix with the flags of scripts/43-archaic.sh, report; + /// the segments as (start, end, length, ancestry). + fn chain(d: &Path, gvcf: Option) -> Vec<(u32, u32, u32, String)> { + let gvcf = + gvcf.unwrap_or_else(|| format!("{VCF_HEAD}{}", ref_block("chr22", 1, 6_000_000))); + write(&p(d, "s.g.vcf"), &gvcf); + let (code, err) = og(&[ + "genotypes", + "--table-dir", + &p(d, "table"), + "--vcf", + &p(d, "s.vcf"), + "--gvcf", + &p(d, "s.g.vcf"), + "--out-dir", + &p(d, "gt"), + ]); + assert_eq!(code, 0, "{err}"); + std::fs::create_dir_all(d.join("ibd")).unwrap(); + let sh = "set -e; gzip -dc /w/gt/chr22.gt.gz > /w/gt/chr22.gt; for a in Altai Vindija Denisova; do \ + ibdmix -g /w/gt/chr22.gt -s /w/gt/samples.txt -n $a -r /w/gt/chr22.$a.mask.bed -o /w/ibd/chr22.$a.txt \ + -d 4 -m 1 -a 0.01 -e 0.002 -c 2; done"; + let o = Command::new("docker") + .args(["run", "--rm", "--network", "none", "-u"]) + .arg(format!("{}:{}", id("-u"), id("-g"))) + .arg("-v") + .arg(format!("{}:/w", d.display())) + .arg(image()) + .args(["sh", "-c", sh]) + .output() + .unwrap_or_else(|e| panic!("docker: {e}")); + assert!( + o.status.success(), + "IBDmix: {}{}", + String::from_utf8_lossy(&o.stderr), + String::from_utf8_lossy(&o.stdout) + ); + let (code, err) = og(&[ + "report", + "--sample", + "S", + "--ibdmix-dir", + &p(d, "ibd"), + "--spans", + &p(d, "gt/spans.tsv"), + "--out-dir", + &p(d, "out"), + ]); + assert_eq!(code, 0, "{err}"); + let body: Vec<(u32, u32, u32, String)> = + std::fs::read_to_string(p(d, "out/S_archaic_segments.tsv")) + .unwrap() + .lines() + .skip(1) + .map(|l| { + let c: Vec<&str> = l.split('\t').collect(); + ( + c[1].parse().unwrap(), + c[2].parse().unwrap(), + c[3].parse().unwrap(), + c[4].to_owned(), + ) + }) + .collect(); + eprintln!("segments in {}: {body:?}", d.display()); + body + } + + fn cover(segs: &[(u32, u32, u32, String)], kind: &str, (lo, hi): (u32, u32)) -> u32 { + segs.iter() + .filter(|s| s.3 == kind) + .map(|s| s.1.min(hi).saturating_sub(s.0.max(lo))) + .sum() + } + + #[test] + fn planted_stretches_come_out_as_one_segment_each() { + if !docker_tests("planted_stretches_come_out_as_one_segment_each") { + return; + } + let d = open_dir("real_planted"); + synthetic(&d, true, None); + let body = chain(&d, None); + let nea: Vec<_> = body + .iter() + .filter(|s| s.3 == "Neanderthal") + .cloned() + .collect(); + let den: Vec<_> = body + .iter() + .filter(|s| s.3 == "Denisovan") + .cloned() + .collect(); + assert_eq!(nea.len(), 1, "{body:?}"); + assert!( + cover(&nea, "Neanderthal", NEA) * 10 >= 9 * (NEA.1 - NEA.0), + "{nea:?}" + ); + assert!( + nea[0].2 <= NEA.1 - NEA.0 + 100_000, + "no more than 100 kb beyond it" + ); + assert_eq!(den.len(), 1, "{body:?}"); + assert!( + cover(&den, "Denisovan", DEN) * 10 >= 9 * (DEN.1 - DEN.0), + "{den:?}" + ); + + let d = open_dir("real_control"); + synthetic(&d, false, None); + assert!(chain(&d, None).is_empty(), "without the planted stretches"); + } + + #[test] + fn masked_mismatches_stop_a_segment_that_unknown_calls_let_through() { + if !docker_tests("masked_mismatches_stop_a_segment_that_unknown_calls_let_through") { + return; + } + let span = MID.1 - MID.0; + let d = open_dir("real_mid_unknown"); + synthetic(&d, true, Some("unknown")); + let unknown = cover(&chain(&d, None), "Neanderthal", MID); + assert!( + unknown * 10 >= 9 * span, + "control: written as unknown the segment runs through ({unknown})" + ); + let d = open_dir("real_mid_masked"); + synthetic(&d, true, Some("masked")); + let masked = cover(&chain(&d, None), "Neanderthal", MID); + assert!( + masked * 10 <= span, + "written as masked no segment covers the middle ({masked})" + ); + } + + #[test] + fn a_stretch_with_no_call_needs_a_reference_block() { + if !docker_tests("a_stretch_with_no_call_needs_a_reference_block") { + return; + } + let (lo, hi) = (200_000u32, 349_000u32); + let mut got = Vec::new(); + for covered in [true, false] { + let d = open_dir(&format!("real_gap_{covered}")); + std::fs::create_dir_all(d.join("table")).unwrap(); + let rows: String = (lo..=hi) + .step_by(1000) + .map(|pos| format!("chr22\t{pos}\tC\tT\t0\t0\t0\t{}\n", ["0.9"; 6].join("\t"))) + .collect(); + gzip( + &p(&d, "table/chr22.tsv.gz"), + &format!("{TABLE_HEADER}\n{rows}"), + ); + write( + &p(&d, "s.vcf"), + &format!("{VCF_HEAD}chr22\t10000000\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\n"), + ); + // Without the block the person has one read site left, at the start. + let block = if covered { + ref_block("chr22", 1, 400_000) + } else { + ref_block("chr22", 1, lo) + }; + let body = chain(&d, Some(format!("{VCF_HEAD}{block}"))); + got.push( + body.iter() + .filter(|s| s.3 == "Neanderthal") + .map(|s| s.2) + .sum::(), + ); + } + assert!( + got[0] >= 100_000, + "control: a stretch read as the reference gives a segment ({got:?})" + ); + assert_eq!(got[1], 0, "no call and no reference block: no segment"); + } +} diff --git a/rust/crates/og-render/tests/ancestry_fine.rs b/rust/crates/og-render/tests/ancestry_fine.rs new file mode 100644 index 000000000..be90b2238 --- /dev/null +++ b/rust/crates/og-render/tests/ancestry_fine.rs @@ -0,0 +1,117 @@ +//! Both reports of step 26b's table (`og ancestry-fine report` writes it): +//! the groups with their bootstrap range, the group the model leaves out, +//! the chromosome RFMix failed on, the painted shares and the painting. +//! It replaces the render part of `tests/test_ancestry_fine_report.py`; +//! og-ancestry's own tests check the table the report command writes. + +use std::path::{Path, PathBuf}; + +use og_render::model::Summary; +use og_render::{html, text}; +use pycompat::json::{Dump, dumps}; + +fn repo() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +/// The table `og ancestry-fine report` wrote for the Python test's inputs. +const TABLE: &str = "level\tcode\tgroup\tfraction\tse\tlow\thigh +continent\tWEA\tWest Eurasia (Europe, Middle East, North Africa)\t0.9000\t0.0332\t0.8025\t0.8975 +continent\tEAS\tEast Asia\t0.0500\t0.0000\t0.0500\t0.0500 +continent\tAFR\tAfrica\t0.0500\t0.0332\t0.0525\t0.1475 +region\tIBE\tIberia\t0.6000\t.\t.\t. +region\tITA\tItaly\t0.4000\t.\t.\t. +painting\tWEA\tWest Eurasia (Europe, Middle East, North Africa)\t0.7000\t.\t.\t. +painting\tAFR\tAfrica\t0.3000\t.\t.\t. +left_out\tOCE\tOceania (continental group)\t.\t.\t.\t. +left_out\tOCE\tOceania (regional group)\t.\t.\t.\t. +not_painted\tchr19\tRFMix failed\t.\t.\t.\t. +"; + +fn sample(tag: &str, painting: bool) -> PathBuf { + let d = std::env::temp_dir().join(format!("og-ancestry-fine-{}-{tag}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(d.join("ancestry_fine")).unwrap(); + std::fs::write(d.join("ancestry_fine/S_ancestry_fine.tsv"), TABLE).unwrap(); + if painting { + std::fs::write( + d.join("ancestry_fine/S_painting.png"), + b"\x89PNG\r\n\x1a\nnot a real picture", + ) + .unwrap(); + } + d +} + +fn reports(d: &Path) -> (String, String) { + let input = og_summary::collect::Input { + sample: "S".to_owned(), + sample_dir: d.display().to_string(), + declared_sex: None, + assets: repo().join("assets").display().to_string(), + group: og_summary::collect::Group::All, + tables: repo().join("bin").display().to_string(), + }; + let summary = dumps( + &og_summary::collect::post::collect(&input).unwrap(), + &Dump::default(), + ) + .unwrap(); + let s = Summary::parse(&summary).unwrap(); + (text::report(&s).unwrap(), html::report(&s).unwrap()) +} + +#[test] +fn both_reports_name_the_group_left_out_and_the_chromosome_not_painted() { + let (txt, page) = reports(&sample("full", true)); + for doc in [&txt, &page] { + assert!( + doc.contains( + "Left out of the model: Oceania (continental and regional group). The panel has \ + fewer than 10" + ), + "{doc}" + ); + assert!(doc.contains("not painted: RFMix failed on chr19"), "{doc}"); + } + assert!(txt.contains("Fine-Grained Ancestry (ADMIXTURE, RFMix)")); + assert!( + txt.contains(" West Eurasia (Europe, Middle East, North Africa) 90% (80% to 90%)"), + "{txt}" + ); + assert!( + txt.contains( + " Chromosome painting, share of the painted genome: West Eurasia (Europe, Middle \ + East, North Africa) 70%, Africa 30%; not painted: RFMix failed on chr19" + ), + "{txt}" + ); + assert!(page.contains("

    Fine-Grained Ancestry

    ")); + assert!(page.contains( + "West Eurasia (Europe, Middle East, North Africa)90%80% to \ + 90%" + )); + assert!( + page.contains("Iberia60%."), + "a level without bootstrap replicates has no range" + ); + assert!( + page.contains("src=\"data:image/png;base64,iVBORw0KGg"), + "the painting" + ); + assert!( + page.contains("■ not painted

    "), + "its legend" + ); +} + +#[test] +fn without_a_picture_the_card_has_no_image() { + let (_, page) = reports(&sample("nopng", false)); + assert!(page.contains("

    Fine-Grained Ancestry

    ")); + assert!(!page.contains("data:image/png;base64,")); + assert!( + !page.contains("not painted

    "), + "no legend without the painting" + ); +} diff --git a/rust/crates/og-render/tests/ancestry_haplogroups.rs b/rust/crates/og-render/tests/ancestry_haplogroups.rs new file mode 100644 index 000000000..b8499ae0a --- /dev/null +++ b/rust/crates/og-render/tests/ancestry_haplogroups.rs @@ -0,0 +1,169 @@ +//! Both reports point to where a haplogroup can be explored further (steps +//! 12 and 37). It replaces `tests/test_haplogroup_explore.py`. +//! +//! Each case writes the haplogroup files of one made-up sample S into an +//! output folder, collects the summary and renders both reports: +//! +//! 1. a male sample (Yleaf table with R-M269, haplogrep3 with H5a7): the Y +//! and the mtDNA links are built from those names, in the text and the +//! HTML report, with the note that nothing is uploaded; +//! 2. Yleaf's paragroup form R-M269*(xR-L23): the links use R-M269; +//! 3. a female sample (no Yleaf table): no Y links; the mtDNA links stay; +//! 4. no call (Hg NA, haplogroup ''): no links at all; +//! 5. a branch name with characters a URL cannot carry raw is encoded; +//! 6. an address that would close the link's attribute and open a tag is +//! escaped in the HTML report, so the page never runs it. +//! +//! The branch names are the fixture's and public examples, not anyone's +//! result. + +use std::path::{Path, PathBuf}; + +use og_render::model::Summary; +use og_render::{html, text}; +use pycompat::json::{Dump, dumps}; + +const Y_HEAD: &str = + "Sample_name\tHg\tHg_marker\tTotal_reads\tValid_markers\tQC-score\tQC-1\tQC-2\tQC-3\n"; + +fn repo() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +fn put(path: &Path, text: &str) { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, text).unwrap(); +} + +/// A sample folder with haplogrep3's call and, for a male, Yleaf's table. +fn sample(name: &str, y_hg: Option<&str>, mt_hg: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("og-ancestry-hg-{}-{name}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + put( + &d.join("mito/S_haplogroup.txt"), + &format!("\"SampleID\"\t\"Haplogroup\"\t\"Rank\"\n\"S\"\t\"{mt_hg}\"\t\"0.98\"\n"), + ); + if let Some(hg) = y_hg { + put( + &d.join("y_haplogroup/S_y_haplogroup.txt"), + &format!("{Y_HEAD}S_sorted\t{hg}\tM269\t100\t42\t0.97\t1\t1\t0.97\n"), + ); + } + d +} + +/// The summary og summary collect writes for S, as text. +fn collect(d: &Path) -> String { + let input = og_summary::collect::Input { + sample: "S".to_owned(), + sample_dir: d.display().to_string(), + declared_sex: None, + assets: repo().join("assets").display().to_string(), + group: og_summary::collect::Group::All, + tables: repo().join("bin").display().to_string(), + }; + dumps( + &og_summary::collect::post::collect(&input).unwrap(), + &Dump::default(), + ) + .unwrap() +} + +/// Both reports of a summary. +fn render(summary: &str) -> (String, String) { + let s = Summary::parse(summary).unwrap(); + (text::report(&s).unwrap(), html::report(&s).unwrap()) +} + +fn reports(d: &Path) -> (String, String) { + render(&collect(d)) +} + +#[test] +fn a_male_sample_gets_the_y_and_the_mtdna_links() { + let (txt, page) = reports(&sample("male", Some("R-M269"), "H5a7")); + for url in [ + "https://discover.familytreedna.com/y-dna/R-M269", + "https://www.yfull.com/tree/R-M269/", + "https://discover.familytreedna.com/mtdna/H5a7", + "https://www.yfull.com/mtree/H5a7/", + ] { + assert!(txt.contains(url), "{url} in the text report:\n{txt}"); + assert!( + page.contains(&format!("
    ")), + "{url} as a link in the HTML report" + ); + } + assert!(txt.contains("docs/37-y-haplogroup.md#explore-your-branch-further")); + assert!(txt.contains("docs/12-mito-haplogroup.md#explore-your-branch-further")); + assert!(txt.contains("Nothing is uploaded by this pipeline")); + assert!(page.contains("Nothing is uploaded by this pipeline")); +} + +#[test] +fn a_paragroup_links_its_named_branch() { + let (txt, _) = reports(&sample("para", Some("R-M269*(xR-L23)"), "H5a7")); + assert!(txt.contains("https://www.yfull.com/tree/R-M269/"), "{txt}"); + assert!(!txt.contains("y-dna/R-M269*"), "{txt}"); + assert!(!txt.contains("%2A"), "{txt}"); +} + +#[test] +fn a_female_sample_has_no_y_link_and_keeps_the_mtdna_ones() { + let (txt, page) = reports(&sample("female", None, "H5a7")); + for t in [&txt, &page] { + assert!(!t.contains("y-dna/"), "{t}"); + assert!(!t.contains("yfull.com/tree/"), "{t}"); + } + assert!(txt.contains("https://discover.familytreedna.com/mtdna/H5a7")); + assert!(page.contains("https://www.yfull.com/mtree/H5a7/")); +} + +#[test] +fn no_call_has_no_link() { + let (txt, page) = reports(&sample("nocall", Some("NA"), "")); + assert!(!txt.contains("discover.familytreedna.com"), "{txt}"); + assert!(!txt.contains("yfull.com"), "{txt}"); + assert!(!page.contains("discover.familytreedna.com")); + assert!( + txt.contains("insufficient markers"), + "the Y section still says why:\n{txt}" + ); +} + +#[test] +fn a_plus_in_the_branch_is_encoded() { + let (txt, _) = reports(&sample("esc", None, "L3e2b+152")); + assert!(txt.contains("mtdna/L3e2b%2B152"), "{txt}"); + assert!(txt.contains("mtree/L3e2b%2B152/"), "{txt}"); +} + +#[test] +fn a_planted_address_cannot_open_a_tag_in_the_html_report() { + let summary = collect(&sample("planted", Some("R-M269"), "H5a7")); + let planted = r#"https://x.example/\"><"), "it ended the attribute"); + assert_eq!( + page.matches(""><script>alert(1)</script>") + .count(), + 4, + "escaped in the address and the text of both links" + ); + assert!( + txt.contains("https://x.example/\">" | .sections.haplogroup.data.explore[1].name = "YFull & co" | .sections.haplogroup.data.contamination_status = "YES" | .sections.haplogroup.data.contamination_level = "0.4<" - | .sections.y_haplogroup.data.explore[1].url = "javascript:alert(\"x\")" + | .sections.y_haplogroup.data.explore[1].url = "https://www.yfull.com/tree/\"x\"&y/" | .sections.y_tree.data.path[0].Branch = "" | .sections.y_tree.data.path[0].Ancestral_SNPs = "M1&M2" | .sections.y_str.data.markers[0].marker = "DYS<393>" | .sections.ancient.data.nearest[0].group = "Spain_" | .sections.ancient.data.nearest[0].locality = "A & B" From c17b6fb41f79f1c947d73e6eb00c6a7bbdcf9d16 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 18 Nov 2026 13:44:03 +0100 Subject: [PATCH 7/7] fix(report): keep the legend CSS, explore links and test folders safe - A painting legend colour that is not #rrggbb shows as a fixed grey, so a summary cannot add a CSS declaration to the legend. - An explore address needs a host to become a link, and only the two fixed docs pages get the plain "opens a website" note. - A PRS row with no score reads "not available" in the HTML result cell, not "raw score only". - The ancestry tests create their scratch folders exclusively and guard them before writing, fail on duplicate genotype rows, compare BED rows by coordinates, fail on an unknown region name, and render the archaic summary through both reports. --- CHANGELOG.md | 7 +- .../crates/og-render/src/sections/ancestry.rs | 23 +++++- .../og-render/src/sections/ancestry_fine.rs | 10 ++- rust/crates/og-render/src/sections/prs.rs | 8 +- .../og-render/tests/ancestry_ancient.rs | 22 ++++- .../og-render/tests/ancestry_archaic.rs | 82 +++++++++++++++---- rust/crates/og-render/tests/ancestry_fine.rs | 59 +++++++++++-- .../og-render/tests/ancestry_haplogroups.rs | 58 ++++++++++++- .../og-render/tests/ancestry_prs_null.rs | 5 +- rust/crates/og-render/tests/ancestry_ystr.rs | 50 ++++++++--- tests/parity/gen/report-corpus.sh | 6 +- 11 files changed, 276 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f57d461ab..ee1255385 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,11 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- `og report render` writes the ancestry chapter and the polygenic score card in Rust: fine-grained ancestry with the painting and its legend, the mitochondrial haplogroup, the Y chromosome (Yleaf, the placement on the current public tree, the Y-STR markers in FamilyTreeDNA's order), archaic and ancient ancestry, and the PRS table with its ancestry and sex notes. Both reports read as `bin/render_report.py` writes them, except: - - A PRS row whose score, matched or total the summary has as null (a short row of pgsc_calc's table) reads "not available" in both reports. The Python wrote "None" in the text report and stopped the whole HTML report on that row. +- `og report render` writes most of the ancestry chapter and the polygenic score card in Rust: fine-grained ancestry with the painting and its legend, the mitochondrial haplogroup, the Y chromosome (Yleaf, the placement on the current public tree, the Y-STR markers in FamilyTreeDNA's order), archaic and ancient ancestry, and the PRS table with its ancestry and sex notes. Runs of homozygosity are not written yet, so both Rust reports leave them out even when the summary has them. Both reports read as `bin/render_report.py` writes them, except: + - A PRS row whose score, matched or total the summary has as null (a short row of pgsc_calc's table) reads "not available" in both reports, and in the HTML result cell when there is no score either. The Python wrote "None" in the text report and stopped the whole HTML report on that row. - The painting and the ancient PCA plot are escaped in their `src` attribute. A real picture is base64 and does not change; a value that is not text renders instead of stopping the report. - - A haplogroup's explore address that is not a web address (`http://` or `https://`), such as a `javascript:` one, shows as text in the HTML report. The Python made it a link, which would run it when clicked. + - A haplogroup's explore address that is not a web address with a host (`http://` or `https://`), such as a `javascript:` one, shows as text in the HTML report. The Python made it a link, which would run it when clicked. Only the two docs pages the collector links get the plain "opens a website" note; any other address gets the note that it carries the haplogroup. + - A painting legend colour that is not `#rrggbb` shows as a fixed grey, so a summary cannot add CSS to the legend. The collector always writes `#rrggbb`. - The charts are not part of these cards yet; `og report charts` will write them. The tests of these sections moved from Python to cargo (`rust/crates/og-render/tests/ancestry_*.rs`): `tests/test_ancient_report.py`, `test_archaic_report.py`, `test_haplogroup_explore.py` and `test_ystr_report.py` are gone, and `test_ancestry_fine_report.py` no longer renders the reports. The IBDmix runs of the archaic test need docker and amd64; they run in CI. - A new page, [User journeys](docs/user-journeys.md), records where a new user gets stuck on each surface: the docs, the demo, the shell and the local site. It gives the outcome of each journey in the first test-as-user run (2026-11-17, commit `65095e97`), the fix for each finding, and what the run could not reach. A fix that lands has a journey to walk again. diff --git a/rust/crates/og-render/src/sections/ancestry.rs b/rust/crates/og-render/src/sections/ancestry.rs index b134da47e..4f5d38853 100644 --- a/rust/crates/og-render/src/sections/ancestry.rs +++ b/rust/crates/og-render/src/sections/ancestry.rs @@ -17,6 +17,12 @@ use crate::notes::EXPLORE_NOTE; /// Every explore address but the docs one carries the haplogroup in it. pub const EXPLORE_DOCS: &str = "https://github.com/GeiserX/OwnGenome/blob/main/docs/"; +/// The pages under [`EXPLORE_DOCS`] the collector links, one per haplogroup. +const EXPLORE_DOC_PAGES: [&str; 2] = [ + "37-y-haplogroup.md#explore-your-branch-further", + "12-mito-haplogroup.md#explore-your-branch-further", +]; + /// The mark of an address that leaves this computer with the haplogroup in it. pub const OFFSITE_HAPLOGROUP: &str = "opens a website; the address carries the haplogroup"; @@ -152,16 +158,27 @@ pub fn explore_html(d: &Value, path: &str) -> Result<(Vec, bool), Malforme let Value::Str(url) = url else { return Err(Malformed(format!("{path}.explore[].url is not a text"))); }; - let why = if url.starts_with(EXPLORE_DOCS) { + // Only the two fixed pages the collector links get the plain + // warning: any other address, even under the docs folder, may carry + // the haplogroup in its query. + let why = if url + .strip_prefix(EXPLORE_DOCS) + .is_some_and(|page| EXPLORE_DOC_PAGES.contains(&page)) + { crate::html::OFFSITE } else { OFFSITE_HAPLOGROUP }; - // Only a web address becomes a link: a `javascript:` or `data:` - // address in a summary would run when clicked, so it shows as text. + // Only a web address with a host becomes a link: a `javascript:` or + // `data:` address in a summary would run when clicked, and a bare + // `https://` leads nowhere, so both show as text. let web = ["http://", "https://"].iter().any(|s| { url.get(..s.len()) .is_some_and(|head| head.eq_ignore_ascii_case(s)) + && url[s.len()..] + .split(['/', '?', '#']) + .next() + .is_some_and(|host| !host.is_empty()) }); let value = if web { offsite_link(&url, &url, why) diff --git a/rust/crates/og-render/src/sections/ancestry_fine.rs b/rust/crates/og-render/src/sections/ancestry_fine.rs index 67b9c2134..08516c1f9 100644 --- a/rust/crates/og-render/src/sections/ancestry_fine.rs +++ b/rust/crates/og-render/src/sections/ancestry_fine.rs @@ -265,7 +265,7 @@ pub fn html(s: &Summary) -> Result { .map(|x| { Ok(format!( "■ {}", - e(&need_text(x, "colour", &at)?), + css_colour(&need_text(x, "colour", &at)?), e(&need_text(x, "group", &at)?) )) }) @@ -282,3 +282,11 @@ pub fn html(s: &Summary) -> Result { files: Vec::new(), }) } + +/// A legend colour as CSS: the `#rrggbb` the collector writes, else a fixed +/// grey. Escaping keeps it inside the attribute but would still let a `;` +/// add a declaration, such as one that hides the swatch. +fn css_colour(c: &str) -> &str { + let hex = c.len() == 7 && c.starts_with('#') && c[1..].bytes().all(|b| b.is_ascii_hexdigit()); + if hex { c } else { "#808080" } +} diff --git a/rust/crates/og-render/src/sections/prs.rs b/rust/crates/og-render/src/sections/prs.rs index 83f06bca9..5a75d2827 100644 --- a/rust/crates/og-render/src/sections/prs.rs +++ b/rust/crates/og-render/src/sections/prs.rs @@ -291,10 +291,12 @@ pub fn html(s: &Summary) -> Result { c("score")?, c("matched")?, c("total")?, - e(if pct.is_empty() { - "raw score only" - } else { + e(if !pct.is_empty() { &pct + } else if matches!(need(x, "score", ROW)?, Value::Null) { + NOT_AVAILABLE + } else { + "raw score only" }) ); if show_effect { diff --git a/rust/crates/og-render/tests/ancestry_ancient.rs b/rust/crates/og-render/tests/ancestry_ancient.rs index 1790e0ec3..47bc88bcc 100644 --- a/rust/crates/og-render/tests/ancestry_ancient.rs +++ b/rust/crates/og-render/tests/ancestry_ancient.rs @@ -64,6 +64,23 @@ impl AsRef for Tmp { } } +impl Tmp { + /// A new folder that only this test owns. `create_dir` fails when the + /// path exists, so a run never shares or deletes another run's folder. + fn new(kind: &str, tag: &str) -> Tmp { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let d = std::env::temp_dir().join(format!( + "og-ancestry-{kind}-{}-{tag}-{nanos}", + std::process::id() + )); + std::fs::create_dir(&d).unwrap(); + Tmp(d) + } +} + impl Drop for Tmp { fn drop(&mut self) { let _ = std::fs::remove_dir_all(&self.0); @@ -71,10 +88,7 @@ impl Drop for Tmp { } fn tmp(tag: &str) -> Tmp { - let d = std::env::temp_dir().join(format!("og-ancestry-ancient-{}-{tag}", std::process::id())); - let _ = std::fs::remove_dir_all(&d); - std::fs::create_dir_all(&d).unwrap(); - Tmp(d) + Tmp::new("ancient", tag) } fn write(d: &Path, name: &str, text: &str) -> String { diff --git a/rust/crates/og-render/tests/ancestry_archaic.rs b/rust/crates/og-render/tests/ancestry_archaic.rs index a83e6fcd4..e6c53a5af 100644 --- a/rust/crates/og-render/tests/ancestry_archaic.rs +++ b/rust/crates/og-render/tests/ancestry_archaic.rs @@ -72,6 +72,23 @@ impl AsRef for Tmp { } } +impl Tmp { + /// A new folder that only this test owns. `create_dir` fails when the + /// path exists, so a run never shares or deletes another run's folder. + fn new(kind: &str, tag: &str) -> Tmp { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let d = std::env::temp_dir().join(format!( + "og-ancestry-{kind}-{}-{tag}-{nanos}", + std::process::id() + )); + std::fs::create_dir(&d).unwrap(); + Tmp(d) + } +} + impl Drop for Tmp { fn drop(&mut self) { let _ = std::fs::remove_dir_all(&self.0); @@ -79,10 +96,7 @@ impl Drop for Tmp { } fn tmp(tag: &str) -> Tmp { - let d = std::env::temp_dir().join(format!("og-ancestry-archaic-{}-{tag}", std::process::id())); - let _ = std::fs::remove_dir_all(&d); - std::fs::create_dir_all(&d).unwrap(); - Tmp(d) + Tmp::new("archaic", tag) } fn p(d: &Path, name: &str) -> String { @@ -387,14 +401,16 @@ fn strict_marks_rows_outside_the_mask_and_keeps_each_position_once() { /// A genotype file by position. fn gt_rows(path: &str) -> BTreeMap> { - gunzip(path) - .lines() - .skip(1) - .map(|l| { - let c: Vec = l.split('\t').map(str::to_owned).collect(); - (c[1].parse().unwrap(), c) - }) - .collect() + let mut rows = BTreeMap::new(); + for l in gunzip(path).lines().skip(1) { + let c: Vec = l.split('\t').map(str::to_owned).collect(); + let pos: u32 = c[1].parse().unwrap(); + assert!( + rows.insert(pos, c).is_none(), + "two genotype rows for position {pos} in {path}" + ); + } + rows } const ROWS: [&str; 5] = [ @@ -731,12 +747,13 @@ fn unsorted_input_stops_and_keeps_the_previous_outputs() { &out, ]); assert!(code == 2 && err.contains("not sorted"), "{code} {err}"); - let left: Vec = std::fs::read_dir(&out) - .map(|r| { - r.map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) - .collect() - }) - .unwrap_or_default(); + let left: Vec = match std::fs::read_dir(&out) { + Ok(r) => r + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect(), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(), + Err(e) => panic!("cannot read {out}: {e}"), + }; assert!( !left .iter() @@ -760,6 +777,27 @@ fn archaic_section(dir: &Path) -> Value { get(&get(&v, "sections"), "archaic") } +/// Both reports of the summary og summary collect writes for S1. +fn reports(dir: &Path) -> (String, String) { + let input = og_summary::collect::Input { + sample: "S1".to_owned(), + sample_dir: dir.display().to_string(), + declared_sex: None, + assets: repo().join("assets").display().to_string(), + group: og_summary::collect::Group::All, + tables: repo().join("bin").display().to_string(), + }; + let v = og_summary::collect::post::collect(&input).unwrap(); + let s = og_render::model::Summary::parse( + &pycompat::json::dumps(&v, &pycompat::json::Dump::default()).unwrap(), + ) + .unwrap(); + ( + og_render::text::report(&s).unwrap(), + og_render::html::report(&s).unwrap(), + ) +} + fn field<'a>(v: &'a Value, k: &str) -> &'a Value { pycompat::value::get(v, k).unwrap().unwrap() } @@ -904,6 +942,14 @@ fn the_report_merges_segments_and_both_reports_read_it() { "{lines:?}" ); assert!(lines[1].starts_with("Denisovan: 2.00%"), "{lines:?}"); + let (txt, page) = reports(&sd); + let shares = "Neanderthal: 5.00% (0.5 Mb in 1 segments)"; + assert!(txt.contains(shares), "the text report: {txt}"); + assert!( + page.contains("

    Archaic Ancestry (Neanderthal, Denisovan)

    ") + && page.contains(shares), + "the HTML report" + ); for damaged in [ "key\tvalue\nneanderthal_percent\tlots\n", "key\tvalue\nneanderthal_segments\n", diff --git a/rust/crates/og-render/tests/ancestry_fine.rs b/rust/crates/og-render/tests/ancestry_fine.rs index 820ffdfa9..6387423ec 100644 --- a/rust/crates/og-render/tests/ancestry_fine.rs +++ b/rust/crates/og-render/tests/ancestry_fine.rs @@ -50,6 +50,23 @@ impl AsRef for Tmp { } } +impl Tmp { + /// A new folder that only this test owns. `create_dir` fails when the + /// path exists, so a run never shares or deletes another run's folder. + fn new(kind: &str, tag: &str) -> Tmp { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let d = std::env::temp_dir().join(format!( + "og-ancestry-{kind}-{}-{tag}-{nanos}", + std::process::id() + )); + std::fs::create_dir(&d).unwrap(); + Tmp(d) + } +} + impl Drop for Tmp { fn drop(&mut self) { let _ = std::fs::remove_dir_all(&self.0); @@ -57,9 +74,8 @@ impl Drop for Tmp { } fn sample(tag: &str, painting: bool) -> Tmp { - let d = std::env::temp_dir().join(format!("og-ancestry-fine-{}-{tag}", std::process::id())); - let _ = std::fs::remove_dir_all(&d); - std::fs::create_dir_all(d.join("ancestry_fine")).unwrap(); + let d = Tmp::new("fine", tag); + std::fs::create_dir(d.join("ancestry_fine")).unwrap(); std::fs::write(d.join("ancestry_fine/S_ancestry_fine.tsv"), TABLE).unwrap(); if painting { std::fs::write( @@ -68,10 +84,15 @@ fn sample(tag: &str, painting: bool) -> Tmp { ) .unwrap(); } - Tmp(d) + d } fn reports(d: &Path) -> (String, String) { + reports_with(d, |s| s) +} + +/// Both reports of the summary og summary collect writes, after `edit`. +fn reports_with(d: &Path, edit: impl Fn(String) -> String) -> (String, String) { let input = og_summary::collect::Input { sample: "S".to_owned(), sample_dir: d.display().to_string(), @@ -85,7 +106,7 @@ fn reports(d: &Path) -> (String, String) { &Dump::default(), ) .unwrap(); - let s = Summary::parse(&summary).unwrap(); + let s = Summary::parse(&edit(summary)).unwrap(); (text::report(&s).unwrap(), html::report(&s).unwrap()) } @@ -131,6 +152,34 @@ fn both_reports_name_the_group_left_out_and_the_chromosome_not_painted() { page.contains("■ not painted

    "), "its legend" ); + let [r, g, b] = og_ancestry::COLOURS + .iter() + .find(|(code, _)| *code == "WEA") + .unwrap() + .1; + let entry = format!( + "■ West Eurasia (Europe, \ + Middle East, North Africa)" + ); + assert!( + page.contains(&entry), + "a group of the legend, in its colour" + ); +} + +#[test] +fn a_legend_colour_that_is_not_a_hex_colour_is_shown_grey() { + let d = sample("colour", true); + let (_, page) = reports_with(&d, |s| { + let at = s.find("\"colour\": \"#").expect("a legend colour"); + let start = at + "\"colour\": \"".len(); + format!("{}red;display:none{}", &s[..start], &s[start + 7..]) + }); + assert!(!page.contains("display:none"), "a planted CSS declaration"); + assert!( + page.contains("■"), + "the fixed grey" + ); } #[test] diff --git a/rust/crates/og-render/tests/ancestry_haplogroups.rs b/rust/crates/og-render/tests/ancestry_haplogroups.rs index 8950b1812..79419f488 100644 --- a/rust/crates/og-render/tests/ancestry_haplogroups.rs +++ b/rust/crates/og-render/tests/ancestry_haplogroups.rs @@ -57,6 +57,23 @@ impl AsRef for Tmp { } } +impl Tmp { + /// A new folder that only this test owns. `create_dir` fails when the + /// path exists, so a run never shares or deletes another run's folder. + fn new(kind: &str, tag: &str) -> Tmp { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let d = std::env::temp_dir().join(format!( + "og-ancestry-{kind}-{}-{tag}-{nanos}", + std::process::id() + )); + std::fs::create_dir(&d).unwrap(); + Tmp(d) + } +} + impl Drop for Tmp { fn drop(&mut self) { let _ = std::fs::remove_dir_all(&self.0); @@ -65,8 +82,7 @@ impl Drop for Tmp { /// A sample folder with haplogrep3's call and, for a male, Yleaf's table. fn sample(name: &str, y_hg: Option<&str>, mt_hg: &str) -> Tmp { - let d = std::env::temp_dir().join(format!("og-ancestry-hg-{}-{name}", std::process::id())); - let _ = std::fs::remove_dir_all(&d); + let d = Tmp::new("hg", name); put( &d.join("mito/S_haplogroup.txt"), &format!("\"SampleID\"\t\"Haplogroup\"\t\"Rank\"\n\"S\"\t\"{mt_hg}\"\t\"0.98\"\n"), @@ -77,7 +93,7 @@ fn sample(name: &str, y_hg: Option<&str>, mt_hg: &str) -> Tmp { &format!("{Y_HEAD}S_sorted\t{hg}\tM269\t100\t42\t0.97\t1\t1\t0.97\n"), ); } - Tmp(d) + d } /// The summary og summary collect writes for S, as text. @@ -217,3 +233,39 @@ fn an_address_that_is_not_a_web_address_is_shown_as_text() { ); assert!(txt.contains("javascript:alert(1)"), "{txt}"); } + +#[test] +fn an_address_with_no_host_is_text_and_only_the_fixed_docs_get_the_plain_warning() { + let summary = collect(&sample("host", Some("R-M269"), "H5a7")); + let mt = "https://www.yfull.com/mtree/H5a7/"; + let docs = "https://github.com/GeiserX/OwnGenome/blob/main/docs/37-y-haplogroup.md\ + #explore-your-branch-further"; + assert!(summary.contains(mt) && summary.contains(docs), "{summary}"); + let query = "https://github.com/GeiserX/OwnGenome/blob/main/docs/x?hg=R-M269"; + let s = summary.replace(mt, "https://").replace(docs, query); + let (_, page) = render(&s); + assert!( + !page.contains("href=\"https://\""), + "an address with no host became a link" + ); + assert!( + page.contains("https://"), + "it is shown as text" + ); + assert!( + page.contains(&format!( + "
    {} (opens a website; the address \ + carries the haplogroup)", + query.replace('&', "&"), + query.replace('&', "&") + )), + "a docs address with a query gets the haplogroup warning" + ); + let (_, plain) = render(&summary); + assert!( + plain.contains(&format!( + "{docs} (opens a website)" + )), + "the fixed docs page keeps the plain warning" + ); +} diff --git a/rust/crates/og-render/tests/ancestry_prs_null.rs b/rust/crates/og-render/tests/ancestry_prs_null.rs index 2c50f1118..662792aba 100644 --- a/rust/crates/og-render/tests/ancestry_prs_null.rs +++ b/rust/crates/og-render/tests/ancestry_prs_null.rs @@ -2,7 +2,8 @@ //! matched and total as null (the golden summary //! tests/parity/golden/summary-collect/all/all.json has one, Short/PGS1). //! Both reports render it, and the row reads "not available" in those -//! cells. bin/render_report.py stops the whole HTML report on that row +//! cells and, in the HTML report, in the result cell: with no score there +//! is no raw score either. bin/render_report.py stops the whole HTML report on that row //! (`html.escape(None)`) and writes "None" in the text report, so this is a //! known difference from the Python, outside the parity corpus. @@ -24,7 +25,7 @@ fn a_null_score_reads_not_available_in_both_reports() { assert!( page.contains( "ShortPGS1not availablenot available/not \ - availableraw score only" + availablenot available" ), "the HTML row" ); diff --git a/rust/crates/og-render/tests/ancestry_ystr.rs b/rust/crates/og-render/tests/ancestry_ystr.rs index 9b29d52f2..8fb6e3d0d 100644 --- a/rust/crates/og-render/tests/ancestry_ystr.rs +++ b/rust/crates/og-render/tests/ancestry_ystr.rs @@ -68,6 +68,23 @@ impl AsRef for Tmp { } } +impl Tmp { + /// A new folder that only this test owns. `create_dir` fails when the + /// path exists, so a run never shares or deletes another run's folder. + fn new(kind: &str, tag: &str) -> Tmp { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let d = std::env::temp_dir().join(format!( + "og-ancestry-{kind}-{}-{tag}-{nanos}", + std::process::id() + )); + std::fs::create_dir(&d).unwrap(); + Tmp(d) + } +} + impl Drop for Tmp { fn drop(&mut self) { let _ = std::fs::remove_dir_all(&self.0); @@ -76,10 +93,7 @@ impl Drop for Tmp { /// A fresh folder for one test. fn tmp(tag: &str) -> Tmp { - let d = std::env::temp_dir().join(format!("og-ancestry-ystr-{}-{tag}", std::process::id())); - let _ = std::fs::remove_dir_all(&d); - std::fs::create_dir_all(&d).unwrap(); - Tmp(d) + Tmp::new("ystr", tag) } /// One region of a panel row: chrom, start, end, period, copies, name. @@ -179,7 +193,8 @@ fn vcf(d: &Path, records: &[(&str, Call)], samples: &[&str]) -> PathBuf { .iter() .flat_map(|r| r.regions.iter()) .find(|x| x.5 == *name) - .map_or(("chrY".to_owned(), 1), |x| (x.0.clone(), x.1)); + .map(|x| (x.0.clone(), x.1)) + .unwrap_or_else(|| panic!("no panel region is named {name}")); let cell = format!("{gt}:{gb}:{q}:{q}:{dp}:0:0|0:{dp}|0:.:{st}:{fl}"); let mut row = vec![ c, @@ -384,16 +399,31 @@ fn the_panel_is_familytreedna_s_111_in_order_and_the_bed_holds_each_region_once( assert_eq!(code, 0, "{err}"); let text = std::fs::read_to_string(&bed).unwrap(); let bl: Vec> = text.lines().map(|l| l.split('\t').collect()).collect(); - let mut got: Vec<&str> = bl.iter().map(|x| x[5]).collect(); + // Each row's name and coordinates, with repeats kept, against the + // panel's regions, each once. + let mut got: Vec<(String, i64, i64, String)> = bl + .iter() + .map(|x| { + ( + x[0].to_owned(), + x[1].parse().unwrap(), + x[2].parse().unwrap(), + x[5].to_owned(), + ) + }) + .collect(); got.sort_unstable(); - let mut regions: Vec<&str> = panel + let regions: Vec<(String, i64, i64, String)> = panel .iter() - .flat_map(|r| r.regions.iter().map(|x| x.5.as_str())) + .flat_map(|r| { + r.regions + .iter() + .map(|x| (x.0.clone(), x.1, x.2, x.5.clone())) + }) .collect::>() .into_iter() .collect(); - regions.sort_unstable(); - assert_eq!(got, regions, "the BED has every region once"); + assert_eq!(got, regions, "the BED has every region once, at its place"); assert!(bl.iter().all(|x| x.len() == 6 && 0 < x[1].parse::().unwrap() && x[1].parse::().unwrap() < x[2].parse::().unwrap())); diff --git a/tests/parity/gen/report-corpus.sh b/tests/parity/gen/report-corpus.sh index 9ed693ea4..48cd74cb9 100644 --- a/tests/parity/gen/report-corpus.sh +++ b/tests/parity/gen/report-corpus.sh @@ -127,7 +127,9 @@ rm -f "$base" # writes text (a text-only case: the Python's HTML report cannot render it). # A planted address that is not a web address (javascript:) is no case: og # shows it as text where the Python made it a link; the cargo test -# ancestry_haplogroups.rs covers it. +# ancestry_haplogroups.rs covers it. Nor is a legend colour that is not +# #rrggbb: og shows it grey where the Python put it in the CSS; the cargo +# test ancestry_fine.rs covers it, and a-escape plants a mixed-case one. [ -s c-full.json ] || { echo "report-corpus.sh: og summary collect wrote no c-full.json" >&2; exit 1; } abase=abase.tmp jq -c '.sections.prs.data.scores |= map(select(.score != null))' c-full.json > "$abase" @@ -146,7 +148,7 @@ amut escape '.sections.haplogroup.data.haplogroup = "H&\"x'"'"'" | .sections.y_str.data.markers[0].marker = "DYS<393>" | .sections.ancient.data.nearest[0].group = "Spain_" | .sections.ancient.data.nearest[0].locality = "A & B" | .sections.ancestry_fine.data.continent[0].group = "West " - | .sections.ancestry_fine.data.legend[0].colour = "red\">" | .sections.ancestry_fine.data.legend[1].group = "W" + | .sections.ancestry_fine.data.legend[0].colour = "#AbCdEf" | .sections.ancestry_fine.data.legend[1].group = "W" | .sections.archaic.data.longest = ["chr1:1-2 "] | .sections.prs.data.scores[0].condition = "CD & \"x\"" | .sections.prs.data.scores[0].effect = "HR <1.5>"' amut painting '.sections.ancestry_fine.data.painting = [{"code": "AFR", "group": "Africa", "fraction": 0.3},